From 417c2635b8d83b859f921b106980e2fb940726f2 Mon Sep 17 00:00:00 2001 From: ebo Date: Thu, 3 Oct 2024 15:30:11 -0400 Subject: [PATCH 001/159] set bind mounts for archive node --- etc/docker-compose.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/etc/docker-compose.yml b/etc/docker-compose.yml index 618aa6f5ae631..668f08daae7ac 100644 --- a/etc/docker-compose.yml +++ b/etc/docker-compose.yml @@ -69,13 +69,25 @@ services: volumes: mainnet_data: driver: local + driver_opts: + device: "/home/ec2-user/data/mainnet_data" sepolia_data: driver: local + driver_opts: + device: "/home/ec2-user/data/sepolia_data" holesky_data: driver: local + driver_opts: + device: "/home/ec2-user/data/holesky_data" logs: driver: local + driver_opts: + device: "/home/ec2-user/data/logs" prometheusdata: driver: local + driver_opts: + device: "/home/ec2-user/data/prometheusdata" grafanadata: driver: local + driver_opts: + device: "/home/ec2-user/data/grafanadata" From f1e489eb0b03523cd139016ebecb263850e5722e Mon Sep 17 00:00:00 2001 From: ebo Date: Thu, 3 Oct 2024 16:24:51 -0400 Subject: [PATCH 002/159] specify type --- etc/docker-compose.yml | 8 +++++++- etc/lighthouse.yml | 5 ++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/etc/docker-compose.yml b/etc/docker-compose.yml index 668f08daae7ac..af4dec6c6c8be 100644 --- a/etc/docker-compose.yml +++ b/etc/docker-compose.yml @@ -65,29 +65,35 @@ services: sh -c "cp -r /etc/grafana/provisioning_temp/dashboards/. /etc/grafana/provisioning/dashboards && find /etc/grafana/provisioning/dashboards/ -name '*.json' -exec sed -i 's/$${DS_PROMETHEUS}/Prometheus/g' {} \+ && /run.sh" - + volumes: mainnet_data: driver: local driver_opts: + type: "ext4" device: "/home/ec2-user/data/mainnet_data" sepolia_data: driver: local driver_opts: + type: "ext4" device: "/home/ec2-user/data/sepolia_data" holesky_data: driver: local driver_opts: + type: "ext4" device: "/home/ec2-user/data/holesky_data" logs: driver: local driver_opts: + type: "ext4" device: "/home/ec2-user/data/logs" prometheusdata: driver: local driver_opts: + type: "ext4" device: "/home/ec2-user/data/prometheusdata" grafanadata: driver: local driver_opts: + type: "ext4" device: "/home/ec2-user/data/grafanadata" diff --git a/etc/lighthouse.yml b/etc/lighthouse.yml index 966e4454db5bc..69c10bd0aff3f 100644 --- a/etc/lighthouse.yml +++ b/etc/lighthouse.yml @@ -44,4 +44,7 @@ services: volumes: lighthousedata: - driver: local \ No newline at end of file + driver: local + driver_opts: + type: "ext4" + device: "/home/ec2-user/data/lighthousedata" From 1c4f0ef4b1254ad57319b5a7cb5678ae07930594 Mon Sep 17 00:00:00 2001 From: ebo Date: Thu, 3 Oct 2024 16:34:51 -0400 Subject: [PATCH 003/159] fix mounts --- etc/docker-compose.yml | 18 ++++++++++++------ etc/lighthouse.yml | 3 ++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/etc/docker-compose.yml b/etc/docker-compose.yml index af4dec6c6c8be..29dc474ddc1c3 100644 --- a/etc/docker-compose.yml +++ b/etc/docker-compose.yml @@ -70,30 +70,36 @@ volumes: mainnet_data: driver: local driver_opts: - type: "ext4" + o: bind + type: none device: "/home/ec2-user/data/mainnet_data" sepolia_data: driver: local driver_opts: - type: "ext4" + o: bind + type: none device: "/home/ec2-user/data/sepolia_data" holesky_data: driver: local driver_opts: - type: "ext4" + o: bind + type: none device: "/home/ec2-user/data/holesky_data" logs: driver: local driver_opts: - type: "ext4" + o: bind + type: none device: "/home/ec2-user/data/logs" prometheusdata: driver: local driver_opts: - type: "ext4" + o: bind + type: none device: "/home/ec2-user/data/prometheusdata" grafanadata: driver: local driver_opts: - type: "ext4" + o: bind + type: none device: "/home/ec2-user/data/grafanadata" diff --git a/etc/lighthouse.yml b/etc/lighthouse.yml index 69c10bd0aff3f..21ad2f48c43d0 100644 --- a/etc/lighthouse.yml +++ b/etc/lighthouse.yml @@ -46,5 +46,6 @@ volumes: lighthousedata: driver: local driver_opts: - type: "ext4" + o: bind + type: none device: "/home/ec2-user/data/lighthousedata" From 69db5b5b2195aa62fb8794016797191d9ecc69df Mon Sep 17 00:00:00 2001 From: ebo Date: Thu, 3 Oct 2024 16:36:07 -0400 Subject: [PATCH 004/159] remove quotes --- etc/docker-compose.yml | 12 ++++++------ etc/lighthouse.yml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/etc/docker-compose.yml b/etc/docker-compose.yml index 29dc474ddc1c3..27c89ab60f938 100644 --- a/etc/docker-compose.yml +++ b/etc/docker-compose.yml @@ -72,34 +72,34 @@ volumes: driver_opts: o: bind type: none - device: "/home/ec2-user/data/mainnet_data" + device: /home/ec2-user/data/mainnet_data sepolia_data: driver: local driver_opts: o: bind type: none - device: "/home/ec2-user/data/sepolia_data" + device: /home/ec2-user/data/sepolia_data holesky_data: driver: local driver_opts: o: bind type: none - device: "/home/ec2-user/data/holesky_data" + device: /home/ec2-user/data/holesky_data logs: driver: local driver_opts: o: bind type: none - device: "/home/ec2-user/data/logs" + device: /home/ec2-user/data/logs prometheusdata: driver: local driver_opts: o: bind type: none - device: "/home/ec2-user/data/prometheusdata" + device: /home/ec2-user/data/prometheusdata grafanadata: driver: local driver_opts: o: bind type: none - device: "/home/ec2-user/data/grafanadata" + device: /home/ec2-user/data/grafanadata diff --git a/etc/lighthouse.yml b/etc/lighthouse.yml index 21ad2f48c43d0..dd5a4ff195789 100644 --- a/etc/lighthouse.yml +++ b/etc/lighthouse.yml @@ -48,4 +48,4 @@ volumes: driver_opts: o: bind type: none - device: "/home/ec2-user/data/lighthousedata" + device: /home/ec2-user/data/lighthousedata From b87abfcdd468e071f98d78032cc2766c318398cc Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Oct 2024 08:34:37 +0200 Subject: [PATCH 005/159] feat: make addons stateful (#11204) Co-authored-by: Arsenii Kulikov --- bin/reth/src/main.rs | 2 +- crates/ethereum/node/src/node.rs | 7 +++++- crates/ethereum/node/tests/it/builder.rs | 4 ++-- crates/ethereum/node/tests/it/exex.rs | 2 +- crates/exex/test-utils/src/lib.rs | 4 ++++ crates/node/builder/src/builder/add_ons.rs | 2 ++ crates/node/builder/src/builder/mod.rs | 11 ++++++---- crates/node/builder/src/builder/states.rs | 4 +++- crates/node/builder/src/launch/engine.rs | 2 +- crates/node/builder/src/launch/mod.rs | 2 +- crates/node/builder/src/node.rs | 24 +++++++++++++++------ crates/optimism/bin/src/main.rs | 2 +- crates/optimism/node/src/node.rs | 20 ++++++++++++++++- crates/optimism/node/tests/it/builder.rs | 2 +- examples/custom-engine-types/src/main.rs | 4 ++++ examples/custom-evm/src/main.rs | 2 +- examples/custom-node-components/src/main.rs | 2 +- examples/custom-payload-builder/src/main.rs | 2 +- examples/stateful-precompile/src/main.rs | 2 +- 19 files changed, 75 insertions(+), 25 deletions(-) diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index 578f2987d73fc..1be2d0efedcc3 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -60,7 +60,7 @@ fn main() { let handle = builder .with_types_and_provider::>() .with_components(EthereumNode::components()) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch_with_fn(|builder| { let launcher = EngineNodeLauncher::new( builder.task_executor().clone(), diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index c1c4653ae6aa6..f17658bb32da7 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -77,7 +77,8 @@ impl NodeTypesWithEngine for EthereumNode { } /// Add-ons w.r.t. l1 ethereum. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] +#[non_exhaustive] pub struct EthereumAddOns; impl NodeAddOns for EthereumAddOns { @@ -104,6 +105,10 @@ where fn components_builder(&self) -> Self::ComponentsBuilder { Self::components() } + + fn add_ons(&self) -> Self::AddOns { + EthereumAddOns::default() + } } /// A regular ethereum evm and executor builder. diff --git a/crates/ethereum/node/tests/it/builder.rs b/crates/ethereum/node/tests/it/builder.rs index 379f66e814b64..218839fbe0190 100644 --- a/crates/ethereum/node/tests/it/builder.rs +++ b/crates/ethereum/node/tests/it/builder.rs @@ -22,7 +22,7 @@ fn test_basic_setup() { .with_database(db) .with_types::() .with_components(EthereumNode::components()) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .on_component_initialized(move |ctx| { let _provider = ctx.provider(); println!("{msg}"); @@ -54,7 +54,7 @@ async fn test_eth_launcher() { NodeTypesWithDBAdapter>>, >>() .with_components(EthereumNode::components()) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch_with_fn(|builder| { let launcher = EngineNodeLauncher::new( tasks.executor(), diff --git a/crates/ethereum/node/tests/it/exex.rs b/crates/ethereum/node/tests/it/exex.rs index db19aaaf36129..856220300c2bb 100644 --- a/crates/ethereum/node/tests/it/exex.rs +++ b/crates/ethereum/node/tests/it/exex.rs @@ -33,7 +33,7 @@ fn basic_exex() { .with_database(db) .with_types::() .with_components(EthereumNode::components()) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .install_exex("dummy", move |ctx| future::ok(DummyExEx { _ctx: ctx })) .check_launch(); } diff --git a/crates/exex/test-utils/src/lib.rs b/crates/exex/test-utils/src/lib.rs index b8be08616b4ac..f4561a6b55f28 100644 --- a/crates/exex/test-utils/src/lib.rs +++ b/crates/exex/test-utils/src/lib.rs @@ -154,6 +154,10 @@ where .consensus(TestConsensusBuilder::default()) .engine_validator(EthereumEngineValidatorBuilder::default()) } + + fn add_ons(&self) -> Self::AddOns { + EthereumAddOns::default() + } } /// A shared [`TempDatabase`] used for testing diff --git a/crates/node/builder/src/builder/add_ons.rs b/crates/node/builder/src/builder/add_ons.rs index 910cd5896efe6..26d7553bb86d4 100644 --- a/crates/node/builder/src/builder/add_ons.rs +++ b/crates/node/builder/src/builder/add_ons.rs @@ -14,6 +14,8 @@ pub struct AddOns> { pub exexs: Vec<(String, Box>)>, /// Additional RPC add-ons. pub rpc: RpcAddOns, + /// Additional captured addons. + pub addons: AddOns, } /// Captures node specific addons that can be installed on top of the type configured node and are diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index 4989589c9f985..d2ce2a1d8e2f6 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -243,7 +243,7 @@ where where N: Node, ChainSpec = ChainSpec>, { - self.with_types().with_components(node.components_builder()).with_add_ons::() + self.with_types().with_components(node.components_builder()).with_add_ons(node.add_ons()) } } @@ -311,7 +311,7 @@ where where N: Node, ChainSpec = ChainSpec>, { - self.with_types().with_components(node.components_builder()).with_add_ons::() + self.with_types().with_components(node.components_builder()).with_add_ons(node.add_ons()) } /// Launches a preconfigured [Node] @@ -375,12 +375,15 @@ where { /// Advances the state of the node builder to the next state where all customizable /// [`NodeAddOns`] types are configured. - pub fn with_add_ons(self) -> WithLaunchContext> + pub fn with_add_ons( + self, + add_ons: AO, + ) -> WithLaunchContext> where AO: NodeAddOns>, { WithLaunchContext { - builder: self.builder.with_add_ons::(), + builder: self.builder.with_add_ons(add_ons), task_executor: self.task_executor, } } diff --git a/crates/node/builder/src/builder/states.rs b/crates/node/builder/src/builder/states.rs index 30ef54c5683e8..80930ef743cda 100644 --- a/crates/node/builder/src/builder/states.rs +++ b/crates/node/builder/src/builder/states.rs @@ -58,6 +58,7 @@ impl NodeBuilderWithTypes { hooks: NodeHooks::default(), rpc: RpcAddOns { hooks: RpcHooks::default() }, exexs: Vec::new(), + addons: (), }, } } @@ -168,7 +169,7 @@ where { /// Advances the state of the node builder to the next state where all customizable /// [`NodeAddOns`] types are configured. - pub fn with_add_ons(self) -> NodeBuilderWithComponents + pub fn with_add_ons(self, addons: AO) -> NodeBuilderWithComponents where AO: NodeAddOns>, { @@ -182,6 +183,7 @@ where hooks: NodeHooks::default(), rpc: RpcAddOns { hooks: RpcHooks::default() }, exexs: Vec::new(), + addons, }, } } diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index e71a0263c52cb..46ffacbf717a2 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -91,7 +91,7 @@ where let NodeBuilderWithComponents { adapter: NodeTypesAdapter { database }, components_builder, - add_ons: AddOns { hooks, rpc, exexs: installed_exex }, + add_ons: AddOns { hooks, rpc, exexs: installed_exex, .. }, config, } = target; let NodeHooks { on_component_initialized, on_node_started, .. } = hooks; diff --git a/crates/node/builder/src/launch/mod.rs b/crates/node/builder/src/launch/mod.rs index db98ffacedeb0..3188cde4b1578 100644 --- a/crates/node/builder/src/launch/mod.rs +++ b/crates/node/builder/src/launch/mod.rs @@ -126,7 +126,7 @@ where let NodeBuilderWithComponents { adapter: NodeTypesAdapter { database }, components_builder, - add_ons: AddOns { hooks, rpc, exexs: installed_exex }, + add_ons: AddOns { hooks, rpc, exexs: installed_exex, .. }, config, } = target; let NodeHooks { on_component_initialized, on_node_started, .. } = hooks; diff --git a/crates/node/builder/src/node.rs b/crates/node/builder/src/node.rs index 5d047f94c9182..3a70c08c10313 100644 --- a/crates/node/builder/src/node.rs +++ b/crates/node/builder/src/node.rs @@ -34,21 +34,29 @@ pub trait Node: NodeTypesWithEngine + Clone { /// Returns a [`NodeComponentsBuilder`] for the node. fn components_builder(&self) -> Self::ComponentsBuilder; + + /// Returns the node add-ons. + fn add_ons(&self) -> Self::AddOns; } /// A [`Node`] type builder #[derive(Clone, Default, Debug)] -pub struct AnyNode(PhantomData<(N, AO)>, C); +pub struct AnyNode(PhantomData, C, AO); -impl AnyNode { +impl AnyNode { /// Configures the types of the node. - pub fn types(self) -> AnyNode { - AnyNode::(PhantomData::<(T, ())>, self.1) + pub fn types(self) -> AnyNode { + AnyNode(PhantomData, self.1, self.2) } /// Sets the node components builder. - pub const fn components_builder(&self, value: T) -> AnyNode { - AnyNode::(PhantomData::<(N, ())>, value) + pub fn components_builder(self, value: T) -> AnyNode { + AnyNode(PhantomData, value, self.2) + } + + /// Sets the node add-ons. + pub fn add_ons(self, value: T) -> AnyNode { + AnyNode(PhantomData, self.1, value) } } @@ -84,6 +92,10 @@ where fn components_builder(&self) -> Self::ComponentsBuilder { self.1.clone() } + + fn add_ons(&self) -> Self::AddOns { + self.2.clone() + } } /// The launched node with all components including RPC handlers. diff --git a/crates/optimism/bin/src/main.rs b/crates/optimism/bin/src/main.rs index cecd01ea88971..6e041e240a74b 100644 --- a/crates/optimism/bin/src/main.rs +++ b/crates/optimism/bin/src/main.rs @@ -35,7 +35,7 @@ fn main() { let handle = builder .with_types_and_provider::>() .with_components(OptimismNode::components(rollup_args)) - .with_add_ons::() + .with_add_ons(OptimismAddOns::new(sequencer_http_arg.clone())) .extend_rpc_modules(move |ctx| { // register sequencer tx forwarder if let Some(sequencer_http) = sequencer_http_arg { diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index d82cb70a3ffec..4e7788569f89e 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -102,6 +102,10 @@ where let Self { args } = self; Self::components(args.clone()) } + + fn add_ons(&self) -> Self::AddOns { + OptimismAddOns::new(self.args.sequencer_http.clone()) + } } impl NodeTypes for OptimismNode { @@ -115,7 +119,21 @@ impl NodeTypesWithEngine for OptimismNode { /// Add-ons w.r.t. optimism. #[derive(Debug, Clone)] -pub struct OptimismAddOns; +pub struct OptimismAddOns { + sequencer_http: Option, +} + +impl OptimismAddOns { + /// Create a new instance with the given `sequencer_http` URL. + pub const fn new(sequencer_http: Option) -> Self { + Self { sequencer_http } + } + + /// Returns the sequencer HTTP URL. + pub fn sequencer_http(&self) -> Option<&str> { + self.sequencer_http.as_deref() + } +} impl NodeAddOns for OptimismAddOns { type EthApi = OpEthApi; diff --git a/crates/optimism/node/tests/it/builder.rs b/crates/optimism/node/tests/it/builder.rs index 20363828e8612..f1dde4c2c0a81 100644 --- a/crates/optimism/node/tests/it/builder.rs +++ b/crates/optimism/node/tests/it/builder.rs @@ -15,7 +15,7 @@ fn test_basic_setup() { .with_database(db) .with_types::() .with_components(OptimismNode::components(Default::default())) - .with_add_ons::() + .with_add_ons(OptimismAddOns::new(None)) .on_component_initialized(move |ctx| { let _provider = ctx.provider(); Ok(()) diff --git a/examples/custom-engine-types/src/main.rs b/examples/custom-engine-types/src/main.rs index 213a156af8fd6..34f8186be7f82 100644 --- a/examples/custom-engine-types/src/main.rs +++ b/examples/custom-engine-types/src/main.rs @@ -253,6 +253,10 @@ where .consensus(EthereumConsensusBuilder::default()) .engine_validator(CustomEngineValidatorBuilder::default()) } + + fn add_ons(&self) -> Self::AddOns { + EthereumAddOns::default() + } } /// A custom payload service builder that supports the custom engine types diff --git a/examples/custom-evm/src/main.rs b/examples/custom-evm/src/main.rs index d931c3b275bf4..9c421f9c6a59e 100644 --- a/examples/custom-evm/src/main.rs +++ b/examples/custom-evm/src/main.rs @@ -226,7 +226,7 @@ async fn main() -> eyre::Result<()> { .executor(MyExecutorBuilder::default()) .payload(MyPayloadBuilder::default()), ) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch() .await .unwrap(); diff --git a/examples/custom-node-components/src/main.rs b/examples/custom-node-components/src/main.rs index 1faca73d25b05..d00b8a70224af 100644 --- a/examples/custom-node-components/src/main.rs +++ b/examples/custom-node-components/src/main.rs @@ -25,7 +25,7 @@ fn main() { // Configure the components of the node // use default ethereum components but use our custom pool .with_components(EthereumNode::components().pool(CustomPoolBuilder::default())) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch() .await?; diff --git a/examples/custom-payload-builder/src/main.rs b/examples/custom-payload-builder/src/main.rs index 5ed414eb850bc..e46b969adaa14 100644 --- a/examples/custom-payload-builder/src/main.rs +++ b/examples/custom-payload-builder/src/main.rs @@ -81,7 +81,7 @@ fn main() { .with_components( EthereumNode::components().payload(CustomPayloadBuilder::default()), ) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch() .await?; diff --git a/examples/stateful-precompile/src/main.rs b/examples/stateful-precompile/src/main.rs index 05a6fd86c9350..26ebdfe4124b8 100644 --- a/examples/stateful-precompile/src/main.rs +++ b/examples/stateful-precompile/src/main.rs @@ -263,7 +263,7 @@ async fn main() -> eyre::Result<()> { .with_types::() // use default ethereum components but with our executor .with_components(EthereumNode::components().executor(MyExecutorBuilder::default())) - .with_add_ons::() + .with_add_ons(EthereumAddOns::default()) .launch() .await .unwrap(); From 86638a0b96373c9d0792705c93604bd3044818bd Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 4 Oct 2024 00:34:29 -0700 Subject: [PATCH 006/159] Relax Trait Bounds on TransactionPool::Transaction and EthPoolTransaction (#11079) Co-authored-by: Matthias Seitz --- crates/net/network/src/transactions/mod.rs | 13 +++--- crates/optimism/node/src/txpool.rs | 2 +- crates/optimism/rpc/src/eth/transaction.rs | 3 +- .../rpc-eth-api/src/helpers/transaction.rs | 8 ++-- crates/rpc/rpc/src/eth/filter.rs | 4 +- crates/rpc/rpc/src/txpool.rs | 8 ++-- crates/transaction-pool/src/lib.rs | 2 + crates/transaction-pool/src/maintain.rs | 14 ++++--- crates/transaction-pool/src/ordering.rs | 11 +---- crates/transaction-pool/src/pool/mod.rs | 21 ++++++++-- crates/transaction-pool/src/traits.rs | 42 +++++++------------ crates/transaction-pool/src/validate/mod.rs | 14 ++----- 12 files changed, 71 insertions(+), 71 deletions(-) diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 7e3b71aa4365a..2fa4ccfbb606d 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -1033,7 +1033,7 @@ where has_bad_transactions = true; } else { // this is a new transaction that should be imported into the pool - let pool_transaction = Pool::Transaction::from_pooled(tx); + let pool_transaction = Pool::Transaction::from_pooled(tx.into()); new_txs.push(pool_transaction); entry.insert(HashSet::from([peer_id])); @@ -1396,11 +1396,14 @@ impl PropagateTransaction { } /// Create a new instance from a pooled transaction - fn new>( - tx: Arc>, - ) -> Self { + fn new(tx: Arc>) -> Self + where + T: PoolTransaction>, + { let size = tx.encoded_length(); - let transaction = Arc::new(tx.transaction.clone().into_consensus().into_signed()); + let recovered: TransactionSignedEcRecovered = + tx.transaction.clone().into_consensus().into(); + let transaction = Arc::new(recovered.into_signed()); Self { size, transaction } } } diff --git a/crates/optimism/node/src/txpool.rs b/crates/optimism/node/src/txpool.rs index 811c37e91cb57..7ed2a161d0e4f 100644 --- a/crates/optimism/node/src/txpool.rs +++ b/crates/optimism/node/src/txpool.rs @@ -140,7 +140,7 @@ where let l1_block_info = self.block_info.l1_block_info.read().clone(); let mut encoded = Vec::with_capacity(valid_tx.transaction().encoded_length()); - valid_tx.transaction().clone().into_consensus().encode_2718(&mut encoded); + valid_tx.transaction().clone().into_consensus().into().encode_2718(&mut encoded); let cost_addition = match l1_block_info.l1_tx_data_fee( &self.chain_spec(), diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index 2556c895783e4..3ccda419cad81 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -34,7 +34,8 @@ where /// Returns the hash of the transaction. async fn send_raw_transaction(&self, tx: Bytes) -> Result { let recovered = recover_raw_transaction(tx.clone())?; - let pool_transaction = ::Transaction::from_pooled(recovered); + let pool_transaction = + ::Transaction::from_pooled(recovered.into()); // On optimism, transactions are forwarded directly to the sequencer to be included in // blocks that it builds. diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index d98cb69bfc30d..d70ea7e541cb5 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -231,7 +231,7 @@ pub trait EthTransactions: LoadTransaction { LoadState::pool(self).get_transaction_by_sender_and_nonce(sender, nonce) { let transaction = tx.transaction.clone().into_consensus(); - return Ok(Some(from_recovered::(transaction))); + return Ok(Some(from_recovered::(transaction.into()))); } } @@ -324,7 +324,7 @@ pub trait EthTransactions: LoadTransaction { async move { let recovered = recover_raw_transaction(tx.clone())?; let pool_transaction = - ::Transaction::from_pooled(recovered); + ::Transaction::from_pooled(recovered.into()); // submit the transaction to the pool with a `Local` origin let hash = self @@ -376,7 +376,7 @@ pub trait EthTransactions: LoadTransaction { let recovered = signed_tx.into_ecrecovered().ok_or(EthApiError::InvalidTransactionSignature)?; - let pool_transaction = <::Pool as TransactionPool>::Transaction::try_from_consensus(recovered).map_err(|_| EthApiError::TransactionConversionError)?; + let pool_transaction = <::Pool as TransactionPool>::Transaction::try_from_consensus(recovered.into()).map_err(|_| EthApiError::TransactionConversionError)?; // submit the transaction to the pool with a `Local` origin let hash = LoadTransaction::pool(self) @@ -518,7 +518,7 @@ pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes { if let Some(tx) = self.pool().get(&hash).map(|tx| tx.transaction.clone().into_consensus()) { - resp = Some(TransactionSource::Pool(tx)); + resp = Some(TransactionSource::Pool(tx.into())); } } diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 274347404c93d..f4ae892641609 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -611,7 +611,7 @@ where /// Returns all new pending transactions received since the last poll. async fn drain(&self) -> FilterChanges where - T: PoolTransaction, + T: PoolTransaction>, { let mut pending_txs = Vec::new(); let mut prepared_stream = self.txs_stream.lock().await; @@ -633,7 +633,7 @@ trait FullTransactionsFilter: fmt::Debug + Send + Sync + Unpin + 'static { impl FullTransactionsFilter for FullTransactionsReceiver where - T: PoolTransaction + 'static, + T: PoolTransaction> + 'static, TxCompat: TransactionCompat + 'static, { async fn drain(&self) -> FilterChanges { diff --git a/crates/rpc/rpc/src/txpool.rs b/crates/rpc/rpc/src/txpool.rs index 4932a563409e6..5e26935ca1ba7 100644 --- a/crates/rpc/rpc/src/txpool.rs +++ b/crates/rpc/rpc/src/txpool.rs @@ -41,12 +41,12 @@ where tx: &Tx, content: &mut BTreeMap>, ) where - Tx: PoolTransaction, + Tx: PoolTransaction>, RpcTxB: TransactionCompat, { content.entry(tx.sender()).or_default().insert( tx.nonce().to_string(), - from_recovered::(tx.clone().into_consensus()), + from_recovered::(tx.clone().into_consensus().into()), ); } @@ -91,12 +91,12 @@ where trace!(target: "rpc::eth", "Serving txpool_inspect"); #[inline] - fn insert>( + fn insert>>( tx: &T, inspect: &mut BTreeMap>, ) { let entry = inspect.entry(tx.sender()).or_default(); - let tx = tx.clone().into_consensus(); + let tx: TransactionSignedEcRecovered = tx.clone().into_consensus().into(); entry.insert( tx.nonce().to_string(), TxpoolInspectSummary { diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 4aec8ab408581..744dd44d2f83a 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -324,6 +324,7 @@ where impl TransactionPool for Pool where V: TransactionValidator, + ::Transaction: EthPoolTransaction, T: TransactionOrdering::Transaction>, S: BlobStore, { @@ -546,6 +547,7 @@ where impl TransactionPoolExt for Pool where V: TransactionValidator, + ::Transaction: EthPoolTransaction, T: TransactionOrdering::Transaction>, S: BlobStore, { diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index da416fd2d43fb..523151956dd10 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -18,6 +18,7 @@ use reth_execution_types::ChangedAccount; use reth_fs_util::FsPathError; use reth_primitives::{ BlockNumberOrTag, PooledTransactionsElementEcRecovered, SealedHeader, TransactionSigned, + TransactionSignedEcRecovered, }; use reth_storage_api::{errors::provider::ProviderError, BlockReaderIdExt, StateProviderFactory}; use reth_tasks::TaskSpawner; @@ -334,11 +335,10 @@ pub async fn maintain_transaction_pool( .ok() }) .map(|tx| { - <

::Transaction as PoolTransaction>::from_pooled(tx) +

::Transaction::from_pooled(tx.into()) }) } else { - - ::try_from_consensus(tx).ok() +

::Transaction::try_from_consensus(tx.into()).ok() } }) .collect::>(); @@ -583,7 +583,7 @@ where .filter_map(|tx| tx.try_ecrecovered()) .filter_map(|tx| { // Filter out errors - ::try_from_consensus(tx).ok() + ::try_from_consensus(tx.into()).ok() }) .collect::>(); @@ -606,7 +606,11 @@ where let local_transactions = local_transactions .into_iter() - .map(|tx| tx.to_recovered_transaction().into_signed()) + .map(|tx| { + let recovered: TransactionSignedEcRecovered = + tx.transaction.clone().into_consensus().into(); + recovered.into_signed() + }) .collect::>(); let num_txs = local_transactions.len(); diff --git a/crates/transaction-pool/src/ordering.rs b/crates/transaction-pool/src/ordering.rs index 3381bb0279472..0ee0c1004a131 100644 --- a/crates/transaction-pool/src/ordering.rs +++ b/crates/transaction-pool/src/ordering.rs @@ -1,6 +1,5 @@ use crate::traits::PoolTransaction; use alloy_primitives::U256; -use reth_primitives::{PooledTransactionsElementEcRecovered, TransactionSignedEcRecovered}; use std::{fmt, marker::PhantomData}; /// Priority of the transaction that can be missing. @@ -32,10 +31,7 @@ pub trait TransactionOrdering: Send + Sync + 'static { type PriorityValue: Ord + Clone + Default + fmt::Debug + Send + Sync; /// The transaction type to determine the priority of. - type Transaction: PoolTransaction< - Pooled = PooledTransactionsElementEcRecovered, - Consensus = TransactionSignedEcRecovered, - >; + type Transaction: PoolTransaction; /// Returns the priority score for the given transaction. fn priority( @@ -55,10 +51,7 @@ pub struct CoinbaseTipOrdering(PhantomData); impl TransactionOrdering for CoinbaseTipOrdering where - T: PoolTransaction< - Pooled = PooledTransactionsElementEcRecovered, - Consensus = TransactionSignedEcRecovered, - > + 'static, + T: PoolTransaction + 'static, { type PriorityValue = U256; type Transaction = T; diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 4c1a7f2c29bbc..b78f5128686c6 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -88,6 +88,7 @@ use reth_execution_types::ChangedAccount; use reth_primitives::{ BlobTransaction, BlobTransactionSidecar, PooledTransactionsElement, TransactionSigned, + TransactionSignedEcRecovered, }; use std::{ collections::{HashMap, HashSet}, @@ -318,13 +319,19 @@ where &self, tx_hashes: Vec, limit: GetPooledTransactionLimit, - ) -> Vec { + ) -> Vec + where + ::Transaction: + PoolTransaction>, + { let transactions = self.get_all(tx_hashes); let mut elements = Vec::with_capacity(transactions.len()); let mut size = 0; for transaction in transactions { let encoded_len = transaction.encoded_length(); - let tx = transaction.to_recovered_transaction().into_signed(); + let recovered: TransactionSignedEcRecovered = + transaction.transaction.clone().into_consensus().into(); + let tx = recovered.into_signed(); let pooled = if tx.is_eip4844() { // for EIP-4844 transactions, we need to fetch the blob sidecar from the blob store if let Some(blob) = self.get_blob_transaction(tx) { @@ -360,9 +367,15 @@ where pub(crate) fn get_pooled_transaction_element( &self, tx_hash: TxHash, - ) -> Option { + ) -> Option + where + ::Transaction: + PoolTransaction>, + { self.get(&tx_hash).and_then(|transaction| { - let tx = transaction.to_recovered_transaction().into_signed(); + let recovered: TransactionSignedEcRecovered = + transaction.transaction.clone().into_consensus().into(); + let tx = recovered.into_signed(); if tx.is_eip4844() { self.get_blob_transaction(tx).map(PooledTransactionsElement::BlobTransaction) } else { diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index d4eabc73bbcea..e522978fb9dc0 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -44,10 +44,7 @@ pub type PeerId = alloy_primitives::B512; #[auto_impl::auto_impl(&, Arc)] pub trait TransactionPool: Send + Sync + Clone { /// The transaction type of the pool - type Transaction: PoolTransaction< - Pooled = PooledTransactionsElementEcRecovered, - Consensus = TransactionSignedEcRecovered, - >; + type Transaction: EthPoolTransaction; /// Returns stats about the pool and all sub-pools. fn pool_size(&self) -> PoolSize; @@ -496,12 +493,12 @@ pub struct AllPoolTransactions { impl AllPoolTransactions { /// Returns an iterator over all pending [`TransactionSignedEcRecovered`] transactions. pub fn pending_recovered(&self) -> impl Iterator + '_ { - self.pending.iter().map(|tx| tx.transaction.clone().into_consensus()) + self.pending.iter().map(|tx| tx.transaction.clone().into()) } /// Returns an iterator over all queued [`TransactionSignedEcRecovered`] transactions. pub fn queued_recovered(&self) -> impl Iterator + '_ { - self.queued.iter().map(|tx| tx.transaction.clone().into_consensus()) + self.queued.iter().map(|tx| tx.transaction.clone().into()) } } @@ -813,19 +810,25 @@ pub trait PoolTransaction: fmt::Debug + Send + Sync + Clone { type TryFromConsensusError; /// Associated type representing the raw consensus variant of the transaction. - type Consensus: From + TryInto; + type Consensus: From + TryInto; /// Associated type representing the recovered pooled variant of the transaction. type Pooled: Into; /// Define a method to convert from the `Consensus` type to `Self` - fn try_from_consensus(tx: Self::Consensus) -> Result; + fn try_from_consensus(tx: Self::Consensus) -> Result { + tx.try_into() + } /// Define a method to convert from the `Self` type to `Consensus` - fn into_consensus(self) -> Self::Consensus; + fn into_consensus(self) -> Self::Consensus { + self.into() + } /// Define a method to convert from the `Pooled` type to `Self` - fn from_pooled(pooled: Self::Pooled) -> Self; + fn from_pooled(pooled: Self::Pooled) -> Self { + pooled.into() + } /// Hash of the transaction. fn hash(&self) -> &TxHash; @@ -921,12 +924,11 @@ pub trait PoolTransaction: fmt::Debug + Send + Sync + Clone { fn chain_id(&self) -> Option; } -/// An extension trait that provides additional interfaces for the -/// [`EthTransactionValidator`](crate::EthTransactionValidator). +/// Super trait for transactions that can be converted to and from Eth transactions pub trait EthPoolTransaction: PoolTransaction< - Pooled = PooledTransactionsElementEcRecovered, - Consensus = TransactionSignedEcRecovered, + Consensus: From + Into, + Pooled: From + Into, > { /// Extracts the blob sidecar from the transaction. @@ -1069,18 +1071,6 @@ impl PoolTransaction for EthPooledTransaction { type Pooled = PooledTransactionsElementEcRecovered; - fn try_from_consensus(tx: Self::Consensus) -> Result { - tx.try_into() - } - - fn into_consensus(self) -> Self::Consensus { - self.into() - } - - fn from_pooled(pooled: Self::Pooled) -> Self { - pooled.into() - } - /// Returns hash of the transaction. fn hash(&self) -> &TxHash { self.transaction.hash_ref() diff --git a/crates/transaction-pool/src/validate/mod.rs b/crates/transaction-pool/src/validate/mod.rs index b8fe7cbb1de02..4395cc97908b7 100644 --- a/crates/transaction-pool/src/validate/mod.rs +++ b/crates/transaction-pool/src/validate/mod.rs @@ -7,10 +7,7 @@ use crate::{ }; use alloy_primitives::{Address, TxHash, B256, U256}; use futures_util::future::Either; -use reth_primitives::{ - BlobTransactionSidecar, PooledTransactionsElementEcRecovered, SealedBlock, - TransactionSignedEcRecovered, -}; +use reth_primitives::{BlobTransactionSidecar, SealedBlock, TransactionSignedEcRecovered}; use std::{fmt, future::Future, time::Instant}; mod constants; @@ -154,10 +151,7 @@ impl ValidTransaction { /// Provides support for validating transaction at any given state of the chain pub trait TransactionValidator: Send + Sync { /// The transaction type to validate. - type Transaction: PoolTransaction< - Pooled = PooledTransactionsElementEcRecovered, - Consensus = TransactionSignedEcRecovered, - >; + type Transaction: PoolTransaction; /// Validates the transaction and returns a [`TransactionValidationOutcome`] describing the /// validity of the given transaction. @@ -380,12 +374,12 @@ impl ValidPoolTransaction { } } -impl> ValidPoolTransaction { +impl>> ValidPoolTransaction { /// Converts to this type into a [`TransactionSignedEcRecovered`]. /// /// Note: this takes `&self` since indented usage is via `Arc`. pub fn to_recovered_transaction(&self) -> TransactionSignedEcRecovered { - self.transaction.clone().into_consensus() + self.transaction.clone().into_consensus().into() } } From 6068fa51a9e7456b3f46d0b92fb0070519ab2310 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 4 Oct 2024 09:36:24 +0200 Subject: [PATCH 007/159] test: add unit tests for `CanonicalChain` (#11472) --- crates/blockchain-tree/src/canonical_chain.rs | 176 +++++++++++++++++- 1 file changed, 167 insertions(+), 9 deletions(-) diff --git a/crates/blockchain-tree/src/canonical_chain.rs b/crates/blockchain-tree/src/canonical_chain.rs index 7dcd466f7d64a..253f799fe0f87 100644 --- a/crates/blockchain-tree/src/canonical_chain.rs +++ b/crates/blockchain-tree/src/canonical_chain.rs @@ -32,15 +32,7 @@ impl CanonicalChain { /// Returns the block number of the (non-finalized) canonical block with the given hash. #[inline] pub(crate) fn canonical_number(&self, block_hash: &BlockHash) -> Option { - self.chain.iter().find_map( - |(number, hash)| { - if hash == block_hash { - Some(*number) - } else { - None - } - }, - ) + self.chain.iter().find_map(|(number, hash)| (hash == block_hash).then_some(*number)) } /// Extends all items from the given iterator to the chain. @@ -81,3 +73,169 @@ impl CanonicalChain { self.chain.into_iter() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_replace_canonical_chain() { + // Initialize a chain with some blocks + let mut initial_chain = BTreeMap::new(); + initial_chain.insert(BlockNumber::from(1u64), BlockHash::from([0x01; 32])); + initial_chain.insert(BlockNumber::from(2u64), BlockHash::from([0x02; 32])); + + let mut canonical_chain = CanonicalChain::new(initial_chain.clone()); + + // Verify initial chain state + assert_eq!(canonical_chain.chain.len(), 2); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(1u64)), + Some(&BlockHash::from([0x01; 32])) + ); + + // Replace with a new chain + let mut new_chain = BTreeMap::new(); + new_chain.insert(BlockNumber::from(3u64), BlockHash::from([0x03; 32])); + new_chain.insert(BlockNumber::from(4u64), BlockHash::from([0x04; 32])); + new_chain.insert(BlockNumber::from(5u64), BlockHash::from([0x05; 32])); + + canonical_chain.replace(new_chain.clone()); + + // Verify replaced chain state + assert_eq!(canonical_chain.chain.len(), 3); + assert!(!canonical_chain.chain.contains_key(&BlockNumber::from(1u64))); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(3u64)), + Some(&BlockHash::from([0x03; 32])) + ); + } + + #[test] + fn test_canonical_hash_canonical_chain() { + // Initialize a chain with some blocks + let mut chain = BTreeMap::new(); + chain.insert(BlockNumber::from(1u64), BlockHash::from([0x01; 32])); + chain.insert(BlockNumber::from(2u64), BlockHash::from([0x02; 32])); + chain.insert(BlockNumber::from(3u64), BlockHash::from([0x03; 32])); + + // Create an instance of a canonical chain + let canonical_chain = CanonicalChain::new(chain.clone()); + + // Check that the function returns the correct hash for a given block number + let block_number = BlockNumber::from(2u64); + let expected_hash = BlockHash::from([0x02; 32]); + assert_eq!(canonical_chain.canonical_hash(&block_number), Some(expected_hash)); + + // Check that a non-existent block returns None + let non_existent_block = BlockNumber::from(5u64); + assert_eq!(canonical_chain.canonical_hash(&non_existent_block), None); + } + + #[test] + fn test_canonical_number_canonical_chain() { + // Initialize a chain with some blocks + let mut chain = BTreeMap::new(); + chain.insert(BlockNumber::from(1u64), BlockHash::from([0x01; 32])); + chain.insert(BlockNumber::from(2u64), BlockHash::from([0x02; 32])); + chain.insert(BlockNumber::from(3u64), BlockHash::from([0x03; 32])); + + // Create an instance of a canonical chain + let canonical_chain = CanonicalChain::new(chain.clone()); + + // Check that the function returns the correct block number for a given block hash + let block_hash = BlockHash::from([0x02; 32]); + let expected_number = BlockNumber::from(2u64); + assert_eq!(canonical_chain.canonical_number(&block_hash), Some(expected_number)); + + // Check that a non-existent block hash returns None + let non_existent_hash = BlockHash::from([0x05; 32]); + assert_eq!(canonical_chain.canonical_number(&non_existent_hash), None); + } + + #[test] + fn test_extend_canonical_chain() { + // Initialize an empty chain + let mut canonical_chain = CanonicalChain::new(BTreeMap::new()); + + // Create an iterator with some blocks + let blocks = vec![ + (BlockNumber::from(1u64), BlockHash::from([0x01; 32])), + (BlockNumber::from(2u64), BlockHash::from([0x02; 32])), + ] + .into_iter(); + + // Extend the chain with the created blocks + canonical_chain.extend(blocks); + + // Check if the blocks were added correctly + assert_eq!(canonical_chain.chain.len(), 2); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(1u64)), + Some(&BlockHash::from([0x01; 32])) + ); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(2u64)), + Some(&BlockHash::from([0x02; 32])) + ); + + // Test extending with additional blocks again + let more_blocks = vec![(BlockNumber::from(3u64), BlockHash::from([0x03; 32]))].into_iter(); + canonical_chain.extend(more_blocks); + + assert_eq!(canonical_chain.chain.len(), 3); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(3u64)), + Some(&BlockHash::from([0x03; 32])) + ); + } + + #[test] + fn test_retain_canonical_chain() { + // Initialize a chain with some blocks + let mut chain = BTreeMap::new(); + chain.insert(BlockNumber::from(1u64), BlockHash::from([0x01; 32])); + chain.insert(BlockNumber::from(2u64), BlockHash::from([0x02; 32])); + chain.insert(BlockNumber::from(3u64), BlockHash::from([0x03; 32])); + + // Create an instance of CanonicalChain + let mut canonical_chain = CanonicalChain::new(chain); + + // Retain only blocks with even block numbers + canonical_chain.retain(|number, _| number % 2 == 0); + + // Check if the chain only contains the block with number 2 + assert_eq!(canonical_chain.chain.len(), 1); + assert_eq!( + canonical_chain.chain.get(&BlockNumber::from(2u64)), + Some(&BlockHash::from([0x02; 32])) + ); + + // Ensure that the blocks with odd numbers were removed + assert_eq!(canonical_chain.chain.get(&BlockNumber::from(1u64)), None); + assert_eq!(canonical_chain.chain.get(&BlockNumber::from(3u64)), None); + } + + #[test] + fn test_tip_canonical_chain() { + // Initialize a chain with some blocks + let mut chain = BTreeMap::new(); + chain.insert(BlockNumber::from(1u64), BlockHash::from([0x01; 32])); + chain.insert(BlockNumber::from(2u64), BlockHash::from([0x02; 32])); + chain.insert(BlockNumber::from(3u64), BlockHash::from([0x03; 32])); + + // Create an instance of a canonical chain + let canonical_chain = CanonicalChain::new(chain); + + // Call the tip method and verify the returned value + let tip = canonical_chain.tip(); + assert_eq!(tip.number, BlockNumber::from(3u64)); + assert_eq!(tip.hash, BlockHash::from([0x03; 32])); + + // Test with an empty chain + let empty_chain = CanonicalChain::new(BTreeMap::new()); + let empty_tip = empty_chain.tip(); + assert_eq!(empty_tip.number, BlockNumber::default()); + assert_eq!(empty_tip.hash, BlockHash::default()); + } +} From 7153645082d0659fcb816b3ec3515f0df848c0ab Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 4 Oct 2024 10:11:38 +0200 Subject: [PATCH 008/159] feat(perf): integrate OnStateHook in executor (#11345) --- crates/ethereum/evm/src/execute.rs | 56 +++++++++++++++++++++++---- crates/evm/src/either.rs | 19 ++++++++- crates/evm/src/execute.rs | 23 +++++++++++ crates/evm/src/noop.rs | 16 +++++++- crates/evm/src/system_calls/mod.rs | 16 +++++--- crates/evm/src/test_utils.rs | 18 ++++++++- crates/optimism/evm/src/execute.rs | 62 ++++++++++++++++++++++++++---- 7 files changed, 185 insertions(+), 25 deletions(-) diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs index f1d7de115d575..8c84fafc25fba 100644 --- a/crates/ethereum/evm/src/execute.rs +++ b/crates/ethereum/evm/src/execute.rs @@ -14,7 +14,7 @@ use reth_evm::{ BatchExecutor, BlockExecutionError, BlockExecutionInput, BlockExecutionOutput, BlockExecutorProvider, BlockValidationError, Executor, ProviderError, }, - system_calls::SystemCaller, + system_calls::{NoopHook, OnStateHook, SystemCaller}, ConfigureEvm, }; use reth_execution_types::ExecutionOutcome; @@ -126,20 +126,25 @@ where /// This applies the pre-execution and post-execution changes that require an [EVM](Evm), and /// executes the transactions. /// + /// The optional `state_hook` will be executed with the state changes if present. + /// /// # Note /// /// It does __not__ apply post-execution changes that do not require an [EVM](Evm), for that see /// [`EthBlockExecutor::post_execution`]. - fn execute_state_transitions( + fn execute_state_transitions( &self, block: &BlockWithSenders, mut evm: Evm<'_, Ext, &mut State>, + state_hook: Option, ) -> Result where DB: Database, DB::Error: Into + Display, + F: OnStateHook, { - let mut system_caller = SystemCaller::new(&self.evm_config, &self.chain_spec); + let mut system_caller = + SystemCaller::new(&self.evm_config, &self.chain_spec).with_state_hook(state_hook); system_caller.apply_pre_execution_changes(block, &mut evm)?; @@ -161,7 +166,7 @@ where self.evm_config.fill_tx_env(evm.tx_mut(), transaction, *sender); // Execute transaction. - let ResultAndState { result, state } = evm.transact().map_err(move |err| { + let result_and_state = evm.transact().map_err(move |err| { let new_err = err.map_db_err(|e| e.into()); // Ensure hash is calculated for error log, if not already done BlockValidationError::EVM { @@ -169,6 +174,8 @@ where error: Box::new(new_err), } })?; + system_caller.on_state(&result_and_state); + let ResultAndState { result, state } = result_and_state; evm.db_mut().commit(state); // append gas used @@ -260,17 +267,31 @@ where EnvWithHandlerCfg::new_with_cfg_env(cfg, block_env, Default::default()) } + /// Convenience method to invoke `execute_without_verification_with_state_hook` setting the + /// state hook as `None`. + fn execute_without_verification( + &mut self, + block: &BlockWithSenders, + total_difficulty: U256, + ) -> Result { + self.execute_without_verification_with_state_hook(block, total_difficulty, None::) + } + /// Execute a single block and apply the state changes to the internal state. /// /// Returns the receipts of the transactions in the block, the total gas used and the list of /// EIP-7685 [requests](Request). /// /// Returns an error if execution fails. - fn execute_without_verification( + fn execute_without_verification_with_state_hook( &mut self, block: &BlockWithSenders, total_difficulty: U256, - ) -> Result { + state_hook: Option, + ) -> Result + where + F: OnStateHook, + { // 1. prepare state on new block self.on_new_block(&block.header); @@ -278,7 +299,7 @@ where let env = self.evm_env_for_block(&block.header, total_difficulty); let output = { let evm = self.executor.evm_config.evm_with_env(&mut self.state, env); - self.executor.execute_state_transitions(block, evm) + self.executor.execute_state_transitions(block, evm, state_hook) }?; // 3. apply post execution changes @@ -368,6 +389,27 @@ where witness(&self.state); Ok(BlockExecutionOutput { state: self.state.take_bundle(), receipts, requests, gas_used }) } + + fn execute_with_state_hook( + mut self, + input: Self::Input<'_>, + state_hook: F, + ) -> Result + where + F: OnStateHook, + { + let BlockExecutionInput { block, total_difficulty } = input; + let EthExecuteOutput { receipts, requests, gas_used } = self + .execute_without_verification_with_state_hook( + block, + total_difficulty, + Some(state_hook), + )?; + + // NOTE: we need to merge keep the reverts for the bundle retention + self.state.merge_transitions(BundleRetention::Reverts); + Ok(BlockExecutionOutput { state: self.state.take_bundle(), receipts, requests, gas_used }) + } } /// An executor for a batch of blocks. /// diff --git a/crates/evm/src/either.rs b/crates/evm/src/either.rs index a3fca50ec7eee..e28ec3887f806 100644 --- a/crates/evm/src/either.rs +++ b/crates/evm/src/either.rs @@ -2,7 +2,10 @@ use core::fmt::Display; -use crate::execute::{BatchExecutor, BlockExecutorProvider, Executor}; +use crate::{ + execute::{BatchExecutor, BlockExecutorProvider, Executor}, + system_calls::OnStateHook, +}; use alloy_primitives::BlockNumber; use reth_execution_errors::BlockExecutionError; use reth_execution_types::{BlockExecutionInput, BlockExecutionOutput, ExecutionOutcome}; @@ -87,6 +90,20 @@ where Self::Right(b) => b.execute_with_state_witness(input, witness), } } + + fn execute_with_state_hook( + self, + input: Self::Input<'_>, + state_hook: F, + ) -> Result + where + F: OnStateHook, + { + match self { + Self::Left(a) => a.execute_with_state_hook(input, state_hook), + Self::Right(b) => b.execute_with_state_hook(input, state_hook), + } + } } impl BatchExecutor for Either diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index ffc08469dc8d6..3fc2975f0ff78 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -12,6 +12,8 @@ use reth_prune_types::PruneModes; use revm::State; use revm_primitives::db::Database; +use crate::system_calls::OnStateHook; + /// A general purpose executor trait that executes an input (e.g. block) and produces an output /// (e.g. state changes and receipts). /// @@ -43,6 +45,16 @@ pub trait Executor { ) -> Result where F: FnMut(&State); + + /// Executes the EVM with the given input and accepts a state hook closure that is invoked with + /// the EVM state after execution. + fn execute_with_state_hook( + self, + input: Self::Input<'_>, + state_hook: F, + ) -> Result + where + F: OnStateHook; } /// A general purpose executor that can execute multiple inputs in sequence, validate the outputs, @@ -199,6 +211,17 @@ mod tests { { Err(BlockExecutionError::msg("execution unavailable for tests")) } + + fn execute_with_state_hook( + self, + _: Self::Input<'_>, + _: F, + ) -> Result + where + F: OnStateHook, + { + Err(BlockExecutionError::msg("execution unavailable for tests")) + } } impl BatchExecutor for TestExecutor { diff --git a/crates/evm/src/noop.rs b/crates/evm/src/noop.rs index 392bfd0bd722d..3e01bfc4cc467 100644 --- a/crates/evm/src/noop.rs +++ b/crates/evm/src/noop.rs @@ -10,7 +10,10 @@ use reth_storage_errors::provider::ProviderError; use revm::State; use revm_primitives::db::Database; -use crate::execute::{BatchExecutor, BlockExecutorProvider, Executor}; +use crate::{ + execute::{BatchExecutor, BlockExecutorProvider, Executor}, + system_calls::OnStateHook, +}; const UNAVAILABLE_FOR_NOOP: &str = "execution unavailable for noop"; @@ -58,6 +61,17 @@ impl Executor for NoopBlockExecutorProvider { { Err(BlockExecutionError::msg(UNAVAILABLE_FOR_NOOP)) } + + fn execute_with_state_hook( + self, + _: Self::Input<'_>, + _: F, + ) -> Result + where + F: OnStateHook, + { + Err(BlockExecutionError::msg(UNAVAILABLE_FOR_NOOP)) + } } impl BatchExecutor for NoopBlockExecutorProvider { diff --git a/crates/evm/src/system_calls/mod.rs b/crates/evm/src/system_calls/mod.rs index ce5fec42184c1..48429cb49596e 100644 --- a/crates/evm/src/system_calls/mod.rs +++ b/crates/evm/src/system_calls/mod.rs @@ -49,22 +49,19 @@ pub struct SystemCaller<'a, EvmConfig, Chainspec, Hook = NoopHook> { hook: Option, } -impl<'a, EvmConfig, Chainspec> SystemCaller<'a, EvmConfig, Chainspec> { +impl<'a, EvmConfig, Chainspec> SystemCaller<'a, EvmConfig, Chainspec, NoopHook> { /// Create a new system caller with the given EVM config, database, and chain spec, and creates /// the EVM with the given initialized config and block environment. pub const fn new(evm_config: &'a EvmConfig, chain_spec: Chainspec) -> Self { Self { evm_config, chain_spec, hook: None } } -} - -impl<'a, EvmConfig, Chainspec, Hook> SystemCaller<'a, EvmConfig, Chainspec, Hook> { /// Installs a custom hook to be called after each state change. pub fn with_state_hook( self, - hook: H, + hook: Option, ) -> SystemCaller<'a, EvmConfig, Chainspec, H> { let Self { evm_config, chain_spec, .. } = self; - SystemCaller { evm_config, chain_spec, hook: Some(hook) } + SystemCaller { evm_config, chain_spec, hook } } /// Convenience method to consume the type and drop borrowed fields pub fn finish(self) {} @@ -321,4 +318,11 @@ where eip7251::post_commit(result_and_state.result) } + + /// Delegate to stored `OnStateHook`, noop if hook is `None`. + pub fn on_state(&mut self, state: &ResultAndState) { + if let Some(ref mut hook) = &mut self.hook { + hook.on_state(state); + } + } } diff --git a/crates/evm/src/test_utils.rs b/crates/evm/src/test_utils.rs index cf45930aece94..45ab2e97734e0 100644 --- a/crates/evm/src/test_utils.rs +++ b/crates/evm/src/test_utils.rs @@ -1,7 +1,10 @@ //! Helpers for testing. -use crate::execute::{ - BatchExecutor, BlockExecutionInput, BlockExecutionOutput, BlockExecutorProvider, Executor, +use crate::{ + execute::{ + BatchExecutor, BlockExecutionInput, BlockExecutionOutput, BlockExecutorProvider, Executor, + }, + system_calls::OnStateHook, }; use alloy_primitives::BlockNumber; use parking_lot::Mutex; @@ -73,6 +76,17 @@ impl Executor for MockExecutorProvider { { unimplemented!() } + + fn execute_with_state_hook( + self, + _: Self::Input<'_>, + _: F, + ) -> Result + where + F: OnStateHook, + { + unimplemented!() + } } impl BatchExecutor for MockExecutorProvider { diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 6ca9cec5260e5..4e5918831f764 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -10,7 +10,7 @@ use reth_evm::{ BatchExecutor, BlockExecutionError, BlockExecutionInput, BlockExecutionOutput, BlockExecutorProvider, BlockValidationError, Executor, ProviderError, }, - system_calls::SystemCaller, + system_calls::{NoopHook, OnStateHook, SystemCaller}, ConfigureEvm, }; use reth_execution_types::ExecutionOutcome; @@ -108,18 +108,23 @@ where /// /// This applies the pre-execution changes, and executes the transactions. /// + /// The optional `state_hook` will be executed with the state changes if present. + /// /// # Note /// /// It does __not__ apply post-execution changes. - fn execute_pre_and_transactions( + fn execute_pre_and_transactions( &self, block: &BlockWithSenders, mut evm: Evm<'_, Ext, &mut State>, + state_hook: Option, ) -> Result<(Vec, u64), BlockExecutionError> where DB: Database + Display>, + F: OnStateHook, { - let mut system_caller = SystemCaller::new(&self.evm_config, &self.chain_spec); + let mut system_caller = + SystemCaller::new(&self.evm_config, &self.chain_spec).with_state_hook(state_hook); // apply pre execution changes system_caller.apply_beacon_root_contract_call( @@ -178,7 +183,7 @@ where self.evm_config.fill_tx_env(evm.tx_mut(), transaction, *sender); // Execute transaction. - let ResultAndState { result, state } = evm.transact().map_err(move |err| { + let result_and_state = evm.transact().map_err(move |err| { let new_err = err.map_db_err(|e| e.into()); // Ensure hash is calculated for error log, if not already done BlockValidationError::EVM { @@ -192,7 +197,8 @@ where ?transaction, "Executed transaction" ); - + system_caller.on_state(&result_and_state); + let ResultAndState { result, state } = result_and_state; evm.db_mut().commit(state); // append gas used @@ -278,16 +284,30 @@ where EnvWithHandlerCfg::new_with_cfg_env(cfg, block_env, Default::default()) } + /// Convenience method to invoke `execute_without_verification_with_state_hook` setting the + /// state hook as `None`. + fn execute_without_verification( + &mut self, + block: &BlockWithSenders, + total_difficulty: U256, + ) -> Result<(Vec, u64), BlockExecutionError> { + self.execute_without_verification_with_state_hook(block, total_difficulty, None::) + } + /// Execute a single block and apply the state changes to the internal state. /// /// Returns the receipts of the transactions in the block and the total gas used. /// /// Returns an error if execution fails. - fn execute_without_verification( + fn execute_without_verification_with_state_hook( &mut self, block: &BlockWithSenders, total_difficulty: U256, - ) -> Result<(Vec, u64), BlockExecutionError> { + state_hook: Option, + ) -> Result<(Vec, u64), BlockExecutionError> + where + F: OnStateHook, + { // 1. prepare state on new block self.on_new_block(&block.header); @@ -296,7 +316,7 @@ where let (receipts, gas_used) = { let evm = self.executor.evm_config.evm_with_env(&mut self.state, env); - self.executor.execute_pre_and_transactions(block, evm) + self.executor.execute_pre_and_transactions(block, evm, state_hook) }?; // 3. apply post execution changes @@ -383,6 +403,32 @@ where gas_used, }) } + + fn execute_with_state_hook( + mut self, + input: Self::Input<'_>, + state_hook: F, + ) -> Result + where + F: OnStateHook, + { + let BlockExecutionInput { block, total_difficulty } = input; + let (receipts, gas_used) = self.execute_without_verification_with_state_hook( + block, + total_difficulty, + Some(state_hook), + )?; + + // NOTE: we need to merge keep the reverts for the bundle retention + self.state.merge_transitions(BundleRetention::Reverts); + + Ok(BlockExecutionOutput { + state: self.state.take_bundle(), + receipts, + requests: vec![], + gas_used, + }) + } } /// An executor for a batch of blocks. From 300d6329f1b73798f808702c0a1d1bc191b95c5b Mon Sep 17 00:00:00 2001 From: Varun Doshi <61531351+varun-doshi@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:44:50 +0530 Subject: [PATCH 009/159] feat: cleaned up prepare_call_env() (#11469) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 29 +++------------------- crates/rpc/rpc/src/debug.rs | 2 -- crates/rpc/rpc/src/trace.rs | 2 -- 3 files changed, 4 insertions(+), 29 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 6189b69492a26..4d12450a1c7fd 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -256,7 +256,6 @@ pub trait EthCall: Call + LoadPendingBlock { )?; let block = block.ok_or(EthApiError::HeaderNotFound(target_block))?; - let gas_limit = self.call_gas_limit(); // we're essentially replaying the transactions in the block here, hence we need the // state that points to the beginning of the block, which is the state at @@ -302,14 +301,7 @@ pub trait EthCall: Call + LoadPendingBlock { let overrides = EvmOverrides::new(state_overrides, block_overrides.clone()); let env = this - .prepare_call_env( - cfg.clone(), - block_env.clone(), - tx, - gas_limit, - &mut db, - overrides, - ) + .prepare_call_env(cfg.clone(), block_env.clone(), tx, &mut db, overrides) .map(Into::into)?; let (res, _) = this.transact(&mut db, env)?; @@ -559,14 +551,7 @@ pub trait Call: LoadState + SpawnBlocking { let mut db = CacheDB::new(StateProviderDatabase::new(StateProviderTraitObjWrapper(&state))); - let env = this.prepare_call_env( - cfg, - block_env, - request, - this.call_gas_limit(), - &mut db, - overrides, - )?; + let env = this.prepare_call_env(cfg, block_env, request, &mut db, overrides)?; f(StateCacheDbRefMutWrapper(&mut db), env) }) @@ -1099,7 +1084,6 @@ pub trait Call: LoadState + SpawnBlocking { mut cfg: CfgEnvWithHandlerCfg, mut block: BlockEnv, mut request: TransactionRequest, - gas_limit: u64, db: &mut CacheDB, overrides: EvmOverrides, ) -> Result @@ -1107,18 +1091,13 @@ pub trait Call: LoadState + SpawnBlocking { DB: DatabaseRef, EthApiError: From<::Error>, { - // TODO(mattsse): cleanup, by not disabling gaslimit and instead use self.call_gas_limit - if request.gas > Some(gas_limit) { + if request.gas > Some(self.call_gas_limit()) { // configured gas exceeds limit return Err( EthApiError::InvalidTransaction(RpcInvalidTransactionError::GasTooHigh).into() ) } - // we want to disable this in eth_call, since this is common practice used by other node - // impls and providers - cfg.disable_block_gas_limit = true; - // Disabled because eth_call is sometimes used with eoa senders // See cfg.disable_eip3607 = true; @@ -1154,7 +1133,7 @@ pub trait Call: LoadState + SpawnBlocking { // trace!(target: "rpc::eth::call", ?env, "Applying gas limit cap as the maximum gas limit"); - env.tx.gas_limit = gas_limit; + env.tx.gas_limit = self.call_gas_limit(); } } diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index b47473c342703..f4aba778f5729 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -506,7 +506,6 @@ where let opts = opts.unwrap_or_default(); let block = block.ok_or(EthApiError::HeaderNotFound(target_block))?; let GethDebugTracingCallOptions { tracing_options, mut state_overrides, .. } = opts; - let gas_limit = self.inner.eth_api.call_gas_limit(); // we're essentially replaying the transactions in the block here, hence we need the state // that points to the beginning of the block, which is the state at the parent block @@ -570,7 +569,6 @@ where cfg.clone(), block_env.clone(), tx, - gas_limit, &mut db, overrides, )?; diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 185383d811b74..e905ff685c863 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -153,7 +153,6 @@ where let at = block_id.unwrap_or(BlockId::pending()); let (cfg, block_env, at) = self.inner.eth_api.evm_env_at(at).await?; - let gas_limit = self.inner.eth_api.call_gas_limit(); let this = self.clone(); // execute all transactions on top of each other and record the traces self.eth_api() @@ -168,7 +167,6 @@ where cfg.clone(), block_env.clone(), call, - gas_limit, &mut db, Default::default(), )?; From 944bb941302550de0b9c964f1402108a0fa62f00 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Oct 2024 10:37:48 +0200 Subject: [PATCH 010/159] chore: use block.body directly (#11474) --- crates/exex/exex/src/backfill/job.rs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/crates/exex/exex/src/backfill/job.rs b/crates/exex/exex/src/backfill/job.rs index 7642edbac30ee..2db897270bfba 100644 --- a/crates/exex/exex/src/backfill/job.rs +++ b/crates/exex/exex/src/backfill/job.rs @@ -8,7 +8,7 @@ use alloy_primitives::BlockNumber; use reth_evm::execute::{ BatchExecutor, BlockExecutionError, BlockExecutionOutput, BlockExecutorProvider, Executor, }; -use reth_primitives::{Block, BlockBody, BlockWithSenders, Receipt}; +use reth_primitives::{Block, BlockWithSenders, Receipt}; use reth_primitives_traits::format_gas_throughput; use reth_provider::{ BlockReader, Chain, HeaderProvider, ProviderError, StateProviderFactory, TransactionVariant, @@ -103,16 +103,8 @@ where // Unseal the block for execution let (block, senders) = block.into_components(); let (unsealed_header, hash) = block.header.split(); - let block = Block { - header: unsealed_header, - body: BlockBody { - transactions: block.body.transactions, - ommers: block.body.ommers, - withdrawals: block.body.withdrawals, - requests: block.body.requests, - }, - } - .with_senders_unchecked(senders); + let block = + Block { header: unsealed_header, body: block.body }.with_senders_unchecked(senders); executor.execute_and_verify_one((&block, td).into())?; execution_duration += execute_start.elapsed(); From 5259b7148d1185cf6d513fc2688c6535b7a76c6b Mon Sep 17 00:00:00 2001 From: garwah <14845405+garwahl@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:12:31 +1000 Subject: [PATCH 011/159] feat: Add metrics to track transactions by type in txpool (#11403) Co-authored-by: garwah Co-authored-by: Matthias Seitz --- crates/transaction-pool/src/metrics.rs | 12 +++++++- crates/transaction-pool/src/pool/txpool.rs | 35 ++++++++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/crates/transaction-pool/src/metrics.rs b/crates/transaction-pool/src/metrics.rs index d323d47e459ac..f5d269b361fc0 100644 --- a/crates/transaction-pool/src/metrics.rs +++ b/crates/transaction-pool/src/metrics.rs @@ -36,8 +36,18 @@ pub struct TxPoolMetrics { /// Total amount of memory used by the transactions in the blob sub-pool in bytes pub(crate) blob_pool_size_bytes: Gauge, - /// Number of all transactions of all sub-pools: pending + basefee + queued + /// Number of all transactions of all sub-pools: pending + basefee + queued + blob pub(crate) total_transactions: Gauge, + /// Number of all legacy transactions in the pool + pub(crate) total_legacy_transactions: Gauge, + /// Number of all EIP-2930 transactions in the pool + pub(crate) total_eip2930_transactions: Gauge, + /// Number of all EIP-1559 transactions in the pool + pub(crate) total_eip1559_transactions: Gauge, + /// Number of all EIP-4844 transactions in the pool + pub(crate) total_eip4844_transactions: Gauge, + /// Number of all EIP-7702 transactions in the pool + pub(crate) total_eip7702_transactions: Gauge, /// How often the pool was updated after the canonical state changed pub(crate) performed_state_updates: Counter, diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 912e04506a19a..10605565c85d3 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -19,8 +19,12 @@ use crate::{ ValidPoolTransaction, U256, }; use alloy_primitives::{Address, TxHash, B256}; -use reth_primitives::constants::{ - eip4844::BLOB_TX_MIN_BLOB_GASPRICE, ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE, +use reth_primitives::{ + constants::{ + eip4844::BLOB_TX_MIN_BLOB_GASPRICE, ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE, + }, + EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, EIP7702_TX_TYPE_ID, + LEGACY_TX_TYPE_ID, }; use rustc_hash::FxHashMap; use smallvec::SmallVec; @@ -429,6 +433,7 @@ impl TxPool { let UpdateOutcome { promoted, discarded } = self.update_accounts(changed_senders); + self.update_transaction_type_metrics(); self.metrics.performed_state_updates.increment(1); OnNewCanonicalStateOutcome { block_hash, mined: mined_transactions, promoted, discarded } @@ -448,6 +453,32 @@ impl TxPool { self.metrics.total_transactions.set(stats.total as f64); } + /// Updates transaction type metrics for the entire pool. + pub(crate) fn update_transaction_type_metrics(&self) { + let mut legacy_count = 0; + let mut eip2930_count = 0; + let mut eip1559_count = 0; + let mut eip4844_count = 0; + let mut eip7702_count = 0; + + for tx in self.all_transactions.transactions_iter() { + match tx.transaction.tx_type() { + LEGACY_TX_TYPE_ID => legacy_count += 1, + EIP2930_TX_TYPE_ID => eip2930_count += 1, + EIP1559_TX_TYPE_ID => eip1559_count += 1, + EIP4844_TX_TYPE_ID => eip4844_count += 1, + EIP7702_TX_TYPE_ID => eip7702_count += 1, + _ => {} // Ignore other types + } + } + + self.metrics.total_legacy_transactions.set(legacy_count as f64); + self.metrics.total_eip2930_transactions.set(eip2930_count as f64); + self.metrics.total_eip1559_transactions.set(eip1559_count as f64); + self.metrics.total_eip4844_transactions.set(eip4844_count as f64); + self.metrics.total_eip7702_transactions.set(eip7702_count as f64); + } + /// Adds the transaction into the pool. /// /// This pool consists of four sub-pools: `Queued`, `Pending`, `BaseFee`, and `Blob`. From db7a630b1d8f22e9df8893f97c6a200bdf652c66 Mon Sep 17 00:00:00 2001 From: greged93 <82421016+greged93@users.noreply.github.com> Date: Fri, 4 Oct 2024 12:06:06 +0200 Subject: [PATCH 012/159] chore: op chainspec (#11415) --- Cargo.lock | 2 - crates/chainspec/Cargo.toml | 9 +- crates/chainspec/src/spec.rs | 407 +++++---------------- crates/optimism/chainspec/Cargo.toml | 5 +- crates/optimism/chainspec/src/constants.rs | 10 + crates/optimism/chainspec/src/lib.rs | 254 ++++++++++++- 6 files changed, 338 insertions(+), 349 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ceeb53a2e4a77..8c66fbf09b38d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6508,13 +6508,11 @@ dependencies = [ "auto_impl", "derive_more 1.0.0", "once_cell", - "op-alloy-rpc-types", "reth-ethereum-forks", "reth-network-peers", "reth-optimism-forks", "reth-primitives-traits", "reth-trie-common", - "serde", "serde_json", ] diff --git a/crates/chainspec/Cargo.toml b/crates/chainspec/Cargo.toml index de04ffd1e6aab..80636d139a1b5 100644 --- a/crates/chainspec/Cargo.toml +++ b/crates/chainspec/Cargo.toml @@ -27,13 +27,9 @@ alloy-genesis.workspace = true alloy-primitives = { workspace = true, features = ["rand", "rlp"] } alloy-trie.workspace = true -# op -op-alloy-rpc-types = { workspace = true, optional = true } - # misc auto_impl.workspace = true once_cell.workspace = true -serde = { workspace = true, optional = true } serde_json.workspace = true derive_more.workspace = true @@ -44,12 +40,9 @@ alloy-eips = { workspace = true, features = ["arbitrary"] } alloy-rlp = { workspace = true, features = ["arrayvec"] } alloy-genesis.workspace = true -# op -op-alloy-rpc-types.workspace = true - [features] default = ["std"] -optimism = ["serde", "dep:op-alloy-rpc-types", "reth-optimism-forks"] +optimism = ["reth-optimism-forks"] std = [ "alloy-chains/std", "alloy-eips/std", diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index 45070db197d44..d6ca92aa24ebe 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -590,14 +590,89 @@ impl ChainSpec { impl From for ChainSpec { fn from(genesis: Genesis) -> Self { - #[cfg(not(feature = "optimism"))] - { - into_ethereum_chain_spec(genesis) + // Block-based hardforks + let hardfork_opts = [ + (EthereumHardfork::Homestead.boxed(), genesis.config.homestead_block), + (EthereumHardfork::Dao.boxed(), genesis.config.dao_fork_block), + (EthereumHardfork::Tangerine.boxed(), genesis.config.eip150_block), + (EthereumHardfork::SpuriousDragon.boxed(), genesis.config.eip155_block), + (EthereumHardfork::Byzantium.boxed(), genesis.config.byzantium_block), + (EthereumHardfork::Constantinople.boxed(), genesis.config.constantinople_block), + (EthereumHardfork::Petersburg.boxed(), genesis.config.petersburg_block), + (EthereumHardfork::Istanbul.boxed(), genesis.config.istanbul_block), + (EthereumHardfork::MuirGlacier.boxed(), genesis.config.muir_glacier_block), + (EthereumHardfork::Berlin.boxed(), genesis.config.berlin_block), + (EthereumHardfork::London.boxed(), genesis.config.london_block), + (EthereumHardfork::ArrowGlacier.boxed(), genesis.config.arrow_glacier_block), + (EthereumHardfork::GrayGlacier.boxed(), genesis.config.gray_glacier_block), + ]; + let mut hardforks = hardfork_opts + .into_iter() + .filter_map(|(hardfork, opt)| opt.map(|block| (hardfork, ForkCondition::Block(block)))) + .collect::>(); + + // Paris + let paris_block_and_final_difficulty = + if let Some(ttd) = genesis.config.terminal_total_difficulty { + hardforks.push(( + EthereumHardfork::Paris.boxed(), + ForkCondition::TTD { + total_difficulty: ttd, + fork_block: genesis.config.merge_netsplit_block, + }, + )); + + genesis.config.merge_netsplit_block.map(|block| (block, ttd)) + } else { + None + }; + + // Time-based hardforks + let time_hardfork_opts = [ + (EthereumHardfork::Shanghai.boxed(), genesis.config.shanghai_time), + (EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time), + (EthereumHardfork::Prague.boxed(), genesis.config.prague_time), + ]; + + let mut time_hardforks = time_hardfork_opts + .into_iter() + .filter_map(|(hardfork, opt)| { + opt.map(|time| (hardfork, ForkCondition::Timestamp(time))) + }) + .collect::>(); + + hardforks.append(&mut time_hardforks); + + // Ordered Hardforks + let mainnet_hardforks: ChainHardforks = EthereumHardfork::mainnet().into(); + let mainnet_order = mainnet_hardforks.forks_iter(); + + let mut ordered_hardforks = Vec::with_capacity(hardforks.len()); + for (hardfork, _) in mainnet_order { + if let Some(pos) = hardforks.iter().position(|(e, _)| **e == *hardfork) { + ordered_hardforks.push(hardforks.remove(pos)); + } } - #[cfg(feature = "optimism")] - { - into_optimism_chain_spec(genesis) + // append the remaining unknown hardforks to ensure we don't filter any out + ordered_hardforks.append(&mut hardforks); + + // NOTE: in full node, we prune all receipts except the deposit contract's. We do not + // have the deployment block in the genesis file, so we use block zero. We use the same + // deposit topic as the mainnet contract if we have the deposit contract address in the + // genesis json. + let deposit_contract = genesis.config.deposit_contract_address.map(|address| { + DepositContract { address, block: 0, topic: MAINNET_DEPOSIT_CONTRACT.topic } + }); + + Self { + chain: genesis.config.chain_id.into(), + genesis, + genesis_hash: OnceCell::new(), + hardforks: ChainHardforks::new(ordered_hardforks), + paris_block_and_final_difficulty, + deposit_contract, + ..Default::default() } } } @@ -637,194 +712,6 @@ impl EthereumHardforks for ChainSpec { #[cfg(feature = "optimism")] impl reth_optimism_forks::OptimismHardforks for ChainSpec {} -/// Convert the given [`Genesis`] into an Ethereum [`ChainSpec`]. -#[cfg(not(feature = "optimism"))] -fn into_ethereum_chain_spec(genesis: Genesis) -> ChainSpec { - // Block-based hardforks - let hardfork_opts = [ - (EthereumHardfork::Homestead.boxed(), genesis.config.homestead_block), - (EthereumHardfork::Dao.boxed(), genesis.config.dao_fork_block), - (EthereumHardfork::Tangerine.boxed(), genesis.config.eip150_block), - (EthereumHardfork::SpuriousDragon.boxed(), genesis.config.eip155_block), - (EthereumHardfork::Byzantium.boxed(), genesis.config.byzantium_block), - (EthereumHardfork::Constantinople.boxed(), genesis.config.constantinople_block), - (EthereumHardfork::Petersburg.boxed(), genesis.config.petersburg_block), - (EthereumHardfork::Istanbul.boxed(), genesis.config.istanbul_block), - (EthereumHardfork::MuirGlacier.boxed(), genesis.config.muir_glacier_block), - (EthereumHardfork::Berlin.boxed(), genesis.config.berlin_block), - (EthereumHardfork::London.boxed(), genesis.config.london_block), - (EthereumHardfork::ArrowGlacier.boxed(), genesis.config.arrow_glacier_block), - (EthereumHardfork::GrayGlacier.boxed(), genesis.config.gray_glacier_block), - ]; - let mut hardforks = hardfork_opts - .into_iter() - .filter_map(|(hardfork, opt)| opt.map(|block| (hardfork, ForkCondition::Block(block)))) - .collect::>(); - - // Paris - let paris_block_and_final_difficulty = - if let Some(ttd) = genesis.config.terminal_total_difficulty { - hardforks.push(( - EthereumHardfork::Paris.boxed(), - ForkCondition::TTD { - total_difficulty: ttd, - fork_block: genesis.config.merge_netsplit_block, - }, - )); - - genesis.config.merge_netsplit_block.map(|block| (block, ttd)) - } else { - None - }; - - // Time-based hardforks - let time_hardfork_opts = [ - (EthereumHardfork::Shanghai.boxed(), genesis.config.shanghai_time), - (EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time), - (EthereumHardfork::Prague.boxed(), genesis.config.prague_time), - ]; - - let mut time_hardforks = time_hardfork_opts - .into_iter() - .filter_map(|(hardfork, opt)| opt.map(|time| (hardfork, ForkCondition::Timestamp(time)))) - .collect::>(); - - hardforks.append(&mut time_hardforks); - - // Ordered Hardforks - let mainnet_hardforks: ChainHardforks = EthereumHardfork::mainnet().into(); - let mainnet_order = mainnet_hardforks.forks_iter(); - - let mut ordered_hardforks = Vec::with_capacity(hardforks.len()); - for (hardfork, _) in mainnet_order { - if let Some(pos) = hardforks.iter().position(|(e, _)| **e == *hardfork) { - ordered_hardforks.push(hardforks.remove(pos)); - } - } - - // append the remaining unknown hardforks to ensure we don't filter any out - ordered_hardforks.append(&mut hardforks); - - // NOTE: in full node, we prune all receipts except the deposit contract's. We do not - // have the deployment block in the genesis file, so we use block zero. We use the same - // deposit topic as the mainnet contract if we have the deposit contract address in the - // genesis json. - let deposit_contract = genesis.config.deposit_contract_address.map(|address| DepositContract { - address, - block: 0, - topic: MAINNET_DEPOSIT_CONTRACT.topic, - }); - - ChainSpec { - chain: genesis.config.chain_id.into(), - genesis, - genesis_hash: OnceCell::new(), - hardforks: ChainHardforks::new(ordered_hardforks), - paris_block_and_final_difficulty, - deposit_contract, - ..Default::default() - } -} - -#[cfg(feature = "optimism")] -/// Convert the given [`Genesis`] into an Optimism [`ChainSpec`]. -fn into_optimism_chain_spec(genesis: Genesis) -> ChainSpec { - use reth_optimism_forks::OptimismHardfork; - let optimism_genesis_info = OptimismGenesisInfo::extract_from(&genesis); - let genesis_info = optimism_genesis_info.optimism_chain_info.genesis_info.unwrap_or_default(); - - // Block-based hardforks - let hardfork_opts = [ - (EthereumHardfork::Homestead.boxed(), genesis.config.homestead_block), - (EthereumHardfork::Tangerine.boxed(), genesis.config.eip150_block), - (EthereumHardfork::SpuriousDragon.boxed(), genesis.config.eip155_block), - (EthereumHardfork::Byzantium.boxed(), genesis.config.byzantium_block), - (EthereumHardfork::Constantinople.boxed(), genesis.config.constantinople_block), - (EthereumHardfork::Petersburg.boxed(), genesis.config.petersburg_block), - (EthereumHardfork::Istanbul.boxed(), genesis.config.istanbul_block), - (EthereumHardfork::MuirGlacier.boxed(), genesis.config.muir_glacier_block), - (EthereumHardfork::Berlin.boxed(), genesis.config.berlin_block), - (EthereumHardfork::London.boxed(), genesis.config.london_block), - (EthereumHardfork::ArrowGlacier.boxed(), genesis.config.arrow_glacier_block), - (EthereumHardfork::GrayGlacier.boxed(), genesis.config.gray_glacier_block), - (OptimismHardfork::Bedrock.boxed(), genesis_info.bedrock_block), - ]; - let mut block_hardforks = hardfork_opts - .into_iter() - .filter_map(|(hardfork, opt)| opt.map(|block| (hardfork, ForkCondition::Block(block)))) - .collect::>(); - - // Paris - let paris_block_and_final_difficulty = - if let Some(ttd) = genesis.config.terminal_total_difficulty { - block_hardforks.push(( - EthereumHardfork::Paris.boxed(), - ForkCondition::TTD { - total_difficulty: ttd, - fork_block: genesis.config.merge_netsplit_block, - }, - )); - - genesis.config.merge_netsplit_block.map(|block| (block, ttd)) - } else { - None - }; - - // Time-based hardforks - let time_hardfork_opts = [ - (EthereumHardfork::Shanghai.boxed(), genesis.config.shanghai_time), - (EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time), - (EthereumHardfork::Prague.boxed(), genesis.config.prague_time), - (OptimismHardfork::Regolith.boxed(), genesis_info.regolith_time), - (OptimismHardfork::Canyon.boxed(), genesis_info.canyon_time), - (OptimismHardfork::Ecotone.boxed(), genesis_info.ecotone_time), - (OptimismHardfork::Fjord.boxed(), genesis_info.fjord_time), - (OptimismHardfork::Granite.boxed(), genesis_info.granite_time), - ]; - - let mut time_hardforks = time_hardfork_opts - .into_iter() - .filter_map(|(hardfork, opt)| opt.map(|time| (hardfork, ForkCondition::Timestamp(time)))) - .collect::>(); - - block_hardforks.append(&mut time_hardforks); - - // Ordered Hardforks - let mainnet_hardforks = OptimismHardfork::op_mainnet(); - let mainnet_order = mainnet_hardforks.forks_iter(); - - let mut ordered_hardforks = Vec::with_capacity(block_hardforks.len()); - for (hardfork, _) in mainnet_order { - if let Some(pos) = block_hardforks.iter().position(|(e, _)| **e == *hardfork) { - ordered_hardforks.push(block_hardforks.remove(pos)); - } - } - - // append the remaining unknown hardforks to ensure we don't filter any out - ordered_hardforks.append(&mut block_hardforks); - - // NOTE: in full node, we prune all receipts except the deposit contract's. We do not - // have the deployment block in the genesis file, so we use block zero. We use the same - // deposit topic as the mainnet contract if we have the deposit contract address in the - // genesis json. - let deposit_contract = genesis.config.deposit_contract_address.map(|address| DepositContract { - address, - block: 0, - topic: MAINNET_DEPOSIT_CONTRACT.topic, - }); - - ChainSpec { - chain: genesis.config.chain_id.into(), - genesis, - genesis_hash: OnceCell::new(), - hardforks: ChainHardforks::new(ordered_hardforks), - paris_block_and_final_difficulty, - deposit_contract, - base_fee_params: optimism_genesis_info.base_fee_params, - ..Default::default() - } -} - /// A trait for reading the current chainspec. #[auto_impl::auto_impl(&, Arc)] pub trait ChainSpecProvider: Send + Sync { @@ -1102,59 +989,6 @@ impl DepositContract { } } -/// Genesis info for Optimism. -#[cfg(feature = "optimism")] -#[derive(Default, Debug, serde::Deserialize)] -#[serde(rename_all = "camelCase")] -struct OptimismGenesisInfo { - optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo, - #[serde(skip)] - base_fee_params: BaseFeeParamsKind, -} - -#[cfg(feature = "optimism")] -impl OptimismGenesisInfo { - fn extract_from(genesis: &Genesis) -> Self { - let mut info = Self { - optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo::extract_from( - &genesis.config.extra_fields, - ) - .unwrap_or_default(), - ..Default::default() - }; - if let Some(optimism_base_fee_info) = &info.optimism_chain_info.base_fee_info { - if let (Some(elasticity), Some(denominator)) = ( - optimism_base_fee_info.eip1559_elasticity, - optimism_base_fee_info.eip1559_denominator, - ) { - let base_fee_params = if let Some(canyon_denominator) = - optimism_base_fee_info.eip1559_denominator_canyon - { - BaseFeeParamsKind::Variable( - vec![ - ( - EthereumHardfork::London.boxed(), - BaseFeeParams::new(denominator as u128, elasticity as u128), - ), - ( - reth_optimism_forks::OptimismHardfork::Canyon.boxed(), - BaseFeeParams::new(canyon_denominator as u128, elasticity as u128), - ), - ] - .into(), - ) - } else { - BaseFeeParams::new(denominator as u128, elasticity as u128).into() - }; - - info.base_fee_params = base_fee_params; - } - } - - info - } -} - /// Verifies [`ChainSpec`] configuration against expected data in given cases. #[cfg(any(test, feature = "test-utils"))] pub fn test_fork_ids(spec: &ChainSpec, cases: &[(Head, ForkId)]) { @@ -2477,7 +2311,6 @@ Post-merge hard forks (timestamp based): } #[test] - #[cfg(not(feature = "optimism"))] fn test_fork_order_ethereum_mainnet() { let genesis = Genesis { config: ChainConfig { @@ -2506,7 +2339,7 @@ Post-merge hard forks (timestamp based): ..Default::default() }; - let chain_spec = into_ethereum_chain_spec(genesis); + let chain_spec: ChainSpec = genesis.into(); let hardforks: Vec<_> = chain_spec.hardforks.forks_iter().map(|(h, _)| h).collect(); let expected_hardforks = vec![ @@ -2534,80 +2367,4 @@ Post-merge hard forks (timestamp based): .all(|(expected, actual)| &**expected == *actual)); assert_eq!(expected_hardforks.len(), hardforks.len()); } - - #[test] - #[cfg(feature = "optimism")] - fn test_fork_order_optimism_mainnet() { - use reth_optimism_forks::OptimismHardfork; - - let genesis = Genesis { - config: ChainConfig { - chain_id: 0, - homestead_block: Some(0), - dao_fork_block: Some(0), - dao_fork_support: false, - eip150_block: Some(0), - eip155_block: Some(0), - eip158_block: Some(0), - byzantium_block: Some(0), - constantinople_block: Some(0), - petersburg_block: Some(0), - istanbul_block: Some(0), - muir_glacier_block: Some(0), - berlin_block: Some(0), - london_block: Some(0), - arrow_glacier_block: Some(0), - gray_glacier_block: Some(0), - merge_netsplit_block: Some(0), - shanghai_time: Some(0), - cancun_time: Some(0), - terminal_total_difficulty: Some(U256::ZERO), - extra_fields: [ - (String::from("bedrockBlock"), 0.into()), - (String::from("regolithTime"), 0.into()), - (String::from("canyonTime"), 0.into()), - (String::from("ecotoneTime"), 0.into()), - (String::from("fjordTime"), 0.into()), - (String::from("graniteTime"), 0.into()), - ] - .into_iter() - .collect(), - ..Default::default() - }, - ..Default::default() - }; - - let chain_spec: ChainSpec = into_optimism_chain_spec(genesis); - - let hardforks: Vec<_> = chain_spec.hardforks.forks_iter().map(|(h, _)| h).collect(); - let expected_hardforks = vec![ - EthereumHardfork::Homestead.boxed(), - EthereumHardfork::Tangerine.boxed(), - EthereumHardfork::SpuriousDragon.boxed(), - EthereumHardfork::Byzantium.boxed(), - EthereumHardfork::Constantinople.boxed(), - EthereumHardfork::Petersburg.boxed(), - EthereumHardfork::Istanbul.boxed(), - EthereumHardfork::MuirGlacier.boxed(), - EthereumHardfork::Berlin.boxed(), - EthereumHardfork::London.boxed(), - EthereumHardfork::ArrowGlacier.boxed(), - EthereumHardfork::GrayGlacier.boxed(), - EthereumHardfork::Paris.boxed(), - OptimismHardfork::Bedrock.boxed(), - OptimismHardfork::Regolith.boxed(), - EthereumHardfork::Shanghai.boxed(), - OptimismHardfork::Canyon.boxed(), - EthereumHardfork::Cancun.boxed(), - OptimismHardfork::Ecotone.boxed(), - OptimismHardfork::Fjord.boxed(), - OptimismHardfork::Granite.boxed(), - ]; - - assert!(expected_hardforks - .iter() - .zip(hardforks.iter()) - .all(|(expected, actual)| &**expected == *actual)); - assert_eq!(expected_hardforks.len(), hardforks.len()); - } } diff --git a/crates/optimism/chainspec/Cargo.toml b/crates/optimism/chainspec/Cargo.toml index 41b54902c201b..e13b95056c7fa 100644 --- a/crates/optimism/chainspec/Cargo.toml +++ b/crates/optimism/chainspec/Cargo.toml @@ -26,12 +26,15 @@ alloy-chains.workspace = true alloy-genesis.workspace = true alloy-primitives.workspace = true +# op +op-alloy-rpc-types.workspace = true + # io serde_json.workspace = true # misc -once_cell.workspace = true derive_more.workspace = true +once_cell.workspace = true [dev-dependencies] reth-chainspec = { workspace = true, features = ["test-utils"] } diff --git a/crates/optimism/chainspec/src/constants.rs b/crates/optimism/chainspec/src/constants.rs index 439c2fed83c0e..8fce63daafef5 100644 --- a/crates/optimism/chainspec/src/constants.rs +++ b/crates/optimism/chainspec/src/constants.rs @@ -1,5 +1,8 @@ //! OP stack variation of chain spec constants. +use alloy_primitives::{address, b256}; +use reth_chainspec::DepositContract; + //------------------------------- BASE MAINNET -------------------------------// /// Max gas limit on Base: @@ -9,3 +12,10 @@ pub const BASE_MAINNET_MAX_GAS_LIMIT: u64 = 105_000_000; /// Max gas limit on Base Sepolia: pub const BASE_SEPOLIA_MAX_GAS_LIMIT: u64 = 45_000_000; + +/// Deposit contract address: `0x00000000219ab540356cbb839cbe05303d7705fa` +pub(crate) const MAINNET_DEPOSIT_CONTRACT: DepositContract = DepositContract::new( + address!("00000000219ab540356cbb839cbe05303d7705fa"), + 11052984, + b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"), +); diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 2668c374500df..76a3ed838b187 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -26,11 +26,14 @@ pub use dev::OP_DEV; pub use op::OP_MAINNET; pub use op_sepolia::OP_SEPOLIA; +use crate::constants::MAINNET_DEPOSIT_CONTRACT; use derive_more::{Constructor, Deref, Into}; +use once_cell::sync::OnceCell; use reth_chainspec::{ - BaseFeeParams, ChainSpec, DepositContract, EthChainSpec, EthereumHardforks, ForkFilter, ForkId, - Hardforks, Head, + BaseFeeParams, BaseFeeParamsKind, ChainSpec, DepositContract, EthChainSpec, EthereumHardforks, + ForkFilter, ForkId, Hardforks, Head, }; +use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition}; use reth_network_peers::NodeRecord; use reth_primitives_traits::Header; @@ -52,14 +55,14 @@ impl EthChainSpec for OpChainSpec { self.inner.chain() } - fn base_fee_params_at_timestamp(&self, timestamp: u64) -> BaseFeeParams { - self.inner.base_fee_params_at_timestamp(timestamp) - } - fn base_fee_params_at_block(&self, block_number: u64) -> BaseFeeParams { self.inner.base_fee_params_at_block(block_number) } + fn base_fee_params_at_timestamp(&self, timestamp: u64) -> BaseFeeParams { + self.inner.base_fee_params_at_timestamp(timestamp) + } + fn deposit_contract(&self) -> Option<&DepositContract> { self.inner.deposit_contract() } @@ -118,20 +121,170 @@ impl Hardforks for OpChainSpec { } impl EthereumHardforks for OpChainSpec { + fn get_final_paris_total_difficulty(&self) -> Option { + self.inner.get_final_paris_total_difficulty() + } + fn final_paris_total_difficulty(&self, block_number: u64) -> Option { self.inner.final_paris_total_difficulty(block_number) } +} - fn get_final_paris_total_difficulty(&self) -> Option { - self.inner.get_final_paris_total_difficulty() +impl From for OpChainSpec { + fn from(genesis: Genesis) -> Self { + use reth_optimism_forks::OptimismHardfork; + let optimism_genesis_info = OptimismGenesisInfo::extract_from(&genesis); + let genesis_info = + optimism_genesis_info.optimism_chain_info.genesis_info.unwrap_or_default(); + + // Block-based hardforks + let hardfork_opts = [ + (EthereumHardfork::Homestead.boxed(), genesis.config.homestead_block), + (EthereumHardfork::Tangerine.boxed(), genesis.config.eip150_block), + (EthereumHardfork::SpuriousDragon.boxed(), genesis.config.eip155_block), + (EthereumHardfork::Byzantium.boxed(), genesis.config.byzantium_block), + (EthereumHardfork::Constantinople.boxed(), genesis.config.constantinople_block), + (EthereumHardfork::Petersburg.boxed(), genesis.config.petersburg_block), + (EthereumHardfork::Istanbul.boxed(), genesis.config.istanbul_block), + (EthereumHardfork::MuirGlacier.boxed(), genesis.config.muir_glacier_block), + (EthereumHardfork::Berlin.boxed(), genesis.config.berlin_block), + (EthereumHardfork::London.boxed(), genesis.config.london_block), + (EthereumHardfork::ArrowGlacier.boxed(), genesis.config.arrow_glacier_block), + (EthereumHardfork::GrayGlacier.boxed(), genesis.config.gray_glacier_block), + (OptimismHardfork::Bedrock.boxed(), genesis_info.bedrock_block), + ]; + let mut block_hardforks = hardfork_opts + .into_iter() + .filter_map(|(hardfork, opt)| opt.map(|block| (hardfork, ForkCondition::Block(block)))) + .collect::>(); + + // Paris + let paris_block_and_final_difficulty = + if let Some(ttd) = genesis.config.terminal_total_difficulty { + block_hardforks.push(( + EthereumHardfork::Paris.boxed(), + ForkCondition::TTD { + total_difficulty: ttd, + fork_block: genesis.config.merge_netsplit_block, + }, + )); + + genesis.config.merge_netsplit_block.map(|block| (block, ttd)) + } else { + None + }; + + // Time-based hardforks + let time_hardfork_opts = [ + (EthereumHardfork::Shanghai.boxed(), genesis.config.shanghai_time), + (EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time), + (EthereumHardfork::Prague.boxed(), genesis.config.prague_time), + (OptimismHardfork::Regolith.boxed(), genesis_info.regolith_time), + (OptimismHardfork::Canyon.boxed(), genesis_info.canyon_time), + (OptimismHardfork::Ecotone.boxed(), genesis_info.ecotone_time), + (OptimismHardfork::Fjord.boxed(), genesis_info.fjord_time), + (OptimismHardfork::Granite.boxed(), genesis_info.granite_time), + ]; + + let mut time_hardforks = time_hardfork_opts + .into_iter() + .filter_map(|(hardfork, opt)| { + opt.map(|time| (hardfork, ForkCondition::Timestamp(time))) + }) + .collect::>(); + + block_hardforks.append(&mut time_hardforks); + + // Ordered Hardforks + let mainnet_hardforks = OptimismHardfork::op_mainnet(); + let mainnet_order = mainnet_hardforks.forks_iter(); + + let mut ordered_hardforks = Vec::with_capacity(block_hardforks.len()); + for (hardfork, _) in mainnet_order { + if let Some(pos) = block_hardforks.iter().position(|(e, _)| **e == *hardfork) { + ordered_hardforks.push(block_hardforks.remove(pos)); + } + } + + // append the remaining unknown hardforks to ensure we don't filter any out + ordered_hardforks.append(&mut block_hardforks); + + // NOTE: in full node, we prune all receipts except the deposit contract's. We do not + // have the deployment block in the genesis file, so we use block zero. We use the same + // deposit topic as the mainnet contract if we have the deposit contract address in the + // genesis json. + let deposit_contract = genesis.config.deposit_contract_address.map(|address| { + DepositContract { address, block: 0, topic: MAINNET_DEPOSIT_CONTRACT.topic } + }); + + Self { + inner: ChainSpec { + chain: genesis.config.chain_id.into(), + genesis, + genesis_hash: OnceCell::new(), + hardforks: ChainHardforks::new(ordered_hardforks), + paris_block_and_final_difficulty, + deposit_contract, + base_fee_params: optimism_genesis_info.base_fee_params, + ..Default::default() + }, + } + } +} + +#[derive(Default, Debug)] +struct OptimismGenesisInfo { + optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo, + base_fee_params: BaseFeeParamsKind, +} + +impl OptimismGenesisInfo { + fn extract_from(genesis: &Genesis) -> Self { + let mut info = Self { + optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo::extract_from( + &genesis.config.extra_fields, + ) + .unwrap_or_default(), + ..Default::default() + }; + if let Some(optimism_base_fee_info) = &info.optimism_chain_info.base_fee_info { + if let (Some(elasticity), Some(denominator)) = ( + optimism_base_fee_info.eip1559_elasticity, + optimism_base_fee_info.eip1559_denominator, + ) { + let base_fee_params = if let Some(canyon_denominator) = + optimism_base_fee_info.eip1559_denominator_canyon + { + BaseFeeParamsKind::Variable( + vec![ + ( + EthereumHardfork::London.boxed(), + BaseFeeParams::new(denominator as u128, elasticity as u128), + ), + ( + reth_optimism_forks::OptimismHardfork::Canyon.boxed(), + BaseFeeParams::new(canyon_denominator as u128, elasticity as u128), + ), + ] + .into(), + ) + } else { + BaseFeeParams::new(denominator as u128, elasticity as u128).into() + }; + + info.base_fee_params = base_fee_params; + } + } + + info } } #[cfg(test)] mod tests { - use alloy_genesis::Genesis; + use alloy_genesis::{ChainConfig, Genesis}; use alloy_primitives::b256; - use reth_chainspec::{test_fork_ids, BaseFeeParams, BaseFeeParamsKind, ChainSpec}; + use reth_chainspec::{test_fork_ids, BaseFeeParams, BaseFeeParamsKind}; use reth_ethereum_forks::{EthereumHardfork, ForkCondition, ForkHash, ForkId, Head}; use reth_optimism_forks::{OptimismHardfork, OptimismHardforks}; @@ -383,7 +536,7 @@ mod tests { }) ); - let chain_spec: ChainSpec = genesis.into(); + let chain_spec: OpChainSpec = genesis.into(); assert_eq!( chain_spec.base_fee_params, @@ -449,7 +602,7 @@ mod tests { }) ); - let chain_spec: ChainSpec = genesis.into(); + let chain_spec: OpChainSpec = genesis.into(); assert_eq!( chain_spec.base_fee_params, @@ -511,7 +664,7 @@ mod tests { } "#; let genesis: Genesis = serde_json::from_str(geth_genesis).unwrap(); - let chainspec = ChainSpec::from(genesis.clone()); + let chainspec = OpChainSpec::from(genesis.clone()); let actual_chain_id = genesis.config.chain_id; assert_eq!(actual_chain_id, 8453); @@ -552,4 +705,79 @@ mod tests { assert!(chainspec.is_fork_active_at_timestamp(OptimismHardfork::Regolith, 20)); } + + #[test] + fn test_fork_order_optimism_mainnet() { + use reth_optimism_forks::OptimismHardfork; + + let genesis = Genesis { + config: ChainConfig { + chain_id: 0, + homestead_block: Some(0), + dao_fork_block: Some(0), + dao_fork_support: false, + eip150_block: Some(0), + eip155_block: Some(0), + eip158_block: Some(0), + byzantium_block: Some(0), + constantinople_block: Some(0), + petersburg_block: Some(0), + istanbul_block: Some(0), + muir_glacier_block: Some(0), + berlin_block: Some(0), + london_block: Some(0), + arrow_glacier_block: Some(0), + gray_glacier_block: Some(0), + merge_netsplit_block: Some(0), + shanghai_time: Some(0), + cancun_time: Some(0), + terminal_total_difficulty: Some(U256::ZERO), + extra_fields: [ + (String::from("bedrockBlock"), 0.into()), + (String::from("regolithTime"), 0.into()), + (String::from("canyonTime"), 0.into()), + (String::from("ecotoneTime"), 0.into()), + (String::from("fjordTime"), 0.into()), + (String::from("graniteTime"), 0.into()), + ] + .into_iter() + .collect(), + ..Default::default() + }, + ..Default::default() + }; + + let chain_spec: OpChainSpec = genesis.into(); + + let hardforks: Vec<_> = chain_spec.hardforks.forks_iter().map(|(h, _)| h).collect(); + let expected_hardforks = vec![ + EthereumHardfork::Homestead.boxed(), + EthereumHardfork::Tangerine.boxed(), + EthereumHardfork::SpuriousDragon.boxed(), + EthereumHardfork::Byzantium.boxed(), + EthereumHardfork::Constantinople.boxed(), + EthereumHardfork::Petersburg.boxed(), + EthereumHardfork::Istanbul.boxed(), + EthereumHardfork::MuirGlacier.boxed(), + EthereumHardfork::Berlin.boxed(), + EthereumHardfork::London.boxed(), + EthereumHardfork::ArrowGlacier.boxed(), + EthereumHardfork::GrayGlacier.boxed(), + EthereumHardfork::Paris.boxed(), + OptimismHardfork::Bedrock.boxed(), + OptimismHardfork::Regolith.boxed(), + EthereumHardfork::Shanghai.boxed(), + OptimismHardfork::Canyon.boxed(), + EthereumHardfork::Cancun.boxed(), + OptimismHardfork::Ecotone.boxed(), + OptimismHardfork::Fjord.boxed(), + OptimismHardfork::Granite.boxed(), + ]; + + assert!(expected_hardforks + .iter() + .zip(hardforks.iter()) + .all(|(expected, actual)| &**expected == *actual)); + assert_eq!(expected_hardforks.len(), hardforks.len()); + } } From 1915bbdb89ee05ce6262eed768004cc64d543f81 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 4 Oct 2024 13:08:04 +0300 Subject: [PATCH 013/159] chore(exex): more backfill debug logs (#11476) --- crates/exex/exex/src/backfill/job.rs | 6 ++++++ crates/exex/exex/src/backfill/stream.rs | 2 ++ 2 files changed, 8 insertions(+) diff --git a/crates/exex/exex/src/backfill/job.rs b/crates/exex/exex/src/backfill/job.rs index 2db897270bfba..7d66ce23d6098 100644 --- a/crates/exex/exex/src/backfill/job.rs +++ b/crates/exex/exex/src/backfill/job.rs @@ -64,6 +64,12 @@ where } fn execute_range(&mut self) -> Result { + debug!( + target: "exex::backfill", + range = ?self.range, + "Executing block range" + ); + let mut executor = self.executor.batch_executor(StateProviderDatabase::new( self.provider.history_by_block_number(self.range.start().saturating_sub(1))?, )); diff --git a/crates/exex/exex/src/backfill/stream.rs b/crates/exex/exex/src/backfill/stream.rs index 07b710b7e389d..75a1e17602cf3 100644 --- a/crates/exex/exex/src/backfill/stream.rs +++ b/crates/exex/exex/src/backfill/stream.rs @@ -15,6 +15,7 @@ use reth_primitives::{BlockWithSenders, Receipt}; use reth_provider::{BlockReader, Chain, HeaderProvider, StateProviderFactory}; use reth_prune_types::PruneModes; use reth_stages_api::ExecutionStageThresholds; +use reth_tracing::tracing::debug; use tokio::task::JoinHandle; /// The default parallelism for active tasks in [`StreamBackfillJob`]. @@ -118,6 +119,7 @@ where // If we have range bounds, then we can spawn a new task for that range if let Some((first, last)) = range_bounds { let range = first..=last; + debug!(target: "exex::backfill", tasks = %this.tasks.len(), ?range, "Spawning new backfill task"); let mut job = BackfillJob { executor: this.executor.clone(), provider: this.provider.clone(), From 6958114c487748966cd4306a80db94380a67897e Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 4 Oct 2024 14:05:57 +0300 Subject: [PATCH 014/159] fix(exex): use thresholds in stream backfill (#11478) --- crates/exex/exex/src/backfill/stream.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/exex/exex/src/backfill/stream.rs b/crates/exex/exex/src/backfill/stream.rs index 75a1e17602cf3..ca4bd326daa07 100644 --- a/crates/exex/exex/src/backfill/stream.rs +++ b/crates/exex/exex/src/backfill/stream.rs @@ -41,6 +41,7 @@ pub struct StreamBackfillJob { tasks: BackfillTasks, parallelism: usize, batch_size: usize, + thresholds: ExecutionStageThresholds, } impl StreamBackfillJob { @@ -124,7 +125,7 @@ where executor: this.executor.clone(), provider: this.provider.clone(), prune_modes: this.prune_modes.clone(), - thresholds: ExecutionStageThresholds::default(), + thresholds: this.thresholds.clone(), range, stream_parallelism: this.parallelism, }; @@ -150,12 +151,14 @@ impl From> for StreamBackfillJob From> for StreamBackfillJob { fn from(job: BackfillJob) -> Self { + let batch_size = job.thresholds.max_blocks.map_or(DEFAULT_BATCH_SIZE, |max| max as usize); Self { executor: job.executor, provider: job.provider, @@ -163,7 +166,11 @@ impl From> for StreamBackfillJob Date: Fri, 4 Oct 2024 15:10:32 +0300 Subject: [PATCH 015/159] chore(db): capture tx opening backtrace in debug mode (#11477) --- .../storage/db/src/implementation/mdbx/tx.rs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/storage/db/src/implementation/mdbx/tx.rs b/crates/storage/db/src/implementation/mdbx/tx.rs index 8feb6c90ab27e..2ff2789ea6982 100644 --- a/crates/storage/db/src/implementation/mdbx/tx.rs +++ b/crates/storage/db/src/implementation/mdbx/tx.rs @@ -180,7 +180,12 @@ struct MetricsHandler { /// If `true`, the backtrace of transaction has already been recorded and logged. /// See [`MetricsHandler::log_backtrace_on_long_read_transaction`]. backtrace_recorded: AtomicBool, + /// Shared database environment metrics. env_metrics: Arc, + /// Backtrace of the location where the transaction has been opened. Reported only with debug + /// assertions, because capturing the backtrace on every transaction opening is expensive. + #[cfg(debug_assertions)] + open_backtrace: Backtrace, _marker: PhantomData, } @@ -193,6 +198,8 @@ impl MetricsHandler { close_recorded: false, record_backtrace: true, backtrace_recorded: AtomicBool::new(false), + #[cfg(debug_assertions)] + open_backtrace: Backtrace::force_capture(), env_metrics, _marker: PhantomData, } @@ -232,11 +239,22 @@ impl MetricsHandler { let open_duration = self.start.elapsed(); if open_duration >= self.long_transaction_duration { self.backtrace_recorded.store(true, Ordering::Relaxed); + #[cfg(debug_assertions)] + let message = format!( + "The database read transaction has been open for too long. Open backtrace:\n{}\n\nCurrent backtrace:\n{}", + self.open_backtrace, + Backtrace::force_capture() + ); + #[cfg(not(debug_assertions))] + let message = format!( + "The database read transaction has been open for too long. Backtrace:\n{}", + Backtrace::force_capture() + ); warn!( target: "storage::db::mdbx", ?open_duration, %self.txn_id, - "The database read transaction has been open for too long. Backtrace:\n{}", Backtrace::force_capture() + "{message}" ); } } From 2e516514c2dedd042ccf89958817165aeb8c0950 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Fri, 4 Oct 2024 15:29:36 +0200 Subject: [PATCH 016/159] chore(sdk): `SealedHeader` generic over header (#11429) --- crates/primitives-traits/src/header/sealed.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/primitives-traits/src/header/sealed.rs b/crates/primitives-traits/src/header/sealed.rs index f2047e079c8d5..85b2cb3220473 100644 --- a/crates/primitives-traits/src/header/sealed.rs +++ b/crates/primitives-traits/src/header/sealed.rs @@ -14,22 +14,24 @@ use serde::{Deserialize, Serialize}; /// to modify header. #[derive(Debug, Clone, PartialEq, Eq, Hash, AsRef, Deref, Serialize, Deserialize)] #[add_arbitrary_tests(rlp)] -pub struct SealedHeader { +pub struct SealedHeader { /// Locked Header hash. hash: BlockHash, /// Locked Header fields. #[as_ref] #[deref] - header: Header, + header: H, } -impl SealedHeader { +impl SealedHeader { /// Creates the sealed header with the corresponding block hash. #[inline] - pub const fn new(header: Header, hash: BlockHash) -> Self { + pub const fn new(header: H, hash: BlockHash) -> Self { Self { header, hash } } +} +impl SealedHeader { /// Returns the sealed Header fields. #[inline] pub const fn header(&self) -> &Header { From 5a0831d8621368b9fe80fb14dcb22f24931f5d5f Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Fri, 4 Oct 2024 15:33:18 +0200 Subject: [PATCH 017/159] chore(lint): fix lint primitives (#11487) --- crates/primitives-traits/src/header/sealed.rs | 2 +- crates/primitives/src/block.rs | 6 +++--- crates/primitives/src/receipt.rs | 6 +++--- crates/primitives/src/transaction/mod.rs | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/primitives-traits/src/header/sealed.rs b/crates/primitives-traits/src/header/sealed.rs index 85b2cb3220473..7119a37e742aa 100644 --- a/crates/primitives-traits/src/header/sealed.rs +++ b/crates/primitives-traits/src/header/sealed.rs @@ -184,7 +184,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for SealedHeader<'a> { + impl SerializeAs for SealedHeader<'_> { fn serialize_as(source: &super::SealedHeader, serializer: S) -> Result where S: Serializer, diff --git a/crates/primitives/src/block.rs b/crates/primitives/src/block.rs index 0464c28dee0fc..b0bc538e512a9 100644 --- a/crates/primitives/src/block.rs +++ b/crates/primitives/src/block.rs @@ -756,7 +756,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for BlockBody<'a> { + impl SerializeAs for BlockBody<'_> { fn serialize_as(source: &super::BlockBody, serializer: S) -> Result where S: Serializer, @@ -807,7 +807,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for SealedBlock<'a> { + impl SerializeAs for SealedBlock<'_> { fn serialize_as(source: &super::SealedBlock, serializer: S) -> Result where S: Serializer, @@ -858,7 +858,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for SealedBlockWithSenders<'a> { + impl SerializeAs for SealedBlockWithSenders<'_> { fn serialize_as( source: &super::SealedBlockWithSenders, serializer: S, diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 5c794be5061e6..cfd831ed0f740 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -373,7 +373,7 @@ impl<'a> ReceiptWithBloomRef<'a> { } } -impl<'a> Encodable for ReceiptWithBloomRef<'a> { +impl Encodable for ReceiptWithBloomRef<'_> { fn encode(&self, out: &mut dyn BufMut) { self.as_encoder().encode_inner(out, true) } @@ -394,7 +394,7 @@ struct ReceiptWithBloomEncoder<'a> { receipt: &'a Receipt, } -impl<'a> ReceiptWithBloomEncoder<'a> { +impl ReceiptWithBloomEncoder<'_> { /// Returns the rlp header for the receipt payload. fn receipt_rlp_header(&self) -> alloy_rlp::Header { let mut rlp_head = alloy_rlp::Header { list: true, payload_length: 0 }; @@ -481,7 +481,7 @@ impl<'a> ReceiptWithBloomEncoder<'a> { } } -impl<'a> Encodable for ReceiptWithBloomEncoder<'a> { +impl Encodable for ReceiptWithBloomEncoder<'_> { fn encode(&self, out: &mut dyn BufMut) { self.encode_inner(out, true) } diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index b16c2c88ab529..7ef2c0c1fb76f 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -1640,7 +1640,7 @@ pub mod serde_bincode_compat { } } - impl<'a> SerializeAs for Transaction<'a> { + impl SerializeAs for Transaction<'_> { fn serialize_as(source: &super::Transaction, serializer: S) -> Result where S: Serializer, @@ -1700,7 +1700,7 @@ pub mod serde_bincode_compat { } } - impl<'a> SerializeAs for TransactionSigned<'a> { + impl SerializeAs for TransactionSigned<'_> { fn serialize_as( source: &super::TransactionSigned, serializer: S, From b4ed35ffc212c419b91377fe293db5b38a8ba46a Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:05:17 +0200 Subject: [PATCH 018/159] chore(provider): add more test coverage on `BlockchainProvider::*by_tx_range` queries (#11480) --- .../src/providers/blockchain_provider.rs | 163 +++++++++--------- 1 file changed, 82 insertions(+), 81 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 45bdb15256905..01d8e4cd977c2 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -318,11 +318,17 @@ impl BlockchainProvider2 { let block_tx_count = block_state.block_ref().block().body.transactions.len(); let remaining = (tx_range.end() - tx_range.start() + 1) as usize; - // This should only be more than 0 in the first iteration, in case of a partial range + // If the transaction range start is higher than this block last transaction, advance + if *tx_range.start() > in_memory_tx_num + block_tx_count as u64 - 1 { + in_memory_tx_num += block_tx_count as u64; + continue + } + + // This should only be more than 0 once, in case of a partial range inside a block. let skip = (tx_range.start() - in_memory_tx_num) as usize; items.extend(fetch_from_block_state( - skip..=(remaining.min(block_tx_count) - 1), + skip..=skip + (remaining.min(block_tx_count - skip) - 1), block_state, )?); @@ -4115,51 +4121,76 @@ mod tests { Ok(()) } - #[test] - fn test_transactions_by_tx_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - 0, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; + macro_rules! test_by_tx_range { + ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, [$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ + let db_tx_count = + $database_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; + let in_mem_tx_count = + $in_memory_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; - // Define a valid transaction range within the database - let start_tx_num = 0; - let end_tx_num = 1; + let db_range = 0..=(db_tx_count - 1); + let in_mem_range = db_tx_count..=(in_mem_tx_count + db_range.end()); - // Retrieve the transactions for this transaction number range - let result = provider.transactions_by_tx_range(start_tx_num..=end_tx_num)?; + $( + // Retrieve the expected database data + let database_data = + $database_blocks.iter().flat_map(|b| $data_extractor(b)).collect::>(); + assert_eq!($provider.$method(db_range.clone())?, database_data); - // Ensure the transactions match the expected transactions in the database - assert_eq!(result.len(), 2); - assert_eq!(result[0], database_blocks[0].body.transactions[0].clone().into()); - assert_eq!(result[1], database_blocks[0].body.transactions[1].clone().into()); + // Retrieve the expected in-memory data + let in_memory_data = + $in_memory_blocks.iter().flat_map(|b| $data_extractor(b)).collect::>(); + assert_eq!($provider.$method(in_mem_range.clone())?, in_memory_data); - // Define an empty range that should return no transactions - let start_tx_num = u64::MAX; - let end_tx_num = u64::MAX; + // Test partial in-memory range + assert_eq!( + &$provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, + &in_memory_data[1..in_memory_data.len() - 1] + ); - // Retrieve the transactions for this range - let result = provider.transactions_by_tx_range(start_tx_num..end_tx_num)?; + // Test range in in-memory to unbounded end + assert_eq!($provider.$method(in_mem_range.start() + 1..)?, &in_memory_data[1..]); - // Ensure no transactions are returned - assert!( - result.is_empty(), - "No transactions should be found for an empty transaction range" - ); + // Test last element in-memory + assert_eq!($provider.$method(in_mem_range.end()..)?, &in_memory_data[in_memory_data.len() -1 ..]); - Ok(()) + // Test range that spans database and in-memory + assert_eq!( + $provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, + database_data[database_data.len() - 2..] + .iter() + .chain(&in_memory_data[..in_memory_data.len() - 1]) + .cloned() + .collect::>() + ); + + // Test range that spans database and in-memory with unbounded end + assert_eq!( + $provider.$method(in_mem_range.start() - 2..)?, + database_data[database_data.len() - 2..] + .iter() + .chain(&in_memory_data[..]) + .cloned() + .collect::>() + ); + + // Test invalid range + let start_tx_num = u64::MAX; + let end_tx_num = u64::MAX; + let result = $provider.$method(start_tx_num..end_tx_num)?; + assert!(result.is_empty(), "No data should be found for an invalid transaction range"); + + // Test empty range + let result = $provider.$method(in_mem_range.end()+10..in_mem_range.end()+20)?; + assert!(result.is_empty(), "No data should be found for an empty transaction range"); + )* + }}; } #[test] - fn test_senders_by_tx_range() -> eyre::Result<()> { + fn test_methods_by_tx_range() -> eyre::Result<()> { let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( + let (provider, database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( &mut rng, TEST_BLOCKS_COUNT, TEST_BLOCKS_COUNT, @@ -4169,51 +4200,21 @@ mod tests { }, )?; - let db_tx_count = - database_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; - let in_mem_tx_count = - in_memory_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; - - let db_range = 0..=(db_tx_count - 1); - let in_mem_range = db_tx_count..=(in_mem_tx_count + db_range.end()); - - // Retrieve the senders for the whole database range - let database_senders = - database_blocks.iter().flat_map(|b| b.senders().unwrap()).collect::>(); - assert_eq!(provider.senders_by_tx_range(db_range)?, database_senders); - - // Retrieve the senders for the whole in-memory range - let in_memory_senders = - in_memory_blocks.iter().flat_map(|b| b.senders().unwrap()).collect::>(); - assert_eq!(provider.senders_by_tx_range(in_mem_range.clone())?, in_memory_senders); - - // Retrieve the senders for a partial in-memory range - assert_eq!( - &provider.senders_by_tx_range(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, - &in_memory_senders[1..in_memory_senders.len() - 1] - ); - - // Retrieve the senders for a range that spans database and in-memory - assert_eq!( - provider.senders_by_tx_range(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, - database_senders[database_senders.len() - 2..] - .iter() - .chain(&in_memory_senders[..in_memory_senders.len() - 1]) - .copied() - .collect::>() - ); - - // Define an empty range that should return no sender addresses - let start_tx_num = u64::MAX; - let end_tx_num = u64::MAX; - - // Retrieve the senders for this range - let result = provider.senders_by_tx_range(start_tx_num..end_tx_num)?; - - // Ensure no sender addresses are returned - assert!( - result.is_empty(), - "No sender addresses should be found for an empty transaction range" + test_by_tx_range!( + provider, + database_blocks, + in_memory_blocks, + [ + (senders_by_tx_range, |block: &SealedBlock| block.senders().unwrap()), + (transactions_by_tx_range, |block: &SealedBlock| block + .body + .transactions + .iter() + .map(|tx| Into::::into(tx.clone())) + .collect::>()), + (receipts_by_tx_range, |block: &SealedBlock| receipts[block.number as usize] + .clone()) + ] ); Ok(()) From 8546bed19bc6912d53bf4186cdb6611e7f8f1302 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Oct 2024 17:43:35 +0200 Subject: [PATCH 019/159] test: ensure default hash matches (#11486) --- crates/primitives/src/block.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/primitives/src/block.rs b/crates/primitives/src/block.rs index b0bc538e512a9..de0817fb025db 100644 --- a/crates/primitives/src/block.rs +++ b/crates/primitives/src/block.rs @@ -1130,4 +1130,13 @@ mod tests { Some(SealedBlockWithSenders { block: sealed, senders: vec![sender] }) ); } + + #[test] + fn test_default_seal() { + let block = SealedBlock::default(); + let sealed = block.hash(); + let block = block.unseal(); + let block = block.seal_slow(); + assert_eq!(sealed, block.hash()); + } } From c8be49f4a7729fdec3c2532046151ed02ff070c3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 4 Oct 2024 17:44:47 +0200 Subject: [PATCH 020/159] chore: rm deposit contract config for op (#11479) --- crates/optimism/chainspec/src/constants.rs | 10 ---------- crates/optimism/chainspec/src/lib.rs | 10 ---------- 2 files changed, 20 deletions(-) diff --git a/crates/optimism/chainspec/src/constants.rs b/crates/optimism/chainspec/src/constants.rs index 8fce63daafef5..439c2fed83c0e 100644 --- a/crates/optimism/chainspec/src/constants.rs +++ b/crates/optimism/chainspec/src/constants.rs @@ -1,8 +1,5 @@ //! OP stack variation of chain spec constants. -use alloy_primitives::{address, b256}; -use reth_chainspec::DepositContract; - //------------------------------- BASE MAINNET -------------------------------// /// Max gas limit on Base: @@ -12,10 +9,3 @@ pub const BASE_MAINNET_MAX_GAS_LIMIT: u64 = 105_000_000; /// Max gas limit on Base Sepolia: pub const BASE_SEPOLIA_MAX_GAS_LIMIT: u64 = 45_000_000; - -/// Deposit contract address: `0x00000000219ab540356cbb839cbe05303d7705fa` -pub(crate) const MAINNET_DEPOSIT_CONTRACT: DepositContract = DepositContract::new( - address!("00000000219ab540356cbb839cbe05303d7705fa"), - 11052984, - b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"), -); diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 76a3ed838b187..22316a234d9b1 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -26,7 +26,6 @@ pub use dev::OP_DEV; pub use op::OP_MAINNET; pub use op_sepolia::OP_SEPOLIA; -use crate::constants::MAINNET_DEPOSIT_CONTRACT; use derive_more::{Constructor, Deref, Into}; use once_cell::sync::OnceCell; use reth_chainspec::{ @@ -209,14 +208,6 @@ impl From for OpChainSpec { // append the remaining unknown hardforks to ensure we don't filter any out ordered_hardforks.append(&mut block_hardforks); - // NOTE: in full node, we prune all receipts except the deposit contract's. We do not - // have the deployment block in the genesis file, so we use block zero. We use the same - // deposit topic as the mainnet contract if we have the deposit contract address in the - // genesis json. - let deposit_contract = genesis.config.deposit_contract_address.map(|address| { - DepositContract { address, block: 0, topic: MAINNET_DEPOSIT_CONTRACT.topic } - }); - Self { inner: ChainSpec { chain: genesis.config.chain_id.into(), @@ -224,7 +215,6 @@ impl From for OpChainSpec { genesis_hash: OnceCell::new(), hardforks: ChainHardforks::new(ordered_hardforks), paris_block_and_final_difficulty, - deposit_contract, base_fee_params: optimism_genesis_info.base_fee_params, ..Default::default() }, From 56a9866c53ed3bc23676d6bc31c536fa33a0b089 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Fri, 4 Oct 2024 19:19:02 +0200 Subject: [PATCH 021/159] chore(lint): fix lint storage (#11485) --- crates/etl/src/lib.rs | 4 ++-- crates/storage/db-api/src/cursor.rs | 16 ++++++++-------- crates/storage/db/src/static_file/cursor.rs | 2 +- crates/storage/libmdbx-rs/src/codec.rs | 2 +- crates/storage/libmdbx-rs/src/cursor.rs | 8 ++++---- crates/storage/nippy-jar/src/compression/zstd.rs | 12 ++++++------ crates/storage/nippy-jar/src/cursor.rs | 2 +- crates/trie/trie/src/forward_cursor.rs | 2 +- crates/trie/trie/src/hashed_cursor/post_state.rs | 6 +++--- crates/trie/trie/src/trie_cursor/in_memory.rs | 4 ++-- crates/trie/trie/src/updates.rs | 4 ++-- 11 files changed, 31 insertions(+), 31 deletions(-) diff --git a/crates/etl/src/lib.rs b/crates/etl/src/lib.rs index 3f978fabeee24..d30f432f9c190 100644 --- a/crates/etl/src/lib.rs +++ b/crates/etl/src/lib.rs @@ -190,14 +190,14 @@ pub struct EtlIter<'a> { files: &'a mut Vec, } -impl<'a> EtlIter<'a> { +impl EtlIter<'_> { /// Peeks into the next element pub fn peek(&self) -> Option<&(Vec, Vec)> { self.heap.peek().map(|(Reverse(entry), _)| entry) } } -impl<'a> Iterator for EtlIter<'a> { +impl Iterator for EtlIter<'_> { type Item = std::io::Result<(Vec, Vec)>; fn next(&mut self) -> Option { diff --git a/crates/storage/db-api/src/cursor.rs b/crates/storage/db-api/src/cursor.rs index 134819a8e2cd8..585aa4947a282 100644 --- a/crates/storage/db-api/src/cursor.rs +++ b/crates/storage/db-api/src/cursor.rs @@ -149,7 +149,7 @@ where } } -impl<'cursor, T: Table, CURSOR: DbCursorRO> Iterator for Walker<'cursor, T, CURSOR> { +impl> Iterator for Walker<'_, T, CURSOR> { type Item = Result, DatabaseError>; fn next(&mut self) -> Option { let start = self.start.take(); @@ -174,7 +174,7 @@ impl<'cursor, T: Table, CURSOR: DbCursorRO> Walker<'cursor, T, CURSOR> { } } -impl<'cursor, T: Table, CURSOR: DbCursorRW + DbCursorRO> Walker<'cursor, T, CURSOR> { +impl + DbCursorRO> Walker<'_, T, CURSOR> { /// Delete current item that walker points to. pub fn delete_current(&mut self) -> Result<(), DatabaseError> { self.start.take(); @@ -217,7 +217,7 @@ impl<'cursor, T: Table, CURSOR: DbCursorRO> ReverseWalker<'cursor, T, CURSOR> } } -impl<'cursor, T: Table, CURSOR: DbCursorRW + DbCursorRO> ReverseWalker<'cursor, T, CURSOR> { +impl + DbCursorRO> ReverseWalker<'_, T, CURSOR> { /// Delete current item that walker points to. pub fn delete_current(&mut self) -> Result<(), DatabaseError> { self.start.take(); @@ -225,7 +225,7 @@ impl<'cursor, T: Table, CURSOR: DbCursorRW + DbCursorRO> ReverseWalker<'cu } } -impl<'cursor, T: Table, CURSOR: DbCursorRO> Iterator for ReverseWalker<'cursor, T, CURSOR> { +impl> Iterator for ReverseWalker<'_, T, CURSOR> { type Item = Result, DatabaseError>; fn next(&mut self) -> Option { @@ -266,7 +266,7 @@ where } } -impl<'cursor, T: Table, CURSOR: DbCursorRO> Iterator for RangeWalker<'cursor, T, CURSOR> { +impl> Iterator for RangeWalker<'_, T, CURSOR> { type Item = Result, DatabaseError>; fn next(&mut self) -> Option { @@ -316,7 +316,7 @@ impl<'cursor, T: Table, CURSOR: DbCursorRO> RangeWalker<'cursor, T, CURSOR> { } } -impl<'cursor, T: Table, CURSOR: DbCursorRW + DbCursorRO> RangeWalker<'cursor, T, CURSOR> { +impl + DbCursorRO> RangeWalker<'_, T, CURSOR> { /// Delete current item that walker points to. pub fn delete_current(&mut self) -> Result<(), DatabaseError> { self.start.take(); @@ -349,7 +349,7 @@ where } } -impl<'cursor, T: DupSort, CURSOR: DbCursorRW + DbDupCursorRO> DupWalker<'cursor, T, CURSOR> { +impl + DbDupCursorRO> DupWalker<'_, T, CURSOR> { /// Delete current item that walker points to. pub fn delete_current(&mut self) -> Result<(), DatabaseError> { self.start.take(); @@ -357,7 +357,7 @@ impl<'cursor, T: DupSort, CURSOR: DbCursorRW + DbDupCursorRO> DupWalker<'c } } -impl<'cursor, T: DupSort, CURSOR: DbDupCursorRO> Iterator for DupWalker<'cursor, T, CURSOR> { +impl> Iterator for DupWalker<'_, T, CURSOR> { type Item = Result, DatabaseError>; fn next(&mut self) -> Option { let start = self.start.take(); diff --git a/crates/storage/db/src/static_file/cursor.rs b/crates/storage/db/src/static_file/cursor.rs index f14e023087739..a9eadd9c12a60 100644 --- a/crates/storage/db/src/static_file/cursor.rs +++ b/crates/storage/db/src/static_file/cursor.rs @@ -115,7 +115,7 @@ impl<'a> From<&'a B256> for KeyOrNumber<'a> { } } -impl<'a> From for KeyOrNumber<'a> { +impl From for KeyOrNumber<'_> { fn from(value: u64) -> Self { KeyOrNumber::Number(value) } diff --git a/crates/storage/libmdbx-rs/src/codec.rs b/crates/storage/libmdbx-rs/src/codec.rs index a97ea28ca2ae9..c78f79db9f944 100644 --- a/crates/storage/libmdbx-rs/src/codec.rs +++ b/crates/storage/libmdbx-rs/src/codec.rs @@ -22,7 +22,7 @@ pub trait TableObject: Sized { } } -impl<'tx> TableObject for Cow<'tx, [u8]> { +impl TableObject for Cow<'_, [u8]> { fn decode(_: &[u8]) -> Result { unreachable!() } diff --git a/crates/storage/libmdbx-rs/src/cursor.rs b/crates/storage/libmdbx-rs/src/cursor.rs index d007cc03e490f..3deff0c249bcd 100644 --- a/crates/storage/libmdbx-rs/src/cursor.rs +++ b/crates/storage/libmdbx-rs/src/cursor.rs @@ -539,7 +539,7 @@ where }, } -impl<'cur, K, Key, Value> IntoIter<'cur, K, Key, Value> +impl IntoIter<'_, K, Key, Value> where K: TransactionKind, Key: TableObject, @@ -551,7 +551,7 @@ where } } -impl<'cur, K, Key, Value> Iterator for IntoIter<'cur, K, Key, Value> +impl Iterator for IntoIter<'_, K, Key, Value> where K: TransactionKind, Key: TableObject, @@ -646,7 +646,7 @@ where } } -impl<'cur, K, Key, Value> Iterator for Iter<'cur, K, Key, Value> +impl Iterator for Iter<'_, K, Key, Value> where K: TransactionKind, Key: TableObject, @@ -736,7 +736,7 @@ where } } -impl<'cur, K, Key, Value> fmt::Debug for IterDup<'cur, K, Key, Value> +impl fmt::Debug for IterDup<'_, K, Key, Value> where K: TransactionKind, Key: TableObject, diff --git a/crates/storage/nippy-jar/src/compression/zstd.rs b/crates/storage/nippy-jar/src/compression/zstd.rs index 494d79de52f6b..500247d176771 100644 --- a/crates/storage/nippy-jar/src/compression/zstd.rs +++ b/crates/storage/nippy-jar/src/compression/zstd.rs @@ -266,13 +266,13 @@ mod dictionaries_serde { #[derive(Serialize, Deserialize, Deref)] pub(crate) struct ZstdDictionaries<'a>(Vec>); -impl<'a> std::fmt::Debug for ZstdDictionaries<'a> { +impl std::fmt::Debug for ZstdDictionaries<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ZstdDictionaries").field("num", &self.len()).finish_non_exhaustive() } } -impl<'a> ZstdDictionaries<'a> { +impl ZstdDictionaries<'_> { #[cfg(test)] /// Creates [`ZstdDictionaries`]. pub(crate) fn new(raw: Vec) -> Self { @@ -321,7 +321,7 @@ pub(crate) enum ZstdDictionary<'a> { Loaded(DecoderDictionary<'a>), } -impl<'a> ZstdDictionary<'a> { +impl ZstdDictionary<'_> { /// Returns a reference to the expected `RawDictionary` pub(crate) const fn raw(&self) -> Option<&RawDictionary> { match self { @@ -339,7 +339,7 @@ impl<'a> ZstdDictionary<'a> { } } -impl<'de, 'a> Deserialize<'de> for ZstdDictionary<'a> { +impl<'de> Deserialize<'de> for ZstdDictionary<'_> { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, @@ -349,7 +349,7 @@ impl<'de, 'a> Deserialize<'de> for ZstdDictionary<'a> { } } -impl<'a> Serialize for ZstdDictionary<'a> { +impl Serialize for ZstdDictionary<'_> { fn serialize(&self, serializer: S) -> Result where S: Serializer, @@ -362,7 +362,7 @@ impl<'a> Serialize for ZstdDictionary<'a> { } #[cfg(test)] -impl<'a> PartialEq for ZstdDictionary<'a> { +impl PartialEq for ZstdDictionary<'_> { fn eq(&self, other: &Self) -> bool { if let (Self::Raw(a), Self::Raw(b)) = (self, &other) { return a == b diff --git a/crates/storage/nippy-jar/src/cursor.rs b/crates/storage/nippy-jar/src/cursor.rs index 7af55fd436e46..2677648272997 100644 --- a/crates/storage/nippy-jar/src/cursor.rs +++ b/crates/storage/nippy-jar/src/cursor.rs @@ -18,7 +18,7 @@ pub struct NippyJarCursor<'a, H = ()> { row: u64, } -impl<'a, H: NippyJarHeader> std::fmt::Debug for NippyJarCursor<'a, H> { +impl std::fmt::Debug for NippyJarCursor<'_, H> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("NippyJarCursor").field("config", &self.jar).finish_non_exhaustive() } diff --git a/crates/trie/trie/src/forward_cursor.rs b/crates/trie/trie/src/forward_cursor.rs index da71326cea905..6db214bb51a33 100644 --- a/crates/trie/trie/src/forward_cursor.rs +++ b/crates/trie/trie/src/forward_cursor.rs @@ -21,7 +21,7 @@ impl<'a, K, V> ForwardInMemoryCursor<'a, K, V> { } } -impl<'a, K, V> ForwardInMemoryCursor<'a, K, V> +impl ForwardInMemoryCursor<'_, K, V> where K: PartialOrd + Clone, V: Clone, diff --git a/crates/trie/trie/src/hashed_cursor/post_state.rs b/crates/trie/trie/src/hashed_cursor/post_state.rs index 53a2cdb3bb645..678914191527e 100644 --- a/crates/trie/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/trie/src/hashed_cursor/post_state.rs @@ -132,7 +132,7 @@ where } } -impl<'a, C> HashedCursor for HashedPostStateAccountCursor<'a, C> +impl HashedCursor for HashedPostStateAccountCursor<'_, C> where C: HashedCursor, { @@ -276,7 +276,7 @@ where } } -impl<'a, C> HashedCursor for HashedPostStateStorageCursor<'a, C> +impl HashedCursor for HashedPostStateStorageCursor<'_, C> where C: HashedStorageCursor, { @@ -304,7 +304,7 @@ where } } -impl<'a, C> HashedStorageCursor for HashedPostStateStorageCursor<'a, C> +impl HashedStorageCursor for HashedPostStateStorageCursor<'_, C> where C: HashedStorageCursor, { diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index 82ffcfb428f91..c313231e66dc1 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -188,7 +188,7 @@ impl<'a, C> InMemoryStorageTrieCursor<'a, C> { } } -impl<'a, C: TrieCursor> InMemoryStorageTrieCursor<'a, C> { +impl InMemoryStorageTrieCursor<'_, C> { fn seek_inner( &mut self, key: Nibbles, @@ -237,7 +237,7 @@ impl<'a, C: TrieCursor> InMemoryStorageTrieCursor<'a, C> { } } -impl<'a, C: TrieCursor> TrieCursor for InMemoryStorageTrieCursor<'a, C> { +impl TrieCursor for InMemoryStorageTrieCursor<'_, C> { fn seek_exact( &mut self, key: Nibbles, diff --git a/crates/trie/trie/src/updates.rs b/crates/trie/trie/src/updates.rs index a2d3d67363ea6..03e80cf52e539 100644 --- a/crates/trie/trie/src/updates.rs +++ b/crates/trie/trie/src/updates.rs @@ -455,7 +455,7 @@ pub mod serde_bincode_compat { } } - impl<'a> SerializeAs for TrieUpdates<'a> { + impl SerializeAs for TrieUpdates<'_> { fn serialize_as(source: &super::TrieUpdates, serializer: S) -> Result where S: Serializer, @@ -515,7 +515,7 @@ pub mod serde_bincode_compat { } } - impl<'a> SerializeAs for StorageTrieUpdates<'a> { + impl SerializeAs for StorageTrieUpdates<'_> { fn serialize_as( source: &super::StorageTrieUpdates, serializer: S, From bdfe35034e232333b25494bb08b440bb6a75206d Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Fri, 4 Oct 2024 20:57:44 +0200 Subject: [PATCH 022/159] chore(provider): add more test coverage on `BlockchainProvider::*by_block_range` queries (#11488) --- .../src/providers/blockchain_provider.rs | 555 +++--------------- 1 file changed, 90 insertions(+), 465 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 01d8e4cd977c2..9a2a3d0a77ee5 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1502,8 +1502,8 @@ mod tests { use reth_db::models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives::{ - BlockWithSenders, Receipt, SealedBlock, SealedBlockWithSenders, StaticFileSegment, - TransactionMeta, TransactionSignedNoHash, Withdrawals, + Receipt, SealedBlock, StaticFileSegment, TransactionMeta, TransactionSignedNoHash, + Withdrawals, }; use reth_storage_api::{ BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, @@ -2250,459 +2250,6 @@ mod tests { Ok(()) } - #[test] - fn test_block_range_in_memory_only() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the range of in-memory blocks - let start_block_number = in_memory_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks = provider.block_range(range)?; - - // Check if the retrieved blocks match the in-memory blocks - assert_eq!(blocks.len(), in_memory_blocks.len()); - // Check if the blocks are equal - for (retrieved_block, expected_block) in blocks.iter().zip(in_memory_blocks.iter()) { - assert_eq!(retrieved_block, &expected_block.clone().unseal()); - } - - // Check for partial in-memory ranges - let blocks = provider.block_range(start_block_number + 1..=end_block_number)?; - assert_eq!(blocks.len(), in_memory_blocks.len() - 1); - for (retrieved_block, expected_block) in blocks.iter().zip(in_memory_blocks.iter().skip(1)) - { - assert_eq!(retrieved_block, &expected_block.clone().unseal()); - } - - let blocks = provider.block_range(start_block_number + 1..=end_block_number - 1)?; - assert_eq!(blocks.len(), in_memory_blocks.len() - 2); - for (retrieved_block, expected_block) in blocks.iter().zip(in_memory_blocks.iter().skip(1)) - { - assert_eq!(retrieved_block, &expected_block.clone().unseal()); - } - - let blocks = provider.block_range(start_block_number + 1..=end_block_number + 1)?; - assert_eq!(blocks.len(), in_memory_blocks.len() - 1); - for (retrieved_block, expected_block) in blocks.iter().zip(in_memory_blocks.iter().skip(1)) - { - assert_eq!(retrieved_block, &expected_block.clone().unseal()); - } - - Ok(()) - } - - #[test] - fn test_block_range_in_database_only() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - 0, // No blocks in memory - BlockRangeParams::default(), - )?; - - // Get the range of database blocks - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = database_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks = provider.block_range(range)?; - - // Check if the retrieved blocks match the database blocks - assert_eq!(blocks.len(), database_blocks.len()); - // Check if the blocks are equal - for (retrieved_block, expected_block) in blocks.iter().zip(database_blocks.iter()) { - assert_eq!(retrieved_block, &expected_block.clone().unseal()); - } - - Ok(()) - } - - #[test] - fn test_block_range_across_memory_and_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let mid_point = TEST_BLOCKS_COUNT / 2; - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - mid_point, - TEST_BLOCKS_COUNT - mid_point, - BlockRangeParams::default(), - )?; - - // Get the range of blocks across memory and database - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks = provider.block_range(range)?; - - // Check if the retrieved blocks match the database and in-memory blocks - assert_eq!(blocks.len(), TEST_BLOCKS_COUNT); - let all_expected_blocks = - database_blocks.iter().chain(in_memory_blocks.iter()).collect::>(); - // Check if the blocks are equal - for (retrieved_block, expected_block) in blocks.iter().zip(all_expected_blocks.iter()) { - assert_eq!(retrieved_block.clone(), (*expected_block).clone().unseal()); - } - - Ok(()) - } - - #[test] - fn test_block_range_non_existent_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Generate a non-existent range - let non_existent_range = 9999..=10000; - let blocks = provider.block_range(non_existent_range)?; - - // The range is non-existent, so the blocks should be empty - assert!(blocks.is_empty()); - - Ok(()) - } - - #[test] - fn test_block_range_partial_overlap() -> eyre::Result<()> { - let mut rng = generators::rng(); - let mid_point = TEST_BLOCKS_COUNT / 2; - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - mid_point, - mid_point, - BlockRangeParams::default(), - )?; - - // Get the range of blocks across memory and database - let start_block_number = database_blocks.last().unwrap().number; - let end_block_number = in_memory_blocks.first().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks = provider.block_range(range)?; - - assert_eq!(blocks.len(), 2); // Only one block from each side of the overlap - assert_eq!(blocks[0], database_blocks.last().unwrap().clone().unseal()); - assert_eq!(blocks[1], in_memory_blocks.first().unwrap().clone().unseal()); - - Ok(()) - } - - #[test] - fn test_block_with_senders_range_across_memory_and_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let mid_point = TEST_BLOCKS_COUNT / 2; - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - mid_point, - TEST_BLOCKS_COUNT - mid_point, - BlockRangeParams::default(), - )?; - - // Get the range of blocks across memory and database - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks_with_senders = provider.block_with_senders_range(range)?; - - // Check if the retrieved blocks match the database and in-memory blocks - assert_eq!(blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let all_expected_blocks_with_senders = database_blocks - .iter() - .chain(in_memory_blocks.iter()) - .map(|sealed_block| BlockWithSenders { - block: sealed_block.clone().unseal(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_block_with_senders, expected_block_with_senders) in - blocks_with_senders.iter().zip(all_expected_blocks_with_senders.iter()) - { - assert_eq!(retrieved_block_with_senders.block, expected_block_with_senders.block); - assert_eq!(retrieved_block_with_senders.senders, expected_block_with_senders.senders); - } - - Ok(()) - } - - #[test] - fn test_block_with_senders_range_only_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the range of in-memory blocks - let start_block_number = in_memory_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks_with_senders = provider.block_with_senders_range(range)?; - - // Check if the retrieved blocks match the in-memory blocks - assert_eq!(blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let expected_blocks_with_senders = in_memory_blocks - .iter() - .map(|sealed_block| BlockWithSenders { - block: sealed_block.clone().unseal(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_block_with_senders, expected_block_with_senders) in - blocks_with_senders.iter().zip(expected_blocks_with_senders.iter()) - { - assert_eq!(retrieved_block_with_senders.block, expected_block_with_senders.block); - assert_eq!(retrieved_block_with_senders.senders, expected_block_with_senders.senders); - } - - Ok(()) - } - - #[test] - fn test_block_with_senders_range_only_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - 0, - BlockRangeParams::default(), - )?; - - // Get the range of database blocks - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = database_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let blocks_with_senders = provider.block_with_senders_range(range)?; - - // Check if the retrieved blocks match the database blocks - assert_eq!(blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let expected_blocks_with_senders = database_blocks - .iter() - .map(|sealed_block| BlockWithSenders { - block: sealed_block.clone().unseal(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_block_with_senders, expected_block_with_senders) in - blocks_with_senders.iter().zip(expected_blocks_with_senders.iter()) - { - assert_eq!(retrieved_block_with_senders.block, expected_block_with_senders.block); - assert_eq!(retrieved_block_with_senders.senders, expected_block_with_senders.senders); - } - - Ok(()) - } - - #[test] - fn test_block_with_senders_range_non_existent_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Assuming this range does not exist - let start_block_number = 1000; - let end_block_number = 2000; - - let range = start_block_number..=end_block_number; - let blocks_with_senders = provider.block_with_senders_range(range)?; - - // The range is non-existent, so the blocks should be empty - assert!(blocks_with_senders.is_empty()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_range_across_memory_and_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let mid_point = TEST_BLOCKS_COUNT / 2; - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - mid_point, - TEST_BLOCKS_COUNT - mid_point, - BlockRangeParams::default(), - )?; - - // Get the range of blocks across memory and database - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let sealed_blocks_with_senders = provider.sealed_block_with_senders_range(range)?; - - // Check if the retrieved blocks match the database and in-memory blocks - assert_eq!(sealed_blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let all_expected_sealed_blocks_with_senders = database_blocks - .iter() - .chain(in_memory_blocks.iter()) - .map(|sealed_block| SealedBlockWithSenders { - block: sealed_block.clone(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_sealed_block_with_senders, expected_sealed_block_with_senders) in - sealed_blocks_with_senders.iter().zip(all_expected_sealed_blocks_with_senders.iter()) - { - assert_eq!( - retrieved_sealed_block_with_senders.block, - expected_sealed_block_with_senders.block - ); - assert_eq!( - retrieved_sealed_block_with_senders.senders, - expected_sealed_block_with_senders.senders - ); - } - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_range_only_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the range of in-memory blocks - let start_block_number = in_memory_blocks.first().unwrap().number; - let end_block_number = in_memory_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let sealed_blocks_with_senders = provider.sealed_block_with_senders_range(range)?; - - // Check if the retrieved blocks match the in-memory blocks - assert_eq!(sealed_blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let expected_sealed_blocks_with_senders = in_memory_blocks - .iter() - .map(|sealed_block| SealedBlockWithSenders { - block: sealed_block.clone(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_sealed_block_with_senders, expected_sealed_block_with_senders) in - sealed_blocks_with_senders.iter().zip(expected_sealed_blocks_with_senders.iter()) - { - assert_eq!( - retrieved_sealed_block_with_senders.block, - expected_sealed_block_with_senders.block - ); - assert_eq!( - retrieved_sealed_block_with_senders.senders, - expected_sealed_block_with_senders.senders - ); - } - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_range_only_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - 0, - BlockRangeParams::default(), - )?; - - // Get the range of database blocks - let start_block_number = database_blocks.first().unwrap().number; - let end_block_number = database_blocks.last().unwrap().number; - - let range = start_block_number..=end_block_number; - let sealed_blocks_with_senders = provider.sealed_block_with_senders_range(range)?; - - // Check if the retrieved blocks match the database blocks - assert_eq!(sealed_blocks_with_senders.len(), TEST_BLOCKS_COUNT); - - let expected_sealed_blocks_with_senders = database_blocks - .iter() - .map(|sealed_block| SealedBlockWithSenders { - block: sealed_block.clone(), - senders: sealed_block.senders().unwrap(), - }) - .collect::>(); - - // Check if the blocks are equal - for (retrieved_sealed_block_with_senders, expected_sealed_block_with_senders) in - sealed_blocks_with_senders.iter().zip(expected_sealed_blocks_with_senders.iter()) - { - assert_eq!( - retrieved_sealed_block_with_senders.block, - expected_sealed_block_with_senders.block - ); - assert_eq!( - retrieved_sealed_block_with_senders.senders, - expected_sealed_block_with_senders.senders - ); - } - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_range_non_existent_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Assuming this range does not exist - let start_block_number = 1000; - let end_block_number = 2000; - - let range = start_block_number..=end_block_number; - let sealed_blocks_with_senders = provider.sealed_block_with_senders_range(range)?; - - // The range is non-existent, so the blocks should be empty - assert!(sealed_blocks_with_senders.is_empty()); - - Ok(()) - } - #[test] fn test_block_hash_reader() -> eyre::Result<()> { let mut rng = generators::rng(); @@ -2773,11 +2320,6 @@ mod tests { Some(in_memory_block.difficulty) ); - assert_eq!( - provider.headers_range(0..=10)?, - blocks.iter().map(|b| b.header().clone()).collect::>() - ); - assert_eq!( provider.sealed_header(database_block.number)?, Some(database_block.header.clone()) @@ -2787,11 +2329,6 @@ mod tests { Some(in_memory_block.header.clone()) ); - assert_eq!( - provider.sealed_headers_range(0..=10)?, - blocks.iter().map(|b| b.header.clone()).collect::>() - ); - assert_eq!( provider.sealed_headers_while(0..=10, |header| header.number <= 8)?, blocks @@ -4220,6 +3757,94 @@ mod tests { Ok(()) } + macro_rules! test_by_block_range { + ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, [$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ + let db_block_count = $database_blocks.len() as u64; + let in_mem_block_count = $in_memory_blocks.len() as u64; + + let db_range = 0..=db_block_count - 1; + let in_mem_range = db_block_count..=(in_mem_block_count + db_range.end()); + + $( + // Retrieve the expected database data + let database_data = + $database_blocks.iter().map(|b| $data_extractor(b)).collect::>(); + assert_eq!($provider.$method(db_range.clone())?, database_data); + + // Retrieve the expected in-memory data + let in_memory_data = + $in_memory_blocks.iter().map(|b| $data_extractor(b)).collect::>(); + assert_eq!($provider.$method(in_mem_range.clone())?, in_memory_data); + + // Test partial in-memory range + assert_eq!( + &$provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, + &in_memory_data[1..in_memory_data.len() - 1] + ); + + // Test range that spans database and in-memory + assert_eq!( + $provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, + database_data[database_data.len() - 2..] + .iter() + .chain(&in_memory_data[..in_memory_data.len() - 1]) + .cloned() + .collect::>() + ); + + // Test invalid range + let start_block_num = u64::MAX; + let end_block_num = u64::MAX; + let result = $provider.$method(start_block_num..=end_block_num-1)?; + assert!(result.is_empty(), "No data should be found for an invalid block range"); + + // Test valid range with empty results + let result = $provider.$method(in_mem_range.end() + 10..=in_mem_range.end() + 20)?; + assert!(result.is_empty(), "No data should be found for an empty block range"); + )* + }}; + } + + #[test] + fn test_methods_by_block_range() -> eyre::Result<()> { + let mut rng = generators::rng(); + let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT, + TEST_BLOCKS_COUNT, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; + + // todo(joshie) add canonical_hashes_range below after changing its interface into range + // instead start end + test_by_block_range!( + provider, + database_blocks, + in_memory_blocks, + [ + (headers_range, |block: &SealedBlock| block.header().clone()), + (sealed_headers_range, |block: &SealedBlock| block.header.clone()), + (block_range, |block: &SealedBlock| block.clone().unseal()), + (block_with_senders_range, |block: &SealedBlock| block + .clone() + .unseal() + .with_senders_unchecked(vec![])), + (sealed_block_with_senders_range, |block: &SealedBlock| block + .clone() + .with_senders_unchecked(vec![])), + (transactions_by_block_range, |block: &SealedBlock| block + .body + .transactions + .clone()), + ] + ); + + Ok(()) + } + #[test] fn transaction_sender_found_in_memory() -> eyre::Result<()> { let mut rng = generators::rng(); From 9d2c7f46e62152cf1c36058eb11a74b934c2fe8a Mon Sep 17 00:00:00 2001 From: Delweng Date: Sat, 5 Oct 2024 16:48:18 +0800 Subject: [PATCH 023/159] fix(rpc-eth-types): incorrect error msg(; -> :) (#11503) Signed-off-by: jsvisa --- crates/rpc/rpc-eth-types/src/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/rpc-eth-types/src/error.rs b/crates/rpc/rpc-eth-types/src/error.rs index 10fa3b7160216..212fca36d9c9a 100644 --- a/crates/rpc/rpc-eth-types/src/error.rs +++ b/crates/rpc/rpc-eth-types/src/error.rs @@ -365,7 +365,7 @@ pub enum RpcInvalidTransactionError { PrecompileOutOfGas(u64), /// An operand to an opcode was invalid or out of range. /// Contains the gas limit. - #[error("out of gas: invalid operand to an opcode; {0}")] + #[error("out of gas: invalid operand to an opcode: {0}")] InvalidOperandOutOfGas(u64), /// Thrown if executing a transaction failed during estimate/call #[error(transparent)] From 565bb9f574c7d86bcef8784133e9d2ac4a716007 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Sat, 5 Oct 2024 02:07:14 -0700 Subject: [PATCH 024/159] Reexport optimism specific crates from `op-reth` (#11499) --- Cargo.lock | 6 +++ crates/optimism/bin/Cargo.toml | 6 +++ crates/optimism/bin/src/lib.rs | 73 +++++++++++++++++++++++++++++++++ crates/optimism/bin/src/main.rs | 1 - 4 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 crates/optimism/bin/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 8c66fbf09b38d..73f500d9d22fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5257,8 +5257,14 @@ dependencies = [ "clap", "reth-cli-util", "reth-node-builder", + "reth-optimism-chainspec", "reth-optimism-cli", + "reth-optimism-consensus", + "reth-optimism-evm", + "reth-optimism-forks", "reth-optimism-node", + "reth-optimism-payload-builder", + "reth-optimism-primitives", "reth-optimism-rpc", "reth-provider", "tracing", diff --git a/crates/optimism/bin/Cargo.toml b/crates/optimism/bin/Cargo.toml index 13e7b1aa6d4d4..ecb92dfc454b3 100644 --- a/crates/optimism/bin/Cargo.toml +++ b/crates/optimism/bin/Cargo.toml @@ -15,6 +15,12 @@ reth-optimism-cli.workspace = true reth-provider.workspace = true reth-optimism-rpc.workspace = true reth-optimism-node.workspace = true +reth-optimism-chainspec.workspace = true +reth-optimism-consensus.workspace = true +reth-optimism-evm.workspace = true +reth-optimism-payload-builder.workspace = true +reth-optimism-primitives.workspace = true +reth-optimism-forks.workspace = true clap = { workspace = true, features = ["derive", "env"] } tracing.workspace = true diff --git a/crates/optimism/bin/src/lib.rs b/crates/optimism/bin/src/lib.rs new file mode 100644 index 0000000000000..21c28f7c5470f --- /dev/null +++ b/crates/optimism/bin/src/lib.rs @@ -0,0 +1,73 @@ +//! Rust Optimism (op-reth) binary executable. +//! +//! ## Feature Flags +//! +//! - `jemalloc`: Uses [jemallocator](https://github.com/tikv/jemallocator) as the global allocator. +//! This is **not recommended on Windows**. See [here](https://rust-lang.github.io/rfcs/1974-global-allocators.html#jemalloc) +//! for more info. +//! - `jemalloc-prof`: Enables [jemallocator's](https://github.com/tikv/jemallocator) heap profiling +//! and leak detection functionality. See [jemalloc's opt.prof](https://jemalloc.net/jemalloc.3.html#opt.prof) +//! documentation for usage details. This is **not recommended on Windows**. See [here](https://rust-lang.github.io/rfcs/1974-global-allocators.html#jemalloc) +//! for more info. +//! - `asm-keccak`: replaces the default, pure-Rust implementation of Keccak256 with one implemented +//! in assembly; see [the `keccak-asm` crate](https://github.com/DaniPopes/keccak-asm) for more +//! details and supported targets +//! - `min-error-logs`: Disables all logs below `error` level. +//! - `min-warn-logs`: Disables all logs below `warn` level. +//! - `min-info-logs`: Disables all logs below `info` level. This can speed up the node, since fewer +//! calls to the logging component is made. +//! - `min-debug-logs`: Disables all logs below `debug` level. +//! - `min-trace-logs`: Disables all logs below `trace` level. +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +// The `optimism` feature must be enabled to use this crate. +#![cfg(feature = "optimism")] + +/// Re-exported from `reth_optimism_cli`. +pub mod cli { + pub use reth_optimism_cli::*; +} + +/// Re-exported from `reth_optimism_chainspec`. +pub mod chainspec { + pub use reth_optimism_chainspec::*; +} + +/// Re-exported from `reth_optimism_consensus`. +pub mod consensus { + pub use reth_optimism_consensus::*; +} + +/// Re-exported from `reth_optimism_evm`. +pub mod evm { + pub use reth_optimism_evm::*; +} + +/// Re-exported from `reth_optimism_forks`. +pub mod forks { + pub use reth_optimism_forks::*; +} + +/// Re-exported from `reth_optimism_node`. +pub mod node { + pub use reth_optimism_node::*; +} + +/// Re-exported from `reth_optimism_payload_builder`. +pub mod payload { + pub use reth_optimism_payload_builder::*; +} + +/// Re-exported from `reth_optimism_primitives`. +pub mod primitives { + pub use reth_optimism_primitives::*; +} + +/// Re-exported from `reth_optimism_rpc`. +pub mod rpc { + pub use reth_optimism_rpc::*; +} diff --git a/crates/optimism/bin/src/main.rs b/crates/optimism/bin/src/main.rs index 6e041e240a74b..6822a6a50ec4e 100644 --- a/crates/optimism/bin/src/main.rs +++ b/crates/optimism/bin/src/main.rs @@ -1,7 +1,6 @@ #![allow(missing_docs, rustdoc::missing_crate_level_docs)] // The `optimism` feature must be enabled to use this crate. #![cfg(feature = "optimism")] -#![cfg_attr(not(test), warn(unused_crate_dependencies))] use clap::Parser; use reth_node_builder::{engine_tree_config::TreeConfig, EngineNodeLauncher}; From a8f420074adf5ef6e4c181fe6a7d84c62c873a46 Mon Sep 17 00:00:00 2001 From: Varun Doshi <61531351+varun-doshi@users.noreply.github.com> Date: Sat, 5 Oct 2024 14:44:06 +0530 Subject: [PATCH 025/159] feat: rpc replace function created (#11501) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-builder/src/lib.rs | 260 +++++++++++++++++++++++------- 1 file changed, 201 insertions(+), 59 deletions(-) diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 113cb93c4630d..e49105ab7d7d4 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -1832,6 +1832,13 @@ impl TransportRpcModules { } } + /// Removes the given methods from the configured http methods. + pub fn remove_http_methods(&mut self, methods: impl IntoIterator) { + for name in methods { + self.remove_http_method(name); + } + } + /// Removes the method with the given name from the configured ws methods. /// /// Returns `true` if the method was found and removed, `false` otherwise. @@ -1847,6 +1854,13 @@ impl TransportRpcModules { } } + /// Removes the given methods from the configured ws methods. + pub fn remove_ws_methods(&mut self, methods: impl IntoIterator) { + for name in methods { + self.remove_ws_method(name); + } + } + /// Removes the method with the given name from the configured ipc methods. /// /// Returns `true` if the method was found and removed, `false` otherwise. @@ -1862,6 +1876,13 @@ impl TransportRpcModules { } } + /// Removes the given methods from the configured ipc methods. + pub fn remove_ipc_methods(&mut self, methods: impl IntoIterator) { + for name in methods { + self.remove_ipc_method(name); + } + } + /// Removes the method with the given name from all configured transports. /// /// Returns `true` if the method was found and removed, `false` otherwise. @@ -1872,6 +1893,56 @@ impl TransportRpcModules { http_removed || ws_removed || ipc_removed } + + /// Replace the given [Methods] in the configured http methods. + /// + /// Fails if any of the methods in other is present already or if the method being removed is + /// not present + /// + /// Returns [Ok(false)] if no http transport is configured. + pub fn replace_http(&mut self, other: impl Into) -> Result { + let other = other.into(); + self.remove_http_methods(other.method_names()); + self.merge_http(other) + } + + /// Replace the given [Methods] in the configured ipc methods. + /// + /// Fails if any of the methods in other is present already or if the method being removed is + /// not present + /// + /// Returns [Ok(false)] if no ipc transport is configured. + pub fn replace_ipc(&mut self, other: impl Into) -> Result { + let other = other.into(); + self.remove_ipc_methods(other.method_names()); + self.merge_ipc(other) + } + + /// Replace the given [Methods] in the configured ws methods. + /// + /// Fails if any of the methods in other is present already or if the method being removed is + /// not present + /// + /// Returns [Ok(false)] if no ws transport is configured. + pub fn replace_ws(&mut self, other: impl Into) -> Result { + let other = other.into(); + self.remove_ws_methods(other.method_names()); + self.merge_ws(other) + } + + /// Replaces the method with the given name from all configured transports. + /// + /// Returns `true` if the method was found and replaced, `false` otherwise + pub fn replace_configured( + &mut self, + other: impl Into, + ) -> Result { + let other = other.into(); + self.replace_http(other.clone())?; + self.replace_ws(other.clone())?; + self.replace_ipc(other)?; + Ok(true) + } } /// A handle to the spawned servers. @@ -2121,81 +2192,152 @@ mod tests { ) } - mod remove_methods { - use super::*; + fn create_test_module() -> RpcModule<()> { + let mut module = RpcModule::new(()); + module.register_method("anything", |_, _, _| "succeed").unwrap(); + module + } - fn create_test_module() -> RpcModule<()> { - let mut module = RpcModule::new(()); - module.register_method("anything", |_, _, _| "succeed").unwrap(); - module - } + #[test] + fn test_remove_http_method() { + let mut modules = + TransportRpcModules { http: Some(create_test_module()), ..Default::default() }; + // Remove a method that exists + assert!(modules.remove_http_method("anything")); + + // Remove a method that does not exist + assert!(!modules.remove_http_method("non_existent_method")); - #[test] - fn test_remove_http_method() { - let mut modules = - TransportRpcModules { http: Some(create_test_module()), ..Default::default() }; - // Remove a method that exists - assert!(modules.remove_http_method("anything")); + // Verify that the method was removed + assert!(modules.http.as_ref().unwrap().method("anything").is_none()); + } - // Remove a method that does not exist - assert!(!modules.remove_http_method("non_existent_method")); + #[test] + fn test_remove_ws_method() { + let mut modules = + TransportRpcModules { ws: Some(create_test_module()), ..Default::default() }; - // Verify that the method was removed - assert!(modules.http.as_ref().unwrap().method("anything").is_none()); - } + // Remove a method that exists + assert!(modules.remove_ws_method("anything")); - #[test] - fn test_remove_ws_method() { - let mut modules = - TransportRpcModules { ws: Some(create_test_module()), ..Default::default() }; + // Remove a method that does not exist + assert!(!modules.remove_ws_method("non_existent_method")); - // Remove a method that exists - assert!(modules.remove_ws_method("anything")); + // Verify that the method was removed + assert!(modules.ws.as_ref().unwrap().method("anything").is_none()); + } - // Remove a method that does not exist - assert!(!modules.remove_ws_method("non_existent_method")); + #[test] + fn test_remove_ipc_method() { + let mut modules = + TransportRpcModules { ipc: Some(create_test_module()), ..Default::default() }; - // Verify that the method was removed - assert!(modules.ws.as_ref().unwrap().method("anything").is_none()); - } + // Remove a method that exists + assert!(modules.remove_ipc_method("anything")); - #[test] - fn test_remove_ipc_method() { - let mut modules = - TransportRpcModules { ipc: Some(create_test_module()), ..Default::default() }; + // Remove a method that does not exist + assert!(!modules.remove_ipc_method("non_existent_method")); - // Remove a method that exists - assert!(modules.remove_ipc_method("anything")); + // Verify that the method was removed + assert!(modules.ipc.as_ref().unwrap().method("anything").is_none()); + } - // Remove a method that does not exist - assert!(!modules.remove_ipc_method("non_existent_method")); + #[test] + fn test_remove_method_from_configured() { + let mut modules = TransportRpcModules { + http: Some(create_test_module()), + ws: Some(create_test_module()), + ipc: Some(create_test_module()), + ..Default::default() + }; - // Verify that the method was removed - assert!(modules.ipc.as_ref().unwrap().method("anything").is_none()); - } + // Remove a method that exists + assert!(modules.remove_method_from_configured("anything")); - #[test] - fn test_remove_method_from_configured() { - let mut modules = TransportRpcModules { - http: Some(create_test_module()), - ws: Some(create_test_module()), - ipc: Some(create_test_module()), - ..Default::default() - }; + // Remove a method that was just removed (it does not exist anymore) + assert!(!modules.remove_method_from_configured("anything")); - // Remove a method that exists - assert!(modules.remove_method_from_configured("anything")); + // Remove a method that does not exist + assert!(!modules.remove_method_from_configured("non_existent_method")); - // Remove a method that was just removed (it does not exist anymore) - assert!(!modules.remove_method_from_configured("anything")); + // Verify that the method was removed from all transports + assert!(modules.http.as_ref().unwrap().method("anything").is_none()); + assert!(modules.ws.as_ref().unwrap().method("anything").is_none()); + assert!(modules.ipc.as_ref().unwrap().method("anything").is_none()); + } - // Remove a method that does not exist - assert!(!modules.remove_method_from_configured("non_existent_method")); + #[test] + fn test_replace_http_method() { + let mut modules = + TransportRpcModules { http: Some(create_test_module()), ..Default::default() }; - // Verify that the method was removed from all transports - assert!(modules.http.as_ref().unwrap().method("anything").is_none()); - assert!(modules.ws.as_ref().unwrap().method("anything").is_none()); - assert!(modules.ipc.as_ref().unwrap().method("anything").is_none()); - } + let mut other_module = RpcModule::new(()); + other_module.register_method("something", |_, _, _| "fails").unwrap(); + + assert!(modules.replace_http(other_module.clone()).unwrap()); + + assert!(modules.http.as_ref().unwrap().method("something").is_some()); + + other_module.register_method("anything", |_, _, _| "fails").unwrap(); + assert!(modules.replace_http(other_module.clone()).unwrap()); + + assert!(modules.http.as_ref().unwrap().method("anything").is_some()); + } + #[test] + fn test_replace_ipc_method() { + let mut modules = + TransportRpcModules { ipc: Some(create_test_module()), ..Default::default() }; + + let mut other_module = RpcModule::new(()); + other_module.register_method("something", |_, _, _| "fails").unwrap(); + + assert!(modules.replace_ipc(other_module.clone()).unwrap()); + + assert!(modules.ipc.as_ref().unwrap().method("something").is_some()); + + other_module.register_method("anything", |_, _, _| "fails").unwrap(); + assert!(modules.replace_ipc(other_module.clone()).unwrap()); + + assert!(modules.ipc.as_ref().unwrap().method("anything").is_some()); + } + #[test] + fn test_replace_ws_method() { + let mut modules = + TransportRpcModules { ws: Some(create_test_module()), ..Default::default() }; + + let mut other_module = RpcModule::new(()); + other_module.register_method("something", |_, _, _| "fails").unwrap(); + + assert!(modules.replace_ws(other_module.clone()).unwrap()); + + assert!(modules.ws.as_ref().unwrap().method("something").is_some()); + + other_module.register_method("anything", |_, _, _| "fails").unwrap(); + assert!(modules.replace_ws(other_module.clone()).unwrap()); + + assert!(modules.ws.as_ref().unwrap().method("anything").is_some()); + } + + #[test] + fn test_replace_configured() { + let mut modules = TransportRpcModules { + http: Some(create_test_module()), + ws: Some(create_test_module()), + ipc: Some(create_test_module()), + ..Default::default() + }; + let mut other_module = RpcModule::new(()); + other_module.register_method("something", |_, _, _| "fails").unwrap(); + + assert!(modules.replace_configured(other_module).unwrap()); + + // Verify that the other_method was added + assert!(modules.http.as_ref().unwrap().method("something").is_some()); + assert!(modules.ipc.as_ref().unwrap().method("something").is_some()); + assert!(modules.ws.as_ref().unwrap().method("something").is_some()); + + assert!(modules.http.as_ref().unwrap().method("anything").is_some()); + assert!(modules.ipc.as_ref().unwrap().method("anything").is_some()); + assert!(modules.ws.as_ref().unwrap().method("anything").is_some()); } } From 7904cab4ff438ce4b20a2505ca2f433f72a211f3 Mon Sep 17 00:00:00 2001 From: Parikalp Bhardwaj <53660958+Parikalp-Bhardwaj@users.noreply.github.com> Date: Sat, 5 Oct 2024 14:11:29 +0400 Subject: [PATCH 026/159] Add metrics for failed deliveries to Grafana dashboard (#11481) --- etc/grafana/dashboards/overview.json | 73 ++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/etc/grafana/dashboards/overview.json b/etc/grafana/dashboards/overview.json index 76d45ead885d2..15786764f4299 100644 --- a/etc/grafana/dashboards/overview.json +++ b/etc/grafana/dashboards/overview.json @@ -5732,6 +5732,79 @@ "title": "Engine API getPayloadBodies Latency", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Counts the number of failed response deliveries due to client request termination.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none" + } + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 198 + }, + "id": 213, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "consensus_engine_beacon_failed_new_payload_response_deliveries{instance=~\"$instance\"}", + "legendFormat": "Failed NewPayload Deliveries", + "refId": "A" + }, + { + "expr": "consensus_engine_beacon_failed_forkchoice_updated_response_deliveries{instance=~\"$instance\"}", + "legendFormat": "Failed ForkchoiceUpdated Deliveries", + "refId": "B" + } + ], + "title": "Failed Engine API Response Deliveries", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { From d6c0b01d4c1f72d789d071fe7b0dac8d852ad3ac Mon Sep 17 00:00:00 2001 From: garwah <14845405+garwahl@users.noreply.github.com> Date: Sat, 5 Oct 2024 20:52:08 +1000 Subject: [PATCH 027/159] chore: Remove duplicate EthereumChainSpecParser in favor of existing EthChainSpecParser (#11412) Co-authored-by: garwah Co-authored-by: Matthias Seitz --- Cargo.lock | 13 +-- bin/reth/Cargo.toml | 1 + bin/reth/src/cli/mod.rs | 7 +- bin/reth/src/lib.rs | 1 + bin/reth/src/main.rs | 3 +- crates/cli/cli/Cargo.toml | 4 + crates/cli/cli/src/chainspec.rs | 20 +++- crates/cli/commands/Cargo.toml | 1 + crates/cli/commands/src/db/mod.rs | 2 +- crates/cli/commands/src/dump_genesis.rs | 2 +- crates/cli/commands/src/import.rs | 2 +- crates/cli/commands/src/node.rs | 7 +- crates/cli/commands/src/stage/unwind.rs | 2 +- crates/ethereum/cli/Cargo.toml | 7 -- crates/ethereum/cli/src/chainspec.rs | 43 +++----- crates/node/core/Cargo.toml | 6 -- crates/node/core/src/args/mod.rs | 2 - crates/node/core/src/args/utils.rs | 99 ------------------- crates/optimism/cli/src/chainspec.rs | 37 ++++--- crates/optimism/cli/src/lib.rs | 5 +- .../beacon-api-sidecar-fetcher/src/main.rs | 2 +- examples/beacon-api-sse/src/main.rs | 2 +- examples/custom-inspector/src/main.rs | 2 +- examples/node-custom-rpc/src/main.rs | 2 +- examples/txpool-tracing/src/main.rs | 2 +- 25 files changed, 85 insertions(+), 189 deletions(-) delete mode 100644 crates/node/core/src/args/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 73f500d9d22fe..e75ea0ec095c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6243,6 +6243,7 @@ dependencies = [ "reth-downloaders", "reth-engine-util", "reth-errors", + "reth-ethereum-cli", "reth-ethereum-payload-builder", "reth-evm", "reth-execution-types", @@ -6526,9 +6527,12 @@ dependencies = [ name = "reth-cli" version = "1.0.8" dependencies = [ + "alloy-genesis", "clap", "eyre", "reth-cli-runner", + "serde_json", + "shellexpand", ] [[package]] @@ -6565,6 +6569,7 @@ dependencies = [ "reth-downloaders", "reth-ecies", "reth-eth-wire", + "reth-ethereum-cli", "reth-evm", "reth-exex", "reth-fs-util", @@ -7229,14 +7234,11 @@ dependencies = [ name = "reth-ethereum-cli" version = "1.0.8" dependencies = [ - "alloy-genesis", "clap", "eyre", "reth-chainspec", "reth-cli", "reth-cli-commands", - "serde_json", - "shellexpand", ] [[package]] @@ -7843,7 +7845,6 @@ dependencies = [ name = "reth-node-core" version = "1.0.8" dependencies = [ - "alloy-genesis", "alloy-primitives", "alloy-rpc-types-engine", "clap", @@ -7856,19 +7857,16 @@ dependencies = [ "proptest", "rand 0.8.5", "reth-chainspec", - "reth-cli", "reth-cli-util", "reth-config", "reth-consensus-common", "reth-db", "reth-discv4", "reth-discv5", - "reth-fs-util", "reth-net-nat", "reth-network", "reth-network-p2p", "reth-network-peers", - "reth-optimism-chainspec", "reth-primitives", "reth-prune-types", "reth-rpc-api", @@ -7883,7 +7881,6 @@ dependencies = [ "reth-transaction-pool", "secp256k1", "serde", - "serde_json", "shellexpand", "strum", "tempfile", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 3d3cbd06f4d79..476f9cd5cec73 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -15,6 +15,7 @@ workspace = true [dependencies] # reth reth-cli.workspace = true +reth-ethereum-cli.workspace = true reth-chainspec.workspace = true reth-config.workspace = true reth-primitives.workspace = true diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index 8bb0971ec0817..cca801da36b27 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -15,8 +15,8 @@ use reth_cli_commands::{ }; use reth_cli_runner::CliRunner; use reth_db::DatabaseEnv; +use reth_ethereum_cli::chainspec::EthereumChainSpecParser; use reth_node_builder::{NodeBuilder, WithLaunchContext}; -use reth_node_core::args::utils::EthereumChainSpecParser; use reth_node_ethereum::{EthExecutorProvider, EthereumNode}; use reth_tracing::FileWorkerGuard; use std::{ffi::OsString, fmt, future::Future, sync::Arc}; @@ -117,7 +117,8 @@ impl, Ext: clap::Args + fmt::Debug> Cl /// /// ```no_run /// use clap::Parser; - /// use reth::{args::utils::EthereumChainSpecParser, cli::Cli}; + /// use reth::cli::Cli; + /// use reth_ethereum_cli::chainspec::EthereumChainSpecParser; /// /// #[derive(Debug, Parser)] /// pub struct MyArgs { @@ -238,7 +239,7 @@ mod tests { use super::*; use crate::args::ColorMode; use clap::CommandFactory; - use reth_node_core::args::utils::SUPPORTED_CHAINS; + use reth_ethereum_cli::chainspec::SUPPORTED_CHAINS; #[test] fn parse_color_mode() { diff --git a/bin/reth/src/lib.rs b/bin/reth/src/lib.rs index 4ff2faa1d65f7..6b71f48de123e 100644 --- a/bin/reth/src/lib.rs +++ b/bin/reth/src/lib.rs @@ -90,6 +90,7 @@ pub mod dirs { /// Re-exported from `reth_chainspec` pub mod chainspec { pub use reth_chainspec::*; + pub use reth_ethereum_cli::chainspec::*; } /// Re-exported from `reth_provider`. diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index 1be2d0efedcc3..7153cdcc6c885 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -4,7 +4,8 @@ static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator(); use clap::{Args, Parser}; -use reth::{args::utils::EthereumChainSpecParser, cli::Cli}; +use reth::cli::Cli; +use reth_ethereum_cli::chainspec::EthereumChainSpecParser; use reth_node_builder::{ engine_tree_config::{ TreeConfig, DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, DEFAULT_PERSISTENCE_THRESHOLD, diff --git a/crates/cli/cli/Cargo.toml b/crates/cli/cli/Cargo.toml index e8f7a1dcbe15a..7eb1f43b1e58a 100644 --- a/crates/cli/cli/Cargo.toml +++ b/crates/cli/cli/Cargo.toml @@ -15,9 +15,13 @@ workspace = true # reth reth-cli-runner.workspace = true +alloy-genesis.workspace = true + # misc clap.workspace = true +shellexpand.workspace = true eyre.workspace = true +serde_json.workspace = true diff --git a/crates/cli/cli/src/chainspec.rs b/crates/cli/cli/src/chainspec.rs index 63705bd28f4a6..8432009409f70 100644 --- a/crates/cli/cli/src/chainspec.rs +++ b/crates/cli/cli/src/chainspec.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{fs, path::PathBuf, sync::Arc}; use clap::builder::TypedValueParser; @@ -61,3 +61,21 @@ pub trait ChainSpecParser: Clone + Send + Sync + 'static { format!("The chain this node is running.\nPossible values are either a built-in chain or the path to a chain specification file.\n\nBuilt-in chains:\n {}", Self::SUPPORTED_CHAINS.join(", ")) } } + +/// A helper to parse a [`Genesis`](alloy_genesis::Genesis) as argument or from disk. +pub fn parse_genesis(s: &str) -> eyre::Result { + // try to read json from path first + let raw = match fs::read_to_string(PathBuf::from(shellexpand::full(s)?.into_owned())) { + Ok(raw) => raw, + Err(io_err) => { + // valid json may start with "\n", but must contain "{" + if s.contains('{') { + s.to_string() + } else { + return Err(io_err.into()) // assume invalid path + } + } + }; + + Ok(serde_json::from_str(&raw)?) +} diff --git a/crates/cli/commands/Cargo.toml b/crates/cli/commands/Cargo.toml index 4835c3d0fa29c..e307859dfd86f 100644 --- a/crates/cli/commands/Cargo.toml +++ b/crates/cli/commands/Cargo.toml @@ -14,6 +14,7 @@ repository.workspace = true reth-beacon-consensus.workspace = true reth-chainspec.workspace = true reth-cli.workspace = true +reth-ethereum-cli.workspace = true reth-cli-runner.workspace = true reth-cli-util.workspace = true reth-config.workspace = true diff --git a/crates/cli/commands/src/db/mod.rs b/crates/cli/commands/src/db/mod.rs index 1c000f56bc250..e1a9a90bacc3f 100644 --- a/crates/cli/commands/src/db/mod.rs +++ b/crates/cli/commands/src/db/mod.rs @@ -160,7 +160,7 @@ impl> Command #[cfg(test)] mod tests { use super::*; - use reth_node_core::args::utils::{EthereumChainSpecParser, SUPPORTED_CHAINS}; + use reth_ethereum_cli::chainspec::{EthereumChainSpecParser, SUPPORTED_CHAINS}; use std::path::Path; #[test] diff --git a/crates/cli/commands/src/dump_genesis.rs b/crates/cli/commands/src/dump_genesis.rs index a5c0675cc7e81..c3e7bb217b57d 100644 --- a/crates/cli/commands/src/dump_genesis.rs +++ b/crates/cli/commands/src/dump_genesis.rs @@ -32,7 +32,7 @@ impl> DumpGenesisCommand { #[cfg(test)] mod tests { use super::*; - use reth_node_core::args::utils::{EthereumChainSpecParser, SUPPORTED_CHAINS}; + use reth_ethereum_cli::chainspec::{EthereumChainSpecParser, SUPPORTED_CHAINS}; #[test] fn parse_dump_genesis_command_chain_args() { diff --git a/crates/cli/commands/src/import.rs b/crates/cli/commands/src/import.rs index 31c6cdc691570..6b750d32a3df6 100644 --- a/crates/cli/commands/src/import.rs +++ b/crates/cli/commands/src/import.rs @@ -231,7 +231,7 @@ where #[cfg(test)] mod tests { use super::*; - use reth_node_core::args::utils::{EthereumChainSpecParser, SUPPORTED_CHAINS}; + use reth_ethereum_cli::chainspec::{EthereumChainSpecParser, SUPPORTED_CHAINS}; #[test] fn parse_common_import_command_chain_args() { diff --git a/crates/cli/commands/src/node.rs b/crates/cli/commands/src/node.rs index fe49b769a3d31..5b1a87e068b35 100644 --- a/crates/cli/commands/src/node.rs +++ b/crates/cli/commands/src/node.rs @@ -6,11 +6,12 @@ use reth_cli::chainspec::ChainSpecParser; use reth_cli_runner::CliContext; use reth_cli_util::parse_socket_address; use reth_db::{init_db, DatabaseEnv}; +use reth_ethereum_cli::chainspec::EthereumChainSpecParser; use reth_node_builder::{NodeBuilder, WithLaunchContext}; use reth_node_core::{ args::{ - utils::EthereumChainSpecParser, DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, NetworkArgs, - PayloadBuilderArgs, PruningArgs, RpcServerArgs, TxPoolArgs, + DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, NetworkArgs, PayloadBuilderArgs, + PruningArgs, RpcServerArgs, TxPoolArgs, }, node_config::NodeConfig, version, @@ -210,7 +211,7 @@ pub struct NoArgs; mod tests { use super::*; use reth_discv4::DEFAULT_DISCOVERY_PORT; - use reth_node_core::args::utils::SUPPORTED_CHAINS; + use reth_ethereum_cli::chainspec::SUPPORTED_CHAINS; use std::{ net::{IpAddr, Ipv4Addr}, path::Path, diff --git a/crates/cli/commands/src/stage/unwind.rs b/crates/cli/commands/src/stage/unwind.rs index ae3ae25008746..19305554eaa3f 100644 --- a/crates/cli/commands/src/stage/unwind.rs +++ b/crates/cli/commands/src/stage/unwind.rs @@ -213,7 +213,7 @@ impl Subcommands { #[cfg(test)] mod tests { - use reth_node_core::args::utils::EthereumChainSpecParser; + use reth_ethereum_cli::chainspec::EthereumChainSpecParser; use super::*; diff --git a/crates/ethereum/cli/Cargo.toml b/crates/ethereum/cli/Cargo.toml index b5fadc684b84d..108f42f539b90 100644 --- a/crates/ethereum/cli/Cargo.toml +++ b/crates/ethereum/cli/Cargo.toml @@ -15,13 +15,6 @@ workspace = true reth-cli.workspace = true reth-chainspec.workspace = true -# ethereum -alloy-genesis.workspace = true - -# io -shellexpand.workspace = true -serde_json.workspace = true - # misc eyre.workspace = true diff --git a/crates/ethereum/cli/src/chainspec.rs b/crates/ethereum/cli/src/chainspec.rs index 05db177df8320..cbcce9f69f61a 100644 --- a/crates/ethereum/cli/src/chainspec.rs +++ b/crates/ethereum/cli/src/chainspec.rs @@ -1,48 +1,33 @@ -use alloy_genesis::Genesis; use reth_chainspec::{ChainSpec, DEV, HOLESKY, MAINNET, SEPOLIA}; -use reth_cli::chainspec::ChainSpecParser; -use std::{fs, path::PathBuf, sync::Arc}; +use reth_cli::chainspec::{parse_genesis, ChainSpecParser}; +use std::sync::Arc; + +/// Chains supported by reth. First value should be used as the default. +pub const SUPPORTED_CHAINS: &[&str] = &["mainnet", "sepolia", "holesky", "dev"]; /// Clap value parser for [`ChainSpec`]s. /// /// The value parser matches either a known chain, the path /// to a json file, or a json formatted string in-memory. The json needs to be a Genesis struct. -fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { +pub fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { Ok(match s { "mainnet" => MAINNET.clone(), "sepolia" => SEPOLIA.clone(), "holesky" => HOLESKY.clone(), "dev" => DEV.clone(), - _ => { - // try to read json from path first - let raw = match fs::read_to_string(PathBuf::from(shellexpand::full(s)?.into_owned())) { - Ok(raw) => raw, - Err(io_err) => { - // valid json may start with "\n", but must contain "{" - if s.contains('{') { - s.to_string() - } else { - return Err(io_err.into()) // assume invalid path - } - } - }; - - // both serialized Genesis and ChainSpec structs supported - let genesis: Genesis = serde_json::from_str(&raw)?; - - Arc::new(genesis.into()) - } + _ => Arc::new(parse_genesis(s)?.into()), }) } /// Ethereum chain specification parser. #[derive(Debug, Clone, Default)] -pub struct EthChainSpecParser; +#[non_exhaustive] +pub struct EthereumChainSpecParser; -impl ChainSpecParser for EthChainSpecParser { +impl ChainSpecParser for EthereumChainSpecParser { type ChainSpec = ChainSpec; - const SUPPORTED_CHAINS: &'static [&'static str] = &["mainnet", "sepolia", "holesky", "dev"]; + const SUPPORTED_CHAINS: &'static [&'static str] = SUPPORTED_CHAINS; fn parse(s: &str) -> eyre::Result> { chain_value_parser(s) @@ -56,8 +41,8 @@ mod tests { #[test] fn parse_known_chain_spec() { - for &chain in EthChainSpecParser::SUPPORTED_CHAINS { - assert!(::parse(chain).is_ok()); + for &chain in EthereumChainSpecParser::SUPPORTED_CHAINS { + assert!(::parse(chain).is_ok()); } } @@ -108,7 +93,7 @@ mod tests { } }"#; - let spec = ::parse(s).unwrap(); + let spec = ::parse(s).unwrap(); assert!(spec.is_shanghai_active_at_timestamp(0)); assert!(spec.is_cancun_active_at_timestamp(0)); assert!(spec.is_prague_active_at_timestamp(0)); diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index 35802cf5165a5..46476273e42d3 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -14,9 +14,7 @@ workspace = true # reth reth-chainspec.workspace = true reth-primitives.workspace = true -reth-cli.workspace = true reth-cli-util.workspace = true -reth-fs-util.workspace = true reth-db = { workspace = true, features = ["mdbx"] } reth-storage-errors.workspace = true reth-storage-api.workspace = true @@ -37,10 +35,8 @@ reth-network-peers.workspace = true reth-consensus-common.workspace = true reth-prune-types.workspace = true reth-stages-types.workspace = true -reth-optimism-chainspec = { workspace = true, optional = true } # ethereum -alloy-genesis.workspace = true alloy-primitives.workspace = true alloy-rpc-types-engine = { workspace = true, features = ["jwt"] } @@ -59,7 +55,6 @@ thiserror.workspace = true # io dirs-next = "2.0.0" shellexpand.workspace = true -serde_json.workspace = true # tracing tracing.workspace = true @@ -85,7 +80,6 @@ optimism = [ "reth-primitives/optimism", "reth-rpc-types-compat/optimism", "reth-rpc-eth-api/optimism", - "dep:reth-optimism-chainspec", ] # Features for vergen to generate correct env vars jemalloc = [] diff --git a/crates/node/core/src/args/mod.rs b/crates/node/core/src/args/mod.rs index 1a647ac65b031..7f1b643615156 100644 --- a/crates/node/core/src/args/mod.rs +++ b/crates/node/core/src/args/mod.rs @@ -56,7 +56,5 @@ pub use datadir_args::DatadirArgs; mod benchmark_args; pub use benchmark_args::BenchmarkArgs; -pub mod utils; - mod error; pub mod types; diff --git a/crates/node/core/src/args/utils.rs b/crates/node/core/src/args/utils.rs deleted file mode 100644 index e6ebda45b7f71..0000000000000 --- a/crates/node/core/src/args/utils.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! Clap parser utilities - -use std::{path::PathBuf, sync::Arc}; - -use alloy_genesis::Genesis; -use reth_chainspec::ChainSpec; -#[cfg(not(feature = "optimism"))] -use reth_chainspec::{DEV, HOLESKY, MAINNET, SEPOLIA}; -use reth_cli::chainspec::ChainSpecParser; -use reth_fs_util as fs; -#[cfg(feature = "optimism")] -use reth_optimism_chainspec::{BASE_MAINNET, BASE_SEPOLIA, OP_DEV, OP_MAINNET, OP_SEPOLIA}; - -#[cfg(feature = "optimism")] -/// Chains supported by op-reth. First value should be used as the default. -pub const SUPPORTED_CHAINS: &[&str] = - &["optimism", "optimism-sepolia", "base", "base-sepolia", "dev"]; -#[cfg(not(feature = "optimism"))] -/// Chains supported by reth. First value should be used as the default. -pub const SUPPORTED_CHAINS: &[&str] = &["mainnet", "sepolia", "holesky", "dev"]; - -/// Clap value parser for [`ChainSpec`]s. -/// -/// The value parser matches either a known chain, the path -/// to a json file, or a json formatted string in-memory. The json needs to be a Genesis struct. -#[cfg(not(feature = "optimism"))] -pub fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { - Ok(match s { - "mainnet" => MAINNET.clone(), - "sepolia" => SEPOLIA.clone(), - "holesky" => HOLESKY.clone(), - "dev" => DEV.clone(), - _ => Arc::new(parse_custom_chain_spec(s)?), - }) -} - -/// Clap value parser for [`OpChainSpec`](reth_optimism_chainspec::OpChainSpec)s. -/// -/// The value parser matches either a known chain, the path -/// to a json file, or a json formatted string in-memory. The json needs to be a Genesis struct. -#[cfg(feature = "optimism")] -pub fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { - Ok(Arc::new(match s { - "optimism" => OP_MAINNET.inner.clone(), - "optimism_sepolia" | "optimism-sepolia" => OP_SEPOLIA.inner.clone(), - "base" => BASE_MAINNET.inner.clone(), - "base_sepolia" | "base-sepolia" => BASE_SEPOLIA.inner.clone(), - "dev" => OP_DEV.inner.clone(), - _ => parse_custom_chain_spec(s)?, - })) -} - -/// Parses a custom [`ChainSpec`]. -pub fn parse_custom_chain_spec(s: &str) -> eyre::Result { - // try to read json from path first - let raw = match fs::read_to_string(PathBuf::from(shellexpand::full(s)?.into_owned())) { - Ok(raw) => raw, - Err(io_err) => { - // valid json may start with "\n", but must contain "{" - if s.contains('{') { - s.to_string() - } else { - return Err(io_err.into()) // assume invalid path - } - } - }; - - // both serialized Genesis and ChainSpec structs supported - let genesis: Genesis = serde_json::from_str(&raw)?; - - Ok(genesis.into()) -} - -/// A chain specification parser for ethereum chains. -#[derive(Debug, Copy, Clone, Default)] -#[non_exhaustive] -pub struct EthereumChainSpecParser; - -impl ChainSpecParser for EthereumChainSpecParser { - type ChainSpec = ChainSpec; - - const SUPPORTED_CHAINS: &'static [&'static str] = SUPPORTED_CHAINS; - - fn parse(s: &str) -> eyre::Result> { - chain_value_parser(s) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_known_chain_spec() { - for chain in SUPPORTED_CHAINS { - chain_value_parser(chain).unwrap(); - } - } -} diff --git a/crates/optimism/cli/src/chainspec.rs b/crates/optimism/cli/src/chainspec.rs index e76bfd5f0656c..329669ab8c9cd 100644 --- a/crates/optimism/cli/src/chainspec.rs +++ b/crates/optimism/cli/src/chainspec.rs @@ -1,28 +1,12 @@ -use std::sync::Arc; - -use reth_cli::chainspec::ChainSpecParser; -use reth_node_core::args::utils::parse_custom_chain_spec; +use reth_cli::chainspec::{parse_genesis, ChainSpecParser}; use reth_optimism_chainspec::{ OpChainSpec, BASE_MAINNET, BASE_SEPOLIA, OP_DEV, OP_MAINNET, OP_SEPOLIA, }; - -/// Clap value parser for [`OpChainSpec`]s. -/// -/// The value parser matches either a known chain, the path -/// to a json file, or a json formatted string in-memory. The json needs to be a Genesis struct. -fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { - Ok(match s { - "dev" => OP_DEV.clone(), - "optimism" => OP_MAINNET.clone(), - "optimism_sepolia" | "optimism-sepolia" => OP_SEPOLIA.clone(), - "base" => BASE_MAINNET.clone(), - "base_sepolia" | "base-sepolia" => BASE_SEPOLIA.clone(), - _ => Arc::new(OpChainSpec { inner: parse_custom_chain_spec(s)? }), - }) -} +use std::sync::Arc; /// Optimism chain specification parser. #[derive(Debug, Clone, Default)] +#[non_exhaustive] pub struct OpChainSpecParser; impl ChainSpecParser for OpChainSpecParser { @@ -43,6 +27,21 @@ impl ChainSpecParser for OpChainSpecParser { } } +/// Clap value parser for [`OpChainSpec`]s. +/// +/// The value parser matches either a known chain, the path +/// to a json file, or a json formatted string in-memory. The json needs to be a Genesis struct. +pub fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { + Ok(match s { + "dev" => OP_DEV.clone(), + "optimism" => OP_MAINNET.clone(), + "optimism_sepolia" | "optimism-sepolia" => OP_SEPOLIA.clone(), + "base" => BASE_MAINNET.clone(), + "base_sepolia" | "base-sepolia" => BASE_SEPOLIA.clone(), + _ => Arc::new(parse_genesis(s)?.into()), + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/optimism/cli/src/lib.rs b/crates/optimism/cli/src/lib.rs index b17cefa63287a..74bc80c426ef9 100644 --- a/crates/optimism/cli/src/lib.rs +++ b/crates/optimism/cli/src/lib.rs @@ -176,13 +176,14 @@ where #[cfg(test)] mod test { + use crate::chainspec::OpChainSpecParser; use clap::Parser; - use reth_cli_commands::NodeCommand; + use reth_cli_commands::{node::NoArgs, NodeCommand}; use reth_optimism_chainspec::OP_DEV; #[test] fn parse_dev() { - let cmd: NodeCommand = NodeCommand::parse_from(["op-reth", "--dev"]); + let cmd = NodeCommand::::parse_from(["op-reth", "--dev"]); let chain = OP_DEV.clone(); assert_eq!(cmd.chain.chain, chain.chain); assert_eq!(cmd.chain.genesis_hash, chain.genesis_hash); diff --git a/examples/beacon-api-sidecar-fetcher/src/main.rs b/examples/beacon-api-sidecar-fetcher/src/main.rs index 7d8880ca185cc..a0b9b6e01ec8c 100644 --- a/examples/beacon-api-sidecar-fetcher/src/main.rs +++ b/examples/beacon-api-sidecar-fetcher/src/main.rs @@ -23,7 +23,7 @@ use clap::Parser; use futures_util::{stream::FuturesUnordered, StreamExt}; use mined_sidecar::MinedSidecarStream; use reth::{ - args::utils::EthereumChainSpecParser, builder::NodeHandle, cli::Cli, + builder::NodeHandle, chainspec::EthereumChainSpecParser, cli::Cli, providers::CanonStateSubscriptions, }; use reth_node_ethereum::EthereumNode; diff --git a/examples/beacon-api-sse/src/main.rs b/examples/beacon-api-sse/src/main.rs index 81535ef6140b5..243511d4960ee 100644 --- a/examples/beacon-api-sse/src/main.rs +++ b/examples/beacon-api-sse/src/main.rs @@ -21,7 +21,7 @@ use alloy_rpc_types_beacon::events::PayloadAttributesEvent; use clap::Parser; use futures_util::stream::StreamExt; use mev_share_sse::{client::EventStream, EventClient}; -use reth::{args::utils::EthereumChainSpecParser, cli::Cli}; +use reth::{chainspec::EthereumChainSpecParser, cli::Cli}; use reth_node_ethereum::EthereumNode; use std::net::{IpAddr, Ipv4Addr}; use tracing::{info, warn}; diff --git a/examples/custom-inspector/src/main.rs b/examples/custom-inspector/src/main.rs index 700de274e6810..12b7620f4adcf 100644 --- a/examples/custom-inspector/src/main.rs +++ b/examples/custom-inspector/src/main.rs @@ -15,8 +15,8 @@ use alloy_rpc_types::state::EvmOverrides; use clap::Parser; use futures_util::StreamExt; use reth::{ - args::utils::EthereumChainSpecParser, builder::NodeHandle, + chainspec::EthereumChainSpecParser, cli::Cli, primitives::BlockNumberOrTag, revm::{ diff --git a/examples/node-custom-rpc/src/main.rs b/examples/node-custom-rpc/src/main.rs index 5aeecfd2915c0..92e0bfea26e9b 100644 --- a/examples/node-custom-rpc/src/main.rs +++ b/examples/node-custom-rpc/src/main.rs @@ -14,7 +14,7 @@ use clap::Parser; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; -use reth::{args::utils::EthereumChainSpecParser, cli::Cli}; +use reth::{chainspec::EthereumChainSpecParser, cli::Cli}; use reth_node_ethereum::EthereumNode; use reth_transaction_pool::TransactionPool; diff --git a/examples/txpool-tracing/src/main.rs b/examples/txpool-tracing/src/main.rs index f8c2e19d203a1..94f800987a967 100644 --- a/examples/txpool-tracing/src/main.rs +++ b/examples/txpool-tracing/src/main.rs @@ -15,7 +15,7 @@ use alloy_rpc_types_trace::{parity::TraceType, tracerequest::TraceCallRequest}; use clap::Parser; use futures_util::StreamExt; use reth::{ - args::utils::EthereumChainSpecParser, builder::NodeHandle, cli::Cli, + builder::NodeHandle, chainspec::EthereumChainSpecParser, cli::Cli, rpc::compat::transaction::transaction_to_call_request, transaction_pool::TransactionPool, }; use reth_node_ethereum::node::EthereumNode; From ecd899c119b5cb083676933b35c27995787083b7 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sat, 5 Oct 2024 13:08:08 +0200 Subject: [PATCH 028/159] chore(lint): fix `clippy::needles_lifetimes` (#11496) --- crates/blockchain-tree/src/bundle.rs | 4 +-- .../ethereum/engine-primitives/src/payload.rs | 2 +- crates/evm/execution-types/src/chain.rs | 8 +++--- crates/evm/src/system_calls/mod.rs | 2 +- crates/exex/types/src/notification.rs | 2 +- crates/net/network/src/peers.rs | 2 +- crates/node/builder/src/rpc.rs | 2 +- crates/optimism/payload/src/payload.rs | 2 +- crates/payload/builder/src/database.rs | 4 +-- .../prune/src/segments/static_file/headers.rs | 2 +- crates/rpc/rpc-eth-types/src/cache/db.rs | 16 +++++------ crates/rpc/rpc-layer/src/auth_layer.rs | 2 +- crates/rpc/rpc-testing-util/src/debug.rs | 8 +++--- crates/rpc/rpc-testing-util/src/trace.rs | 28 +++++++++---------- .../src/providers/state/historical.rs | 12 ++++---- .../provider/src/providers/state/latest.rs | 12 ++++---- .../provider/src/providers/static_file/jar.rs | 10 +++---- .../src/providers/static_file/writer.rs | 4 +-- .../storage/provider/src/writer/database.rs | 2 +- crates/storage/provider/src/writer/mod.rs | 10 +++---- .../provider/src/writer/static_file.rs | 2 +- crates/transaction-pool/src/traits.rs | 2 +- crates/trie/db/src/hashed_cursor.rs | 4 +-- crates/trie/db/src/prefix_set.rs | 2 +- crates/trie/db/src/trie_cursor.rs | 4 +-- crates/trie/trie/src/trie_cursor/in_memory.rs | 2 +- 26 files changed, 75 insertions(+), 75 deletions(-) diff --git a/crates/blockchain-tree/src/bundle.rs b/crates/blockchain-tree/src/bundle.rs index 6f62d4136bb76..3745753d3f473 100644 --- a/crates/blockchain-tree/src/bundle.rs +++ b/crates/blockchain-tree/src/bundle.rs @@ -18,7 +18,7 @@ pub struct BundleStateDataRef<'a> { pub canonical_fork: ForkBlock, } -impl<'a> ExecutionDataProvider for BundleStateDataRef<'a> { +impl ExecutionDataProvider for BundleStateDataRef<'_> { fn execution_outcome(&self) -> &ExecutionOutcome { self.execution_outcome } @@ -33,7 +33,7 @@ impl<'a> ExecutionDataProvider for BundleStateDataRef<'a> { } } -impl<'a> BlockExecutionForkProvider for BundleStateDataRef<'a> { +impl BlockExecutionForkProvider for BundleStateDataRef<'_> { fn canonical_fork(&self) -> ForkBlock { self.canonical_fork } diff --git a/crates/ethereum/engine-primitives/src/payload.rs b/crates/ethereum/engine-primitives/src/payload.rs index dd0b7b405e9fc..ae370fdb9d7bb 100644 --- a/crates/ethereum/engine-primitives/src/payload.rs +++ b/crates/ethereum/engine-primitives/src/payload.rs @@ -89,7 +89,7 @@ impl BuiltPayload for EthBuiltPayload { } } -impl<'a> BuiltPayload for &'a EthBuiltPayload { +impl BuiltPayload for &EthBuiltPayload { fn block(&self) -> &SealedBlock { (**self).block() } diff --git a/crates/evm/execution-types/src/chain.rs b/crates/evm/execution-types/src/chain.rs index 5db5495de59ff..467bd4c0ec7cf 100644 --- a/crates/evm/execution-types/src/chain.rs +++ b/crates/evm/execution-types/src/chain.rs @@ -355,7 +355,7 @@ impl Chain { #[derive(Debug)] pub struct DisplayBlocksChain<'a>(pub &'a BTreeMap); -impl<'a> fmt::Display for DisplayBlocksChain<'a> { +impl fmt::Display for DisplayBlocksChain<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut list = f.debug_list(); let mut values = self.0.values().map(|block| block.num_hash()); @@ -376,7 +376,7 @@ pub struct ChainBlocks<'a> { blocks: Cow<'a, BTreeMap>, } -impl<'a> ChainBlocks<'a> { +impl ChainBlocks<'_> { /// Creates a consuming iterator over all blocks in the chain with increasing block number. /// /// Note: this always yields at least one block. @@ -442,7 +442,7 @@ impl<'a> ChainBlocks<'a> { } } -impl<'a> IntoIterator for ChainBlocks<'a> { +impl IntoIterator for ChainBlocks<'_> { type Item = (BlockNumber, SealedBlockWithSenders); type IntoIter = std::collections::btree_map::IntoIter; @@ -571,7 +571,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for Chain<'a> { + impl SerializeAs for Chain<'_> { fn serialize_as(source: &super::Chain, serializer: S) -> Result where S: Serializer, diff --git a/crates/evm/src/system_calls/mod.rs b/crates/evm/src/system_calls/mod.rs index 48429cb49596e..43baa1c766c29 100644 --- a/crates/evm/src/system_calls/mod.rs +++ b/crates/evm/src/system_calls/mod.rs @@ -85,7 +85,7 @@ where .build() } -impl<'a, EvmConfig, Chainspec, Hook> SystemCaller<'a, EvmConfig, Chainspec, Hook> +impl SystemCaller<'_, EvmConfig, Chainspec, Hook> where EvmConfig: ConfigureEvm

, Chainspec: EthereumHardforks, diff --git a/crates/exex/types/src/notification.rs b/crates/exex/types/src/notification.rs index 53411250270d8..61d42a3319be3 100644 --- a/crates/exex/types/src/notification.rs +++ b/crates/exex/types/src/notification.rs @@ -136,7 +136,7 @@ pub(super) mod serde_bincode_compat { } } - impl<'a> SerializeAs for ExExNotification<'a> { + impl SerializeAs for ExExNotification<'_> { fn serialize_as( source: &super::ExExNotification, serializer: S, diff --git a/crates/net/network/src/peers.rs b/crates/net/network/src/peers.rs index b9196d29e8e4d..3d5ff7a0d43ac 100644 --- a/crates/net/network/src/peers.rs +++ b/crates/net/network/src/peers.rs @@ -1131,7 +1131,7 @@ mod tests { peers: &'a mut PeersManager, } - impl<'a> Future for PeerActionFuture<'a> { + impl Future for PeerActionFuture<'_> { type Output = PeerAction; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index 607a33147e940..cb8d8f355dac9 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -254,7 +254,7 @@ pub struct RpcContext<'a, Node: FullNodeComponents, EthApi: EthApiTypes> { pub auth_module: &'a mut AuthRpcModule, } -impl<'a, Node, EthApi> RpcContext<'a, Node, EthApi> +impl RpcContext<'_, Node, EthApi> where Node: FullNodeComponents, EthApi: EthApiTypes, diff --git a/crates/optimism/payload/src/payload.rs b/crates/optimism/payload/src/payload.rs index cb3b939136f30..f1ba24435092c 100644 --- a/crates/optimism/payload/src/payload.rs +++ b/crates/optimism/payload/src/payload.rs @@ -183,7 +183,7 @@ impl BuiltPayload for OptimismBuiltPayload { } } -impl<'a> BuiltPayload for &'a OptimismBuiltPayload { +impl BuiltPayload for &OptimismBuiltPayload { fn block(&self) -> &SealedBlock { (**self).block() } diff --git a/crates/payload/builder/src/database.rs b/crates/payload/builder/src/database.rs index ea1ae08543487..d63f7322dee21 100644 --- a/crates/payload/builder/src/database.rs +++ b/crates/payload/builder/src/database.rs @@ -67,7 +67,7 @@ pub struct CachedReadsDbMut<'a, DB> { pub db: DB, } -impl<'a, DB: DatabaseRef> Database for CachedReadsDbMut<'a, DB> { +impl Database for CachedReadsDbMut<'_, DB> { type Error = ::Error; fn basic(&mut self, address: Address) -> Result, Self::Error> { @@ -130,7 +130,7 @@ pub struct CachedReadsDBRef<'a, DB> { pub inner: RefCell>, } -impl<'a, DB: DatabaseRef> DatabaseRef for CachedReadsDBRef<'a, DB> { +impl DatabaseRef for CachedReadsDBRef<'_, DB> { type Error = ::Error; fn basic_ref(&self, address: Address) -> Result, Self::Error> { diff --git a/crates/prune/prune/src/segments/static_file/headers.rs b/crates/prune/prune/src/segments/static_file/headers.rs index a3daf504e667c..8700a653b1115 100644 --- a/crates/prune/prune/src/segments/static_file/headers.rs +++ b/crates/prune/prune/src/segments/static_file/headers.rs @@ -137,7 +137,7 @@ where } } -impl<'a, Provider> Iterator for HeaderTablesIter<'a, Provider> +impl Iterator for HeaderTablesIter<'_, Provider> where Provider: DBProvider, { diff --git a/crates/rpc/rpc-eth-types/src/cache/db.rs b/crates/rpc/rpc-eth-types/src/cache/db.rs index ad9804893a706..9731e3845768d 100644 --- a/crates/rpc/rpc-eth-types/src/cache/db.rs +++ b/crates/rpc/rpc-eth-types/src/cache/db.rs @@ -20,7 +20,7 @@ pub type StateCacheDb<'a> = CacheDB(pub &'a dyn StateProvider); -impl<'a> reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper<'a> { +impl reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper<'_> { fn state_root( &self, hashed_state: reth_trie::HashedPostState, @@ -50,7 +50,7 @@ impl<'a> reth_storage_api::StateRootProvider for StateProviderTraitObjWrapper<'a } } -impl<'a> reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper<'a> { +impl reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper<'_> { fn storage_root( &self, address: Address, @@ -60,7 +60,7 @@ impl<'a> reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper< } } -impl<'a> reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<'a> { +impl reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<'_> { fn proof( &self, input: reth_trie::TrieInput, @@ -88,7 +88,7 @@ impl<'a> reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<' } } -impl<'a> reth_storage_api::AccountReader for StateProviderTraitObjWrapper<'a> { +impl reth_storage_api::AccountReader for StateProviderTraitObjWrapper<'_> { fn basic_account( &self, address: revm_primitives::Address, @@ -97,7 +97,7 @@ impl<'a> reth_storage_api::AccountReader for StateProviderTraitObjWrapper<'a> { } } -impl<'a> reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'a> { +impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'_> { fn block_hash( &self, block_number: alloy_primitives::BlockNumber, @@ -121,7 +121,7 @@ impl<'a> reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper<'a> } } -impl<'a> StateProvider for StateProviderTraitObjWrapper<'a> { +impl StateProvider for StateProviderTraitObjWrapper<'_> { fn account_balance( &self, addr: revm_primitives::Address, @@ -164,7 +164,7 @@ impl<'a> StateProvider for StateProviderTraitObjWrapper<'a> { #[allow(missing_debug_implementations)] pub struct StateCacheDbRefMutWrapper<'a, 'b>(pub &'b mut StateCacheDb<'a>); -impl<'a, 'b> Database for StateCacheDbRefMutWrapper<'a, 'b> { +impl<'a> Database for StateCacheDbRefMutWrapper<'a, '_> { type Error = as Database>::Error; fn basic( &mut self, @@ -190,7 +190,7 @@ impl<'a, 'b> Database for StateCacheDbRefMutWrapper<'a, 'b> { } } -impl<'a, 'b> DatabaseRef for StateCacheDbRefMutWrapper<'a, 'b> { +impl<'a> DatabaseRef for StateCacheDbRefMutWrapper<'a, '_> { type Error = as Database>::Error; fn basic_ref( diff --git a/crates/rpc/rpc-layer/src/auth_layer.rs b/crates/rpc/rpc-layer/src/auth_layer.rs index 41ebce32dfb35..cdca181cbd061 100644 --- a/crates/rpc/rpc-layer/src/auth_layer.rs +++ b/crates/rpc/rpc-layer/src/auth_layer.rs @@ -176,7 +176,7 @@ mod tests { missing_jwt_error().await; wrong_jwt_signature_error().await; invalid_issuance_timestamp_error().await; - jwt_decode_error().await; + jwt_decode_error().await } async fn valid_jwt() { diff --git a/crates/rpc/rpc-testing-util/src/debug.rs b/crates/rpc/rpc-testing-util/src/debug.rs index cdcb454a2486e..f50064e80ce95 100644 --- a/crates/rpc/rpc-testing-util/src/debug.rs +++ b/crates/rpc/rpc-testing-util/src/debug.rs @@ -304,7 +304,7 @@ impl<'a> DebugTraceTransactionsStream<'a> { } } -impl<'a> Stream for DebugTraceTransactionsStream<'a> { +impl Stream for DebugTraceTransactionsStream<'_> { type Item = TraceTransactionResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -312,7 +312,7 @@ impl<'a> Stream for DebugTraceTransactionsStream<'a> { } } -impl<'a> std::fmt::Debug for DebugTraceTransactionsStream<'a> { +impl std::fmt::Debug for DebugTraceTransactionsStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DebugTraceTransactionsStream").finish_non_exhaustive() } @@ -336,7 +336,7 @@ impl<'a> DebugTraceBlockStream<'a> { } } -impl<'a> Stream for DebugTraceBlockStream<'a> { +impl Stream for DebugTraceBlockStream<'_> { type Item = DebugTraceBlockResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -344,7 +344,7 @@ impl<'a> Stream for DebugTraceBlockStream<'a> { } } -impl<'a> std::fmt::Debug for DebugTraceBlockStream<'a> { +impl std::fmt::Debug for DebugTraceBlockStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DebugTraceBlockStream").finish_non_exhaustive() } diff --git a/crates/rpc/rpc-testing-util/src/trace.rs b/crates/rpc/rpc-testing-util/src/trace.rs index 13914a59eb3eb..c6dc16cf10635 100644 --- a/crates/rpc/rpc-testing-util/src/trace.rs +++ b/crates/rpc/rpc-testing-util/src/trace.rs @@ -114,7 +114,7 @@ pub struct TraceCallStream<'a> { stream: Pin + 'a>>, } -impl<'a> Stream for TraceCallStream<'a> { +impl Stream for TraceCallStream<'_> { type Item = TraceCallResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -122,7 +122,7 @@ impl<'a> Stream for TraceCallStream<'a> { } } -impl<'a> std::fmt::Debug for TraceCallStream<'a> { +impl std::fmt::Debug for TraceCallStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TraceCallStream").finish() } @@ -134,7 +134,7 @@ pub struct TraceFilterStream<'a> { stream: Pin + 'a>>, } -impl<'a> Stream for TraceFilterStream<'a> { +impl Stream for TraceFilterStream<'_> { type Item = TraceFilterResult; /// Attempts to pull out the next value of the stream. @@ -143,7 +143,7 @@ impl<'a> Stream for TraceFilterStream<'a> { } } -impl<'a> std::fmt::Debug for TraceFilterStream<'a> { +impl std::fmt::Debug for TraceFilterStream<'_> { /// Provides a debug representation of the `TraceFilterStream`. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TraceFilterStream").finish_non_exhaustive() @@ -157,7 +157,7 @@ pub struct TraceGetStream<'a> { stream: Pin + 'a>>, } -impl<'a> Stream for TraceGetStream<'a> { +impl Stream for TraceGetStream<'_> { type Item = TraceGetResult; /// Attempts to pull out the next item of the stream @@ -166,7 +166,7 @@ impl<'a> Stream for TraceGetStream<'a> { } } -impl<'a> std::fmt::Debug for TraceGetStream<'a> { +impl std::fmt::Debug for TraceGetStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TraceGetStream").finish_non_exhaustive() } @@ -180,7 +180,7 @@ pub struct CallManyTraceStream<'a> { stream: Pin + 'a>>, } -impl<'a> Stream for CallManyTraceStream<'a> { +impl Stream for CallManyTraceStream<'_> { type Item = CallManyTraceResult; /// Polls for the next item from the stream. @@ -189,7 +189,7 @@ impl<'a> Stream for CallManyTraceStream<'a> { } } -impl<'a> std::fmt::Debug for CallManyTraceStream<'a> { +impl std::fmt::Debug for CallManyTraceStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("CallManyTraceStream").finish() } @@ -201,7 +201,7 @@ pub struct RawTransactionTraceStream<'a> { stream: RawTransactionTraceResult<'a>, } -impl<'a> Stream for RawTransactionTraceStream<'a> { +impl Stream for RawTransactionTraceStream<'_> { type Item = Result<(TraceResults, Bytes), (RpcError, Bytes)>; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -209,7 +209,7 @@ impl<'a> Stream for RawTransactionTraceStream<'a> { } } -impl<'a> std::fmt::Debug for RawTransactionTraceStream<'a> { +impl std::fmt::Debug for RawTransactionTraceStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RawTransactionTraceStream").finish() } @@ -221,7 +221,7 @@ pub struct ReplayTransactionStream<'a> { stream: Pin + 'a>>, } -impl<'a> Stream for ReplayTransactionStream<'a> { +impl Stream for ReplayTransactionStream<'_> { type Item = ReplayTransactionResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -229,7 +229,7 @@ impl<'a> Stream for ReplayTransactionStream<'a> { } } -impl<'a> std::fmt::Debug for ReplayTransactionStream<'a> { +impl std::fmt::Debug for ReplayTransactionStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ReplayTransactionStream").finish() } @@ -393,7 +393,7 @@ impl<'a> TraceBlockStream<'a> { } } -impl<'a> Stream for TraceBlockStream<'a> { +impl Stream for TraceBlockStream<'_> { type Item = TraceBlockResult; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -401,7 +401,7 @@ impl<'a> Stream for TraceBlockStream<'a> { } } -impl<'a> std::fmt::Debug for TraceBlockStream<'a> { +impl std::fmt::Debug for TraceBlockStream<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TraceBlockStream").finish_non_exhaustive() } diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index de30f89c98ee5..22f40f6f951f0 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -227,7 +227,7 @@ impl<'b, TX: DbTx> HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> AccountReader for HistoricalStateProviderRef<'b, TX> { +impl AccountReader for HistoricalStateProviderRef<'_, TX> { /// Get basic account information. fn basic_account(&self, address: Address) -> ProviderResult> { match self.account_history_lookup(address)? { @@ -249,7 +249,7 @@ impl<'b, TX: DbTx> AccountReader for HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> BlockHashReader for HistoricalStateProviderRef<'b, TX> { +impl BlockHashReader for HistoricalStateProviderRef<'_, TX> { /// Get block hash by number. fn block_hash(&self, number: u64) -> ProviderResult> { self.static_file_provider.get_with_static_file_or_database( @@ -285,7 +285,7 @@ impl<'b, TX: DbTx> BlockHashReader for HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateRootProvider for HistoricalStateProviderRef<'b, TX> { +impl StateRootProvider for HistoricalStateProviderRef<'_, TX> { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { let mut revert_state = self.revert_state()?; revert_state.extend(hashed_state); @@ -319,7 +319,7 @@ impl<'b, TX: DbTx> StateRootProvider for HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StorageRootProvider for HistoricalStateProviderRef<'b, TX> { +impl StorageRootProvider for HistoricalStateProviderRef<'_, TX> { fn storage_root( &self, address: Address, @@ -332,7 +332,7 @@ impl<'b, TX: DbTx> StorageRootProvider for HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateProofProvider for HistoricalStateProviderRef<'b, TX> { +impl StateProofProvider for HistoricalStateProviderRef<'_, TX> { /// Get account and storage proofs. fn proof( &self, @@ -364,7 +364,7 @@ impl<'b, TX: DbTx> StateProofProvider for HistoricalStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateProvider for HistoricalStateProviderRef<'b, TX> { +impl StateProvider for HistoricalStateProviderRef<'_, TX> { /// Get storage. fn storage( &self, diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index f63eaee23862c..9fbe00cbd5ee9 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -36,14 +36,14 @@ impl<'b, TX: DbTx> LatestStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> AccountReader for LatestStateProviderRef<'b, TX> { +impl AccountReader for LatestStateProviderRef<'_, TX> { /// Get basic account information. fn basic_account(&self, address: Address) -> ProviderResult> { self.tx.get::(address).map_err(Into::into) } } -impl<'b, TX: DbTx> BlockHashReader for LatestStateProviderRef<'b, TX> { +impl BlockHashReader for LatestStateProviderRef<'_, TX> { /// Get block hash by number. fn block_hash(&self, number: u64) -> ProviderResult> { self.static_file_provider.get_with_static_file_or_database( @@ -79,7 +79,7 @@ impl<'b, TX: DbTx> BlockHashReader for LatestStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateRootProvider for LatestStateProviderRef<'b, TX> { +impl StateRootProvider for LatestStateProviderRef<'_, TX> { fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult { StateRoot::overlay_root(self.tx, hashed_state) .map_err(|err| ProviderError::Database(err.into())) @@ -107,7 +107,7 @@ impl<'b, TX: DbTx> StateRootProvider for LatestStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StorageRootProvider for LatestStateProviderRef<'b, TX> { +impl StorageRootProvider for LatestStateProviderRef<'_, TX> { fn storage_root( &self, address: Address, @@ -118,7 +118,7 @@ impl<'b, TX: DbTx> StorageRootProvider for LatestStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateProofProvider for LatestStateProviderRef<'b, TX> { +impl StateProofProvider for LatestStateProviderRef<'_, TX> { fn proof( &self, input: TrieInput, @@ -146,7 +146,7 @@ impl<'b, TX: DbTx> StateProofProvider for LatestStateProviderRef<'b, TX> { } } -impl<'b, TX: DbTx> StateProvider for LatestStateProviderRef<'b, TX> { +impl StateProvider for LatestStateProviderRef<'_, TX> { /// Get storage. fn storage( &self, diff --git a/crates/storage/provider/src/providers/static_file/jar.rs b/crates/storage/provider/src/providers/static_file/jar.rs index 6372bad244244..8d1dbd117cfb9 100644 --- a/crates/storage/provider/src/providers/static_file/jar.rs +++ b/crates/storage/provider/src/providers/static_file/jar.rs @@ -75,7 +75,7 @@ impl<'a> StaticFileJarProvider<'a> { } } -impl<'a> HeaderProvider for StaticFileJarProvider<'a> { +impl HeaderProvider for StaticFileJarProvider<'_> { fn header(&self, block_hash: &BlockHash) -> ProviderResult> { Ok(self .cursor()? @@ -147,7 +147,7 @@ impl<'a> HeaderProvider for StaticFileJarProvider<'a> { } } -impl<'a> BlockHashReader for StaticFileJarProvider<'a> { +impl BlockHashReader for StaticFileJarProvider<'_> { fn block_hash(&self, number: u64) -> ProviderResult> { self.cursor()?.get_one::>(number.into()) } @@ -169,7 +169,7 @@ impl<'a> BlockHashReader for StaticFileJarProvider<'a> { } } -impl<'a> BlockNumReader for StaticFileJarProvider<'a> { +impl BlockNumReader for StaticFileJarProvider<'_> { fn chain_info(&self) -> ProviderResult { // Information on live database Err(ProviderError::UnsupportedProvider) @@ -194,7 +194,7 @@ impl<'a> BlockNumReader for StaticFileJarProvider<'a> { } } -impl<'a> TransactionsProvider for StaticFileJarProvider<'a> { +impl TransactionsProvider for StaticFileJarProvider<'_> { fn transaction_id(&self, hash: TxHash) -> ProviderResult> { let mut cursor = self.cursor()?; @@ -290,7 +290,7 @@ impl<'a> TransactionsProvider for StaticFileJarProvider<'a> { } } -impl<'a> ReceiptProvider for StaticFileJarProvider<'a> { +impl ReceiptProvider for StaticFileJarProvider<'_> { fn receipt(&self, num: TxNumber) -> ProviderResult> { self.cursor()?.get_one::>(num.into()) } diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index d086c5693ca52..3858f1b140233 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -67,14 +67,14 @@ pub struct StaticFileProviderRWRefMut<'a>( pub(crate) RwLockWriteGuard<'a, RawRwLock, Option>, ); -impl<'a> std::ops::DerefMut for StaticFileProviderRWRefMut<'a> { +impl std::ops::DerefMut for StaticFileProviderRWRefMut<'_> { fn deref_mut(&mut self) -> &mut Self::Target { // This is always created by [`StaticFileWriters::get_or_create`] self.0.as_mut().expect("static file writer provider should be init") } } -impl<'a> std::ops::Deref for StaticFileProviderRWRefMut<'a> { +impl std::ops::Deref for StaticFileProviderRWRefMut<'_> { type Target = StaticFileProviderRW; fn deref(&self) -> &Self::Target { diff --git a/crates/storage/provider/src/writer/database.rs b/crates/storage/provider/src/writer/database.rs index 3ae42b4bf1cb4..1436fb8a6ab95 100644 --- a/crates/storage/provider/src/writer/database.rs +++ b/crates/storage/provider/src/writer/database.rs @@ -9,7 +9,7 @@ use reth_storage_api::ReceiptWriter; pub(crate) struct DatabaseWriter<'a, W>(pub(crate) &'a mut W); -impl<'a, W> ReceiptWriter for DatabaseWriter<'a, W> +impl ReceiptWriter for DatabaseWriter<'_, W> where W: DbCursorRO + DbCursorRW, { diff --git a/crates/storage/provider/src/writer/mod.rs b/crates/storage/provider/src/writer/mod.rs index ecb1de335559f..5b16b2da4e5a2 100644 --- a/crates/storage/provider/src/writer/mod.rs +++ b/crates/storage/provider/src/writer/mod.rs @@ -147,7 +147,7 @@ impl UnifiedStorageWriter<'_, (), ()> { } } -impl<'a, 'b, ProviderDB> UnifiedStorageWriter<'a, ProviderDB, &'b StaticFileProvider> +impl UnifiedStorageWriter<'_, ProviderDB, &StaticFileProvider> where ProviderDB: DBProvider + BlockWriter @@ -318,7 +318,7 @@ where } } -impl<'a, 'b, ProviderDB> UnifiedStorageWriter<'a, ProviderDB, StaticFileProviderRWRefMut<'b>> +impl UnifiedStorageWriter<'_, ProviderDB, StaticFileProviderRWRefMut<'_>> where ProviderDB: DBProvider + HeaderProvider, { @@ -429,7 +429,7 @@ where } } -impl<'a, 'b, ProviderDB> UnifiedStorageWriter<'a, ProviderDB, StaticFileProviderRWRefMut<'b>> +impl UnifiedStorageWriter<'_, ProviderDB, StaticFileProviderRWRefMut<'_>> where ProviderDB: DBProvider + HeaderProvider, { @@ -510,8 +510,8 @@ where } } -impl<'a, 'b, ProviderDB> StateWriter - for UnifiedStorageWriter<'a, ProviderDB, StaticFileProviderRWRefMut<'b>> +impl StateWriter + for UnifiedStorageWriter<'_, ProviderDB, StaticFileProviderRWRefMut<'_>> where ProviderDB: DBProvider + StateChangeWriter + HeaderProvider, { diff --git a/crates/storage/provider/src/writer/static_file.rs b/crates/storage/provider/src/writer/static_file.rs index aca226ca9b752..5514e211e58f9 100644 --- a/crates/storage/provider/src/writer/static_file.rs +++ b/crates/storage/provider/src/writer/static_file.rs @@ -6,7 +6,7 @@ use reth_storage_api::ReceiptWriter; pub(crate) struct StaticFileWriter<'a, W>(pub(crate) &'a mut W); -impl<'a> ReceiptWriter for StaticFileWriter<'a, StaticFileProviderRWRefMut<'_>> { +impl ReceiptWriter for StaticFileWriter<'_, StaticFileProviderRWRefMut<'_>> { fn append_block_receipts( &mut self, first_tx_index: TxNumber, diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index e522978fb9dc0..f44e66e457a1f 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -644,7 +644,7 @@ pub struct CanonicalStateUpdate<'a> { pub mined_transactions: Vec, } -impl<'a> CanonicalStateUpdate<'a> { +impl CanonicalStateUpdate<'_> { /// Returns the number of the tip block. pub fn number(&self) -> u64 { self.new_tip.number diff --git a/crates/trie/db/src/hashed_cursor.rs b/crates/trie/db/src/hashed_cursor.rs index bf0341c8884dc..6d0b79e5a02b4 100644 --- a/crates/trie/db/src/hashed_cursor.rs +++ b/crates/trie/db/src/hashed_cursor.rs @@ -11,7 +11,7 @@ use reth_trie::hashed_cursor::{HashedCursor, HashedCursorFactory, HashedStorageC #[derive(Debug)] pub struct DatabaseHashedCursorFactory<'a, TX>(&'a TX); -impl<'a, TX> Clone for DatabaseHashedCursorFactory<'a, TX> { +impl Clone for DatabaseHashedCursorFactory<'_, TX> { fn clone(&self) -> Self { Self(self.0) } @@ -24,7 +24,7 @@ impl<'a, TX> DatabaseHashedCursorFactory<'a, TX> { } } -impl<'a, TX: DbTx> HashedCursorFactory for DatabaseHashedCursorFactory<'a, TX> { +impl HashedCursorFactory for DatabaseHashedCursorFactory<'_, TX> { type AccountCursor = DatabaseHashedAccountCursor<::Cursor>; type StorageCursor = DatabaseHashedStorageCursor<::DupCursor>; diff --git a/crates/trie/db/src/prefix_set.rs b/crates/trie/db/src/prefix_set.rs index 07b87016d2b4f..079fe393764d2 100644 --- a/crates/trie/db/src/prefix_set.rs +++ b/crates/trie/db/src/prefix_set.rs @@ -26,7 +26,7 @@ impl<'a, TX> PrefixSetLoader<'a, TX> { } } -impl<'a, TX: DbTx> PrefixSetLoader<'a, TX> { +impl PrefixSetLoader<'_, TX> { /// Load all account and storage changes for the given block range. pub fn load(self, range: RangeInclusive) -> Result { // Initialize prefix sets. diff --git a/crates/trie/db/src/trie_cursor.rs b/crates/trie/db/src/trie_cursor.rs index 601100b3faee3..bfded342ba04d 100644 --- a/crates/trie/db/src/trie_cursor.rs +++ b/crates/trie/db/src/trie_cursor.rs @@ -19,7 +19,7 @@ use reth_trie_common::StorageTrieEntry; #[derive(Debug)] pub struct DatabaseTrieCursorFactory<'a, TX>(&'a TX); -impl<'a, TX> Clone for DatabaseTrieCursorFactory<'a, TX> { +impl Clone for DatabaseTrieCursorFactory<'_, TX> { fn clone(&self) -> Self { Self(self.0) } @@ -33,7 +33,7 @@ impl<'a, TX> DatabaseTrieCursorFactory<'a, TX> { } /// Implementation of the trie cursor factory for a database transaction. -impl<'a, TX: DbTx> TrieCursorFactory for DatabaseTrieCursorFactory<'a, TX> { +impl TrieCursorFactory for DatabaseTrieCursorFactory<'_, TX> { type AccountTrieCursor = DatabaseAccountTrieCursor<::Cursor>; type StorageTrieCursor = DatabaseStorageTrieCursor<::DupCursor>; diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index c313231e66dc1..0f00191378ba8 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -113,7 +113,7 @@ impl<'a, C: TrieCursor> InMemoryAccountTrieCursor<'a, C> { } } -impl<'a, C: TrieCursor> TrieCursor for InMemoryAccountTrieCursor<'a, C> { +impl TrieCursor for InMemoryAccountTrieCursor<'_, C> { fn seek_exact( &mut self, key: Nibbles, From 3ace7f4c359dd15c336b9b1c3191fb4a34e54d30 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Oct 2024 19:44:33 +0200 Subject: [PATCH 029/159] chore: rm from genesis impl (#11509) --- crates/net/eth-wire-types/Cargo.toml | 2 +- crates/net/eth-wire-types/src/status.rs | 26 ++++--------------------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/crates/net/eth-wire-types/Cargo.toml b/crates/net/eth-wire-types/Cargo.toml index 6ce51786f282a..82c9fe37a44db 100644 --- a/crates/net/eth-wire-types/Cargo.toml +++ b/crates/net/eth-wire-types/Cargo.toml @@ -20,7 +20,6 @@ reth-primitives.workspace = true # ethereum alloy-chains = { workspace = true, features = ["rlp"] } alloy-eips.workspace = true -alloy-genesis.workspace = true alloy-primitives.workspace = true alloy-rlp = { workspace = true, features = ["derive"] } @@ -36,6 +35,7 @@ proptest-arbitrary-interop = { workspace = true, optional = true } [dev-dependencies] reth-primitives = { workspace = true, features = ["arbitrary"] } +alloy-genesis.workspace = true alloy-chains = { workspace = true, features = ["arbitrary"] } arbitrary = { workspace = true, features = ["derive"] } proptest.workspace = true diff --git a/crates/net/eth-wire-types/src/status.rs b/crates/net/eth-wire-types/src/status.rs index baf1e2991522f..a5e7530ec09a4 100644 --- a/crates/net/eth-wire-types/src/status.rs +++ b/crates/net/eth-wire-types/src/status.rs @@ -1,9 +1,8 @@ use crate::EthVersion; use alloy_chains::{Chain, NamedChain}; -use alloy_genesis::Genesis; use alloy_primitives::{hex, B256, U256}; use alloy_rlp::{RlpDecodable, RlpEncodable}; -use reth_chainspec::{ChainSpec, EthChainSpec, Hardforks, MAINNET}; +use reth_chainspec::{EthChainSpec, Hardforks, MAINNET}; use reth_codecs_derive::add_arbitrary_tests; use reth_primitives::{EthereumHardfork, ForkId, Head}; use std::fmt::{Debug, Display}; @@ -43,23 +42,6 @@ pub struct Status { pub forkid: ForkId, } -impl From for Status { - fn from(genesis: Genesis) -> Self { - let chain = genesis.config.chain_id; - let total_difficulty = genesis.difficulty; - let chainspec = ChainSpec::from(genesis); - - Self { - version: EthVersion::Eth68 as u8, - chain: Chain::from_id(chain), - total_difficulty, - blockhash: chainspec.genesis_hash(), - genesis: chainspec.genesis_hash(), - forkid: chainspec.fork_id(&Head::default()), - } - } -} - impl Status { /// Helper for returning a builder for the status message. pub fn builder() -> StatusBuilder { @@ -71,10 +53,10 @@ impl Status { self.version = version as u8; } - /// Create a [`StatusBuilder`] from the given [`ChainSpec`] and head block. + /// Create a [`StatusBuilder`] from the given [`EthChainSpec`] and head block. /// - /// Sets the `chain` and `genesis`, `blockhash`, and `forkid` fields based on the [`ChainSpec`] - /// and head. + /// Sets the `chain` and `genesis`, `blockhash`, and `forkid` fields based on the + /// [`EthChainSpec`] and head. pub fn spec_builder(spec: Spec, head: &Head) -> StatusBuilder where Spec: EthChainSpec + Hardforks, From 8056e1b8e4dcf8375e34a4f4469193649cad9f89 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Oct 2024 20:06:21 +0200 Subject: [PATCH 030/159] fix: cap gas limit properly (#11505) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 26 ++++++++++------------ 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 4d12450a1c7fd..819aff546d426 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -1023,7 +1023,14 @@ pub trait Call: LoadState + SpawnBlocking { block_env.get_blob_gasprice().map(U256::from), )?; - let gas_limit = gas.unwrap_or_else(|| block_env.gas_limit.min(U256::from(u64::MAX)).to()); + let gas_limit = gas.unwrap_or_else(|| { + // Use maximum allowed gas limit. The reason for this + // is that both Erigon and Geth use pre-configured gas cap even if + // it's possible to derive the gas limit from the block: + // + block_env.gas_limit.saturating_to() + }); #[allow(clippy::needless_update)] let env = TxEnv { @@ -1065,7 +1072,7 @@ pub trait Call: LoadState + SpawnBlocking { Ok(EnvWithHandlerCfg::new_with_cfg_env(cfg, block, tx)) } - /// Prepares the [`EnvWithHandlerCfg`] for execution. + /// Prepares the [`EnvWithHandlerCfg`] for execution of calls. /// /// Does not commit any changes to the underlying database. /// @@ -1076,9 +1083,6 @@ pub trait Call: LoadState + SpawnBlocking { /// - `disable_eip3607` is set to `true` /// - `disable_base_fee` is set to `true` /// - `nonce` is set to `None` - /// - /// Additionally, the block gas limit so that higher tx gas limits can be used in `eth_call`. - /// - `disable_block_gas_limit` is set to `true` fn prepare_call_env( &self, mut cfg: CfgEnvWithHandlerCfg, @@ -1125,15 +1129,9 @@ pub trait Call: LoadState + SpawnBlocking { if env.tx.gas_price > U256::ZERO { // If gas price is specified, cap transaction gas limit with caller allowance trace!(target: "rpc::eth::call", ?env, "Applying gas limit cap with caller allowance"); - cap_tx_gas_limit_with_caller_allowance(db, &mut env.tx)?; - } else { - // If no gas price is specified, use maximum allowed gas limit. The reason for this - // is that both Erigon and Geth use pre-configured gas cap even if - // it's possible to derive the gas limit from the block: - // - trace!(target: "rpc::eth::call", ?env, "Applying gas limit cap as the maximum gas limit"); - env.tx.gas_limit = self.call_gas_limit(); + let cap = caller_gas_allowance(db, &env.tx)?; + // ensure we cap gas_limit to the block's + env.tx.gas_limit = cap.min(env.block.gas_limit).saturating_to(); } } From 9810940c06fd6878c1f9d5a0606f03a82c5dbdbd Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Oct 2024 20:11:28 +0200 Subject: [PATCH 031/159] feat: add PoolBuilderConfigOverrides (#11507) --- crates/node/builder/src/components/pool.rs | 62 ++++++++++++++++++++-- crates/optimism/node/src/node.rs | 13 +++-- crates/transaction-pool/src/config.rs | 2 +- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/crates/node/builder/src/components/pool.rs b/crates/node/builder/src/components/pool.rs index af31f29307ebf..234455913c6f9 100644 --- a/crates/node/builder/src/components/pool.rs +++ b/crates/node/builder/src/components/pool.rs @@ -1,8 +1,8 @@ //! Pool component for the node builder. -use std::future::Future; - -use reth_transaction_pool::TransactionPool; +use alloy_primitives::Address; +use reth_transaction_pool::{PoolConfig, SubPoolLimit, TransactionPool}; +use std::{collections::HashSet, future::Future}; use crate::{BuilderContext, FullNodeTypes}; @@ -34,3 +34,59 @@ where self(ctx) } } + +/// Convenience type to override cli or default pool configuration during build. +#[derive(Debug, Clone, Default)] +pub struct PoolBuilderConfigOverrides { + /// Max number of transaction in the pending sub-pool + pub pending_limit: Option, + /// Max number of transaction in the basefee sub-pool + pub basefee_limit: Option, + /// Max number of transaction in the queued sub-pool + pub queued_limit: Option, + /// Max number of transactions in the blob sub-pool + pub blob_limit: Option, + /// Max number of executable transaction slots guaranteed per account + pub max_account_slots: Option, + /// Minimum base fee required by the protocol. + pub minimal_protocol_basefee: Option, + /// Addresses that will be considered as local. Above exemptions apply. + pub local_addresses: HashSet
, +} + +impl PoolBuilderConfigOverrides { + /// Applies the configured overrides to the given [`PoolConfig`]. + pub fn apply(self, mut config: PoolConfig) -> PoolConfig { + let Self { + pending_limit, + basefee_limit, + queued_limit, + blob_limit, + max_account_slots, + minimal_protocol_basefee, + local_addresses, + } = self; + + if let Some(pending_limit) = pending_limit { + config.pending_limit = pending_limit; + } + if let Some(basefee_limit) = basefee_limit { + config.basefee_limit = basefee_limit; + } + if let Some(queued_limit) = queued_limit { + config.queued_limit = queued_limit; + } + if let Some(blob_limit) = blob_limit { + config.blob_limit = blob_limit; + } + if let Some(max_account_slots) = max_account_slots { + config.max_account_slots = max_account_slots; + } + if let Some(minimal_protocol_basefee) = minimal_protocol_basefee { + config.minimal_protocol_basefee = minimal_protocol_basefee; + } + config.local_transactions_config.local_addresses.extend(local_addresses); + + config + } +} diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 4e7788569f89e..8ae47897084bf 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -9,7 +9,7 @@ use reth_node_api::{EngineValidator, FullNodeComponents, NodeAddOns}; use reth_node_builder::{ components::{ ComponentsBuilder, ConsensusBuilder, EngineValidatorBuilder, ExecutorBuilder, - NetworkBuilder, PayloadServiceBuilder, PoolBuilder, + NetworkBuilder, PayloadServiceBuilder, PoolBuilder, PoolBuilderConfigOverrides, }, node::{FullNodeTypes, NodeTypes, NodeTypesWithEngine}, BuilderContext, Node, PayloadBuilderConfig, @@ -166,9 +166,11 @@ where /// /// This contains various settings that can be configured and take precedence over the node's /// config. -#[derive(Debug, Default, Clone, Copy)] -#[non_exhaustive] -pub struct OptimismPoolBuilder; +#[derive(Debug, Default, Clone)] +pub struct OptimismPoolBuilder { + /// Enforced overrides that are applied to the pool config. + pub pool_config_overrides: PoolBuilderConfigOverrides, +} impl PoolBuilder for OptimismPoolBuilder where @@ -177,6 +179,7 @@ where type Pool = OpTransactionPool; async fn build_pool(self, ctx: &BuilderContext) -> eyre::Result { + let Self { pool_config_overrides } = self; let data_dir = ctx.config().datadir(); let blob_store = DiskFileBlobStore::open(data_dir.blobstore(), Default::default())?; @@ -198,7 +201,7 @@ where validator, CoinbaseTipOrdering::default(), blob_store, - ctx.pool_config(), + pool_config_overrides.apply(ctx.pool_config()), ); info!(target: "reth::cli", "Transaction pool initialized"); let transactions_path = data_dir.txpool_transactions(); diff --git a/crates/transaction-pool/src/config.rs b/crates/transaction-pool/src/config.rs index 623493e6c9d43..67328c7f1be16 100644 --- a/crates/transaction-pool/src/config.rs +++ b/crates/transaction-pool/src/config.rs @@ -157,7 +157,7 @@ pub struct LocalTransactionConfig { /// - no price exemptions /// - no eviction exemptions pub no_exemptions: bool, - /// Addresses that will be considered as local . Above exemptions apply + /// Addresses that will be considered as local. Above exemptions apply. pub local_addresses: HashSet
, /// Flag indicating whether local transactions should be propagated. pub propagate_local_transactions: bool, From 68f45d5d5a6ef9f1fab2aaa533fb6d0aeaf7d277 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 5 Oct 2024 20:12:28 +0200 Subject: [PATCH 032/159] feat: expose Op node network_config helper (#11506) --- crates/net/network/src/config.rs | 8 +++++++ crates/optimism/node/src/node.rs | 41 ++++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index 8217a02a1bab6..60b559b9dbc34 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -106,6 +106,14 @@ impl NetworkConfig { NetworkConfig::builder(secret_key).build(client) } + /// Apply a function to the config. + pub fn apply(self, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + f(self) + } + /// Sets the config to use for the discovery v4 protocol. pub fn set_discovery_v4(mut self, discovery_config: Discv4Config) -> Self { self.discovery_v4_config = Some(discovery_config); diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index 8ae47897084bf..caeab3741a2e6 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -3,8 +3,9 @@ use std::sync::Arc; use reth_basic_payload_builder::{BasicPayloadJobGenerator, BasicPayloadJobGeneratorConfig}; +use reth_chainspec::{EthChainSpec, Hardforks}; use reth_evm::ConfigureEvm; -use reth_network::{NetworkHandle, NetworkManager}; +use reth_network::{NetworkConfig, NetworkHandle, NetworkManager}; use reth_node_api::{EngineValidator, FullNodeComponents, NodeAddOns}; use reth_node_builder::{ components::{ @@ -330,18 +331,18 @@ pub struct OptimismNetworkBuilder { pub disable_discovery_v4: bool, } -impl NetworkBuilder for OptimismNetworkBuilder -where - Node: FullNodeTypes>, - Pool: TransactionPool + Unpin + 'static, -{ - async fn build_network( - self, +impl OptimismNetworkBuilder { + /// Returns the [`NetworkConfig`] that contains the settings to launch the p2p network. + /// + /// This applies the configured [`OptimismNetworkBuilder`] settings. + pub fn network_config( + &self, ctx: &BuilderContext, - pool: Pool, - ) -> eyre::Result { - let Self { disable_txpool_gossip, disable_discovery_v4 } = self; - + ) -> eyre::Result::Provider>> + where + Node: FullNodeTypes>, + { + let Self { disable_txpool_gossip, disable_discovery_v4 } = self.clone(); let args = &ctx.config().network; let network_builder = ctx .network_config_builder()? @@ -374,8 +375,22 @@ where // gossip to prevent other parties in the network from learning about them. network_config.tx_gossip_disabled = disable_txpool_gossip; - let network = NetworkManager::builder(network_config).await?; + Ok(network_config) + } +} +impl NetworkBuilder for OptimismNetworkBuilder +where + Node: FullNodeTypes>, + Pool: TransactionPool + Unpin + 'static, +{ + async fn build_network( + self, + ctx: &BuilderContext, + pool: Pool, + ) -> eyre::Result { + let network_config = self.network_config(ctx)?; + let network = NetworkManager::builder(network_config).await?; let handle = ctx.start_network(network, pool); Ok(handle) From 12d8754f1dc396c58d4a0673bc3c2d650460e2de Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 6 Oct 2024 14:28:31 +0000 Subject: [PATCH 033/159] chore(deps): weekly `cargo update` (#11518) Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com> --- Cargo.lock | 182 ++++++++++++++++++++++------------------------------- 1 file changed, 75 insertions(+), 107 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e75ea0ec095c3..590f01a416fde 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -97,9 +97,9 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy-chains" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8158b4878c67837e5413721cc44298e6a2d88d39203175ea025e51892a16ba4c" +checksum = "94c225801d42099570d0674701dddd4142f0ef715282aeb5985042e2ec962df7" dependencies = [ "alloy-rlp", "arbitrary", @@ -296,7 +296,7 @@ dependencies = [ "getrandom 0.2.15", "hashbrown 0.14.5", "hex-literal", - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "k256", "keccak-asm", @@ -619,7 +619,7 @@ dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.5.0", + "indexmap 2.6.0", "proc-macro-error2", "proc-macro2", "quote", @@ -1016,9 +1016,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" +checksum = "7e614738943d3f68c628ae3dbce7c3daffb196665f82f8c8ea6b65de73c79429" dependencies = [ "brotli", "flate2", @@ -1203,26 +1203,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" -dependencies = [ - "bitflags 2.6.0", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.79", -] - [[package]] name = "bindgen" version = "0.70.1" @@ -1333,7 +1313,7 @@ dependencies = [ "bitflags 2.6.0", "boa_interner", "boa_macros", - "indexmap 2.5.0", + "indexmap 2.6.0", "num-bigint", "rustc-hash 2.0.0", ] @@ -1359,7 +1339,7 @@ dependencies = [ "fast-float", "hashbrown 0.14.5", "icu_normalizer", - "indexmap 2.5.0", + "indexmap 2.6.0", "intrusive-collections", "itertools 0.13.0", "num-bigint", @@ -1405,7 +1385,7 @@ dependencies = [ "boa_gc", "boa_macros", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "once_cell", "phf", "rustc-hash 2.0.0", @@ -1473,9 +1453,9 @@ dependencies = [ [[package]] name = "brotli" -version = "6.0.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1619,9 +1599,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.24" +version = "1.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" +checksum = "e8d9e0b4957f635b8d3da819d0db5603620467ecf1f692d22a8c2717ce27e6d8" dependencies = [ "jobserver", "libc", @@ -3133,9 +3113,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -3148,9 +3128,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -3158,15 +3138,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -3175,9 +3155,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -3196,9 +3176,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -3207,15 +3187,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" @@ -3229,9 +3209,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -3311,9 +3291,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -3390,7 +3370,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3436,6 +3416,12 @@ dependencies = [ "serde", ] +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "hashlink" version = "0.8.4" @@ -3963,13 +3949,13 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "arbitrary", "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "serde", ] @@ -3986,7 +3972,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", - "indexmap 2.5.0", + "indexmap 2.6.0", "is-terminal", "itoa", "log", @@ -4084,9 +4070,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "iri-string" @@ -4124,15 +4110,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -4449,12 +4426,6 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.159" @@ -4498,11 +4469,11 @@ dependencies = [ [[package]] name = "libproc" -version = "0.14.8" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9ea4b75e1a81675429dafe43441df1caea70081e82246a8cccf514884a88bb" +checksum = "78cca3586d5efa98fba425856ba227950fd4287525dd5317b352f476ca7d603b" dependencies = [ - "bindgen 0.69.4", + "bindgen", "errno", "libc", ] @@ -4723,7 +4694,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" dependencies = [ "base64 0.22.1", - "indexmap 2.5.0", + "indexmap 2.6.0", "metrics", "metrics-util", "quanta", @@ -5134,21 +5105,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" @@ -5494,18 +5462,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", @@ -6041,9 +6009,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.1.0" +version = "11.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" dependencies = [ "bitflags 2.6.0", ] @@ -7559,7 +7527,7 @@ dependencies = [ "criterion", "dashmap 6.1.0", "derive_more 1.0.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "parking_lot 0.12.3", "pprof", "rand 0.8.5", @@ -7575,7 +7543,7 @@ dependencies = [ name = "reth-mdbx-sys" version = "1.0.8" dependencies = [ - "bindgen 0.70.1", + "bindgen", "cc", ] @@ -9479,9 +9447,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ "log", "once_cell", @@ -9780,7 +9748,7 @@ version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "memchr", "ryu", @@ -9832,15 +9800,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.10.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9720086b3357bcb44fce40117d769a4d068c70ecfa190850a980a71755f66fcc" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_derive", "serde_json", @@ -9850,9 +9818,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.10.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f1abbfe725f27678f4663bcacb75a83e829fd464c25d78dd038a3a29e307cec" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling", "proc-macro2", @@ -10650,7 +10618,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -10999,9 +10967,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" From 40dc52bfeb92fb674d0e5a9e358ebb8638ff1f68 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:02:10 +0200 Subject: [PATCH 034/159] test: add unit tests for `PruneLimiter` (#11517) --- crates/prune/types/src/limiter.rs | 267 ++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) diff --git a/crates/prune/types/src/limiter.rs b/crates/prune/types/src/limiter.rs index 09f0c4cd3f2d8..3a1059949300a 100644 --- a/crates/prune/types/src/limiter.rs +++ b/crates/prune/types/src/limiter.rs @@ -120,3 +120,270 @@ impl PruneLimiter { self.is_deleted_entries_limit_reached() || self.is_time_limit_reached() } } + +#[cfg(test)] +mod tests { + use super::*; + use std::thread::sleep; + + #[test] + fn test_prune_deleted_entries_limit_initial_state() { + let limit_tracker = PruneDeletedEntriesLimit::new(10); + // Limit should be set properly + assert_eq!(limit_tracker.limit, 10); + // No entries should be deleted + assert_eq!(limit_tracker.deleted, 0); + assert!(!limit_tracker.is_limit_reached()); + } + + #[test] + fn test_prune_deleted_entries_limit_is_limit_reached() { + // Test when the deleted entries are less than the limit + let mut limit_tracker = PruneDeletedEntriesLimit::new(5); + limit_tracker.deleted = 3; + assert!(!limit_tracker.is_limit_reached()); + + // Test when the deleted entries are equal to the limit + limit_tracker.deleted = 5; + assert!(limit_tracker.is_limit_reached()); + + // Test when the deleted entries exceed the limit + limit_tracker.deleted = 6; + assert!(limit_tracker.is_limit_reached()); + } + + #[test] + fn test_prune_time_limit_initial_state() { + let time_limit = PruneTimeLimit::new(Duration::from_secs(10)); + // The limit should be set correctly + assert_eq!(time_limit.limit, Duration::from_secs(10)); + // The elapsed time should be very small right after creation + assert!(time_limit.start.elapsed() < Duration::from_secs(1)); + // Limit should not be reached initially + assert!(!time_limit.is_limit_reached()); + } + + #[test] + fn test_prune_time_limit_is_limit_reached() { + let time_limit = PruneTimeLimit::new(Duration::from_millis(50)); + + // Simulate waiting for some time (less than the limit) + std::thread::sleep(Duration::from_millis(30)); + assert!(!time_limit.is_limit_reached()); + + // Simulate waiting for time greater than the limit + std::thread::sleep(Duration::from_millis(30)); + assert!(time_limit.is_limit_reached()); + } + + #[test] + fn test_set_deleted_entries_limit_initial_state() { + let pruner = PruneLimiter::default().set_deleted_entries_limit(100); + // The deleted_entries_limit should be set with the correct limit + assert!(pruner.deleted_entries_limit.is_some()); + let deleted_entries_limit = pruner.deleted_entries_limit.unwrap(); + assert_eq!(deleted_entries_limit.limit, 100); + // The deleted count should be initially zero + assert_eq!(deleted_entries_limit.deleted, 0); + // The limit should not be reached initially + assert!(!deleted_entries_limit.is_limit_reached()); + } + + #[test] + fn test_set_deleted_entries_limit_overwrite_existing() { + let mut pruner = PruneLimiter::default().set_deleted_entries_limit(50); + // Overwrite the existing limit + pruner = pruner.set_deleted_entries_limit(200); + + assert!(pruner.deleted_entries_limit.is_some()); + let deleted_entries_limit = pruner.deleted_entries_limit.unwrap(); + // Check that the limit has been overwritten correctly + assert_eq!(deleted_entries_limit.limit, 200); + // Deleted count should still be zero + assert_eq!(deleted_entries_limit.deleted, 0); + assert!(!deleted_entries_limit.is_limit_reached()); + } + + #[test] + fn test_set_deleted_entries_limit_when_limit_is_reached() { + let mut pruner = PruneLimiter::default().set_deleted_entries_limit(5); + assert!(pruner.deleted_entries_limit.is_some()); + let mut deleted_entries_limit = pruner.deleted_entries_limit.clone().unwrap(); + + // Simulate deletion of entries + deleted_entries_limit.deleted = 5; + assert!(deleted_entries_limit.is_limit_reached()); + + // Overwrite the limit and check if it resets correctly + pruner = pruner.set_deleted_entries_limit(10); + deleted_entries_limit = pruner.deleted_entries_limit.unwrap(); + assert_eq!(deleted_entries_limit.limit, 10); + // Deletion count should reset + assert_eq!(deleted_entries_limit.deleted, 0); + assert!(!deleted_entries_limit.is_limit_reached()); + } + + #[test] + fn test_floor_deleted_entries_limit_to_multiple_of() { + let limiter = PruneLimiter::default().set_deleted_entries_limit(15); + let denominator = NonZeroUsize::new(4).unwrap(); + + // Floor limit to the largest multiple of 4 less than or equal to 15 (that is 12) + let updated_limiter = limiter.floor_deleted_entries_limit_to_multiple_of(denominator); + assert_eq!(updated_limiter.deleted_entries_limit.unwrap().limit, 12); + + // Test when the limit is already a multiple of the denominator + let limiter = PruneLimiter::default().set_deleted_entries_limit(16); + let updated_limiter = limiter.floor_deleted_entries_limit_to_multiple_of(denominator); + assert_eq!(updated_limiter.deleted_entries_limit.unwrap().limit, 16); + + // Test when there's no limit set (should not panic) + let limiter = PruneLimiter::default(); + let updated_limiter = limiter.floor_deleted_entries_limit_to_multiple_of(denominator); + assert!(updated_limiter.deleted_entries_limit.is_none()); + } + + #[test] + fn test_is_deleted_entries_limit_reached() { + // Limit is not set, should return false + let limiter = PruneLimiter::default(); + assert!(!limiter.is_deleted_entries_limit_reached()); + + // Limit is set but not reached, should return false + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10); + limiter.deleted_entries_limit.as_mut().unwrap().deleted = 5; + // 5 entries deleted out of 10 + assert!(!limiter.is_deleted_entries_limit_reached()); + + // Limit is reached, should return true + limiter.deleted_entries_limit.as_mut().unwrap().deleted = 10; + // 10 entries deleted out of 10 + assert!(limiter.is_deleted_entries_limit_reached()); + + // Deleted entries exceed the limit, should return true + limiter.deleted_entries_limit.as_mut().unwrap().deleted = 12; + // 12 entries deleted out of 10 + assert!(limiter.is_deleted_entries_limit_reached()); + } + + #[test] + fn test_increment_deleted_entries_count_by() { + // Increment when no limit is set + let mut limiter = PruneLimiter::default(); + limiter.increment_deleted_entries_count_by(5); + assert_eq!(limiter.deleted_entries_limit.as_ref().map(|l| l.deleted), None); // Still None + + // Increment when limit is set + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10); + limiter.increment_deleted_entries_count_by(3); + assert_eq!(limiter.deleted_entries_limit.as_ref().unwrap().deleted, 3); // Now 3 deleted + + // Increment again + limiter.increment_deleted_entries_count_by(2); + assert_eq!(limiter.deleted_entries_limit.as_ref().unwrap().deleted, 5); // Now 5 deleted + } + + #[test] + fn test_increment_deleted_entries_count() { + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(5); + assert_eq!(limiter.deleted_entries_limit.as_ref().unwrap().deleted, 0); // Initially 0 + + limiter.increment_deleted_entries_count(); // Increment by 1 + assert_eq!(limiter.deleted_entries_limit.as_ref().unwrap().deleted, 1); // Now 1 + } + + #[test] + fn test_deleted_entries_limit_left() { + // Test when limit is set and some entries are deleted + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(10); + limiter.increment_deleted_entries_count_by(3); // Simulate 3 deleted entries + assert_eq!(limiter.deleted_entries_limit_left(), Some(7)); // 10 - 3 = 7 + + // Test when no entries are deleted + limiter = PruneLimiter::default().set_deleted_entries_limit(5); + assert_eq!(limiter.deleted_entries_limit_left(), Some(5)); // 5 - 0 = 5 + + // Test when limit is reached + limiter.increment_deleted_entries_count_by(5); // Simulate deleting 5 entries + assert_eq!(limiter.deleted_entries_limit_left(), Some(0)); // 5 - 5 = 0 + + // Test when limit is not set + limiter = PruneLimiter::default(); // No limit set + assert_eq!(limiter.deleted_entries_limit_left(), None); // Should be None + } + + #[test] + fn test_set_time_limit() { + // Create a PruneLimiter instance with no time limit set + let mut limiter = PruneLimiter::default(); + + // Set a time limit of 5 seconds + limiter = limiter.set_time_limit(Duration::new(5, 0)); + + // Verify that the time limit is set correctly + assert!(limiter.time_limit.is_some()); + let time_limit = limiter.time_limit.as_ref().unwrap(); + assert_eq!(time_limit.limit, Duration::new(5, 0)); + // Ensure the start time is recent + assert!(time_limit.start.elapsed() < Duration::new(1, 0)); + } + + #[test] + fn test_is_time_limit_reached() { + // Create a PruneLimiter instance and set a time limit of 10 milliseconds + let mut limiter = PruneLimiter::default(); + + // Time limit should not be reached initially + assert!(!limiter.is_time_limit_reached(), "Time limit should not be reached yet"); + + limiter = limiter.set_time_limit(Duration::new(0, 10_000_000)); // 10 milliseconds + + // Sleep for 5 milliseconds (less than the time limit) + sleep(Duration::new(0, 5_000_000)); // 5 milliseconds + assert!(!limiter.is_time_limit_reached(), "Time limit should not be reached yet"); + + // Sleep for an additional 10 milliseconds (totaling 15 milliseconds) + sleep(Duration::new(0, 10_000_000)); // 10 milliseconds + assert!(limiter.is_time_limit_reached(), "Time limit should be reached now"); + } + + #[test] + fn test_is_limit_reached() { + // Create a PruneLimiter instance + let mut limiter = PruneLimiter::default(); + + // Test when no limits are set + assert!(!limiter.is_limit_reached(), "Limit should not be reached with no limits set"); + + // Set a deleted entries limit + limiter = limiter.set_deleted_entries_limit(5); + assert!( + !limiter.is_limit_reached(), + "Limit should not be reached when deleted entries are less than limit" + ); + + // Increment deleted entries count to reach the limit + limiter.increment_deleted_entries_count_by(5); + assert!( + limiter.is_limit_reached(), + "Limit should be reached when deleted entries equal the limit" + ); + + // Reset the limiter + limiter = PruneLimiter::default(); + + // Set a time limit and check + limiter = limiter.set_time_limit(Duration::new(0, 10_000_000)); // 10 milliseconds + + // Sleep for 5 milliseconds (less than the time limit) + sleep(Duration::new(0, 5_000_000)); // 5 milliseconds + assert!( + !limiter.is_limit_reached(), + "Limit should not be reached when time limit not reached" + ); + + // Sleep for another 10 milliseconds (totaling 15 milliseconds) + sleep(Duration::new(0, 10_000_000)); // 10 milliseconds + assert!(limiter.is_limit_reached(), "Limit should be reached when time limit is reached"); + } +} From 1f7818bb9f61bef36030f30c9d83e4a9e0a88213 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:06:28 +0200 Subject: [PATCH 035/159] feat(provider): add `test_race` to `BlockchainProvider2` tests (#11523) --- Cargo.lock | 1 + crates/storage/db/Cargo.toml | 4 +- crates/storage/db/src/lib.rs | 46 +++++- .../src/providers/blockchain_provider.rs | 148 +++++++++++++++++- 4 files changed, 188 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 590f01a416fde..a66887900cb46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6702,6 +6702,7 @@ dependencies = [ "iai-callgrind", "metrics", "page_size", + "parking_lot 0.12.3", "paste", "pprof", "proptest", diff --git a/crates/storage/db/Cargo.toml b/crates/storage/db/Cargo.toml index ba012cf68af16..a075f77246378 100644 --- a/crates/storage/db/Cargo.toml +++ b/crates/storage/db/Cargo.toml @@ -50,6 +50,7 @@ derive_more.workspace = true paste.workspace = true rustc-hash = { workspace = true, optional = true } sysinfo = { version = "0.31", default-features = false, features = ["system"] } +parking_lot = { workspace = true, optional = true } # arbitrary utils strum = { workspace = true, features = ["derive"], optional = true } @@ -61,6 +62,7 @@ rand.workspace = true serde_json.workspace = true tempfile.workspace = true test-fuzz.workspace = true +parking_lot.workspace = true pprof = { workspace = true, features = [ "flamegraph", @@ -88,7 +90,7 @@ mdbx = [ "dep:strum", "dep:rustc-hash", ] -test-utils = ["dep:tempfile", "arbitrary"] +test-utils = ["dep:tempfile", "arbitrary", "parking_lot"] bench = [] arbitrary = ["reth-primitives/arbitrary", "reth-db-api/arbitrary"] optimism = [] diff --git a/crates/storage/db/src/lib.rs b/crates/storage/db/src/lib.rs index a9f073d7b5462..7090b4262fd78 100644 --- a/crates/storage/db/src/lib.rs +++ b/crates/storage/db/src/lib.rs @@ -44,6 +44,7 @@ pub use reth_db_api::*; pub mod test_utils { use super::*; use crate::mdbx::DatabaseArguments; + use parking_lot::RwLock; use reth_db_api::{ database::Database, database_metrics::{DatabaseMetadata, DatabaseMetadataValue, DatabaseMetrics}, @@ -52,6 +53,7 @@ pub mod test_utils { use reth_fs_util; use reth_libmdbx::MaxReadTransactionDuration; use std::{ + fmt::Formatter, path::{Path, PathBuf}, sync::Arc, }; @@ -69,10 +71,19 @@ pub mod test_utils { pub const ERROR_TEMPDIR: &str = "Not able to create a temporary directory."; /// A database will delete the db dir when dropped. - #[derive(Debug)] pub struct TempDatabase { db: Option, path: PathBuf, + /// Executed right before a database transaction is created. + pre_tx_hook: RwLock>, + /// Executed right after a database transaction is created. + post_tx_hook: RwLock>, + } + + impl std::fmt::Debug for TempDatabase { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TempDatabase").field("db", &self.db).field("path", &self.path).finish() + } } impl Drop for TempDatabase { @@ -85,6 +96,16 @@ pub mod test_utils { } impl TempDatabase { + /// Create new [`TempDatabase`] instance. + pub fn new(db: DB, path: PathBuf) -> Self { + Self { + db: Some(db), + path, + pre_tx_hook: RwLock::new(Box::new(|| ())), + post_tx_hook: RwLock::new(Box::new(|| ())), + } + } + /// Returns the reference to inner db. pub fn db(&self) -> &DB { self.db.as_ref().unwrap() @@ -99,13 +120,28 @@ pub mod test_utils { pub fn into_inner_db(mut self) -> DB { self.db.take().unwrap() // take out db to avoid clean path in drop fn } + + /// Sets [`TempDatabase`] new pre transaction creation hook. + pub fn set_pre_transaction_hook(&self, hook: Box) { + let mut db_hook = self.pre_tx_hook.write(); + *db_hook = hook; + } + + /// Sets [`TempDatabase`] new post transaction creation hook. + pub fn set_post_transaction_hook(&self, hook: Box) { + let mut db_hook = self.post_tx_hook.write(); + *db_hook = hook; + } } impl Database for TempDatabase { type TX = ::TX; type TXMut = ::TXMut; fn tx(&self) -> Result { - self.db().tx() + self.pre_tx_hook.read()(); + let tx = self.db().tx()?; + self.post_tx_hook.read()(); + Ok(tx) } fn tx_mut(&self) -> Result { @@ -150,7 +186,7 @@ pub mod test_utils { ) .expect(&emsg); - Arc::new(TempDatabase { db: Some(db), path }) + Arc::new(TempDatabase::new(db, path)) } /// Create read/write database for testing @@ -162,7 +198,7 @@ pub mod test_utils { .with_max_read_transaction_duration(Some(MaxReadTransactionDuration::Unbounded)), ) .expect(ERROR_DB_CREATION); - Arc::new(TempDatabase { db: Some(db), path }) + Arc::new(TempDatabase::new(db, path)) } /// Create read only database for testing @@ -175,7 +211,7 @@ pub mod test_utils { init_db(path.as_path(), args.clone()).expect(ERROR_DB_CREATION); } let db = open_db_read_only(path.as_path(), args).expect(ERROR_DB_OPEN); - Arc::new(TempDatabase { db: Some(db), path }) + Arc::new(TempDatabase::new(db, path)) } } diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 9a2a3d0a77ee5..fdd31cd848f77 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1486,20 +1486,26 @@ mod tests { MockNodeTypesWithDB, }, writer::UnifiedStorageWriter, - BlockWriter, CanonChainTracker, StaticFileProviderFactory, StaticFileWriter, + BlockWriter, CanonChainTracker, ProviderFactory, StaticFileProviderFactory, + StaticFileWriter, }; use alloy_eips::{BlockHashOrNumber, BlockNumHash, BlockNumberOrTag}; - use alloy_primitives::B256; + use alloy_primitives::{BlockNumber, B256}; use itertools::Itertools; use rand::Rng; use reth_chain_state::{ test_utils::TestBlockBuilder, CanonStateNotification, CanonStateSubscriptions, - ExecutedBlock, NewCanonicalChain, + CanonicalInMemoryState, ExecutedBlock, NewCanonicalChain, }; use reth_chainspec::{ ChainSpec, ChainSpecBuilder, ChainSpecProvider, EthereumHardfork, MAINNET, }; - use reth_db::models::{AccountBeforeTx, StoredBlockBodyIndices}; + use reth_db::{ + models::{AccountBeforeTx, StoredBlockBodyIndices}, + tables, + }; + use reth_db_api::{cursor::DbCursorRO, transaction::DbTx}; + use reth_errors::ProviderError; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives::{ Receipt, SealedBlock, StaticFileSegment, TransactionMeta, TransactionSignedNoHash, @@ -1586,9 +1592,27 @@ mod tests { let factory = create_test_provider_factory_with_chain_spec(chain_spec); let provider_rw = factory.database_provider_rw()?; + let static_file_provider = factory.static_file_provider(); + + // Write transactions to static files with the right `tx_num`` + let mut bodies_cursor = provider_rw.tx_ref().cursor_read::()?; + let mut tx_num = bodies_cursor + .seek_exact(database_blocks.first().as_ref().unwrap().number.saturating_sub(1))? + .map(|(_, indices)| indices.next_tx_num()) + .unwrap_or_default(); // Insert blocks into the database for block in &database_blocks { + // TODO: this should be moved inside `insert_historical_block`: + let mut transactions_writer = + static_file_provider.latest_writer(StaticFileSegment::Transactions)?; + transactions_writer.increment_block(block.number)?; + for tx in block.body.transactions() { + let tx: TransactionSignedNoHash = tx.clone().into(); + transactions_writer.append_transaction(tx_num, &tx)?; + tx_num += 1; + } + provider_rw.insert_historical_block( block.clone().seal_with_senders().expect("failed to seal block with senders"), )?; @@ -1602,7 +1626,9 @@ mod tests { .append_receipts_from_blocks( // The initial block number is required database_blocks.first().map(|b| b.number).unwrap_or_default(), - receipts.iter().map(|vec| vec.clone().into_iter().map(Some).collect::>()), + receipts[..database_blocks.len()] + .iter() + .map(|vec| vec.clone().into_iter().map(Some).collect::>()), )?; // Commit to both storages: database and static files @@ -1668,6 +1694,42 @@ mod tests { ) } + /// This will persist the last block in-memory and delete it from + /// `canonical_in_memory_state` right after a database read transaction is created. + /// + /// This simulates a RPC method having a different view than when its database transaction was + /// created. + fn persist_block_after_db_tx_creation( + provider: Arc>, + block_number: BlockNumber, + ) { + let hook_provider = provider.clone(); + provider.database.db_ref().set_post_transaction_hook(Box::new(move || { + if let Some(state) = hook_provider.canonical_in_memory_state.head_state() { + if state.anchor().number + 1 == block_number { + let mut lowest_memory_block = + state.parent_state_chain().last().expect("qed").block(); + let num_hash = lowest_memory_block.block().num_hash(); + + let mut execution_output = (*lowest_memory_block.execution_output).clone(); + execution_output.first_block = lowest_memory_block.block().number; + lowest_memory_block.execution_output = Arc::new(execution_output); + + // Push to disk + let provider_rw = hook_provider.database_provider_rw().unwrap(); + UnifiedStorageWriter::from(&provider_rw, &hook_provider.static_file_provider()) + .save_blocks(&[lowest_memory_block]) + .unwrap(); + UnifiedStorageWriter::commit(provider_rw, hook_provider.static_file_provider()) + .unwrap(); + + // Remove from memory + hook_provider.canonical_in_memory_state.remove_persisted_blocks(num_hash); + } + } + })); + } + #[test] fn test_block_reader_find_block_by_hash() -> eyre::Result<()> { // Initialize random number generator and provider factory @@ -3908,4 +3970,80 @@ mod tests { Ok(()) } + + #[test] + fn test_race() -> eyre::Result<()> { + let mut rng = generators::rng(); + let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT - 1, + TEST_BLOCKS_COUNT + 1, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; + + let provider = Arc::new(provider); + + // Old implementation was querying the database first. This is problematic, if there are + // changes AFTER the database transaction is created. + let old_transaction_hash_fn = + |hash: B256, + canonical_in_memory_state: CanonicalInMemoryState, + factory: ProviderFactory| { + assert!(factory.transaction_by_hash(hash)?.is_none(), "should not be in database"); + Ok::<_, ProviderError>(canonical_in_memory_state.transaction_by_hash(hash)) + }; + + // Correct implementation queries in-memory first + let correct_transaction_hash_fn = + |hash: B256, + canonical_in_memory_state: CanonicalInMemoryState, + _factory: ProviderFactory| { + if let Some(tx) = canonical_in_memory_state.transaction_by_hash(hash) { + return Ok::<_, ProviderError>(Some(tx)) + } + panic!("should not be in database"); + // _factory.transaction_by_hash(hash) + }; + + // OLD BEHAVIOUR + { + // This will persist block 1 AFTER a database is created. Moving it from memory to + // storage. + persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[0].number); + let to_be_persisted_tx = in_memory_blocks[0].body.transactions[0].clone(); + + // Even though the block exists, given the order of provider queries done in the method + // above, we do not see it. + assert_eq!( + old_transaction_hash_fn( + to_be_persisted_tx.hash(), + provider.canonical_in_memory_state(), + provider.database.clone() + ), + Ok(None) + ); + } + + // CORRECT BEHAVIOUR + { + // This will persist block 1 AFTER a database is created. Moving it from memory to + // storage. + persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[1].number); + let to_be_persisted_tx = in_memory_blocks[1].body.transactions[0].clone(); + + assert_eq!( + correct_transaction_hash_fn( + to_be_persisted_tx.hash(), + provider.canonical_in_memory_state(), + provider.database.clone() + ), + Ok(Some(to_be_persisted_tx)) + ); + } + + Ok(()) + } } From 831c4486b8465c83241addc6026fb6aca3821251 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 7 Oct 2024 05:12:35 -0400 Subject: [PATCH 036/159] fix(tree): make state methods work for historical blocks (#11265) Co-authored-by: Roman Krasiuk Co-authored-by: Federico Gimenez --- .../src/providers/blockchain_provider.rs | 239 +++++++++++++++++- .../src/providers/database/provider.rs | 17 +- crates/storage/storage-api/src/storage.rs | 11 + 3 files changed, 258 insertions(+), 9 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index fdd31cd848f77..9d075015397fc 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -14,21 +14,27 @@ use reth_chain_state::{ BlockState, CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, MemoryOverlayStateProvider, }; -use reth_chainspec::ChainInfo; +use reth_chainspec::{ChainInfo, EthereumHardforks}; +use reth_db::models::BlockNumberAddress; use reth_db_api::models::{AccountBeforeTx, StoredBlockBodyIndices}; use reth_evm::ConfigureEvmEnv; -use reth_execution_types::ExecutionOutcome; +use reth_execution_types::{BundleStateInit, ExecutionOutcome, RevertsInit}; use reth_node_types::NodeTypesWithDB; use reth_primitives::{ - Account, Block, BlockWithSenders, EthereumHardforks, Header, Receipt, SealedBlock, - SealedBlockWithSenders, SealedHeader, TransactionMeta, TransactionSigned, - TransactionSignedNoHash, Withdrawal, Withdrawals, + Account, Block, BlockWithSenders, Header, Receipt, SealedBlock, SealedBlockWithSenders, + SealedHeader, StorageEntry, TransactionMeta, TransactionSigned, TransactionSignedNoHash, + Withdrawal, Withdrawals, }; use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; +use reth_storage_api::StorageChangeSetReader; use reth_storage_errors::provider::ProviderResult; -use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg}; +use revm::{ + db::states::PlainStorageRevert, + primitives::{BlockEnv, CfgEnvWithHandlerCfg}, +}; use std::{ + collections::{hash_map, HashMap}, ops::{Add, Bound, RangeBounds, RangeInclusive, Sub}, sync::Arc, time::Instant, @@ -122,6 +128,145 @@ impl BlockchainProvider2 { (start, end) } + /// Return the last N blocks of state, recreating the [`ExecutionOutcome`]. + /// + /// If the range is empty, or there are no blocks for the given range, then this returns `None`. + pub fn get_state( + &self, + range: RangeInclusive, + ) -> ProviderResult> { + if range.is_empty() { + return Ok(None) + } + let start_block_number = *range.start(); + let end_block_number = *range.end(); + + // We are not removing block meta as it is used to get block changesets. + let mut block_bodies = Vec::new(); + for block_num in range.clone() { + let block_body = self + .block_body_indices(block_num)? + .ok_or(ProviderError::BlockBodyIndicesNotFound(block_num))?; + block_bodies.push((block_num, block_body)) + } + + // get transaction receipts + let Some(from_transaction_num) = block_bodies.first().map(|body| body.1.first_tx_num()) + else { + return Ok(None) + }; + let Some(to_transaction_num) = block_bodies.last().map(|body| body.1.last_tx_num()) else { + return Ok(None) + }; + + let mut account_changeset = Vec::new(); + for block_num in range.clone() { + let changeset = + self.account_block_changeset(block_num)?.into_iter().map(|elem| (block_num, elem)); + account_changeset.extend(changeset); + } + + let mut storage_changeset = Vec::new(); + for block_num in range { + let changeset = self.storage_changeset(block_num)?; + storage_changeset.extend(changeset); + } + + let (state, reverts) = + self.populate_bundle_state(account_changeset, storage_changeset, end_block_number)?; + + let mut receipt_iter = + self.receipts_by_tx_range(from_transaction_num..=to_transaction_num)?.into_iter(); + + let mut receipts = Vec::with_capacity(block_bodies.len()); + // loop break if we are at the end of the blocks. + for (_, block_body) in block_bodies { + let mut block_receipts = Vec::with_capacity(block_body.tx_count as usize); + for tx_num in block_body.tx_num_range() { + let receipt = + receipt_iter.next().ok_or(ProviderError::ReceiptNotFound(tx_num.into()))?; + block_receipts.push(Some(receipt)); + } + receipts.push(block_receipts); + } + + Ok(Some(ExecutionOutcome::new_init( + state, + reverts, + // We skip new contracts since we never delete them from the database + Vec::new(), + receipts.into(), + start_block_number, + Vec::new(), + ))) + } + + /// Populate a [`BundleStateInit`] and [`RevertsInit`] using cursors over the + /// [`reth_db::PlainAccountState`] and [`reth_db::PlainStorageState`] tables, based on the given + /// storage and account changesets. + fn populate_bundle_state( + &self, + account_changeset: Vec<(u64, AccountBeforeTx)>, + storage_changeset: Vec<(BlockNumberAddress, StorageEntry)>, + block_range_end: BlockNumber, + ) -> ProviderResult<(BundleStateInit, RevertsInit)> { + let mut state: BundleStateInit = HashMap::new(); + let mut reverts: RevertsInit = HashMap::new(); + let state_provider = self.state_by_block_number_or_tag(block_range_end.into())?; + + // add account changeset changes + for (block_number, account_before) in account_changeset.into_iter().rev() { + let AccountBeforeTx { info: old_info, address } = account_before; + match state.entry(address) { + hash_map::Entry::Vacant(entry) => { + let new_info = state_provider.basic_account(address)?; + entry.insert((old_info, new_info, HashMap::new())); + } + hash_map::Entry::Occupied(mut entry) => { + // overwrite old account state. + entry.get_mut().0 = old_info; + } + } + // insert old info into reverts. + reverts.entry(block_number).or_default().entry(address).or_default().0 = Some(old_info); + } + + // add storage changeset changes + for (block_and_address, old_storage) in storage_changeset.into_iter().rev() { + let BlockNumberAddress((block_number, address)) = block_and_address; + // get account state or insert from plain state. + let account_state = match state.entry(address) { + hash_map::Entry::Vacant(entry) => { + let present_info = state_provider.basic_account(address)?; + entry.insert((present_info, present_info, HashMap::new())) + } + hash_map::Entry::Occupied(entry) => entry.into_mut(), + }; + + // match storage. + match account_state.2.entry(old_storage.key) { + hash_map::Entry::Vacant(entry) => { + let new_storage_value = + state_provider.storage(address, old_storage.key)?.unwrap_or_default(); + entry.insert((old_storage.value, new_storage_value)); + } + hash_map::Entry::Occupied(mut entry) => { + entry.get_mut().0 = old_storage.value; + } + }; + + reverts + .entry(block_number) + .or_default() + .entry(address) + .or_default() + .1 + .push(old_storage); + } + + Ok((state, reverts)) + } + /// Fetches a range of data from both in-memory state and persistent storage while a predicate /// is met. /// @@ -1426,6 +1571,57 @@ impl ForkChoiceSubscriptions for BlockchainProvider2 { } } +impl StorageChangeSetReader for BlockchainProvider2 { + fn storage_changeset( + &self, + block_number: BlockNumber, + ) -> ProviderResult> { + if let Some(state) = self.canonical_in_memory_state.state_by_number(block_number) { + let changesets = state + .block() + .execution_output + .bundle + .reverts + .clone() + .into_plain_state_reverts() + .storage + .into_iter() + .flatten() + .flat_map(|revert: PlainStorageRevert| { + revert.storage_revert.into_iter().map(move |(key, value)| { + ( + BlockNumberAddress((block_number, revert.address)), + StorageEntry { key: key.into(), value: value.to_previous_value() }, + ) + }) + }) + .collect(); + Ok(changesets) + } else { + // Perform checks on whether or not changesets exist for the block. + let provider = self.database.provider()?; + + // No prune checkpoint means history should exist and we should `unwrap_or(true)` + let storage_history_exists = provider + .get_prune_checkpoint(PruneSegment::StorageHistory)? + .and_then(|checkpoint| { + // return true if the block number is ahead of the prune checkpoint. + // + // The checkpoint stores the highest pruned block number, so we should make + // sure the block_number is strictly greater. + checkpoint.block_number.map(|checkpoint| block_number > checkpoint) + }) + .unwrap_or(true); + + if !storage_history_exists { + return Err(ProviderError::StateAtBlockPruned(block_number)) + } + + provider.storage_changeset(block_number) + } + } +} + impl ChangeSetReader for BlockchainProvider2 { fn account_block_changeset( &self, @@ -1446,7 +1642,25 @@ impl ChangeSetReader for BlockchainProvider2 { .collect(); Ok(changesets) } else { - self.database.provider()?.account_block_changeset(block_number) + // Perform checks on whether or not changesets exist for the block. + let provider = self.database.provider()?; + // No prune checkpoint means history should exist and we should `unwrap_or(true)` + let account_history_exists = provider + .get_prune_checkpoint(PruneSegment::AccountHistory)? + .and_then(|checkpoint| { + // return true if the block number is ahead of the prune checkpoint. + // + // The checkpoint stores the highest pruned block number, so we should make + // sure the block_number is strictly greater. + checkpoint.block_number.map(|checkpoint| block_number > checkpoint) + }) + .unwrap_or(true); + + if !account_history_exists { + return Err(ProviderError::StateAtBlockPruned(block_number)) + } + + provider.account_block_changeset(block_number) } } } @@ -1461,12 +1675,21 @@ impl AccountReader for BlockchainProvider2 { } impl StateReader for BlockchainProvider2 { + /// Re-constructs the [`ExecutionOutcome`] from in-memory and database state, if necessary. + /// + /// If data for the block does not exist, this will return [`None`]. + /// + /// NOTE: This cannot be called safely in a loop outside of the blockchain tree thread. This is + /// because the [`CanonicalInMemoryState`] could change during a reorg, causing results to be + /// inconsistent. Currently this can safely be called within the blockchain tree thread, + /// because the tree thread is responsible for modifying the [`CanonicalInMemoryState`] in the + /// first place. fn get_state(&self, block: BlockNumber) -> ProviderResult> { if let Some(state) = self.canonical_in_memory_state.state_by_number(block) { let state = state.block_ref().execution_outcome().clone(); Ok(Some(state)) } else { - self.database.provider()?.get_state(block..=block) + self.get_state(block..=block) } } } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 8627cacabb4ca..1afd4da3fa8cc 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -46,7 +46,7 @@ use reth_primitives::{ }; use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; -use reth_storage_api::TryIntoHistoricalStateProvider; +use reth_storage_api::{StorageChangeSetReader, TryIntoHistoricalStateProvider}; use reth_storage_errors::provider::{ProviderResult, RootMismatch}; use reth_trie::{ prefix_set::{PrefixSet, PrefixSetMut, TriePrefixSets}, @@ -1414,6 +1414,21 @@ impl AccountExtReader for DatabaseProvider StorageChangeSetReader for DatabaseProvider { + fn storage_changeset( + &self, + block_number: BlockNumber, + ) -> ProviderResult> { + let range = block_number..=block_number; + let storage_range = BlockNumberAddress::range(range); + self.tx + .cursor_dup_read::()? + .walk_range(storage_range)? + .map(|result| -> ProviderResult<_> { Ok(result?) }) + .collect() + } +} + impl ChangeSetReader for DatabaseProvider { fn account_block_changeset( &self, diff --git a/crates/storage/storage-api/src/storage.rs b/crates/storage/storage-api/src/storage.rs index 91d0bc8c73535..e1443347e4bb8 100644 --- a/crates/storage/storage-api/src/storage.rs +++ b/crates/storage/storage-api/src/storage.rs @@ -1,4 +1,5 @@ use alloy_primitives::{Address, BlockNumber, B256}; +use reth_db_api::models::BlockNumberAddress; use reth_primitives::StorageEntry; use reth_storage_errors::provider::ProviderResult; use std::{ @@ -30,3 +31,13 @@ pub trait StorageReader: Send + Sync { range: RangeInclusive, ) -> ProviderResult>>; } + +/// Storage ChangeSet reader +#[auto_impl::auto_impl(&, Arc, Box)] +pub trait StorageChangeSetReader: Send + Sync { + /// Iterate over storage changesets and return the storage state from before this block. + fn storage_changeset( + &self, + block_number: BlockNumber, + ) -> ProviderResult>; +} From e79ec74c38e1187fc720ddf0d0a54473922e4fe8 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Mon, 7 Oct 2024 11:41:30 +0200 Subject: [PATCH 037/159] rpc: use `eth_api()` method (#11516) --- crates/rpc/rpc/src/debug.rs | 45 +++++++++++++------------------- crates/rpc/rpc/src/eth/bundle.rs | 16 +++++++----- crates/rpc/rpc/src/trace.rs | 30 +++++++++------------ 3 files changed, 40 insertions(+), 51 deletions(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index f4aba778f5729..a83ba68b42f5e 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -208,8 +208,8 @@ where .ok_or(EthApiError::HeaderNotFound(block_id))?; let ((cfg, block_env, _), block) = futures::try_join!( - self.inner.eth_api.evm_env_at(block_hash.into()), - self.inner.eth_api.block_with_senders(block_id), + self.eth_api().evm_env_at(block_hash.into()), + self.eth_api().block_with_senders(block_id), )?; let block = block.ok_or(EthApiError::HeaderNotFound(block_id))?; @@ -235,11 +235,11 @@ where tx_hash: B256, opts: GethDebugTracingOptions, ) -> Result { - let (transaction, block) = match self.inner.eth_api.transaction_and_block(tx_hash).await? { + let (transaction, block) = match self.eth_api().transaction_and_block(tx_hash).await? { None => return Err(EthApiError::TransactionNotFound.into()), Some(res) => res, }; - let (cfg, block_env, _) = self.inner.eth_api.evm_env_at(block.hash().into()).await?; + let (cfg, block_env, _) = self.eth_api().evm_env_at(block.hash().into()).await?; // we need to get the state of the parent block because we're essentially replaying the // block the transaction is included in @@ -248,8 +248,7 @@ where let block_txs = block.into_transactions_ecrecovered(); let this = self.clone(); - self.inner - .eth_api + self.eth_api() .spawn_with_state_at_block(state_at, move |state| { // configure env for the target transaction let tx = transaction.into_recovered(); @@ -312,8 +311,7 @@ where GethDebugBuiltInTracerType::FourByteTracer => { let mut inspector = FourByteInspector::default(); let inspector = self - .inner - .eth_api + .eth_api() .spawn_with_call_at(call, at, overrides, move |db, env| { this.eth_api().inspect(db, env, &mut inspector)?; Ok(inspector) @@ -331,8 +329,7 @@ where ); let frame = self - .inner - .eth_api + .eth_api() .spawn_with_call_at(call, at, overrides, move |db, env| { let (res, env) = this.eth_api().inspect(db, env, &mut inspector)?; let frame = inspector @@ -353,8 +350,7 @@ where ); let frame = self - .inner - .eth_api + .eth_api() .spawn_with_call_at(call, at, overrides, move |db, env| { // wrapper is hack to get around 'higher-ranked lifetime error', // see @@ -434,11 +430,10 @@ where GethDebugTracerType::JsTracer(code) => { let config = tracer_config.into_json(); - let (_, _, at) = self.inner.eth_api.evm_env_at(at).await?; + let (_, _, at) = self.eth_api().evm_env_at(at).await?; let res = self - .inner - .eth_api + .eth_api() .spawn_with_call_at(call, at, overrides, move |db, env| { // wrapper is hack to get around 'higher-ranked lifetime error', see // @@ -464,8 +459,7 @@ where let mut inspector = TracingInspector::new(inspector_config); let (res, tx_gas_limit, inspector) = self - .inner - .eth_api + .eth_api() .spawn_with_call_at(call, at, overrides, move |db, env| { let (res, env) = this.eth_api().inspect(db, env, &mut inspector)?; Ok((res, env.tx.gas_limit, inspector)) @@ -499,8 +493,8 @@ where let target_block = block_number.unwrap_or_default(); let ((cfg, mut block_env, _), block) = futures::try_join!( - self.inner.eth_api.evm_env_at(target_block), - self.inner.eth_api.block_with_senders(target_block), + self.eth_api().evm_env_at(target_block), + self.eth_api().block_with_senders(target_block), )?; let opts = opts.unwrap_or_default(); @@ -524,8 +518,7 @@ where let this = self.clone(); - self.inner - .eth_api + self.eth_api() .spawn_with_state_at_block(at.into(), move |state| { // the outer vec for the bundles let mut all_bundles = Vec::with_capacity(bundles.len()); @@ -546,7 +539,7 @@ where ), handler_cfg: cfg.handler_cfg, }; - let (res, _) = this.inner.eth_api.transact(&mut db, env)?; + let (res, _) = this.eth_api().transact(&mut db, env)?; db.commit(res.state); } } @@ -604,14 +597,12 @@ where ) -> Result { let this = self.clone(); let block = this - .inner - .eth_api + .eth_api() .block_with_senders(block_id.into()) .await? .ok_or(EthApiError::HeaderNotFound(block_id.into()))?; - self.inner - .eth_api + self.eth_api() .spawn_with_state_at_block(block.parent_hash.into(), move |state_provider| { let db = StateProviderDatabase::new(&state_provider); let block_executor = this.inner.block_executor.executor(db); @@ -868,7 +859,7 @@ where /// /// Returns the bytes of the transaction for the given hash. async fn raw_transaction(&self, hash: B256) -> RpcResult> { - self.inner.eth_api.raw_transaction_by_hash(hash).await.map_err(Into::into) + self.eth_api().raw_transaction_by_hash(hash).await.map_err(Into::into) } /// Handler for `debug_getRawTransactions` diff --git a/crates/rpc/rpc/src/eth/bundle.rs b/crates/rpc/rpc/src/eth/bundle.rs index bede4599e1a1b..e97497786ede4 100644 --- a/crates/rpc/rpc/src/eth/bundle.rs +++ b/crates/rpc/rpc/src/eth/bundle.rs @@ -37,6 +37,11 @@ impl EthBundle { pub fn new(eth_api: Eth, blocking_task_guard: BlockingTaskGuard) -> Self { Self { inner: Arc::new(EthBundleInner { eth_api, blocking_task_guard }) } } + + /// Access the underlying `Eth` API. + pub fn eth_api(&self) -> &Eth { + &self.inner.eth_api + } } impl EthBundle @@ -103,7 +108,7 @@ where let block_id: alloy_rpc_types::BlockId = state_block_number.into(); // Note: the block number is considered the `parent` block: - let (cfg, mut block_env, at) = self.inner.eth_api.evm_env_at(block_id).await?; + let (cfg, mut block_env, at) = self.eth_api().evm_env_at(block_id).await?; // need to adjust the timestamp for the next block if let Some(timestamp) = timestamp { @@ -125,12 +130,12 @@ where } else if cfg.handler_cfg.spec_id.is_enabled_in(SpecId::LONDON) { let parent_block = block_env.number.saturating_to::(); // here we need to fetch the _next_ block's basefee based on the parent block - let parent = LoadPendingBlock::provider(&self.inner.eth_api) + let parent = LoadPendingBlock::provider(self.eth_api()) .header_by_number(parent_block) .map_err(Eth::Error::from_eth_err)? .ok_or(EthApiError::HeaderNotFound(parent_block.into()))?; if let Some(base_fee) = parent.next_block_base_fee( - LoadPendingBlock::provider(&self.inner.eth_api) + LoadPendingBlock::provider(self.eth_api()) .chain_spec() .base_fee_params_at_block(parent_block), ) { @@ -142,10 +147,9 @@ where // use the block number of the request block_env.number = U256::from(block_number); - let eth_api = self.inner.eth_api.clone(); + let eth_api = self.eth_api().clone(); - self.inner - .eth_api + self.eth_api() .spawn_with_state_at_block(at, move |state| { let coinbase = block_env.coinbase; let basefee = Some(block_env.basefee.to::()); diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index e905ff685c863..687762b74b5ed 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -120,7 +120,7 @@ where ) -> Result { let tx = recover_raw_transaction(tx)?; - let (cfg, block, at) = self.inner.eth_api.evm_env_at(block_id.unwrap_or_default()).await?; + let (cfg, block, at) = self.eth_api().evm_env_at(block_id.unwrap_or_default()).await?; let env = EnvWithHandlerCfg::new_with_cfg_env( cfg, @@ -130,8 +130,7 @@ where let config = TracingInspectorConfig::from_parity_config(&trace_types); - self.inner - .eth_api + self.eth_api() .spawn_trace_at_with_state(env, config, at, move |inspector, res, db| { inspector .into_parity_builder() @@ -151,7 +150,7 @@ where block_id: Option, ) -> Result, Eth::Error> { let at = block_id.unwrap_or(BlockId::pending()); - let (cfg, block_env, at) = self.inner.eth_api.evm_env_at(at).await?; + let (cfg, block_env, at) = self.eth_api().evm_env_at(at).await?; let this = self.clone(); // execute all transactions on top of each other and record the traces @@ -202,8 +201,7 @@ where trace_types: HashSet, ) -> Result { let config = TracingInspectorConfig::from_parity_config(&trace_types); - self.inner - .eth_api + self.eth_api() .spawn_trace_transaction_in_block(hash, config, move |_, inspector, res, db| { let trace_res = inspector .into_parity_builder() @@ -285,7 +283,7 @@ where let mut block_traces = Vec::with_capacity(blocks.len()); for block in &blocks { let matcher = matcher.clone(); - let traces = self.inner.eth_api.trace_block_until( + let traces = self.eth_api().trace_block_until( block.number.into(), None, TracingInspectorConfig::default_parity(), @@ -345,8 +343,7 @@ where &self, hash: B256, ) -> Result>, Eth::Error> { - self.inner - .eth_api + self.eth_api() .spawn_trace_transaction_in_block( hash, TracingInspectorConfig::default_parity(), @@ -364,7 +361,7 @@ where &self, block_id: BlockId, ) -> Result>, Eth::Error> { - let traces = self.inner.eth_api.trace_block_with( + let traces = self.eth_api().trace_block_with( block_id, TracingInspectorConfig::default_parity(), |tx_info, inspector, _, _, _| { @@ -374,7 +371,7 @@ where }, ); - let block = self.inner.eth_api.block(block_id); + let block = self.eth_api().block(block_id); let (maybe_traces, maybe_block) = futures::try_join!(traces, block)?; let mut maybe_traces = @@ -399,8 +396,7 @@ where block_id: BlockId, trace_types: HashSet, ) -> Result>, Eth::Error> { - self.inner - .eth_api + self.eth_api() .trace_block_with( block_id, TracingInspectorConfig::from_parity_config(&trace_types), @@ -431,8 +427,7 @@ where &self, tx_hash: B256, ) -> Result, Eth::Error> { - self.inner - .eth_api + self.eth_api() .spawn_trace_transaction_in_block_with_inspector( tx_hash, OpcodeGasInspector::default(), @@ -456,8 +451,7 @@ where block_id: BlockId, ) -> Result, Eth::Error> { let res = self - .inner - .eth_api + .eth_api() .trace_block_inspector( block_id, OpcodeGasInspector::default, @@ -473,7 +467,7 @@ where let Some(transactions) = res else { return Ok(None) }; - let Some(block) = self.inner.eth_api.block(block_id).await? else { return Ok(None) }; + let Some(block) = self.eth_api().block(block_id).await? else { return Ok(None) }; Ok(Some(BlockOpcodeGas { block_hash: block.hash(), From b5e4ecb2d65b97f77b6a329a297d8dd4f81d4118 Mon Sep 17 00:00:00 2001 From: crazykissshout Date: Mon, 7 Oct 2024 13:25:24 +0200 Subject: [PATCH 038/159] chore: delete rpc-types (#11528) --- docs/repo/layout.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/repo/layout.md b/docs/repo/layout.md index 6ba14e652bf1c..8eb0782b988c7 100644 --- a/docs/repo/layout.md +++ b/docs/repo/layout.md @@ -132,7 +132,6 @@ The IPC transport lives in [`rpc/ipc`](../../crates/rpc/ipc). - [`rpc/rpc-api`](../../crates/rpc/rpc-api): RPC traits - Supported transports: HTTP, WS, IPC - Supported namespaces: `eth_`, `engine_`, `debug_` -- [`rpc/rpc-types`](../../crates/rpc/rpc-types): Types relevant for the RPC endpoints above, grouped by namespace - [`rpc/rpc-eth-api`](../../crates/rpc/rpc-eth-api/): Reth RPC 'eth' namespace API (including interface and implementation), this crate is re-exported by `rpc/rpc-api` - [`rpc/rpc-eth-types`](../../crates/rpc/rpc-eth-types/): Types `supporting implementation` of 'eth' namespace RPC server API - [`rpc/rpc-server-types`](../../crates/rpc/rpc-server-types/): RPC server types and constants From 0d7883f5a0fdaea08b6d1897641e9060c66163ef Mon Sep 17 00:00:00 2001 From: tedison <76473430+edisontim@users.noreply.github.com> Date: Mon, 7 Oct 2024 07:42:56 -0400 Subject: [PATCH 039/159] feat: add get_highest_tx_by_sender to pools (#11514) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-eth-api/src/helpers/state.rs | 30 ++++++++++----------- crates/transaction-pool/src/lib.rs | 7 +++++ crates/transaction-pool/src/noop.rs | 7 +++++ crates/transaction-pool/src/pool/mod.rs | 9 +++++++ crates/transaction-pool/src/traits.rs | 6 +++++ 5 files changed, 44 insertions(+), 15 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index dee4e58956385..4c77aec1ae15d 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -13,7 +13,7 @@ use reth_provider::{ }; use reth_rpc_eth_types::{EthApiError, EthStateCache, PendingBlockEnv, RpcInvalidTransactionError}; use reth_rpc_types_compat::proof::from_primitive_account_proof; -use reth_transaction_pool::{PoolTransaction, TransactionPool}; +use reth_transaction_pool::TransactionPool; use revm_primitives::{BlockEnv, CfgEnvWithHandlerCfg, SpecId}; use crate::{EthApiTypes, FromEthApiError}; @@ -280,23 +280,23 @@ pub trait LoadState: EthApiTypes { if block_id == Some(BlockId::pending()) { // for pending tag we need to find the highest nonce in the pool - let address_txs = this.pool().get_transactions_by_sender(address); - if let Some(highest_pool_nonce) = - address_txs.iter().map(|item| item.transaction.nonce()).max() + if let Some(highest_pool_tx) = + this.pool().get_highest_transaction_by_sender(address) { - // and the corresponding txcount is nonce + 1 - let next_nonce = - nonce.max(highest_pool_nonce).checked_add(1).ok_or_else(|| { - Self::Error::from(EthApiError::InvalidTransaction( - RpcInvalidTransactionError::NonceMaxValue, - )) - })?; - - let tx_count = nonce.max(next_nonce); - return Ok(U256::from(tx_count)) + { + // and the corresponding txcount is nonce + 1 + let next_nonce = + nonce.max(highest_pool_tx.nonce()).checked_add(1).ok_or_else(|| { + Self::Error::from(EthApiError::InvalidTransaction( + RpcInvalidTransactionError::NonceMaxValue, + )) + })?; + + let tx_count = nonce.max(next_nonce); + return Ok(U256::from(tx_count)); + } } } - Ok(U256::from(nonce)) }) } diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 744dd44d2f83a..a5acd6edba5a8 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -489,6 +489,13 @@ where self.pool.get_transactions_by_sender(sender) } + fn get_highest_transaction_by_sender( + &self, + sender: Address, + ) -> Option>> { + self.pool.get_highest_transaction_by_sender(sender) + } + fn get_transaction_by_sender_and_nonce( &self, sender: Address, diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index 0c4caa5731402..ddab4f6227415 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -206,6 +206,13 @@ impl TransactionPool for NoopTransactionPool { vec![] } + fn get_highest_transaction_by_sender( + &self, + _sender: Address, + ) -> Option>> { + None + } + fn get_transaction_by_sender_and_nonce( &self, _sender: Address, diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index b78f5128686c6..090b92fb65949 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -744,6 +744,15 @@ where self.get_pool_data().get_transactions_by_sender(sender_id) } + /// Returns the highest transaction of the address + pub(crate) fn get_highest_transaction_by_sender( + &self, + sender: Address, + ) -> Option>> { + let sender_id = self.get_sender_id(sender); + self.get_pool_data().get_highest_transaction_by_sender(sender_id) + } + /// Returns all transactions that where submitted with the given [`TransactionOrigin`] pub(crate) fn get_transactions_by_origin( &self, diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index f44e66e457a1f..1598877ff3b6b 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -334,6 +334,12 @@ pub trait TransactionPool: Send + Sync + Clone { sender: Address, ) -> Vec>>; + /// Returns the highest transaction sent by a given user + fn get_highest_transaction_by_sender( + &self, + sender: Address, + ) -> Option>>; + /// Returns a transaction sent by a given user and a nonce fn get_transaction_by_sender_and_nonce( &self, From 458632d293e59316c1f841c6ae588313a9a9f0a8 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:52:01 +0200 Subject: [PATCH 040/159] ci: add `windows` cargo check (#11468) --- .github/workflows/windows.yml | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/windows.yml diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml new file mode 100644 index 0000000000000..03c491b368a65 --- /dev/null +++ b/.github/workflows/windows.yml @@ -0,0 +1,47 @@ +# Windows build + +name: windows + +on: + push: + branches: [main] + pull_request: + branches: [main] + merge_group: + +jobs: + check-reth: + runs-on: ubuntu-20.04 + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + target: x86_64-pc-windows-gnu + - uses: taiki-e/install-action@cross + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: mingw-w64 + run: sudo apt-get install -y mingw-w64 + - name: Check Reth + run: cargo check --target x86_64-pc-windows-gnu + + check-op-reth: + runs-on: ubuntu-20.04 + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + target: x86_64-pc-windows-gnu + - uses: taiki-e/install-action@cross + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: mingw-w64 + run: sudo apt-get install -y mingw-w64 + - name: Check OP-Reth + run: cargo check -p op-reth --features optimism --target x86_64-pc-windows-gnu From f155ef1c426f0ddfe3375b248a996b8b62e39046 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Oct 2024 16:10:39 +0200 Subject: [PATCH 041/159] fix: acquire permit first (#11537) --- crates/rpc/rpc-eth-api/src/helpers/state.rs | 27 +++++++++++---------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 4c77aec1ae15d..0008454c93a49 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -92,25 +92,26 @@ pub trait EthState: LoadState + SpawnBlocking { where Self: EthApiSpec, { - let chain_info = self.chain_info().map_err(Self::Error::from_eth_err)?; - let block_id = block_id.unwrap_or_default(); - - // Check whether the distance to the block exceeds the maximum configured window. - let block_number = LoadState::provider(self) - .block_number_for_id(block_id) - .map_err(Self::Error::from_eth_err)? - .ok_or(EthApiError::HeaderNotFound(block_id))?; - let max_window = self.max_proof_window(); - if chain_info.best_number.saturating_sub(block_number) > max_window { - return Err(EthApiError::ExceedsMaxProofWindow.into()) - } - Ok(async move { let _permit = self .acquire_owned() .await .map_err(RethError::other) .map_err(EthApiError::Internal)?; + + let chain_info = self.chain_info().map_err(Self::Error::from_eth_err)?; + let block_id = block_id.unwrap_or_default(); + + // Check whether the distance to the block exceeds the maximum configured window. + let block_number = LoadState::provider(self) + .block_number_for_id(block_id) + .map_err(Self::Error::from_eth_err)? + .ok_or(EthApiError::HeaderNotFound(block_id))?; + let max_window = self.max_proof_window(); + if chain_info.best_number.saturating_sub(block_number) > max_window { + return Err(EthApiError::ExceedsMaxProofWindow.into()) + } + self.spawn_blocking_io(move |this| { let state = this.state_at_block_id(block_id)?; let storage_keys = keys.iter().map(|key| key.0).collect::>(); From 96d2dc35ab19cb6b571901f95ceb25b357c356fb Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Oct 2024 16:47:52 +0200 Subject: [PATCH 042/159] chore: dont fail on ttd (#11539) --- crates/rpc/rpc-eth-api/src/helpers/block.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 3a26536cc7cf7..d7e081d80f807 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -6,7 +6,7 @@ use alloy_rpc_types::{Header, Index}; use futures::Future; use reth_primitives::{BlockId, Receipt, SealedBlock, SealedBlockWithSenders}; use reth_provider::{BlockIdReader, BlockReader, BlockReaderIdExt, HeaderProvider}; -use reth_rpc_eth_types::{EthApiError, EthStateCache}; +use reth_rpc_eth_types::EthStateCache; use reth_rpc_types_compat::block::{from_block, uncle_block_from_header}; use crate::{FromEthApiError, FullEthApiTypes, RpcBlock, RpcReceipt}; @@ -47,13 +47,20 @@ pub trait EthBlocks: LoadBlock { async move { let Some(block) = self.block_with_senders(block_id).await? else { return Ok(None) }; let block_hash = block.hash(); - let total_difficulty = EthBlocks::provider(self) + let mut total_difficulty = EthBlocks::provider(self) .header_td_by_number(block.number) - .map_err(Self::Error::from_eth_err)? - .ok_or(EthApiError::HeaderNotFound(block_id))?; + .map_err(Self::Error::from_eth_err)?; + if total_difficulty.is_none() { + // if we failed to find td after we successfully loaded the block, try again using + // the hash this only matters if the chain is currently transitioning the merge block and there's a reorg: + total_difficulty = EthBlocks::provider(self) + .header_td(&block.hash()) + .map_err(Self::Error::from_eth_err)?; + } + let block = from_block::( block.unseal(), - total_difficulty, + total_difficulty.unwrap_or_default(), full.into(), Some(block_hash), ) From 04097bdbb81cbe47e0739a93a69a06f8153a51dd Mon Sep 17 00:00:00 2001 From: David <39963997+No0key@users.noreply.github.com> Date: Mon, 7 Oct 2024 18:12:52 +0300 Subject: [PATCH 043/159] grafana: add metrics of all transactions in pool by type (#11515) Co-authored-by: Emilia Hane Co-authored-by: Emilia Hane --- etc/grafana/dashboards/reth-mempool.json | 172 ++++++++++++++++++++++- 1 file changed, 171 insertions(+), 1 deletion(-) diff --git a/etc/grafana/dashboards/reth-mempool.json b/etc/grafana/dashboards/reth-mempool.json index 41be1dc411ebc..9e3f5ae81930f 100644 --- a/etc/grafana/dashboards/reth-mempool.json +++ b/etc/grafana/dashboards/reth-mempool.json @@ -2679,6 +2679,176 @@ "title": "Fetch Hashes Pending Fetch Duration", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of all transactions of all sub-pools by type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "cps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 218, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(reth_transaction_pool_total_legacy_transactions{instance=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "Legacy", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(reth_transaction_pool_total_eip2930_transactions{instance=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-2930", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(reth_transaction_pool_total_eip1559_transactions{instance=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-1559", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(reth_transaction_pool_total_eip4844_transactions{instance=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-4844", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(reth_transaction_pool_total_eip7702_transactions{instance=\"$instance\"}[$__rate_interval])", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "EIP-7702", + "range": true, + "refId": "E", + "useBackend": false + } + ], + "title": "Transactions by Type in Pool", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -3524,6 +3694,6 @@ "timezone": "", "title": "Reth - Transaction Pool", "uid": "bee34f59-c79c-4669-a000-198057b3703d", - "version": 3, + "version": 4, "weekStart": "" } \ No newline at end of file From 9eb8ffa62580fccc1251042ec079fa6de803e6ce Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 7 Oct 2024 18:31:15 +0300 Subject: [PATCH 044/159] feat(exex): subscribe to notifications with head using `ExExContext` (#11500) --- crates/exex/exex/src/context.rs | 15 +- crates/exex/exex/src/manager.rs | 119 +++++++++----- crates/exex/exex/src/notifications.rs | 221 +++++++++++++++++--------- 3 files changed, 236 insertions(+), 119 deletions(-) diff --git a/crates/exex/exex/src/context.rs b/crates/exex/exex/src/context.rs index bdb2d4c27b99c..9af12e260a728 100644 --- a/crates/exex/exex/src/context.rs +++ b/crates/exex/exex/src/context.rs @@ -1,5 +1,6 @@ use std::fmt::Debug; +use reth_exex_types::ExExHead; use reth_node_api::{FullNodeComponents, NodeTypes, NodeTypesWithEngine}; use reth_node_core::node_config::NodeConfig; use reth_primitives::Head; @@ -32,7 +33,7 @@ pub struct ExExContext { /// considered delivered by the node. pub notifications: ExExNotifications, - /// node components + /// Node components pub components: Node, } @@ -92,4 +93,16 @@ impl ExExContext { pub fn task_executor(&self) -> &TaskExecutor { self.components.task_executor() } + + /// Sets notifications stream to [`crate::ExExNotificationsWithoutHead`], a stream of + /// notifications without a head. + pub fn set_notifications_without_head(&mut self) { + self.notifications.set_without_head(); + } + + /// Sets notifications stream to [`crate::ExExNotificationsWithHead`], a stream of notifications + /// with the provided head. + pub fn set_notifications_with_head(&mut self, head: ExExHead) { + self.notifications.set_with_head(head); + } } diff --git a/crates/exex/exex/src/manager.rs b/crates/exex/exex/src/manager.rs index e8e24c09db024..31dc822222b63 100644 --- a/crates/exex/exex/src/manager.rs +++ b/crates/exex/exex/src/manager.rs @@ -610,12 +610,16 @@ impl Clone for ExExManagerHandle { mod tests { use super::*; use alloy_primitives::B256; - use eyre::OptionExt; - use futures::{FutureExt, StreamExt}; + use futures::StreamExt; use rand::Rng; + use reth_db_common::init::init_genesis; + use reth_evm_ethereum::execute::EthExecutorProvider; use reth_primitives::SealedBlockWithSenders; - use reth_provider::{test_utils::create_test_provider_factory, BlockWriter, Chain}; - use reth_testing_utils::generators::{self, random_block}; + use reth_provider::{ + providers::BlockchainProvider2, test_utils::create_test_provider_factory, BlockReader, + Chain, TransactionVariant, + }; + use reth_testing_utils::generators; fn empty_finalized_header_stream() -> ForkChoiceStream { let (tx, rx) = watch::channel(None); @@ -975,11 +979,20 @@ mod tests { #[tokio::test] async fn exex_handle_new() { + let provider_factory = create_test_provider_factory(); + init_genesis(&provider_factory).unwrap(); + let provider = BlockchainProvider2::new(provider_factory).unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); let wal = Wal::new(temp_dir.path()).unwrap(); - let (mut exex_handle, _, mut notifications) = - ExExHandle::new("test_exex".to_string(), Head::default(), (), (), wal.handle()); + let (mut exex_handle, _, mut notifications) = ExExHandle::new( + "test_exex".to_string(), + Head::default(), + provider, + EthExecutorProvider::mainnet(), + wal.handle(), + ); // Check initial state assert_eq!(exex_handle.id, "test_exex"); @@ -1008,7 +1021,7 @@ mod tests { // Send a notification and ensure it's received correctly match exex_handle.send(&mut cx, &(22, notification.clone())) { Poll::Ready(Ok(())) => { - let received_notification = notifications.next().await.unwrap(); + let received_notification = notifications.next().await.unwrap().unwrap(); assert_eq!(received_notification, notification); } Poll::Pending => panic!("Notification send is pending"), @@ -1021,11 +1034,20 @@ mod tests { #[tokio::test] async fn test_notification_if_finished_height_gt_chain_tip() { + let provider_factory = create_test_provider_factory(); + init_genesis(&provider_factory).unwrap(); + let provider = BlockchainProvider2::new(provider_factory).unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); let wal = Wal::new(temp_dir.path()).unwrap(); - let (mut exex_handle, _, mut notifications) = - ExExHandle::new("test_exex".to_string(), Head::default(), (), (), wal.handle()); + let (mut exex_handle, _, mut notifications) = ExExHandle::new( + "test_exex".to_string(), + Head::default(), + provider, + EthExecutorProvider::mainnet(), + wal.handle(), + ); // Set finished_height to a value higher than the block tip exex_handle.finished_height = Some(BlockNumHash::new(15, B256::random())); @@ -1046,11 +1068,7 @@ mod tests { poll_fn(|cx| { // The notification should be skipped, so nothing should be sent. // Check that the receiver channel is indeed empty - assert_eq!( - notifications.poll_next_unpin(cx), - Poll::Pending, - "Receiver channel should be empty" - ); + assert!(notifications.poll_next_unpin(cx).is_pending()); Poll::Ready(()) }) .await; @@ -1066,11 +1084,20 @@ mod tests { #[tokio::test] async fn test_sends_chain_reorged_notification() { + let provider_factory = create_test_provider_factory(); + init_genesis(&provider_factory).unwrap(); + let provider = BlockchainProvider2::new(provider_factory).unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); let wal = Wal::new(temp_dir.path()).unwrap(); - let (mut exex_handle, _, mut notifications) = - ExExHandle::new("test_exex".to_string(), Head::default(), (), (), wal.handle()); + let (mut exex_handle, _, mut notifications) = ExExHandle::new( + "test_exex".to_string(), + Head::default(), + provider, + EthExecutorProvider::mainnet(), + wal.handle(), + ); let notification = ExExNotification::ChainReorged { old: Arc::new(Chain::default()), @@ -1086,7 +1113,7 @@ mod tests { // Send the notification match exex_handle.send(&mut cx, &(22, notification.clone())) { Poll::Ready(Ok(())) => { - let received_notification = notifications.next().await.unwrap(); + let received_notification = notifications.next().await.unwrap().unwrap(); assert_eq!(received_notification, notification); } Poll::Pending | Poll::Ready(Err(_)) => { @@ -1100,11 +1127,20 @@ mod tests { #[tokio::test] async fn test_sends_chain_reverted_notification() { + let provider_factory = create_test_provider_factory(); + init_genesis(&provider_factory).unwrap(); + let provider = BlockchainProvider2::new(provider_factory).unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); let wal = Wal::new(temp_dir.path()).unwrap(); - let (mut exex_handle, _, mut notifications) = - ExExHandle::new("test_exex".to_string(), Head::default(), (), (), wal.handle()); + let (mut exex_handle, _, mut notifications) = ExExHandle::new( + "test_exex".to_string(), + Head::default(), + provider, + EthExecutorProvider::mainnet(), + wal.handle(), + ); let notification = ExExNotification::ChainReverted { old: Arc::new(Chain::default()) }; @@ -1117,7 +1153,7 @@ mod tests { // Send the notification match exex_handle.send(&mut cx, &(22, notification.clone())) { Poll::Ready(Ok(())) => { - let received_notification = notifications.next().await.unwrap(); + let received_notification = notifications.next().await.unwrap().unwrap(); assert_eq!(received_notification, notification); } Poll::Pending | Poll::Ready(Err(_)) => { @@ -1135,30 +1171,34 @@ mod tests { let mut rng = generators::rng(); + let provider_factory = create_test_provider_factory(); + let genesis_hash = init_genesis(&provider_factory).unwrap(); + let genesis_block = provider_factory + .sealed_block_with_senders(genesis_hash.into(), TransactionVariant::NoHash) + .unwrap() + .ok_or_else(|| eyre::eyre!("genesis block not found"))?; + let provider = BlockchainProvider2::new(provider_factory).unwrap(); + let temp_dir = tempfile::tempdir().unwrap(); let wal = Wal::new(temp_dir.path()).unwrap(); - let provider_factory = create_test_provider_factory(); - - let block = random_block(&mut rng, 0, Default::default()) - .seal_with_senders() - .ok_or_eyre("failed to recover senders")?; - let provider_rw = provider_factory.provider_rw()?; - provider_rw.insert_block(block.clone())?; - provider_rw.commit()?; + let (exex_handle, events_tx, mut notifications) = ExExHandle::new( + "test_exex".to_string(), + Head::default(), + provider.clone(), + EthExecutorProvider::mainnet(), + wal.handle(), + ); let notification = ExExNotification::ChainCommitted { - new: Arc::new(Chain::new(vec![block.clone()], Default::default(), None)), + new: Arc::new(Chain::new(vec![genesis_block.clone()], Default::default(), None)), }; let (finalized_headers_tx, rx) = watch::channel(None); let finalized_header_stream = ForkChoiceStream::new(rx); - let (exex_handle, events_tx, mut notifications) = - ExExHandle::new("test_exex".to_string(), Head::default(), (), (), wal.handle()); - let mut exex_manager = std::pin::pin!(ExExManager::new( - provider_factory, + provider, vec![exex_handle], 1, wal, @@ -1170,16 +1210,13 @@ mod tests { exex_manager.handle().send(notification.clone())?; assert!(exex_manager.as_mut().poll(&mut cx)?.is_pending()); - assert_eq!( - notifications.next().poll_unpin(&mut cx), - Poll::Ready(Some(notification.clone())) - ); + assert_eq!(notifications.next().await.unwrap().unwrap(), notification.clone()); assert_eq!( exex_manager.wal.iter_notifications()?.collect::>>()?, [notification.clone()] ); - finalized_headers_tx.send(Some(block.header.clone()))?; + finalized_headers_tx.send(Some(genesis_block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL isn't finalized because the ExEx didn't emit the `FinishedHeight` event assert_eq!( @@ -1192,7 +1229,7 @@ mod tests { .send(ExExEvent::FinishedHeight((rng.gen::(), rng.gen::()).into())) .unwrap(); - finalized_headers_tx.send(Some(block.header.clone()))?; + finalized_headers_tx.send(Some(genesis_block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL isn't finalized because the ExEx emitted a `FinishedHeight` event with a // non-canonical block @@ -1202,9 +1239,9 @@ mod tests { ); // Send a `FinishedHeight` event with a canonical block - events_tx.send(ExExEvent::FinishedHeight(block.num_hash())).unwrap(); + events_tx.send(ExExEvent::FinishedHeight(genesis_block.num_hash())).unwrap(); - finalized_headers_tx.send(Some(block.header.clone()))?; + finalized_headers_tx.send(Some(genesis_block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL is finalized assert!(exex_manager.wal.iter_notifications()?.next().is_none()); diff --git a/crates/exex/exex/src/notifications.rs b/crates/exex/exex/src/notifications.rs index 116dac95422bc..6efdb1775cf14 100644 --- a/crates/exex/exex/src/notifications.rs +++ b/crates/exex/exex/src/notifications.rs @@ -13,27 +13,28 @@ use std::{ }; use tokio::sync::mpsc::Receiver; -/// A stream of [`ExExNotification`]s. The stream will emit notifications for all blocks. +/// A stream of [`ExExNotification`]s. The stream will emit notifications for all blocks. If the +/// stream is configured with a head via [`ExExNotifications::set_with_head`] or +/// [`ExExNotifications::with_head`], it will run backfill jobs to catch up to the node head. +#[derive(Debug)] pub struct ExExNotifications { - node_head: Head, - provider: P, - executor: E, - notifications: Receiver, - wal_handle: WalHandle, + inner: ExExNotificationsInner, } -impl Debug for ExExNotifications { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ExExNotifications") - .field("provider", &self.provider) - .field("executor", &self.executor) - .field("notifications", &self.notifications) - .finish() - } +#[derive(Debug)] +enum ExExNotificationsInner { + /// A stream of [`ExExNotification`]s. The stream will emit notifications for all blocks. + WithoutHead(ExExNotificationsWithoutHead), + /// A stream of [`ExExNotification`]s. The stream will only emit notifications for blocks that + /// are committed or reverted after the given head. + WithHead(ExExNotificationsWithHead), + /// Internal state used when transitioning between [`ExExNotificationsInner::WithoutHead`] and + /// [`ExExNotificationsInner::WithHead`]. + Invalid, } impl ExExNotifications { - /// Creates a new instance of [`ExExNotifications`]. + /// Creates a new stream of [`ExExNotifications`] without a head. pub const fn new( node_head: Head, provider: P, @@ -41,73 +42,131 @@ impl ExExNotifications { notifications: Receiver, wal_handle: WalHandle, ) -> Self { - Self { node_head, provider, executor, notifications, wal_handle } + Self { + inner: ExExNotificationsInner::WithoutHead(ExExNotificationsWithoutHead::new( + node_head, + provider, + executor, + notifications, + wal_handle, + )), + } } - /// Receives the next value for this receiver. + /// Sets [`ExExNotifications`] to a stream of [`ExExNotification`]s without a head. /// - /// This method returns `None` if the channel has been closed and there are - /// no remaining messages in the channel's buffer. This indicates that no - /// further values can ever be received from this `Receiver`. The channel is - /// closed when all senders have been dropped, or when [`Receiver::close`] is called. + /// It's a no-op if the stream has already been configured without a head. /// - /// # Cancel safety - /// - /// This method is cancel safe. If `recv` is used as the event in a - /// [`tokio::select!`] statement and some other branch - /// completes first, it is guaranteed that no messages were received on this - /// channel. - /// - /// For full documentation, see [`Receiver::recv`]. - #[deprecated(note = "use `ExExNotifications::next` and its `Stream` implementation instead")] - pub async fn recv(&mut self) -> Option { - self.notifications.recv().await + /// See the documentation of [`ExExNotificationsWithoutHead`] for more details. + pub fn set_without_head(&mut self) { + let current = std::mem::replace(&mut self.inner, ExExNotificationsInner::Invalid); + self.inner = ExExNotificationsInner::WithoutHead(match current { + ExExNotificationsInner::WithoutHead(notifications) => notifications, + ExExNotificationsInner::WithHead(notifications) => ExExNotificationsWithoutHead::new( + notifications.node_head, + notifications.provider, + notifications.executor, + notifications.notifications, + notifications.wal_handle, + ), + ExExNotificationsInner::Invalid => unreachable!(), + }); } - /// Polls to receive the next message on this channel. + /// Returns a new [`ExExNotifications`] without a head. /// - /// This method returns: - /// - /// * `Poll::Pending` if no messages are available but the channel is not closed, or if a - /// spurious failure happens. - /// * `Poll::Ready(Some(message))` if a message is available. - /// * `Poll::Ready(None)` if the channel has been closed and all messages sent before it was - /// closed have been received. + /// See the documentation of [`ExExNotificationsWithoutHead`] for more details. + pub fn without_head(mut self) -> Self { + self.set_without_head(); + self + } + + /// Sets [`ExExNotifications`] to a stream of [`ExExNotification`]s with the provided head. /// - /// When the method returns `Poll::Pending`, the `Waker` in the provided - /// `Context` is scheduled to receive a wakeup when a message is sent on any - /// receiver, or when the channel is closed. Note that on multiple calls to - /// `poll_recv` or `poll_recv_many`, only the `Waker` from the `Context` - /// passed to the most recent call is scheduled to receive a wakeup. + /// It's a no-op if the stream has already been configured with a head. /// - /// If this method returns `Poll::Pending` due to a spurious failure, then - /// the `Waker` will be notified when the situation causing the spurious - /// failure has been resolved. Note that receiving such a wakeup does not - /// guarantee that the next call will succeed — it could fail with another - /// spurious failure. + /// See the documentation of [`ExExNotificationsWithHead`] for more details. + pub fn set_with_head(&mut self, exex_head: ExExHead) { + let current = std::mem::replace(&mut self.inner, ExExNotificationsInner::Invalid); + self.inner = ExExNotificationsInner::WithHead(match current { + ExExNotificationsInner::WithoutHead(notifications) => { + notifications.with_head(exex_head) + } + ExExNotificationsInner::WithHead(notifications) => ExExNotificationsWithHead::new( + notifications.node_head, + notifications.provider, + notifications.executor, + notifications.notifications, + notifications.wal_handle, + exex_head, + ), + ExExNotificationsInner::Invalid => unreachable!(), + }); + } + + /// Returns a new [`ExExNotifications`] with the provided head. /// - /// For full documentation, see [`Receiver::poll_recv`]. - #[deprecated( - note = "use `ExExNotifications::poll_next` and its `Stream` implementation instead" - )] - pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll> { - self.notifications.poll_recv(cx) + /// See the documentation of [`ExExNotificationsWithHead`] for more details. + pub fn with_head(mut self, exex_head: ExExHead) -> Self { + self.set_with_head(exex_head); + self } } -impl ExExNotifications +impl Stream for ExExNotifications where P: BlockReader + HeaderProvider + StateProviderFactory + Clone + Unpin + 'static, E: BlockExecutorProvider + Clone + Unpin + 'static, { - /// Subscribe to notifications with the given head. This head is the ExEx's - /// latest view of the host chain. - /// - /// Notifications will be sent starting from the head, not inclusive. For - /// example, if `head.number == 10`, then the first notification will be - /// with `block.number == 11`. A `head.number` of 10 indicates that the ExEx - /// has processed up to block 10, and is ready to process block 11. - pub fn with_head(self, head: ExExHead) -> ExExNotificationsWithHead { + type Item = eyre::Result; + + fn poll_next( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + match &mut self.get_mut().inner { + ExExNotificationsInner::WithoutHead(notifications) => { + notifications.poll_next_unpin(cx).map(|result| result.map(Ok)) + } + ExExNotificationsInner::WithHead(notifications) => notifications.poll_next_unpin(cx), + ExExNotificationsInner::Invalid => unreachable!(), + } + } +} + +/// A stream of [`ExExNotification`]s. The stream will emit notifications for all blocks. +pub struct ExExNotificationsWithoutHead { + node_head: Head, + provider: P, + executor: E, + notifications: Receiver, + wal_handle: WalHandle, +} + +impl Debug for ExExNotificationsWithoutHead { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExExNotifications") + .field("provider", &self.provider) + .field("executor", &self.executor) + .field("notifications", &self.notifications) + .finish() + } +} + +impl ExExNotificationsWithoutHead { + /// Creates a new instance of [`ExExNotificationsWithoutHead`]. + const fn new( + node_head: Head, + provider: P, + executor: E, + notifications: Receiver, + wal_handle: WalHandle, + ) -> Self { + Self { node_head, provider, executor, notifications, wal_handle } + } + + /// Subscribe to notifications with the given head. + fn with_head(self, head: ExExHead) -> ExExNotificationsWithHead { ExExNotificationsWithHead::new( self.node_head, self.provider, @@ -119,7 +178,7 @@ where } } -impl Stream for ExExNotifications { +impl Stream for ExExNotificationsWithoutHead { type Item = ExExNotification; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -128,7 +187,13 @@ impl Stream for ExExNotifications { } /// A stream of [`ExExNotification`]s. The stream will only emit notifications for blocks that are -/// committed or reverted after the given head. +/// committed or reverted after the given head. The head is the ExEx's latest view of the host +/// chain. +/// +/// Notifications will be sent starting from the head, not inclusive. For example, if +/// `exex_head.number == 10`, then the first notification will be with `block.number == 11`. An +/// `exex_head.number` of 10 indicates that the ExEx has processed up to block 10, and is ready to +/// process block 11. #[derive(Debug)] pub struct ExExNotificationsWithHead { node_head: Head, @@ -147,13 +212,9 @@ pub struct ExExNotificationsWithHead { backfill_job: Option>, } -impl ExExNotificationsWithHead -where - P: BlockReader + HeaderProvider + StateProviderFactory + Clone + Unpin + 'static, - E: BlockExecutorProvider + Clone + Unpin + 'static, -{ +impl ExExNotificationsWithHead { /// Creates a new [`ExExNotificationsWithHead`]. - pub const fn new( + const fn new( node_head: Head, provider: P, executor: E, @@ -173,7 +234,13 @@ where backfill_job: None, } } +} +impl ExExNotificationsWithHead +where + P: BlockReader + HeaderProvider + StateProviderFactory + Clone + Unpin + 'static, + E: BlockExecutorProvider + Clone + Unpin + 'static, +{ /// Checks if the ExEx head is on the canonical chain. /// /// If the head block is not found in the database or it's ahead of the node head, it means @@ -367,7 +434,7 @@ mod tests { notifications_tx.send(notification.clone()).await?; - let mut notifications = ExExNotifications::new( + let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, EthExecutorProvider::mainnet(), @@ -438,7 +505,7 @@ mod tests { notifications_tx.send(notification.clone()).await?; - let mut notifications = ExExNotifications::new( + let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, EthExecutorProvider::mainnet(), @@ -528,7 +595,7 @@ mod tests { notifications_tx.send(new_notification.clone()).await?; - let mut notifications = ExExNotifications::new( + let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, EthExecutorProvider::mainnet(), @@ -609,7 +676,7 @@ mod tests { notifications_tx.send(new_notification.clone()).await?; - let mut notifications = ExExNotifications::new( + let mut notifications = ExExNotificationsWithoutHead::new( node_head, provider, EthExecutorProvider::mainnet(), From 5e3ed0683e4aaa3e705f1a955fc0f3ad71a32534 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Oct 2024 17:35:47 +0200 Subject: [PATCH 045/159] chore: enforce window (#11540) --- crates/rpc/rpc-eth-api/src/helpers/state.rs | 16 ++++++++++++++-- crates/rpc/rpc/src/eth/helpers/state.rs | 21 +++++---------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 0008454c93a49..d601e43d90a81 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -9,7 +9,8 @@ use reth_errors::RethError; use reth_evm::ConfigureEvmEnv; use reth_primitives::{BlockId, Header, KECCAK_EMPTY}; use reth_provider::{ - BlockIdReader, ChainSpecProvider, StateProvider, StateProviderBox, StateProviderFactory, + BlockIdReader, BlockNumReader, ChainSpecProvider, StateProvider, StateProviderBox, + StateProviderFactory, }; use reth_rpc_eth_types::{EthApiError, EthStateCache, PendingBlockEnv, RpcInvalidTransactionError}; use reth_rpc_types_compat::proof::from_primitive_account_proof; @@ -132,10 +133,21 @@ pub trait EthState: LoadState + SpawnBlocking { ) -> impl Future, Self::Error>> + Send { self.spawn_blocking_io(move |this| { let state = this.state_at_block_id(block_id)?; - let account = state.basic_account(address).map_err(Self::Error::from_eth_err)?; let Some(account) = account else { return Ok(None) }; + // Check whether the distance to the block exceeds the maximum configured proof window. + let chain_info = + LoadState::provider(&this).chain_info().map_err(Self::Error::from_eth_err)?; + let block_number = LoadState::provider(&this) + .block_number_for_id(block_id) + .map_err(Self::Error::from_eth_err)? + .ok_or(EthApiError::HeaderNotFound(block_id))?; + let max_window = this.max_proof_window(); + if chain_info.best_number.saturating_sub(block_number) > max_window { + return Err(EthApiError::ExceedsMaxProofWindow.into()) + } + let balance = account.balance; let nonce = account.nonce; let code_hash = account.bytecode_hash.unwrap_or(KECCAK_EMPTY); diff --git a/crates/rpc/rpc/src/eth/helpers/state.rs b/crates/rpc/rpc/src/eth/helpers/state.rs index 006a0192f73a0..8a35842798bcc 100644 --- a/crates/rpc/rpc/src/eth/helpers/state.rs +++ b/crates/rpc/rpc/src/eth/helpers/state.rs @@ -48,7 +48,8 @@ mod tests { use alloy_primitives::{Address, StorageKey, StorageValue, U256}; use reth_chainspec::MAINNET; use reth_evm_ethereum::EthEvmConfig; - use reth_primitives::{constants::ETHEREUM_BLOCK_GAS_LIMIT, KECCAK_EMPTY}; + use reth_network_api::noop::NoopNetwork; + use reth_primitives::constants::ETHEREUM_BLOCK_GAS_LIMIT; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider, NoopProvider}; use reth_rpc_eth_api::helpers::EthState; use reth_rpc_eth_types::{ @@ -61,7 +62,7 @@ mod tests { use reth_transaction_pool::test_utils::{testing_pool, TestPool}; use std::collections::HashMap; - fn noop_eth_api() -> EthApi { + fn noop_eth_api() -> EthApi { let pool = testing_pool(); let evm_config = EthEvmConfig::new(MAINNET.clone()); @@ -70,7 +71,7 @@ mod tests { EthApi::new( NoopProvider::default(), pool, - (), + NoopNetwork::default(), cache.clone(), GasPriceOracle::new(NoopProvider::default(), Default::default(), cache.clone()), ETHEREUM_BLOCK_GAS_LIMIT, @@ -102,7 +103,7 @@ mod tests { GasPriceOracle::new(mock_provider, Default::default(), cache.clone()), ETHEREUM_BLOCK_GAS_LIMIT, DEFAULT_MAX_SIMULATE_BLOCKS, - DEFAULT_ETH_PROOF_WINDOW, + DEFAULT_ETH_PROOF_WINDOW + 1, BlockingTaskPool::build().expect("failed to build tracing pool"), FeeHistoryCache::new(cache, FeeHistoryCacheConfig::default()), evm_config, @@ -139,16 +140,4 @@ mod tests { let account = eth_api.get_account(address, Default::default()).await.unwrap(); assert!(account.is_none()); } - - #[tokio::test] - async fn test_get_account_empty() { - let address = Address::random(); - let accounts = HashMap::from([(address, ExtendedAccount::new(0, U256::ZERO))]); - let eth_api = mock_eth_api(accounts); - - let account = eth_api.get_account(address, Default::default()).await.unwrap(); - let expected_account = - alloy_rpc_types::Account { code_hash: KECCAK_EMPTY, ..Default::default() }; - assert_eq!(Some(expected_account), account); - } } From 6dce31a469f4a1d95dad126a320c9492d5884401 Mon Sep 17 00:00:00 2001 From: Parikalp Bhardwaj <53660958+Parikalp-Bhardwaj@users.noreply.github.com> Date: Mon, 7 Oct 2024 20:11:59 +0400 Subject: [PATCH 046/159] Refactor get_payload_bodies_by_hash_with to be non-blocking (#11511) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-engine-api/src/engine_api.rs | 51 +++++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 907297de1776b..252808c14a77f 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -464,48 +464,59 @@ where } /// Called to retrieve execution payload bodies by hashes. - fn get_payload_bodies_by_hash_with( + async fn get_payload_bodies_by_hash_with( &self, hashes: Vec, f: F, ) -> EngineApiResult>> where - F: Fn(Block) -> R, + F: Fn(Block) -> R + Send + 'static, + R: Send + 'static, { let len = hashes.len() as u64; if len > MAX_PAYLOAD_BODIES_LIMIT { - return Err(EngineApiError::PayloadRequestTooLarge { len }) + return Err(EngineApiError::PayloadRequestTooLarge { len }); } - let mut result = Vec::with_capacity(hashes.len()); - for hash in hashes { - let block = self - .inner - .provider - .block(BlockHashOrNumber::Hash(hash)) - .map_err(|err| EngineApiError::Internal(Box::new(err)))?; - result.push(block.map(&f)); - } + let (tx, rx) = oneshot::channel(); + let inner = self.inner.clone(); - Ok(result) + self.inner.task_spawner.spawn_blocking(Box::pin(async move { + let mut result = Vec::with_capacity(hashes.len()); + for hash in hashes { + let block_result = inner.provider.block(BlockHashOrNumber::Hash(hash)); + match block_result { + Ok(block) => { + result.push(block.map(&f)); + } + Err(err) => { + let _ = tx.send(Err(EngineApiError::Internal(Box::new(err)))); + return; + } + } + } + tx.send(Ok(result)).ok(); + })); + + rx.await.map_err(|err| EngineApiError::Internal(Box::new(err)))? } /// Called to retrieve execution payload bodies by hashes. - pub fn get_payload_bodies_by_hash_v1( + pub async fn get_payload_bodies_by_hash_v1( &self, hashes: Vec, ) -> EngineApiResult { - self.get_payload_bodies_by_hash_with(hashes, convert_to_payload_body_v1) + self.get_payload_bodies_by_hash_with(hashes, convert_to_payload_body_v1).await } /// Called to retrieve execution payload bodies by hashes. /// /// Same as [`Self::get_payload_bodies_by_hash_v1`] but as [`ExecutionPayloadBodiesV2`]. - pub fn get_payload_bodies_by_hash_v2( + pub async fn get_payload_bodies_by_hash_v2( &self, hashes: Vec, ) -> EngineApiResult { - self.get_payload_bodies_by_hash_with(hashes, convert_to_payload_body_v2) + self.get_payload_bodies_by_hash_with(hashes, convert_to_payload_body_v2).await } /// Called to verify network configuration parameters and ensure that Consensus and Execution @@ -832,7 +843,7 @@ where let start = Instant::now(); let res = Self::get_payload_bodies_by_hash_v1(self, block_hashes); self.inner.metrics.latency.get_payload_bodies_by_hash_v1.record(start.elapsed()); - Ok(res?) + Ok(res.await?) } async fn get_payload_bodies_by_hash_v2( @@ -843,7 +854,7 @@ where let start = Instant::now(); let res = Self::get_payload_bodies_by_hash_v2(self, block_hashes); self.inner.metrics.latency.get_payload_bodies_by_hash_v2.record(start.elapsed()); - Ok(res?) + Ok(res.await?) } /// Handler for `engine_getPayloadBodiesByRangeV1` @@ -1147,7 +1158,7 @@ mod tests { .collect::>(); let hashes = blocks.iter().map(|b| b.hash()).collect(); - let res = api.get_payload_bodies_by_hash_v1(hashes).unwrap(); + let res = api.get_payload_bodies_by_hash_v1(hashes).await.unwrap(); assert_eq!(res, expected); } } From a9a7b17c6bfe5f75f15563582cb11eff2b8d39ec Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:30:57 -0600 Subject: [PATCH 047/159] Introduce Eth PayloadTypes Impl (#11519) Co-authored-by: Matthias Seitz --- crates/ethereum/engine-primitives/src/lib.rs | 31 ++++++++++++++++---- crates/node/types/src/lib.rs | 1 + 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/crates/ethereum/engine-primitives/src/lib.rs b/crates/ethereum/engine-primitives/src/lib.rs index 69d73a021747b..034a8c6bffbb4 100644 --- a/crates/ethereum/engine-primitives/src/lib.rs +++ b/crates/ethereum/engine-primitives/src/lib.rs @@ -26,21 +26,40 @@ use reth_payload_primitives::{ /// The types used in the default mainnet ethereum beacon consensus engine. #[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] #[non_exhaustive] -pub struct EthEngineTypes; +pub struct EthEngineTypes { + _marker: std::marker::PhantomData, +} -impl PayloadTypes for EthEngineTypes { - type BuiltPayload = EthBuiltPayload; - type PayloadAttributes = EthPayloadAttributes; - type PayloadBuilderAttributes = EthPayloadBuilderAttributes; +impl PayloadTypes for EthEngineTypes { + type BuiltPayload = T::BuiltPayload; + type PayloadAttributes = T::PayloadAttributes; + type PayloadBuilderAttributes = T::PayloadBuilderAttributes; } -impl EngineTypes for EthEngineTypes { +impl EngineTypes for EthEngineTypes +where + T::BuiltPayload: TryInto + + TryInto + + TryInto + + TryInto, +{ type ExecutionPayloadV1 = ExecutionPayloadV1; type ExecutionPayloadV2 = ExecutionPayloadEnvelopeV2; type ExecutionPayloadV3 = ExecutionPayloadEnvelopeV3; type ExecutionPayloadV4 = ExecutionPayloadEnvelopeV4; } +/// A default payload type for [`EthEngineTypes`] +#[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] +#[non_exhaustive] +pub struct EthPayloadTypes; + +impl PayloadTypes for EthPayloadTypes { + type BuiltPayload = EthBuiltPayload; + type PayloadAttributes = EthPayloadAttributes; + type PayloadBuilderAttributes = EthPayloadBuilderAttributes; +} + /// Validator for the ethereum engine API. #[derive(Debug, Clone)] pub struct EthereumEngineValidator { diff --git a/crates/node/types/src/lib.rs b/crates/node/types/src/lib.rs index 2c72e02d3edc7..0f6d5a1a370f3 100644 --- a/crates/node/types/src/lib.rs +++ b/crates/node/types/src/lib.rs @@ -40,6 +40,7 @@ pub trait NodeTypes: Send + Sync + Unpin + 'static { pub trait NodeTypesWithEngine: NodeTypes { /// The node's engine types, defining the interaction with the consensus engine. type Engine: EngineTypes; + // type Engine: EngineTypes; } /// A helper trait that is downstream of the [`NodeTypesWithEngine`] trait and adds database to the From 0dcc814b1f9002591f61e672d2cfe6ad31f805d8 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:41:22 -0400 Subject: [PATCH 048/159] fix(op-reth): add jemalloc feature to optimism-cli for version (#11543) --- Cargo.lock | 1 + crates/optimism/bin/Cargo.toml | 2 +- crates/optimism/cli/Cargo.toml | 9 +++++++++ crates/optimism/cli/src/lib.rs | 4 ++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index a66887900cb46..93ae84221192b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7996,6 +7996,7 @@ dependencies = [ "reth-node-builder", "reth-node-core", "reth-node-events", + "reth-node-metrics", "reth-optimism-chainspec", "reth-optimism-evm", "reth-optimism-node", diff --git a/crates/optimism/bin/Cargo.toml b/crates/optimism/bin/Cargo.toml index ecb92dfc454b3..2de0bb6ee181d 100644 --- a/crates/optimism/bin/Cargo.toml +++ b/crates/optimism/bin/Cargo.toml @@ -31,7 +31,7 @@ workspace = true [features] default = ["jemalloc"] -jemalloc = ["reth-cli-util/jemalloc"] +jemalloc = ["reth-cli-util/jemalloc", "reth-optimism-cli/jemalloc"] jemalloc-prof = ["reth-cli-util/jemalloc-prof"] tracy-allocator = ["reth-cli-util/tracy-allocator"] diff --git a/crates/optimism/cli/Cargo.toml b/crates/optimism/cli/Cargo.toml index 99d1641e36438..d53270cd62f82 100644 --- a/crates/optimism/cli/Cargo.toml +++ b/crates/optimism/cli/Cargo.toml @@ -27,6 +27,9 @@ reth-node-core.workspace = true reth-optimism-node.workspace = true reth-primitives.workspace = true +# so jemalloc metrics can be included +reth-node-metrics.workspace = true + ## optimism reth-optimism-primitives.workspace = true reth-optimism-chainspec.workspace = true @@ -82,3 +85,9 @@ asm-keccak = [ "reth-optimism-node/asm-keccak", "reth-primitives/asm-keccak", ] + +# Jemalloc feature for vergen to generate correct env vars +jemalloc = [ + "reth-node-core/jemalloc", + "reth-node-metrics/jemalloc" +] diff --git a/crates/optimism/cli/src/lib.rs b/crates/optimism/cli/src/lib.rs index 74bc80c426ef9..e6eed86bf7fc5 100644 --- a/crates/optimism/cli/src/lib.rs +++ b/crates/optimism/cli/src/lib.rs @@ -51,6 +51,10 @@ use reth_optimism_node::OptimismNode; use reth_tracing::FileWorkerGuard; use tracing::info; +// This allows us to manually enable node metrics features, required for proper jemalloc metric +// reporting +use reth_node_metrics as _; + /// The main op-reth cli interface. /// /// This is the entrypoint to the executable. From 3a54d2ac36ea3583251badf248cc74db0fb89553 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Mon, 7 Oct 2024 18:42:09 +0200 Subject: [PATCH 049/159] fix(grafana): remove rate function from panel "Transactions by Type in Pool" (#11542) --- etc/grafana/dashboards/reth-mempool.json | 52 +++++++++++++++++------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/etc/grafana/dashboards/reth-mempool.json b/etc/grafana/dashboards/reth-mempool.json index 9e3f5ae81930f..ebb693184a5a6 100644 --- a/etc/grafana/dashboards/reth-mempool.json +++ b/etc/grafana/dashboards/reth-mempool.json @@ -15,7 +15,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "11.1.3" + "version": "11.2.0" }, { "type": "panel", @@ -131,7 +131,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -201,7 +201,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -271,7 +271,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -341,7 +341,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -411,7 +411,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -481,7 +481,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -534,6 +534,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -686,6 +687,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -817,6 +819,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -969,6 +972,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1099,6 +1103,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1261,6 +1266,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1357,6 +1363,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1504,6 +1511,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1605,6 +1613,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1756,6 +1765,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1852,6 +1862,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1971,6 +1982,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2101,6 +2113,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2323,6 +2336,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2442,6 +2456,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2578,6 +2593,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2697,6 +2713,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2736,7 +2753,7 @@ } ] }, - "unit": "cps" + "unit": "none" }, "overrides": [] }, @@ -2766,8 +2783,8 @@ "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(reth_transaction_pool_total_legacy_transactions{instance=\"$instance\"}[$__rate_interval])", + "editorMode": "code", + "expr": "reth_transaction_pool_total_legacy_transactions{instance=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -2784,7 +2801,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(reth_transaction_pool_total_eip2930_transactions{instance=\"$instance\"}[$__rate_interval])", + "expr": "reth_transaction_pool_total_eip2930_transactions{instance=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -2801,7 +2818,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(reth_transaction_pool_total_eip1559_transactions{instance=\"$instance\"}[$__rate_interval])", + "expr": "reth_transaction_pool_total_eip1559_transactions{instance=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -2818,7 +2835,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(reth_transaction_pool_total_eip4844_transactions{instance=\"$instance\"}[$__rate_interval])", + "expr": "reth_transaction_pool_total_eip4844_transactions{instance=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -2835,7 +2852,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "rate(reth_transaction_pool_total_eip7702_transactions{instance=\"$instance\"}[$__rate_interval])", + "expr": "reth_transaction_pool_total_eip7702_transactions{instance=\"$instance\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, @@ -2867,6 +2884,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3035,6 +3053,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3130,6 +3149,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3272,6 +3292,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3578,6 +3599,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3694,6 +3716,6 @@ "timezone": "", "title": "Reth - Transaction Pool", "uid": "bee34f59-c79c-4669-a000-198057b3703d", - "version": 4, + "version": 2, "weekStart": "" } \ No newline at end of file From db40811d209142244d147f3a45598952af54acee Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 7 Oct 2024 23:54:42 +0200 Subject: [PATCH 050/159] chore: move ethfiltererror (#11552) --- crates/rpc/rpc-eth-types/src/lib.rs | 1 - crates/rpc/rpc-eth-types/src/logs_utils.rs | 56 +--------------------- crates/rpc/rpc/src/eth/filter.rs | 53 +++++++++++++++++++- 3 files changed, 53 insertions(+), 57 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/lib.rs b/crates/rpc/rpc-eth-types/src/lib.rs index fba893c15f590..fa36dae4c8810 100644 --- a/crates/rpc/rpc-eth-types/src/lib.rs +++ b/crates/rpc/rpc-eth-types/src/lib.rs @@ -36,7 +36,6 @@ pub use gas_oracle::{ GasCap, GasPriceOracle, GasPriceOracleConfig, GasPriceOracleResult, RPC_DEFAULT_GAS_CAP, }; pub use id_provider::EthSubscriptionIdProvider; -pub use logs_utils::EthFilterError; pub use pending_block::{PendingBlock, PendingBlockEnv, PendingBlockEnvOrigin}; pub use receipt::ReceiptBuilder; pub use transaction::TransactionSource; diff --git a/crates/rpc/rpc-eth-types/src/logs_utils.rs b/crates/rpc/rpc-eth-types/src/logs_utils.rs index bb44dc0e6669b..f26555bb70daf 100644 --- a/crates/rpc/rpc-eth-types/src/logs_utils.rs +++ b/crates/rpc/rpc-eth-types/src/logs_utils.rs @@ -3,64 +3,12 @@ //! Log parsing for building filter. use alloy_primitives::TxHash; -use alloy_rpc_types::{FilterId, FilteredParams, Log}; +use alloy_rpc_types::{FilteredParams, Log}; use reth_chainspec::ChainInfo; use reth_errors::ProviderError; use reth_primitives::{BlockNumHash, Receipt}; -use reth_rpc_server_types::result::rpc_error_with_code; use reth_storage_api::BlockReader; -use crate::EthApiError; - -/// Errors that can occur in the handler implementation -#[derive(Debug, thiserror::Error)] -pub enum EthFilterError { - /// Filter not found. - #[error("filter not found")] - FilterNotFound(FilterId), - /// Invalid block range. - #[error("invalid block range params")] - InvalidBlockRangeParams, - /// Query scope is too broad. - #[error("query exceeds max block range {0}")] - QueryExceedsMaxBlocks(u64), - /// Query result is too large. - #[error("query exceeds max results {0}")] - QueryExceedsMaxResults(usize), - /// Error serving request in `eth_` namespace. - #[error(transparent)] - EthAPIError(#[from] EthApiError), - /// Error thrown when a spawned task failed to deliver a response. - #[error("internal filter error")] - InternalError, -} - -// convert the error -impl From for jsonrpsee_types::error::ErrorObject<'static> { - fn from(err: EthFilterError) -> Self { - match err { - EthFilterError::FilterNotFound(_) => { - rpc_error_with_code(jsonrpsee_types::error::INVALID_PARAMS_CODE, "filter not found") - } - err @ EthFilterError::InternalError => { - rpc_error_with_code(jsonrpsee_types::error::INTERNAL_ERROR_CODE, err.to_string()) - } - EthFilterError::EthAPIError(err) => err.into(), - err @ (EthFilterError::InvalidBlockRangeParams | - EthFilterError::QueryExceedsMaxBlocks(_) | - EthFilterError::QueryExceedsMaxResults(_)) => { - rpc_error_with_code(jsonrpsee_types::error::INVALID_PARAMS_CODE, err.to_string()) - } - } - } -} - -impl From for EthFilterError { - fn from(err: ProviderError) -> Self { - Self::EthAPIError(err.into()) - } -} - /// Returns all matching of a block's receipts when the transaction hashes are known. pub fn matching_block_logs_with_tx_hashes<'a, I>( filter: &FilteredParams, @@ -107,7 +55,7 @@ pub fn append_matching_block_logs( receipts: &[Receipt], removed: bool, block_timestamp: u64, -) -> Result<(), EthFilterError> { +) -> Result<(), ProviderError> { // Tracks the index of a log in the entire block. let mut log_index: u64 = 0; diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index f4ae892641609..c5581f42a06c2 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -24,9 +24,9 @@ use reth_provider::{BlockIdReader, BlockReader, EvmEnvProvider, ProviderError}; use reth_rpc_eth_api::{EthFilterApiServer, FullEthApiTypes, RpcTransaction, TransactionCompat}; use reth_rpc_eth_types::{ logs_utils::{self, append_matching_block_logs}, - EthApiError, EthFilterConfig, EthFilterError, EthStateCache, EthSubscriptionIdProvider, + EthApiError, EthFilterConfig, EthStateCache, EthSubscriptionIdProvider, }; -use reth_rpc_server_types::ToRpcResult; +use reth_rpc_server_types::{result::rpc_error_with_code, ToRpcResult}; use reth_rpc_types_compat::transaction::from_recovered; use reth_tasks::TaskSpawner; use reth_transaction_pool::{NewSubpoolTransactionStream, PoolTransaction, TransactionPool}; @@ -695,6 +695,55 @@ impl Iterator for BlockRangeInclusiveIter { } } +/// Errors that can occur in the handler implementation +#[derive(Debug, thiserror::Error)] +pub enum EthFilterError { + /// Filter not found. + #[error("filter not found")] + FilterNotFound(FilterId), + /// Invalid block range. + #[error("invalid block range params")] + InvalidBlockRangeParams, + /// Query scope is too broad. + #[error("query exceeds max block range {0}")] + QueryExceedsMaxBlocks(u64), + /// Query result is too large. + #[error("query exceeds max results {0}")] + QueryExceedsMaxResults(usize), + /// Error serving request in `eth_` namespace. + #[error(transparent)] + EthAPIError(#[from] EthApiError), + /// Error thrown when a spawned task failed to deliver a response. + #[error("internal filter error")] + InternalError, +} + +impl From for jsonrpsee::types::error::ErrorObject<'static> { + fn from(err: EthFilterError) -> Self { + match err { + EthFilterError::FilterNotFound(_) => rpc_error_with_code( + jsonrpsee::types::error::INVALID_PARAMS_CODE, + "filter not found", + ), + err @ EthFilterError::InternalError => { + rpc_error_with_code(jsonrpsee::types::error::INTERNAL_ERROR_CODE, err.to_string()) + } + EthFilterError::EthAPIError(err) => err.into(), + err @ (EthFilterError::InvalidBlockRangeParams | + EthFilterError::QueryExceedsMaxBlocks(_) | + EthFilterError::QueryExceedsMaxResults(_)) => { + rpc_error_with_code(jsonrpsee::types::error::INVALID_PARAMS_CODE, err.to_string()) + } + } + } +} + +impl From for EthFilterError { + fn from(err: ProviderError) -> Self { + Self::EthAPIError(err.into()) + } +} + #[cfg(test)] mod tests { use super::*; From f172a58c3d9f213a24ab57f164aa94e35f656768 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 03:06:16 +0200 Subject: [PATCH 051/159] chore: rm redundant type hint (#11557) --- crates/rpc/rpc/src/eth/helpers/signer.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/rpc/rpc/src/eth/helpers/signer.rs b/crates/rpc/rpc/src/eth/helpers/signer.rs index b5109d09017a1..a5818aa494fd3 100644 --- a/crates/rpc/rpc/src/eth/helpers/signer.rs +++ b/crates/rpc/rpc/src/eth/helpers/signer.rs @@ -3,7 +3,6 @@ use std::collections::HashMap; use crate::EthApi; -use alloy_consensus::TxEnvelope; use alloy_dyn_abi::TypedData; use alloy_eips::eip2718::Decodable2718; use alloy_network::{eip2718::Encodable2718, EthereumWallet, TransactionBuilder}; @@ -91,7 +90,7 @@ impl EthSigner for DevSigner { let wallet = EthereumWallet::from(signer); // build and sign transaction with signer - let txn_envelope: TxEnvelope = + let txn_envelope = request.build(&wallet).await.map_err(|_| SignError::InvalidTransactionRequest)?; // decode transaction into signed transaction type From a4c4b7c079e0e08bcf9c4cc4842f2fbc8e23875b Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Tue, 8 Oct 2024 02:42:14 -0600 Subject: [PATCH 052/159] Introduce Op PayloadTypes Impl (#11558) Co-authored-by: Matthias Seitz --- crates/optimism/node/src/engine.rs | 47 +++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index a9adadd3068e8..11c8f15cfdac1 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -21,7 +21,39 @@ use reth_optimism_payload_builder::{OptimismBuiltPayload, OptimismPayloadBuilder /// The types used in the optimism beacon consensus engine. #[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] #[non_exhaustive] -pub struct OptimismEngineTypes; +pub struct OptimismEngineTypes { + _marker: std::marker::PhantomData, +} + +impl PayloadTypes for OptimismEngineTypes { + type BuiltPayload = T::BuiltPayload; + type PayloadAttributes = T::PayloadAttributes; + type PayloadBuilderAttributes = T::PayloadBuilderAttributes; +} + +impl EngineTypes for OptimismEngineTypes +where + T::BuiltPayload: TryInto + + TryInto + + TryInto + + TryInto, +{ + type ExecutionPayloadV1 = ExecutionPayloadV1; + type ExecutionPayloadV2 = ExecutionPayloadEnvelopeV2; + type ExecutionPayloadV3 = OptimismExecutionPayloadEnvelopeV3; + type ExecutionPayloadV4 = OptimismExecutionPayloadEnvelopeV4; +} + +/// A default payload type for [`OptimismEngineTypes`] +#[derive(Debug, Default, Clone, serde::Deserialize, serde::Serialize)] +#[non_exhaustive] +pub struct OptimismPayloadTypes; + +impl PayloadTypes for OptimismPayloadTypes { + type BuiltPayload = OptimismBuiltPayload; + type PayloadAttributes = OptimismPayloadAttributes; + type PayloadBuilderAttributes = OptimismPayloadBuilderAttributes; +} /// Validator for Optimism engine API. #[derive(Debug, Clone)] @@ -36,19 +68,6 @@ impl OptimismEngineValidator { } } -impl PayloadTypes for OptimismEngineTypes { - type BuiltPayload = OptimismBuiltPayload; - type PayloadAttributes = OptimismPayloadAttributes; - type PayloadBuilderAttributes = OptimismPayloadBuilderAttributes; -} - -impl EngineTypes for OptimismEngineTypes { - type ExecutionPayloadV1 = ExecutionPayloadV1; - type ExecutionPayloadV2 = ExecutionPayloadEnvelopeV2; - type ExecutionPayloadV3 = OptimismExecutionPayloadEnvelopeV3; - type ExecutionPayloadV4 = OptimismExecutionPayloadEnvelopeV4; -} - /// Validates the presence of the `withdrawals` field according to the payload timestamp. /// /// After Canyon, withdrawals field must be [Some]. From 946324c2b3870b9ad43e0f40aa91884de00df23b Mon Sep 17 00:00:00 2001 From: greged93 <82421016+greged93@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:02:32 +0200 Subject: [PATCH 053/159] chore: chain manual serialisation implementation (#11538) --- crates/evm/execution-types/src/chain.rs | 48 ++++++++++++++++++------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/crates/evm/execution-types/src/chain.rs b/crates/evm/execution-types/src/chain.rs index 467bd4c0ec7cf..25bc39ea32570 100644 --- a/crates/evm/execution-types/src/chain.rs +++ b/crates/evm/execution-types/src/chain.rs @@ -516,7 +516,7 @@ pub(super) mod serde_bincode_compat { use alloy_primitives::BlockNumber; use reth_primitives::serde_bincode_compat::SealedBlockWithSenders; use reth_trie::serde_bincode_compat::updates::TrieUpdates; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer}; use serde_with::{DeserializeAs, SerializeAs}; use crate::ExecutionOutcome; @@ -538,19 +538,47 @@ pub(super) mod serde_bincode_compat { /// ``` #[derive(Debug, Serialize, Deserialize)] pub struct Chain<'a> { - blocks: BTreeMap>, + blocks: SealedBlocksWithSenders<'a>, execution_outcome: Cow<'a, ExecutionOutcome>, trie_updates: Option>, } + #[derive(Debug)] + struct SealedBlocksWithSenders<'a>( + Cow<'a, BTreeMap>, + ); + + impl Serialize for SealedBlocksWithSenders<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut state = serializer.serialize_map(Some(self.0.len()))?; + + for (block_number, block) in self.0.iter() { + state.serialize_entry(block_number, &SealedBlockWithSenders::<'_>::from(block))?; + } + + state.end() + } + } + + impl<'de> Deserialize<'de> for SealedBlocksWithSenders<'_> { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(Self(Cow::Owned( + BTreeMap::>::deserialize(deserializer) + .map(|blocks| blocks.into_iter().map(|(n, b)| (n, b.into())).collect())?, + ))) + } + } + impl<'a> From<&'a super::Chain> for Chain<'a> { fn from(value: &'a super::Chain) -> Self { Self { - blocks: value - .blocks - .iter() - .map(|(block_number, block)| (*block_number, block.into())) - .collect(), + blocks: SealedBlocksWithSenders(Cow::Borrowed(&value.blocks)), execution_outcome: Cow::Borrowed(&value.execution_outcome), trie_updates: value.trie_updates.as_ref().map(Into::into), } @@ -560,11 +588,7 @@ pub(super) mod serde_bincode_compat { impl<'a> From> for super::Chain { fn from(value: Chain<'a>) -> Self { Self { - blocks: value - .blocks - .into_iter() - .map(|(block_number, block)| (block_number, block.into())) - .collect(), + blocks: value.blocks.0.into_owned(), execution_outcome: value.execution_outcome.into_owned(), trie_updates: value.trie_updates.map(Into::into), } From 887b1b15be998ca5942ad55f97fc5c3b458c0b14 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 11:03:50 +0200 Subject: [PATCH 054/159] chore: rm unused optimism feature (#11559) --- crates/node/core/Cargo.toml | 1 - crates/optimism/rpc/Cargo.toml | 1 - crates/rpc/rpc-eth-api/Cargo.toml | 5 ----- crates/rpc/rpc/Cargo.toml | 1 - 4 files changed, 8 deletions(-) diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index 46476273e42d3..f6d6864519051 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -79,7 +79,6 @@ tempfile.workspace = true optimism = [ "reth-primitives/optimism", "reth-rpc-types-compat/optimism", - "reth-rpc-eth-api/optimism", ] # Features for vergen to generate correct env vars jemalloc = [] diff --git a/crates/optimism/rpc/Cargo.toml b/crates/optimism/rpc/Cargo.toml index 24c3eb02d247c..65dce7510b0e9 100644 --- a/crates/optimism/rpc/Cargo.toml +++ b/crates/optimism/rpc/Cargo.toml @@ -65,6 +65,5 @@ optimism = [ "reth-optimism-evm/optimism", "reth-primitives/optimism", "reth-provider/optimism", - "reth-rpc-eth-api/optimism", "revm/optimism", ] diff --git a/crates/rpc/rpc-eth-api/Cargo.toml b/crates/rpc/rpc-eth-api/Cargo.toml index 23dd46baecf19..e59ee39a694b4 100644 --- a/crates/rpc/rpc-eth-api/Cargo.toml +++ b/crates/rpc/rpc-eth-api/Cargo.toml @@ -59,8 +59,3 @@ tracing.workspace = true [features] js-tracer = ["revm-inspectors/js-tracer", "reth-rpc-eth-types/js-tracer"] client = ["jsonrpsee/client", "jsonrpsee/async-client"] -optimism = [ - "reth-primitives/optimism", - "revm/optimism", - "reth-provider/optimism", -] diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index 4665cd002ca4e..b0994a33e0aa7 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -102,6 +102,5 @@ optimism = [ "reth-primitives/optimism", "reth-rpc-types-compat/optimism", "reth-provider/optimism", - "reth-rpc-eth-api/optimism", "reth-revm/optimism", ] From 11b09ada8fb2948758eba9d2f4564a5d44385161 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Tue, 8 Oct 2024 11:47:15 +0200 Subject: [PATCH 055/159] feat(trie): expose storage proofs (#11550) --- crates/chain-state/src/in_memory.rs | 11 +++- crates/chain-state/src/memory_overlay.rs | 17 ++++++- crates/revm/src/test_utils.rs | 12 ++++- crates/rpc/rpc-eth-types/src/cache/db.rs | 9 ++++ .../src/providers/bundle_state_provider.rs | 36 +++++++++---- .../src/providers/state/historical.rs | 20 ++++++-- .../provider/src/providers/state/latest.rs | 21 ++++++-- .../provider/src/providers/state/macros.rs | 3 +- .../storage/provider/src/test_utils/mock.rs | 12 ++++- .../storage/provider/src/test_utils/noop.rs | 9 ++++ crates/storage/storage-api/src/trie.rs | 12 ++++- crates/trie/db/src/lib.rs | 2 +- crates/trie/db/src/proof.rs | 50 ++++++++++++++++++- crates/trie/trie/src/proof.rs | 33 +++++++++++- crates/trie/trie/src/witness.rs | 2 +- 15 files changed, 221 insertions(+), 28 deletions(-) diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index f47417aa385df..07120cf8ee310 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -872,7 +872,7 @@ mod tests { AccountReader, BlockHashReader, StateProofProvider, StateProvider, StateRootProvider, StorageRootProvider, }; - use reth_trie::{AccountProof, HashedStorage, MultiProof, TrieInput}; + use reth_trie::{AccountProof, HashedStorage, MultiProof, StorageProof, TrieInput}; fn create_mock_state( test_block_builder: &mut TestBlockBuilder, @@ -973,6 +973,15 @@ mod tests { ) -> ProviderResult { Ok(B256::random()) } + + fn storage_proof( + &self, + _address: Address, + slot: B256, + _hashed_storage: HashedStorage, + ) -> ProviderResult { + Ok(StorageProof::new(slot)) + } } impl StateProofProvider for MockStateProvider { diff --git a/crates/chain-state/src/memory_overlay.rs b/crates/chain-state/src/memory_overlay.rs index 2712d1259e85b..eb125dad115ee 100644 --- a/crates/chain-state/src/memory_overlay.rs +++ b/crates/chain-state/src/memory_overlay.rs @@ -133,11 +133,26 @@ impl StateRootProvider for MemoryOverlayStateProvider { impl StorageRootProvider for MemoryOverlayStateProvider { // TODO: Currently this does not reuse available in-memory trie nodes. fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { + let state = &self.trie_state().state; let mut hashed_storage = - self.trie_state().state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); + state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); hashed_storage.extend(&storage); self.historical.storage_root(address, hashed_storage) } + + // TODO: Currently this does not reuse available in-memory trie nodes. + fn storage_proof( + &self, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> ProviderResult { + let state = &self.trie_state().state; + let mut hashed_storage = + state.storages.get(&keccak256(address)).cloned().unwrap_or_default(); + hashed_storage.extend(&storage); + self.historical.storage_proof(address, slot, hashed_storage) + } } impl StateProofProvider for MemoryOverlayStateProvider { diff --git a/crates/revm/src/test_utils.rs b/crates/revm/src/test_utils.rs index d42ec49599076..813997c72d11b 100644 --- a/crates/revm/src/test_utils.rs +++ b/crates/revm/src/test_utils.rs @@ -11,7 +11,8 @@ use reth_storage_api::{ }; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ - updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, TrieInput, + updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, StorageProof, + TrieInput, }; /// Mock state for testing @@ -102,6 +103,15 @@ impl StorageRootProvider for StateProviderTest { ) -> ProviderResult { unimplemented!("storage root is not supported") } + + fn storage_proof( + &self, + _address: Address, + _slot: B256, + _hashed_storage: HashedStorage, + ) -> ProviderResult { + unimplemented!("proof generation is not supported") + } } impl StateProofProvider for StateProviderTest { diff --git a/crates/rpc/rpc-eth-types/src/cache/db.rs b/crates/rpc/rpc-eth-types/src/cache/db.rs index 9731e3845768d..7422dcfb8a7b2 100644 --- a/crates/rpc/rpc-eth-types/src/cache/db.rs +++ b/crates/rpc/rpc-eth-types/src/cache/db.rs @@ -58,6 +58,15 @@ impl reth_storage_api::StorageRootProvider for StateProviderTraitObjWrapper<'_> ) -> ProviderResult { self.0.storage_root(address, hashed_storage) } + + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + self.0.storage_proof(address, slot, hashed_storage) + } } impl reth_storage_api::StateProofProvider for StateProviderTraitObjWrapper<'_> { diff --git a/crates/storage/provider/src/providers/bundle_state_provider.rs b/crates/storage/provider/src/providers/bundle_state_provider.rs index 6fe3fa85cb895..296dae8c6ab7e 100644 --- a/crates/storage/provider/src/providers/bundle_state_provider.rs +++ b/crates/storage/provider/src/providers/bundle_state_provider.rs @@ -31,6 +31,20 @@ impl BundleStateProvider pub const fn new(state_provider: SP, block_execution_data_provider: EDP) -> Self { Self { state_provider, block_execution_data_provider } } + + /// Retrieve hashed storage for target address. + fn get_hashed_storage(&self, address: Address) -> HashedStorage { + let bundle_state = self.block_execution_data_provider.execution_outcome().state(); + bundle_state + .account(&address) + .map(|account| { + HashedStorage::from_plain_storage( + account.status, + account.storage.iter().map(|(slot, value)| (slot, &value.present_value)), + ) + }) + .unwrap_or_else(|| HashedStorage::new(false)) + } } /* Implement StateProvider traits */ @@ -109,19 +123,21 @@ impl StorageRootProvider address: Address, hashed_storage: HashedStorage, ) -> ProviderResult { - let bundle_state = self.block_execution_data_provider.execution_outcome().state(); - let mut storage = bundle_state - .account(&address) - .map(|account| { - HashedStorage::from_plain_storage( - account.status, - account.storage.iter().map(|(slot, value)| (slot, &value.present_value)), - ) - }) - .unwrap_or_else(|| HashedStorage::new(false)); + let mut storage = self.get_hashed_storage(address); storage.extend(&hashed_storage); self.state_provider.storage_root(address, storage) } + + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + let mut storage = self.get_hashed_storage(address); + storage.extend(&hashed_storage); + self.state_provider.storage_proof(address, slot, storage) + } } impl StateProofProvider diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 22f40f6f951f0..781a11f6decac 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -17,12 +17,14 @@ use reth_primitives::{constants::EPOCH_SLOTS, Account, Bytecode, StaticFileSegme use reth_storage_api::{StateProofProvider, StorageRootProvider}; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ - proof::Proof, updates::TrieUpdates, witness::TrieWitness, AccountProof, HashedPostState, - HashedStorage, MultiProof, StateRoot, StorageRoot, TrieInput, + proof::{Proof, StorageProof}, + updates::TrieUpdates, + witness::TrieWitness, + AccountProof, HashedPostState, HashedStorage, MultiProof, StateRoot, StorageRoot, TrieInput, }; use reth_trie_db::{ DatabaseHashedPostState, DatabaseHashedStorage, DatabaseProof, DatabaseStateRoot, - DatabaseStorageRoot, DatabaseTrieWitness, + DatabaseStorageProof, DatabaseStorageRoot, DatabaseTrieWitness, }; use std::fmt::Debug; @@ -330,6 +332,18 @@ impl StorageRootProvider for HistoricalStateProviderRef<'_, TX> { StorageRoot::overlay_root(self.tx, address, revert_storage) .map_err(|err| ProviderError::Database(err.into())) } + + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + let mut revert_storage = self.revert_storage(address)?; + revert_storage.extend(&hashed_storage); + StorageProof::overlay_storage_proof(self.tx, address, slot, revert_storage) + .map_err(Into::::into) + } } impl StateProofProvider for HistoricalStateProviderRef<'_, TX> { diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index 9fbe00cbd5ee9..fdcbfc4937fec 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -15,10 +15,15 @@ use reth_primitives::{Account, Bytecode, StaticFileSegment}; use reth_storage_api::{StateProofProvider, StorageRootProvider}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use reth_trie::{ - proof::Proof, updates::TrieUpdates, witness::TrieWitness, AccountProof, HashedPostState, - HashedStorage, MultiProof, StateRoot, StorageRoot, TrieInput, + proof::{Proof, StorageProof}, + updates::TrieUpdates, + witness::TrieWitness, + AccountProof, HashedPostState, HashedStorage, MultiProof, StateRoot, StorageRoot, TrieInput, +}; +use reth_trie_db::{ + DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, + DatabaseTrieWitness, }; -use reth_trie_db::{DatabaseProof, DatabaseStateRoot, DatabaseStorageRoot, DatabaseTrieWitness}; /// State provider over latest state that takes tx reference. #[derive(Debug)] @@ -116,6 +121,16 @@ impl StorageRootProvider for LatestStateProviderRef<'_, TX> { StorageRoot::overlay_root(self.tx, address, hashed_storage) .map_err(|err| ProviderError::Database(err.into())) } + + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + StorageProof::overlay_storage_proof(self.tx, address, slot, hashed_storage) + .map_err(Into::::into) + } } impl StateProofProvider for LatestStateProviderRef<'_, TX> { diff --git a/crates/storage/provider/src/providers/state/macros.rs b/crates/storage/provider/src/providers/state/macros.rs index 388b59ab0a1e1..b90924354c434 100644 --- a/crates/storage/provider/src/providers/state/macros.rs +++ b/crates/storage/provider/src/providers/state/macros.rs @@ -48,7 +48,8 @@ macro_rules! delegate_provider_impls { fn state_root_from_nodes_with_updates(&self, input: reth_trie::TrieInput) -> reth_storage_errors::provider::ProviderResult<(alloy_primitives::B256, reth_trie::updates::TrieUpdates)>; } StorageRootProvider $(where [$($generics)*])? { - fn storage_root(&self, address: alloy_primitives::Address, storage: reth_trie::HashedStorage) -> reth_storage_errors::provider::ProviderResult; + fn storage_root(&self, address: alloy_primitives::Address, storage: reth_trie::HashedStorage) -> reth_storage_errors::provider::ProviderResult; + fn storage_proof(&self, address: alloy_primitives::Address, slot: alloy_primitives::B256, storage: reth_trie::HashedStorage) -> reth_storage_errors::provider::ProviderResult; } StateProofProvider $(where [$($generics)*])? { fn proof(&self, input: reth_trie::TrieInput, address: alloy_primitives::Address, slots: &[alloy_primitives::B256]) -> reth_storage_errors::provider::ProviderResult; diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 4f2faad8abe4f..3325d3ae9edba 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -31,7 +31,8 @@ use reth_storage_api::{ }; use reth_storage_errors::provider::{ConsistentViewError, ProviderError, ProviderResult}; use reth_trie::{ - updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, TrieInput, + updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, StorageProof, + TrieInput, }; use revm::primitives::{BlockEnv, CfgEnvWithHandlerCfg}; use std::{ @@ -639,6 +640,15 @@ impl StorageRootProvider for MockEthProvider { ) -> ProviderResult { Ok(EMPTY_ROOT_HASH) } + + fn storage_proof( + &self, + _address: Address, + slot: B256, + _hashed_storage: HashedStorage, + ) -> ProviderResult { + Ok(StorageProof::new(slot)) + } } impl StateProofProvider for MockEthProvider { diff --git a/crates/storage/provider/src/test_utils/noop.rs b/crates/storage/provider/src/test_utils/noop.rs index e8b7760b880b7..0a205389c9b61 100644 --- a/crates/storage/provider/src/test_utils/noop.rs +++ b/crates/storage/provider/src/test_utils/noop.rs @@ -356,6 +356,15 @@ impl StorageRootProvider for NoopProvider { ) -> ProviderResult { Ok(B256::default()) } + + fn storage_proof( + &self, + _address: Address, + slot: B256, + _hashed_storage: HashedStorage, + ) -> ProviderResult { + Ok(reth_trie::StorageProof::new(slot)) + } } impl StateProofProvider for NoopProvider { diff --git a/crates/storage/storage-api/src/trie.rs b/crates/storage/storage-api/src/trie.rs index d989def8bb0d0..f7d41066d0699 100644 --- a/crates/storage/storage-api/src/trie.rs +++ b/crates/storage/storage-api/src/trie.rs @@ -4,7 +4,8 @@ use alloy_primitives::{ }; use reth_storage_errors::provider::ProviderResult; use reth_trie::{ - updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, TrieInput, + updates::TrieUpdates, AccountProof, HashedPostState, HashedStorage, MultiProof, StorageProof, + TrieInput, }; /// A type that can compute the state root of a given post state. @@ -46,6 +47,15 @@ pub trait StorageRootProvider: Send + Sync { /// state. fn storage_root(&self, address: Address, hashed_storage: HashedStorage) -> ProviderResult; + + /// Returns the storage proof of the `HashedStorage` for target slot on top of the current + /// state. + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult; } /// A type that can generate state proof on top of a given post state. diff --git a/crates/trie/db/src/lib.rs b/crates/trie/db/src/lib.rs index aceea1da86602..3a9b1e328239e 100644 --- a/crates/trie/db/src/lib.rs +++ b/crates/trie/db/src/lib.rs @@ -12,7 +12,7 @@ pub use hashed_cursor::{ DatabaseHashedAccountCursor, DatabaseHashedCursorFactory, DatabaseHashedStorageCursor, }; pub use prefix_set::PrefixSetLoader; -pub use proof::DatabaseProof; +pub use proof::{DatabaseProof, DatabaseStorageProof}; pub use state::{DatabaseHashedPostState, DatabaseStateRoot}; pub use storage::{DatabaseHashedStorage, DatabaseStorageRoot}; pub use trie_cursor::{ diff --git a/crates/trie/db/src/proof.rs b/crates/trie/db/src/proof.rs index 1d5fda84cc5bc..9bf08fe136f74 100644 --- a/crates/trie/db/src/proof.rs +++ b/crates/trie/db/src/proof.rs @@ -1,13 +1,16 @@ use crate::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory}; use alloy_primitives::{ + keccak256, map::{HashMap, HashSet}, Address, B256, }; use reth_db_api::transaction::DbTx; use reth_execution_errors::StateProofError; use reth_trie::{ - hashed_cursor::HashedPostStateCursorFactory, proof::Proof, - trie_cursor::InMemoryTrieCursorFactory, MultiProof, TrieInput, + hashed_cursor::HashedPostStateCursorFactory, + proof::{Proof, StorageProof}, + trie_cursor::InMemoryTrieCursorFactory, + HashedPostStateSorted, HashedStorage, MultiProof, TrieInput, }; use reth_trie_common::AccountProof; @@ -81,3 +84,46 @@ impl<'a, TX: DbTx> DatabaseProof<'a, TX> .multiproof(targets) } } + +/// Extends [`StorageProof`] with operations specific for working with a database transaction. +pub trait DatabaseStorageProof<'a, TX> { + /// Create a new [`StorageProof`] from database transaction and account address. + fn from_tx(tx: &'a TX, address: Address) -> Self; + + /// Generates the storage proof for target slot based on [`TrieInput`]. + fn overlay_storage_proof( + tx: &'a TX, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> Result; +} + +impl<'a, TX: DbTx> DatabaseStorageProof<'a, TX> + for StorageProof, DatabaseHashedCursorFactory<'a, TX>> +{ + fn from_tx(tx: &'a TX, address: Address) -> Self { + Self::new(DatabaseTrieCursorFactory::new(tx), DatabaseHashedCursorFactory::new(tx), address) + } + + fn overlay_storage_proof( + tx: &'a TX, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> Result { + let hashed_address = keccak256(address); + let prefix_set = storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from([(hashed_address, storage.into_sorted())]), + ); + Self::from_tx(tx, address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + DatabaseHashedCursorFactory::new(tx), + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_proof(slot) + } +} diff --git a/crates/trie/trie/src/proof.rs b/crates/trie/trie/src/proof.rs index 95d9505218bf5..bff681a3498df 100644 --- a/crates/trie/trie/src/proof.rs +++ b/crates/trie/trie/src/proof.rs @@ -124,7 +124,7 @@ where hashed_address, ) .with_prefix_set_mut(storage_prefix_set) - .storage_proof(proof_targets)?; + .storage_multiproof(proof_targets)?; // Encode account account_rlp.clear(); @@ -170,6 +170,26 @@ impl StorageProof { } } + /// Set the trie cursor factory. + pub fn with_trie_cursor_factory(self, trie_cursor_factory: TF) -> StorageProof { + StorageProof { + trie_cursor_factory, + hashed_cursor_factory: self.hashed_cursor_factory, + hashed_address: self.hashed_address, + prefix_set: self.prefix_set, + } + } + + /// Set the hashed cursor factory. + pub fn with_hashed_cursor_factory(self, hashed_cursor_factory: HF) -> StorageProof { + StorageProof { + trie_cursor_factory: self.trie_cursor_factory, + hashed_cursor_factory, + hashed_address: self.hashed_address, + prefix_set: self.prefix_set, + } + } + /// Set the changed prefixes. pub fn with_prefix_set_mut(mut self, prefix_set: PrefixSetMut) -> Self { self.prefix_set = prefix_set; @@ -182,8 +202,17 @@ where T: TrieCursorFactory, H: HashedCursorFactory, { - /// Generate storage proof. + /// Generate an account proof from intermediate nodes. pub fn storage_proof( + self, + slot: B256, + ) -> Result { + let targets = HashSet::from_iter([keccak256(slot)]); + Ok(self.storage_multiproof(targets)?.storage_proof(slot)?) + } + + /// Generate storage proof. + pub fn storage_multiproof( mut self, targets: HashSet, ) -> Result { diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index b0fcfb021ae1f..d668946e62381 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -178,7 +178,7 @@ where hashed_address, ) .with_prefix_set_mut(storage_prefix_set) - .storage_proof(HashSet::from_iter([target_key]))?; + .storage_multiproof(HashSet::from_iter([target_key]))?; // The subtree only contains the proof for a single target. let node = From e2e9f7dac1acc92ef879ce9599a2abf2b9891074 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 12:02:34 +0200 Subject: [PATCH 056/159] chore: rm unused optimism feature from compat (#11560) --- crates/node/core/Cargo.toml | 3 +-- crates/optimism/node/Cargo.toml | 1 - crates/optimism/payload/Cargo.toml | 1 - crates/rpc/rpc-types-compat/Cargo.toml | 5 +---- crates/rpc/rpc/Cargo.toml | 1 - 5 files changed, 2 insertions(+), 9 deletions(-) diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index f6d6864519051..3ac90a888705a 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -77,8 +77,7 @@ tempfile.workspace = true [features] optimism = [ - "reth-primitives/optimism", - "reth-rpc-types-compat/optimism", + "reth-primitives/optimism" ] # Features for vergen to generate correct env vars jemalloc = [] diff --git a/crates/optimism/node/Cargo.toml b/crates/optimism/node/Cargo.toml index cb74bcbc09040..3c298bea9cee0 100644 --- a/crates/optimism/node/Cargo.toml +++ b/crates/optimism/node/Cargo.toml @@ -81,7 +81,6 @@ optimism = [ "reth-chainspec/optimism", "reth-primitives/optimism", "reth-provider/optimism", - "reth-rpc-types-compat/optimism", "reth-optimism-evm/optimism", "reth-optimism-payload-builder/optimism", "reth-beacon-consensus/optimism", diff --git a/crates/optimism/payload/Cargo.toml b/crates/optimism/payload/Cargo.toml index 117f63201a48a..d5f8b520e822a 100644 --- a/crates/optimism/payload/Cargo.toml +++ b/crates/optimism/payload/Cargo.toml @@ -52,7 +52,6 @@ optimism = [ "reth-chainspec/optimism", "reth-primitives/optimism", "reth-provider/optimism", - "reth-rpc-types-compat/optimism", "reth-optimism-evm/optimism", "reth-revm/optimism", ] diff --git a/crates/rpc/rpc-types-compat/Cargo.toml b/crates/rpc/rpc-types-compat/Cargo.toml index a9d82d95779c8..81b4def204f98 100644 --- a/crates/rpc/rpc-types-compat/Cargo.toml +++ b/crates/rpc/rpc-types-compat/Cargo.toml @@ -26,7 +26,4 @@ alloy-serde.workspace = true alloy-rpc-types-engine.workspace = true [dev-dependencies] -serde_json.workspace = true - -[features] -optimism = ["reth-primitives/optimism"] +serde_json.workspace = true \ No newline at end of file diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index b0994a33e0aa7..5399e50ce28a0 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -100,7 +100,6 @@ jsonrpsee = { workspace = true, features = ["client"] } js-tracer = ["revm-inspectors/js-tracer", "reth-rpc-eth-types/js-tracer"] optimism = [ "reth-primitives/optimism", - "reth-rpc-types-compat/optimism", "reth-provider/optimism", "reth-revm/optimism", ] From c9ad5bbb7946ebc5d089e58fa91196a251dfe6c1 Mon Sep 17 00:00:00 2001 From: Debjit Bhowal Date: Tue, 8 Oct 2024 15:57:18 +0530 Subject: [PATCH 057/159] Added InternalBlockExecutionError to execute.rs exports (#11525) Co-authored-by: Matthias Seitz --- crates/evm/src/execute.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index 3fc2975f0ff78..1bd79378127e2 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -1,7 +1,9 @@ //! Traits for execution. // Re-export execution types -pub use reth_execution_errors::{BlockExecutionError, BlockValidationError}; +pub use reth_execution_errors::{ + BlockExecutionError, BlockValidationError, InternalBlockExecutionError, +}; pub use reth_execution_types::{BlockExecutionInput, BlockExecutionOutput, ExecutionOutcome}; pub use reth_storage_errors::provider::ProviderError; From 980137b3a39f96b5c0be06996cc5895c460e8ac9 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 8 Oct 2024 11:46:53 +0100 Subject: [PATCH 058/159] docs(exex): include code for ExEx book from real files (#11545) --- .github/workflows/lint.yml | 12 ++- .github/workflows/unit.yml | 5 ++ .gitignore | 5 +- Cargo.toml | 1 + book/developers/exex/hello-world.md | 86 +--------------------- book/sources/Cargo.toml | 9 +++ book/sources/exex/hello-world/Cargo.toml | 13 ++++ book/sources/exex/hello-world/src/bin/1.rs | 9 +++ book/sources/exex/hello-world/src/bin/2.rs | 20 +++++ book/sources/exex/hello-world/src/bin/3.rs | 39 ++++++++++ 10 files changed, 114 insertions(+), 85 deletions(-) create mode 100644 book/sources/Cargo.toml create mode 100644 book/sources/exex/hello-world/Cargo.toml create mode 100644 book/sources/exex/hello-world/src/bin/1.rs create mode 100644 book/sources/exex/hello-world/src/bin/2.rs create mode 100644 book/sources/exex/hello-world/src/bin/3.rs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4bab77b915eec..e8161557fde9c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,7 +30,12 @@ jobs: - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true - - run: cargo clippy --bin "${{ matrix.binary }}" --workspace --features "${{ matrix.network }} asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" + - name: Run clippy on binaries + run: cargo clippy --bin "${{ matrix.binary }}" --workspace --features "${{ matrix.network }} asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" + env: + RUSTFLAGS: -D warnings + - name: Run clippy on book binary sources + run: cargo clippy --manifest-path book/sources/Cargo.toml --workspace --bins env: RUSTFLAGS: -D warnings @@ -128,7 +133,10 @@ jobs: - uses: dtolnay/rust-toolchain@nightly with: components: rustfmt - - run: cargo fmt --all --check + - name: Run fmt + run: cargo fmt --all --check + - name: Run fmt on book sources + run: cargo fmt --manifest-path book/sources/Cargo.toml --all --check udeps: name: udeps diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index a6663aea8843f..1056a6fb58c80 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -42,6 +42,11 @@ jobs: --workspace --exclude ef-tests \ --partition hash:${{ matrix.partition }}/2 \ -E "!kind(test)" + - name: Run tests on book sources + run: | + cargo nextest run \ + --manifest-path book/sources/Cargo.toml --workspace \ + -E "!kind(test)" state: name: Ethereum state tests diff --git a/.gitignore b/.gitignore index 2d5d851a5055e..00f7765424220 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,7 @@ jwttoken/ crates/storage/libmdbx-rs/mdbx-sys/libmdbx/cmake-build-debug # Rust bug report -rustc-ice-* \ No newline at end of file +rustc-ice-* + +# Book sources should be able to build with the latest version +book/sources/Cargo.lock diff --git a/Cargo.toml b/Cargo.toml index 8c1ee8c4514c1..b6eed127119b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -148,6 +148,7 @@ members = [ "testing/testing-utils", ] default-members = ["bin/reth"] +exclude = ["book/sources"] # Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 # https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html diff --git a/book/developers/exex/hello-world.md b/book/developers/exex/hello-world.md index facb07e5307f2..c1f3e5af94430 100644 --- a/book/developers/exex/hello-world.md +++ b/book/developers/exex/hello-world.md @@ -14,19 +14,7 @@ cd my-exex And add Reth as a dependency in `Cargo.toml` ```toml -[package] -name = "my-exex" -version = "0.1.0" -edition = "2021" - -[dependencies] -reth = { git = "https://github.com/paradigmxyz/reth.git" } # Reth -reth-exex = { git = "https://github.com/paradigmxyz/reth.git" } # Execution Extensions -reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git" } # Ethereum Node implementation -reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } # Logging - -eyre = "0.6" # Easy error handling -futures-util = "0.3" # Stream utilities for consuming notifications +{{#include ../../sources/exex/hello-world/Cargo.toml}} ``` ### Default Reth node @@ -34,15 +22,7 @@ futures-util = "0.3" # Stream utilities for consuming notifications Now, let's jump to our `main.rs` and start by initializing and launching a default Reth node ```rust,norun,noplayground,ignore -use reth_node_ethereum::EthereumNode; - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let handle = builder.node(EthereumNode::default()).launch().await?; - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/hello-world/src/bin/1.rs}} ``` You can already test that it works by running the binary and initializing the Holesky node in a custom datadir @@ -63,26 +43,7 @@ $ cargo run -- init --chain holesky --datadir data The simplest ExEx is just an async function that never returns. We need to install it into our node ```rust,norun,noplayground,ignore -use reth::api::FullNodeComponents; -use reth_exex::{ExExContext, ExExEvent, ExExNotification}; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; - -async fn my_exex(mut _ctx: ExExContext) -> eyre::Result<()> { - loop {} -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let handle = builder - .node(EthereumNode::default()) - .install_exex("my-exex", |ctx| async move { Ok(my_exex(ctx)) }) - .launch() - .await?; - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/hello-world/src/bin/2.rs}} ``` See that unused `_ctx`? That's the context that we'll use to listen to new notifications coming from the main node, @@ -103,46 +64,7 @@ If you try running a node with an ExEx that exits, the node will exit as well. Now, let's extend our simplest ExEx and start actually listening to new notifications, log them, and send events back to the main node ```rust,norun,noplayground,ignore -use futures_util::StreamExt; -use reth::api::FullNodeComponents; -use reth_exex::{ExExContext, ExExEvent, ExExNotification}; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; - -async fn my_exex(mut ctx: ExExContext) -> eyre::Result<()> { - while let Some(notification) = ctx.notifications.next().await { - match ¬ification { - ExExNotification::ChainCommitted { new } => { - info!(committed_chain = ?new.range(), "Received commit"); - } - ExExNotification::ChainReorged { old, new } => { - info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); - } - ExExNotification::ChainReverted { old } => { - info!(reverted_chain = ?old.range(), "Received revert"); - } - }; - - if let Some(committed_chain) = notification.committed_chain() { - ctx.events - .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; - } - } - - Ok(()) -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let handle = builder - .node(EthereumNode::default()) - .install_exex("my-exex", |ctx| async move { Ok(my_exex(ctx)) }) - .launch() - .await?; - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/hello-world/src/bin/3.rs}} ``` Woah, there's a lot of new stuff here! Let's go through it step by step: diff --git a/book/sources/Cargo.toml b/book/sources/Cargo.toml new file mode 100644 index 0000000000000..c04c8567f94d0 --- /dev/null +++ b/book/sources/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +members = [ + "exex/hello-world", +] + +# Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 +# https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html +resolver = "2" + diff --git a/book/sources/exex/hello-world/Cargo.toml b/book/sources/exex/hello-world/Cargo.toml new file mode 100644 index 0000000000000..e5d32a1405499 --- /dev/null +++ b/book/sources/exex/hello-world/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "my-exex" +version = "0.1.0" +edition = "2021" + +[dependencies] +reth = { git = "https://github.com/paradigmxyz/reth.git" } # Reth +reth-exex = { git = "https://github.com/paradigmxyz/reth.git" } # Execution Extensions +reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git" } # Ethereum Node implementation +reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } # Logging + +eyre = "0.6" # Easy error handling +futures-util = "0.3" # Stream utilities for consuming notifications diff --git a/book/sources/exex/hello-world/src/bin/1.rs b/book/sources/exex/hello-world/src/bin/1.rs new file mode 100644 index 0000000000000..794609bfd7d11 --- /dev/null +++ b/book/sources/exex/hello-world/src/bin/1.rs @@ -0,0 +1,9 @@ +use reth_node_ethereum::EthereumNode; + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let handle = builder.node(EthereumNode::default()).launch().await?; + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/hello-world/src/bin/2.rs b/book/sources/exex/hello-world/src/bin/2.rs new file mode 100644 index 0000000000000..6ab5fc49650ee --- /dev/null +++ b/book/sources/exex/hello-world/src/bin/2.rs @@ -0,0 +1,20 @@ +use reth::api::FullNodeComponents; +use reth_exex::ExExContext; +use reth_node_ethereum::EthereumNode; + +async fn my_exex(mut _ctx: ExExContext) -> eyre::Result<()> { + #[allow(clippy::empty_loop)] + loop {} +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let handle = builder + .node(EthereumNode::default()) + .install_exex("my-exex", |ctx| async move { Ok(my_exex(ctx)) }) + .launch() + .await?; + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/hello-world/src/bin/3.rs b/book/sources/exex/hello-world/src/bin/3.rs new file mode 100644 index 0000000000000..21bd25a56dbf3 --- /dev/null +++ b/book/sources/exex/hello-world/src/bin/3.rs @@ -0,0 +1,39 @@ +use futures_util::TryStreamExt; +use reth::api::FullNodeComponents; +use reth_exex::{ExExContext, ExExEvent, ExExNotification}; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; + +async fn my_exex(mut ctx: ExExContext) -> eyre::Result<()> { + while let Some(notification) = ctx.notifications.try_next().await? { + match ¬ification { + ExExNotification::ChainCommitted { new } => { + info!(committed_chain = ?new.range(), "Received commit"); + } + ExExNotification::ChainReorged { old, new } => { + info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); + } + ExExNotification::ChainReverted { old } => { + info!(reverted_chain = ?old.range(), "Received revert"); + } + }; + + if let Some(committed_chain) = notification.committed_chain() { + ctx.events.send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + } + } + + Ok(()) +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let handle = builder + .node(EthereumNode::default()) + .install_exex("my-exex", |ctx| async move { Ok(my_exex(ctx)) }) + .launch() + .await?; + + handle.wait_for_node_exit().await + }) +} From 87b2b04ed34a95723332d2b672b436b87b00af3a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 13:04:48 +0200 Subject: [PATCH 059/159] fix: actually configure the custom gas limit (#11565) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 819aff546d426..c5703bcceef30 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -716,6 +716,7 @@ pub trait Call: LoadState + SpawnBlocking { // Keep a copy of gas related request values let tx_request_gas_limit = request.gas; let tx_request_gas_price = request.gas_price; + // the gas limit of the corresponding block let block_env_gas_limit = block.gas_limit; // Determine the highest possible gas limit, considering both the request's specified limit @@ -1083,6 +1084,8 @@ pub trait Call: LoadState + SpawnBlocking { /// - `disable_eip3607` is set to `true` /// - `disable_base_fee` is set to `true` /// - `nonce` is set to `None` + /// + /// In addition, this changes the block's gas limit to the configured [`Self::call_gas_limit`]. fn prepare_call_env( &self, mut cfg: CfgEnvWithHandlerCfg, @@ -1102,6 +1105,9 @@ pub trait Call: LoadState + SpawnBlocking { ) } + // apply configured gas cap + block.gas_limit = U256::from(self.call_gas_limit()); + // Disabled because eth_call is sometimes used with eoa senders // See cfg.disable_eip3607 = true; From f3899cbb6db0d61f8658c8dd9132e7f130b523f6 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 8 Oct 2024 14:54:22 +0200 Subject: [PATCH 060/159] chore: relax trait bound for `EthTransactions` (#11571) --- crates/rpc/rpc-eth-api/src/helpers/transaction.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index d70ea7e541cb5..afcb1b2eeb256 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -22,8 +22,7 @@ use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool} use crate::{FromEthApiError, FullEthApiTypes, IntoEthApiError, RpcReceipt, RpcTransaction}; use super::{ - Call, EthApiSpec, EthSigner, LoadBlock, LoadFee, LoadPendingBlock, LoadReceipt, LoadState, - SpawnBlocking, + Call, EthApiSpec, EthSigner, LoadBlock, LoadPendingBlock, LoadReceipt, LoadState, SpawnBlocking, }; /// Transaction related functions for the [`EthApiServer`](crate::EthApiServer) trait in @@ -344,7 +343,7 @@ pub trait EthTransactions: LoadTransaction { mut request: TransactionRequest, ) -> impl Future> + Send where - Self: EthApiSpec + LoadBlock + LoadPendingBlock + LoadFee + Call, + Self: EthApiSpec + LoadBlock + LoadPendingBlock + Call, { async move { let from = match request.from { From eff50db504cea67487a902936872ffff6f75a55c Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 8 Oct 2024 21:54:47 +0900 Subject: [PATCH 061/159] fix(provider): fix sub overflow on `tx_range` queries for empty blocks (#11568) --- crates/storage/provider/src/providers/blockchain_provider.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 9d075015397fc..c52d2d58b1e0f 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -463,8 +463,9 @@ impl BlockchainProvider2 { let block_tx_count = block_state.block_ref().block().body.transactions.len(); let remaining = (tx_range.end() - tx_range.start() + 1) as usize; - // If the transaction range start is higher than this block last transaction, advance - if *tx_range.start() > in_memory_tx_num + block_tx_count as u64 - 1 { + // If the transaction range start is equal or higher than the next block first + // transaction, advance + if *tx_range.start() >= in_memory_tx_num + block_tx_count as u64 { in_memory_tx_num += block_tx_count as u64; continue } From bf5818f943dcbfb0f547b04b8d40445fbe9a7eaf Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 8 Oct 2024 22:17:57 +0900 Subject: [PATCH 062/159] chore(provider): add more test coverage on `BlockchainProvider` non-range queries (#11564) --- .../src/providers/blockchain_provider.rs | 1203 +++++------------ 1 file changed, 323 insertions(+), 880 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index c52d2d58b1e0f..d25c3f84bd1a8 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1714,7 +1714,7 @@ mod tests { StaticFileWriter, }; use alloy_eips::{BlockHashOrNumber, BlockNumHash, BlockNumberOrTag}; - use alloy_primitives::{BlockNumber, B256}; + use alloy_primitives::{BlockNumber, TxNumber, B256}; use itertools::Itertools; use rand::Rng; use reth_chain_state::{ @@ -1732,8 +1732,7 @@ mod tests { use reth_errors::ProviderError; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives::{ - Receipt, SealedBlock, StaticFileSegment, TransactionMeta, TransactionSignedNoHash, - Withdrawals, + Receipt, SealedBlock, StaticFileSegment, TransactionSignedNoHash, Withdrawals, }; use reth_storage_api::{ BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, @@ -2258,284 +2257,6 @@ mod tests { Ok(()) } - #[test] - fn test_block_with_senders_by_hash_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first in-memory block - let first_in_mem_block = in_memory_blocks.first().unwrap(); - let block_hash = first_in_mem_block.hash(); - - // Get the block with senders by hash and check if it matches the first in-memory block - let block_with_senders = provider - .block_with_senders(BlockHashOrNumber::Hash(block_hash), TransactionVariant::WithHash)? - .unwrap(); - assert_eq!(block_with_senders.block.seal(block_hash), first_in_mem_block.clone()); - assert_eq!(block_with_senders.senders, first_in_mem_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_block_with_senders_by_number_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first in-memory block - let first_in_mem_block = in_memory_blocks.first().unwrap(); - let block_number = first_in_mem_block.number; - - // Get the block with senders by number and check if it matches the first in-memory block - let block_with_senders = provider - .block_with_senders( - BlockHashOrNumber::Number(block_number), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!( - block_with_senders.block.seal(first_in_mem_block.hash()), - first_in_mem_block.clone() - ); - assert_eq!(block_with_senders.senders, first_in_mem_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_block_with_senders_by_hash_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first database block - let first_db_block = database_blocks.first().unwrap(); - let block_hash = first_db_block.hash(); - - // Get the block with senders by hash and check if it matches the first database block - let block_with_senders = provider - .block_with_senders(BlockHashOrNumber::Hash(block_hash), TransactionVariant::WithHash)? - .unwrap(); - assert_eq!(block_with_senders.block.seal(block_hash), first_db_block.clone()); - assert_eq!(block_with_senders.senders, first_db_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_block_with_senders_by_number_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first database block - let first_db_block = database_blocks.first().unwrap(); - let block_number = first_db_block.number; - - // Get the block with senders by number and check if it matches the first database block - let block_with_senders = provider - .block_with_senders( - BlockHashOrNumber::Number(block_number), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!(block_with_senders.block.seal(first_db_block.hash()), first_db_block.clone()); - assert_eq!(block_with_senders.senders, first_db_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_block_with_senders_non_existent_block() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Generate a random hash (non-existent block) - let non_existent_hash = B256::random(); - let result = provider.block_with_senders( - BlockHashOrNumber::Hash(non_existent_hash), - TransactionVariant::WithHash, - )?; - // The block should not be found - assert!(result.is_none()); - - // Generate a random number (non-existent block) - let non_existent_number = 9999; - let result = provider.block_with_senders( - BlockHashOrNumber::Number(non_existent_number), - TransactionVariant::WithHash, - )?; - // The block should not be found - assert!(result.is_none()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_by_hash_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first in-memory block - let first_in_mem_block = in_memory_blocks.first().unwrap(); - let block_hash = first_in_mem_block.hash(); - - // Get the sealed block with senders by hash and check if it matches the first in-memory - // block - let sealed_block_with_senders = provider - .sealed_block_with_senders( - BlockHashOrNumber::Hash(block_hash), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!(sealed_block_with_senders.block, first_in_mem_block.clone()); - assert_eq!(sealed_block_with_senders.senders, first_in_mem_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_by_number_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first in-memory block - let first_in_mem_block = in_memory_blocks.first().unwrap(); - let block_number = first_in_mem_block.number; - - // Get the sealed block with senders by number and check if it matches the first in-memory - let sealed_block_with_senders = provider - .sealed_block_with_senders( - BlockHashOrNumber::Number(block_number), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!(sealed_block_with_senders.block, first_in_mem_block.clone()); - assert_eq!(sealed_block_with_senders.senders, first_in_mem_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_by_hash_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first database block - let first_db_block = database_blocks.first().unwrap(); - let block_hash = first_db_block.hash(); - - // Get the sealed block with senders by hash and check if it matches the first database - // block - let sealed_block_with_senders = provider - .sealed_block_with_senders( - BlockHashOrNumber::Hash(block_hash), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!(sealed_block_with_senders.block, first_db_block.clone()); - assert_eq!(sealed_block_with_senders.senders, first_db_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_by_number_in_database() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Get the first database block - let first_db_block = database_blocks.first().unwrap(); - let block_number = first_db_block.number; - - // Get the sealed block with senders by number and check if it matches the first database - // block - let sealed_block_with_senders = provider - .sealed_block_with_senders( - BlockHashOrNumber::Number(block_number), - TransactionVariant::WithHash, - )? - .unwrap(); - assert_eq!(sealed_block_with_senders.block, first_db_block.clone()); - assert_eq!(sealed_block_with_senders.senders, first_db_block.senders().unwrap()); - - Ok(()) - } - - #[test] - fn test_sealed_block_with_senders_non_existent_block() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, _, _, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams::default(), - )?; - - // Generate a random hash (non-existent block) - let non_existent_hash = B256::random(); - let result = provider.sealed_block_with_senders( - BlockHashOrNumber::Hash(non_existent_hash), - TransactionVariant::WithHash, - )?; - // The block should not be found - assert!(result.is_none()); - - // Generate a random number (non-existent block) - let non_existent_number = 9999; - let result = provider.sealed_block_with_senders( - BlockHashOrNumber::Number(non_existent_number), - TransactionVariant::WithHash, - )?; - // The block should not be found - assert!(result.is_none()); - - Ok(()) - } - #[test] fn test_block_hash_reader() -> eyre::Result<()> { let mut rng = generators::rng(); @@ -2582,39 +2303,16 @@ mod tests { let blocks = [database_blocks, in_memory_blocks].concat(); - assert_eq!(provider.header(&database_block.hash())?, Some(database_block.header().clone())); - assert_eq!( - provider.header(&in_memory_block.hash())?, - Some(in_memory_block.header().clone()) - ); - - assert_eq!( - provider.header_by_number(database_block.number)?, - Some(database_block.header().clone()) - ); - assert_eq!( - provider.header_by_number(in_memory_block.number)?, - Some(in_memory_block.header().clone()) - ); - assert_eq!( provider.header_td_by_number(database_block.number)?, Some(database_block.difficulty) ); + assert_eq!( provider.header_td_by_number(in_memory_block.number)?, Some(in_memory_block.difficulty) ); - assert_eq!( - provider.sealed_header(database_block.number)?, - Some(database_block.header.clone()) - ); - assert_eq!( - provider.sealed_header(in_memory_block.number)?, - Some(in_memory_block.header.clone()) - ); - assert_eq!( provider.sealed_headers_while(0..=10, |header| header.number <= 8)?, blocks @@ -2936,37 +2634,6 @@ mod tests { Ok(()) } - #[test] - fn test_receipt_provider() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { tx_count: 1..3, ..Default::default() }, - )?; - - let blocks = [database_blocks, in_memory_blocks].concat(); - - for block in blocks { - let block_number = block.number as usize; - for (txn_number, _) in block.body.transactions.iter().enumerate() { - let txn_hash = block.body.transactions.get(txn_number).unwrap().hash(); - let txn_id = provider.transaction_id(txn_hash)?.unwrap(); - assert_eq!( - provider.receipt(txn_id)?.unwrap(), - receipts.get(block_number).unwrap().clone().get(txn_number).unwrap().clone() - ); - assert_eq!( - provider.receipt_by_hash(txn_hash)?.unwrap(), - receipts.get(block_number).unwrap().clone().get(txn_number).unwrap().clone() - ); - } - } - - Ok(()) - } - #[test] fn test_receipt_provider_id_ext_receipts_by_block_id() -> eyre::Result<()> { let mut rng = generators::rng(); @@ -3448,502 +3115,6 @@ mod tests { Ok(()) } - #[test] - fn test_transaction_id() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // Database - // Choose a random transaction from the database blocks - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Ensure the transaction ID can be found in the database - let result = provider.transaction_id(tx_hash)?; - assert_eq!(result, Some(0)); - - // In memory - // Choose a random transaction from the in-memory blocks - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Ensure the transaction ID can be found in the in-memory state - let result = provider.transaction_id(tx_hash)?; - assert!(result.is_some(), "Transaction ID should be found in the in-memory state"); - - // Check that the transaction ID is greater than the last database transaction ID - let last_db_tx_id = provider.database.last_block_number()?; - let last_db_tx_id = - provider.database.block_body_indices(last_db_tx_id)?.unwrap().last_tx_num(); - - assert!( - result.unwrap() > last_db_tx_id, - "In-memory transaction ID should be greater than the last database transaction ID" - ); - assert_eq!(result, Some(last_db_tx_id + 1)); - - // Generate a random hash not present in any transaction - let random_tx_hash = B256::random(); - - // Ensure the transaction ID is not found - let result = provider.transaction_id(random_tx_hash)?; - assert!(result.is_none(), "Transaction ID should not be found"); - - Ok(()) - } - - #[test] - fn test_transaction_by_id() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random transaction ID from in-memory blocks - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Ensure the transaction can be retrieved by its ID - let result = provider.transaction_by_id(tx_id)?; - assert_eq!( - result.unwrap(), - *tx, - "The retrieved transaction should match the expected transaction" - ); - - // Database - // Choose a random transaction ID from the database blocks - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Ensure the transaction can be retrieved by its ID - let result = provider.transaction_by_id(tx_id)?; - assert!(result.is_some(), "Transaction should be found in the database"); - assert_eq!( - result.unwrap(), - *tx, - "The retrieved transaction should match the expected transaction" - ); - - // Generate a random transaction ID not present in any block - let random_tx_id = 999999; - - // Ensure the transaction is not found - let result = provider.transaction_by_id(random_tx_id)?; - assert!(result.is_none(), "Transaction should not be found"); - - Ok(()) - } - - #[test] - fn test_transaction_by_id_no_hash() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random transaction ID from in-memory blocks - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Ensure the transaction can be retrieved by its ID without hash - let result = provider.transaction_by_id_no_hash(tx_id)?; - let expected_tx: TransactionSignedNoHash = tx.clone().into(); - assert_eq!( - result.unwrap(), - expected_tx, - "The retrieved transaction without hash should match the expected transaction" - ); - - // Database - // Choose a random transaction ID from the database blocks - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Ensure the transaction can be retrieved by its ID without hash - let result = provider.transaction_by_id_no_hash(tx_id)?; - let expected_tx: TransactionSignedNoHash = tx.clone().into(); - assert_eq!( - result.unwrap(), - expected_tx, - "The retrieved transaction without hash should match the expected transaction" - ); - - // Generate a random transaction ID not present in any block - let random_tx_id = 7656898; - - // Ensure the transaction is not found without hash - let result = provider.transaction_by_id_no_hash(random_tx_id)?; - assert!(result.is_none(), "Transaction should not be found without hash"); - - Ok(()) - } - - #[test] - fn test_transaction_by_hash() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random transaction hash from the in-memory blocks - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Ensure the transaction can be retrieved by its hash from the in-memory state - let result = provider.transaction_by_hash(tx_hash)?; - assert_eq!( - result.unwrap(), - *tx, - "The retrieved transaction should match the expected transaction" - ); - - // Database - // Choose a random transaction hash from the database blocks - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Ensure the transaction can be retrieved by its hash from the database - let result = provider.transaction_by_hash(tx_hash)?; - assert_eq!( - result.unwrap(), - *tx, - "The retrieved transaction should match the expected transaction" - ); - - // Generate a random hash not present in any transaction - let random_tx_hash = B256::random(); - - // Ensure the transaction is not found by the random hash - let result = provider.transaction_by_hash(random_tx_hash)?; - assert!(result.is_none(), "Transaction should not be found"); - - Ok(()) - } - - #[test] - fn test_transaction_by_hash_with_meta() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random transaction from the in-memory block - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Create the expected metadata for this transaction - let meta = TransactionMeta { - tx_hash, - index: 0, - block_hash: in_memory_blocks[0].header.hash(), - block_number: in_memory_blocks[0].header.number, - base_fee: in_memory_blocks[0].header.base_fee_per_gas, - excess_blob_gas: None, - timestamp: in_memory_blocks[0].header.timestamp, - }; - - // Ensure the transaction and its metadata can be retrieved from the in-memory state - let result = provider.transaction_by_hash_with_meta(tx_hash)?; - let (retrieved_tx, retrieved_meta) = result.unwrap(); - assert_eq!( - retrieved_tx, *tx, - "The retrieved transaction should match the expected transaction" - ); - assert_eq!( - retrieved_meta, meta, - "The retrieved metadata should match the expected metadata" - ); - - // Database - // Choose a random transaction from the database blocks - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Create the expected metadata for this transaction - let meta = TransactionMeta { - tx_hash, - index: 0, - block_hash: database_blocks[0].header.hash(), - block_number: database_blocks[0].header.number, - base_fee: database_blocks[0].header.base_fee_per_gas, - excess_blob_gas: None, - timestamp: database_blocks[0].header.timestamp, - }; - - // Ensure the transaction and its metadata can be retrieved from the database - let result = provider.transaction_by_hash_with_meta(tx_hash)?; - let (retrieved_tx, retrieved_meta) = result.unwrap(); - assert_eq!( - retrieved_tx, *tx, - "The retrieved transaction should match the expected transaction" - ); - assert_eq!( - retrieved_meta, meta, - "The retrieved metadata should match the expected metadata" - ); - - // Generate a random hash not present in any transaction - let random_tx_hash = B256::random(); - - // Ensure the transaction with metadata is not found by the random hash - let result = provider.transaction_by_hash_with_meta(random_tx_hash)?; - assert!(result.is_none(), "Transaction with metadata should not be found"); - - Ok(()) - } - - #[test] - fn test_transaction_block() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random transaction ID from in-memory blocks - let tx = &in_memory_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Retrieve the block number for this transaction - let result = provider.transaction_block(tx_id)?; - let block_number = result.unwrap(); - - // Ensure the block number matches the expected block number - assert_eq!( - block_number, in_memory_blocks[0].header.number, - "The block number should match the in-memory block number" - ); - - // Database - // Choose a random transaction from the database block - let tx = &database_blocks[0].body.transactions[0]; - let tx_hash = tx.hash(); - - // Fetch the transaction ID - let tx_id = provider.transaction_id(tx_hash)?.unwrap(); - - // Retrieve the block number for this transaction - let result = provider.transaction_block(tx_id)?; - assert_eq!(Some(0), result, "The block number should match the database block number"); - - // Ensure that invalid transaction ID returns None - let result = provider.transaction_block(67675657)?; - - assert!(result.is_none(), "Block number should not be found for an invalid transaction ID"); - - Ok(()) - } - - #[test] - fn transactions_found_by_block_hash() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random block hash from in-memory blocks - let block_hash = in_memory_blocks[0].header.hash(); - - // Retrieve the transactions for this block using the block hash - let result = provider.transactions_by_block(BlockHashOrNumber::Hash(block_hash))?; - let transactions = result.unwrap(); - - // Ensure the transactions match the expected transactions in the block - assert_eq!( - transactions, in_memory_blocks[0].body.transactions, - "The transactions should match the in-memory block transactions" - ); - - // Database - // Choose a random block hash from the database blocks - let block_hash = database_blocks[0].header.hash(); - - // Retrieve the transactions for this block using the block hash - let result = provider.transactions_by_block(BlockHashOrNumber::Hash(block_hash))?; - let transactions = result.unwrap(); - - // Ensure the transactions match the expected transactions in the block - assert_eq!( - transactions, database_blocks[0].body.transactions, - "The transactions should match the database block transactions" - ); - - // Generate a random block hash that does not exist - let random_block_hash = B256::random(); - - // Try to retrieve transactions for a non-existent block hash - let result = provider.transactions_by_block(BlockHashOrNumber::Hash(random_block_hash))?; - - // Ensure no transactions are found - assert!(result.is_none(), "No transactions should be found for a non-existent block hash"); - - Ok(()) - } - - #[test] - fn transactions_found_by_block_number() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Choose a random block number from in-memory blocks - let block_number = in_memory_blocks[0].header.number; - - // Retrieve the transactions for this block using the block number - let result = provider.transactions_by_block(BlockHashOrNumber::Number(block_number))?; - let transactions = result.unwrap(); - - // Ensure the transactions match the expected transactions in the block - assert_eq!( - transactions, in_memory_blocks[0].body.transactions, - "The transactions should match the in-memory block transactions" - ); - - // Database - // Choose a random block number from the database blocks - let block_number = database_blocks[0].header.number; - - // Retrieve the transactions for this block using the block number - let result = provider.transactions_by_block(BlockHashOrNumber::Number(block_number))?; - let transactions = result.unwrap(); - - // Ensure the transactions match the expected transactions in the block - assert_eq!( - transactions, database_blocks[0].body.transactions, - "The transactions should match the database block transactions" - ); - - // Generate a block number that is out of range (non-existent) - let non_existent_block_number = u64::MAX; - - // Try to retrieve transactions for a non-existent block number - let result = - provider.transactions_by_block(BlockHashOrNumber::Number(non_existent_block_number))?; - - // Ensure no transactions are found - assert!( - result.is_none(), - "No transactions should be found for a non-existent block number" - ); - - Ok(()) - } - - #[test] - fn transactions_found_entirely_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - // In memory - // Define a block range entirely within in-memory blocks - let start_block = in_memory_blocks[0].header.number; - let end_block = in_memory_blocks[1].header.number; - - // Retrieve the transactions for this block range - let result = provider.transactions_by_block_range(start_block..=end_block)?; - - // Ensure the transactions match the expected transactions in the in-memory blocks - assert_eq!(result.len(), 2); - assert_eq!(result[0], in_memory_blocks[0].body.transactions); - assert_eq!(result[1], in_memory_blocks[1].body.transactions); - - // Database - // Define a block range entirely within database blocks - let start_block = database_blocks[0].header.number; - let end_block = database_blocks[1].header.number; - - // Retrieve the transactions for this block range - let result = provider.transactions_by_block_range(start_block..=end_block)?; - - // Ensure the transactions match the expected transactions in the database blocks - assert_eq!(result.len(), 2); - assert_eq!(result[0], database_blocks[0].body.transactions); - assert_eq!(result[1], database_blocks[1].body.transactions); - - Ok(()) - } - macro_rules! test_by_tx_range { ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, [$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ let db_tx_count = @@ -4131,65 +3302,337 @@ mod tests { Ok(()) } - #[test] - fn transaction_sender_found_in_memory() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; + /// Helper macro to call a provider method based on argument count and check its result + macro_rules! call_method { + ($provider:expr, $method:ident, ($($args:expr),*), $expected_item:expr) => {{ + let result = $provider.$method($($args),*)?; + assert_eq!( + result, + $expected_item, + "{}: item does not match the expected item for arguments {:?}", + stringify!($method), + ($($args),*) + ); + }}; - // In memory - // Choose a random transaction from the in-memory block - let tx = &in_memory_blocks[0].body.transactions[0]; + // Handle valid or invalid arguments for one argument + (ONE, $provider:expr, $method:ident, $item_extractor:expr, $txnum:expr, $txhash:expr, $block:expr, $receipts:expr) => {{ + let (arg, expected_item) = $item_extractor($block, $txnum($block), $txhash($block), $receipts); + call_method!($provider, $method, (arg), expected_item); + }}; - // Retrieve the transaction ID - let tx_id = provider.transaction_id(tx.hash())?.unwrap(); + // Handle valid or invalid arguments for two arguments + (TWO, $provider:expr, $method:ident, $item_extractor:expr, $txnum:expr, $txhash:expr, $block:expr, $receipts:expr) => {{ + let ((arg1, arg2), expected_item) = $item_extractor($block, $txnum($block), $txhash($block), $receipts); + call_method!($provider, $method, (arg1, arg2), expected_item); + }}; + } - // Retrieve the sender address for this transaction - let result = provider.transaction_sender(tx_id)?; + /// Macro to test non-range methods. + /// + /// ( `NUMBER_ARGUMENTS`, METHOD, FN -> ((`METHOD_ARGUMENT(s)`,...), `EXPECTED_RESULT`), + /// `INVALID_ARGUMENTS`) + macro_rules! test_non_range { + ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, $receipts:expr, [$(($arg_count:ident, $method:ident, $item_extractor:expr, $invalid_args:expr)),* $(,)?]) => {{ + $( + let tx_hash = |block: &SealedBlock| block.body.transactions[0].hash(); + let tx_num = |block: &SealedBlock| { + $database_blocks + .iter() + .chain($in_memory_blocks.iter()) + .take_while(|b| b.number < block.number) + .map(|b| b.body.transactions.len()) + .sum::() as u64 + }; - // Ensure the sender address matches the expected sender address - let expected_sender = tx.recover_signer().unwrap(); - assert_eq!( - result, - Some(expected_sender), - "The sender address should match the expected sender address" - ); + // Ensure that the first generated in-memory block exists + // In the future this block will be persisted to disk and removed from memory AFTER the firsk database query. + // This ensures that we query the in-memory state before the database avoiding any race condition. + { + call_method!($arg_count, $provider, $method, $item_extractor, tx_num, tx_hash, &$in_memory_blocks[0], &$receipts); + } - // Database - // Choose a random transaction from the database block - let tx = &database_blocks[0].body.transactions[0]; + // Invalid/Non-existent argument should return `None` + { + call_method!($arg_count, $provider, $method, |_,_,_,_| ( ($invalid_args, None)), tx_num, tx_hash, &$in_memory_blocks[0], &$receipts); + } - // Retrieve the transaction ID - let tx_id = provider.transaction_id(tx.hash())?.unwrap(); + // Check that the item is only in memory and not in database + { + let last_mem_block = &$in_memory_blocks[$in_memory_blocks.len() - 1]; - // Retrieve the sender address for this transaction - let result = provider.transaction_sender(tx_id)?; + let (args, expected_item) = $item_extractor(last_mem_block, tx_num(last_mem_block), tx_hash(last_mem_block), &$receipts); + call_method!($arg_count, $provider, $method, |_,_,_,_| (args.clone(), expected_item), tx_num, tx_hash, last_mem_block, &$receipts); - // Ensure the sender address matches the expected sender address - let expected_sender = tx.recover_signer().unwrap(); - assert_eq!( - result, - Some(expected_sender), - "The sender address should match the expected sender address" - ); + // Ensure the item is not in storage + call_method!($arg_count, $provider.database, $method, |_,_,_,_| ( (args, None)), tx_num, tx_hash, last_mem_block, &$receipts); + } + )* + }}; +} - // Generate a random transaction ID that does not exist - let invalid_tx_id = u64::MAX; + #[test] + fn test_non_range_methods() -> eyre::Result<()> { + let test_tx_index = 0; - // Attempt to retrieve the sender address for this invalid transaction ID - let result = provider.transaction_sender(invalid_tx_id)?; + let mut rng = generators::rng(); + let (provider, database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT, + TEST_BLOCKS_COUNT, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; - // Ensure no sender address is found - assert!( - result.is_none(), - "No sender address should be found for an invalid transaction ID" + test_non_range!( + provider, + database_blocks, + in_memory_blocks, + receipts, + [ + // TODO: header should use B256 like others instead of &B256 + // ( + // ONE, + // header, + // |block: &SealedBlock, tx_num: TxNumber, tx_hash: B256, receipts: &Vec>| (&block.hash(), Some(block.header.header().clone())), + // (&B256::random()) + // ), + ( + ONE, + header_by_number, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.header.header().clone()) + ), + u64::MAX + ), + ( + ONE, + sealed_header, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.header.clone()) + ), + u64::MAX + ), + ( + ONE, + block_hash, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.hash()) + ), + u64::MAX + ), + ( + ONE, + block_number, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.hash(), + Some(block.number) + ), + B256::random() + ), + ( + ONE, + block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(block.clone().unseal()) + ), + BlockHashOrNumber::Hash(B256::random()) + ), + ( + ONE, + block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(block.clone().unseal()) + ), + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + block_body_indices, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(StoredBlockBodyIndices { + first_tx_num: tx_num, + tx_count: block.body.transactions.len() as u64 + }) + ), + u64::MAX + ), + ( + TWO, + block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), + block.clone().unseal().with_recovered_senders() + ), + (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + ), + ( + TWO, + block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), + block.clone().unseal().with_recovered_senders() + ), + (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + ), + ( + TWO, + sealed_block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), + Some( + block + .clone() + .unseal() + .with_recovered_senders() + .unwrap() + .seal(block.hash()) + ) + ), + (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + ), + ( + TWO, + sealed_block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), + Some( + block + .clone() + .unseal() + .with_recovered_senders() + .unwrap() + .seal(block.hash()) + ) + ), + (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + ), + ( + ONE, + transaction_id, + |_: &SealedBlock, tx_num: TxNumber, tx_hash: B256, _: &Vec>| ( + tx_hash, + Some(tx_num) + ), + B256::random() + ), + ( + ONE, + transaction_by_id, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(block.body.transactions[test_tx_index].clone()) + ), + u64::MAX + ), + ( + ONE, + transaction_by_id_no_hash, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(Into::::into( + block.body.transactions[test_tx_index].clone() + )) + ), + u64::MAX + ), + ( + ONE, + transaction_by_hash, + |block: &SealedBlock, _: TxNumber, tx_hash: B256, _: &Vec>| ( + tx_hash, + Some(block.body.transactions[test_tx_index].clone()) + ), + B256::random() + ), + ( + ONE, + transaction_block, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(block.number) + ), + u64::MAX + ), + ( + ONE, + transactions_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(block.body.transactions.clone()) + ), + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + transactions_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(block.body.transactions.clone()) + ), + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + transaction_sender, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + block.body.transactions[test_tx_index].recover_signer() + ), + u64::MAX + ), + ( + ONE, + receipt, + |block: &SealedBlock, + tx_num: TxNumber, + _: B256, + receipts: &Vec>| ( + tx_num, + Some(receipts[block.number as usize][test_tx_index].clone()) + ), + u64::MAX + ), + ( + ONE, + receipt_by_hash, + |block: &SealedBlock, + _: TxNumber, + tx_hash: B256, + receipts: &Vec>| ( + tx_hash, + Some(receipts[block.number as usize][test_tx_index].clone()) + ), + B256::random() + ), + ( + ONE, + receipts_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(receipts[block.number as usize].clone()) + ), + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + receipts_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(receipts[block.number as usize].clone()) + ), + BlockHashOrNumber::Hash(B256::random()) + ), + // TODO: withdrawals, requests, ommers + ] ); Ok(()) From 139776fe8e4607fe31ebde07766971b06c765461 Mon Sep 17 00:00:00 2001 From: Luca Provini Date: Tue, 8 Oct 2024 15:27:42 +0200 Subject: [PATCH 063/159] feat: adding a new method to network config builder (#11569) --- crates/exex/test-utils/src/lib.rs | 1 + crates/net/network/src/config.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/crates/exex/test-utils/src/lib.rs b/crates/exex/test-utils/src/lib.rs index f4561a6b55f28..e219f55031dbb 100644 --- a/crates/exex/test-utils/src/lib.rs +++ b/crates/exex/test-utils/src/lib.rs @@ -271,6 +271,7 @@ pub async fn test_exex_context_with_chain_spec( let network_manager = NetworkManager::new( NetworkConfigBuilder::new(SecretKey::new(&mut rand::thread_rng())) + .with_unused_discovery_port() .build(provider_factory.clone()), ) .await?; diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index 60b559b9dbc34..9d9183edcf986 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -347,6 +347,18 @@ impl NetworkConfigBuilder { self } + /// Sets the discovery port to an unused port. + /// This is useful for testing. + pub fn with_unused_discovery_port(self) -> Self { + self.discovery_port(0) + } + + /// Sets the listener port to an unused port. + /// This is useful for testing. + pub fn with_unused_listener_port(self) -> Self { + self.listener_port(0) + } + /// Sets the external ip resolver to use for discovery v4. /// /// If no [`Discv4ConfigBuilder`] is set via [`Self::discovery`], this will create a new one. From 2cfb1c88d3c3d5d5a0511428085e4180a0de33ee Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 16:23:54 +0200 Subject: [PATCH 064/159] chore: rm unused optimism feature from engine api (#11577) --- crates/rpc/rpc-engine-api/Cargo.toml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/rpc/rpc-engine-api/Cargo.toml b/crates/rpc/rpc-engine-api/Cargo.toml index 4463d375a0345..00503f2c1dd7f 100644 --- a/crates/rpc/rpc-engine-api/Cargo.toml +++ b/crates/rpc/rpc-engine-api/Cargo.toml @@ -55,7 +55,4 @@ reth-testing-utils.workspace = true alloy-rlp.workspace = true -assert_matches.workspace = true - -[features] -optimism = ["reth-primitives/optimism"] +assert_matches.workspace = true \ No newline at end of file From 8760f3a2cb7c7ae89deb52b998d98707e5cb2d84 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 16:23:56 +0200 Subject: [PATCH 065/159] chore: replace some revm deps (#11579) --- crates/primitives/benches/validate_blob_tx.rs | 7 +++---- crates/primitives/src/alloy_compat.rs | 3 +-- crates/primitives/src/transaction/util.rs | 12 ++++-------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/crates/primitives/benches/validate_blob_tx.rs b/crates/primitives/benches/validate_blob_tx.rs index 61fe161f2f74f..50498a9420fdd 100644 --- a/crates/primitives/benches/validate_blob_tx.rs +++ b/crates/primitives/benches/validate_blob_tx.rs @@ -1,7 +1,7 @@ #![allow(missing_docs)] use alloy_consensus::TxEip4844; -use alloy_eips::eip4844::env_settings::EnvKzgSettings; +use alloy_eips::eip4844::{env_settings::EnvKzgSettings, MAX_BLOBS_PER_BLOCK}; use alloy_primitives::hex; use criterion::{ criterion_group, criterion_main, measurement::WallTime, BenchmarkGroup, Criterion, @@ -13,7 +13,6 @@ use proptest::{ }; use proptest_arbitrary_interop::arb; use reth_primitives::BlobTransactionSidecar; -use revm_primitives::MAX_BLOB_NUMBER_PER_BLOCK; // constant seed to use for the rng const SEED: [u8; 32] = hex!("1337133713371337133713371337133713371337133713371337133713371337"); @@ -22,9 +21,9 @@ const SEED: [u8; 32] = hex!("133713371337133713371337133713371337133713371337133 fn blob_validation(c: &mut Criterion) { let mut group = c.benchmark_group("Blob Transaction KZG validation"); - for num_blobs in 1..=MAX_BLOB_NUMBER_PER_BLOCK { + for num_blobs in 1..=MAX_BLOBS_PER_BLOCK { println!("Benchmarking validation for tx with {num_blobs} blobs"); - validate_blob_tx(&mut group, "ValidateBlob", num_blobs, EnvKzgSettings::Default); + validate_blob_tx(&mut group, "ValidateBlob", num_blobs as u64, EnvKzgSettings::Default); } } diff --git a/crates/primitives/src/alloy_compat.rs b/crates/primitives/src/alloy_compat.rs index bf7f557b799af..f43af019e75b1 100644 --- a/crates/primitives/src/alloy_compat.rs +++ b/crates/primitives/src/alloy_compat.rs @@ -278,9 +278,8 @@ impl TryFrom> for TransactionSigne #[cfg(feature = "optimism")] mod tests { use super::*; - use alloy_primitives::{B256, U256}; + use alloy_primitives::{address, Address, B256, U256}; use alloy_rpc_types::Transaction as AlloyTransaction; - use revm_primitives::{address, Address}; #[test] fn optimism_deposit_tx_conversion_no_mint() { diff --git a/crates/primitives/src/transaction/util.rs b/crates/primitives/src/transaction/util.rs index 6205ec886ca09..7569400e94b46 100644 --- a/crates/primitives/src/transaction/util.rs +++ b/crates/primitives/src/transaction/util.rs @@ -1,6 +1,5 @@ use crate::Signature; use alloy_primitives::Address; -use revm_primitives::B256; #[cfg(feature = "secp256k1")] pub(crate) mod secp256k1 { @@ -20,8 +19,7 @@ mod impl_secp256k1 { ecdsa::{RecoverableSignature, RecoveryId}, Message, PublicKey, SecretKey, SECP256K1, }; - use alloy_primitives::{keccak256, Parity}; - use revm_primitives::U256; + use alloy_primitives::{keccak256, Parity, B256, U256}; /// Recovers the address of the sender using secp256k1 pubkey recovery. /// @@ -65,10 +63,9 @@ mod impl_secp256k1 { #[cfg_attr(feature = "secp256k1", allow(unused, unreachable_pub))] mod impl_k256 { use super::*; - use alloy_primitives::{keccak256, Parity}; + use alloy_primitives::{keccak256, Parity, B256, U256}; pub(crate) use k256::ecdsa::Error; use k256::ecdsa::{RecoveryId, SigningKey, VerifyingKey}; - use revm_primitives::U256; /// Recovers the address of the sender using secp256k1 pubkey recovery. /// @@ -117,11 +114,12 @@ mod impl_k256 { #[cfg(test)] mod tests { + use alloy_primitives::{keccak256, B256}; + #[cfg(feature = "secp256k1")] #[test] fn sanity_ecrecover_call_secp256k1() { use super::impl_secp256k1::*; - use revm_primitives::{keccak256, B256}; let (secret, public) = secp256k1::generate_keypair(&mut rand::thread_rng()); let signer = public_key_to_address(public); @@ -143,7 +141,6 @@ mod tests { #[test] fn sanity_ecrecover_call_k256() { use super::impl_k256::*; - use revm_primitives::{keccak256, B256}; let secret = k256::ecdsa::SigningKey::random(&mut rand::thread_rng()); let public = *secret.verifying_key(); @@ -165,7 +162,6 @@ mod tests { #[test] fn sanity_secp256k1_k256_compat() { use super::{impl_k256, impl_secp256k1}; - use revm_primitives::{keccak256, B256}; let (secp256k1_secret, secp256k1_public) = secp256k1::generate_keypair(&mut rand::thread_rng()); From 3f99124a767096788a050545853e43e7b15b018a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 16:25:00 +0200 Subject: [PATCH 066/159] chore: rm bad cap function (#11562) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 7 ++++--- crates/rpc/rpc-eth-types/src/revm_utils.rs | 16 +++------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index c5703bcceef30..5bc7d73b2221a 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -28,8 +28,8 @@ use reth_rpc_eth_types::{ cache::db::{StateCacheDbRefMutWrapper, StateProviderTraitObjWrapper}, error::ensure_success, revm_utils::{ - apply_block_overrides, apply_state_overrides, caller_gas_allowance, - cap_tx_gas_limit_with_caller_allowance, get_precompiles, CallFees, + apply_block_overrides, apply_state_overrides, caller_gas_allowance, get_precompiles, + CallFees, }, simulate::{self, EthSimulateError}, EthApiError, RevertError, RpcInvalidTransactionError, StateCacheDb, @@ -379,8 +379,9 @@ pub trait EthCall: Call + LoadPendingBlock { let mut db = CacheDB::new(StateProviderDatabase::new(state)); if request.gas.is_none() && env.tx.gas_price > U256::ZERO { + let cap = caller_gas_allowance(&mut db, &env.tx)?; // no gas limit was provided in the request, so we need to cap the request's gas limit - cap_tx_gas_limit_with_caller_allowance(&mut db, &mut env.tx)?; + env.tx.gas_limit = cap.min(env.block.gas_limit).saturating_to(); } let from = request.from.unwrap_or_default(); diff --git a/crates/rpc/rpc-eth-types/src/revm_utils.rs b/crates/rpc/rpc-eth-types/src/revm_utils.rs index f6fbdc2f7ac8b..25c54fd467772 100644 --- a/crates/rpc/rpc-eth-types/src/revm_utils.rs +++ b/crates/rpc/rpc-eth-types/src/revm_utils.rs @@ -23,25 +23,15 @@ pub fn get_precompiles(spec_id: SpecId) -> impl IntoIterator { Precompiles::new(spec).addresses().copied().map(Address::from) } -/// Caps the configured [`TxEnv`] `gas_limit` with the allowance of the caller. -pub fn cap_tx_gas_limit_with_caller_allowance(db: &mut DB, env: &mut TxEnv) -> EthResult<()> -where - DB: Database, - EthApiError: From<::Error>, -{ - if let Ok(gas_limit) = caller_gas_allowance(db, env)?.try_into() { - env.gas_limit = gas_limit; - } - - Ok(()) -} - /// Calculates the caller gas allowance. /// /// `allowance = (account.balance - tx.value) / tx.gas_price` /// /// Returns an error if the caller has insufficient funds. /// Caution: This assumes non-zero `env.gas_price`. Otherwise, zero allowance will be returned. +/// +/// Note: this takes the mut [Database] trait because the loaded sender can be reused for the +/// following operation like `eth_call`. pub fn caller_gas_allowance(db: &mut DB, env: &TxEnv) -> EthResult where DB: Database, From ba8ccf5d4a93a67d12679f3684c0875540ba56e8 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Tue, 8 Oct 2024 18:33:31 +0400 Subject: [PATCH 067/159] feat: impl `Encodable2718` and `Decodable2718` for `PooledTransactionsElement` (#11482) --- Cargo.lock | 1 + .../src/commands/debug_cmd/build_block.rs | 2 +- crates/net/eth-wire/Cargo.toml | 1 + .../net/eth-wire/tests/pooled_transactions.rs | 3 +- crates/primitives/src/transaction/pooled.rs | 396 ++++++------------ crates/primitives/src/transaction/sidecar.rs | 11 +- .../rpc-eth-api/src/helpers/transaction.rs | 2 +- crates/rpc/rpc-eth-types/src/utils.rs | 5 +- crates/transaction-pool/src/maintain.rs | 3 +- crates/transaction-pool/src/traits.rs | 2 +- crates/transaction-pool/src/validate/eth.rs | 3 +- 11 files changed, 150 insertions(+), 279 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 93ae84221192b..bbf487026d2b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7146,6 +7146,7 @@ dependencies = [ name = "reth-eth-wire" version = "1.0.8" dependencies = [ + "alloy-eips", "alloy-primitives", "alloy-rlp", "arbitrary", diff --git a/bin/reth/src/commands/debug_cmd/build_block.rs b/bin/reth/src/commands/debug_cmd/build_block.rs index a7f75c02a8f82..455d8356aff94 100644 --- a/bin/reth/src/commands/debug_cmd/build_block.rs +++ b/bin/reth/src/commands/debug_cmd/build_block.rs @@ -194,7 +194,7 @@ impl> Command { ) .expect("should not fail to convert blob tx if it is already eip4844"); let pooled = PooledTransactionsElement::BlobTransaction(tx); - let encoded_length = pooled.length_without_header(); + let encoded_length = pooled.encode_2718_len(); // insert the blob into the store blob_store.insert(transaction.hash, sidecar)?; diff --git a/crates/net/eth-wire/Cargo.toml b/crates/net/eth-wire/Cargo.toml index 13224a37ada17..6eea4bc4ac659 100644 --- a/crates/net/eth-wire/Cargo.toml +++ b/crates/net/eth-wire/Cargo.toml @@ -62,6 +62,7 @@ proptest.workspace = true proptest-arbitrary-interop.workspace = true async-stream.workspace = true serde.workspace = true +alloy-eips.workspace = true [features] arbitrary = [ diff --git a/crates/net/eth-wire/tests/pooled_transactions.rs b/crates/net/eth-wire/tests/pooled_transactions.rs index 7927384c95b02..6690f42631a66 100644 --- a/crates/net/eth-wire/tests/pooled_transactions.rs +++ b/crates/net/eth-wire/tests/pooled_transactions.rs @@ -1,5 +1,6 @@ //! Decoding tests for [`PooledTransactions`] +use alloy_eips::eip2718::Decodable2718; use alloy_primitives::hex; use alloy_rlp::{Decodable, Encodable}; use reth_eth_wire::{EthVersion, PooledTransactions, ProtocolMessage}; @@ -72,5 +73,5 @@ fn decode_blob_rpc_transaction() { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata/rpc_blob_transaction"); let data = fs::read_to_string(network_data_path).expect("Unable to read file"); let hex_data = hex::decode(data.trim()).unwrap(); - let _txs = PooledTransactionsElement::decode_enveloped(&mut hex_data.as_ref()).unwrap(); + let _txs = PooledTransactionsElement::decode_2718(&mut hex_data.as_ref()).unwrap(); } diff --git a/crates/primitives/src/transaction/pooled.rs b/crates/primitives/src/transaction/pooled.rs index cc2dc57663948..ec49f44a680bc 100644 --- a/crates/primitives/src/transaction/pooled.rs +++ b/crates/primitives/src/transaction/pooled.rs @@ -10,14 +10,13 @@ use crate::{ BlobTransaction, BlobTransactionSidecar, Signature, Transaction, TransactionSigned, TransactionSignedEcRecovered, EIP4844_TX_TYPE_ID, }; -use alloc::vec::Vec; use alloy_consensus::{ transaction::{TxEip1559, TxEip2930, TxEip4844, TxLegacy}, SignableTransaction, TxEip4844WithSidecar, }; -use alloy_eips::eip2718::{Decodable2718, Eip2718Error}; -use alloy_primitives::{Address, Bytes, TxHash, B256}; -use alloy_rlp::{Decodable, Encodable, Error as RlpError, Header, EMPTY_LIST_CODE}; +use alloy_eips::eip2718::{Decodable2718, Eip2718Result, Encodable2718}; +use alloy_primitives::{Address, TxHash, B256}; +use alloy_rlp::{Decodable, Encodable, Error as RlpError, Header}; use bytes::Buf; use derive_more::{AsRef, Deref}; use serde::{Deserialize, Serialize}; @@ -183,101 +182,6 @@ impl PooledTransactionsElement { } } - /// Decodes the "raw" format of transaction (e.g. `eth_sendRawTransaction`). - /// - /// This should be used for `eth_sendRawTransaction`, for any transaction type. Blob - /// transactions **must** include the blob sidecar as part of the raw encoding. - /// - /// This method can not be used for decoding the `transactions` field of `engine_newPayload`, - /// because EIP-4844 transactions for that method do not include the blob sidecar. The blobs - /// are supplied in an argument separate from the payload. - /// - /// A raw transaction is either a legacy transaction or EIP-2718 typed transaction, with a - /// special case for EIP-4844 transactions. - /// - /// For legacy transactions, the format is encoded as: `rlp(tx)`. This format will start with a - /// RLP list header. - /// - /// For EIP-2718 typed transactions, the format is encoded as the type of the transaction - /// followed by the rlp of the transaction: `type || rlp(tx)`. - /// - /// For EIP-4844 transactions, the format includes a blob sidecar (the blobs, commitments, and - /// proofs) after the transaction: - /// `type || rlp([tx_payload_body, blobs, commitments, proofs])` - /// - /// Where `tx_payload_body` is encoded as a RLP list: - /// `[chain_id, nonce, max_priority_fee_per_gas, ..., y_parity, r, s]` - pub fn decode_enveloped(data: &mut &[u8]) -> alloy_rlp::Result { - if data.is_empty() { - return Err(RlpError::InputTooShort) - } - - // Check if the tx is a list - tx types are less than EMPTY_LIST_CODE (0xc0) - if data[0] >= EMPTY_LIST_CODE { - // decode as legacy transaction - let (transaction, hash, signature) = - TransactionSigned::decode_rlp_legacy_transaction_tuple(data)?; - - Ok(Self::Legacy { transaction, signature, hash }) - } else { - // decode the type byte, only decode BlobTransaction if it is a 4844 transaction - let tx_type = *data.first().ok_or(RlpError::InputTooShort)?; - - // First, we advance the buffer past the type byte - data.advance(1); - - if tx_type == EIP4844_TX_TYPE_ID { - // Recall that the blob transaction response `TransactionPayload` is encoded like - // this: `rlp([tx_payload_body, blobs, commitments, proofs])` - // - // Note that `tx_payload_body` is a list: - // `[chain_id, nonce, max_priority_fee_per_gas, ..., y_parity, r, s]` - // - // This makes the full encoding: - // `tx_type (0x03) || rlp([[chain_id, nonce, ...], blobs, commitments, proofs])` - - // Now, we decode the inner blob transaction: - // `rlp([[chain_id, nonce, ...], blobs, commitments, proofs])` - let blob_tx = BlobTransaction::decode_inner(data)?; - Ok(Self::BlobTransaction(blob_tx)) - } else { - let typed_tx = - TransactionSigned::typed_decode(tx_type, data).map_err(|err| match err { - Eip2718Error::RlpError(err) => err, - _ => RlpError::Custom("failed to decode EIP-2718 transaction"), - })?; - - // because we checked the tx type, we can be sure that the transaction is not a - // blob transaction or legacy - match typed_tx.transaction { - Transaction::Legacy(_) => Err(RlpError::Custom( - "legacy transactions should not be a result of EIP-2718 decoding", - )), - Transaction::Eip4844(_) => Err(RlpError::Custom( - "EIP-4844 transactions can only be decoded with transaction type 0x03", - )), - Transaction::Eip2930(tx) => Ok(Self::Eip2930 { - transaction: tx, - signature: typed_tx.signature, - hash: typed_tx.hash, - }), - Transaction::Eip1559(tx) => Ok(Self::Eip1559 { - transaction: tx, - signature: typed_tx.signature, - hash: typed_tx.hash, - }), - Transaction::Eip7702(tx) => {Ok(Self::Eip7702 { - transaction: tx, - signature: typed_tx.signature, - hash: typed_tx.hash, - })}, - #[cfg(feature = "optimism")] - Transaction::Deposit(_) => Err(RlpError::Custom("Optimism deposit transaction cannot be decoded to PooledTransactionsElement")) - } - } - } - } - /// Create [`TransactionSignedEcRecovered`] by converting this transaction into /// [`TransactionSigned`] and [`Address`] of the signer. pub fn into_ecrecovered_transaction(self, signer: Address) -> TransactionSignedEcRecovered { @@ -309,83 +213,6 @@ impl PooledTransactionsElement { } } - /// Returns the length without an RLP header - this is used for eth/68 sizes. - pub fn length_without_header(&self) -> usize { - match self { - Self::Legacy { transaction, signature, .. } => { - // method computes the payload len with a RLP header - transaction.encoded_len_with_signature(&with_eip155_parity( - signature, - transaction.chain_id, - )) - } - Self::Eip2930 { transaction, signature, .. } => { - // method computes the payload len without a RLP header - transaction.encoded_len_with_signature(signature, false) - } - Self::Eip1559 { transaction, signature, .. } => { - // method computes the payload len without a RLP header - transaction.encoded_len_with_signature(signature, false) - } - Self::Eip7702 { transaction, signature, .. } => { - // method computes the payload len without a RLP header - transaction.encoded_len_with_signature(signature, false) - } - Self::BlobTransaction(blob_tx) => { - // the encoding does not use a header, so we set `with_header` to false - blob_tx.payload_len_with_type(false) - } - } - } - - /// Returns the enveloped encoded transactions. - /// - /// See also [`alloy_eips::eip2718::Encodable2718::encoded_2718`] - pub fn envelope_encoded(&self) -> Bytes { - let mut buf = Vec::new(); - self.encode_enveloped(&mut buf); - buf.into() - } - - /// Encodes the transaction into the "raw" format (e.g. `eth_sendRawTransaction`). - /// This format is also referred to as "binary" encoding. - /// - /// For legacy transactions, it encodes the RLP of the transaction into the buffer: - /// `rlp(tx-data)` - /// For EIP-2718 typed it encodes the type of the transaction followed by the rlp of the - /// transaction: `tx-type || rlp(tx-data)` - pub fn encode_enveloped(&self, out: &mut dyn bytes::BufMut) { - // The encoding of `tx-data` depends on the transaction type. Refer to these docs for more - // information on the exact format: - // - Legacy: TxLegacy::encode_with_signature - // - EIP-2930: TxEip2930::encode_with_signature - // - EIP-1559: TxEip1559::encode_with_signature - // - EIP-4844: BlobTransaction::encode_with_type_inner - // - EIP-7702: TxEip7702::encode_with_signature - match self { - Self::Legacy { transaction, signature, .. } => transaction - .encode_with_signature_fields( - &with_eip155_parity(signature, transaction.chain_id), - out, - ), - Self::Eip2930 { transaction, signature, .. } => { - transaction.encode_with_signature(signature, out, false) - } - Self::Eip1559 { transaction, signature, .. } => { - transaction.encode_with_signature(signature, out, false) - } - Self::Eip7702 { transaction, signature, .. } => { - transaction.encode_with_signature(signature, out, false) - } - Self::BlobTransaction(blob_tx) => { - // The inner encoding is used with `with_header` set to true, making the final - // encoding: - // `tx_type || rlp([transaction_payload_body, blobs, commitments, proofs]))` - blob_tx.encode_with_type_inner(out, false); - } - } - } - /// Returns true if the transaction is an EIP-4844 transaction. #[inline] pub const fn is_eip4844(&self) -> bool { @@ -481,73 +308,25 @@ impl PooledTransactionsElement { } impl Encodable for PooledTransactionsElement { - /// Encodes an enveloped post EIP-4844 [`PooledTransactionsElement`]. + /// This encodes the transaction _with_ the signature, and an rlp header. /// - /// For legacy transactions, this encodes the transaction as `rlp(tx-data)`. + /// For legacy transactions, it encodes the transaction data: + /// `rlp(tx-data)` /// - /// For EIP-2718 transactions, this encodes the transaction as `rlp(tx_type || rlp(tx-data)))`, - /// ___including__ the RLP-header for the entire transaction. + /// For EIP-2718 typed transactions, it encodes the transaction type followed by the rlp of the + /// transaction: + /// `rlp(tx-type || rlp(tx-data))` fn encode(&self, out: &mut dyn bytes::BufMut) { - // The encoding of `tx-data` depends on the transaction type. Refer to these docs for more - // information on the exact format: - // - Legacy: TxLegacy::encode_with_signature - // - EIP-2930: TxEip2930::encode_with_signature - // - EIP-1559: TxEip1559::encode_with_signature - // - EIP-4844: BlobTransaction::encode_with_type_inner - // - EIP-7702: TxEip7702::encode_with_signature - match self { - Self::Legacy { transaction, signature, .. } => transaction - .encode_with_signature_fields( - &with_eip155_parity(signature, transaction.chain_id), - out, - ), - Self::Eip2930 { transaction, signature, .. } => { - // encodes with string header - transaction.encode_with_signature(signature, out, true) - } - Self::Eip1559 { transaction, signature, .. } => { - // encodes with string header - transaction.encode_with_signature(signature, out, true) - } - Self::Eip7702 { transaction, signature, .. } => { - // encodes with string header - transaction.encode_with_signature(signature, out, true) - } - Self::BlobTransaction(blob_tx) => { - // The inner encoding is used with `with_header` set to true, making the final - // encoding: - // `rlp(tx_type || rlp([transaction_payload_body, blobs, commitments, proofs]))` - blob_tx.encode_with_type_inner(out, true); - } - } + self.network_encode(out); } fn length(&self) -> usize { - match self { - Self::Legacy { transaction, signature, .. } => { - // method computes the payload len with a RLP header - transaction.encoded_len_with_signature(&with_eip155_parity( - signature, - transaction.chain_id, - )) - } - Self::Eip2930 { transaction, signature, .. } => { - // method computes the payload len with a RLP header - transaction.encoded_len_with_signature(signature, true) - } - Self::Eip1559 { transaction, signature, .. } => { - // method computes the payload len with a RLP header - transaction.encoded_len_with_signature(signature, true) - } - Self::Eip7702 { transaction, signature, .. } => { - // method computes the payload len with a RLP header - transaction.encoded_len_with_signature(signature, true) - } - Self::BlobTransaction(blob_tx) => { - // the encoding uses a header, so we set `with_header` to true - blob_tx.payload_len_with_type(true) - } + let mut payload_length = self.encode_2718_len(); + if !self.is_legacy() { + payload_length += Header { list: false, payload_length }.length(); } + + payload_length } } @@ -581,23 +360,110 @@ impl Decodable for PooledTransactionsElement { // Check if the tx is a list if header.list { // decode as legacy transaction - let (transaction, hash, signature) = - TransactionSigned::decode_rlp_legacy_transaction_tuple(&mut original_encoding)?; + let tx = Self::fallback_decode(&mut original_encoding)?; // advance the buffer by however long the legacy transaction decoding advanced the // buffer *buf = original_encoding; - Ok(Self::Legacy { transaction, signature, hash }) + Ok(tx) } else { // decode the type byte, only decode BlobTransaction if it is a 4844 transaction let tx_type = *buf.first().ok_or(RlpError::InputTooShort)?; let remaining_len = buf.len(); - // Aadvance the buffer past the type byte + // Advance the buffer past the type byte buf.advance(1); - if tx_type == EIP4844_TX_TYPE_ID { + let tx = Self::typed_decode(tx_type, buf).map_err(RlpError::from)?; + + // check that the bytes consumed match the payload length + let bytes_consumed = remaining_len - buf.len(); + if bytes_consumed != header.payload_length { + return Err(RlpError::UnexpectedLength) + } + + Ok(tx) + } + } +} + +impl Encodable2718 for PooledTransactionsElement { + fn type_flag(&self) -> Option { + match self { + Self::Legacy { .. } => None, + Self::Eip2930 { .. } => Some(0x01), + Self::Eip1559 { .. } => Some(0x02), + Self::BlobTransaction { .. } => Some(0x03), + Self::Eip7702 { .. } => Some(0x04), + } + } + + fn encode_2718_len(&self) -> usize { + match self { + Self::Legacy { transaction, signature, .. } => { + // method computes the payload len with a RLP header + transaction.encoded_len_with_signature(&with_eip155_parity( + signature, + transaction.chain_id, + )) + } + Self::Eip2930 { transaction, signature, .. } => { + // method computes the payload len without a RLP header + transaction.encoded_len_with_signature(signature, false) + } + Self::Eip1559 { transaction, signature, .. } => { + // method computes the payload len without a RLP header + transaction.encoded_len_with_signature(signature, false) + } + Self::Eip7702 { transaction, signature, .. } => { + // method computes the payload len without a RLP header + transaction.encoded_len_with_signature(signature, false) + } + Self::BlobTransaction(blob_tx) => { + // the encoding does not use a header, so we set `with_header` to false + blob_tx.payload_len_with_type(false) + } + } + } + + fn encode_2718(&self, out: &mut dyn alloy_rlp::BufMut) { + // The encoding of `tx-data` depends on the transaction type. Refer to these docs for more + // information on the exact format: + // - Legacy: TxLegacy::encode_with_signature + // - EIP-2930: TxEip2930::encode_with_signature + // - EIP-1559: TxEip1559::encode_with_signature + // - EIP-4844: BlobTransaction::encode_with_type_inner + // - EIP-7702: TxEip7702::encode_with_signature + match self { + Self::Legacy { transaction, signature, .. } => transaction + .encode_with_signature_fields( + &with_eip155_parity(signature, transaction.chain_id), + out, + ), + Self::Eip2930 { transaction, signature, .. } => { + transaction.encode_with_signature(signature, out, false) + } + Self::Eip1559 { transaction, signature, .. } => { + transaction.encode_with_signature(signature, out, false) + } + Self::Eip7702 { transaction, signature, .. } => { + transaction.encode_with_signature(signature, out, false) + } + Self::BlobTransaction(blob_tx) => { + // The inner encoding is used with `with_header` set to true, making the final + // encoding: + // `tx_type || rlp([transaction_payload_body, blobs, commitments, proofs]))` + blob_tx.encode_with_type_inner(out, false); + } + } + } +} + +impl Decodable2718 for PooledTransactionsElement { + fn typed_decode(ty: u8, buf: &mut &[u8]) -> Eip2718Result { + match ty { + EIP4844_TX_TYPE_ID => { // Recall that the blob transaction response `TransactionPayload` is encoded like // this: `rlp([tx_payload_body, blobs, commitments, proofs])` // @@ -607,36 +473,23 @@ impl Decodable for PooledTransactionsElement { // This makes the full encoding: // `tx_type (0x03) || rlp([[chain_id, nonce, ...], blobs, commitments, proofs])` - // Decode the inner blob transaction: + // Now, we decode the inner blob transaction: // `rlp([[chain_id, nonce, ...], blobs, commitments, proofs])` let blob_tx = BlobTransaction::decode_inner(buf)?; - - // check that the bytes consumed match the payload length - let bytes_consumed = remaining_len - buf.len(); - if bytes_consumed != header.payload_length { - return Err(RlpError::UnexpectedLength) - } - Ok(Self::BlobTransaction(blob_tx)) - } else { - let typed_tx = - TransactionSigned::typed_decode(tx_type, buf).map_err(RlpError::from)?; - - // check that the bytes consumed match the payload length - let bytes_consumed = remaining_len - buf.len(); - if bytes_consumed != header.payload_length { - return Err(RlpError::UnexpectedLength) - } + } + tx_type => { + let typed_tx = TransactionSigned::typed_decode(tx_type, buf)?; - // because we checked the tx type, we can be sure that the transaction is not a - // blob transaction or legacy match typed_tx.transaction { Transaction::Legacy(_) => Err(RlpError::Custom( - "legacy transactions should not be a result of EIP-2718 decoding", - )), + "legacy transactions should not be a result of typed decoding", + ).into()), + // because we checked the tx type, we can be sure that the transaction is not a + // blob transaction Transaction::Eip4844(_) => Err(RlpError::Custom( "EIP-4844 transactions can only be decoded with transaction type 0x03", - )), + ).into()), Transaction::Eip2930(tx) => Ok(Self::Eip2930 { transaction: tx, signature: typed_tx.signature, @@ -653,11 +506,19 @@ impl Decodable for PooledTransactionsElement { hash: typed_tx.hash, }), #[cfg(feature = "optimism")] - Transaction::Deposit(_) => Err(RlpError::Custom("Optimism deposit transaction cannot be decoded to PooledTransactionsElement")) + Transaction::Deposit(_) => Err(RlpError::Custom("Optimism deposit transaction cannot be decoded to PooledTransactionsElement").into()) } } } } + + fn fallback_decode(buf: &mut &[u8]) -> Eip2718Result { + // decode as legacy transaction + let (transaction, hash, signature) = + TransactionSigned::decode_rlp_legacy_transaction_tuple(buf)?; + + Ok(Self::Legacy { transaction, signature, hash }) + } } impl TryFrom for PooledTransactionsElement { @@ -773,6 +634,7 @@ mod tests { use super::*; use alloy_primitives::{address, hex}; use assert_matches::assert_matches; + use bytes::Bytes; #[test] fn invalid_legacy_pooled_decoding_input_too_short() { @@ -804,7 +666,7 @@ mod tests { // this is a legacy tx so we can attempt the same test with decode_enveloped let input_rlp = &mut &hex_data[..]; - let res = PooledTransactionsElement::decode_enveloped(input_rlp); + let res = PooledTransactionsElement::decode_2718(input_rlp); assert!( res.is_err(), @@ -820,7 +682,7 @@ mod tests { let data = hex!("02f903d382426882ba09832dc6c0848674742682ed9694714b6a4ea9b94a8a7d9fd362ed72630688c8898c80b90364492d24749189822d8512430d3f3ff7a2ede675ac08265c08e2c56ff6fdaa66dae1cdbe4a5d1d7809f3e99272d067364e597542ac0c369d69e22a6399c3e9bee5da4b07e3f3fdc34c32c3d88aa2268785f3e3f8086df0934b10ef92cfffc2e7f3d90f5e83302e31382e302d64657600000000000000000000000000000000000000000000569e75fc77c1a856f6daaf9e69d8a9566ca34aa47f9133711ce065a571af0cfd000000000000000000000000e1e210594771824dad216568b91c9cb4ceed361c00000000000000000000000000000000000000000000000000000000000546e00000000000000000000000000000000000000000000000000000000000e4e1c00000000000000000000000000000000000000000000000000000000065d6750c00000000000000000000000000000000000000000000000000000000000f288000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002cf600000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000000f1628e56fa6d8c50e5b984a58c0df14de31c7b857ce7ba499945b99252976a93d06dcda6776fc42167fbe71cb59f978f5ef5b12577a90b132d14d9c6efa528076f0161d7bf03643cfc5490ec5084f4a041db7f06c50bd97efa08907ba79ddcac8b890f24d12d8db31abbaaf18985d54f400449ee0559a4452afe53de5853ce090000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000028000000000000000000000000000000000000000000000000000000000000003e800000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000064ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00000000000000000000000000000000000000000000000000000000c080a01428023fc54a27544abc421d5d017b9a7c5936ad501cbdecd0d9d12d04c1a033a0753104bbf1c87634d6ff3f0ffa0982710612306003eb022363b57994bdef445a" ); - let res = PooledTransactionsElement::decode_enveloped(&mut &data[..]).unwrap(); + let res = PooledTransactionsElement::decode_2718(&mut &data[..]).unwrap(); assert_eq!( res.into_transaction().to(), Some(address!("714b6a4ea9b94a8a7d9fd362ed72630688c8898c")) @@ -854,7 +716,7 @@ mod tests { assert!(input_rlp.is_empty()); // we can also decode_enveloped - let res = PooledTransactionsElement::decode_enveloped(&mut &data[..]); + let res = PooledTransactionsElement::decode_2718(&mut &data[..]); assert_matches!(res, Ok(_tx)); } } diff --git a/crates/primitives/src/transaction/sidecar.rs b/crates/primitives/src/transaction/sidecar.rs index 52c3c68ef9db7..87b8c1fbf3e80 100644 --- a/crates/primitives/src/transaction/sidecar.rs +++ b/crates/primitives/src/transaction/sidecar.rs @@ -283,7 +283,10 @@ pub fn generate_blob_sidecar(blobs: Vec) -> BlobTransactionSidecar mod tests { use super::*; use crate::{kzg::Blob, PooledTransactionsElement}; - use alloy_eips::eip4844::Bytes48; + use alloy_eips::{ + eip2718::{Decodable2718, Encodable2718}, + eip4844::Bytes48, + }; use alloy_primitives::hex; use alloy_rlp::Encodable; use std::{fs, path::PathBuf, str::FromStr}; @@ -435,15 +438,15 @@ mod tests { let entry = entry.unwrap(); let content = fs::read_to_string(entry.path()).unwrap(); let raw = hex::decode(content.trim()).unwrap(); - let tx = PooledTransactionsElement::decode_enveloped(&mut raw.as_ref()) + let tx = PooledTransactionsElement::decode_2718(&mut raw.as_ref()) .map_err(|err| { panic!("Failed to decode transaction: {:?} {:?}", err, entry.path()); }) .unwrap(); // We want to test only EIP-4844 transactions assert!(tx.is_eip4844()); - let encoded = tx.envelope_encoded(); - assert_eq!(encoded.as_ref(), &raw[..], "{:?}", entry.path()); + let encoded = tx.encoded_2718(); + assert_eq!(encoded.as_slice(), &raw[..], "{:?}", entry.path()); } } } diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index afcb1b2eeb256..5e6aadd778bd5 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -97,7 +97,7 @@ pub trait EthTransactions: LoadTransaction { async move { // Note: this is mostly used to fetch pooled transactions so we check the pool first if let Some(tx) = - self.pool().get_pooled_transaction_element(hash).map(|tx| tx.envelope_encoded()) + self.pool().get_pooled_transaction_element(hash).map(|tx| tx.encoded_2718().into()) { return Ok(Some(tx)) } diff --git a/crates/rpc/rpc-eth-types/src/utils.rs b/crates/rpc/rpc-eth-types/src/utils.rs index bb7c3d64846ee..596acc74ce1ee 100644 --- a/crates/rpc/rpc-eth-types/src/utils.rs +++ b/crates/rpc/rpc-eth-types/src/utils.rs @@ -1,5 +1,6 @@ //! Commonly used code snippets +use alloy_eips::eip2718::Decodable2718; use alloy_primitives::Bytes; use reth_primitives::{PooledTransactionsElement, PooledTransactionsElementEcRecovered}; use std::future::Future; @@ -8,13 +9,13 @@ use super::{EthApiError, EthResult}; /// Recovers a [`PooledTransactionsElementEcRecovered`] from an enveloped encoded byte stream. /// -/// See [`PooledTransactionsElement::decode_enveloped`] +/// See [`Decodable2718::decode_2718`] pub fn recover_raw_transaction(data: Bytes) -> EthResult { if data.is_empty() { return Err(EthApiError::EmptyRawTransactionData) } - let transaction = PooledTransactionsElement::decode_enveloped(&mut data.as_ref()) + let transaction = PooledTransactionsElement::decode_2718(&mut data.as_ref()) .map_err(|_| EthApiError::FailedToDecodeSignedTransaction)?; transaction.try_into_ecrecovered().or(Err(EthApiError::InvalidTransactionSignature)) diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index 523151956dd10..66b98614737e0 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -676,6 +676,7 @@ mod tests { blobstore::InMemoryBlobStore, validate::EthTransactionValidatorBuilder, CoinbaseTipOrdering, EthPooledTransaction, Pool, TransactionOrigin, }; + use alloy_eips::eip2718::Decodable2718; use alloy_primitives::{hex, U256}; use reth_chainspec::MAINNET; use reth_fs_util as fs; @@ -699,7 +700,7 @@ mod tests { let temp_dir = tempfile::tempdir().unwrap(); let transactions_path = temp_dir.path().join(FILENAME).with_extension(EXTENSION); let tx_bytes = hex!("02f87201830655c2808505ef61f08482565f94388c818ca8b9251b393131c08a736a67ccb192978801049e39c4b5b1f580c001a01764ace353514e8abdfb92446de356b260e3c1225b73fc4c8876a6258d12a129a04f02294aa61ca7676061cd99f29275491218b4754b46a0248e5e42bc5091f507"); - let tx = PooledTransactionsElement::decode_enveloped(&mut &tx_bytes[..]).unwrap(); + let tx = PooledTransactionsElement::decode_2718(&mut &tx_bytes[..]).unwrap(); let provider = MockEthProvider::default(); let transaction: EthPooledTransaction = tx.try_into_ecrecovered().unwrap().into(); let tx_to_cmp = transaction.clone(); diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 1598877ff3b6b..adae238e46b3c 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1051,7 +1051,7 @@ impl EthPooledTransaction { /// Conversion from the network transaction type to the pool transaction type. impl From for EthPooledTransaction { fn from(tx: PooledTransactionsElementEcRecovered) -> Self { - let encoded_length = tx.length_without_header(); + let encoded_length = tx.encode_2718_len(); let (tx, signer) = tx.into_components(); match tx { PooledTransactionsElement::BlobTransaction(tx) => { diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index acfe46d6e88a0..49165f189a057 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -833,6 +833,7 @@ mod tests { blobstore::InMemoryBlobStore, error::PoolErrorKind, CoinbaseTipOrdering, EthPooledTransaction, Pool, TransactionPool, }; + use alloy_eips::eip2718::Decodable2718; use alloy_primitives::{hex, U256}; use reth_chainspec::MAINNET; use reth_primitives::PooledTransactionsElement; @@ -842,7 +843,7 @@ mod tests { let raw = "0x02f914950181ad84b2d05e0085117553845b830f7df88080b9143a6040608081523462000414576200133a803803806200001e8162000419565b9283398101608082820312620004145781516001600160401b03908181116200041457826200004f9185016200043f565b92602092838201519083821162000414576200006d9183016200043f565b8186015190946001600160a01b03821692909183900362000414576060015190805193808511620003145760038054956001938488811c9816801562000409575b89891014620003f3578190601f988981116200039d575b50899089831160011462000336576000926200032a575b505060001982841b1c191690841b1781555b8751918211620003145760049788548481811c9116801562000309575b89821014620002f457878111620002a9575b5087908784116001146200023e5793839491849260009562000232575b50501b92600019911b1c19161785555b6005556007805460ff60a01b19169055600880546001600160a01b0319169190911790553015620001f3575060025469d3c21bcecceda100000092838201809211620001de57506000917fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef9160025530835282815284832084815401905584519384523093a351610e889081620004b28239f35b601190634e487b7160e01b6000525260246000fd5b90606493519262461bcd60e51b845283015260248201527f45524332303a206d696e7420746f20746865207a65726f2061646472657373006044820152fd5b0151935038806200013a565b9190601f198416928a600052848a6000209460005b8c8983831062000291575050501062000276575b50505050811b0185556200014a565b01519060f884600019921b161c191690553880808062000267565b86860151895590970196948501948893500162000253565b89600052886000208880860160051c8201928b8710620002ea575b0160051c019085905b828110620002dd5750506200011d565b60008155018590620002cd565b92508192620002c4565b60228a634e487b7160e01b6000525260246000fd5b90607f16906200010b565b634e487b7160e01b600052604160045260246000fd5b015190503880620000dc565b90869350601f19831691856000528b6000209260005b8d8282106200038657505084116200036d575b505050811b018155620000ee565b015160001983861b60f8161c191690553880806200035f565b8385015186558a979095019493840193016200034c565b90915083600052896000208980850160051c8201928c8610620003e9575b918891869594930160051c01915b828110620003d9575050620000c5565b60008155859450889101620003c9565b92508192620003bb565b634e487b7160e01b600052602260045260246000fd5b97607f1697620000ae565b600080fd5b6040519190601f01601f191682016001600160401b038111838210176200031457604052565b919080601f84011215620004145782516001600160401b038111620003145760209062000475601f8201601f1916830162000419565b92818452828287010111620004145760005b8181106200049d57508260009394955001015290565b85810183015184820184015282016200048756fe608060408181526004918236101561001657600080fd5b600092833560e01c91826306fdde0314610a1c57508163095ea7b3146109f257816318160ddd146109d35781631b4c84d2146109ac57816323b872dd14610833578163313ce5671461081757816339509351146107c357816370a082311461078c578163715018a6146107685781638124f7ac146107495781638da5cb5b1461072057816395d89b411461061d578163a457c2d714610575578163a9059cbb146104e4578163c9567bf914610120575063dd62ed3e146100d557600080fd5b3461011c578060031936011261011c57806020926100f1610b5a565b6100f9610b75565b6001600160a01b0391821683526001865283832091168252845220549051908152f35b5080fd5b905082600319360112610338576008546001600160a01b039190821633036104975760079283549160ff8360a01c1661045557737a250d5630b4cf539739df2c5dacb4c659f2488d92836bffffffffffffffffffffffff60a01b8092161786553087526020938785528388205430156104065730895260018652848920828a52865280858a205584519081527f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925863092a38554835163c45a015560e01b815290861685828581845afa9182156103dd57849187918b946103e7575b5086516315ab88c960e31b815292839182905afa9081156103dd576044879289928c916103c0575b508b83895196879586946364e329cb60e11b8652308c870152166024850152165af19081156103b6579086918991610389575b50169060065416176006558385541660604730895288865260c4858a20548860085416928751958694859363f305d71960e01b8552308a86015260248501528d60448501528d606485015260848401524260a48401525af1801561037f579084929161034c575b50604485600654169587541691888551978894859363095ea7b360e01b855284015260001960248401525af1908115610343575061030c575b5050805460ff60a01b1916600160a01b17905580f35b81813d831161033c575b6103208183610b8b565b8101031261033857518015150361011c5738806102f6565b8280fd5b503d610316565b513d86823e3d90fd5b6060809293503d8111610378575b6103648183610b8b565b81010312610374578290386102bd565b8580fd5b503d61035a565b83513d89823e3d90fd5b6103a99150863d88116103af575b6103a18183610b8b565b810190610e33565b38610256565b503d610397565b84513d8a823e3d90fd5b6103d79150843d86116103af576103a18183610b8b565b38610223565b85513d8b823e3d90fd5b6103ff919450823d84116103af576103a18183610b8b565b92386101fb565b845162461bcd60e51b81528085018790526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f206164646044820152637265737360e01b6064820152608490fd5b6020606492519162461bcd60e51b8352820152601760248201527f74726164696e6720697320616c7265616479206f70656e0000000000000000006044820152fd5b608490602084519162461bcd60e51b8352820152602160248201527f4f6e6c79206f776e65722063616e2063616c6c20746869732066756e6374696f6044820152603760f91b6064820152fd5b9050346103385781600319360112610338576104fe610b5a565b9060243593303303610520575b602084610519878633610bc3565b5160018152f35b600594919454808302908382041483151715610562576127109004820391821161054f5750925080602061050b565b634e487b7160e01b815260118552602490fd5b634e487b7160e01b825260118652602482fd5b9050823461061a578260031936011261061a57610590610b5a565b918360243592338152600160205281812060018060a01b03861682526020522054908282106105c9576020856105198585038733610d31565b608490602086519162461bcd60e51b8352820152602560248201527f45524332303a2064656372656173656420616c6c6f77616e63652062656c6f77604482015264207a65726f60d81b6064820152fd5b80fd5b83833461011c578160031936011261011c57805191809380549160019083821c92828516948515610716575b6020958686108114610703578589529081156106df5750600114610687575b6106838787610679828c0383610b8b565b5191829182610b11565b0390f35b81529295507f8a35acfbc15ff81a39ae7d344fd709f28e8600b4aa8c65c6b64bfe7fe36bd19b5b8284106106cc57505050826106839461067992820101948680610668565b80548685018801529286019281016106ae565b60ff19168887015250505050151560051b8301019250610679826106838680610668565b634e487b7160e01b845260228352602484fd5b93607f1693610649565b50503461011c578160031936011261011c5760085490516001600160a01b039091168152602090f35b50503461011c578160031936011261011c576020906005549051908152f35b833461061a578060031936011261061a57600880546001600160a01b031916905580f35b50503461011c57602036600319011261011c5760209181906001600160a01b036107b4610b5a565b16815280845220549051908152f35b82843461061a578160031936011261061a576107dd610b5a565b338252600160209081528383206001600160a01b038316845290528282205460243581019290831061054f57602084610519858533610d31565b50503461011c578160031936011261011c576020905160128152f35b83833461011c57606036600319011261011c5761084e610b5a565b610856610b75565b6044359160018060a01b0381169485815260209560018752858220338352875285822054976000198903610893575b505050906105199291610bc3565b85891061096957811561091a5733156108cc5750948481979861051997845260018a528284203385528a52039120558594938780610885565b865162461bcd60e51b8152908101889052602260248201527f45524332303a20617070726f766520746f20746865207a65726f206164647265604482015261737360f01b6064820152608490fd5b865162461bcd60e51b81529081018890526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f206164646044820152637265737360e01b6064820152608490fd5b865162461bcd60e51b8152908101889052601d60248201527f45524332303a20696e73756666696369656e7420616c6c6f77616e63650000006044820152606490fd5b50503461011c578160031936011261011c5760209060ff60075460a01c1690519015158152f35b50503461011c578160031936011261011c576020906002549051908152f35b50503461011c578060031936011261011c57602090610519610a12610b5a565b6024359033610d31565b92915034610b0d5783600319360112610b0d57600354600181811c9186908281168015610b03575b6020958686108214610af05750848852908115610ace5750600114610a75575b6106838686610679828b0383610b8b565b929550600383527fc2575a0e9e593c00f959f8c92f12db2869c3395a3b0502d05e2516446f71f85b5b828410610abb575050508261068394610679928201019438610a64565b8054868501880152928601928101610a9e565b60ff191687860152505050151560051b83010192506106798261068338610a64565b634e487b7160e01b845260229052602483fd5b93607f1693610a44565b8380fd5b6020808252825181830181905290939260005b828110610b4657505060409293506000838284010152601f8019910116010190565b818101860151848201604001528501610b24565b600435906001600160a01b0382168203610b7057565b600080fd5b602435906001600160a01b0382168203610b7057565b90601f8019910116810190811067ffffffffffffffff821117610bad57604052565b634e487b7160e01b600052604160045260246000fd5b6001600160a01b03908116918215610cde5716918215610c8d57600082815280602052604081205491808310610c3957604082827fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef958760209652828652038282205586815220818154019055604051908152a3565b60405162461bcd60e51b815260206004820152602660248201527f45524332303a207472616e7366657220616d6f756e7420657863656564732062604482015265616c616e636560d01b6064820152608490fd5b60405162461bcd60e51b815260206004820152602360248201527f45524332303a207472616e7366657220746f20746865207a65726f206164647260448201526265737360e81b6064820152608490fd5b60405162461bcd60e51b815260206004820152602560248201527f45524332303a207472616e736665722066726f6d20746865207a65726f206164604482015264647265737360d81b6064820152608490fd5b6001600160a01b03908116918215610de25716918215610d925760207f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925918360005260018252604060002085600052825280604060002055604051908152a3565b60405162461bcd60e51b815260206004820152602260248201527f45524332303a20617070726f766520746f20746865207a65726f206164647265604482015261737360f01b6064820152608490fd5b60405162461bcd60e51b8152602060048201526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f206164646044820152637265737360e01b6064820152608490fd5b90816020910312610b7057516001600160a01b0381168103610b70579056fea2646970667358221220285c200b3978b10818ff576bb83f2dc4a2a7c98dfb6a36ea01170de792aa652764736f6c63430008140033000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000c0000000000000000000000000d3fd4f95820a9aa848ce716d6c200eaefb9a2e4900000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000003543131000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000035431310000000000000000000000000000000000000000000000000000000000c001a04e551c75810ffdfe6caff57da9f5a8732449f42f0f4c57f935b05250a76db3b6a046cd47e6d01914270c1ec0d9ac7fae7dfb240ec9a8b6ec7898c4d6aa174388f2"; let data = hex::decode(raw).unwrap(); - let tx = PooledTransactionsElement::decode_enveloped(&mut data.as_ref()).unwrap(); + let tx = PooledTransactionsElement::decode_2718(&mut data.as_ref()).unwrap(); tx.try_into_ecrecovered().unwrap().into() } From 8f3865b41b23464d774c7518c12387f9c0505bb8 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 8 Oct 2024 16:52:36 +0100 Subject: [PATCH 068/159] fix(exex): exhaust backfill job when using a stream (#11578) --- Cargo.lock | 182 ++++++++++++++---------- crates/exex/exex/src/backfill/job.rs | 10 +- crates/exex/exex/src/backfill/stream.rs | 114 +++++++++------ crates/exex/exex/src/notifications.rs | 6 +- 4 files changed, 190 insertions(+), 122 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbf487026d2b4..6c135f294c58e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.24.2" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] @@ -97,9 +97,9 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy-chains" -version = "0.1.36" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94c225801d42099570d0674701dddd4142f0ef715282aeb5985042e2ec962df7" +checksum = "8158b4878c67837e5413721cc44298e6a2d88d39203175ea025e51892a16ba4c" dependencies = [ "alloy-rlp", "arbitrary", @@ -296,7 +296,7 @@ dependencies = [ "getrandom 0.2.15", "hashbrown 0.14.5", "hex-literal", - "indexmap 2.6.0", + "indexmap 2.5.0", "itoa", "k256", "keccak-asm", @@ -619,7 +619,7 @@ dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.6.0", + "indexmap 2.5.0", "proc-macro-error2", "proc-macro2", "quote", @@ -1016,9 +1016,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.13" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e614738943d3f68c628ae3dbce7c3daffb196665f82f8c8ea6b65de73c79429" +checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ "brotli", "flate2", @@ -1203,6 +1203,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.79", +] + [[package]] name = "bindgen" version = "0.70.1" @@ -1313,7 +1333,7 @@ dependencies = [ "bitflags 2.6.0", "boa_interner", "boa_macros", - "indexmap 2.6.0", + "indexmap 2.5.0", "num-bigint", "rustc-hash 2.0.0", ] @@ -1339,7 +1359,7 @@ dependencies = [ "fast-float", "hashbrown 0.14.5", "icu_normalizer", - "indexmap 2.6.0", + "indexmap 2.5.0", "intrusive-collections", "itertools 0.13.0", "num-bigint", @@ -1385,7 +1405,7 @@ dependencies = [ "boa_gc", "boa_macros", "hashbrown 0.14.5", - "indexmap 2.6.0", + "indexmap 2.5.0", "once_cell", "phf", "rustc-hash 2.0.0", @@ -1453,9 +1473,9 @@ dependencies = [ [[package]] name = "brotli" -version = "7.0.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1599,9 +1619,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.25" +version = "1.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d9e0b4957f635b8d3da819d0db5603620467ecf1f692d22a8c2717ce27e6d8" +checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" dependencies = [ "jobserver", "libc", @@ -3113,9 +3133,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -3128,9 +3148,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -3138,15 +3158,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -3155,9 +3175,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -3176,9 +3196,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -3187,15 +3207,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -3209,9 +3229,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -3291,9 +3311,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "glob" @@ -3370,7 +3390,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.6.0", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", @@ -3416,12 +3436,6 @@ dependencies = [ "serde", ] -[[package]] -name = "hashbrown" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" - [[package]] name = "hashlink" version = "0.8.4" @@ -3949,13 +3963,13 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "arbitrary", "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.14.5", "serde", ] @@ -3972,7 +3986,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", - "indexmap 2.6.0", + "indexmap 2.5.0", "is-terminal", "itoa", "log", @@ -4070,9 +4084,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "iri-string" @@ -4110,6 +4124,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -4426,6 +4449,12 @@ dependencies = [ "spin", ] +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.159" @@ -4469,11 +4498,11 @@ dependencies = [ [[package]] name = "libproc" -version = "0.14.9" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cca3586d5efa98fba425856ba227950fd4287525dd5317b352f476ca7d603b" +checksum = "ae9ea4b75e1a81675429dafe43441df1caea70081e82246a8cccf514884a88bb" dependencies = [ - "bindgen", + "bindgen 0.69.4", "errno", "libc", ] @@ -4694,7 +4723,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" dependencies = [ "base64 0.22.1", - "indexmap 2.6.0", + "indexmap 2.5.0", "metrics", "metrics-util", "quanta", @@ -5105,18 +5134,21 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.2" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" +dependencies = [ + "portable-atomic", +] [[package]] name = "oorandom" @@ -5462,18 +5494,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.6" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.6" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", @@ -6009,9 +6041,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.2.0" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" +checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" dependencies = [ "bitflags 2.6.0", ] @@ -7529,7 +7561,7 @@ dependencies = [ "criterion", "dashmap 6.1.0", "derive_more 1.0.0", - "indexmap 2.6.0", + "indexmap 2.5.0", "parking_lot 0.12.3", "pprof", "rand 0.8.5", @@ -7545,7 +7577,7 @@ dependencies = [ name = "reth-mdbx-sys" version = "1.0.8" dependencies = [ - "bindgen", + "bindgen 0.70.1", "cc", ] @@ -9450,9 +9482,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.14" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "log", "once_cell", @@ -9751,7 +9783,7 @@ version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.5.0", "itoa", "memchr", "ryu", @@ -9803,15 +9835,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.11.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" +checksum = "9720086b3357bcb44fce40117d769a4d068c70ecfa190850a980a71755f66fcc" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.6.0", + "indexmap 2.5.0", "serde", "serde_derive", "serde_json", @@ -9821,9 +9853,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.11.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" +checksum = "5f1abbfe725f27678f4663bcacb75a83e829fd464c25d78dd038a3a29e307cec" dependencies = [ "darling", "proc-macro2", @@ -10621,7 +10653,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.5.0", "serde", "serde_spanned", "toml_datetime", @@ -10970,9 +11002,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.17" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" diff --git a/crates/exex/exex/src/backfill/job.rs b/crates/exex/exex/src/backfill/job.rs index 7d66ce23d6098..77a7b50477b00 100644 --- a/crates/exex/exex/src/backfill/job.rs +++ b/crates/exex/exex/src/backfill/job.rs @@ -18,6 +18,8 @@ use reth_revm::database::StateProviderDatabase; use reth_stages_api::ExecutionStageThresholds; use reth_tracing::tracing::{debug, trace}; +pub(super) type BackfillJobResult = Result; + /// Backfill job started for a specific range. /// /// It implements [`Iterator`] that executes blocks in batches according to the provided thresholds @@ -37,7 +39,7 @@ where E: BlockExecutorProvider, P: HeaderProvider + BlockReader + StateProviderFactory, { - type Item = Result; + type Item = BackfillJobResult; fn next(&mut self) -> Option { if self.range.is_empty() { @@ -63,7 +65,7 @@ where self.into() } - fn execute_range(&mut self) -> Result { + fn execute_range(&mut self) -> BackfillJobResult { debug!( target: "exex::backfill", range = ?self.range, @@ -165,7 +167,7 @@ where E: BlockExecutorProvider, P: HeaderProvider + BlockReader + StateProviderFactory, { - type Item = Result<(BlockWithSenders, BlockExecutionOutput), BlockExecutionError>; + type Item = BackfillJobResult<(BlockWithSenders, BlockExecutionOutput)>; fn next(&mut self) -> Option { self.range.next().map(|block_number| self.execute_block(block_number)) @@ -187,7 +189,7 @@ where pub(crate) fn execute_block( &self, block_number: u64, - ) -> Result<(BlockWithSenders, BlockExecutionOutput), BlockExecutionError> { + ) -> BackfillJobResult<(BlockWithSenders, BlockExecutionOutput)> { let td = self .provider .header_td_by_number(block_number)? diff --git a/crates/exex/exex/src/backfill/stream.rs b/crates/exex/exex/src/backfill/stream.rs index ca4bd326daa07..c55b8651daf11 100644 --- a/crates/exex/exex/src/backfill/stream.rs +++ b/crates/exex/exex/src/backfill/stream.rs @@ -18,12 +18,25 @@ use reth_stages_api::ExecutionStageThresholds; use reth_tracing::tracing::debug; use tokio::task::JoinHandle; +use super::job::BackfillJobResult; + /// The default parallelism for active tasks in [`StreamBackfillJob`]. pub(crate) const DEFAULT_PARALLELISM: usize = 4; /// The default batch size for active tasks in [`StreamBackfillJob`]. const DEFAULT_BATCH_SIZE: usize = 100; -type BackfillTasks = FuturesOrdered>>; +/// Boxed thread-safe iterator that yields [`BackfillJobResult`]s. +type BackfillTaskIterator = + Box> + Send + Sync + 'static>; + +/// Backfill task output. +struct BackfillTaskOutput { + job: BackfillTaskIterator, + result: Option>, +} + +/// Ordered queue of [`JoinHandle`]s that yield [`BackfillTaskOutput`]s. +type BackfillTasks = FuturesOrdered>>; type SingleBlockStreamItem = (BlockWithSenders, BlockExecutionOutput); type BatchBlockStreamItem = Chain; @@ -44,7 +57,10 @@ pub struct StreamBackfillJob { thresholds: ExecutionStageThresholds, } -impl StreamBackfillJob { +impl StreamBackfillJob +where + T: Send + Sync + 'static, +{ /// Configures the parallelism of the [`StreamBackfillJob`] to handle active tasks. pub const fn with_parallelism(mut self, parallelism: usize) -> Self { self.parallelism = parallelism; @@ -57,14 +73,30 @@ impl StreamBackfillJob { self } - fn poll_next_task( - &mut self, - cx: &mut Context<'_>, - ) -> Poll>> { - match ready!(self.tasks.poll_next_unpin(cx)) { - Some(res) => Poll::Ready(Some(res.map_err(BlockExecutionError::other)?)), - None => Poll::Ready(None), + /// Spawns a new task calling the [`BackfillTaskIterator::next`] method and pushes it to the + /// [`BackfillTasks`] queue. + fn push_task(&mut self, mut job: BackfillTaskIterator) { + self.tasks.push_back(tokio::task::spawn_blocking(move || BackfillTaskOutput { + result: job.next(), + job, + })); + } + + /// Polls the next task in the [`BackfillTasks`] queue until it returns a non-empty result. + fn poll_next_task(&mut self, cx: &mut Context<'_>) -> Poll>> { + while let Some(res) = ready!(self.tasks.poll_next_unpin(cx)) { + let task_result = res.map_err(BlockExecutionError::other)?; + + if let BackfillTaskOutput { result: Some(job_result), job } = task_result { + // If the task returned a non-empty result, a new task advancing the job is created + // and pushed to the front of the queue. + self.push_task(job); + + return Poll::Ready(Some(job_result)) + }; } + + Poll::Ready(None) } } @@ -73,27 +105,28 @@ where E: BlockExecutorProvider + Clone + Send + 'static, P: HeaderProvider + BlockReader + StateProviderFactory + Clone + Send + Unpin + 'static, { - type Item = Result; + type Item = BackfillJobResult; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); // Spawn new tasks only if we are below the parallelism configured. while this.tasks.len() < this.parallelism { - // If we have a block number, then we can spawn a new task for that block - if let Some(block_number) = this.range.next() { - let mut job = SingleBlockBackfillJob { - executor: this.executor.clone(), - provider: this.provider.clone(), - range: block_number..=block_number, - stream_parallelism: this.parallelism, - }; - let task = - tokio::task::spawn_blocking(move || job.next().expect("non-empty range")); - this.tasks.push_back(task); - } else { + // Get the next block number from the range. If it is empty, we are done. + let Some(block_number) = this.range.next() else { + debug!(target: "exex::backfill", tasks = %this.tasks.len(), range = ?this.range, "No more single blocks to backfill"); break; - } + }; + + // Spawn a new task for that block + debug!(target: "exex::backfill", tasks = %this.tasks.len(), ?block_number, "Spawning new single block backfill task"); + let job = Box::new(SingleBlockBackfillJob { + executor: this.executor.clone(), + provider: this.provider.clone(), + range: block_number..=block_number, + stream_parallelism: this.parallelism, + }) as BackfillTaskIterator<_>; + this.push_task(job); } this.poll_next_task(cx) @@ -105,7 +138,7 @@ where E: BlockExecutorProvider + Clone + Send + 'static, P: HeaderProvider + BlockReader + StateProviderFactory + Clone + Send + Unpin + 'static, { - type Item = Result; + type Item = BackfillJobResult; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = self.get_mut(); @@ -117,24 +150,23 @@ where let start = range.next(); let range_bounds = start.zip(range.last().or(start)); - // If we have range bounds, then we can spawn a new task for that range - if let Some((first, last)) = range_bounds { - let range = first..=last; - debug!(target: "exex::backfill", tasks = %this.tasks.len(), ?range, "Spawning new backfill task"); - let mut job = BackfillJob { - executor: this.executor.clone(), - provider: this.provider.clone(), - prune_modes: this.prune_modes.clone(), - thresholds: this.thresholds.clone(), - range, - stream_parallelism: this.parallelism, - }; - let task = - tokio::task::spawn_blocking(move || job.next().expect("non-empty range")); - this.tasks.push_back(task); - } else { + // Create the range from the range bounds. If it is empty, we are done. + let Some(range) = range_bounds.map(|(first, last)| first..=last) else { + debug!(target: "exex::backfill", tasks = %this.tasks.len(), range = ?this.range, "No more block batches to backfill"); break; - } + }; + + // Spawn a new task for that range + debug!(target: "exex::backfill", tasks = %this.tasks.len(), ?range, "Spawning new block batch backfill task"); + let job = Box::new(BackfillJob { + executor: this.executor.clone(), + provider: this.provider.clone(), + prune_modes: this.prune_modes.clone(), + thresholds: this.thresholds.clone(), + range, + stream_parallelism: this.parallelism, + }) as BackfillTaskIterator<_>; + this.push_task(job); } this.poll_next_task(cx) diff --git a/crates/exex/exex/src/notifications.rs b/crates/exex/exex/src/notifications.rs index 6efdb1775cf14..d0c94d34f6442 100644 --- a/crates/exex/exex/src/notifications.rs +++ b/crates/exex/exex/src/notifications.rs @@ -338,9 +338,11 @@ where } if let Some(backfill_job) = &mut this.backfill_job { - if let Some(chain) = ready!(backfill_job.poll_next_unpin(cx)) { + debug!(target: "exex::notifications", "Polling backfill job"); + if let Some(chain) = ready!(backfill_job.poll_next_unpin(cx)).transpose()? { + debug!(target: "exex::notifications", range = ?chain.range(), "Backfill job returned a chain"); return Poll::Ready(Some(Ok(ExExNotification::ChainCommitted { - new: Arc::new(chain?), + new: Arc::new(chain), }))) } From 47400d819e8b820b1da854c226571b50513fee51 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Tue, 8 Oct 2024 20:18:27 +0400 Subject: [PATCH 069/159] fix: in-memory trie updates pruning (#11580) Co-authored-by: Matthias Seitz --- crates/engine/tree/src/tree/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 47be69a3dee43..fd3f2df5c01b3 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -282,7 +282,7 @@ impl TreeState { } // remove trie updates that are below the finalized block - self.persisted_trie_updates.retain(|_, (block_num, _)| *block_num < finalized_num); + self.persisted_trie_updates.retain(|_, (block_num, _)| *block_num > finalized_num); // The only block that should remain at the `finalized` number now, is the finalized // block, if it exists. From bfcc00dca4fd81005f6ba6c04cf9b2df77946c3b Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 9 Oct 2024 02:10:23 +0900 Subject: [PATCH 070/159] chore(providers): test race condition on all `BlockchainProvider2` macro tests (#11574) --- .../src/providers/blockchain_provider.rs | 734 +++++++++--------- 1 file changed, 379 insertions(+), 355 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index d25c3f84bd1a8..54f28b77b9125 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -3116,66 +3116,96 @@ mod tests { } macro_rules! test_by_tx_range { - ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, [$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ - let db_tx_count = - $database_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; - let in_mem_tx_count = - $in_memory_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; + ([$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ - let db_range = 0..=(db_tx_count - 1); - let in_mem_range = db_tx_count..=(in_mem_tx_count + db_range.end()); + // Get the number methods being tested. + // Since each method tested will move a block from memory to storage, this ensures we have enough. + let extra_blocks = [$(stringify!($method)),*].len(); + + let mut rng = generators::rng(); + let (provider, mut database_blocks, mut in_memory_blocks, receipts) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT, + TEST_BLOCKS_COUNT + extra_blocks, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; + let provider = Arc::new(provider); $( + // Since data moves for each tried method, need to recalculate everything + let db_tx_count = + database_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; + let in_mem_tx_count = + in_memory_blocks.iter().map(|b| b.body.transactions.len()).sum::() as u64; + + let db_range = 0..=(db_tx_count - 1); + let in_mem_range = db_tx_count..=(in_mem_tx_count + db_range.end()); + // Retrieve the expected database data let database_data = - $database_blocks.iter().flat_map(|b| $data_extractor(b)).collect::>(); - assert_eq!($provider.$method(db_range.clone())?, database_data); + database_blocks.iter().flat_map(|b| $data_extractor(b, &receipts)).collect::>(); + assert_eq!(provider.$method(db_range.clone())?, database_data, "full db data"); // Retrieve the expected in-memory data let in_memory_data = - $in_memory_blocks.iter().flat_map(|b| $data_extractor(b)).collect::>(); - assert_eq!($provider.$method(in_mem_range.clone())?, in_memory_data); + in_memory_blocks.iter().flat_map(|b| $data_extractor(b, &receipts)).collect::>(); + assert_eq!(provider.$method(in_mem_range.clone())?, in_memory_data, "full mem data"); // Test partial in-memory range assert_eq!( - &$provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, - &in_memory_data[1..in_memory_data.len() - 1] + &provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, + &in_memory_data[1..in_memory_data.len() - 1], + "partial mem data" ); // Test range in in-memory to unbounded end - assert_eq!($provider.$method(in_mem_range.start() + 1..)?, &in_memory_data[1..]); + assert_eq!(provider.$method(in_mem_range.start() + 1..)?, &in_memory_data[1..], "unbounded mem data"); // Test last element in-memory - assert_eq!($provider.$method(in_mem_range.end()..)?, &in_memory_data[in_memory_data.len() -1 ..]); - - // Test range that spans database and in-memory - assert_eq!( - $provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, - database_data[database_data.len() - 2..] - .iter() - .chain(&in_memory_data[..in_memory_data.len() - 1]) - .cloned() - .collect::>() - ); + assert_eq!(provider.$method(in_mem_range.end()..)?, &in_memory_data[in_memory_data.len() -1 ..], "last mem data"); // Test range that spans database and in-memory with unbounded end assert_eq!( - $provider.$method(in_mem_range.start() - 2..)?, + provider.$method(in_mem_range.start() - 2..)?, database_data[database_data.len() - 2..] .iter() .chain(&in_memory_data[..]) .cloned() - .collect::>() + .collect::>(), + "unbounded span data" ); + // Test range that spans database and in-memory + #[allow(unused_assignments)] + { + // This block will be persisted to disk and removed from memory AFTER the firsk database query. This ensures that we query the in-memory state before the database avoiding any race condition. + persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[0].number); + + assert_eq!( + provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, + database_data[database_data.len() - 2..] + .iter() + .chain(&in_memory_data[..in_memory_data.len() - 1]) + .cloned() + .collect::>(), + "span data" + ); + + // Adjust our blocks accordingly + database_blocks.push(in_memory_blocks.remove(0)); + } + // Test invalid range let start_tx_num = u64::MAX; let end_tx_num = u64::MAX; - let result = $provider.$method(start_tx_num..end_tx_num)?; + let result = provider.$method(start_tx_num..end_tx_num)?; assert!(result.is_empty(), "No data should be found for an invalid transaction range"); // Test empty range - let result = $provider.$method(in_mem_range.end()+10..in_mem_range.end()+20)?; + let result = provider.$method(in_mem_range.end()+10..in_mem_range.end()+20)?; assert!(result.is_empty(), "No data should be found for an empty transaction range"); )* }}; @@ -3183,80 +3213,93 @@ mod tests { #[test] fn test_methods_by_tx_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - test_by_tx_range!( - provider, - database_blocks, - in_memory_blocks, - [ - (senders_by_tx_range, |block: &SealedBlock| block.senders().unwrap()), - (transactions_by_tx_range, |block: &SealedBlock| block - .body - .transactions - .iter() - .map(|tx| Into::::into(tx.clone())) - .collect::>()), - (receipts_by_tx_range, |block: &SealedBlock| receipts[block.number as usize] - .clone()) - ] - ); + test_by_tx_range!([ + (senders_by_tx_range, |block: &SealedBlock, _: &Vec>| block + .senders() + .unwrap()), + (transactions_by_tx_range, |block: &SealedBlock, _: &Vec>| block + .body + .transactions + .iter() + .map(|tx| Into::::into(tx.clone())) + .collect::>()), + (receipts_by_tx_range, |block: &SealedBlock, receipts: &Vec>| receipts + [block.number as usize] + .clone()) + ]); Ok(()) } macro_rules! test_by_block_range { - ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, [$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ - let db_block_count = $database_blocks.len() as u64; - let in_mem_block_count = $in_memory_blocks.len() as u64; + ([$(($method:ident, $data_extractor:expr)),* $(,)?]) => {{ + // Get the number methods being tested. + // Since each method tested will move a block from memory to storage, this ensures we have enough. + let extra_blocks = [$(stringify!($method)),*].len(); - let db_range = 0..=db_block_count - 1; - let in_mem_range = db_block_count..=(in_mem_block_count + db_range.end()); + let mut rng = generators::rng(); + let (provider, mut database_blocks, mut in_memory_blocks, _) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT, + TEST_BLOCKS_COUNT + extra_blocks, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; + let provider = Arc::new(provider); $( + // Since data moves for each tried method, need to recalculate everything + let db_block_count = database_blocks.len() as u64; + let in_mem_block_count = in_memory_blocks.len() as u64; + + let db_range = 0..=db_block_count - 1; + let in_mem_range = db_block_count..=(in_mem_block_count + db_range.end()); + // Retrieve the expected database data let database_data = - $database_blocks.iter().map(|b| $data_extractor(b)).collect::>(); - assert_eq!($provider.$method(db_range.clone())?, database_data); + database_blocks.iter().map(|b| $data_extractor(b)).collect::>(); + assert_eq!(provider.$method(db_range.clone())?, database_data); // Retrieve the expected in-memory data let in_memory_data = - $in_memory_blocks.iter().map(|b| $data_extractor(b)).collect::>(); - assert_eq!($provider.$method(in_mem_range.clone())?, in_memory_data); + in_memory_blocks.iter().map(|b| $data_extractor(b)).collect::>(); + assert_eq!(provider.$method(in_mem_range.clone())?, in_memory_data); // Test partial in-memory range assert_eq!( - &$provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, + &provider.$method(in_mem_range.start() + 1..=in_mem_range.end() - 1)?, &in_memory_data[1..in_memory_data.len() - 1] ); // Test range that spans database and in-memory - assert_eq!( - $provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, - database_data[database_data.len() - 2..] - .iter() - .chain(&in_memory_data[..in_memory_data.len() - 1]) - .cloned() - .collect::>() - ); + { + + // This block will be persisted to disk and removed from memory AFTER the firsk database query. This ensures that we query the in-memory state before the database avoiding any race condition. + persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[0].number); + + assert_eq!( + provider.$method(in_mem_range.start() - 2..=in_mem_range.end() - 1)?, + database_data[database_data.len() - 2..] + .iter() + .chain(&in_memory_data[..in_memory_data.len() - 1]) + .cloned() + .collect::>() + ); + + // Adjust our blocks accordingly + database_blocks.push(in_memory_blocks.remove(0)); + } // Test invalid range let start_block_num = u64::MAX; let end_block_num = u64::MAX; - let result = $provider.$method(start_block_num..=end_block_num-1)?; + let result = provider.$method(start_block_num..=end_block_num-1)?; assert!(result.is_empty(), "No data should be found for an invalid block range"); // Test valid range with empty results - let result = $provider.$method(in_mem_range.end() + 10..=in_mem_range.end() + 20)?; + let result = provider.$method(in_mem_range.end() + 10..=in_mem_range.end() + 20)?; assert!(result.is_empty(), "No data should be found for an empty block range"); )* }}; @@ -3264,40 +3307,21 @@ mod tests { #[test] fn test_methods_by_block_range() -> eyre::Result<()> { - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, _) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - // todo(joshie) add canonical_hashes_range below after changing its interface into range // instead start end - test_by_block_range!( - provider, - database_blocks, - in_memory_blocks, - [ - (headers_range, |block: &SealedBlock| block.header().clone()), - (sealed_headers_range, |block: &SealedBlock| block.header.clone()), - (block_range, |block: &SealedBlock| block.clone().unseal()), - (block_with_senders_range, |block: &SealedBlock| block - .clone() - .unseal() - .with_senders_unchecked(vec![])), - (sealed_block_with_senders_range, |block: &SealedBlock| block - .clone() - .with_senders_unchecked(vec![])), - (transactions_by_block_range, |block: &SealedBlock| block - .body - .transactions - .clone()), - ] - ); + test_by_block_range!([ + (headers_range, |block: &SealedBlock| block.header().clone()), + (sealed_headers_range, |block: &SealedBlock| block.header.clone()), + (block_range, |block: &SealedBlock| block.clone().unseal()), + (block_with_senders_range, |block: &SealedBlock| block + .clone() + .unseal() + .with_senders_unchecked(vec![])), + (sealed_block_with_senders_range, |block: &SealedBlock| block + .clone() + .with_senders_unchecked(vec![])), + (transactions_by_block_range, |block: &SealedBlock| block.body.transactions.clone()), + ]); Ok(()) } @@ -3333,39 +3357,72 @@ mod tests { /// ( `NUMBER_ARGUMENTS`, METHOD, FN -> ((`METHOD_ARGUMENT(s)`,...), `EXPECTED_RESULT`), /// `INVALID_ARGUMENTS`) macro_rules! test_non_range { - ($provider:expr, $database_blocks:expr, $in_memory_blocks:expr, $receipts:expr, [$(($arg_count:ident, $method:ident, $item_extractor:expr, $invalid_args:expr)),* $(,)?]) => {{ + ([$(($arg_count:ident, $method:ident, $item_extractor:expr, $invalid_args:expr)),* $(,)?]) => {{ + + // Get the number methods being tested. + // Since each method tested will move a block from memory to storage, this ensures we have enough. + let extra_blocks = [$(stringify!($arg_count)),*].len(); + + let mut rng = generators::rng(); + let (provider, mut database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT, + TEST_BLOCKS_COUNT + extra_blocks, + BlockRangeParams { + tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, + ..Default::default() + }, + )?; + let provider = Arc::new(provider); + + let mut in_memory_blocks: std::collections::VecDeque<_> = in_memory_blocks.into(); + $( let tx_hash = |block: &SealedBlock| block.body.transactions[0].hash(); let tx_num = |block: &SealedBlock| { - $database_blocks + database_blocks .iter() - .chain($in_memory_blocks.iter()) + .chain(in_memory_blocks.iter()) .take_while(|b| b.number < block.number) .map(|b| b.body.transactions.len()) .sum::() as u64 }; // Ensure that the first generated in-memory block exists - // In the future this block will be persisted to disk and removed from memory AFTER the firsk database query. - // This ensures that we query the in-memory state before the database avoiding any race condition. { - call_method!($arg_count, $provider, $method, $item_extractor, tx_num, tx_hash, &$in_memory_blocks[0], &$receipts); + // This block will be persisted to disk and removed from memory AFTER the firsk database query. This ensures that we query the in-memory state before the database avoiding any race condition. + persist_block_after_db_tx_creation(provider.clone(), in_memory_blocks[0].number); + + call_method!($arg_count, provider, $method, $item_extractor, tx_num, tx_hash, &in_memory_blocks[0], &receipts); + + // Move the block as well in our own structures + database_blocks.push(in_memory_blocks.pop_front().unwrap()); } + // database_blocks is changed above + let tx_num = |block: &SealedBlock| { + database_blocks + .iter() + .chain(in_memory_blocks.iter()) + .take_while(|b| b.number < block.number) + .map(|b| b.body.transactions.len()) + .sum::() as u64 + }; + // Invalid/Non-existent argument should return `None` { - call_method!($arg_count, $provider, $method, |_,_,_,_| ( ($invalid_args, None)), tx_num, tx_hash, &$in_memory_blocks[0], &$receipts); + call_method!($arg_count, provider, $method, |_,_,_,_| ( ($invalid_args, None)), tx_num, tx_hash, &in_memory_blocks[0], &receipts); } // Check that the item is only in memory and not in database { - let last_mem_block = &$in_memory_blocks[$in_memory_blocks.len() - 1]; + let last_mem_block = &in_memory_blocks[in_memory_blocks.len() - 1]; - let (args, expected_item) = $item_extractor(last_mem_block, tx_num(last_mem_block), tx_hash(last_mem_block), &$receipts); - call_method!($arg_count, $provider, $method, |_,_,_,_| (args.clone(), expected_item), tx_num, tx_hash, last_mem_block, &$receipts); + let (args, expected_item) = $item_extractor(last_mem_block, tx_num(last_mem_block), tx_hash(last_mem_block), &receipts); + call_method!($arg_count, provider, $method, |_,_,_,_| (args.clone(), expected_item), tx_num, tx_hash, last_mem_block, &receipts); // Ensure the item is not in storage - call_method!($arg_count, $provider.database, $method, |_,_,_,_| ( (args, None)), tx_num, tx_hash, last_mem_block, &$receipts); + call_method!($arg_count, provider.database, $method, |_,_,_,_| ( (args, None)), tx_num, tx_hash, last_mem_block, &receipts); } )* }}; @@ -3375,265 +3432,232 @@ mod tests { fn test_non_range_methods() -> eyre::Result<()> { let test_tx_index = 0; - let mut rng = generators::rng(); - let (provider, database_blocks, in_memory_blocks, receipts) = provider_with_random_blocks( - &mut rng, - TEST_BLOCKS_COUNT, - TEST_BLOCKS_COUNT, - BlockRangeParams { - tx_count: TEST_TRANSACTIONS_COUNT..TEST_TRANSACTIONS_COUNT, - ..Default::default() - }, - )?; - - test_non_range!( - provider, - database_blocks, - in_memory_blocks, - receipts, - [ - // TODO: header should use B256 like others instead of &B256 - // ( - // ONE, - // header, - // |block: &SealedBlock, tx_num: TxNumber, tx_hash: B256, receipts: &Vec>| (&block.hash(), Some(block.header.header().clone())), - // (&B256::random()) - // ), - ( - ONE, - header_by_number, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - block.number, - Some(block.header.header().clone()) - ), - u64::MAX + test_non_range!([ + // TODO: header should use B256 like others instead of &B256 + // ( + // ONE, + // header, + // |block: &SealedBlock, tx_num: TxNumber, tx_hash: B256, receipts: &Vec>| (&block.hash(), Some(block.header.header().clone())), + // (&B256::random()) + // ), + ( + ONE, + header_by_number, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.header.header().clone()) ), - ( - ONE, - sealed_header, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - block.number, - Some(block.header.clone()) - ), - u64::MAX + u64::MAX + ), + ( + ONE, + sealed_header, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.header.clone()) ), - ( - ONE, - block_hash, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - block.number, - Some(block.hash()) - ), - u64::MAX + u64::MAX + ), + ( + ONE, + block_hash, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(block.hash()) ), - ( - ONE, - block_number, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - block.hash(), - Some(block.number) - ), - B256::random() + u64::MAX + ), + ( + ONE, + block_number, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + block.hash(), + Some(block.number) ), - ( - ONE, - block, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - BlockHashOrNumber::Hash(block.hash()), - Some(block.clone().unseal()) - ), - BlockHashOrNumber::Hash(B256::random()) + B256::random() + ), + ( + ONE, + block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(block.clone().unseal()) ), - ( - ONE, - block, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - BlockHashOrNumber::Number(block.number), - Some(block.clone().unseal()) - ), - BlockHashOrNumber::Number(u64::MAX) + BlockHashOrNumber::Hash(B256::random()) + ), + ( + ONE, + block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(block.clone().unseal()) ), - ( - ONE, - block_body_indices, - |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( - block.number, - Some(StoredBlockBodyIndices { - first_tx_num: tx_num, - tx_count: block.body.transactions.len() as u64 - }) - ), - u64::MAX + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + block_body_indices, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + block.number, + Some(StoredBlockBodyIndices { + first_tx_num: tx_num, + tx_count: block.body.transactions.len() as u64 + }) ), - ( - TWO, - block_with_senders, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), - block.clone().unseal().with_recovered_senders() - ), - (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + u64::MAX + ), + ( + TWO, + block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), + block.clone().unseal().with_recovered_senders() ), - ( - TWO, - block_with_senders, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), - block.clone().unseal().with_recovered_senders() - ), - (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + ), + ( + TWO, + block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), + block.clone().unseal().with_recovered_senders() ), - ( - TWO, - sealed_block_with_senders, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), - Some( - block - .clone() - .unseal() - .with_recovered_senders() - .unwrap() - .seal(block.hash()) - ) - ), - (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + ), + ( + TWO, + sealed_block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Number(block.number), TransactionVariant::WithHash), + Some( + block.clone().unseal().with_recovered_senders().unwrap().seal(block.hash()) + ) ), - ( - TWO, - sealed_block_with_senders, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), - Some( - block - .clone() - .unseal() - .with_recovered_senders() - .unwrap() - .seal(block.hash()) - ) - ), - (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + (BlockHashOrNumber::Number(u64::MAX), TransactionVariant::WithHash) + ), + ( + TWO, + sealed_block_with_senders, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + (BlockHashOrNumber::Hash(block.hash()), TransactionVariant::WithHash), + Some( + block.clone().unseal().with_recovered_senders().unwrap().seal(block.hash()) + ) ), - ( - ONE, - transaction_id, - |_: &SealedBlock, tx_num: TxNumber, tx_hash: B256, _: &Vec>| ( - tx_hash, - Some(tx_num) - ), - B256::random() + (BlockHashOrNumber::Hash(B256::random()), TransactionVariant::WithHash) + ), + ( + ONE, + transaction_id, + |_: &SealedBlock, tx_num: TxNumber, tx_hash: B256, _: &Vec>| ( + tx_hash, + Some(tx_num) ), - ( - ONE, - transaction_by_id, - |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( - tx_num, - Some(block.body.transactions[test_tx_index].clone()) - ), - u64::MAX + B256::random() + ), + ( + ONE, + transaction_by_id, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(block.body.transactions[test_tx_index].clone()) ), - ( - ONE, - transaction_by_id_no_hash, - |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( - tx_num, - Some(Into::::into( - block.body.transactions[test_tx_index].clone() - )) - ), - u64::MAX + u64::MAX + ), + ( + ONE, + transaction_by_id_no_hash, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(Into::::into( + block.body.transactions[test_tx_index].clone() + )) ), - ( - ONE, - transaction_by_hash, - |block: &SealedBlock, _: TxNumber, tx_hash: B256, _: &Vec>| ( - tx_hash, - Some(block.body.transactions[test_tx_index].clone()) - ), - B256::random() + u64::MAX + ), + ( + ONE, + transaction_by_hash, + |block: &SealedBlock, _: TxNumber, tx_hash: B256, _: &Vec>| ( + tx_hash, + Some(block.body.transactions[test_tx_index].clone()) ), - ( - ONE, - transaction_block, - |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( - tx_num, - Some(block.number) - ), - u64::MAX + B256::random() + ), + ( + ONE, + transaction_block, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + Some(block.number) ), - ( - ONE, - transactions_by_block, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - BlockHashOrNumber::Number(block.number), - Some(block.body.transactions.clone()) - ), - BlockHashOrNumber::Number(u64::MAX) + u64::MAX + ), + ( + ONE, + transactions_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(block.body.transactions.clone()) ), - ( - ONE, - transactions_by_block, - |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( - BlockHashOrNumber::Hash(block.hash()), - Some(block.body.transactions.clone()) - ), - BlockHashOrNumber::Number(u64::MAX) + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + transactions_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, _: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(block.body.transactions.clone()) ), - ( - ONE, - transaction_sender, - |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( - tx_num, - block.body.transactions[test_tx_index].recover_signer() - ), - u64::MAX + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + transaction_sender, + |block: &SealedBlock, tx_num: TxNumber, _: B256, _: &Vec>| ( + tx_num, + block.body.transactions[test_tx_index].recover_signer() ), - ( - ONE, - receipt, - |block: &SealedBlock, - tx_num: TxNumber, - _: B256, - receipts: &Vec>| ( - tx_num, - Some(receipts[block.number as usize][test_tx_index].clone()) - ), - u64::MAX + u64::MAX + ), + ( + ONE, + receipt, + |block: &SealedBlock, tx_num: TxNumber, _: B256, receipts: &Vec>| ( + tx_num, + Some(receipts[block.number as usize][test_tx_index].clone()) ), - ( - ONE, - receipt_by_hash, - |block: &SealedBlock, - _: TxNumber, - tx_hash: B256, - receipts: &Vec>| ( - tx_hash, - Some(receipts[block.number as usize][test_tx_index].clone()) - ), - B256::random() + u64::MAX + ), + ( + ONE, + receipt_by_hash, + |block: &SealedBlock, _: TxNumber, tx_hash: B256, receipts: &Vec>| ( + tx_hash, + Some(receipts[block.number as usize][test_tx_index].clone()) ), - ( - ONE, - receipts_by_block, - |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( - BlockHashOrNumber::Number(block.number), - Some(receipts[block.number as usize].clone()) - ), - BlockHashOrNumber::Number(u64::MAX) + B256::random() + ), + ( + ONE, + receipts_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( + BlockHashOrNumber::Number(block.number), + Some(receipts[block.number as usize].clone()) ), - ( - ONE, - receipts_by_block, - |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( - BlockHashOrNumber::Hash(block.hash()), - Some(receipts[block.number as usize].clone()) - ), - BlockHashOrNumber::Hash(B256::random()) + BlockHashOrNumber::Number(u64::MAX) + ), + ( + ONE, + receipts_by_block, + |block: &SealedBlock, _: TxNumber, _: B256, receipts: &Vec>| ( + BlockHashOrNumber::Hash(block.hash()), + Some(receipts[block.number as usize].clone()) ), - // TODO: withdrawals, requests, ommers - ] - ); + BlockHashOrNumber::Hash(B256::random()) + ), + // TODO: withdrawals, requests, ommers + ]); Ok(()) } From 137a3f60385e40a3f2ac863519c1dabf54079561 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 20:31:37 +0200 Subject: [PATCH 071/159] chore: bump alloy primitives 0 8 7 (#11586) --- Cargo.lock | 22 +++++++++++++++++++--- Cargo.toml | 2 +- crates/primitives-traits/src/withdrawal.rs | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c135f294c58e..863f0f35c18d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "260d3ff3bff0bb84599f032a2f2c6828180b0ea0cd41fdaf44f39cef3ba41861" +checksum = "8ecb848c43f6b06ae3de2e4a67496cbbabd78ae87db0f1248934f15d76192c6a" dependencies = [ "alloy-rlp", "arbitrary", @@ -293,8 +293,9 @@ dependencies = [ "const-hex", "derive_arbitrary", "derive_more 1.0.0", + "foldhash", "getrandom 0.2.15", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "hex-literal", "indexmap 2.5.0", "itoa", @@ -3101,6 +3102,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -3433,6 +3440,15 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "foldhash", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index b6eed127119b9..5c772a844429e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -420,7 +420,7 @@ revm-primitives = { version = "10.0.0", features = [ # eth alloy-chains = "0.1.32" alloy-dyn-abi = "0.8.0" -alloy-primitives = { version = "0.8.4", default-features = false } +alloy-primitives = { version = "0.8.7", default-features = false } alloy-rlp = "0.3.4" alloy-sol-types = "0.8.0" alloy-trie = { version = "0.6", default-features = false } diff --git a/crates/primitives-traits/src/withdrawal.rs b/crates/primitives-traits/src/withdrawal.rs index f6b0607e7f032..995e60292c6e1 100644 --- a/crates/primitives-traits/src/withdrawal.rs +++ b/crates/primitives-traits/src/withdrawal.rs @@ -133,7 +133,7 @@ mod tests { // #[test] fn test_withdrawal_serde_roundtrip() { - let input = r#"[{"index":"0x0","validatorIndex":"0x0","address":"0x0000000000000000000000000000000000001000","amount":"0x1"},{"index":"0x1","validatorIndex":"0x1","address":"0x0000000000000000000000000000000000001001","amount":"0x1"},{"index":"0x2","validatorIndex":"0x2","address":"0x0000000000000000000000000000000000001002","amount":"0x1"},{"index":"0x3","validatorIndex":"0x3","address":"0x0000000000000000000000000000000000001003","amount":"0x1"},{"index":"0x4","validatorIndex":"0x4","address":"0x0000000000000000000000000000000000001004","amount":"0x1"},{"index":"0x5","validatorIndex":"0x5","address":"0x0000000000000000000000000000000000001005","amount":"0x1"},{"index":"0x6","validatorIndex":"0x6","address":"0x0000000000000000000000000000000000001006","amount":"0x1"},{"index":"0x7","validatorIndex":"0x7","address":"0x0000000000000000000000000000000000001007","amount":"0x1"},{"index":"0x8","validatorIndex":"0x8","address":"0x0000000000000000000000000000000000001008","amount":"0x1"},{"index":"0x9","validatorIndex":"0x9","address":"0x0000000000000000000000000000000000001009","amount":"0x1"},{"index":"0xa","validatorIndex":"0xa","address":"0x000000000000000000000000000000000000100A","amount":"0x1"},{"index":"0xb","validatorIndex":"0xb","address":"0x000000000000000000000000000000000000100b","amount":"0x1"},{"index":"0xc","validatorIndex":"0xc","address":"0x000000000000000000000000000000000000100C","amount":"0x1"},{"index":"0xd","validatorIndex":"0xd","address":"0x000000000000000000000000000000000000100D","amount":"0x1"},{"index":"0xe","validatorIndex":"0xe","address":"0x000000000000000000000000000000000000100e","amount":"0x1"},{"index":"0xf","validatorIndex":"0xf","address":"0x000000000000000000000000000000000000100f","amount":"0x1"}]"#; + let input = r#"[{"index":"0x0","validatorIndex":"0x0","address":"0x0000000000000000000000000000000000001000","amount":"0x1"},{"index":"0x1","validatorIndex":"0x1","address":"0x0000000000000000000000000000000000001001","amount":"0x1"},{"index":"0x2","validatorIndex":"0x2","address":"0x0000000000000000000000000000000000001002","amount":"0x1"},{"index":"0x3","validatorIndex":"0x3","address":"0x0000000000000000000000000000000000001003","amount":"0x1"},{"index":"0x4","validatorIndex":"0x4","address":"0x0000000000000000000000000000000000001004","amount":"0x1"},{"index":"0x5","validatorIndex":"0x5","address":"0x0000000000000000000000000000000000001005","amount":"0x1"},{"index":"0x6","validatorIndex":"0x6","address":"0x0000000000000000000000000000000000001006","amount":"0x1"},{"index":"0x7","validatorIndex":"0x7","address":"0x0000000000000000000000000000000000001007","amount":"0x1"},{"index":"0x8","validatorIndex":"0x8","address":"0x0000000000000000000000000000000000001008","amount":"0x1"},{"index":"0x9","validatorIndex":"0x9","address":"0x0000000000000000000000000000000000001009","amount":"0x1"},{"index":"0xa","validatorIndex":"0xa","address":"0x000000000000000000000000000000000000100a","amount":"0x1"},{"index":"0xb","validatorIndex":"0xb","address":"0x000000000000000000000000000000000000100b","amount":"0x1"},{"index":"0xc","validatorIndex":"0xc","address":"0x000000000000000000000000000000000000100c","amount":"0x1"},{"index":"0xd","validatorIndex":"0xd","address":"0x000000000000000000000000000000000000100d","amount":"0x1"},{"index":"0xe","validatorIndex":"0xe","address":"0x000000000000000000000000000000000000100e","amount":"0x1"},{"index":"0xf","validatorIndex":"0xf","address":"0x000000000000000000000000000000000000100f","amount":"0x1"}]"#; let withdrawals: Vec = serde_json::from_str(input).unwrap(); let s = serde_json::to_string(&withdrawals).unwrap(); From 75c3c2e1d746d3cbfe8b4f23222d59987ec3578f Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Tue, 8 Oct 2024 20:31:55 +0200 Subject: [PATCH 072/159] chore(ci): remove expected failures related to checksummed addresses (#11589) --- .github/assets/hive/expected_failures.yaml | 17 ----------------- .../hive/expected_failures_experimental.yaml | 17 ----------------- 2 files changed, 34 deletions(-) diff --git a/.github/assets/hive/expected_failures.yaml b/.github/assets/hive/expected_failures.yaml index ddf1383ff4d9f..7a212a51dd5ef 100644 --- a/.github/assets/hive/expected_failures.yaml +++ b/.github/assets/hive/expected_failures.yaml @@ -20,23 +20,6 @@ rpc-compat: - eth_getBlockByNumber/get-latest (reth) - eth_getBlockByNumber/get-safe (reth) - - eth_createAccessList/create-al-contract-eip1559 (reth) - - eth_createAccessList/create-al-contract (reth) - - eth_getProof/get-account-proof-blockhash (reth) - - eth_getProof/get-account-proof-latest (reth) - - eth_getProof/get-account-proof-with-storage (reth) - - eth_getTransactionByBlockHashAndIndex/get-block-n (reth) - - eth_getTransactionByBlockNumberAndIndex/get-block-n (reth) - - eth_getTransactionByHash/get-access-list (reth) - - eth_getTransactionByHash/get-blob-tx (reth) - - eth_getTransactionByHash/get-dynamic-fee (reth) - - eth_getTransactionByHash/get-legacy-create (reth) - - eth_getTransactionByHash/get-legacy-input (reth) - - eth_getTransactionByHash/get-legacy-tx (reth) - - eth_getTransactionReceipt/get-legacy-contract (reth) - - eth_getTransactionReceipt/get-legacy-input (reth) - - eth_getTransactionReceipt/get-legacy-receipt (reth)' - # https://github.com/paradigmxyz/reth/issues/8732 engine-withdrawals: - Withdrawals Fork On Genesis (Paris) (reth) diff --git a/.github/assets/hive/expected_failures_experimental.yaml b/.github/assets/hive/expected_failures_experimental.yaml index 91fd1a88ab834..d4b3d2bcbd3c1 100644 --- a/.github/assets/hive/expected_failures_experimental.yaml +++ b/.github/assets/hive/expected_failures_experimental.yaml @@ -20,23 +20,6 @@ rpc-compat: - eth_getBlockByNumber/get-latest (reth) - eth_getBlockByNumber/get-safe (reth) - - eth_createAccessList/create-al-contract-eip1559 (reth) - - eth_createAccessList/create-al-contract (reth) - - eth_getProof/get-account-proof-blockhash (reth) - - eth_getProof/get-account-proof-latest (reth) - - eth_getProof/get-account-proof-with-storage (reth) - - eth_getTransactionByBlockHashAndIndex/get-block-n (reth) - - eth_getTransactionByBlockNumberAndIndex/get-block-n (reth) - - eth_getTransactionByHash/get-access-list (reth) - - eth_getTransactionByHash/get-blob-tx (reth) - - eth_getTransactionByHash/get-dynamic-fee (reth) - - eth_getTransactionByHash/get-legacy-create (reth) - - eth_getTransactionByHash/get-legacy-input (reth) - - eth_getTransactionByHash/get-legacy-tx (reth) - - eth_getTransactionReceipt/get-legacy-contract (reth) - - eth_getTransactionReceipt/get-legacy-input (reth) - - eth_getTransactionReceipt/get-legacy-receipt (reth)' - # https://github.com/paradigmxyz/reth/issues/8732 engine-withdrawals: - Withdrawals Fork On Genesis (Paris) (reth) From f8769f91a6a2633d47027a8aaeabef6790ed4140 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 9 Oct 2024 03:26:44 +0900 Subject: [PATCH 073/159] chore(rpc): use `block_hash` instead on fetching `debug_trace_block` block (#11587) --- crates/rpc/rpc/src/debug.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index a83ba68b42f5e..9fbde05bb400d 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -209,7 +209,7 @@ where let ((cfg, block_env, _), block) = futures::try_join!( self.eth_api().evm_env_at(block_hash.into()), - self.eth_api().block_with_senders(block_id), + self.eth_api().block_with_senders(block_hash.into()), )?; let block = block.ok_or(EthApiError::HeaderNotFound(block_id))?; From 8cd38c2ec134fd5f4b34504a27898c89e5af723b Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 20:30:30 +0200 Subject: [PATCH 074/159] chore: also derive arb for test (#11588) --- crates/primitives-traits/src/storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/primitives-traits/src/storage.rs b/crates/primitives-traits/src/storage.rs index 801edba088f4c..39b6155ee2841 100644 --- a/crates/primitives-traits/src/storage.rs +++ b/crates/primitives-traits/src/storage.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; /// /// `key` is the subkey when used as a value in the `StorageChangeSets` table. #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Ord)] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] +#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))] #[add_arbitrary_tests(compact)] pub struct StorageEntry { /// Storage key. From 4c81f1411ca46b59772888c262d19e60fbb045e8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 8 Oct 2024 21:03:47 +0200 Subject: [PATCH 075/159] feat: add mul support for SubPoolLimit (#11591) Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> --- crates/transaction-pool/src/config.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/transaction-pool/src/config.rs b/crates/transaction-pool/src/config.rs index 67328c7f1be16..1b4b010a8e14f 100644 --- a/crates/transaction-pool/src/config.rs +++ b/crates/transaction-pool/src/config.rs @@ -7,7 +7,8 @@ use reth_primitives::{ constants::{ETHEREUM_BLOCK_GAS_LIMIT, MIN_PROTOCOL_BASE_FEE}, EIP4844_TX_TYPE_ID, }; -use std::collections::HashSet; +use std::{collections::HashSet, ops::Mul}; + /// Guarantees max transactions for one sender, compatible with geth/erigon pub const TXPOOL_MAX_ACCOUNT_SLOTS_PER_SENDER: usize = 16; @@ -107,6 +108,15 @@ impl SubPoolLimit { } } +impl Mul for SubPoolLimit { + type Output = Self; + + fn mul(self, rhs: usize) -> Self::Output { + let Self { max_txs, max_size } = self; + Self { max_txs: max_txs * rhs, max_size: max_size * rhs } + } +} + impl Default for SubPoolLimit { fn default() -> Self { // either 10k transactions or 20MB @@ -318,4 +328,14 @@ mod tests { let new_config = config.set_propagate_local_transactions(false); assert!(!new_config.propagate_local_transactions); } + + #[test] + fn scale_pool_limit() { + let limit = SubPoolLimit::default(); + let double = limit * 2; + assert_eq!( + double, + SubPoolLimit { max_txs: limit.max_txs * 2, max_size: limit.max_size * 2 } + ) + } } From e08e44d0a083e1a40e879168b7e110b908627ea9 Mon Sep 17 00:00:00 2001 From: John Date: Wed, 9 Oct 2024 01:27:08 +0600 Subject: [PATCH 076/159] docs: delete missing part path (#11590) Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> --- docs/repo/layout.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/repo/layout.md b/docs/repo/layout.md index 8eb0782b988c7..6ed91e79656f9 100644 --- a/docs/repo/layout.md +++ b/docs/repo/layout.md @@ -168,7 +168,7 @@ Small utility crates. - [`tasks`](../../crates/tasks): An executor-agnostic task abstraction, used to spawn tasks on different async executors. Supports blocking tasks and handles panics gracefully. A tokio implementation is provided by default. - [`metrics/common`](../../crates/metrics/src/common): Common metrics types (e.g. metered channels) -- [`metrics/metrics-derive`](../../crates/metrics/metrics-derive): A derive-style API for creating metrics +- [`metrics/metrics-derive`](https://github.com/rkrasiuk/metrics-derive): A derive-style API for creating metrics - [`tracing`](../../crates/tracing): A small utility crate to install a uniform [`tracing`][tracing] subscriber [libmdbx-rs]: https://crates.io/crates/libmdbx From b272c4118579e65d2a60d7be89d8a6d6d7a174c2 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Wed, 9 Oct 2024 06:04:16 +0400 Subject: [PATCH 077/159] fix: simplify reorg handling (#11592) --- crates/engine/tree/src/tree/mod.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index fd3f2df5c01b3..0478c73c90d28 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -899,16 +899,9 @@ where return Ok(None); } - if old_hash == current_hash { - // We've found the fork point - break; - } - if let Some(block) = self.executed_block_by_hash(current_hash)? { - if self.is_fork(block.block.hash())? { - current_hash = block.block.parent_hash; - new_chain.push(block); - } + current_hash = block.block.parent_hash; + new_chain.push(block); } else { // This shouldn't happen as we've already walked this path warn!(target: "engine::tree", invalid_hash=?current_hash, "New chain block not found in TreeState"); From 05910f2f270fc023bad6d1329ad625c59c378715 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 04:48:36 +0200 Subject: [PATCH 078/159] fix: use original bytes for codes (#11593) --- crates/rpc/rpc/src/debug.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 9fbde05bb400d..fb04070397768 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -619,7 +619,7 @@ where .cache .contracts .iter() - .map(|(hash, code)| (*hash, code.bytes())) + .map(|(hash, code)| (*hash, code.original_bytes())) .collect(); for (address, account) in &statedb.cache.accounts { From 24272b2ce25d63d884b81c7cec320b56b9811313 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 9 Oct 2024 17:33:29 +0900 Subject: [PATCH 079/159] perf(rpc): use `Arc>>>; /// The type that can send the response to a requested [`BlockWithSenders`] -type BlockWithSendersResponseSender = oneshot::Sender>>; +type BlockWithSendersResponseSender = + oneshot::Sender>>>; /// The type that can send the response to the requested receipts of a block. type ReceiptsResponseSender = oneshot::Sender>>>>; @@ -48,7 +49,7 @@ type EnvResponseSender = oneshot::Sender = MultiConsumerLruCache< B256, - BlockWithSenders, + Arc, L, Either, >; @@ -151,7 +152,7 @@ impl EthStateCache { rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)?; if let Ok(Some(block_with_senders)) = block_with_senders_res { - Ok(Some(block_with_senders.block)) + Ok(Some(block_with_senders.block.clone())) } else { Ok(None) } @@ -186,7 +187,7 @@ impl EthStateCache { Ok(self .get_block_with_senders(block_hash) .await? - .map(|block| block.into_transactions_ecrecovered().collect())) + .map(|block| (*block).clone().into_transactions_ecrecovered().collect())) } /// Fetches both transactions and receipts for the given block hash. @@ -208,7 +209,7 @@ impl EthStateCache { pub async fn get_block_with_senders( &self, block_hash: B256, - ) -> ProviderResult> { + ) -> ProviderResult>> { let (response_tx, rx) = oneshot::channel(); let _ = self.to_service.send(CacheAction::GetBlockWithSenders { block_hash, response_tx }); rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? @@ -221,7 +222,10 @@ impl EthStateCache { &self, block_hash: B256, ) -> ProviderResult> { - Ok(self.get_block_with_senders(block_hash).await?.map(|block| block.seal(block_hash))) + Ok(self + .get_block_with_senders(block_hash) + .await? + .map(|block| (*block).clone().seal(block_hash))) } /// Requests the [Receipt] for the block hash @@ -288,7 +292,7 @@ pub(crate) struct EthStateCacheService< LimitReceipts = ByLength, LimitEnvs = ByLength, > where - LimitBlocks: Limiter, + LimitBlocks: Limiter>, LimitReceipts: Limiter>>, LimitEnvs: Limiter, { @@ -318,7 +322,11 @@ where Tasks: TaskSpawner + Clone + 'static, EvmConfig: ConfigureEvm
, { - fn on_new_block(&mut self, block_hash: B256, res: ProviderResult>) { + fn on_new_block( + &mut self, + block_hash: B256, + res: ProviderResult>>, + ) { if let Some(queued) = self.full_block_cache.remove(&block_hash) { // send the response to queued senders for tx in queued { @@ -328,7 +336,7 @@ where } Either::Right(transaction_tx) => { let _ = transaction_tx.send(res.clone().map(|maybe_block| { - maybe_block.map(|block| block.block.body.transactions) + maybe_block.map(|block| block.block.body.transactions.clone()) })); } } @@ -360,6 +368,7 @@ where } fn on_reorg_block(&mut self, block_hash: B256, res: ProviderResult>) { + let res = res.map(|b| b.map(Arc::new)); if let Some(queued) = self.full_block_cache.remove(&block_hash) { // send the response to queued senders for tx in queued { @@ -369,7 +378,7 @@ where } Either::Right(transaction_tx) => { let _ = transaction_tx.send(res.clone().map(|maybe_block| { - maybe_block.map(|block| block.block.body.transactions) + maybe_block.map(|block| block.block.body.transactions.clone()) })); } } @@ -431,10 +440,12 @@ where let _permit = rate_limiter.acquire().await; // Only look in the database to prevent situations where we // looking up the tree is blocking - let block_sender = provider.block_with_senders( - BlockHashOrNumber::Hash(block_hash), - TransactionVariant::WithHash, - ); + let block_sender = provider + .block_with_senders( + BlockHashOrNumber::Hash(block_hash), + TransactionVariant::WithHash, + ) + .map(|maybe_block| maybe_block.map(Arc::new)); let _ = action_tx.send(CacheAction::BlockWithSendersResult { block_hash, res: block_sender, @@ -459,10 +470,12 @@ where let _permit = rate_limiter.acquire().await; // Only look in the database to prevent situations where we // looking up the tree is blocking - let res = provider.block_with_senders( - BlockHashOrNumber::Hash(block_hash), - TransactionVariant::WithHash, - ); + let res = provider + .block_with_senders( + BlockHashOrNumber::Hash(block_hash), + TransactionVariant::WithHash, + ) + .map(|b| b.map(Arc::new)); let _ = action_tx.send(CacheAction::BlockWithSendersResult { block_hash, res, @@ -561,7 +574,7 @@ where } CacheAction::CacheNewCanonicalChain { chain_change } => { for block in chain_change.blocks { - this.on_new_block(block.hash(), Ok(Some(block.unseal()))); + this.on_new_block(block.hash(), Ok(Some(Arc::new(block.unseal())))); } for block_receipts in chain_change.receipts { @@ -601,7 +614,7 @@ enum CacheAction { GetBlockTransactions { block_hash: B256, response_tx: BlockTransactionsResponseSender }, GetEnv { block_hash: B256, response_tx: EnvResponseSender }, GetReceipts { block_hash: B256, response_tx: ReceiptsResponseSender }, - BlockWithSendersResult { block_hash: B256, res: ProviderResult> }, + BlockWithSendersResult { block_hash: B256, res: ProviderResult>> }, ReceiptsResult { block_hash: B256, res: ProviderResult>>> }, EnvResult { block_hash: B256, res: Box> }, CacheNewCanonicalChain { chain_change: ChainChange }, From 6bbcd0af5f27a21a33788aeec4c8bfde1938dacd Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 12:03:25 +0200 Subject: [PATCH 080/159] fix: active inflight count (#11598) --- crates/net/network/src/transactions/fetcher.rs | 11 ++++++++--- crates/net/network/src/transactions/mod.rs | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index e15972df08b28..0b849b659101f 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -98,6 +98,11 @@ pub struct TransactionFetcher { // === impl TransactionFetcher === impl TransactionFetcher { + /// Removes the peer from the active set. + pub(crate) fn remove_peer(&mut self, peer_id: &PeerId) { + self.active_peers.remove(peer_id); + } + /// Updates metrics. #[inline] pub fn update_metrics(&self) { @@ -157,7 +162,7 @@ impl TransactionFetcher { fn decrement_inflight_request_count_for(&mut self, peer_id: &PeerId) { let remove = || -> bool { if let Some(inflight_count) = self.active_peers.get(peer_id) { - *inflight_count -= 1; + *inflight_count = inflight_count.saturating_sub(1); if *inflight_count == 0 { return true } @@ -659,8 +664,6 @@ impl TransactionFetcher { return Some(new_announced_hashes) } - *inflight_count += 1; - #[cfg(debug_assertions)] { for hash in &new_announced_hashes { @@ -695,6 +698,8 @@ impl TransactionFetcher { } } } + + *inflight_count += 1; // stores a new request future for the request self.inflight_requests.push(GetPooledTxRequestFut::new(peer_id, new_announced_hashes, rx)); diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 2fa4ccfbb606d..24cc8f61407dc 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -904,6 +904,7 @@ where NetworkEvent::SessionClosed { peer_id, .. } => { // remove the peer self.peers.remove(&peer_id); + self.transaction_fetcher.remove_peer(&peer_id); } NetworkEvent::SessionEstablished { peer_id, client_version, messages, version, .. From 1092b5182eb5717d181c14b1f233d7509408ca81 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Wed, 9 Oct 2024 12:32:18 +0200 Subject: [PATCH 081/159] fix(net): max inflight tx reqs default (#11602) --- crates/net/network/src/transactions/fetcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index 0b849b659101f..cf5c09045f288 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -1325,7 +1325,7 @@ impl TransactionFetcherInfo { impl Default for TransactionFetcherInfo { fn default() -> Self { Self::new( - DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS as usize * DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER as usize, + DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS as usize, DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, From fa7fc4fbbfe0c17f11ae4e3ab08a683ea5eba926 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 13:00:17 +0200 Subject: [PATCH 082/159] fix: set deposit gasprice correctly (#11603) --- crates/optimism/rpc/src/eth/transaction.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index 3ccda419cad81..a4b4c064970da 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -105,7 +105,11 @@ impl TransactionCompat for OpTxBuilder { fn fill(tx: TransactionSignedEcRecovered, tx_info: TransactionInfo) -> Self::Transaction { let signed_tx = tx.clone().into_signed(); - let inner = EthTxBuilder::fill(tx, tx_info).inner; + let mut inner = EthTxBuilder::fill(tx, tx_info).inner; + + if signed_tx.is_deposit() { + inner.gas_price = Some(signed_tx.max_fee_per_gas()) + } Transaction { inner, From fceca64037fd3db6fae6a025d69bdb0b9ca6ee45 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 13:12:28 +0200 Subject: [PATCH 083/159] fix: set system tx correctly (#11601) --- crates/optimism/rpc/src/eth/transaction.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/optimism/rpc/src/eth/transaction.rs b/crates/optimism/rpc/src/eth/transaction.rs index a4b4c064970da..ab7525016a1ac 100644 --- a/crates/optimism/rpc/src/eth/transaction.rs +++ b/crates/optimism/rpc/src/eth/transaction.rs @@ -116,7 +116,8 @@ impl TransactionCompat for OpTxBuilder { source_hash: signed_tx.source_hash(), mint: signed_tx.mint(), // only include is_system_tx if true: - is_system_tx: signed_tx.is_deposit().then_some(signed_tx.is_system_transaction()), + is_system_tx: (signed_tx.is_deposit() && signed_tx.is_system_transaction()) + .then_some(true), deposit_receipt_version: None, // todo: how to fill this field? } } From aa430266d42932f9571003290218ae6fa14801b4 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Wed, 9 Oct 2024 13:20:58 +0200 Subject: [PATCH 084/159] fix(trie): prefix set extension (#11605) --- crates/trie/trie/src/prefix_set.rs | 21 +++++++++++++++++---- crates/trie/trie/src/proof.rs | 4 ++-- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/crates/trie/trie/src/prefix_set.rs b/crates/trie/trie/src/prefix_set.rs index 4997228050a3a..af0fb173d98a5 100644 --- a/crates/trie/trie/src/prefix_set.rs +++ b/crates/trie/trie/src/prefix_set.rs @@ -20,9 +20,9 @@ pub struct TriePrefixSetsMut { impl TriePrefixSetsMut { /// Extends prefix sets with contents of another prefix set. pub fn extend(&mut self, other: Self) { - self.account_prefix_set.extend(other.account_prefix_set.keys); + self.account_prefix_set.extend(other.account_prefix_set); for (hashed_address, prefix_set) in other.storage_prefix_sets { - self.storage_prefix_sets.entry(hashed_address).or_default().extend(prefix_set.keys); + self.storage_prefix_sets.entry(hashed_address).or_default().extend(prefix_set); } self.destroyed_accounts.extend(other.destroyed_accounts); } @@ -115,12 +115,18 @@ impl PrefixSetMut { self.keys.push(nibbles); } + /// Extend prefix set with contents of another prefix set. + pub fn extend(&mut self, other: Self) { + self.all |= other.all; + self.keys.extend(other.keys); + } + /// Extend prefix set keys with contents of provided iterator. - pub fn extend(&mut self, nibbles_iter: I) + pub fn extend_keys(&mut self, keys: I) where I: IntoIterator, { - self.keys.extend(nibbles_iter); + self.keys.extend(keys); } /// Returns the number of elements in the set. @@ -270,4 +276,11 @@ mod tests { assert_eq!(prefix_set.keys.len(), 3); // Length should be 3 (excluding duplicate) assert_eq!(prefix_set.keys.capacity(), 3); // Capacity should be 3 after shrinking } + + #[test] + fn test_prefix_set_all_extend() { + let mut prefix_set_mut = PrefixSetMut::default(); + prefix_set_mut.extend(PrefixSetMut::all()); + assert!(prefix_set_mut.all); + } } diff --git a/crates/trie/trie/src/proof.rs b/crates/trie/trie/src/proof.rs index bff681a3498df..d31d63fd9a8be 100644 --- a/crates/trie/trie/src/proof.rs +++ b/crates/trie/trie/src/proof.rs @@ -96,7 +96,7 @@ where // Create the walker. let mut prefix_set = self.prefix_sets.account_prefix_set.clone(); - prefix_set.extend(targets.keys().map(Nibbles::unpack)); + prefix_set.extend_keys(targets.keys().map(Nibbles::unpack)); let walker = TrieWalker::new(trie_cursor, prefix_set.freeze()); // Create a hash builder to rebuild the root node since it is not available in the database. @@ -225,7 +225,7 @@ where } let target_nibbles = targets.into_iter().map(Nibbles::unpack).collect::>(); - self.prefix_set.extend(target_nibbles.clone()); + self.prefix_set.extend_keys(target_nibbles.clone()); let trie_cursor = self.trie_cursor_factory.storage_trie_cursor(self.hashed_address)?; let walker = TrieWalker::new(trie_cursor, self.prefix_set.freeze()); From e95084f601c41df5b3eaf7b2cc8a2f6aa41a28d1 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 13:42:55 +0200 Subject: [PATCH 085/159] feat: add tx propagation mode (#11594) --- crates/net/network/src/transactions/config.rs | 28 +++++++++++++++++++ crates/net/network/src/transactions/mod.rs | 16 +++++------ crates/node/core/src/args/network.rs | 1 + 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/crates/net/network/src/transactions/config.rs b/crates/net/network/src/transactions/config.rs index b8023ca7928c4..81ec293ea1ff9 100644 --- a/crates/net/network/src/transactions/config.rs +++ b/crates/net/network/src/transactions/config.rs @@ -18,6 +18,9 @@ pub struct TransactionsManagerConfig { pub transaction_fetcher_config: TransactionFetcherConfig, /// Max number of seen transactions to store for each peer. pub max_transactions_seen_by_peer_history: u32, + /// How new pending transactions are propagated. + #[cfg_attr(feature = "serde", serde(default))] + pub propagation_mode: TransactionPropagationMode, } impl Default for TransactionsManagerConfig { @@ -25,6 +28,31 @@ impl Default for TransactionsManagerConfig { Self { transaction_fetcher_config: TransactionFetcherConfig::default(), max_transactions_seen_by_peer_history: DEFAULT_MAX_COUNT_TRANSACTIONS_SEEN_BY_PEER, + propagation_mode: TransactionPropagationMode::default(), + } + } +} + +/// Determines how new pending transactions are propagated to other peers in full. +#[derive(Debug, Clone, Default)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum TransactionPropagationMode { + /// Send full transactions to sqrt of current peers. + #[default] + Sqrt, + /// Always send transactions in full. + All, + /// Send full transactions to a maximum number of peers + Max(usize), +} + +impl TransactionPropagationMode { + /// Returns the number of peers that should + pub(crate) fn full_peer_count(&self, peer_count: usize) -> usize { + match self { + Self::Sqrt => (peer_count as f64).sqrt().round() as usize, + Self::All => peer_count, + Self::Max(max) => peer_count.min(*max), } } } diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 24cc8f61407dc..0c488ff919dc8 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -12,7 +12,7 @@ pub use self::constants::{ tx_fetcher::DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, }; -pub use config::{TransactionFetcherConfig, TransactionsManagerConfig}; +pub use config::{TransactionFetcherConfig, TransactionPropagationMode, TransactionsManagerConfig}; pub use validation::*; pub(crate) use fetcher::{FetchEvent, TransactionFetcher}; @@ -246,8 +246,8 @@ pub struct TransactionsManager { pending_transactions: ReceiverStream, /// Incoming events from the [`NetworkManager`](crate::NetworkManager). transaction_events: UnboundedMeteredReceiver, - /// Max number of seen transactions to store for each peer. - max_transactions_seen_by_peer_history: u32, + /// How the `TransactionsManager` is configured. + config: TransactionsManagerConfig, /// `TransactionsManager` metrics metrics: TransactionsManagerMetrics, } @@ -298,8 +298,7 @@ impl TransactionsManager { from_network, NETWORK_POOL_TRANSACTIONS_SCOPE, ), - max_transactions_seen_by_peer_history: transactions_manager_config - .max_transactions_seen_by_peer_history, + config: transactions_manager_config, metrics, } } @@ -424,9 +423,8 @@ where return propagated } - // send full transactions to a fraction of the connected peers (square root of the total - // number of connected peers) - let max_num_full = (self.peers.len() as f64).sqrt().round() as usize; + // send full transactions to a set of the connected peers based on the configured mode + let max_num_full = self.config.propagation_mode.full_peer_count(self.peers.len()); // Note: Assuming ~random~ order due to random state of the peers map hasher for (peer_idx, (peer_id, peer)) in self.peers.iter_mut().enumerate() { @@ -914,7 +912,7 @@ where messages, version, client_version, - self.max_transactions_seen_by_peer_history, + self.config.max_transactions_seen_by_peer_history, ); let peer = match self.peers.entry(peer_id) { Entry::Occupied(mut entry) => { diff --git a/crates/node/core/src/args/network.rs b/crates/node/core/src/args/network.rs index 0f1465bc5795b..04153b93ecdd4 100644 --- a/crates/node/core/src/args/network.rs +++ b/crates/node/core/src/args/network.rs @@ -226,6 +226,7 @@ impl NetworkArgs { self.max_capacity_cache_txns_pending_fetch, ), max_transactions_seen_by_peer_history: self.max_seen_tx_history, + propagation_mode: Default::default(), }; // Configure basic network stack From 441d88d485add9cbcf61e24c760d0b4683914b0b Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 14:12:54 +0200 Subject: [PATCH 086/159] feat: add helper function to provde the tx manager config (#11608) --- crates/node/builder/src/builder/mod.rs | 40 +++++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index d2ce2a1d8e2f6..8b57781ea97c3 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -9,6 +9,13 @@ pub use states::*; use std::sync::Arc; +use crate::{ + common::WithConfigs, + components::NodeComponentsBuilder, + node::FullNode, + rpc::{EthApiBuilderProvider, RethRpcServerHandles, RpcContext}, + DefaultNodeLauncher, LaunchNode, Node, NodeHandle, +}; use futures::Future; use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks}; use reth_cli_util::get_secret_key; @@ -18,7 +25,8 @@ use reth_db_api::{ }; use reth_exex::ExExContext; use reth_network::{ - NetworkBuilder, NetworkConfig, NetworkConfigBuilder, NetworkHandle, NetworkManager, + transactions::TransactionsManagerConfig, NetworkBuilder, NetworkConfig, NetworkConfigBuilder, + NetworkHandle, NetworkManager, }; use reth_node_api::{ FullNodeTypes, FullNodeTypesAdapter, NodeAddOns, NodeTypes, NodeTypesWithDBAdapter, @@ -38,14 +46,6 @@ use reth_transaction_pool::{PoolConfig, TransactionPool}; use secp256k1::SecretKey; use tracing::{info, trace, warn}; -use crate::{ - common::WithConfigs, - components::NodeComponentsBuilder, - node::FullNode, - rpc::{EthApiBuilderProvider, RethRpcServerHandles, RpcContext}, - DefaultNodeLauncher, LaunchNode, Node, NodeHandle, -}; - /// The adapter type for a reth node with the builtin provider type // Note: we need to hardcode this because custom components might depend on it in associated types. pub type RethFullAdapter = FullNodeTypesAdapter< @@ -583,16 +583,34 @@ impl BuilderContext { self.config().builder.clone() } - /// Convenience function to start the network. + /// Convenience function to start the network tasks. /// /// Spawns the configured network and associated tasks and returns the [`NetworkHandle`] /// connected to that network. pub fn start_network(&self, builder: NetworkBuilder<(), ()>, pool: Pool) -> NetworkHandle + where + Pool: TransactionPool + Unpin + 'static, + { + self.start_network_with(builder, pool, Default::default()) + } + + /// Convenience function to start the network tasks. + /// + /// Accepts the config for the transaction task. + /// + /// Spawns the configured network and associated tasks and returns the [`NetworkHandle`] + /// connected to that network. + pub fn start_network_with( + &self, + builder: NetworkBuilder<(), ()>, + pool: Pool, + tx_config: TransactionsManagerConfig, + ) -> NetworkHandle where Pool: TransactionPool + Unpin + 'static, { let (handle, network, txpool, eth) = builder - .transactions(pool, Default::default()) + .transactions(pool, tx_config) .request_handler(self.provider().clone()) .split_with_handle(); From 226b9422e064724529a4fb78688c38723e3418d5 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Wed, 9 Oct 2024 13:12:44 +0100 Subject: [PATCH 087/159] fix(grafana): set instance variable from `reth_info` metric (#11607) --- etc/grafana/dashboards/reth-performance.json | 28 +++++++++----------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/etc/grafana/dashboards/reth-performance.json b/etc/grafana/dashboards/reth-performance.json index e0ff5865dd599..02d890dceeff0 100644 --- a/etc/grafana/dashboards/reth-performance.json +++ b/etc/grafana/dashboards/reth-performance.json @@ -15,7 +15,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "11.2.0" + "version": "11.1.0" }, { "type": "datasource", @@ -83,7 +83,6 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -137,9 +136,9 @@ "options": { "legend": { "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -202,7 +201,6 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -256,9 +254,9 @@ "options": { "legend": { "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -315,7 +313,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "definition": "label_values($instance)", + "definition": "query_result(reth_info)", "hide": 0, "includeAll": false, "label": "instance", @@ -323,12 +321,12 @@ "name": "instance", "options": [], "query": { - "qryType": 1, - "query": "label_values($instance)", + "qryType": 3, + "query": "query_result(reth_info)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, - "regex": "", + "regex": "/.*instance=\\\"([^\\\"]*).*/", "skipUrlSync": false, "sort": 0, "type": "query" @@ -341,8 +339,8 @@ }, "timepicker": {}, "timezone": "browser", - "title": "Reth Performance", + "title": "Reth - Performance", "uid": "bdywb3xjphfy8a", "version": 2, "weekStart": "" -} \ No newline at end of file +} From 861c7feb6e8381ea0d74c65ec3e749954b3e1162 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Wed, 9 Oct 2024 14:32:16 +0200 Subject: [PATCH 088/159] fix(net): add concurrency param from config to `TransactionFetcherInfo` (#11600) Co-authored-by: Matthias Seitz --- .../net/network/src/transactions/fetcher.rs | 79 +++++++++++-------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index cf5c09045f288..9276219d593b7 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -130,20 +130,27 @@ impl TransactionFetcher { /// Sets up transaction fetcher with config pub fn with_transaction_fetcher_config(config: &TransactionFetcherConfig) -> Self { - let mut tx_fetcher = Self::default(); + let TransactionFetcherConfig { + max_inflight_requests, + max_capacity_cache_txns_pending_fetch, + .. + } = *config; - tx_fetcher.info.soft_limit_byte_size_pooled_transactions_response = - config.soft_limit_byte_size_pooled_transactions_response; - tx_fetcher.info.soft_limit_byte_size_pooled_transactions_response_on_pack_request = - config.soft_limit_byte_size_pooled_transactions_response_on_pack_request; - tx_fetcher - .metrics - .capacity_inflight_requests - .increment(tx_fetcher.info.max_inflight_requests as u64); - tx_fetcher.info.max_capacity_cache_txns_pending_fetch = - config.max_capacity_cache_txns_pending_fetch; + let info = config.clone().into(); - tx_fetcher + let metrics = TransactionFetcherMetrics::default(); + metrics.capacity_inflight_requests.increment(max_inflight_requests as u64); + + Self { + active_peers: LruMap::new(max_inflight_requests), + hashes_pending_fetch: LruCache::new(max_capacity_cache_txns_pending_fetch), + hashes_fetch_inflight_and_pending_fetch: LruMap::new( + max_inflight_requests + max_capacity_cache_txns_pending_fetch, + ), + info, + metrics, + ..Default::default() + } } /// Removes the specified hashes from inflight tracking. @@ -178,7 +185,7 @@ impl TransactionFetcher { /// Returns `true` if peer is idle with respect to `self.inflight_requests`. pub fn is_idle(&self, peer_id: &PeerId) -> bool { let Some(inflight_count) = self.active_peers.peek(peer_id) else { return true }; - if *inflight_count < DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER { + if *inflight_count < self.info.max_inflight_requests_per_peer { return true } false @@ -653,12 +660,12 @@ impl TransactionFetcher { return Some(new_announced_hashes) }; - if *inflight_count >= DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER { + if *inflight_count >= self.info.max_inflight_requests_per_peer { trace!(target: "net::tx", peer_id=format!("{peer_id:#}"), hashes=?*new_announced_hashes, %conn_eth_version, - max_concurrent_tx_reqs_per_peer=DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER, + max_concurrent_tx_reqs_per_peer=self.info.max_inflight_requests_per_peer, "limit for concurrent `GetPooledTransactions` requests per peer reached" ); return Some(new_announced_hashes) @@ -1288,10 +1295,12 @@ pub enum VerificationOutcome { } /// Tracks stats about the [`TransactionFetcher`]. -#[derive(Debug)] +#[derive(Debug, Constructor)] pub struct TransactionFetcherInfo { /// Max inflight [`GetPooledTransactions`] requests. pub max_inflight_requests: usize, + /// Max inflight [`GetPooledTransactions`] requests per peer. + pub max_inflight_requests_per_peer: u8, /// Soft limit for the byte size of the expected [`PooledTransactions`] response, upon packing /// a [`GetPooledTransactions`] request with hashes (by default less than 2 MiB worth of /// transactions is requested). @@ -1305,27 +1314,11 @@ pub struct TransactionFetcherInfo { pub max_capacity_cache_txns_pending_fetch: u32, } -impl TransactionFetcherInfo { - /// Creates a new max - pub const fn new( - max_inflight_requests: usize, - soft_limit_byte_size_pooled_transactions_response_on_pack_request: usize, - soft_limit_byte_size_pooled_transactions_response: usize, - max_capacity_cache_txns_pending_fetch: u32, - ) -> Self { - Self { - max_inflight_requests, - soft_limit_byte_size_pooled_transactions_response_on_pack_request, - soft_limit_byte_size_pooled_transactions_response, - max_capacity_cache_txns_pending_fetch, - } - } -} - impl Default for TransactionFetcherInfo { fn default() -> Self { Self::new( DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS as usize, + DEFAULT_MAX_COUNT_CONCURRENT_REQUESTS_PER_PEER, DEFAULT_SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESP_ON_PACK_GET_POOLED_TRANSACTIONS_REQ, SOFT_LIMIT_BYTE_SIZE_POOLED_TRANSACTIONS_RESPONSE, DEFAULT_MAX_CAPACITY_CACHE_PENDING_FETCH, @@ -1333,6 +1326,26 @@ impl Default for TransactionFetcherInfo { } } +impl From for TransactionFetcherInfo { + fn from(config: TransactionFetcherConfig) -> Self { + let TransactionFetcherConfig { + max_inflight_requests, + max_inflight_requests_per_peer, + soft_limit_byte_size_pooled_transactions_response, + soft_limit_byte_size_pooled_transactions_response_on_pack_request, + max_capacity_cache_txns_pending_fetch, + } = config; + + Self::new( + max_inflight_requests as usize, + max_inflight_requests_per_peer, + soft_limit_byte_size_pooled_transactions_response_on_pack_request, + soft_limit_byte_size_pooled_transactions_response, + max_capacity_cache_txns_pending_fetch, + ) + } +} + #[derive(Debug, Default)] struct TxFetcherSearchDurations { find_idle_peer: Duration, From c25cc572d601a7ad51291feea8f9f8dbfe8c98c6 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 9 Oct 2024 22:45:40 +0900 Subject: [PATCH 089/159] perf(rpc): optimistically retrieve block if near the tip on `eth_getLogs` (#11582) --- crates/rpc/rpc-eth-types/src/logs_utils.rs | 67 +++++++++++------- crates/rpc/rpc/src/eth/filter.rs | 81 ++++++++++++---------- 2 files changed, 87 insertions(+), 61 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/logs_utils.rs b/crates/rpc/rpc-eth-types/src/logs_utils.rs index f26555bb70daf..c64bbe055b79d 100644 --- a/crates/rpc/rpc-eth-types/src/logs_utils.rs +++ b/crates/rpc/rpc-eth-types/src/logs_utils.rs @@ -6,7 +6,7 @@ use alloy_primitives::TxHash; use alloy_rpc_types::{FilteredParams, Log}; use reth_chainspec::ChainInfo; use reth_errors::ProviderError; -use reth_primitives::{BlockNumHash, Receipt}; +use reth_primitives::{BlockNumHash, Receipt, SealedBlock}; use reth_storage_api::BlockReader; /// Returns all matching of a block's receipts when the transaction hashes are known. @@ -45,11 +45,20 @@ where all_logs } +/// Helper enum to fetch a transaction either from a block or from the provider. +#[derive(Debug)] +pub enum ProviderOrBlock<'a, P: BlockReader> { + /// Provider + Provider(&'a P), + /// [`SealedBlock`] + Block(SealedBlock), +} + /// Appends all matching logs of a block's receipts. /// If the log matches, look up the corresponding transaction hash. -pub fn append_matching_block_logs( +pub fn append_matching_block_logs( all_logs: &mut Vec, - provider: impl BlockReader, + provider_or_block: ProviderOrBlock<'_, P>, filter: &FilteredParams, block_num_hash: BlockNumHash, receipts: &[Receipt], @@ -60,8 +69,8 @@ pub fn append_matching_block_logs( let mut log_index: u64 = 0; // Lazy loaded number of the first transaction in the block. - // This is useful for blocks with multiple matching logs because it prevents - // re-querying the block body indices. + // This is useful for blocks with multiple matching logs because it + // prevents re-querying the block body indices. let mut loaded_first_tx_num = None; // Iterate over receipts and append matching logs. @@ -71,27 +80,37 @@ pub fn append_matching_block_logs( for log in &receipt.logs { if log_matches_filter(block_num_hash, log, filter) { - let first_tx_num = match loaded_first_tx_num { - Some(num) => num, - None => { - let block_body_indices = - provider.block_body_indices(block_num_hash.number)?.ok_or( - ProviderError::BlockBodyIndicesNotFound(block_num_hash.number), - )?; - loaded_first_tx_num = Some(block_body_indices.first_tx_num); - block_body_indices.first_tx_num - } - }; - // if this is the first match in the receipt's logs, look up the transaction hash if transaction_hash.is_none() { - // This is safe because Transactions and Receipts have the same keys. - let transaction_id = first_tx_num + receipt_idx as u64; - let transaction = provider - .transaction_by_id(transaction_id)? - .ok_or_else(|| ProviderError::TransactionNotFound(transaction_id.into()))?; - - transaction_hash = Some(transaction.hash()); + transaction_hash = match &provider_or_block { + ProviderOrBlock::Block(block) => { + block.body.transactions.get(receipt_idx).map(|t| t.hash()) + } + ProviderOrBlock::Provider(provider) => { + let first_tx_num = match loaded_first_tx_num { + Some(num) => num, + None => { + let block_body_indices = provider + .block_body_indices(block_num_hash.number)? + .ok_or(ProviderError::BlockBodyIndicesNotFound( + block_num_hash.number, + ))?; + loaded_first_tx_num = Some(block_body_indices.first_tx_num); + block_body_indices.first_tx_num + } + }; + + // This is safe because Transactions and Receipts have the same + // keys. + let transaction_id = first_tx_num + receipt_idx as u64; + let transaction = + provider.transaction_by_id(transaction_id)?.ok_or_else(|| { + ProviderError::TransactionNotFound(transaction_id.into()) + })?; + + Some(transaction.hash()) + } + }; } let log = Log { diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index c5581f42a06c2..9efecf3dae7f3 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -19,11 +19,11 @@ use async_trait::async_trait; use jsonrpsee::{core::RpcResult, server::IdProvider}; use reth_chainspec::ChainInfo; use reth_node_api::EthApiTypes; -use reth_primitives::TransactionSignedEcRecovered; +use reth_primitives::{Receipt, SealedBlock, TransactionSignedEcRecovered}; use reth_provider::{BlockIdReader, BlockReader, EvmEnvProvider, ProviderError}; use reth_rpc_eth_api::{EthFilterApiServer, FullEthApiTypes, RpcTransaction, TransactionCompat}; use reth_rpc_eth_types::{ - logs_utils::{self, append_matching_block_logs}, + logs_utils::{self, append_matching_block_logs, ProviderOrBlock}, EthApiError, EthFilterConfig, EthStateCache, EthSubscriptionIdProvider, }; use reth_rpc_server_types::{result::rpc_error_with_code, ToRpcResult}; @@ -376,29 +376,34 @@ where FilterBlockOption::AtBlockHash(block_hash) => { // for all matching logs in the block // get the block header with the hash - let block = self + let header = self .provider .header_by_hash_or_number(block_hash.into())? .ok_or_else(|| ProviderError::HeaderNotFound(block_hash.into()))?; + let block_num_hash = BlockNumHash::new(header.number, block_hash); + // we also need to ensure that the receipts are available and return an error if // not, in case the block hash been reorged - let receipts = self - .eth_cache - .get_receipts(block_hash) + let (receipts, maybe_block) = self + .receipts_and_maybe_block( + &block_num_hash, + self.provider.chain_info()?.best_number, + ) .await? .ok_or(EthApiError::HeaderNotFound(block_hash.into()))?; let mut all_logs = Vec::new(); - let filter = FilteredParams::new(Some(filter)); - logs_utils::append_matching_block_logs( + append_matching_block_logs( &mut all_logs, - &self.provider, - &filter, - (block_hash, block.number).into(), + maybe_block + .map(|b| ProviderOrBlock::Block(b)) + .unwrap_or_else(|| ProviderOrBlock::Provider(&self.provider)), + &FilteredParams::new(Some(filter)), + block_num_hash, &receipts, false, - block.timestamp, + header.timestamp, )?; Ok(all_logs) @@ -454,7 +459,6 @@ where chain_info: ChainInfo, ) -> Result, EthFilterError> { trace!(target: "rpc::eth::filter", from=from_block, to=to_block, ?filter, "finding logs in range"); - let best_number = chain_info.best_number; if to_block < from_block { return Err(EthFilterError::InvalidBlockRangeParams) @@ -467,27 +471,6 @@ where let mut all_logs = Vec::new(); let filter_params = FilteredParams::new(Some(filter.clone())); - if (to_block == best_number) && (from_block == best_number) { - // only one block to check and it's the current best block which we can fetch directly - // Note: In case of a reorg, the best block's hash might have changed, hence we only - // return early of we were able to fetch the best block's receipts - // perf: we're fetching the best block here which is expected to be cached - if let Some((block, receipts)) = - self.eth_cache.get_block_and_receipts(chain_info.best_hash).await? - { - logs_utils::append_matching_block_logs( - &mut all_logs, - &self.provider, - &filter_params, - chain_info.into(), - &receipts, - false, - block.header.timestamp, - )?; - } - return Ok(all_logs) - } - // derive bloom filters from filter input, so we can check headers for matching logs let address_filter = FilteredParams::address_filter(&filter.address); let topics_filter = FilteredParams::topics_filter(&filter.topics); @@ -514,12 +497,17 @@ where .ok_or_else(|| ProviderError::HeaderNotFound(header.number.into()))?, }; - if let Some(receipts) = self.eth_cache.get_receipts(block_hash).await? { + let num_hash = BlockNumHash::new(header.number, block_hash); + if let Some((receipts, maybe_block)) = + self.receipts_and_maybe_block(&num_hash, chain_info.best_number).await? + { append_matching_block_logs( &mut all_logs, - &self.provider, + maybe_block + .map(|block| ProviderOrBlock::Block(block)) + .unwrap_or_else(|| ProviderOrBlock::Provider(&self.provider)), &filter_params, - BlockNumHash::new(header.number, block_hash), + num_hash, &receipts, false, header.timestamp, @@ -540,6 +528,25 @@ where Ok(all_logs) } + + /// Retrieves receipts and block from cache if near the tip (4 blocks), otherwise only receipts. + async fn receipts_and_maybe_block( + &self, + block_num_hash: &BlockNumHash, + best_number: u64, + ) -> Result>, Option)>, EthFilterError> { + // The last 4 blocks are most likely cached, so we can just fetch them + let cached_range = best_number.saturating_sub(4)..=best_number; + let receipts_block = if cached_range.contains(&block_num_hash.number) { + self.eth_cache + .get_block_and_receipts(block_num_hash.hash) + .await? + .map(|(b, r)| (r, Some(b))) + } else { + self.eth_cache.get_receipts(block_num_hash.hash).await?.map(|r| (r, None)) + }; + Ok(receipts_block) + } } /// All active filters From a38353768ca09b7ccbf071ddcd338b866a03ef14 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Thu, 10 Oct 2024 00:28:35 +0900 Subject: [PATCH 090/159] chore(rpc): use `block_hash` as `BlockId` on `eth_callMany` (#11595) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 5bc7d73b2221a..0aa38a36e9dac 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -22,7 +22,7 @@ use reth_primitives::{ }, Header, TransactionSignedEcRecovered, }; -use reth_provider::{ChainSpecProvider, HeaderProvider, StateProvider}; +use reth_provider::{BlockIdReader, ChainSpecProvider, HeaderProvider, StateProvider}; use reth_revm::{database::StateProviderDatabase, db::CacheDB, DatabaseRef}; use reth_rpc_eth_types::{ cache::db::{StateCacheDbRefMutWrapper, StateProviderTraitObjWrapper}, @@ -247,9 +247,19 @@ pub trait EthCall: Call + LoadPendingBlock { state_context.unwrap_or_default(); let transaction_index = transaction_index.unwrap_or_default(); - let target_block = block_number.unwrap_or_default(); + let mut target_block = block_number.unwrap_or_default(); let is_block_target_pending = target_block.is_pending(); + // if it's not pending, we should always use block_hash over block_number to ensure that + // different provider calls query data related to the same block. + if !is_block_target_pending { + target_block = LoadBlock::provider(self) + .block_hash_for_id(target_block) + .map_err(|_| EthApiError::HeaderNotFound(target_block))? + .ok_or_else(|| EthApiError::HeaderNotFound(target_block))? + .into(); + } + let ((cfg, block_env, _), block) = futures::try_join!( self.evm_env_at(target_block), self.block_with_senders(target_block) From 1def1deb12ab216e52ce5b588f4986bedefda240 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 17:51:25 +0200 Subject: [PATCH 091/159] fix: 7702 gas fields (#11614) --- crates/rpc/rpc-types-compat/src/transaction/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc-types-compat/src/transaction/mod.rs b/crates/rpc/rpc-types-compat/src/transaction/mod.rs index 4a00bf7a43905..9bb2a8b5d92a8 100644 --- a/crates/rpc/rpc-types-compat/src/transaction/mod.rs +++ b/crates/rpc/rpc-types-compat/src/transaction/mod.rs @@ -37,11 +37,12 @@ pub trait TransactionCompat: Send + Sync + Unpin + Clone + fmt::Debug { /// Formats gas price and max fee per gas for RPC transaction response w.r.t. network specific /// transaction type. fn gas_price(signed_tx: &TransactionSigned, base_fee: Option) -> GasPrice { + #[allow(unreachable_patterns)] match signed_tx.tx_type() { TxType::Legacy | TxType::Eip2930 => { GasPrice { gas_price: Some(signed_tx.max_fee_per_gas()), max_fee_per_gas: None } } - TxType::Eip1559 | TxType::Eip4844 => { + TxType::Eip1559 | TxType::Eip4844 | TxType::Eip7702 => { // the gas price field for EIP1559 is set to `min(tip, gasFeeCap - baseFee) + // baseFee` let gas_price = base_fee From b48be36d1016a1896871df0a3e8a04b240e5ce6b Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Wed, 9 Oct 2024 17:59:28 +0200 Subject: [PATCH 092/159] fix(witness): destroyed slots as proof targets (#11596) --- crates/trie/db/tests/witness.rs | 38 +++++++++++++++++++++++++++- crates/trie/trie/src/witness.rs | 45 ++++++++++++++++++++++++--------- 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/crates/trie/db/tests/witness.rs b/crates/trie/db/tests/witness.rs index cc921f6570877..59656383de4c9 100644 --- a/crates/trie/db/tests/witness.rs +++ b/crates/trie/db/tests/witness.rs @@ -6,7 +6,7 @@ use alloy_primitives::{ Address, Bytes, B256, U256, }; use alloy_rlp::EMPTY_STRING_CODE; -use reth_primitives::{constants::EMPTY_ROOT_HASH, Account}; +use reth_primitives::{constants::EMPTY_ROOT_HASH, Account, StorageEntry}; use reth_provider::{test_utils::create_test_provider_factory, HashingWriter}; use reth_trie::{proof::Proof, witness::TrieWitness, HashedPostState, HashedStorage, StateRoot}; use reth_trie_db::{DatabaseProof, DatabaseStateRoot, DatabaseTrieWitness}; @@ -55,3 +55,39 @@ fn includes_empty_node_preimage() { // witness includes empty state trie root node assert_eq!(witness.get(&EMPTY_ROOT_HASH), Some(&Bytes::from([EMPTY_STRING_CODE]))); } + +#[test] +fn includes_nodes_for_destroyed_storage_nodes() { + let factory = create_test_provider_factory(); + let provider = factory.provider_rw().unwrap(); + + let address = Address::random(); + let hashed_address = keccak256(address); + let slot = B256::random(); + let hashed_slot = keccak256(slot); + + // Insert account and slot into database + provider.insert_account_for_hashing([(address, Some(Account::default()))]).unwrap(); + provider + .insert_storage_for_hashing([(address, [StorageEntry { key: slot, value: U256::from(1) }])]) + .unwrap(); + + let state_root = StateRoot::from_tx(provider.tx_ref()).root().unwrap(); + let multiproof = Proof::from_tx(provider.tx_ref()) + .multiproof(HashMap::from_iter([(hashed_address, HashSet::from_iter([hashed_slot]))])) + .unwrap(); + + let witness = TrieWitness::from_tx(provider.tx_ref()) + .compute(HashedPostState { + accounts: HashMap::from([(hashed_address, Some(Account::default()))]), + storages: HashMap::from([(hashed_address, HashedStorage::from_iter(true, []))]), // destroyed + }) + .unwrap(); + assert!(witness.contains_key(&state_root)); + for node in multiproof.account_subtree.values() { + assert_eq!(witness.get(&keccak256(node)), Some(node)); + } + for node in multiproof.storages.iter().flat_map(|(_, storage)| storage.subtree.values()) { + assert_eq!(witness.get(&keccak256(node)), Some(node)); + } +} diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index d668946e62381..c042a0d8213b3 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use crate::{ - hashed_cursor::HashedCursorFactory, + hashed_cursor::{HashedCursor, HashedCursorFactory}, prefix_set::TriePrefixSetsMut, proof::{Proof, StorageProof}, trie_cursor::TrieCursorFactory, @@ -14,7 +14,7 @@ use alloy_primitives::{ }; use alloy_rlp::{BufMut, Decodable, Encodable}; use itertools::{Either, Itertools}; -use reth_execution_errors::TrieWitnessError; +use reth_execution_errors::{StateProofError, TrieWitnessError}; use reth_primitives::constants::EMPTY_ROOT_HASH; use reth_trie_common::{ BranchNode, HashBuilder, Nibbles, StorageMultiProof, TrieAccount, TrieNode, CHILD_INDEX_RANGE, @@ -90,16 +90,7 @@ where return Ok(self.witness) } - let proof_targets = HashMap::from_iter( - state - .accounts - .keys() - .map(|hashed_address| (*hashed_address, HashSet::default())) - .chain(state.storages.iter().map(|(hashed_address, storage)| { - (*hashed_address, storage.storage.keys().copied().collect()) - })), - ); - + let proof_targets = self.get_proof_targets(&state)?; let mut account_multiproof = Proof::new(self.trie_cursor_factory.clone(), self.hashed_cursor_factory.clone()) .with_prefix_sets_mut(self.prefix_sets.clone()) @@ -257,6 +248,36 @@ where Ok(trie_nodes) } + /// Retrieve proof targets for incoming hashed state. + /// This method will aggregate all accounts and slots present in the hash state as well as + /// select all existing slots from the database for the accounts that have been destroyed. + fn get_proof_targets( + &self, + state: &HashedPostState, + ) -> Result>, StateProofError> { + let mut proof_targets = HashMap::default(); + for hashed_address in state.accounts.keys() { + proof_targets.insert(*hashed_address, HashSet::default()); + } + for (hashed_address, storage) in &state.storages { + let mut storage_keys = storage.storage.keys().copied().collect::>(); + if storage.wiped { + // storage for this account was destroyed, gather all slots from the current state + let mut storage_cursor = + self.hashed_cursor_factory.hashed_storage_cursor(*hashed_address)?; + // position cursor at the start + if let Some((hashed_slot, _)) = storage_cursor.seek(B256::ZERO)? { + storage_keys.insert(hashed_slot); + } + while let Some((hashed_slot, _)) = storage_cursor.next()? { + storage_keys.insert(hashed_slot); + } + } + proof_targets.insert(*hashed_address, storage_keys); + } + Ok(proof_targets) + } + fn next_root_from_proofs( trie_nodes: BTreeMap>>, mut trie_node_provider: impl FnMut(Nibbles) -> Result, From 1f9405a72204e021c11d37d9088f627ed839769d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 18:40:48 +0200 Subject: [PATCH 093/159] chore: bump op-alloy (#11617) --- Cargo.lock | 24 +++++++++---------- Cargo.toml | 8 +++---- crates/e2e-test-utils/src/traits.rs | 4 ++-- crates/optimism/chainspec/src/lib.rs | 10 ++++---- crates/optimism/node/src/engine.rs | 19 +++++++-------- crates/optimism/payload/src/lib.rs | 4 +--- crates/optimism/payload/src/payload.rs | 21 +++++++--------- crates/optimism/rpc/src/eth/receipt.rs | 32 +++++++++++-------------- crates/payload/primitives/src/traits.rs | 4 ++-- crates/primitives/src/alloy_compat.rs | 2 +- 10 files changed, 58 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 863f0f35c18d8..9bf1ea2d666da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5174,9 +5174,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "op-alloy-consensus" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f7f318f885db6e1455370ca91f74b7faed152c8142f6418f0936d606e582ff" +checksum = "7ea7162170c6f3cad8f67f4dd7108e3f78349fd553da5b8bebff1e7ef8f38896" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5192,9 +5192,9 @@ dependencies = [ [[package]] name = "op-alloy-genesis" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8215c87b74d2fbbaff0fd2887868a8341df33a3c495ee01f813e5ddd5be9c46" +checksum = "9f3d31dfbbd8dd898c7512f8ce7d30103980485416f668566100b0ed0994b958" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5206,9 +5206,9 @@ dependencies = [ [[package]] name = "op-alloy-network" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd514c4ccd0b3c69fa3e7050cde77db842d4c308ae48f9a3e1ce263e823e45e" +checksum = "d113b325527ba7da271a8793f1c14bdf7f035ce9e0611e668c36fc6812568c7f" dependencies = [ "alloy-consensus", "alloy-network", @@ -5220,9 +5220,9 @@ dependencies = [ [[package]] name = "op-alloy-protocol" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa5c397fbe35e07f9c95a571440ca2e90df754e198496d82ff4127de00b89dd9" +checksum = "310873e4fbfc41986716c4fb6000a8b49d025d932d2c261af58271c434b05288" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5237,9 +5237,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "547d29c5ab957ff32e14edddb93652dad748d2ef6cbe4b0fe8615ce06b0a3ddb" +checksum = "323c65880e2561aa87f74f8af260fd15b9cc930c448c88a60ae95af86c88c634" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5254,9 +5254,9 @@ dependencies = [ [[package]] name = "op-alloy-rpc-types-engine" -version = "0.3.3" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041122e20b76644cc690bba688671eecdc4626e6384a76eb740535d6ddcef14" +checksum = "349e7b420f45d1a00216ec4c65fcf3f0057a841bc39732c405c85ae782b94121" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", diff --git a/Cargo.toml b/Cargo.toml index 5c772a844429e..22ce3e92f6a6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -460,10 +460,10 @@ alloy-transport-ipc = { version = "0.4.2", default-features = false } alloy-transport-ws = { version = "0.4.2", default-features = false } # op -op-alloy-rpc-types = "0.3.2" -op-alloy-rpc-types-engine = "0.3.2" -op-alloy-network = "0.3.2" -op-alloy-consensus = "0.3.2" +op-alloy-rpc-types = "0.4" +op-alloy-rpc-types-engine = "0.4" +op-alloy-network = "0.4" +op-alloy-consensus = "0.4" # misc aquamarine = "0.5" diff --git a/crates/e2e-test-utils/src/traits.rs b/crates/e2e-test-utils/src/traits.rs index 7bd5a5b3222f9..6786492140b08 100644 --- a/crates/e2e-test-utils/src/traits.rs +++ b/crates/e2e-test-utils/src/traits.rs @@ -1,4 +1,4 @@ -use op_alloy_rpc_types_engine::OptimismExecutionPayloadEnvelopeV3; +use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelopeV3; use reth::rpc::types::engine::{ExecutionPayloadEnvelopeV3, ExecutionPayloadV3}; /// The execution payload envelope type. @@ -7,7 +7,7 @@ pub trait PayloadEnvelopeExt: Send + Sync + std::fmt::Debug { fn execution_payload(&self) -> ExecutionPayloadV3; } -impl PayloadEnvelopeExt for OptimismExecutionPayloadEnvelopeV3 { +impl PayloadEnvelopeExt for OpExecutionPayloadEnvelopeV3 { fn execution_payload(&self) -> ExecutionPayloadV3 { self.execution_payload.clone() } diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 22316a234d9b1..ebd9392b5330a 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -224,14 +224,14 @@ impl From for OpChainSpec { #[derive(Default, Debug)] struct OptimismGenesisInfo { - optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo, + optimism_chain_info: op_alloy_rpc_types::genesis::OpChainInfo, base_fee_params: BaseFeeParamsKind, } impl OptimismGenesisInfo { fn extract_from(genesis: &Genesis) -> Self { let mut info = Self { - optimism_chain_info: op_alloy_rpc_types::genesis::OptimismChainInfo::extract_from( + optimism_chain_info: op_alloy_rpc_types::genesis::OpChainInfo::extract_from( &genesis.config.extra_fields, ) .unwrap_or_default(), @@ -622,7 +622,7 @@ mod tests { #[test] fn parse_genesis_optimism_with_variable_base_fee_params() { - use op_alloy_rpc_types::genesis::OptimismBaseFeeInfo; + use op_alloy_rpc_types::genesis::OpBaseFeeInfo; let geth_genesis = r#" { @@ -673,11 +673,11 @@ mod tests { let optimism_object = genesis.config.extra_fields.get("optimism").unwrap(); let optimism_base_fee_info = - serde_json::from_value::(optimism_object.clone()).unwrap(); + serde_json::from_value::(optimism_object.clone()).unwrap(); assert_eq!( optimism_base_fee_info, - OptimismBaseFeeInfo { + OpBaseFeeInfo { eip1559_elasticity: Some(6), eip1559_denominator: Some(50), eip1559_denominator_canyon: None, diff --git a/crates/optimism/node/src/engine.rs b/crates/optimism/node/src/engine.rs index 11c8f15cfdac1..a83f4c696a1c9 100644 --- a/crates/optimism/node/src/engine.rs +++ b/crates/optimism/node/src/engine.rs @@ -2,8 +2,7 @@ use std::sync::Arc; use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV2, ExecutionPayloadV1}; use op_alloy_rpc_types_engine::{ - OptimismExecutionPayloadEnvelopeV3, OptimismExecutionPayloadEnvelopeV4, - OptimismPayloadAttributes, + OpExecutionPayloadEnvelopeV3, OpExecutionPayloadEnvelopeV4, OpPayloadAttributes, }; use reth_chainspec::ChainSpec; use reth_node_api::{ @@ -35,13 +34,13 @@ impl EngineTypes for OptimismEngineTypes where T::BuiltPayload: TryInto + TryInto - + TryInto - + TryInto, + + TryInto + + TryInto, { type ExecutionPayloadV1 = ExecutionPayloadV1; type ExecutionPayloadV2 = ExecutionPayloadEnvelopeV2; - type ExecutionPayloadV3 = OptimismExecutionPayloadEnvelopeV3; - type ExecutionPayloadV4 = OptimismExecutionPayloadEnvelopeV4; + type ExecutionPayloadV3 = OpExecutionPayloadEnvelopeV3; + type ExecutionPayloadV4 = OpExecutionPayloadEnvelopeV4; } /// A default payload type for [`OptimismEngineTypes`] @@ -51,7 +50,7 @@ pub struct OptimismPayloadTypes; impl PayloadTypes for OptimismPayloadTypes { type BuiltPayload = OptimismBuiltPayload; - type PayloadAttributes = OptimismPayloadAttributes; + type PayloadAttributes = OpPayloadAttributes; type PayloadBuilderAttributes = OptimismPayloadBuilderAttributes; } @@ -112,12 +111,12 @@ pub fn validate_withdrawals_presence( impl EngineValidator for OptimismEngineValidator where - Types: EngineTypes, + Types: EngineTypes, { fn validate_version_specific_fields( &self, version: EngineApiMessageVersion, - payload_or_attrs: PayloadOrAttributes<'_, OptimismPayloadAttributes>, + payload_or_attrs: PayloadOrAttributes<'_, OpPayloadAttributes>, ) -> Result<(), EngineObjectValidationError> { validate_withdrawals_presence( &self.chain_spec, @@ -138,7 +137,7 @@ where fn ensure_well_formed_attributes( &self, version: EngineApiMessageVersion, - attributes: &OptimismPayloadAttributes, + attributes: &OpPayloadAttributes, ) -> Result<(), EngineObjectValidationError> { validate_version_specific_fields(&self.chain_spec, version, attributes.into())?; diff --git a/crates/optimism/payload/src/lib.rs b/crates/optimism/payload/src/lib.rs index 2bb60594287a0..c06b49c537627 100644 --- a/crates/optimism/payload/src/lib.rs +++ b/crates/optimism/payload/src/lib.rs @@ -15,6 +15,4 @@ pub mod builder; pub use builder::OptimismPayloadBuilder; pub mod error; pub mod payload; -pub use payload::{ - OptimismBuiltPayload, OptimismPayloadAttributes, OptimismPayloadBuilderAttributes, -}; +pub use payload::{OpPayloadAttributes, OptimismBuiltPayload, OptimismPayloadBuilderAttributes}; diff --git a/crates/optimism/payload/src/payload.rs b/crates/optimism/payload/src/payload.rs index f1ba24435092c..122c2fde52694 100644 --- a/crates/optimism/payload/src/payload.rs +++ b/crates/optimism/payload/src/payload.rs @@ -7,10 +7,8 @@ use alloy_primitives::{Address, B256, U256}; use alloy_rlp::Encodable; use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV2, ExecutionPayloadV1, PayloadId}; /// Re-export for use in downstream arguments. -pub use op_alloy_rpc_types_engine::OptimismPayloadAttributes; -use op_alloy_rpc_types_engine::{ - OptimismExecutionPayloadEnvelopeV3, OptimismExecutionPayloadEnvelopeV4, -}; +pub use op_alloy_rpc_types_engine::OpPayloadAttributes; +use op_alloy_rpc_types_engine::{OpExecutionPayloadEnvelopeV3, OpExecutionPayloadEnvelopeV4}; use reth_chain_state::ExecutedBlock; use reth_chainspec::EthereumHardforks; use reth_optimism_chainspec::OpChainSpec; @@ -40,13 +38,13 @@ pub struct OptimismPayloadBuilderAttributes { } impl PayloadBuilderAttributes for OptimismPayloadBuilderAttributes { - type RpcPayloadAttributes = OptimismPayloadAttributes; + type RpcPayloadAttributes = OpPayloadAttributes; type Error = alloy_rlp::Error; /// Creates a new payload builder for the given parent block and the attributes. /// /// Derives the unique [`PayloadId`] for the given parent and attributes - fn try_new(parent: B256, attributes: OptimismPayloadAttributes) -> Result { + fn try_new(parent: B256, attributes: OpPayloadAttributes) -> Result { let id = payload_id_optimism(&parent, &attributes); let transactions = attributes @@ -213,7 +211,7 @@ impl From for ExecutionPayloadEnvelopeV2 { } } -impl From for OptimismExecutionPayloadEnvelopeV3 { +impl From for OpExecutionPayloadEnvelopeV3 { fn from(value: OptimismBuiltPayload) -> Self { let OptimismBuiltPayload { block, fees, sidecars, chain_spec, attributes, .. } = value; @@ -240,7 +238,7 @@ impl From for OptimismExecutionPayloadEnvelopeV3 { } } } -impl From for OptimismExecutionPayloadEnvelopeV4 { +impl From for OpExecutionPayloadEnvelopeV4 { fn from(value: OptimismBuiltPayload) -> Self { let OptimismBuiltPayload { block, fees, sidecars, chain_spec, attributes, .. } = value; @@ -268,13 +266,10 @@ impl From for OptimismExecutionPayloadEnvelopeV4 { } } -/// Generates the payload id for the configured payload from the [`OptimismPayloadAttributes`]. +/// Generates the payload id for the configured payload from the [`OpPayloadAttributes`]. /// /// Returns an 8-byte identifier by hashing the payload components with sha256 hash. -pub(crate) fn payload_id_optimism( - parent: &B256, - attributes: &OptimismPayloadAttributes, -) -> PayloadId { +pub(crate) fn payload_id_optimism(parent: &B256, attributes: &OpPayloadAttributes) -> PayloadId { use sha2::Digest; let mut hasher = sha2::Sha256::new(); hasher.update(parent.as_slice()); diff --git a/crates/optimism/rpc/src/eth/receipt.rs b/crates/optimism/rpc/src/eth/receipt.rs index 76b03e845a09a..fac8d220c761f 100644 --- a/crates/optimism/rpc/src/eth/receipt.rs +++ b/crates/optimism/rpc/src/eth/receipt.rs @@ -3,9 +3,7 @@ use alloy_eips::eip2718::Encodable2718; use alloy_rpc_types::{AnyReceiptEnvelope, Log, TransactionReceipt}; use op_alloy_consensus::{OpDepositReceipt, OpDepositReceiptWithBloom, OpReceiptEnvelope}; -use op_alloy_rpc_types::{ - receipt::L1BlockInfo, OpTransactionReceipt, OptimismTransactionReceiptFields, -}; +use op_alloy_rpc_types::{receipt::L1BlockInfo, OpTransactionReceipt, OpTransactionReceiptFields}; use reth_chainspec::ChainSpec; use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_optimism_chainspec::OpChainSpec; @@ -68,7 +66,7 @@ where tx: &TransactionSigned, l1_block_info: revm::L1BlockInfo, receipt: &Receipt, - ) -> Result { + ) -> Result { Ok(OpReceiptFieldsBuilder::default() .l1_block_info(&self.inner.provider().chain_spec(), tx, l1_block_info)? .deposit_nonce(receipt.deposit_nonce) @@ -162,8 +160,8 @@ impl OpReceiptFieldsBuilder { self } - /// Builds the [`OptimismTransactionReceiptFields`] object. - pub const fn build(self) -> OptimismTransactionReceiptFields { + /// Builds the [`OpTransactionReceiptFields`] object. + pub const fn build(self) -> OpTransactionReceiptFields { let Self { l1_block_timestamp: _, // used to compute other fields l1_fee, @@ -177,7 +175,7 @@ impl OpReceiptFieldsBuilder { l1_blob_base_fee_scalar, } = self; - OptimismTransactionReceiptFields { + OpTransactionReceiptFields { l1_block_info: L1BlockInfo { l1_gas_price, l1_gas_used, @@ -201,7 +199,7 @@ pub struct OpReceiptBuilder { /// Transaction type. pub tx_type: TxType, /// Additional OP receipt fields. - pub op_receipt_fields: OptimismTransactionReceiptFields, + pub op_receipt_fields: OpTransactionReceiptFields, } impl OpReceiptBuilder { @@ -234,11 +232,8 @@ impl OpReceiptBuilder { pub fn build(self) -> OpTransactionReceipt { let Self { core_receipt, tx_type, op_receipt_fields } = self; - let OptimismTransactionReceiptFields { - l1_block_info, - deposit_nonce, - deposit_receipt_version, - } = op_receipt_fields; + let OpTransactionReceiptFields { l1_block_info, deposit_nonce, deposit_receipt_version } = + op_receipt_fields; let TransactionReceipt { inner: AnyReceiptEnvelope { inner: receipt_with_bloom, .. }, @@ -261,10 +256,11 @@ impl OpReceiptBuilder { TxType::Legacy => OpReceiptEnvelope::::Legacy(receipt_with_bloom), TxType::Eip2930 => OpReceiptEnvelope::::Eip2930(receipt_with_bloom), TxType::Eip1559 => OpReceiptEnvelope::::Eip1559(receipt_with_bloom), - TxType::Eip4844 => OpReceiptEnvelope::::Eip4844(receipt_with_bloom), - TxType::Eip7702 => { - unimplemented!("not implemented yet for OpReceiptEnvelope") + TxType::Eip4844 => { + // TODO: unreachable + OpReceiptEnvelope::::Eip1559(receipt_with_bloom) } + TxType::Eip7702 => OpReceiptEnvelope::::Eip7702(receipt_with_bloom), TxType::Deposit => { OpReceiptEnvelope::::Deposit(OpDepositReceiptWithBloom:: { receipt: OpDepositReceipt:: { @@ -325,8 +321,8 @@ mod test { /// L1 block info for transaction at index 1 in block 124665056. /// /// - const TX_META_TX_1_OP_MAINNET_BLOCK_124665056: OptimismTransactionReceiptFields = - OptimismTransactionReceiptFields { + const TX_META_TX_1_OP_MAINNET_BLOCK_124665056: OpTransactionReceiptFields = + OpTransactionReceiptFields { l1_block_info: L1BlockInfo { l1_gas_price: Some(1055991687), // since bedrock l1 base fee l1_gas_used: Some(4471), diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index b0647691f760b..6ae6361fdb286 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -4,7 +4,7 @@ use alloy_rpc_types::{ engine::{PayloadAttributes as EthPayloadAttributes, PayloadId}, Withdrawal, }; -use op_alloy_rpc_types_engine::OptimismPayloadAttributes; +use op_alloy_rpc_types_engine::OpPayloadAttributes; use reth_chain_state::ExecutedBlock; use reth_primitives::{SealedBlock, Withdrawals}; use std::{future::Future, pin::Pin}; @@ -145,7 +145,7 @@ impl PayloadAttributes for EthPayloadAttributes { } } -impl PayloadAttributes for OptimismPayloadAttributes { +impl PayloadAttributes for OpPayloadAttributes { fn timestamp(&self) -> u64 { self.payload_attributes.timestamp } diff --git a/crates/primitives/src/alloy_compat.rs b/crates/primitives/src/alloy_compat.rs index f43af019e75b1..65099967e6e9e 100644 --- a/crates/primitives/src/alloy_compat.rs +++ b/crates/primitives/src/alloy_compat.rs @@ -194,7 +194,7 @@ impl TryFrom> for Transaction { #[cfg(feature = "optimism")] Some(TxType::Deposit) => { let fields = other - .deserialize_into::() + .deserialize_into::() .map_err(|e| ConversionError::Custom(e.to_string()))?; Ok(Self::Deposit(op_alloy_consensus::TxDeposit { source_hash: fields From 24be120a872167992e26e059c9e288f382c33657 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 9 Oct 2024 21:41:50 +0200 Subject: [PATCH 094/159] test: add sanity test for local txs args (#11620) --- crates/node/core/src/args/txpool.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/node/core/src/args/txpool.rs b/crates/node/core/src/args/txpool.rs index 98e125ab9f394..63f6c566ca2bc 100644 --- a/crates/node/core/src/args/txpool.rs +++ b/crates/node/core/src/args/txpool.rs @@ -170,4 +170,15 @@ mod tests { let args = CommandParser::::parse_from(["reth"]).args; assert_eq!(args, default_args); } + + #[test] + fn txpool_parse_locals() { + let args = CommandParser::::parse_from([ + "reth", + "--txpool.locals", + "0x0000000000000000000000000000000000000000", + ]) + .args; + assert_eq!(args.locals, vec![Address::ZERO]); + } } From ba533715c7d762b82b28722ce28b20e09a2b6afe Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 02:54:39 +0200 Subject: [PATCH 095/159] fix: persist finalized block (#11623) --- crates/engine/tree/src/persistence.rs | 17 ++++++++++++++++- crates/engine/tree/src/tree/mod.rs | 9 ++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index a748ece8a7c16..231945b3369ac 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -4,7 +4,7 @@ use reth_chain_state::ExecutedBlock; use reth_errors::ProviderError; use reth_provider::{ providers::ProviderNodeTypes, writer::UnifiedStorageWriter, BlockHashReader, - DatabaseProviderFactory, ProviderFactory, StaticFileProviderFactory, + DatabaseProviderFactory, FinalizedBlockWriter, ProviderFactory, StaticFileProviderFactory, }; use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory}; use reth_stages_api::{MetricEvent, MetricEventsSender}; @@ -92,6 +92,10 @@ impl PersistenceService { // we ignore the error because the caller may or may not care about the result let _ = sender.send(res); } + PersistenceAction::SaveFinalizedBlock(finalized_block) => self + .provider + .database_provider_rw()? + .save_finalized_block_number(finalized_block)?, } } Ok(()) @@ -168,6 +172,9 @@ pub enum PersistenceAction { /// Prune associated block data before the given block number, according to already-configured /// prune modes. PruneBefore(u64, oneshot::Sender), + + /// Update the persisted finalized block on disk + SaveFinalizedBlock(u64), } /// A handle to the persistence service @@ -235,6 +242,14 @@ impl PersistenceHandle { self.send_action(PersistenceAction::SaveBlocks(blocks, tx)) } + /// Persists the finalized block number on disk. + pub fn save_finalized_block_number( + &self, + finalized_block: u64, + ) -> Result<(), SendError> { + self.send_action(PersistenceAction::SaveFinalizedBlock(finalized_block)) + } + /// Tells the persistence service to remove blocks above a certain block number. The removed /// blocks are returned by the service. /// diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 0478c73c90d28..5a39bc9901769 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2358,7 +2358,14 @@ where return Err(OnForkChoiceUpdated::invalid_state()) } Ok(Some(finalized)) => { - self.canonical_in_memory_state.set_finalized(finalized); + if Some(finalized.num_hash()) != + self.canonical_in_memory_state.get_finalized_num_hash() + { + // we're also persisting the finalized block on disk so we can reload it on + // restart this is required by optimism which queries the finalized block: + let _ = self.persistence.save_finalized_block_number(finalized.number); + self.canonical_in_memory_state.set_finalized(finalized); + } } Err(err) => { error!(target: "engine::tree", %err, "Failed to fetch finalized block header"); From 2959c37e63403c085341d13c9b94b9204d40b414 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 9 Oct 2024 21:14:39 -0400 Subject: [PATCH 096/159] chore(metrics): remove redundant `starting metrics` log (#11621) --- crates/node/metrics/src/server.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/node/metrics/src/server.rs b/crates/node/metrics/src/server.rs index 06bb490a43658..87521349d4dee 100644 --- a/crates/node/metrics/src/server.rs +++ b/crates/node/metrics/src/server.rs @@ -11,7 +11,6 @@ use metrics_process::Collector; use reth_metrics::metrics::Unit; use reth_tasks::TaskExecutor; use std::{convert::Infallible, net::SocketAddr, sync::Arc}; -use tracing::info; /// Configuration for the [`MetricServer`] #[derive(Debug)] @@ -53,8 +52,6 @@ impl MetricServer { let MetricServerConfig { listen_addr, hooks, task_executor, version_info, chain_spec_info } = &self.config; - info!(target: "reth::cli", addr = %listen_addr, "Starting metrics endpoint"); - let hooks = hooks.clone(); self.start_endpoint( *listen_addr, From b07bbc3c79b6e52f6cfb591d3a8ccfa63711f8ff Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Thu, 10 Oct 2024 09:50:40 +0200 Subject: [PATCH 097/159] chore(ci): disable hive discord alerts (#11625) --- .github/workflows/hive.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 3155fd4af2d19..12224a951a424 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -259,11 +259,6 @@ jobs: runs-on: group: Reth steps: - - name: Discord Webhook Action - uses: tsickert/discord-webhook@v6.0.0 - with: - webhook-url: ${{ secrets.RETH_ALERTS_WEBHOOK }} - content: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}" - name: Slack Webhook Action uses: rtCamp/action-slack-notify@v2 env: From 25082967942078efaa0269786f8729dbb6cade5c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 10:15:53 +0200 Subject: [PATCH 098/159] fix: actually commit (#11626) --- crates/engine/tree/src/persistence.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 231945b3369ac..dfddfcfaa89b3 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -92,10 +92,11 @@ impl PersistenceService { // we ignore the error because the caller may or may not care about the result let _ = sender.send(res); } - PersistenceAction::SaveFinalizedBlock(finalized_block) => self - .provider - .database_provider_rw()? - .save_finalized_block_number(finalized_block)?, + PersistenceAction::SaveFinalizedBlock(finalized_block) => { + let provider = self.provider.database_provider_rw()?; + provider.save_finalized_block_number(finalized_block)?; + provider.commit()?; + } } } Ok(()) From cbe69402f25ef7db9463360d6caa4a65bdedc191 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 10 Oct 2024 09:27:01 +0100 Subject: [PATCH 099/159] feat(evm, trie): more metrics (#11613) --- crates/engine/tree/src/tree/metrics.rs | 7 +- crates/engine/tree/src/tree/mod.rs | 9 +- crates/ethereum/evm/src/execute.rs | 2 +- crates/evm/src/either.rs | 6 +- crates/evm/src/execute.rs | 10 +- crates/evm/src/metrics.rs | 101 ++++++++++++++++--- crates/evm/src/noop.rs | 2 +- crates/evm/src/test_utils.rs | 10 +- crates/optimism/evm/src/execute.rs | 2 +- crates/rpc/rpc/src/debug.rs | 2 +- crates/stages/stages/src/stages/execution.rs | 2 +- 11 files changed, 116 insertions(+), 37 deletions(-) diff --git a/crates/engine/tree/src/tree/metrics.rs b/crates/engine/tree/src/tree/metrics.rs index d46c2f05a028d..dba80a17914a2 100644 --- a/crates/engine/tree/src/tree/metrics.rs +++ b/crates/engine/tree/src/tree/metrics.rs @@ -4,6 +4,7 @@ use reth_metrics::{ metrics::{Counter, Gauge, Histogram}, Metrics, }; +use reth_trie::updates::TrieUpdates; /// Metrics for the `EngineApi`. #[derive(Debug, Default)] @@ -49,6 +50,8 @@ pub(crate) struct EngineMetrics { #[derive(Metrics)] #[metrics(scope = "sync.block_validation")] pub(crate) struct BlockValidationMetrics { + /// Total number of storage tries updated in the state root calculation + pub(crate) state_root_storage_tries_updated_total: Counter, /// Histogram of state root duration pub(crate) state_root_histogram: Histogram, /// Latest state root duration @@ -57,7 +60,9 @@ pub(crate) struct BlockValidationMetrics { impl BlockValidationMetrics { /// Records a new state root time, updating both the histogram and state root gauge - pub(crate) fn record_state_root(&self, elapsed_as_secs: f64) { + pub(crate) fn record_state_root(&self, trie_output: &TrieUpdates, elapsed_as_secs: f64) { + self.state_root_storage_tries_updated_total + .increment(trie_output.storage_tries_ref().len() as u64); self.state_root_duration.set(elapsed_as_secs); self.state_root_histogram.record(elapsed_as_secs); } diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 5a39bc9901769..a4491fde2927a 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -28,7 +28,7 @@ use reth_chainspec::EthereumHardforks; use reth_consensus::{Consensus, PostExecutionInput}; use reth_engine_primitives::EngineTypes; use reth_errors::{ConsensusError, ProviderResult}; -use reth_evm::execute::{BlockExecutorProvider, Executor}; +use reth_evm::execute::BlockExecutorProvider; use reth_payload_builder::PayloadBuilderHandle; use reth_payload_primitives::{PayloadAttributes, PayloadBuilder, PayloadBuilderAttributes}; use reth_payload_validator::ExecutionPayloadValidator; @@ -2160,10 +2160,7 @@ where let block = block.unseal(); let exec_time = Instant::now(); - let output = self - .metrics - .executor - .metered((&block, U256::MAX).into(), |input| executor.execute(input))?; + let output = self.metrics.executor.execute_metered(executor, (&block, U256::MAX).into())?; trace!(target: "engine::tree", elapsed=?exec_time.elapsed(), ?block_number, "Executed block"); if let Err(err) = self.consensus.validate_block_post_execution( @@ -2227,7 +2224,7 @@ where } let root_elapsed = root_time.elapsed(); - self.metrics.block_validation.record_state_root(root_elapsed.as_secs_f64()); + self.metrics.block_validation.record_state_root(&trie_output, root_elapsed.as_secs_f64()); debug!(target: "engine::tree", ?root_elapsed, ?block_number, "Calculated state root"); let executed = ExecutedBlock { diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs index 8c84fafc25fba..6513cf75fad55 100644 --- a/crates/ethereum/evm/src/execute.rs +++ b/crates/ethereum/evm/src/execute.rs @@ -372,7 +372,7 @@ where Ok(BlockExecutionOutput { state: self.state.take_bundle(), receipts, requests, gas_used }) } - fn execute_with_state_witness( + fn execute_with_state_closure( mut self, input: Self::Input<'_>, mut witness: F, diff --git a/crates/evm/src/either.rs b/crates/evm/src/either.rs index e28ec3887f806..8022c68c43dcc 100644 --- a/crates/evm/src/either.rs +++ b/crates/evm/src/either.rs @@ -77,7 +77,7 @@ where } } - fn execute_with_state_witness( + fn execute_with_state_closure( self, input: Self::Input<'_>, witness: F, @@ -86,8 +86,8 @@ where F: FnMut(&State), { match self { - Self::Left(a) => a.execute_with_state_witness(input, witness), - Self::Right(b) => b.execute_with_state_witness(input, witness), + Self::Left(a) => a.execute_with_state_closure(input, witness), + Self::Right(b) => b.execute_with_state_closure(input, witness), } } diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index 1bd79378127e2..145eca29c92bc 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -38,12 +38,12 @@ pub trait Executor { /// The output of the block execution. fn execute(self, input: Self::Input<'_>) -> Result; - /// Executes the EVM with the given input and accepts a witness closure that is invoked with the - /// EVM state after execution. - fn execute_with_state_witness( + /// Executes the EVM with the given input and accepts a state closure that is invoked with + /// the EVM state after execution. + fn execute_with_state_closure( self, input: Self::Input<'_>, - witness: F, + state: F, ) -> Result where F: FnMut(&State); @@ -203,7 +203,7 @@ mod tests { Err(BlockExecutionError::msg("execution unavailable for tests")) } - fn execute_with_state_witness( + fn execute_with_state_closure( self, _: Self::Input<'_>, _: F, diff --git a/crates/evm/src/metrics.rs b/crates/evm/src/metrics.rs index d6ffe0d79c69d..fbb2b858b1589 100644 --- a/crates/evm/src/metrics.rs +++ b/crates/evm/src/metrics.rs @@ -1,16 +1,18 @@ //! Executor metrics. //! -//! Block processing related to syncing should take care to update the metrics by using e.g. -//! [`ExecutorMetrics::metered`]. +//! Block processing related to syncing should take care to update the metrics by using either +//! [`ExecutorMetrics::execute_metered`] or [`ExecutorMetrics::metered_one`]. use std::time::Instant; use metrics::{Counter, Gauge, Histogram}; -use reth_execution_types::BlockExecutionInput; +use reth_execution_types::{BlockExecutionInput, BlockExecutionOutput}; use reth_metrics::Metrics; use reth_primitives::BlockWithSenders; +use crate::execute::Executor; + /// Executor metrics. -// TODO(onbjerg): add sload/sstore, acc load/acc change, bytecode metrics +// TODO(onbjerg): add sload/sstore #[derive(Metrics, Clone)] #[metrics(scope = "sync.execution")] pub struct ExecutorMetrics { @@ -18,31 +20,106 @@ pub struct ExecutorMetrics { pub gas_processed_total: Counter, /// The instantaneous amount of gas processed per second. pub gas_per_second: Gauge, + /// The Histogram for amount of time taken to execute blocks. pub execution_histogram: Histogram, /// The total amount of time it took to execute the latest block. pub execution_duration: Gauge, + + /// The Histogram for number of accounts loaded when executing the latest block. + pub accounts_loaded_histogram: Histogram, + /// The Histogram for number of storage slots loaded when executing the latest block. + pub storage_slots_loaded_histogram: Histogram, + /// The Histogram for number of bytecodes loaded when executing the latest block. + pub bytecodes_loaded_histogram: Histogram, + + /// The Histogram for number of accounts updated when executing the latest block. + pub accounts_updated_histogram: Histogram, + /// The Histogram for number of storage slots updated when executing the latest block. + pub storage_slots_updated_histogram: Histogram, + /// The Histogram for number of bytecodes updated when executing the latest block. + pub bytecodes_updated_histogram: Histogram, } impl ExecutorMetrics { - /// Execute the given block and update metrics for the execution. - pub fn metered(&self, input: BlockExecutionInput<'_, BlockWithSenders>, f: F) -> R + fn metered(&self, block: &BlockWithSenders, f: F) -> R where - F: FnOnce(BlockExecutionInput<'_, BlockWithSenders>) -> R, + F: FnOnce() -> R, { - let gas_used = input.block.gas_used; - // Execute the block and record the elapsed time. let execute_start = Instant::now(); - let output = f(input); + let output = f(); let execution_duration = execute_start.elapsed().as_secs_f64(); // Update gas metrics. - self.gas_processed_total.increment(gas_used); - self.gas_per_second.set(gas_used as f64 / execution_duration); + self.gas_processed_total.increment(block.gas_used); + self.gas_per_second.set(block.gas_used as f64 / execution_duration); self.execution_histogram.record(execution_duration); self.execution_duration.set(execution_duration); output } + + /// Execute the given block using the provided [`Executor`] and update metrics for the + /// execution. + /// + /// Compared to [`Self::metered_one`], this method additionally updates metrics for the number + /// of accounts, storage slots and bytecodes loaded and updated. + pub fn execute_metered<'a, E, DB, O, Error>( + &self, + executor: E, + input: BlockExecutionInput<'a, BlockWithSenders>, + ) -> Result, Error> + where + E: Executor< + DB, + Input<'a> = BlockExecutionInput<'a, BlockWithSenders>, + Output = BlockExecutionOutput, + Error = Error, + >, + { + let output = self.metered(input.block, || { + executor.execute_with_state_closure(input, |state: &revm::db::State| { + // Update the metrics for the number of accounts, storage slots and bytecodes + // loaded + let accounts = state.cache.accounts.len(); + let storage_slots = state + .cache + .accounts + .values() + .filter_map(|account| { + account.account.as_ref().map(|account| account.storage.len()) + }) + .sum::(); + let bytecodes = state.cache.contracts.len(); + + // Record all state present in the cache state as loaded even though some might have + // been newly created. + // TODO: Consider spitting these into loaded and newly created. + self.accounts_loaded_histogram.record(accounts as f64); + self.storage_slots_loaded_histogram.record(storage_slots as f64); + self.bytecodes_loaded_histogram.record(bytecodes as f64); + }) + })?; + + // Update the metrics for the number of accounts, storage slots and bytecodes updated + let accounts = output.state.state.len(); + let storage_slots = + output.state.state.values().map(|account| account.storage.len()).sum::(); + let bytecodes = output.state.contracts.len(); + + self.accounts_updated_histogram.record(accounts as f64); + self.storage_slots_updated_histogram.record(storage_slots as f64); + self.bytecodes_updated_histogram.record(bytecodes as f64); + + Ok(output) + } + + /// Execute the given block and update metrics for the execution. + pub fn metered_one(&self, input: BlockExecutionInput<'_, BlockWithSenders>, f: F) -> R + where + F: FnOnce(BlockExecutionInput<'_, BlockWithSenders>) -> R, + { + self.metered(input.block, || f(input)) + } } diff --git a/crates/evm/src/noop.rs b/crates/evm/src/noop.rs index 3e01bfc4cc467..4fdc6d367a277 100644 --- a/crates/evm/src/noop.rs +++ b/crates/evm/src/noop.rs @@ -51,7 +51,7 @@ impl Executor for NoopBlockExecutorProvider { Err(BlockExecutionError::msg(UNAVAILABLE_FOR_NOOP)) } - fn execute_with_state_witness( + fn execute_with_state_closure( self, _: Self::Input<'_>, _: F, diff --git a/crates/evm/src/test_utils.rs b/crates/evm/src/test_utils.rs index 45ab2e97734e0..fc620bb42c335 100644 --- a/crates/evm/src/test_utils.rs +++ b/crates/evm/src/test_utils.rs @@ -66,26 +66,26 @@ impl Executor for MockExecutorProvider { }) } - fn execute_with_state_witness( + fn execute_with_state_closure( self, - _: Self::Input<'_>, + input: Self::Input<'_>, _: F, ) -> Result where F: FnMut(&State), { - unimplemented!() + >::execute(self, input) } fn execute_with_state_hook( self, - _: Self::Input<'_>, + input: Self::Input<'_>, _: F, ) -> Result where F: OnStateHook, { - unimplemented!() + >::execute(self, input) } } diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 4e5918831f764..2491e99b5038e 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -381,7 +381,7 @@ where }) } - fn execute_with_state_witness( + fn execute_with_state_closure( mut self, input: Self::Input<'_>, mut witness: F, diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index fb04070397768..b26459c4f0406 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -612,7 +612,7 @@ where let mut codes = HashMap::default(); let _ = block_executor - .execute_with_state_witness( + .execute_with_state_closure( (&block.clone().unseal(), block.difficulty).into(), |statedb| { codes = statedb diff --git a/crates/stages/stages/src/stages/execution.rs b/crates/stages/stages/src/stages/execution.rs index a99d8a572e5b7..df234e542be63 100644 --- a/crates/stages/stages/src/stages/execution.rs +++ b/crates/stages/stages/src/stages/execution.rs @@ -275,7 +275,7 @@ where // Execute the block let execute_start = Instant::now(); - self.metrics.metered((&block, td).into(), |input| { + self.metrics.metered_one((&block, td).into(), |input| { let sealed = block.header.clone().seal_slow(); let (header, seal) = sealed.into_parts(); From d435daff552a6f8bc18f51cb8c380fb27636d97e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 11:39:40 +0200 Subject: [PATCH 100/159] fix: always handle payload building for opstack (#11629) --- crates/chainspec/src/api.rs | 9 +++++++++ crates/engine/service/Cargo.toml | 1 + crates/engine/service/src/service.rs | 7 ++++++- crates/engine/tree/src/engine.rs | 17 +++++++++++++++-- crates/engine/tree/src/tree/mod.rs | 24 +++++++++++++++++++++--- crates/optimism/chainspec/src/lib.rs | 4 ++++ 6 files changed, 56 insertions(+), 6 deletions(-) diff --git a/crates/chainspec/src/api.rs b/crates/chainspec/src/api.rs index fb64e08ae1ef8..e481fbede70dd 100644 --- a/crates/chainspec/src/api.rs +++ b/crates/chainspec/src/api.rs @@ -46,6 +46,11 @@ pub trait EthChainSpec: Send + Sync + Unpin + Debug { /// The bootnodes for the chain, if any. fn bootnodes(&self) -> Option>; + + /// Returns `true` if this chain contains Optimism configuration. + fn is_optimism(&self) -> bool { + self.chain().is_optimism() + } } impl EthChainSpec for ChainSpec { @@ -92,4 +97,8 @@ impl EthChainSpec for ChainSpec { fn bootnodes(&self) -> Option> { self.bootnodes() } + + fn is_optimism(&self) -> bool { + Self::is_optimism(self) + } } diff --git a/crates/engine/service/Cargo.toml b/crates/engine/service/Cargo.toml index 63d5f1fb97663..c6098bfe66717 100644 --- a/crates/engine/service/Cargo.toml +++ b/crates/engine/service/Cargo.toml @@ -24,6 +24,7 @@ reth-prune.workspace = true reth-stages-api.workspace = true reth-tasks.workspace = true reth-node-types.workspace = true +reth-chainspec.workspace = true # async futures.workspace = true diff --git a/crates/engine/service/src/service.rs b/crates/engine/service/src/service.rs index 880be5c0261b8..ed9c1aa1c27e0 100644 --- a/crates/engine/service/src/service.rs +++ b/crates/engine/service/src/service.rs @@ -1,11 +1,12 @@ use futures::{Stream, StreamExt}; use pin_project::pin_project; use reth_beacon_consensus::{BeaconConsensusEngineEvent, BeaconEngineMessage, EngineNodeTypes}; +use reth_chainspec::EthChainSpec; use reth_consensus::Consensus; use reth_engine_tree::{ backfill::PipelineSync, download::BasicBlockDownloader, - engine::{EngineApiRequest, EngineApiRequestHandler, EngineHandler}, + engine::{EngineApiKind, EngineApiRequest, EngineApiRequestHandler, EngineHandler}, persistence::PersistenceHandle, tree::{EngineApiTreeHandler, InvalidBlockHook, TreeConfig}, }; @@ -79,6 +80,9 @@ where invalid_block_hook: Box, sync_metrics_tx: MetricEventsSender, ) -> Self { + let engine_kind = + if chain_spec.is_optimism() { EngineApiKind::OpStack } else { EngineApiKind::Ethereum }; + let downloader = BasicBlockDownloader::new(client, consensus.clone()); let persistence_handle = @@ -97,6 +101,7 @@ where canonical_in_memory_state, tree_config, invalid_block_hook, + engine_kind, ); let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree); diff --git a/crates/engine/tree/src/engine.rs b/crates/engine/tree/src/engine.rs index 8172a44691739..c1571ed821784 100644 --- a/crates/engine/tree/src/engine.rs +++ b/crates/engine/tree/src/engine.rs @@ -212,15 +212,28 @@ where } } -/// The type for specifying the kind of engine api -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// The type for specifying the kind of engine api. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum EngineApiKind { /// The chain contains Ethereum configuration. + #[default] Ethereum, /// The chain contains Optimism configuration. OpStack, } +impl EngineApiKind { + /// Returns true if this is the ethereum variant + pub const fn is_ethereum(&self) -> bool { + matches!(self, Self::Ethereum) + } + + /// Returns true if this is the ethereum variant + pub const fn is_opstack(&self) -> bool { + matches!(self, Self::OpStack) + } +} + /// The request variants that the engine API handler can receive. #[derive(Debug)] pub enum EngineApiRequest { diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index a4491fde2927a..bc1da6369459a 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -66,7 +66,10 @@ pub mod config; mod invalid_block_hook; mod metrics; mod persistence_state; -use crate::{engine::EngineApiRequest, tree::metrics::EngineApiMetrics}; +use crate::{ + engine::{EngineApiKind, EngineApiRequest}, + tree::metrics::EngineApiMetrics, +}; pub use config::TreeConfig; pub use invalid_block_hook::{InvalidBlockHooks, NoopInvalidBlockHook}; pub use persistence_state::PersistenceState; @@ -498,6 +501,8 @@ pub struct EngineApiTreeHandler { metrics: EngineApiMetrics, /// An invalid block hook. invalid_block_hook: Box, + /// The engine API variant of this handler + engine_kind: EngineApiKind, } impl std::fmt::Debug @@ -519,6 +524,7 @@ impl std::fmt::Debug .field("config", &self.config) .field("metrics", &self.metrics) .field("invalid_block_hook", &format!("{:p}", self.invalid_block_hook)) + .field("engine_kind", &self.engine_kind) .finish() } } @@ -545,6 +551,7 @@ where persistence_state: PersistenceState, payload_builder: PayloadBuilderHandle, config: TreeConfig, + engine_kind: EngineApiKind, ) -> Self { let (incoming_tx, incoming) = std::sync::mpsc::channel(); @@ -565,6 +572,7 @@ where metrics: Default::default(), incoming_tx, invalid_block_hook: Box::new(NoopInvalidBlockHook), + engine_kind, } } @@ -589,6 +597,7 @@ where canonical_in_memory_state: CanonicalInMemoryState, config: TreeConfig, invalid_block_hook: Box, + kind: EngineApiKind, ) -> (Sender>>, UnboundedReceiver) { let best_block_number = provider.best_block_number().unwrap_or(0); let header = provider.sealed_header(best_block_number).ok().flatten().unwrap_or_default(); @@ -618,6 +627,7 @@ where persistence_state, payload_builder, config, + kind, ); task.set_invalid_block_hook(invalid_block_hook); let incoming = task.incoming_tx.clone(); @@ -1041,8 +1051,15 @@ where if let Ok(Some(canonical_header)) = self.find_canonical_header(state.head_block_hash) { debug!(target: "engine::tree", head = canonical_header.number, "fcu head block is already canonical"); - // TODO(mattsse): for optimism we'd need to trigger a build job as well because on - // Optimism, the proposers are allowed to reorg their own chain at will. + // For OpStack the proposers are allowed to reorg their own chain at will, so we need to + // always trigger a new payload job if requested. + if self.engine_kind.is_opstack() { + if let Some(attr) = attrs { + debug!(target: "engine::tree", head = canonical_header.number, "handling payload attributes for canonical head"); + let updated = self.process_payload_attributes(attr, &canonical_header, state); + return Ok(TreeOutcome::new(updated)) + } + } // 2. Client software MAY skip an update of the forkchoice state and MUST NOT begin a // payload build process if `forkchoiceState.headBlockHash` references a `VALID` @@ -2680,6 +2697,7 @@ mod tests { PersistenceState::default(), payload_builder, TreeConfig::default(), + EngineApiKind::Ethereum, ); let block_builder = TestBlockBuilder::default().with_chain_spec((*chain_spec).clone()); diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index ebd9392b5330a..9b2d1c8c11ba6 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -93,6 +93,10 @@ impl EthChainSpec for OpChainSpec { fn bootnodes(&self) -> Option> { self.inner.bootnodes() } + + fn is_optimism(&self) -> bool { + true + } } impl Hardforks for OpChainSpec { From 73e1e3f01b06f0901a4fb70c89e88ef6eb82f4c0 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 10 Oct 2024 12:28:46 +0200 Subject: [PATCH 101/159] chore(trie): make initialization of in-memory trie cursors `pub` (#11628) --- crates/trie/trie/src/trie_cursor/in_memory.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/trie/trie/src/trie_cursor/in_memory.rs b/crates/trie/trie/src/trie_cursor/in_memory.rs index 0f00191378ba8..851670f4267ad 100644 --- a/crates/trie/trie/src/trie_cursor/in_memory.rs +++ b/crates/trie/trie/src/trie_cursor/in_memory.rs @@ -50,7 +50,7 @@ impl<'a, CF: TrieCursorFactory> TrieCursorFactory for InMemoryTrieCursorFactory< /// It will always give precedence to the data from the trie updates. #[derive(Debug)] pub struct InMemoryAccountTrieCursor<'a, C> { - /// The database cursor. + /// The underlying cursor. cursor: C, /// Forward-only in-memory cursor over storage trie nodes. in_memory_cursor: ForwardInMemoryCursor<'a, Nibbles, BranchNodeCompact>, @@ -61,7 +61,9 @@ pub struct InMemoryAccountTrieCursor<'a, C> { } impl<'a, C: TrieCursor> InMemoryAccountTrieCursor<'a, C> { - const fn new(cursor: C, trie_updates: &'a TrieUpdatesSorted) -> Self { + /// Create new account trie cursor from underlying cursor and reference to + /// [`TrieUpdatesSorted`]. + pub const fn new(cursor: C, trie_updates: &'a TrieUpdatesSorted) -> Self { let in_memory_cursor = ForwardInMemoryCursor::new(&trie_updates.account_nodes); Self { cursor, @@ -160,7 +162,7 @@ impl TrieCursor for InMemoryAccountTrieCursor<'_, C> { pub struct InMemoryStorageTrieCursor<'a, C> { /// The hashed address of the account that trie belongs to. hashed_address: B256, - /// The database cursor. + /// The underlying cursor. cursor: C, /// Forward-only in-memory cursor over storage trie nodes. in_memory_cursor: Option>, @@ -173,7 +175,13 @@ pub struct InMemoryStorageTrieCursor<'a, C> { } impl<'a, C> InMemoryStorageTrieCursor<'a, C> { - fn new(hashed_address: B256, cursor: C, updates: Option<&'a StorageTrieUpdatesSorted>) -> Self { + /// Create new storage trie cursor from underlying cursor and reference to + /// [`StorageTrieUpdatesSorted`]. + pub fn new( + hashed_address: B256, + cursor: C, + updates: Option<&'a StorageTrieUpdatesSorted>, + ) -> Self { let in_memory_cursor = updates.map(|u| ForwardInMemoryCursor::new(&u.storage_nodes)); let removed_nodes = updates.map(|u| &u.removed_nodes); let storage_trie_cleared = updates.map_or(false, |u| u.is_deleted); From e3b316257e749bb32ff3ec09960b6433fc5d6e13 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 10 Oct 2024 12:28:57 +0200 Subject: [PATCH 102/159] feat(trie): noop hashed cursors (#11627) --- crates/trie/trie/src/hashed_cursor/mod.rs | 3 + crates/trie/trie/src/hashed_cursor/noop.rs | 65 ++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 crates/trie/trie/src/hashed_cursor/noop.rs diff --git a/crates/trie/trie/src/hashed_cursor/mod.rs b/crates/trie/trie/src/hashed_cursor/mod.rs index 9b539cdce7d1e..1101f507888f7 100644 --- a/crates/trie/trie/src/hashed_cursor/mod.rs +++ b/crates/trie/trie/src/hashed_cursor/mod.rs @@ -6,6 +6,9 @@ use reth_storage_errors::db::DatabaseError; mod post_state; pub use post_state::*; +/// Implementation of noop hashed state cursor. +pub mod noop; + /// The factory trait for creating cursors over the hashed state. pub trait HashedCursorFactory { /// The hashed account cursor type. diff --git a/crates/trie/trie/src/hashed_cursor/noop.rs b/crates/trie/trie/src/hashed_cursor/noop.rs new file mode 100644 index 0000000000000..4783d5afd9d41 --- /dev/null +++ b/crates/trie/trie/src/hashed_cursor/noop.rs @@ -0,0 +1,65 @@ +use super::{HashedCursor, HashedCursorFactory, HashedStorageCursor}; +use alloy_primitives::{B256, U256}; +use reth_primitives::Account; +use reth_storage_errors::db::DatabaseError; + +/// Noop hashed cursor factory. +#[derive(Default, Debug)] +#[non_exhaustive] +pub struct NoopHashedCursorFactory; + +impl HashedCursorFactory for NoopHashedCursorFactory { + type AccountCursor = NoopHashedAccountCursor; + type StorageCursor = NoopHashedStorageCursor; + + fn hashed_account_cursor(&self) -> Result { + Ok(NoopHashedAccountCursor::default()) + } + + fn hashed_storage_cursor( + &self, + _hashed_address: B256, + ) -> Result { + Ok(NoopHashedStorageCursor::default()) + } +} + +/// Noop account hashed cursor. +#[derive(Default, Debug)] +#[non_exhaustive] +pub struct NoopHashedAccountCursor; + +impl HashedCursor for NoopHashedAccountCursor { + type Value = Account; + + fn next(&mut self) -> Result, DatabaseError> { + Ok(None) + } + + fn seek(&mut self, _key: B256) -> Result, DatabaseError> { + Ok(None) + } +} + +/// Noop account hashed cursor. +#[derive(Default, Debug)] +#[non_exhaustive] +pub struct NoopHashedStorageCursor; + +impl HashedCursor for NoopHashedStorageCursor { + type Value = U256; + + fn next(&mut self) -> Result, DatabaseError> { + Ok(None) + } + + fn seek(&mut self, _key: B256) -> Result, DatabaseError> { + Ok(None) + } +} + +impl HashedStorageCursor for NoopHashedStorageCursor { + fn is_storage_empty(&mut self) -> Result { + Ok(true) + } +} From 7010b6e5fd853e1e7aa3f6e4f4cdadd44e924548 Mon Sep 17 00:00:00 2001 From: Steven <112043913+stevencartavia@users.noreply.github.com> Date: Thu, 10 Oct 2024 04:31:57 -0600 Subject: [PATCH 103/159] Remove duplicate EngineTypes comment (#11624) --- crates/node/types/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/node/types/src/lib.rs b/crates/node/types/src/lib.rs index 0f6d5a1a370f3..2c72e02d3edc7 100644 --- a/crates/node/types/src/lib.rs +++ b/crates/node/types/src/lib.rs @@ -40,7 +40,6 @@ pub trait NodeTypes: Send + Sync + Unpin + 'static { pub trait NodeTypesWithEngine: NodeTypes { /// The node's engine types, defining the interaction with the consensus engine. type Engine: EngineTypes; - // type Engine: EngineTypes; } /// A helper trait that is downstream of the [`NodeTypesWithEngine`] trait and adds database to the From ab78a82d7ff2a6e60675c8e33d98fe420ed0bf4c Mon Sep 17 00:00:00 2001 From: cody-wang-cb Date: Thu, 10 Oct 2024 07:12:27 -0400 Subject: [PATCH 104/159] Broadcast external IP from NAT in enode record (#10274) Co-authored-by: Matthias Seitz --- book/cli/reth/debug/execution.md | 3 +++ book/cli/reth/debug/in-memory-merkle.md | 3 +++ book/cli/reth/debug/merkle.md | 3 +++ book/cli/reth/debug/replay-engine.md | 3 +++ book/cli/reth/node.md | 3 +++ book/cli/reth/p2p.md | 3 +++ book/cli/reth/stage/run.md | 3 +++ book/cli/reth/stage/unwind.md | 3 +++ crates/net/nat/src/lib.rs | 8 ++++++ crates/net/network/src/config.rs | 22 ++++++++++++++++- crates/net/network/src/manager.rs | 2 ++ crates/net/network/src/network.rs | 17 +++++++++---- crates/net/network/tests/it/startup.rs | 33 +++++++++++++++++++++++-- crates/node/core/src/args/network.rs | 11 ++++++++- 14 files changed, 108 insertions(+), 9 deletions(-) diff --git a/book/cli/reth/debug/execution.md b/book/cli/reth/debug/execution.md index 76755a1121e17..202e1452a8aef 100644 --- a/book/cli/reth/debug/execution.md +++ b/book/cli/reth/debug/execution.md @@ -82,6 +82,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/debug/in-memory-merkle.md b/book/cli/reth/debug/in-memory-merkle.md index 19061826da5d6..534e6d46c69d6 100644 --- a/book/cli/reth/debug/in-memory-merkle.md +++ b/book/cli/reth/debug/in-memory-merkle.md @@ -82,6 +82,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/debug/merkle.md b/book/cli/reth/debug/merkle.md index bdcbc820f6233..19bc38acceb9d 100644 --- a/book/cli/reth/debug/merkle.md +++ b/book/cli/reth/debug/merkle.md @@ -82,6 +82,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/debug/replay-engine.md b/book/cli/reth/debug/replay-engine.md index 3d3ae85d72a14..7a14b9cf09d4d 100644 --- a/book/cli/reth/debug/replay-engine.md +++ b/book/cli/reth/debug/replay-engine.md @@ -82,6 +82,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/node.md b/book/cli/reth/node.md index 0e89667644633..231e665cb8d96 100644 --- a/book/cli/reth/node.md +++ b/book/cli/reth/node.md @@ -74,6 +74,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/p2p.md b/book/cli/reth/p2p.md index 475e6bdbbb2ca..01253705b2336 100644 --- a/book/cli/reth/p2p.md +++ b/book/cli/reth/p2p.md @@ -59,6 +59,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/stage/run.md b/book/cli/reth/stage/run.md index 5a7a13d37ed4e..bfe5ff9d6c638 100644 --- a/book/cli/reth/stage/run.md +++ b/book/cli/reth/stage/run.md @@ -125,6 +125,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/book/cli/reth/stage/unwind.md b/book/cli/reth/stage/unwind.md index 2ca3303648c76..d181b3bcade6b 100644 --- a/book/cli/reth/stage/unwind.md +++ b/book/cli/reth/stage/unwind.md @@ -87,6 +87,9 @@ Networking: --enable-discv5-discovery Enable Discv5 discovery + --disable-nat + Disable Nat discovery + --discovery.addr The UDP address to use for devp2p peer discovery version 4 diff --git a/crates/net/nat/src/lib.rs b/crates/net/nat/src/lib.rs index b49677ef87c29..600ba97cd2dda 100644 --- a/crates/net/nat/src/lib.rs +++ b/crates/net/nat/src/lib.rs @@ -61,6 +61,14 @@ impl NatResolver { pub async fn external_addr(self) -> Option { external_addr_with(self).await } + + /// Returns the external ip, if it is [`NatResolver::ExternalIp`] + pub const fn as_external_ip(self) -> Option { + match self { + Self::ExternalIp(ip) => Some(ip), + _ => None, + } + } } impl fmt::Display for NatResolver { diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index 9d9183edcf986..72627f5b657d1 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -81,6 +81,8 @@ pub struct NetworkConfig { pub tx_gossip_disabled: bool, /// How to instantiate transactions manager. pub transactions_manager_config: TransactionsManagerConfig, + /// The NAT resolver for external IP + pub nat: Option, } // === impl NetworkConfig === @@ -197,6 +199,8 @@ pub struct NetworkConfigBuilder { block_import: Option>, /// How to instantiate transactions manager. transactions_manager_config: TransactionsManagerConfig, + /// The NAT resolver for external IP + nat: Option, } // === impl NetworkConfigBuilder === @@ -228,6 +232,7 @@ impl NetworkConfigBuilder { tx_gossip_disabled: false, block_import: None, transactions_manager_config: Default::default(), + nat: None, } } @@ -369,6 +374,7 @@ impl NetworkConfigBuilder { self.discovery_v4_builder .get_or_insert_with(Discv4Config::builder) .external_ip_resolver(Some(resolver)); + self.nat = Some(resolver); self } @@ -417,9 +423,15 @@ impl NetworkConfigBuilder { self } + // Disable nat + pub const fn disable_nat(mut self) -> Self { + self.nat = None; + self + } + /// Disables all discovery. pub fn disable_discovery(self) -> Self { - self.disable_discv4_discovery().disable_dns_discovery() + self.disable_discv4_discovery().disable_dns_discovery().disable_nat() } /// Disables all discovery if the given condition is true. @@ -485,6 +497,12 @@ impl NetworkConfigBuilder { self.build(NoopBlockReader::new(chain_spec)) } + /// Sets the NAT resolver for external IP. + pub const fn add_nat(mut self, nat: Option) -> Self { + self.nat = nat; + self + } + /// Consumes the type and creates the actual [`NetworkConfig`] /// for the given client type that can interact with the chain. /// @@ -515,6 +533,7 @@ impl NetworkConfigBuilder { tx_gossip_disabled, block_import, transactions_manager_config, + nat, } = self; discovery_v5_builder = discovery_v5_builder.map(|mut builder| { @@ -581,6 +600,7 @@ impl NetworkConfigBuilder { fork_filter, tx_gossip_disabled, transactions_manager_config, + nat, } } } diff --git a/crates/net/network/src/manager.rs b/crates/net/network/src/manager.rs index e8ffb81c5e1a6..3a7f94985fc93 100644 --- a/crates/net/network/src/manager.rs +++ b/crates/net/network/src/manager.rs @@ -188,6 +188,7 @@ impl NetworkManager { extra_protocols, tx_gossip_disabled, transactions_manager_config: _, + nat, } = config; let peers_manager = PeersManager::new(peers_config); @@ -267,6 +268,7 @@ impl NetworkManager { discv4, discv5, event_sender.clone(), + nat, ); Ok(Self { diff --git a/crates/net/network/src/network.rs b/crates/net/network/src/network.rs index de3a0ab7432bb..594ad4d155de4 100644 --- a/crates/net/network/src/network.rs +++ b/crates/net/network/src/network.rs @@ -9,7 +9,7 @@ use std::{ use alloy_primitives::B256; use enr::Enr; use parking_lot::Mutex; -use reth_discv4::Discv4; +use reth_discv4::{Discv4, NatResolver}; use reth_discv5::Discv5; use reth_eth_wire::{DisconnectReason, NewBlock, NewPooledTransactionHashes, SharedTransactions}; use reth_network_api::{ @@ -65,6 +65,7 @@ impl NetworkHandle { discv4: Option, discv5: Option, event_sender: EventSender, + nat: Option, ) -> Self { let inner = NetworkInner { num_active_peers, @@ -81,6 +82,7 @@ impl NetworkHandle { discv4, discv5, event_sender, + nat, }; Self { inner: Arc::new(inner) } } @@ -216,10 +218,13 @@ impl PeersInfo for NetworkHandle { } else if let Some(record) = self.inner.discv5.as_ref().and_then(|d| d.node_record()) { record } else { - let id = *self.peer_id(); - let mut socket_addr = *self.inner.listener_address.lock(); + let external_ip = self.inner.nat.and_then(|nat| nat.as_external_ip()); - if socket_addr.ip().is_unspecified() { + let mut socket_addr = *self.inner.listener_address.lock(); + if let Some(ip) = external_ip { + // if able to resolve external ip, use it instead and also set the local address + socket_addr.set_ip(ip) + } else if socket_addr.ip().is_unspecified() { // zero address is invalid if socket_addr.ip().is_ipv4() { socket_addr.set_ip(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); @@ -228,7 +233,7 @@ impl PeersInfo for NetworkHandle { } } - NodeRecord::new(socket_addr, id) + NodeRecord::new(socket_addr, *self.peer_id()) } } @@ -432,6 +437,8 @@ struct NetworkInner { discv5: Option, /// Sender for high level network events. event_sender: EventSender, + /// The NAT resolver + nat: Option, } /// Provides access to modify the network's additional protocol handlers. diff --git a/crates/net/network/tests/it/startup.rs b/crates/net/network/tests/it/startup.rs index 8f7e4ff8c78ce..89889a8694608 100644 --- a/crates/net/network/tests/it/startup.rs +++ b/crates/net/network/tests/it/startup.rs @@ -1,10 +1,10 @@ use std::{ io, - net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}, }; use reth_chainspec::MAINNET; -use reth_discv4::Discv4Config; +use reth_discv4::{Discv4Config, NatResolver}; use reth_network::{ error::{NetworkError, ServiceKind}, Discovery, NetworkConfigBuilder, NetworkManager, @@ -105,3 +105,32 @@ async fn test_tcp_port_node_record_discovery() { assert_eq!(record.tcp_port, local_addr.port()); assert_ne!(record.udp_port, 0); } + +#[tokio::test(flavor = "multi_thread")] +async fn test_node_record_address_with_nat() { + let secret_key = SecretKey::new(&mut rand::thread_rng()); + let config = NetworkConfigBuilder::new(secret_key) + .add_nat(Some(NatResolver::ExternalIp("10.1.1.1".parse().unwrap()))) + .disable_discv4_discovery() + .disable_dns_discovery() + .build_with_noop_provider(MAINNET.clone()); + + let network = NetworkManager::new(config).await.unwrap(); + let record = network.handle().local_node_record(); + + assert_eq!(record.address, IpAddr::V4(Ipv4Addr::new(10, 1, 1, 1))); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_node_record_address_with_nat_disable_discovery() { + let secret_key = SecretKey::new(&mut rand::thread_rng()); + let config = NetworkConfigBuilder::new(secret_key) + .add_nat(Some(NatResolver::ExternalIp("10.1.1.1".parse().unwrap()))) + .disable_discovery() + .build_with_noop_provider(MAINNET.clone()); + + let network = NetworkManager::new(config).await.unwrap(); + let record = network.handle().local_node_record(); + + assert_eq!(record.address, IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); +} diff --git a/crates/node/core/src/args/network.rs b/crates/node/core/src/args/network.rs index 04153b93ecdd4..d359cfb3f1afb 100644 --- a/crates/node/core/src/args/network.rs +++ b/crates/node/core/src/args/network.rs @@ -172,7 +172,7 @@ impl NetworkArgs { DEFAULT_DISCOVERY_ADDR } - } + }; } self.addr @@ -355,6 +355,10 @@ pub struct DiscoveryArgs { #[arg(long, conflicts_with = "disable_discovery")] pub enable_discv5_discovery: bool, + /// Disable Nat discovery. + #[arg(long, conflicts_with = "disable_discovery")] + pub disable_nat: bool, + /// The UDP address to use for devp2p peer discovery version 4. #[arg(id = "discovery.addr", long = "discovery.addr", value_name = "DISCOVERY_ADDR", default_value_t = DEFAULT_DISCOVERY_ADDR)] pub addr: IpAddr, @@ -418,6 +422,10 @@ impl DiscoveryArgs { network_config_builder = network_config_builder.disable_discv4_discovery(); } + if self.disable_discovery || self.disable_nat { + network_config_builder = network_config_builder.disable_nat(); + } + if !self.disable_discovery && self.enable_discv5_discovery { network_config_builder = network_config_builder .discovery_v5(self.discovery_v5_builder(rlpx_tcp_socket, boot_nodes)); @@ -494,6 +502,7 @@ impl Default for DiscoveryArgs { disable_dns_discovery: false, disable_discv4_discovery: false, enable_discv5_discovery: false, + disable_nat: false, addr: DEFAULT_DISCOVERY_ADDR, port: DEFAULT_DISCOVERY_PORT, discv5_addr: None, From f5f768b86bba368afc1eca4555fc295a7a79ec1f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 14:04:18 +0200 Subject: [PATCH 105/159] docs: clarify op-mainnet --debug.tip (#11634) --- book/run/sync-op-mainnet.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/book/run/sync-op-mainnet.md b/book/run/sync-op-mainnet.md index 057860c334943..2a862314a1d53 100644 --- a/book/run/sync-op-mainnet.md +++ b/book/run/sync-op-mainnet.md @@ -9,9 +9,15 @@ To sync OP mainnet, bedrock state needs to be imported as a starting point. Ther **The state snapshot at Bedrock block is required.** It can be exported from [op-geth](https://github.com/testinprod-io/op-erigon/blob/pcw109550/bedrock-db-migration/bedrock-migration.md#export-state) (**.jsonl**) or downloaded directly from [here](https://mega.nz/file/GdZ1xbAT#a9cBv3AqzsTGXYgX7nZc_3fl--tcBmOAIwIA5ND6kwc). +Import the state snapshot + ```sh $ op-reth init-state --without-ovm --chain optimism --datadir op-mainnet world_trie_state.jsonl +``` +Sync the node to a recent finalized block (e.g. 125200000) to catch up close to the tip, before pairing with op-node. + +```sh $ op-reth node --chain optimism --datadir op-mainnet --debug.tip 0x098f87b75c8b861c775984f9d5dbe7b70cbbbc30fc15adb03a5044de0144f2d0 # block #125200000 ``` From ac77fe648d94ad1ab367705e4a16cd9d151a9ecb Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Thu, 10 Oct 2024 14:24:24 +0200 Subject: [PATCH 106/159] feat(ci): add Kurtosis workflow (#11615) --- .github/assets/kurtosis_network_params.yaml | 15 ++++ .github/workflows/kurtosis.yml | 95 +++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 .github/assets/kurtosis_network_params.yaml create mode 100644 .github/workflows/kurtosis.yml diff --git a/.github/assets/kurtosis_network_params.yaml b/.github/assets/kurtosis_network_params.yaml new file mode 100644 index 0000000000000..9c104de495005 --- /dev/null +++ b/.github/assets/kurtosis_network_params.yaml @@ -0,0 +1,15 @@ +participants: + - el_type: geth + cl_type: lighthouse + - el_type: reth + el_extra_params: + - --engine.experimental + el_image: "ghcr.io/paradigmxyz/reth:kurtosis-ci" + cl_type: teku +additional_services: + - assertoor +assertoor_params: + run_block_proposal_check: true + run_transaction_test: true + run_blob_transaction_test: true + run_opcodes_transaction_test: true diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml new file mode 100644 index 0000000000000..74d26dbd3eeaf --- /dev/null +++ b/.github/workflows/kurtosis.yml @@ -0,0 +1,95 @@ +# Runs `assertoor` tests on a `kurtosis` testnet. + +name: kurtosis + +on: + workflow_dispatch: + schedule: + # every day + - cron: "0 1 * * *" + +env: + CARGO_TERM_COLOR: always + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + prepare-reth: + if: github.repository == 'paradigmxyz/reth' + timeout-minutes: 45 + runs-on: + group: Reth + steps: + - uses: actions/checkout@v4 + - run: mkdir artifacts + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + - name: Build reth + run: | + cargo build --features asm-keccak --profile hivetests --bin reth --locked + mkdir dist && cp ./target/hivetests/reth ./dist/reth + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and export reth image + uses: docker/build-push-action@v6 + with: + context: . + file: .github/assets/hive/Dockerfile + tags: ghcr.io/paradigmxyz/reth:kurtosis-ci + outputs: type=docker,dest=./artifacts/reth_image.tar + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Upload reth image + uses: actions/upload-artifact@v4 + with: + name: artifacts + path: ./artifacts + + test: + timeout-minutes: 60 + strategy: + fail-fast: false + name: run kurtosis + runs-on: + group: Reth + needs: + - prepare-reth + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download reth image + uses: actions/download-artifact@v4 + with: + name: artifacts + path: /tmp + + - name: Load Docker image + run: | + docker load -i /tmp/reth_image.tar & + wait + docker image ls -a + + - name: Run kurtosis + uses: ethpandaops/kurtosis-assertoor-github-action@v1 + with: + ethereum_package_args: '.github/assets/kurtosis_network_params.yaml' + + notify-on-error: + needs: test + if: failure() + runs-on: + group: Reth + steps: + - name: Slack Webhook Action + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}" + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} From c035f906164f41793def51990e5dd2566ce2abcb Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 10 Oct 2024 14:13:14 +0100 Subject: [PATCH 107/159] feat(exex): commit only notifications with unfinalized blocks to WAL (#11638) --- crates/exex/exex/src/manager.rs | 124 ++++++++++++++----- crates/node/builder/src/launch/exex.rs | 6 +- crates/stages/stages/src/stages/execution.rs | 15 ++- 3 files changed, 108 insertions(+), 37 deletions(-) diff --git a/crates/exex/exex/src/manager.rs b/crates/exex/exex/src/manager.rs index 31dc822222b63..8c1518f3090f1 100644 --- a/crates/exex/exex/src/manager.rs +++ b/crates/exex/exex/src/manager.rs @@ -34,6 +34,18 @@ use tokio_util::sync::{PollSendError, PollSender, ReusableBoxFuture}; /// or 17 minutes of 1-second blocks. pub const DEFAULT_EXEX_MANAGER_CAPACITY: usize = 1024; +/// The source of the notification. +/// +/// This distinguishment is needed to not commit any pipeline notificatations to [WAL](`Wal`), +/// because they are already finalized. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExExNotificationSource { + /// The notification was sent from the pipeline. + Pipeline, + /// The notification was sent from the blockchain tree. + BlockchainTree, +} + /// Metrics for an `ExEx`. #[derive(Metrics)] #[metrics(scope = "exex")] @@ -197,7 +209,7 @@ pub struct ExExManager

{ exex_handles: Vec, /// [`ExExNotification`] channel from the [`ExExManagerHandle`]s. - handle_rx: UnboundedReceiver, + handle_rx: UnboundedReceiver<(ExExNotificationSource, ExExNotification)>, /// The minimum notification ID currently present in the buffer. min_id: usize, @@ -429,14 +441,23 @@ where // Drain handle notifications while this.buffer.len() < this.max_capacity { - if let Poll::Ready(Some(notification)) = this.handle_rx.poll_recv(cx) { - debug!( - target: "exex::manager", - committed_tip = ?notification.committed_chain().map(|chain| chain.tip().number), - reverted_tip = ?notification.reverted_chain().map(|chain| chain.tip().number), - "Received new notification" - ); - this.wal.commit(¬ification)?; + if let Poll::Ready(Some((source, notification))) = this.handle_rx.poll_recv(cx) { + let committed_tip = notification.committed_chain().map(|chain| chain.tip().number); + let reverted_tip = notification.reverted_chain().map(|chain| chain.tip().number); + debug!(target: "exex::manager", ?committed_tip, ?reverted_tip, "Received new notification"); + + // Commit to WAL only notifications from blockchain tree. Pipeline notifications + // always contain only finalized blocks. + match source { + ExExNotificationSource::BlockchainTree => { + debug!(target: "exex::manager", ?committed_tip, ?reverted_tip, "Committing notification to WAL"); + this.wal.commit(¬ification)?; + } + ExExNotificationSource::Pipeline => { + debug!(target: "exex::manager", ?committed_tip, ?reverted_tip, "Notification was sent from pipeline, skipping WAL commit"); + } + } + this.push_notification(notification); continue } @@ -491,7 +512,7 @@ where #[derive(Debug)] pub struct ExExManagerHandle { /// Channel to send notifications to the `ExEx` manager. - exex_tx: UnboundedSender, + exex_tx: UnboundedSender<(ExExNotificationSource, ExExNotification)>, /// The number of `ExEx`'s running on the node. num_exexs: usize, /// A watch channel denoting whether the manager is ready for new notifications or not. @@ -533,8 +554,12 @@ impl ExExManagerHandle { /// Synchronously send a notification over the channel to all execution extensions. /// /// Senders should call [`Self::has_capacity`] first. - pub fn send(&self, notification: ExExNotification) -> Result<(), SendError> { - self.exex_tx.send(notification) + pub fn send( + &self, + source: ExExNotificationSource, + notification: ExExNotification, + ) -> Result<(), SendError<(ExExNotificationSource, ExExNotification)>> { + self.exex_tx.send((source, notification)) } /// Asynchronously send a notification over the channel to all execution extensions. @@ -543,10 +568,11 @@ impl ExExManagerHandle { /// capacity in the channel, the future will wait. pub async fn send_async( &mut self, + source: ExExNotificationSource, notification: ExExNotification, - ) -> Result<(), SendError> { + ) -> Result<(), SendError<(ExExNotificationSource, ExExNotification)>> { self.ready().await; - self.exex_tx.send(notification) + self.exex_tx.send((source, notification)) } /// Get the current capacity of the `ExEx` manager's internal notification buffer. @@ -610,16 +636,16 @@ impl Clone for ExExManagerHandle { mod tests { use super::*; use alloy_primitives::B256; - use futures::StreamExt; + use futures::{StreamExt, TryStreamExt}; use rand::Rng; use reth_db_common::init::init_genesis; use reth_evm_ethereum::execute::EthExecutorProvider; use reth_primitives::SealedBlockWithSenders; use reth_provider::{ providers::BlockchainProvider2, test_utils::create_test_provider_factory, BlockReader, - Chain, TransactionVariant, + BlockWriter, Chain, DatabaseProviderFactory, TransactionVariant, }; - use reth_testing_utils::generators; + use reth_testing_utils::generators::{self, random_block, BlockParams}; fn empty_finalized_header_stream() -> ForkChoiceStream { let (tx, rx) = watch::channel(None); @@ -959,9 +985,21 @@ mod tests { }; // Send notifications to go over the max capacity - exex_manager.handle.exex_tx.send(notification.clone()).unwrap(); - exex_manager.handle.exex_tx.send(notification.clone()).unwrap(); - exex_manager.handle.exex_tx.send(notification).unwrap(); + exex_manager + .handle + .exex_tx + .send((ExExNotificationSource::BlockchainTree, notification.clone())) + .unwrap(); + exex_manager + .handle + .exex_tx + .send((ExExNotificationSource::BlockchainTree, notification.clone())) + .unwrap(); + exex_manager + .handle + .exex_tx + .send((ExExNotificationSource::BlockchainTree, notification)) + .unwrap(); // Pin the ExExManager to call the poll method let mut pinned_manager = std::pin::pin!(exex_manager); @@ -1177,6 +1215,18 @@ mod tests { .sealed_block_with_senders(genesis_hash.into(), TransactionVariant::NoHash) .unwrap() .ok_or_else(|| eyre::eyre!("genesis block not found"))?; + + let block = random_block( + &mut rng, + genesis_block.number + 1, + BlockParams { parent: Some(genesis_hash), ..Default::default() }, + ) + .seal_with_senders() + .unwrap(); + let provider_rw = provider_factory.database_provider_rw().unwrap(); + provider_rw.insert_block(block.clone()).unwrap(); + provider_rw.commit().unwrap(); + let provider = BlockchainProvider2::new(provider_factory).unwrap(); let temp_dir = tempfile::tempdir().unwrap(); @@ -1190,33 +1240,49 @@ mod tests { wal.handle(), ); - let notification = ExExNotification::ChainCommitted { + let genesis_notification = ExExNotification::ChainCommitted { new: Arc::new(Chain::new(vec![genesis_block.clone()], Default::default(), None)), }; + let notification = ExExNotification::ChainCommitted { + new: Arc::new(Chain::new(vec![block.clone()], Default::default(), None)), + }; let (finalized_headers_tx, rx) = watch::channel(None); + finalized_headers_tx.send(Some(genesis_block.header.clone()))?; let finalized_header_stream = ForkChoiceStream::new(rx); let mut exex_manager = std::pin::pin!(ExExManager::new( provider, vec![exex_handle], - 1, + 2, wal, finalized_header_stream )); let mut cx = Context::from_waker(futures::task::noop_waker_ref()); - exex_manager.handle().send(notification.clone())?; + exex_manager + .handle() + .send(ExExNotificationSource::Pipeline, genesis_notification.clone())?; + exex_manager.handle().send(ExExNotificationSource::BlockchainTree, notification.clone())?; assert!(exex_manager.as_mut().poll(&mut cx)?.is_pending()); - assert_eq!(notifications.next().await.unwrap().unwrap(), notification.clone()); + assert_eq!( + notifications.try_poll_next_unpin(&mut cx)?, + Poll::Ready(Some(genesis_notification)) + ); + assert!(exex_manager.as_mut().poll(&mut cx)?.is_pending()); + assert_eq!( + notifications.try_poll_next_unpin(&mut cx)?, + Poll::Ready(Some(notification.clone())) + ); + // WAL shouldn't contain the genesis notification, because it's finalized assert_eq!( exex_manager.wal.iter_notifications()?.collect::>>()?, [notification.clone()] ); - finalized_headers_tx.send(Some(genesis_block.header.clone()))?; + finalized_headers_tx.send(Some(block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL isn't finalized because the ExEx didn't emit the `FinishedHeight` event assert_eq!( @@ -1229,7 +1295,7 @@ mod tests { .send(ExExEvent::FinishedHeight((rng.gen::(), rng.gen::()).into())) .unwrap(); - finalized_headers_tx.send(Some(genesis_block.header.clone()))?; + finalized_headers_tx.send(Some(block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL isn't finalized because the ExEx emitted a `FinishedHeight` event with a // non-canonical block @@ -1239,12 +1305,12 @@ mod tests { ); // Send a `FinishedHeight` event with a canonical block - events_tx.send(ExExEvent::FinishedHeight(genesis_block.num_hash())).unwrap(); + events_tx.send(ExExEvent::FinishedHeight(block.num_hash())).unwrap(); - finalized_headers_tx.send(Some(genesis_block.header.clone()))?; + finalized_headers_tx.send(Some(block.header.clone()))?; assert!(exex_manager.as_mut().poll(&mut cx).is_pending()); // WAL is finalized - assert!(exex_manager.wal.iter_notifications()?.next().is_none()); + assert_eq!(exex_manager.wal.iter_notifications()?.next().transpose()?, None); Ok(()) } diff --git a/crates/node/builder/src/launch/exex.rs b/crates/node/builder/src/launch/exex.rs index 5441f393d3c94..a3640690c1dc4 100644 --- a/crates/node/builder/src/launch/exex.rs +++ b/crates/node/builder/src/launch/exex.rs @@ -6,7 +6,8 @@ use futures::future; use reth_chain_state::ForkChoiceSubscriptions; use reth_chainspec::EthChainSpec; use reth_exex::{ - ExExContext, ExExHandle, ExExManager, ExExManagerHandle, Wal, DEFAULT_EXEX_MANAGER_CAPACITY, + ExExContext, ExExHandle, ExExManager, ExExManagerHandle, ExExNotificationSource, Wal, + DEFAULT_EXEX_MANAGER_CAPACITY, }; use reth_node_api::{FullNodeComponents, NodeTypes}; use reth_primitives::Head; @@ -47,6 +48,7 @@ impl ExExLauncher { return Ok(None) } + info!(target: "reth::cli", "Loading ExEx Write-Ahead Log..."); let exex_wal = Wal::new( config_container .config @@ -127,7 +129,7 @@ impl ExExLauncher { async move { while let Ok(notification) = canon_state_notifications.recv().await { handle - .send_async(notification.into()) + .send_async(ExExNotificationSource::BlockchainTree, notification.into()) .await .expect("blockchain tree notification could not be sent to exex manager"); } diff --git a/crates/stages/stages/src/stages/execution.rs b/crates/stages/stages/src/stages/execution.rs index df234e542be63..7bb6ebc59e09b 100644 --- a/crates/stages/stages/src/stages/execution.rs +++ b/crates/stages/stages/src/stages/execution.rs @@ -9,7 +9,7 @@ use reth_evm::{ metrics::ExecutorMetrics, }; use reth_execution_types::{Chain, ExecutionOutcome}; -use reth_exex::{ExExManagerHandle, ExExNotification}; +use reth_exex::{ExExManagerHandle, ExExNotification, ExExNotificationSource}; use reth_primitives::{Header, SealedHeader, StaticFileSegment}; use reth_primitives_traits::format_gas_throughput; use reth_provider::{ @@ -389,9 +389,10 @@ where // NOTE: We can ignore the error here, since an error means that the channel is closed, // which means the manager has died, which then in turn means the node is shutting down. - let _ = self - .exex_manager_handle - .send(ExExNotification::ChainCommitted { new: Arc::new(chain) }); + let _ = self.exex_manager_handle.send( + ExExNotificationSource::Pipeline, + ExExNotification::ChainCommitted { new: Arc::new(chain) }, + ); Ok(()) } @@ -477,8 +478,10 @@ where // NOTE: We can ignore the error here, since an error means that the channel is closed, // which means the manager has died, which then in turn means the node is shutting down. - let _ = - self.exex_manager_handle.send(ExExNotification::ChainReverted { old: Arc::new(chain) }); + let _ = self.exex_manager_handle.send( + ExExNotificationSource::Pipeline, + ExExNotification::ChainReverted { old: Arc::new(chain) }, + ); Ok(()) } From fa87b9859f17b9d7543763d0e8818539adae9fe3 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 10 Oct 2024 14:48:30 +0100 Subject: [PATCH 108/159] feat(ci): move book clippy and tests to matrix (#11618) --- .github/workflows/lint.yml | 25 +++++++++++++++---------- .github/workflows/unit.yml | 37 +++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e8161557fde9c..fd71a8c636e4e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,16 +11,21 @@ env: jobs: clippy-binaries: - name: clippy / ${{ matrix.network }} + name: clippy binaries / ${{ matrix.type }} runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: include: - - binary: reth - network: ethereum - - binary: op-reth - network: optimism + - type: ethereum + args: --bin reth --workspace + features: "ethereum asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" + - type: optimism + args: --bin op-reth --workspace + features: "optimism asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" + - type: book + args: --manifest-path book/sources/Cargo.toml --workspace --bins + features: "" steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@clippy @@ -30,12 +35,12 @@ jobs: - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true + - if: "${{ matrix.type == 'book' }}" + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run clippy on binaries - run: cargo clippy --bin "${{ matrix.binary }}" --workspace --features "${{ matrix.network }} asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" - env: - RUSTFLAGS: -D warnings - - name: Run clippy on book binary sources - run: cargo clippy --manifest-path book/sources/Cargo.toml --workspace --bins + run: cargo clippy ${{ matrix.args }} --features "${{ matrix.features }}" env: RUSTFLAGS: -D warnings diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 1056a6fb58c80..defd9a6f535d6 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -18,15 +18,34 @@ concurrency: jobs: test: - name: test / ${{ matrix.network }} (${{ matrix.partition }}/2) + name: test / ${{ matrix.type }} (${{ matrix.partition }}/${{ matrix.total_partitions }}) runs-on: group: Reth env: RUST_BACKTRACE: 1 strategy: matrix: - partition: [1, 2] - network: ["ethereum", "optimism"] + include: + - type: ethereum + args: --features "asm-keccak ethereum" --locked + partition: 1 + total_partitions: 2 + - type: ethereum + args: --features "asm-keccak ethereum" --locked + partition: 2 + total_partitions: 2 + - type: optimism + args: --features "asm-keccak optimism" --locked + partition: 1 + total_partitions: 2 + - type: optimism + args: --features "asm-keccak optimism" --locked + partition: 2 + total_partitions: 2 + - type: book + args: --manifest-path book/sources/Cargo.toml + partition: 1 + total_partitions: 1 timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -35,18 +54,16 @@ jobs: with: cache-on-failure: true - uses: taiki-e/install-action@nextest + - if: "${{ matrix.type == 'book' }}" + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Run tests run: | cargo nextest run \ - --locked --features "asm-keccak ${{ matrix.network }}" \ - --workspace --exclude ef-tests \ + ${{ matrix.args }} --workspace --exclude ef-tests \ --partition hash:${{ matrix.partition }}/2 \ -E "!kind(test)" - - name: Run tests on book sources - run: | - cargo nextest run \ - --manifest-path book/sources/Cargo.toml --workspace \ - -E "!kind(test)" state: name: Ethereum state tests From 7f54263356340aef484a6183f9977493e045f623 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 16:59:04 +0200 Subject: [PATCH 109/159] chore: release 1.1.0 (#11640) --- Cargo.lock | 236 ++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 119 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9bf1ea2d666da..37da7b57510b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2564,7 +2564,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -5268,7 +5268,7 @@ dependencies = [ [[package]] name = "op-reth" -version = "1.0.8" +version = "1.1.0" dependencies = [ "clap", "reth-cli-util", @@ -6230,7 +6230,7 @@ dependencies = [ [[package]] name = "reth" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6303,7 +6303,7 @@ dependencies = [ [[package]] name = "reth-auto-seal-consensus" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -6333,7 +6333,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -6356,7 +6356,7 @@ dependencies = [ [[package]] name = "reth-beacon-consensus" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -6407,7 +6407,7 @@ dependencies = [ [[package]] name = "reth-bench" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-json-rpc", @@ -6442,7 +6442,7 @@ dependencies = [ [[package]] name = "reth-blockchain-tree" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6480,7 +6480,7 @@ dependencies = [ [[package]] name = "reth-blockchain-tree-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-consensus", @@ -6492,7 +6492,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6520,7 +6520,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-eips", @@ -6541,7 +6541,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-genesis", "clap", @@ -6553,7 +6553,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "1.0.8" +version = "1.1.0" dependencies = [ "ahash", "alloy-eips", @@ -6614,7 +6614,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "1.0.8" +version = "1.1.0" dependencies = [ "reth-tasks", "tokio", @@ -6623,7 +6623,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -6640,7 +6640,7 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6663,7 +6663,7 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "1.0.8" +version = "1.1.0" dependencies = [ "convert_case", "proc-macro2", @@ -6674,7 +6674,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "eyre", @@ -6690,7 +6690,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "auto_impl", @@ -6700,7 +6700,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -6715,7 +6715,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -6738,7 +6738,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -6779,7 +6779,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -6806,7 +6806,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -6834,7 +6834,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -6850,7 +6850,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -6876,7 +6876,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -6900,7 +6900,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-primitives", @@ -6928,7 +6928,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -6965,7 +6965,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7003,7 +7003,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "1.0.8" +version = "1.1.0" dependencies = [ "aes", "alloy-primitives", @@ -7033,7 +7033,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -7063,7 +7063,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-execution-types", @@ -7075,7 +7075,7 @@ dependencies = [ [[package]] name = "reth-engine-service" -version = "1.0.8" +version = "1.1.0" dependencies = [ "futures", "pin-project", @@ -7103,7 +7103,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7150,7 +7150,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -7180,7 +7180,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "1.0.8" +version = "1.1.0" dependencies = [ "reth-blockchain-tree-api", "reth-consensus", @@ -7192,7 +7192,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7227,7 +7227,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7250,7 +7250,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "1.0.8" +version = "1.1.0" dependencies = [ "clap", "eyre", @@ -7261,7 +7261,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-chainspec", @@ -7273,7 +7273,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7292,7 +7292,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-primitives", @@ -7311,7 +7311,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-basic-payload-builder", @@ -7335,7 +7335,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "rayon", @@ -7345,7 +7345,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7367,7 +7367,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7390,7 +7390,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7405,7 +7405,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7422,7 +7422,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7467,7 +7467,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "1.0.8" +version = "1.1.0" dependencies = [ "eyre", "futures-util", @@ -7499,7 +7499,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7515,7 +7515,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "1.0.8" +version = "1.1.0" dependencies = [ "serde", "serde_json", @@ -7524,7 +7524,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7548,7 +7548,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "1.0.8" +version = "1.1.0" dependencies = [ "async-trait", "bytes", @@ -7570,7 +7570,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "1.0.8" +version = "1.1.0" dependencies = [ "bitflags 2.6.0", "byteorder", @@ -7591,7 +7591,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "1.0.8" +version = "1.1.0" dependencies = [ "bindgen 0.70.1", "cc", @@ -7599,7 +7599,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "1.0.8" +version = "1.1.0" dependencies = [ "futures", "metrics", @@ -7610,14 +7610,14 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", ] [[package]] name = "reth-net-nat" -version = "1.0.8" +version = "1.1.0" dependencies = [ "futures-util", "if-addrs", @@ -7631,7 +7631,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7691,7 +7691,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-admin", @@ -7713,7 +7713,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7733,7 +7733,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7749,7 +7749,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "humantime-serde", "reth-ethereum-forks", @@ -7762,7 +7762,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "1.0.8" +version = "1.1.0" dependencies = [ "anyhow", "bincode", @@ -7780,7 +7780,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "reth-engine-primitives", "reth-evm", @@ -7797,7 +7797,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -7861,7 +7861,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -7911,7 +7911,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -7947,7 +7947,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -7969,7 +7969,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "1.0.8" +version = "1.1.0" dependencies = [ "eyre", "http", @@ -7995,7 +7995,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "reth-chainspec", "reth-db-api", @@ -8004,7 +8004,7 @@ dependencies = [ [[package]] name = "reth-optimism-chainspec" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-genesis", @@ -8022,7 +8022,7 @@ dependencies = [ [[package]] name = "reth-optimism-cli" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8066,7 +8066,7 @@ dependencies = [ [[package]] name = "reth-optimism-consensus" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-chainspec", @@ -8081,7 +8081,7 @@ dependencies = [ [[package]] name = "reth-optimism-evm" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8107,7 +8107,7 @@ dependencies = [ [[package]] name = "reth-optimism-forks" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8118,7 +8118,7 @@ dependencies = [ [[package]] name = "reth-optimism-node" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-genesis", @@ -8171,7 +8171,7 @@ dependencies = [ [[package]] name = "reth-optimism-payload-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8204,7 +8204,7 @@ dependencies = [ [[package]] name = "reth-optimism-primitives" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-primitives", @@ -8213,7 +8213,7 @@ dependencies = [ [[package]] name = "reth-optimism-rpc" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8252,7 +8252,7 @@ dependencies = [ [[package]] name = "reth-optimism-storage" -version = "1.0.8" +version = "1.1.0" dependencies = [ "reth-codecs", "reth-db-api", @@ -8263,7 +8263,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -8284,7 +8284,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -8305,7 +8305,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-rpc-types", "reth-chainspec", @@ -8315,7 +8315,7 @@ dependencies = [ [[package]] name = "reth-primitives" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8360,7 +8360,7 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8388,7 +8388,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8438,7 +8438,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "assert_matches", @@ -8467,7 +8467,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -8487,7 +8487,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "reth-chainspec", @@ -8504,7 +8504,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -8572,7 +8572,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-json-rpc", @@ -8598,7 +8598,7 @@ dependencies = [ [[package]] name = "reth-rpc-api-testing-util" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types", @@ -8617,7 +8617,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-network", "alloy-primitives", @@ -8669,7 +8669,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8705,7 +8705,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-dyn-abi", "alloy-eips", @@ -8745,7 +8745,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8788,7 +8788,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-rpc-types-engine", "http", @@ -8803,7 +8803,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -8818,7 +8818,7 @@ dependencies = [ [[package]] name = "reth-rpc-types-compat" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8834,7 +8834,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8884,7 +8884,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "aquamarine", @@ -8912,7 +8912,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -8929,7 +8929,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "assert_matches", @@ -8954,7 +8954,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "clap", @@ -8965,7 +8965,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8983,7 +8983,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8995,7 +8995,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "1.0.8" +version = "1.1.0" dependencies = [ "auto_impl", "dyn-clone", @@ -9012,7 +9012,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9025,7 +9025,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "1.0.8" +version = "1.1.0" dependencies = [ "tokio", "tokio-stream", @@ -9034,7 +9034,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "1.0.8" +version = "1.1.0" dependencies = [ "clap", "eyre", @@ -9048,7 +9048,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9093,7 +9093,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9124,7 +9124,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -9148,7 +9148,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9183,7 +9183,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "1.0.8" +version = "1.1.0" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index 22ce3e92f6a6a..db115c89cccdd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "1.0.8" +version = "1.1.0" edition = "2021" rust-version = "1.81" license = "MIT OR Apache-2.0" From 772c1b4765477dad6f89acf0f3d0f4a9e9f5f115 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 10 Oct 2024 16:04:31 +0100 Subject: [PATCH 110/159] feat(bin): make experimental engine default (#11612) --- bin/reth/src/main.rs | 25 ++++++++++++++++++++----- book/cli/reth/node.md | 7 ++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index 7153cdcc6c885..f424163a24fed 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -14,15 +14,24 @@ use reth_node_builder::{ }; use reth_node_ethereum::{node::EthereumAddOns, EthereumNode}; use reth_provider::providers::BlockchainProvider2; +use reth_tracing::tracing::warn; +use tracing::info; /// Parameters for configuring the engine #[derive(Debug, Clone, Args, PartialEq, Eq)] #[command(next_help_heading = "Engine")] pub struct EngineArgs { - /// Enable the engine2 experimental features on reth binary + /// Enable the experimental engine features on reth binary + /// + /// DEPRECATED: experimental engine is default now, use --engine.legacy to enable the legacy + /// functionality #[arg(long = "engine.experimental", default_value = "false")] pub experimental: bool, + /// Enable the legacy engine on reth binary + #[arg(long = "engine.legacy", default_value = "false")] + pub legacy: bool, + /// Configure persistence threshold for engine experimental. #[arg(long = "engine.persistence-threshold", requires = "experimental", default_value_t = DEFAULT_PERSISTENCE_THRESHOLD)] pub persistence_threshold: u64, @@ -36,6 +45,7 @@ impl Default for EngineArgs { fn default() -> Self { Self { experimental: false, + legacy: false, persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD, memory_block_buffer_target: DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, } @@ -52,9 +62,13 @@ fn main() { if let Err(err) = Cli::::parse().run(|builder, engine_args| async move { - let enable_engine2 = engine_args.experimental; - match enable_engine2 { - true => { + if engine_args.experimental { + warn!(target: "reth::cli", "Experimental engine is default now, and the --engine.experimental flag is deprecated. To enable the legacy functionality, use --engine.legacy."); + } + + let use_legacy_engine = engine_args.legacy; + match use_legacy_engine { + false => { let engine_tree_config = TreeConfig::default() .with_persistence_threshold(engine_args.persistence_threshold) .with_memory_block_buffer_target(engine_args.memory_block_buffer_target); @@ -73,7 +87,8 @@ fn main() { .await?; handle.node_exit_future.await } - false => { + true => { + info!(target: "reth::cli", "Running with legacy engine"); let handle = builder.launch_node(EthereumNode::default()).await?; handle.node_exit_future.await } diff --git a/book/cli/reth/node.md b/book/cli/reth/node.md index 231e665cb8d96..73a8063a852b6 100644 --- a/book/cli/reth/node.md +++ b/book/cli/reth/node.md @@ -666,7 +666,12 @@ Pruning: Engine: --engine.experimental - Enable the engine2 experimental features on reth binary + Enable the experimental engine features on reth binary + + DEPRECATED: experimental engine is default now, use --engine.legacy to enable the legacy functionality + + --engine.legacy + Enable the legacy engine on reth binary --engine.persistence-threshold Configure persistence threshold for engine experimental From f9ac5e5cde4a6ff52945b5f8c952ca39d4ff2ff9 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Fri, 11 Oct 2024 00:43:58 +0900 Subject: [PATCH 111/159] chore(rpc): don't recover sender if we have it (#11645) --- crates/rpc/rpc-eth-api/src/helpers/transaction.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index c25b18c2f67b4..94dd04414874d 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -370,12 +370,9 @@ pub trait EthTransactions: LoadTransaction { let gas_limit = estimated_gas; request.set_gas_limit(gas_limit.to()); - let signed_tx = self.sign_request(&from, request).await?; + let transaction = self.sign_request(&from, request).await?.with_signer(from); - let recovered = - signed_tx.into_ecrecovered().ok_or(EthApiError::InvalidTransactionSignature)?; - - let pool_transaction = <::Pool as TransactionPool>::Transaction::try_from_consensus(recovered.into()).map_err(|_| EthApiError::TransactionConversionError)?; + let pool_transaction = <::Pool as TransactionPool>::Transaction::try_from_consensus(transaction.into()).map_err(|_| EthApiError::TransactionConversionError)?; // submit the transaction to the pool with a `Local` origin let hash = LoadTransaction::pool(self) From a336efdee52dd86912a77cbd29d8ee683f929ffb Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 18:54:06 +0200 Subject: [PATCH 112/159] fix: don't unwrap missing requests (#11646) --- crates/storage/codecs/src/alloy/header.rs | 10 ++++++++++ .../provider/src/providers/database/provider.rs | 12 ++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/crates/storage/codecs/src/alloy/header.rs b/crates/storage/codecs/src/alloy/header.rs index 526bc69b13696..3a17ed1fdcd2c 100644 --- a/crates/storage/codecs/src/alloy/header.rs +++ b/crates/storage/codecs/src/alloy/header.rs @@ -181,4 +181,14 @@ mod tests { let len = header.to_compact(&mut encoded_header); assert_eq!(header, Header::from_compact(&encoded_header, len).0); } + + #[test] + fn test_extra_fields_missing() { + let mut header = HOLESKY_BLOCK; + header.extra_fields = None; + + let mut encoded_header = vec![]; + let len = header.to_compact(&mut encoded_header); + assert_eq!(header, Header::from_compact(&encoded_header, len).0); + } } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 1afd4da3fa8cc..767f92db98f2f 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -2151,10 +2151,8 @@ impl RequestsProvider ) -> ProviderResult> { if self.chain_spec.is_prague_active_at_timestamp(timestamp) { if let Some(number) = self.convert_hash_or_number(id)? { - // If we are past Prague, then all blocks should have a requests list, even if - // empty - let requests = self.tx.get::(number)?.unwrap_or_default(); - return Ok(Some(requests)) + let requests = self.tx.get::(number)?; + return Ok(requests) } } Ok(None) @@ -3483,10 +3481,8 @@ impl(block_number, requests)?; - durations_recorder.record_relative(metrics::Action::InsertBlockRequests); - } + self.tx.put::(block_number, requests)?; + durations_recorder.record_relative(metrics::Action::InsertBlockRequests); } let block_indices = StoredBlockBodyIndices { first_tx_num, tx_count }; From b362a8ed2216187c4021201056ee14f11345f6eb Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 19:15:06 +0200 Subject: [PATCH 113/159] chore: preempt single block downloading (#11647) --- crates/net/p2p/src/full_block.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/net/p2p/src/full_block.rs b/crates/net/p2p/src/full_block.rs index 91b786e410ca2..0116f13488104 100644 --- a/crates/net/p2p/src/full_block.rs +++ b/crates/net/p2p/src/full_block.rs @@ -177,6 +177,9 @@ where fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); + // preemptive yield point + let mut budget = 4; + loop { match ready!(this.request.poll(cx)) { ResponseResult::Header(res) => { @@ -232,6 +235,14 @@ where if let Some(res) = this.take_block() { return Poll::Ready(res) } + + // ensure we still have enough budget for another iteration + budget -= 1; + if budget == 0 { + // make sure we're woken up again + cx.waker().wake_by_ref(); + return Poll::Pending + } } } } From 02963bf5803e813fb4f7d4907b42ab3e38a955f9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 10 Oct 2024 19:32:15 +0200 Subject: [PATCH 114/159] feat: store safe block num as well (#11648) --- crates/blockchain-tree/src/externals.rs | 2 +- crates/chain-state/src/chain_info.rs | 8 +++- crates/chain-state/src/in_memory.rs | 17 ++++++--- crates/cli/commands/src/stage/unwind.rs | 2 +- .../consensus/beacon/src/engine/test_utils.rs | 9 ++++- crates/engine/tree/src/persistence.rs | 18 ++++++++- crates/engine/tree/src/tree/mod.rs | 13 +++++-- crates/stages/api/src/pipeline/mod.rs | 4 +- crates/storage/db/src/tables/mod.rs | 11 ++++-- .../src/providers/blockchain_provider.rs | 20 ++++++++-- .../src/providers/database/provider.rs | 37 ++++++++++++++----- crates/storage/provider/src/providers/mod.rs | 16 ++++++-- .../provider/src/traits/finalized_block.rs | 15 ++++++-- crates/storage/provider/src/traits/mod.rs | 2 +- 14 files changed, 130 insertions(+), 44 deletions(-) diff --git a/crates/blockchain-tree/src/externals.rs b/crates/blockchain-tree/src/externals.rs index a4f72f6d33d7d..719852c12ac0e 100644 --- a/crates/blockchain-tree/src/externals.rs +++ b/crates/blockchain-tree/src/externals.rs @@ -7,7 +7,7 @@ use reth_db_api::{cursor::DbCursorRO, transaction::DbTx}; use reth_node_types::NodeTypesWithDB; use reth_primitives::StaticFileSegment; use reth_provider::{ - providers::ProviderNodeTypes, FinalizedBlockReader, FinalizedBlockWriter, ProviderFactory, + providers::ProviderNodeTypes, ChainStateBlockReader, ChainStateBlockWriter, ProviderFactory, StaticFileProviderFactory, StatsReader, }; use reth_storage_errors::provider::ProviderResult; diff --git a/crates/chain-state/src/chain_info.rs b/crates/chain-state/src/chain_info.rs index d9e2c03e27383..d36d9f47e4387 100644 --- a/crates/chain-state/src/chain_info.rs +++ b/crates/chain-state/src/chain_info.rs @@ -21,9 +21,13 @@ pub struct ChainInfoTracker { impl ChainInfoTracker { /// Create a new chain info container for the given canonical head and finalized header if it /// exists. - pub fn new(head: SealedHeader, finalized: Option) -> Self { + pub fn new( + head: SealedHeader, + finalized: Option, + safe: Option, + ) -> Self { let (finalized_block, _) = watch::channel(finalized); - let (safe_block, _) = watch::channel(None); + let (safe_block, _) = watch::channel(safe); Self { inner: Arc::new(ChainInfoInner { diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index 07120cf8ee310..fb67608ebda77 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -173,12 +173,13 @@ impl CanonicalInMemoryState { numbers: BTreeMap, pending: Option, finalized: Option, + safe: Option, ) -> Self { let in_memory_state = InMemoryState::new(blocks, numbers, pending); let header = in_memory_state .head_state() .map_or_else(SealedHeader::default, |state| state.block_ref().block().header.clone()); - let chain_info_tracker = ChainInfoTracker::new(header, finalized); + let chain_info_tracker = ChainInfoTracker::new(header, finalized, safe); let (canon_state_notification_sender, _) = broadcast::channel(CANON_STATE_NOTIFICATION_CHANNEL_SIZE); @@ -193,13 +194,17 @@ impl CanonicalInMemoryState { /// Create an empty state. pub fn empty() -> Self { - Self::new(HashMap::default(), BTreeMap::new(), None, None) + Self::new(HashMap::default(), BTreeMap::new(), None, None, None) } /// Create a new in memory state with the given local head and finalized header /// if it exists. - pub fn with_head(head: SealedHeader, finalized: Option) -> Self { - let chain_info_tracker = ChainInfoTracker::new(head, finalized); + pub fn with_head( + head: SealedHeader, + finalized: Option, + safe: Option, + ) -> Self { + let chain_info_tracker = ChainInfoTracker::new(head, finalized, safe); let in_memory_state = InMemoryState::default(); let (canon_state_notification_sender, _) = broadcast::channel(CANON_STATE_NOTIFICATION_CHANNEL_SIZE); @@ -1255,7 +1260,7 @@ mod tests { numbers.insert(2, block2.block().hash()); numbers.insert(3, block3.block().hash()); - let canonical_state = CanonicalInMemoryState::new(blocks, numbers, None, None); + let canonical_state = CanonicalInMemoryState::new(blocks, numbers, None, None, None); let historical: StateProviderBox = Box::new(MockStateProvider); @@ -1297,7 +1302,7 @@ mod tests { let mut numbers = BTreeMap::new(); numbers.insert(1, hash); - let state = CanonicalInMemoryState::new(blocks, numbers, None, None); + let state = CanonicalInMemoryState::new(blocks, numbers, None, None, None); let chain: Vec<_> = state.canonical_chain().collect(); assert_eq!(chain.len(), 1); diff --git a/crates/cli/commands/src/stage/unwind.rs b/crates/cli/commands/src/stage/unwind.rs index 19305554eaa3f..a5c9956c95b2a 100644 --- a/crates/cli/commands/src/stage/unwind.rs +++ b/crates/cli/commands/src/stage/unwind.rs @@ -17,7 +17,7 @@ use reth_node_builder::{NodeTypesWithDB, NodeTypesWithEngine}; use reth_node_core::args::NetworkArgs; use reth_provider::{ providers::ProviderNodeTypes, BlockExecutionWriter, BlockNumReader, ChainSpecProvider, - FinalizedBlockReader, FinalizedBlockWriter, ProviderFactory, StaticFileProviderFactory, + ChainStateBlockReader, ChainStateBlockWriter, ProviderFactory, StaticFileProviderFactory, }; use reth_prune::PruneModes; use reth_stages::{ diff --git a/crates/consensus/beacon/src/engine/test_utils.rs b/crates/consensus/beacon/src/engine/test_utils.rs index 4dfd9c87d3212..633ae03d8ad9e 100644 --- a/crates/consensus/beacon/src/engine/test_utils.rs +++ b/crates/consensus/beacon/src/engine/test_utils.rs @@ -398,8 +398,13 @@ where let (header, seal) = sealed.into_parts(); let genesis_block = SealedHeader::new(header, seal); - let blockchain_provider = - BlockchainProvider::with_blocks(provider_factory.clone(), tree, genesis_block, None); + let blockchain_provider = BlockchainProvider::with_blocks( + provider_factory.clone(), + tree, + genesis_block, + None, + None, + ); let pruner = Pruner::new_with_factory( provider_factory.clone(), diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index dfddfcfaa89b3..25c1f0ed7030c 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -4,7 +4,7 @@ use reth_chain_state::ExecutedBlock; use reth_errors::ProviderError; use reth_provider::{ providers::ProviderNodeTypes, writer::UnifiedStorageWriter, BlockHashReader, - DatabaseProviderFactory, FinalizedBlockWriter, ProviderFactory, StaticFileProviderFactory, + ChainStateBlockWriter, DatabaseProviderFactory, ProviderFactory, StaticFileProviderFactory, }; use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory}; use reth_stages_api::{MetricEvent, MetricEventsSender}; @@ -97,6 +97,11 @@ impl PersistenceService { provider.save_finalized_block_number(finalized_block)?; provider.commit()?; } + PersistenceAction::SaveSafeBlock(safe_block) => { + let provider = self.provider.database_provider_rw()?; + provider.save_safe_block_number(safe_block)?; + provider.commit()?; + } } } Ok(()) @@ -176,6 +181,9 @@ pub enum PersistenceAction { /// Update the persisted finalized block on disk SaveFinalizedBlock(u64), + + /// Update the persisted safe block on disk + SaveSafeBlock(u64), } /// A handle to the persistence service @@ -251,6 +259,14 @@ impl PersistenceHandle { self.send_action(PersistenceAction::SaveFinalizedBlock(finalized_block)) } + /// Persists the finalized block number on disk. + pub fn save_safe_block_number( + &self, + safe_block: u64, + ) -> Result<(), SendError> { + self.send_action(PersistenceAction::SaveSafeBlock(safe_block)) + } + /// Tells the persistence service to remove blocks above a certain block number. The removed /// blocks are returned by the service. /// diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index bc1da6369459a..3eadbbd522db2 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -2401,8 +2401,13 @@ where // if the safe block is not known, we can't update the safe block return Err(OnForkChoiceUpdated::invalid_state()) } - Ok(Some(finalized)) => { - self.canonical_in_memory_state.set_safe(finalized); + Ok(Some(safe)) => { + if Some(safe.num_hash()) != self.canonical_in_memory_state.get_safe_num_hash() { + // we're also persisting the safe block on disk so we can reload it on + // restart this is required by optimism which queries the safe block: + let _ = self.persistence.save_safe_block_number(safe.number); + self.canonical_in_memory_state.set_safe(safe); + } } Err(err) => { error!(target: "engine::tree", %err, "Failed to fetch safe block header"); @@ -2680,7 +2685,7 @@ mod tests { let (header, seal) = sealed.into_parts(); let header = SealedHeader::new(header, seal); let engine_api_tree_state = EngineApiTreeState::new(10, 10, header.num_hash()); - let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None); + let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None, None); let (to_payload_service, _payload_command_rx) = unbounded_channel(); let payload_builder = PayloadBuilderHandle::new(to_payload_service); @@ -2744,7 +2749,7 @@ mod tests { let last_executed_block = blocks.last().unwrap().clone(); let pending = Some(BlockState::new(last_executed_block)); self.tree.canonical_in_memory_state = - CanonicalInMemoryState::new(state_by_hash, hash_by_number, pending, None); + CanonicalInMemoryState::new(state_by_hash, hash_by_number, pending, None, None); self.blocks = blocks.clone(); self.persist_blocks( diff --git a/crates/stages/api/src/pipeline/mod.rs b/crates/stages/api/src/pipeline/mod.rs index 19b68b384853e..1f6d9341ad6d2 100644 --- a/crates/stages/api/src/pipeline/mod.rs +++ b/crates/stages/api/src/pipeline/mod.rs @@ -7,8 +7,8 @@ pub use event::*; use futures_util::Future; use reth_primitives_traits::constants::BEACON_CONSENSUS_REORG_UNWIND_DEPTH; use reth_provider::{ - providers::ProviderNodeTypes, writer::UnifiedStorageWriter, DatabaseProviderFactory, - FinalizedBlockReader, FinalizedBlockWriter, ProviderFactory, StageCheckpointReader, + providers::ProviderNodeTypes, writer::UnifiedStorageWriter, ChainStateBlockReader, + ChainStateBlockWriter, DatabaseProviderFactory, ProviderFactory, StageCheckpointReader, StageCheckpointWriter, StaticFileProviderFactory, }; use reth_prune::PrunerBuilder; diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 384139618163f..83a063903e089 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -416,6 +416,8 @@ tables! { pub enum ChainStateKey { /// Last finalized block key LastFinalizedBlock, + /// Last finalized block key + LastSafeBlockBlock, } impl Encode for ChainStateKey { @@ -424,16 +426,17 @@ impl Encode for ChainStateKey { fn encode(self) -> Self::Encoded { match self { Self::LastFinalizedBlock => [0], + Self::LastSafeBlockBlock => [1], } } } impl Decode for ChainStateKey { fn decode(value: &[u8]) -> Result { - if value == [0] { - Ok(Self::LastFinalizedBlock) - } else { - Err(reth_db_api::DatabaseError::Decode) + match value { + [0] => Ok(Self::LastFinalizedBlock), + [1] => Ok(Self::LastSafeBlockBlock), + _ => Err(reth_db_api::DatabaseError::Decode), } } } diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 54f28b77b9125..a0811db2aee26 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1,8 +1,8 @@ use crate::{ providers::StaticFileProvider, AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, - CanonStateSubscriptions, ChainSpecProvider, ChangeSetReader, DatabaseProviderFactory, - DatabaseProviderRO, EvmEnvProvider, FinalizedBlockReader, HeaderProvider, ProviderError, + CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, + DatabaseProviderFactory, DatabaseProviderRO, EvmEnvProvider, HeaderProvider, ProviderError, ProviderFactory, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, RequestsProvider, StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, WithdrawalsProvider, @@ -93,9 +93,23 @@ impl BlockchainProvider2 { .map(|num| provider.sealed_header(num)) .transpose()? .flatten(); + let safe_header = provider + .last_safe_block_number()? + .or_else(|| { + // for the purpose of this we can also use the finalized block if we don't have the + // safe block + provider.last_finalized_block_number().ok().flatten() + }) + .map(|num| provider.sealed_header(num)) + .transpose()? + .flatten(); Ok(Self { database, - canonical_in_memory_state: CanonicalInMemoryState::with_head(latest, finalized_header), + canonical_in_memory_state: CanonicalInMemoryState::with_head( + latest, + finalized_header, + safe_header, + ), }) } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 767f92db98f2f..c9b0af3d33ca0 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -7,13 +7,14 @@ use crate::{ }, writer::UnifiedStorageWriter, AccountReader, BlockExecutionReader, BlockExecutionWriter, BlockHashReader, BlockNumReader, - BlockReader, BlockWriter, BundleStateInit, DBProvider, EvmEnvProvider, FinalizedBlockReader, - FinalizedBlockWriter, HashingWriter, HeaderProvider, HeaderSyncGap, HeaderSyncGapProvider, - HistoricalStateProvider, HistoryWriter, LatestStateProvider, OriginalValuesKnown, - ProviderError, PruneCheckpointReader, PruneCheckpointWriter, RequestsProvider, RevertsInit, - StageCheckpointReader, StateChangeWriter, StateProviderBox, StateReader, StateWriter, - StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, - TransactionsProvider, TransactionsProviderExt, TrieWriter, WithdrawalsProvider, + BlockReader, BlockWriter, BundleStateInit, ChainStateBlockReader, ChainStateBlockWriter, + DBProvider, EvmEnvProvider, HashingWriter, HeaderProvider, HeaderSyncGap, + HeaderSyncGapProvider, HistoricalStateProvider, HistoryWriter, LatestStateProvider, + OriginalValuesKnown, ProviderError, PruneCheckpointReader, PruneCheckpointWriter, + RequestsProvider, RevertsInit, StageCheckpointReader, StateChangeWriter, StateProviderBox, + StateReader, StateWriter, StaticFileProviderFactory, StatsReader, StorageReader, + StorageTrieWriter, TransactionVariant, TransactionsProvider, TransactionsProviderExt, + TrieWriter, WithdrawalsProvider, }; use alloy_eips::BlockHashOrNumber; use alloy_primitives::{keccak256, Address, BlockHash, BlockNumber, TxHash, TxNumber, B256, U256}; @@ -3596,7 +3597,7 @@ impl StatsReader for DatabaseProvider { } } -impl FinalizedBlockReader for DatabaseProvider { +impl ChainStateBlockReader for DatabaseProvider { fn last_finalized_block_number(&self) -> ProviderResult> { let mut finalized_blocks = self .tx @@ -3608,14 +3609,32 @@ impl FinalizedBlockReader for DatabaseProvider ProviderResult> { + let mut finalized_blocks = self + .tx + .cursor_read::()? + .walk(Some(tables::ChainStateKey::LastSafeBlockBlock))? + .take(1) + .collect::, _>>()?; + + let last_finalized_block_number = finalized_blocks.pop_first().map(|pair| pair.1); + Ok(last_finalized_block_number) + } } -impl FinalizedBlockWriter for DatabaseProvider { +impl ChainStateBlockWriter for DatabaseProvider { fn save_finalized_block_number(&self, block_number: BlockNumber) -> ProviderResult<()> { Ok(self .tx .put::(tables::ChainStateKey::LastFinalizedBlock, block_number)?) } + + fn save_safe_block_number(&self, block_number: BlockNumber) -> ProviderResult<()> { + Ok(self + .tx + .put::(tables::ChainStateKey::LastSafeBlockBlock, block_number)?) + } } impl DBProvider for DatabaseProvider { diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 5fed81c155d10..561e1d9743623 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -1,9 +1,9 @@ use crate::{ AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, BlockchainTreePendingStateProvider, CanonChainTracker, CanonStateNotifications, - CanonStateSubscriptions, ChainSpecProvider, ChangeSetReader, DatabaseProviderFactory, - EvmEnvProvider, FinalizedBlockReader, FullExecutionDataProvider, HeaderProvider, ProviderError, - PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, RequestsProvider, + CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, + DatabaseProviderFactory, EvmEnvProvider, FullExecutionDataProvider, HeaderProvider, + ProviderError, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, RequestsProvider, StageCheckpointReader, StateProviderBox, StateProviderFactory, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, TreeViewer, WithdrawalsProvider, }; @@ -109,8 +109,9 @@ impl BlockchainProvider { tree: Arc, latest: SealedHeader, finalized: Option, + safe: Option, ) -> Self { - Self { database, tree, chain_info: ChainInfoTracker::new(latest, finalized) } + Self { database, tree, chain_info: ChainInfoTracker::new(latest, finalized, safe) } } /// Create a new provider using only the database and the tree, fetching the latest header from @@ -128,11 +129,18 @@ impl BlockchainProvider { .transpose()? .flatten(); + let safe_header = provider + .last_safe_block_number()? + .map(|num| provider.sealed_header(num)) + .transpose()? + .flatten(); + Ok(Self::with_blocks( database, tree, SealedHeader::new(latest_header, best.best_hash), finalized_header, + safe_header, )) } diff --git a/crates/storage/provider/src/traits/finalized_block.rs b/crates/storage/provider/src/traits/finalized_block.rs index 5509db0aa9394..98a6d9d0e3433 100644 --- a/crates/storage/provider/src/traits/finalized_block.rs +++ b/crates/storage/provider/src/traits/finalized_block.rs @@ -1,16 +1,23 @@ use alloy_primitives::BlockNumber; use reth_errors::ProviderResult; -/// Functionality to read the last known finalized block from the database. -pub trait FinalizedBlockReader: Send + Sync { +/// Functionality to read the last known chain blocks from the database. +pub trait ChainStateBlockReader: Send + Sync { /// Returns the last finalized block number. /// /// If no finalized block has been written yet, this returns `None`. fn last_finalized_block_number(&self) -> ProviderResult>; + /// Returns the last safe block number. + /// + /// If no safe block has been written yet, this returns `None`. + fn last_safe_block_number(&self) -> ProviderResult>; } -/// Functionality to write the last known finalized block to the database. -pub trait FinalizedBlockWriter: Send + Sync { +/// Functionality to write the last known chain blocks to the database. +pub trait ChainStateBlockWriter: Send + Sync { /// Saves the given finalized block number in the DB. fn save_finalized_block_number(&self, block_number: BlockNumber) -> ProviderResult<()>; + + /// Saves the given safe block number in the DB. + fn save_safe_block_number(&self, block_number: BlockNumber) -> ProviderResult<()>; } diff --git a/crates/storage/provider/src/traits/mod.rs b/crates/storage/provider/src/traits/mod.rs index 8bae4f67ae85f..c31c7c1e2f218 100644 --- a/crates/storage/provider/src/traits/mod.rs +++ b/crates/storage/provider/src/traits/mod.rs @@ -42,4 +42,4 @@ mod tree_viewer; pub use tree_viewer::TreeViewer; mod finalized_block; -pub use finalized_block::{FinalizedBlockReader, FinalizedBlockWriter}; +pub use finalized_block::{ChainStateBlockReader, ChainStateBlockWriter}; From f4b4ffa65b41031e41e2156b9bbf6c409bd3688b Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 11 Oct 2024 04:53:19 +0200 Subject: [PATCH 115/159] docs: `LoadFee::eip1559_fees` returns base fee, not max fee per gas (#11652) --- crates/rpc/rpc-eth-api/src/helpers/fee.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/fee.rs b/crates/rpc/rpc-eth-api/src/helpers/fee.rs index b6dcef4708eff..52ccabebf6208 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/fee.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/fee.rs @@ -284,15 +284,15 @@ pub trait LoadFee: LoadBlock { /// Returns the EIP-1559 fees if they are set, otherwise fetches a suggested gas price for /// EIP-1559 transactions. /// - /// Returns (`max_fee`, `priority_fee`) + /// Returns (`base_fee`, `priority_fee`) fn eip1559_fees( &self, - max_fee_per_gas: Option, + base_fee: Option, max_priority_fee_per_gas: Option, ) -> impl Future> + Send { async move { - let max_fee_per_gas = match max_fee_per_gas { - Some(max_fee_per_gas) => max_fee_per_gas, + let base_fee = match base_fee { + Some(base_fee) => base_fee, None => { // fetch pending base fee let base_fee = self @@ -311,7 +311,7 @@ pub trait LoadFee: LoadBlock { Some(max_priority_fee_per_gas) => max_priority_fee_per_gas, None => self.suggested_priority_fee().await?, }; - Ok((max_fee_per_gas, max_priority_fee_per_gas)) + Ok((base_fee, max_priority_fee_per_gas)) } } From 57873fa33ca92278a8721cb4e23270cee023a45e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 11 Oct 2024 08:28:07 +0200 Subject: [PATCH 116/159] feat: propagate helper (#11654) Co-authored-by: Oliver --- crates/net/network/src/transactions/mod.rs | 25 ++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index 0c488ff919dc8..bc77de9f463bf 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -153,6 +153,14 @@ impl TransactionsHandle { self.send(TransactionsCommand::PropagateTransactionsTo(transactions, peer)) } + /// Manually propagate the given transactions to all peers. + /// + /// It's up to the [`TransactionsManager`] whether the transactions are sent as hashes or in + /// full. + pub fn propagate_transactions(&self, transactions: Vec) { + self.send(TransactionsCommand::PropagateTransactions(transactions)) + } + /// Request the transaction hashes known by specific peers. pub async fn get_transaction_hashes( &self, @@ -398,8 +406,14 @@ where trace!(target: "net::tx", num_hashes=?hashes.len(), "Start propagating transactions"); - // This fetches all transaction from the pool, including the 4844 blob transactions but - // __without__ their sidecar, because 4844 transactions are only ever announced as hashes. + self.propagate_all(hashes); + } + + /// Propagates the given transactions to the peers + /// + /// This fetches all transaction from the pool, including the 4844 blob transactions but + /// __without__ their sidecar, because 4844 transactions are only ever announced as hashes. + fn propagate_all(&mut self, hashes: Vec) { let propagated = self.propagate_transactions( self.pool.get_all(hashes).into_iter().map(PropagateTransaction::new).collect(), ); @@ -872,11 +886,12 @@ where let peers = self.peers.keys().copied().collect::>(); tx.send(peers).ok(); } - TransactionsCommand::PropagateTransactionsTo(txs, _peer) => { - if let Some(propagated) = self.propagate_full_transactions_to_peer(txs, _peer) { + TransactionsCommand::PropagateTransactionsTo(txs, peer) => { + if let Some(propagated) = self.propagate_full_transactions_to_peer(txs, peer) { self.pool.on_propagated(propagated); } } + TransactionsCommand::PropagateTransactions(txs) => self.propagate_all(txs), TransactionsCommand::GetTransactionHashes { peers, tx } => { let mut res = HashMap::with_capacity(peers.len()); for peer_id in peers { @@ -1653,6 +1668,8 @@ enum TransactionsCommand { GetActivePeers(oneshot::Sender>), /// Propagate a collection of full transactions to a specific peer. PropagateTransactionsTo(Vec, PeerId), + /// Propagate a collection of full transactions to all peers. + PropagateTransactions(Vec), /// Request transaction hashes known by specific peers from the [`TransactionsManager`]. GetTransactionHashes { peers: Vec, From 7d904e10b0eb557babb722a33b99d79c05245cd4 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Fri, 11 Oct 2024 10:21:59 +0200 Subject: [PATCH 117/159] chore(ci): remove assertoor workflow (#11656) --- .github/workflows/assertoor.yml | 230 -------------------------------- 1 file changed, 230 deletions(-) delete mode 100644 .github/workflows/assertoor.yml diff --git a/.github/workflows/assertoor.yml b/.github/workflows/assertoor.yml deleted file mode 100644 index a5028f7ff31a0..0000000000000 --- a/.github/workflows/assertoor.yml +++ /dev/null @@ -1,230 +0,0 @@ -name: Assertoor Tests - -on: - workflow_dispatch: - schedule: - - cron: '0 0 * * *' - -jobs: - get_tests: - name: "Run assertoor tests on reth pairs" - runs-on: ubuntu-latest - outputs: - test_result: ${{ steps.test_result.outputs.test_result }} - test_status: ${{ steps.test_result.outputs.test_status }} - failed_test_status: ${{ steps.test_result.outputs.failed_test_status }} - if: github.repository == 'paradigmxyz/reth' - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - name: Setup Kurtosis - shell: bash - run: | - echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list - sudo apt update - sudo apt install kurtosis-cli - kurtosis analytics disable - - - name: Run Kurtosis - shell: bash - id: services - run: | - export github_sha=${{ github.sha }} - export github_repository=${{ github.repository }} - - cat etc/assertoor/assertoor-template.yaml | envsubst > etc/assertoor/assertoor.yaml - - kurtosis run github.com/ethpandaops/ethereum-package --enclave assertoor-${{ github.run_id }} --args-file etc/assertoor/assertoor.yaml - - enclave_dump=$(kurtosis enclave inspect assertoor-${{ github.run_id }}) - - assertoor_url=$(echo "$enclave_dump" | grep assertoor | grep http | sed 's/.*\(http:\/\/[0-9.:]\+\).*/\1/') - echo "assertoor_url: ${assertoor_url}" - echo "assertoor_url=${assertoor_url}" >> $GITHUB_OUTPUT - - - name: Await test completion - shell: bash - id: test_result - run: | - assertoor_url="${{ steps.services.outputs.assertoor_url }}" - - YELLOW='\033[1;33m' - GRAY='\033[0;37m' - GREEN='\033[0;32m' - RED='\033[0;31m' - NC='\033[0m' - - # print assertor logs - assertoor_container=$(docker container list | grep assertoor | sed 's/^\([^ ]\+\) .*$/\1/') - docker logs -f $assertoor_container & - - # helper to fetch task status for specific test id - get_tasks_status() { - tasks=$(curl -s ${assertoor_url}/api/v1/test_run/$1 | jq -c ".data.tasks[] | {index, parent_index, name, title, status, result}") - declare -A task_graph_map - task_graph_map[0]="" - - while read task; do - task_id=$(echo "$task" | jq -r ".index") - task_parent=$(echo "$task" | jq -r ".parent_index") - task_name=$(echo "$task" | jq -r ".name") - task_title=$(echo "$task" | jq -r ".title") - task_status=$(echo "$task" | jq -r ".status") - task_result=$(echo "$task" | jq -r ".result") - - task_graph="${task_graph_map[$task_parent]}" - task_graph_map[$task_id]="$task_graph |" - if [ ! -z "$task_graph" ]; then - task_graph="${task_graph}- " - fi - - if [ "$task_status" == "pending" ]; then - task_status="${GRAY}pending ${NC}" - elif [ "$task_status" == "running" ]; then - task_status="${YELLOW}running ${NC}" - elif [ "$task_status" == "complete" ]; then - task_status="${GREEN}complete${NC}" - fi - - if [ "$task_result" == "none" ]; then - task_result="${GRAY}none ${NC}" - elif [ "$task_result" == "success" ]; then - task_result="${GREEN}success${NC}" - elif [ "$task_result" == "failure" ]; then - task_result="${RED}failure${NC}" - fi - - echo -e " $(printf '%-4s' "$task_id")\t$task_status\t$task_result\t$(printf '%-50s' "$task_graph$task_name") \t$task_title" - done <<< $(echo "$tasks") - } - - # poll & check test status - final_test_result="" - failed_test_id="" - while true - do - pending_tests=0 - failed_tests=0 - total_tests=0 - running_test="" - - status_lines=() - task_lines="" - status_lines+=("$(date +'%Y-%m-%d %H:%M:%S') Test Status:") - - tests=$(curl -s ${assertoor_url}/api/v1/test_runs | jq -c ".data[] | {run_id, test_id, name, status}") - while read test; do - if [ -z "$test" ]; then - continue - fi - run_id=$(echo "$test" | jq -r ".run_id") - test_id=$(echo "$test" | jq -r ".test_id") - test_name=$(echo "$test" | jq -r ".name") - test_status=$(echo "$test" | jq -r ".status") - - if [ "$test_status" == "pending" ]; then - pending_tests=$(expr $pending_tests + 1) - status_name="${GRAY}pending${NC}" - elif [ "$test_status" == "running" ]; then - pending_tests=$(expr $pending_tests + 1) - running_test="$run_id" - status_name="${YELLOW}running${NC}" - - elif [ "$test_status" == "success" ]; then - status_name="${GREEN}success${NC}" - elif [ "$test_status" == "failure" ]; then - failed_tests=$(expr $failed_tests + 1) - failed_test_id="$run_id" - status_name="${RED}failure${NC}" - else - status_name="$test_status" - fi - status_lines+=(" $(printf '%-3s' "$test_id") $status_name \t$test_name") - total_tests=$(expr $total_tests + 1) - done <<< $(echo "$tests") - - for status_line in "${status_lines[@]}" - do - echo -e "$status_line" - done - - if ! [ -z "$running_test" ]; then - task_lines=$(get_tasks_status "$running_test") - echo "Active Test Task Status:" - echo "$task_lines" - fi - - if [ $failed_tests -gt 0 ]; then - final_test_result="failure" - break - fi - if [ $total_tests -gt 0 ] && [ $pending_tests -le 0 ]; then - final_test_result="success" - break - fi - - sleep 60 - done - - # save test results & status to github output - echo "test_result=$(echo "$final_test_result")" >> $GITHUB_OUTPUT - echo "test_status<> $GITHUB_OUTPUT - for status_line in "${status_lines[@]}" - do - echo -e "$status_line" >> $GITHUB_OUTPUT - done - echo "EOF" >> $GITHUB_OUTPUT - - if ! [ -z "$failed_test_id" ]; then - echo "failed_test_status<> $GITHUB_OUTPUT - get_tasks_status "$failed_test_id" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - else - echo "failed_test_status=" >> $GITHUB_OUTPUT - fi - - - name: Generate dump and remove kurtosis enclave - shell: bash - run: | - mkdir -p ./temp/dump - cd ./temp/dump - cp ../../etc/assertoor/assertoor.yaml ./kurtosis-params.yaml - - kurtosis enclave dump assertoor-${{ github.run_id }} - kurtosis enclave rm -f assertoor-${{ github.run_id }} - - - name: Upload dump artifact - uses: actions/upload-artifact@v4 - with: - name: "kurtosis-enclave-dump-${{ github.run_id }}" - path: ./temp/dump - - - name: Return test result - shell: bash - run: | - test_result="${{ steps.test_result.outputs.test_result }}" - test_status=$( - cat <<"EOF" - ${{ steps.test_result.outputs.test_status }} - EOF - ) - failed_test_status=$( - cat <<"EOF" - ${{ steps.test_result.outputs.failed_test_status }} - EOF - ) - - echo "Test Result: $test_result" - echo "$test_status" - - if ! [ "$test_result" == "success" ]; then - echo "" - echo "Failed Test Task Status:" - echo "$failed_test_status" - - echo "" - echo "See 'Await test completion' task for detailed logs about this failure!" - echo "" - - exit 1 # fail action - fi From 53315518640995840634e263a66523adcfa240e3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 11 Oct 2024 11:45:44 +0200 Subject: [PATCH 118/159] feat: add override for additional_validation_tasks (#11655) --- crates/node/builder/src/components/pool.rs | 3 +++ crates/optimism/node/src/node.rs | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/node/builder/src/components/pool.rs b/crates/node/builder/src/components/pool.rs index 234455913c6f9..436a80c52e00a 100644 --- a/crates/node/builder/src/components/pool.rs +++ b/crates/node/builder/src/components/pool.rs @@ -52,6 +52,8 @@ pub struct PoolBuilderConfigOverrides { pub minimal_protocol_basefee: Option, /// Addresses that will be considered as local. Above exemptions apply. pub local_addresses: HashSet

, + /// Additional tasks to validate new transactions. + pub additional_validation_tasks: Option, } impl PoolBuilderConfigOverrides { @@ -65,6 +67,7 @@ impl PoolBuilderConfigOverrides { max_account_slots, minimal_protocol_basefee, local_addresses, + additional_validation_tasks: _, } = self; if let Some(pending_limit) = pending_limit { diff --git a/crates/optimism/node/src/node.rs b/crates/optimism/node/src/node.rs index caeab3741a2e6..b6e64f7e0e434 100644 --- a/crates/optimism/node/src/node.rs +++ b/crates/optimism/node/src/node.rs @@ -189,7 +189,11 @@ where )) .with_head_timestamp(ctx.head().timestamp) .kzg_settings(ctx.kzg_settings()?) - .with_additional_tasks(ctx.config().txpool.additional_validation_tasks) + .with_additional_tasks( + pool_config_overrides + .additional_validation_tasks + .unwrap_or_else(|| ctx.config().txpool.additional_validation_tasks), + ) .build_with_tasks(ctx.provider().clone(), ctx.task_executor().clone(), blob_store.clone()) .map(|validator| { OpTransactionValidator::new(validator) From c40d0133548e244e9c3ac85870a539817c41a1c3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 11 Oct 2024 14:17:07 +0200 Subject: [PATCH 119/159] Revert "fix(net): batch `P2PStream` sends" (#11658) --- crates/net/eth-wire/src/p2pstream.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index 76075838bc76f..9bb8fe7c399c9 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -614,24 +614,19 @@ where /// Returns `Poll::Ready(Ok(()))` when no buffered items remain. fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let mut this = self.project(); - let poll_res = loop { - match this.inner.as_mut().poll_ready(cx) { - Poll::Pending => break Poll::Pending, - Poll::Ready(Err(err)) => break Poll::Ready(Err(err.into())), - Poll::Ready(Ok(())) => { + loop { + match ready!(this.inner.as_mut().poll_flush(cx)) { + Err(err) => return Poll::Ready(Err(err.into())), + Ok(()) => { let Some(message) = this.outgoing_messages.pop_front() else { - break Poll::Ready(Ok(())) + return Poll::Ready(Ok(())) }; if let Err(err) = this.inner.as_mut().start_send(message) { - break Poll::Ready(Err(err.into())) + return Poll::Ready(Err(err.into())) } } } - }; - - ready!(this.inner.as_mut().poll_flush(cx))?; - - poll_res + } } fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { From 175b9de7e9a2707d336db3095fafbd49f858bcd2 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Fri, 11 Oct 2024 15:34:52 +0200 Subject: [PATCH 120/159] chore(net): log p2p stream flush error (#11659) --- crates/net/eth-wire/src/p2pstream.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index 9bb8fe7c399c9..9882e39787e21 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -616,7 +616,13 @@ where let mut this = self.project(); loop { match ready!(this.inner.as_mut().poll_flush(cx)) { - Err(err) => return Poll::Ready(Err(err.into())), + Err(err) => { + trace!(target: "net::p2p", + %err, + "error flushing p2p stream" + ); + return Poll::Ready(Err(err.into())) + } Ok(()) => { let Some(message) = this.outgoing_messages.pop_front() else { return Poll::Ready(Ok(())) From b34de4662adf53d149f76404391cb2460ed374bf Mon Sep 17 00:00:00 2001 From: Ayene <2958807+ayenesimo1i@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:31:11 +0300 Subject: [PATCH 121/159] fix(docs): remove ci link (#11665) Co-authored-by: Federico Gimenez --- docs/repo/ci.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/repo/ci.md b/docs/repo/ci.md index da84e001f3761..5ed2cec0091e0 100644 --- a/docs/repo/ci.md +++ b/docs/repo/ci.md @@ -26,7 +26,7 @@ The CI runs a couple of workflows: ### Integration Testing -- **[assertoor]**: Runs Assertoor tests on Reth pairs. +- **[kurtosis]**: Spins up a Kurtosis testnet and runs Assertoor tests on Reth pairs. - **[hive]**: Runs `ethereum/hive` tests. ### Linting and Checks @@ -48,7 +48,7 @@ The CI runs a couple of workflows: [dependencies]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/dependencies.yml [stale]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/stale.yml [docker]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/docker.yml -[assertoor]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/assertoor.yml +[kurtosis]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/kurtosis.yml [hive]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/hive.yml [lint]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/lint.yml [lint-actions]: https://github.com/paradigmxyz/reth/blob/main/.github/workflows/lint-actions.yml From fd34baf92f9f766c21ccd452f9d1e0cda2d62533 Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Sat, 12 Oct 2024 00:27:24 +0800 Subject: [PATCH 122/159] chore(test): use collect void realloc (#11669) --- crates/transaction-pool/src/test_utils/mock.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index 6b470bb6fb1f1..e2b5f373e4467 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -1338,10 +1338,8 @@ impl MockTransactionDistribution { nonce_range: Range, rng: &mut impl rand::Rng, ) -> MockTransactionSet { - let mut txs = Vec::new(); - for nonce in nonce_range { - txs.push(self.tx(nonce, rng).with_sender(sender)); - } + let txs = + nonce_range.map(|nonce| self.tx(nonce, rng).with_sender(sender)).collect::>(); MockTransactionSet::new(txs) } From e4b0c82b1ba77f20ecd2542458b6118ffc791e1b Mon Sep 17 00:00:00 2001 From: nk_ysg Date: Sat, 12 Oct 2024 01:49:29 +0800 Subject: [PATCH 123/159] transaction-pool:rm redundance clone (#11667) --- crates/transaction-pool/src/pool/best.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index 5880a73f5101e..52f25a9db8dcf 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -132,9 +132,8 @@ impl BestTransactions { /// created and inserts them fn add_new_transactions(&mut self) { while let Some(pending_tx) = self.try_recv() { - let tx = pending_tx.transaction.clone(); // same logic as PendingPool::add_transaction/PendingPool::best_with_unlocked - let tx_id = *tx.id(); + let tx_id = *pending_tx.transaction.id(); if self.ancestor(&tx_id).is_none() { self.independent.insert(pending_tx.clone()); } From 84ba24ab7454d443b310db1c8d1254bccb0a4da5 Mon Sep 17 00:00:00 2001 From: Francis Li Date: Fri, 11 Oct 2024 10:55:22 -0700 Subject: [PATCH 124/159] fix(rpc): add missing codes for witness (#11673) --- crates/rpc/rpc/src/debug.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index b26459c4f0406..738b91ad0ecac 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -34,7 +34,7 @@ use reth_rpc_server_types::{result::internal_rpc_err, ToRpcResult}; use reth_tasks::pool::BlockingTaskGuard; use reth_trie::{HashedPostState, HashedStorage}; use revm::{ - db::CacheDB, + db::{CacheDB, State}, primitives::{db::DatabaseCommit, BlockEnv, CfgEnvWithHandlerCfg, Env, EnvWithHandlerCfg}, }; use revm_inspectors::tracing::{ @@ -614,12 +614,23 @@ where let _ = block_executor .execute_with_state_closure( (&block.clone().unseal(), block.difficulty).into(), - |statedb| { + |statedb: &State<_>| { codes = statedb .cache .contracts .iter() .map(|(hash, code)| (*hash, code.original_bytes())) + .chain( + // cache state does not have all the contracts, especially when + // a contract is created within the block + // the contract only exists in bundle state, therefore we need + // to include them as well + statedb + .bundle_state + .contracts + .iter() + .map(|(hash, code)| (*hash, code.original_bytes())), + ) .collect(); for (address, account) in &statedb.cache.accounts { From 64a3958c5fc8c05256c34c73ba2bec98e6165be8 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 11 Oct 2024 19:56:33 +0200 Subject: [PATCH 125/159] clippy: add `from_iter_instead_of_collect` warn (#11666) --- Cargo.toml | 1 + crates/evm/execution-types/src/chain.rs | 2 +- crates/rpc/rpc-builder/src/metrics.rs | 9 ++-- crates/rpc/rpc/src/debug.rs | 6 +-- .../src/providers/database/provider.rs | 8 ++-- .../storage/provider/src/test_utils/blocks.rs | 47 +++++++++---------- crates/transaction-pool/benches/reorder.rs | 10 ++-- crates/transaction-pool/src/maintain.rs | 2 +- crates/trie/db/src/state.rs | 32 +++++++------ crates/trie/db/tests/post_state.rs | 44 +++++++++-------- crates/trie/trie/src/proof.rs | 2 +- crates/trie/trie/src/updates.rs | 2 +- 12 files changed, 85 insertions(+), 80 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index db115c89cccdd..efae22f8ed117 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -183,6 +183,7 @@ equatable_if_let = "warn" explicit_into_iter_loop = "warn" explicit_iter_loop = "warn" flat_map_option = "warn" +from_iter_instead_of_collect = "warn" if_not_else = "warn" implicit_clone = "warn" imprecise_flops = "warn" diff --git a/crates/evm/execution-types/src/chain.rs b/crates/evm/execution-types/src/chain.rs index 25bc39ea32570..30f1f4cd2fc3b 100644 --- a/crates/evm/execution-types/src/chain.rs +++ b/crates/evm/execution-types/src/chain.rs @@ -52,7 +52,7 @@ impl Chain { execution_outcome: ExecutionOutcome, trie_updates: Option, ) -> Self { - let blocks = BTreeMap::from_iter(blocks.into_iter().map(|b| (b.number, b))); + let blocks = blocks.into_iter().map(|b| (b.number, b)).collect::>(); debug_assert!(!blocks.is_empty(), "Chain should have at least one block"); Self { blocks, execution_outcome, trie_updates } diff --git a/crates/rpc/rpc-builder/src/metrics.rs b/crates/rpc/rpc-builder/src/metrics.rs index 08fd38898558a..57283ded3796b 100644 --- a/crates/rpc/rpc-builder/src/metrics.rs +++ b/crates/rpc/rpc-builder/src/metrics.rs @@ -30,9 +30,12 @@ impl RpcRequestMetrics { Self { inner: Arc::new(RpcServerMetricsInner { connection_metrics: transport.connection_metrics(), - call_metrics: HashMap::from_iter(module.method_names().map(|method| { - (method, RpcServerCallMetrics::new_with_labels(&[("method", method)])) - })), + call_metrics: module + .method_names() + .map(|method| { + (method, RpcServerCallMetrics::new_with_labels(&[("method", method)])) + }) + .collect(), }), } } diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 738b91ad0ecac..fbef6f7a7e0b3 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -663,11 +663,7 @@ where let state = state_provider.witness(Default::default(), hashed_state).map_err(Into::into)?; - Ok(ExecutionWitness { - state: HashMap::from_iter(state.into_iter()), - codes, - keys: Some(keys), - }) + Ok(ExecutionWitness { state: state.into_iter().collect(), codes, keys: Some(keys) }) }) .await } diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index c9b0af3d33ca0..33fed1e80ce1e 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -2914,10 +2914,10 @@ impl HashingWriter for DatabaseProvider()?; // Hash the address and key and apply them to HashedStorage (if Storage is None diff --git a/crates/storage/provider/src/test_utils/blocks.rs b/crates/storage/provider/src/test_utils/blocks.rs index daed906646d3b..f7928319b48c1 100644 --- a/crates/storage/provider/src/test_utils/blocks.rs +++ b/crates/storage/provider/src/test_utils/blocks.rs @@ -324,11 +324,10 @@ fn block3( ) .state_storage( address, - HashMap::from_iter( - slot_range - .clone() - .map(|slot| (U256::from(slot), (U256::ZERO, U256::from(slot)))), - ), + slot_range + .clone() + .map(|slot| (U256::from(slot), (U256::ZERO, U256::from(slot)))) + .collect(), ) .revert_account_info(number, address, Some(None)) .revert_storage(number, address, Vec::new()); @@ -393,20 +392,18 @@ fn block4( ) .state_storage( address, - HashMap::from_iter( - slot_range.clone().map(|slot| { - (U256::from(slot), (U256::from(slot), U256::from(slot * 2))) - }), - ), + slot_range + .clone() + .map(|slot| (U256::from(slot), (U256::from(slot), U256::from(slot * 2)))) + .collect(), ) } else { bundle_state_builder.state_address(address).state_storage( address, - HashMap::from_iter( - slot_range - .clone() - .map(|slot| (U256::from(slot), (U256::from(slot), U256::ZERO))), - ), + slot_range + .clone() + .map(|slot| (U256::from(slot), (U256::from(slot), U256::ZERO))) + .collect(), ) }; // record previous account info @@ -423,7 +420,7 @@ fn block4( .revert_storage( number, address, - Vec::from_iter(slot_range.clone().map(|slot| (U256::from(slot), U256::from(slot)))), + slot_range.clone().map(|slot| (U256::from(slot), U256::from(slot))).collect(), ); } let execution_outcome = ExecutionOutcome::new( @@ -485,12 +482,11 @@ fn block5( ) .state_storage( address, - HashMap::from_iter( - slot_range - .clone() - .take(50) - .map(|slot| (U256::from(slot), (U256::from(slot), U256::from(slot * 4)))), - ), + slot_range + .clone() + .take(50) + .map(|slot| (U256::from(slot), (U256::from(slot), U256::from(slot * 4)))) + .collect(), ); bundle_state_builder = if idx % 2 == 0 { bundle_state_builder @@ -506,9 +502,10 @@ fn block5( .revert_storage( number, address, - Vec::from_iter( - slot_range.clone().map(|slot| (U256::from(slot), U256::from(slot * 2))), - ), + slot_range + .clone() + .map(|slot| (U256::from(slot), U256::from(slot * 2))) + .collect(), ) } else { bundle_state_builder.revert_address(number, address) diff --git a/crates/transaction-pool/benches/reorder.rs b/crates/transaction-pool/benches/reorder.rs index 1836ce40c0855..9fc21629753fd 100644 --- a/crates/transaction-pool/benches/reorder.rs +++ b/crates/transaction-pool/benches/reorder.rs @@ -206,10 +206,12 @@ mod implementations { self.base_fee = Some(base_fee); let drained = self.inner.drain(); - self.inner = BinaryHeap::from_iter(drained.map(|mock| { - let priority = mock.tx.effective_tip_per_gas(base_fee).expect("set"); - MockTransactionWithPriority { tx: mock.tx, priority } - })); + self.inner = drained + .map(|mock| { + let priority = mock.tx.effective_tip_per_gas(base_fee).expect("set"); + MockTransactionWithPriority { tx: mock.tx, priority } + }) + .collect(); } } } diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index 66b98614737e0..36c7067d4a835 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -490,7 +490,7 @@ impl MaintainedPoolState { } } -/// A unique `ChangedAccount` identified by its address that can be used for deduplication +/// A unique [`ChangedAccount`] identified by its address that can be used for deduplication #[derive(Eq)] struct ChangedAccountEntry(ChangedAccount); diff --git a/crates/trie/db/src/state.rs b/crates/trie/db/src/state.rs index 5acb9e0d1b491..4d46183dfda48 100644 --- a/crates/trie/db/src/state.rs +++ b/crates/trie/db/src/state.rs @@ -244,22 +244,24 @@ impl DatabaseHashedPostState for HashedPostState { } } - let hashed_accounts = HashMap::from_iter( - accounts.into_iter().map(|(address, info)| (keccak256(address), info)), - ); + let hashed_accounts = + accounts.into_iter().map(|(address, info)| (keccak256(address), info)).collect(); - let hashed_storages = HashMap::from_iter(storages.into_iter().map(|(address, storage)| { - ( - keccak256(address), - HashedStorage::from_iter( - // The `wiped` flag indicates only whether previous storage entries - // should be looked up in db or not. For reverts it's a noop since all - // wiped changes had been written as storage reverts. - false, - storage.into_iter().map(|(slot, value)| (keccak256(slot), value)), - ), - ) - })); + let hashed_storages = storages + .into_iter() + .map(|(address, storage)| { + ( + keccak256(address), + HashedStorage::from_iter( + // The `wiped` flag indicates only whether previous storage entries + // should be looked up in db or not. For reverts it's a noop since all + // wiped changes had been written as storage reverts. + false, + storage.into_iter().map(|(slot, value)| (keccak256(slot), value)), + ), + ) + }) + .collect(); Ok(Self { accounts: hashed_accounts, storages: hashed_storages }) } diff --git a/crates/trie/db/tests/post_state.rs b/crates/trie/db/tests/post_state.rs index ceadf7cde590c..ce6f10d76aedc 100644 --- a/crates/trie/db/tests/post_state.rs +++ b/crates/trie/db/tests/post_state.rs @@ -55,7 +55,7 @@ fn assert_storage_cursor_order( #[test] fn post_state_only_accounts() { let accounts = - Vec::from_iter((1..11).map(|key| (B256::with_last_byte(key), Account::default()))); + (1..11).map(|key| (B256::with_last_byte(key), Account::default())).collect::>(); let mut hashed_post_state = HashedPostState::default(); for (hashed_address, account) in &accounts { @@ -73,7 +73,7 @@ fn post_state_only_accounts() { #[test] fn db_only_accounts() { let accounts = - Vec::from_iter((1..11).map(|key| (B256::with_last_byte(key), Account::default()))); + (1..11).map(|key| (B256::with_last_byte(key), Account::default())).collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -96,7 +96,7 @@ fn db_only_accounts() { fn account_cursor_correct_order() { // odd keys are in post state, even keys are in db let accounts = - Vec::from_iter((1..111).map(|key| (B256::with_last_byte(key), Account::default()))); + (1..111).map(|key| (B256::with_last_byte(key), Account::default())).collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -121,9 +121,9 @@ fn account_cursor_correct_order() { fn removed_accounts_are_discarded() { // odd keys are in post state, even keys are in db let accounts = - Vec::from_iter((1..111).map(|key| (B256::with_last_byte(key), Account::default()))); + (1..111).map(|key| (B256::with_last_byte(key), Account::default())).collect::>(); // accounts 5, 9, 11 should be considered removed from post state - let removed_keys = Vec::from_iter([5, 9, 11].into_iter().map(B256::with_last_byte)); + let removed_keys = [5, 9, 11].into_iter().map(B256::with_last_byte).collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -150,9 +150,9 @@ fn removed_accounts_are_discarded() { #[test] fn post_state_accounts_take_precedence() { - let accounts = Vec::from_iter((1..10).map(|key| { - (B256::with_last_byte(key), Account { nonce: key as u64, ..Default::default() }) - })); + let accounts = (1..10) + .map(|key| (B256::with_last_byte(key), Account { nonce: key as u64, ..Default::default() })) + .collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -224,7 +224,7 @@ fn storage_is_empty() { } let db_storage = - BTreeMap::from_iter((0..10).map(|key| (B256::with_last_byte(key), U256::from(key)))); + (0..10).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); db.update(|tx| { for (slot, value) in &db_storage { // insert zero value accounts to the database @@ -299,9 +299,10 @@ fn storage_is_empty() { fn storage_cursor_correct_order() { let address = B256::random(); let db_storage = - BTreeMap::from_iter((1..11).map(|key| (B256::with_last_byte(key), U256::from(key)))); - let post_state_storage = - BTreeMap::from_iter((11..21).map(|key| (B256::with_last_byte(key), U256::from(key)))); + (1..11).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); + let post_state_storage = (11..21) + .map(|key| (B256::with_last_byte(key), U256::from(key))) + .collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -334,10 +335,12 @@ fn storage_cursor_correct_order() { fn zero_value_storage_entries_are_discarded() { let address = B256::random(); let db_storage = - BTreeMap::from_iter((0..10).map(|key| (B256::with_last_byte(key), U256::from(key)))); // every even number is changed to zero value - let post_state_storage = BTreeMap::from_iter((0..10).map(|key| { - (B256::with_last_byte(key), if key % 2 == 0 { U256::ZERO } else { U256::from(key) }) - })); + (0..10).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); // every even number is changed to zero value + let post_state_storage = (0..10) + .map(|key| { + (B256::with_last_byte(key), if key % 2 == 0 { U256::ZERO } else { U256::from(key) }) + }) + .collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -371,9 +374,10 @@ fn zero_value_storage_entries_are_discarded() { fn wiped_storage_is_discarded() { let address = B256::random(); let db_storage = - BTreeMap::from_iter((1..11).map(|key| (B256::with_last_byte(key), U256::from(key)))); - let post_state_storage = - BTreeMap::from_iter((11..21).map(|key| (B256::with_last_byte(key), U256::from(key)))); + (1..11).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); + let post_state_storage = (11..21) + .map(|key| (B256::with_last_byte(key), U256::from(key))) + .collect::>(); let db = create_test_rw_db(); db.update(|tx| { @@ -404,7 +408,7 @@ fn wiped_storage_is_discarded() { fn post_state_storages_take_precedence() { let address = B256::random(); let storage = - BTreeMap::from_iter((1..10).map(|key| (B256::with_last_byte(key), U256::from(key)))); + (1..10).map(|key| (B256::with_last_byte(key), U256::from(key))).collect::>(); let db = create_test_rw_db(); db.update(|tx| { diff --git a/crates/trie/trie/src/proof.rs b/crates/trie/trie/src/proof.rs index d31d63fd9a8be..e99d686aca7fb 100644 --- a/crates/trie/trie/src/proof.rs +++ b/crates/trie/trie/src/proof.rs @@ -100,7 +100,7 @@ where let walker = TrieWalker::new(trie_cursor, prefix_set.freeze()); // Create a hash builder to rebuild the root node since it is not available in the database. - let retainer = ProofRetainer::from_iter(targets.keys().map(Nibbles::unpack)); + let retainer = targets.keys().map(Nibbles::unpack).collect(); let mut hash_builder = HashBuilder::default().with_proof_retainer(retainer); let mut storages = HashMap::default(); diff --git a/crates/trie/trie/src/updates.rs b/crates/trie/trie/src/updates.rs index 03e80cf52e539..6d1bcab63d8f1 100644 --- a/crates/trie/trie/src/updates.rs +++ b/crates/trie/trie/src/updates.rs @@ -236,7 +236,7 @@ mod serde_nibbles_set { S: Serializer, { let mut storage_nodes = - Vec::from_iter(map.iter().map(|elem| alloy_primitives::hex::encode(elem.pack()))); + map.iter().map(|elem| alloy_primitives::hex::encode(elem.pack())).collect::>(); storage_nodes.sort_unstable(); storage_nodes.serialize(serializer) } From 0349397fd0df32a2af2fa0e9c0e5c57f42912137 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:11:58 +0200 Subject: [PATCH 126/159] tx-pool: simplify `FinalizedBlockTracker` update logic (#11664) --- crates/transaction-pool/src/maintain.rs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index 36c7067d4a835..aaf2d6d12d85f 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -454,20 +454,11 @@ impl FinalizedBlockTracker { /// Updates the tracked finalized block and returns the new finalized block if it changed fn update(&mut self, finalized_block: Option) -> Option { - match (self.last_finalized_block, finalized_block) { - (Some(last), Some(finalized)) => { - self.last_finalized_block = Some(finalized); - if last < finalized { - Some(finalized) - } else { - None - } - } - (None, Some(finalized)) => { - self.last_finalized_block = Some(finalized); - Some(finalized) - } - _ => None, + let finalized = finalized_block?; + if self.last_finalized_block.replace(finalized).map_or(true, |last| last < finalized) { + Some(finalized) + } else { + None } } } From 63bf698425f501d5ce89d7334eb70a4f7c59ea62 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:22:20 +0200 Subject: [PATCH 127/159] refac: small refactor in `BlockchainProvider2` (#11660) --- .../provider/src/providers/blockchain_provider.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index a0811db2aee26..dc4264210b275 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -681,7 +681,7 @@ impl HeaderProvider for BlockchainProvider2 { { last_finalized_num_hash.number } else { - self.database.last_block_number()? + self.last_block_number()? } } else { // Otherwise, return what we have on disk for the input block @@ -841,12 +841,7 @@ impl BlockReader for BlockchainProvider2 { id, |db_provider| db_provider.ommers(id), |block_state| { - if self - .database - .chain_spec() - .final_paris_total_difficulty(block_state.number()) - .is_some() - { + if self.chain_spec().final_paris_total_difficulty(block_state.number()).is_some() { return Ok(Some(Vec::new())) } @@ -1174,7 +1169,7 @@ impl WithdrawalsProvider for BlockchainProvider2 { id: BlockHashOrNumber, timestamp: u64, ) -> ProviderResult> { - if !self.database.chain_spec().is_shanghai_active_at_timestamp(timestamp) { + if !self.chain_spec().is_shanghai_active_at_timestamp(timestamp) { return Ok(None) } @@ -1210,7 +1205,7 @@ impl RequestsProvider for BlockchainProvider2 { id: BlockHashOrNumber, timestamp: u64, ) -> ProviderResult> { - if !self.database.chain_spec().is_prague_active_at_timestamp(timestamp) { + if !self.chain_spec().is_prague_active_at_timestamp(timestamp) { return Ok(None) } From d46e6b645dba0a5c979650f7523ba9524145e7ef Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Fri, 11 Oct 2024 20:52:36 +0200 Subject: [PATCH 128/159] test: add unit tests for `ChainInfoTracker` (#11642) Co-authored-by: Alexey Shekhirin --- Cargo.lock | 1 + crates/chain-state/Cargo.toml | 1 + crates/chain-state/src/chain_info.rs | 203 ++++++++++++++++++++++++++- 3 files changed, 201 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37da7b57510b2..22e1360a22e77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6511,6 +6511,7 @@ dependencies = [ "reth-metrics", "reth-primitives", "reth-storage-api", + "reth-testing-utils", "reth-trie", "revm", "tokio", diff --git a/crates/chain-state/Cargo.toml b/crates/chain-state/Cargo.toml index 63016918c5cba..c9691bec411cc 100644 --- a/crates/chain-state/Cargo.toml +++ b/crates/chain-state/Cargo.toml @@ -47,6 +47,7 @@ rand = { workspace = true, optional = true } revm = { workspace = true, optional = true } [dev-dependencies] +reth-testing-utils.workspace = true alloy-signer.workspace = true alloy-signer-local.workspace = true alloy-consensus.workspace = true diff --git a/crates/chain-state/src/chain_info.rs b/crates/chain-state/src/chain_info.rs index d36d9f47e4387..3c75544ac460c 100644 --- a/crates/chain-state/src/chain_info.rs +++ b/crates/chain-state/src/chain_info.rs @@ -95,14 +95,12 @@ impl ChainInfoTracker { /// Returns the safe header of the chain. pub fn get_safe_num_hash(&self) -> Option { - let h = self.inner.safe_block.borrow(); - h.as_ref().map(|h| h.num_hash()) + self.inner.safe_block.borrow().as_ref().map(SealedHeader::num_hash) } /// Returns the finalized header of the chain. pub fn get_finalized_num_hash(&self) -> Option { - let h = self.inner.finalized_block.borrow(); - h.as_ref().map(|h| h.num_hash()) + self.inner.finalized_block.borrow().as_ref().map(SealedHeader::num_hash) } /// Sets the canonical head of the chain. @@ -169,3 +167,200 @@ struct ChainInfoInner { /// The block that the beacon node considers finalized. finalized_block: watch::Sender>, } + +#[cfg(test)] +mod tests { + use super::*; + use reth_testing_utils::{generators, generators::random_header}; + + #[test] + fn test_chain_info() { + // Create a random header + let mut rng = generators::rng(); + let header = random_header(&mut rng, 10, None); + + // Create a new chain info tracker with the header + let tracker = ChainInfoTracker::new(header.clone(), None, None); + + // Fetch the chain information from the tracker + let chain_info = tracker.chain_info(); + + // Verify that the chain information matches the header + assert_eq!(chain_info.best_number, header.number); + assert_eq!(chain_info.best_hash, header.hash()); + } + + #[test] + fn test_on_forkchoice_update_received() { + // Create a random block header + let mut rng = generators::rng(); + let header = random_header(&mut rng, 10, None); + + // Create a new chain info tracker with the header + let tracker = ChainInfoTracker::new(header, None, None); + + // Assert that there has been no forkchoice update yet (the timestamp is None) + assert!(tracker.last_forkchoice_update_received_at().is_none()); + + // Call the method to record the receipt of a forkchoice update + tracker.on_forkchoice_update_received(); + + // Assert that there is now a timestamp indicating when the forkchoice update was received + assert!(tracker.last_forkchoice_update_received_at().is_some()); + } + + #[test] + fn test_on_transition_configuration_exchanged() { + // Create a random header + let mut rng = generators::rng(); + let header = random_header(&mut rng, 10, None); + + // Create a new chain info tracker with the header + let tracker = ChainInfoTracker::new(header, None, None); + + // Assert that there has been no transition configuration exchange yet (the timestamp is + // None) + assert!(tracker.last_transition_configuration_exchanged_at().is_none()); + + // Call the method to record the transition configuration exchange + tracker.on_transition_configuration_exchanged(); + + // Assert that there is now a timestamp indicating when the transition configuration + // exchange occurred + assert!(tracker.last_transition_configuration_exchanged_at().is_some()); + } + + #[test] + fn test_set_canonical_head() { + // Create a random number generator + let mut rng = generators::rng(); + // Generate two random headers for testing + let header1 = random_header(&mut rng, 10, None); + let header2 = random_header(&mut rng, 20, None); + + // Create a new chain info tracker with the first header + let tracker = ChainInfoTracker::new(header1, None, None); + + // Set the second header as the canonical head of the tracker + tracker.set_canonical_head(header2.clone()); + + // Assert that the tracker now uses the second header as its canonical head + let canonical_head = tracker.get_canonical_head(); + assert_eq!(canonical_head, header2); + } + + #[test] + fn test_set_safe() { + // Create a random number generator + let mut rng = generators::rng(); + + // Case 1: basic test + // Generate two random headers for the test + let header1 = random_header(&mut rng, 10, None); + let header2 = random_header(&mut rng, 20, None); + + // Create a new chain info tracker with the first header (header1) + let tracker = ChainInfoTracker::new(header1, None, None); + + // Call the set_safe method with the second header (header2) + tracker.set_safe(header2.clone()); + + // Verify that the tracker now has header2 as the safe block + let safe_header = tracker.get_safe_header(); + assert!(safe_header.is_some()); // Ensure a safe header is present + let safe_header = safe_header.unwrap(); + assert_eq!(safe_header, header2); + + // Case 2: call with the same header as the current safe block + // Call set_safe again with the same header (header2) + tracker.set_safe(header2.clone()); + + // Verify that nothing changes and the safe header remains the same + let same_safe_header = tracker.get_safe_header(); + assert!(same_safe_header.is_some()); + let same_safe_header = same_safe_header.unwrap(); + assert_eq!(same_safe_header, header2); + + // Case 3: call with a different (new) header + // Generate a third header with a higher block number + let header3 = random_header(&mut rng, 30, None); + + // Call set_safe with this new header (header3) + tracker.set_safe(header3.clone()); + + // Verify that the safe header is updated with the new header + let updated_safe_header = tracker.get_safe_header(); + assert!(updated_safe_header.is_some()); + let updated_safe_header = updated_safe_header.unwrap(); + assert_eq!(updated_safe_header, header3); + } + + #[test] + fn test_set_finalized() { + // Create a random number generator + let mut rng = generators::rng(); + + // Generate random headers for testing + let header1 = random_header(&mut rng, 10, None); + let header2 = random_header(&mut rng, 20, None); + let header3 = random_header(&mut rng, 30, None); + + // Create a new chain info tracker with the first header + let tracker = ChainInfoTracker::new(header1, None, None); + + // Initial state: finalize header should be None + assert!(tracker.get_finalized_header().is_none()); + + // Set the second header as the finalized header + tracker.set_finalized(header2.clone()); + + // Assert that the tracker now uses the second header as its finalized block + let finalized_header = tracker.get_finalized_header(); + assert!(finalized_header.is_some()); + let finalized_header = finalized_header.unwrap(); + assert_eq!(finalized_header, header2); + + // Case 2: attempt to set the same finalized header again + tracker.set_finalized(header2.clone()); + + // The finalized header should remain unchanged + let unchanged_finalized_header = tracker.get_finalized_header(); + assert_eq!(unchanged_finalized_header.unwrap(), header2); // Should still be header2 + + // Case 3: set a higher block number as finalized + tracker.set_finalized(header3.clone()); + + // The finalized header should now be updated to header3 + let updated_finalized_header = tracker.get_finalized_header(); + assert!(updated_finalized_header.is_some()); + assert_eq!(updated_finalized_header.unwrap(), header3); + } + + #[test] + fn test_get_finalized_num_hash() { + // Create a random header + let mut rng = generators::rng(); + let finalized_header = random_header(&mut rng, 10, None); + + // Create a new chain info tracker with the finalized header + let tracker = + ChainInfoTracker::new(finalized_header.clone(), Some(finalized_header.clone()), None); + + // Assert that the BlockNumHash returned matches the finalized header + assert_eq!(tracker.get_finalized_num_hash(), Some(finalized_header.num_hash())); + } + + #[test] + fn test_get_safe_num_hash() { + // Create a random header + let mut rng = generators::rng(); + let safe_header = random_header(&mut rng, 10, None); + + // Create a new chain info tracker with the safe header + let tracker = ChainInfoTracker::new(safe_header.clone(), None, None); + tracker.set_safe(safe_header.clone()); + + // Assert that the BlockNumHash returned matches the safe header + assert_eq!(tracker.get_safe_num_hash(), Some(safe_header.num_hash())); + } +} From c13b069990695448f4da47fe2aac79cf418a2a6b Mon Sep 17 00:00:00 2001 From: 0xDmtri <0xDmtri@protonmail.com> Date: Fri, 11 Oct 2024 20:53:34 +0200 Subject: [PATCH 129/159] chore(examples): fix db-access example with RO provider (#11670) Co-authored-by: Matthias Seitz --- examples/db-access/src/main.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/db-access/src/main.rs b/examples/db-access/src/main.rs index ab018a0b07a69..5772461bd7a78 100644 --- a/examples/db-access/src/main.rs +++ b/examples/db-access/src/main.rs @@ -1,6 +1,7 @@ use alloy_primitives::{Address, Sealable, B256}; use alloy_rpc_types::{Filter, FilteredParams}; use reth_chainspec::ChainSpecBuilder; +use reth_db::{open_db_read_only, DatabaseEnv}; use reth_node_ethereum::EthereumNode; use reth_node_types::NodeTypesWithDBAdapter; use reth_primitives::SealedHeader; @@ -8,7 +9,7 @@ use reth_provider::{ providers::StaticFileProvider, AccountReader, BlockReader, BlockSource, HeaderProvider, ProviderFactory, ReceiptProvider, StateProvider, TransactionsProvider, }; -use std::path::Path; +use std::{path::Path, sync::Arc}; // Providers are zero cost abstractions on top of an opened MDBX Transaction // exposing a familiar API to query the chain's information without requiring knowledge @@ -20,17 +21,16 @@ fn main() -> eyre::Result<()> { // Opens a RO handle to the database file. let db_path = std::env::var("RETH_DB_PATH")?; let db_path = Path::new(&db_path); + let db = open_db_read_only(db_path.join("db").as_path(), Default::default())?; // Instantiate a provider factory for Ethereum mainnet using the provided DB. // TODO: Should the DB version include the spec so that you do not need to specify it here? let spec = ChainSpecBuilder::mainnet().build(); - let factory = - ProviderFactory::>::new_with_database_path( - db_path, - spec.into(), - Default::default(), - StaticFileProvider::read_only(db_path.join("static_files"), false)?, - )?; + let factory = ProviderFactory::>>::new( + db.into(), + spec.into(), + StaticFileProvider::read_only(db_path.join("static_files"), true)?, + ); // This call opens a RO transaction on the database. To write to the DB you'd need to call // the `provider_rw` function and look for the `Writer` variants of the traits. From 1bc6c0de061c497a70aca8861bff9da375a7a1e5 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Sat, 12 Oct 2024 08:24:06 +0400 Subject: [PATCH 130/159] fix: always poll new pool imports (#11675) --- crates/net/network/src/transactions/mod.rs | 46 +++++++++++----------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index bc77de9f463bf..439f92bada9a6 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -1255,29 +1255,6 @@ where // yield back control to tokio. See `NetworkManager` for more context on the design // pattern. - // Advance pool imports (flush txns to pool). - // - // Note, this is done in batches. A batch is filled from one `Transactions` - // broadcast messages or one `PooledTransactions` response at a time. The - // minimum batch size is 1 transaction (and might often be the case with blob - // transactions). - // - // The smallest decodable transaction is an empty legacy transaction, 10 bytes - // (2 MiB / 10 bytes > 200k transactions). - // - // Since transactions aren't validated until they are inserted into the pool, - // this can potentially validate >200k transactions. More if the message size - // is bigger than the soft limit on a `PooledTransactions` response which is - // 2 MiB (`Transactions` broadcast messages is smaller, 128 KiB). - let maybe_more_pool_imports = metered_poll_nested_stream_with_budget!( - poll_durations.acc_pending_imports, - "net::tx", - "Batched pool imports stream", - DEFAULT_BUDGET_TRY_DRAIN_PENDING_POOL_IMPORTS, - this.pool_imports.poll_next_unpin(cx), - |batch_results| this.on_batch_import_result(batch_results) - ); - // Advance network/peer related events (update peers map). let maybe_more_network_events = metered_poll_nested_stream_with_budget!( poll_durations.acc_network_events, @@ -1351,6 +1328,29 @@ where |event| this.on_network_tx_event(event), ); + // Advance pool imports (flush txns to pool). + // + // Note, this is done in batches. A batch is filled from one `Transactions` + // broadcast messages or one `PooledTransactions` response at a time. The + // minimum batch size is 1 transaction (and might often be the case with blob + // transactions). + // + // The smallest decodable transaction is an empty legacy transaction, 10 bytes + // (2 MiB / 10 bytes > 200k transactions). + // + // Since transactions aren't validated until they are inserted into the pool, + // this can potentially validate >200k transactions. More if the message size + // is bigger than the soft limit on a `PooledTransactions` response which is + // 2 MiB (`Transactions` broadcast messages is smaller, 128 KiB). + let maybe_more_pool_imports = metered_poll_nested_stream_with_budget!( + poll_durations.acc_pending_imports, + "net::tx", + "Batched pool imports stream", + DEFAULT_BUDGET_TRY_DRAIN_PENDING_POOL_IMPORTS, + this.pool_imports.poll_next_unpin(cx), + |batch_results| this.on_batch_import_result(batch_results) + ); + // Tries to drain hashes pending fetch cache if the tx manager currently has // capacity for this (fetch txns). // From 8e6d958f4e419bab2098ef14174fe2dfcefc792d Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Sat, 12 Oct 2024 10:06:19 +0200 Subject: [PATCH 131/159] fix(net): decrease budget for header reqs to process before yielding thread (#11636) --- crates/net/network/src/budget.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/net/network/src/budget.rs b/crates/net/network/src/budget.rs index 0b5e3d3a90bb3..2f878cdb2e880 100644 --- a/crates/net/network/src/budget.rs +++ b/crates/net/network/src/budget.rs @@ -5,8 +5,8 @@ pub const DEFAULT_BUDGET_TRY_DRAIN_STREAM: u32 = 10; /// Default budget to try and drain headers and bodies download streams. /// -/// Default is 4 iterations. -pub const DEFAULT_BUDGET_TRY_DRAIN_DOWNLOADERS: u32 = 4; +/// Default is 1 iteration. +pub const DEFAULT_BUDGET_TRY_DRAIN_DOWNLOADERS: u32 = 1; /// Default budget to try and drain [`Swarm`](crate::swarm::Swarm). /// From ec9f4fd3371550c334468debc630dfa73fa2a16e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 12 Oct 2024 13:19:48 +0200 Subject: [PATCH 132/159] fix: dont remove txs manually (#11683) --- crates/consensus/auto-seal/src/task.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/consensus/auto-seal/src/task.rs b/crates/consensus/auto-seal/src/task.rs index e4873615f1d0f..cb0586d44407d 100644 --- a/crates/consensus/auto-seal/src/task.rs +++ b/crates/consensus/auto-seal/src/task.rs @@ -113,7 +113,6 @@ where let to_engine = this.to_engine.clone(); let client = this.client.clone(); let chain_spec = Arc::clone(&this.chain_spec); - let pool = this.pool.clone(); let events = this.pipe_line_events.take(); let executor = this.block_executor.clone(); @@ -139,11 +138,6 @@ where &executor, ) { Ok((new_header, _bundle_state)) => { - // clear all transactions from pool - pool.remove_transactions( - transactions.iter().map(|tx| tx.hash()).collect(), - ); - let state = ForkchoiceState { head_block_hash: new_header.hash(), finalized_block_hash: new_header.hash(), From 4f32af7e2ec351844136faf397b706c2a3a57648 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 12 Oct 2024 14:42:46 +0200 Subject: [PATCH 133/159] fix: only +1 on the pool nonce (#11680) --- crates/rpc/rpc-eth-api/src/helpers/state.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index d601e43d90a81..7b11ce6afe632 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -284,8 +284,8 @@ pub trait LoadState: EthApiTypes { Self: SpawnBlocking, { self.spawn_blocking_io(move |this| { - // first fetch the on chain nonce - let nonce = this + // first fetch the on chain nonce of the account + let on_chain_account_nonce = this .state_at_block_id_or_latest(block_id)? .account_nonce(address) .map_err(Self::Error::from_eth_err)? @@ -297,20 +297,24 @@ pub trait LoadState: EthApiTypes { this.pool().get_highest_transaction_by_sender(address) { { - // and the corresponding txcount is nonce + 1 - let next_nonce = - nonce.max(highest_pool_tx.nonce()).checked_add(1).ok_or_else(|| { + // and the corresponding txcount is nonce + 1 of the highest tx in the pool + // (on chain nonce is increased after tx) + let next_tx_nonce = + highest_pool_tx.nonce().checked_add(1).ok_or_else(|| { Self::Error::from(EthApiError::InvalidTransaction( RpcInvalidTransactionError::NonceMaxValue, )) })?; - let tx_count = nonce.max(next_nonce); + // guard against drifts in the pool + let next_tx_nonce = on_chain_account_nonce.max(next_tx_nonce); + + let tx_count = on_chain_account_nonce.max(next_tx_nonce); return Ok(U256::from(tx_count)); } } } - Ok(U256::from(nonce)) + Ok(U256::from(on_chain_account_nonce)) }) } From 4e4bca66746d44fcc14731b64768d9dfd6fa73fd Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Sat, 12 Oct 2024 14:46:03 +0200 Subject: [PATCH 134/159] test: more unit tests for `HashedPostState` (#11663) --- crates/trie/trie/src/state.rs | 148 ++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/crates/trie/trie/src/state.rs b/crates/trie/trie/src/state.rs index 3b0af5cd879b6..2af48dfff7983 100644 --- a/crates/trie/trie/src/state.rs +++ b/crates/trie/trie/src/state.rs @@ -347,6 +347,15 @@ impl HashedStorageSorted { #[cfg(test)] mod tests { + use alloy_primitives::Bytes; + use revm::{ + db::{ + states::{plain_account::PlainStorage, StorageSlot}, + PlainAccount, StorageWithOriginalValues, + }, + primitives::{AccountInfo, Bytecode}, + }; + use super::*; #[test] @@ -422,4 +431,143 @@ mod tests { ); assert_eq!(account_storage.map(|st| st.wiped), Some(true)); } + + #[test] + fn test_hashed_post_state_from_bundle_state() { + // Prepare a random Ethereum address as a key for the account. + let address = Address::random(); + + // Create a mock account info object. + let account_info = AccountInfo { + balance: U256::from(123), + nonce: 42, + code_hash: B256::random(), + code: Some(Bytecode::LegacyRaw(Bytes::from(vec![1, 2]))), + }; + + let mut storage = StorageWithOriginalValues::default(); + storage.insert( + U256::from(1), + StorageSlot { present_value: U256::from(4), ..Default::default() }, + ); + + // Create a `BundleAccount` struct to represent the account and its storage. + let account = BundleAccount { + status: AccountStatus::Changed, + info: Some(account_info.clone()), + storage, + original_info: None, + }; + + // Create a vector of tuples representing the bundle state. + let state = vec![(&address, &account)]; + + // Convert the bundle state into a hashed post state. + let hashed_state = HashedPostState::from_bundle_state(state); + + // Validate the hashed post state. + assert_eq!(hashed_state.accounts.len(), 1); + assert_eq!(hashed_state.storages.len(), 1); + + // Validate the account info. + assert_eq!( + *hashed_state.accounts.get(&keccak256(address)).unwrap(), + Some(account_info.into()) + ); + } + + #[test] + fn test_hashed_post_state_from_cache_state() { + // Prepare a random Ethereum address. + let address = Address::random(); + + // Create mock account info. + let account_info = AccountInfo { + balance: U256::from(500), + nonce: 5, + code_hash: B256::random(), + code: None, + }; + + let mut storage = PlainStorage::default(); + storage.insert(U256::from(1), U256::from(35636)); + + // Create a `CacheAccount` with the mock account info. + let account = CacheAccount { + account: Some(PlainAccount { info: account_info.clone(), storage }), + status: AccountStatus::Changed, + }; + + // Create a vector of tuples representing the cache state. + let state = vec![(&address, &account)]; + + // Convert the cache state into a hashed post state. + let hashed_state = HashedPostState::from_cache_state(state); + + // Validate the hashed post state. + assert_eq!(hashed_state.accounts.len(), 1); + assert_eq!(hashed_state.storages.len(), 1); + + // Validate the account info. + assert_eq!( + *hashed_state.accounts.get(&keccak256(address)).unwrap(), + Some(account_info.into()) + ); + } + + #[test] + fn test_hashed_post_state_with_accounts() { + // Prepare random addresses and mock account info. + let address_1 = Address::random(); + let address_2 = Address::random(); + + let account_info_1 = AccountInfo { + balance: U256::from(1000), + nonce: 1, + code_hash: B256::random(), + code: None, + }; + + // Create hashed accounts with addresses. + let account_1 = (keccak256(address_1), Some(account_info_1.into())); + let account_2 = (keccak256(address_2), None); + + // Add accounts to the hashed post state. + let hashed_state = HashedPostState::default().with_accounts(vec![account_1, account_2]); + + // Validate the hashed post state. + assert_eq!(hashed_state.accounts.len(), 2); + assert!(hashed_state.accounts.contains_key(&keccak256(address_1))); + assert!(hashed_state.accounts.contains_key(&keccak256(address_2))); + } + + #[test] + fn test_hashed_post_state_with_storages() { + // Prepare random addresses and mock storage entries. + let address_1 = Address::random(); + let address_2 = Address::random(); + + let storage_1 = (keccak256(address_1), HashedStorage::new(false)); + let storage_2 = (keccak256(address_2), HashedStorage::new(true)); + + // Add storages to the hashed post state. + let hashed_state = HashedPostState::default().with_storages(vec![storage_1, storage_2]); + + // Validate the hashed post state. + assert_eq!(hashed_state.storages.len(), 2); + assert!(hashed_state.storages.contains_key(&keccak256(address_1))); + assert!(hashed_state.storages.contains_key(&keccak256(address_2))); + } + + #[test] + fn test_hashed_post_state_is_empty() { + // Create an empty hashed post state and validate it's empty. + let empty_state = HashedPostState::default(); + assert!(empty_state.is_empty()); + + // Add an account and validate the state is no longer empty. + let non_empty_state = HashedPostState::default() + .with_accounts(vec![(keccak256(Address::random()), Some(Account::default()))]); + assert!(!non_empty_state.is_empty()); + } } From f88f2faea544dd5e8c0b64aee126ab8e56b4d463 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 12 Oct 2024 15:27:52 +0200 Subject: [PATCH 135/159] docs: complete sentence (#11685) --- crates/net/network/src/transactions/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/net/network/src/transactions/config.rs b/crates/net/network/src/transactions/config.rs index 81ec293ea1ff9..b838f7cfe71be 100644 --- a/crates/net/network/src/transactions/config.rs +++ b/crates/net/network/src/transactions/config.rs @@ -47,7 +47,7 @@ pub enum TransactionPropagationMode { } impl TransactionPropagationMode { - /// Returns the number of peers that should + /// Returns the number of peers full transactions should be propagated to. pub(crate) fn full_peer_count(&self, peer_count: usize) -> usize { match self { Self::Sqrt => (peer_count as f64).sqrt().round() as usize, From a67818b990c744f742b509e87df191a29cf8312f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 12 Oct 2024 15:28:08 +0200 Subject: [PATCH 136/159] chore: we dont need sat here (#11678) --- crates/net/network/src/budget.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/net/network/src/budget.rs b/crates/net/network/src/budget.rs index 2f878cdb2e880..6eba076f6b9f1 100644 --- a/crates/net/network/src/budget.rs +++ b/crates/net/network/src/budget.rs @@ -55,7 +55,7 @@ macro_rules! poll_nested_stream_with_budget { let mut f = $on_ready_some; f(item); - budget = budget.saturating_sub(1); + budget -= 1; if budget == 0 { break true } From 901f20dbc349e642b86571275b5782b28a795f0d Mon Sep 17 00:00:00 2001 From: Delweng Date: Sat, 12 Oct 2024 21:29:20 +0800 Subject: [PATCH 137/159] chore(stages): reduce the progress logging (#11653) Signed-off-by: jsvisa --- .../stages/src/stages/hashing_account.rs | 20 +++++---- .../stages/src/stages/hashing_storage.rs | 20 +++++---- crates/stages/stages/src/stages/headers.rs | 24 +++++----- crates/stages/stages/src/stages/tx_lookup.rs | 8 +++- crates/stages/stages/src/stages/utils.rs | 44 ++++++++++++++----- 5 files changed, 74 insertions(+), 42 deletions(-) diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index 14afb37d81db5..2fdccd7837fba 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -1,3 +1,4 @@ +use crate::log_progress; use alloy_primitives::{keccak256, B256}; use itertools::Itertools; use reth_config::config::{EtlConfig, HashingConfig}; @@ -18,6 +19,7 @@ use std::{ fmt::Debug, ops::{Range, RangeInclusive}, sync::mpsc::{self, Receiver}, + time::Instant, }; use tracing::*; @@ -186,16 +188,16 @@ where let mut hashed_account_cursor = tx.cursor_write::>()?; - let total_hashes = collector.len(); - let interval = (total_hashes / 10).max(1); + let total = collector.len(); + let mut last_log = Instant::now(); for (index, item) in collector.iter()?.enumerate() { - if index > 0 && index % interval == 0 { - info!( - target: "sync::stages::hashing_account", - progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), - "Inserting hashes" - ); - } + log_progress!( + "sync::stages::hashing_account", + index, + total, + last_log, + "Inserting hashes" + ); let (key, value) = item?; hashed_account_cursor diff --git a/crates/stages/stages/src/stages/hashing_storage.rs b/crates/stages/stages/src/stages/hashing_storage.rs index ef070d30c6d68..0646032935f11 100644 --- a/crates/stages/stages/src/stages/hashing_storage.rs +++ b/crates/stages/stages/src/stages/hashing_storage.rs @@ -1,3 +1,4 @@ +use crate::log_progress; use alloy_primitives::{bytes::BufMut, keccak256, B256}; use itertools::Itertools; use reth_config::config::{EtlConfig, HashingConfig}; @@ -19,6 +20,7 @@ use reth_storage_errors::provider::ProviderResult; use std::{ fmt::Debug, sync::mpsc::{self, Receiver}, + time::Instant, }; use tracing::*; @@ -117,17 +119,17 @@ where collect(&mut channels, &mut collector)?; - let total_hashes = collector.len(); - let interval = (total_hashes / 10).max(1); + let total = collector.len(); + let mut last_log = Instant::now(); let mut cursor = tx.cursor_dup_write::()?; for (index, item) in collector.iter()?.enumerate() { - if index > 0 && index % interval == 0 { - info!( - target: "sync::stages::hashing_storage", - progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), - "Inserting hashes" - ); - } + log_progress!( + "sync::stages::hashing_storage", + index, + total, + last_log, + "Inserting hashes" + ); let (addr_key, value) = item?; cursor.append_dup( diff --git a/crates/stages/stages/src/stages/headers.rs b/crates/stages/stages/src/stages/headers.rs index 199e015c2dce9..eaa2ea01f0d85 100644 --- a/crates/stages/stages/src/stages/headers.rs +++ b/crates/stages/stages/src/stages/headers.rs @@ -1,3 +1,4 @@ +use crate::log_progress; use alloy_primitives::{BlockHash, BlockNumber, Bytes, B256}; use futures_util::StreamExt; use reth_config::config::EtlConfig; @@ -25,6 +26,7 @@ use reth_storage_errors::provider::ProviderError; use std::{ sync::Arc, task::{ready, Context, Poll}, + time::Instant, }; use tokio::sync::watch; use tracing::*; @@ -95,9 +97,9 @@ where provider: &impl DBProvider, static_file_provider: StaticFileProvider, ) -> Result { - let total_headers = self.header_collector.len(); + let total = self.header_collector.len(); - info!(target: "sync::stages::headers", total = total_headers, "Writing headers"); + info!(target: "sync::stages::headers", total, "Writing headers"); // Consistency check of expected headers in static files vs DB is done on provider::sync_gap // when poll_execute_ready is polled. @@ -113,13 +115,11 @@ where // Although headers were downloaded in reverse order, the collector iterates it in ascending // order let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; - let interval = (total_headers / 10).max(1); + let mut last_log = Instant::now(); for (index, header) in self.header_collector.iter()?.enumerate() { let (_, header_buf) = header?; - if index > 0 && index % interval == 0 && total_headers > 100 { - info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers"); - } + log_progress!("sync::stages::headers", index, total, last_log, "Writing headers"); let sealed_header: SealedHeader = bincode::deserialize::>(&header_buf) @@ -147,7 +147,7 @@ where writer.append_header(&header, td, &header_hash)?; } - info!(target: "sync::stages::headers", total = total_headers, "Writing headers hash index"); + info!(target: "sync::stages::headers", total, "Writing headers hash index"); let mut cursor_header_numbers = provider.tx_ref().cursor_write::>()?; @@ -168,9 +168,13 @@ where for (index, hash_to_number) in self.hash_collector.iter()?.enumerate() { let (hash, number) = hash_to_number?; - if index > 0 && index % interval == 0 && total_headers > 100 { - info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers hash index"); - } + log_progress!( + "sync::stages::headers", + index, + total, + last_log, + "Writing headers hash index" + ); if first_sync { cursor_header_numbers.append( diff --git a/crates/stages/stages/src/stages/tx_lookup.rs b/crates/stages/stages/src/stages/tx_lookup.rs index 60c958abf8623..c5ccff54c525d 100644 --- a/crates/stages/stages/src/stages/tx_lookup.rs +++ b/crates/stages/stages/src/stages/tx_lookup.rs @@ -1,3 +1,4 @@ +use super::utils::LOG_INTERVAL; use alloy_primitives::{TxHash, TxNumber}; use num_traits::Zero; use reth_config::config::{EtlConfig, TransactionLookupConfig}; @@ -17,6 +18,7 @@ use reth_stages_api::{ UnwindInput, UnwindOutput, }; use reth_storage_errors::provider::ProviderError; +use std::time::Instant; use tracing::*; /// The transaction lookup stage. @@ -147,16 +149,18 @@ where .cursor_write::>()?; let total_hashes = hash_collector.len(); - let interval = (total_hashes / 10).max(1); + let mut last_log = Instant::now(); for (index, hash_to_number) in hash_collector.iter()?.enumerate() { let (hash, number) = hash_to_number?; - if index > 0 && index % interval == 0 { + let now = Instant::now(); + if now.duration_since(last_log) >= LOG_INTERVAL { info!( target: "sync::stages::transaction_lookup", ?append_only, progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), "Inserting hashes" ); + last_log = now; } let key = RawKey::::from_vec(hash); diff --git a/crates/stages/stages/src/stages/utils.rs b/crates/stages/stages/src/stages/utils.rs index caf039faca108..af6594cd78611 100644 --- a/crates/stages/stages/src/stages/utils.rs +++ b/crates/stages/stages/src/stages/utils.rs @@ -12,12 +12,36 @@ use reth_db_api::{ use reth_etl::Collector; use reth_provider::DBProvider; use reth_stages_api::StageError; -use std::{collections::HashMap, hash::Hash, ops::RangeBounds}; +use std::{ + collections::HashMap, + hash::Hash, + ops::RangeBounds, + time::{Duration, Instant}, +}; use tracing::info; /// Number of blocks before pushing indices from cache to [`Collector`] const DEFAULT_CACHE_THRESHOLD: u64 = 100_000; +/// Log interval for progress. +pub(crate) const LOG_INTERVAL: Duration = Duration::from_secs(5); + +/// Log progress at a regular interval. +#[macro_export] +macro_rules! log_progress { + ($target:expr, $index:expr, $total:expr, $last_log:expr, $message:expr) => { + let now = std::time::Instant::now(); + if now.duration_since($last_log) >= $crate::stages::utils::LOG_INTERVAL { + info!( + target: $target, + progress = %format!("{:.2}%", ($index as f64 / $total as f64) * 100.0), + $message + ); + $last_log = now; + } + } +} + /// Collects all history (`H`) indices for a range of changesets (`CS`) and stores them in a /// [`Collector`]. /// @@ -65,18 +89,16 @@ where }; // observability - let total_changesets = provider.tx_ref().entries::()?; - let interval = (total_changesets / 1000).max(1); + let total = provider.tx_ref().entries::()?; + let mut last_log = Instant::now(); let mut flush_counter = 0; let mut current_block_number = u64::MAX; - for (idx, entry) in changeset_cursor.walk_range(range)?.enumerate() { + for (index, entry) in changeset_cursor.walk_range(range)?.enumerate() { let (block_number, key) = partial_key_factory(entry?); cache.entry(key).or_default().push(block_number); - if idx > 0 && idx % interval == 0 && total_changesets > 1000 { - info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices"); - } + log_progress!("sync::stages::index_history", index, total, last_log, "Collecting indices"); // Make sure we only flush the cache every DEFAULT_CACHE_THRESHOLD blocks. if current_block_number != block_number { @@ -120,17 +142,15 @@ where let mut current_list = Vec::::new(); // observability - let total_entries = collector.len(); - let interval = (total_entries / 100).max(1); + let total = collector.len(); + let mut last_log = Instant::now(); for (index, element) in collector.iter()?.enumerate() { let (k, v) = element?; let sharded_key = decode_key(k)?; let new_list = BlockNumberList::decompress_owned(v)?; - if index > 0 && index % interval == 0 && total_entries > 100 { - info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices"); - } + log_progress!("sync::stages::index_history", index, total, last_log, "Writing indices"); // AccountsHistory: `Address`. // StorageHistory: `Address.StorageKey`. From efd42a83248ef5ce4593d3b3059c18cb147a3f05 Mon Sep 17 00:00:00 2001 From: Delweng Date: Sat, 12 Oct 2024 23:18:43 +0800 Subject: [PATCH 138/159] chore(forkid): simplify and add comment of the set_head_priv (#11686) Signed-off-by: jsvisa --- crates/ethereum-forks/src/forkid.rs | 34 +++++++++++------------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/crates/ethereum-forks/src/forkid.rs b/crates/ethereum-forks/src/forkid.rs index 8a344e2dd2cca..6876d0eb926d1 100644 --- a/crates/ethereum-forks/src/forkid.rs +++ b/crates/ethereum-forks/src/forkid.rs @@ -258,32 +258,24 @@ impl ForkFilter { } fn set_head_priv(&mut self, head: Head) -> Option { - let recompute_cache = { - let head_in_past = match self.cache.epoch_start { - ForkFilterKey::Block(epoch_start_block) => head.number < epoch_start_block, - ForkFilterKey::Time(epoch_start_time) => head.timestamp < epoch_start_time, - }; - let head_in_future = match self.cache.epoch_end { - Some(ForkFilterKey::Block(epoch_end_block)) => head.number >= epoch_end_block, - Some(ForkFilterKey::Time(epoch_end_time)) => head.timestamp >= epoch_end_time, - None => false, - }; - - head_in_past || head_in_future + let head_in_past = match self.cache.epoch_start { + ForkFilterKey::Block(epoch_start_block) => head.number < epoch_start_block, + ForkFilterKey::Time(epoch_start_time) => head.timestamp < epoch_start_time, }; - - // recompute the cache - let transition = if recompute_cache { - let past = self.current(); - self.cache = Cache::compute_cache(&self.forks, head); - Some(ForkTransition { current: self.current(), past }) - } else { - None + let head_in_future = match self.cache.epoch_end { + Some(ForkFilterKey::Block(epoch_end_block)) => head.number >= epoch_end_block, + Some(ForkFilterKey::Time(epoch_end_time)) => head.timestamp >= epoch_end_time, + None => false, }; self.head = head; - transition + // Recompute the cache if the head is in the past or future epoch. + (head_in_past || head_in_future).then(|| { + let past = self.current(); + self.cache = Cache::compute_cache(&self.forks, head); + ForkTransition { current: self.current(), past } + }) } /// Set the current head. From 406f23fa893691da08c886a1d5a627954a4d6d7d Mon Sep 17 00:00:00 2001 From: Kien Trinh <51135161+kien6034@users.noreply.github.com> Date: Sat, 12 Oct 2024 22:45:47 +0700 Subject: [PATCH 139/159] feat: add helper function to modify node builder (#11682) --- crates/node/builder/src/builder/mod.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index 8b57781ea97c3..15bdc730a85d6 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -162,6 +162,14 @@ impl NodeBuilder<(), ChainSpec> { pub const fn new(config: NodeConfig) -> Self { Self { config, database: () } } + + /// Apply a function to the builder + pub fn apply(self, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + f(self) + } } impl NodeBuilder { @@ -400,6 +408,14 @@ where &self.builder.config } + /// Apply a function to the builder + pub fn apply(self, f: F) -> Self + where + F: FnOnce(Self) -> Self, + { + f(self) + } + /// Sets the hook that is run once the node's components are initialized. pub fn on_component_initialized(self, hook: F) -> Self where From 6587663bda2de589592dbb4d6ed165e4e489e25a Mon Sep 17 00:00:00 2001 From: greged93 <82421016+greged93@users.noreply.github.com> Date: Sat, 12 Oct 2024 17:57:25 +0200 Subject: [PATCH 140/159] feat(op): opchainspec builder (#11630) Co-authored-by: Matthias Seitz --- crates/chainspec/src/spec.rs | 67 ++-------- crates/optimism/chainspec/src/lib.rs | 165 ++++++++++++++++++++++-- crates/optimism/evm/src/config.rs | 15 ++- crates/optimism/evm/src/execute.rs | 19 +-- crates/optimism/node/tests/e2e/utils.rs | 14 +- 5 files changed, 179 insertions(+), 101 deletions(-) diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index d6ca92aa24ebe..f80a20924394c 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -755,13 +755,19 @@ impl ChainSpecBuilder { } /// Add the given fork with the given activation condition to the spec. - pub fn with_fork(mut self, fork: EthereumHardfork, condition: ForkCondition) -> Self { + pub fn with_fork(mut self, fork: H, condition: ForkCondition) -> Self { self.hardforks.insert(fork, condition); self } + /// Add the given chain hardforks to the spec. + pub fn with_forks(mut self, forks: ChainHardforks) -> Self { + self.hardforks = forks; + self + } + /// Remove the given fork from the spec. - pub fn without_fork(mut self, fork: EthereumHardfork) -> Self { + pub fn without_fork(mut self, fork: H) -> Self { self.hardforks.remove(fork); self } @@ -876,63 +882,6 @@ impl ChainSpecBuilder { self } - /// Enable Bedrock at genesis - #[cfg(feature = "optimism")] - pub fn bedrock_activated(mut self) -> Self { - self = self.paris_activated(); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Bedrock, ForkCondition::Block(0)); - self - } - - /// Enable Regolith at genesis - #[cfg(feature = "optimism")] - pub fn regolith_activated(mut self) -> Self { - self = self.bedrock_activated(); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Regolith, ForkCondition::Timestamp(0)); - self - } - - /// Enable Canyon at genesis - #[cfg(feature = "optimism")] - pub fn canyon_activated(mut self) -> Self { - self = self.regolith_activated(); - // Canyon also activates changes from L1's Shanghai hardfork - self.hardforks.insert(EthereumHardfork::Shanghai, ForkCondition::Timestamp(0)); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Canyon, ForkCondition::Timestamp(0)); - self - } - - /// Enable Ecotone at genesis - #[cfg(feature = "optimism")] - pub fn ecotone_activated(mut self) -> Self { - self = self.canyon_activated(); - self.hardforks.insert(EthereumHardfork::Cancun, ForkCondition::Timestamp(0)); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Ecotone, ForkCondition::Timestamp(0)); - self - } - - /// Enable Fjord at genesis - #[cfg(feature = "optimism")] - pub fn fjord_activated(mut self) -> Self { - self = self.ecotone_activated(); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Fjord, ForkCondition::Timestamp(0)); - self - } - - /// Enable Granite at genesis - #[cfg(feature = "optimism")] - pub fn granite_activated(mut self) -> Self { - self = self.fjord_activated(); - self.hardforks - .insert(reth_optimism_forks::OptimismHardfork::Granite, ForkCondition::Timestamp(0)); - self - } - /// Build the resulting [`ChainSpec`]. /// /// # Panics diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index 9b2d1c8c11ba6..5ebc18f676886 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -16,25 +16,154 @@ mod dev; mod op; mod op_sepolia; -use std::fmt::Display; - +use alloy_chains::Chain; use alloy_genesis::Genesis; use alloy_primitives::{Parity, Signature, B256, U256}; pub use base::BASE_MAINNET; pub use base_sepolia::BASE_SEPOLIA; +use derive_more::{Constructor, Deref, From, Into}; pub use dev::OP_DEV; +use once_cell::sync::OnceCell; pub use op::OP_MAINNET; pub use op_sepolia::OP_SEPOLIA; - -use derive_more::{Constructor, Deref, Into}; -use once_cell::sync::OnceCell; use reth_chainspec::{ - BaseFeeParams, BaseFeeParamsKind, ChainSpec, DepositContract, EthChainSpec, EthereumHardforks, - ForkFilter, ForkId, Hardforks, Head, + BaseFeeParams, BaseFeeParamsKind, ChainSpec, ChainSpecBuilder, DepositContract, EthChainSpec, + EthereumHardforks, ForkFilter, ForkId, Hardforks, Head, }; -use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition}; +use reth_ethereum_forks::{ChainHardforks, EthereumHardfork, ForkCondition, Hardfork}; use reth_network_peers::NodeRecord; use reth_primitives_traits::Header; +use std::fmt::Display; + +/// Chain spec builder for a OP stack chain. +#[derive(Debug, Default, From)] +pub struct OpChainSpecBuilder { + /// [`ChainSpecBuilder`] + inner: ChainSpecBuilder, +} + +impl OpChainSpecBuilder { + /// Construct a new builder from the base mainnet chain spec. + pub fn base_mainnet() -> Self { + let mut inner = ChainSpecBuilder::default() + .chain(BASE_MAINNET.chain) + .genesis(BASE_MAINNET.genesis.clone()); + let forks = BASE_MAINNET.hardforks.clone(); + inner = inner.with_forks(forks); + + Self { inner } + } + + /// Construct a new builder from the optimism mainnet chain spec. + pub fn optimism_mainnet() -> Self { + let mut inner = + ChainSpecBuilder::default().chain(OP_MAINNET.chain).genesis(OP_MAINNET.genesis.clone()); + let forks = OP_MAINNET.hardforks.clone(); + inner = inner.with_forks(forks); + + Self { inner } + } +} + +impl OpChainSpecBuilder { + /// Set the chain ID + pub fn chain(mut self, chain: Chain) -> Self { + self.inner = self.inner.chain(chain); + self + } + + /// Set the genesis block. + pub fn genesis(mut self, genesis: Genesis) -> Self { + self.inner = self.inner.genesis(genesis); + self + } + + /// Add the given fork with the given activation condition to the spec. + pub fn with_fork(mut self, fork: H, condition: ForkCondition) -> Self { + self.inner = self.inner.with_fork(fork, condition); + self + } + + /// Add the given forks with the given activation condition to the spec. + pub fn with_forks(mut self, forks: ChainHardforks) -> Self { + self.inner = self.inner.with_forks(forks); + self + } + + /// Remove the given fork from the spec. + pub fn without_fork(mut self, fork: reth_optimism_forks::OptimismHardfork) -> Self { + self.inner = self.inner.without_fork(fork); + self + } + + /// Enable Bedrock at genesis + pub fn bedrock_activated(mut self) -> Self { + self.inner = self.inner.paris_activated(); + self.inner = self + .inner + .with_fork(reth_optimism_forks::OptimismHardfork::Bedrock, ForkCondition::Block(0)); + self + } + + /// Enable Regolith at genesis + pub fn regolith_activated(mut self) -> Self { + self = self.bedrock_activated(); + self.inner = self.inner.with_fork( + reth_optimism_forks::OptimismHardfork::Regolith, + ForkCondition::Timestamp(0), + ); + self + } + + /// Enable Canyon at genesis + pub fn canyon_activated(mut self) -> Self { + self = self.regolith_activated(); + // Canyon also activates changes from L1's Shanghai hardfork + self.inner = self.inner.with_fork(EthereumHardfork::Shanghai, ForkCondition::Timestamp(0)); + self.inner = self + .inner + .with_fork(reth_optimism_forks::OptimismHardfork::Canyon, ForkCondition::Timestamp(0)); + self + } + + /// Enable Ecotone at genesis + pub fn ecotone_activated(mut self) -> Self { + self = self.canyon_activated(); + self.inner = self.inner.with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(0)); + self.inner = self + .inner + .with_fork(reth_optimism_forks::OptimismHardfork::Ecotone, ForkCondition::Timestamp(0)); + self + } + + /// Enable Fjord at genesis + pub fn fjord_activated(mut self) -> Self { + self = self.ecotone_activated(); + self.inner = self + .inner + .with_fork(reth_optimism_forks::OptimismHardfork::Fjord, ForkCondition::Timestamp(0)); + self + } + + /// Enable Granite at genesis + pub fn granite_activated(mut self) -> Self { + self = self.fjord_activated(); + self.inner = self + .inner + .with_fork(reth_optimism_forks::OptimismHardfork::Granite, ForkCondition::Timestamp(0)); + self + } + + /// Build the resulting [`OpChainSpec`]. + /// + /// # Panics + /// + /// This function panics if the chain ID and genesis is not set ([`Self::chain`] and + /// [`Self::genesis`]) + pub fn build(self) -> OpChainSpec { + OpChainSpec { inner: self.inner.build() } + } +} /// OP stack chain spec type. #[derive(Debug, Clone, Deref, Into, Constructor, PartialEq, Eq)] @@ -286,6 +415,8 @@ mod tests { #[test] fn base_mainnet_forkids() { + let base_mainnet = OpChainSpecBuilder::base_mainnet().build(); + let _ = base_mainnet.genesis_hash.set(BASE_MAINNET.genesis_hash.get().copied().unwrap()); test_fork_ids( &BASE_MAINNET, &[ @@ -372,8 +503,12 @@ mod tests { #[test] fn op_mainnet_forkids() { + let op_mainnet = OpChainSpecBuilder::optimism_mainnet().build(); + // for OP mainnet we have to do this because the genesis header can't be properly computed + // from the genesis.json file + let _ = op_mainnet.genesis_hash.set(OP_MAINNET.genesis_hash()); test_fork_ids( - &OP_MAINNET, + &op_mainnet, &[ ( Head { number: 0, ..Default::default() }, @@ -483,9 +618,19 @@ mod tests { ) } + #[test] + fn latest_base_mainnet_fork_id_with_builder() { + let base_mainnet = OpChainSpecBuilder::base_mainnet().build(); + assert_eq!( + ForkId { hash: ForkHash([0xbc, 0x38, 0xf9, 0xca]), next: 0 }, + base_mainnet.latest_fork_id() + ) + } + #[test] fn is_bedrock_active() { - assert!(!OP_MAINNET.is_bedrock_active_at_block(1)) + let op_mainnet = OpChainSpecBuilder::optimism_mainnet().build(); + assert!(!op_mainnet.is_bedrock_active_at_block(1)) } #[test] diff --git a/crates/optimism/evm/src/config.rs b/crates/optimism/evm/src/config.rs index 95d2c9a66fa9a..668fcba4ddc4b 100644 --- a/crates/optimism/evm/src/config.rs +++ b/crates/optimism/evm/src/config.rs @@ -1,5 +1,5 @@ -use reth_chainspec::ChainSpec; use reth_ethereum_forks::{EthereumHardfork, Head}; +use reth_optimism_chainspec::OpChainSpec; use reth_optimism_forks::OptimismHardfork; /// Returns the revm [`SpecId`](revm_primitives::SpecId) at the given timestamp. @@ -9,7 +9,7 @@ use reth_optimism_forks::OptimismHardfork; /// This is only intended to be used after the Bedrock, when hardforks are activated by /// timestamp. pub fn revm_spec_by_timestamp_after_bedrock( - chain_spec: &ChainSpec, + chain_spec: &OpChainSpec, timestamp: u64, ) -> revm_primitives::SpecId { if chain_spec.fork(OptimismHardfork::Granite).active_at_timestamp(timestamp) { @@ -28,7 +28,7 @@ pub fn revm_spec_by_timestamp_after_bedrock( } /// Map the latest active hardfork at the given block to a revm [`SpecId`](revm_primitives::SpecId). -pub fn revm_spec(chain_spec: &ChainSpec, block: &Head) -> revm_primitives::SpecId { +pub fn revm_spec(chain_spec: &OpChainSpec, block: &Head) -> revm_primitives::SpecId { if chain_spec.fork(OptimismHardfork::Granite).active_at_head(block) { revm_primitives::GRANITE } else if chain_spec.fork(OptimismHardfork::Fjord).active_at_head(block) { @@ -79,12 +79,13 @@ pub fn revm_spec(chain_spec: &ChainSpec, block: &Head) -> revm_primitives::SpecI mod tests { use super::*; use reth_chainspec::ChainSpecBuilder; + use reth_optimism_chainspec::{OpChainSpec, OpChainSpecBuilder}; #[test] fn test_revm_spec_by_timestamp_after_merge() { #[inline(always)] - fn op_cs(f: impl FnOnce(ChainSpecBuilder) -> ChainSpecBuilder) -> ChainSpec { - let cs = ChainSpecBuilder::mainnet().chain(reth_chainspec::Chain::from_id(10)); + fn op_cs(f: impl FnOnce(OpChainSpecBuilder) -> OpChainSpecBuilder) -> OpChainSpec { + let cs = ChainSpecBuilder::mainnet().chain(reth_chainspec::Chain::from_id(10)).into(); f(cs).build() } assert_eq!( @@ -116,8 +117,8 @@ mod tests { #[test] fn test_to_revm_spec() { #[inline(always)] - fn op_cs(f: impl FnOnce(ChainSpecBuilder) -> ChainSpecBuilder) -> ChainSpec { - let cs = ChainSpecBuilder::mainnet().chain(reth_chainspec::Chain::from_id(10)); + fn op_cs(f: impl FnOnce(OpChainSpecBuilder) -> OpChainSpecBuilder) -> OpChainSpec { + let cs = ChainSpecBuilder::mainnet().chain(reth_chainspec::Chain::from_id(10)).into(); f(cs).build() } assert_eq!( diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index 2491e99b5038e..f4abb8c887e24 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -513,8 +513,8 @@ mod tests { use crate::OpChainSpec; use alloy_consensus::TxEip1559; use alloy_primitives::{b256, Address, StorageKey, StorageValue}; - use reth_chainspec::{ChainSpecBuilder, MIN_TRANSACTION_GAS}; - use reth_optimism_chainspec::{optimism_deposit_tx_signature, BASE_MAINNET}; + use reth_chainspec::MIN_TRANSACTION_GAS; + use reth_optimism_chainspec::{optimism_deposit_tx_signature, OpChainSpecBuilder}; use reth_primitives::{Account, Block, BlockBody, Signature, Transaction, TransactionSigned}; use reth_revm::{ database::StateProviderDatabase, test_utils::StateProviderTest, L1_BLOCK_CONTRACT, @@ -548,8 +548,7 @@ mod tests { db } - fn executor_provider(chain_spec: Arc) -> OpExecutorProvider { - let chain_spec = Arc::new(OpChainSpec::new(Arc::unwrap_or_clone(chain_spec))); + fn executor_provider(chain_spec: Arc) -> OpExecutorProvider { OpExecutorProvider { evm_config: OptimismEvmConfig::new(chain_spec.clone()), chain_spec } } @@ -572,11 +571,7 @@ mod tests { let account = Account { balance: U256::MAX, ..Account::default() }; db.insert_account(addr, account, None, HashMap::default()); - let chain_spec = Arc::new( - ChainSpecBuilder::from(&Arc::new(BASE_MAINNET.inner.clone())) - .regolith_activated() - .build(), - ); + let chain_spec = Arc::new(OpChainSpecBuilder::base_mainnet().regolith_activated().build()); let tx = TransactionSigned::from_transaction_and_signature( Transaction::Eip1559(TxEip1559 { @@ -656,11 +651,7 @@ mod tests { db.insert_account(addr, account, None, HashMap::default()); - let chain_spec = Arc::new( - ChainSpecBuilder::from(&Arc::new(BASE_MAINNET.inner.clone())) - .canyon_activated() - .build(), - ); + let chain_spec = Arc::new(OpChainSpecBuilder::base_mainnet().canyon_activated().build()); let tx = TransactionSigned::from_transaction_and_signature( Transaction::Eip1559(TxEip1559 { diff --git a/crates/optimism/node/tests/e2e/utils.rs b/crates/optimism/node/tests/e2e/utils.rs index 1e9ffa652f1cc..863bf254e4949 100644 --- a/crates/optimism/node/tests/e2e/utils.rs +++ b/crates/optimism/node/tests/e2e/utils.rs @@ -1,15 +1,13 @@ -use std::sync::Arc; - use alloy_genesis::Genesis; use alloy_primitives::{Address, B256}; use reth::{rpc::types::engine::PayloadAttributes, tasks::TaskManager}; -use reth_chainspec::ChainSpecBuilder; use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet::Wallet, NodeHelperType}; -use reth_optimism_chainspec::{OpChainSpec, BASE_MAINNET}; +use reth_optimism_chainspec::OpChainSpecBuilder; use reth_optimism_node::{ node::OptimismAddOns, OptimismBuiltPayload, OptimismNode, OptimismPayloadBuilderAttributes, }; use reth_payload_builder::EthPayloadBuilderAttributes; +use std::sync::Arc; use tokio::sync::Mutex; /// Optimism Node Helper type @@ -19,13 +17,7 @@ pub(crate) async fn setup(num_nodes: usize) -> eyre::Result<(Vec, TaskMa let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap(); reth_e2e_test_utils::setup( num_nodes, - Arc::new(OpChainSpec::new( - ChainSpecBuilder::default() - .chain(BASE_MAINNET.chain) - .genesis(genesis) - .ecotone_activated() - .build(), - )), + Arc::new(OpChainSpecBuilder::base_mainnet().genesis(genesis).ecotone_activated().build()), false, ) .await From d22647a97778f521381b961c184ec07159bafecc Mon Sep 17 00:00:00 2001 From: Delweng Date: Sun, 13 Oct 2024 02:53:33 +0800 Subject: [PATCH 141/159] chore(clippy): fix the very complex type used (#11689) Signed-off-by: jsvisa Co-authored-by: Matthias Seitz --- crates/rpc/rpc-eth-api/src/helpers/block.rs | 9 +++++++-- crates/rpc/rpc-eth-api/src/helpers/call.rs | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index d7e081d80f807..898037f4a5867 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -13,6 +13,11 @@ use crate::{FromEthApiError, FullEthApiTypes, RpcBlock, RpcReceipt}; use super::{LoadPendingBlock, LoadReceipt, SpawnBlocking}; +/// Result type of the fetched block receipts. +pub type BlockReceiptsResult = Result>>, E>; +/// Result type of the fetched block and its receipts. +pub type BlockAndReceiptsResult = Result>)>, E>; + /// Block related functions for the [`EthApiServer`](crate::EthApiServer) trait in the /// `eth_` namespace. pub trait EthBlocks: LoadBlock { @@ -108,7 +113,7 @@ pub trait EthBlocks: LoadBlock { fn block_receipts( &self, block_id: BlockId, - ) -> impl Future>>, Self::Error>> + Send + ) -> impl Future> + Send where Self: LoadReceipt; @@ -116,7 +121,7 @@ pub trait EthBlocks: LoadBlock { fn load_block_and_receipts( &self, block_id: BlockId, - ) -> impl Future>)>, Self::Error>> + Send + ) -> impl Future> + Send where Self: LoadReceipt, { diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 0aa38a36e9dac..5f6f3d77f5558 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -41,6 +41,9 @@ use tracing::trace; use super::{LoadBlock, LoadPendingBlock, LoadState, LoadTransaction, SpawnBlocking, Trace}; +/// Result type for `eth_simulateV1` RPC method. +pub type SimulatedBlocksResult = Result>>, E>; + /// Execution related functions for the [`EthApiServer`](crate::EthApiServer) trait in /// the `eth_` namespace. pub trait EthCall: Call + LoadPendingBlock { @@ -62,8 +65,7 @@ pub trait EthCall: Call + LoadPendingBlock { &self, payload: SimulatePayload, block: Option, - ) -> impl Future>>, Self::Error>> - + Send + ) -> impl Future> + Send where Self: LoadBlock + FullEthApiTypes, { From 4ff7a2e77102d8f060107fb03d8f6c0f73ef2e2c Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Sun, 13 Oct 2024 11:25:56 +0200 Subject: [PATCH 142/159] chore(ci): unpin clippy (#11697) --- .github/workflows/lint.yml | 2 -- crates/storage/provider/src/providers/blockchain_provider.rs | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index fd71a8c636e4e..efa38857e06b2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,7 +30,6 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@clippy with: - toolchain: nightly-2024-09-25 components: clippy - uses: Swatinem/rust-cache@v2 with: @@ -52,7 +51,6 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@nightly with: - toolchain: nightly-2024-09-25 components: clippy - uses: Swatinem/rust-cache@v2 with: diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index dc4264210b275..9b88cab136f5c 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -197,8 +197,9 @@ impl BlockchainProvider2 { for (_, block_body) in block_bodies { let mut block_receipts = Vec::with_capacity(block_body.tx_count as usize); for tx_num in block_body.tx_num_range() { - let receipt = - receipt_iter.next().ok_or(ProviderError::ReceiptNotFound(tx_num.into()))?; + let receipt = receipt_iter + .next() + .ok_or_else(|| ProviderError::ReceiptNotFound(tx_num.into()))?; block_receipts.push(Some(receipt)); } receipts.push(block_receipts); From 20e63de516104a7886e0aa8e2bb03863f580c3a1 Mon Sep 17 00:00:00 2001 From: Thomas Coratger <60488569+tcoratger@users.noreply.github.com> Date: Sun, 13 Oct 2024 12:58:55 +0200 Subject: [PATCH 143/159] refactor(tree): small refac for `BlockBuffer` (#11691) --- crates/blockchain-tree/src/block_buffer.rs | 30 ++++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/crates/blockchain-tree/src/block_buffer.rs b/crates/blockchain-tree/src/block_buffer.rs index e116463e4af67..5d4ca2705cb10 100644 --- a/crates/blockchain-tree/src/block_buffer.rs +++ b/crates/blockchain-tree/src/block_buffer.rs @@ -2,7 +2,7 @@ use crate::metrics::BlockBufferMetrics; use alloy_primitives::{BlockHash, BlockNumber}; use reth_network::cache::LruCache; use reth_primitives::SealedBlockWithSenders; -use std::collections::{btree_map, hash_map, BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; /// Contains the tree of pending blocks that cannot be executed due to missing parent. /// It allows to store unconnected blocks for potential future inclusion. @@ -83,6 +83,7 @@ impl BlockBuffer { } self.metrics.blocks.set(self.blocks.len() as f64); } + /// Removes the given block from the buffer and also all the children of the block. /// /// This is used to get all the blocks that are dependent on the block that is included. @@ -93,10 +94,11 @@ impl BlockBuffer { &mut self, parent_hash: &BlockHash, ) -> Vec { - // remove parent block if present - let mut removed = self.remove_block(parent_hash).into_iter().collect::>(); - - removed.extend(self.remove_children(vec![*parent_hash])); + let removed = self + .remove_block(parent_hash) + .into_iter() + .chain(self.remove_children(vec![*parent_hash])) + .collect(); self.metrics.blocks.set(self.blocks.len() as f64); removed } @@ -126,10 +128,10 @@ impl BlockBuffer { /// Remove block entry fn remove_from_earliest_blocks(&mut self, number: BlockNumber, hash: &BlockHash) { - if let btree_map::Entry::Occupied(mut entry) = self.earliest_blocks.entry(number) { - entry.get_mut().remove(hash); - if entry.get().is_empty() { - entry.remove(); + if let Some(entry) = self.earliest_blocks.get_mut(&number) { + entry.remove(hash); + if entry.is_empty() { + self.earliest_blocks.remove(&number); } } } @@ -137,13 +139,13 @@ impl BlockBuffer { /// Remove from parent child connection. This method does not remove children. fn remove_from_parent(&mut self, parent_hash: BlockHash, hash: &BlockHash) { // remove from parent to child connection, but only for this block parent. - if let hash_map::Entry::Occupied(mut entry) = self.parent_to_child.entry(parent_hash) { - entry.get_mut().remove(hash); + if let Some(entry) = self.parent_to_child.get_mut(&parent_hash) { + entry.remove(hash); // if set is empty remove block entry. - if entry.get().is_empty() { - entry.remove(); + if entry.is_empty() { + self.parent_to_child.remove(&parent_hash); } - }; + } } /// Removes block from inner collections. From 984db4e4c461ce9c9cc38ad4e1604b3dac96c791 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 13 Oct 2024 16:32:52 +0200 Subject: [PATCH 144/159] chore: set request budget to 2 (#11699) Co-authored-by: Oliver --- crates/net/network/src/budget.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/net/network/src/budget.rs b/crates/net/network/src/budget.rs index 6eba076f6b9f1..5f5d88861815c 100644 --- a/crates/net/network/src/budget.rs +++ b/crates/net/network/src/budget.rs @@ -5,8 +5,8 @@ pub const DEFAULT_BUDGET_TRY_DRAIN_STREAM: u32 = 10; /// Default budget to try and drain headers and bodies download streams. /// -/// Default is 1 iteration. -pub const DEFAULT_BUDGET_TRY_DRAIN_DOWNLOADERS: u32 = 1; +/// Default is 2 iterations. +pub const DEFAULT_BUDGET_TRY_DRAIN_DOWNLOADERS: u32 = 2; /// Default budget to try and drain [`Swarm`](crate::swarm::Swarm). /// From ca8491188b030f6e27685bd7f2673207e71d2c2a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 13 Oct 2024 14:39:44 +0000 Subject: [PATCH 145/159] chore(deps): weekly `cargo update` (#11696) Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com> --- Cargo.lock | 352 ++++++++++++++++++++++++++--------------------------- 1 file changed, 170 insertions(+), 182 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22e1360a22e77..538816c7c4043 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -97,10 +97,11 @@ checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy-chains" -version = "0.1.34" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8158b4878c67837e5413721cc44298e6a2d88d39203175ea025e51892a16ba4c" +checksum = "156bfc5dcd52ef9a5f33381701fa03310317e14c65093a9430d3e3557b08dcd3" dependencies = [ + "alloy-primitives", "alloy-rlp", "arbitrary", "num_enum", @@ -129,9 +130,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b499852e1d0e9b8c6db0f24c48998e647c0d5762a01090f955106a7700e4611" +checksum = "f95d76a38cae906fd394a5afb0736aaceee5432efe76addfd71048e623e208af" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -205,9 +206,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a438d4486b5d525df3b3004188f9d5cd1d65cd30ecc41e5a3ccef6f6342e8af9" +checksum = "03c66eec1acdd96b39b995b8f5ee5239bc0c871d62c527ae1ac9fd1d7fecd455" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -297,7 +298,7 @@ dependencies = [ "getrandom 0.2.15", "hashbrown 0.15.0", "hex-literal", - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "k256", "keccak-asm", @@ -599,9 +600,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68e7f6e8fe5b443f82b3f1e15abfa191128f71569148428e49449d01f6f49e8b" +checksum = "661c516eb1fa3294cc7f2fb8955b3b609d639c282ac81a4eedb14d3046db503a" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -613,14 +614,14 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b96ce28d2fde09abb6135f410c41fad670a3a770b6776869bd852f1df102e6f" +checksum = "ecbabb8fc3d75a0c2cea5215be22e7a267e3efde835b0f2a8922f5e3f5d47683" dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.5.0", + "indexmap 2.6.0", "proc-macro-error2", "proc-macro2", "quote", @@ -631,9 +632,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-input" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "906746396a8296537745711630d9185746c0b50c033d5e9d18b0a6eba3d53f90" +checksum = "16517f2af03064485150d89746b8ffdcdbc9b6eeb3d536fb66efd7c2846fbc75" dependencies = [ "const-hex", "dunce", @@ -646,9 +647,9 @@ dependencies = [ [[package]] name = "alloy-sol-type-parser" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc85178909a49c8827ffccfc9103a7ce1767ae66a801b69bdc326913870bf8e6" +checksum = "c07ebb0c1674ff8cbb08378d7c2e0e27919d2a2dae07ad3bca26174deda8d389" dependencies = [ "serde", "winnow", @@ -656,9 +657,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d86a533ce22525969661b25dfe296c112d35eb6861f188fd284f8bd4bb3842ae" +checksum = "8e448d879903624863f608c552d10efb0e0905ddbee98b0049412799911eb062" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -1017,9 +1018,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.12" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" +checksum = "998282f8f49ccd6116b0ed8a4de0fbd3151697920e7c7533416d6e25e76434a7" dependencies = [ "brotli", "flate2", @@ -1204,26 +1205,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" -dependencies = [ - "bitflags 2.6.0", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.79", -] - [[package]] name = "bindgen" version = "0.70.1" @@ -1334,7 +1315,7 @@ dependencies = [ "bitflags 2.6.0", "boa_interner", "boa_macros", - "indexmap 2.5.0", + "indexmap 2.6.0", "num-bigint", "rustc-hash 2.0.0", ] @@ -1360,7 +1341,7 @@ dependencies = [ "fast-float", "hashbrown 0.14.5", "icu_normalizer", - "indexmap 2.5.0", + "indexmap 2.6.0", "intrusive-collections", "itertools 0.13.0", "num-bigint", @@ -1406,7 +1387,7 @@ dependencies = [ "boa_gc", "boa_macros", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "once_cell", "phf", "rustc-hash 2.0.0", @@ -1474,9 +1455,9 @@ dependencies = [ [[package]] name = "brotli" -version = "6.0.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1526,9 +1507,9 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26" +checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" dependencies = [ "proc-macro2", "quote", @@ -1620,9 +1601,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.24" +version = "1.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" +checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" dependencies = [ "jobserver", "libc", @@ -1715,9 +1696,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -1725,9 +1706,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -3140,9 +3121,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -3155,9 +3136,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -3165,15 +3146,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -3182,9 +3163,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -3203,9 +3184,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -3214,15 +3195,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" @@ -3236,9 +3217,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -3318,9 +3299,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -3397,7 +3378,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3448,6 +3429,8 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", "serde", ] @@ -3979,13 +3962,13 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "arbitrary", "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "serde", ] @@ -4002,7 +3985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", - "indexmap 2.5.0", + "indexmap 2.6.0", "is-terminal", "itoa", "log", @@ -4100,15 +4083,15 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "iri-string" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44bd7eced44cfe2cebc674adb2a7124a754a4b5269288d22e9f39f8fada3562d" +checksum = "dc0f0a572e8ffe56e2ff4f769f32ffe919282c3916799f8b68688b6030063bea" dependencies = [ "memchr", "serde", @@ -4140,15 +4123,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -4195,18 +4169,18 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] [[package]] name = "jsonrpsee" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126b48a5acc3c52fbd5381a77898cb60e145123179588a29e7ac48f9c06e401b" +checksum = "02f01f48e04e0d7da72280ab787c9943695699c9b32b99158ece105e8ad0afea" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", @@ -4222,9 +4196,9 @@ dependencies = [ [[package]] name = "jsonrpsee-client-transport" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf679a8e0e083c77997f7c4bb4ca826577105906027ae462aac70ff348d02c6a" +checksum = "d80eccbd47a7b9f1e67663fd846928e941cb49c65236e297dd11c9ea3c5e3387" dependencies = [ "base64 0.22.1", "futures-channel", @@ -4247,9 +4221,9 @@ dependencies = [ [[package]] name = "jsonrpsee-core" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0e503369a76e195b65af35058add0e6900b794a4e9a9316900ddd3a87a80477" +checksum = "3c2709a32915d816a6e8f625bf72cf74523ebe5d8829f895d6b041b1d3137818" dependencies = [ "async-trait", "bytes", @@ -4274,9 +4248,9 @@ dependencies = [ [[package]] name = "jsonrpsee-http-client" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c0caba4a6a8efbafeec9baa986aa22a75a96c29d3e4b0091b0098d6470efb5" +checksum = "cc54db939002b030e794fbfc9d5a925aa2854889c5a2f0352b0bffa54681707e" dependencies = [ "async-trait", "base64 0.22.1", @@ -4299,9 +4273,9 @@ dependencies = [ [[package]] name = "jsonrpsee-proc-macros" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc660a9389e2748e794a40673a4155d501f32db667757cdb80edeff0306b489b" +checksum = "3a9a4b2eaba8cc928f49c4ccf4fcfa65b690a73997682da99ed08f3393b51f07" dependencies = [ "heck", "proc-macro-crate", @@ -4312,9 +4286,9 @@ dependencies = [ [[package]] name = "jsonrpsee-server" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6e6c9b6d975edcb443565d648b605f3e85a04ec63aa6941811a8894cc9cded" +checksum = "e30110d0f2d7866c8cc6c86483bdab2eb9f4d2f0e20db55518b2bca84651ba8e" dependencies = [ "futures-util", "http", @@ -4339,9 +4313,9 @@ dependencies = [ [[package]] name = "jsonrpsee-types" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8fb16314327cbc94fdf7965ef7e4422509cd5597f76d137bd104eb34aeede67" +checksum = "1ca331cd7b3fe95b33432825c2d4c9f5a43963e207fdc01ae67f9fd80ab0930f" dependencies = [ "http", "serde", @@ -4351,9 +4325,9 @@ dependencies = [ [[package]] name = "jsonrpsee-wasm-client" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0da62b43702bd5640ea305d35df95da30abc878e79a7b4b01feda3beaf35d3c" +checksum = "5c603d97578071dc44d79d3cfaf0775437638fd5adc33c6b622dfe4fa2ec812d" dependencies = [ "jsonrpsee-client-transport", "jsonrpsee-core", @@ -4362,9 +4336,9 @@ dependencies = [ [[package]] name = "jsonrpsee-ws-client" -version = "0.24.5" +version = "0.24.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39aabf5d6c6f22da8d5b808eea1fab0736059f11fb42f71f141b14f404e5046a" +checksum = "755ca3da1c67671f1fae01cd1a47f41dfb2233a8f19a643e587ab0a663942044" dependencies = [ "http", "jsonrpsee-client-transport", @@ -4465,12 +4439,6 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.159" @@ -4514,11 +4482,11 @@ dependencies = [ [[package]] name = "libproc" -version = "0.14.8" +version = "0.14.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9ea4b75e1a81675429dafe43441df1caea70081e82246a8cccf514884a88bb" +checksum = "e78a09b56be5adbcad5aa1197371688dc6bb249a26da3bca2011ee2fb987ebfb" dependencies = [ - "bindgen 0.69.4", + "bindgen", "errno", "libc", ] @@ -4640,11 +4608,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.0", ] [[package]] @@ -4739,7 +4707,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" dependencies = [ "base64 0.22.1", - "indexmap 2.5.0", + "indexmap 2.6.0", "metrics", "metrics-util", "quanta", @@ -4748,17 +4716,18 @@ dependencies = [ [[package]] name = "metrics-process" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb524e5438255eaa8aa74214d5a62713b77b2c3c6e3c0bbeee65cfd9a58948ba" +checksum = "e69e6ced169644e186e060ddc15f3923fdf06862c811a867bb1e5e7c7824f4d0" dependencies = [ + "libc", "libproc", "mach2", "metrics", "once_cell", - "procfs", + "procfs 0.17.0", "rlimit", - "windows 0.57.0", + "windows 0.58.0", ] [[package]] @@ -5150,21 +5119,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" @@ -5510,18 +5476,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", @@ -5777,9 +5743,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ "unicode-ident", ] @@ -5795,7 +5761,19 @@ dependencies = [ "flate2", "hex", "lazy_static", - "procfs-core", + "procfs-core 0.16.0", + "rustix", +] + +[[package]] +name = "procfs" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" +dependencies = [ + "bitflags 2.6.0", + "hex", + "procfs-core 0.17.0", "rustix", ] @@ -5810,6 +5788,16 @@ dependencies = [ "hex", ] +[[package]] +name = "procfs-core" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" +dependencies = [ + "bitflags 2.6.0", + "hex", +] + [[package]] name = "proptest" version = "1.5.0" @@ -6057,9 +6045,9 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.1.0" +version = "11.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" dependencies = [ "bitflags 2.6.0", ] @@ -7578,7 +7566,7 @@ dependencies = [ "criterion", "dashmap 6.1.0", "derive_more 1.0.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "parking_lot 0.12.3", "pprof", "rand 0.8.5", @@ -7594,7 +7582,7 @@ dependencies = [ name = "reth-mdbx-sys" version = "1.1.0" dependencies = [ - "bindgen 0.70.1", + "bindgen", "cc", ] @@ -7979,7 +7967,7 @@ dependencies = [ "metrics-exporter-prometheus", "metrics-process", "metrics-util", - "procfs", + "procfs 0.16.0", "reqwest", "reth-chainspec", "reth-db-api", @@ -9499,9 +9487,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ "log", "once_cell", @@ -9632,18 +9620,18 @@ dependencies = [ [[package]] name = "scc" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836f1e0f4963ef5288b539b643b35e043e76a32d0f4e47e67febf69576527f50" +checksum = "553f8299af7450cda9a52d3a370199904e7a46b5ffd1bef187c4a6af3bb6db69" dependencies = [ "sdd", ] [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ "windows-sys 0.59.0", ] @@ -9673,9 +9661,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sdd" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a7b59a5d9b0099720b417b6325d91a52cbf5b3dcb5041d864be53eefa58abc" +checksum = "49c1eeaf4b6a87c7479688c6d52b9f1153cedd3c489300564f932b065c6eab95" [[package]] name = "sec1" @@ -9800,7 +9788,7 @@ version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "memchr", "ryu", @@ -9852,15 +9840,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.10.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9720086b3357bcb44fce40117d769a4d068c70ecfa190850a980a71755f66fcc" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_derive", "serde_json", @@ -9870,9 +9858,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.10.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f1abbfe725f27678f4663bcacb75a83e829fd464c25d78dd038a3a29e307cec" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling", "proc-macro2", @@ -10260,9 +10248,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab661c8148c2261222a4d641ad5477fd4bea79406a99056096a0b41b35617a5" +checksum = "20e7b52ad118b2153644eea95c6fc740b6c1555b2344fdab763fc9de4075f665" dependencies = [ "paste", "proc-macro2", @@ -10670,7 +10658,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -11019,9 +11007,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" @@ -11214,9 +11202,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", @@ -11225,9 +11213,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", @@ -11240,9 +11228,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ "cfg-if", "js-sys", @@ -11252,9 +11240,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -11262,9 +11250,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", @@ -11275,9 +11263,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "wasm-streams" @@ -11294,9 +11282,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", From 91abdaaea1ea186b65037b4ff4f91bc60481f273 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Oct 2024 10:29:58 +0200 Subject: [PATCH 146/159] Revert "chore(stages): reduce the progress logging " (#11698) --- .../stages/src/stages/hashing_account.rs | 20 ++++----- .../stages/src/stages/hashing_storage.rs | 20 ++++----- crates/stages/stages/src/stages/headers.rs | 24 +++++----- crates/stages/stages/src/stages/tx_lookup.rs | 8 +--- crates/stages/stages/src/stages/utils.rs | 44 +++++-------------- 5 files changed, 42 insertions(+), 74 deletions(-) diff --git a/crates/stages/stages/src/stages/hashing_account.rs b/crates/stages/stages/src/stages/hashing_account.rs index 2fdccd7837fba..14afb37d81db5 100644 --- a/crates/stages/stages/src/stages/hashing_account.rs +++ b/crates/stages/stages/src/stages/hashing_account.rs @@ -1,4 +1,3 @@ -use crate::log_progress; use alloy_primitives::{keccak256, B256}; use itertools::Itertools; use reth_config::config::{EtlConfig, HashingConfig}; @@ -19,7 +18,6 @@ use std::{ fmt::Debug, ops::{Range, RangeInclusive}, sync::mpsc::{self, Receiver}, - time::Instant, }; use tracing::*; @@ -188,16 +186,16 @@ where let mut hashed_account_cursor = tx.cursor_write::>()?; - let total = collector.len(); - let mut last_log = Instant::now(); + let total_hashes = collector.len(); + let interval = (total_hashes / 10).max(1); for (index, item) in collector.iter()?.enumerate() { - log_progress!( - "sync::stages::hashing_account", - index, - total, - last_log, - "Inserting hashes" - ); + if index > 0 && index % interval == 0 { + info!( + target: "sync::stages::hashing_account", + progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), + "Inserting hashes" + ); + } let (key, value) = item?; hashed_account_cursor diff --git a/crates/stages/stages/src/stages/hashing_storage.rs b/crates/stages/stages/src/stages/hashing_storage.rs index 0646032935f11..ef070d30c6d68 100644 --- a/crates/stages/stages/src/stages/hashing_storage.rs +++ b/crates/stages/stages/src/stages/hashing_storage.rs @@ -1,4 +1,3 @@ -use crate::log_progress; use alloy_primitives::{bytes::BufMut, keccak256, B256}; use itertools::Itertools; use reth_config::config::{EtlConfig, HashingConfig}; @@ -20,7 +19,6 @@ use reth_storage_errors::provider::ProviderResult; use std::{ fmt::Debug, sync::mpsc::{self, Receiver}, - time::Instant, }; use tracing::*; @@ -119,17 +117,17 @@ where collect(&mut channels, &mut collector)?; - let total = collector.len(); - let mut last_log = Instant::now(); + let total_hashes = collector.len(); + let interval = (total_hashes / 10).max(1); let mut cursor = tx.cursor_dup_write::()?; for (index, item) in collector.iter()?.enumerate() { - log_progress!( - "sync::stages::hashing_storage", - index, - total, - last_log, - "Inserting hashes" - ); + if index > 0 && index % interval == 0 { + info!( + target: "sync::stages::hashing_storage", + progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), + "Inserting hashes" + ); + } let (addr_key, value) = item?; cursor.append_dup( diff --git a/crates/stages/stages/src/stages/headers.rs b/crates/stages/stages/src/stages/headers.rs index eaa2ea01f0d85..199e015c2dce9 100644 --- a/crates/stages/stages/src/stages/headers.rs +++ b/crates/stages/stages/src/stages/headers.rs @@ -1,4 +1,3 @@ -use crate::log_progress; use alloy_primitives::{BlockHash, BlockNumber, Bytes, B256}; use futures_util::StreamExt; use reth_config::config::EtlConfig; @@ -26,7 +25,6 @@ use reth_storage_errors::provider::ProviderError; use std::{ sync::Arc, task::{ready, Context, Poll}, - time::Instant, }; use tokio::sync::watch; use tracing::*; @@ -97,9 +95,9 @@ where provider: &impl DBProvider, static_file_provider: StaticFileProvider, ) -> Result { - let total = self.header_collector.len(); + let total_headers = self.header_collector.len(); - info!(target: "sync::stages::headers", total, "Writing headers"); + info!(target: "sync::stages::headers", total = total_headers, "Writing headers"); // Consistency check of expected headers in static files vs DB is done on provider::sync_gap // when poll_execute_ready is polled. @@ -115,11 +113,13 @@ where // Although headers were downloaded in reverse order, the collector iterates it in ascending // order let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; - let mut last_log = Instant::now(); + let interval = (total_headers / 10).max(1); for (index, header) in self.header_collector.iter()?.enumerate() { let (_, header_buf) = header?; - log_progress!("sync::stages::headers", index, total, last_log, "Writing headers"); + if index > 0 && index % interval == 0 && total_headers > 100 { + info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers"); + } let sealed_header: SealedHeader = bincode::deserialize::>(&header_buf) @@ -147,7 +147,7 @@ where writer.append_header(&header, td, &header_hash)?; } - info!(target: "sync::stages::headers", total, "Writing headers hash index"); + info!(target: "sync::stages::headers", total = total_headers, "Writing headers hash index"); let mut cursor_header_numbers = provider.tx_ref().cursor_write::>()?; @@ -168,13 +168,9 @@ where for (index, hash_to_number) in self.hash_collector.iter()?.enumerate() { let (hash, number) = hash_to_number?; - log_progress!( - "sync::stages::headers", - index, - total, - last_log, - "Writing headers hash index" - ); + if index > 0 && index % interval == 0 && total_headers > 100 { + info!(target: "sync::stages::headers", progress = %format!("{:.2}%", (index as f64 / total_headers as f64) * 100.0), "Writing headers hash index"); + } if first_sync { cursor_header_numbers.append( diff --git a/crates/stages/stages/src/stages/tx_lookup.rs b/crates/stages/stages/src/stages/tx_lookup.rs index c5ccff54c525d..60c958abf8623 100644 --- a/crates/stages/stages/src/stages/tx_lookup.rs +++ b/crates/stages/stages/src/stages/tx_lookup.rs @@ -1,4 +1,3 @@ -use super::utils::LOG_INTERVAL; use alloy_primitives::{TxHash, TxNumber}; use num_traits::Zero; use reth_config::config::{EtlConfig, TransactionLookupConfig}; @@ -18,7 +17,6 @@ use reth_stages_api::{ UnwindInput, UnwindOutput, }; use reth_storage_errors::provider::ProviderError; -use std::time::Instant; use tracing::*; /// The transaction lookup stage. @@ -149,18 +147,16 @@ where .cursor_write::>()?; let total_hashes = hash_collector.len(); - let mut last_log = Instant::now(); + let interval = (total_hashes / 10).max(1); for (index, hash_to_number) in hash_collector.iter()?.enumerate() { let (hash, number) = hash_to_number?; - let now = Instant::now(); - if now.duration_since(last_log) >= LOG_INTERVAL { + if index > 0 && index % interval == 0 { info!( target: "sync::stages::transaction_lookup", ?append_only, progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0), "Inserting hashes" ); - last_log = now; } let key = RawKey::::from_vec(hash); diff --git a/crates/stages/stages/src/stages/utils.rs b/crates/stages/stages/src/stages/utils.rs index af6594cd78611..caf039faca108 100644 --- a/crates/stages/stages/src/stages/utils.rs +++ b/crates/stages/stages/src/stages/utils.rs @@ -12,36 +12,12 @@ use reth_db_api::{ use reth_etl::Collector; use reth_provider::DBProvider; use reth_stages_api::StageError; -use std::{ - collections::HashMap, - hash::Hash, - ops::RangeBounds, - time::{Duration, Instant}, -}; +use std::{collections::HashMap, hash::Hash, ops::RangeBounds}; use tracing::info; /// Number of blocks before pushing indices from cache to [`Collector`] const DEFAULT_CACHE_THRESHOLD: u64 = 100_000; -/// Log interval for progress. -pub(crate) const LOG_INTERVAL: Duration = Duration::from_secs(5); - -/// Log progress at a regular interval. -#[macro_export] -macro_rules! log_progress { - ($target:expr, $index:expr, $total:expr, $last_log:expr, $message:expr) => { - let now = std::time::Instant::now(); - if now.duration_since($last_log) >= $crate::stages::utils::LOG_INTERVAL { - info!( - target: $target, - progress = %format!("{:.2}%", ($index as f64 / $total as f64) * 100.0), - $message - ); - $last_log = now; - } - } -} - /// Collects all history (`H`) indices for a range of changesets (`CS`) and stores them in a /// [`Collector`]. /// @@ -89,16 +65,18 @@ where }; // observability - let total = provider.tx_ref().entries::()?; - let mut last_log = Instant::now(); + let total_changesets = provider.tx_ref().entries::()?; + let interval = (total_changesets / 1000).max(1); let mut flush_counter = 0; let mut current_block_number = u64::MAX; - for (index, entry) in changeset_cursor.walk_range(range)?.enumerate() { + for (idx, entry) in changeset_cursor.walk_range(range)?.enumerate() { let (block_number, key) = partial_key_factory(entry?); cache.entry(key).or_default().push(block_number); - log_progress!("sync::stages::index_history", index, total, last_log, "Collecting indices"); + if idx > 0 && idx % interval == 0 && total_changesets > 1000 { + info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices"); + } // Make sure we only flush the cache every DEFAULT_CACHE_THRESHOLD blocks. if current_block_number != block_number { @@ -142,15 +120,17 @@ where let mut current_list = Vec::::new(); // observability - let total = collector.len(); - let mut last_log = Instant::now(); + let total_entries = collector.len(); + let interval = (total_entries / 100).max(1); for (index, element) in collector.iter()?.enumerate() { let (k, v) = element?; let sharded_key = decode_key(k)?; let new_list = BlockNumberList::decompress_owned(v)?; - log_progress!("sync::stages::index_history", index, total, last_log, "Writing indices"); + if index > 0 && index % interval == 0 && total_entries > 100 { + info!(target: "sync::stages::index_history", progress = %format!("{:.2}%", (index as f64 / total_entries as f64) * 100.0), "Writing indices"); + } // AccountsHistory: `Address`. // StorageHistory: `Address.StorageKey`. From 985eb4303ee2e5160ba63af727a2926cafb82cbb Mon Sep 17 00:00:00 2001 From: Kien Trinh <51135161+kien6034@users.noreply.github.com> Date: Mon, 14 Oct 2024 16:11:24 +0700 Subject: [PATCH 147/159] tests(node-builder): basic exex test update for apply function (#11695) Co-authored-by: Matthias Seitz --- crates/node/builder/src/builder/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index 15bdc730a85d6..9f7254bad7372 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -177,6 +177,11 @@ impl NodeBuilder { pub const fn config(&self) -> &NodeConfig { &self.config } + + /// Returns a mutable reference to the node builder's config. + pub fn config_mut(&mut self) -> &mut NodeConfig { + &mut self.config + } } impl NodeBuilder { From b066b96fc1fcd66570606065da41958ee7664f55 Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Mon, 14 Oct 2024 11:12:29 +0200 Subject: [PATCH 148/159] chore(sdk): define trait `Receipt` (#11643) --- crates/primitives-traits/Cargo.toml | 2 +- crates/primitives-traits/src/lib.rs | 3 +++ crates/primitives-traits/src/receipt.rs | 29 +++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 crates/primitives-traits/src/receipt.rs diff --git a/crates/primitives-traits/Cargo.toml b/crates/primitives-traits/Cargo.toml index 2fec75666568f..b34987327ee8e 100644 --- a/crates/primitives-traits/Cargo.toml +++ b/crates/primitives-traits/Cargo.toml @@ -64,4 +64,4 @@ arbitrary = [ "dep:proptest", "dep:proptest-arbitrary-interop", ] -serde-bincode-compat = ["serde_with", "alloy-consensus/serde-bincode-compat"] +serde-bincode-compat = ["serde_with", "alloy-consensus/serde-bincode-compat"] \ No newline at end of file diff --git a/crates/primitives-traits/src/lib.rs b/crates/primitives-traits/src/lib.rs index ccc3ea13baf6b..9c244226b2c71 100644 --- a/crates/primitives-traits/src/lib.rs +++ b/crates/primitives-traits/src/lib.rs @@ -20,6 +20,9 @@ pub use constants::gas_units::{format_gas, format_gas_throughput}; pub mod account; pub use account::{Account, Bytecode}; +pub mod receipt; +pub use receipt::Receipt; + mod integer_list; pub use integer_list::{IntegerList, IntegerListError}; diff --git a/crates/primitives-traits/src/receipt.rs b/crates/primitives-traits/src/receipt.rs new file mode 100644 index 0000000000000..e2d19e4d4ff5d --- /dev/null +++ b/crates/primitives-traits/src/receipt.rs @@ -0,0 +1,29 @@ +//! Receipt abstraction + +use alloc::fmt; + +use alloy_consensus::TxReceipt; +use reth_codecs::Compact; +use serde::{Deserialize, Serialize}; + +/// Helper trait that unifies all behaviour required by receipt to support full node operations. +pub trait FullReceipt: Receipt + Compact {} + +impl FullReceipt for T where T: Receipt + Compact {} + +/// Abstraction of a receipt. +pub trait Receipt: + TxReceipt + + Clone + + fmt::Debug + + PartialEq + + Eq + + Default + + alloy_rlp::Encodable + + alloy_rlp::Decodable + + Serialize + + for<'de> Deserialize<'de> +{ + /// Returns transaction type. + fn tx_type(&self) -> u8; +} From 232e5772c888dbf51da348424e3c84f77b6f74c3 Mon Sep 17 00:00:00 2001 From: Varun Doshi <61531351+varun-doshi@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:49:04 +0530 Subject: [PATCH 149/159] feat: reset pruned numbers on stage drop (#11491) Co-authored-by: Alexey Shekhirin --- crates/cli/commands/src/stage/drop.rs | 114 ++++++++++++-------------- 1 file changed, 51 insertions(+), 63 deletions(-) diff --git a/crates/cli/commands/src/stage/drop.rs b/crates/cli/commands/src/stage/drop.rs index e324c9851502f..9e0396404b376 100644 --- a/crates/cli/commands/src/stage/drop.rs +++ b/crates/cli/commands/src/stage/drop.rs @@ -4,7 +4,7 @@ use clap::Parser; use itertools::Itertools; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_cli::chainspec::ChainSpecParser; -use reth_db::{static_file::iter_static_files, tables}; +use reth_db::{mdbx::tx::Tx, static_file::iter_static_files, tables, DatabaseError}; use reth_db_api::transaction::{DbTx, DbTxMut}; use reth_db_common::{ init::{insert_genesis_header, insert_genesis_history, insert_genesis_state}, @@ -69,42 +69,28 @@ impl> Command tx.clear::()?; tx.clear::()?; tx.clear::()?; - tx.put::( - StageId::Headers.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::Headers)?; + insert_genesis_header(&provider_rw.0, &static_file_provider, &self.env.chain)?; } StageEnum::Bodies => { tx.clear::()?; tx.clear::()?; + reset_prune_checkpoint(tx, PruneSegment::Transactions)?; + tx.clear::()?; tx.clear::()?; tx.clear::()?; tx.clear::()?; - tx.put::( - StageId::Bodies.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::Bodies)?; + insert_genesis_header(&provider_rw.0, &static_file_provider, &self.env.chain)?; } StageEnum::Senders => { tx.clear::()?; // Reset pruned numbers to not count them in the next rerun's stage progress - if let Some(mut prune_checkpoint) = - tx.get::(PruneSegment::SenderRecovery)? - { - prune_checkpoint.block_number = None; - prune_checkpoint.tx_number = None; - tx.put::( - PruneSegment::SenderRecovery, - prune_checkpoint, - )?; - } - tx.put::( - StageId::SenderRecovery.to_string(), - Default::default(), - )?; + reset_prune_checkpoint(tx, PruneSegment::SenderRecovery)?; + reset_stage_checkpoint(tx, StageId::SenderRecovery)?; } StageEnum::Execution => { tx.clear::()?; @@ -113,53 +99,38 @@ impl> Command tx.clear::()?; tx.clear::()?; tx.clear::()?; - tx.put::( - StageId::Execution.to_string(), - Default::default(), - )?; + + reset_prune_checkpoint(tx, PruneSegment::Receipts)?; + reset_prune_checkpoint(tx, PruneSegment::ContractLogs)?; + reset_stage_checkpoint(tx, StageId::Execution)?; + let alloc = &self.env.chain.genesis().alloc; insert_genesis_state(&provider_rw.0, alloc.iter())?; } StageEnum::AccountHashing => { tx.clear::()?; - tx.put::( - StageId::AccountHashing.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::AccountHashing)?; } StageEnum::StorageHashing => { tx.clear::()?; - tx.put::( - StageId::StorageHashing.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::StorageHashing)?; } StageEnum::Hashing => { // Clear hashed accounts tx.clear::()?; - tx.put::( - StageId::AccountHashing.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::AccountHashing)?; // Clear hashed storages tx.clear::()?; - tx.put::( - StageId::StorageHashing.to_string(), - Default::default(), - )?; + reset_stage_checkpoint(tx, StageId::StorageHashing)?; } StageEnum::Merkle => { tx.clear::()?; tx.clear::()?; - tx.put::( - StageId::MerkleExecute.to_string(), - Default::default(), - )?; - tx.put::( - StageId::MerkleUnwind.to_string(), - Default::default(), - )?; + + reset_stage_checkpoint(tx, StageId::MerkleExecute)?; + reset_stage_checkpoint(tx, StageId::MerkleUnwind)?; + tx.delete::( StageId::MerkleExecute.to_string(), None, @@ -168,22 +139,17 @@ impl> Command StageEnum::AccountHistory | StageEnum::StorageHistory => { tx.clear::()?; tx.clear::()?; - tx.put::( - StageId::IndexAccountHistory.to_string(), - Default::default(), - )?; - tx.put::( - StageId::IndexStorageHistory.to_string(), - Default::default(), - )?; + + reset_stage_checkpoint(tx, StageId::IndexAccountHistory)?; + reset_stage_checkpoint(tx, StageId::IndexStorageHistory)?; + insert_genesis_history(&provider_rw.0, self.env.chain.genesis().alloc.iter())?; } StageEnum::TxLookup => { tx.clear::()?; - tx.put::( - StageId::TransactionLookup.to_string(), - Default::default(), - )?; + reset_prune_checkpoint(tx, PruneSegment::TransactionLookup)?; + + reset_stage_checkpoint(tx, StageId::TransactionLookup)?; insert_genesis_header(&provider_rw.0, &static_file_provider, &self.env.chain)?; } } @@ -195,3 +161,25 @@ impl> Command Ok(()) } } + +fn reset_prune_checkpoint( + tx: &Tx, + prune_segment: PruneSegment, +) -> Result<(), DatabaseError> { + if let Some(mut prune_checkpoint) = tx.get::(prune_segment)? { + prune_checkpoint.block_number = None; + prune_checkpoint.tx_number = None; + tx.put::(prune_segment, prune_checkpoint)?; + } + + Ok(()) +} + +fn reset_stage_checkpoint( + tx: &Tx, + stage_id: StageId, +) -> Result<(), DatabaseError> { + tx.put::(stage_id.to_string(), Default::default())?; + + Ok(()) +} From 2af6728b0b9f120df83260f70f55fa3373714ae8 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 14 Oct 2024 11:22:27 +0200 Subject: [PATCH 150/159] feat: new `reth-trie-sparse` crate (#11707) --- Cargo.lock | 4 ++++ Cargo.toml | 1 + crates/trie/sparse/Cargo.toml | 12 ++++++++++++ crates/trie/sparse/src/lib.rs | 1 + 4 files changed, 18 insertions(+) create mode 100644 crates/trie/sparse/Cargo.toml create mode 100644 crates/trie/sparse/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 538816c7c4043..3e3fd18a8442c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9197,6 +9197,10 @@ dependencies = [ "tracing", ] +[[package]] +name = "reth-trie-sparse" +version = "1.1.0" + [[package]] name = "revm" version = "14.0.3" diff --git a/Cargo.toml b/Cargo.toml index efae22f8ed117..628d7d47b2251 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,6 +123,7 @@ members = [ "crates/trie/common", "crates/trie/db", "crates/trie/parallel/", + "crates/trie/sparse", "crates/trie/trie", "examples/beacon-api-sidecar-fetcher/", "examples/beacon-api-sse/", diff --git a/crates/trie/sparse/Cargo.toml b/crates/trie/sparse/Cargo.toml new file mode 100644 index 0000000000000..4ebb56145e1fd --- /dev/null +++ b/crates/trie/sparse/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "reth-trie-sparse" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "Sparse MPT implementation" + +[lints] +workspace = true diff --git a/crates/trie/sparse/src/lib.rs b/crates/trie/sparse/src/lib.rs new file mode 100644 index 0000000000000..5d3d4a5b6f8d5 --- /dev/null +++ b/crates/trie/sparse/src/lib.rs @@ -0,0 +1 @@ +//! The implementation of sparse MPT. From d48dd32302640becd5a484c50791a06149f6472d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 14 Oct 2024 11:46:00 +0200 Subject: [PATCH 151/159] fix: respect --debug.terminate --debug.max-block (#11710) --- crates/node/builder/src/launch/common.rs | 9 +++++++++ crates/node/builder/src/launch/engine.rs | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 99e9b29368a0d..3e8f92e707c06 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -806,6 +806,15 @@ where Ok(initial_target) } + /// Returns true if the node should terminate after the initial backfill run. + /// + /// This is the case if any of these configs are set: + /// `--debug.max-block` + /// `--debug.terminate` + pub const fn terminate_after_initial_backfill(&self) -> bool { + self.node_config().debug.terminate || self.node_config().debug.max_block.is_some() + } + /// Check if the pipeline is consistent (all stages have the checkpoint block numbers no less /// than the checkpoint of the first stage). /// diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 46ffacbf717a2..782cc7bbb1b0a 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -334,6 +334,8 @@ where .fuse(); let chainspec = ctx.chain_spec(); let (exit, rx) = oneshot::channel(); + let terminate_after_backfill = ctx.terminate_after_initial_backfill(); + info!(target: "reth::cli", "Starting consensus engine"); ctx.task_executor().spawn_critical("consensus engine", async move { if let Some(initial_target) = initial_target { @@ -357,6 +359,11 @@ where debug!(target: "reth::cli", "Event: {event}"); match event { ChainEvent::BackfillSyncFinished => { + if terminate_after_backfill { + debug!(target: "reth::cli", "Terminating after initial backfill"); + break + } + network_handle.update_sync_state(SyncState::Idle); } ChainEvent::BackfillSyncStarted => { From 11c56695ba0616278965da8b393f502ba37ce355 Mon Sep 17 00:00:00 2001 From: caglarkaya Date: Mon, 14 Oct 2024 12:50:07 +0300 Subject: [PATCH 152/159] docs: move ExEx book examples (#11616) Co-authored-by: Alexey Shekhirin --- book/developers/exex/remote.md | 349 +----------------- book/developers/exex/tracking-state.md | 138 +------ book/sources/Cargo.toml | 2 + book/sources/exex/remote/Cargo.toml | 52 +++ book/sources/exex/remote/build.rs | 4 + book/sources/exex/remote/proto/exex.proto | 13 + book/sources/exex/remote/src/consumer.rs | 32 ++ book/sources/exex/remote/src/exex.rs | 87 +++++ book/sources/exex/remote/src/exex_1.rs | 40 ++ book/sources/exex/remote/src/exex_2.rs | 49 +++ book/sources/exex/remote/src/exex_3.rs | 65 ++++ book/sources/exex/remote/src/exex_4.rs | 84 +++++ book/sources/exex/remote/src/lib.rs | 3 + book/sources/exex/tracking-state/Cargo.toml | 14 + book/sources/exex/tracking-state/src/bin/1.rs | 57 +++ book/sources/exex/tracking-state/src/bin/2.rs | 73 ++++ 16 files changed, 587 insertions(+), 475 deletions(-) create mode 100644 book/sources/exex/remote/Cargo.toml create mode 100644 book/sources/exex/remote/build.rs create mode 100644 book/sources/exex/remote/proto/exex.proto create mode 100644 book/sources/exex/remote/src/consumer.rs create mode 100644 book/sources/exex/remote/src/exex.rs create mode 100644 book/sources/exex/remote/src/exex_1.rs create mode 100644 book/sources/exex/remote/src/exex_2.rs create mode 100644 book/sources/exex/remote/src/exex_3.rs create mode 100644 book/sources/exex/remote/src/exex_4.rs create mode 100644 book/sources/exex/remote/src/lib.rs create mode 100644 book/sources/exex/tracking-state/Cargo.toml create mode 100644 book/sources/exex/tracking-state/src/bin/1.rs create mode 100644 book/sources/exex/tracking-state/src/bin/2.rs diff --git a/book/developers/exex/remote.md b/book/developers/exex/remote.md index 4344e28b34fc8..0ec704308ff49 100644 --- a/book/developers/exex/remote.md +++ b/book/developers/exex/remote.md @@ -25,41 +25,7 @@ We will also need a bunch of dependencies. Some of them you know from the [Hello but some of specific to what we need now. ```toml -[package] -name = "remote-exex" -version = "0.1.0" -edition = "2021" - -[dependencies] -# reth -reth = { git = "https://github.com/paradigmxyz/reth.git" } -reth-exex = { git = "https://github.com/paradigmxyz/reth.git", features = ["serde"] } -reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git"} -reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } - -# async -tokio = { version = "1", features = ["full"] } -tokio-stream = "0.1" -futures-util = "0.3" - -# grpc -tonic = "0.11" -prost = "0.12" -bincode = "1" - -# misc -eyre = "0.6" - -[build-dependencies] -tonic-build = "0.11" - -[[bin]] -name = "exex" -path = "src/exex.rs" - -[[bin]] -name = "consumer" -path = "src/consumer.rs" +{{#include ../../sources/exex/remote/Cargo.toml}} ``` We also added a build dependency for Tonic. We will use it to generate the Rust code for our @@ -87,26 +53,12 @@ For an example of a full schema, see the [Remote ExEx](https://github.com/paradi ```protobuf -syntax = "proto3"; - -package exex; - -service RemoteExEx { - rpc Subscribe(SubscribeRequest) returns (stream ExExNotification) {} -} - -message SubscribeRequest {} - -message ExExNotification { - bytes data = 1; -} +{{#include ../../sources/exex/remote/proto/exex.proto}} ``` To instruct Tonic to generate the Rust code using this `.proto`, add the following lines to your `lib.rs` file: ```rust,norun,noplayground,ignore -pub mod proto { - tonic::include_proto!("exex"); -} +{{#include ../../sources/exex/remote/src/lib.rs}} ``` ## ExEx and gRPC server @@ -119,52 +71,7 @@ Let's create a minimal gRPC server that listens on the port `:10000`, and spawn the [NodeBuilder](https://reth.rs/docs/reth/builder/struct.NodeBuilder.html)'s [task executor](https://reth.rs/docs/reth/tasks/struct.TaskExecutor.html). ```rust,norun,noplayground,ignore -use remote_exex::proto::{ - self, - remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, -}; -use reth_exex::ExExNotification; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; -use std::sync::Arc; -use tokio::sync::{broadcast, mpsc}; -use tokio_stream::wrappers::ReceiverStream; -use tonic::{transport::Server, Request, Response, Status}; - -struct ExExService {} - -#[tonic::async_trait] -impl RemoteExEx for ExExService { - type SubscribeStream = ReceiverStream>; - - async fn subscribe( - &self, - _request: Request, - ) -> Result, Status> { - let (_tx, rx) = mpsc::channel(1); - - Ok(Response::new(ReceiverStream::new(rx))) - } -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let server = Server::builder() - .add_service(RemoteExExServer::new(ExExService {})) - .serve("[::1]:10000".parse().unwrap()); - - let handle = builder.node(EthereumNode::default()).launch().await?; - - handle - .node - .task_executor - .spawn_critical("gRPC server", async move { - server.await.expect("failed to start gRPC server") - }); - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/remote/src/exex_1.rs}} ``` Currently, it does not send anything on the stream. @@ -175,40 +82,7 @@ Let's create this channel in the `main` function where we will have both gRPC se and save the sender part (that way we will be able to create new receivers) of this channel in our gRPC server. ```rust,norun,noplayground,ignore -// ... -use reth_exex::{ExExNotification}; - -struct ExExService { - notifications: Arc>, -} - -... - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let notifications = Arc::new(broadcast::channel(1).0); - - let server = Server::builder() - .add_service(RemoteExExServer::new(ExExService { - notifications: notifications.clone(), - })) - .serve("[::1]:10000".parse().unwrap()); - - let handle = builder - .node(EthereumNode::default()) - .launch() - .await?; - - handle - .node - .task_executor - .spawn_critical("gRPC server", async move { - server.await.expect("failed to start gRPC server") - }); - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/remote/src/exex_2.rs}} ``` And with that, we're ready to handle incoming notifications, serialize them with [bincode](https://docs.rs/bincode/) @@ -218,37 +92,7 @@ For each incoming request, we spawn a separate tokio task that will run in the b and then return the stream receiver to the client. ```rust,norun,noplayground,ignore -// ... - -#[tonic::async_trait] -impl RemoteExEx for ExExService { - type SubscribeStream = ReceiverStream>; - - async fn subscribe( - &self, - _request: Request, - ) -> Result, Status> { - let (tx, rx) = mpsc::channel(1); - - let mut notifications = self.notifications.subscribe(); - tokio::spawn(async move { - while let Ok(notification) = notifications.recv().await { - let proto_notification = proto::ExExNotification { - data: bincode::serialize(¬ification).expect("failed to serialize"), - }; - tx.send(Ok(proto_notification)) - .await - .expect("failed to send notification to client"); - - info!("Notification sent to the gRPC client"); - } - }); - - Ok(Response::new(ReceiverStream::new(rx))) - } -} - -// ... +{{#rustdoc_include ../../sources/exex/remote/src/exex_3.rs:snippet}} ``` That's it for the gRPC server part! It doesn't receive anything on the `notifications` channel yet, @@ -267,65 +111,14 @@ Don't forget to emit `ExExEvent::FinishedHeight` ```rust,norun,noplayground,ignore -// ... - -use futures_util::StreamExt; -use reth_exex::{ExExContext, ExExEvent}; - -async fn remote_exex( - mut ctx: ExExContext, - notifications: Arc>, -) -> eyre::Result<()> { - while let Some(notification) = ctx.notifications.next().await { - if let Some(committed_chain) = notification.committed_chain() { - ctx.events - .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; - } - - info!("Notification sent to the gRPC server"); - let _ = notifications.send(notification); - } - - Ok(()) -} - -// ... +{{#rustdoc_include ../../sources/exex/remote/src/exex_4.rs:snippet}} ``` All that's left is to connect all pieces together: install our ExEx in the node and pass the sender part of communication channel to it. ```rust,norun,noplayground,ignore -// ... - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let notifications = Arc::new(broadcast::channel(1).0); - - let server = Server::builder() - .add_service(RemoteExExServer::new(ExExService { - notifications: notifications.clone(), - })) - .serve("[::1]:10000".parse().unwrap()); - - let handle = builder - .node(EthereumNode::default()) - .install_exex("remote-exex", |ctx| async move { - Ok(remote_exex(ctx, notifications)) - }) - .launch() - .await?; - - handle - .node - .task_executor - .spawn_critical("gRPC server", async move { - server.await.expect("failed to start gRPC server") - }); - - handle.wait_for_node_exit().await - }) -} +{{#rustdoc_include ../../sources/exex/remote/src/exex.rs:snippet}} ``` ### Full `exex.rs` code @@ -334,98 +127,7 @@ fn main() -> eyre::Result<()> { Click to expand ```rust,norun,noplayground,ignore -use std::sync::Arc; - -use futures_util::StreamExt; -use remote_exex::proto::{ - self, - remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, -}; -use reth::api::FullNodeComponents; -use reth_exex::{ExExContext, ExExEvent, ExExNotification}; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; -use tokio::sync::{broadcast, mpsc}; -use tokio_stream::wrappers::ReceiverStream; -use tonic::{transport::Server, Request, Response, Status}; - -struct ExExService { - notifications: Arc>, -} - -#[tonic::async_trait] -impl RemoteExEx for ExExService { - type SubscribeStream = ReceiverStream>; - - async fn subscribe( - &self, - _request: Request, - ) -> Result, Status> { - let (tx, rx) = mpsc::channel(1); - - let mut notifications = self.notifications.subscribe(); - tokio::spawn(async move { - while let Ok(notification) = notifications.recv().await { - let proto_notification = proto::ExExNotification { - data: bincode::serialize(¬ification).expect("failed to serialize"), - }; - tx.send(Ok(proto_notification)) - .await - .expect("failed to send notification to client"); - - info!(?notification, "Notification sent to the gRPC client"); - } - }); - - Ok(Response::new(ReceiverStream::new(rx))) - } -} - -async fn remote_exex( - mut ctx: ExExContext, - notifications: Arc>, -) -> eyre::Result<()> { - while let Some(notification) = ctx.notifications.next().await { - if let Some(committed_chain) = notification.committed_chain() { - ctx.events - .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; - } - - info!(?notification, "Notification sent to the gRPC server"); - let _ = notifications.send(notification); - } - - Ok(()) -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let notifications = Arc::new(broadcast::channel(1).0); - - let server = Server::builder() - .add_service(RemoteExExServer::new(ExExService { - notifications: notifications.clone(), - })) - .serve("[::1]:10000".parse().unwrap()); - - let handle = builder - .node(EthereumNode::default()) - .install_exex("remote-exex", |ctx| async move { - Ok(remote_exex(ctx, notifications)) - }) - .launch() - .await?; - - handle - .node - .task_executor - .spawn_critical("gRPC server", async move { - server.await.expect("failed to start gRPC server") - }); - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/remote/src/exex.rs}} ``` @@ -442,38 +144,7 @@ because notifications can get very heavy ```rust,norun,noplayground,ignore -use remote_exex::proto::{remote_ex_ex_client::RemoteExExClient, SubscribeRequest}; -use reth_exex::ExExNotification; -use reth_tracing::{tracing::info, RethTracer, Tracer}; - -#[tokio::main] -async fn main() -> eyre::Result<()> { - let _ = RethTracer::new().init()?; - - let mut client = RemoteExExClient::connect("http://[::1]:10000") - .await? - .max_encoding_message_size(usize::MAX) - .max_decoding_message_size(usize::MAX); - - let mut stream = client.subscribe(SubscribeRequest {}).await?.into_inner(); - while let Some(notification) = stream.message().await? { - let notification: ExExNotification = bincode::deserialize(¬ification.data)?; - - match notification { - ExExNotification::ChainCommitted { new } => { - info!(committed_chain = ?new.range(), "Received commit"); - } - ExExNotification::ChainReorged { old, new } => { - info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); - } - ExExNotification::ChainReverted { old } => { - info!(reverted_chain = ?old.range(), "Received revert"); - } - }; - } - - Ok(()) -} +{{#include ../../sources/exex/remote/src/consumer.rs}} ``` ## Running diff --git a/book/developers/exex/tracking-state.md b/book/developers/exex/tracking-state.md index 52c73e6180297..d2a9fe6ca3e35 100644 --- a/book/developers/exex/tracking-state.md +++ b/book/developers/exex/tracking-state.md @@ -19,63 +19,7 @@ because you can't access variables inside the function to assert the state of yo ```rust,norun,noplayground,ignore -use std::{ - future::Future, - pin::Pin, - task::{ready, Context, Poll}, -}; - -use futures_util::StreamExt; -use reth::api::FullNodeComponents; -use reth_exex::{ExExContext, ExExEvent, ExExNotification}; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; - -struct MyExEx { - ctx: ExExContext, -} - -impl Future for MyExEx { - type Output = eyre::Result<()>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.get_mut(); - - while let Some(notification) = ready!(this.ctx.notifications.poll_next_unpin(cx)) { - match ¬ification { - ExExNotification::ChainCommitted { new } => { - info!(committed_chain = ?new.range(), "Received commit"); - } - ExExNotification::ChainReorged { old, new } => { - info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); - } - ExExNotification::ChainReverted { old } => { - info!(reverted_chain = ?old.range(), "Received revert"); - } - }; - - if let Some(committed_chain) = notification.committed_chain() { - this.ctx - .events - .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; - } - } - - Poll::Ready(Ok(())) - } -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let handle = builder - .node(EthereumNode::default()) - .install_exex("my-exex", |ctx| async move { Ok(MyExEx { ctx }) }) - .launch() - .await?; - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/tracking-state/src/bin/1.rs}} ``` For those who are not familiar with how async Rust works on a lower level, that may seem scary, @@ -96,85 +40,7 @@ With all that done, we're now free to add more fields to our `MyExEx` struct, an Our ExEx will count the number of transactions in each block and log it to the console. ```rust,norun,noplayground,ignore -use std::{ - future::Future, - pin::Pin, - task::{ready, Context, Poll}, -}; - -use futures_util::StreamExt; -use reth::{api::FullNodeComponents, primitives::BlockNumber}; -use reth_exex::{ExExContext, ExExEvent}; -use reth_node_ethereum::EthereumNode; -use reth_tracing::tracing::info; - -struct MyExEx { - ctx: ExExContext, - /// First block that was committed since the start of the ExEx. - first_block: Option, - /// Total number of transactions committed. - transactions: u64, -} - -impl MyExEx { - fn new(ctx: ExExContext) -> Self { - Self { - ctx, - first_block: None, - transactions: 0, - } - } -} - -impl Future for MyExEx { - type Output = eyre::Result<()>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.get_mut(); - - while let Some(notification) = ready!(this.ctx.notifications.poll_next_unpin(cx)) { - if let Some(reverted_chain) = notification.reverted_chain() { - this.transactions = this.transactions.saturating_sub( - reverted_chain - .blocks_iter() - .map(|b| b.body.len() as u64) - .sum(), - ); - } - - if let Some(committed_chain) = notification.committed_chain() { - this.first_block.get_or_insert(committed_chain.first().number); - - this.transactions += committed_chain - .blocks_iter() - .map(|b| b.body.len() as u64) - .sum::(); - - this.ctx - .events - .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; - } - - if let Some(first_block) = this.first_block { - info!(%first_block, transactions = %this.transactions, "Total number of transactions"); - } - } - - Poll::Ready(Ok(())) - } -} - -fn main() -> eyre::Result<()> { - reth::cli::Cli::parse_args().run(|builder, _| async move { - let handle = builder - .node(EthereumNode::default()) - .install_exex("my-exex", |ctx| async move { Ok(MyExEx::new(ctx)) }) - .launch() - .await?; - - handle.wait_for_node_exit().await - }) -} +{{#include ../../sources/exex/tracking-state/src/bin/2.rs}} ``` As you can see, we added two fields to our ExEx struct: diff --git a/book/sources/Cargo.toml b/book/sources/Cargo.toml index c04c8567f94d0..1529af952b923 100644 --- a/book/sources/Cargo.toml +++ b/book/sources/Cargo.toml @@ -1,6 +1,8 @@ [workspace] members = [ "exex/hello-world", + "exex/remote", + "exex/tracking-state", ] # Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021 diff --git a/book/sources/exex/remote/Cargo.toml b/book/sources/exex/remote/Cargo.toml new file mode 100644 index 0000000000000..6eeb848cacfe2 --- /dev/null +++ b/book/sources/exex/remote/Cargo.toml @@ -0,0 +1,52 @@ +[package] +name = "remote-exex" +version = "0.1.0" +edition = "2021" + +[dependencies] +# reth +reth = { git = "https://github.com/paradigmxyz/reth.git" } +reth-exex = { git = "https://github.com/paradigmxyz/reth.git", features = ["serde"] } +reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git"} +reth-node-api = { git = "https://github.com/paradigmxyz/reth.git"} +reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } + +# async +tokio = { version = "1", features = ["full"] } +tokio-stream = "0.1" +futures-util = "0.3" + +# grpc +tonic = "0.11" +prost = "0.12" +bincode = "1" + +# misc +eyre = "0.6" + +[build-dependencies] +tonic-build = "0.11" + +[[bin]] +name = "exex_1" +path = "src/exex_1.rs" + +[[bin]] +name = "exex_2" +path = "src/exex_2.rs" + +[[bin]] +name = "exex_3" +path = "src/exex_3.rs" + +[[bin]] +name = "exex_4" +path = "src/exex_4.rs" + +[[bin]] +name = "exex" +path = "src/exex.rs" + +[[bin]] +name = "consumer" +path = "src/consumer.rs" \ No newline at end of file diff --git a/book/sources/exex/remote/build.rs b/book/sources/exex/remote/build.rs new file mode 100644 index 0000000000000..8e66f2a30a6b6 --- /dev/null +++ b/book/sources/exex/remote/build.rs @@ -0,0 +1,4 @@ +fn main() -> Result<(), Box> { + tonic_build::compile_protos("proto/exex.proto")?; + Ok(()) +} diff --git a/book/sources/exex/remote/proto/exex.proto b/book/sources/exex/remote/proto/exex.proto new file mode 100644 index 0000000000000..9bb180b5f215f --- /dev/null +++ b/book/sources/exex/remote/proto/exex.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package exex; + +service RemoteExEx { + rpc Subscribe(SubscribeRequest) returns (stream ExExNotification) {} +} + +message SubscribeRequest {} + +message ExExNotification { + bytes data = 1; +} \ No newline at end of file diff --git a/book/sources/exex/remote/src/consumer.rs b/book/sources/exex/remote/src/consumer.rs new file mode 100644 index 0000000000000..a0400f4bbfde0 --- /dev/null +++ b/book/sources/exex/remote/src/consumer.rs @@ -0,0 +1,32 @@ +use remote_exex::proto::{remote_ex_ex_client::RemoteExExClient, SubscribeRequest}; +use reth_exex::ExExNotification; +use reth_tracing::{tracing::info, RethTracer, Tracer}; + +#[tokio::main] +async fn main() -> eyre::Result<()> { + let _ = RethTracer::new().init()?; + + let mut client = RemoteExExClient::connect("http://[::1]:10000") + .await? + .max_encoding_message_size(usize::MAX) + .max_decoding_message_size(usize::MAX); + + let mut stream = client.subscribe(SubscribeRequest {}).await?.into_inner(); + while let Some(notification) = stream.message().await? { + let notification: ExExNotification = bincode::deserialize(¬ification.data)?; + + match notification { + ExExNotification::ChainCommitted { new } => { + info!(committed_chain = ?new.range(), "Received commit"); + } + ExExNotification::ChainReorged { old, new } => { + info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); + } + ExExNotification::ChainReverted { old } => { + info!(reverted_chain = ?old.range(), "Received revert"); + } + }; + } + + Ok(()) +} diff --git a/book/sources/exex/remote/src/exex.rs b/book/sources/exex/remote/src/exex.rs new file mode 100644 index 0000000000000..1ae4785db8b86 --- /dev/null +++ b/book/sources/exex/remote/src/exex.rs @@ -0,0 +1,87 @@ +use futures_util::TryStreamExt; +use remote_exex::proto::{ + self, + remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, +}; +use reth_exex::{ExExContext, ExExEvent, ExExNotification}; +use reth_node_api::FullNodeComponents; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; +use std::sync::Arc; +use tokio::sync::{broadcast, mpsc}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{transport::Server, Request, Response, Status}; + +struct ExExService { + notifications: Arc>, +} + +#[tonic::async_trait] +impl RemoteExEx for ExExService { + type SubscribeStream = ReceiverStream>; + + async fn subscribe( + &self, + _request: Request, + ) -> Result, Status> { + let (tx, rx) = mpsc::channel(1); + + let mut notifications = self.notifications.subscribe(); + tokio::spawn(async move { + while let Ok(notification) = notifications.recv().await { + let proto_notification = proto::ExExNotification { + data: bincode::serialize(¬ification).expect("failed to serialize"), + }; + tx.send(Ok(proto_notification)) + .await + .expect("failed to send notification to client"); + + info!("Notification sent to the gRPC client"); + } + }); + + Ok(Response::new(ReceiverStream::new(rx))) + } +} + +async fn remote_exex( + mut ctx: ExExContext, + notifications: Arc>, +) -> eyre::Result<()> { + while let Some(notification) = ctx.notifications.try_next().await? { + if let Some(committed_chain) = notification.committed_chain() { + ctx.events.send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + } + + info!("Notification sent to the gRPC server"); + let _ = notifications.send(notification); + } + + Ok(()) +} + +// ANCHOR: snippet +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let notifications = Arc::new(broadcast::channel(1).0); + + let server = Server::builder() + .add_service(RemoteExExServer::new(ExExService { + notifications: notifications.clone(), + })) + .serve("[::1]:10000".parse().unwrap()); + + let handle = builder + .node(EthereumNode::default()) + .install_exex("remote-exex", |ctx| async move { Ok(remote_exex(ctx, notifications)) }) + .launch() + .await?; + + handle.node.task_executor.spawn_critical("gRPC server", async move { + server.await.expect("failed to start gRPC server") + }); + + handle.wait_for_node_exit().await + }) +} +// ANCHOR_END: snippet diff --git a/book/sources/exex/remote/src/exex_1.rs b/book/sources/exex/remote/src/exex_1.rs new file mode 100644 index 0000000000000..09a4bcc064d52 --- /dev/null +++ b/book/sources/exex/remote/src/exex_1.rs @@ -0,0 +1,40 @@ +use remote_exex::proto::{ + self, + remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, +}; +use reth_node_ethereum::EthereumNode; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{transport::Server, Request, Response, Status}; + +struct ExExService {} + +#[tonic::async_trait] +impl RemoteExEx for ExExService { + type SubscribeStream = ReceiverStream>; + + async fn subscribe( + &self, + _request: Request, + ) -> Result, Status> { + let (_tx, rx) = mpsc::channel(1); + + Ok(Response::new(ReceiverStream::new(rx))) + } +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let server = Server::builder() + .add_service(RemoteExExServer::new(ExExService {})) + .serve("[::1]:10000".parse().unwrap()); + + let handle = builder.node(EthereumNode::default()).launch().await?; + + handle.node.task_executor.spawn_critical("gRPC server", async move { + server.await.expect("failed to start gRPC server") + }); + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/remote/src/exex_2.rs b/book/sources/exex/remote/src/exex_2.rs new file mode 100644 index 0000000000000..c4f51ddaea339 --- /dev/null +++ b/book/sources/exex/remote/src/exex_2.rs @@ -0,0 +1,49 @@ +use remote_exex::proto::{ + self, + remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, +}; +use reth_exex::ExExNotification; +use reth_node_ethereum::EthereumNode; +use std::sync::Arc; +use tokio::sync::{broadcast, mpsc}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{transport::Server, Request, Response, Status}; + +#[allow(dead_code)] +struct ExExService { + notifications: Arc>, +} + +#[tonic::async_trait] +impl RemoteExEx for ExExService { + type SubscribeStream = ReceiverStream>; + + async fn subscribe( + &self, + _request: Request, + ) -> Result, Status> { + let (_tx, rx) = mpsc::channel(1); + + Ok(Response::new(ReceiverStream::new(rx))) + } +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let notifications = Arc::new(broadcast::channel(1).0); + + let server = Server::builder() + .add_service(RemoteExExServer::new(ExExService { + notifications: notifications.clone(), + })) + .serve("[::1]:10000".parse().unwrap()); + + let handle = builder.node(EthereumNode::default()).launch().await?; + + handle.node.task_executor.spawn_critical("gRPC server", async move { + server.await.expect("failed to start gRPC server") + }); + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/remote/src/exex_3.rs b/book/sources/exex/remote/src/exex_3.rs new file mode 100644 index 0000000000000..9f264cf345317 --- /dev/null +++ b/book/sources/exex/remote/src/exex_3.rs @@ -0,0 +1,65 @@ +use remote_exex::proto::{ + self, + remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, +}; +use reth_exex::ExExNotification; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; +use std::sync::Arc; +use tokio::sync::{broadcast, mpsc}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{transport::Server, Request, Response, Status}; + +struct ExExService { + notifications: Arc>, +} + +// ANCHOR: snippet +#[tonic::async_trait] +impl RemoteExEx for ExExService { + type SubscribeStream = ReceiverStream>; + + async fn subscribe( + &self, + _request: Request, + ) -> Result, Status> { + let (tx, rx) = mpsc::channel(1); + + let mut notifications = self.notifications.subscribe(); + tokio::spawn(async move { + while let Ok(notification) = notifications.recv().await { + let proto_notification = proto::ExExNotification { + data: bincode::serialize(¬ification).expect("failed to serialize"), + }; + tx.send(Ok(proto_notification)) + .await + .expect("failed to send notification to client"); + + info!("Notification sent to the gRPC client"); + } + }); + + Ok(Response::new(ReceiverStream::new(rx))) + } +} +// ANCHOR_END: snippet + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let notifications = Arc::new(broadcast::channel(1).0); + + let server = Server::builder() + .add_service(RemoteExExServer::new(ExExService { + notifications: notifications.clone(), + })) + .serve("[::1]:10000".parse().unwrap()); + + let handle = builder.node(EthereumNode::default()).launch().await?; + + handle.node.task_executor.spawn_critical("gRPC server", async move { + server.await.expect("failed to start gRPC server") + }); + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/remote/src/exex_4.rs b/book/sources/exex/remote/src/exex_4.rs new file mode 100644 index 0000000000000..24c7bf2c2f11c --- /dev/null +++ b/book/sources/exex/remote/src/exex_4.rs @@ -0,0 +1,84 @@ +use futures_util::TryStreamExt; +use remote_exex::proto::{ + self, + remote_ex_ex_server::{RemoteExEx, RemoteExExServer}, +}; +use reth_exex::{ExExContext, ExExEvent, ExExNotification}; +use reth_node_api::FullNodeComponents; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; +use std::sync::Arc; +use tokio::sync::{broadcast, mpsc}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{transport::Server, Request, Response, Status}; + +struct ExExService { + notifications: Arc>, +} + +#[tonic::async_trait] +impl RemoteExEx for ExExService { + type SubscribeStream = ReceiverStream>; + + async fn subscribe( + &self, + _request: Request, + ) -> Result, Status> { + let (tx, rx) = mpsc::channel(1); + + let mut notifications = self.notifications.subscribe(); + tokio::spawn(async move { + while let Ok(notification) = notifications.recv().await { + let proto_notification = proto::ExExNotification { + data: bincode::serialize(¬ification).expect("failed to serialize"), + }; + tx.send(Ok(proto_notification)) + .await + .expect("failed to send notification to client"); + + info!("Notification sent to the gRPC client"); + } + }); + + Ok(Response::new(ReceiverStream::new(rx))) + } +} + +// ANCHOR: snippet +#[allow(dead_code)] +async fn remote_exex( + mut ctx: ExExContext, + notifications: Arc>, +) -> eyre::Result<()> { + while let Some(notification) = ctx.notifications.try_next().await? { + if let Some(committed_chain) = notification.committed_chain() { + ctx.events.send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + } + + info!("Notification sent to the gRPC server"); + let _ = notifications.send(notification); + } + + Ok(()) +} +// ANCHOR_END: snippet + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let notifications = Arc::new(broadcast::channel(1).0); + + let server = Server::builder() + .add_service(RemoteExExServer::new(ExExService { + notifications: notifications.clone(), + })) + .serve("[::1]:10000".parse().unwrap()); + + let handle = builder.node(EthereumNode::default()).launch().await?; + + handle.node.task_executor.spawn_critical("gRPC server", async move { + server.await.expect("failed to start gRPC server") + }); + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/remote/src/lib.rs b/book/sources/exex/remote/src/lib.rs new file mode 100644 index 0000000000000..9abb458bd3c07 --- /dev/null +++ b/book/sources/exex/remote/src/lib.rs @@ -0,0 +1,3 @@ +pub mod proto { + tonic::include_proto!("exex"); +} diff --git a/book/sources/exex/tracking-state/Cargo.toml b/book/sources/exex/tracking-state/Cargo.toml new file mode 100644 index 0000000000000..3ce21b0c34032 --- /dev/null +++ b/book/sources/exex/tracking-state/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "tracking-state" +version = "0.1.0" +edition = "2021" + +[dependencies] +reth = { git = "https://github.com/paradigmxyz/reth.git" } +reth-exex = { git = "https://github.com/paradigmxyz/reth.git", features = ["serde"] } +reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth.git"} +reth-tracing = { git = "https://github.com/paradigmxyz/reth.git" } + +eyre = "0.6" # Easy error handling +futures-util = "0.3" # Stream utilities for consuming notifications +alloy-primitives = "0.8.7" diff --git a/book/sources/exex/tracking-state/src/bin/1.rs b/book/sources/exex/tracking-state/src/bin/1.rs new file mode 100644 index 0000000000000..0d42e0791a177 --- /dev/null +++ b/book/sources/exex/tracking-state/src/bin/1.rs @@ -0,0 +1,57 @@ +use std::{ + future::Future, + pin::Pin, + task::{ready, Context, Poll}, +}; + +use futures_util::{FutureExt, TryStreamExt}; +use reth::api::FullNodeComponents; +use reth_exex::{ExExContext, ExExEvent, ExExNotification}; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; + +struct MyExEx { + ctx: ExExContext, +} + +impl Future for MyExEx { + type Output = eyre::Result<()>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + while let Some(notification) = ready!(this.ctx.notifications.try_next().poll_unpin(cx))? { + match ¬ification { + ExExNotification::ChainCommitted { new } => { + info!(committed_chain = ?new.range(), "Received commit"); + } + ExExNotification::ChainReorged { old, new } => { + info!(from_chain = ?old.range(), to_chain = ?new.range(), "Received reorg"); + } + ExExNotification::ChainReverted { old } => { + info!(reverted_chain = ?old.range(), "Received revert"); + } + }; + + if let Some(committed_chain) = notification.committed_chain() { + this.ctx + .events + .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + } + } + + Poll::Ready(Ok(())) + } +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let handle = builder + .node(EthereumNode::default()) + .install_exex("my-exex", |ctx| async move { Ok(MyExEx { ctx }) }) + .launch() + .await?; + + handle.wait_for_node_exit().await + }) +} diff --git a/book/sources/exex/tracking-state/src/bin/2.rs b/book/sources/exex/tracking-state/src/bin/2.rs new file mode 100644 index 0000000000000..9416810668f51 --- /dev/null +++ b/book/sources/exex/tracking-state/src/bin/2.rs @@ -0,0 +1,73 @@ +use std::{ + future::Future, + pin::Pin, + task::{ready, Context, Poll}, +}; + +use alloy_primitives::BlockNumber; +use futures_util::{FutureExt, TryStreamExt}; +use reth::api::FullNodeComponents; +use reth_exex::{ExExContext, ExExEvent}; +use reth_node_ethereum::EthereumNode; +use reth_tracing::tracing::info; + +struct MyExEx { + ctx: ExExContext, + /// First block that was committed since the start of the ExEx. + first_block: Option, + /// Total number of transactions committed. + transactions: u64, +} + +impl MyExEx { + fn new(ctx: ExExContext) -> Self { + Self { ctx, first_block: None, transactions: 0 } + } +} + +impl Future for MyExEx { + type Output = eyre::Result<()>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + while let Some(notification) = ready!(this.ctx.notifications.try_next().poll_unpin(cx))? { + if let Some(reverted_chain) = notification.reverted_chain() { + this.transactions = this.transactions.saturating_sub( + reverted_chain.blocks_iter().map(|b| b.body.transactions.len() as u64).sum(), + ); + } + + if let Some(committed_chain) = notification.committed_chain() { + this.first_block.get_or_insert(committed_chain.first().number); + + this.transactions += committed_chain + .blocks_iter() + .map(|b| b.body.transactions.len() as u64) + .sum::(); + + this.ctx + .events + .send(ExExEvent::FinishedHeight(committed_chain.tip().num_hash()))?; + } + + if let Some(first_block) = this.first_block { + info!(%first_block, transactions = %this.transactions, "Total number of transactions"); + } + } + + Poll::Ready(Ok(())) + } +} + +fn main() -> eyre::Result<()> { + reth::cli::Cli::parse_args().run(|builder, _| async move { + let handle = builder + .node(EthereumNode::default()) + .install_exex("my-exex", |ctx| async move { Ok(MyExEx::new(ctx)) }) + .launch() + .await?; + + handle.wait_for_node_exit().await + }) +} From 9563ac2fc59169d5b39dee1cfcc06e8f1c407ebb Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 14 Oct 2024 11:58:15 +0100 Subject: [PATCH 153/159] perf(rpc): do not clone filter matcher on every block tracing (#11714) --- crates/rpc/rpc/src/trace.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 687762b74b5ed..f7b2f7ab7eaca 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -251,7 +251,8 @@ where &self, filter: TraceFilter, ) -> Result, Eth::Error> { - let matcher = filter.matcher(); + // We'll reuse the matcher across multiple blocks that are traced in parallel + let matcher = Arc::new(filter.matcher()); let TraceFilter { from_block, to_block, after, count, .. } = filter; let start = from_block.unwrap_or(0); let end = if let Some(to_block) = to_block { From 5e57390aea7b46ae9ac33ad30eb7d893bbaeeebc Mon Sep 17 00:00:00 2001 From: Emilia Hane Date: Mon, 14 Oct 2024 13:41:50 +0200 Subject: [PATCH 154/159] fix(net): remove outdated debug assert `TransactionFetcher` (#11713) --- crates/net/network/src/transactions/fetcher.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/crates/net/network/src/transactions/fetcher.rs b/crates/net/network/src/transactions/fetcher.rs index 9276219d593b7..3e856951552ac 100644 --- a/crates/net/network/src/transactions/fetcher.rs +++ b/crates/net/network/src/transactions/fetcher.rs @@ -912,17 +912,6 @@ impl TransactionFetcher { // fallback peers let GetPooledTxResponse { peer_id, mut requested_hashes, result } = response; - debug_assert!( - self.active_peers.get(&peer_id).is_some(), - "`{}` has been removed from `@active_peers` before inflight request(s) resolved, broken invariant `@active_peers` and `@inflight_requests`, `%peer_id`: {}, `@hashes_fetch_inflight_and_pending_fetch` for `%requested_hashes`: {:?}", - peer_id, - peer_id, - requested_hashes.iter().map(|hash| { - let metadata = self.hashes_fetch_inflight_and_pending_fetch.get(hash); - (*hash, metadata.map(|m| (m.retries, m.tx_encoded_length))) - }).collect::)>)>>() - ); - self.decrement_inflight_request_count_for(&peer_id); match result { From 28725334bc40caff747ac77d7c2b68ee89fa3557 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:44:28 +0900 Subject: [PATCH 155/159] perf(rpc): use `Arc` on cache and rpc (#11635) --- Cargo.lock | 1 + crates/ethereum/payload/src/lib.rs | 2 +- crates/evm/src/lib.rs | 10 +- crates/optimism/evm/src/l1.rs | 13 +- crates/optimism/node/src/txpool.rs | 2 +- crates/optimism/payload/src/builder.rs | 4 +- crates/optimism/rpc/src/eth/block.rs | 2 +- crates/optimism/rpc/src/eth/receipt.rs | 5 +- crates/primitives/src/transaction/mod.rs | 5 + crates/rpc/rpc-eth-api/src/helpers/block.rs | 25 +-- crates/rpc/rpc-eth-api/src/helpers/call.rs | 27 +-- crates/rpc/rpc-eth-api/src/helpers/fee.rs | 14 +- .../rpc-eth-api/src/helpers/pending_block.rs | 2 +- crates/rpc/rpc-eth-api/src/helpers/trace.rs | 10 +- .../rpc-eth-api/src/helpers/transaction.rs | 30 +-- crates/rpc/rpc-eth-types/Cargo.toml | 1 + crates/rpc/rpc-eth-types/src/cache/mod.rs | 180 +++++------------- crates/rpc/rpc-eth-types/src/fee_history.rs | 17 +- crates/rpc/rpc-eth-types/src/gas_oracle.rs | 14 +- crates/rpc/rpc-eth-types/src/logs_utils.rs | 7 +- crates/rpc/rpc/src/debug.rs | 17 +- crates/rpc/rpc/src/eth/filter.rs | 5 +- crates/rpc/rpc/src/trace.rs | 10 +- 23 files changed, 168 insertions(+), 235 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e3fd18a8442c..3c5639eadab0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8745,6 +8745,7 @@ dependencies = [ "alloy-sol-types", "derive_more 1.0.0", "futures", + "itertools 0.13.0", "jsonrpsee-core", "jsonrpsee-types", "metrics", diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 5d2fcde2f7e5d..97237efa8b60c 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -228,7 +228,7 @@ where let env = EnvWithHandlerCfg::new_with_cfg_env( initialized_cfg.clone(), initialized_block_env.clone(), - evm_config.tx_env(&tx), + evm_config.tx_env(tx.as_signed(), tx.signer()), ); // Configure the environment for the block. diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index 6fcb3d9f8c3d1..e0d45f04cd31b 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -11,11 +11,9 @@ extern crate alloc; -use core::ops::Deref; - use crate::builder::RethEvmBuilder; use alloy_primitives::{Address, Bytes, B256, U256}; -use reth_primitives::{TransactionSigned, TransactionSignedEcRecovered}; +use reth_primitives::TransactionSigned; use reth_primitives_traits::BlockHeader; use revm::{Database, Evm, GetInspector}; use revm_primitives::{BlockEnv, CfgEnvWithHandlerCfg, Env, EnvWithHandlerCfg, SpecId, TxEnv}; @@ -111,10 +109,10 @@ pub trait ConfigureEvmEnv: Send + Sync + Unpin + Clone + 'static { /// The header type used by the EVM. type Header: BlockHeader; - /// Returns a [`TxEnv`] from a [`TransactionSignedEcRecovered`]. - fn tx_env(&self, transaction: &TransactionSignedEcRecovered) -> TxEnv { + /// Returns a [`TxEnv`] from a [`TransactionSigned`] and [`Address`]. + fn tx_env(&self, transaction: &TransactionSigned, signer: Address) -> TxEnv { let mut tx_env = TxEnv::default(); - self.fill_tx_env(&mut tx_env, transaction.deref(), transaction.signer()); + self.fill_tx_env(&mut tx_env, transaction, signer); tx_env } diff --git a/crates/optimism/evm/src/l1.rs b/crates/optimism/evm/src/l1.rs index 18ccbed9518a9..3412501eb99be 100644 --- a/crates/optimism/evm/src/l1.rs +++ b/crates/optimism/evm/src/l1.rs @@ -6,7 +6,7 @@ use reth_chainspec::ChainSpec; use reth_execution_errors::BlockExecutionError; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_forks::OptimismHardfork; -use reth_primitives::Block; +use reth_primitives::BlockBody; use revm::{ primitives::{Bytecode, HashMap, SpecId}, DatabaseCommit, L1BlockInfo, @@ -31,9 +31,8 @@ const L1_BLOCK_ECOTONE_SELECTOR: [u8; 4] = hex!("440a5e20"); /// transaction in the L2 block. /// /// Returns an error if the L1 info transaction is not found, if the block is empty. -pub fn extract_l1_info(block: &Block) -> Result { - let l1_info_tx_data = block - .body +pub fn extract_l1_info(body: &BlockBody) -> Result { + let l1_info_tx_data = body .transactions .first() .ok_or_else(|| OptimismBlockExecutionError::L1BlockInfoError { @@ -302,7 +301,7 @@ mod tests { use alloy_eips::eip2718::Decodable2718; use reth_optimism_chainspec::OP_MAINNET; use reth_optimism_forks::OptimismHardforks; - use reth_primitives::{BlockBody, TransactionSigned}; + use reth_primitives::{Block, BlockBody, TransactionSigned}; use super::*; @@ -318,7 +317,7 @@ mod tests { body: BlockBody { transactions: vec![l1_info_tx], ..Default::default() }, }; - let l1_info: L1BlockInfo = extract_l1_info(&mock_block).unwrap(); + let l1_info: L1BlockInfo = extract_l1_info(&mock_block.body).unwrap(); assert_eq!(l1_info.l1_base_fee, U256::from(652_114)); assert_eq!(l1_info.l1_fee_overhead, Some(U256::from(2100))); assert_eq!(l1_info.l1_base_fee_scalar, U256::from(1_000_000)); @@ -358,7 +357,7 @@ mod tests { // test - let l1_block_info: L1BlockInfo = extract_l1_info(&block).unwrap(); + let l1_block_info: L1BlockInfo = extract_l1_info(&block.body).unwrap(); assert_eq!(l1_block_info.l1_base_fee, expected_l1_base_fee); assert_eq!(l1_block_info.l1_base_fee_scalar, expected_l1_base_fee_scalar); diff --git a/crates/optimism/node/src/txpool.rs b/crates/optimism/node/src/txpool.rs index 7ed2a161d0e4f..09aa76fefb8bf 100644 --- a/crates/optimism/node/src/txpool.rs +++ b/crates/optimism/node/src/txpool.rs @@ -99,7 +99,7 @@ where /// Update the L1 block info. fn update_l1_block_info(&self, block: &Block) { self.block_info.timestamp.store(block.timestamp, Ordering::Relaxed); - if let Ok(cost_addition) = reth_optimism_evm::extract_l1_info(block) { + if let Ok(cost_addition) = reth_optimism_evm::extract_l1_info(&block.body) { *self.block_info.l1_block_info.write() = cost_addition; } } diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 5e9c1b5d18be7..2ff3d33420378 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -275,7 +275,7 @@ where let env = EnvWithHandlerCfg::new_with_cfg_env( initialized_cfg.clone(), initialized_block_env.clone(), - evm_config.tx_env(&sequencer_tx), + evm_config.tx_env(sequencer_tx.as_signed(), sequencer_tx.signer()), ); let mut evm = evm_config.evm_with_env(&mut db, env); @@ -356,7 +356,7 @@ where let env = EnvWithHandlerCfg::new_with_cfg_env( initialized_cfg.clone(), initialized_block_env.clone(), - evm_config.tx_env(&tx), + evm_config.tx_env(tx.as_signed(), tx.signer()), ); // Configure the environment for the block. diff --git a/crates/optimism/rpc/src/eth/block.rs b/crates/optimism/rpc/src/eth/block.rs index d5066be0c620b..dfdd096085638 100644 --- a/crates/optimism/rpc/src/eth/block.rs +++ b/crates/optimism/rpc/src/eth/block.rs @@ -45,7 +45,7 @@ where let block = block.unseal(); let l1_block_info = - reth_optimism_evm::extract_l1_info(&block).map_err(OpEthApiError::from)?; + reth_optimism_evm::extract_l1_info(&block.body).map_err(OpEthApiError::from)?; return block .body diff --git a/crates/optimism/rpc/src/eth/receipt.rs b/crates/optimism/rpc/src/eth/receipt.rs index fac8d220c761f..cc685104133c3 100644 --- a/crates/optimism/rpc/src/eth/receipt.rs +++ b/crates/optimism/rpc/src/eth/receipt.rs @@ -40,9 +40,8 @@ where meta.block_hash.into(), )))?; - let block = block.unseal(); let l1_block_info = - reth_optimism_evm::extract_l1_info(&block).map_err(OpEthApiError::from)?; + reth_optimism_evm::extract_l1_info(&block.body).map_err(OpEthApiError::from)?; Ok(OpReceiptBuilder::new( &self.inner.provider().chain_spec(), @@ -353,7 +352,7 @@ mod test { }; let l1_block_info = - reth_optimism_evm::extract_l1_info(&block).expect("should extract l1 info"); + reth_optimism_evm::extract_l1_info(&block.body).expect("should extract l1 info"); // test assert!(OP_MAINNET.is_fjord_active_at_timestamp(BLOCK_124665056_TIMESTAMP)); diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 7ef2c0c1fb76f..3622b953107be 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -1472,6 +1472,11 @@ impl TransactionSignedEcRecovered { self.signer } + /// Returns a reference to [`TransactionSigned`] + pub const fn as_signed(&self) -> &TransactionSigned { + &self.signed_transaction + } + /// Transform back to [`TransactionSigned`] pub fn into_signed(self) -> TransactionSigned { self.signed_transaction diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 898037f4a5867..5d8496abdae28 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -64,7 +64,7 @@ pub trait EthBlocks: LoadBlock { } let block = from_block::( - block.unseal(), + (*block).clone().unseal(), total_difficulty.unwrap_or_default(), full.into(), Some(block_hash), @@ -100,10 +100,10 @@ pub trait EthBlocks: LoadBlock { Ok(self .cache() - .get_block_transactions(block_hash) + .get_sealed_block_with_senders(block_hash) .await .map_err(Self::Error::from_eth_err)? - .map(|txs| txs.len())) + .map(|b| b.body.transactions.len())) } } @@ -150,6 +150,7 @@ pub trait EthBlocks: LoadBlock { .get_block_and_receipts(block_hash) .await .map_err(Self::Error::from_eth_err) + .map(|b| b.map(|(b, r)| (b.block.clone(), r))) } Ok(None) @@ -208,23 +209,11 @@ pub trait LoadBlock: LoadPendingBlock + SpawnBlocking { /// Data access in default (L1) trait method implementations. fn cache(&self) -> &EthStateCache; - /// Returns the block object for the given block id. - fn block( - &self, - block_id: BlockId, - ) -> impl Future, Self::Error>> + Send { - async move { - self.block_with_senders(block_id) - .await - .map(|maybe_block| maybe_block.map(|block| block.block)) - } - } - /// Returns the block object for the given block id. fn block_with_senders( &self, block_id: BlockId, - ) -> impl Future, Self::Error>> + Send { + ) -> impl Future>, Self::Error>> + Send { async move { if block_id.is_pending() { // Pending block can be fetched directly without need for caching @@ -232,11 +221,11 @@ pub trait LoadBlock: LoadPendingBlock + SpawnBlocking { .pending_block_with_senders() .map_err(Self::Error::from_eth_err)?; return if maybe_pending.is_some() { - Ok(maybe_pending) + Ok(maybe_pending.map(Arc::new)) } else { // If no pending block from provider, try to get local pending block return match self.local_pending_block().await? { - Some((block, _)) => Ok(Some(block)), + Some((block, _)) => Ok(Some(Arc::new(block))), None => Ok(None), }; }; diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 5f6f3d77f5558..ff6e930069f87 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -5,7 +5,7 @@ use crate::{ AsEthApiError, FromEthApiError, FromEvmError, FullEthApiTypes, IntoEthApiError, RpcBlock, }; use alloy_eips::{eip1559::calc_next_block_base_fee, eip2930::AccessListResult}; -use alloy_primitives::{Bytes, TxKind, B256, U256}; +use alloy_primitives::{Address, Bytes, TxKind, B256, U256}; use alloy_rpc_types::{ simulate::{SimBlock, SimulatePayload, SimulatedBlock}, state::{EvmOverrides, StateOverride}, @@ -20,7 +20,7 @@ use reth_primitives::{ BlockEnv, CfgEnvWithHandlerCfg, EnvWithHandlerCfg, ExecutionResult, HaltReason, ResultAndState, TransactTo, TxEnv, }, - Header, TransactionSignedEcRecovered, + Header, TransactionSigned, }; use reth_provider::{BlockIdReader, ChainSpecProvider, HeaderProvider, StateProvider}; use reth_revm::{database::StateProviderDatabase, db::CacheDB, DatabaseRef}; @@ -92,7 +92,8 @@ pub trait EthCall: Call + LoadPendingBlock { // Gas cap for entire operation let total_gas_limit = self.call_gas_limit(); - let base_block = self.block(block).await?.ok_or(EthApiError::HeaderNotFound(block))?; + let base_block = + self.block_with_senders(block).await?.ok_or(EthApiError::HeaderNotFound(block))?; let mut parent_hash = base_block.header.hash(); let total_difficulty = LoadPendingBlock::provider(self) .header_td_by_number(block_env.number.to()) @@ -292,12 +293,12 @@ pub trait EthCall: Call + LoadPendingBlock { if replay_block_txs { // only need to replay the transactions in the block if not all transactions are // to be replayed - let transactions = block.into_transactions_ecrecovered().take(num_txs); - for tx in transactions { + let transactions = block.transactions_with_sender().take(num_txs); + for (signer, tx) in transactions { let env = EnvWithHandlerCfg::new_with_cfg_env( cfg.clone(), block_env.clone(), - Call::evm_config(&this).tx_env(&tx), + Call::evm_config(&this).tx_env(tx, *signer), ); let (res, _) = this.transact(&mut db, env)?; db.commit(res.state); @@ -605,11 +606,11 @@ pub trait Call: LoadState + SpawnBlocking { // we need to get the state of the parent block because we're essentially replaying the // block the transaction is included in let parent_block = block.parent_hash; - let block_txs = block.into_transactions_ecrecovered(); let this = self.clone(); self.spawn_with_state_at_block(parent_block.into(), move |state| { let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let block_txs = block.transactions_with_sender(); // replay all transactions prior to the targeted transaction this.replay_transactions_until( @@ -623,7 +624,7 @@ pub trait Call: LoadState + SpawnBlocking { let env = EnvWithHandlerCfg::new_with_cfg_env( cfg, block_env, - Call::evm_config(&this).tx_env(&tx), + Call::evm_config(&this).tx_env(tx.as_signed(), tx.signer()), ); let (res, _) = this.transact(&mut db, env)?; @@ -641,30 +642,30 @@ pub trait Call: LoadState + SpawnBlocking { /// /// Note: This assumes the target transaction is in the given iterator. /// Returns the index of the target transaction in the given iterator. - fn replay_transactions_until( + fn replay_transactions_until<'a, DB, I>( &self, db: &mut CacheDB, cfg: CfgEnvWithHandlerCfg, block_env: BlockEnv, - transactions: impl IntoIterator, + transactions: I, target_tx_hash: B256, ) -> Result where DB: DatabaseRef, EthApiError: From, + I: IntoIterator, { let env = EnvWithHandlerCfg::new_with_cfg_env(cfg, block_env, Default::default()); let mut evm = self.evm_config().evm_with_env(db, env); let mut index = 0; - for tx in transactions { + for (sender, tx) in transactions { if tx.hash() == target_tx_hash { // reached the target transaction break } - let sender = tx.signer(); - self.evm_config().fill_tx_env(evm.tx_mut(), &tx.into_signed(), sender); + self.evm_config().fill_tx_env(evm.tx_mut(), tx, *sender); evm.transact_commit().map_err(Self::Error::from_evm_err)?; index += 1; } diff --git a/crates/rpc/rpc-eth-api/src/helpers/fee.rs b/crates/rpc/rpc-eth-api/src/helpers/fee.rs index 52ccabebf6208..34ba6dc7e4ec0 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/fee.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/fee.rs @@ -172,8 +172,8 @@ pub trait EthFees: LoadFee { // Percentiles were specified, so we need to collect reward percentile ino if let Some(percentiles) = &reward_percentiles { - let (transactions, receipts) = LoadFee::cache(self) - .get_transactions_and_receipts(header.hash()) + let (block, receipts) = LoadFee::cache(self) + .get_block_and_receipts(header.hash()) .await .map_err(Self::Error::from_eth_err)? .ok_or(EthApiError::InvalidBlockRange)?; @@ -182,7 +182,7 @@ pub trait EthFees: LoadFee { percentiles, header.gas_used, header.base_fee_per_gas.unwrap_or_default(), - &transactions, + &block.body.transactions, &receipts, ) .unwrap_or_default(), @@ -296,7 +296,7 @@ pub trait LoadFee: LoadBlock { None => { // fetch pending base fee let base_fee = self - .block(BlockNumberOrTag::Pending.into()) + .block_with_senders(BlockNumberOrTag::Pending.into()) .await? .ok_or(EthApiError::HeaderNotFound(BlockNumberOrTag::Pending.into()))? .base_fee_per_gas @@ -332,7 +332,7 @@ pub trait LoadFee: LoadBlock { /// /// See also: fn gas_price(&self) -> impl Future> + Send { - let header = self.block(BlockNumberOrTag::Latest.into()); + let header = self.block_with_senders(BlockNumberOrTag::Latest.into()); let suggested_tip = self.suggested_priority_fee(); async move { let (header, suggested_tip) = futures::try_join!(header, suggested_tip)?; @@ -344,9 +344,9 @@ pub trait LoadFee: LoadBlock { /// Returns a suggestion for a base fee for blob transactions. fn blob_base_fee(&self) -> impl Future> + Send { async move { - self.block(BlockNumberOrTag::Latest.into()) + self.block_with_senders(BlockNumberOrTag::Latest.into()) .await? - .and_then(|h: reth_primitives::SealedBlock| h.next_block_blob_fee()) + .and_then(|h| h.next_block_blob_fee()) .ok_or(EthApiError::ExcessBlobGasNotSet.into()) .map(U256::from) } diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 8014851e3f937..03597289f1517 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -323,7 +323,7 @@ pub trait LoadPendingBlock: EthApiTypes { let env = Env::boxed( cfg.cfg_env.clone(), block_env.clone(), - Self::evm_config(self).tx_env(&tx), + Self::evm_config(self).tx_env(tx.as_signed(), tx.signer()), ); let mut evm = revm::Evm::builder().with_env(env).with_db(&mut db).build(); diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index 7b5d13b2a7dd6..2dc1c5b4746da 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -191,11 +191,11 @@ pub trait Trace: LoadState { // block the transaction is included in let parent_block = block.parent_hash; let parent_beacon_block_root = block.parent_beacon_block_root; - let block_txs = block.into_transactions_ecrecovered(); let this = self.clone(); self.spawn_with_state_at_block(parent_block.into(), move |state| { let mut db = CacheDB::new(StateProviderDatabase::new(state)); + let block_txs = block.transactions_with_sender(); // apply relevant system calls let mut system_caller = SystemCaller::new( @@ -227,7 +227,7 @@ pub trait Trace: LoadState { let env = EnvWithHandlerCfg::new_with_cfg_env( cfg, block_env, - Call::evm_config(&this).tx_env(&tx), + Call::evm_config(&this).tx_env(tx.as_signed(), tx.signer()), ); let (res, _) = this.inspect(StateCacheDbRefMutWrapper(&mut db), env, &mut inspector)?; @@ -356,10 +356,10 @@ pub trait Trace: LoadState { let mut results = Vec::with_capacity(max_transactions); let mut transactions = block - .into_transactions_ecrecovered() + .transactions_with_sender() .take(max_transactions) .enumerate() - .map(|(idx, tx)| { + .map(|(idx, (signer, tx))| { let tx_info = TransactionInfo { hash: Some(tx.hash()), index: Some(idx as u64), @@ -367,7 +367,7 @@ pub trait Trace: LoadState { block_number: Some(block_number), base_fee: Some(base_fee), }; - let tx_env = Trace::evm_config(&this).tx_env(&tx); + let tx_env = Trace::evm_config(&this).tx_env(tx, *signer); (tx_info, tx_env) }) .peekable(); diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 94dd04414874d..4c2493717a04e 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -18,6 +18,7 @@ use reth_rpc_eth_types::{ }; use reth_rpc_types_compat::transaction::{from_recovered, from_recovered_with_block_context}; use reth_transaction_pool::{PoolTransaction, TransactionOrigin, TransactionPool}; +use std::sync::Arc; use crate::{FromEthApiError, FullEthApiTypes, IntoEthApiError, RpcReceipt, RpcTransaction}; @@ -79,7 +80,11 @@ pub trait EthTransactions: LoadTransaction { block: B256, ) -> impl Future>, Self::Error>> + Send { async move { - self.cache().get_block_transactions(block).await.map_err(Self::Error::from_eth_err) + self.cache() + .get_sealed_block_with_senders(block) + .await + .map(|b| b.map(|b| b.body.transactions.clone())) + .map_err(Self::Error::from_eth_err) } } @@ -194,7 +199,7 @@ pub trait EthTransactions: LoadTransaction { let block_hash = block.hash(); let block_number = block.number; let base_fee_per_gas = block.base_fee_per_gas; - if let Some(tx) = block.into_transactions_ecrecovered().nth(index) { + if let Some((signer, tx)) = block.transactions_with_sender().nth(index) { let tx_info = TransactionInfo { hash: Some(tx.hash()), block_hash: Some(block_hash), @@ -204,7 +209,8 @@ pub trait EthTransactions: LoadTransaction { }; return Ok(Some(from_recovered_with_block_context::( - tx, tx_info, + tx.clone().with_signer(*signer), + tx_info, ))) } } @@ -270,10 +276,10 @@ pub trait EthTransactions: LoadTransaction { let base_fee_per_gas = block.base_fee_per_gas; block - .into_transactions_ecrecovered() + .transactions_with_sender() .enumerate() - .find(|(_, tx)| tx.signer() == sender && tx.nonce() == nonce) - .map(|(index, tx)| { + .find(|(_, (signer, tx))| **signer == sender && tx.nonce() == nonce) + .map(|(index, (signer, tx))| { let tx_info = TransactionInfo { hash: Some(tx.hash()), block_hash: Some(block_hash), @@ -282,7 +288,8 @@ pub trait EthTransactions: LoadTransaction { index: Some(index as u64), }; from_recovered_with_block_context::( - tx, tx_info, + tx.clone().with_signer(*signer), + tx_info, ) }) }) @@ -544,8 +551,9 @@ pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes { fn transaction_and_block( &self, hash: B256, - ) -> impl Future, Self::Error>> - + Send { + ) -> impl Future< + Output = Result)>, Self::Error>, + > + Send { async move { let (transaction, at) = match self.transaction_by_hash_at(hash).await? { None => return Ok(None), @@ -559,10 +567,10 @@ pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes { }; let block = self .cache() - .get_block_with_senders(block_hash) + .get_sealed_block_with_senders(block_hash) .await .map_err(Self::Error::from_eth_err)?; - Ok(block.map(|block| (transaction, (*block).clone().seal(block_hash)))) + Ok(block.map(|block| (transaction, block))) } } } diff --git a/crates/rpc/rpc-eth-types/Cargo.toml b/crates/rpc/rpc-eth-types/Cargo.toml index 2c6f51ff462b0..46a0d7b5c323d 100644 --- a/crates/rpc/rpc-eth-types/Cargo.toml +++ b/crates/rpc/rpc-eth-types/Cargo.toml @@ -58,6 +58,7 @@ derive_more.workspace = true schnellru.workspace = true rand.workspace = true tracing.workspace = true +itertools.workspace = true [dev-dependencies] serde_json.workspace = true diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index e8353ecc4e166..cbf05f2764e4e 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -7,8 +7,7 @@ use reth_errors::{ProviderError, ProviderResult}; use reth_evm::{provider::EvmEnvProvider, ConfigureEvm}; use reth_execution_types::Chain; use reth_primitives::{ - Block, BlockHashOrNumber, BlockWithSenders, Header, Receipt, SealedBlock, - SealedBlockWithSenders, TransactionSigned, TransactionSignedEcRecovered, + BlockHashOrNumber, Header, Receipt, SealedBlockWithSenders, TransactionSigned, }; use reth_storage_api::{BlockReader, StateProviderFactory, TransactionVariant}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; @@ -33,13 +32,13 @@ pub mod db; pub mod metrics; pub mod multi_consumer; -/// The type that can send the response to a requested [`Block`] +/// The type that can send the response to a requested [`SealedBlockWithSenders`] type BlockTransactionsResponseSender = oneshot::Sender>>>; -/// The type that can send the response to a requested [`BlockWithSenders`] +/// The type that can send the response to a requested [`SealedBlockWithSenders`] type BlockWithSendersResponseSender = - oneshot::Sender>>>; + oneshot::Sender>>>; /// The type that can send the response to the requested receipts of a block. type ReceiptsResponseSender = oneshot::Sender>>>>; @@ -49,7 +48,7 @@ type EnvResponseSender = oneshot::Sender = MultiConsumerLruCache< B256, - Arc, + Arc, L, Either, >; @@ -142,92 +141,18 @@ impl EthStateCache { this } - /// Requests the [`Block`] for the block hash - /// - /// Returns `None` if the block does not exist. - pub async fn get_block(&self, block_hash: B256) -> ProviderResult> { - let (response_tx, rx) = oneshot::channel(); - let _ = self.to_service.send(CacheAction::GetBlockWithSenders { block_hash, response_tx }); - let block_with_senders_res = - rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)?; - - if let Ok(Some(block_with_senders)) = block_with_senders_res { - Ok(Some(block_with_senders.block.clone())) - } else { - Ok(None) - } - } - - /// Requests the [`Block`] for the block hash, sealed with the given block hash. - /// - /// Returns `None` if the block does not exist. - pub async fn get_sealed_block(&self, block_hash: B256) -> ProviderResult> { - Ok(self.get_block(block_hash).await?.map(|block| block.seal(block_hash))) - } - - /// Requests the transactions of the [`Block`] - /// - /// Returns `None` if the block does not exist. - pub async fn get_block_transactions( - &self, - block_hash: B256, - ) -> ProviderResult>> { - let (response_tx, rx) = oneshot::channel(); - let _ = self.to_service.send(CacheAction::GetBlockTransactions { block_hash, response_tx }); - rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? - } - - /// Requests the ecrecovered transactions of the [`Block`] - /// - /// Returns `None` if the block does not exist. - pub async fn get_block_transactions_ecrecovered( - &self, - block_hash: B256, - ) -> ProviderResult>> { - Ok(self - .get_block_with_senders(block_hash) - .await? - .map(|block| (*block).clone().into_transactions_ecrecovered().collect())) - } - - /// Fetches both transactions and receipts for the given block hash. - pub async fn get_transactions_and_receipts( - &self, - block_hash: B256, - ) -> ProviderResult, Arc>)>> { - let transactions = self.get_block_transactions(block_hash); - let receipts = self.get_receipts(block_hash); - - let (transactions, receipts) = futures::try_join!(transactions, receipts)?; - - Ok(transactions.zip(receipts)) - } - - /// Requests the [`BlockWithSenders`] for the block hash + /// Requests the [`SealedBlockWithSenders`] for the block hash /// /// Returns `None` if the block does not exist. - pub async fn get_block_with_senders( + pub async fn get_sealed_block_with_senders( &self, block_hash: B256, - ) -> ProviderResult>> { + ) -> ProviderResult>> { let (response_tx, rx) = oneshot::channel(); let _ = self.to_service.send(CacheAction::GetBlockWithSenders { block_hash, response_tx }); rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? } - /// Requests the [`SealedBlockWithSenders`] for the block hash - /// - /// Returns `None` if the block does not exist. - pub async fn get_sealed_block_with_senders( - &self, - block_hash: B256, - ) -> ProviderResult> { - Ok(self - .get_block_with_senders(block_hash) - .await? - .map(|block| (*block).clone().seal(block_hash))) - } - /// Requests the [Receipt] for the block hash /// /// Returns `None` if the block was not found. @@ -244,8 +169,8 @@ impl EthStateCache { pub async fn get_block_and_receipts( &self, block_hash: B256, - ) -> ProviderResult>)>> { - let block = self.get_sealed_block(block_hash); + ) -> ProviderResult, Arc>)>> { + let block = self.get_sealed_block_with_senders(block_hash); let receipts = self.get_receipts(block_hash); let (block, receipts) = futures::try_join!(block, receipts)?; @@ -292,7 +217,7 @@ pub(crate) struct EthStateCacheService< LimitReceipts = ByLength, LimitEnvs = ByLength, > where - LimitBlocks: Limiter>, + LimitBlocks: Limiter>, LimitReceipts: Limiter>>, LimitEnvs: Limiter, { @@ -325,7 +250,7 @@ where fn on_new_block( &mut self, block_hash: B256, - res: ProviderResult>>, + res: ProviderResult>>, ) { if let Some(queued) = self.full_block_cache.remove(&block_hash) { // send the response to queued senders @@ -367,7 +292,11 @@ where } } - fn on_reorg_block(&mut self, block_hash: B256, res: ProviderResult>) { + fn on_reorg_block( + &mut self, + block_hash: B256, + res: ProviderResult>, + ) { let res = res.map(|b| b.map(Arc::new)); if let Some(queued) = self.full_block_cache.remove(&block_hash) { // send the response to queued senders @@ -441,7 +370,7 @@ where // Only look in the database to prevent situations where we // looking up the tree is blocking let block_sender = provider - .block_with_senders( + .sealed_block_with_senders( BlockHashOrNumber::Hash(block_hash), TransactionVariant::WithHash, ) @@ -453,36 +382,6 @@ where })); } } - CacheAction::GetBlockTransactions { block_hash, response_tx } => { - // check if block is cached - if let Some(block) = this.full_block_cache.get(&block_hash) { - let _ = response_tx.send(Ok(Some(block.body.transactions.clone()))); - continue - } - - // block is not in the cache, request it if this is the first consumer - if this.full_block_cache.queue(block_hash, Either::Right(response_tx)) { - let provider = this.provider.clone(); - let action_tx = this.action_tx.clone(); - let rate_limiter = this.rate_limiter.clone(); - this.action_task_spawner.spawn_blocking(Box::pin(async move { - // Acquire permit - let _permit = rate_limiter.acquire().await; - // Only look in the database to prevent situations where we - // looking up the tree is blocking - let res = provider - .block_with_senders( - BlockHashOrNumber::Hash(block_hash), - TransactionVariant::WithHash, - ) - .map(|b| b.map(Arc::new)); - let _ = action_tx.send(CacheAction::BlockWithSendersResult { - block_hash, - res, - }); - })); - } - } CacheAction::GetReceipts { block_hash, response_tx } => { // check if block is cached if let Some(receipts) = this.receipts_cache.get(&block_hash).cloned() { @@ -574,7 +473,7 @@ where } CacheAction::CacheNewCanonicalChain { chain_change } => { for block in chain_change.blocks { - this.on_new_block(block.hash(), Ok(Some(Arc::new(block.unseal())))); + this.on_new_block(block.hash(), Ok(Some(Arc::new(block)))); } for block_receipts in chain_change.receipts { @@ -588,7 +487,7 @@ where } CacheAction::RemoveReorgedChain { chain_change } => { for block in chain_change.blocks { - this.on_reorg_block(block.hash(), Ok(Some(block.unseal()))); + this.on_reorg_block(block.hash(), Ok(Some(block))); } for block_receipts in chain_change.receipts { @@ -610,15 +509,36 @@ where /// All message variants sent through the channel enum CacheAction { - GetBlockWithSenders { block_hash: B256, response_tx: BlockWithSendersResponseSender }, - GetBlockTransactions { block_hash: B256, response_tx: BlockTransactionsResponseSender }, - GetEnv { block_hash: B256, response_tx: EnvResponseSender }, - GetReceipts { block_hash: B256, response_tx: ReceiptsResponseSender }, - BlockWithSendersResult { block_hash: B256, res: ProviderResult>> }, - ReceiptsResult { block_hash: B256, res: ProviderResult>>> }, - EnvResult { block_hash: B256, res: Box> }, - CacheNewCanonicalChain { chain_change: ChainChange }, - RemoveReorgedChain { chain_change: ChainChange }, + GetBlockWithSenders { + block_hash: B256, + response_tx: BlockWithSendersResponseSender, + }, + GetEnv { + block_hash: B256, + response_tx: EnvResponseSender, + }, + GetReceipts { + block_hash: B256, + response_tx: ReceiptsResponseSender, + }, + BlockWithSendersResult { + block_hash: B256, + res: ProviderResult>>, + }, + ReceiptsResult { + block_hash: B256, + res: ProviderResult>>>, + }, + EnvResult { + block_hash: B256, + res: Box>, + }, + CacheNewCanonicalChain { + chain_change: ChainChange, + }, + RemoveReorgedChain { + chain_change: ChainChange, + }, } struct BlockReceipts { diff --git a/crates/rpc/rpc-eth-types/src/fee_history.rs b/crates/rpc/rpc-eth-types/src/fee_history.rs index 08ac56845ffe0..57dd276e5cf43 100644 --- a/crates/rpc/rpc-eth-types/src/fee_history.rs +++ b/crates/rpc/rpc-eth-types/src/fee_history.rs @@ -73,16 +73,16 @@ impl FeeHistoryCache { } /// Insert block data into the cache. - async fn insert_blocks(&self, blocks: I) + async fn insert_blocks<'a, I>(&self, blocks: I) where - I: IntoIterator>)>, + I: IntoIterator>)>, { let mut entries = self.inner.entries.write().await; let percentiles = self.predefined_percentiles(); // Insert all new blocks and calculate approximated rewards for (block, receipts) in blocks { - let mut fee_history_entry = FeeHistoryEntry::new(&block); + let mut fee_history_entry = FeeHistoryEntry::new(block); fee_history_entry.rewards = calculate_reward_percentiles_for_block( &percentiles, fee_history_entry.gas_used, @@ -237,7 +237,9 @@ pub async fn fee_history_cache_new_blocks_task( tokio::select! { res = &mut fetch_missing_block => { if let Ok(res) = res { - fee_history_cache.insert_blocks(res.into_iter()).await; + fee_history_cache.insert_blocks(res.as_ref() + .map(|(b, r)| (&b.block, r.clone())) + .into_iter()).await; } } event = events.next() => { @@ -245,11 +247,12 @@ pub async fn fee_history_cache_new_blocks_task( // the stream ended, we are done break; }; - let (blocks, receipts): (Vec<_>, Vec<_>) = event - .committed() + + let committed = event .committed(); + let (blocks, receipts): (Vec<_>, Vec<_>) = committed .blocks_and_receipts() .map(|(block, receipts)| { - (block.block.clone(), Arc::new(receipts.iter().flatten().cloned().collect::>())) + (&block.block, Arc::new(receipts.iter().flatten().cloned().collect::>())) }) .unzip(); fee_history_cache.insert_blocks(blocks.into_iter().zip(receipts)).await; diff --git a/crates/rpc/rpc-eth-types/src/gas_oracle.rs b/crates/rpc/rpc-eth-types/src/gas_oracle.rs index 01591bc4de384..84e7ab8306df2 100644 --- a/crates/rpc/rpc-eth-types/src/gas_oracle.rs +++ b/crates/rpc/rpc-eth-types/src/gas_oracle.rs @@ -1,16 +1,16 @@ //! An implementation of the eth gas price oracle, used for providing gas price estimates based on //! previous blocks. -use std::fmt::{self, Debug, Formatter}; - use alloy_primitives::{B256, U256}; use alloy_rpc_types::BlockId; use derive_more::{Deref, DerefMut, From, Into}; +use itertools::Itertools; use reth_primitives::{constants::GWEI_TO_WEI, BlockNumberOrTag}; use reth_rpc_server_types::constants; use reth_storage_api::BlockReaderIdExt; use schnellru::{ByLength, LruMap}; use serde::{Deserialize, Serialize}; +use std::fmt::{self, Debug, Formatter}; use tokio::sync::Mutex; use tracing::warn; @@ -212,7 +212,7 @@ where limit: usize, ) -> EthResult)>> { // check the cache (this will hit the disk if the block is not cached) - let mut block = match self.cache.get_block(block_hash).await? { + let block = match self.cache.get_sealed_block_with_senders(block_hash).await? { Some(block) => block, None => return Ok(None), }; @@ -221,11 +221,15 @@ where let parent_hash = block.parent_hash; // sort the functions by ascending effective tip first - block.body.transactions.sort_by_cached_key(|tx| tx.effective_tip_per_gas(base_fee_per_gas)); + let sorted_transactions = block + .body + .transactions + .iter() + .sorted_by_cached_key(|tx| tx.effective_tip_per_gas(base_fee_per_gas)); let mut prices = Vec::with_capacity(limit); - for tx in block.body.transactions() { + for tx in sorted_transactions { let mut effective_gas_tip = None; // ignore transactions with a tip under the configured threshold if let Some(ignore_under) = self.ignore_price { diff --git a/crates/rpc/rpc-eth-types/src/logs_utils.rs b/crates/rpc/rpc-eth-types/src/logs_utils.rs index c64bbe055b79d..205e2bba37be7 100644 --- a/crates/rpc/rpc-eth-types/src/logs_utils.rs +++ b/crates/rpc/rpc-eth-types/src/logs_utils.rs @@ -6,8 +6,9 @@ use alloy_primitives::TxHash; use alloy_rpc_types::{FilteredParams, Log}; use reth_chainspec::ChainInfo; use reth_errors::ProviderError; -use reth_primitives::{BlockNumHash, Receipt, SealedBlock}; +use reth_primitives::{BlockNumHash, Receipt, SealedBlockWithSenders}; use reth_storage_api::BlockReader; +use std::sync::Arc; /// Returns all matching of a block's receipts when the transaction hashes are known. pub fn matching_block_logs_with_tx_hashes<'a, I>( @@ -50,8 +51,8 @@ where pub enum ProviderOrBlock<'a, P: BlockReader> { /// Provider Provider(&'a P), - /// [`SealedBlock`] - Block(SealedBlock), + /// [`SealedBlockWithSenders`] + Block(Arc), } /// Appends all matching logs of a block's receipts. diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index fbef6f7a7e0b3..acf215b3b2cc6 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -119,7 +119,7 @@ where env: Env::boxed( cfg.cfg_env.clone(), block_env.clone(), - Call::evm_config(this.eth_api()).tx_env(&tx), + Call::evm_config(this.eth_api()).tx_env(tx.as_signed(), tx.signer()), ), handler_cfg: cfg.handler_cfg, }; @@ -219,7 +219,7 @@ where self.trace_block( state_at.into(), - block.into_transactions_ecrecovered().collect(), + (*block).clone().into_transactions_ecrecovered().collect(), cfg, block_env, opts, @@ -245,11 +245,12 @@ where // block the transaction is included in let state_at: BlockId = block.parent_hash.into(); let block_hash = block.hash(); - let block_txs = block.into_transactions_ecrecovered(); let this = self.clone(); self.eth_api() .spawn_with_state_at_block(state_at, move |state| { + let block_txs = block.transactions_with_sender(); + // configure env for the target transaction let tx = transaction.into_recovered(); @@ -267,7 +268,7 @@ where env: Env::boxed( cfg.cfg_env.clone(), block_env, - Call::evm_config(this.eth_api()).tx_env(&tx), + Call::evm_config(this.eth_api()).tx_env(tx.as_signed(), tx.signer()), ), handler_cfg: cfg.handler_cfg, }; @@ -527,15 +528,15 @@ where if replay_block_txs { // only need to replay the transactions in the block if not all transactions are // to be replayed - let transactions = block.into_transactions_ecrecovered().take(num_txs); + let transactions = block.transactions_with_sender().take(num_txs); // Execute all transactions until index - for tx in transactions { + for (signer, tx) in transactions { let env = EnvWithHandlerCfg { env: Env::boxed( cfg.cfg_env.clone(), block_env.clone(), - Call::evm_config(this.eth_api()).tx_env(&tx), + Call::evm_config(this.eth_api()).tx_env(tx, *signer), ), handler_cfg: cfg.handler_cfg, }; @@ -613,7 +614,7 @@ where let _ = block_executor .execute_with_state_closure( - (&block.clone().unseal(), block.difficulty).into(), + (&(*block).clone().unseal(), block.difficulty).into(), |statedb: &State<_>| { codes = statedb .cache diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 9efecf3dae7f3..b136861c79611 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -19,7 +19,7 @@ use async_trait::async_trait; use jsonrpsee::{core::RpcResult, server::IdProvider}; use reth_chainspec::ChainInfo; use reth_node_api::EthApiTypes; -use reth_primitives::{Receipt, SealedBlock, TransactionSignedEcRecovered}; +use reth_primitives::{Receipt, SealedBlockWithSenders, TransactionSignedEcRecovered}; use reth_provider::{BlockIdReader, BlockReader, EvmEnvProvider, ProviderError}; use reth_rpc_eth_api::{EthFilterApiServer, FullEthApiTypes, RpcTransaction, TransactionCompat}; use reth_rpc_eth_types::{ @@ -534,7 +534,8 @@ where &self, block_num_hash: &BlockNumHash, best_number: u64, - ) -> Result>, Option)>, EthFilterError> { + ) -> Result>, Option>)>, EthFilterError> + { // The last 4 blocks are most likely cached, so we can just fetch them let cached_range = best_number.saturating_sub(4)..=best_number; let receipts_block = if cached_range.contains(&block_num_hash.number) { diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index f7b2f7ab7eaca..05588d4dabea3 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -118,14 +118,14 @@ where trace_types: HashSet, block_id: Option, ) -> Result { - let tx = recover_raw_transaction(tx)?; + let tx = recover_raw_transaction(tx)?.into_ecrecovered_transaction(); let (cfg, block, at) = self.eth_api().evm_env_at(block_id.unwrap_or_default()).await?; let env = EnvWithHandlerCfg::new_with_cfg_env( cfg, block, - Call::evm_config(self.eth_api()).tx_env(&tx.into_ecrecovered_transaction()), + Call::evm_config(self.eth_api()).tx_env(tx.as_signed(), tx.signer()), ); let config = TracingInspectorConfig::from_parity_config(&trace_types); @@ -372,7 +372,7 @@ where }, ); - let block = self.eth_api().block(block_id); + let block = self.eth_api().block_with_senders(block_id); let (maybe_traces, maybe_block) = futures::try_join!(traces, block)?; let mut maybe_traces = @@ -468,7 +468,9 @@ where let Some(transactions) = res else { return Ok(None) }; - let Some(block) = self.eth_api().block(block_id).await? else { return Ok(None) }; + let Some(block) = self.eth_api().block_with_senders(block_id).await? else { + return Ok(None) + }; Ok(Some(BlockOpcodeGas { block_hash: block.hash(), From a352c8cab48bdfc464041e1775178356e37ca4cc Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Mon, 14 Oct 2024 17:21:41 +0400 Subject: [PATCH 156/159] feat: refactor and integrate local engine into `EngineNodeLauncher` (#11703) --- Cargo.lock | 17 +- .../consensus/beacon/src/engine/forkchoice.rs | 9 +- crates/e2e-test-utils/Cargo.toml | 1 + crates/engine/local/Cargo.toml | 24 +- crates/engine/local/src/lib.rs | 13 + crates/engine/local/src/miner.rs | 193 +++++++- crates/engine/local/src/payload.rs | 63 ++- crates/engine/local/src/service.rs | 427 +++++------------- crates/engine/service/src/service.rs | 2 +- crates/ethereum/node/tests/e2e/dev.rs | 53 ++- crates/ethereum/node/tests/e2e/utils.rs | 5 - crates/node/builder/Cargo.toml | 1 + crates/node/builder/src/launch/engine.rs | 79 +++- crates/node/events/src/node.rs | 2 +- crates/optimism/node/Cargo.toml | 2 + crates/payload/primitives/src/traits.rs | 9 +- 16 files changed, 500 insertions(+), 400 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c5639eadab0e..d690cf536aa3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6971,6 +6971,7 @@ dependencies = [ "reth", "reth-chainspec", "reth-db", + "reth-engine-local", "reth-network-peers", "reth-node-builder", "reth-node-ethereum", @@ -7028,22 +7029,22 @@ dependencies = [ "alloy-rpc-types-engine", "eyre", "futures-util", + "op-alloy-rpc-types-engine", "reth-beacon-consensus", - "reth-chain-state", "reth-chainspec", - "reth-config", - "reth-db", + "reth-consensus", + "reth-engine-primitives", + "reth-engine-service", "reth-engine-tree", "reth-ethereum-engine-primitives", - "reth-exex-test-utils", - "reth-node-types", + "reth-evm", "reth-payload-builder", "reth-payload-primitives", - "reth-primitives", + "reth-payload-validator", "reth-provider", "reth-prune", + "reth-rpc-types-compat", "reth-stages-api", - "reth-tracing", "reth-transaction-pool", "tokio", "tokio-stream", @@ -7809,6 +7810,7 @@ dependencies = [ "reth-db-api", "reth-db-common", "reth-downloaders", + "reth-engine-local", "reth-engine-service", "reth-engine-tree", "reth-engine-util", @@ -8131,6 +8133,7 @@ dependencies = [ "reth-db", "reth-discv5", "reth-e2e-test-utils", + "reth-engine-local", "reth-evm", "reth-network", "reth-node-api", diff --git a/crates/consensus/beacon/src/engine/forkchoice.rs b/crates/consensus/beacon/src/engine/forkchoice.rs index 975c2ee3bc452..7e49714ba3756 100644 --- a/crates/consensus/beacon/src/engine/forkchoice.rs +++ b/crates/consensus/beacon/src/engine/forkchoice.rs @@ -150,15 +150,18 @@ pub enum ForkchoiceStatus { } impl ForkchoiceStatus { - pub(crate) const fn is_valid(&self) -> bool { + /// Returns `true` if the forkchoice state is [`ForkchoiceStatus::Valid`]. + pub const fn is_valid(&self) -> bool { matches!(self, Self::Valid) } - pub(crate) const fn is_invalid(&self) -> bool { + /// Returns `true` if the forkchoice state is [`ForkchoiceStatus::Invalid`]. + pub const fn is_invalid(&self) -> bool { matches!(self, Self::Invalid) } - pub(crate) const fn is_syncing(&self) -> bool { + /// Returns `true` if the forkchoice state is [`ForkchoiceStatus::Syncing`]. + pub const fn is_syncing(&self) -> bool { matches!(self, Self::Syncing) } diff --git a/crates/e2e-test-utils/Cargo.toml b/crates/e2e-test-utils/Cargo.toml index a10162e78ff64..2742d704054ef 100644 --- a/crates/e2e-test-utils/Cargo.toml +++ b/crates/e2e-test-utils/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] reth.workspace = true reth-chainspec.workspace = true +reth-engine-local.workspace = true reth-primitives.workspace = true reth-tracing.workspace = true reth-db = { workspace = true, features = ["test-utils"] } diff --git a/crates/engine/local/Cargo.toml b/crates/engine/local/Cargo.toml index d7a5d05091f14..f22ab1f8d5603 100644 --- a/crates/engine/local/Cargo.toml +++ b/crates/engine/local/Cargo.toml @@ -11,15 +11,19 @@ exclude.workspace = true [dependencies] # reth reth-beacon-consensus.workspace = true -reth-chain-state.workspace = true +reth-chainspec.workspace = true +reth-consensus.workspace = true +reth-engine-primitives.workspace = true +reth-engine-service.workspace = true reth-engine-tree.workspace = true +reth-evm.workspace = true reth-ethereum-engine-primitives.workspace = true -reth-node-types.workspace = true reth-payload-builder.workspace = true reth-payload-primitives.workspace = true -reth-primitives.workspace = true +reth-payload-validator.workspace = true reth-provider.workspace = true reth-prune.workspace = true +reth-rpc-types-compat.workspace = true reth-transaction-pool.workspace = true reth-stages-api.workspace = true @@ -36,16 +40,10 @@ futures-util.workspace = true eyre.workspace = true tracing.workspace = true -[dev-dependencies] -reth-chainspec.workspace = true -reth-chain-state.workspace = true -reth-config.workspace = true -reth-db = { workspace = true, features = ["test-utils"] } -reth-ethereum-engine-primitives.workspace = true -reth-exex-test-utils.workspace = true -reth-payload-builder = { workspace = true, features = ["test-utils"] } -reth-provider = { workspace = true, features = ["test-utils"] } -reth-tracing.workspace = true +op-alloy-rpc-types-engine = { workspace = true, optional = true } [lints] workspace = true + +[features] +optimism = ["op-alloy-rpc-types-engine"] diff --git a/crates/engine/local/src/lib.rs b/crates/engine/local/src/lib.rs index 1b84c8a113f43..26c84d50c851d 100644 --- a/crates/engine/local/src/lib.rs +++ b/crates/engine/local/src/lib.rs @@ -1,4 +1,17 @@ //! A local engine service that can be used to drive a dev chain. + +#![doc( + html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png", + html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", + issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" +)] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] + pub mod miner; pub mod payload; pub mod service; + +pub use miner::MiningMode; +pub use payload::LocalPayloadAttributesBuilder; +pub use service::LocalEngineService; diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index de3d8cb8d0132..e12a2a50d0366 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -1,16 +1,31 @@ //! Contains the implementation of the mining mode for the local engine. -use alloy_primitives::TxHash; +use alloy_primitives::{TxHash, B256}; +use alloy_rpc_types_engine::{CancunPayloadFields, ForkchoiceState}; +use eyre::OptionExt; use futures_util::{stream::Fuse, StreamExt}; +use reth_beacon_consensus::BeaconEngineMessage; +use reth_chainspec::EthereumHardforks; +use reth_engine_primitives::EngineTypes; +use reth_payload_builder::PayloadBuilderHandle; +use reth_payload_primitives::{ + BuiltPayload, PayloadAttributesBuilder, PayloadBuilder, PayloadTypes, +}; +use reth_provider::{BlockReader, ChainSpecProvider}; +use reth_rpc_types_compat::engine::payload::block_to_payload; use reth_transaction_pool::TransactionPool; use std::{ future::Future, pin::Pin, task::{Context, Poll}, - time::Duration, + time::{Duration, UNIX_EPOCH}, +}; +use tokio::{ + sync::{mpsc::UnboundedSender, oneshot}, + time::Interval, }; -use tokio::time::Interval; use tokio_stream::wrappers::ReceiverStream; +use tracing::error; /// A mining mode for the local dev engine. #[derive(Debug)] @@ -58,3 +73,175 @@ impl Future for MiningMode { } } } + +/// Local miner advancing the chain/ +#[derive(Debug)] +pub struct LocalMiner { + /// Provider to read the current tip of the chain. + provider: Provider, + /// The payload attribute builder for the engine + payload_attributes_builder: B, + /// Sender for events to engine. + to_engine: UnboundedSender>, + /// The mining mode for the engine + mode: MiningMode, + /// The payload builder for the engine + payload_builder: PayloadBuilderHandle, + /// Timestamp for the next block. + last_timestamp: u64, + /// Stores latest mined blocks. + last_block_hashes: Vec, +} + +impl LocalMiner +where + EngineT: EngineTypes, + Provider: BlockReader + ChainSpecProvider + 'static, + B: PayloadAttributesBuilder<::PayloadAttributes>, +{ + /// Spawns a new [`LocalMiner`] with the given parameters. + pub fn spawn_new( + provider: Provider, + payload_attributes_builder: B, + to_engine: UnboundedSender>, + mode: MiningMode, + payload_builder: PayloadBuilderHandle, + ) { + let latest_header = + provider.sealed_header(provider.best_block_number().unwrap()).unwrap().unwrap(); + + let miner = Self { + provider, + payload_attributes_builder, + to_engine, + mode, + payload_builder, + last_timestamp: latest_header.timestamp, + last_block_hashes: vec![latest_header.hash()], + }; + + // Spawn the miner + tokio::spawn(miner.run()); + } + + /// Runs the [`LocalMiner`] in a loop, polling the miner and building payloads. + async fn run(mut self) { + let mut fcu_interval = tokio::time::interval(Duration::from_secs(1)); + loop { + tokio::select! { + // Wait for the interval or the pool to receive a transaction + _ = &mut self.mode => { + if let Err(e) = self.advance().await { + error!(target: "engine::local", "Error advancing the chain: {:?}", e); + } + } + // send FCU once in a while + _ = fcu_interval.tick() => { + if let Err(e) = self.update_forkchoice_state().await { + error!(target: "engine::local", "Error updating fork choice: {:?}", e); + } + } + } + } + } + + /// Returns current forkchoice state. + fn forkchoice_state(&self) -> ForkchoiceState { + ForkchoiceState { + head_block_hash: *self.last_block_hashes.last().expect("at least 1 block exists"), + safe_block_hash: *self + .last_block_hashes + .get(self.last_block_hashes.len().saturating_sub(32)) + .expect("at least 1 block exists"), + finalized_block_hash: *self + .last_block_hashes + .get(self.last_block_hashes.len().saturating_sub(64)) + .expect("at least 1 block exists"), + } + } + + /// Sends a FCU to the engine. + async fn update_forkchoice_state(&self) -> eyre::Result<()> { + let (tx, rx) = oneshot::channel(); + self.to_engine.send(BeaconEngineMessage::ForkchoiceUpdated { + state: self.forkchoice_state(), + payload_attrs: None, + tx, + })?; + + let res = rx.await??; + if !res.forkchoice_status().is_valid() { + eyre::bail!("Invalid fork choice update") + } + + Ok(()) + } + + /// Generates payload attributes for a new block, passes them to FCU and inserts built payload + /// through newPayload. + async fn advance(&mut self) -> eyre::Result<()> { + let timestamp = std::cmp::max( + self.last_timestamp + 1, + std::time::SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("cannot be earlier than UNIX_EPOCH") + .as_secs(), + ); + + let (tx, rx) = oneshot::channel(); + self.to_engine.send(BeaconEngineMessage::ForkchoiceUpdated { + state: self.forkchoice_state(), + payload_attrs: Some(self.payload_attributes_builder.build(timestamp)), + tx, + })?; + + let res = rx.await??.await?; + if !res.payload_status.is_valid() { + eyre::bail!("Invalid payload status") + } + + let payload_id = res.payload_id.ok_or_eyre("No payload id")?; + + // wait for some time to let the payload be built + tokio::time::sleep(Duration::from_millis(200)).await; + + let Some(Ok(payload)) = self.payload_builder.best_payload(payload_id).await else { + eyre::bail!("No payload") + }; + + let block = payload.block(); + + let cancun_fields = + if self.provider.chain_spec().is_cancun_active_at_timestamp(block.timestamp) { + Some(CancunPayloadFields { + parent_beacon_block_root: block.parent_beacon_block_root.unwrap(), + versioned_hashes: block.blob_versioned_hashes().into_iter().copied().collect(), + }) + } else { + None + }; + + let (tx, rx) = oneshot::channel(); + self.to_engine.send(BeaconEngineMessage::NewPayload { + payload: block_to_payload(payload.block().clone()), + cancun_fields, + tx, + })?; + + let res = rx.await??; + + if !res.is_valid() { + eyre::bail!("Invalid payload") + } + + self.last_timestamp = timestamp; + self.last_block_hashes.push(block.hash()); + // ensure we keep at most 64 blocks + if self.last_block_hashes.len() > 64 { + self.last_block_hashes = + self.last_block_hashes.split_off(self.last_block_hashes.len() - 64); + } + + Ok(()) + } +} diff --git a/crates/engine/local/src/payload.rs b/crates/engine/local/src/payload.rs index 4fd49f53fb49d..15d5ff2cf0084 100644 --- a/crates/engine/local/src/payload.rs +++ b/crates/engine/local/src/payload.rs @@ -2,29 +2,62 @@ //! [`LocalEngineService`](super::service::LocalEngineService). use alloy_primitives::{Address, B256}; +use reth_chainspec::EthereumHardforks; use reth_ethereum_engine_primitives::EthPayloadAttributes; use reth_payload_primitives::PayloadAttributesBuilder; -use std::{convert::Infallible, time::UNIX_EPOCH}; +use std::sync::Arc; /// The attributes builder for local Ethereum payload. #[derive(Debug)] -pub struct EthLocalPayloadAttributesBuilder; - -impl PayloadAttributesBuilder for EthLocalPayloadAttributesBuilder { - type PayloadAttributes = EthPayloadAttributes; - type Error = Infallible; +#[non_exhaustive] +pub struct LocalPayloadAttributesBuilder { + chain_spec: Arc, +} - fn build(&self) -> Result { - let ts = std::time::SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("cannot be earlier than UNIX_EPOCH"); +impl LocalPayloadAttributesBuilder { + /// Creates a new instance of the builder. + pub const fn new(chain_spec: Arc) -> Self { + Self { chain_spec } + } +} - Ok(EthPayloadAttributes { - timestamp: ts.as_secs(), +impl PayloadAttributesBuilder + for LocalPayloadAttributesBuilder +where + ChainSpec: Send + Sync + EthereumHardforks + 'static, +{ + fn build(&self, timestamp: u64) -> EthPayloadAttributes { + EthPayloadAttributes { + timestamp, prev_randao: B256::random(), suggested_fee_recipient: Address::random(), - withdrawals: None, - parent_beacon_block_root: None, - }) + withdrawals: if self.chain_spec.is_shanghai_active_at_timestamp(timestamp) { + Some(Default::default()) + } else { + None + }, + parent_beacon_block_root: if self.chain_spec.is_cancun_active_at_timestamp(timestamp) { + Some(B256::random()) + } else { + None + }, + } + } +} + +#[cfg(feature = "optimism")] +impl PayloadAttributesBuilder + for LocalPayloadAttributesBuilder +where + ChainSpec: Send + Sync + EthereumHardforks + 'static, +{ + fn build(&self, timestamp: u64) -> op_alloy_rpc_types_engine::OpPayloadAttributes { + op_alloy_rpc_types_engine::OpPayloadAttributes { + payload_attributes: self.build(timestamp), + transactions: None, + no_tx_pool: None, + gas_limit: None, + eip_1559_params: None, + } } } diff --git a/crates/engine/local/src/service.rs b/crates/engine/local/src/service.rs index c9794ecfabb0f..93a9cf11ecc4b 100644 --- a/crates/engine/local/src/service.rs +++ b/crates/engine/local/src/service.rs @@ -6,357 +6,154 @@ //! with a single transaction. The `Interval` mode will initiate block //! building at a fixed interval. -use crate::miner::MiningMode; -use eyre::eyre; -use reth_beacon_consensus::EngineNodeTypes; -use reth_chain_state::{CanonicalInMemoryState, ExecutedBlock, NewCanonicalChain}; -use reth_engine_tree::persistence::PersistenceHandle; -use reth_payload_builder::PayloadBuilderHandle; -use reth_payload_primitives::{ - BuiltPayload, PayloadAttributesBuilder, PayloadBuilder, PayloadBuilderAttributes, PayloadTypes, +use core::fmt; +use std::{ + fmt::{Debug, Formatter}, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use crate::miner::{LocalMiner, MiningMode}; +use futures_util::{Stream, StreamExt}; +use reth_beacon_consensus::{BeaconConsensusEngineEvent, BeaconEngineMessage, EngineNodeTypes}; +use reth_chainspec::EthChainSpec; +use reth_consensus::Consensus; +use reth_engine_service::service::EngineMessageStream; +use reth_engine_tree::{ + chain::{ChainEvent, HandlerEvent}, + engine::{ + EngineApiKind, EngineApiRequest, EngineApiRequestHandler, EngineRequestHandler, FromEngine, + RequestHandlerEvent, + }, + persistence::PersistenceHandle, + tree::{EngineApiTreeHandler, InvalidBlockHook, TreeConfig}, }; -use reth_provider::ProviderFactory; +use reth_evm::execute::BlockExecutorProvider; +use reth_payload_builder::PayloadBuilderHandle; +use reth_payload_primitives::{PayloadAttributesBuilder, PayloadTypes}; +use reth_payload_validator::ExecutionPayloadValidator; +use reth_provider::{providers::BlockchainProvider2, ChainSpecProvider, ProviderFactory}; use reth_prune::PrunerWithFactory; use reth_stages_api::MetricEventsSender; -use tokio::sync::oneshot; -use tracing::debug; +use tokio::sync::mpsc::UnboundedSender; +use tracing::error; /// Provides a local dev service engine that can be used to drive the /// chain forward. -#[derive(Debug)] -pub struct LocalEngineService +/// +/// This service both produces and consumes [`BeaconEngineMessage`]s. This is done to allow +/// modifications of the stream +pub struct LocalEngineService where N: EngineNodeTypes, - B: PayloadAttributesBuilder::PayloadAttributes>, { - /// The payload builder for the engine - payload_builder: PayloadBuilderHandle, - /// The payload attribute builder for the engine - payload_attributes_builder: B, - /// Keep track of the Canonical chain state that isn't persisted on disk yet - canonical_in_memory_state: CanonicalInMemoryState, - /// A handle to the persistence layer - persistence_handle: PersistenceHandle, - /// The mining mode for the engine - mode: MiningMode, + /// Processes requests. + /// + /// This type is responsible for processing incoming requests. + handler: EngineApiRequestHandler>, + /// Receiver for incoming requests (from the engine API endpoint) that need to be processed. + incoming_requests: EngineMessageStream, } -impl LocalEngineService +impl LocalEngineService where N: EngineNodeTypes, - B: PayloadAttributesBuilder::PayloadAttributes>, { /// Constructor for [`LocalEngineService`]. - pub fn new( - payload_builder: PayloadBuilderHandle, - payload_attributes_builder: B, + #[allow(clippy::too_many_arguments)] + pub fn new( + consensus: Arc, + executor_factory: impl BlockExecutorProvider, provider: ProviderFactory, + blockchain_db: BlockchainProvider2, pruner: PrunerWithFactory>, - canonical_in_memory_state: CanonicalInMemoryState, + payload_builder: PayloadBuilderHandle, + tree_config: TreeConfig, + invalid_block_hook: Box, sync_metrics_tx: MetricEventsSender, + to_engine: UnboundedSender>, + from_engine: EngineMessageStream, mode: MiningMode, - ) -> Self { + payload_attributes_builder: B, + ) -> Self + where + B: PayloadAttributesBuilder<::PayloadAttributes>, + { + let chain_spec = provider.chain_spec(); + let engine_kind = + if chain_spec.is_optimism() { EngineApiKind::OpStack } else { EngineApiKind::Ethereum }; + let persistence_handle = PersistenceHandle::spawn_service(provider, pruner, sync_metrics_tx); + let payload_validator = ExecutionPayloadValidator::new(chain_spec); - Self { - payload_builder, - payload_attributes_builder, - canonical_in_memory_state, + let canonical_in_memory_state = blockchain_db.canonical_in_memory_state(); + + let (to_tree_tx, from_tree) = EngineApiTreeHandler::spawn_new( + blockchain_db.clone(), + executor_factory, + consensus, + payload_validator, persistence_handle, - mode, - } - } + payload_builder.clone(), + canonical_in_memory_state, + tree_config, + invalid_block_hook, + engine_kind, + ); - /// Spawn the [`LocalEngineService`] on a tokio green thread. The service will poll the payload - /// builder with two varying modes, [`MiningMode::Instant`] or [`MiningMode::Interval`] - /// which will respectively either execute the block as soon as it finds a - /// transaction in the pool or build the block based on an interval. - pub fn spawn_new( - payload_builder: PayloadBuilderHandle, - payload_attributes_builder: B, - provider: ProviderFactory, - pruner: PrunerWithFactory>, - canonical_in_memory_state: CanonicalInMemoryState, - sync_metrics_tx: MetricEventsSender, - mode: MiningMode, - ) { - let engine = Self::new( - payload_builder, + let handler = EngineApiRequestHandler::new(to_tree_tx, from_tree); + + LocalMiner::spawn_new( + blockchain_db, payload_attributes_builder, - provider, - pruner, - canonical_in_memory_state, - sync_metrics_tx, + to_engine, mode, + payload_builder, ); - // Spawn the engine - tokio::spawn(engine.run()); + Self { handler, incoming_requests: from_engine } } +} - /// Runs the [`LocalEngineService`] in a loop, polling the miner and building - /// payloads. - async fn run(mut self) { - loop { - // Wait for the interval or the pool to receive a transaction - (&mut self.mode).await; - - // Start a new payload building job - let executed_block = self.build_and_save_payload().await; - - if executed_block.is_err() { - debug!(target: "local_engine", err = ?executed_block.unwrap_err(), "failed payload building"); - continue - } - let block = executed_block.expect("not error"); - - let res = self.update_canonical_in_memory_state(block); - if res.is_err() { - debug!(target: "local_engine", err = ?res.unwrap_err(), "failed canonical state update"); +impl Stream for LocalEngineService +where + N: EngineNodeTypes, +{ + type Item = ChainEvent; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + + if let Poll::Ready(ev) = this.handler.poll(cx) { + return match ev { + RequestHandlerEvent::HandlerEvent(ev) => match ev { + HandlerEvent::BackfillAction(_) => { + error!(target: "engine::local", "received backfill request in local engine"); + Poll::Ready(Some(ChainEvent::FatalError)) + } + HandlerEvent::Event(ev) => Poll::Ready(Some(ChainEvent::Handler(ev))), + HandlerEvent::FatalError => Poll::Ready(Some(ChainEvent::FatalError)), + }, + RequestHandlerEvent::Download(_) => { + error!(target: "engine::local", "received download request in local engine"); + Poll::Ready(Some(ChainEvent::FatalError)) + } } } - } - - /// Builds a payload by initiating a new payload job via the [`PayloadBuilderHandle`], - /// saving the execution outcome to persistence and returning the executed block. - async fn build_and_save_payload(&self) -> eyre::Result { - let payload_attributes = self.payload_attributes_builder.build()?; - let parent = self.canonical_in_memory_state.get_canonical_head().hash(); - let payload_builder_attributes = - ::PayloadBuilderAttributes::try_new( - parent, - payload_attributes, - ) - .map_err(|_| eyre::eyre!("failed to fetch payload attributes"))?; - - let payload = self - .payload_builder - .send_and_resolve_payload(payload_builder_attributes) - .await? - .await?; - - let executed_block = - payload.executed_block().ok_or_else(|| eyre!("missing executed block"))?; - let (tx, rx) = oneshot::channel(); - - let _ = self.persistence_handle.save_blocks(vec![executed_block.clone()], tx); - - // Wait for the persistence_handle to complete - let _ = rx.await?.ok_or_else(|| eyre!("missing new head"))?; - Ok(executed_block) - } - - /// Update the canonical in memory state and send notification for a new canon state to - /// all the listeners. - fn update_canonical_in_memory_state(&self, executed_block: ExecutedBlock) -> eyre::Result<()> { - let chain = NewCanonicalChain::Commit { new: vec![executed_block] }; - let tip = chain.tip().header.clone(); - let notification = chain.to_chain_notification(); - - // Update the tracked in-memory state with the new chain - self.canonical_in_memory_state.update_chain(chain); - self.canonical_in_memory_state.set_canonical_head(tip); - - // Sends an event to all active listeners about the new canonical chain - self.canonical_in_memory_state.notify_canon_state(notification); - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use reth_chainspec::MAINNET; - use reth_config::PruneConfig; - use reth_db::test_utils::{create_test_rw_db, create_test_static_files_dir}; - use reth_ethereum_engine_primitives::EthEngineTypes; - use reth_exex_test_utils::TestNode; - use reth_node_types::NodeTypesWithDBAdapter; - use reth_payload_builder::test_utils::spawn_test_payload_service; - use reth_provider::{providers::StaticFileProvider, BlockReader, ProviderFactory}; - use reth_prune::PrunerBuilder; - use reth_transaction_pool::{ - test_utils::{testing_pool, MockTransaction}, - TransactionPool, - }; - use std::{convert::Infallible, time::Duration}; - use tokio::sync::mpsc::unbounded_channel; - - #[derive(Debug)] - struct TestPayloadAttributesBuilder; - - impl PayloadAttributesBuilder for TestPayloadAttributesBuilder { - type PayloadAttributes = alloy_rpc_types_engine::PayloadAttributes; - type Error = Infallible; - - fn build(&self) -> Result { - Ok(alloy_rpc_types_engine::PayloadAttributes { - timestamp: 0, - prev_randao: Default::default(), - suggested_fee_recipient: Default::default(), - withdrawals: None, - parent_beacon_block_root: None, - }) + // forward incoming requests to the handler + while let Poll::Ready(Some(req)) = this.incoming_requests.poll_next_unpin(cx) { + this.handler.on_event(FromEngine::Request(req.into())); } - } - - #[tokio::test] - async fn test_local_engine_service_interval() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - // Start the provider and the pruner - let (_, static_dir_path) = create_test_static_files_dir(); - let provider = ProviderFactory::>::new( - create_test_rw_db(), - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path)?, - ); - let pruner = PrunerBuilder::new(PruneConfig::default()) - .build_with_provider_factory(provider.clone()); - - // Create an empty canonical in memory state - let canonical_in_memory_state = CanonicalInMemoryState::empty(); - - // Start the payload builder service - let payload_handle = spawn_test_payload_service::(); - - // Sync metric channel - let (sync_metrics_tx, _) = unbounded_channel(); - - // Launch the LocalEngineService in interval mode - let period = Duration::from_secs(1); - LocalEngineService::spawn_new( - payload_handle, - TestPayloadAttributesBuilder, - provider.clone(), - pruner, - canonical_in_memory_state, - sync_metrics_tx, - MiningMode::interval(period), - ); - - // Check that we have no block for now - let block = provider.block_by_number(0)?; - assert!(block.is_none()); - // Wait 4 intervals - tokio::time::sleep(2 * period).await; - - // Assert a block has been build - let block = provider.block_by_number(0)?; - assert!(block.is_some()); - - Ok(()) + Poll::Pending } +} - #[tokio::test] - async fn test_local_engine_service_instant() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - // Start the provider and the pruner - let (_, static_dir_path) = create_test_static_files_dir(); - let provider = ProviderFactory::>::new( - create_test_rw_db(), - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path)?, - ); - let pruner = PrunerBuilder::new(PruneConfig::default()) - .build_with_provider_factory(provider.clone()); - - // Create an empty canonical in memory state - let canonical_in_memory_state = CanonicalInMemoryState::empty(); - - // Start the payload builder service - let payload_handle = spawn_test_payload_service::(); - - // Start a transaction pool - let pool = testing_pool(); - - // Sync metric channel - let (sync_metrics_tx, _) = unbounded_channel(); - - // Launch the LocalEngineService in instant mode - LocalEngineService::spawn_new( - payload_handle, - TestPayloadAttributesBuilder, - provider.clone(), - pruner, - canonical_in_memory_state, - sync_metrics_tx, - MiningMode::instant(pool.clone()), - ); - - // Wait for a small period to assert block building is - // triggered by adding a transaction to the pool - let period = Duration::from_millis(500); - tokio::time::sleep(period).await; - let block = provider.block_by_number(0)?; - assert!(block.is_none()); - - // Add a transaction to the pool - let transaction = MockTransaction::legacy().with_gas_price(10); - pool.add_transaction(Default::default(), transaction).await?; - - // Wait for block building - let period = Duration::from_secs(2); - tokio::time::sleep(period).await; - - // Assert a block has been build - let block = provider.block_by_number(0)?; - assert!(block.is_some()); - - Ok(()) - } - - #[tokio::test] - async fn test_canonical_chain_subscription() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - // Start the provider and the pruner - let (_, static_dir_path) = create_test_static_files_dir(); - let provider = ProviderFactory::>::new( - create_test_rw_db(), - MAINNET.clone(), - StaticFileProvider::read_write(static_dir_path)?, - ); - let pruner = PrunerBuilder::new(PruneConfig::default()) - .build_with_provider_factory(provider.clone()); - - // Create an empty canonical in memory state - let canonical_in_memory_state = CanonicalInMemoryState::empty(); - let mut notifications = canonical_in_memory_state.subscribe_canon_state(); - - // Start the payload builder service - let payload_handle = spawn_test_payload_service::(); - - // Start a transaction pool - let pool = testing_pool(); - - // Sync metric channel - let (sync_metrics_tx, _) = unbounded_channel(); - - // Launch the LocalEngineService in instant mode - LocalEngineService::spawn_new( - payload_handle, - TestPayloadAttributesBuilder, - provider.clone(), - pruner, - canonical_in_memory_state, - sync_metrics_tx, - MiningMode::instant(pool.clone()), - ); - - // Add a transaction to the pool - let transaction = MockTransaction::legacy().with_gas_price(10); - pool.add_transaction(Default::default(), transaction).await?; - - // Check a notification is received for block 0 - let res = notifications.recv().await?; - - assert_eq!(res.tip().number, 0); - - Ok(()) +impl Debug for LocalEngineService { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("LocalEngineService").finish_non_exhaustive() } } diff --git a/crates/engine/service/src/service.rs b/crates/engine/service/src/service.rs index ed9c1aa1c27e0..026476a8260f6 100644 --- a/crates/engine/service/src/service.rs +++ b/crates/engine/service/src/service.rs @@ -31,7 +31,7 @@ use std::{ }; /// Alias for consensus engine stream. -type EngineMessageStream = Pin> + Send + Sync>>; +pub type EngineMessageStream = Pin> + Send + Sync>>; /// Alias for chain orchestrator. type EngineServiceType = ChainOrchestrator< diff --git a/crates/ethereum/node/tests/e2e/dev.rs b/crates/ethereum/node/tests/e2e/dev.rs index 2ef6e08c7e42a..6b4733b6f9b41 100644 --- a/crates/ethereum/node/tests/e2e/dev.rs +++ b/crates/ethereum/node/tests/e2e/dev.rs @@ -3,29 +3,64 @@ use std::sync::Arc; use alloy_genesis::Genesis; use alloy_primitives::{b256, hex}; use futures::StreamExt; -use reth::core::rpc::eth::helpers::EthTransactions; +use reth::{args::DevArgs, core::rpc::eth::helpers::EthTransactions}; use reth_chainspec::ChainSpec; use reth_e2e_test_utils::setup; -use reth_provider::CanonStateSubscriptions; - -use crate::utils::EthNode; +use reth_node_api::{FullNodeComponents, NodeAddOns}; +use reth_node_builder::{EngineNodeLauncher, FullNode, NodeBuilder, NodeConfig, NodeHandle}; +use reth_node_ethereum::{node::EthereumAddOns, EthereumNode}; +use reth_provider::{providers::BlockchainProvider2, CanonStateSubscriptions}; +use reth_tasks::TaskManager; #[tokio::test] async fn can_run_dev_node() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let (mut nodes, _tasks, _) = setup(1, custom_chain(), true).await?; + let (mut nodes, _tasks, _) = setup::(1, custom_chain(), true).await?; + + assert_chain_advances(nodes.pop().unwrap().inner).await; + Ok(()) +} + +#[tokio::test] +async fn can_run_dev_node_new_engine() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + let tasks = TaskManager::current(); + let exec = tasks.executor(); + + let node_config = NodeConfig::test() + .with_chain(custom_chain()) + .with_dev(DevArgs { dev: true, ..Default::default() }); + let NodeHandle { node, .. } = NodeBuilder::new(node_config.clone()) + .testing_node(exec.clone()) + .with_types_and_provider::>() + .with_components(EthereumNode::components()) + .with_add_ons(EthereumAddOns::default()) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + Default::default(), + ); + builder.launch_with(launcher) + }) + .await?; + + assert_chain_advances(node).await; - assert_chain_advances(nodes.pop().unwrap()).await; Ok(()) } -async fn assert_chain_advances(node: EthNode) { - let mut notifications = node.inner.provider.canonical_state_stream(); +async fn assert_chain_advances(node: FullNode) +where + N: FullNodeComponents, + AddOns: NodeAddOns, +{ + let mut notifications = node.provider.canonical_state_stream(); // submit tx through rpc let raw_tx = hex!("02f876820a28808477359400847735940082520894ab0840c0e43688012c1adb0f5e3fc665188f83d28a029d394a5d630544000080c080a0a044076b7e67b5deecc63f61a8d7913fab86ca365b344b5759d1fe3563b4c39ea019eab979dd000da04dfc72bb0377c092d30fd9e1cab5ae487de49586cc8b0090"); - let eth_api = node.inner.rpc_registry.eth_api(); + let eth_api = node.rpc_registry.eth_api(); let hash = eth_api.send_raw_transaction(raw_tx.into()).await.unwrap(); diff --git a/crates/ethereum/node/tests/e2e/utils.rs b/crates/ethereum/node/tests/e2e/utils.rs index 5a79509991893..6e534f5dc0ed3 100644 --- a/crates/ethereum/node/tests/e2e/utils.rs +++ b/crates/ethereum/node/tests/e2e/utils.rs @@ -1,12 +1,7 @@ use alloy_primitives::{Address, B256}; use reth::rpc::types::engine::PayloadAttributes; -use reth_e2e_test_utils::NodeHelperType; -use reth_node_ethereum::{node::EthereumAddOns, EthereumNode}; use reth_payload_builder::EthPayloadBuilderAttributes; -/// Ethereum Node Helper type -pub(crate) type EthNode = NodeHelperType; - /// Helper function to create a new eth payload attributes pub(crate) fn eth_payload_attributes(timestamp: u64) -> EthPayloadBuilderAttributes { let attributes = PayloadAttributes { diff --git a/crates/node/builder/Cargo.toml b/crates/node/builder/Cargo.toml index 1bf2ba2337398..53e53cd2b8563 100644 --- a/crates/node/builder/Cargo.toml +++ b/crates/node/builder/Cargo.toml @@ -26,6 +26,7 @@ reth-db = { workspace = true, features = ["mdbx"], optional = true } reth-db-api.workspace = true reth-db-common.workspace = true reth-downloaders.workspace = true +reth-engine-local.workspace = true reth-engine-service.workspace = true reth-engine-tree.workspace = true reth-engine-util.workspace = true diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 782cc7bbb1b0a..f9e26f202fc4e 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -9,6 +9,7 @@ use reth_beacon_consensus::{ use reth_blockchain_tree::BlockchainTreeConfig; use reth_chainspec::EthChainSpec; use reth_consensus_debug_client::{DebugConsensusClient, EtherscanBlockProvider}; +use reth_engine_local::{LocalEngineService, LocalPayloadAttributesBuilder, MiningMode}; use reth_engine_service::service::{ChainEvent, EngineService}; use reth_engine_tree::{ engine::{EngineApiRequest, EngineRequestHandler}, @@ -18,7 +19,10 @@ use reth_engine_util::EngineMessageStreamExt; use reth_exex::ExExManagerHandle; use reth_network::{NetworkSyncUpdater, SyncState}; use reth_network_api::{BlockDownloaderProvider, NetworkEventListenerProvider}; -use reth_node_api::{BuiltPayload, FullNodeTypes, NodeAddOns, NodeTypesWithEngine}; +use reth_node_api::{ + BuiltPayload, FullNodeTypes, NodeAddOns, NodeTypesWithEngine, PayloadAttributesBuilder, + PayloadTypes, +}; use reth_node_core::{ dirs::{ChainPath, DataDirPath}, exit::NodeExitFuture, @@ -80,6 +84,9 @@ where + FullEthApiServer + AddDevSigners, >, + LocalPayloadAttributesBuilder: PayloadAttributesBuilder< + <::Engine as PayloadTypes>::PayloadAttributes, + >, { type Node = NodeHandle, AO>; @@ -210,23 +217,49 @@ where let pruner_events = pruner.events(); info!(target: "reth::cli", prune_config=?ctx.prune_config().unwrap_or_default(), "Pruner initialized"); - // Configure the consensus engine - let mut eth_service = EngineService::new( - ctx.consensus(), - ctx.components().block_executor().clone(), - ctx.chain_spec(), - network_client.clone(), - Box::pin(consensus_engine_stream), - pipeline, - Box::new(ctx.task_executor().clone()), - ctx.provider_factory().clone(), - ctx.blockchain_db().clone(), - pruner, - ctx.components().payload_builder().clone(), - engine_tree_config, - ctx.invalid_block_hook()?, - ctx.sync_metrics_tx(), - ); + let mut engine_service = if ctx.is_dev() { + let mining_mode = if let Some(block_time) = ctx.node_config().dev.block_time { + MiningMode::interval(block_time) + } else { + MiningMode::instant(ctx.components().pool().clone()) + }; + let eth_service = LocalEngineService::new( + ctx.consensus(), + ctx.components().block_executor().clone(), + ctx.provider_factory().clone(), + ctx.blockchain_db().clone(), + pruner, + ctx.components().payload_builder().clone(), + engine_tree_config, + ctx.invalid_block_hook()?, + ctx.sync_metrics_tx(), + consensus_engine_tx.clone(), + Box::pin(consensus_engine_stream), + mining_mode, + LocalPayloadAttributesBuilder::new(ctx.chain_spec()), + ); + + Either::Left(eth_service) + } else { + let eth_service = EngineService::new( + ctx.consensus(), + ctx.components().block_executor().clone(), + ctx.chain_spec(), + network_client.clone(), + Box::pin(consensus_engine_stream), + pipeline, + Box::new(ctx.task_executor().clone()), + ctx.provider_factory().clone(), + ctx.blockchain_db().clone(), + pruner, + ctx.components().payload_builder().clone(), + engine_tree_config, + ctx.invalid_block_hook()?, + ctx.sync_metrics_tx(), + ); + + Either::Right(eth_service) + }; let event_sender = EventSender::default(); @@ -340,7 +373,9 @@ where ctx.task_executor().spawn_critical("consensus engine", async move { if let Some(initial_target) = initial_target { debug!(target: "reth::cli", %initial_target, "start backfill sync"); - eth_service.orchestrator_mut().start_backfill_sync(initial_target); + if let Either::Right(eth_service) = &mut engine_service { + eth_service.orchestrator_mut().start_backfill_sync(initial_target); + } } let mut res = Ok(()); @@ -351,10 +386,12 @@ where payload = built_payloads.select_next_some() => { if let Some(executed_block) = payload.executed_block() { debug!(target: "reth::cli", block=?executed_block.block().num_hash(), "inserting built payload"); - eth_service.orchestrator_mut().handler_mut().handler_mut().on_event(EngineApiRequest::InsertExecutedBlock(executed_block).into()); + if let Either::Right(eth_service) = &mut engine_service { + eth_service.orchestrator_mut().handler_mut().handler_mut().on_event(EngineApiRequest::InsertExecutedBlock(executed_block).into()); + } } } - event = eth_service.next() => { + event = engine_service.next() => { let Some(event) = event else { break }; debug!(target: "reth::cli", "Event: {event}"); match event { diff --git a/crates/node/events/src/node.rs b/crates/node/events/src/node.rs index c856c0ec9e187..e10caaee7c53f 100644 --- a/crates/node/events/src/node.rs +++ b/crates/node/events/src/node.rs @@ -504,7 +504,7 @@ where } else if let Some(latest_block) = this.state.latest_block { let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); - if now - this.state.latest_block_time.unwrap_or(0) > 60 { + if now.saturating_sub(this.state.latest_block_time.unwrap_or(0)) > 60 { // Once we start receiving consensus nodes, don't emit status unless stalled for // 1 minute info!( diff --git a/crates/optimism/node/Cargo.toml b/crates/optimism/node/Cargo.toml index 3c298bea9cee0..4675029b64c87 100644 --- a/crates/optimism/node/Cargo.toml +++ b/crates/optimism/node/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] # reth reth-chainspec.workspace = true +reth-engine-local.workspace = true reth-primitives.workspace = true reth-payload-builder.workspace = true reth-auto-seal-consensus.workspace = true @@ -87,6 +88,7 @@ optimism = [ "reth-revm/optimism", "reth-auto-seal-consensus/optimism", "reth-optimism-rpc/optimism", + "reth-engine-local/optimism", ] asm-keccak = ["reth-primitives/asm-keccak"] test-utils = ["reth-node-builder/test-utils"] diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index 6ae6361fdb286..494ed68aa4eef 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -160,12 +160,7 @@ impl PayloadAttributes for OpPayloadAttributes { } /// A builder that can return the current payload attribute. -pub trait PayloadAttributesBuilder: std::fmt::Debug + Send + Sync + 'static { - /// The payload attributes type returned by the builder. - type PayloadAttributes: PayloadAttributes; - /// The error type returned by [`PayloadAttributesBuilder::build`]. - type Error: core::error::Error + Send + Sync; - +pub trait PayloadAttributesBuilder: Send + Sync + 'static { /// Return a new payload attribute from the builder. - fn build(&self) -> Result; + fn build(&self, timestamp: u64) -> Attributes; } From ab35d48ae57b66db17f50484cbe98be1751f7387 Mon Sep 17 00:00:00 2001 From: Delweng Date: Mon, 14 Oct 2024 23:45:26 +0800 Subject: [PATCH 157/159] chore(clippy): enable if_then_some_else_none lint (#11679) Signed-off-by: jsvisa Co-authored-by: Matthias Seitz --- Cargo.toml | 1 + crates/chainspec/src/spec.rs | 7 ++-- crates/consensus/auto-seal/src/lib.rs | 32 +++++++------------ crates/engine/local/src/miner.rs | 10 +++--- crates/engine/local/src/payload.rs | 18 +++++------ crates/ethereum/evm/src/lib.rs | 11 ++----- crates/net/network/src/cache.rs | 2 +- crates/net/p2p/src/test_utils/full_block.rs | 18 +++++------ crates/node/core/src/args/log.rs | 2 +- crates/optimism/evm/src/lib.rs | 11 ++----- .../src/segments/user/receipts_by_logs.rs | 6 +--- .../rpc-eth-api/src/helpers/pending_block.rs | 2 +- crates/storage/db-models/src/accounts.rs | 8 ++--- .../storage/db/src/implementation/mdbx/mod.rs | 7 ++-- .../src/providers/database/provider.rs | 14 ++++---- .../src/providers/static_file/manager.rs | 2 +- .../src/providers/static_file/writer.rs | 20 ++++++------ .../storage/provider/src/test_utils/mock.rs | 15 ++------- crates/transaction-pool/src/maintain.rs | 9 +++--- crates/trie/common/src/subnode.rs | 10 +++--- crates/trie/trie/src/trie_cursor/subnode.rs | 2 +- crates/trie/trie/src/witness.rs | 15 ++++----- 22 files changed, 87 insertions(+), 135 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 628d7d47b2251..ac0ab2d39228f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -186,6 +186,7 @@ explicit_iter_loop = "warn" flat_map_option = "warn" from_iter_instead_of_collect = "warn" if_not_else = "warn" +if_then_some_else_none = "warn" implicit_clone = "warn" imprecise_flops = "warn" iter_on_empty_collections = "warn" diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index f80a20924394c..e43e0bc78d864 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -297,11 +297,8 @@ impl ChainSpec { }; // If Prague is activated at genesis we set requests root to an empty trie root. - let requests_root = if self.is_prague_active_at_timestamp(self.genesis.timestamp) { - Some(EMPTY_ROOT_HASH) - } else { - None - }; + let requests_root = + self.is_prague_active_at_timestamp(self.genesis.timestamp).then_some(EMPTY_ROOT_HASH); Header { gas_limit: self.genesis.gas_limit, diff --git a/crates/consensus/auto-seal/src/lib.rs b/crates/consensus/auto-seal/src/lib.rs index f1ef64c8c0faa..261227f1074ed 100644 --- a/crates/consensus/auto-seal/src/lib.rs +++ b/crates/consensus/auto-seal/src/lib.rs @@ -282,17 +282,13 @@ impl StorageInner { parent.next_block_base_fee(chain_spec.base_fee_params_at_timestamp(timestamp)) }); - let blob_gas_used = if chain_spec.is_cancun_active_at_timestamp(timestamp) { - let mut sum_blob_gas_used = 0; - for tx in transactions { - if let Some(blob_tx) = tx.transaction.as_eip4844() { - sum_blob_gas_used += blob_tx.blob_gas(); - } - } - Some(sum_blob_gas_used) - } else { - None - }; + let blob_gas_used = chain_spec.is_cancun_active_at_timestamp(timestamp).then(|| { + transactions + .iter() + .filter_map(|tx| tx.transaction.as_eip4844()) + .map(|blob_tx| blob_tx.blob_gas()) + .sum::() + }); let mut header = Header { parent_hash: self.best_hash, @@ -304,7 +300,7 @@ impl StorageInner { gas_limit: chain_spec.max_gas_limit(), timestamp, base_fee_per_gas, - blob_gas_used: blob_gas_used.map(Into::into), + blob_gas_used, requests_root: requests.map(|r| proofs::calculate_requests_root(&r.0)), ..Default::default() }; @@ -316,14 +312,10 @@ impl StorageInner { header.blob_gas_used = Some(0); let (parent_excess_blob_gas, parent_blob_gas_used) = match parent { - Some(parent_block) - if chain_spec.is_cancun_active_at_timestamp(parent_block.timestamp) => - { - ( - parent_block.excess_blob_gas.unwrap_or_default(), - parent_block.blob_gas_used.unwrap_or_default(), - ) - } + Some(parent) if chain_spec.is_cancun_active_at_timestamp(parent.timestamp) => ( + parent.excess_blob_gas.unwrap_or_default(), + parent.blob_gas_used.unwrap_or_default(), + ), _ => (0, 0), }; header.excess_blob_gas = diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index e12a2a50d0366..f20d70b14893f 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -212,14 +212,12 @@ where let block = payload.block(); let cancun_fields = - if self.provider.chain_spec().is_cancun_active_at_timestamp(block.timestamp) { - Some(CancunPayloadFields { + self.provider.chain_spec().is_cancun_active_at_timestamp(block.timestamp).then(|| { + CancunPayloadFields { parent_beacon_block_root: block.parent_beacon_block_root.unwrap(), versioned_hashes: block.blob_versioned_hashes().into_iter().copied().collect(), - }) - } else { - None - }; + } + }); let (tx, rx) = oneshot::channel(); self.to_engine.send(BeaconEngineMessage::NewPayload { diff --git a/crates/engine/local/src/payload.rs b/crates/engine/local/src/payload.rs index 15d5ff2cf0084..5111360d5bf6b 100644 --- a/crates/engine/local/src/payload.rs +++ b/crates/engine/local/src/payload.rs @@ -31,16 +31,14 @@ where timestamp, prev_randao: B256::random(), suggested_fee_recipient: Address::random(), - withdrawals: if self.chain_spec.is_shanghai_active_at_timestamp(timestamp) { - Some(Default::default()) - } else { - None - }, - parent_beacon_block_root: if self.chain_spec.is_cancun_active_at_timestamp(timestamp) { - Some(B256::random()) - } else { - None - }, + withdrawals: self + .chain_spec + .is_shanghai_active_at_timestamp(timestamp) + .then(Default::default), + parent_beacon_block_root: self + .chain_spec + .is_cancun_active_at_timestamp(timestamp) + .then(B256::random), } } } diff --git a/crates/ethereum/evm/src/lib.rs b/crates/ethereum/evm/src/lib.rs index a71f26f703e4f..8ea39f93deefe 100644 --- a/crates/ethereum/evm/src/lib.rs +++ b/crates/ethereum/evm/src/lib.rs @@ -134,17 +134,10 @@ impl ConfigureEvmEnv for EthEvmConfig { let spec_id = revm_spec_by_timestamp_after_merge(&self.chain_spec, attributes.timestamp); // if the parent block did not have excess blob gas (i.e. it was pre-cancun), but it is - // cancun now, we need to set the excess blob gas to the default value + // cancun now, we need to set the excess blob gas to the default value(0) let blob_excess_gas_and_price = parent .next_block_excess_blob_gas() - .or_else(|| { - if spec_id == SpecId::CANCUN { - // default excess blob gas is zero - Some(0) - } else { - None - } - }) + .or_else(|| (spec_id == SpecId::CANCUN).then_some(0)) .map(BlobExcessGasAndPrice::new); let mut basefee = parent.next_block_base_fee( diff --git a/crates/net/network/src/cache.rs b/crates/net/network/src/cache.rs index fb2daca666e47..758b491679089 100644 --- a/crates/net/network/src/cache.rs +++ b/crates/net/network/src/cache.rs @@ -42,7 +42,7 @@ impl LruCache { pub fn insert_and_get_evicted(&mut self, entry: T) -> (bool, Option) { let new = self.inner.peek(&entry).is_none(); let evicted = - if new && (self.limit as usize) <= self.inner.len() { self.remove_lru() } else { None }; + (new && (self.limit as usize) <= self.inner.len()).then(|| self.remove_lru()).flatten(); _ = self.inner.get_or_insert(entry, || ()); (new, evicted) diff --git a/crates/net/p2p/src/test_utils/full_block.rs b/crates/net/p2p/src/test_utils/full_block.rs index acc01a60ef804..8a13f69325dc5 100644 --- a/crates/net/p2p/src/test_utils/full_block.rs +++ b/crates/net/p2p/src/test_utils/full_block.rs @@ -185,15 +185,15 @@ impl HeadersClient for TestFullBlockClient { .filter_map(|_| { headers.iter().find_map(|(hash, header)| { // Checks if the header matches the specified block or number. - if BlockNumHash::new(header.number, *hash).matches_block_or_num(&block) { - match request.direction { - HeadersDirection::Falling => block = header.parent_hash.into(), - HeadersDirection::Rising => block = (header.number + 1).into(), - } - Some(header.clone()) - } else { - None - } + BlockNumHash::new(header.number, *hash).matches_block_or_num(&block).then( + || { + match request.direction { + HeadersDirection::Falling => block = header.parent_hash.into(), + HeadersDirection::Rising => block = (header.number + 1).into(), + } + header.clone() + }, + ) }) }) .collect::>(); diff --git a/crates/node/core/src/args/log.rs b/crates/node/core/src/args/log.rs index aa2e0cf5f1a38..3d124fba229f8 100644 --- a/crates/node/core/src/args/log.rs +++ b/crates/node/core/src/args/log.rs @@ -78,7 +78,7 @@ impl LogArgs { format, self.verbosity.directive().to_string(), filter, - if use_color { Some(self.color.to_string()) } else { None }, + use_color.then(|| self.color.to_string()), ) } diff --git a/crates/optimism/evm/src/lib.rs b/crates/optimism/evm/src/lib.rs index b220b6056de0d..158ed2e8929e1 100644 --- a/crates/optimism/evm/src/lib.rs +++ b/crates/optimism/evm/src/lib.rs @@ -139,17 +139,10 @@ impl ConfigureEvmEnv for OptimismEvmConfig { let spec_id = revm_spec_by_timestamp_after_bedrock(&self.chain_spec, attributes.timestamp); // if the parent block did not have excess blob gas (i.e. it was pre-cancun), but it is - // cancun now, we need to set the excess blob gas to the default value + // cancun now, we need to set the excess blob gas to the default value(0) let blob_excess_gas_and_price = parent .next_block_excess_blob_gas() - .or_else(|| { - if spec_id.is_enabled_in(SpecId::CANCUN) { - // default excess blob gas is zero - Some(0) - } else { - None - } - }) + .or_else(|| (spec_id.is_enabled_in(SpecId::CANCUN)).then_some(0)) .map(BlobExcessGasAndPrice::new); let block_env = BlockEnv { diff --git a/crates/prune/prune/src/segments/user/receipts_by_logs.rs b/crates/prune/prune/src/segments/user/receipts_by_logs.rs index 489df7e722736..05bc40b6c7b0b 100644 --- a/crates/prune/prune/src/segments/user/receipts_by_logs.rs +++ b/crates/prune/prune/src/segments/user/receipts_by_logs.rs @@ -267,11 +267,7 @@ mod tests { let mut receipt = random_receipt(&mut rng, transaction, Some(1)); receipt.logs.push(random_log( &mut rng, - if txi == (block.body.transactions.len() - 1) { - Some(deposit_contract_addr) - } else { - None - }, + (txi == (block.body.transactions.len() - 1)).then_some(deposit_contract_addr), Some(1), )); receipts.push((receipts.len() as u64, receipt)); diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 03597289f1517..85ad7fd181076 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -415,7 +415,7 @@ pub trait LoadPendingBlock: EthApiTypes { // check if cancun is activated to set eip4844 header fields correctly let blob_gas_used = - if cfg.handler_cfg.spec_id >= SpecId::CANCUN { Some(sum_blob_gas_used) } else { None }; + (cfg.handler_cfg.spec_id >= SpecId::CANCUN).then_some(sum_blob_gas_used); // note(onbjerg): the rpc spec has not been changed to include requests, so for now we just // set these to empty diff --git a/crates/storage/db-models/src/accounts.rs b/crates/storage/db-models/src/accounts.rs index e1f4773960fac..b0099d22d5fb0 100644 --- a/crates/storage/db-models/src/accounts.rs +++ b/crates/storage/db-models/src/accounts.rs @@ -39,13 +39,11 @@ impl Compact for AccountBeforeTx { let address = Address::from_slice(&buf[..20]); buf.advance(20); - let info = if len - 20 > 0 { + let info = (len - 20 > 0).then(|| { let (acc, advanced_buf) = Account::from_compact(buf, len - 20); buf = advanced_buf; - Some(acc) - } else { - None - }; + acc + }); (Self { address, info }, buf) } diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 1deb86ba614f7..65b804e6a58d3 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -256,10 +256,9 @@ impl DatabaseEnv { args: DatabaseArguments, ) -> Result { let _lock_file = if kind.is_rw() { - Some( - StorageLock::try_acquire(path) - .map_err(|err| DatabaseError::Other(err.to_string()))?, - ) + StorageLock::try_acquire(path) + .map_err(|err| DatabaseError::Other(err.to_string()))? + .into() } else { None }; diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 33fed1e80ce1e..3b074a5afffa9 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -542,18 +542,18 @@ impl DatabaseProvider { // even if empty let withdrawals = if self.chain_spec.is_shanghai_active_at_timestamp(header_ref.timestamp) { - Some( - withdrawals_cursor - .seek_exact(header_ref.number)? - .map(|(_, w)| w.withdrawals) - .unwrap_or_default(), - ) + withdrawals_cursor + .seek_exact(header_ref.number)? + .map(|(_, w)| w.withdrawals) + .unwrap_or_default() + .into() } else { None }; let requests = if self.chain_spec.is_prague_active_at_timestamp(header_ref.timestamp) { - Some(requests_cursor.seek_exact(header_ref.number)?.unwrap_or_default().1) + (requests_cursor.seek_exact(header_ref.number)?.unwrap_or_default().1) + .into() } else { None }; diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index ec76e9504d194..e233332a0e951 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -222,7 +222,7 @@ impl StaticFileProviderInner { /// Creates a new [`StaticFileProviderInner`]. fn new(path: impl AsRef, access: StaticFileAccess) -> ProviderResult { let _lock_file = if access.is_read_write() { - Some(StorageLock::try_acquire(path.as_ref())?) + StorageLock::try_acquire(path.as_ref())?.into() } else { None }; diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index 3858f1b140233..8c31c021f218b 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -289,16 +289,16 @@ impl StaticFileProviderRW { // // If that expected block start is 0, then it means that there's no actual block data, and // there's no block data in static files. - let segment_max_block = match self.writer.user_header().block_range() { - Some(block_range) => Some(block_range.end()), - None => { - if self.writer.user_header().expected_block_start() > 0 { - Some(self.writer.user_header().expected_block_start() - 1) - } else { - None - } - } - }; + let segment_max_block = self + .writer + .user_header() + .block_range() + .as_ref() + .map(|block_range| block_range.end()) + .or_else(|| { + (self.writer.user_header().expected_block_start() > 0) + .then(|| self.writer.user_header().expected_block_start() - 1) + }); self.reader().update_index(self.writer.user_header().segment(), segment_max_block) } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 3325d3ae9edba..c7c94b939ac39 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -344,13 +344,8 @@ impl TransactionsProvider for MockEthProvider { .values() .flat_map(|block| &block.body.transactions) .enumerate() - .filter_map(|(tx_number, tx)| { - if range.contains(&(tx_number as TxNumber)) { - Some(tx.clone().into()) - } else { - None - } - }) + .filter(|&(tx_number, _)| range.contains(&(tx_number as TxNumber))) + .map(|(_, tx)| tx.clone().into()) .collect(); Ok(transactions) @@ -366,11 +361,7 @@ impl TransactionsProvider for MockEthProvider { .flat_map(|block| &block.body.transactions) .enumerate() .filter_map(|(tx_number, tx)| { - if range.contains(&(tx_number as TxNumber)) { - Some(tx.recover_signer()?) - } else { - None - } + range.contains(&(tx_number as TxNumber)).then(|| tx.recover_signer()).flatten() }) .collect(); diff --git a/crates/transaction-pool/src/maintain.rs b/crates/transaction-pool/src/maintain.rs index aaf2d6d12d85f..23a8d0dc66a58 100644 --- a/crates/transaction-pool/src/maintain.rs +++ b/crates/transaction-pool/src/maintain.rs @@ -455,11 +455,10 @@ impl FinalizedBlockTracker { /// Updates the tracked finalized block and returns the new finalized block if it changed fn update(&mut self, finalized_block: Option) -> Option { let finalized = finalized_block?; - if self.last_finalized_block.replace(finalized).map_or(true, |last| last < finalized) { - Some(finalized) - } else { - None - } + self.last_finalized_block + .replace(finalized) + .map_or(true, |last| last < finalized) + .then_some(finalized) } } diff --git a/crates/trie/common/src/subnode.rs b/crates/trie/common/src/subnode.rs index 98ce76a323e63..c64b2317cf309 100644 --- a/crates/trie/common/src/subnode.rs +++ b/crates/trie/common/src/subnode.rs @@ -51,16 +51,14 @@ impl Compact for StoredSubNode { buf.advance(key_len); let nibbles_exists = buf.get_u8() != 0; - let nibble = if nibbles_exists { Some(buf.get_u8()) } else { None }; + let nibble = nibbles_exists.then(|| buf.get_u8()); let node_exists = buf.get_u8() != 0; - let node = if node_exists { + let node = node_exists.then(|| { let (node, rest) = BranchNodeCompact::from_compact(buf, 0); buf = rest; - Some(node) - } else { - None - }; + node + }); (Self { key, nibble, node }, buf) } diff --git a/crates/trie/trie/src/trie_cursor/subnode.rs b/crates/trie/trie/src/trie_cursor/subnode.rs index c2ba839ebf2e1..9d5a2770b2682 100644 --- a/crates/trie/trie/src/trie_cursor/subnode.rs +++ b/crates/trie/trie/src/trie_cursor/subnode.rs @@ -49,7 +49,7 @@ impl From for CursorSubNode { impl From for StoredSubNode { fn from(value: CursorSubNode) -> Self { - let nibble = if value.nibble >= 0 { Some(value.nibble as u8) } else { None }; + let nibble = (value.nibble >= 0).then_some(value.nibble as u8); Self { key: value.key.to_vec(), nibble, node: value.node } } } diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index c042a0d8213b3..582319c7fc51e 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -111,14 +111,13 @@ where .accounts .get(&hashed_address) .ok_or(TrieWitnessError::MissingAccount(hashed_address))?; - let value = if account.is_some() || storage_multiproof.root != EMPTY_ROOT_HASH { - account_rlp.clear(); - TrieAccount::from((account.unwrap_or_default(), storage_multiproof.root)) - .encode(&mut account_rlp as &mut dyn BufMut); - Some(account_rlp.clone()) - } else { - None - }; + let value = + (account.is_some() || storage_multiproof.root != EMPTY_ROOT_HASH).then(|| { + account_rlp.clear(); + TrieAccount::from((account.unwrap_or_default(), storage_multiproof.root)) + .encode(&mut account_rlp as &mut dyn BufMut); + account_rlp.clone() + }); let key = Nibbles::unpack(hashed_address); account_trie_nodes.extend( self.target_nodes( From 2dc5d873647754665b7c18d9b1ecd97471501686 Mon Sep 17 00:00:00 2001 From: Federico Gimenez Date: Mon, 14 Oct 2024 17:52:34 +0200 Subject: [PATCH 158/159] feat: update SystemCaller (#11718) --- .../engine/invalid-block-hooks/src/witness.rs | 3 ++- crates/engine/util/src/reorg.rs | 2 +- crates/ethereum/evm/src/execute.rs | 12 +++++---- crates/ethereum/payload/src/lib.rs | 2 +- crates/evm/src/either.rs | 2 +- crates/evm/src/execute.rs | 2 +- crates/evm/src/system_calls/mod.rs | 26 +++++++++---------- crates/optimism/evm/src/execute.rs | 12 +++++---- crates/optimism/payload/src/builder.rs | 2 +- .../rpc-eth-api/src/helpers/pending_block.rs | 3 +-- crates/rpc/rpc-eth-api/src/helpers/trace.rs | 4 +-- 11 files changed, 36 insertions(+), 34 deletions(-) diff --git a/crates/engine/invalid-block-hooks/src/witness.rs b/crates/engine/invalid-block-hooks/src/witness.rs index 51978311faade..bb227e304198b 100644 --- a/crates/engine/invalid-block-hooks/src/witness.rs +++ b/crates/engine/invalid-block-hooks/src/witness.rs @@ -84,7 +84,8 @@ where EnvWithHandlerCfg::new_with_cfg_env(cfg, block_env, Default::default()), ); - let mut system_caller = SystemCaller::new(&self.evm_config, self.provider.chain_spec()); + let mut system_caller = + SystemCaller::new(self.evm_config.clone(), self.provider.chain_spec()); // Apply pre-block system contract calls. system_caller.apply_pre_execution_changes(&block.clone().unseal(), &mut evm)?; diff --git a/crates/engine/util/src/reorg.rs b/crates/engine/util/src/reorg.rs index bd7b1b95642aa..611095101ff3c 100644 --- a/crates/engine/util/src/reorg.rs +++ b/crates/engine/util/src/reorg.rs @@ -286,7 +286,7 @@ where let mut evm = evm_config.evm_with_env(&mut state, env); // apply eip-4788 pre block contract call - let mut system_caller = SystemCaller::new(evm_config, chain_spec); + let mut system_caller = SystemCaller::new(evm_config.clone(), chain_spec); system_caller.apply_beacon_root_contract_call( reorg_target.timestamp, diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs index 6513cf75fad55..108e1f87c455e 100644 --- a/crates/ethereum/evm/src/execute.rs +++ b/crates/ethereum/evm/src/execute.rs @@ -141,10 +141,12 @@ where where DB: Database, DB::Error: Into + Display, - F: OnStateHook, + F: OnStateHook + 'static, { - let mut system_caller = - SystemCaller::new(&self.evm_config, &self.chain_spec).with_state_hook(state_hook); + let mut system_caller = SystemCaller::new(self.evm_config.clone(), &self.chain_spec); + if let Some(hook) = state_hook { + system_caller.with_state_hook(Some(Box::new(hook) as Box)); + } system_caller.apply_pre_execution_changes(block, &mut evm)?; @@ -290,7 +292,7 @@ where state_hook: Option, ) -> Result where - F: OnStateHook, + F: OnStateHook + 'static, { // 1. prepare state on new block self.on_new_block(&block.header); @@ -396,7 +398,7 @@ where state_hook: F, ) -> Result where - F: OnStateHook, + F: OnStateHook + 'static, { let BlockExecutionInput { block, total_difficulty } = input; let EthExecuteOutput { receipts, requests, gas_used } = self diff --git a/crates/ethereum/payload/src/lib.rs b/crates/ethereum/payload/src/lib.rs index 97237efa8b60c..248aa3486de45 100644 --- a/crates/ethereum/payload/src/lib.rs +++ b/crates/ethereum/payload/src/lib.rs @@ -160,7 +160,7 @@ where let block_number = initialized_block_env.number.to::(); - let mut system_caller = SystemCaller::new(&evm_config, chain_spec.clone()); + let mut system_caller = SystemCaller::new(evm_config.clone(), chain_spec.clone()); // apply eip-4788 pre block contract call system_caller diff --git a/crates/evm/src/either.rs b/crates/evm/src/either.rs index 8022c68c43dcc..82f84301f03a5 100644 --- a/crates/evm/src/either.rs +++ b/crates/evm/src/either.rs @@ -97,7 +97,7 @@ where state_hook: F, ) -> Result where - F: OnStateHook, + F: OnStateHook + 'static, { match self { Self::Left(a) => a.execute_with_state_hook(input, state_hook), diff --git a/crates/evm/src/execute.rs b/crates/evm/src/execute.rs index 145eca29c92bc..476c695e7ac6b 100644 --- a/crates/evm/src/execute.rs +++ b/crates/evm/src/execute.rs @@ -56,7 +56,7 @@ pub trait Executor { state_hook: F, ) -> Result where - F: OnStateHook; + F: OnStateHook + 'static; } /// A general purpose executor that can execute multiple inputs in sequence, validate the outputs, diff --git a/crates/evm/src/system_calls/mod.rs b/crates/evm/src/system_calls/mod.rs index 43baa1c766c29..5dc3f35bd3a27 100644 --- a/crates/evm/src/system_calls/mod.rs +++ b/crates/evm/src/system_calls/mod.rs @@ -1,7 +1,7 @@ //! System contract call functions. use crate::ConfigureEvm; -use alloc::vec::Vec; +use alloc::{boxed::Box, vec::Vec}; use core::fmt::Display; use reth_chainspec::EthereumHardforks; use reth_execution_errors::BlockExecutionError; @@ -42,27 +42,26 @@ impl OnStateHook for NoopHook { /// /// This can be used to chain system transaction calls. #[allow(missing_debug_implementations)] -pub struct SystemCaller<'a, EvmConfig, Chainspec, Hook = NoopHook> { - evm_config: &'a EvmConfig, +pub struct SystemCaller { + evm_config: EvmConfig, chain_spec: Chainspec, /// Optional hook to be called after each state change. - hook: Option, + hook: Option>, } -impl<'a, EvmConfig, Chainspec> SystemCaller<'a, EvmConfig, Chainspec, NoopHook> { +impl SystemCaller { /// Create a new system caller with the given EVM config, database, and chain spec, and creates /// the EVM with the given initialized config and block environment. - pub const fn new(evm_config: &'a EvmConfig, chain_spec: Chainspec) -> Self { + pub const fn new(evm_config: EvmConfig, chain_spec: Chainspec) -> Self { Self { evm_config, chain_spec, hook: None } } + /// Installs a custom hook to be called after each state change. - pub fn with_state_hook( - self, - hook: Option, - ) -> SystemCaller<'a, EvmConfig, Chainspec, H> { - let Self { evm_config, chain_spec, .. } = self; - SystemCaller { evm_config, chain_spec, hook } + pub fn with_state_hook(&mut self, hook: Option>) -> &mut Self { + self.hook = hook; + self } + /// Convenience method to consume the type and drop borrowed fields pub fn finish(self) {} } @@ -85,11 +84,10 @@ where .build() } -impl SystemCaller<'_, EvmConfig, Chainspec, Hook> +impl SystemCaller where EvmConfig: ConfigureEvm
, Chainspec: EthereumHardforks, - Hook: OnStateHook, { /// Apply pre execution changes. pub fn apply_pre_execution_changes( diff --git a/crates/optimism/evm/src/execute.rs b/crates/optimism/evm/src/execute.rs index f4abb8c887e24..ee0d028e4251e 100644 --- a/crates/optimism/evm/src/execute.rs +++ b/crates/optimism/evm/src/execute.rs @@ -121,10 +121,12 @@ where ) -> Result<(Vec, u64), BlockExecutionError> where DB: Database + Display>, - F: OnStateHook, + F: OnStateHook + 'static, { - let mut system_caller = - SystemCaller::new(&self.evm_config, &self.chain_spec).with_state_hook(state_hook); + let mut system_caller = SystemCaller::new(self.evm_config.clone(), &self.chain_spec); + if let Some(hook) = state_hook { + system_caller.with_state_hook(Some(Box::new(hook) as Box)); + } // apply pre execution changes system_caller.apply_beacon_root_contract_call( @@ -306,7 +308,7 @@ where state_hook: Option, ) -> Result<(Vec, u64), BlockExecutionError> where - F: OnStateHook, + F: OnStateHook + 'static, { // 1. prepare state on new block self.on_new_block(&block.header); @@ -410,7 +412,7 @@ where state_hook: F, ) -> Result where - F: OnStateHook, + F: OnStateHook + 'static, { let BlockExecutionInput { block, total_difficulty } = input; let (receipts, gas_used) = self.execute_without_verification_with_state_hook( diff --git a/crates/optimism/payload/src/builder.rs b/crates/optimism/payload/src/builder.rs index 2ff3d33420378..0a8dcdb124492 100644 --- a/crates/optimism/payload/src/builder.rs +++ b/crates/optimism/payload/src/builder.rs @@ -201,7 +201,7 @@ where ); // apply eip-4788 pre block contract call - let mut system_caller = SystemCaller::new(&evm_config, &chain_spec); + let mut system_caller = SystemCaller::new(evm_config.clone(), &chain_spec); system_caller .pre_block_beacon_root_contract_call( diff --git a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs index 85ad7fd181076..26e0ffc7412a7 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/pending_block.rs @@ -260,8 +260,7 @@ pub trait LoadPendingBlock: EthApiTypes { let chain_spec = self.provider().chain_spec(); - let evm_config = self.evm_config().clone(); - let mut system_caller = SystemCaller::new(&evm_config, chain_spec.clone()); + let mut system_caller = SystemCaller::new(self.evm_config().clone(), chain_spec.clone()); let parent_beacon_block_root = if origin.is_actual_pending() { // apply eip-4788 pre block contract call if we got the block from the CL with the real diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index 2dc1c5b4746da..457cbb4811f3b 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -199,7 +199,7 @@ pub trait Trace: LoadState { // apply relevant system calls let mut system_caller = SystemCaller::new( - Trace::evm_config(&this), + Trace::evm_config(&this).clone(), LoadState::provider(&this).chain_spec(), ); system_caller @@ -332,7 +332,7 @@ pub trait Trace: LoadState { // apply relevant system calls let mut system_caller = SystemCaller::new( - Trace::evm_config(&this), + Trace::evm_config(&this).clone(), LoadState::provider(&this).chain_spec(), ); system_caller From 8b6ab9630136cac85811aa6fce967ad5d0ef63f7 Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 15 Oct 2024 01:24:20 +0800 Subject: [PATCH 159/159] fix(rpc/trace): return empty if after >= traces (#11715) Signed-off-by: jsvisa Co-authored-by: Matthias Seitz --- crates/rpc/rpc/src/trace.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 05588d4dabea3..0cd94ef15b893 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -322,13 +322,18 @@ where } } - // apply after and count to traces if specified, this allows for a pagination style. - // only consider traces after - if let Some(after) = after.map(|a| a as usize).filter(|a| *a < all_traces.len()) { - all_traces = all_traces.split_off(after); + // Skips the first `after` number of matching traces. + // If `after` is greater than or equal to the number of matched traces, it returns an empty + // array. + if let Some(after) = after.map(|a| a as usize) { + if after < all_traces.len() { + all_traces.drain(..after); + } else { + return Ok(vec![]) + } } - // at most, return count of traces + // Return at most `count` of traces if let Some(count) = count { let count = count as usize; if count < all_traces.len() {