merge queue: embarking unstable (c7e5dd1) and [#5103 + #5135 + #5120 + #5065 + #5018] together

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -4070,7 +4070,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .task_executor
             .spawn_blocking_handle(
                 move || chain.load_state_for_block_production(slot),
-                "produce_partial_beacon_block",
+                "load_state_for_block_production",
             )
             .ok_or(BlockProductionError::ShuttingDown)?
             .await

beacon_node/beacon_chain/src/data_availability_checker.rs

@@ -545,6 +545,18 @@ pub struct AvailableBlock<E: EthSpec> {
 }
 
 impl<E: EthSpec> AvailableBlock<E> {
+    pub fn __new_for_testing(
+        block_root: Hash256,
+        block: Arc<SignedBeaconBlock<E>>,
+        blobs: Option<BlobSidecarList<E>>,
+    ) -> Self {
+        Self {
+            block_root,
+            block,
+            blobs,
+        }
+    }
+
     pub fn block(&self) -> &SignedBeaconBlock<E> {
         &self.block
     }

beacon_node/beacon_chain/src/historical_blocks.rs

@@ -135,20 +135,20 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
             prev_block_slot = block.slot();
             expected_block_root = block.message().parent_root();
+            signed_blocks.push(block);
 
             // If we've reached genesis, add the genesis block root to the batch for all slots
             // between 0 and the first block slot, and set the anchor slot to 0 to indicate
             // completion.
             if expected_block_root == self.genesis_block_root {
                 let genesis_slot = self.spec.genesis_slot;
-                for slot in genesis_slot.as_usize()..block.slot().as_usize() {
+                for slot in genesis_slot.as_usize()..prev_block_slot.as_usize() {
                     chunk_writer.set(slot, self.genesis_block_root, &mut cold_batch)?;
                 }
                 prev_block_slot = genesis_slot;
                 expected_block_root = Hash256::zero();
                 break;
             }
-            signed_blocks.push(block);
         }
         chunk_writer.write(&mut cold_batch)?;
         // these were pushed in reverse order so we reverse again

beacon_node/beacon_chain/src/observed_aggregates.rs

@@ -43,7 +43,7 @@ impl<T: EthSpec> Consts for Attestation<T> {
 
     /// We need to keep attestations for each slot of the current epoch.
     fn max_slot_capacity() -> usize {
-        T::slots_per_epoch() as usize
+        2 * T::slots_per_epoch() as usize
     }
 
     /// As a DoS protection measure, the maximum number of distinct `Attestations` or

beacon_node/beacon_chain/src/observed_attesters.rs

@@ -24,18 +24,16 @@ use types::{Epoch, EthSpec, Hash256, Slot, Unsigned};
 
 /// The maximum capacity of the `AutoPruningEpochContainer`.
 ///
-/// Fits the next, current and previous epochs. We require the next epoch due to the
-/// `MAXIMUM_GOSSIP_CLOCK_DISPARITY`. We require the previous epoch since the specification
-/// declares:
+/// If the current epoch is N, this fits epoch N + 1, N, N - 1, and N - 2. We require the next epoch due
+/// to the `MAXIMUM_GOSSIP_CLOCK_DISPARITY`. We require the N - 2 epoch since the specification declares:
 ///
 /// ```ignore
-/// aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE
-///      >= current_slot >= aggregate.data.slot
+/// the epoch of `aggregate.data.slot` is either the current or previous epoch
 /// ```
 ///
-/// This means that during the current epoch we will always accept an attestation
-/// from at least one slot in the previous epoch.
-pub const MAX_CACHED_EPOCHS: u64 = 3;
+/// This means that during the current epoch we will always accept an attestation from
+/// at least one slot in the epoch prior to the previous epoch.
+pub const MAX_CACHED_EPOCHS: u64 = 4;
 
 pub type ObservedAttesters<E> = AutoPruningEpochContainer<EpochBitfield, E>;
 pub type ObservedSyncContributors<E> =

beacon_node/beacon_chain/tests/store_tests.rs

@@ -3,6 +3,7 @@
 use beacon_chain::attestation_verification::Error as AttnError;
 use beacon_chain::block_verification_types::RpcBlock;
 use beacon_chain::builder::BeaconChainBuilder;
+use beacon_chain::data_availability_checker::AvailableBlock;
 use beacon_chain::schema_change::migrate_schema;
 use beacon_chain::test_utils::{
     mock_execution_layer_from_parts, test_spec, AttestationStrategy, BeaconChainHarness,
@@ -2547,6 +2548,25 @@ async fn weak_subjectivity_sync_test(slots: Vec<Slot>, checkpoint_slot: Slot) {
         }
     }
 
+    // Corrupt the signature on the 1st block to ensure that the backfill processor is checking
+    // signatures correctly. Regression test for https://github.com/sigp/lighthouse/pull/5120.
+    let mut batch_with_invalid_first_block = available_blocks.clone();
+    batch_with_invalid_first_block[0] = {
+        let (block_root, block, blobs) = available_blocks[0].clone().deconstruct();
+        let mut corrupt_block = (*block).clone();
+        *corrupt_block.signature_mut() = Signature::empty();
+        AvailableBlock::__new_for_testing(block_root, Arc::new(corrupt_block), blobs)
+    };
+
+    // Importing the invalid batch should error.
+    assert!(matches!(
+        beacon_chain
+            .import_historical_block_batch(batch_with_invalid_first_block)
+            .unwrap_err(),
+        BeaconChainError::HistoricalBlockError(HistoricalBlockError::InvalidSignature)
+    ));
+
+    // Importing the batch with valid signatures should succeed.
     beacon_chain
         .import_historical_block_batch(available_blocks.clone())
         .unwrap();

beacon_node/http_api/src/lib.rs

@@ -4267,36 +4267,6 @@ pub fn serve<T: BeaconChainTypes>(
             },
         );
 
-    // GET lighthouse/beacon/states/{state_id}/ssz
-    let get_lighthouse_beacon_states_ssz = warp::path("lighthouse")
-        .and(warp::path("beacon"))
-        .and(warp::path("states"))
-        .and(warp::path::param::<StateId>())
-        .and(warp::path("ssz"))
-        .and(warp::path::end())
-        .and(task_spawner_filter.clone())
-        .and(chain_filter.clone())
-        .then(
-            |state_id: StateId,
-             task_spawner: TaskSpawner<T::EthSpec>,
-             chain: Arc<BeaconChain<T>>| {
-                task_spawner.blocking_response_task(Priority::P1, move || {
-                    // This debug endpoint provides no indication of optimistic status.
-                    let (state, _execution_optimistic, _finalized) = state_id.state(&chain)?;
-                    Response::builder()
-                        .status(200)
-                        .body(state.as_ssz_bytes().into())
-                        .map(|res: Response<Body>| add_ssz_content_type_header(res))
-                        .map_err(|e| {
-                            warp_utils::reject::custom_server_error(format!(
-                                "failed to create response: {}",
-                                e
-                            ))
-                        })
-                })
-            },
-        );
-
     // GET lighthouse/staking
     let get_lighthouse_staking = warp::path("lighthouse")
         .and(warp::path("staking"))
@@ -4631,7 +4601,6 @@ pub fn serve<T: BeaconChainTypes>(
                 .uor(get_lighthouse_eth1_syncing)
                 .uor(get_lighthouse_eth1_block_cache)
                 .uor(get_lighthouse_eth1_deposit_cache)
-                .uor(get_lighthouse_beacon_states_ssz)
                 .uor(get_lighthouse_staking)
                 .uor(get_lighthouse_database_info)
                 .uor(get_lighthouse_block_rewards)

beacon_node/http_api/tests/tests.rs

@@ -5057,26 +5057,6 @@ impl ApiTester {
         self
     }
 
-    pub async fn test_get_lighthouse_beacon_states_ssz(self) -> Self {
-        for state_id in self.interesting_state_ids() {
-            let result = self
-                .client
-                .get_lighthouse_beacon_states_ssz(&state_id.0, &self.chain.spec)
-                .await
-                .unwrap();
-
-            let mut expected = state_id
-                .state(&self.chain)
-                .ok()
-                .map(|(state, _execution_optimistic, _finalized)| state);
-            expected.as_mut().map(|state| state.drop_all_caches());
-
-            assert_eq!(result, expected, "{:?}", state_id);
-        }
-
-        self
-    }
-
     pub async fn test_get_lighthouse_staking(self) -> Self {
         let result = self.client.get_lighthouse_staking().await.unwrap();
 
@@ -6373,8 +6353,6 @@ async fn lighthouse_endpoints() {
         .await
         .test_get_lighthouse_eth1_deposit_cache()
         .await
-        .test_get_lighthouse_beacon_states_ssz()
-        .await
         .test_get_lighthouse_staking()
         .await
         .test_get_lighthouse_database_info()

book/src/contributing.md

@@ -49,7 +49,7 @@ into the canonical spec.
 ## Rust
 
 Lighthouse adheres to Rust code conventions as outlined in the [**Rust
-Styleguide**](https://github.com/rust-dev-tools/fmt-rfcs/blob/master/guide/guide.md).
+Styleguide**](https://doc.rust-lang.org/nightly/style-guide/).
 
 Please use [clippy](https://github.com/rust-lang/rust-clippy) and
 [rustfmt](https://github.com/rust-lang/rustfmt) to detect common mistakes and

book/src/merge-migration.md

@@ -207,6 +207,6 @@ guidance for specific setups.
 
 - [Ethereum.org: The Merge](https://ethereum.org/en/upgrades/merge/)
 - [Ethereum Staking Launchpad: Merge Readiness](https://launchpad.ethereum.org/en/merge-readiness).
-- [CoinCashew: Ethereum Merge Upgrade Checklist](https://www.coincashew.com/coins/overview-eth/ethereum-merge-upgrade-checklist-for-home-stakers-and-validators)
+- [CoinCashew: Ethereum Merge Upgrade Checklist](https://www.coincashew.com/coins/overview-eth/archived-guides/ethereum-merge-upgrade-checklist-for-home-stakers-and-validators)
 - [EthDocker: Merge Preparation](https://eth-docker.net/About/MergePrep/)
 - [Remy Roy: How to join the Goerli/Prater merge testnet](https://github.com/remyroy/ethstaker/blob/main/merge-goerli-prater.md)

common/eth2/src/lighthouse.rs

@@ -8,15 +8,12 @@ mod standard_block_rewards;
 mod sync_committee_rewards;
 
 use crate::{
-    ok_or_error,
     types::{
-        BeaconState, ChainSpec, DepositTreeSnapshot, Epoch, EthSpec, FinalizedExecutionBlock,
-        GenericResponse, ValidatorId,
+        DepositTreeSnapshot, Epoch, EthSpec, FinalizedExecutionBlock, GenericResponse, ValidatorId,
     },
-    BeaconNodeHttpClient, DepositData, Error, Eth1Data, Hash256, Slot, StateId, StatusCode,
+    BeaconNodeHttpClient, DepositData, Error, Eth1Data, Hash256, Slot,
 };
 use proto_array::core::ProtoArray;
-use reqwest::IntoUrl;
 use serde::{Deserialize, Serialize};
 use ssz::four_byte_option_impl;
 use ssz_derive::{Decode, Encode};
@@ -371,27 +368,6 @@ pub struct DatabaseInfo {
 }
 
 impl BeaconNodeHttpClient {
-    /// Perform a HTTP GET request, returning `None` on a 404 error.
-    async fn get_bytes_opt<U: IntoUrl>(&self, url: U) -> Result<Option<Vec<u8>>, Error> {
-        let response = self.client.get(url).send().await.map_err(Error::from)?;
-        match ok_or_error(response).await {
-            Ok(resp) => Ok(Some(
-                resp.bytes()
-                    .await
-                    .map_err(Error::from)?
-                    .into_iter()
-                    .collect::<Vec<_>>(),
-            )),
-            Err(err) => {
-                if err.status() == Some(StatusCode::NOT_FOUND) {
-                    Ok(None)
-                } else {
-                    Err(err)
-                }
-            }
-        }
-    }
-
     /// `GET lighthouse/health`
     pub async fn get_lighthouse_health(&self) -> Result<GenericResponse<Health>, Error> {
         let mut path = self.server.full.clone();
@@ -516,28 +492,6 @@ impl BeaconNodeHttpClient {
         self.get(path).await
     }
 
-    /// `GET lighthouse/beacon/states/{state_id}/ssz`
-    pub async fn get_lighthouse_beacon_states_ssz<E: EthSpec>(
-        &self,
-        state_id: &StateId,
-        spec: &ChainSpec,
-    ) -> Result<Option<BeaconState<E>>, Error> {
-        let mut path = self.server.full.clone();
-
-        path.path_segments_mut()
-            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
-            .push("lighthouse")
-            .push("beacon")
-            .push("states")
-            .push(&state_id.to_string())
-            .push("ssz");
-
-        self.get_bytes_opt(path)
-            .await?
-            .map(|bytes| BeaconState::from_ssz_bytes(&bytes, spec).map_err(Error::InvalidSsz))
-            .transpose()
-    }
-
     /// `GET lighthouse/staking`
     pub async fn get_lighthouse_staking(&self) -> Result<bool, Error> {
         let mut path = self.server.full.clone();