Update to spec v0.9.1 (#597)

* Update to spec v0.9.0

* Update to v0.9.1

* Bump spec tags for v0.9.1

* Formatting, fix CI failures

* Resolve accidental KeyPair merge conflict

* Document new BeaconState functions

* Fix incorrect cache drops in `advance_caches`

* Update fork choice for v0.9.1

* Clean up some FIXMEs

* Fix a few docs/logs

.gitlab-ci.yml

@@ -4,7 +4,7 @@ default:
   image: 'sigp/lighthouse:latest'
   cache:
     paths:
-      - tests/ef_tests/*-v0.8.3.tar.gz
+      - tests/ef_tests/*-v0.9.1.tar.gz
 
 stages:
   - test
@@ -47,7 +47,7 @@ test-ef-fake-crypto:
     GIT_SUBMODULE_STRATEGY: normal
   script:
     - make make-ef-tests
-    - cargo test --manifest-path tests/ef_tests/Cargo.toml --release --features ef_tests fake_crypto
+    - cargo test --manifest-path tests/ef_tests/Cargo.toml --release --features ef_tests,fake_crypto
 
 documentation:
   stage: document

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -16,7 +16,7 @@ use ssz::Encode;
 use state_processing::per_block_processing::{
     errors::{
         AttestationValidationError, AttesterSlashingValidationError, DepositValidationError,
-        ExitValidationError, ProposerSlashingValidationError, TransferValidationError,
+        ExitValidationError, ProposerSlashingValidationError,
     },
     verify_attestation_for_state, VerifySignatures,
 };
@@ -218,6 +218,44 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         ReverseBlockRootIterator::new((head.beacon_block_root, head.beacon_block.slot), iter)
     }
 
+    /// Traverse backwards from `block_root` to find the block roots of its ancestors.
+    ///
+    /// ## Notes
+    ///
+    /// `slot` always decreases by `1`.
+    /// - Skipped slots contain the root of the closest prior
+    ///     non-skipped slot (identical to the way they are stored in `state.block_roots`) .
+    /// - Iterator returns `(Hash256, Slot)`.
+    /// - The provided `block_root` is included as the first item in the iterator.
+    pub fn rev_iter_block_roots_from(
+        &self,
+        block_root: Hash256,
+    ) -> Result<ReverseBlockRootIterator<T::EthSpec, T::Store>, Error> {
+        let block = self
+            .get_block(&block_root)?
+            .ok_or_else(|| Error::MissingBeaconBlock(block_root))?;
+        let state = self
+            .get_state(&block.state_root)?
+            .ok_or_else(|| Error::MissingBeaconState(block.state_root))?;
+        let iter = BlockRootsIterator::owned(self.store.clone(), state);
+        Ok(ReverseBlockRootIterator::new(
+            (block_root, block.slot),
+            iter,
+        ))
+    }
+
+    /// Traverse backwards from `block_root` to find the root of the ancestor block at `slot`.
+    pub fn get_ancestor_block_root(
+        &self,
+        block_root: Hash256,
+        slot: Slot,
+    ) -> Result<Option<Hash256>, Error> {
+        Ok(self
+            .rev_iter_block_roots_from(block_root)?
+            .find(|(_, ancestor_slot)| *ancestor_slot == slot)
+            .map(|(ancestor_block_root, _)| ancestor_block_root))
+    }
+
     /// Iterates across all `(state_root, slot)` pairs from the head of the chain (inclusive) to
     /// the earliest reachable ancestor (may or may not be genesis).
     ///
@@ -248,6 +286,18 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         Ok(self.store.get(block_root)?)
     }
 
+    /// Returns the state at the given root, if any.
+    ///
+    /// ## Errors
+    ///
+    /// May return a database error.
+    pub fn get_state(
+        &self,
+        state_root: &Hash256,
+    ) -> Result<Option<BeaconState<T::EthSpec>>, Error> {
+        Ok(self.store.get(state_root)?)
+    }
+
     /// Returns a `Checkpoint` representing the head block and state. Contains the "best block";
     /// the head of the canonical `BeaconChain`.
     ///
@@ -385,15 +435,15 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         }
 
         state
-            .get_beacon_proposer_index(slot, RelativeEpoch::Current, &self.spec)
+            .get_beacon_proposer_index(slot, &self.spec)
             .map_err(Into::into)
     }
 
-    /// Returns the attestation slot and shard for a given validator index.
+    /// Returns the attestation slot and committee index for a given validator index.
     ///
     /// Information is read from the current state, so only information from the present and prior
     /// epoch is available.
-    pub fn validator_attestation_slot_and_shard(
+    pub fn validator_attestation_slot_and_index(
         &self,
         validator_index: usize,
         epoch: Epoch,
@@ -420,25 +470,25 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         if let Some(attestation_duty) =
             state.get_attestation_duties(validator_index, RelativeEpoch::Current)?
         {
-            Ok(Some((attestation_duty.slot, attestation_duty.shard)))
+            Ok(Some((attestation_duty.slot, attestation_duty.index)))
         } else {
             Ok(None)
         }
     }
 
-    /// Produce an `AttestationData` that is valid for the given `slot` `shard`.
+    /// Produce an `AttestationData` that is valid for the given `slot`, `index`.
     ///
     /// Always attests to the canonical chain.
     pub fn produce_attestation_data(
         &self,
-        shard: u64,
         slot: Slot,
+        index: CommitteeIndex,
     ) -> Result<AttestationData, Error> {
         let state = self.state_at_slot(slot)?;
         let head = self.head();
 
         self.produce_attestation_data_for_block(
-            shard,
+            index,
             head.beacon_block_root,
             head.beacon_block.slot,
             &state,
@@ -451,7 +501,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// function should be used as it attests to the canonical chain.
     pub fn produce_attestation_data_for_block(
         &self,
-        shard: u64,
+        index: CommitteeIndex,
         head_block_root: Hash256,
         head_block_slot: Slot,
         state: &BeaconState<T::EthSpec>,
@@ -492,18 +542,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             root: target_root,
         };
 
-        let parent_crosslink = state.get_current_crosslink(shard)?;
-        let crosslink = Crosslink {
-            shard,
-            parent_root: Hash256::from_slice(&parent_crosslink.tree_hash_root()),
-            start_epoch: parent_crosslink.end_epoch,
-            end_epoch: std::cmp::min(
-                target.epoch,
-                parent_crosslink.end_epoch + self.spec.max_epochs_per_crosslink,
-            ),
-            data_root: Hash256::zero(),
-        };
-
         // Collect some metrics.
         metrics::inc_counter(&metrics::ATTESTATION_PRODUCTION_SUCCESSES);
         metrics::stop_timer(timer);
@@ -512,15 +550,16 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             self.log,
             "Produced beacon attestation data";
             "beacon_block_root" => format!("{}", head_block_root),
-            "shard" => shard,
-            "slot" => state.slot
+            "slot" => state.slot,
+            "index" => index
         );
 
         Ok(AttestationData {
+            slot: state.slot,
+            index,
             beacon_block_root: head_block_root,
             source: state.current_justified_checkpoint.clone(),
             target,
-            crosslink,
         })
     }
 
@@ -549,7 +588,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         self.log,
                         "Beacon attestation imported";
                         "target_epoch" => attestation.data.target.epoch,
-                        "shard" => attestation.data.crosslink.shard,
+                        "index" => attestation.data.index,
                     );
                     let _ = self
                         .event_handler
@@ -668,16 +707,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
             state.build_committee_cache(RelativeEpoch::Current, &self.spec)?;
 
-            let attestation_slot = state.get_attestation_data_slot(&attestation.data)?;
-
             // Reject any attestation where the `state` loaded from `data.beacon_block_root`
             // has a higher slot than the attestation.
             //
             // Permitting this would allow for attesters to vote on _future_ slots.
-            if state.slot > attestation_slot {
+            if state.slot > attestation.data.slot {
                 Ok(AttestationProcessingOutcome::AttestsToFutureState {
                     state: state.slot,
-                    attestation: attestation_slot,
+                    attestation: attestation.data.slot,
                 })
             } else {
                 self.process_attestation_for_state_and_block(
@@ -776,20 +813,27 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
 
             Ok(AttestationProcessingOutcome::Invalid(e))
         } else {
-            // Provide the attestation to fork choice, updating the validator latest messages but
-            // _without_ finding and updating the head.
-            if let Err(e) = self
-                .fork_choice
-                .process_attestation(&state, &attestation, block)
+            // If the attestation is from the current or previous epoch, supply it to the fork
+            // choice. This is FMD GHOST.
+            let current_epoch = self.epoch()?;
+            if attestation.data.target.epoch == current_epoch
+                || attestation.data.target.epoch == current_epoch - 1
             {
-                error!(
-                    self.log,
-                    "Add attestation to fork choice failed";
-                    "fork_choice_integrity" => format!("{:?}", self.fork_choice.verify_integrity()),
-                    "beacon_block_root" =>  format!("{}", attestation.data.beacon_block_root),
-                    "error" => format!("{:?}", e)
-                );
-                return Err(e.into());
+                // Provide the attestation to fork choice, updating the validator latest messages but
+                // _without_ finding and updating the head.
+                if let Err(e) = self
+                    .fork_choice
+                    .process_attestation(&state, &attestation, block)
+                {
+                    error!(
+                        self.log,
+                        "Add attestation to fork choice failed";
+                        "fork_choice_integrity" => format!("{:?}", self.fork_choice.verify_integrity()),
+                        "beacon_block_root" =>  format!("{}", attestation.data.beacon_block_root),
+                        "error" => format!("{:?}", e)
+                    );
+                    return Err(e.into());
+                }
             }
 
             // Provide the valid attestation to op pool, which may choose to retain the
@@ -829,22 +873,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         }
     }
 
-    /// Accept some transfer and queue it for inclusion in an appropriate block.
-    pub fn process_transfer(&self, transfer: Transfer) -> Result<(), TransferValidationError> {
-        match self.wall_clock_state() {
-            Ok(state) => self.op_pool.insert_transfer(transfer, &state, &self.spec),
-            Err(e) => {
-                error!(
-                    &self.log,
-                    "Unable to process transfer";
-                    "error" => format!("{:?}", e),
-                    "reason" => "no state"
-                );
-                Ok(())
-            }
-        }
-    }
-
     /// Accept some proposer slashing and queue it for inclusion in an appropriate block.
     pub fn process_proposer_slashing(
         &self,
@@ -905,7 +933,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         self.log,
                         "Beacon block imported";
                         "block_root" => format!("{:?}", block_root),
-                        "block_slot" => format!("{:?}", block_root),
+                        "block_slot" => format!("{:?}", block.slot.as_u64()),
                     );
                     let _ = self.event_handler.register(EventKind::BeaconBlockImported {
                         block_root: *block_root,
@@ -1115,7 +1143,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             metrics::start_timer(&metrics::BLOCK_PROCESSING_FORK_CHOICE_REGISTER);
 
         // Register the new block with the fork choice service.
-        if let Err(e) = self.fork_choice.process_block(&state, &block, block_root) {
+        if let Err(e) = self
+            .fork_choice
+            .process_block(self, &state, &block, block_root)
+        {
             error!(
                 self.log,
                 "Add block to fork choice failed";
@@ -1230,7 +1261,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                     .deposits_for_block_inclusion(&state, &self.spec)?
                     .into(),
                 voluntary_exits: self.op_pool.get_voluntary_exits(&state, &self.spec).into(),
-                transfers: self.op_pool.get_transfers(&state, &self.spec).into(),
             },
         };
 

beacon_node/beacon_chain/src/builder.rs

@@ -231,7 +231,12 @@ where
             .genesis_block_root
             .ok_or_else(|| "fork_choice_backend requires a genesis_block_root")?;
 
-        self.fork_choice = Some(ForkChoice::new(store, backend, genesis_block_root));
+        self.fork_choice = Some(ForkChoice::new(
+            store,
+            backend,
+            genesis_block_root,
+            self.spec.genesis_slot,
+        ));
 
         Ok(self)
     }

beacon_node/beacon_chain/src/errors.rs

@@ -18,7 +18,6 @@ macro_rules! easy_from_to {
 #[derive(Debug, PartialEq)]
 pub enum BeaconChainError {
     InsufficientValidators,
-    BadRecentBlockRoots,
     UnableToReadSlot,
     RevertedFinalizedEpoch {
         previous_epoch: Epoch,

beacon_node/beacon_chain/src/eth1_chain.rs

@@ -487,7 +487,7 @@ mod test {
             let keypair = generate_deterministic_keypair(i as usize);
             let mut builder =
                 TestingDepositBuilder::new(keypair.pk.clone(), spec.max_effective_balance);
-            builder.sign(&DepositTestTask::Valid, &keypair, spec);
+            builder.sign(DepositTestTask::Valid, &keypair, spec);
             let deposit_data = builder.build().data;
 
             DepositLog {