Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

no-std implementation #167

Open
wants to merge 37 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 30 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
779a1e7
started no-std support
dharjeezy Dec 27, 2022
3e66ff7
invalid operation error
dharjeezy Dec 27, 2022
0b79485
introduce prelude which contains the no_std imports
dharjeezy Dec 28, 2022
7492dde
transform all state_transition errors to no_std
dharjeezy Dec 28, 2022
8221123
introduce BTreeSet and BTreeMap in prelude
dharjeezy Dec 28, 2022
ac1b571
further no_std imports in prelude
dharjeezy Dec 28, 2022
cd52ded
introduce hashbrown crate
dharjeezy Dec 29, 2022
0d05be0
serde features on structs
dharjeezy Jan 3, 2023
54f8133
make use of milagro_bls for no_std support
dharjeezy Jan 4, 2023
eeb1d5e
make ssz-rs default feature
dharjeezy Jan 4, 2023
8849fcb
fix on comments
dharjeezy Jan 4, 2023
2c0d3dd
use signature for aggregate verify
dharjeezy Jan 4, 2023
32883ab
use signature for aggregate verify
dharjeezy Jan 4, 2023
04f9bcc
introduce key validation failed
dharjeezy Jan 4, 2023
a6c3687
build successful in std and no_std context
dharjeezy Jan 5, 2023
1f0e6e2
all test passes
dharjeezy Jan 5, 2023
71710e9
remove duplication
dharjeezy Jan 8, 2023
df115f5
implement Eq and PartialEq traits for DomainType enum
dharjeezy Jan 13, 2023
7875009
remove saturating_div
dharjeezy Jan 19, 2023
33f236a
Merge branch 'main' of https://github.com/polytope-labs/ethereum-cons…
dharjeezy Jan 19, 2023
8c50522
merge changes
dharjeezy Jan 19, 2023
e5c0d21
fix warnings
dharjeezy Jan 19, 2023
2f46baf
remove prelude.rs
dharjeezy Jan 21, 2023
248b5d1
remove excess clones
Wizdave97 Jan 24, 2023
63b847d
introduce default macro for execution payload
dharjeezy Jan 26, 2023
9177be3
remove default
dharjeezy Jan 26, 2023
6b971e2
include default and alias
dharjeezy Jan 30, 2023
918fda3
use rev
dharjeezy Feb 1, 2023
a850739
Merge branch 'main' of https://github.com/polytope-labs/ethereum-cons…
dharjeezy Feb 1, 2023
8be43ea
no-std for capella fork
dharjeezy Feb 1, 2023
9862cab
change ssz-rs repo temporarily
dharjeezy Feb 1, 2023
e334085
Merge branch 'main' of https://github.com/polytope-labs/ethereum-cons…
dharjeezy Feb 25, 2023
6b7f33d
merge conflict fixed
dharjeezy Feb 25, 2023
b6ced43
fix std and no std builds
Wizdave97 Feb 27, 2023
23cd342
extra fixes
Wizdave97 Feb 27, 2023
bd19d7c
point ssz-rs to original crate
Wizdave97 Feb 27, 2023
34bb57f
point ssz-rs to polytope labs
Wizdave97 Mar 13, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 24 additions & 9 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,22 +13,37 @@ spec-tests = ["serde", "serde_yaml"]
gen-spec = ["syn", "prettyplease", "quote"]
gen-tests = ["walkdir", "convert_case"]

std = [
"rand/std",
"sha2/std",
"multihash/std",
"bs58/std",
"getrandom/std",
"ssz-rs/std",
"serde/std"
]

[dependencies]
ssz-rs = { git = "https://github.com/ralexstokes/ssz-rs", rev = "5946af4a65a1e8547a8fc7cfb62e12637421b8f2" }
blst = "0.3.6"
rand = "0.8.4"
thiserror = "1.0.30"
sha2 = "0.9.8"
integer-sqrt = "0.1.5"
enr = "0.6.2"
getrandom = { version = "0.2.8", default-features=false, features = ["js"] }
ssz-rs = { git = "https://github.com/ralexstokes/ssz-rs", rev = "5946af4a65a1e8547a8fc7cfb62e12637421b8f2", default-features=false, features=["serde", "std"] }
rand = {version = "0.8.4", default-features = false}
thiserror = {version = "1.0.30"}
error-chain={version = "0.12.4", default-features=false}
sha2 = { version ="0.9.8", default-features = false }
integer-sqrt = {version = "0.1.5", default-features = false }
enr = {version = "0.6.2" }
multihash = { version = "0.16", default-features = false, features = ["std", "multihash-impl", "identity", "sha2"] }
multiaddr = "0.14.0"
multiaddr = {version="0.14.0" }
hashbrown = {version="0.13.1"}

serde = { version = "1.0", features = ["derive"], optional = true }
serde_json = { version = "1.0.81", optional = true }
serde_yaml = { version = "0.8", optional = true }
hex = {version = "0.4.3", optional = true }

blst = {version = "0.3.6", optional=true}
milagro_bls = { git = "https://github.com/sigp/milagro_bls", default-features = false }

tokio = { version = "1.18.2", features = ["full"], optional = true }
tokio-stream = { version = "0.1.8", optional = true }
async-stream = { version = "0.3.3", optional = true }
Expand All @@ -39,7 +54,7 @@ quote = { version = "1.0.18", optional = true }

walkdir = { version = "2.3.2", optional = true }
convert_case = { version = "0.5.0", optional = true }
bs58 = "0.4.0"
bs58 = {version="0.4.0", default-features = false }

[dev-dependencies]
serde_with = "1.13.0"
Expand Down
8 changes: 4 additions & 4 deletions examples/state_transition_across_multiple_forks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,18 @@ fn main() -> std::result::Result<(), Box<dyn Error>> {

let mut block = phase0::SignedBeaconBlock::default();
block.message.slot = 1;
executor.apply_block(&mut block.into())?;
executor.apply_block(&mut block.into()).unwrap();

let mut block = altair::SignedBeaconBlock::default();
block.message.slot = executor.context.altair_fork_epoch * executor.context.slots_per_epoch;
executor.apply_block(&mut block.into())?;
executor.apply_block(&mut block.into()).unwrap();

let mut block = bellatrix::SignedBeaconBlock::default();
block.message.slot = executor.context.bellatrix_fork_epoch * executor.context.slots_per_epoch;
executor.apply_block(&mut block.into())?;
executor.apply_block(&mut block.into()).unwrap();

let mut state = executor.state.bellatrix().unwrap();
let state_root = state.hash_tree_root()?;
let state_root = state.hash_tree_root().unwrap();
dbg!(state_root);
Ok(())
}
5 changes: 3 additions & 2 deletions src/altair/beacon_block.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::altair::SyncAggregate;
use crate::lib::*;
use crate::phase0::{
Attestation, AttesterSlashing, Deposit, Eth1Data, ProposerSlashing, SignedVoluntaryExit,
};
Expand Down Expand Up @@ -39,9 +40,9 @@ pub struct BeaconBlock<
const MAX_VOLUNTARY_EXITS: usize,
const SYNC_COMMITTEE_SIZE: usize,
> {
#[serde(with = "crate::serde::as_string")]
#[cfg_attr(feature = "serde", serde(with = "crate::serde::as_string"))]
pub slot: Slot,
#[serde(with = "crate::serde::as_string")]
#[cfg_attr(feature = "serde", serde(with = "crate::serde::as_string"))]
pub proposer_index: ValidatorIndex,
pub parent_root: Root,
pub state_root: Root,
Expand Down
32 changes: 24 additions & 8 deletions src/altair/beacon_state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use crate::altair::{
BeaconBlockHeader, Checkpoint, Eth1Data, Fork, SyncCommittee, Validator,
JUSTIFICATION_BITS_LENGTH,
};
use crate::lib::*;
use crate::primitives::{Bytes32, Gwei, ParticipationFlags, Root, Slot};
use ssz_rs::prelude::*;

Expand All @@ -17,10 +18,10 @@ pub struct BeaconState<
const MAX_VALIDATORS_PER_COMMITTEE: usize,
const SYNC_COMMITTEE_SIZE: usize,
> {
#[serde(with = "crate::serde::as_string")]
#[cfg_attr(feature = "serde", serde(with = "crate::serde::as_string"))]
pub genesis_time: u64,
pub genesis_validators_root: Root,
#[serde(with = "crate::serde::as_string")]
#[cfg_attr(feature = "serde", serde(with = "crate::serde::as_string"))]
pub slot: Slot,
pub fork: Fork,
pub latest_block_header: BeaconBlockHeader,
Expand All @@ -29,23 +30,38 @@ pub struct BeaconState<
pub historical_roots: List<Root, HISTORICAL_ROOTS_LIMIT>,
pub eth1_data: Eth1Data,
pub eth1_data_votes: List<Eth1Data, ETH1_DATA_VOTES_BOUND>,
#[serde(with = "crate::serde::as_string")]
#[cfg_attr(feature = "serde", serde(with = "crate::serde::as_string"))]
pub eth1_deposit_index: u64,
pub validators: List<Validator, VALIDATOR_REGISTRY_LIMIT>,
#[serde(with = "crate::serde::collection_over_string")]
#[cfg_attr(
feature = "serde",
serde(with = "crate::serde::collection_over_string")
)]
pub balances: List<Gwei, VALIDATOR_REGISTRY_LIMIT>,
pub randao_mixes: Vector<Bytes32, EPOCHS_PER_HISTORICAL_VECTOR>,
#[serde(with = "crate::serde::collection_over_string")]
#[cfg_attr(
feature = "serde",
serde(with = "crate::serde::collection_over_string")
)]
pub slashings: Vector<Gwei, EPOCHS_PER_SLASHINGS_VECTOR>,
#[serde(with = "crate::serde::collection_over_string")]
#[cfg_attr(
feature = "serde",
serde(with = "crate::serde::collection_over_string")
)]
pub previous_epoch_participation: List<ParticipationFlags, VALIDATOR_REGISTRY_LIMIT>,
#[serde(with = "crate::serde::collection_over_string")]
#[cfg_attr(
feature = "serde",
serde(with = "crate::serde::collection_over_string")
)]
pub current_epoch_participation: List<ParticipationFlags, VALIDATOR_REGISTRY_LIMIT>,
pub justification_bits: Bitvector<JUSTIFICATION_BITS_LENGTH>,
pub previous_justified_checkpoint: Checkpoint,
pub current_justified_checkpoint: Checkpoint,
pub finalized_checkpoint: Checkpoint,
#[serde(with = "crate::serde::collection_over_string")]
#[cfg_attr(
feature = "serde",
serde(with = "crate::serde::collection_over_string")
)]
pub inactivity_scores: List<u64, VALIDATOR_REGISTRY_LIMIT>,
pub current_sync_committee: SyncCommittee<SYNC_COMMITTEE_SIZE>,
pub next_sync_committee: SyncCommittee<SYNC_COMMITTEE_SIZE>,
Expand Down
43 changes: 24 additions & 19 deletions src/altair/block_processing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use crate::altair as spec;

use crate::crypto::{eth_fast_aggregate_verify, verify_signature};
use crate::domains::DomainType;
use crate::lib::*;
use crate::primitives::{BlsPublicKey, ParticipationFlags, ValidatorIndex};
use crate::signing::compute_signing_root;
use crate::state_transition::{
Expand All @@ -20,8 +21,6 @@ use spec::{
PARTICIPATION_FLAG_WEIGHTS, PROPOSER_WEIGHT, SYNC_REWARD_WEIGHT, WEIGHT_DENOMINATOR,
};
use ssz_rs::prelude::*;
use std::collections::{HashMap, HashSet};
use std::iter::zip;

pub fn process_attestation<
const SLOTS_PER_HISTORICAL_ROOT: usize,
Expand Down Expand Up @@ -202,9 +201,12 @@ pub fn process_deposit<

let public_key = &deposit.data.public_key;
let amount = deposit.data.amount;
let validator_public_keys: HashSet<&BlsPublicKey> =
HashSet::from_iter(state.validators.iter().map(|v| &v.public_key));
if !validator_public_keys.contains(public_key) {
let validator_check = {
let validator_public_keys: HashSet<&BlsPublicKey> =
HashSet::from_iter(state.validators.iter().map(|v| &v.public_key));
validator_public_keys.contains(public_key)
};
if !validator_check {
let mut deposit_message = DepositMessage {
public_key: public_key.clone(),
withdrawal_credentials: deposit.data.withdrawal_credentials.clone(),
Expand Down Expand Up @@ -316,20 +318,23 @@ pub fn process_sync_aggregate<
participant_reward * PROPOSER_WEIGHT / (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT);

// Apply participant and proposer rewards
let all_public_keys = state
.validators
.iter()
.enumerate()
.map(|(i, v)| (&v.public_key, i))
.collect::<HashMap<&BlsPublicKey, usize>>();
let mut committee_indices: Vec<ValidatorIndex> = Vec::default();
for public_key in state.current_sync_committee.public_keys.iter() {
committee_indices.push(
*all_public_keys
.get(public_key)
.expect("validator public_key should exist"),
);
}
let committee_indices = {
let all_public_keys = state
.validators
.iter()
.enumerate()
.map(|(i, v)| (&v.public_key, i))
.collect::<HashMap<_, _>>();
let mut committee_indices: Vec<ValidatorIndex> = Vec::default();
for public_key in state.current_sync_committee.public_keys.iter() {
committee_indices.push(
*all_public_keys
.get(public_key)
.expect("validator public_key should exist"),
);
}
committee_indices
};
for (participant_index, participation_bit) in zip(
committee_indices.iter(),
sync_aggregate.sync_committee_bits.iter(),
Expand Down
2 changes: 1 addition & 1 deletion src/altair/epoch_processing.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::altair as spec;

use crate::lib::*;
use crate::primitives::{Gwei, ParticipationFlags, ValidatorIndex, GENESIS_EPOCH};
use crate::state_transition::{Context, Result};
use spec::{
Expand All @@ -12,7 +13,6 @@ use spec::{
weigh_justification_and_finalization, BeaconState, PARTICIPATION_FLAG_WEIGHTS,
TIMELY_TARGET_FLAG_INDEX,
};
use std::mem;

// Return the base reward for the validator defined by `index` with respect to the current `state`
pub fn get_base_reward<
Expand Down
1 change: 1 addition & 0 deletions src/altair/fork.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use crate::altair::{
add_flag, get_attestation_participation_flag_indices, get_attesting_indices,
get_next_sync_committee, BeaconState, Fork,
};
use crate::lib::*;
use crate::phase0;
use crate::state_transition::{Context, Result};
use ssz_rs::prelude::*;
Expand Down
3 changes: 2 additions & 1 deletion src/altair/genesis.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::altair as spec;

use crate::lib::*;
use crate::phase0::DEPOSIT_DATA_LIST_BOUND;
use crate::primitives::{Gwei, Hash32, GENESIS_EPOCH};
use crate::state_transition::{Context, Result};
Expand Down Expand Up @@ -65,7 +66,7 @@ pub fn initialize_beacon_state_from_eth1<
..Default::default()
};
let randao_mixes = Vector::try_from(
std::iter::repeat(eth1_block_hash)
repeat(eth1_block_hash)
.take(context.epochs_per_historical_vector as usize)
.collect::<Vec<_>>(),
)
Expand Down
2 changes: 1 addition & 1 deletion src/altair/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use crate::altair as spec;

use crate::crypto::{eth_aggregate_public_keys, hash};
use crate::domains::DomainType;
use crate::lib::*;
use crate::primitives::{BlsPublicKey, Epoch, Gwei, ParticipationFlags, ValidatorIndex};
use crate::state_transition::{
invalid_operation_error, Context, Error, InvalidAttestation, InvalidOperation, Result,
Expand All @@ -16,7 +17,6 @@ use spec::{
TIMELY_HEAD_FLAG_INDEX, TIMELY_SOURCE_FLAG_INDEX, TIMELY_TARGET_FLAG_INDEX, WEIGHT_DENOMINATOR,
};
use ssz_rs::Vector;
use std::collections::HashSet;

// Return a new ``ParticipationFlags`` adding ``flag_index`` to ``flags``
pub fn add_flag(flags: ParticipationFlags, flag_index: usize) -> ParticipationFlags {
Expand Down
1 change: 1 addition & 0 deletions src/altair/light_client.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use crate::altair::{SyncAggregate, SyncCommittee};
use crate::lib::*;
use crate::phase0::BeaconBlockHeader;
use crate::primitives::{Bytes32, Version};
use ssz_rs::prelude::*;
Expand Down
2 changes: 1 addition & 1 deletion src/altair/state_transition/block_processing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ pub use crate::altair::block_processing::process_sync_aggregate;
use crate::crypto::{hash, verify_signature};
use crate::signing::compute_signing_root;

use crate::lib::*;
use crate::primitives::{Bytes32, DomainType, Gwei, ValidatorIndex, FAR_FUTURE_EPOCH};
use crate::ssz::ByteVector;
use crate::state_transition::{
Expand All @@ -22,7 +23,6 @@ use spec::{
SignedVoluntaryExit, Validator,
};
use ssz_rs::prelude::*;
use std::collections::HashSet;
pub fn get_validator_from_deposit(deposit: &Deposit, context: &Context) -> Validator {
let amount = deposit.data.amount;
let effective_balance = Gwei::min(
Expand Down
2 changes: 2 additions & 0 deletions src/altair/state_transition/epoch_processing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ pub use crate::altair::epoch_processing::process_rewards_and_penalties;
pub use crate::altair::epoch_processing::process_slashings;
pub use crate::altair::epoch_processing::process_sync_committee_updates;
use crate::primitives::{Epoch, Gwei, ValidatorIndex};

use crate::lib::*;
use crate::state_transition::{Context, Result};

use ssz_rs::prelude::*;
Expand Down
11 changes: 8 additions & 3 deletions src/altair/state_transition/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ pub use crate::altair::helpers::get_unslashed_participating_indices;
pub use crate::altair::helpers::has_flag;
pub use crate::altair::helpers::slash_validator;
use crate::crypto::{fast_aggregate_verify, hash, verify_signature};
use crate::lib::*;
use crate::primitives::{
Bytes32, CommitteeIndex, Domain, DomainType, Epoch, ForkDigest, Gwei, Root, Slot,
ValidatorIndex, Version, FAR_FUTURE_EPOCH, GENESIS_EPOCH,
Expand All @@ -25,8 +26,7 @@ use spec::{
Validator,
};
use ssz_rs::prelude::*;
use std::cmp;
use std::collections::HashSet;

pub fn compute_activation_exit_epoch(epoch: Epoch, context: &Context) -> Epoch {
epoch + 1 + context.max_seed_lookahead
}
Expand Down Expand Up @@ -117,7 +117,12 @@ pub fn compute_proposer_index<
loop {
let shuffled_index = compute_shuffled_index(i % total, total, seed, context)?;
let candidate_index = indices[shuffled_index];
#[cfg(not(feature = "serde"))]
let i_bytes: [u8; 4] = (i / 32).to_le_bytes();

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm curious what is the purpose of this? It seems to be broken at the moment if the "serde" feature is not enabled

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dharjeezy why is there a serde feature flag here?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So, our target compilation is wasm32-unknown-unknown
I had to include the feature flag because of that else it errors out

@Wizdave97 @willemolding

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Collecting into a dynamic type like a Vec might work better in this case.

Copy link

@willemolding willemolding Feb 25, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe better to use #[cfg(target_arch = "wasm32")] if it is a wasm specific thing? As it stands it errors out building for x86 with serde disabled

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh wait an even better solution is to remove the intermediate variable entirely e.g.

hash_input[32..].copy_from_slice(&((i / 32) as u64).to_le_bytes());

This will build either way regardless of the size of a u64 on the given target


#[cfg(feature = "serde")]
let i_bytes: [u8; 8] = (i / 32).to_le_bytes();

hash_input[32..].copy_from_slice(&i_bytes);
let random_byte = hash(hash_input).as_ref()[(i % 32)] as u64;
let effective_balance = state.validators[candidate_index].effective_balance;
Expand Down Expand Up @@ -254,7 +259,7 @@ pub fn get_attesting_indices<
},
)));
}
let mut indices = HashSet::with_capacity(bits.capacity());
let mut indices = HashSet::new();
for (i, validator_index) in committee.iter().enumerate() {
if bits[i] {
indices.insert(*validator_index);
Expand Down
5 changes: 3 additions & 2 deletions src/altair/sync.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use crate::lib::*;
use crate::primitives::{BlsPublicKey, BlsSignature};
use ssz_rs::prelude::*;

Expand All @@ -11,8 +12,8 @@ pub struct SyncAggregate<const SYNC_COMMITTEE_SIZE: usize> {
#[derive(Default, Debug, SimpleSerialize, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct SyncCommittee<const SYNC_COMMITTEE_SIZE: usize> {
#[serde(rename = "pubkeys")]
#[cfg_attr(feature = "serde", serde(rename = "pubkeys"))]
pub public_keys: Vector<BlsPublicKey, SYNC_COMMITTEE_SIZE>,
#[serde(rename = "aggregate_pubkey")]
#[cfg_attr(feature = "serde", serde(rename = "aggregate_pubkey"))]
pub aggregate_public_key: BlsPublicKey,
}
Loading