From 4be2362df12fbf9dff25fce49d3d089bdc010714 Mon Sep 17 00:00:00 2001 From: willesxm Date: Wed, 15 Mar 2023 18:22:28 +0800 Subject: [PATCH 01/27] add cron fields --- .github/workflows/{tests.yml => ci.yml} | 8 ++-- atomic-exec/src/lib.rs | 17 ++++--- gateway/src/cross.rs | 4 +- gateway/src/lib.rs | 41 ++++++++-------- gateway/src/state.rs | 12 ++++- gateway/src/types.rs | 12 +++++ gateway/tests/harness.rs | 9 ++++ sdk/src/lib.rs | 59 +++++++++++++++++++++++ subnet-actor/src/lib.rs | 4 +- subnet-actor/src/state.rs | 5 +- subnet-actor/src/types.rs | 62 ------------------------- 11 files changed, 129 insertions(+), 104 deletions(-) rename .github/workflows/{tests.yml => ci.yml} (89%) diff --git a/.github/workflows/tests.yml b/.github/workflows/ci.yml similarity index 89% rename from .github/workflows/tests.yml rename to .github/workflows/ci.yml index ba1bb2c..e2692af 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/ci.yml @@ -20,8 +20,8 @@ jobs: target: wasm32-unknown-unknown toolchain: nightly override: true - - run: cargo b --all --release - - run: cargo t --all --release + - run: cargo b --all + - run: cargo t --all fmt: name: Rustfmt @@ -38,7 +38,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: fmt - args: --all -- --check + args: --all --check clippy: name: Clippy @@ -55,4 +55,4 @@ jobs: - uses: actions-rs/cargo@v1 with: command: clippy - args: -- -D warnings + args: --all -- -D clippy::all diff --git a/atomic-exec/src/lib.rs b/atomic-exec/src/lib.rs index 8cd2c7d..4f2a754 100644 --- a/atomic-exec/src/lib.rs +++ b/atomic-exec/src/lib.rs @@ -87,7 +87,7 @@ impl Actor { } let msgs = rt.transaction(|st: &mut State, rt| { - st.modify_atomic_exec(rt.store(), &exec_id, &actors, |entry| { + st.modify_atomic_exec(rt.store(), exec_id, actors, |entry| { // Record the pre-commitment entry.insert(from.to_string().unwrap(), params.commit); @@ -135,13 +135,12 @@ impl Actor { // Remove the atomic execution entry rt.transaction(|st: &mut State, rt| { - st.rm_atomic_exec(rt.store(), &exec_id, &actors) - .map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - "failed to remove atomic exec from registry", - ) - }) + st.rm_atomic_exec(rt.store(), exec_id, actors).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + "failed to remove atomic exec from registry", + ) + }) })?; Ok(true) @@ -180,7 +179,7 @@ impl Actor { } let msg = rt.transaction(|st: &mut State, rt| { - st.modify_atomic_exec(rt.store(), &exec_id, &actors, |entry| { + st.modify_atomic_exec(rt.store(), exec_id, actors, |entry| { // Remove the pre-commitment entry.remove_entry(&from.to_string().unwrap()); diff --git a/gateway/src/cross.rs b/gateway/src/cross.rs index 79187fd..4f8bdd2 100644 --- a/gateway/src/cross.rs +++ b/gateway/src/cross.rs @@ -158,7 +158,7 @@ impl CrossMsgs { } pub(crate) fn cid(&self) -> anyhow::Result>> { - TCid::new_link(&MemoryBlockstore::new(), &self) + TCid::new_link(&MemoryBlockstore::new(), self) } /// Appends a cross-message to cross-msgs @@ -211,7 +211,7 @@ pub(crate) fn distribute_crossmsg_fee( fee: TokenAmount, ) -> Result<(), ActorError> { if !fee.is_zero() { - rt.send(&subnet_actor, SUBNET_ACTOR_REWARD_METHOD, None, fee)?; + rt.send(subnet_actor, SUBNET_ACTOR_REWARD_METHOD, None, fee)?; } Ok(()) } diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index a86e274..8e0ad08 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -70,7 +70,7 @@ impl Actor { fn constructor(rt: &mut impl Runtime, params: ConstructorParams) -> Result<(), ActorError> { rt.validate_immediate_caller_is(std::iter::once(&INIT_ACTOR_ADDR))?; - let st = State::new(rt.store(), params).map_err(|e| { + let st = State::new(rt.store(), params, rt.curr_epoch()).map_err(|e| { e.downcast_default( ExitCode::USR_ILLEGAL_STATE, "Failed to create SCA actor state", @@ -328,30 +328,27 @@ impl Actor { // commit cross-message in checkpoint to either execute them or // queue them for propagation if there are cross-msgs availble. - match commit.cross_msgs() { - Some(cross_msg) => { - // if tcid not default it means cross-msgs are being propagated. - if cross_msg.msgs_cid != TCid::default() { - st.store_bottomup_msg(rt.store(), cross_msg).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - "error storing bottom_up messages from checkpoint", - ) - })?; - } - - // release circulating supply - sub.release_supply(&cross_msg.value).map_err(|e| { + if let Some(cross_msg) = commit.cross_msgs() { + // if tcid not default it means cross-msgs are being propagated. + if cross_msg.msgs_cid != TCid::default() { + st.store_bottomup_msg(rt.store(), cross_msg).map_err(|e| { e.downcast_default( ExitCode::USR_ILLEGAL_STATE, - "error releasing circulating supply", + "error storing bottom_up messages from checkpoint", ) })?; - - // distribute fee - fee = cross_msg.fee.clone(); } - None => {} + + // release circulating supply + sub.release_supply(&cross_msg.value).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + "error releasing circulating supply", + ) + })?; + + // distribute fee + fee = cross_msg.fee.clone(); } // append new checkpoint to the list of childs @@ -466,7 +463,7 @@ impl Actor { rt.transaction(|st: &mut State, rt| { let fee = &CROSS_MSG_FEE; // collect fees - st.collect_cross_fee(&mut value, &fee)?; + st.collect_cross_fee(&mut value, fee)?; // Create release message let r_msg = CrossMsg { @@ -486,7 +483,7 @@ impl Actor { }; // Commit bottom-up message. - st.commit_bottomup_msg(rt.store(), &r_msg, &fee, rt.curr_epoch()) + st.commit_bottomup_msg(rt.store(), &r_msg, fee, rt.curr_epoch()) .map_err(|e| { e.downcast_default( ExitCode::USR_ILLEGAL_STATE, diff --git a/gateway/src/state.rs b/gateway/src/state.rs index 231a1cb..93875f7 100644 --- a/gateway/src/state.rs +++ b/gateway/src/state.rs @@ -46,6 +46,10 @@ pub struct State { pub bottomup_msg_meta: TCid>, pub applied_bottomup_nonce: u64, pub applied_topdown_nonce: u64, + /// The epoch that this actor is deployed + pub genesis_epoch: ChainEpoch, + /// How often cron checkpoints will be submitted by validator in the child subnet + pub cron_period: ChainEpoch, } lazy_static! { @@ -53,7 +57,11 @@ lazy_static! { } impl State { - pub fn new(store: &BS, params: ConstructorParams) -> anyhow::Result { + pub fn new( + store: &BS, + params: ConstructorParams, + current_epoch: ChainEpoch, + ) -> anyhow::Result { Ok(State { network_name: SubnetID::from_str(¶ms.network_name)?, total_subnets: Default::default(), @@ -73,6 +81,8 @@ impl State { // We first increase to the subsequent and then execute for bottom-up messages applied_bottomup_nonce: MAX_NONCE, applied_topdown_nonce: Default::default(), + genesis_epoch: current_epoch, + cron_period: params.cron_period, }) } diff --git a/gateway/src/types.rs b/gateway/src/types.rs index dd13483..47dcb18 100644 --- a/gateway/src/types.rs +++ b/gateway/src/types.rs @@ -7,11 +7,13 @@ use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use ipc_sdk::subnet_id::SubnetID; +use ipc_sdk::ValidatorSet; use multihash::MultihashDigest; use primitives::CodeType; use crate::checkpoint::{Checkpoint, CrossMsgMeta}; use crate::cross::CrossMsg; +use crate::StorableMsg; /// ID used in the builtin-actors bundle manifest pub const MANIFEST_ID: &str = "ipc_gateway"; @@ -30,6 +32,7 @@ pub type CrossMsgArray<'bs, BS> = Array<'bs, CrossMsg, BS>; pub struct ConstructorParams { pub network_name: String, pub checkpoint_period: ChainEpoch, + pub cron_period: ChainEpoch, } #[derive(Serialize_tuple, Deserialize_tuple, Clone)] @@ -99,6 +102,13 @@ impl PostBoxItem { } } +#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] +pub struct CronCheckpoint { + pub epoch: ChainEpoch, + pub membership: ValidatorSet, + pub top_down_msgs: Vec, +} + #[cfg(test)] mod tests { use crate::ConstructorParams; @@ -109,6 +119,7 @@ mod tests { let p = ConstructorParams { network_name: "/root".to_string(), checkpoint_period: 100, + cron_period: 20, }; let bytes = fil_actors_runtime::util::cbor::serialize(&p, "").unwrap(); let serialized = base64::encode(bytes.bytes()); @@ -120,5 +131,6 @@ mod tests { assert_eq!(p.network_name, deserialized.network_name); assert_eq!(p.checkpoint_period, deserialized.checkpoint_period); + assert_eq!(p.cron_period, deserialized.cron_period); } } diff --git a/gateway/tests/harness.rs b/gateway/tests/harness.rs index 5224370..e867c1b 100644 --- a/gateway/tests/harness.rs +++ b/gateway/tests/harness.rs @@ -35,6 +35,7 @@ use ipc_gateway::{ use lazy_static::lazy_static; use primitives::{TCid, TCidContent}; use std::str::FromStr; +use fvm_shared::clock::ChainEpoch; lazy_static! { pub static ref ROOTNET_ID: SubnetID = @@ -46,6 +47,7 @@ lazy_static! { Address::new_bls(&[1; fvm_shared::address::BLS_PUB_LEN]).unwrap(); pub static ref ACTOR: Address = Address::new_actor("actor".as_bytes()); pub static ref SIG_TYPES: Vec = vec![*ACCOUNT_ACTOR_CODE_ID, *MULTISIG_ACTOR_CODE_ID]; + pub static ref DEFAULT_CRON_PERIOD: ChainEpoch = 20; } pub fn new_runtime() -> MockRuntime { @@ -83,6 +85,7 @@ impl Harness { let params = ConstructorParams { network_name: self.net_name.to_string(), checkpoint_period: 10, + cron_period: *DEFAULT_CRON_PERIOD, }; rt.set_caller(*INIT_ACTOR_CODE_ID, INIT_ACTOR_ADDR); rt.call::( @@ -93,7 +96,11 @@ impl Harness { } pub fn construct_and_verify(&self, rt: &mut MockRuntime) { + let chain_epoch = 10; + rt.set_epoch(chain_epoch); + self.construct(rt); + let st: State = rt.get_state(); let store = &rt.store; @@ -106,6 +113,8 @@ impl Harness { assert_eq!(st.check_period, DEFAULT_CHECKPOINT_PERIOD); assert_eq!(st.applied_bottomup_nonce, MAX_NONCE); assert_eq!(st.bottomup_msg_meta.cid(), empty_bottomup_array); + assert_eq!(st.genesis_epoch, chain_epoch); + assert_eq!(st.cron_period, *DEFAULT_CRON_PERIOD); verify_empty_map(rt, st.subnets.cid()); verify_empty_map(rt, st.checkpoints.cid()); verify_empty_map(rt, st.check_msg_registry.cid()); diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs index 79beea4..15aac66 100644 --- a/sdk/src/lib.rs +++ b/sdk/src/lib.rs @@ -1,3 +1,7 @@ +use fvm_ipld_encoding::tuple::{Deserialize_tuple, Serialize_tuple}; +use fvm_shared::address::Address; +use fvm_shared::econ::TokenAmount; + pub mod address; pub mod error; pub mod subnet_id; @@ -6,3 +10,58 @@ pub mod account { /// Public key account actor method. pub const PUBKEY_ADDRESS_METHOD: u64 = 2; } + +#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] +pub struct Validator { + pub addr: Address, + pub net_addr: String, + // voting power for the validator determined by its stake in the + // network. + pub weight: TokenAmount, +} + +#[derive(Clone, Default, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] +pub struct ValidatorSet { + validators: Vec, + // sequence number that uniquely identifies a validator set + configuration_number: u64, +} + +impl ValidatorSet { + pub fn validators(&self) -> &Vec { + &self.validators + } + + pub fn validators_mut(&mut self) -> &mut Vec { + &mut self.validators + } + + pub fn config_number(&self) -> u64 { + self.configuration_number + } + + /// Push a new validator to the validator set. + pub fn push(&mut self, val: Validator) { + self.validators.push(val); + // update the config_number with every update + // we allow config_number to overflow if that scenario ever comes. + self.configuration_number += 1; + } + + /// Remove a validator from validator set by address + pub fn rm(&mut self, val: &Address) { + self.validators.retain(|x| x.addr != *val); + // update the config_number with every update + // we allow config_number to overflow if that scenario ever comes. + self.configuration_number += 1; + } + + pub fn update_weight(&mut self, val: &Address, weight: &TokenAmount) { + self.validators_mut() + .iter_mut() + .filter(|x| x.addr == *val) + .for_each(|x| x.weight = weight.clone()); + + self.configuration_number += 1; + } +} diff --git a/subnet-actor/src/lib.rs b/subnet-actor/src/lib.rs index 8033e7e..c67efbc 100644 --- a/subnet-actor/src/lib.rs +++ b/subnet-actor/src/lib.rs @@ -340,7 +340,7 @@ impl SubnetActor for Actor { // complex and fair policies to incentivize certain behaviors. // we may even have a default one for IPC. let div = { - if st.validator_set.validators().len() == 0 { + if st.validator_set.validators().is_empty() { return Err(actor_error!(illegal_state, "no validators in subnet")); }; match BigInt::from_usize(st.validator_set.validators().len()) { @@ -351,7 +351,7 @@ impl SubnetActor for Actor { } }; let rew_amount = amount.div_floor(div); - for v in st.validator_set.validators().into_iter() { + for v in st.validator_set.validators().iter() { rt.send(&v.addr, METHOD_SEND, None, rew_amount.clone())?; } Ok(None) diff --git a/subnet-actor/src/state.rs b/subnet-actor/src/state.rs index 4467882..09120e9 100644 --- a/subnet-actor/src/state.rs +++ b/subnet-actor/src/state.rs @@ -10,6 +10,7 @@ use fvm_shared::bigint::Zero; use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use ipc_gateway::{Checkpoint, SubnetID, DEFAULT_CHECKPOINT_PERIOD, MIN_COLLATERAL_AMOUNT}; +use ipc_sdk::{Validator, ValidatorSet}; use lazy_static::lazy_static; use num::rational::Ratio; use num::BigInt; @@ -78,7 +79,7 @@ impl State { checkpoints: TCid::new_hamt(store)?, stake: TCid::new_hamt(store)?, window_checks: TCid::new_hamt(store)?, - validator_set: ValidatorSet::new(), + validator_set: ValidatorSet::default(), }; Ok(state) @@ -391,7 +392,7 @@ impl Default for State { checkpoints: TCid::default(), stake: TCid::default(), window_checks: TCid::default(), - validator_set: ValidatorSet::new(), + validator_set: ValidatorSet::default(), min_validators: 0, } } diff --git a/subnet-actor/src/types.rs b/subnet-actor/src/types.rs index 7203180..f379835 100644 --- a/subnet-actor/src/types.rs +++ b/subnet-actor/src/types.rs @@ -19,68 +19,6 @@ pub const MANIFEST_ID: &str = "ipc_subnet_actor"; pub const LEAVING_COEFF: u64 = 1; pub const TESTING_ID: u64 = 339; -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] -pub struct Validator { - pub addr: Address, - pub net_addr: String, - // voting power for the validator determined by its stake in the - // network. - pub weight: TokenAmount, -} - -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] -pub struct ValidatorSet { - validators: Vec, - // sequence number that uniquely identifies a validator set - configuration_number: u64, -} - -impl ValidatorSet { - pub fn new() -> Self { - Self { - validators: Vec::new(), - configuration_number: 0, - } - } - - pub fn validators(&self) -> &Vec { - &self.validators - } - - pub fn validators_mut(&mut self) -> &mut Vec { - &mut self.validators - } - - pub fn config_number(&self) -> u64 { - self.configuration_number - } - - /// Push a new validator to the validator set. - pub fn push(&mut self, val: Validator) { - self.validators.push(val); - // update the config_number with every update - // we allow config_number to overflow if that scenario ever comes. - self.configuration_number += 1; - } - - /// Remove a validator from validator set by address - pub fn rm(&mut self, val: &Address) { - self.validators.retain(|x| x.addr != *val); - // update the config_number with every update - // we allow config_number to overflow if that scenario ever comes. - self.configuration_number += 1; - } - - pub fn update_weight(&mut self, val: &Address, weight: &TokenAmount) { - self.validators_mut() - .iter_mut() - .filter(|x| x.addr == *val) - .for_each(|x| x.weight = weight.clone()); - - self.configuration_number += 1; - } -} - #[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] pub struct Votes { pub validators: Vec
, From 2cedc625a67bf397daf697b2dab036b875618880 Mon Sep 17 00:00:00 2001 From: willesxm Date: Wed, 15 Mar 2023 18:30:54 +0800 Subject: [PATCH 02/27] fmt code --- gateway/tests/harness.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway/tests/harness.rs b/gateway/tests/harness.rs index e867c1b..5d66e78 100644 --- a/gateway/tests/harness.rs +++ b/gateway/tests/harness.rs @@ -20,6 +20,7 @@ use fvm_ipld_encoding::RawBytes; use fvm_shared::address::Address; use fvm_shared::bigint::bigint_ser::BigIntDe; use fvm_shared::bigint::Zero; +use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use fvm_shared::error::ExitCode; use fvm_shared::MethodNum; @@ -35,7 +36,6 @@ use ipc_gateway::{ use lazy_static::lazy_static; use primitives::{TCid, TCidContent}; use std::str::FromStr; -use fvm_shared::clock::ChainEpoch; lazy_static! { pub static ref ROOTNET_ID: SubnetID = From 6dc45bdd863501888da8e50c22bbdf55342d6e2a Mon Sep 17 00:00:00 2001 From: cryptoAtwill <108330426+cryptoAtwill@users.noreply.github.com> Date: Wed, 15 Mar 2023 20:15:41 +0800 Subject: [PATCH 03/27] Update gateway/src/state.rs Co-authored-by: adlrocha --- gateway/src/state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway/src/state.rs b/gateway/src/state.rs index 93875f7..0dc266f 100644 --- a/gateway/src/state.rs +++ b/gateway/src/state.rs @@ -46,7 +46,7 @@ pub struct State { pub bottomup_msg_meta: TCid>, pub applied_bottomup_nonce: u64, pub applied_topdown_nonce: u64, - /// The epoch that this actor is deployed + /// The epoch that the subnet actor is deployed pub genesis_epoch: ChainEpoch, /// How often cron checkpoints will be submitted by validator in the child subnet pub cron_period: ChainEpoch, From 51ec87f458b045bde842952ee19cc080cdf564af Mon Sep 17 00:00:00 2001 From: cryptoAtwill <108330426+cryptoAtwill@users.noreply.github.com> Date: Wed, 15 Mar 2023 20:15:48 +0800 Subject: [PATCH 04/27] Update gateway/src/types.rs Co-authored-by: adlrocha --- gateway/src/types.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gateway/src/types.rs b/gateway/src/types.rs index 47dcb18..3ca99e0 100644 --- a/gateway/src/types.rs +++ b/gateway/src/types.rs @@ -102,6 +102,8 @@ impl PostBoxItem { } } +/// Checkpoints propagated from parent to child to signal the "final view" of the parent chain +/// from the different validators in the subnet. #[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] pub struct CronCheckpoint { pub epoch: ChainEpoch, From 304d549828d1cdd3f3345c48978ddacbd38e85e7 Mon Sep 17 00:00:00 2001 From: cryptoAtwill <108330426+cryptoAtwill@users.noreply.github.com> Date: Wed, 15 Mar 2023 20:49:06 +0800 Subject: [PATCH 05/27] update cron (#65) * update cron * fix lint --------- Co-authored-by: willesxm --- gateway/src/lib.rs | 2 +- gateway/src/state.rs | 8 ++------ gateway/src/types.rs | 5 ++++- gateway/tests/harness.rs | 7 +++---- subnet-actor/src/lib.rs | 2 +- subnet-actor/src/state.rs | 9 ++++++++- subnet-actor/tests/actor_test.rs | 5 +++++ 7 files changed, 24 insertions(+), 14 deletions(-) diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 8e0ad08..5498811 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -70,7 +70,7 @@ impl Actor { fn constructor(rt: &mut impl Runtime, params: ConstructorParams) -> Result<(), ActorError> { rt.validate_immediate_caller_is(std::iter::once(&INIT_ACTOR_ADDR))?; - let st = State::new(rt.store(), params, rt.curr_epoch()).map_err(|e| { + let st = State::new(rt.store(), params).map_err(|e| { e.downcast_default( ExitCode::USR_ILLEGAL_STATE, "Failed to create SCA actor state", diff --git a/gateway/src/state.rs b/gateway/src/state.rs index 0dc266f..6b06207 100644 --- a/gateway/src/state.rs +++ b/gateway/src/state.rs @@ -57,11 +57,7 @@ lazy_static! { } impl State { - pub fn new( - store: &BS, - params: ConstructorParams, - current_epoch: ChainEpoch, - ) -> anyhow::Result { + pub fn new(store: &BS, params: ConstructorParams) -> anyhow::Result { Ok(State { network_name: SubnetID::from_str(¶ms.network_name)?, total_subnets: Default::default(), @@ -81,7 +77,7 @@ impl State { // We first increase to the subsequent and then execute for bottom-up messages applied_bottomup_nonce: MAX_NONCE, applied_topdown_nonce: Default::default(), - genesis_epoch: current_epoch, + genesis_epoch: params.genesis_epoch, cron_period: params.cron_period, }) } diff --git a/gateway/src/types.rs b/gateway/src/types.rs index 3ca99e0..65f9068 100644 --- a/gateway/src/types.rs +++ b/gateway/src/types.rs @@ -33,6 +33,7 @@ pub struct ConstructorParams { pub network_name: String, pub checkpoint_period: ChainEpoch, pub cron_period: ChainEpoch, + pub genesis_epoch: ChainEpoch, } #[derive(Serialize_tuple, Deserialize_tuple, Clone)] @@ -103,7 +104,7 @@ impl PostBoxItem { } /// Checkpoints propagated from parent to child to signal the "final view" of the parent chain -/// from the different validators in the subnet. +/// from the different validators in the subnet. #[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] pub struct CronCheckpoint { pub epoch: ChainEpoch, @@ -122,6 +123,7 @@ mod tests { network_name: "/root".to_string(), checkpoint_period: 100, cron_period: 20, + genesis_epoch: 10, }; let bytes = fil_actors_runtime::util::cbor::serialize(&p, "").unwrap(); let serialized = base64::encode(bytes.bytes()); @@ -134,5 +136,6 @@ mod tests { assert_eq!(p.network_name, deserialized.network_name); assert_eq!(p.checkpoint_period, deserialized.checkpoint_period); assert_eq!(p.cron_period, deserialized.cron_period); + assert_eq!(p.genesis_epoch, deserialized.genesis_epoch); } } diff --git a/gateway/tests/harness.rs b/gateway/tests/harness.rs index 5d66e78..133e6cd 100644 --- a/gateway/tests/harness.rs +++ b/gateway/tests/harness.rs @@ -48,6 +48,7 @@ lazy_static! { pub static ref ACTOR: Address = Address::new_actor("actor".as_bytes()); pub static ref SIG_TYPES: Vec = vec![*ACCOUNT_ACTOR_CODE_ID, *MULTISIG_ACTOR_CODE_ID]; pub static ref DEFAULT_CRON_PERIOD: ChainEpoch = 20; + pub static ref DEFAULT_GENESIS_EPOCH: ChainEpoch = 1; } pub fn new_runtime() -> MockRuntime { @@ -86,6 +87,7 @@ impl Harness { network_name: self.net_name.to_string(), checkpoint_period: 10, cron_period: *DEFAULT_CRON_PERIOD, + genesis_epoch: *DEFAULT_GENESIS_EPOCH, }; rt.set_caller(*INIT_ACTOR_CODE_ID, INIT_ACTOR_ADDR); rt.call::( @@ -96,9 +98,6 @@ impl Harness { } pub fn construct_and_verify(&self, rt: &mut MockRuntime) { - let chain_epoch = 10; - rt.set_epoch(chain_epoch); - self.construct(rt); let st: State = rt.get_state(); @@ -113,8 +112,8 @@ impl Harness { assert_eq!(st.check_period, DEFAULT_CHECKPOINT_PERIOD); assert_eq!(st.applied_bottomup_nonce, MAX_NONCE); assert_eq!(st.bottomup_msg_meta.cid(), empty_bottomup_array); - assert_eq!(st.genesis_epoch, chain_epoch); assert_eq!(st.cron_period, *DEFAULT_CRON_PERIOD); + assert_eq!(st.genesis_epoch, *DEFAULT_GENESIS_EPOCH); verify_empty_map(rt, st.subnets.cid()); verify_empty_map(rt, st.checkpoints.cid()); verify_empty_map(rt, st.check_msg_registry.cid()); diff --git a/subnet-actor/src/lib.rs b/subnet-actor/src/lib.rs index c67efbc..083b0a8 100644 --- a/subnet-actor/src/lib.rs +++ b/subnet-actor/src/lib.rs @@ -79,7 +79,7 @@ impl SubnetActor for Actor { fn constructor(rt: &mut impl Runtime, params: ConstructParams) -> Result<(), ActorError> { rt.validate_immediate_caller_is(std::iter::once(&INIT_ACTOR_ADDR))?; - let st = State::new(rt.store(), params).map_err(|e| { + let st = State::new(rt.store(), params, rt.curr_epoch()).map_err(|e| { e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "Failed to create actor state") })?; diff --git a/subnet-actor/src/state.rs b/subnet-actor/src/state.rs index 09120e9..29a8984 100644 --- a/subnet-actor/src/state.rs +++ b/subnet-actor/src/state.rs @@ -47,13 +47,18 @@ pub struct State { pub window_checks: TCid>, pub validator_set: ValidatorSet, pub min_validators: u64, + pub genesis_epoch: ChainEpoch, } /// We should probably have a derive macro to mark an object as a state object, /// and have load and save methods automatically generated for them as part of a /// StateObject trait (i.e. impl StateObject for State). impl State { - pub fn new(store: &BS, params: ConstructParams) -> anyhow::Result { + pub fn new( + store: &BS, + params: ConstructParams, + current_epoch: ChainEpoch, + ) -> anyhow::Result { let min_stake = TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT); let state = State { @@ -80,6 +85,7 @@ impl State { stake: TCid::new_hamt(store)?, window_checks: TCid::new_hamt(store)?, validator_set: ValidatorSet::default(), + genesis_epoch: current_epoch, }; Ok(state) @@ -394,6 +400,7 @@ impl Default for State { window_checks: TCid::default(), validator_set: ValidatorSet::default(), min_validators: 0, + genesis_epoch: 0, } } } diff --git a/subnet-actor/tests/actor_test.rs b/subnet-actor/tests/actor_test.rs index 5314a02..51403bd 100644 --- a/subnet-actor/tests/actor_test.rs +++ b/subnet-actor/tests/actor_test.rs @@ -10,6 +10,7 @@ mod test { use fvm_ipld_encoding::ipld_block::IpldBlock; use fvm_ipld_encoding::RawBytes; use fvm_shared::address::Address; + use fvm_shared::clock::ChainEpoch; use fvm_shared::crypto::signature::Signature; use fvm_shared::econ::TokenAmount; use fvm_shared::error::ExitCode; @@ -28,6 +29,7 @@ mod test { // just a test address const IPC_GATEWAY_ADDR: u64 = 1024; const NETWORK_NAME: &'static str = "test"; + const DEFAULT_CHAIN_EPOCH: ChainEpoch = 10; lazy_static! { pub static ref SIG_TYPES: Vec = vec![*ACCOUNT_ACTOR_CODE_ID, *MULTISIG_ACTOR_CODE_ID]; @@ -63,6 +65,8 @@ mod test { runtime.expect_validate_caller_addr(vec![INIT_ACTOR_ADDR]); + runtime.set_epoch(DEFAULT_CHAIN_EPOCH); + runtime .call::( Method::Constructor as u64, @@ -88,6 +92,7 @@ mod test { assert_eq!(state.ipc_gateway_addr, Address::new_id(IPC_GATEWAY_ADDR)); assert_eq!(state.total_stake, TokenAmount::zero()); assert_eq!(state.validator_set.validators().is_empty(), true); + assert_eq!(state.genesis_epoch, DEFAULT_CHAIN_EPOCH); } #[test] From 27007e1a07d402bd0773bb96ba2dd610e1fc147f Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Thu, 16 Mar 2023 16:30:38 +0800 Subject: [PATCH 06/27] add submit cron impl --- gateway/src/cron.rs | 253 +++++++++++++++++++++++++++++++++++++++++++ gateway/src/lib.rs | 75 ++++++++++++- gateway/src/state.rs | 3 + gateway/src/types.rs | 11 -- 4 files changed, 330 insertions(+), 12 deletions(-) create mode 100644 gateway/src/cron.rs diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs new file mode 100644 index 0000000..6bab748 --- /dev/null +++ b/gateway/src/cron.rs @@ -0,0 +1,253 @@ +use crate::StorableMsg; +use anyhow::anyhow; +use cid::multihash::Code; +use cid::multihash::MultihashDigest; +use fvm_ipld_blockstore::Blockstore; +use fvm_ipld_encoding::to_vec; +use fvm_ipld_encoding::tuple::{Deserialize_tuple, Serialize_tuple}; +use fvm_ipld_hamt::BytesKey; +use fvm_shared::address::Address; +use fvm_shared::clock::ChainEpoch; +use ipc_sdk::ValidatorSet; +use primitives::{TCid, THamt}; +use std::cmp::Ordering; +use std::collections::HashSet; + +pub type HashOutput = Vec; +const RATIO_NUMERATOR: u16 = 2; +const RATIO_DENOMINATOR: u16 = 3; + +/// Checkpoints propagated from parent to child to signal the "final view" of the parent chain +/// from the different validators in the subnet. +#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] +pub struct CronCheckpoint { + pub epoch: ChainEpoch, + pub validators: ValidatorSet, + pub top_down_msgs: Vec, +} + +impl CronCheckpoint { + /// Hash the checkpoint. + /// + /// To compare the cron checkpoint and ensure they are the same, we need to make sure the + /// validators and top_down_msgs are the same. However, the top_down_msgs and validators are vec, + /// they may contain the same content, but their orders are different. In this case, we need to + /// ensure the same order is maintained in the cron checkpoint submission. + /// + /// To ensure we have the same consistent output for different submissions, we require: + /// - validators are sorted by `net_addr` in string ascending order + /// - top down messages are sorted by (from, to, nonce) in descending order + /// + /// Actor will not perform sorting to save gas. Client should do it, actor just check. + fn hash(&self) -> anyhow::Result { + // check validators + let validators = self.validators.validators(); + for i in 1..validators.len() { + match validators[i - 1].net_addr.cmp(&validators[i].net_addr) { + Ordering::Less => {} + Ordering::Equal => return Err(anyhow!("validators not unique")), + Ordering::Greater => return Err(anyhow!("validators not sorted")), + }; + } + + // check top down msgs + for i in 1..self.top_down_msgs.len() { + match compare_top_down_msg(&self.top_down_msgs[i - 1], &self.top_down_msgs[i])? { + Ordering::Less => {} + Ordering::Equal => return Err(anyhow!("top down messages not distinct")), + Ordering::Greater => return Err(anyhow!("top down messages not sorted")), + }; + } + + let mh_code = Code::Blake2b256; + Ok(mh_code.digest(&to_vec(self).unwrap()).to_bytes()) + } +} + +/// Track all the cron checkpoint submissions of an epoch +#[derive(Serialize_tuple, Deserialize_tuple, PartialEq, Eq, Clone)] +pub struct CronSubmission { + /// The total number of submitters + total_submitters: u16, + /// All the submitters + submitters: TCid>, + /// The most submitted hash. Using set because there might be a tie + most_submitted_hashes: Option>, + /// The binary heap to track the max submitted + submission_counts: TCid>, + /// The different cron checkpoints, with cron checkpoint hash as key + submissions: TCid>, +} + +impl CronSubmission { + pub fn new(store: &BS) -> anyhow::Result { + Ok(CronSubmission { + total_submitters: 0, + submitters: TCid::new_hamt(store)?, + most_submitted_hashes: None, + submission_counts: Default::default(), + submissions: TCid::new_hamt(store)?, + }) + } + + pub fn submit( + &mut self, + store: &BS, + submitter: Address, + checkpoint: CronCheckpoint, + ) -> anyhow::Result { + let total_submitters = self.update_submitters(store, submitter)?; + + let checkpoint_hash = self.insert_checkpoint(store, checkpoint)?; + let most_submitted_count = self.update_submission_count(store, checkpoint_hash)?; + + // use u16 numerator and denominator to avoid floating point calculation and external crate + if total_submitters * RATIO_NUMERATOR / RATIO_DENOMINATOR > most_submitted_count { + return Ok(false); + } + + Ok(true) + } + + pub fn load_most_submitted_checkpoint( + &self, + store: &BS, + ) -> anyhow::Result> { + if let Some(most_submitted_hashes) = &self.most_submitted_hashes { + // we will only have one entry in the `most_submitted` set + let hash = most_submitted_hashes.iter().next().unwrap(); + self.get_submission(store, hash) + } else { + Ok(None) + } + } + + pub fn get_submission( + &self, + store: &BS, + hash: &HashOutput, + ) -> anyhow::Result> { + let hamt = self.submissions.load(store)?; + let key = BytesKey::from(hash.as_slice()); + Ok(hamt.get(&key)?.cloned()) + } + + /// Update the total submitters, returns the latest total number of submitters + fn update_submitters( + &mut self, + store: &BS, + submitter: Address, + ) -> anyhow::Result { + let addr_byte_key = BytesKey::from(submitter.to_bytes()); + self.submitters.modify(store, |hamt| { + // check the submitter has not submitted before + if hamt.contains_key(&addr_byte_key)? { + return Err(anyhow!("already submitted")); + } + + // now the submitter has not submitted before, mark as submitted + hamt.set(addr_byte_key, ())?; + self.total_submitters += 1; + + Ok(self.total_submitters) + }) + } + + /// Insert the checkpoint to store if it has not been submitted before. Returns the hash of the checkpoint. + fn insert_checkpoint( + &mut self, + store: &BS, + checkpoint: CronCheckpoint, + ) -> anyhow::Result { + let hash = checkpoint.hash()?; + let hash_key = BytesKey::from(hash.as_slice()); + self.submissions.modify(store, |hamt| { + if hamt.contains_key(&hash_key)? { + return Ok(()); + } + + // checkpoint has not submitted before + hamt.set(hash_key, checkpoint)?; + + Ok(()) + })?; + Ok(hash) + } + + /// Update submission count of the hash. Returns the currently most submitted submission count. + fn update_submission_count( + &mut self, + store: &BS, + hash: HashOutput, + ) -> anyhow::Result { + let hash_byte_key = BytesKey::from(hash.as_slice()); + + self.submission_counts.modify(store, |hamt| { + let new_count = hamt.get(&hash_byte_key)?.map(|v| v + 1).unwrap_or(1); + + // update the new count + hamt.set(hash_byte_key, new_count)?; + + // now we compare with the most submitted hash or cron checkpoint + if self.most_submitted_hashes.is_none() { + // no most submitted hash set yet, set to current + let mut set = HashSet::new(); + set.insert(hash); + self.most_submitted_hashes = Some(set); + return Ok(new_count); + } + + let most_submitted_hashes = self.most_submitted_hashes.as_mut().unwrap(); + + // the current submission is already one of the most submitted entries + if most_submitted_hashes.contains(&hash) { + if most_submitted_hashes.len() != 1 { + // we have more than 1 checkpoint with most number of submissions + // now, with the new submission, the current checkpoint will be the most + // submitted checkpoint, remove other submissions. + most_submitted_hashes.clear(); + most_submitted_hashes.insert(hash); + } + + // the current submission is already the only one submission, no need update + + // return the current checkpoint's count as the current most submitted checkpoint + return Ok(new_count); + } + + // the current submission is not part of the most submitted entries, need to check + // the most submitted entry to compare if the current submission is exceeding + + // save to unwrap at the set cannot be empty + let most_submitted_hash = most_submitted_hashes.iter().next().unwrap(); + let most_submitted_key = BytesKey::from(most_submitted_hash.as_slice()); + + // safe to unwrap as the hamt must contain the key + let most_submitted_count = hamt.get(&most_submitted_key)?.unwrap(); + + // current submission was not found in the most submitted checkpoints, the count gas is + // at least 1, new_count > *most_submitted_count will not happen + // if new_count < *most_submitted_count, we do nothing as the new count is not close to the most submitted + if new_count == *most_submitted_count { + most_submitted_hashes.insert(hash); + } + + Ok(*most_submitted_count) + }) + } +} + +/// Compare the ordering of two storable messages. +fn compare_top_down_msg(a: &StorableMsg, b: &StorableMsg) -> anyhow::Result { + let ordering = a.from.raw_addr()?.cmp(&b.from.raw_addr()?); + if ordering != Ordering::Equal { + return Ok(ordering); + } + + let ordering = a.to.raw_addr()?.cmp(&b.to.raw_addr()?); + if ordering != Ordering::Equal { + return Ok(ordering); + } + + Ok(a.nonce.cmp(&b.nonce)) +} diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 5498811..7147db8 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -1,10 +1,14 @@ #![feature(let_chains)] // For some simpler syntax for if let Some conditions +extern crate core; + pub use self::checkpoint::{Checkpoint, CrossMsgMeta}; pub use self::cross::{is_bottomup, CrossMsg, CrossMsgs, IPCMsgType, StorableMsg}; pub use self::state::*; pub use self::subnet::*; pub use self::types::*; +use crate::cron::CronSubmission; +use cron::CronCheckpoint; use cross::{burn_bu_funds, cross_msg_side_effects, distribute_crossmsg_fee}; use fil_actors_runtime::runtime::fvm::resolve_secp_bls; use fil_actors_runtime::runtime::{ActorCode, Runtime}; @@ -13,6 +17,7 @@ use fil_actors_runtime::{ CALLER_TYPES_SIGNABLE, INIT_ACTOR_ADDR, SYSTEM_ACTOR_ADDR, }; use fvm_ipld_encoding::RawBytes; +use fvm_ipld_hamt::BytesKey; use fvm_shared::address::Address; use fvm_shared::bigint::Zero; use fvm_shared::econ::TokenAmount; @@ -30,6 +35,7 @@ use primitives::TCid; fil_actors_runtime::wasm_trampoline!(Actor); pub mod checkpoint; +mod cron; mod cross; mod error; #[doc(hidden)] @@ -60,6 +66,7 @@ pub enum Method { ApplyMessage = frc42_dispatch::method_hash!("ApplyMessage"), Propagate = frc42_dispatch::method_hash!("Propagate"), WhiteListPropagator = frc42_dispatch::method_hash!("WhiteListPropagator"), + SubmitCron = frc42_dispatch::method_hash!("SubmitCron"), } /// Gateway Actor @@ -594,9 +601,11 @@ impl Actor { /// - And updated the latest nonce applied for future checks. fn apply_msg(rt: &mut impl Runtime, params: ApplyMsgParams) -> Result { rt.validate_immediate_caller_is([&SYSTEM_ACTOR_ADDR as &Address])?; - let ApplyMsgParams { cross_msg } = params; + Self::apply_msg_inner(rt, cross_msg) + } + fn apply_msg_inner(rt: &mut impl Runtime, cross_msg: CrossMsg) -> Result { let rto = match cross_msg.msg.to.raw_addr() { Ok(to) => to, Err(_) => { @@ -787,6 +796,69 @@ impl Actor { Ok(()) } + fn submit_cron(rt: &mut impl Runtime, params: CronCheckpoint) -> Result { + // submit cron can only be performed by signable addresses + rt.validate_immediate_caller_type(CALLER_TYPES_SIGNABLE.iter())?; + + let msgs = rt.transaction(|st: &mut State, rt| { + // first we check the epoch is the correct one + let genesis_epoch = st.genesis_epoch; + + // we process only it's multiple of cron_period since genesis_epoch + if (params.epoch - genesis_epoch) % st.cron_period != 0 { + return Err(actor_error!(illegal_argument, "epoch not allowed")); + } + + let store = rt.store(); + let submitter = rt.message().caller(); + + st.cron_submissions + .modify(store, |hamt| { + let epoch_key = BytesKey::from(params.epoch.to_be_bytes().as_slice()); + let mut submission = match hamt.get(&epoch_key)? { + Some(s) => s.clone(), + None => CronSubmission::new(store)?, + }; + + let reached_limit = submission.submit(store, submitter, params)?; + + if !reached_limit { + return Ok(None); + } + + let msgs = submission + .load_most_submitted_checkpoint(store)? + .unwrap() + .top_down_msgs; + + hamt.set(epoch_key, submission)?; + + Ok(Some(msgs)) + }) + .map_err(|e| { + log::error!( + "encountered error processing submit cron checkpoint: {:?}", + e + ); + actor_error!(unhandled_message, e.to_string()) + }) + })?; + + if let Some(msgs) = msgs { + // TODO: we might need to batch the execution so that it's atomic + for m in msgs { + Self::apply_msg_inner( + rt, + CrossMsg { + msg: m, + wrapped: false, + }, + )?; + } + } + Ok(RawBytes::default()) + } + /// Commit the cross message to storage. It outputs a flag signaling /// if the committed messages was bottom-up and some funds need to be /// burnt or if a top-down message fee needs to be distributed. @@ -880,5 +952,6 @@ impl ActorCode for Actor { ApplyMessage => apply_msg, Propagate => propagate, WhiteListPropagator => whitelist_propagator, + SubmitCron => submit_cron, } } diff --git a/gateway/src/state.rs b/gateway/src/state.rs index 6b06207..61e92e2 100644 --- a/gateway/src/state.rs +++ b/gateway/src/state.rs @@ -16,6 +16,7 @@ use primitives::{TAmt, TCid, THamt, TLink}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; use std::str::FromStr; +use crate::cron::CronSubmission; use ipc_sdk::subnet_id::SubnetID; use super::checkpoint::*; @@ -50,6 +51,7 @@ pub struct State { pub genesis_epoch: ChainEpoch, /// How often cron checkpoints will be submitted by validator in the child subnet pub cron_period: ChainEpoch, + pub cron_submissions: TCid>, } lazy_static! { @@ -79,6 +81,7 @@ impl State { applied_topdown_nonce: Default::default(), genesis_epoch: params.genesis_epoch, cron_period: params.cron_period, + cron_submissions: TCid::new_hamt(store)?, }) } diff --git a/gateway/src/types.rs b/gateway/src/types.rs index 65f9068..1389fe6 100644 --- a/gateway/src/types.rs +++ b/gateway/src/types.rs @@ -7,13 +7,11 @@ use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use ipc_sdk::subnet_id::SubnetID; -use ipc_sdk::ValidatorSet; use multihash::MultihashDigest; use primitives::CodeType; use crate::checkpoint::{Checkpoint, CrossMsgMeta}; use crate::cross::CrossMsg; -use crate::StorableMsg; /// ID used in the builtin-actors bundle manifest pub const MANIFEST_ID: &str = "ipc_gateway"; @@ -103,15 +101,6 @@ impl PostBoxItem { } } -/// Checkpoints propagated from parent to child to signal the "final view" of the parent chain -/// from the different validators in the subnet. -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] -pub struct CronCheckpoint { - pub epoch: ChainEpoch, - pub membership: ValidatorSet, - pub top_down_msgs: Vec, -} - #[cfg(test)] mod tests { use crate::ConstructorParams; From 7e22f00ec36ee03356f345f8b11eed1381588564 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Thu, 16 Mar 2023 16:43:18 +0800 Subject: [PATCH 07/27] add more checks --- gateway/src/lib.rs | 15 +++++++++++++-- gateway/src/state.rs | 3 +++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 7147db8..f0e7464 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -796,6 +796,14 @@ impl Actor { Ok(()) } + /// Submit a new cron checkpoint + /// + /// It only accepts submission at multiples of `cron_period` since `genesis_epoch`, which are + /// set during construction. Each checkpoint will have its number of submissions tracked. The + /// same address cannot submit twice. Once the number of submissions is more than or equal to 2/3 + /// of the total number of validators, the messages will be applied. + /// + /// Each cron checkpoint will be checked against each other using blake hashing. fn submit_cron(rt: &mut impl Runtime, params: CronCheckpoint) -> Result { // submit cron can only be performed by signable addresses rt.validate_immediate_caller_type(CALLER_TYPES_SIGNABLE.iter())?; @@ -820,17 +828,20 @@ impl Actor { None => CronSubmission::new(store)?, }; + let epoch = params.epoch; let reached_limit = submission.submit(store, submitter, params)?; - if !reached_limit { + if !reached_limit || st.last_cron_executed_epoch + st.cron_period != epoch{ + hamt.set(epoch_key, submission)?; return Ok(None); } + st.last_cron_executed_epoch = epoch; + let msgs = submission .load_most_submitted_checkpoint(store)? .unwrap() .top_down_msgs; - hamt.set(epoch_key, submission)?; Ok(Some(msgs)) diff --git a/gateway/src/state.rs b/gateway/src/state.rs index 61e92e2..229fa77 100644 --- a/gateway/src/state.rs +++ b/gateway/src/state.rs @@ -51,6 +51,8 @@ pub struct State { pub genesis_epoch: ChainEpoch, /// How often cron checkpoints will be submitted by validator in the child subnet pub cron_period: ChainEpoch, + /// The last submit cron epoch that was executed + pub last_cron_executed_epoch: ChainEpoch, pub cron_submissions: TCid>, } @@ -81,6 +83,7 @@ impl State { applied_topdown_nonce: Default::default(), genesis_epoch: params.genesis_epoch, cron_period: params.cron_period, + last_cron_executed_epoch: params.genesis_epoch, cron_submissions: TCid::new_hamt(store)?, }) } From 25e8044223abb8e6fadf88ffa20871f35a745c8a Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Thu, 16 Mar 2023 16:54:58 +0800 Subject: [PATCH 08/27] add some todo --- gateway/src/cron.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index 6bab748..00add02 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -60,6 +60,9 @@ impl CronCheckpoint { } let mh_code = Code::Blake2b256; + // TODO: to avoid serialization again, maybe we should perform deserialization in the actor + // TODO: dispatch call to save gas? The actor dispatching contains the raw serialized data, + // TODO: which we dont have to serialize here again Ok(mh_code.digest(&to_vec(self).unwrap()).to_bytes()) } } From 5d35dbe2595b83c6c83d8595153d6fc0aadd3f35 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Thu, 16 Mar 2023 17:03:19 +0800 Subject: [PATCH 09/27] derive total validators --- gateway/src/cron.rs | 16 ++++++++-------- gateway/src/lib.rs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index 00add02..1dec7cc 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -70,8 +70,6 @@ impl CronCheckpoint { /// Track all the cron checkpoint submissions of an epoch #[derive(Serialize_tuple, Deserialize_tuple, PartialEq, Eq, Clone)] pub struct CronSubmission { - /// The total number of submitters - total_submitters: u16, /// All the submitters submitters: TCid>, /// The most submitted hash. Using set because there might be a tie @@ -85,7 +83,6 @@ pub struct CronSubmission { impl CronSubmission { pub fn new(store: &BS) -> anyhow::Result { Ok(CronSubmission { - total_submitters: 0, submitters: TCid::new_hamt(store)?, most_submitted_hashes: None, submission_counts: Default::default(), @@ -99,13 +96,17 @@ impl CronSubmission { submitter: Address, checkpoint: CronCheckpoint, ) -> anyhow::Result { - let total_submitters = self.update_submitters(store, submitter)?; + // TODO: validation of validator set is correct + let total_validators = checkpoint.validators.validators().len(); + + self.update_submitters(store, submitter)?; let checkpoint_hash = self.insert_checkpoint(store, checkpoint)?; let most_submitted_count = self.update_submission_count(store, checkpoint_hash)?; // use u16 numerator and denominator to avoid floating point calculation and external crate - if total_submitters * RATIO_NUMERATOR / RATIO_DENOMINATOR > most_submitted_count { + // total validators should be within u16::MAX. + if total_validators as u16 * RATIO_NUMERATOR / RATIO_DENOMINATOR > most_submitted_count { return Ok(false); } @@ -140,7 +141,7 @@ impl CronSubmission { &mut self, store: &BS, submitter: Address, - ) -> anyhow::Result { + ) -> anyhow::Result<()> { let addr_byte_key = BytesKey::from(submitter.to_bytes()); self.submitters.modify(store, |hamt| { // check the submitter has not submitted before @@ -150,9 +151,8 @@ impl CronSubmission { // now the submitter has not submitted before, mark as submitted hamt.set(addr_byte_key, ())?; - self.total_submitters += 1; - Ok(self.total_submitters) + Ok(()) }) } diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index f0e7464..f9838f5 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -831,7 +831,7 @@ impl Actor { let epoch = params.epoch; let reached_limit = submission.submit(store, submitter, params)?; - if !reached_limit || st.last_cron_executed_epoch + st.cron_period != epoch{ + if !reached_limit || st.last_cron_executed_epoch + st.cron_period != epoch { hamt.set(epoch_key, submission)?; return Ok(None); } From 8182b7bb75dea0b21678b8593d245be57f6f6e64 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Thu, 16 Mar 2023 17:04:56 +0800 Subject: [PATCH 10/27] add todo --- gateway/src/cron.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index 1dec7cc..d4e331c 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -96,7 +96,7 @@ impl CronSubmission { submitter: Address, checkpoint: CronCheckpoint, ) -> anyhow::Result { - // TODO: validation of validator set is correct + // TODO: Add validation of validator set logic so that we know the set of validators is correct let total_validators = checkpoint.validators.validators().len(); self.update_submitters(store, submitter)?; From 63d1817e64bc84fec35d9d0cf5f440b3496f0c43 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Thu, 16 Mar 2023 17:31:42 +0800 Subject: [PATCH 11/27] specify rust tool chain --- rust-toolchain.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 rust-toolchain.toml diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..4518cf1 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "nightly-2022-10-03" +components = ["clippy", "llvm-tools-preview", "rustfmt"] +targets = ["wasm32-unknown-unknown"] From 5c1c68c21c7d8398d83acfa3438d911a97ed835c Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Fri, 17 Mar 2023 14:30:00 +0800 Subject: [PATCH 12/27] add tests --- gateway/src/cron.rs | 187 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 179 insertions(+), 8 deletions(-) diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index d4e331c..9eb941d 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -74,7 +74,7 @@ pub struct CronSubmission { submitters: TCid>, /// The most submitted hash. Using set because there might be a tie most_submitted_hashes: Option>, - /// The binary heap to track the max submitted + /// The map to track the max submitted submission_counts: TCid>, /// The different cron checkpoints, with cron checkpoint hash as key submissions: TCid>, @@ -90,6 +90,8 @@ impl CronSubmission { }) } + /// Submit a cron checkpoint as the submitter. Returns `true` if the submission threshold + /// is reached, else `false`. pub fn submit( &mut self, store: &BS, @@ -118,7 +120,7 @@ impl CronSubmission { store: &BS, ) -> anyhow::Result> { if let Some(most_submitted_hashes) = &self.most_submitted_hashes { - // we will only have one entry in the `most_submitted` set + // we will only have one entry in the `most_submitted` set if more than 2/3 has reached let hash = most_submitted_hashes.iter().next().unwrap(); self.get_submission(store, hash) } else { @@ -164,16 +166,18 @@ impl CronSubmission { ) -> anyhow::Result { let hash = checkpoint.hash()?; let hash_key = BytesKey::from(hash.as_slice()); - self.submissions.modify(store, |hamt| { - if hamt.contains_key(&hash_key)? { - return Ok(()); - } - // checkpoint has not submitted before - hamt.set(hash_key, checkpoint)?; + let hamt = self.submissions.load(store)?; + if hamt.contains_key(&hash_key)? { + return Ok(hash); + } + // checkpoint has not submitted before + self.submissions.modify(store, |hamt| { + hamt.set(hash_key, checkpoint)?; Ok(()) })?; + Ok(hash) } @@ -238,6 +242,42 @@ impl CronSubmission { Ok(*most_submitted_count) }) } + + /// Checks if the submitter has already submitted the checkpoint. Currently used only in + /// tests, but can be used in prod as well. + #[cfg(test)] + fn has_submitted( + &self, + store: &BS, + submitter: &Address, + ) -> anyhow::Result { + let addr_byte_key = BytesKey::from(submitter.to_bytes()); + let hamt = self.submitters.load(store)?; + Ok(hamt.contains_key(&addr_byte_key)?) + } + + /// Checks if the checkpoint hash has already inserted in the store + #[cfg(test)] + fn has_checkpoint_inserted( + &self, + store: &BS, + hash: &HashOutput + ) -> anyhow::Result { + let hamt = self.submissions.load(store)?; + Ok(hamt.contains_key(&BytesKey::from(hash.as_slice()))?) + } + + /// Checks if the checkpoint hash has already inserted in the store + #[cfg(test)] + fn get_submission_count( + &self, + store: &BS, + hash: &HashOutput + ) -> anyhow::Result> { + let hamt = self.submission_counts.load(store)?; + let r = hamt.get(&BytesKey::from(hash.as_slice()))?; + Ok(r.cloned()) + } } /// Compare the ordering of two storable messages. @@ -254,3 +294,134 @@ fn compare_top_down_msg(a: &StorableMsg, b: &StorableMsg) -> anyhow::Result { + { + let mut h = std::collections::HashSet::new(); + $( + h.insert($x); + )* + Some(h) + } + } + } + + #[test] + fn test_new_works() { + let store = MemoryBlockstore::new(); + let r = CronSubmission::new(&store); + assert!(r.is_ok()); + } + + #[test] + fn test_compare_top_down_msg() { + let a = StorableMsg{ + from: IPCAddress::new(&ROOTNET_ID, &Address::new_id(0)).unwrap(), + to: IPCAddress::new(&ROOTNET_ID, &Address::new_id(1)).unwrap(), + method: 0, + params: RawBytes::default(), + value: TokenAmount::from_whole(1), + nonce: 0, + }; + + let b = StorableMsg{ + from: IPCAddress::new(&ROOTNET_ID, &Address::new_id(0)).unwrap(), + to: IPCAddress::new(&ROOTNET_ID, &Address::new_id(1)).unwrap(), + method: 0, + params: RawBytes::default(), + value: TokenAmount::from_whole(1), + nonce: 2, + }; + + assert_eq!(compare_top_down_msg(&a, &b).unwrap(), Ordering::Less); + } + + #[test] + fn test_update_submitters() { + let store = MemoryBlockstore::new(); + let mut submission = CronSubmission::new(&store).unwrap(); + + let submitter = Address::new_id(0); + submission.update_submitters(&store, submitter).unwrap(); + assert!(submission.has_submitted(&store, &submitter).unwrap()); + + // now submit again, but should fail + assert!(submission.update_submitters(&store, submitter).is_err()); + } + + #[test] + fn test_insert_checkpoint() { + let store = MemoryBlockstore::new(); + let mut submission = CronSubmission::new(&store).unwrap(); + + let checkpoint = CronCheckpoint{ + epoch: 100, + validators: Default::default(), + top_down_msgs: vec![] + }; + + let hash = checkpoint.hash().unwrap(); + + submission.insert_checkpoint(&store, checkpoint.clone()).unwrap(); + assert!(submission.has_checkpoint_inserted(&store, &hash).unwrap()); + + // insert again should not have caused any error + submission.insert_checkpoint(&store, checkpoint.clone()).unwrap(); + + let inserted_checkpoint = submission.get_submission(&store, &hash).unwrap().unwrap(); + assert_eq!(inserted_checkpoint, checkpoint); + } + + #[test] + fn test_update_submission_count() { + let store = MemoryBlockstore::new(); + let mut submission = CronSubmission::new(&store).unwrap(); + + let hash1 = vec![1, 2, 1]; + let hash2 = vec![1, 2, 2]; + let hash3 = vec![1, 2, 3]; + + // insert hash1, should have only one item + assert_eq!(submission.most_submitted_hashes, None); + assert_eq!(submission.update_submission_count(&store, hash1.clone()).unwrap(), 1); + assert_eq!(submission.get_submission_count(&store, &hash1).unwrap().unwrap(), 1); + assert_eq!(submission.most_submitted_hashes, some_hashset!(hash1.clone())); + + // insert hash2, we should have two items, and there is a tie + assert_eq!(submission.update_submission_count(&store, hash2.clone()).unwrap(), 1); + assert_eq!(submission.get_submission_count(&store, &hash2).unwrap().unwrap(), 1); + assert_eq!(submission.get_submission_count(&store, &hash1).unwrap().unwrap(), 1); + assert_eq!(submission.most_submitted_hashes, some_hashset!(hash1.clone(), hash2.clone())); + + // insert hash3, we should have three items, and there is still a tie + assert_eq!(submission.update_submission_count(&store, hash3.clone()).unwrap(), 1); + assert_eq!(submission.get_submission_count(&store, &hash3).unwrap().unwrap(), 1); + assert_eq!(submission.get_submission_count(&store, &hash2).unwrap().unwrap(), 1); + assert_eq!(submission.get_submission_count(&store, &hash1).unwrap().unwrap(), 1); + assert_eq!(submission.most_submitted_hashes, some_hashset!(hash3.clone(), hash1.clone(), hash2.clone())); + + // insert hash1 again, we should have only 1 most submitted hash + assert_eq!(submission.update_submission_count(&store, hash1.clone()).unwrap(), 2); + assert_eq!(submission.get_submission_count(&store, &hash1).unwrap().unwrap(), 2); + assert_eq!(submission.most_submitted_hashes, some_hashset!(hash1.clone())); + + // insert hash1 again, we should have only 1 most submitted hash, but count incr by 1 + assert_eq!(submission.update_submission_count(&store, hash1.clone()).unwrap(), 3); + assert_eq!(submission.get_submission_count(&store, &hash1).unwrap().unwrap(), 3); + assert_eq!(submission.most_submitted_hashes, some_hashset!(hash1.clone())); + assert_eq!(submission.most_submitted_hashes.unwrap().len(), 1); + } +} \ No newline at end of file From b2bf4d47e8c9c0a6f78683e479ff6c1f3442626b Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Mon, 20 Mar 2023 14:55:11 +0800 Subject: [PATCH 13/27] support abort --- .github/workflows/ci.yml | 4 +- gateway/src/cron.rs | 363 ++++++++++++++++++++++++++++----------- gateway/src/lib.rs | 30 +++- 3 files changed, 294 insertions(+), 103 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2692af..43b1c5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: with: profile: minimal target: wasm32-unknown-unknown - toolchain: nightly + toolchain: nightly-2022-10-03 override: true - run: cargo b --all - run: cargo t --all @@ -49,7 +49,7 @@ jobs: with: profile: minimal target: wasm32-unknown-unknown - toolchain: nightly + toolchain: nightly-2022-10-03 override: true - run: rustup component add clippy - uses: actions-rs/cargo@v1 diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index 9eb941d..df92ca9 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -11,7 +11,6 @@ use fvm_shared::clock::ChainEpoch; use ipc_sdk::ValidatorSet; use primitives::{TCid, THamt}; use std::cmp::Ordering; -use std::collections::HashSet; pub type HashOutput = Vec; const RATIO_NUMERATOR: u16 = 2; @@ -70,10 +69,14 @@ impl CronCheckpoint { /// Track all the cron checkpoint submissions of an epoch #[derive(Serialize_tuple, Deserialize_tuple, PartialEq, Eq, Clone)] pub struct CronSubmission { - /// All the submitters + /// Whether the voting has been executed once consensus has reached + executed: bool, + /// Total number of submissions + total_submissions: u16, + /// The most submitted hash. + most_voted_hash: Option, + /// The addresses of all the submitters submitters: TCid>, - /// The most submitted hash. Using set because there might be a tie - most_submitted_hashes: Option>, /// The map to track the max submitted submission_counts: TCid>, /// The different cron checkpoints, with cron checkpoint hash as key @@ -83,13 +86,39 @@ pub struct CronSubmission { impl CronSubmission { pub fn new(store: &BS) -> anyhow::Result { Ok(CronSubmission { + executed: false, + total_submissions: 0, submitters: TCid::new_hamt(store)?, - most_submitted_hashes: None, - submission_counts: Default::default(), + most_voted_hash: None, + submission_counts: TCid::new_hamt(store)?, submissions: TCid::new_hamt(store)?, }) } + /// Abort the current round and reset the submission data. + pub fn abort(&mut self, store: &BS) -> anyhow::Result<()> { + // not need to reset executed as it's still false + + self.total_submissions = 0; + self.submitters = TCid::new_hamt(store)?; + self.most_voted_hash = None; + self.submission_counts = TCid::new_hamt(store)?; + + // no need reset submissions, we can still reuse the previous submissions + // new submissions will be inserted, old submission will not be inserted to save + // gas. + + Ok(()) + } + + pub fn executed(&mut self) { + self.executed = true; + } + + pub fn is_executed(&self) -> bool { + self.executed + } + /// Submit a cron checkpoint as the submitter. Returns `true` if the submission threshold /// is reached, else `false`. pub fn submit( @@ -97,31 +126,27 @@ impl CronSubmission { store: &BS, submitter: Address, checkpoint: CronCheckpoint, - ) -> anyhow::Result { - // TODO: Add validation of validator set logic so that we know the set of validators is correct + ) -> anyhow::Result { + // TODO: remove this once tracking of validators are added let total_validators = checkpoint.validators.validators().len(); - self.update_submitters(store, submitter)?; - + let total_submissions = self.update_submitters(store, submitter)?; let checkpoint_hash = self.insert_checkpoint(store, checkpoint)?; let most_submitted_count = self.update_submission_count(store, checkpoint_hash)?; - // use u16 numerator and denominator to avoid floating point calculation and external crate - // total validators should be within u16::MAX. - if total_validators as u16 * RATIO_NUMERATOR / RATIO_DENOMINATOR > most_submitted_count { - return Ok(false); - } - - Ok(true) + Ok(Self::derive_execution_status( + total_validators as u16, + total_submissions, + most_submitted_count, + )) } pub fn load_most_submitted_checkpoint( &self, store: &BS, ) -> anyhow::Result> { - if let Some(most_submitted_hashes) = &self.most_submitted_hashes { - // we will only have one entry in the `most_submitted` set if more than 2/3 has reached - let hash = most_submitted_hashes.iter().next().unwrap(); + // we will only have one entry in the `most_submitted` set if more than 2/3 has reached + if let Some(hash) = &self.most_voted_hash { self.get_submission(store, hash) } else { Ok(None) @@ -137,13 +162,72 @@ impl CronSubmission { let key = BytesKey::from(hash.as_slice()); Ok(hamt.get(&key)?.cloned()) } +} + +/// The status indicating if the voting should be executed +#[derive(Eq, PartialEq, Debug)] +pub enum VoteExecutionStatus { + /// The execution threshold has yet to be reached + ThresholdNotReached, + /// The voting threshold has reached, but consensus has yet to be reached, needs more + /// voting to reach consensus + ReachingConsensus, + /// Consensus cannot be reached in this round + RoundAbort, + /// Execution threshold reached + ConsensusReached, +} + +impl CronSubmission { + fn derive_execution_status( + total_validators: u16, + total_submissions: u16, + most_voted_count: u16, + ) -> VoteExecutionStatus { + // use u16 numerator and denominator to avoid floating point calculation and external crate + // total validators should be within u16::MAX. + let threshold = total_validators as u16 * RATIO_NUMERATOR / RATIO_DENOMINATOR; + + // note that we require THRESHOLD to be surpassed, equality is not enough! + + if total_submissions <= threshold { + return VoteExecutionStatus::ThresholdNotReached; + } + + // now we have reached the threshold + + // consensus reached + if most_voted_count > threshold { + return VoteExecutionStatus::ConsensusReached; + } + + // now the total submissions has reached the threshold, but the most submitted vote + // has yet to reach the threshold, that means consensus has not reached. + + // we do a early termination check, to see if consensus will ever be reached. + // + // consider an example that consensus will never be reached: + // + // -------- | -------------------------|--------------- | ------------- | + // MOST_VOTED THRESHOLD TOTAL_SUBMISSIONS TOTAL_VALIDATORS + // + // we see MOST_VOTED is smaller than THRESHOLD, TOTAL_SUBMISSIONS and TOTAL_VALIDATORS, if + // the potential extra votes any vote can obtain, i.e. TOTAL_VALIDATORS - TOTAL_SUBMISSIONS, + // is smaller than or equal to the potential extra vote the most voted can obtain, i.e. + // THRESHOLD - MOST_VOTED, then consensus will never be reached, no point voting, just abort. + if threshold - most_voted_count >= total_validators - total_submissions { + VoteExecutionStatus::RoundAbort + } else { + VoteExecutionStatus::ReachingConsensus + } + } /// Update the total submitters, returns the latest total number of submitters fn update_submitters( &mut self, store: &BS, submitter: Address, - ) -> anyhow::Result<()> { + ) -> anyhow::Result { let addr_byte_key = BytesKey::from(submitter.to_bytes()); self.submitters.modify(store, |hamt| { // check the submitter has not submitted before @@ -153,8 +237,9 @@ impl CronSubmission { // now the submitter has not submitted before, mark as submitted hamt.set(addr_byte_key, ())?; + self.total_submissions += 1; - Ok(()) + Ok(self.total_submissions) }) } @@ -196,26 +281,16 @@ impl CronSubmission { hamt.set(hash_byte_key, new_count)?; // now we compare with the most submitted hash or cron checkpoint - if self.most_submitted_hashes.is_none() { + if self.most_voted_hash.is_none() { // no most submitted hash set yet, set to current - let mut set = HashSet::new(); - set.insert(hash); - self.most_submitted_hashes = Some(set); + self.most_voted_hash = Some(hash); return Ok(new_count); } - let most_submitted_hashes = self.most_submitted_hashes.as_mut().unwrap(); + let most_submitted_hash = self.most_voted_hash.as_mut().unwrap(); // the current submission is already one of the most submitted entries - if most_submitted_hashes.contains(&hash) { - if most_submitted_hashes.len() != 1 { - // we have more than 1 checkpoint with most number of submissions - // now, with the new submission, the current checkpoint will be the most - // submitted checkpoint, remove other submissions. - most_submitted_hashes.clear(); - most_submitted_hashes.insert(hash); - } - + if most_submitted_hash == &hash { // the current submission is already the only one submission, no need update // return the current checkpoint's count as the current most submitted checkpoint @@ -225,21 +300,19 @@ impl CronSubmission { // the current submission is not part of the most submitted entries, need to check // the most submitted entry to compare if the current submission is exceeding - // save to unwrap at the set cannot be empty - let most_submitted_hash = most_submitted_hashes.iter().next().unwrap(); let most_submitted_key = BytesKey::from(most_submitted_hash.as_slice()); // safe to unwrap as the hamt must contain the key let most_submitted_count = hamt.get(&most_submitted_key)?.unwrap(); - // current submission was not found in the most submitted checkpoints, the count gas is - // at least 1, new_count > *most_submitted_count will not happen + // current submission is not the most voted checkpoints // if new_count < *most_submitted_count, we do nothing as the new count is not close to the most submitted - if new_count == *most_submitted_count { - most_submitted_hashes.insert(hash); + if new_count > *most_submitted_count { + *most_submitted_hash = hash; + Ok(new_count) + } else { + Ok(*most_submitted_count) } - - Ok(*most_submitted_count) }) } @@ -261,7 +334,7 @@ impl CronSubmission { fn has_checkpoint_inserted( &self, store: &BS, - hash: &HashOutput + hash: &HashOutput, ) -> anyhow::Result { let hamt = self.submissions.load(store)?; Ok(hamt.contains_key(&BytesKey::from(hash.as_slice()))?) @@ -272,7 +345,7 @@ impl CronSubmission { fn get_submission_count( &self, store: &BS, - hash: &HashOutput + hash: &HashOutput, ) -> anyhow::Result> { let hamt = self.submission_counts.load(store)?; let r = hamt.get(&BytesKey::from(hash.as_slice()))?; @@ -297,27 +370,15 @@ fn compare_top_down_msg(a: &StorableMsg, b: &StorableMsg) -> anyhow::Result { - { - let mut h = std::collections::HashSet::new(); - $( - h.insert($x); - )* - Some(h) - } - } - } + use std::cmp::Ordering; #[test] fn test_new_works() { @@ -328,7 +389,7 @@ mod tests { #[test] fn test_compare_top_down_msg() { - let a = StorableMsg{ + let a = StorableMsg { from: IPCAddress::new(&ROOTNET_ID, &Address::new_id(0)).unwrap(), to: IPCAddress::new(&ROOTNET_ID, &Address::new_id(1)).unwrap(), method: 0, @@ -337,7 +398,7 @@ mod tests { nonce: 0, }; - let b = StorableMsg{ + let b = StorableMsg { from: IPCAddress::new(&ROOTNET_ID, &Address::new_id(0)).unwrap(), to: IPCAddress::new(&ROOTNET_ID, &Address::new_id(1)).unwrap(), method: 0, @@ -367,19 +428,23 @@ mod tests { let store = MemoryBlockstore::new(); let mut submission = CronSubmission::new(&store).unwrap(); - let checkpoint = CronCheckpoint{ + let checkpoint = CronCheckpoint { epoch: 100, validators: Default::default(), - top_down_msgs: vec![] + top_down_msgs: vec![], }; let hash = checkpoint.hash().unwrap(); - submission.insert_checkpoint(&store, checkpoint.clone()).unwrap(); + submission + .insert_checkpoint(&store, checkpoint.clone()) + .unwrap(); assert!(submission.has_checkpoint_inserted(&store, &hash).unwrap()); // insert again should not have caused any error - submission.insert_checkpoint(&store, checkpoint.clone()).unwrap(); + submission + .insert_checkpoint(&store, checkpoint.clone()) + .unwrap(); let inserted_checkpoint = submission.get_submission(&store, &hash).unwrap().unwrap(); assert_eq!(inserted_checkpoint, checkpoint); @@ -392,36 +457,140 @@ mod tests { let hash1 = vec![1, 2, 1]; let hash2 = vec![1, 2, 2]; - let hash3 = vec![1, 2, 3]; // insert hash1, should have only one item - assert_eq!(submission.most_submitted_hashes, None); - assert_eq!(submission.update_submission_count(&store, hash1.clone()).unwrap(), 1); - assert_eq!(submission.get_submission_count(&store, &hash1).unwrap().unwrap(), 1); - assert_eq!(submission.most_submitted_hashes, some_hashset!(hash1.clone())); - - // insert hash2, we should have two items, and there is a tie - assert_eq!(submission.update_submission_count(&store, hash2.clone()).unwrap(), 1); - assert_eq!(submission.get_submission_count(&store, &hash2).unwrap().unwrap(), 1); - assert_eq!(submission.get_submission_count(&store, &hash1).unwrap().unwrap(), 1); - assert_eq!(submission.most_submitted_hashes, some_hashset!(hash1.clone(), hash2.clone())); - - // insert hash3, we should have three items, and there is still a tie - assert_eq!(submission.update_submission_count(&store, hash3.clone()).unwrap(), 1); - assert_eq!(submission.get_submission_count(&store, &hash3).unwrap().unwrap(), 1); - assert_eq!(submission.get_submission_count(&store, &hash2).unwrap().unwrap(), 1); - assert_eq!(submission.get_submission_count(&store, &hash1).unwrap().unwrap(), 1); - assert_eq!(submission.most_submitted_hashes, some_hashset!(hash3.clone(), hash1.clone(), hash2.clone())); - - // insert hash1 again, we should have only 1 most submitted hash - assert_eq!(submission.update_submission_count(&store, hash1.clone()).unwrap(), 2); - assert_eq!(submission.get_submission_count(&store, &hash1).unwrap().unwrap(), 2); - assert_eq!(submission.most_submitted_hashes, some_hashset!(hash1.clone())); - - // insert hash1 again, we should have only 1 most submitted hash, but count incr by 1 - assert_eq!(submission.update_submission_count(&store, hash1.clone()).unwrap(), 3); - assert_eq!(submission.get_submission_count(&store, &hash1).unwrap().unwrap(), 3); - assert_eq!(submission.most_submitted_hashes, some_hashset!(hash1.clone())); - assert_eq!(submission.most_submitted_hashes.unwrap().len(), 1); + assert_eq!(submission.most_voted_hash, None); + assert_eq!( + submission + .update_submission_count(&store, hash1.clone()) + .unwrap(), + 1 + ); + assert_eq!( + submission + .get_submission_count(&store, &hash1) + .unwrap() + .unwrap(), + 1 + ); + assert_eq!(submission.most_voted_hash, Some(hash1.clone())); + + // insert hash2, we should have two items, and there is a tie, hash1 still the most voted + assert_eq!( + submission + .update_submission_count(&store, hash2.clone()) + .unwrap(), + 1 + ); + assert_eq!( + submission + .get_submission_count(&store, &hash2) + .unwrap() + .unwrap(), + 1 + ); + assert_eq!( + submission + .get_submission_count(&store, &hash1) + .unwrap() + .unwrap(), + 1 + ); + assert_eq!(submission.most_voted_hash, Some(hash1.clone())); + + // insert hash2 again, we should have only 1 most submitted hash + assert_eq!( + submission + .update_submission_count(&store, hash2.clone()) + .unwrap(), + 2 + ); + assert_eq!( + submission + .get_submission_count(&store, &hash2) + .unwrap() + .unwrap(), + 2 + ); + assert_eq!(submission.most_voted_hash, Some(hash2.clone())); + + // insert hash2 again, we should have only 1 most submitted hash, but count incr by 1 + assert_eq!( + submission + .update_submission_count(&store, hash2.clone()) + .unwrap(), + 3 + ); + assert_eq!( + submission + .get_submission_count(&store, &hash2) + .unwrap() + .unwrap(), + 3 + ); + assert_eq!(submission.most_voted_hash, Some(hash2.clone())); + } + + #[test] + fn test_derive_execution_status() { + let total_validators = 35; + let total_submissions = 10; + let most_voted_count = 5; + assert_eq!( + CronSubmission::derive_execution_status( + total_validators, + total_submissions, + most_voted_count + ), + VoteExecutionStatus::ThresholdNotReached, + ); + + // We could have 3 submissions: A, B, C + // Current submissions and their counts are: A - 2, B - 2. + // If the threshold is 1 / 2, we could have: + // If the last vote is C, then we should abort. + // If the last vote is any of A or B, we can execute. + // If the threshold is 1 / 3, we have to abort. + let total_validators = 5; + let total_submissions = 4; + let most_voted_count = 2; + assert_eq!( + CronSubmission::derive_execution_status( + total_validators, + total_submissions, + most_voted_count + ), + VoteExecutionStatus::RoundAbort, + ); + + // We could have 1 submission: A + // Current submissions and their counts are: A - 4. + let total_validators = 5; + let total_submissions = 4; + let most_voted_count = 4; + assert_eq!( + CronSubmission::derive_execution_status( + total_validators, + total_submissions, + most_voted_count + ), + VoteExecutionStatus::ConsensusReached, + ); + + // We could have 2 submission: A, B + // Current submissions and their counts are: A - 3, B - 1. + // Say the threshold is 2 / 3. If the last vote is B, we should abort, if the last vote is + // A, then we have reached consensus. The current votes are in conclusive. + let total_validators = 5; + let total_submissions = 4; + let most_voted_count = 3; + assert_eq!( + CronSubmission::derive_execution_status( + total_validators, + total_submissions, + most_voted_count + ), + VoteExecutionStatus::ReachingConsensus, + ); } -} \ No newline at end of file +} diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index f9838f5..580181a 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -7,7 +7,8 @@ pub use self::cross::{is_bottomup, CrossMsg, CrossMsgs, IPCMsgType, StorableMsg} pub use self::state::*; pub use self::subnet::*; pub use self::types::*; -use crate::cron::CronSubmission; +use crate::cron::{CronSubmission, VoteExecutionStatus}; +use anyhow::anyhow; use cron::CronCheckpoint; use cross::{burn_bu_funds, cross_msg_side_effects, distribute_crossmsg_fee}; use fil_actors_runtime::runtime::fvm::resolve_secp_bls; @@ -828,20 +829,42 @@ impl Actor { None => CronSubmission::new(store)?, }; + if submission.is_executed() { + return Err(anyhow!("epoch already executed")); + } + let epoch = params.epoch; - let reached_limit = submission.submit(store, submitter, params)?; + let execution_status = submission.submit(store, submitter, params)?; - if !reached_limit || st.last_cron_executed_epoch + st.cron_period != epoch { + if st.last_cron_executed_epoch + st.cron_period != epoch { + // there are pending epoch to be executed, + // just store the submission and skip execution hamt.set(epoch_key, submission)?; return Ok(None); } + match execution_status { + VoteExecutionStatus::ThresholdNotReached + | VoteExecutionStatus::ReachingConsensus => { + // threshold or consensus not reached, store submission and return + hamt.set(epoch_key, submission)?; + return Ok(None); + } + VoteExecutionStatus::RoundAbort => { + submission.abort(store)?; + hamt.set(epoch_key, submission)?; + return Ok(None); + } + VoteExecutionStatus::ConsensusReached => {} + } + st.last_cron_executed_epoch = epoch; let msgs = submission .load_most_submitted_checkpoint(store)? .unwrap() .top_down_msgs; + submission.executed(); hamt.set(epoch_key, submission)?; Ok(Some(msgs)) @@ -856,7 +879,6 @@ impl Actor { })?; if let Some(msgs) = msgs { - // TODO: we might need to batch the execution so that it's atomic for m in msgs { Self::apply_msg_inner( rt, From 1e712421ac9bc58a09254adbd6c7ba9872e66ee0 Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Mon, 20 Mar 2023 17:22:39 +0800 Subject: [PATCH 14/27] simplify impl --- gateway/src/cron.rs | 90 +++++++-------------------------------------- gateway/src/lib.rs | 13 +++---- 2 files changed, 19 insertions(+), 84 deletions(-) diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index df92ca9..eb105c0 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -29,29 +29,21 @@ impl CronCheckpoint { /// Hash the checkpoint. /// /// To compare the cron checkpoint and ensure they are the same, we need to make sure the - /// validators and top_down_msgs are the same. However, the top_down_msgs and validators are vec, - /// they may contain the same content, but their orders are different. In this case, we need to - /// ensure the same order is maintained in the cron checkpoint submission. + /// top_down_msgs are the same. However, the top_down_msgs are vec, they may contain the same + /// content, but their orders are different. In this case, we need to ensure the same order is + /// maintained in the cron checkpoint submission. /// /// To ensure we have the same consistent output for different submissions, we require: - /// - validators are sorted by `net_addr` in string ascending order - /// - top down messages are sorted by (from, to, nonce) in descending order + /// - top down messages are sorted by `nonce` in descending order /// /// Actor will not perform sorting to save gas. Client should do it, actor just check. fn hash(&self) -> anyhow::Result { - // check validators - let validators = self.validators.validators(); - for i in 1..validators.len() { - match validators[i - 1].net_addr.cmp(&validators[i].net_addr) { - Ordering::Less => {} - Ordering::Equal => return Err(anyhow!("validators not unique")), - Ordering::Greater => return Err(anyhow!("validators not sorted")), - }; - } - // check top down msgs for i in 1..self.top_down_msgs.len() { - match compare_top_down_msg(&self.top_down_msgs[i - 1], &self.top_down_msgs[i])? { + match self.top_down_msgs[i - 1] + .nonce + .cmp(&self.top_down_msgs[i].nonce) + { Ordering::Less => {} Ordering::Equal => return Err(anyhow!("top down messages not distinct")), Ordering::Greater => return Err(anyhow!("top down messages not sorted")), @@ -69,9 +61,7 @@ impl CronCheckpoint { /// Track all the cron checkpoint submissions of an epoch #[derive(Serialize_tuple, Deserialize_tuple, PartialEq, Eq, Clone)] pub struct CronSubmission { - /// Whether the voting has been executed once consensus has reached - executed: bool, - /// Total number of submissions + /// Total number of submissions from validators total_submissions: u16, /// The most submitted hash. most_voted_hash: Option, @@ -86,7 +76,6 @@ pub struct CronSubmission { impl CronSubmission { pub fn new(store: &BS) -> anyhow::Result { Ok(CronSubmission { - executed: false, total_submissions: 0, submitters: TCid::new_hamt(store)?, most_voted_hash: None, @@ -97,28 +86,18 @@ impl CronSubmission { /// Abort the current round and reset the submission data. pub fn abort(&mut self, store: &BS) -> anyhow::Result<()> { - // not need to reset executed as it's still false - self.total_submissions = 0; self.submitters = TCid::new_hamt(store)?; self.most_voted_hash = None; self.submission_counts = TCid::new_hamt(store)?; - // no need reset submissions, we can still reuse the previous submissions + // no need reset `self.submissions`, we can still reuse the previous self.submissions // new submissions will be inserted, old submission will not be inserted to save // gas. Ok(()) } - pub fn executed(&mut self) { - self.executed = true; - } - - pub fn is_executed(&self) -> bool { - self.executed - } - /// Submit a cron checkpoint as the submitter. Returns `true` if the submission threshold /// is reached, else `false`. pub fn submit( @@ -127,7 +106,8 @@ impl CronSubmission { submitter: Address, checkpoint: CronCheckpoint, ) -> anyhow::Result { - // TODO: remove this once tracking of validators are added + // TODO: remove this once tracking of validators are added, we will get the validator + // TODO: set from a field in the gateway after the next PR. let total_validators = checkpoint.validators.validators().len(); let total_submissions = self.update_submitters(store, submitter)?; @@ -353,32 +333,11 @@ impl CronSubmission { } } -/// Compare the ordering of two storable messages. -fn compare_top_down_msg(a: &StorableMsg, b: &StorableMsg) -> anyhow::Result { - let ordering = a.from.raw_addr()?.cmp(&b.from.raw_addr()?); - if ordering != Ordering::Equal { - return Ok(ordering); - } - - let ordering = a.to.raw_addr()?.cmp(&b.to.raw_addr()?); - if ordering != Ordering::Equal { - return Ok(ordering); - } - - Ok(a.nonce.cmp(&b.nonce)) -} - #[cfg(test)] mod tests { - use crate::cron::compare_top_down_msg; - use crate::{CronCheckpoint, CronSubmission, StorableMsg, VoteExecutionStatus}; + use crate::{CronCheckpoint, CronSubmission, VoteExecutionStatus}; use fvm_ipld_blockstore::MemoryBlockstore; - use fvm_ipld_encoding::RawBytes; use fvm_shared::address::Address; - use fvm_shared::econ::TokenAmount; - use ipc_sdk::address::IPCAddress; - use ipc_sdk::subnet_id::ROOTNET_ID; - use std::cmp::Ordering; #[test] fn test_new_works() { @@ -387,29 +346,6 @@ mod tests { assert!(r.is_ok()); } - #[test] - fn test_compare_top_down_msg() { - let a = StorableMsg { - from: IPCAddress::new(&ROOTNET_ID, &Address::new_id(0)).unwrap(), - to: IPCAddress::new(&ROOTNET_ID, &Address::new_id(1)).unwrap(), - method: 0, - params: RawBytes::default(), - value: TokenAmount::from_whole(1), - nonce: 0, - }; - - let b = StorableMsg { - from: IPCAddress::new(&ROOTNET_ID, &Address::new_id(0)).unwrap(), - to: IPCAddress::new(&ROOTNET_ID, &Address::new_id(1)).unwrap(), - method: 0, - params: RawBytes::default(), - value: TokenAmount::from_whole(1), - nonce: 2, - }; - - assert_eq!(compare_top_down_msg(&a, &b).unwrap(), Ordering::Less); - } - #[test] fn test_update_submitters() { let store = MemoryBlockstore::new(); diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 580181a..28a5e2f 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -8,7 +8,6 @@ pub use self::state::*; pub use self::subnet::*; pub use self::types::*; use crate::cron::{CronSubmission, VoteExecutionStatus}; -use anyhow::anyhow; use cron::CronCheckpoint; use cross::{burn_bu_funds, cross_msg_side_effects, distribute_crossmsg_fee}; use fil_actors_runtime::runtime::fvm::resolve_secp_bls; @@ -818,6 +817,10 @@ impl Actor { return Err(actor_error!(illegal_argument, "epoch not allowed")); } + if st.last_cron_executed_epoch >= params.epoch { + return Err(actor_error!(illegal_argument, "epoch already executed")); + } + let store = rt.store(); let submitter = rt.message().caller(); @@ -829,10 +832,6 @@ impl Actor { None => CronSubmission::new(store)?, }; - if submission.is_executed() { - return Err(anyhow!("epoch already executed")); - } - let epoch = params.epoch; let execution_status = submission.submit(store, submitter, params)?; @@ -858,14 +857,14 @@ impl Actor { VoteExecutionStatus::ConsensusReached => {} } + // we reach consensus in the checkpoints submission st.last_cron_executed_epoch = epoch; let msgs = submission .load_most_submitted_checkpoint(store)? .unwrap() .top_down_msgs; - submission.executed(); - hamt.set(epoch_key, submission)?; + hamt.delete(&epoch_key)?; Ok(Some(msgs)) }) From 2cc63cc708f1a86d09f701c41f4a0652764b6f9a Mon Sep 17 00:00:00 2001 From: cryptoAtwill <108330426+cryptoAtwill@users.noreply.github.com> Date: Mon, 20 Mar 2023 21:59:34 +0800 Subject: [PATCH 15/27] Track validators (#70) * track validators * add validator check to submit cron * update impl --- gateway/src/cron.rs | 127 ++++++++++++++++++++++--------------------- gateway/src/lib.rs | 25 ++++++++- gateway/src/state.rs | 13 ++++- 3 files changed, 102 insertions(+), 63 deletions(-) diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index eb105c0..c66f0e5 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -8,7 +8,9 @@ use fvm_ipld_encoding::tuple::{Deserialize_tuple, Serialize_tuple}; use fvm_ipld_hamt::BytesKey; use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; +use fvm_shared::econ::TokenAmount; use ipc_sdk::ValidatorSet; +use num_traits::Zero; use primitives::{TCid, THamt}; use std::cmp::Ordering; @@ -16,12 +18,38 @@ pub type HashOutput = Vec; const RATIO_NUMERATOR: u16 = 2; const RATIO_DENOMINATOR: u16 = 3; +/// Validators tracks all the validator in the subnet. It is useful in handling cron checkpoints. +#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] +pub struct Validators { + /// The validator set that holds all the validators + pub validators: ValidatorSet, + /// Tracks the total weight of the validators + pub total_weight: TokenAmount, +} + +impl Validators { + pub fn new(validators: ValidatorSet) -> Self { + let mut weight = TokenAmount::zero(); + for v in validators.validators() { + weight += v.weight.clone(); + } + Self { + validators, + total_weight: weight, + } + } + + /// Checks if an address is a validator + pub fn is_validator(&self, addr: &Address) -> bool { + self.validators.validators().iter().any(|x| x.addr == *addr) + } +} + /// Checkpoints propagated from parent to child to signal the "final view" of the parent chain /// from the different validators in the subnet. #[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] pub struct CronCheckpoint { pub epoch: ChainEpoch, - pub validators: ValidatorSet, pub top_down_msgs: Vec, } @@ -74,7 +102,7 @@ pub struct CronSubmission { } impl CronSubmission { - pub fn new(store: &BS) -> anyhow::Result { + pub fn new(store: &BS) -> anyhow::Result { Ok(CronSubmission { total_submissions: 0, submitters: TCid::new_hamt(store)?, @@ -98,27 +126,16 @@ impl CronSubmission { Ok(()) } - /// Submit a cron checkpoint as the submitter. Returns `true` if the submission threshold - /// is reached, else `false`. + /// Submit a cron checkpoint as the submitter. pub fn submit( &mut self, store: &BS, submitter: Address, checkpoint: CronCheckpoint, - ) -> anyhow::Result { - // TODO: remove this once tracking of validators are added, we will get the validator - // TODO: set from a field in the gateway after the next PR. - let total_validators = checkpoint.validators.validators().len(); - - let total_submissions = self.update_submitters(store, submitter)?; + ) -> anyhow::Result { + self.update_submitters(store, submitter)?; let checkpoint_hash = self.insert_checkpoint(store, checkpoint)?; - let most_submitted_count = self.update_submission_count(store, checkpoint_hash)?; - - Ok(Self::derive_execution_status( - total_validators as u16, - total_submissions, - most_submitted_count, - )) + self.update_submission_count(store, checkpoint_hash) } pub fn load_most_submitted_checkpoint( @@ -142,26 +159,10 @@ impl CronSubmission { let key = BytesKey::from(hash.as_slice()); Ok(hamt.get(&key)?.cloned()) } -} -/// The status indicating if the voting should be executed -#[derive(Eq, PartialEq, Debug)] -pub enum VoteExecutionStatus { - /// The execution threshold has yet to be reached - ThresholdNotReached, - /// The voting threshold has reached, but consensus has yet to be reached, needs more - /// voting to reach consensus - ReachingConsensus, - /// Consensus cannot be reached in this round - RoundAbort, - /// Execution threshold reached - ConsensusReached, -} - -impl CronSubmission { - fn derive_execution_status( + pub fn derive_execution_status( + &self, total_validators: u16, - total_submissions: u16, most_voted_count: u16, ) -> VoteExecutionStatus { // use u16 numerator and denominator to avoid floating point calculation and external crate @@ -169,8 +170,7 @@ impl CronSubmission { let threshold = total_validators as u16 * RATIO_NUMERATOR / RATIO_DENOMINATOR; // note that we require THRESHOLD to be surpassed, equality is not enough! - - if total_submissions <= threshold { + if self.total_submissions <= threshold { return VoteExecutionStatus::ThresholdNotReached; } @@ -195,13 +195,29 @@ impl CronSubmission { // the potential extra votes any vote can obtain, i.e. TOTAL_VALIDATORS - TOTAL_SUBMISSIONS, // is smaller than or equal to the potential extra vote the most voted can obtain, i.e. // THRESHOLD - MOST_VOTED, then consensus will never be reached, no point voting, just abort. - if threshold - most_voted_count >= total_validators - total_submissions { + if threshold - most_voted_count >= total_validators - self.total_submissions { VoteExecutionStatus::RoundAbort } else { VoteExecutionStatus::ReachingConsensus } } +} + +/// The status indicating if the voting should be executed +#[derive(Eq, PartialEq, Debug)] +pub enum VoteExecutionStatus { + /// The execution threshold has yet to be reached + ThresholdNotReached, + /// The voting threshold has reached, but consensus has yet to be reached, needs more + /// voting to reach consensus + ReachingConsensus, + /// Consensus cannot be reached in this round + RoundAbort, + /// Execution threshold reached + ConsensusReached, +} +impl CronSubmission { /// Update the total submitters, returns the latest total number of submitters fn update_submitters( &mut self, @@ -366,7 +382,6 @@ mod tests { let checkpoint = CronCheckpoint { epoch: 100, - validators: Default::default(), top_down_msgs: vec![], }; @@ -469,15 +484,16 @@ mod tests { #[test] fn test_derive_execution_status() { + let store = MemoryBlockstore::new(); + let mut s = CronSubmission::new(&store).unwrap(); + let total_validators = 35; let total_submissions = 10; let most_voted_count = 5; + + s.total_submissions = total_submissions; assert_eq!( - CronSubmission::derive_execution_status( - total_validators, - total_submissions, - most_voted_count - ), + s.derive_execution_status(total_validators, most_voted_count), VoteExecutionStatus::ThresholdNotReached, ); @@ -490,26 +506,19 @@ mod tests { let total_validators = 5; let total_submissions = 4; let most_voted_count = 2; + s.total_submissions = total_submissions; assert_eq!( - CronSubmission::derive_execution_status( - total_validators, - total_submissions, - most_voted_count - ), + s.derive_execution_status(total_submissions, most_voted_count), VoteExecutionStatus::RoundAbort, ); // We could have 1 submission: A // Current submissions and their counts are: A - 4. - let total_validators = 5; let total_submissions = 4; let most_voted_count = 4; + s.total_submissions = total_submissions; assert_eq!( - CronSubmission::derive_execution_status( - total_validators, - total_submissions, - most_voted_count - ), + s.derive_execution_status(total_validators, most_voted_count), VoteExecutionStatus::ConsensusReached, ); @@ -517,15 +526,11 @@ mod tests { // Current submissions and their counts are: A - 3, B - 1. // Say the threshold is 2 / 3. If the last vote is B, we should abort, if the last vote is // A, then we have reached consensus. The current votes are in conclusive. - let total_validators = 5; let total_submissions = 4; let most_voted_count = 3; + s.total_submissions = total_submissions; assert_eq!( - CronSubmission::derive_execution_status( - total_validators, - total_submissions, - most_voted_count - ), + s.derive_execution_status(total_validators, most_voted_count), VoteExecutionStatus::ReachingConsensus, ); } diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 28a5e2f..61d9cc9 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -26,6 +26,7 @@ use fvm_shared::METHOD_SEND; use fvm_shared::{MethodNum, METHOD_CONSTRUCTOR}; pub use ipc_sdk::address::IPCAddress; pub use ipc_sdk::subnet_id::SubnetID; +use ipc_sdk::ValidatorSet; use lazy_static::lazy_static; use num_derive::FromPrimitive; use num_traits::FromPrimitive; @@ -67,6 +68,7 @@ pub enum Method { Propagate = frc42_dispatch::method_hash!("Propagate"), WhiteListPropagator = frc42_dispatch::method_hash!("WhiteListPropagator"), SubmitCron = frc42_dispatch::method_hash!("SubmitCron"), + SetMembership = frc42_dispatch::method_hash!("SetMembership"), } /// Gateway Actor @@ -796,6 +798,18 @@ impl Actor { Ok(()) } + /// Set the memberships of the validators + fn set_membership( + rt: &mut impl Runtime, + validator_set: ValidatorSet, + ) -> Result { + rt.validate_immediate_caller_is([&SYSTEM_ACTOR_ADDR as &Address])?; + rt.transaction(|st: &mut State, _| { + st.set_membership(validator_set); + Ok(RawBytes::default()) + }) + } + /// Submit a new cron checkpoint /// /// It only accepts submission at multiples of `cron_period` since `genesis_epoch`, which are @@ -824,6 +838,12 @@ impl Actor { let store = rt.store(); let submitter = rt.message().caller(); + if !st.validators.is_validator(&submitter) { + return Err(actor_error!(illegal_argument, "caller not validator")); + } + + let total_validators = st.total_validators(); + st.cron_submissions .modify(store, |hamt| { let epoch_key = BytesKey::from(params.epoch.to_be_bytes().as_slice()); @@ -833,7 +853,9 @@ impl Actor { }; let epoch = params.epoch; - let execution_status = submission.submit(store, submitter, params)?; + let most_voted_count = submission.submit(store, submitter, params)?; + let execution_status = + submission.derive_execution_status(total_validators, most_voted_count); if st.last_cron_executed_epoch + st.cron_period != epoch { // there are pending epoch to be executed, @@ -985,5 +1007,6 @@ impl ActorCode for Actor { Propagate => propagate, WhiteListPropagator => whitelist_propagator, SubmitCron => submit_cron, + SetMembership => set_membership, } } diff --git a/gateway/src/state.rs b/gateway/src/state.rs index 229fa77..b47af0c 100644 --- a/gateway/src/state.rs +++ b/gateway/src/state.rs @@ -16,8 +16,9 @@ use primitives::{TAmt, TCid, THamt, TLink}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; use std::str::FromStr; -use crate::cron::CronSubmission; +use crate::cron::{CronSubmission, Validators}; use ipc_sdk::subnet_id::SubnetID; +use ipc_sdk::ValidatorSet; use super::checkpoint::*; use super::cross::*; @@ -54,6 +55,7 @@ pub struct State { /// The last submit cron epoch that was executed pub last_cron_executed_epoch: ChainEpoch, pub cron_submissions: TCid>, + pub validators: Validators, } lazy_static! { @@ -85,6 +87,7 @@ impl State { cron_period: params.cron_period, last_cron_executed_epoch: params.genesis_epoch, cron_submissions: TCid::new_hamt(store)?, + validators: Validators::new(ValidatorSet::default()), }) } @@ -460,6 +463,14 @@ impl State { *balance -= fee; Ok(()) } + + pub fn set_membership(&mut self, validator_set: ValidatorSet) { + self.validators = Validators::new(validator_set); + } + + pub(crate) fn total_validators(&self) -> u16 { + self.validators.validators.validators().len() as u16 + } } pub fn set_subnet( From 976b71060dab0f4ed7f30f0aeb338512ba58180c Mon Sep 17 00:00:00 2001 From: cryptoAtwill <108330426+cryptoAtwill@users.noreply.github.com> Date: Tue, 21 Mar 2023 22:04:23 +0800 Subject: [PATCH 16/27] Weighted vote (#71) * track validators * add validator check to submit cron * update impl * weighted vote * Update gateway/src/cron.rs Co-authored-by: adlrocha * update method name --------- Co-authored-by: adlrocha --- gateway/src/cron.rs | 173 ++++++++++++++++++++++++------------------- gateway/src/lib.rs | 15 ++-- gateway/src/state.rs | 4 - 3 files changed, 103 insertions(+), 89 deletions(-) diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index c66f0e5..9d38977 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -10,13 +10,18 @@ use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use ipc_sdk::ValidatorSet; +use lazy_static::lazy_static; use num_traits::Zero; use primitives::{TCid, THamt}; use std::cmp::Ordering; +use std::ops::Mul; pub type HashOutput = Vec; -const RATIO_NUMERATOR: u16 = 2; -const RATIO_DENOMINATOR: u16 = 3; + +lazy_static! { + pub static ref RATIO_NUMERATOR: u64 = 2; + pub static ref RATIO_DENOMINATOR: u64 = 3; +} /// Validators tracks all the validator in the subnet. It is useful in handling cron checkpoints. #[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] @@ -39,9 +44,13 @@ impl Validators { } } - /// Checks if an address is a validator - pub fn is_validator(&self, addr: &Address) -> bool { - self.validators.validators().iter().any(|x| x.addr == *addr) + /// Get the weight of a validator + pub fn get_validator_weight(&self, addr: &Address) -> Option { + self.validators + .validators() + .iter() + .find(|x| x.addr == *addr) + .map(|v| v.weight.clone()) } } @@ -89,14 +98,14 @@ impl CronCheckpoint { /// Track all the cron checkpoint submissions of an epoch #[derive(Serialize_tuple, Deserialize_tuple, PartialEq, Eq, Clone)] pub struct CronSubmission { - /// Total number of submissions from validators - total_submissions: u16, + /// The summation of the weights from all validator submissions + total_submission_weight: TokenAmount, /// The most submitted hash. most_voted_hash: Option, /// The addresses of all the submitters submitters: TCid>, - /// The map to track the max submitted - submission_counts: TCid>, + /// The map to track the submission weight of each hash + submission_weights: TCid>, /// The different cron checkpoints, with cron checkpoint hash as key submissions: TCid>, } @@ -104,20 +113,20 @@ pub struct CronSubmission { impl CronSubmission { pub fn new(store: &BS) -> anyhow::Result { Ok(CronSubmission { - total_submissions: 0, + total_submission_weight: TokenAmount::zero(), submitters: TCid::new_hamt(store)?, most_voted_hash: None, - submission_counts: TCid::new_hamt(store)?, + submission_weights: TCid::new_hamt(store)?, submissions: TCid::new_hamt(store)?, }) } /// Abort the current round and reset the submission data. pub fn abort(&mut self, store: &BS) -> anyhow::Result<()> { - self.total_submissions = 0; + self.total_submission_weight = TokenAmount::zero(); self.submitters = TCid::new_hamt(store)?; self.most_voted_hash = None; - self.submission_counts = TCid::new_hamt(store)?; + self.submission_weights = TCid::new_hamt(store)?; // no need reset `self.submissions`, we can still reuse the previous self.submissions // new submissions will be inserted, old submission will not be inserted to save @@ -131,11 +140,13 @@ impl CronSubmission { &mut self, store: &BS, submitter: Address, + submitter_weight: TokenAmount, checkpoint: CronCheckpoint, - ) -> anyhow::Result { + ) -> anyhow::Result { self.update_submitters(store, submitter)?; + self.total_submission_weight += &submitter_weight; let checkpoint_hash = self.insert_checkpoint(store, checkpoint)?; - self.update_submission_count(store, checkpoint_hash) + self.update_submission_weight(store, checkpoint_hash, submitter_weight) } pub fn load_most_submitted_checkpoint( @@ -162,22 +173,23 @@ impl CronSubmission { pub fn derive_execution_status( &self, - total_validators: u16, - most_voted_count: u16, + total_weight: TokenAmount, + most_voted_weight: TokenAmount, ) -> VoteExecutionStatus { - // use u16 numerator and denominator to avoid floating point calculation and external crate - // total validators should be within u16::MAX. - let threshold = total_validators as u16 * RATIO_NUMERATOR / RATIO_DENOMINATOR; + let threshold = total_weight + .clone() + .mul(*RATIO_NUMERATOR) + .div_floor(*RATIO_DENOMINATOR); // note that we require THRESHOLD to be surpassed, equality is not enough! - if self.total_submissions <= threshold { + if self.total_submission_weight <= threshold { return VoteExecutionStatus::ThresholdNotReached; } // now we have reached the threshold // consensus reached - if most_voted_count > threshold { + if most_voted_weight > threshold { return VoteExecutionStatus::ConsensusReached; } @@ -189,13 +201,13 @@ impl CronSubmission { // consider an example that consensus will never be reached: // // -------- | -------------------------|--------------- | ------------- | - // MOST_VOTED THRESHOLD TOTAL_SUBMISSIONS TOTAL_VALIDATORS + // MOST_VOTED THRESHOLD TOTAL_SUBMISSIONS TOTAL_WEIGHT // - // we see MOST_VOTED is smaller than THRESHOLD, TOTAL_SUBMISSIONS and TOTAL_VALIDATORS, if - // the potential extra votes any vote can obtain, i.e. TOTAL_VALIDATORS - TOTAL_SUBMISSIONS, + // we see MOST_VOTED is smaller than THRESHOLD, TOTAL_SUBMISSIONS and TOTAL_WEIGHT, if + // the potential extra votes any vote can obtain, i.e. TOTAL_WEIGHT - TOTAL_SUBMISSIONS, // is smaller than or equal to the potential extra vote the most voted can obtain, i.e. // THRESHOLD - MOST_VOTED, then consensus will never be reached, no point voting, just abort. - if threshold - most_voted_count >= total_validators - self.total_submissions { + if threshold - most_voted_weight >= total_weight - &self.total_submission_weight { VoteExecutionStatus::RoundAbort } else { VoteExecutionStatus::ReachingConsensus @@ -223,7 +235,7 @@ impl CronSubmission { &mut self, store: &BS, submitter: Address, - ) -> anyhow::Result { + ) -> anyhow::Result<()> { let addr_byte_key = BytesKey::from(submitter.to_bytes()); self.submitters.modify(store, |hamt| { // check the submitter has not submitted before @@ -233,9 +245,8 @@ impl CronSubmission { // now the submitter has not submitted before, mark as submitted hamt.set(addr_byte_key, ())?; - self.total_submissions += 1; - Ok(self.total_submissions) + Ok(()) }) } @@ -262,25 +273,30 @@ impl CronSubmission { Ok(hash) } - /// Update submission count of the hash. Returns the currently most submitted submission count. - fn update_submission_count( + /// Update submission weight of the hash. Returns the currently most submitted submission count. + fn update_submission_weight( &mut self, store: &BS, hash: HashOutput, - ) -> anyhow::Result { + weight: TokenAmount, + ) -> anyhow::Result { let hash_byte_key = BytesKey::from(hash.as_slice()); - self.submission_counts.modify(store, |hamt| { - let new_count = hamt.get(&hash_byte_key)?.map(|v| v + 1).unwrap_or(1); + self.submission_weights.modify(store, |hamt| { + let new_weight = hamt + .get(&hash_byte_key)? + .cloned() + .unwrap_or_else(TokenAmount::zero) + + weight; // update the new count - hamt.set(hash_byte_key, new_count)?; + hamt.set(hash_byte_key, new_weight.clone())?; // now we compare with the most submitted hash or cron checkpoint if self.most_voted_hash.is_none() { // no most submitted hash set yet, set to current self.most_voted_hash = Some(hash); - return Ok(new_count); + return Ok(new_weight); } let most_submitted_hash = self.most_voted_hash.as_mut().unwrap(); @@ -290,7 +306,7 @@ impl CronSubmission { // the current submission is already the only one submission, no need update // return the current checkpoint's count as the current most submitted checkpoint - return Ok(new_count); + return Ok(new_weight); } // the current submission is not part of the most submitted entries, need to check @@ -303,11 +319,11 @@ impl CronSubmission { // current submission is not the most voted checkpoints // if new_count < *most_submitted_count, we do nothing as the new count is not close to the most submitted - if new_count > *most_submitted_count { + if new_weight > *most_submitted_count { *most_submitted_hash = hash; - Ok(new_count) + Ok(new_weight) } else { - Ok(*most_submitted_count) + Ok(most_submitted_count.clone()) } }) } @@ -338,12 +354,12 @@ impl CronSubmission { /// Checks if the checkpoint hash has already inserted in the store #[cfg(test)] - fn get_submission_count( + fn get_submission_weight( &self, store: &BS, hash: &HashOutput, - ) -> anyhow::Result> { - let hamt = self.submission_counts.load(store)?; + ) -> anyhow::Result> { + let hamt = self.submission_weights.load(store)?; let r = hamt.get(&BytesKey::from(hash.as_slice()))?; Ok(r.cloned()) } @@ -354,6 +370,7 @@ mod tests { use crate::{CronCheckpoint, CronSubmission, VoteExecutionStatus}; use fvm_ipld_blockstore::MemoryBlockstore; use fvm_shared::address::Address; + use fvm_shared::econ::TokenAmount; #[test] fn test_new_works() { @@ -413,71 +430,71 @@ mod tests { assert_eq!(submission.most_voted_hash, None); assert_eq!( submission - .update_submission_count(&store, hash1.clone()) + .update_submission_weight(&store, hash1.clone(), TokenAmount::from_atto(1)) .unwrap(), - 1 + TokenAmount::from_atto(1) ); assert_eq!( submission - .get_submission_count(&store, &hash1) + .get_submission_weight(&store, &hash1) .unwrap() .unwrap(), - 1 + TokenAmount::from_atto(1) ); assert_eq!(submission.most_voted_hash, Some(hash1.clone())); // insert hash2, we should have two items, and there is a tie, hash1 still the most voted assert_eq!( submission - .update_submission_count(&store, hash2.clone()) + .update_submission_weight(&store, hash2.clone(), TokenAmount::from_atto(1)) .unwrap(), - 1 + TokenAmount::from_atto(1) ); assert_eq!( submission - .get_submission_count(&store, &hash2) + .get_submission_weight(&store, &hash2) .unwrap() .unwrap(), - 1 + TokenAmount::from_atto(1) ); assert_eq!( submission - .get_submission_count(&store, &hash1) + .get_submission_weight(&store, &hash1) .unwrap() .unwrap(), - 1 + TokenAmount::from_atto(1) ); assert_eq!(submission.most_voted_hash, Some(hash1.clone())); // insert hash2 again, we should have only 1 most submitted hash assert_eq!( submission - .update_submission_count(&store, hash2.clone()) + .update_submission_weight(&store, hash2.clone(), TokenAmount::from_atto(1)) .unwrap(), - 2 + TokenAmount::from_atto(2) ); assert_eq!( submission - .get_submission_count(&store, &hash2) + .get_submission_weight(&store, &hash2) .unwrap() .unwrap(), - 2 + TokenAmount::from_atto(2) ); assert_eq!(submission.most_voted_hash, Some(hash2.clone())); // insert hash2 again, we should have only 1 most submitted hash, but count incr by 1 assert_eq!( submission - .update_submission_count(&store, hash2.clone()) + .update_submission_weight(&store, hash2.clone(), TokenAmount::from_atto(1)) .unwrap(), - 3 + TokenAmount::from_atto(3) ); assert_eq!( submission - .get_submission_count(&store, &hash2) + .get_submission_weight(&store, &hash2) .unwrap() .unwrap(), - 3 + TokenAmount::from_atto(3) ); assert_eq!(submission.most_voted_hash, Some(hash2.clone())); } @@ -487,11 +504,11 @@ mod tests { let store = MemoryBlockstore::new(); let mut s = CronSubmission::new(&store).unwrap(); - let total_validators = 35; - let total_submissions = 10; - let most_voted_count = 5; + let total_validators = TokenAmount::from_atto(35); + let total_submissions = TokenAmount::from_atto(10); + let most_voted_count = TokenAmount::from_atto(5); - s.total_submissions = total_submissions; + s.total_submission_weight = total_submissions; assert_eq!( s.derive_execution_status(total_validators, most_voted_count), VoteExecutionStatus::ThresholdNotReached, @@ -502,23 +519,23 @@ mod tests { // If the threshold is 1 / 2, we could have: // If the last vote is C, then we should abort. // If the last vote is any of A or B, we can execute. - // If the threshold is 1 / 3, we have to abort. - let total_validators = 5; - let total_submissions = 4; - let most_voted_count = 2; - s.total_submissions = total_submissions; + // If the threshold is 2 / 3, we have to abort. + let total_validators = TokenAmount::from_atto(5); + let total_submissions = TokenAmount::from_atto(4); + let most_voted_count = TokenAmount::from_atto(2); + s.total_submission_weight = total_submissions.clone(); assert_eq!( - s.derive_execution_status(total_submissions, most_voted_count), + s.derive_execution_status(total_submissions.clone(), most_voted_count), VoteExecutionStatus::RoundAbort, ); // We could have 1 submission: A // Current submissions and their counts are: A - 4. - let total_submissions = 4; - let most_voted_count = 4; - s.total_submissions = total_submissions; + let total_submissions = TokenAmount::from_atto(4); + let most_voted_count = TokenAmount::from_atto(4); + s.total_submission_weight = total_submissions; assert_eq!( - s.derive_execution_status(total_validators, most_voted_count), + s.derive_execution_status(total_validators.clone(), most_voted_count), VoteExecutionStatus::ConsensusReached, ); @@ -526,9 +543,9 @@ mod tests { // Current submissions and their counts are: A - 3, B - 1. // Say the threshold is 2 / 3. If the last vote is B, we should abort, if the last vote is // A, then we have reached consensus. The current votes are in conclusive. - let total_submissions = 4; - let most_voted_count = 3; - s.total_submissions = total_submissions; + let total_submissions = TokenAmount::from_atto(4); + let most_voted_count = TokenAmount::from_atto(3); + s.total_submission_weight = total_submissions; assert_eq!( s.derive_execution_status(total_validators, most_voted_count), VoteExecutionStatus::ReachingConsensus, diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 61d9cc9..80b0795 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -838,11 +838,11 @@ impl Actor { let store = rt.store(); let submitter = rt.message().caller(); - if !st.validators.is_validator(&submitter) { - return Err(actor_error!(illegal_argument, "caller not validator")); - } - - let total_validators = st.total_validators(); + let validator_weight = match st.validators.get_validator_weight(&submitter) { + None => return Err(actor_error!(illegal_argument, "caller not validator")), + Some(w) => w, + }; + let total_weight = st.validators.total_weight.clone(); st.cron_submissions .modify(store, |hamt| { @@ -853,9 +853,10 @@ impl Actor { }; let epoch = params.epoch; - let most_voted_count = submission.submit(store, submitter, params)?; + let most_voted_weight = + submission.submit(store, submitter, validator_weight, params)?; let execution_status = - submission.derive_execution_status(total_validators, most_voted_count); + submission.derive_execution_status(total_weight, most_voted_weight); if st.last_cron_executed_epoch + st.cron_period != epoch { // there are pending epoch to be executed, diff --git a/gateway/src/state.rs b/gateway/src/state.rs index b47af0c..a729401 100644 --- a/gateway/src/state.rs +++ b/gateway/src/state.rs @@ -467,10 +467,6 @@ impl State { pub fn set_membership(&mut self, validator_set: ValidatorSet) { self.validators = Validators::new(validator_set); } - - pub(crate) fn total_validators(&self) -> u16 { - self.validators.validators.validators().len() as u16 - } } pub fn set_subnet( From d1d09ae7e27a893435407a2091b8d73f050217fb Mon Sep 17 00:00:00 2001 From: cryptoAtwill <108330426+cryptoAtwill@users.noreply.github.com> Date: Thu, 23 Mar 2023 11:08:40 +0800 Subject: [PATCH 17/27] Cron submit tests (#73) * track validators * add validator check to submit cron * update impl * weighted vote * Update gateway/src/cron.rs Co-authored-by: adlrocha * update method name * add tests * refactor pending epoches * fix clippy * add more tests --------- Co-authored-by: adlrocha --- gateway/src/cron.rs | 52 +++--- gateway/src/lib.rs | 256 +++++++++++++++++++------- gateway/src/state.rs | 16 ++ gateway/tests/gateway_test.rs | 336 +++++++++++++++++++++++++++++++++- gateway/tests/harness.rs | 40 +++- sdk/src/lib.rs | 7 + 6 files changed, 612 insertions(+), 95 deletions(-) diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index 9d38977..6408de2 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -74,7 +74,7 @@ impl CronCheckpoint { /// - top down messages are sorted by `nonce` in descending order /// /// Actor will not perform sorting to save gas. Client should do it, actor just check. - fn hash(&self) -> anyhow::Result { + pub fn hash(&self) -> anyhow::Result { // check top down msgs for i in 1..self.top_down_msgs.len() { match self.top_down_msgs[i - 1] @@ -161,6 +161,17 @@ impl CronSubmission { } } + pub fn most_voted_weight(&self, store: &BS) -> anyhow::Result { + // we will only have one entry in the `most_submitted` set if more than 2/3 has reached + if let Some(hash) = &self.most_voted_hash { + Ok(self + .get_submission_weight(store, hash)? + .unwrap_or_else(TokenAmount::zero)) + } else { + Ok(TokenAmount::zero()) + } + } + pub fn get_submission( &self, store: &BS, @@ -213,6 +224,17 @@ impl CronSubmission { VoteExecutionStatus::ReachingConsensus } } + + /// Checks if the submitter has already submitted the checkpoint. + pub fn has_submitted( + &self, + store: &BS, + submitter: &Address, + ) -> anyhow::Result { + let addr_byte_key = BytesKey::from(submitter.to_bytes()); + let hamt = self.submitters.load(store)?; + Ok(hamt.contains_key(&addr_byte_key)?) + } } /// The status indicating if the voting should be executed @@ -328,17 +350,15 @@ impl CronSubmission { }) } - /// Checks if the submitter has already submitted the checkpoint. Currently used only in - /// tests, but can be used in prod as well. - #[cfg(test)] - fn has_submitted( + /// Checks if the checkpoint hash has already inserted in the store + fn get_submission_weight( &self, store: &BS, - submitter: &Address, - ) -> anyhow::Result { - let addr_byte_key = BytesKey::from(submitter.to_bytes()); - let hamt = self.submitters.load(store)?; - Ok(hamt.contains_key(&addr_byte_key)?) + hash: &HashOutput, + ) -> anyhow::Result> { + let hamt = self.submission_weights.load(store)?; + let r = hamt.get(&BytesKey::from(hash.as_slice()))?; + Ok(r.cloned()) } /// Checks if the checkpoint hash has already inserted in the store @@ -351,18 +371,6 @@ impl CronSubmission { let hamt = self.submissions.load(store)?; Ok(hamt.contains_key(&BytesKey::from(hash.as_slice()))?) } - - /// Checks if the checkpoint hash has already inserted in the store - #[cfg(test)] - fn get_submission_weight( - &self, - store: &BS, - hash: &HashOutput, - ) -> anyhow::Result> { - let hamt = self.submission_weights.load(store)?; - let r = hamt.get(&BytesKey::from(hash.as_slice()))?; - Ok(r.cloned()) - } } #[cfg(test)] diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 80b0795..8375413 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -1,4 +1,5 @@ -#![feature(let_chains)] // For some simpler syntax for if let Some conditions +#![feature(let_chains)] +#![feature(map_first_last)] // For some simpler syntax for if let Some conditions extern crate core; @@ -7,8 +8,8 @@ pub use self::cross::{is_bottomup, CrossMsg, CrossMsgs, IPCMsgType, StorableMsg} pub use self::state::*; pub use self::subnet::*; pub use self::types::*; -use crate::cron::{CronSubmission, VoteExecutionStatus}; -use cron::CronCheckpoint; +pub use crate::cron::{CronSubmission, VoteExecutionStatus}; +pub use cron::CronCheckpoint; use cross::{burn_bu_funds, cross_msg_side_effects, distribute_crossmsg_fee}; use fil_actors_runtime::runtime::fvm::resolve_secp_bls; use fil_actors_runtime::runtime::{ActorCode, Runtime}; @@ -16,10 +17,12 @@ use fil_actors_runtime::{ actor_dispatch, actor_error, restrict_internal_api, ActorDowncast, ActorError, CALLER_TYPES_SIGNABLE, INIT_ACTOR_ADDR, SYSTEM_ACTOR_ADDR, }; +use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::RawBytes; use fvm_ipld_hamt::BytesKey; use fvm_shared::address::Address; use fvm_shared::bigint::Zero; +use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use fvm_shared::error::ExitCode; use fvm_shared::METHOD_SEND; @@ -818,79 +821,19 @@ impl Actor { /// of the total number of validators, the messages will be applied. /// /// Each cron checkpoint will be checked against each other using blake hashing. - fn submit_cron(rt: &mut impl Runtime, params: CronCheckpoint) -> Result { + fn submit_cron( + rt: &mut impl Runtime, + checkpoint: CronCheckpoint, + ) -> Result { // submit cron can only be performed by signable addresses rt.validate_immediate_caller_type(CALLER_TYPES_SIGNABLE.iter())?; let msgs = rt.transaction(|st: &mut State, rt| { - // first we check the epoch is the correct one - let genesis_epoch = st.genesis_epoch; - - // we process only it's multiple of cron_period since genesis_epoch - if (params.epoch - genesis_epoch) % st.cron_period != 0 { - return Err(actor_error!(illegal_argument, "epoch not allowed")); - } - - if st.last_cron_executed_epoch >= params.epoch { - return Err(actor_error!(illegal_argument, "epoch already executed")); - } - - let store = rt.store(); let submitter = rt.message().caller(); + let submitter_weight = Self::validate_submitter(st, checkpoint.epoch, &submitter)?; + let store = rt.store(); - let validator_weight = match st.validators.get_validator_weight(&submitter) { - None => return Err(actor_error!(illegal_argument, "caller not validator")), - Some(w) => w, - }; - let total_weight = st.validators.total_weight.clone(); - - st.cron_submissions - .modify(store, |hamt| { - let epoch_key = BytesKey::from(params.epoch.to_be_bytes().as_slice()); - let mut submission = match hamt.get(&epoch_key)? { - Some(s) => s.clone(), - None => CronSubmission::new(store)?, - }; - - let epoch = params.epoch; - let most_voted_weight = - submission.submit(store, submitter, validator_weight, params)?; - let execution_status = - submission.derive_execution_status(total_weight, most_voted_weight); - - if st.last_cron_executed_epoch + st.cron_period != epoch { - // there are pending epoch to be executed, - // just store the submission and skip execution - hamt.set(epoch_key, submission)?; - return Ok(None); - } - - match execution_status { - VoteExecutionStatus::ThresholdNotReached - | VoteExecutionStatus::ReachingConsensus => { - // threshold or consensus not reached, store submission and return - hamt.set(epoch_key, submission)?; - return Ok(None); - } - VoteExecutionStatus::RoundAbort => { - submission.abort(store)?; - hamt.set(epoch_key, submission)?; - return Ok(None); - } - VoteExecutionStatus::ConsensusReached => {} - } - - // we reach consensus in the checkpoints submission - st.last_cron_executed_epoch = epoch; - - let msgs = submission - .load_most_submitted_checkpoint(store)? - .unwrap() - .top_down_msgs; - hamt.delete(&epoch_key)?; - - Ok(Some(msgs)) - }) + Self::handle_cron_submission(store, st, checkpoint, submitter, submitter_weight) .map_err(|e| { log::error!( "encountered error processing submit cron checkpoint: {:?}", @@ -900,7 +843,12 @@ impl Actor { }) })?; + // we only `execute_next_cron_epoch(rt)` if there is no execution for the current submission + // so that we don't blow up the gas. if let Some(msgs) = msgs { + if msgs.is_empty() { + Self::execute_next_cron_epoch(rt)?; + } for m in msgs { Self::apply_msg_inner( rt, @@ -910,7 +858,10 @@ impl Actor { }, )?; } + } else { + Self::execute_next_cron_epoch(rt)?; } + Ok(RawBytes::default()) } @@ -991,6 +942,171 @@ impl Actor { } } +/// All the validator code for the actor calls +impl Actor { + /// Validate the submitter's submission against the state, also returns the weight of the validator + fn validate_submitter( + st: &State, + epoch: ChainEpoch, + submitter: &Address, + ) -> Result { + // first we check the epoch is the correct one, we process only it's multiple + // of cron_period since genesis_epoch + if (epoch - st.genesis_epoch) % st.cron_period != 0 { + return Err(actor_error!(illegal_argument, "epoch not allowed")); + } + + if st.last_cron_executed_epoch >= epoch { + return Err(actor_error!(illegal_argument, "epoch already executed")); + } + + st.validators + .get_validator_weight(submitter) + .ok_or_else(|| actor_error!(illegal_argument, "caller not validator")) + } +} + +/// Contains private method invocation +impl Actor { + fn handle_cron_submission( + store: &BS, + st: &mut State, + checkpoint: CronCheckpoint, + submitter: Address, + submitter_weight: TokenAmount, + ) -> anyhow::Result>> { + let total_weight = st.validators.total_weight.clone(); + let params_epoch = checkpoint.epoch; + + // We are doing this manually because we have to modify `state` while processing the `hamt`. + // The current `st.cron_submissions.modify(...)` does not allow us to modify state in the + // function closure passed to modify. + let mut hamt = st.cron_submissions.load(store)?; + + let epoch_key = BytesKey::from(params_epoch.to_be_bytes().as_slice()); + let mut submission = match hamt.get(&epoch_key)? { + Some(s) => s.clone(), + None => CronSubmission::new(store)?, + }; + + let most_voted_weight = + submission.submit(store, submitter, submitter_weight, checkpoint)?; + let execution_status = submission.derive_execution_status(total_weight, most_voted_weight); + + let messages = match execution_status { + VoteExecutionStatus::ThresholdNotReached | VoteExecutionStatus::ReachingConsensus => { + // threshold or consensus not reached, store submission and return + hamt.set(epoch_key, submission)?; + None + } + VoteExecutionStatus::RoundAbort => { + submission.abort(store)?; + hamt.set(epoch_key, submission)?; + None + } + VoteExecutionStatus::ConsensusReached => { + if st.last_cron_executed_epoch + st.cron_period != params_epoch { + // there are pending epochs to be executed, + // just store the submission and skip execution + hamt.set(epoch_key, submission)?; + st.insert_executable_epoch(params_epoch); + return Ok(None); + } + + // we reach consensus in the checkpoints submission + st.last_cron_executed_epoch = params_epoch; + + let msgs = submission + .load_most_submitted_checkpoint(store)? + .unwrap() + .top_down_msgs; + hamt.delete(&epoch_key)?; + + Some(msgs) + } + }; + + // don't forget to flush + st.cron_submissions = TCid::from(hamt.flush()?); + + Ok(messages) + } + + /// Execute the next approved cron checkpoint. + /// This is an edge case to ensure none of the epoches will be stuck. Consider the following example: + /// + /// Epoch 10 and 20 are two cron epoch to be executed. However, all the validators have submitted + /// epoch 20, and the status is to be executed, however, epoch 10 has yet to be executed. Now, + /// epoch 10 has reached consensus and executed, but epoch 20 cannot be executed because every + /// validator has already voted, no one can vote again to trigger the execution. Epoch 20 is stuck. + fn execute_next_cron_epoch(rt: &mut impl Runtime) -> Result<(), ActorError> { + let msgs = rt.transaction(|st: &mut State, rt| { + let epoch_queue = match st.executable_epoch_queue.as_mut() { + None => return Ok(None), + Some(queue) => queue, + }; + + match epoch_queue.first() { + None => { + unreachable!("`epoch_queue` is not None, it should not be empty, report bug") + } + Some(epoch) => { + if *epoch > st.last_cron_executed_epoch + st.cron_period { + log::debug!("earliest executable epoch not the same cron period"); + return Ok(None); + } + } + } + + let store = rt.store(); + let epoch = epoch_queue.pop_first().unwrap(); + + if epoch_queue.is_empty() { + st.executable_epoch_queue = None; + } + + st.cron_submissions + .modify(store, |hamt| { + let epoch_key = BytesKey::from(epoch.to_be_bytes().as_slice()); + let submission = match hamt.get(&epoch_key)? { + Some(s) => s, + None => unreachable!("Submission in epoch not found, report bug"), + }; + + st.last_cron_executed_epoch = epoch; + + let msgs = submission + .load_most_submitted_checkpoint(store)? + .unwrap() + .top_down_msgs; + hamt.delete(&epoch_key)?; + + Ok(Some(msgs)) + }) + .map_err(|e| { + log::error!( + "encountered error processing submit cron checkpoint: {:?}", + e + ); + actor_error!(unhandled_message, e.to_string()) + }) + })?; + + if let Some(msgs) = msgs { + for m in msgs { + Self::apply_msg_inner( + rt, + CrossMsg { + msg: m, + wrapped: false, + }, + )?; + } + } + Ok(()) + } +} + impl ActorCode for Actor { type Methods = Method; diff --git a/gateway/src/state.rs b/gateway/src/state.rs index a729401..3a7a70e 100644 --- a/gateway/src/state.rs +++ b/gateway/src/state.rs @@ -14,6 +14,7 @@ use lazy_static::lazy_static; use num_traits::Zero; use primitives::{TAmt, TCid, THamt, TLink}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; +use std::collections::BTreeSet; use std::str::FromStr; use crate::cron::{CronSubmission, Validators}; @@ -54,6 +55,11 @@ pub struct State { pub cron_period: ChainEpoch, /// The last submit cron epoch that was executed pub last_cron_executed_epoch: ChainEpoch, + /// Contains the executable epochs that are ready to be executed, but has yet to be executed. + /// This usually happens when previous submission epoch has not executed, but the next submission + /// epoch is ready to be executed. Most of the time this should be empty, we are wrapping with + /// Option instead of empty VecDeque just to save some storage space. + pub executable_epoch_queue: Option>, pub cron_submissions: TCid>, pub validators: Validators, } @@ -86,6 +92,7 @@ impl State { genesis_epoch: params.genesis_epoch, cron_period: params.cron_period, last_cron_executed_epoch: params.genesis_epoch, + executable_epoch_queue: None, cron_submissions: TCid::new_hamt(store)?, validators: Validators::new(ValidatorSet::default()), }) @@ -467,6 +474,15 @@ impl State { pub fn set_membership(&mut self, validator_set: ValidatorSet) { self.validators = Validators::new(validator_set); } + + pub fn insert_executable_epoch(&mut self, epoch: ChainEpoch) { + match self.executable_epoch_queue.as_mut() { + None => self.executable_epoch_queue = Some(BTreeSet::from([epoch])), + Some(queue) => { + queue.insert(epoch); + } + } + } } pub fn set_subnet( diff --git a/gateway/tests/gateway_test.rs b/gateway/tests/gateway_test.rs index a18e2ab..52fca28 100644 --- a/gateway/tests/gateway_test.rs +++ b/gateway/tests/gateway_test.rs @@ -1,7 +1,9 @@ use cid::Cid; use fil_actors_runtime::runtime::Runtime; +use fil_actors_runtime::test_utils::MockRuntime; use fil_actors_runtime::BURNT_FUNDS_ACTOR_ADDR; use fvm_ipld_encoding::RawBytes; +use fvm_ipld_hamt::BytesKey; use fvm_shared::address::Address; use fvm_shared::bigint::Zero; use fvm_shared::clock::ChainEpoch; @@ -10,11 +12,13 @@ use fvm_shared::error::ExitCode; use fvm_shared::METHOD_SEND; use ipc_gateway::Status::{Active, Inactive}; use ipc_gateway::{ - get_topdown_msg, Checkpoint, CrossMsg, IPCAddress, State, StorableMsg, CROSS_MSG_FEE, - DEFAULT_CHECKPOINT_PERIOD, SUBNET_ACTOR_REWARD_METHOD, + get_topdown_msg, Checkpoint, CronCheckpoint, CronSubmission, CrossMsg, IPCAddress, State, + StorableMsg, CROSS_MSG_FEE, DEFAULT_CHECKPOINT_PERIOD, SUBNET_ACTOR_REWARD_METHOD, }; use ipc_sdk::subnet_id::SubnetID; +use ipc_sdk::{Validator, ValidatorSet}; use primitives::TCid; +use std::collections::BTreeSet; use std::ops::Mul; use std::str::FromStr; @@ -1143,3 +1147,331 @@ fn test_apply_msg_match_target_subnet() { // TODO: Trying to release over circulating supply } + +#[test] +fn test_set_membership() { + let (h, mut rt) = setup_root(); + + let weights = vec![1000, 2000]; + let mut index = 0; + let validators = weights + .iter() + .map(|weight| { + let v = Validator { + addr: Address::new_id(index), + net_addr: index.to_string(), + weight: TokenAmount::from_atto(*weight), + }; + index += 1; + v + }) + .collect(); + let validator_set = ValidatorSet::new(validators, 10); + h.set_membership(&mut rt, validator_set.clone()).unwrap(); + + let st: State = rt.get_state(); + + assert_eq!(st.validators.validators, validator_set); + assert_eq!( + st.validators.total_weight, + TokenAmount::from_atto(weights.iter().sum::()) + ); +} + +fn setup_membership(h: &Harness, rt: &mut MockRuntime) { + let weights = vec![1000; 5]; + let mut index = 0; + let validators = weights + .iter() + .map(|weight| { + let v = Validator { + addr: Address::new_id(index), + net_addr: index.to_string(), + weight: TokenAmount::from_atto(*weight), + }; + index += 1; + v + }) + .collect(); + let validator_set = ValidatorSet::new(validators, 10); + h.set_membership(rt, validator_set.clone()).unwrap(); +} + +#[test] +fn test_submit_cron_checking_errors() { + let (h, mut rt) = setup_root(); + + setup_membership(&h, &mut rt); + + let submitter = Address::new_id(10000); + let checkpoint = CronCheckpoint { + epoch: *DEFAULT_GENESIS_EPOCH + 1, + top_down_msgs: vec![], + }; + let r = h.submit_cron(&mut rt, submitter, checkpoint); + assert!(r.is_err()); + assert_eq!(r.unwrap_err().msg(), "epoch not allowed"); + + let checkpoint = CronCheckpoint { + epoch: *DEFAULT_GENESIS_EPOCH, + top_down_msgs: vec![], + }; + let r = h.submit_cron(&mut rt, submitter, checkpoint); + assert!(r.is_err()); + assert_eq!(r.unwrap_err().msg(), "epoch already executed"); + + let checkpoint = CronCheckpoint { + epoch: *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD, + top_down_msgs: vec![], + }; + let r = h.submit_cron(&mut rt, submitter, checkpoint); + assert!(r.is_err()); + assert_eq!(r.unwrap_err().msg(), "caller not validator"); +} + +fn get_epoch_submissions(rt: &mut MockRuntime, epoch: ChainEpoch) -> Option { + let st: State = rt.get_state(); + let hamt = st.cron_submissions.load(rt.store()).unwrap(); + let bytes_key = BytesKey::from(epoch.to_be_bytes().as_slice()); + hamt.get(&bytes_key).unwrap().cloned() +} + +#[test] +fn test_submit_cron_works_with_execution() { + let (h, mut rt) = setup_root(); + + setup_membership(&h, &mut rt); + + let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD; + let msg = storable_msg(0); + let checkpoint = CronCheckpoint { + epoch, + top_down_msgs: vec![msg.clone()], + }; + + // first submission + let submitter = Address::new_id(0); + let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + assert!(r.is_ok()); + let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); + assert_eq!( + submission + .get_submission(rt.store(), &checkpoint.hash().unwrap()) + .unwrap() + .unwrap(), + checkpoint + ); + let st: State = rt.get_state(); + assert_eq!(st.last_cron_executed_epoch, *DEFAULT_GENESIS_EPOCH); // not executed yet + + // already submitted + let submitter = Address::new_id(0); + let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + assert!(r.is_err()); + assert_eq!(r.unwrap_err().msg(), "already submitted"); + + // second submission + let submitter = Address::new_id(1); + let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + assert!(r.is_ok()); + let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); + assert_eq!( + submission + .get_submission(rt.store(), &checkpoint.hash().unwrap()) + .unwrap() + .unwrap(), + checkpoint + ); + let st: State = rt.get_state(); + assert_eq!(st.last_cron_executed_epoch, *DEFAULT_GENESIS_EPOCH); // not executed yet + + // third submission + let submitter = Address::new_id(2); + let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + assert!(r.is_ok()); + let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); + assert_eq!( + submission + .get_submission(rt.store(), &checkpoint.hash().unwrap()) + .unwrap() + .unwrap(), + checkpoint + ); + let st: State = rt.get_state(); + assert_eq!(st.last_cron_executed_epoch, *DEFAULT_GENESIS_EPOCH); // not executed yet + + // fourth submission, executed + let submitter = Address::new_id(3); + rt.expect_send( + msg.to.raw_addr().unwrap(), + msg.method, + None, + msg.value, + None, + ExitCode::OK, + ); + let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + assert!(r.is_ok()); + let submission = get_epoch_submissions(&mut rt, epoch); + assert!(submission.is_none()); + let st: State = rt.get_state(); + assert_eq!(st.last_cron_executed_epoch, epoch); +} + +fn storable_msg(nonce: u64) -> StorableMsg { + StorableMsg { + from: IPCAddress::new(&ROOTNET_ID, &Address::new_id(10)).unwrap(), + to: IPCAddress::new(&ROOTNET_ID, &Address::new_id(20)).unwrap(), + method: 0, + params: Default::default(), + value: Default::default(), + nonce, + } +} + +#[test] +fn test_submit_cron_abort() { + let (h, mut rt) = setup_root(); + + setup_membership(&h, &mut rt); + + let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD; + + // first submission + let submitter = Address::new_id(0); + let checkpoint = CronCheckpoint { + epoch, + top_down_msgs: vec![], + }; + let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + assert!(r.is_ok()); + + // second submission + let submitter = Address::new_id(1); + let checkpoint = CronCheckpoint { + epoch, + top_down_msgs: vec![storable_msg(1)], + }; + let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + assert!(r.is_ok()); + + // third submission + let submitter = Address::new_id(2); + let checkpoint = CronCheckpoint { + epoch, + top_down_msgs: vec![storable_msg(1), storable_msg(2)], + }; + let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + assert!(r.is_ok()); + + // fourth submission, aborted + let submitter = Address::new_id(3); + let checkpoint = CronCheckpoint { + epoch, + top_down_msgs: vec![storable_msg(1), storable_msg(2), storable_msg(3)], + }; + let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + assert!(r.is_ok()); + + // check aborted + let st: State = rt.get_state(); + assert_eq!(st.last_cron_executed_epoch, *DEFAULT_GENESIS_EPOCH); // not executed yet + let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); + for i in 0..4 { + assert_eq!( + submission + .has_submitted(rt.store(), &Address::new_id(i)) + .unwrap(), + false + ); + } +} + +#[test] +fn test_submit_cron_sequential_execution() { + let (h, mut rt) = setup_root(); + + setup_membership(&h, &mut rt); + + let pending_epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD * 2; + let checkpoint = CronCheckpoint { + epoch: pending_epoch, + top_down_msgs: vec![], + }; + + // first submission + let submitter = Address::new_id(0); + h.submit_cron(&mut rt, submitter, checkpoint.clone()) + .unwrap(); + + // second submission + let submitter = Address::new_id(1); + h.submit_cron(&mut rt, submitter, checkpoint.clone()) + .unwrap(); + + // third submission + let submitter = Address::new_id(2); + h.submit_cron(&mut rt, submitter, checkpoint.clone()) + .unwrap(); + + // fourth submission, not executed + let submitter = Address::new_id(3); + h.submit_cron(&mut rt, submitter, checkpoint.clone()) + .unwrap(); + let st: State = rt.get_state(); + assert_eq!(st.last_cron_executed_epoch, *DEFAULT_GENESIS_EPOCH); // not executed yet + assert_eq!( + st.executable_epoch_queue, + Some(BTreeSet::from([pending_epoch])) + ); // not executed yet + + // now we execute the previous epoch + let msg = storable_msg(0); + let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD; + let checkpoint = CronCheckpoint { + epoch, + top_down_msgs: vec![msg.clone()], + }; + + // first submission + let submitter = Address::new_id(0); + h.submit_cron(&mut rt, submitter, checkpoint.clone()) + .unwrap(); + // second submission + let submitter = Address::new_id(1); + h.submit_cron(&mut rt, submitter, checkpoint.clone()) + .unwrap(); + // third submission + let submitter = Address::new_id(2); + h.submit_cron(&mut rt, submitter, checkpoint.clone()) + .unwrap(); + // fourth submission, executed + let submitter = Address::new_id(3); + // define expected send + rt.expect_send( + msg.to.raw_addr().unwrap(), + msg.method, + None, + msg.value, + None, + ExitCode::OK, + ); + h.submit_cron(&mut rt, submitter, checkpoint.clone()) + .unwrap(); + let submission = get_epoch_submissions(&mut rt, epoch); + assert!(submission.is_none()); + let st: State = rt.get_state(); + assert_eq!(st.last_cron_executed_epoch, epoch); + + // now we submit to the next epoch + let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD * 3; + let checkpoint = CronCheckpoint { + epoch, + top_down_msgs: vec![], + }; + h.submit_cron(&mut rt, submitter, checkpoint.clone()) + .unwrap(); + let st: State = rt.get_state(); + assert_eq!(st.last_cron_executed_epoch, pending_epoch); + assert_eq!(st.executable_epoch_queue, None); +} diff --git a/gateway/tests/harness.rs b/gateway/tests/harness.rs index 133e6cd..737ec01 100644 --- a/gateway/tests/harness.rs +++ b/gateway/tests/harness.rs @@ -26,13 +26,14 @@ use fvm_shared::error::ExitCode; use fvm_shared::MethodNum; use fvm_shared::METHOD_SEND; use ipc_gateway::checkpoint::ChildCheck; -use ipc_gateway::SUBNET_ACTOR_REWARD_METHOD; use ipc_gateway::{ ext, get_topdown_msg, is_bottomup, Actor, ApplyMsgParams, Checkpoint, ConstructorParams, CrossMsg, CrossMsgMeta, CrossMsgParams, CrossMsgs, FundParams, IPCAddress, IPCMsgType, Method, PropagateParams, State, StorableMsg, Subnet, SubnetID, CROSSMSG_AMT_BITWIDTH, CROSS_MSG_FEE, DEFAULT_CHECKPOINT_PERIOD, MAX_NONCE, MIN_COLLATERAL_AMOUNT, }; +use ipc_gateway::{CronCheckpoint, SUBNET_ACTOR_REWARD_METHOD}; +use ipc_sdk::ValidatorSet; use lazy_static::lazy_static; use primitives::{TCid, TCidContent}; use std::str::FromStr; @@ -719,6 +720,43 @@ impl Harness { pub fn get_subnet(&self, rt: &MockRuntime, id: &SubnetID) -> Option { get_subnet(rt, id) } + + pub fn set_membership( + &self, + rt: &mut MockRuntime, + validator_set: ValidatorSet, + ) -> Result<(), ActorError> { + rt.set_caller(*SYSTEM_ACTOR_CODE_ID, SYSTEM_ACTOR_ADDR); + rt.expect_validate_caller_addr(vec![SYSTEM_ACTOR_ADDR]); + + rt.call::( + Method::SetMembership as MethodNum, + IpldBlock::serialize_cbor(&validator_set).unwrap(), + ) + .unwrap(); + rt.verify(); + + Ok(()) + } + + pub fn submit_cron( + &self, + rt: &mut MockRuntime, + submitter: Address, + checkpoint: CronCheckpoint, + ) -> Result<(), ActorError> { + rt.set_caller(*ACCOUNT_ACTOR_CODE_ID, submitter); + rt.expect_validate_caller_type(SIG_TYPES.clone()); + + rt.call::( + Method::SubmitCron as MethodNum, + IpldBlock::serialize_cbor(&checkpoint).unwrap(), + )?; + + rt.verify(); + + Ok(()) + } } pub fn get_subnet(rt: &MockRuntime, id: &SubnetID) -> Option { diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs index 15aac66..ec3e5dc 100644 --- a/sdk/src/lib.rs +++ b/sdk/src/lib.rs @@ -28,6 +28,13 @@ pub struct ValidatorSet { } impl ValidatorSet { + pub fn new(validators: Vec, configuration_number: u64) -> Self { + Self { + validators, + configuration_number, + } + } + pub fn validators(&self) -> &Vec { &self.validators } From de4875bf7732b37a6d88ad55e488fee43b78a6d7 Mon Sep 17 00:00:00 2001 From: cryptoAtwill <108330426+cryptoAtwill@users.noreply.github.com> Date: Mon, 3 Apr 2023 13:59:58 +0800 Subject: [PATCH 18/27] Refactor checkpoints (#74) * track validators * add validator check to submit cron * update impl * weighted vote * Update gateway/src/cron.rs Co-authored-by: adlrocha * update method name * add tests * refactor pending epoches * fix clippy * add more tests * initial commit * Cross execution (#75) * update bottom up execution * update cross message execution * fix fmt * update review and clean up * check message ordering * Cross execution tests (#76) * fix clippy * fmt code --------- Co-authored-by: adlrocha --- gateway/src/checkpoint.rs | 57 +++- gateway/src/cron.rs | 15 +- gateway/src/cross.rs | 32 +- gateway/src/lib.rs | 321 +++++++++---------- gateway/src/state.rs | 148 +-------- gateway/src/subnet.rs | 2 +- gateway/src/types.rs | 21 +- gateway/tests/gateway_test.rs | 519 ++++++++++++++++++------------- gateway/tests/harness.rs | 237 +------------- subnet-actor/src/lib.rs | 33 ++ subnet-actor/tests/actor_test.rs | 73 +++++ 11 files changed, 672 insertions(+), 786 deletions(-) diff --git a/gateway/src/checkpoint.rs b/gateway/src/checkpoint.rs index ded6136..2671c48 100644 --- a/gateway/src/checkpoint.rs +++ b/gateway/src/checkpoint.rs @@ -6,10 +6,11 @@ use fvm_ipld_encoding::{serde_bytes, to_vec}; use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use ipc_sdk::subnet_id::SubnetID; +use num_traits::Zero; use primitives::{TCid, TLink}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; -use crate::CrossMsgs; +use crate::{ensure_message_sorted, CrossMsg, CrossMsgs}; #[derive(PartialEq, Eq, Clone, Debug, Serialize_tuple, Deserialize_tuple)] pub struct Checkpoint { @@ -62,19 +63,44 @@ impl Checkpoint { &self.data.prev_check } - /// return cross_msg included in the checkpoint. - pub fn cross_msgs(&self) -> Option<&CrossMsgMeta> { - self.data.cross_msgs.as_ref() + /// Take the cross messages out of the checkpoint. This will empty the `self.data.cross_msgs` + /// and replace with None. + pub fn take_cross_msgs(&mut self) -> Option> { + self.data.cross_msgs.cross_msgs.take() } - /// set cross_msg included in the checkpoint. - pub fn set_cross_msgs(&mut self, cm: CrossMsgMeta) { - self.data.cross_msgs = Some(cm) + pub fn ensure_cross_msgs_sorted(&self) -> anyhow::Result<()> { + match self.data.cross_msgs.cross_msgs.as_ref() { + None => Ok(()), + Some(v) => ensure_message_sorted(v), + } } - /// return cross_msg included in the checkpoint as mutable reference - pub fn cross_msgs_mut(&mut self) -> Option<&mut CrossMsgMeta> { - self.data.cross_msgs.as_mut() + /// Get the sum of values in cross messages + pub fn total_value(&self) -> TokenAmount { + match &self.data.cross_msgs.cross_msgs { + None => TokenAmount::zero(), + Some(cross_msgs) => { + let mut value = TokenAmount::zero(); + cross_msgs.iter().for_each(|cross_msg| { + value += &cross_msg.msg.value; + }); + value + } + } + } + + /// Get the total fee of the cross messages + pub fn total_fee(&self) -> &TokenAmount { + &self.data.cross_msgs.fee + } + + pub fn push_cross_msgs(&mut self, cross_msg: CrossMsg, fee: &TokenAmount) { + self.data.cross_msgs.fee += fee; + match self.data.cross_msgs.cross_msgs.as_mut() { + None => self.data.cross_msgs.cross_msgs = Some(vec![cross_msg]), + Some(v) => v.push(cross_msg), + }; } /// Add the cid of a checkpoint from a child subnet for further propagation @@ -121,8 +147,15 @@ pub struct CheckData { pub epoch: ChainEpoch, pub prev_check: TCid>, pub children: Vec, - pub cross_msgs: Option, + pub cross_msgs: BatchCrossMsgs, +} + +#[derive(Default, PartialEq, Eq, Clone, Debug, Serialize_tuple, Deserialize_tuple)] +pub struct BatchCrossMsgs { + pub cross_msgs: Option>, + pub fee: TokenAmount, } + impl CheckData { pub fn new(id: SubnetID, epoch: ChainEpoch) -> Self { Self { @@ -131,7 +164,7 @@ impl CheckData { epoch, prev_check: TCid::default(), children: Vec::new(), - cross_msgs: None, + cross_msgs: BatchCrossMsgs::default(), } } } diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index 6408de2..d971482 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -1,4 +1,4 @@ -use crate::StorableMsg; +use crate::{ensure_message_sorted, StorableMsg}; use anyhow::anyhow; use cid::multihash::Code; use cid::multihash::MultihashDigest; @@ -13,7 +13,6 @@ use ipc_sdk::ValidatorSet; use lazy_static::lazy_static; use num_traits::Zero; use primitives::{TCid, THamt}; -use std::cmp::Ordering; use std::ops::Mul; pub type HashOutput = Vec; @@ -75,17 +74,7 @@ impl CronCheckpoint { /// /// Actor will not perform sorting to save gas. Client should do it, actor just check. pub fn hash(&self) -> anyhow::Result { - // check top down msgs - for i in 1..self.top_down_msgs.len() { - match self.top_down_msgs[i - 1] - .nonce - .cmp(&self.top_down_msgs[i].nonce) - { - Ordering::Less => {} - Ordering::Equal => return Err(anyhow!("top down messages not distinct")), - Ordering::Greater => return Err(anyhow!("top down messages not sorted")), - }; - } + ensure_message_sorted(&self.top_down_msgs)?; let mh_code = Code::Blake2b256; // TODO: to avoid serialization again, maybe we should perform deserialization in the actor diff --git a/gateway/src/cross.rs b/gateway/src/cross.rs index 4f8bdd2..ea0c058 100644 --- a/gateway/src/cross.rs +++ b/gateway/src/cross.rs @@ -1,11 +1,10 @@ -use crate::ApplyMsgParams; use crate::State; use crate::SUBNET_ACTOR_REWARD_METHOD; +use crate::{ApplyMsgParams, ExecutableMessage}; use anyhow::anyhow; use fil_actors_runtime::runtime::Runtime; use fil_actors_runtime::ActorError; use fil_actors_runtime::BURNT_FUNDS_ACTOR_ADDR; -use fvm_ipld_blockstore::MemoryBlockstore; use fvm_ipld_encoding::ipld_block::IpldBlock; use fvm_ipld_encoding::RawBytes; use fvm_shared::address::Address; @@ -14,7 +13,6 @@ use fvm_shared::MethodNum; use fvm_shared::METHOD_SEND; use ipc_sdk::address::IPCAddress; use ipc_sdk::subnet_id::SubnetID; -use primitives::{TCid, TLink}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; use std::path::Path; @@ -35,12 +33,24 @@ pub struct StorableMsg { pub nonce: u64, } +impl ExecutableMessage for StorableMsg { + fn nonce(&self) -> u64 { + self.nonce + } +} + #[derive(PartialEq, Eq, Clone, Debug, Serialize_tuple, Deserialize_tuple)] pub struct CrossMsg { pub msg: StorableMsg, pub wrapped: bool, } +impl ExecutableMessage for CrossMsg { + fn nonce(&self) -> u64 { + self.msg.nonce() + } +} + #[derive(PartialEq, Eq)] pub enum IPCMsgType { BottomUp, @@ -71,7 +81,6 @@ impl StorableMsg { sub_id: &SubnetID, sig_addr: &Address, value: TokenAmount, - nonce: u64, ) -> anyhow::Result { let to = IPCAddress::new( &match sub_id.parent() { @@ -87,7 +96,7 @@ impl StorableMsg { method: METHOD_SEND, params: RawBytes::default(), value, - nonce, + nonce: 0, }) } @@ -156,19 +165,6 @@ impl CrossMsgs { pub fn new() -> Self { Self::default() } - - pub(crate) fn cid(&self) -> anyhow::Result>> { - TCid::new_link(&MemoryBlockstore::new(), self) - } - - /// Appends a cross-message to cross-msgs - pub(crate) fn add_msg(&mut self, msg: &CrossMsg) -> bool { - if !self.msgs.contains(msg) { - self.msgs.push(msg.clone()); - return true; - } - false - } } /// Transaction side-effects from the commitment of a cross-net message. It burns funds diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 8375413..f9f1d00 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -67,7 +67,6 @@ pub enum Method { Fund = frc42_dispatch::method_hash!("Fund"), Release = frc42_dispatch::method_hash!("Release"), SendCross = frc42_dispatch::method_hash!("SendCross"), - ApplyMessage = frc42_dispatch::method_hash!("ApplyMessage"), Propagate = frc42_dispatch::method_hash!("Propagate"), WhiteListPropagator = frc42_dispatch::method_hash!("WhiteListPropagator"), SubmitCron = frc42_dispatch::method_hash!("SubmitCron"), @@ -278,11 +277,16 @@ impl Actor { /// CommitChildCheck propagates the commitment of a checkpoint from a child subnet, /// process the cross-messages directed to the subnet. - fn commit_child_check(rt: &mut impl Runtime, params: Checkpoint) -> Result<(), ActorError> { + fn commit_child_check(rt: &mut impl Runtime, mut commit: Checkpoint) -> Result<(), ActorError> { + // This must be called by a subnet actor, once we have a way to identify subnet actor, + // we should update here. rt.validate_immediate_caller_accept_any()?; + commit + .ensure_cross_msgs_sorted() + .map_err(|_| actor_error!(illegal_argument, "cross messages not ordered by nonce"))?; + let subnet_addr = rt.message().caller(); - let commit = params; let subnet_actor = commit.source().subnet_actor(); // check if the checkpoint belongs to the subnet @@ -293,13 +297,12 @@ impl Actor { )); } - let fee = rt.transaction(|st: &mut State, rt| { + let (fee, cross_msgs) = rt.transaction(|st: &mut State, rt| { let shid = SubnetID::new_from_parent(&st.network_name, subnet_addr); let sub = st.get_subnet(rt.store(), &shid).map_err(|e| { e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "failed to load subnet") })?; - let mut fee = TokenAmount::zero(); match sub { Some(mut sub) => { // check if subnet active @@ -333,35 +336,22 @@ impl Actor { if commit.prev_check().cid() != prev_checkpoint.cid() { return Err(actor_error!( illegal_argument, - "previous checkpoint not consistente with previous one" + "previous checkpoint not consistent with previous one" )); } } - // commit cross-message in checkpoint to either execute them or - // queue them for propagation if there are cross-msgs availble. - if let Some(cross_msg) = commit.cross_msgs() { - // if tcid not default it means cross-msgs are being propagated. - if cross_msg.msgs_cid != TCid::default() { - st.store_bottomup_msg(rt.store(), cross_msg).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - "error storing bottom_up messages from checkpoint", - ) - })?; - } - - // release circulating supply - sub.release_supply(&cross_msg.value).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - "error releasing circulating supply", - ) - })?; + // commit cross-message in checkpoint to execute them. + let fee = commit.total_fee().clone(); + let value = commit.total_value() + &fee; - // distribute fee - fee = cross_msg.fee.clone(); - } + // release circulating supply + sub.release_supply(&value).map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + "error releasing circulating supply", + ) + })?; // append new checkpoint to the list of childs ch.add_child_check(&commit).map_err(|e| { @@ -376,25 +366,30 @@ impl Actor { e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "error flushing checkpoint") })?; + let cross_msgs = commit.take_cross_msgs(); + // update prev_check for child sub.prev_checkpoint = Some(commit); // flush subnet st.flush_subnet(rt.store(), &sub).map_err(|e| { e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "error flushing subnet") })?; + Ok((fee, cross_msgs)) } - None => { - return Err(actor_error!( - illegal_argument, - "subnet with id {} not registered", - shid - )); - } + None => Err(actor_error!( + illegal_argument, + "subnet with id {} not registered", + shid + )), } - - Ok(fee) })?; + if let Some(msgs) = cross_msgs { + for cross_msg in msgs { + Self::apply_msg_inner(rt, cross_msg)?; + } + } + // distribute rewards distribute_crossmsg_fee(rt, &subnet_actor, fee) } @@ -459,9 +454,6 @@ impl Actor { // funds can only be moved between subnets by signable addresses rt.validate_immediate_caller_type(CALLER_TYPES_SIGNABLE.iter())?; - // FIXME: Only supporting cross-messages initiated by signable addresses for - // now. Consider supporting also send-cross messages initiated by actors. - let mut value = rt.message().value_received(); if value <= TokenAmount::zero() { return Err(actor_error!( @@ -479,23 +471,18 @@ impl Actor { // Create release message let r_msg = CrossMsg { - msg: StorableMsg::new_release_msg( - &st.network_name, - &sig_addr, - value.clone(), - st.nonce, - ) - .map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - "error creating release cross-message", - ) - })?, + msg: StorableMsg::new_release_msg(&st.network_name, &sig_addr, value.clone()) + .map_err(|e| { + e.downcast_default( + ExitCode::USR_ILLEGAL_STATE, + "error creating release cross-message", + ) + })?, wrapped: false, }; // Commit bottom-up message. - st.commit_bottomup_msg(rt.store(), &r_msg, fee, rt.curr_epoch()) + st.commit_bottomup_msg(rt.store(), &r_msg, rt.curr_epoch(), fee) .map_err(|e| { e.downcast_default( ExitCode::USR_ILLEGAL_STATE, @@ -596,123 +583,6 @@ impl Actor { Ok(()) } - /// ApplyMessage triggers the execution of a cross-subnet message validated through the consensus. - /// - /// This function can only be triggered using `ApplyImplicitMessage`, and the source needs to - /// be the SystemActor. Cross messages are applied similarly to how rewards are applied once - /// a block has been validated. This function: - /// - Determines the type of cross-message. - /// - Performs the corresponding state changes. - /// - And updated the latest nonce applied for future checks. - fn apply_msg(rt: &mut impl Runtime, params: ApplyMsgParams) -> Result { - rt.validate_immediate_caller_is([&SYSTEM_ACTOR_ADDR as &Address])?; - let ApplyMsgParams { cross_msg } = params; - Self::apply_msg_inner(rt, cross_msg) - } - - fn apply_msg_inner(rt: &mut impl Runtime, cross_msg: CrossMsg) -> Result { - let rto = match cross_msg.msg.to.raw_addr() { - Ok(to) => to, - Err(_) => { - return Err(actor_error!( - illegal_argument, - "error getting raw address from msg" - )); - } - }; - let sto = match cross_msg.msg.to.subnet() { - Ok(to) => to, - Err(_) => { - return Err(actor_error!( - illegal_argument, - "error getting subnet from msg" - )); - } - }; - - let st: State = rt.state()?; - - log::debug!("sto: {:?}, network: {:?}", sto, st.network_name); - - match cross_msg.msg.apply_type(&st.network_name) { - Ok(IPCMsgType::BottomUp) => { - // if directed to current network, execute message. - if sto == st.network_name { - rt.transaction(|st: &mut State, _| { - st.bottomup_state_transition(&cross_msg.msg).map_err(|e| { - e.downcast_default( - ExitCode::USR_ILLEGAL_STATE, - "failed applying bottomup message", - ) - })?; - Ok(()) - })?; - return cross_msg.send(rt, &rto); - } - } - Ok(IPCMsgType::TopDown) => { - // Mint funds for the gateway, as any topdown message - // including tokens traversing the subnet will use - // some balance from the gateway to increase the circ_supply. - // check if the gateway has enough funds to mint new FIL, - // if not fail right-away and do not allow the execution of - // the message. - // TODO: It may be a good idea in the future to decouple the - // balance provisioned in the gateway to mint new circulating supply - // in an independent actor. Minting tokens would require a call to the new actor actor to unlock - // additional circulating supply. This prevents an attacker from being able token - // vulnerabilities in the gateway - // from draining the whole balance. - if rt.current_balance() < cross_msg.msg.value { - return Err(actor_error!( - illegal_state, - "not enough balance to mint new tokens as part of the cross-message" - )); - } - - if sto == st.network_name { - if st.applied_topdown_nonce != cross_msg.msg.nonce { - return Err(actor_error!( - illegal_state, - "the top-down message being applied doesn't hold the subsequent nonce" - )); - } - - rt.transaction(|st: &mut State, _| { - st.applied_topdown_nonce += 1; - Ok(()) - })?; - - // We can return the send result - return cross_msg.send(rt, &rto); - } - } - _ => { - return Err(actor_error!( - illegal_argument, - "cross-message to apply dosen't have the right type" - )) - } - }; - - let cid = rt.transaction(|st: &mut State, rt| { - let owner = cross_msg - .msg - .from - .raw_addr() - .map_err(|_| actor_error!(illegal_argument, "invalid address"))?; - let r = st - .insert_postbox(rt.store(), Some(vec![owner]), cross_msg) - .map_err(|e| { - e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "error save topdown messages") - })?; - Ok(r) - })?; - - // it is safe to just unwrap. If `transaction` fails, cid is None and wont reach here. - Ok(RawBytes::new(cid.to_bytes())) - } - /// Whitelist a series of addresses as propagator of a cross net message. /// This is basically adding this list of addresses to the `PostBoxItem::owners`. /// Only existing owners can perform this operation. @@ -916,7 +786,7 @@ impl Actor { if cross_msg.msg.value > TokenAmount::zero() { do_burn = true; } - st.commit_bottomup_msg(rt.store(), cross_msg, &fee, rt.curr_epoch()) + st.commit_bottomup_msg(rt.store(), cross_msg, rt.curr_epoch(), &fee) }; r.map_err(|e| { @@ -968,6 +838,112 @@ impl Actor { /// Contains private method invocation impl Actor { + fn apply_msg_inner(rt: &mut impl Runtime, cross_msg: CrossMsg) -> Result { + let rto = match cross_msg.msg.to.raw_addr() { + Ok(to) => to, + Err(_) => { + return Err(actor_error!( + illegal_argument, + "error getting raw address from msg" + )); + } + }; + let sto = match cross_msg.msg.to.subnet() { + Ok(to) => to, + Err(_) => { + return Err(actor_error!( + illegal_argument, + "error getting subnet from msg" + )); + } + }; + + let st: State = rt.state()?; + + log::debug!("sto: {:?}, network: {:?}", sto, st.network_name); + + match cross_msg.msg.apply_type(&st.network_name) { + Ok(IPCMsgType::BottomUp) => { + // if directed to current network, execute message. + if sto == st.network_name { + rt.transaction(|st: &mut State, _| { + if st.applied_bottomup_nonce != cross_msg.msg.nonce { + return Err(actor_error!( + illegal_state, + "the bottom-up message being applied doesn't hold the subsequent nonce" + )); + } + + st.applied_bottomup_nonce += 1; + + Ok(()) + })?; + return cross_msg.send(rt, &rto); + } + } + Ok(IPCMsgType::TopDown) => { + // Mint funds for the gateway, as any topdown message + // including tokens traversing the subnet will use + // some balance from the gateway to increase the circ_supply. + // check if the gateway has enough funds to mint new FIL, + // if not fail right-away and do not allow the execution of + // the message. + // TODO: It may be a good idea in the future to decouple the + // balance provisioned in the gateway to mint new circulating supply + // in an independent actor. Minting tokens would require a call to the new actor actor to unlock + // additional circulating supply. This prevents an attacker from being able token + // vulnerabilities in the gateway + // from draining the whole balance. + if rt.current_balance() < cross_msg.msg.value { + return Err(actor_error!( + illegal_state, + "not enough balance to mint new tokens as part of the cross-message" + )); + } + + if sto == st.network_name { + rt.transaction(|st: &mut State, _| { + if st.applied_topdown_nonce != cross_msg.msg.nonce { + return Err(actor_error!( + illegal_state, + "the top-down message being applied doesn't hold the subsequent nonce" + )); + } + + st.applied_topdown_nonce += 1; + Ok(()) + })?; + + // We can return the send result + return cross_msg.send(rt, &rto); + } + } + _ => { + return Err(actor_error!( + illegal_argument, + "cross-message to apply dosen't have the right type" + )) + } + }; + + let cid = rt.transaction(|st: &mut State, rt| { + let owner = cross_msg + .msg + .from + .raw_addr() + .map_err(|_| actor_error!(illegal_argument, "invalid address"))?; + let r = st + .insert_postbox(rt.store(), Some(vec![owner]), cross_msg) + .map_err(|e| { + e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "error save topdown messages") + })?; + Ok(r) + })?; + + // it is safe to just unwrap. If `transaction` fails, cid is None and wont reach here. + Ok(RawBytes::new(cid.to_bytes())) + } + fn handle_cron_submission( store: &BS, st: &mut State, @@ -1120,7 +1096,6 @@ impl ActorCode for Actor { Fund => fund, Release => release, SendCross => send_cross, - ApplyMessage => apply_msg, Propagate => propagate, WhiteListPropagator => whitelist_propagator, SubmitCron => submit_cron, diff --git a/gateway/src/state.rs b/gateway/src/state.rs index 3a7a70e..9a16f69 100644 --- a/gateway/src/state.rs +++ b/gateway/src/state.rs @@ -12,7 +12,7 @@ use fvm_shared::econ::TokenAmount; use fvm_shared::error::ExitCode; use lazy_static::lazy_static; use num_traits::Zero; -use primitives::{TAmt, TCid, THamt, TLink}; +use primitives::{TCid, THamt}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; use std::collections::BTreeSet; use std::str::FromStr; @@ -40,13 +40,10 @@ pub struct State { pub subnets: TCid>, pub check_period: ChainEpoch, pub checkpoints: TCid>, - pub check_msg_registry: TCid>, CrossMsgs>>, /// `postbox` keeps track for an EOA of all the cross-net messages triggered by /// an actor that need to be propagated further through the hierarchy. pub postbox: PostBox, - pub nonce: u64, pub bottomup_nonce: u64, - pub bottomup_msg_meta: TCid>, pub applied_bottomup_nonce: u64, pub applied_topdown_nonce: u64, /// The epoch that the subnet actor is deployed @@ -80,14 +77,11 @@ impl State { false => DEFAULT_CHECKPOINT_PERIOD, }, checkpoints: TCid::new_hamt(store)?, - check_msg_registry: TCid::new_hamt(store)?, postbox: TCid::new_hamt(store)?, - nonce: Default::default(), bottomup_nonce: Default::default(), - bottomup_msg_meta: TCid::new_amt(store)?, // This way we ensure that the first message to execute has nonce= 0, if not it would expect 1 and fail for the first nonce // We first increase to the subsequent and then execute for bottom-up messages - applied_bottomup_nonce: MAX_NONCE, + applied_bottomup_nonce: Default::default(), applied_topdown_nonce: Default::default(), genesis_epoch: params.genesis_epoch, cron_period: params.cron_period, @@ -130,7 +124,7 @@ impl State { top_down_msgs: TCid::new_amt(rt.store())?, circ_supply: TokenAmount::zero(), status: Status::Active, - nonce: 0, + topdown_nonce: 0, prev_checkpoint: None, }; set_subnet(subnets, id, subnet)?; @@ -205,31 +199,18 @@ impl State { &mut self, store: &BS, cross_msg: &CrossMsg, - fee: &TokenAmount, curr_epoch: ChainEpoch, + fee: &TokenAmount, ) -> anyhow::Result<()> { let mut ch = self.get_window_checkpoint(store, curr_epoch)?; - match ch.cross_msgs_mut() { - Some(msgmeta) => { - let prev_cid = &msgmeta.msgs_cid; - let m_cid = self.append_msg_to_meta(store, prev_cid, cross_msg)?; - msgmeta.msgs_cid = m_cid; - msgmeta.value += &cross_msg.msg.value + fee; - msgmeta.fee += fee; - } - None => self.check_msg_registry.modify(store, |cross_reg| { - let mut msgmeta = CrossMsgMeta::default(); - let mut crossmsgs = CrossMsgs::new(); - let _ = crossmsgs.add_msg(cross_msg); - let m_cid = put_msgmeta(cross_reg, crossmsgs)?; - msgmeta.msgs_cid = m_cid; - msgmeta.value += &cross_msg.msg.value + fee; - msgmeta.fee += fee; - ch.set_cross_msgs(msgmeta); - Ok(()) - })?, - }; + let mut cross_msg = cross_msg.clone(); + cross_msg.msg.nonce = self.bottomup_nonce; + + ch.push_cross_msgs(cross_msg, fee); + + // increment nonce + self.bottomup_nonce += 1; // flush checkpoint self.flush_checkpoint(store, &ch).map_err(|e| { @@ -239,55 +220,6 @@ impl State { Ok(()) } - /// append cross-msg and/or fee reward to a specific message meta. - pub(crate) fn append_msg_to_meta( - &mut self, - store: &BS, - meta_cid: &TCid>, - cross_msg: &CrossMsg, - ) -> anyhow::Result>> { - self.check_msg_registry.modify(store, |cross_reg| { - // get previous meta stored - let mut prev_crossmsgs = match cross_reg.get(&meta_cid.cid().to_bytes())? { - Some(m) => m.clone(), - None => { - // double-check that is not found because there were no messages - // in meta and not because we messed-up something. - if meta_cid.cid() != Cid::default() { - return Err(anyhow!("no msgmeta found for cid")); - } - // return empty messages - CrossMsgs::new() - } - }; - let added = prev_crossmsgs.add_msg(cross_msg); - if !added { - return Ok(meta_cid.clone()); - } - replace_msgmeta(cross_reg, meta_cid, prev_crossmsgs) - }) - } - - /// release circulating supply from a subent - /// - /// This is triggered through bottom-up messages sending subnet tokens - /// to some other subnet in the hierarchy. - /// store bottomup messages for their execution in the subnet - pub(crate) fn store_bottomup_msg( - &mut self, - store: &BS, - meta: &CrossMsgMeta, - ) -> anyhow::Result<()> { - let mut new_meta = meta.clone(); - new_meta.nonce = self.bottomup_nonce; - self.bottomup_nonce += 1; - self.bottomup_msg_meta.update(store, |crossmsgs| { - crossmsgs - .set(new_meta.nonce, new_meta) - .map_err(|e| anyhow!("failed to set crossmsg meta array: {:?}", e)) - }) - } - /// commit topdown messages for their execution in the subnet pub(crate) fn commit_topdown_msg( &mut self, @@ -310,9 +242,9 @@ impl State { })?; match sub { Some(mut sub) => { - cross_msg.msg.nonce = sub.nonce; + cross_msg.msg.nonce = sub.topdown_nonce; sub.store_topdown_msg(store, cross_msg)?; - sub.nonce += 1; + sub.topdown_nonce += 1; sub.circ_supply += &cross_msg.msg.value; self.flush_subnet(store, &sub)?; } @@ -330,37 +262,11 @@ impl State { &mut self, store: &BS, msg: &CrossMsg, - fee: &TokenAmount, curr_epoch: ChainEpoch, + fee: &TokenAmount, ) -> anyhow::Result<()> { // store bottom-up msg and fee in checkpoint for propagation - self.store_msg_in_checkpoint(store, msg, fee, curr_epoch)?; - // increment nonce - self.nonce += 1; - - Ok(()) - } - - pub fn bottomup_state_transition(&mut self, msg: &StorableMsg) -> anyhow::Result<()> { - // Bottom-up messages include the nonce of their message meta. Several messages - // will include the same nonce. They need to be applied in order of nonce. - - // As soon as we see a message with the next msgMeta nonce, we increment the nonce - // and start accepting the one for the next nonce. - if self.applied_bottomup_nonce == u64::MAX && msg.nonce == 0 { - self.applied_bottomup_nonce = 0; - } else if self.applied_bottomup_nonce.wrapping_add(1) == msg.nonce { - // wrapping add is used to prevent overflow. - self.applied_bottomup_nonce = self.applied_bottomup_nonce.wrapping_add(1); - }; - - if self.applied_bottomup_nonce != msg.nonce { - return Err(anyhow!( - "the bottom-up message being applied doesn't hold the subsequent nonce: nonce={} applied={}", - msg.nonce, - self.applied_bottomup_nonce, - )); - } + self.store_msg_in_checkpoint(store, msg, curr_epoch, fee)?; Ok(()) } @@ -525,30 +431,6 @@ fn get_checkpoint<'m, BS: Blockstore>( .map_err(|e| e.downcast_wrap(format!("failed to get checkpoint for id {}", epoch))) } -fn put_msgmeta( - registry: &mut Map, - metas: CrossMsgs, -) -> anyhow::Result>> { - let m_cid = metas.cid()?; - registry - .set(m_cid.cid().to_bytes().into(), metas) - .map_err(|e| e.downcast_wrap(format!("failed to set crossmsg meta for cid {}", m_cid)))?; - Ok(m_cid) -} - -/// insert a message meta and remove the old one. -fn replace_msgmeta( - registry: &mut Map, - prev_cid: &TCid>, - meta: CrossMsgs, -) -> anyhow::Result>> { - // add new meta - let m_cid = put_msgmeta(registry, meta)?; - // remove the previous one - registry.delete(&prev_cid.cid().to_bytes())?; - Ok(m_cid) -} - pub fn get_bottomup_msg<'m, BS: Blockstore>( crossmsgs: &'m CrossMsgMetaArray, nonce: u64, diff --git a/gateway/src/subnet.rs b/gateway/src/subnet.rs index 13b7331..bef22b1 100644 --- a/gateway/src/subnet.rs +++ b/gateway/src/subnet.rs @@ -26,7 +26,7 @@ pub struct Subnet { pub id: SubnetID, pub stake: TokenAmount, pub top_down_msgs: TCid>, - pub nonce: u64, + pub topdown_nonce: u64, pub circ_supply: TokenAmount, pub status: Status, pub prev_checkpoint: Option, diff --git a/gateway/src/types.rs b/gateway/src/types.rs index 1389fe6..aab7fe2 100644 --- a/gateway/src/types.rs +++ b/gateway/src/types.rs @@ -1,3 +1,4 @@ +use anyhow::anyhow; use cid::multihash::Code; use cid::{multihash, Cid}; use fil_actors_runtime::{cbor, ActorError, Array}; @@ -9,6 +10,7 @@ use fvm_shared::econ::TokenAmount; use ipc_sdk::subnet_id::SubnetID; use multihash::MultihashDigest; use primitives::CodeType; +use std::cmp::Ordering; use crate::checkpoint::{Checkpoint, CrossMsgMeta}; use crate::cross::CrossMsg; @@ -18,7 +20,6 @@ pub const MANIFEST_ID: &str = "ipc_gateway"; pub const CROSSMSG_AMT_BITWIDTH: u32 = 3; pub const DEFAULT_CHECKPOINT_PERIOD: ChainEpoch = 10; -pub const MAX_NONCE: u64 = u64::MAX; pub const MIN_COLLATERAL_AMOUNT: u64 = 10_u64.pow(18); pub const SUBNET_ACTOR_REWARD_METHOD: u64 = frc42_dispatch::method_hash!("Reward"); @@ -26,6 +27,12 @@ pub const SUBNET_ACTOR_REWARD_METHOD: u64 = frc42_dispatch::method_hash!("Reward pub type CrossMsgMetaArray<'bs, BS> = Array<'bs, CrossMsgMeta, BS>; pub type CrossMsgArray<'bs, BS> = Array<'bs, CrossMsg, BS>; +/// The executable message trait +pub trait ExecutableMessage { + /// Get the nonce of the message + fn nonce(&self) -> u64; +} + #[derive(Serialize_tuple, Deserialize_tuple)] pub struct ConstructorParams { pub network_name: String, @@ -101,6 +108,18 @@ impl PostBoxItem { } } +pub(crate) fn ensure_message_sorted(messages: &[E]) -> anyhow::Result<()> { + // check top down msgs + for i in 1..messages.len() { + match messages[i - 1].nonce().cmp(&messages[i].nonce()) { + Ordering::Less => {} + Ordering::Equal => return Err(anyhow!("top down messages not distinct")), + Ordering::Greater => return Err(anyhow!("top down messages not sorted")), + }; + } + Ok(()) +} + #[cfg(test)] mod tests { use crate::ConstructorParams; diff --git a/gateway/tests/gateway_test.rs b/gateway/tests/gateway_test.rs index 52fca28..9c247eb 100644 --- a/gateway/tests/gateway_test.rs +++ b/gateway/tests/gateway_test.rs @@ -10,10 +10,11 @@ use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use fvm_shared::error::ExitCode; use fvm_shared::METHOD_SEND; +use ipc_gateway::checkpoint::BatchCrossMsgs; use ipc_gateway::Status::{Active, Inactive}; use ipc_gateway::{ - get_topdown_msg, Checkpoint, CronCheckpoint, CronSubmission, CrossMsg, IPCAddress, State, - StorableMsg, CROSS_MSG_FEE, DEFAULT_CHECKPOINT_PERIOD, SUBNET_ACTOR_REWARD_METHOD, + get_topdown_msg, Checkpoint, CronCheckpoint, CronSubmission, CrossMsg, IPCAddress, PostBoxItem, + State, StorableMsg, CROSS_MSG_FEE, DEFAULT_CHECKPOINT_PERIOD, SUBNET_ACTOR_REWARD_METHOD, }; use ipc_sdk::subnet_id::SubnetID; use ipc_sdk::{Validator, ValidatorSet}; @@ -335,18 +336,30 @@ fn checkpoint_crossmsgs() { assert_eq!(subnet.status, Active); h.check_state(); + // found some to the subnet + let funder = Address::new_id(1001); + let amount = TokenAmount::from_atto(10_u64.pow(18)); + h.fund( + &mut rt, + &funder, + &shid, + ExitCode::OK, + amount.clone(), + 1, + &amount, + ) + .unwrap(); + // Commit first checkpoint for first window in first subnet let epoch: ChainEpoch = 10; rt.set_epoch(epoch); let mut ch = Checkpoint::new(shid.clone(), epoch + 9); - // and include some fees in msgmeta. + // and include some fees. let fee = TokenAmount::from_atto(5); - set_msg_meta( - &mut ch, - "rand1".as_bytes().to_vec(), - TokenAmount::zero(), - fee.clone(), - ); + ch.data.cross_msgs = BatchCrossMsgs { + cross_msgs: None, + fee: fee.clone(), + }; rt.expect_send( shid.subnet_actor(), @@ -457,27 +470,10 @@ fn test_release() { // Release funds let r_amount = TokenAmount::from_atto(5_u64.pow(18)); rt.set_balance(2 * r_amount.clone()); - let prev_cid = h - .release( - &mut rt, - &releaser, - ExitCode::OK, - r_amount.clone(), - 0, - &Cid::default(), - CROSS_MSG_FEE.clone(), - ) + h.release(&mut rt, &releaser, ExitCode::OK, r_amount.clone(), 0) + .unwrap(); + h.release(&mut rt, &releaser, ExitCode::OK, r_amount, 1) .unwrap(); - h.release( - &mut rt, - &releaser, - ExitCode::OK, - r_amount, - 1, - &prev_cid, - 2 * CROSS_MSG_FEE.clone(), - ) - .unwrap(); } #[test] @@ -585,11 +581,29 @@ fn test_send_cross() { /// This test covers the case where a bottom up cross_msg's target subnet is the SAME as that of /// the gateway. It should directly commit the message and will not save in postbox. #[test] -fn test_apply_msg_bu_target_subnet() { +fn test_commit_child_check_bu_target_subnet() { // ============== Register subnet ============== let shid = SubnetID::new_from_parent(&ROOTNET_ID, *SUBNET_ONE); let (h, mut rt) = setup(ROOTNET_ID.clone()); + h.register( + &mut rt, + &SUBNET_ONE, + &TokenAmount::from_atto(10_u64.pow(18)), + ExitCode::OK, + ) + .unwrap(); + h.fund( + &mut rt, + &Address::new_id(1001), + &shid, + ExitCode::OK, + TokenAmount::from_atto(10_u64.pow(18)), + 1, + &TokenAmount::from_atto(10_u64.pow(18)), + ) + .unwrap(); + let from = Address::new_bls(&[3; fvm_shared::address::BLS_PUB_LEN]).unwrap(); let to = Address::new_bls(&[4; fvm_shared::address::BLS_PUB_LEN]).unwrap(); @@ -604,7 +618,7 @@ fn test_apply_msg_bu_target_subnet() { let msg_nonce = 0; // Only system code is allowed to this method - let params = StorableMsg { + let msg = StorableMsg { to: tt.clone(), from: ff.clone(), method: METHOD_SEND, @@ -612,51 +626,82 @@ fn test_apply_msg_bu_target_subnet() { params: RawBytes::default(), nonce: msg_nonce, }; - let sto = tt.raw_addr().unwrap(); - - let cid = h - .apply_cross_execute_only( - &mut rt, - value.clone(), - params, - Some(Box::new(move |rt| { - rt.expect_send( - sto.clone(), - METHOD_SEND, - None, - value.clone(), - None, - ExitCode::OK, - ); - })), - ) + + let epoch: ChainEpoch = 10; + rt.set_epoch(epoch); + let mut ch = Checkpoint::new(shid.clone(), epoch + 9); + // and include some fees. + let fee = TokenAmount::from_atto(5); + ch.data.cross_msgs = BatchCrossMsgs { + cross_msgs: Some(vec![CrossMsg { + msg: msg.clone(), + wrapped: false, + }]), + fee: fee.clone(), + }; + + // execute bottom up + rt.expect_send( + msg.to.raw_addr().unwrap(), + msg.method, + None, + msg.value, + None, + ExitCode::OK, + ); + // distribute fee + rt.expect_send( + shid.subnet_actor(), + SUBNET_ACTOR_REWARD_METHOD, + None, + fee, + None, + ExitCode::OK, + ); + h.commit_child_check(&mut rt, &shid, &ch, ExitCode::OK) .unwrap(); - assert_eq!(cid.is_none(), true); } /// This test covers the case where a bottom up cross_msg's target subnet is NOT the same as that of /// the gateway. It will save it in the postbox. #[test] -fn test_apply_msg_bu_not_target_subnet() { +fn test_commit_child_check_bu_not_target_subnet() { // ============== Register subnet ============== - let shid = SubnetID::new_from_parent(&ROOTNET_ID, *SUBNET_ONE); - let (h, mut rt) = setup(shid.clone()); + let parent = SubnetID::new_from_parent(&ROOTNET_ID, *SUBNET_ONE); + let shid = SubnetID::new_from_parent(&parent, *SUBNET_TWO); + let (h, mut rt) = setup(parent); + + h.register( + &mut rt, + &shid.subnet_actor(), + &TokenAmount::from_atto(10_u64.pow(18)), + ExitCode::OK, + ) + .unwrap(); + h.fund( + &mut rt, + &Address::new_id(1001), + &shid, + ExitCode::OK, + TokenAmount::from_atto(10_u64.pow(18)), + 1, + &TokenAmount::from_atto(10_u64.pow(18)), + ) + .unwrap(); let from = Address::new_bls(&[3; fvm_shared::address::BLS_PUB_LEN]).unwrap(); let to = Address::new_bls(&[4; fvm_shared::address::BLS_PUB_LEN]).unwrap(); - let sub = shid.clone(); - // ================ Setup =============== let value = TokenAmount::from_atto(10_u64.pow(17)); // ================= Bottom-Up =============== - let ff = IPCAddress::new(&sub, &to).unwrap(); + let ff = IPCAddress::new(&shid.clone(), &to).unwrap(); let tt = IPCAddress::new(&ROOTNET_ID, &from).unwrap(); let msg_nonce = 0; // Only system code is allowed to this method - let params = StorableMsg { + let msg = StorableMsg { to: tt.clone(), from: ff.clone(), method: METHOD_SEND, @@ -664,43 +709,72 @@ fn test_apply_msg_bu_not_target_subnet() { params: RawBytes::default(), nonce: msg_nonce, }; - let cid = h - .apply_cross_execute_only(&mut rt, value.clone(), params.clone(), None) - .unwrap() + + let epoch: ChainEpoch = 10; + rt.set_epoch(epoch); + let mut ch = Checkpoint::new(shid.clone(), epoch + 9); + // and include some fees. + let fee = TokenAmount::from_atto(5); + ch.data.cross_msgs = BatchCrossMsgs { + cross_msgs: Some(vec![CrossMsg { + msg: msg.clone(), + wrapped: false, + }]), + fee: fee.clone(), + }; + + // distribute fee + rt.expect_send( + shid.subnet_actor(), + SUBNET_ACTOR_REWARD_METHOD, + None, + fee, + None, + ExitCode::OK, + ); + h.commit_child_check(&mut rt, &shid, &ch, ExitCode::OK) .unwrap(); // Part 1: test the message is stored in postbox let st: State = rt.get_state(); assert_ne!(tt.subnet().unwrap(), st.network_name); - // Check 1: `tt` is in `sub1`, which is not in that of `runtime` of gateway, will store in postbox - let item = st.load_from_postbox(rt.store(), cid.clone()).unwrap(); - assert_eq!(item.owners, Some(vec![ff.clone().raw_addr().unwrap()])); - let msg = item.cross_msg.msg; - assert_eq!(msg.to, tt); - // the nonce should not have changed at all - assert_eq!(msg.nonce, msg_nonce); - assert_eq!(msg.value, value); + // Check 1: `tt` is in `parent`, which is not in that of `runtime` of gateway, will store in postbox + let postbox = st.postbox.load(rt.store()).unwrap(); + let mut cid = None; + postbox + .for_each(|k, v| { + let item = PostBoxItem::deserialize(v.clone()).unwrap(); + assert_eq!(item.owners, Some(vec![ff.clone().raw_addr().unwrap()])); + let msg = item.cross_msg.msg; + assert_eq!(msg.to, tt); + // the nonce should not have changed at all + assert_eq!(msg.nonce, msg_nonce); + assert_eq!(msg.value, value); + + cid = Some(Cid::try_from(k.clone().to_vec()).unwrap()); + Ok(()) + }) + .unwrap(); // Part 2: Now we propagate from postbox // get the original subnet nonce first let caller = ff.clone().raw_addr().unwrap(); - let old_state: State = rt.get_state(); // propagating a bottom-up message triggers the // funds included in the message to be burnt. rt.expect_send( BURNT_FUNDS_ACTOR_ADDR, METHOD_SEND, None, - params.clone().value, + msg.clone().value, None, ExitCode::OK, ); h.propagate( &mut rt, caller, - cid.clone(), - ¶ms.value, + cid.unwrap().clone(), + &msg.value, TokenAmount::zero(), ) .unwrap(); @@ -709,11 +783,10 @@ fn test_apply_msg_bu_not_target_subnet() { let new_state: State = rt.get_state(); // cid should be removed from postbox - let r = new_state.load_from_postbox(rt.store(), cid.clone()); + let r = new_state.load_from_postbox(rt.store(), cid.unwrap()); assert_eq!(r.is_err(), true); let err = r.unwrap_err(); assert_eq!(err.to_string(), "cid not found in postbox"); - assert_eq!(new_state.nonce, old_state.nonce + 1); } /// This test covers the case where the amount send in the propagate @@ -722,8 +795,27 @@ fn test_apply_msg_bu_not_target_subnet() { #[test] fn test_propagate_with_remainder() { // ============== Register subnet ============== - let shid = SubnetID::new_from_parent(&ROOTNET_ID, *SUBNET_ONE); - let (h, mut rt) = setup(shid.clone()); + let parent = SubnetID::new_from_parent(&ROOTNET_ID, *SUBNET_ONE); + let shid = SubnetID::new_from_parent(&parent, *SUBNET_TWO); + + let (h, mut rt) = setup(parent); + h.register( + &mut rt, + &shid.subnet_actor(), + &TokenAmount::from_atto(10_u64.pow(18)), + ExitCode::OK, + ) + .unwrap(); + h.fund( + &mut rt, + &Address::new_id(1001), + &shid, + ExitCode::OK, + TokenAmount::from_atto(10_u64.pow(18)), + 1, + &TokenAmount::from_atto(10_u64.pow(18)), + ) + .unwrap(); let from = Address::new_bls(&[3; fvm_shared::address::BLS_PUB_LEN]).unwrap(); let to = Address::new_bls(&[4; fvm_shared::address::BLS_PUB_LEN]).unwrap(); @@ -747,30 +839,59 @@ fn test_propagate_with_remainder() { params: RawBytes::default(), nonce: msg_nonce, }; - let cid = h - .apply_cross_execute_only(&mut rt, value.clone(), params.clone(), None) - .unwrap() + + let epoch: ChainEpoch = 10; + rt.set_epoch(epoch); + let mut ch = Checkpoint::new(shid.clone(), epoch + 9); + // and include some fees. + let fee = TokenAmount::from_atto(5); + ch.data.cross_msgs = BatchCrossMsgs { + cross_msgs: Some(vec![CrossMsg { + msg: params.clone(), + wrapped: false, + }]), + fee: fee.clone(), + }; + + // distribute fee + rt.expect_send( + shid.subnet_actor(), + SUBNET_ACTOR_REWARD_METHOD, + None, + fee, + None, + ExitCode::OK, + ); + h.commit_child_check(&mut rt, &shid, &ch, ExitCode::OK) .unwrap(); // Part 1: test the message is stored in postbox let st: State = rt.get_state(); assert_ne!(tt.subnet().unwrap(), st.network_name); - // Check 1: `tt` is in `sub1`, which is not in that of `runtime` of gateway, will store in postbox - let item = st.load_from_postbox(rt.store(), cid.clone()).unwrap(); - assert_eq!(item.owners, Some(vec![ff.clone().raw_addr().unwrap()])); - let msg = item.cross_msg.msg; - assert_eq!(msg.to, tt); - // the nonce should not have changed at all - assert_eq!(msg.nonce, msg_nonce); - assert_eq!(msg.value, value); + // Check 1: `tt` is in `parent`, which is not in that of `runtime` of gateway, will store in postbox + let postbox = st.postbox.load(rt.store()).unwrap(); + let mut cid = None; + postbox + .for_each(|k, v| { + let item = PostBoxItem::deserialize(v.clone()).unwrap(); + assert_eq!(item.owners, Some(vec![ff.clone().raw_addr().unwrap()])); + let msg = item.cross_msg.msg; + assert_eq!(msg.to, tt); + // the nonce should not have changed at all + assert_eq!(msg.nonce, msg_nonce); + assert_eq!(msg.value, value); + + cid = Some(Cid::try_from(k.clone().to_vec()).unwrap()); + Ok(()) + }) + .unwrap(); // Part 2: Now we propagate from postbox // get the original subnet nonce first with an // excess to check that there is a remainder // to be returned let caller = ff.clone().raw_addr().unwrap(); - let old_state: State = rt.get_state(); // propagating a bottom-up message triggers the // funds included in the message to be burnt. rt.expect_send( @@ -781,25 +902,30 @@ fn test_propagate_with_remainder() { None, ExitCode::OK, ); - h.propagate(&mut rt, caller, cid.clone(), ¶ms.value, value.clone()) - .unwrap(); + h.propagate( + &mut rt, + caller, + cid.clone().unwrap(), + ¶ms.value, + value.clone(), + ) + .unwrap(); // state should be updated, load again let new_state: State = rt.get_state(); // cid should be removed from postbox - let r = new_state.load_from_postbox(rt.store(), cid.clone()); + let r = new_state.load_from_postbox(rt.store(), cid.unwrap()); assert_eq!(r.is_err(), true); let err = r.unwrap_err(); assert_eq!(err.to_string(), "cid not found in postbox"); - assert_eq!(new_state.nonce, old_state.nonce + 1); } /// This test covers the case where a bottom up cross_msg's target subnet is NOT the same as that of /// the gateway. It would save in postbox. Also, the gateway is the nearest parent, a switch to /// top down cross msg should occur. #[test] -fn test_apply_msg_bu_switch_td() { +fn test_commit_child_check_bu_switch_td() { // ============== Register subnet ============== let parent_sub = SubnetID::new_from_parent(&ROOTNET_ID, *SUBNET_ONE); let (h, mut rt) = setup(parent_sub.clone()); @@ -856,7 +982,7 @@ fn test_apply_msg_bu_switch_td() { let starting_nonce = get_subnet(&rt, &tt.subnet().unwrap().down(&h.net_name).unwrap()) .unwrap() - .nonce; + .topdown_nonce; // propagated as top-down, so it should distribute a fee in this subnet rt.expect_send( @@ -893,7 +1019,7 @@ fn test_apply_msg_bu_switch_td() { // the cross msg should have been committed to the next subnet, check this! let sub = get_subnet(&rt, &tt.subnet().unwrap().down(&h.net_name).unwrap()).unwrap(); - assert_eq!(sub.nonce, starting_nonce + 1); + assert_eq!(sub.topdown_nonce, starting_nonce + 1); let crossmsgs = sub.top_down_msgs.load(rt.store()).unwrap(); let msg = get_topdown_msg(&crossmsgs, starting_nonce).unwrap(); assert_eq!(msg.is_some(), true); @@ -907,22 +1033,38 @@ fn test_apply_msg_bu_switch_td() { /// This test covers the case where the cross_msg's target subnet is the SAME as that of /// the gateway. It would directly commit the message and will not save in postbox. #[test] -fn test_apply_msg_tp_target_subnet() { +fn test_commit_child_check_tp_target_subnet() { // ============== Register subnet ============== let shid = SubnetID::new_from_parent(&ROOTNET_ID, *SUBNET_ONE); - let (h, mut rt) = setup(shid.clone()); + let (h, mut rt) = setup(ROOTNET_ID.clone()); + + h.register( + &mut rt, + &SUBNET_ONE, + &TokenAmount::from_atto(10_u64.pow(18)), + ExitCode::OK, + ) + .unwrap(); + h.fund( + &mut rt, + &Address::new_id(1001), + &shid, + ExitCode::OK, + TokenAmount::from_atto(10_u64.pow(18)), + 1, + &TokenAmount::from_atto(10_u64.pow(18)), + ) + .unwrap(); let from = Address::new_bls(&[3; fvm_shared::address::BLS_PUB_LEN]).unwrap(); let to = Address::new_bls(&[4; fvm_shared::address::BLS_PUB_LEN]).unwrap(); - let sub = shid.clone(); - // ================ Setup =============== let value = TokenAmount::from_atto(10_u64.pow(17)); // ================= Top-Down =============== let ff = IPCAddress::new(&ROOTNET_ID, &from).unwrap(); - let tt = IPCAddress::new(&sub, &to).unwrap(); + let tt = IPCAddress::new(&shid.clone(), &to).unwrap(); let msg_nonce = 0; // Only system code is allowed to this method @@ -934,32 +1076,36 @@ fn test_apply_msg_tp_target_subnet() { params: RawBytes::default(), nonce: msg_nonce, }; - let sto = tt.raw_addr().unwrap(); - let v = value.clone(); - let cid = h - .apply_cross_execute_only( - &mut rt, - value.clone(), - params, - Some(Box::new(move |rt| { - rt.expect_send( - sto.clone(), - METHOD_SEND, - None, - v.clone(), - None, - ExitCode::OK, - ); - })), - ) + let epoch: ChainEpoch = 10; + rt.set_epoch(epoch); + let mut ch = Checkpoint::new(shid.clone(), epoch + 9); + // and include some fees. + let fee = TokenAmount::from_atto(5); + ch.data.cross_msgs = BatchCrossMsgs { + cross_msgs: Some(vec![CrossMsg { + msg: params.clone(), + wrapped: false, + }]), + fee: fee.clone(), + }; + + // distribute fee + rt.expect_send( + shid.subnet_actor(), + SUBNET_ACTOR_REWARD_METHOD, + None, + fee, + None, + ExitCode::OK, + ); + h.commit_child_check(&mut rt, &shid, &ch, ExitCode::OK) .unwrap(); - assert_eq!(cid.is_none(), true); } /// This test covers the case where the cross_msg's target subnet is not the same as that of /// the gateway. #[test] -fn test_apply_msg_tp_not_target_subnet() { +fn test_commit_child_check_tp_not_target_subnet() { // ============== Define Parameters ============== // gateway: /root/sub1 let shid = SubnetID::new_from_parent(&ROOTNET_ID, *SUBNET_ONE); @@ -1004,30 +1150,58 @@ fn test_apply_msg_tp_not_target_subnet() { params: RawBytes::default(), nonce: msg_nonce, }; - // cid is expected, should not be None - let cid = h - .apply_cross_execute_only(&mut rt, value.clone(), params.clone(), None) - .unwrap() + let epoch: ChainEpoch = 10; + rt.set_epoch(epoch); + let mut ch = Checkpoint::new(shid.clone(), epoch + 9); + // and include some fees. + let fee = TokenAmount::from_atto(5); + ch.data.cross_msgs = BatchCrossMsgs { + cross_msgs: Some(vec![CrossMsg { + msg: params.clone(), + wrapped: false, + }]), + fee: fee.clone(), + }; + + // distribute fee + rt.expect_send( + shid.subnet_actor(), + SUBNET_ACTOR_REWARD_METHOD, + None, + fee, + None, + ExitCode::OK, + ); + h.commit_child_check(&mut rt, &shid, &ch, ExitCode::OK) .unwrap(); // Part 1: test the message is stored in postbox let st: State = rt.get_state(); assert_ne!(tt.subnet().unwrap(), st.network_name); - // Check 1: `tt` is in `sub1`, which is not in that of `runtime` of gateway, will store in postbox - let item = st.load_from_postbox(rt.store(), cid.clone()).unwrap(); - assert_eq!(item.owners, Some(vec![ff.clone().raw_addr().unwrap()])); - let msg = item.cross_msg.msg; - assert_eq!(msg.to, tt); - // the nonce should not have changed at all - assert_eq!(msg.nonce, msg_nonce); - assert_eq!(msg.value, value); + // Check 1: `tt` is in `parent`, which is not in that of `runtime` of gateway, will store in postbox + let postbox = st.postbox.load(rt.store()).unwrap(); + let mut cid = None; + postbox + .for_each(|k, v| { + let item = PostBoxItem::deserialize(v.clone()).unwrap(); + assert_eq!(item.owners, Some(vec![ff.clone().raw_addr().unwrap()])); + let msg = item.cross_msg.msg; + assert_eq!(msg.to, tt); + // the nonce should not have changed at all + assert_eq!(msg.nonce, msg_nonce); + assert_eq!(msg.value, value); + + cid = Some(Cid::try_from(k.clone().to_vec()).unwrap()); + Ok(()) + }) + .unwrap(); // Part 2: Now we propagate from postbox // get the original subnet nonce first let starting_nonce = get_subnet(&rt, &tt.subnet().unwrap().down(&h.net_name).unwrap()) .unwrap() - .nonce; + .topdown_nonce; let caller = ff.clone().raw_addr().unwrap(); // propagated as top-down, so it should distribute a fee in this subnet @@ -1047,7 +1221,7 @@ fn test_apply_msg_tp_not_target_subnet() { h.propagate( &mut rt, caller, - cid.clone(), + cid.clone().unwrap(), ¶ms.value, TokenAmount::zero(), ) @@ -1057,14 +1231,14 @@ fn test_apply_msg_tp_not_target_subnet() { let st: State = rt.get_state(); // cid should be removed from postbox - let r = st.load_from_postbox(rt.store(), cid.clone()); + let r = st.load_from_postbox(rt.store(), cid.unwrap()); assert_eq!(r.is_err(), true); let err = r.unwrap_err(); assert_eq!(err.to_string(), "cid not found in postbox"); // the cross msg should have been committed to the next subnet, check this! let sub = get_subnet(&rt, &tt.subnet().unwrap().down(&h.net_name).unwrap()).unwrap(); - assert_eq!(sub.nonce, starting_nonce + 1); + assert_eq!(sub.topdown_nonce, starting_nonce + 1); let crossmsgs = sub.top_down_msgs.load(rt.store()).unwrap(); let msg = get_topdown_msg(&crossmsgs, starting_nonce).unwrap(); assert_eq!(msg.is_some(), true); @@ -1075,79 +1249,6 @@ fn test_apply_msg_tp_not_target_subnet() { assert_eq!(msg.value, value); } -#[test] -fn test_apply_msg_match_target_subnet() { - let (h, mut rt) = setup_root(); - - // Register a subnet with 1FIL collateral - let value = TokenAmount::from_atto(10_u64.pow(18)); - h.register(&mut rt, &SUBNET_ONE, &value, ExitCode::OK) - .unwrap(); - let shid = SubnetID::new_from_parent(&h.net_name, *SUBNET_ONE); - - // inject some funds - let funder_id = Address::new_id(1001); - let funder = IPCAddress::new( - &shid.parent().unwrap(), - &Address::new_bls(&[3; fvm_shared::address::BLS_PUB_LEN]).unwrap(), - ) - .unwrap(); - let amount = TokenAmount::from_atto(10_u64.pow(18)); - h.fund( - &mut rt, - &funder_id, - &shid, - ExitCode::OK, - amount.clone(), - 1, - &amount, - ) - .unwrap(); - - // Apply fund messages - for i in 0..5 { - h.apply_cross_msg(&mut rt, &funder, &funder, value.clone(), i, i, ExitCode::OK) - .unwrap(); - } - // Apply release messages - let from = IPCAddress::new(&shid, &BURNT_FUNDS_ACTOR_ADDR).unwrap(); - // with the same nonce - for _ in 0..5 { - h.apply_cross_msg(&mut rt, &from, &funder, value.clone(), 0, 0, ExitCode::OK) - .unwrap(); - } - // with increasing nonce - for i in 0..5 { - h.apply_cross_msg(&mut rt, &from, &funder, value.clone(), i, i, ExitCode::OK) - .unwrap(); - } - - // trying to apply non-subsequent nonce. - h.apply_cross_msg( - &mut rt, - &from, - &funder, - value.clone(), - 10, - 0, - ExitCode::USR_ILLEGAL_STATE, - ) - .unwrap(); - // trying already applied nonce - h.apply_cross_msg( - &mut rt, - &from, - &funder, - value.clone(), - 0, - 0, - ExitCode::USR_ILLEGAL_STATE, - ) - .unwrap(); - - // TODO: Trying to release over circulating supply -} - #[test] fn test_set_membership() { let (h, mut rt) = setup_root(); diff --git a/gateway/tests/harness.rs b/gateway/tests/harness.rs index 737ec01..fcedb20 100644 --- a/gateway/tests/harness.rs +++ b/gateway/tests/harness.rs @@ -1,6 +1,3 @@ -use anyhow::anyhow; -use cid::multihash::Code; -use cid::multihash::MultihashDigest; use cid::Cid; use fil_actors_runtime::builtin::HAMT_BIT_WIDTH; use fil_actors_runtime::deserialize_block; @@ -10,11 +7,10 @@ use fil_actors_runtime::test_utils::{ MockRuntime, ACCOUNT_ACTOR_CODE_ID, INIT_ACTOR_CODE_ID, MULTISIG_ACTOR_CODE_ID, SUBNET_ACTOR_CODE_ID, SYSTEM_ACTOR_CODE_ID, }; +use fil_actors_runtime::INIT_ACTOR_ADDR; use fil_actors_runtime::{ - make_map_with_root_and_bitwidth, ActorError, Map, BURNT_FUNDS_ACTOR_ADDR, SYSTEM_ACTOR_ADDR, + make_map_with_root_and_bitwidth, ActorError, BURNT_FUNDS_ACTOR_ADDR, SYSTEM_ACTOR_ADDR, }; -use fil_actors_runtime::{Array, INIT_ACTOR_ADDR}; -use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::ipld_block::IpldBlock; use fvm_ipld_encoding::RawBytes; use fvm_shared::address::Address; @@ -27,10 +23,9 @@ use fvm_shared::MethodNum; use fvm_shared::METHOD_SEND; use ipc_gateway::checkpoint::ChildCheck; use ipc_gateway::{ - ext, get_topdown_msg, is_bottomup, Actor, ApplyMsgParams, Checkpoint, ConstructorParams, - CrossMsg, CrossMsgMeta, CrossMsgParams, CrossMsgs, FundParams, IPCAddress, IPCMsgType, Method, - PropagateParams, State, StorableMsg, Subnet, SubnetID, CROSSMSG_AMT_BITWIDTH, CROSS_MSG_FEE, - DEFAULT_CHECKPOINT_PERIOD, MAX_NONCE, MIN_COLLATERAL_AMOUNT, + ext, get_topdown_msg, is_bottomup, Actor, Checkpoint, ConstructorParams, CrossMsg, + CrossMsgParams, FundParams, IPCAddress, Method, PropagateParams, State, StorableMsg, Subnet, + SubnetID, CROSS_MSG_FEE, DEFAULT_CHECKPOINT_PERIOD, MIN_COLLATERAL_AMOUNT, }; use ipc_gateway::{CronCheckpoint, SUBNET_ACTOR_REWARD_METHOD}; use ipc_sdk::ValidatorSet; @@ -102,22 +97,15 @@ impl Harness { self.construct(rt); let st: State = rt.get_state(); - let store = &rt.store; - - let empty_bottomup_array = Array::<(), _>::new_with_bit_width(store, CROSSMSG_AMT_BITWIDTH) - .flush() - .unwrap(); assert_eq!(st.network_name, self.net_name); assert_eq!(st.min_stake, TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT)); assert_eq!(st.check_period, DEFAULT_CHECKPOINT_PERIOD); - assert_eq!(st.applied_bottomup_nonce, MAX_NONCE); - assert_eq!(st.bottomup_msg_meta.cid(), empty_bottomup_array); + assert_eq!(st.applied_bottomup_nonce, 0); assert_eq!(st.cron_period, *DEFAULT_CRON_PERIOD); assert_eq!(st.genesis_epoch, *DEFAULT_GENESIS_EPOCH); verify_empty_map(rt, st.subnets.cid()); verify_empty_map(rt, st.checkpoints.cid()); - verify_empty_map(rt, st.check_msg_registry.cid()); } pub fn register( @@ -334,7 +322,7 @@ impl Harness { .unwrap() .unwrap(); assert_eq!(&sub.circ_supply, expected_circ_sup); - assert_eq!(sub.nonce, expected_nonce); + assert_eq!(sub.topdown_nonce, expected_nonce); let from = IPCAddress::new(&self.net_name, &*TEST_BLS).unwrap(); let to = IPCAddress::new(&id, &TEST_BLS).unwrap(); assert_eq!(msg.from, from); @@ -352,8 +340,6 @@ impl Harness { code: ExitCode, value: TokenAmount, expected_nonce: u64, - prev_meta: &Cid, - expected_fee: TokenAmount, ) -> Result { rt.set_caller(*ACCOUNT_ACTOR_CODE_ID, *releaser); rt.expect_validate_caller_type(SIG_TYPES.clone()); @@ -393,30 +379,14 @@ impl Harness { let to = IPCAddress::new(&parent, &TEST_BLS).unwrap(); rt.set_epoch(0); let ch = st.get_window_checkpoint(rt.store(), 0).unwrap(); - let chmeta = ch.cross_msgs().unwrap(); - // check that fees are collected - assert_eq!(chmeta.fee, expected_fee); - - let cross_reg = st.check_msg_registry.load(rt.store()).unwrap(); - let meta = get_cross_msgs(&cross_reg, &chmeta.msgs_cid.cid()) - .unwrap() - .unwrap(); - let msg = meta.msgs[expected_nonce as usize].clone(); - assert_eq!(meta.msgs.len(), (expected_nonce + 1) as usize); + let msg = ch.data.cross_msgs.cross_msgs.unwrap()[expected_nonce as usize].clone(); assert_eq!(msg.msg.from, from); assert_eq!(msg.msg.to, to); assert_eq!(msg.msg.nonce, expected_nonce); assert_eq!(msg.msg.value, value); - if prev_meta != &Cid::default() { - match get_cross_msgs(&cross_reg, &prev_meta).unwrap() { - Some(_) => panic!("previous meta should have been removed"), - None => {} - } - } - - Ok(chmeta.msgs_cid.cid()) + Ok(Cid::default()) } pub fn send_cross( @@ -500,15 +470,8 @@ impl Harness { let to = IPCAddress::new(&dest, &to).unwrap(); rt.set_epoch(0); let ch = st.get_window_checkpoint(rt.store(), 0).unwrap(); - let chmeta = ch.cross_msgs(); - let cross_reg = st.check_msg_registry.load(rt.store()).unwrap(); - let meta = get_cross_msgs(&cross_reg, &chmeta.unwrap().msgs_cid.cid()) - .unwrap() - .unwrap(); - let msg = meta.msgs[nonce as usize].clone(); - - assert_eq!(meta.msgs.len(), (nonce + 1) as usize); + let msg = ch.data.cross_msgs.cross_msgs.unwrap()[nonce as usize].clone(); assert_eq!(msg.msg.from, from); assert_eq!(msg.msg.to, to); assert_eq!(msg.msg.nonce, nonce); @@ -521,7 +484,7 @@ impl Harness { let crossmsgs = sub.top_down_msgs.load(rt.store()).unwrap(); let msg = get_topdown_msg(&crossmsgs, nonce - 1).unwrap().unwrap(); assert_eq!(&sub.circ_supply, expected_circ_sup); - assert_eq!(sub.nonce, nonce); + assert_eq!(sub.topdown_nonce, nonce); let from = IPCAddress::new(&self.net_name, &SYSTEM_ACTOR_ADDR).unwrap(); let to = IPCAddress::new(&dest, &to).unwrap(); assert_eq!(msg.from, from); @@ -533,39 +496,6 @@ impl Harness { Ok(()) } - pub fn apply_cross_execute_only( - &self, - rt: &mut MockRuntime, - balance: TokenAmount, - params: StorableMsg, - append_expected_send: Option>, - ) -> Result, ActorError> { - rt.set_caller(*SYSTEM_ACTOR_CODE_ID, SYSTEM_ACTOR_ADDR); - rt.expect_validate_caller_addr(vec![SYSTEM_ACTOR_ADDR.clone()]); - rt.set_balance(balance); - - if let Some(f) = append_expected_send { - f(rt) - } - let cid_blk = rt.call::( - Method::ApplyMessage as MethodNum, - IpldBlock::serialize_cbor(&ApplyMsgParams { - cross_msg: CrossMsg { - msg: params.clone(), - wrapped: false, - }, - })?, - )?; - rt.verify(); - - let cid: RawBytes = deserialize_block(cid_blk).unwrap(); - if cid.is_empty() { - Ok(None) - } else { - Ok(Some(Cid::try_from(cid.to_vec().as_slice()).unwrap())) - } - } - pub fn propagate( &self, rt: &mut MockRuntime, @@ -592,127 +522,6 @@ impl Harness { Ok(()) } - pub fn apply_cross_msg( - &self, - rt: &mut MockRuntime, - from: &IPCAddress, - to: &IPCAddress, - value: TokenAmount, - msg_nonce: u64, - td_nonce: u64, - code: ExitCode, - ) -> Result<(), ActorError> { - rt.set_caller(*SYSTEM_ACTOR_CODE_ID, SYSTEM_ACTOR_ADDR); - rt.expect_validate_caller_addr(vec![SYSTEM_ACTOR_ADDR.clone()]); - - rt.set_balance(value.clone()); - let params = StorableMsg { - to: to.clone(), - from: from.clone(), - method: METHOD_SEND, - value: value.clone(), - params: RawBytes::default(), - nonce: msg_nonce, - }; - - let st: State = rt.get_state(); - let sto = params.to.subnet().unwrap(); - let rto = to.raw_addr().unwrap(); - - // if expected code is not ok - if code != ExitCode::OK { - expect_abort( - code, - rt.call::( - Method::ApplyMessage as MethodNum, - IpldBlock::serialize_cbor(&ApplyMsgParams { - cross_msg: CrossMsg { - msg: params.clone(), - wrapped: false, - }, - }) - .unwrap(), - ), - ); - rt.verify(); - return Ok(()); - } - - if params.apply_type(&st.network_name).unwrap() == IPCMsgType::BottomUp { - if sto == st.network_name { - rt.expect_send( - rto, - METHOD_SEND, - None, - params.value.clone(), - None, - ExitCode::OK, - ); - } - - rt.call::( - Method::ApplyMessage as MethodNum, - IpldBlock::serialize_cbor(&ApplyMsgParams { - cross_msg: CrossMsg { - msg: params.clone(), - wrapped: false, - }, - }) - .unwrap(), - )?; - rt.verify(); - let st: State = rt.get_state(); - assert_eq!(st.applied_bottomup_nonce, msg_nonce); - } else { - if sto == st.network_name { - rt.expect_send( - rto, - METHOD_SEND, - None, - params.value.clone(), - None, - ExitCode::OK, - ); - } - let cid_blk = rt.call::( - Method::ApplyMessage as MethodNum, - IpldBlock::serialize_cbor(&ApplyMsgParams { - cross_msg: CrossMsg { - msg: params.clone(), - wrapped: false, - }, - }) - .unwrap(), - )?; - rt.verify(); - let st: State = rt.get_state(); - - if sto != st.network_name { - let sub = self - .get_subnet(rt, &sto.down(&self.net_name).unwrap()) - .unwrap(); - assert_eq!(sub.nonce, td_nonce); - let crossmsgs = sub.top_down_msgs.load(rt.store()).unwrap(); - let msg = get_topdown_msg(&crossmsgs, td_nonce).unwrap(); - assert_eq!(msg.is_none(), true); - - let cid: RawBytes = deserialize_block(cid_blk).unwrap(); - let cid_ref = cid.to_vec(); - let item = st - .load_from_postbox(rt.store(), Cid::try_from(cid_ref.as_slice()).unwrap()) - .unwrap(); - assert_eq!(item.owners, Some(vec![from.clone().raw_addr().unwrap()])); - let msg = item.cross_msg.msg; - assert_eq!(&msg.to, to); - assert_eq!(msg.nonce, msg_nonce); - assert_eq!(msg.value, value); - } else { - assert_eq!(st.applied_topdown_nonce, msg_nonce + 1); - } - } - Ok(()) - } - pub fn check_state(&self) { // TODO: https://github.com/filecoin-project/builtin-actors/issues/44 } @@ -783,15 +592,6 @@ pub fn has_cid<'a, T: TCidContent>(children: &'a Vec>, cid: &Cid) -> boo children.iter().any(|c| c.cid() == *cid) } -pub fn get_cross_msgs<'m, BS: Blockstore>( - registry: &'m Map, - cid: &Cid, -) -> anyhow::Result> { - registry - .get(&cid.to_bytes()) - .map_err(|e| anyhow!("error getting fross messages: {:?}", e)) -} - fn set_rt_value_with_cross_fee(rt: &mut MockRuntime, value: &TokenAmount) { rt.set_value(if value.clone() != TokenAmount::zero() { value.clone() + &*CROSS_MSG_FEE @@ -799,18 +599,3 @@ fn set_rt_value_with_cross_fee(rt: &mut MockRuntime, value: &TokenAmount) { value.clone() }); } - -pub fn set_msg_meta(ch: &mut Checkpoint, rand: Vec, value: TokenAmount, fee: TokenAmount) { - let mh_code = Code::Blake2b256; - let c = TCid::from(Cid::new_v1( - fvm_ipld_encoding::DAG_CBOR, - mh_code.digest(&rand), - )); - let meta = CrossMsgMeta { - msgs_cid: c, - nonce: 0, - value, - fee, - }; - ch.set_cross_msgs(meta); -} diff --git a/subnet-actor/src/lib.rs b/subnet-actor/src/lib.rs index 083b0a8..dabc8f9 100644 --- a/subnet-actor/src/lib.rs +++ b/subnet-actor/src/lib.rs @@ -33,6 +33,7 @@ pub enum Method { Leave = frc42_dispatch::method_hash!("Leave"), Kill = frc42_dispatch::method_hash!("Kill"), SubmitCheckpoint = frc42_dispatch::method_hash!("SubmitCheckpoint"), + SetValidatorNetAddr = frc42_dispatch::method_hash!("SetValidatorNetAddr"), Reward = frc42_dispatch::method_hash!("Reward"), } @@ -358,6 +359,37 @@ impl SubnetActor for Actor { } } +/// This impl includes methods that are not required by the subnet actor +/// trait. +impl Actor { + /// Sets a new net address to an existing validator + pub fn set_validator_net_addr( + rt: &mut impl Runtime, + params: JoinParams, + ) -> Result, ActorError> { + rt.validate_immediate_caller_type(CALLER_TYPES_SIGNABLE.iter())?; + let caller = rt.message().caller(); + + rt.transaction(|st: &mut State, _rt| { + // if the caller is a validator allow him to change his net addr + if let Some(index) = st + .validator_set + .validators() + .iter() + .position(|x| x.addr == caller) + { + if let Some(x) = st.validator_set.validators_mut().get_mut(index) { + x.net_addr = params.validator_net_addr; + } + } else { + return Err(actor_error!(forbidden, "caller is not a validator")); + } + Ok(()) + })?; + Ok(None) + } +} + impl ActorCode for Actor { type Methods = Method; @@ -368,5 +400,6 @@ impl ActorCode for Actor { Kill => kill, SubmitCheckpoint => submit_checkpoint, Reward => reward, + SetValidatorNetAddr => set_validator_net_addr, } } diff --git a/subnet-actor/tests/actor_test.rs b/subnet-actor/tests/actor_test.rs index 51403bd..c2c1843 100644 --- a/subnet-actor/tests/actor_test.rs +++ b/subnet-actor/tests/actor_test.rs @@ -115,6 +115,79 @@ mod test { ); } + #[test] + fn test_set_net_addr_works() { + let mut runtime = construct_runtime(); + + let caller = Address::new_id(10); + let validator = Address::new_id(100); + let params = JoinParams { + validator_net_addr: validator.to_string(), + }; + let gateway = Address::new_id(IPC_GATEWAY_ADDR); + + // join + let value = TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT); + runtime.set_value(value.clone()); + runtime.set_balance(value.clone()); + runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, caller.clone()); + runtime.expect_validate_caller_type(SIG_TYPES.clone()); + runtime.expect_send( + gateway.clone(), + ipc_gateway::Method::Register as u64, + None, + TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT), + None, + ExitCode::new(0), + ); + runtime + .call::( + Method::Join as u64, + IpldBlock::serialize_cbor(¶ms).unwrap(), + ) + .unwrap(); + + // modify net address + let new_addr = String::from("test_addr"); + let params = JoinParams { + validator_net_addr: new_addr.clone(), + }; + + runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, caller.clone()); + runtime.expect_validate_caller_type(SIG_TYPES.clone()); + runtime + .call::( + Method::SetValidatorNetAddr as u64, + IpldBlock::serialize_cbor(¶ms).unwrap(), + ) + .unwrap(); + + let st: State = runtime.get_state(); + + if let Some(val) = st + .validator_set + .validators() + .iter() + .find(|x| x.addr == caller) + { + assert_eq!(val.net_addr, new_addr); + } else { + panic!("validator address not set correctly") + } + + // user which is not a validator tries to change address + let caller = Address::new_id(11); + runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, caller.clone()); + runtime.expect_validate_caller_type(SIG_TYPES.clone()); + expect_abort( + ExitCode::USR_FORBIDDEN, + runtime.call::( + Method::SetValidatorNetAddr as u64, + IpldBlock::serialize_cbor(¶ms).unwrap(), + ), + ); + } + #[test] fn test_join_works() { let mut runtime = construct_runtime(); From a119ac8b9ffc8ac48fe4af6bd86c684037872d14 Mon Sep 17 00:00:00 2001 From: Alfonso de la Rocha Date: Tue, 4 Apr 2023 11:08:25 +0200 Subject: [PATCH 19/27] cargo fmt --- gateway/src/checkpoint.rs | 2 +- sdk/src/lib.rs | 4 ++-- subnet-actor/src/state.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gateway/src/checkpoint.rs b/gateway/src/checkpoint.rs index eb0dba3..dcd8f60 100644 --- a/gateway/src/checkpoint.rs +++ b/gateway/src/checkpoint.rs @@ -7,8 +7,8 @@ use fvm_ipld_encoding::{serde_bytes, to_vec}; use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use ipc_sdk::subnet_id::SubnetID; -use num_traits::Zero; use lazy_static::lazy_static; +use num_traits::Zero; use primitives::{TCid, TLink}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs index 3ebf876..543bb62 100644 --- a/sdk/src/lib.rs +++ b/sdk/src/lib.rs @@ -1,6 +1,6 @@ -use fvm_ipld_encoding::tuple::{Deserialize_tuple, Serialize_tuple}; -use fvm_shared::{econ::TokenAmount, address::Address}; use fil_actors_runtime::fvm_ipld_hamt::BytesKey; +use fvm_ipld_encoding::tuple::{Deserialize_tuple, Serialize_tuple}; +use fvm_shared::{address::Address, econ::TokenAmount}; use fvm_shared::{ address::{set_current_network, Network}, clock::ChainEpoch, diff --git a/subnet-actor/src/state.rs b/subnet-actor/src/state.rs index fdb61e0..4333d0a 100644 --- a/subnet-actor/src/state.rs +++ b/subnet-actor/src/state.rs @@ -11,8 +11,8 @@ use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use ipc_gateway::checkpoint::CHECKPOINT_GENESIS_CID; use ipc_gateway::{Checkpoint, SubnetID, DEFAULT_CHECKPOINT_PERIOD, MIN_COLLATERAL_AMOUNT}; -use ipc_sdk::{Validator, ValidatorSet}; use ipc_sdk::epoch_key; +use ipc_sdk::{Validator, ValidatorSet}; use lazy_static::lazy_static; use num::rational::Ratio; use num::BigInt; From de81961965b98c4589ca7064ee33b6e1bc2d59a9 Mon Sep 17 00:00:00 2001 From: cryptoAtwill <108330426+cryptoAtwill@users.noreply.github.com> Date: Wed, 5 Apr 2023 14:52:48 +0800 Subject: [PATCH 20/27] Vote checkpoints (#81) * track validators * add validator check to submit cron * update impl * weighted vote * Update gateway/src/cron.rs Co-authored-by: adlrocha * update method name * add tests * refactor pending epoches * fix clippy * add more tests * initial commit * Cross execution (#75) * update bottom up execution * update cross message execution * fix fmt * update review and clean up * check message ordering * Cross execution tests (#76) * fix clippy * fmt code * generics for cron submission * migrate to sdk * format code * remove wip field * work in progress * local changes * reorg code * update comment * update tests * format code and clippy * fix error --------- Co-authored-by: adlrocha --- Cargo.toml | 1 + common/Cargo.toml | 26 ++ common/src/lib.rs | 3 + common/src/vote/mod.rs | 13 + common/src/vote/submission.rs | 590 +++++++++++++++++++++++++++++++ common/src/vote/voting.rs | 466 ++++++++++++++++++++++++ gateway/Cargo.toml | 1 + gateway/src/checkpoint.rs | 31 +- gateway/src/cron.rs | 486 +------------------------ gateway/src/lib.rs | 185 +++------- gateway/src/state.rs | 46 +-- gateway/src/subnet.rs | 3 +- gateway/src/types.rs | 3 +- gateway/tests/gateway_test.rs | 85 +++-- gateway/tests/harness.rs | 10 +- sdk/Cargo.toml | 12 +- subnet-actor/Cargo.toml | 1 + subnet-actor/src/lib.rs | 141 +++++--- subnet-actor/src/state.rs | 175 ++++----- subnet-actor/src/types.rs | 7 +- subnet-actor/tests/actor_test.rs | 334 ++++++++++++++--- 21 files changed, 1687 insertions(+), 932 deletions(-) create mode 100644 common/Cargo.toml create mode 100644 common/src/lib.rs create mode 100644 common/src/vote/mod.rs create mode 100644 common/src/vote/submission.rs create mode 100644 common/src/vote/voting.rs diff --git a/Cargo.toml b/Cargo.toml index d2a2294..ca33ffd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ members = [ "gateway", "subnet-actor", "sdk", + "common", "atomic-exec", "atomic-exec/primitives", "atomic-exec/examples/fungible-token", diff --git a/common/Cargo.toml b/common/Cargo.toml new file mode 100644 index 0000000..aaee9f6 --- /dev/null +++ b/common/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "ipc-actor-common" +description = "The common code used by both gateway actor and subnet actor, but not by sdk exposed to users" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.56" +log = "0.4.17" +primitives = { git = "https://github.com/consensus-shipyard/fvm-utils" } +ipc-sdk = { path = "../sdk" } +num-traits = "0.2.14" +fvm_ipld_blockstore = "0.1.1" +fvm_ipld_encoding = "0.3.3" +lazy_static = "1.4.0" +serde_tuple = "0.5" +serde = { version = "1.0.136", features = ["derive"] } +fvm_shared = { version = "=3.0.0-alpha.17", default-features = false } +thiserror = "1.0.38" +fil_actors_runtime = { git = "https://github.com/consensus-shipyard/fvm-utils", features = ["fil-actor"] } +integer-encoding = { version = "3.0.3", default-features = false } + +[dev-dependencies] +serde_json = "1.0.95" \ No newline at end of file diff --git a/common/src/lib.rs b/common/src/lib.rs new file mode 100644 index 0000000..ac19ff0 --- /dev/null +++ b/common/src/lib.rs @@ -0,0 +1,3 @@ +#![feature(map_first_last)] + +pub mod vote; diff --git a/common/src/vote/mod.rs b/common/src/vote/mod.rs new file mode 100644 index 0000000..aba570c --- /dev/null +++ b/common/src/vote/mod.rs @@ -0,0 +1,13 @@ +mod submission; +mod voting; + +pub use crate::vote::submission::EpochVoteSubmissions; +pub use crate::vote::voting::Voting; + +pub type UniqueBytesKey = Vec; + +/// The vote trait that requires each vote to be unique by `unique_key`. +pub trait UniqueVote: PartialEq + Clone { + /// Outputs the unique bytes key of the vote + fn unique_key(&self) -> anyhow::Result; +} diff --git a/common/src/vote/submission.rs b/common/src/vote/submission.rs new file mode 100644 index 0000000..ff513ba --- /dev/null +++ b/common/src/vote/submission.rs @@ -0,0 +1,590 @@ +//! Contains the inner implementation of the voting process + +use crate::vote::{UniqueBytesKey, UniqueVote}; +use anyhow::anyhow; +use fil_actors_runtime::fvm_ipld_hamt::BytesKey; +use fvm_ipld_blockstore::Blockstore; +use fvm_shared::address::Address; +use fvm_shared::econ::TokenAmount; +use num_traits::Zero; +use primitives::{TCid, THamt}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::ops::Mul; + +pub type Ratio = (u64, u64); + +/// Track all the vote submissions of an epoch +#[derive(PartialEq, Eq, Clone, Debug)] +pub struct EpochVoteSubmissions { + /// The summation of the weights from all validator submissions + pub total_submission_weight: TokenAmount, + /// The most submitted unique key. + pub most_voted_key: Option, + /// The addresses of all the submitters + pub submitters: TCid>, + /// The map to track the submission weight of each unique key + pub submission_weights: TCid>, + /// The different cron checkpoints, with vote's unique key as key + pub submissions: TCid>, +} + +/// The status indicating if the voting should be executed +#[derive(Eq, PartialEq, Debug)] +pub enum VoteExecutionStatus { + /// The execution threshold has yet to be reached + ThresholdNotReached, + /// The voting threshold has reached, but consensus has yet to be reached, needs more + /// voting to reach consensus + ReachingConsensus, + /// Consensus cannot be reached in this round + RoundAbort, + /// Execution threshold reached + ConsensusReached, +} + +impl EpochVoteSubmissions { + pub fn new(store: &BS) -> anyhow::Result { + Ok(EpochVoteSubmissions { + total_submission_weight: TokenAmount::zero(), + submitters: TCid::new_hamt(store)?, + most_voted_key: None, + submission_weights: TCid::new_hamt(store)?, + submissions: TCid::new_hamt(store)?, + }) + } + + /// Abort the current round and reset the submission data. + pub fn abort(&mut self, store: &BS) -> anyhow::Result<()> { + self.total_submission_weight = TokenAmount::zero(); + self.submitters = TCid::new_hamt(store)?; + self.most_voted_key = None; + self.submission_weights = TCid::new_hamt(store)?; + + // no need reset `self.submissions`, we can still reuse the previous self.submissions + // new submissions will be inserted, old submission will not be inserted to save + // gas. + + Ok(()) + } + + /// Submit a vote as the submitter. + pub fn submit( + &mut self, + store: &BS, + submitter: Address, + submitter_weight: TokenAmount, + vote: T, + ) -> anyhow::Result { + self.update_submitters(store, submitter)?; + self.total_submission_weight += &submitter_weight; + let checkpoint_hash = self.insert_vote(store, vote)?; + self.update_submission_weight(store, checkpoint_hash, submitter_weight) + } + + pub fn load_most_voted_submission( + &self, + store: &BS, + ) -> anyhow::Result> { + // we will only have one entry in the `most_submitted` set if more than 2/3 has reached + if let Some(unique_key) = &self.most_voted_key { + self.get_submission(store, unique_key) + } else { + Ok(None) + } + } + + pub fn load_most_voted_weight( + &self, + store: &BS, + ) -> anyhow::Result> { + // we will only have one entry in the `most_submitted` set if more than 2/3 has reached + if let Some(unique_key) = &self.most_voted_key { + self.get_submission_weight(store, unique_key) + } else { + Ok(None) + } + } + + pub fn get_submission( + &self, + store: &BS, + unique_key: &UniqueBytesKey, + ) -> anyhow::Result> { + let hamt = self.submissions.load(store)?; + let key = BytesKey::from(unique_key.as_slice()); + Ok(hamt.get(&key)?.cloned()) + } + + pub fn derive_execution_status( + &self, + total_weight: TokenAmount, + most_voted_weight: TokenAmount, + ratio: &Ratio, + ) -> VoteExecutionStatus { + // threshold keeps track of the weight of validators that have already + // voted for the checkpoint + let threshold = total_weight.clone().mul(ratio.0).div_floor(ratio.1); + + // note that we require THRESHOLD to be surpassed, equality is not enough! + if self.total_submission_weight <= threshold { + return VoteExecutionStatus::ThresholdNotReached; + } + + // now we have reached the threshold + + // consensus reached + if most_voted_weight > threshold { + return VoteExecutionStatus::ConsensusReached; + } + + // now the total submissions has reached the threshold, but the most submitted vote + // has yet to reach the threshold, that means consensus has not reached. + + // we do a early termination check, to see if consensus will ever be reached. + // + // consider an example that consensus will never be reached: + // + // -------- | -------------------------|--------------- | ------------- | + // MOST_VOTED THRESHOLD TOTAL_SUBMISSIONS TOTAL_WEIGHT + // + // we see MOST_VOTED is smaller than THRESHOLD, TOTAL_SUBMISSIONS and TOTAL_WEIGHT, if + // the potential extra votes any vote can obtain, i.e. TOTAL_WEIGHT - TOTAL_SUBMISSIONS, + // is smaller than or equal to the potential extra vote the most voted can obtain, i.e. + // THRESHOLD - MOST_VOTED, then consensus will never be reached, no point voting, just abort. + if threshold - most_voted_weight >= total_weight - &self.total_submission_weight { + VoteExecutionStatus::RoundAbort + } else { + VoteExecutionStatus::ReachingConsensus + } + } + + /// Checks if the submitter has already submitted the checkpoint. + pub fn has_submitted( + &self, + store: &BS, + submitter: &Address, + ) -> anyhow::Result { + let addr_byte_key = BytesKey::from(submitter.to_bytes()); + let hamt = self.submitters.load(store)?; + Ok(hamt.contains_key(&addr_byte_key)?) + } +} + +// Private and internal implementations +impl EpochVoteSubmissions { + /// Checks if the checkpoint unique key has already inserted in the store + fn get_submission_weight( + &self, + store: &BS, + unique_key: &UniqueBytesKey, + ) -> anyhow::Result> { + let hamt = self.submission_weights.load(store)?; + let r = hamt.get(&BytesKey::from(unique_key.as_slice()))?; + Ok(r.cloned()) + } + + /// Update the total submitters, returns the latest total number of submitters + fn update_submitters( + &mut self, + store: &BS, + submitter: Address, + ) -> anyhow::Result<()> { + let addr_byte_key = BytesKey::from(submitter.to_bytes()); + self.submitters.modify(store, |hamt| { + // check the submitter has not submitted before + if hamt.contains_key(&addr_byte_key)? { + return Err(anyhow!("already submitted")); + } + + // now the submitter has not submitted before, mark as submitted + hamt.set(addr_byte_key, ())?; + + Ok(()) + }) + } + + /// Insert the vote to store if it has not been submitted before. Returns the unique of the checkpoint. + fn insert_vote( + &mut self, + store: &BS, + vote: T, + ) -> anyhow::Result { + let unique_key = vote.unique_key()?; + let hash_key = BytesKey::from(unique_key.as_slice()); + + let hamt = self.submissions.load(store)?; + if hamt.contains_key(&hash_key)? { + return Ok(unique_key); + } + + // checkpoint has not been submitted before + self.submissions.modify(store, |hamt| { + hamt.set(hash_key, vote)?; + Ok(()) + })?; + + Ok(unique_key) + } + + /// Update submission weight of the unique key. Returns the currently most submitted submission count. + fn update_submission_weight( + &mut self, + store: &BS, + unique_key: UniqueBytesKey, + weight: TokenAmount, + ) -> anyhow::Result { + let hash_byte_key = BytesKey::from(unique_key.as_slice()); + + self.submission_weights.modify(store, |hamt| { + let new_weight = hamt + .get(&hash_byte_key)? + .cloned() + .unwrap_or_else(TokenAmount::zero) + + weight; + + // update the new count + hamt.set(hash_byte_key, new_weight.clone())?; + + // now we compare with the most submitted unique key or vote + if self.most_voted_key.is_none() { + // no most submitted unique_key set yet, set to current + self.most_voted_key = Some(unique_key); + return Ok(new_weight); + } + + let most_voted_key = self.most_voted_key.as_mut().unwrap(); + + // the current submission is already one of the most submitted entries + if most_voted_key == &unique_key { + // the current submission is already the only one submission, no need update + + // return the current checkpoint's count as the current most submitted checkpoint + return Ok(new_weight); + } + + // the current submission is not part of the most submitted entries, need to check + // the most submitted entry to compare if the current submission is exceeding + + let most_submitted_key = BytesKey::from(most_voted_key.as_slice()); + + // safe to unwrap as the hamt must contain the key + let most_submitted_weight = hamt.get(&most_submitted_key)?.unwrap(); + // current submission is not the most voted checkpoints + // if new_count < *most_submitted_count, we do nothing as the new count is not close to the most submitted + if new_weight > *most_submitted_weight { + *most_voted_key = unique_key; + Ok(new_weight) + } else { + Ok(most_submitted_weight.clone()) + } + }) + } + + /// Checks if the checkpoint unique key has already inserted in the store + #[cfg(test)] + fn is_vote_inserted( + &self, + store: &BS, + unique_key: &UniqueBytesKey, + ) -> anyhow::Result { + let hamt = self.submissions.load(store)?; + Ok(hamt.contains_key(&BytesKey::from(unique_key.as_slice()))?) + } +} + +impl Serialize for EpochVoteSubmissions { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let inner = ( + &self.total_submission_weight, + &self.most_voted_key, + &self.submitters, + &self.submission_weights, + &self.submissions, + ); + serde::Serialize::serialize(&inner, serde_tuple::Serializer(serializer)) + } +} + +impl<'de, T: DeserializeOwned> Deserialize<'de> for EpochVoteSubmissions { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + type Inner = ( + TokenAmount, + Option, + TCid>, + TCid>, + TCid>, + ); + let inner = >::deserialize(serde_tuple::Deserializer(deserializer))?; + + Ok(EpochVoteSubmissions { + total_submission_weight: inner.0, + most_voted_key: inner.1, + submitters: inner.2, + submission_weights: inner.3, + submissions: inner.4, + }) + } +} + +#[cfg(test)] +mod tests { + use crate::vote::submission::VoteExecutionStatus; + use crate::vote::{EpochVoteSubmissions, UniqueBytesKey, UniqueVote}; + use fil_actors_runtime::builtin::HAMT_BIT_WIDTH; + use fil_actors_runtime::fvm_ipld_hamt::BytesKey; + use fil_actors_runtime::make_empty_map; + use fvm_ipld_blockstore::MemoryBlockstore; + use fvm_shared::address::Address; + use fvm_shared::econ::TokenAmount; + use primitives::{TCid, THamt}; + use serde_tuple::{Deserialize_tuple, Serialize_tuple}; + + #[derive(PartialEq, Clone, Deserialize_tuple, Serialize_tuple, Debug)] + struct DummyVote { + key: UniqueBytesKey, + } + + impl UniqueVote for DummyVote { + fn unique_key(&self) -> anyhow::Result { + Ok(self.key.clone()) + } + } + + #[test] + fn test_serialization() { + #[derive(Deserialize_tuple, Serialize_tuple, PartialEq, Eq, Clone, Debug)] + struct DummySubmissions { + total_submission_weight: TokenAmount, + most_voted_key: Option, + submitters: TCid>, + submission_weights: TCid>, + submissions: TCid>, + } + + let dummy_submissions = DummySubmissions { + total_submission_weight: TokenAmount::from_atto(100), + most_voted_key: Some(vec![1, 2, 3]), + submitters: Default::default(), + submission_weights: Default::default(), + submissions: Default::default(), + }; + + let submissions = EpochVoteSubmissions:: { + total_submission_weight: TokenAmount::from_atto(100), + most_voted_key: Some(vec![1, 2, 3]), + submitters: Default::default(), + submission_weights: Default::default(), + submissions: Default::default(), + }; + + let json1 = serde_json::to_string(&dummy_submissions).unwrap(); + let json2 = serde_json::to_string(&submissions).unwrap(); + assert_eq!(json1, json2); + } + + #[test] + fn test_storage() { + let store = MemoryBlockstore::new(); + let mut hamt = make_empty_map(&store, HAMT_BIT_WIDTH); + + let submissions = EpochVoteSubmissions:: { + total_submission_weight: TokenAmount::from_atto(100), + most_voted_key: Some(vec![1, 2, 3, 4]), + submitters: Default::default(), + submission_weights: Default::default(), + submissions: Default::default(), + }; + + let key = BytesKey::from("1"); + hamt.set(key.clone(), submissions.clone()).unwrap(); + let fetched = hamt.get(&key).unwrap().unwrap(); + assert_eq!( + fetched.total_submission_weight, + submissions.total_submission_weight + ); + assert_eq!(fetched.most_voted_key, submissions.most_voted_key); + assert_eq!(fetched.submitters, submissions.submitters); + assert_eq!(fetched.submission_weights, submissions.submission_weights); + assert_eq!(fetched.submissions, submissions.submissions); + } + + #[test] + fn test_new_works() { + let store = MemoryBlockstore::new(); + let r = EpochVoteSubmissions::::new(&store); + assert!(r.is_ok()); + } + + #[test] + fn test_update_submitters() { + let store = MemoryBlockstore::new(); + let mut submission = EpochVoteSubmissions::::new(&store).unwrap(); + + let submitter = Address::new_id(0); + submission.update_submitters(&store, submitter).unwrap(); + assert!(submission.has_submitted(&store, &submitter).unwrap()); + + // now submit again, but should fail + assert!(submission.update_submitters(&store, submitter).is_err()); + } + + #[test] + fn test_insert_checkpoint() { + let store = MemoryBlockstore::new(); + let mut submission = EpochVoteSubmissions::::new(&store).unwrap(); + + let checkpoint = DummyVote { key: vec![0] }; + + let hash = checkpoint.unique_key().unwrap(); + + submission.insert_vote(&store, checkpoint.clone()).unwrap(); + assert!(submission.is_vote_inserted(&store, &hash).unwrap()); + + // insert again should not have caused any error + submission.insert_vote(&store, checkpoint.clone()).unwrap(); + + let inserted_checkpoint = submission.get_submission(&store, &hash).unwrap().unwrap(); + assert_eq!(inserted_checkpoint, checkpoint); + } + + #[test] + fn test_update_submission_count() { + let store = MemoryBlockstore::new(); + let mut submission = EpochVoteSubmissions::::new(&store).unwrap(); + + let hash1 = vec![1, 2, 1]; + let hash2 = vec![1, 2, 2]; + + // insert hash1, should have only one item + assert_eq!(submission.most_voted_key, None); + assert_eq!( + submission + .update_submission_weight(&store, hash1.clone(), TokenAmount::from_atto(1)) + .unwrap(), + TokenAmount::from_atto(1) + ); + assert_eq!( + submission + .get_submission_weight(&store, &hash1) + .unwrap() + .unwrap(), + TokenAmount::from_atto(1) + ); + assert_eq!(submission.most_voted_key, Some(hash1.clone())); + + // insert hash2, we should have two items, and there is a tie, hash1 still the most voted + assert_eq!( + submission + .update_submission_weight(&store, hash2.clone(), TokenAmount::from_atto(1)) + .unwrap(), + TokenAmount::from_atto(1) + ); + assert_eq!( + submission + .get_submission_weight(&store, &hash2) + .unwrap() + .unwrap(), + TokenAmount::from_atto(1) + ); + assert_eq!( + submission + .get_submission_weight(&store, &hash1) + .unwrap() + .unwrap(), + TokenAmount::from_atto(1) + ); + assert_eq!(submission.most_voted_key, Some(hash1.clone())); + + // insert hash2 again, we should have only 1 most submitted hash + assert_eq!( + submission + .update_submission_weight(&store, hash2.clone(), TokenAmount::from_atto(1)) + .unwrap(), + TokenAmount::from_atto(2) + ); + assert_eq!( + submission + .get_submission_weight(&store, &hash2) + .unwrap() + .unwrap(), + TokenAmount::from_atto(2) + ); + assert_eq!(submission.most_voted_key, Some(hash2.clone())); + + // insert hash2 again, we should have only 1 most submitted hash, but count incr by 1 + assert_eq!( + submission + .update_submission_weight(&store, hash2.clone(), TokenAmount::from_atto(1)) + .unwrap(), + TokenAmount::from_atto(3) + ); + assert_eq!( + submission + .get_submission_weight(&store, &hash2) + .unwrap() + .unwrap(), + TokenAmount::from_atto(3) + ); + assert_eq!(submission.most_voted_key, Some(hash2.clone())); + } + + #[test] + fn test_derive_execution_status() { + let store = MemoryBlockstore::new(); + let mut s = EpochVoteSubmissions::::new(&store).unwrap(); + + let total_validators = TokenAmount::from_atto(35); + let total_submissions = TokenAmount::from_atto(10); + let most_voted_count = TokenAmount::from_atto(5); + + s.total_submission_weight = total_submissions; + assert_eq!( + s.derive_execution_status(total_validators, most_voted_count, &(2, 3)), + VoteExecutionStatus::ThresholdNotReached, + ); + + // We could have 3 submissions: A, B, C + // Current submissions and their counts are: A - 2, B - 2. + // If the threshold is 1 / 2, we could have: + // If the last vote is C, then we should abort. + // If the last vote is any of A or B, we can execute. + // If the threshold is 2 / 3, we have to abort. + let total_validators = TokenAmount::from_atto(5); + let total_submissions = TokenAmount::from_atto(4); + let most_voted_count = TokenAmount::from_atto(2); + s.total_submission_weight = total_submissions.clone(); + assert_eq!( + s.derive_execution_status(total_submissions.clone(), most_voted_count, &(2, 3)), + VoteExecutionStatus::RoundAbort, + ); + + // We could have 1 submission: A + // Current submissions and their counts are: A - 4. + let total_submissions = TokenAmount::from_atto(4); + let most_voted_count = TokenAmount::from_atto(4); + s.total_submission_weight = total_submissions; + assert_eq!( + s.derive_execution_status(total_validators.clone(), most_voted_count, &(2, 3)), + VoteExecutionStatus::ConsensusReached, + ); + + // We could have 2 submission: A, B + // Current submissions and their counts are: A - 3, B - 1. + // Say the threshold is 2 / 3. If the last vote is B, we should abort, if the last vote is + // A, then we have reached consensus. The current votes are in conclusive. + let total_submissions = TokenAmount::from_atto(4); + let most_voted_count = TokenAmount::from_atto(3); + s.total_submission_weight = total_submissions; + assert_eq!( + s.derive_execution_status(total_validators, most_voted_count, &(2, 3)), + VoteExecutionStatus::ReachingConsensus, + ); + } +} diff --git a/common/src/vote/voting.rs b/common/src/vote/voting.rs new file mode 100644 index 0000000..c7cfd88 --- /dev/null +++ b/common/src/vote/voting.rs @@ -0,0 +1,466 @@ +use crate::vote::submission::{EpochVoteSubmissions, Ratio, VoteExecutionStatus}; +use crate::vote::UniqueVote; +use anyhow::anyhow; +use fvm_ipld_blockstore::Blockstore; +use fvm_shared::address::Address; +use fvm_shared::clock::ChainEpoch; +use fvm_shared::econ::TokenAmount; +use ipc_sdk::epoch_key; +use primitives::{TCid, THamt}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::collections::BTreeSet; // numerator and denominator + +const DEFAULT_THRESHOLD_RATIO: Ratio = (2, 3); + +/// Handle the epoch voting +#[derive(PartialEq, Eq, Clone, Debug)] +pub struct Voting { + /// The epoch that the voting started + pub genesis_epoch: ChainEpoch, + /// How often the voting should be submitted by validators + pub submission_period: ChainEpoch, + /// The last voting epoch that was executed + pub last_voting_executed_epoch: ChainEpoch, + /// Contains the executable epochs that are ready to be executed, but has yet to be executed. + /// This usually happens when previous submission epoch has not executed, but the next submission + /// epoch is ready to be executed. Most of the time this should be empty, we are wrapping with + /// Option instead of empty BTreeSet just to save some storage space. + pub executable_epoch_queue: Option>, + pub epoch_vote_submissions: TCid>>, + /// The voting execution threshold + pub threshold_ratio: Ratio, +} + +impl Default for Voting { + fn default() -> Self { + Voting { + genesis_epoch: 0, + submission_period: 0, + last_voting_executed_epoch: 0, + executable_epoch_queue: None, + epoch_vote_submissions: TCid::default(), + threshold_ratio: DEFAULT_THRESHOLD_RATIO, + } + } +} + +impl Voting { + pub fn new( + store: &BS, + genesis_epoch: ChainEpoch, + period: ChainEpoch, + ) -> anyhow::Result> { + Self::new_with_ratio( + store, + genesis_epoch, + period, + DEFAULT_THRESHOLD_RATIO.0, + DEFAULT_THRESHOLD_RATIO.1, + ) + } + + pub fn new_with_ratio( + store: &BS, + genesis_epoch: ChainEpoch, + period: ChainEpoch, + ratio_numerator: u64, + ratio_denominator: u64, + ) -> anyhow::Result> { + Ok(Self { + genesis_epoch, + submission_period: period, + last_voting_executed_epoch: genesis_epoch, + executable_epoch_queue: None, + epoch_vote_submissions: TCid::new_hamt(store)?, + threshold_ratio: (ratio_numerator, ratio_denominator), + }) + } + + /// Submit a vote at a specific epoch. If the validator threshold is reached, this method would + /// return the most voted vote, else it returns None. + /// + /// Note that this struct does not track the weight, it needs to be managed by external caller. + pub fn submit_vote( + &mut self, + store: &BS, + vote: T, + epoch: ChainEpoch, + submitter: Address, + submitter_weight: TokenAmount, + total_weight: TokenAmount, + ) -> anyhow::Result> { + // first we check the epoch is the correct one, we process only it's multiple + // of cron_period since genesis_epoch + if !self.epoch_can_vote(epoch) { + return Err(anyhow!("epoch not allowed")); + } + + if self.is_epoch_executed(epoch) { + return Err(anyhow!("epoch already executed")); + } + + // We are doing this manually because we have to modify `state` while processing the `hamt`. + // The current `self.epoch_vote_submissions.modify(...)` does not allow us to modify state in the + // function closure passed to modify. + let mut hamt = self.epoch_vote_submissions.load(store)?; + + let epoch_key = epoch_key(epoch); + let mut submission = match hamt.get(&epoch_key)? { + Some(s) => s.clone(), + None => EpochVoteSubmissions::::new(store)?, + }; + + let most_voted_weight = submission.submit(store, submitter, submitter_weight, vote)?; + let execution_status = submission.derive_execution_status( + total_weight, + most_voted_weight, + &self.threshold_ratio, + ); + + let messages = match execution_status { + VoteExecutionStatus::ThresholdNotReached | VoteExecutionStatus::ReachingConsensus => { + // threshold or consensus not reached, store submission and return + hamt.set(epoch_key, submission)?; + None + } + VoteExecutionStatus::RoundAbort => { + submission.abort(store)?; + hamt.set(epoch_key, submission)?; + None + } + VoteExecutionStatus::ConsensusReached => { + if self.last_voting_executed_epoch + self.submission_period != epoch { + // there are pending epochs to be executed, + // just store the submission and skip execution + hamt.set(epoch_key, submission)?; + self.insert_executable_epoch(epoch); + None + } else { + let msgs = submission.load_most_voted_submission(store)?.unwrap(); + Some(msgs) + } + } + }; + + // don't forget to flush + self.epoch_vote_submissions = TCid::from(hamt.flush()?); + + Ok(messages) + } + + /// Checks the `epoch` is the next executable epoch. + pub fn is_next_executable_epoch(&self, epoch: ChainEpoch) -> bool { + self.last_voting_executed_epoch + self.submission_period == epoch + } + + /// Abort a specific epoch. + pub fn abort_epoch( + &mut self, + store: &BS, + epoch: ChainEpoch, + ) -> anyhow::Result<()> { + self.remove_epoch_from_queue(epoch); + + let epoch_key = epoch_key(epoch); + self.epoch_vote_submissions.modify(store, |hamt| { + let mut submission = match hamt.get(&epoch_key)? { + Some(s) => s.clone(), + None => return Ok(()), + }; + + submission.abort(store)?; + hamt.set(epoch_key, submission)?; + + Ok(()) + }) + } + + /// Marks the epoch executed, removes the epoch from the `self.executable_epoch_queue` and clears all + /// the submissions in `self.epoch_vote_submissions`. + pub fn mark_epoch_executed( + &mut self, + store: &BS, + epoch: ChainEpoch, + ) -> anyhow::Result<()> { + if !self.is_next_executable_epoch(epoch) { + return Err(anyhow!("epoch not the next executable epoch")); + } + + if let Some(queue) = &self.executable_epoch_queue { + if queue.contains(&epoch) && queue.first() != Some(&epoch) { + return Err(anyhow!("epoch not the next executable epoch queue")); + } + } + + self.last_voting_executed_epoch = epoch; + self.remove_epoch_from_queue(epoch); + + let epoch_key = epoch_key(epoch); + self.epoch_vote_submissions.modify(store, |hamt| { + hamt.delete(&epoch_key)?; + Ok(()) + }) + } + + /// Load the next executable epoch and the content to be executed. + /// This ensures none of the epochs will be stuck. Consider the following example: + /// + /// Epoch 10 and 20 are two epochs to be executed. However, all the validators have submitted + /// epoch 20, and the status is to be executed. However, epoch 10 has yet to be executed. Now, + /// epoch 10 has reached consensus and executed, but epoch 20 cannot be executed because every + /// validator has already voted, no one can vote again to trigger the execution. Epoch 20 is stuck. + /// + /// This method lets one check if the next epoch can be executed, returns Some(T) if executable. + pub fn get_next_executable_vote( + &mut self, + store: &BS, + ) -> anyhow::Result> { + let epoch_queue = match self.executable_epoch_queue.as_mut() { + None => return Ok(None), + Some(queue) => queue, + }; + + let epoch = match epoch_queue.first() { + None => { + unreachable!("`epoch_queue` is not None, it should not be empty, report bug") + } + Some(epoch) => { + if *epoch > self.last_voting_executed_epoch + self.submission_period { + log::debug!("earliest executable epoch not the same cron period"); + return Ok(None); + } + *epoch + } + }; + + let hamt = self.epoch_vote_submissions.load(store)?; + + let epoch_key = epoch_key(epoch); + let submission = match hamt.get(&epoch_key)? { + Some(s) => s, + None => unreachable!("Submission in epoch not found, report bug"), + }; + + let vote = submission.load_most_voted_submission(store)?.unwrap(); + + Ok(Some(vote)) + } + + pub fn submission_period(&self) -> ChainEpoch { + self.submission_period + } + + pub fn epoch_vote_submissions(&self) -> TCid>> { + self.epoch_vote_submissions.clone() + } + + pub fn last_voting_executed_epoch(&self) -> ChainEpoch { + self.last_voting_executed_epoch + } + + pub fn executable_epoch_queue(&self) -> &Option> { + &self.executable_epoch_queue + } + + pub fn genesis_epoch(&self) -> ChainEpoch { + self.genesis_epoch + } + + /// Checks if the current epoch is votable + pub fn epoch_can_vote(&self, epoch: ChainEpoch) -> bool { + (epoch - self.genesis_epoch) % self.submission_period == 0 + } + + /// Checks if the epoch has already executed + pub fn is_epoch_executed(&self, epoch: ChainEpoch) -> bool { + self.last_voting_executed_epoch >= epoch + } + + /// Load the most voted submission at a specific epoch + pub fn load_most_voted_submission( + &self, + store: &impl Blockstore, + epoch: ChainEpoch, + ) -> anyhow::Result> { + let hamt = self.epoch_vote_submissions.load(store)?; + + let epoch_key = epoch_key(epoch); + + if let Some(submission) = hamt.get(&epoch_key)? { + submission.load_most_voted_submission(store) + } else { + Ok(None) + } + } + + /// Load the most voted weight at a specific epoch + pub fn load_most_voted_weight( + &self, + store: &impl Blockstore, + epoch: ChainEpoch, + ) -> anyhow::Result> { + let hamt = self.epoch_vote_submissions.load(store)?; + + let epoch_key = epoch_key(epoch); + + if let Some(submission) = hamt.get(&epoch_key)? { + submission.load_most_voted_weight(store) + } else { + Ok(None) + } + } + + fn remove_epoch_from_queue(&mut self, epoch: ChainEpoch) { + if let Some(queue) = self.executable_epoch_queue.as_mut() { + queue.remove(&epoch); + if queue.is_empty() { + self.executable_epoch_queue = None; + } + } + } + + fn insert_executable_epoch(&mut self, epoch: ChainEpoch) { + match self.executable_epoch_queue.as_mut() { + None => self.executable_epoch_queue = Some(BTreeSet::from([epoch])), + Some(queue) => { + queue.insert(epoch); + } + } + } +} + +impl Serialize for Voting { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let inner = ( + &self.genesis_epoch, + &self.submission_period, + &self.last_voting_executed_epoch, + &self.executable_epoch_queue, + &self.epoch_vote_submissions, + &self.threshold_ratio, + ); + inner.serialize(serde_tuple::Serializer(serializer)) + } +} + +impl<'de, T: DeserializeOwned> Deserialize<'de> for Voting { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + type Inner = ( + ChainEpoch, + ChainEpoch, + ChainEpoch, + Option>, + TCid>>, + Ratio, + ); + let inner = >::deserialize(serde_tuple::Deserializer(deserializer))?; + Ok(Voting { + genesis_epoch: inner.0, + submission_period: inner.1, + last_voting_executed_epoch: inner.2, + executable_epoch_queue: inner.3, + epoch_vote_submissions: inner.4, + threshold_ratio: inner.5, + }) + } +} + +#[cfg(test)] +mod tests { + use crate::vote::submission::Ratio; + use crate::vote::{EpochVoteSubmissions, UniqueBytesKey, UniqueVote, Voting}; + use fil_actors_runtime::builtin::HAMT_BIT_WIDTH; + use fil_actors_runtime::fvm_ipld_hamt::BytesKey; + use fil_actors_runtime::make_empty_map; + use fvm_ipld_blockstore::MemoryBlockstore; + use fvm_shared::clock::ChainEpoch; + use primitives::{TCid, THamt}; + use serde_tuple::{Deserialize_tuple, Serialize_tuple}; + use std::collections::BTreeSet; + + #[derive(PartialEq, Clone, Deserialize_tuple, Serialize_tuple, Debug)] + struct DummyVote { + key: UniqueBytesKey, + } + + impl UniqueVote for DummyVote { + fn unique_key(&self) -> anyhow::Result { + Ok(self.key.clone()) + } + } + + #[test] + fn test_serialization() { + #[derive(Deserialize_tuple, Serialize_tuple, PartialEq, Clone, Debug)] + struct DummyVoting { + genesis_epoch: ChainEpoch, + submission_period: ChainEpoch, + last_voting_executed_epoch: ChainEpoch, + executable_epoch_queue: Option>, + epoch_vote_submissions: TCid>>, + threshold_ratio: Ratio, + } + + let dummy_voting = DummyVoting { + genesis_epoch: 1, + submission_period: 2, + last_voting_executed_epoch: 3, + executable_epoch_queue: Some(BTreeSet::from([1])), + epoch_vote_submissions: Default::default(), + threshold_ratio: (2, 3), + }; + + let voting = Voting:: { + genesis_epoch: 1, + submission_period: 2, + last_voting_executed_epoch: 3, + executable_epoch_queue: Some(BTreeSet::from([1])), + epoch_vote_submissions: Default::default(), + threshold_ratio: (2, 3), + }; + + let json1 = serde_json::to_string(&dummy_voting).unwrap(); + let json2 = serde_json::to_string(&voting).unwrap(); + assert_eq!(json1, json2); + } + + #[test] + fn test_storage() { + let store = MemoryBlockstore::new(); + let mut hamt = make_empty_map(&store, HAMT_BIT_WIDTH); + + let voting = Voting:: { + genesis_epoch: 1, + submission_period: 2, + last_voting_executed_epoch: 3, + executable_epoch_queue: Some(BTreeSet::from([1])), + epoch_vote_submissions: Default::default(), + threshold_ratio: (2, 3), + }; + + let key = BytesKey::from("1"); + hamt.set(key.clone(), voting.clone()).unwrap(); + let fetched = hamt.get(&key).unwrap().unwrap(); + assert_eq!(fetched.genesis_epoch, voting.genesis_epoch); + assert_eq!(fetched.submission_period, voting.submission_period); + assert_eq!( + fetched.last_voting_executed_epoch, + voting.last_voting_executed_epoch + ); + assert_eq!( + fetched.executable_epoch_queue, + voting.executable_epoch_queue + ); + assert_eq!( + fetched.epoch_vote_submissions, + voting.epoch_vote_submissions + ); + } +} diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 46d312b..9487671 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -19,6 +19,7 @@ frc42_dispatch = "3.0.0" primitives = { git = "https://github.com/consensus-shipyard/fvm-utils" } fvm_shared = { version = "=3.0.0-alpha.17", default-features = false } ipc-sdk = { path = "../sdk" } +ipc-actor-common = { path = "../common" } fvm_ipld_hamt = "0.5.1" num-traits = "0.2.14" num-derive = "0.3.3" diff --git a/gateway/src/checkpoint.rs b/gateway/src/checkpoint.rs index dcd8f60..12ef05a 100644 --- a/gateway/src/checkpoint.rs +++ b/gateway/src/checkpoint.rs @@ -6,13 +6,14 @@ use fvm_ipld_encoding::DAG_CBOR; use fvm_ipld_encoding::{serde_bytes, to_vec}; use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; +use ipc_actor_common::vote::{UniqueBytesKey, UniqueVote}; use ipc_sdk::subnet_id::SubnetID; use lazy_static::lazy_static; use num_traits::Zero; use primitives::{TCid, TLink}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; -use crate::{ensure_message_sorted, CrossMsg, CrossMsgs}; +use crate::{ensure_message_sorted, CrossMsg}; lazy_static! { // Default CID used for the genesis checkpoint. Using @@ -29,6 +30,12 @@ pub struct Checkpoint { pub sig: Vec, } +impl UniqueVote for Checkpoint { + fn unique_key(&self) -> anyhow::Result { + Ok(self.cid().to_bytes()) + } +} + impl Checkpoint { pub fn new(id: SubnetID, epoch: ChainEpoch) -> Self { Self { @@ -172,33 +179,13 @@ impl CheckData { source: id, proof: Vec::new(), epoch, - prev_check: CHECKPOINT_GENESIS_CID.clone().into(), + prev_check: (*CHECKPOINT_GENESIS_CID).into(), children: Vec::new(), cross_msgs: BatchCrossMsgs::default(), } } } -// CrossMsgMeta sends an aggregate of all messages being propagated up in -// the checkpoint. -#[derive(PartialEq, Eq, Clone, Debug, Default, Serialize_tuple, Deserialize_tuple)] -pub struct CrossMsgMeta { - pub msgs_cid: TCid>, - pub nonce: u64, - pub value: TokenAmount, - pub fee: TokenAmount, -} - -impl CrossMsgMeta { - pub fn new() -> Self { - Self::default() - } - - pub fn set_nonce(&mut self, nonce: u64) { - self.nonce = nonce; - } -} - #[derive(PartialEq, Eq, Clone, Debug, Serialize_tuple, Deserialize_tuple)] pub struct ChildCheck { pub source: SubnetID, diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index d971482..eabeecc 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -1,26 +1,14 @@ use crate::{ensure_message_sorted, StorableMsg}; -use anyhow::anyhow; use cid::multihash::Code; use cid::multihash::MultihashDigest; -use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::to_vec; use fvm_ipld_encoding::tuple::{Deserialize_tuple, Serialize_tuple}; -use fvm_ipld_hamt::BytesKey; use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; +use ipc_actor_common::vote::{UniqueBytesKey, UniqueVote}; use ipc_sdk::ValidatorSet; -use lazy_static::lazy_static; use num_traits::Zero; -use primitives::{TCid, THamt}; -use std::ops::Mul; - -pub type HashOutput = Vec; - -lazy_static! { - pub static ref RATIO_NUMERATOR: u64 = 2; - pub static ref RATIO_DENOMINATOR: u64 = 3; -} /// Validators tracks all the validator in the subnet. It is useful in handling cron checkpoints. #[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] @@ -61,8 +49,8 @@ pub struct CronCheckpoint { pub top_down_msgs: Vec, } -impl CronCheckpoint { - /// Hash the checkpoint. +impl UniqueVote for CronCheckpoint { + /// Derive the unique key of the checkpoint using hash function. /// /// To compare the cron checkpoint and ensure they are the same, we need to make sure the /// top_down_msgs are the same. However, the top_down_msgs are vec, they may contain the same @@ -73,7 +61,7 @@ impl CronCheckpoint { /// - top down messages are sorted by `nonce` in descending order /// /// Actor will not perform sorting to save gas. Client should do it, actor just check. - pub fn hash(&self) -> anyhow::Result { + fn unique_key(&self) -> anyhow::Result { ensure_message_sorted(&self.top_down_msgs)?; let mh_code = Code::Blake2b256; @@ -83,469 +71,3 @@ impl CronCheckpoint { Ok(mh_code.digest(&to_vec(self).unwrap()).to_bytes()) } } - -/// Track all the cron checkpoint submissions of an epoch -#[derive(Serialize_tuple, Deserialize_tuple, PartialEq, Eq, Clone)] -pub struct CronSubmission { - /// The summation of the weights from all validator submissions - total_submission_weight: TokenAmount, - /// The most submitted hash. - most_voted_hash: Option, - /// The addresses of all the submitters - submitters: TCid>, - /// The map to track the submission weight of each hash - submission_weights: TCid>, - /// The different cron checkpoints, with cron checkpoint hash as key - submissions: TCid>, -} - -impl CronSubmission { - pub fn new(store: &BS) -> anyhow::Result { - Ok(CronSubmission { - total_submission_weight: TokenAmount::zero(), - submitters: TCid::new_hamt(store)?, - most_voted_hash: None, - submission_weights: TCid::new_hamt(store)?, - submissions: TCid::new_hamt(store)?, - }) - } - - /// Abort the current round and reset the submission data. - pub fn abort(&mut self, store: &BS) -> anyhow::Result<()> { - self.total_submission_weight = TokenAmount::zero(); - self.submitters = TCid::new_hamt(store)?; - self.most_voted_hash = None; - self.submission_weights = TCid::new_hamt(store)?; - - // no need reset `self.submissions`, we can still reuse the previous self.submissions - // new submissions will be inserted, old submission will not be inserted to save - // gas. - - Ok(()) - } - - /// Submit a cron checkpoint as the submitter. - pub fn submit( - &mut self, - store: &BS, - submitter: Address, - submitter_weight: TokenAmount, - checkpoint: CronCheckpoint, - ) -> anyhow::Result { - self.update_submitters(store, submitter)?; - self.total_submission_weight += &submitter_weight; - let checkpoint_hash = self.insert_checkpoint(store, checkpoint)?; - self.update_submission_weight(store, checkpoint_hash, submitter_weight) - } - - pub fn load_most_submitted_checkpoint( - &self, - store: &BS, - ) -> anyhow::Result> { - // we will only have one entry in the `most_submitted` set if more than 2/3 has reached - if let Some(hash) = &self.most_voted_hash { - self.get_submission(store, hash) - } else { - Ok(None) - } - } - - pub fn most_voted_weight(&self, store: &BS) -> anyhow::Result { - // we will only have one entry in the `most_submitted` set if more than 2/3 has reached - if let Some(hash) = &self.most_voted_hash { - Ok(self - .get_submission_weight(store, hash)? - .unwrap_or_else(TokenAmount::zero)) - } else { - Ok(TokenAmount::zero()) - } - } - - pub fn get_submission( - &self, - store: &BS, - hash: &HashOutput, - ) -> anyhow::Result> { - let hamt = self.submissions.load(store)?; - let key = BytesKey::from(hash.as_slice()); - Ok(hamt.get(&key)?.cloned()) - } - - pub fn derive_execution_status( - &self, - total_weight: TokenAmount, - most_voted_weight: TokenAmount, - ) -> VoteExecutionStatus { - let threshold = total_weight - .clone() - .mul(*RATIO_NUMERATOR) - .div_floor(*RATIO_DENOMINATOR); - - // note that we require THRESHOLD to be surpassed, equality is not enough! - if self.total_submission_weight <= threshold { - return VoteExecutionStatus::ThresholdNotReached; - } - - // now we have reached the threshold - - // consensus reached - if most_voted_weight > threshold { - return VoteExecutionStatus::ConsensusReached; - } - - // now the total submissions has reached the threshold, but the most submitted vote - // has yet to reach the threshold, that means consensus has not reached. - - // we do a early termination check, to see if consensus will ever be reached. - // - // consider an example that consensus will never be reached: - // - // -------- | -------------------------|--------------- | ------------- | - // MOST_VOTED THRESHOLD TOTAL_SUBMISSIONS TOTAL_WEIGHT - // - // we see MOST_VOTED is smaller than THRESHOLD, TOTAL_SUBMISSIONS and TOTAL_WEIGHT, if - // the potential extra votes any vote can obtain, i.e. TOTAL_WEIGHT - TOTAL_SUBMISSIONS, - // is smaller than or equal to the potential extra vote the most voted can obtain, i.e. - // THRESHOLD - MOST_VOTED, then consensus will never be reached, no point voting, just abort. - if threshold - most_voted_weight >= total_weight - &self.total_submission_weight { - VoteExecutionStatus::RoundAbort - } else { - VoteExecutionStatus::ReachingConsensus - } - } - - /// Checks if the submitter has already submitted the checkpoint. - pub fn has_submitted( - &self, - store: &BS, - submitter: &Address, - ) -> anyhow::Result { - let addr_byte_key = BytesKey::from(submitter.to_bytes()); - let hamt = self.submitters.load(store)?; - Ok(hamt.contains_key(&addr_byte_key)?) - } -} - -/// The status indicating if the voting should be executed -#[derive(Eq, PartialEq, Debug)] -pub enum VoteExecutionStatus { - /// The execution threshold has yet to be reached - ThresholdNotReached, - /// The voting threshold has reached, but consensus has yet to be reached, needs more - /// voting to reach consensus - ReachingConsensus, - /// Consensus cannot be reached in this round - RoundAbort, - /// Execution threshold reached - ConsensusReached, -} - -impl CronSubmission { - /// Update the total submitters, returns the latest total number of submitters - fn update_submitters( - &mut self, - store: &BS, - submitter: Address, - ) -> anyhow::Result<()> { - let addr_byte_key = BytesKey::from(submitter.to_bytes()); - self.submitters.modify(store, |hamt| { - // check the submitter has not submitted before - if hamt.contains_key(&addr_byte_key)? { - return Err(anyhow!("already submitted")); - } - - // now the submitter has not submitted before, mark as submitted - hamt.set(addr_byte_key, ())?; - - Ok(()) - }) - } - - /// Insert the checkpoint to store if it has not been submitted before. Returns the hash of the checkpoint. - fn insert_checkpoint( - &mut self, - store: &BS, - checkpoint: CronCheckpoint, - ) -> anyhow::Result { - let hash = checkpoint.hash()?; - let hash_key = BytesKey::from(hash.as_slice()); - - let hamt = self.submissions.load(store)?; - if hamt.contains_key(&hash_key)? { - return Ok(hash); - } - - // checkpoint has not submitted before - self.submissions.modify(store, |hamt| { - hamt.set(hash_key, checkpoint)?; - Ok(()) - })?; - - Ok(hash) - } - - /// Update submission weight of the hash. Returns the currently most submitted submission count. - fn update_submission_weight( - &mut self, - store: &BS, - hash: HashOutput, - weight: TokenAmount, - ) -> anyhow::Result { - let hash_byte_key = BytesKey::from(hash.as_slice()); - - self.submission_weights.modify(store, |hamt| { - let new_weight = hamt - .get(&hash_byte_key)? - .cloned() - .unwrap_or_else(TokenAmount::zero) - + weight; - - // update the new count - hamt.set(hash_byte_key, new_weight.clone())?; - - // now we compare with the most submitted hash or cron checkpoint - if self.most_voted_hash.is_none() { - // no most submitted hash set yet, set to current - self.most_voted_hash = Some(hash); - return Ok(new_weight); - } - - let most_submitted_hash = self.most_voted_hash.as_mut().unwrap(); - - // the current submission is already one of the most submitted entries - if most_submitted_hash == &hash { - // the current submission is already the only one submission, no need update - - // return the current checkpoint's count as the current most submitted checkpoint - return Ok(new_weight); - } - - // the current submission is not part of the most submitted entries, need to check - // the most submitted entry to compare if the current submission is exceeding - - let most_submitted_key = BytesKey::from(most_submitted_hash.as_slice()); - - // safe to unwrap as the hamt must contain the key - let most_submitted_count = hamt.get(&most_submitted_key)?.unwrap(); - - // current submission is not the most voted checkpoints - // if new_count < *most_submitted_count, we do nothing as the new count is not close to the most submitted - if new_weight > *most_submitted_count { - *most_submitted_hash = hash; - Ok(new_weight) - } else { - Ok(most_submitted_count.clone()) - } - }) - } - - /// Checks if the checkpoint hash has already inserted in the store - fn get_submission_weight( - &self, - store: &BS, - hash: &HashOutput, - ) -> anyhow::Result> { - let hamt = self.submission_weights.load(store)?; - let r = hamt.get(&BytesKey::from(hash.as_slice()))?; - Ok(r.cloned()) - } - - /// Checks if the checkpoint hash has already inserted in the store - #[cfg(test)] - fn has_checkpoint_inserted( - &self, - store: &BS, - hash: &HashOutput, - ) -> anyhow::Result { - let hamt = self.submissions.load(store)?; - Ok(hamt.contains_key(&BytesKey::from(hash.as_slice()))?) - } -} - -#[cfg(test)] -mod tests { - use crate::{CronCheckpoint, CronSubmission, VoteExecutionStatus}; - use fvm_ipld_blockstore::MemoryBlockstore; - use fvm_shared::address::Address; - use fvm_shared::econ::TokenAmount; - - #[test] - fn test_new_works() { - let store = MemoryBlockstore::new(); - let r = CronSubmission::new(&store); - assert!(r.is_ok()); - } - - #[test] - fn test_update_submitters() { - let store = MemoryBlockstore::new(); - let mut submission = CronSubmission::new(&store).unwrap(); - - let submitter = Address::new_id(0); - submission.update_submitters(&store, submitter).unwrap(); - assert!(submission.has_submitted(&store, &submitter).unwrap()); - - // now submit again, but should fail - assert!(submission.update_submitters(&store, submitter).is_err()); - } - - #[test] - fn test_insert_checkpoint() { - let store = MemoryBlockstore::new(); - let mut submission = CronSubmission::new(&store).unwrap(); - - let checkpoint = CronCheckpoint { - epoch: 100, - top_down_msgs: vec![], - }; - - let hash = checkpoint.hash().unwrap(); - - submission - .insert_checkpoint(&store, checkpoint.clone()) - .unwrap(); - assert!(submission.has_checkpoint_inserted(&store, &hash).unwrap()); - - // insert again should not have caused any error - submission - .insert_checkpoint(&store, checkpoint.clone()) - .unwrap(); - - let inserted_checkpoint = submission.get_submission(&store, &hash).unwrap().unwrap(); - assert_eq!(inserted_checkpoint, checkpoint); - } - - #[test] - fn test_update_submission_count() { - let store = MemoryBlockstore::new(); - let mut submission = CronSubmission::new(&store).unwrap(); - - let hash1 = vec![1, 2, 1]; - let hash2 = vec![1, 2, 2]; - - // insert hash1, should have only one item - assert_eq!(submission.most_voted_hash, None); - assert_eq!( - submission - .update_submission_weight(&store, hash1.clone(), TokenAmount::from_atto(1)) - .unwrap(), - TokenAmount::from_atto(1) - ); - assert_eq!( - submission - .get_submission_weight(&store, &hash1) - .unwrap() - .unwrap(), - TokenAmount::from_atto(1) - ); - assert_eq!(submission.most_voted_hash, Some(hash1.clone())); - - // insert hash2, we should have two items, and there is a tie, hash1 still the most voted - assert_eq!( - submission - .update_submission_weight(&store, hash2.clone(), TokenAmount::from_atto(1)) - .unwrap(), - TokenAmount::from_atto(1) - ); - assert_eq!( - submission - .get_submission_weight(&store, &hash2) - .unwrap() - .unwrap(), - TokenAmount::from_atto(1) - ); - assert_eq!( - submission - .get_submission_weight(&store, &hash1) - .unwrap() - .unwrap(), - TokenAmount::from_atto(1) - ); - assert_eq!(submission.most_voted_hash, Some(hash1.clone())); - - // insert hash2 again, we should have only 1 most submitted hash - assert_eq!( - submission - .update_submission_weight(&store, hash2.clone(), TokenAmount::from_atto(1)) - .unwrap(), - TokenAmount::from_atto(2) - ); - assert_eq!( - submission - .get_submission_weight(&store, &hash2) - .unwrap() - .unwrap(), - TokenAmount::from_atto(2) - ); - assert_eq!(submission.most_voted_hash, Some(hash2.clone())); - - // insert hash2 again, we should have only 1 most submitted hash, but count incr by 1 - assert_eq!( - submission - .update_submission_weight(&store, hash2.clone(), TokenAmount::from_atto(1)) - .unwrap(), - TokenAmount::from_atto(3) - ); - assert_eq!( - submission - .get_submission_weight(&store, &hash2) - .unwrap() - .unwrap(), - TokenAmount::from_atto(3) - ); - assert_eq!(submission.most_voted_hash, Some(hash2.clone())); - } - - #[test] - fn test_derive_execution_status() { - let store = MemoryBlockstore::new(); - let mut s = CronSubmission::new(&store).unwrap(); - - let total_validators = TokenAmount::from_atto(35); - let total_submissions = TokenAmount::from_atto(10); - let most_voted_count = TokenAmount::from_atto(5); - - s.total_submission_weight = total_submissions; - assert_eq!( - s.derive_execution_status(total_validators, most_voted_count), - VoteExecutionStatus::ThresholdNotReached, - ); - - // We could have 3 submissions: A, B, C - // Current submissions and their counts are: A - 2, B - 2. - // If the threshold is 1 / 2, we could have: - // If the last vote is C, then we should abort. - // If the last vote is any of A or B, we can execute. - // If the threshold is 2 / 3, we have to abort. - let total_validators = TokenAmount::from_atto(5); - let total_submissions = TokenAmount::from_atto(4); - let most_voted_count = TokenAmount::from_atto(2); - s.total_submission_weight = total_submissions.clone(); - assert_eq!( - s.derive_execution_status(total_submissions.clone(), most_voted_count), - VoteExecutionStatus::RoundAbort, - ); - - // We could have 1 submission: A - // Current submissions and their counts are: A - 4. - let total_submissions = TokenAmount::from_atto(4); - let most_voted_count = TokenAmount::from_atto(4); - s.total_submission_weight = total_submissions; - assert_eq!( - s.derive_execution_status(total_validators.clone(), most_voted_count), - VoteExecutionStatus::ConsensusReached, - ); - - // We could have 2 submission: A, B - // Current submissions and their counts are: A - 3, B - 1. - // Say the threshold is 2 / 3. If the last vote is B, we should abort, if the last vote is - // A, then we have reached consensus. The current votes are in conclusive. - let total_submissions = TokenAmount::from_atto(4); - let most_voted_count = TokenAmount::from_atto(3); - s.total_submission_weight = total_submissions; - assert_eq!( - s.derive_execution_status(total_validators, most_voted_count), - VoteExecutionStatus::ReachingConsensus, - ); - } -} diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 5b98727..9c1856f 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -2,12 +2,11 @@ extern crate core; -pub use self::checkpoint::{Checkpoint, CrossMsgMeta, CHECKPOINT_GENESIS_CID}; +pub use self::checkpoint::{Checkpoint, CHECKPOINT_GENESIS_CID}; pub use self::cross::{is_bottomup, CrossMsg, CrossMsgs, IPCMsgType, StorableMsg}; pub use self::state::*; pub use self::subnet::*; pub use self::types::*; -pub use crate::cron::{CronSubmission, VoteExecutionStatus}; pub use cron::CronCheckpoint; use cross::{burn_bu_funds, cross_msg_side_effects, distribute_crossmsg_fee}; use fil_actors_runtime::runtime::fvm::resolve_secp_bls; @@ -16,12 +15,9 @@ use fil_actors_runtime::{ actor_dispatch, actor_error, restrict_internal_api, ActorDowncast, ActorError, CALLER_TYPES_SIGNABLE, INIT_ACTOR_ADDR, SYSTEM_ACTOR_ADDR, }; -use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::RawBytes; -use fvm_ipld_hamt::BytesKey; use fvm_shared::address::Address; use fvm_shared::bigint::Zero; -use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use fvm_shared::error::ExitCode; use fvm_shared::METHOD_SEND; @@ -32,7 +28,6 @@ use ipc_sdk::ValidatorSet; use lazy_static::lazy_static; use num_derive::FromPrimitive; use num_traits::FromPrimitive; -use primitives::TCid; #[cfg(feature = "fil-gateway-actor")] fil_actors_runtime::wasm_trampoline!(Actor); @@ -697,28 +692,49 @@ impl Actor { // submit cron can only be performed by signable addresses rt.validate_immediate_caller_type(CALLER_TYPES_SIGNABLE.iter())?; - let msgs = rt.transaction(|st: &mut State, rt| { + let to_execute = rt.transaction(|st: &mut State, rt| { let submitter = rt.message().caller(); - let submitter_weight = Self::validate_submitter(st, checkpoint.epoch, &submitter)?; + let submitter_weight = Self::validate_submitter(st, &submitter)?; let store = rt.store(); - Self::handle_cron_submission(store, st, checkpoint, submitter, submitter_weight) + let epoch = checkpoint.epoch; + let total_weight = st.validators.total_weight.clone(); + let ch = st + .cron_checkpoint_voting + .submit_vote( + store, + checkpoint, + epoch, + submitter, + submitter_weight, + total_weight, + ) .map_err(|e| { log::error!( "encountered error processing submit cron checkpoint: {:?}", e ); actor_error!(unhandled_message, e.to_string()) - }) + })?; + if ch.is_some() { + st.cron_checkpoint_voting + .mark_epoch_executed(store, epoch) + .map_err(|e| { + log::error!("encountered error marking epoch executed: {:?}", e); + actor_error!(unhandled_message, e.to_string()) + })?; + } + + Ok(ch) })?; // we only `execute_next_cron_epoch(rt)` if there is no execution for the current submission // so that we don't blow up the gas. - if let Some(msgs) = msgs { - if msgs.is_empty() { + if let Some(checkpoint) = to_execute { + if checkpoint.top_down_msgs.is_empty() { Self::execute_next_cron_epoch(rt)?; } - for m in msgs { + for m in checkpoint.top_down_msgs { Self::apply_msg_inner( rt, CrossMsg { @@ -814,21 +830,7 @@ impl Actor { /// All the validator code for the actor calls impl Actor { /// Validate the submitter's submission against the state, also returns the weight of the validator - fn validate_submitter( - st: &State, - epoch: ChainEpoch, - submitter: &Address, - ) -> Result { - // first we check the epoch is the correct one, we process only it's multiple - // of cron_period since genesis_epoch - if (epoch - st.genesis_epoch) % st.cron_period != 0 { - return Err(actor_error!(illegal_argument, "epoch not allowed")); - } - - if st.last_cron_executed_epoch >= epoch { - return Err(actor_error!(illegal_argument, "epoch already executed")); - } - + fn validate_submitter(st: &State, submitter: &Address) -> Result { st.validators .get_validator_weight(submitter) .ok_or_else(|| actor_error!(illegal_argument, "caller not validator")) @@ -943,70 +945,6 @@ impl Actor { Ok(RawBytes::new(cid.to_bytes())) } - fn handle_cron_submission( - store: &BS, - st: &mut State, - checkpoint: CronCheckpoint, - submitter: Address, - submitter_weight: TokenAmount, - ) -> anyhow::Result>> { - let total_weight = st.validators.total_weight.clone(); - let params_epoch = checkpoint.epoch; - - // We are doing this manually because we have to modify `state` while processing the `hamt`. - // The current `st.cron_submissions.modify(...)` does not allow us to modify state in the - // function closure passed to modify. - let mut hamt = st.cron_submissions.load(store)?; - - let epoch_key = BytesKey::from(params_epoch.to_be_bytes().as_slice()); - let mut submission = match hamt.get(&epoch_key)? { - Some(s) => s.clone(), - None => CronSubmission::new(store)?, - }; - - let most_voted_weight = - submission.submit(store, submitter, submitter_weight, checkpoint)?; - let execution_status = submission.derive_execution_status(total_weight, most_voted_weight); - - let messages = match execution_status { - VoteExecutionStatus::ThresholdNotReached | VoteExecutionStatus::ReachingConsensus => { - // threshold or consensus not reached, store submission and return - hamt.set(epoch_key, submission)?; - None - } - VoteExecutionStatus::RoundAbort => { - submission.abort(store)?; - hamt.set(epoch_key, submission)?; - None - } - VoteExecutionStatus::ConsensusReached => { - if st.last_cron_executed_epoch + st.cron_period != params_epoch { - // there are pending epochs to be executed, - // just store the submission and skip execution - hamt.set(epoch_key, submission)?; - st.insert_executable_epoch(params_epoch); - return Ok(None); - } - - // we reach consensus in the checkpoints submission - st.last_cron_executed_epoch = params_epoch; - - let msgs = submission - .load_most_submitted_checkpoint(store)? - .unwrap() - .top_down_msgs; - hamt.delete(&epoch_key)?; - - Some(msgs) - } - }; - - // don't forget to flush - st.cron_submissions = TCid::from(hamt.flush()?); - - Ok(messages) - } - /// Execute the next approved cron checkpoint. /// This is an edge case to ensure none of the epoches will be stuck. Consider the following example: /// @@ -1015,60 +953,31 @@ impl Actor { /// epoch 10 has reached consensus and executed, but epoch 20 cannot be executed because every /// validator has already voted, no one can vote again to trigger the execution. Epoch 20 is stuck. fn execute_next_cron_epoch(rt: &mut impl Runtime) -> Result<(), ActorError> { - let msgs = rt.transaction(|st: &mut State, rt| { - let epoch_queue = match st.executable_epoch_queue.as_mut() { - None => return Ok(None), - Some(queue) => queue, - }; - - match epoch_queue.first() { - None => { - unreachable!("`epoch_queue` is not None, it should not be empty, report bug") - } - Some(epoch) => { - if *epoch > st.last_cron_executed_epoch + st.cron_period { - log::debug!("earliest executable epoch not the same cron period"); - return Ok(None); - } - } - } - - let store = rt.store(); - let epoch = epoch_queue.pop_first().unwrap(); - - if epoch_queue.is_empty() { - st.executable_epoch_queue = None; - } - - st.cron_submissions - .modify(store, |hamt| { - let epoch_key = BytesKey::from(epoch.to_be_bytes().as_slice()); - let submission = match hamt.get(&epoch_key)? { - Some(s) => s, - None => unreachable!("Submission in epoch not found, report bug"), - }; - - st.last_cron_executed_epoch = epoch; - - let msgs = submission - .load_most_submitted_checkpoint(store)? - .unwrap() - .top_down_msgs; - hamt.delete(&epoch_key)?; - - Ok(Some(msgs)) - }) + let checkpoint = rt.transaction(|st: &mut State, rt| { + let cp = st + .cron_checkpoint_voting + .get_next_executable_vote(rt.store()) .map_err(|e| { log::error!( "encountered error processing submit cron checkpoint: {:?}", e ); actor_error!(unhandled_message, e.to_string()) - }) + })?; + if let Some(cp) = &cp { + st.cron_checkpoint_voting + .mark_epoch_executed(rt.store(), cp.epoch) + .map_err(|e| { + log::error!("encountered error marking epoch executed: {:?}", e); + actor_error!(unhandled_message, e.to_string()) + })?; + } + + Ok(cp) })?; - if let Some(msgs) = msgs { - for m in msgs { + if let Some(checkpoint) = checkpoint { + for m in checkpoint.top_down_msgs { Self::apply_msg_inner( rt, CrossMsg { diff --git a/gateway/src/state.rs b/gateway/src/state.rs index 7d34598..9536c71 100644 --- a/gateway/src/state.rs +++ b/gateway/src/state.rs @@ -15,10 +15,11 @@ use lazy_static::lazy_static; use num_traits::Zero; use primitives::{TCid, THamt}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; -use std::collections::BTreeSet; use std::str::FromStr; -use crate::cron::{CronSubmission, Validators}; +use crate::cron::Validators; +use crate::CronCheckpoint; +use ipc_actor_common::vote::Voting; use ipc_sdk::subnet_id::SubnetID; use ipc_sdk::ValidatorSet; @@ -49,18 +50,7 @@ pub struct State { pub bottomup_nonce: u64, pub applied_bottomup_nonce: u64, pub applied_topdown_nonce: u64, - /// The epoch that the subnet actor is deployed - pub genesis_epoch: ChainEpoch, - /// How often cron checkpoints will be submitted by validator in the child subnet - pub cron_period: ChainEpoch, - /// The last submit cron epoch that was executed - pub last_cron_executed_epoch: ChainEpoch, - /// Contains the executable epochs that are ready to be executed, but has yet to be executed. - /// This usually happens when previous submission epoch has not executed, but the next submission - /// epoch is ready to be executed. Most of the time this should be empty, we are wrapping with - /// Option instead of empty VecDeque just to save some storage space. - pub executable_epoch_queue: Option>, - pub cron_submissions: TCid>, + pub cron_checkpoint_voting: Voting, pub validators: Validators, } @@ -86,11 +76,11 @@ impl State { // We first increase to the subsequent and then execute for bottom-up messages applied_bottomup_nonce: Default::default(), applied_topdown_nonce: Default::default(), - genesis_epoch: params.genesis_epoch, - cron_period: params.cron_period, - last_cron_executed_epoch: params.genesis_epoch, - executable_epoch_queue: None, - cron_submissions: TCid::new_hamt(store)?, + cron_checkpoint_voting: Voting::::new( + store, + params.genesis_epoch, + params.cron_period, + )?, validators: Validators::new(ValidatorSet::default()), }) } @@ -383,15 +373,6 @@ impl State { pub fn set_membership(&mut self, validator_set: ValidatorSet) { self.validators = Validators::new(validator_set); } - - pub fn insert_executable_epoch(&mut self, epoch: ChainEpoch) { - match self.executable_epoch_queue.as_mut() { - None => self.executable_epoch_queue = Some(BTreeSet::from([epoch])), - Some(queue) => { - queue.insert(epoch); - } - } - } } pub fn set_subnet( @@ -434,15 +415,6 @@ pub fn get_checkpoint<'m, BS: Blockstore>( .map_err(|e| e.downcast_wrap(format!("failed to get checkpoint for id {}", epoch))) } -pub fn get_bottomup_msg<'m, BS: Blockstore>( - crossmsgs: &'m CrossMsgMetaArray, - nonce: u64, -) -> anyhow::Result> { - crossmsgs - .get(nonce) - .map_err(|e| anyhow!("failed to get crossmsg meta by nonce: {:?}", e)) -} - pub fn get_topdown_msg<'m, BS: Blockstore>( crossmsgs: &'m CrossMsgArray, nonce: u64, diff --git a/gateway/src/subnet.rs b/gateway/src/subnet.rs index bef22b1..edfcc74 100644 --- a/gateway/src/subnet.rs +++ b/gateway/src/subnet.rs @@ -6,12 +6,11 @@ use fvm_shared::econ::TokenAmount; use primitives::{TAmt, TCid}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; -use crate::CROSSMSG_AMT_BITWIDTH; +use crate::{State, CROSSMSG_AMT_BITWIDTH}; use ipc_sdk::subnet_id::SubnetID; use super::checkpoint::*; use super::cross::CrossMsg; -use super::state::State; #[derive(PartialEq, Eq, Clone, Copy, Debug, Deserialize_repr, Serialize_repr)] #[repr(i32)] diff --git a/gateway/src/types.rs b/gateway/src/types.rs index aab7fe2..03cbe85 100644 --- a/gateway/src/types.rs +++ b/gateway/src/types.rs @@ -12,7 +12,7 @@ use multihash::MultihashDigest; use primitives::CodeType; use std::cmp::Ordering; -use crate::checkpoint::{Checkpoint, CrossMsgMeta}; +use crate::checkpoint::Checkpoint; use crate::cross::CrossMsg; /// ID used in the builtin-actors bundle manifest @@ -24,7 +24,6 @@ pub const MIN_COLLATERAL_AMOUNT: u64 = 10_u64.pow(18); pub const SUBNET_ACTOR_REWARD_METHOD: u64 = frc42_dispatch::method_hash!("Reward"); -pub type CrossMsgMetaArray<'bs, BS> = Array<'bs, CrossMsgMeta, BS>; pub type CrossMsgArray<'bs, BS> = Array<'bs, CrossMsg, BS>; /// The executable message trait diff --git a/gateway/tests/gateway_test.rs b/gateway/tests/gateway_test.rs index e7112c7..a3666e4 100644 --- a/gateway/tests/gateway_test.rs +++ b/gateway/tests/gateway_test.rs @@ -3,21 +3,21 @@ use fil_actors_runtime::runtime::Runtime; use fil_actors_runtime::test_utils::MockRuntime; use fil_actors_runtime::BURNT_FUNDS_ACTOR_ADDR; use fvm_ipld_encoding::RawBytes; -use fvm_ipld_hamt::BytesKey; use fvm_shared::address::Address; use fvm_shared::bigint::Zero; use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use fvm_shared::error::ExitCode; use fvm_shared::METHOD_SEND; +use ipc_actor_common::vote::{EpochVoteSubmissions, UniqueVote}; use ipc_gateway::checkpoint::BatchCrossMsgs; use ipc_gateway::Status::{Active, Inactive}; use ipc_gateway::{ - get_topdown_msg, Checkpoint, CronCheckpoint, CronSubmission, CrossMsg, IPCAddress, PostBoxItem, - State, StorableMsg, CROSS_MSG_FEE, DEFAULT_CHECKPOINT_PERIOD, SUBNET_ACTOR_REWARD_METHOD, + get_topdown_msg, Checkpoint, CronCheckpoint, CrossMsg, IPCAddress, PostBoxItem, State, + StorableMsg, CROSS_MSG_FEE, DEFAULT_CHECKPOINT_PERIOD, SUBNET_ACTOR_REWARD_METHOD, }; use ipc_sdk::subnet_id::SubnetID; -use ipc_sdk::{Validator, ValidatorSet}; +use ipc_sdk::{epoch_key, Validator, ValidatorSet}; use primitives::TCid; use std::collections::BTreeSet; use std::ops::Mul; @@ -1305,35 +1305,44 @@ fn test_submit_cron_checking_errors() { setup_membership(&h, &mut rt); let submitter = Address::new_id(10000); + let checkpoint = CronCheckpoint { - epoch: *DEFAULT_GENESIS_EPOCH + 1, + epoch: *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD, top_down_msgs: vec![], }; let r = h.submit_cron(&mut rt, submitter, checkpoint); assert!(r.is_err()); - assert_eq!(r.unwrap_err().msg(), "epoch not allowed"); + assert_eq!(r.unwrap_err().msg(), "caller not validator"); + let submitter = Address::new_id(0); let checkpoint = CronCheckpoint { - epoch: *DEFAULT_GENESIS_EPOCH, + epoch: *DEFAULT_GENESIS_EPOCH + 1, top_down_msgs: vec![], }; let r = h.submit_cron(&mut rt, submitter, checkpoint); assert!(r.is_err()); - assert_eq!(r.unwrap_err().msg(), "epoch already executed"); + assert_eq!(r.unwrap_err().msg(), "epoch not allowed"); let checkpoint = CronCheckpoint { - epoch: *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD, + epoch: *DEFAULT_GENESIS_EPOCH, top_down_msgs: vec![], }; let r = h.submit_cron(&mut rt, submitter, checkpoint); assert!(r.is_err()); - assert_eq!(r.unwrap_err().msg(), "caller not validator"); + assert_eq!(r.unwrap_err().msg(), "epoch already executed"); } -fn get_epoch_submissions(rt: &mut MockRuntime, epoch: ChainEpoch) -> Option { +fn get_epoch_submissions( + rt: &mut MockRuntime, + epoch: ChainEpoch, +) -> Option> { let st: State = rt.get_state(); - let hamt = st.cron_submissions.load(rt.store()).unwrap(); - let bytes_key = BytesKey::from(epoch.to_be_bytes().as_slice()); + let hamt = st + .cron_checkpoint_voting + .epoch_vote_submissions() + .load(rt.store()) + .unwrap(); + let bytes_key = epoch_key(epoch); hamt.get(&bytes_key).unwrap().cloned() } @@ -1357,13 +1366,16 @@ fn test_submit_cron_works_with_execution() { let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); assert_eq!( submission - .get_submission(rt.store(), &checkpoint.hash().unwrap()) + .get_submission(rt.store(), &checkpoint.unique_key().unwrap()) .unwrap() .unwrap(), checkpoint ); let st: State = rt.get_state(); - assert_eq!(st.last_cron_executed_epoch, *DEFAULT_GENESIS_EPOCH); // not executed yet + assert_eq!( + st.cron_checkpoint_voting.last_voting_executed_epoch(), + *DEFAULT_GENESIS_EPOCH + ); // not executed yet // already submitted let submitter = Address::new_id(0); @@ -1378,13 +1390,16 @@ fn test_submit_cron_works_with_execution() { let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); assert_eq!( submission - .get_submission(rt.store(), &checkpoint.hash().unwrap()) + .get_submission(rt.store(), &checkpoint.unique_key().unwrap()) .unwrap() .unwrap(), checkpoint ); let st: State = rt.get_state(); - assert_eq!(st.last_cron_executed_epoch, *DEFAULT_GENESIS_EPOCH); // not executed yet + assert_eq!( + st.cron_checkpoint_voting.last_voting_executed_epoch(), + *DEFAULT_GENESIS_EPOCH + ); // not executed yet // third submission let submitter = Address::new_id(2); @@ -1393,13 +1408,16 @@ fn test_submit_cron_works_with_execution() { let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); assert_eq!( submission - .get_submission(rt.store(), &checkpoint.hash().unwrap()) + .get_submission(rt.store(), &checkpoint.unique_key().unwrap()) .unwrap() .unwrap(), checkpoint ); let st: State = rt.get_state(); - assert_eq!(st.last_cron_executed_epoch, *DEFAULT_GENESIS_EPOCH); // not executed yet + assert_eq!( + st.cron_checkpoint_voting.last_voting_executed_epoch(), + *DEFAULT_GENESIS_EPOCH + ); // not executed yet // fourth submission, executed let submitter = Address::new_id(3); @@ -1416,7 +1434,10 @@ fn test_submit_cron_works_with_execution() { let submission = get_epoch_submissions(&mut rt, epoch); assert!(submission.is_none()); let st: State = rt.get_state(); - assert_eq!(st.last_cron_executed_epoch, epoch); + assert_eq!( + st.cron_checkpoint_voting.last_voting_executed_epoch(), + epoch + ); } fn storable_msg(nonce: u64) -> StorableMsg { @@ -1476,7 +1497,10 @@ fn test_submit_cron_abort() { // check aborted let st: State = rt.get_state(); - assert_eq!(st.last_cron_executed_epoch, *DEFAULT_GENESIS_EPOCH); // not executed yet + assert_eq!( + st.cron_checkpoint_voting.last_voting_executed_epoch(), + *DEFAULT_GENESIS_EPOCH + ); // not executed yet let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); for i in 0..4 { assert_eq!( @@ -1520,9 +1544,12 @@ fn test_submit_cron_sequential_execution() { h.submit_cron(&mut rt, submitter, checkpoint.clone()) .unwrap(); let st: State = rt.get_state(); - assert_eq!(st.last_cron_executed_epoch, *DEFAULT_GENESIS_EPOCH); // not executed yet assert_eq!( - st.executable_epoch_queue, + st.cron_checkpoint_voting.last_voting_executed_epoch(), + *DEFAULT_GENESIS_EPOCH + ); // not executed yet + assert_eq!( + *st.cron_checkpoint_voting.executable_epoch_queue(), Some(BTreeSet::from([pending_epoch])) ); // not executed yet @@ -1562,7 +1589,10 @@ fn test_submit_cron_sequential_execution() { let submission = get_epoch_submissions(&mut rt, epoch); assert!(submission.is_none()); let st: State = rt.get_state(); - assert_eq!(st.last_cron_executed_epoch, epoch); + assert_eq!( + st.cron_checkpoint_voting.last_voting_executed_epoch(), + epoch + ); // now we submit to the next epoch let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD * 3; @@ -1573,6 +1603,9 @@ fn test_submit_cron_sequential_execution() { h.submit_cron(&mut rt, submitter, checkpoint.clone()) .unwrap(); let st: State = rt.get_state(); - assert_eq!(st.last_cron_executed_epoch, pending_epoch); - assert_eq!(st.executable_epoch_queue, None); + assert_eq!( + st.cron_checkpoint_voting.last_voting_executed_epoch(), + pending_epoch + ); + assert_eq!(*st.cron_checkpoint_voting.executable_epoch_queue(), None); } diff --git a/gateway/tests/harness.rs b/gateway/tests/harness.rs index fcedb20..77c0da4 100644 --- a/gateway/tests/harness.rs +++ b/gateway/tests/harness.rs @@ -102,8 +102,14 @@ impl Harness { assert_eq!(st.min_stake, TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT)); assert_eq!(st.check_period, DEFAULT_CHECKPOINT_PERIOD); assert_eq!(st.applied_bottomup_nonce, 0); - assert_eq!(st.cron_period, *DEFAULT_CRON_PERIOD); - assert_eq!(st.genesis_epoch, *DEFAULT_GENESIS_EPOCH); + assert_eq!( + st.cron_checkpoint_voting.submission_period(), + *DEFAULT_CRON_PERIOD + ); + assert_eq!( + st.cron_checkpoint_voting.genesis_epoch(), + *DEFAULT_GENESIS_EPOCH + ); verify_empty_map(rt, st.subnets.cid()); verify_empty_map(rt, st.checkpoints.cid()); } diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index 07de9ca..da4db7b 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -11,12 +11,20 @@ keywords = ["filecoin", "web3", "wasm", "ipc"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = "1.0.56" +log = "0.4.17" +primitives = { git = "https://github.com/consensus-shipyard/fvm-utils" } +num-traits = "0.2.14" +fvm_ipld_blockstore = "0.1.1" fvm_ipld_encoding = "0.3.3" lazy_static = "1.4.0" serde_tuple = "0.5" serde = { version = "1.0.136", features = ["derive"] } fvm_shared = { version = "=3.0.0-alpha.17", default-features = false } thiserror = "1.0.38" -num-traits = "0.2.14" fil_actors_runtime = { git = "https://github.com/consensus-shipyard/fvm-utils", features = ["fil-actor"] } -integer-encoding = { version = "3.0.3", default-features = false } \ No newline at end of file +integer-encoding = { version = "3.0.3", default-features = false } + +[dev-dependencies] +serde_json = "1.0.95" + diff --git a/subnet-actor/Cargo.toml b/subnet-actor/Cargo.toml index 7780769..55c5a3d 100644 --- a/subnet-actor/Cargo.toml +++ b/subnet-actor/Cargo.toml @@ -18,6 +18,7 @@ fil_actors_runtime = { git = "https://github.com/consensus-shipyard/fvm-utils", frc42_dispatch = "3.0.0 " ipc-gateway = { path = "../gateway" } ipc-sdk = { path = "../sdk" } +ipc-actor-common = { path = "../common" } primitives = { git = "https://github.com/consensus-shipyard/fvm-utils"} fvm_shared = { version = "=3.0.0-alpha.17", default-features = false } fvm_ipld_hamt = "0.5.1" diff --git a/subnet-actor/src/lib.rs b/subnet-actor/src/lib.rs index 8d451c6..aa029bc 100644 --- a/subnet-actor/src/lib.rs +++ b/subnet-actor/src/lib.rs @@ -8,6 +8,7 @@ use fil_actors_runtime::{ actor_dispatch, actor_error, restrict_internal_api, ActorDowncast, ActorError, CALLER_TYPES_SIGNABLE, INIT_ACTOR_ADDR, }; +use fvm_ipld_blockstore::Blockstore; use fvm_ipld_encoding::ipld_block::IpldBlock; use fvm_ipld_encoding::RawBytes; use fvm_shared::econ::TokenAmount; @@ -259,63 +260,46 @@ impl SubnetActor for Actor { return Err(actor_error!(illegal_state, "not validator")); } - state.verify_checkpoint(rt, &ch).map_err(|e| { - actor_error!( - illegal_state, - format!("checkpoint failed: {}", e.to_string()) - ) - })?; - - let mut msg = None; - - rt.transaction(|st: &mut State, rt| { - let ch_cid = ch.cid(); - - let mut found = false; - let mut votes = match st.get_votes(rt.store(), &ch_cid)? { - Some(v) => { - found = true; - v - } - None => Votes { - validators: Vec::new(), - }, - }; - - if votes.validators.iter().any(|x| x == &caller) { - return Err(actor_error!( - illegal_state, - "miner has already voted the checkpoint" - )); - } - - // add miner vote - votes.validators.push(caller); - - // if has majority - if st.has_majority_vote(rt.store(), &votes)? { - // commit checkpoint - st.flush_checkpoint(rt.store(), &ch) - .map_err(|_| actor_error!(illegal_state, "cannot flush checkpoint"))?; - - // prepare the message - msg = Some(CrossActorPayload::new( - st.ipc_gateway_addr, - ipc_gateway::Method::CommitChildCheckpoint as u64, - IpldBlock::serialize_cbor(&ch)?, - TokenAmount::zero(), - )); + state + .verify_checkpoint(rt, &ch) + .map_err(|e| actor_error!(illegal_state, format!("checkpoint failed: {}", e)))?; + + let msg = rt.transaction(|st: &mut State, rt| { + let store = rt.store(); + + let total_validator_weight = st.total_stake.clone(); + let submitter_weight = st + .get_stake(store, &caller) + .map_err(|_| actor_error!(illegal_state, "cannot get validator stake"))? + .unwrap_or_else(TokenAmount::zero); + let submission_epoch = ch.epoch(); + + let some_checkpoint = st + .epoch_checkpoint_voting + .submit_vote( + rt.store(), + ch, + submission_epoch, + caller, + submitter_weight, + total_validator_weight, + ) + .map_err(|e| { + log::error!("encountered error submitting checkpoint: {:?}", e); + actor_error!(illegal_state, e.to_string()) + })?; - // remove votes used for commitment - if found { - st.remove_votes(rt.store(), &ch_cid)?; - } + if let Some(ch) = some_checkpoint { + commit_checkpoint(st, store, &ch) + } else if let Some(ch) = st + .epoch_checkpoint_voting + .get_next_executable_vote(store) + .map_err(|_| actor_error!(illegal_state, "cannot check previous checkpoint"))? + { + commit_checkpoint(st, store, &ch) } else { - // if no majority store vote and return - st.set_votes(rt.store(), &ch_cid, votes)?; + Ok(None) } - - Ok(()) })?; // propagate to gateway @@ -381,10 +365,9 @@ impl Actor { .iter() .position(|x| x.addr == caller) { - st.validator_set - .validators_mut() - .get_mut(index) - .map(|x| x.net_addr = params.validator_net_addr); + if let Some(x) = st.validator_set.validators_mut().get_mut(index) { + x.net_addr = params.validator_net_addr; + } } else { return Err(actor_error!(forbidden, "caller is not a validator")); } @@ -407,3 +390,45 @@ impl ActorCode for Actor { SetValidatorNetAddr => set_validator_net_addr, } } + +/// The checkpoint to be committed should be the same as the previous executed checkpoint's cid before execution +fn commit_checkpoint( + st: &mut State, + store: &impl Blockstore, + ch: &Checkpoint, +) -> Result, ActorError> { + match st.ensure_checkpoint_chained(store, ch) { + Ok(is_chained) => { + if !is_chained { + return Ok(None); + } + } + Err(e) => { + log::error!("encountered error checking epoch chained: {:?}", e); + return Err(actor_error!(unhandled_message, e.to_string())); + } + }; + + st.epoch_checkpoint_voting + .mark_epoch_executed(store, ch.epoch()) + .map_err(|e| { + log::error!("encountered error marking epoch executed: {:?}", e); + actor_error!(unhandled_message, e.to_string()) + })?; + + st.previous_executed_checkpoint_cid = ch.cid(); + + // commit checkpoint + st.flush_checkpoint(store, ch) + .map_err(|_| actor_error!(illegal_state, "cannot flush checkpoint"))?; + + // prepare the message + let msg = Some(CrossActorPayload::new( + st.ipc_gateway_addr, + ipc_gateway::Method::CommitChildCheckpoint as u64, + IpldBlock::serialize_cbor(&ch)?, + TokenAmount::zero(), + )); + + Ok(msg) +} diff --git a/subnet-actor/src/state.rs b/subnet-actor/src/state.rs index 4333d0a..6ff094c 100644 --- a/subnet-actor/src/state.rs +++ b/subnet-actor/src/state.rs @@ -9,8 +9,10 @@ use fvm_shared::address::Address; use fvm_shared::bigint::Zero; use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; -use ipc_gateway::checkpoint::CHECKPOINT_GENESIS_CID; -use ipc_gateway::{Checkpoint, SubnetID, DEFAULT_CHECKPOINT_PERIOD, MIN_COLLATERAL_AMOUNT}; +use ipc_actor_common::vote::Voting; +use ipc_gateway::{ + Checkpoint, SubnetID, CHECKPOINT_GENESIS_CID, DEFAULT_CHECKPOINT_PERIOD, MIN_COLLATERAL_AMOUNT, +}; use ipc_sdk::epoch_key; use ipc_sdk::{Validator, ValidatorSet}; use lazy_static::lazy_static; @@ -44,14 +46,18 @@ pub struct State { #[serde(with = "serde_bytes")] pub genesis: Vec, pub finality_threshold: ChainEpoch, + + // duplicated definition for easier data access in client applications pub check_period: ChainEpoch, + pub genesis_epoch: ChainEpoch, + // FIXME: Consider making checkpoints a HAMT instead of an AMT so we use // the AMT index instead of and epoch k for object indexing. - pub checkpoints: TCid>, - pub window_checks: TCid>, + pub committed_checkpoints: TCid>, pub validator_set: ValidatorSet, pub min_validators: u64, - pub genesis_epoch: ChainEpoch, + pub previous_executed_checkpoint_cid: Cid, + pub epoch_checkpoint_voting: Voting, } /// We should probably have a derive macro to mark an object as a state object, @@ -64,7 +70,11 @@ impl State { current_epoch: ChainEpoch, ) -> anyhow::Result { let min_stake = TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT); - + let check_period = if params.check_period < DEFAULT_CHECKPOINT_PERIOD { + DEFAULT_CHECKPOINT_PERIOD + } else { + params.check_period + }; let state = State { name: params.name, parent_id: params.parent, @@ -78,70 +88,26 @@ impl State { }, min_validators: params.min_validators, finality_threshold: params.finality_threshold, - check_period: if params.check_period < DEFAULT_CHECKPOINT_PERIOD { - DEFAULT_CHECKPOINT_PERIOD - } else { - params.check_period - }, + check_period, + committed_checkpoints: TCid::new_hamt(store)?, genesis: params.genesis, status: Status::Instantiated, - checkpoints: TCid::new_hamt(store)?, stake: TCid::new_hamt(store)?, - window_checks: TCid::new_hamt(store)?, validator_set: ValidatorSet::default(), genesis_epoch: current_epoch, + previous_executed_checkpoint_cid: *CHECKPOINT_GENESIS_CID, + epoch_checkpoint_voting: Voting::::new_with_ratio( + store, + current_epoch, + check_period, + 1, + 2, + )?, }; Ok(state) } - pub fn get_votes( - &self, - store: &BS, - cid: &Cid, - ) -> Result, ActorError> { - let hamt = self - .window_checks - .load(store) - .map_err(|_| actor_error!(illegal_state, "cannot load votes hamt"))?; - let votes = hamt - .get(&BytesKey::from(cid.to_bytes())) - .map_err(|_| actor_error!(illegal_state, "cannot read votes"))?; - Ok(votes.cloned()) - } - - pub fn remove_votes( - &mut self, - store: &BS, - cid: &Cid, - ) -> Result<(), ActorError> { - self.window_checks - .modify(store, |hamt| { - hamt.delete(&BytesKey::from(cid.to_bytes())) - .map_err(|_| actor_error!(illegal_state, "cannot remove votes from hamt"))?; - Ok(true) - }) - .map_err(|_| actor_error!(illegal_state, "cannot modify window checks"))?; - - Ok(()) - } - - pub fn set_votes( - &mut self, - store: &BS, - cid: &Cid, - votes: Votes, - ) -> Result<(), ActorError> { - self.window_checks - .modify(store, |hamt| { - hamt.set(BytesKey::from(cid.to_bytes()), votes) - .map_err(|_| actor_error!(illegal_state, "cannot set votes in hamt"))?; - Ok(true) - }) - .map_err(|_| actor_error!(illegal_state, "cannot modify window checks"))?; - Ok(()) - } - /// Get the stake of an address. pub fn get_stake( &self, @@ -282,22 +248,6 @@ impl State { } } - fn get_checkpoint( - &self, - store: &BS, - epoch: ChainEpoch, - ) -> anyhow::Result> { - let hamt = self - .checkpoints - .load(store) - .map_err(|e| anyhow!("failed to load checkpoints: {}", e))?; - let checkpoint = hamt - .get(&epoch_key(epoch)) - .map_err(|e| anyhow!("failed to get checkpoint for id {}: {:?}", epoch, e))? - .cloned(); - Ok(checkpoint) - } - pub fn is_validator(&self, addr: &Address) -> bool { self.validator_set .validators() @@ -314,28 +264,20 @@ impl State { )); } - // check that a checkpoint for the epoch doesn't exist already. - if self.get_checkpoint(rt.store(), ch.epoch())?.is_some() { - return Err(anyhow!("cannot submit checkpoint for epoch")); - }; - - // check that the epoch is correct - if ch.epoch() % self.check_period != 0 { - return Err(anyhow!( - "epoch in checkpoint doesn't correspond with a signing window" - )); - } - // check the source is correct if *ch.source() != SubnetID::new_from_parent(&self.parent_id, rt.message().receiver()) { return Err(anyhow!("submitting checkpoint with the wrong source")); } - // check previous checkpoint - if self.prev_checkpoint_cid(rt.store(), &ch.epoch())? != ch.prev_check().cid() { - return Err(anyhow!( - "previous checkpoint not consistent with previously committed" - )); + // the epoch being submitted is the next executable epoch, we perform a check to ensure + // the checkpoints are chained. This is an early termination check to ensure the checkpoints + // are actually chained. + if self + .epoch_checkpoint_voting + .is_next_executable_epoch(ch.epoch()) + && self.previous_executed_checkpoint_cid != ch.prev_check().cid() + { + return Err(anyhow!("checkpoint not chained")); } // check signature @@ -354,21 +296,22 @@ impl State { Ok(()) } - fn prev_checkpoint_cid( - &self, - store: &BS, - epoch: &ChainEpoch, - ) -> anyhow::Result { - let mut epoch = epoch - self.check_period; - while epoch >= 0 { - match self.get_checkpoint(store, epoch)? { - Some(ch) => return Ok(ch.cid()), - None => { - epoch -= self.check_period; - } - } - } - Ok(CHECKPOINT_GENESIS_CID.clone()) + /// Ensures the checkpoints are chained, aka checkpoint.prev_check() should be the previous executed + /// checkpoint cid. If not, should abort the current checkpoint. + pub fn ensure_checkpoint_chained( + &mut self, + store: &impl Blockstore, + ch: &Checkpoint, + ) -> anyhow::Result { + Ok( + if self.previous_executed_checkpoint_cid != ch.prev_check().cid() { + self.epoch_checkpoint_voting + .abort_epoch(store, ch.data.epoch)?; + false + } else { + true + }, + ) } pub fn flush_checkpoint( @@ -377,7 +320,7 @@ impl State { ch: &Checkpoint, ) -> anyhow::Result<()> { let epoch = ch.epoch(); - self.checkpoints.modify(store, |hamt| { + self.committed_checkpoints.modify(store, |hamt| { hamt.set(epoch_key(epoch), ch.clone()) .map_err(|e| anyhow!("failed to set checkpoint: {:?}", e))?; Ok(true) @@ -396,15 +339,23 @@ impl Default for State { min_validator_stake: TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT), total_stake: TokenAmount::zero(), finality_threshold: 5, - check_period: 10, + check_period: 0, genesis: Vec::new(), status: Status::Instantiated, - checkpoints: TCid::default(), stake: TCid::default(), - window_checks: TCid::default(), validator_set: ValidatorSet::default(), min_validators: 0, genesis_epoch: 0, + previous_executed_checkpoint_cid: *CHECKPOINT_GENESIS_CID, + epoch_checkpoint_voting: Voting { + genesis_epoch: 0, + submission_period: 0, + last_voting_executed_epoch: 0, + executable_epoch_queue: None, + epoch_vote_submissions: TCid::default(), + threshold_ratio: (2, 3), + }, + committed_checkpoints: TCid::default(), } } } diff --git a/subnet-actor/src/types.rs b/subnet-actor/src/types.rs index 73bc90c..ffddbb9 100644 --- a/subnet-actor/src/types.rs +++ b/subnet-actor/src/types.rs @@ -30,7 +30,7 @@ pub struct Validator { pub weight: TokenAmount, } -#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] +#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, Default, PartialEq, Eq)] pub struct ValidatorSet { validators: Vec, // sequence number that uniquely identifies a validator set @@ -39,10 +39,7 @@ pub struct ValidatorSet { impl ValidatorSet { pub fn new() -> Self { - Self { - validators: Vec::new(), - configuration_number: 0, - } + Self::default() } pub fn validators(&self) -> &Vec { diff --git a/subnet-actor/tests/actor_test.rs b/subnet-actor/tests/actor_test.rs index af538f1..6ac822a 100644 --- a/subnet-actor/tests/actor_test.rs +++ b/subnet-actor/tests/actor_test.rs @@ -3,8 +3,8 @@ mod test { use cid::Cid; use fil_actors_runtime::runtime::Runtime; use fil_actors_runtime::test_utils::{ - expect_abort, MockRuntime, ACCOUNT_ACTOR_CODE_ID, INIT_ACTOR_CODE_ID, - MULTISIG_ACTOR_CODE_ID, + expect_abort, expect_abort_contains_message, MockRuntime, ACCOUNT_ACTOR_CODE_ID, + INIT_ACTOR_CODE_ID, MULTISIG_ACTOR_CODE_ID, }; use fil_actors_runtime::{ActorError, INIT_ACTOR_ADDR}; use fvm_ipld_encoding::ipld_block::IpldBlock; @@ -15,7 +15,9 @@ mod test { use fvm_shared::econ::TokenAmount; use fvm_shared::error::ExitCode; use fvm_shared::METHOD_SEND; - use ipc_gateway::{get_checkpoint, Checkpoint, FundParams, SubnetID, MIN_COLLATERAL_AMOUNT}; + use ipc_gateway::{ + Checkpoint, FundParams, SubnetID, CHECKPOINT_GENESIS_CID, MIN_COLLATERAL_AMOUNT, + }; use ipc_subnet_actor::{ Actor, ConsensusType, ConstructParams, JoinParams, Method, State, Status, }; @@ -24,6 +26,7 @@ mod test { use num_traits::FromPrimitive; use num_traits::Zero; use primitives::TCid; + use std::collections::BTreeSet; use std::str::FromStr; // just a test address @@ -641,7 +644,7 @@ mod test { } #[test] - fn test_submit_checkpoint() { + fn test_submit_checkpoint_works() { let test_actor_address = Address::new_id(9999); let mut runtime = construct_runtime_with_receiver(test_actor_address.clone()); @@ -650,10 +653,6 @@ mod test { Address::new_id(20), Address::new_id(30), ]; - let validator = Address::new_id(100); - let params = JoinParams { - validator_net_addr: validator.to_string(), - }; // first miner joins the subnet let value = TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT); @@ -684,6 +683,10 @@ mod test { ); } + let params = JoinParams { + validator_net_addr: caller.to_string(), + }; + runtime .call::( Method::Join as u64, @@ -702,7 +705,8 @@ mod test { // Generate the check point let root_subnet = SubnetID::from_str("/root").unwrap(); let subnet = SubnetID::new_from_parent(&root_subnet, test_actor_address); - let epoch = 10; + // we are targeting the next submission period + let epoch = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period; let mut checkpoint_0 = Checkpoint::new(subnet.clone(), epoch); checkpoint_0.set_signature( RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) @@ -727,29 +731,21 @@ mod test { let sender = miners.get(0).cloned().unwrap(); send_checkpoint(&mut runtime, sender.clone(), &checkpoint_0, false).unwrap(); - let st: State = runtime.get_state(); - let votes = st - .get_votes(runtime.store(), &checkpoint_0.cid()) - .unwrap() - .unwrap(); - assert_eq!(votes.validators, vec![sender.clone()]); - expect_abort( + // Already voted, should not vote again + expect_abort_contains_message( ExitCode::USR_ILLEGAL_STATE, + "already submitted", send_checkpoint(&mut runtime, sender.clone(), &checkpoint_0, false), ); // Send second checkpoint let sender2 = miners.get(1).cloned().unwrap(); - send_checkpoint(&mut runtime, sender2.clone(), &checkpoint_0, true).unwrap(); - let st: State = runtime.get_state(); - let votes = st.get_votes(runtime.store(), &checkpoint_0.cid()).unwrap(); - assert_eq!(votes.is_none(), true); - // check if the checkpoint is committed - let checkpoints = st.checkpoints.load(runtime.store()).unwrap(); - get_checkpoint(&checkpoints, epoch).unwrap(); + // This should have triggered commit + send_checkpoint(&mut runtime, sender2.clone(), &checkpoint_0, true).unwrap(); - // Trying to submit an already committed checkpoint should fail + // Trying to submit an already committed checkpoint should fail, i.e. if the epoch is already + // committed, then we should not allow voting again let sender2 = miners.get(2).cloned().unwrap(); runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, sender2.clone()); runtime.expect_validate_caller_type(SIG_TYPES.clone()); @@ -761,7 +757,8 @@ mod test { ), ); - // If the epoch is wrong in the next checkpoint, it should be rejected. + // If the epoch is wrong in the next checkpoint, it should be rejected. Not multiple of the + // execution period. let prev_cid = checkpoint_0.cid(); let mut checkpoint_1 = Checkpoint::new(subnet.clone(), epoch + 1); checkpoint_1.data.prev_check = TCid::from(prev_cid.clone()); @@ -775,38 +772,287 @@ mod test { ), ); - // Submit checkpoint with invalid previous cid - let epoch = 20; - let mut checkpoint_3 = Checkpoint::new(subnet.clone(), epoch); - checkpoint_3.data.prev_check = TCid::from(Cid::default()); - runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, sender.clone()); + // Start the voting for a new epoch, checking we can proceed with new epoch number. + let epoch = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2; + let prev_cid = checkpoint_0.cid(); + let mut checkpoint_4 = Checkpoint::new(subnet.clone(), epoch); + checkpoint_4.data.prev_check = TCid::from(prev_cid); + checkpoint_4.set_signature( + RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) + .unwrap() + .bytes() + .to_vec(), + ); + send_checkpoint(&mut runtime, sender.clone(), &checkpoint_4, false).unwrap(); + } + + /// Tests the checkpoint will abort when checkpoints are not chained and the submitted epoch is the + /// next executable epoch, we stop the epoch from submission + #[test] + fn test_submit_checkpoint_aborts_not_chained_early_termination() { + let test_actor_address = Address::new_id(9999); + let mut runtime = construct_runtime_with_receiver(test_actor_address.clone()); + + let miners = vec![ + Address::new_id(10), + Address::new_id(20), + Address::new_id(30), + ]; + + // first miner joins the subnet + let value = TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT); + + let mut i = 0; + for caller in &miners { + runtime.set_value(value.clone()); + runtime.set_balance(TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT)); + runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, caller.clone()); + runtime.expect_validate_caller_type(SIG_TYPES.clone()); + if i == 0 { + runtime.expect_send( + Address::new_id(IPC_GATEWAY_ADDR), + ipc_gateway::Method::Register as u64, + None, + TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT), + None, + ExitCode::new(0), + ); + } else { + runtime.expect_send( + Address::new_id(IPC_GATEWAY_ADDR), + ipc_gateway::Method::AddStake as u64, + None, + TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT), + None, + ExitCode::new(0), + ); + } + + let params = JoinParams { + validator_net_addr: caller.to_string(), + }; + + runtime + .call::( + Method::Join as u64, + IpldBlock::serialize_cbor(¶ms).unwrap(), + ) + .unwrap(); + + i += 1; + } + + // verify that we have an active subnet with 3 validators. + let st: State = runtime.get_state(); + assert_eq!(st.validator_set.validators().len(), 3); + assert_eq!(st.status, Status::Active); + + // Generate the check point + let root_subnet = SubnetID::from_str("/root").unwrap(); + let subnet = SubnetID::new_from_parent(&root_subnet, test_actor_address); + // we are targeting the next submission period + let epoch = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period; + let mut checkpoint_0 = Checkpoint::new(subnet.clone(), epoch); + checkpoint_0.data.prev_check = TCid::default(); + checkpoint_0.set_signature( + RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) + .unwrap() + .bytes() + .to_vec(), + ); + + // Reject the submission as checkpoints are not chained + let non_miner = Address::new_id(10); + runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, non_miner.clone()); runtime.expect_validate_caller_type(SIG_TYPES.clone()); - expect_abort( + expect_abort_contains_message( ExitCode::USR_ILLEGAL_STATE, + "checkpoint not chained", runtime.call::( Method::SubmitCheckpoint as u64, - IpldBlock::serialize_cbor(&checkpoint_3).unwrap(), + IpldBlock::serialize_cbor(&checkpoint_0).unwrap(), ), ); + } - // Send correct payload - let epoch = 20; - let prev_cid = checkpoint_0.cid(); - let mut checkpoint_4 = Checkpoint::new(subnet.clone(), epoch); - checkpoint_4.data.prev_check = TCid::from(prev_cid); - checkpoint_4.set_signature( + /// Tests the checkpoint will abort when checkpoints are not chained and the submitted epoch is NOT the + /// next executable epoch, we need to reset the epoch. + /// + /// Test flows like the below: + /// 1. Create 3 validators and register them to the subnet with equal weight + /// + /// 2. Submit to epoch `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2`, we are skipping + /// the first epoch and ensure this is executable. The previous checkpoint cid is set to some value `cid_a`. + /// We should see the epoch number being stored in the next executable queue. + /// Checks at step 2: + /// - executable_queue should contain `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2` + /// - last_executed_epoch is still `DEFAULT_CHAIN_EPOCH` + /// + /// 3. Submit to epoch `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 1`, i.e. the previous + /// epoch in step 2. This would lead to the epoch being committed. The key is the checkpoint cid of the current + /// epoch should be different from that in step 2, i.e. any value other than `cid_a` + /// Checks at step 3: + /// - executable_queue should contain `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2` + /// - last_executed_epoch is still `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 1` + /// + /// 4. Submit to any epoch after `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 1`, should + /// trigger a reset in submission of epoch `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2`. + /// Checks at step 4: + /// - executable_queue should have removed `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2` + /// - last_executed_epoch is still `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 1` + /// - submission at `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2` is cleared + #[test] + fn test_submit_checkpoint_aborts_not_chained_reset_epoch() { + let test_actor_address = Address::new_id(9999); + let mut runtime = construct_runtime_with_receiver(test_actor_address.clone()); + + // Step 1 + let miners = vec![ + Address::new_id(10), + Address::new_id(20), + Address::new_id(30), + ]; + + // first miner joins the subnet + let value = TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT); + + let mut i = 0; + for caller in &miners { + runtime.set_value(value.clone()); + runtime.set_balance(TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT)); + runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, caller.clone()); + runtime.expect_validate_caller_type(SIG_TYPES.clone()); + if i == 0 { + runtime.expect_send( + Address::new_id(IPC_GATEWAY_ADDR), + ipc_gateway::Method::Register as u64, + None, + TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT), + None, + ExitCode::new(0), + ); + } else { + runtime.expect_send( + Address::new_id(IPC_GATEWAY_ADDR), + ipc_gateway::Method::AddStake as u64, + None, + TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT), + None, + ExitCode::new(0), + ); + } + + let params = JoinParams { + validator_net_addr: caller.to_string(), + }; + + runtime + .call::( + Method::Join as u64, + IpldBlock::serialize_cbor(¶ms).unwrap(), + ) + .unwrap(); + + i += 1; + } + + let root_subnet = SubnetID::from_str("/root").unwrap(); + let subnet = SubnetID::new_from_parent(&root_subnet, test_actor_address); + + // Step 2 + let st: State = runtime.get_state(); + let epoch_2 = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2; + let prev_cid = Cid::default(); + let mut checkpoint_2 = Checkpoint::new(subnet.clone(), epoch_2); + checkpoint_2.data.prev_check = TCid::from(prev_cid); + checkpoint_2.set_signature( RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) .unwrap() .bytes() .to_vec(), ); - send_checkpoint(&mut runtime, sender.clone(), &checkpoint_4, false).unwrap(); + + send_checkpoint(&mut runtime, miners[0].clone(), &checkpoint_2, false).unwrap(); + send_checkpoint(&mut runtime, miners[1].clone(), &checkpoint_2, false).unwrap(); + + // performing checks let st: State = runtime.get_state(); - let votes = st - .get_votes(runtime.store(), &checkpoint_4.cid()) - .unwrap() - .unwrap(); - assert_eq!(votes.validators, vec![sender.clone()]); + assert_eq!( + st.previous_executed_checkpoint_cid, + CHECKPOINT_GENESIS_CID.clone() + ); + assert_eq!( + st.epoch_checkpoint_voting.last_voting_executed_epoch, + DEFAULT_CHAIN_EPOCH + ); + assert_eq!( + st.epoch_checkpoint_voting.executable_epoch_queue, + Some(BTreeSet::from([epoch_2])) + ); + assert_eq!( + st.epoch_checkpoint_voting + .load_most_voted_submission(runtime.store(), epoch_2) + .unwrap(), + Some(checkpoint_2.clone()) + ); + assert_eq!( + st.epoch_checkpoint_voting + .load_most_voted_weight(runtime.store(), epoch_2) + .unwrap(), + Some(TokenAmount::from_whole(2)) + ); + + // Step 3 + let epoch_1 = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 1; + let mut checkpoint_1 = Checkpoint::new(subnet.clone(), epoch_1); + checkpoint_1.set_signature( + RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) + .unwrap() + .bytes() + .to_vec(), + ); + + send_checkpoint(&mut runtime, miners[0].clone(), &checkpoint_1, false).unwrap(); + send_checkpoint(&mut runtime, miners[1].clone(), &checkpoint_1, true).unwrap(); + + // performing checks + let st: State = runtime.get_state(); + assert_eq!(st.previous_executed_checkpoint_cid, checkpoint_1.cid()); + assert_eq!( + st.epoch_checkpoint_voting.last_voting_executed_epoch, + epoch_1 + ); + assert_eq!( + st.epoch_checkpoint_voting.executable_epoch_queue, + Some(BTreeSet::from([ + DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2 + ])) + ); + assert_eq!( + st.epoch_checkpoint_voting + .load_most_voted_weight(runtime.store(), epoch_2) + .unwrap(), + Some(TokenAmount::from_whole(2)) + ); + assert_eq!( + st.epoch_checkpoint_voting + .load_most_voted_weight(runtime.store(), epoch_1) + .unwrap(), + None + ); + + // Step 4 + checkpoint_2.data.prev_check = TCid::from(checkpoint_1.cid()); + send_checkpoint(&mut runtime, miners[2].clone(), &checkpoint_2, false).unwrap(); + + // perform checks + let st: State = runtime.get_state(); + assert_eq!(st.previous_executed_checkpoint_cid, checkpoint_1.cid()); + assert_eq!( + st.epoch_checkpoint_voting.last_voting_executed_epoch, + epoch_1 + ); + assert_eq!(st.epoch_checkpoint_voting.executable_epoch_queue, None); } fn send_checkpoint( From 779963ed4838c1a8f00649d207684ec67c727a66 Mon Sep 17 00:00:00 2001 From: Alfonso de la Rocha Date: Wed, 5 Apr 2023 11:17:25 +0200 Subject: [PATCH 21/27] rename checkpoints to bottomup and topdown --- common/src/vote/submission.rs | 4 +- common/src/vote/voting.rs | 4 +- gateway/src/checkpoint.rs | 81 ++++++++++++++++--- gateway/src/cron.rs | 10 +-- gateway/src/lib.rs | 50 ++++++------ gateway/src/state.rs | 45 ++++++----- gateway/src/subnet.rs | 2 +- gateway/src/types.rs | 18 ++--- gateway/tests/gateway_test.rs | 132 +++++++++++++++---------------- gateway/tests/harness.rs | 30 +++---- subnet-actor/src/lib.rs | 8 +- subnet-actor/src/state.rs | 44 +++++++---- subnet-actor/src/types.rs | 4 +- subnet-actor/tests/actor_test.rs | 20 ++--- 14 files changed, 263 insertions(+), 189 deletions(-) diff --git a/common/src/vote/submission.rs b/common/src/vote/submission.rs index ff513ba..df19efa 100644 --- a/common/src/vote/submission.rs +++ b/common/src/vote/submission.rs @@ -25,7 +25,7 @@ pub struct EpochVoteSubmissions { pub submitters: TCid>, /// The map to track the submission weight of each unique key pub submission_weights: TCid>, - /// The different cron checkpoints, with vote's unique key as key + /// The different checkpoints, with vote's unique key as key pub submissions: TCid>, } @@ -346,7 +346,7 @@ mod tests { use primitives::{TCid, THamt}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; - #[derive(PartialEq, Clone, Deserialize_tuple, Serialize_tuple, Debug)] + #[derive(PartialEq, Eq, Clone, Deserialize_tuple, Serialize_tuple, Debug)] struct DummyVote { key: UniqueBytesKey, } diff --git a/common/src/vote/voting.rs b/common/src/vote/voting.rs index c7cfd88..2548c1d 100644 --- a/common/src/vote/voting.rs +++ b/common/src/vote/voting.rs @@ -91,7 +91,7 @@ impl Voting { total_weight: TokenAmount, ) -> anyhow::Result> { // first we check the epoch is the correct one, we process only it's multiple - // of cron_period since genesis_epoch + // of topdown_check_period since genesis_epoch if !self.epoch_can_vote(epoch) { return Err(anyhow!("epoch not allowed")); } @@ -227,7 +227,7 @@ impl Voting { } Some(epoch) => { if *epoch > self.last_voting_executed_epoch + self.submission_period { - log::debug!("earliest executable epoch not the same cron period"); + log::debug!("earliest executable epoch not the same checkpoint period"); return Ok(None); } *epoch diff --git a/gateway/src/checkpoint.rs b/gateway/src/checkpoint.rs index 12ef05a..1666181 100644 --- a/gateway/src/checkpoint.rs +++ b/gateway/src/checkpoint.rs @@ -1,20 +1,21 @@ +use crate::{ensure_message_sorted, CrossMsg, StorableMsg}; use anyhow::anyhow; use cid::multihash::Code; use cid::multihash::MultihashDigest; use cid::Cid; use fvm_ipld_encoding::DAG_CBOR; use fvm_ipld_encoding::{serde_bytes, to_vec}; +use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use ipc_actor_common::vote::{UniqueBytesKey, UniqueVote}; use ipc_sdk::subnet_id::SubnetID; +use ipc_sdk::ValidatorSet; use lazy_static::lazy_static; use num_traits::Zero; use primitives::{TCid, TLink}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; -use crate::{ensure_message_sorted, CrossMsg}; - lazy_static! { // Default CID used for the genesis checkpoint. Using // TCid::default() leads to corrupting the fvm datastore @@ -24,19 +25,19 @@ lazy_static! { } #[derive(PartialEq, Eq, Clone, Debug, Serialize_tuple, Deserialize_tuple)] -pub struct Checkpoint { +pub struct BottomUpCheckpoint { pub data: CheckData, #[serde(with = "serde_bytes")] pub sig: Vec, } -impl UniqueVote for Checkpoint { +impl UniqueVote for BottomUpCheckpoint { fn unique_key(&self) -> anyhow::Result { Ok(self.cid().to_bytes()) } } -impl Checkpoint { +impl BottomUpCheckpoint { pub fn new(id: SubnetID, epoch: ChainEpoch) -> Self { Self { data: CheckData::new(id, epoch), @@ -76,7 +77,7 @@ impl Checkpoint { } /// return the cid of the previous checkpoint this checkpoint points to. - pub fn prev_check(&self) -> &TCid> { + pub fn prev_check(&self) -> &TCid> { &self.data.prev_check } @@ -122,7 +123,7 @@ impl Checkpoint { /// Add the cid of a checkpoint from a child subnet for further propagation /// to the upper layerse of the hierarchy. - pub fn add_child_check(&mut self, commit: &Checkpoint) -> anyhow::Result<()> { + pub fn add_child_check(&mut self, commit: &BottomUpCheckpoint) -> anyhow::Result<()> { let cid = TCid::from(commit.cid()); match self .data @@ -162,7 +163,7 @@ pub struct CheckData { #[serde(with = "serde_bytes")] pub proof: Vec, pub epoch: ChainEpoch, - pub prev_check: TCid>, + pub prev_check: TCid>, pub children: Vec, pub cross_msgs: BatchCrossMsgs, } @@ -189,7 +190,7 @@ impl CheckData { #[derive(PartialEq, Eq, Clone, Debug, Serialize_tuple, Deserialize_tuple)] pub struct ChildCheck { pub source: SubnetID, - pub checks: Vec>>, + pub checks: Vec>>, } /// CheckpointEpoch returns the epoch of the next checkpoint @@ -210,3 +211,65 @@ pub fn window_epoch(epoch: ChainEpoch, period: ChainEpoch) -> ChainEpoch { let ind = epoch / period; period * (ind + 1) } + +/// Validators tracks all the validator in the subnet. It is useful in handling top-down checkpoints. +#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] +pub struct Validators { + /// The validator set that holds all the validators + pub validators: ValidatorSet, + /// Tracks the total weight of the validators + pub total_weight: TokenAmount, +} + +impl Validators { + pub fn new(validators: ValidatorSet) -> Self { + let mut weight = TokenAmount::zero(); + for v in validators.validators() { + weight += v.weight.clone(); + } + Self { + validators, + total_weight: weight, + } + } + + /// Get the weight of a validator + pub fn get_validator_weight(&self, addr: &Address) -> Option { + self.validators + .validators() + .iter() + .find(|x| x.addr == *addr) + .map(|v| v.weight.clone()) + } +} + +/// Checkpoints propagated from parent to child to signal the "final view" of the parent chain +/// from the different validators in the subnet. +#[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] +pub struct TopDownCheckpoint { + pub epoch: ChainEpoch, + pub top_down_msgs: Vec, +} + +impl UniqueVote for TopDownCheckpoint { + /// Derive the unique key of the checkpoint using hash function. + /// + /// To compare the top-down checkpoint and ensure they are the same, we need to make sure the + /// top_down_msgs are the same. However, the top_down_msgs are vec, they may contain the same + /// content, but their orders are different. In this case, we need to ensure the same order is + /// maintained in the top-down checkpoint submission. + /// + /// To ensure we have the same consistent output for different submissions, we require: + /// - top down messages are sorted by `nonce` in descending order + /// + /// Actor will not perform sorting to save gas. Client should do it, actor just check. + fn unique_key(&self) -> anyhow::Result { + ensure_message_sorted(&self.top_down_msgs)?; + + let mh_code = Code::Blake2b256; + // TODO: to avoid serialization again, maybe we should perform deserialization in the actor + // TODO: dispatch call to save gas? The actor dispatching contains the raw serialized data, + // TODO: which we dont have to serialize here again + Ok(mh_code.digest(&to_vec(self).unwrap()).to_bytes()) + } +} diff --git a/gateway/src/cron.rs b/gateway/src/cron.rs index eabeecc..8d8087b 100644 --- a/gateway/src/cron.rs +++ b/gateway/src/cron.rs @@ -10,7 +10,7 @@ use ipc_actor_common::vote::{UniqueBytesKey, UniqueVote}; use ipc_sdk::ValidatorSet; use num_traits::Zero; -/// Validators tracks all the validator in the subnet. It is useful in handling cron checkpoints. +/// Validators tracks all the validator in the subnet. It is useful in handling top-down checkpoints. #[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple)] pub struct Validators { /// The validator set that holds all the validators @@ -44,18 +44,18 @@ impl Validators { /// Checkpoints propagated from parent to child to signal the "final view" of the parent chain /// from the different validators in the subnet. #[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] -pub struct CronCheckpoint { +pub struct TopDownCheckpoint { pub epoch: ChainEpoch, pub top_down_msgs: Vec, } -impl UniqueVote for CronCheckpoint { +impl UniqueVote for TopDownCheckpoint { /// Derive the unique key of the checkpoint using hash function. /// - /// To compare the cron checkpoint and ensure they are the same, we need to make sure the + /// To compare the top-down checkpoint and ensure they are the same, we need to make sure the /// top_down_msgs are the same. However, the top_down_msgs are vec, they may contain the same /// content, but their orders are different. In this case, we need to ensure the same order is - /// maintained in the cron checkpoint submission. + /// maintained in the top-down checkpoint submission. /// /// To ensure we have the same consistent output for different submissions, we require: /// - top down messages are sorted by `nonce` in descending order diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 9c1856f..f13c541 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -2,12 +2,12 @@ extern crate core; -pub use self::checkpoint::{Checkpoint, CHECKPOINT_GENESIS_CID}; +pub use self::checkpoint::{BottomUpCheckpoint, CHECKPOINT_GENESIS_CID}; pub use self::cross::{is_bottomup, CrossMsg, CrossMsgs, IPCMsgType, StorableMsg}; pub use self::state::*; pub use self::subnet::*; pub use self::types::*; -pub use cron::CronCheckpoint; +pub use checkpoint::TopDownCheckpoint; use cross::{burn_bu_funds, cross_msg_side_effects, distribute_crossmsg_fee}; use fil_actors_runtime::runtime::fvm::resolve_secp_bls; use fil_actors_runtime::runtime::{ActorCode, Runtime}; @@ -33,7 +33,6 @@ use num_traits::FromPrimitive; fil_actors_runtime::wasm_trampoline!(Actor); pub mod checkpoint; -mod cron; mod cross; mod error; #[doc(hidden)] @@ -63,7 +62,7 @@ pub enum Method { SendCross = frc42_dispatch::method_hash!("SendCross"), Propagate = frc42_dispatch::method_hash!("Propagate"), WhiteListPropagator = frc42_dispatch::method_hash!("WhiteListPropagator"), - SubmitCron = frc42_dispatch::method_hash!("SubmitCron"), + SubmitTopDownCheckpoint = frc42_dispatch::method_hash!("SubmitTopDownCheckpoint"), SetMembership = frc42_dispatch::method_hash!("SetMembership"), } @@ -271,7 +270,10 @@ impl Actor { /// CommitChildCheck propagates the commitment of a checkpoint from a child subnet, /// process the cross-messages directed to the subnet. - fn commit_child_check(rt: &mut impl Runtime, mut commit: Checkpoint) -> Result<(), ActorError> { + fn commit_child_check( + rt: &mut impl Runtime, + mut commit: BottomUpCheckpoint, + ) -> Result<(), ActorError> { // This must be called by a subnet actor, once we have a way to identify subnet actor, // we should update here. rt.validate_immediate_caller_accept_any()?; @@ -677,19 +679,19 @@ impl Actor { }) } - /// Submit a new cron checkpoint + /// Submit a new topdown checkpoint /// - /// It only accepts submission at multiples of `cron_period` since `genesis_epoch`, which are + /// It only accepts submission at multiples of `topdown_check_period` since `genesis_epoch`, which are /// set during construction. Each checkpoint will have its number of submissions tracked. The /// same address cannot submit twice. Once the number of submissions is more than or equal to 2/3 /// of the total number of validators, the messages will be applied. /// - /// Each cron checkpoint will be checked against each other using blake hashing. - fn submit_cron( + /// Each topdown checkpoint will be checked against each other using blake hashing. + fn submit_topdown_check( rt: &mut impl Runtime, - checkpoint: CronCheckpoint, + checkpoint: TopDownCheckpoint, ) -> Result { - // submit cron can only be performed by signable addresses + // submit topdown can only be performed by signable addresses rt.validate_immediate_caller_type(CALLER_TYPES_SIGNABLE.iter())?; let to_execute = rt.transaction(|st: &mut State, rt| { @@ -700,7 +702,7 @@ impl Actor { let epoch = checkpoint.epoch; let total_weight = st.validators.total_weight.clone(); let ch = st - .cron_checkpoint_voting + .topdown_checkpoint_voting .submit_vote( store, checkpoint, @@ -711,13 +713,13 @@ impl Actor { ) .map_err(|e| { log::error!( - "encountered error processing submit cron checkpoint: {:?}", + "encountered error processing submit topdown checkpoint: {:?}", e ); actor_error!(unhandled_message, e.to_string()) })?; if ch.is_some() { - st.cron_checkpoint_voting + st.topdown_checkpoint_voting .mark_epoch_executed(store, epoch) .map_err(|e| { log::error!("encountered error marking epoch executed: {:?}", e); @@ -728,11 +730,11 @@ impl Actor { Ok(ch) })?; - // we only `execute_next_cron_epoch(rt)` if there is no execution for the current submission + // we only `execute_next_topdown_epoch(rt)` if there is no execution for the current submission // so that we don't blow up the gas. if let Some(checkpoint) = to_execute { if checkpoint.top_down_msgs.is_empty() { - Self::execute_next_cron_epoch(rt)?; + Self::execute_next_topdown_epoch(rt)?; } for m in checkpoint.top_down_msgs { Self::apply_msg_inner( @@ -744,7 +746,7 @@ impl Actor { )?; } } else { - Self::execute_next_cron_epoch(rt)?; + Self::execute_next_topdown_epoch(rt)?; } Ok(RawBytes::default()) @@ -945,27 +947,27 @@ impl Actor { Ok(RawBytes::new(cid.to_bytes())) } - /// Execute the next approved cron checkpoint. + /// Execute the next approved topdown checkpoint. /// This is an edge case to ensure none of the epoches will be stuck. Consider the following example: /// - /// Epoch 10 and 20 are two cron epoch to be executed. However, all the validators have submitted + /// Epoch 10 and 20 are two topdown epoch to be executed. However, all the validators have submitted /// epoch 20, and the status is to be executed, however, epoch 10 has yet to be executed. Now, /// epoch 10 has reached consensus and executed, but epoch 20 cannot be executed because every /// validator has already voted, no one can vote again to trigger the execution. Epoch 20 is stuck. - fn execute_next_cron_epoch(rt: &mut impl Runtime) -> Result<(), ActorError> { + fn execute_next_topdown_epoch(rt: &mut impl Runtime) -> Result<(), ActorError> { let checkpoint = rt.transaction(|st: &mut State, rt| { let cp = st - .cron_checkpoint_voting + .topdown_checkpoint_voting .get_next_executable_vote(rt.store()) .map_err(|e| { log::error!( - "encountered error processing submit cron checkpoint: {:?}", + "encountered error processing submit topdown checkpoint: {:?}", e ); actor_error!(unhandled_message, e.to_string()) })?; if let Some(cp) = &cp { - st.cron_checkpoint_voting + st.topdown_checkpoint_voting .mark_epoch_executed(rt.store(), cp.epoch) .map_err(|e| { log::error!("encountered error marking epoch executed: {:?}", e); @@ -1006,7 +1008,7 @@ impl ActorCode for Actor { SendCross => send_cross, Propagate => propagate, WhiteListPropagator => whitelist_propagator, - SubmitCron => submit_cron, + SubmitTopDownCheckpoint => submit_topdown_check, SetMembership => set_membership, } } diff --git a/gateway/src/state.rs b/gateway/src/state.rs index 9536c71..df806db 100644 --- a/gateway/src/state.rs +++ b/gateway/src/state.rs @@ -17,8 +17,8 @@ use primitives::{TCid, THamt}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; use std::str::FromStr; -use crate::cron::Validators; -use crate::CronCheckpoint; +use crate::checkpoint::Validators; +use crate::TopDownCheckpoint; use ipc_actor_common::vote::Voting; use ipc_sdk::subnet_id::SubnetID; use ipc_sdk::ValidatorSet; @@ -40,17 +40,18 @@ pub struct State { pub total_subnets: u64, pub min_stake: TokenAmount, pub subnets: TCid>, - pub check_period: ChainEpoch, + pub bottomup_check_period: ChainEpoch, + pub topdown_check_period: ChainEpoch, // FIXME: Consider making checkpoints a HAMT instead of an AMT so we use // the AMT index instead of and epoch k for object indexing. - pub checkpoints: TCid>, + pub bottomup_checkpoints: TCid>, /// `postbox` keeps track for an EOA of all the cross-net messages triggered by /// an actor that need to be propagated further through the hierarchy. pub postbox: PostBox, pub bottomup_nonce: u64, pub applied_bottomup_nonce: u64, pub applied_topdown_nonce: u64, - pub cron_checkpoint_voting: Voting, + pub topdown_checkpoint_voting: Voting, pub validators: Validators, } @@ -65,21 +66,25 @@ impl State { total_subnets: Default::default(), min_stake: MIN_SUBNET_COLLATERAL.clone(), subnets: TCid::new_hamt(store)?, - check_period: match params.checkpoint_period > DEFAULT_CHECKPOINT_PERIOD { - true => params.checkpoint_period, + bottomup_check_period: match params.bottomup_check_period > DEFAULT_CHECKPOINT_PERIOD { + true => params.bottomup_check_period, false => DEFAULT_CHECKPOINT_PERIOD, }, - checkpoints: TCid::new_hamt(store)?, + topdown_check_period: match params.topdown_check_period > DEFAULT_CHECKPOINT_PERIOD { + true => params.topdown_check_period, + false => DEFAULT_CHECKPOINT_PERIOD, + }, + bottomup_checkpoints: TCid::new_hamt(store)?, postbox: TCid::new_hamt(store)?, bottomup_nonce: Default::default(), // This way we ensure that the first message to execute has nonce= 0, if not it would expect 1 and fail for the first nonce // We first increase to the subsequent and then execute for bottom-up messages applied_bottomup_nonce: Default::default(), applied_topdown_nonce: Default::default(), - cron_checkpoint_voting: Voting::::new( + topdown_checkpoint_voting: Voting::::new( store, params.genesis_epoch, - params.cron_period, + params.topdown_check_period, )?, validators: Validators::new(ValidatorSet::default()), }) @@ -163,9 +168,9 @@ impl State { pub(crate) fn flush_checkpoint( &mut self, store: &BS, - ch: &Checkpoint, + ch: &BottomUpCheckpoint, ) -> anyhow::Result<()> { - self.checkpoints + self.bottomup_checkpoints .update(store, |checkpoints| set_checkpoint(checkpoints, ch.clone())) } @@ -174,16 +179,16 @@ impl State { &self, store: &BS, epoch: ChainEpoch, - ) -> anyhow::Result { + ) -> anyhow::Result { if epoch < 0 { return Err(anyhow!("epoch can't be negative")); } - let ch_epoch = checkpoint_epoch(epoch, self.check_period); - let checkpoints = self.checkpoints.load(store)?; + let ch_epoch = checkpoint_epoch(epoch, self.bottomup_check_period); + let checkpoints = self.bottomup_checkpoints.load(store)?; Ok(match get_checkpoint(&checkpoints, ch_epoch)? { Some(ch) => ch.clone(), - None => Checkpoint::new(self.network_name.clone(), ch_epoch), + None => BottomUpCheckpoint::new(self.network_name.clone(), ch_epoch), }) } @@ -396,8 +401,8 @@ fn get_subnet<'m, BS: Blockstore>( } pub fn set_checkpoint( - checkpoints: &mut Map, - ch: Checkpoint, + checkpoints: &mut Map, + ch: BottomUpCheckpoint, ) -> anyhow::Result<()> { let epoch = ch.epoch(); checkpoints @@ -407,9 +412,9 @@ pub fn set_checkpoint( } pub fn get_checkpoint<'m, BS: Blockstore>( - checkpoints: &'m Map, + checkpoints: &'m Map, epoch: ChainEpoch, -) -> anyhow::Result> { +) -> anyhow::Result> { checkpoints .get(&epoch_key(epoch)) .map_err(|e| e.downcast_wrap(format!("failed to get checkpoint for id {}", epoch))) diff --git a/gateway/src/subnet.rs b/gateway/src/subnet.rs index edfcc74..bbcb799 100644 --- a/gateway/src/subnet.rs +++ b/gateway/src/subnet.rs @@ -28,7 +28,7 @@ pub struct Subnet { pub topdown_nonce: u64, pub circ_supply: TokenAmount, pub status: Status, - pub prev_checkpoint: Option, + pub prev_checkpoint: Option, } impl Subnet { diff --git a/gateway/src/types.rs b/gateway/src/types.rs index 03cbe85..87d1032 100644 --- a/gateway/src/types.rs +++ b/gateway/src/types.rs @@ -12,7 +12,6 @@ use multihash::MultihashDigest; use primitives::CodeType; use std::cmp::Ordering; -use crate::checkpoint::Checkpoint; use crate::cross::CrossMsg; /// ID used in the builtin-actors bundle manifest @@ -35,8 +34,8 @@ pub trait ExecutableMessage { #[derive(Serialize_tuple, Deserialize_tuple)] pub struct ConstructorParams { pub network_name: String, - pub checkpoint_period: ChainEpoch, - pub cron_period: ChainEpoch, + pub bottomup_check_period: ChainEpoch, + pub topdown_check_period: ChainEpoch, pub genesis_epoch: ChainEpoch, } @@ -45,11 +44,6 @@ pub struct FundParams { pub value: TokenAmount, } -#[derive(Debug, Serialize_tuple, Deserialize_tuple)] -pub struct CheckpointParams { - pub checkpoint: Checkpoint, -} - #[derive(Serialize_tuple, Deserialize_tuple, Clone)] pub struct CrossMsgParams { pub cross_msg: CrossMsg, @@ -128,8 +122,8 @@ mod tests { fn serialize_params() { let p = ConstructorParams { network_name: "/root".to_string(), - checkpoint_period: 100, - cron_period: 20, + bottomup_check_period: 100, + topdown_check_period: 20, genesis_epoch: 10, }; let bytes = fil_actors_runtime::util::cbor::serialize(&p, "").unwrap(); @@ -141,8 +135,8 @@ mod tests { .unwrap(); assert_eq!(p.network_name, deserialized.network_name); - assert_eq!(p.checkpoint_period, deserialized.checkpoint_period); - assert_eq!(p.cron_period, deserialized.cron_period); + assert_eq!(p.bottomup_check_period, deserialized.bottomup_check_period); + assert_eq!(p.topdown_check_period, deserialized.topdown_check_period); assert_eq!(p.genesis_epoch, deserialized.genesis_epoch); } } diff --git a/gateway/tests/gateway_test.rs b/gateway/tests/gateway_test.rs index a3666e4..3c565f1 100644 --- a/gateway/tests/gateway_test.rs +++ b/gateway/tests/gateway_test.rs @@ -13,8 +13,8 @@ use ipc_actor_common::vote::{EpochVoteSubmissions, UniqueVote}; use ipc_gateway::checkpoint::BatchCrossMsgs; use ipc_gateway::Status::{Active, Inactive}; use ipc_gateway::{ - get_topdown_msg, Checkpoint, CronCheckpoint, CrossMsg, IPCAddress, PostBoxItem, State, - StorableMsg, CROSS_MSG_FEE, DEFAULT_CHECKPOINT_PERIOD, SUBNET_ACTOR_REWARD_METHOD, + get_topdown_msg, BottomUpCheckpoint, CrossMsg, IPCAddress, PostBoxItem, State, StorableMsg, + TopDownCheckpoint, CROSS_MSG_FEE, DEFAULT_CHECKPOINT_PERIOD, SUBNET_ACTOR_REWARD_METHOD, }; use ipc_sdk::subnet_id::SubnetID; use ipc_sdk::{epoch_key, Validator, ValidatorSet}; @@ -258,7 +258,7 @@ fn checkpoint_commit() { // Commit first checkpoint for first window in first subnet let epoch: ChainEpoch = 10; rt.set_epoch(epoch); - let ch = Checkpoint::new(shid.clone(), epoch + 9); + let ch = BottomUpCheckpoint::new(shid.clone(), epoch + 9); h.commit_child_check(&mut rt, &shid, &ch, ExitCode::OK) .unwrap(); @@ -275,7 +275,7 @@ fn checkpoint_commit() { let prev_cid = ch.cid(); // Append a new checkpoint for the same subnet - let mut ch = Checkpoint::new(shid.clone(), epoch + 11); + let mut ch = BottomUpCheckpoint::new(shid.clone(), epoch + 11); ch.data.prev_check = TCid::from(prev_cid); h.commit_child_check(&mut rt, &shid, &ch, ExitCode::OK) .unwrap(); @@ -298,14 +298,14 @@ fn checkpoint_commit() { h.check_state(); // Trying to commit from the wrong subnet - let ch = Checkpoint::new(shid.clone(), epoch + 9); + let ch = BottomUpCheckpoint::new(shid.clone(), epoch + 9); h.commit_child_check(&mut rt, &shid_two, &ch, ExitCode::USR_ILLEGAL_ARGUMENT) .unwrap(); // Commit first checkpoint for first window in second subnet let epoch: ChainEpoch = 10; rt.set_epoch(epoch); - let ch = Checkpoint::new(shid_two.clone(), epoch + 9); + let ch = BottomUpCheckpoint::new(shid_two.clone(), epoch + 9); h.commit_child_check(&mut rt, &shid_two, &ch, ExitCode::OK) .unwrap(); @@ -353,7 +353,7 @@ fn checkpoint_crossmsgs() { // Commit first checkpoint for first window in first subnet let epoch: ChainEpoch = 10; rt.set_epoch(epoch); - let mut ch = Checkpoint::new(shid.clone(), epoch + 9); + let mut ch = BottomUpCheckpoint::new(shid.clone(), epoch + 9); // and include some fees. let fee = TokenAmount::from_atto(5); ch.data.cross_msgs = BatchCrossMsgs { @@ -629,7 +629,7 @@ fn test_commit_child_check_bu_target_subnet() { let epoch: ChainEpoch = 10; rt.set_epoch(epoch); - let mut ch = Checkpoint::new(shid.clone(), epoch + 9); + let mut ch = BottomUpCheckpoint::new(shid.clone(), epoch + 9); // and include some fees. let fee = TokenAmount::from_atto(5); ch.data.cross_msgs = BatchCrossMsgs { @@ -712,7 +712,7 @@ fn test_commit_child_check_bu_not_target_subnet() { let epoch: ChainEpoch = 10; rt.set_epoch(epoch); - let mut ch = Checkpoint::new(shid.clone(), epoch + 9); + let mut ch = BottomUpCheckpoint::new(shid.clone(), epoch + 9); // and include some fees. let fee = TokenAmount::from_atto(5); ch.data.cross_msgs = BatchCrossMsgs { @@ -842,7 +842,7 @@ fn test_propagate_with_remainder() { let epoch: ChainEpoch = 10; rt.set_epoch(epoch); - let mut ch = Checkpoint::new(shid.clone(), epoch + 9); + let mut ch = BottomUpCheckpoint::new(shid.clone(), epoch + 9); // and include some fees. let fee = TokenAmount::from_atto(5); ch.data.cross_msgs = BatchCrossMsgs { @@ -1078,7 +1078,7 @@ fn test_commit_child_check_tp_target_subnet() { }; let epoch: ChainEpoch = 10; rt.set_epoch(epoch); - let mut ch = Checkpoint::new(shid.clone(), epoch + 9); + let mut ch = BottomUpCheckpoint::new(shid.clone(), epoch + 9); // and include some fees. let fee = TokenAmount::from_atto(5); ch.data.cross_msgs = BatchCrossMsgs { @@ -1152,7 +1152,7 @@ fn test_commit_child_check_tp_not_target_subnet() { }; let epoch: ChainEpoch = 10; rt.set_epoch(epoch); - let mut ch = Checkpoint::new(shid.clone(), epoch + 9); + let mut ch = BottomUpCheckpoint::new(shid.clone(), epoch + 9); // and include some fees. let fee = TokenAmount::from_atto(5); ch.data.cross_msgs = BatchCrossMsgs { @@ -1299,35 +1299,35 @@ fn setup_membership(h: &Harness, rt: &mut MockRuntime) { } #[test] -fn test_submit_cron_checking_errors() { +fn test_submit_topdown_check_checking_errors() { let (h, mut rt) = setup_root(); setup_membership(&h, &mut rt); let submitter = Address::new_id(10000); - let checkpoint = CronCheckpoint { - epoch: *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD, + let checkpoint = TopDownCheckpoint { + epoch: *DEFAULT_GENESIS_EPOCH + *DEFAULT_TOPDOWN_PERIOD, top_down_msgs: vec![], }; - let r = h.submit_cron(&mut rt, submitter, checkpoint); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint); assert!(r.is_err()); assert_eq!(r.unwrap_err().msg(), "caller not validator"); let submitter = Address::new_id(0); - let checkpoint = CronCheckpoint { + let checkpoint = TopDownCheckpoint { epoch: *DEFAULT_GENESIS_EPOCH + 1, top_down_msgs: vec![], }; - let r = h.submit_cron(&mut rt, submitter, checkpoint); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint); assert!(r.is_err()); assert_eq!(r.unwrap_err().msg(), "epoch not allowed"); - let checkpoint = CronCheckpoint { + let checkpoint = TopDownCheckpoint { epoch: *DEFAULT_GENESIS_EPOCH, top_down_msgs: vec![], }; - let r = h.submit_cron(&mut rt, submitter, checkpoint); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint); assert!(r.is_err()); assert_eq!(r.unwrap_err().msg(), "epoch already executed"); } @@ -1335,10 +1335,10 @@ fn test_submit_cron_checking_errors() { fn get_epoch_submissions( rt: &mut MockRuntime, epoch: ChainEpoch, -) -> Option> { +) -> Option> { let st: State = rt.get_state(); let hamt = st - .cron_checkpoint_voting + .topdown_checkpoint_voting .epoch_vote_submissions() .load(rt.store()) .unwrap(); @@ -1347,21 +1347,21 @@ fn get_epoch_submissions( } #[test] -fn test_submit_cron_works_with_execution() { +fn test_submit_topdown_check_works_with_execution() { let (h, mut rt) = setup_root(); setup_membership(&h, &mut rt); - let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD; + let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_TOPDOWN_PERIOD; let msg = storable_msg(0); - let checkpoint = CronCheckpoint { + let checkpoint = TopDownCheckpoint { epoch, top_down_msgs: vec![msg.clone()], }; // first submission let submitter = Address::new_id(0); - let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_ok()); let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); assert_eq!( @@ -1373,19 +1373,19 @@ fn test_submit_cron_works_with_execution() { ); let st: State = rt.get_state(); assert_eq!( - st.cron_checkpoint_voting.last_voting_executed_epoch(), + st.topdown_checkpoint_voting.last_voting_executed_epoch(), *DEFAULT_GENESIS_EPOCH ); // not executed yet // already submitted let submitter = Address::new_id(0); - let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_err()); assert_eq!(r.unwrap_err().msg(), "already submitted"); // second submission let submitter = Address::new_id(1); - let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_ok()); let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); assert_eq!( @@ -1397,13 +1397,13 @@ fn test_submit_cron_works_with_execution() { ); let st: State = rt.get_state(); assert_eq!( - st.cron_checkpoint_voting.last_voting_executed_epoch(), + st.topdown_checkpoint_voting.last_voting_executed_epoch(), *DEFAULT_GENESIS_EPOCH ); // not executed yet // third submission let submitter = Address::new_id(2); - let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_ok()); let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); assert_eq!( @@ -1415,7 +1415,7 @@ fn test_submit_cron_works_with_execution() { ); let st: State = rt.get_state(); assert_eq!( - st.cron_checkpoint_voting.last_voting_executed_epoch(), + st.topdown_checkpoint_voting.last_voting_executed_epoch(), *DEFAULT_GENESIS_EPOCH ); // not executed yet @@ -1429,13 +1429,13 @@ fn test_submit_cron_works_with_execution() { None, ExitCode::OK, ); - let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_ok()); let submission = get_epoch_submissions(&mut rt, epoch); assert!(submission.is_none()); let st: State = rt.get_state(); assert_eq!( - st.cron_checkpoint_voting.last_voting_executed_epoch(), + st.topdown_checkpoint_voting.last_voting_executed_epoch(), epoch ); } @@ -1452,53 +1452,53 @@ fn storable_msg(nonce: u64) -> StorableMsg { } #[test] -fn test_submit_cron_abort() { +fn test_submit_topdown_check_abort() { let (h, mut rt) = setup_root(); setup_membership(&h, &mut rt); - let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD; + let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_TOPDOWN_PERIOD; // first submission let submitter = Address::new_id(0); - let checkpoint = CronCheckpoint { + let checkpoint = TopDownCheckpoint { epoch, top_down_msgs: vec![], }; - let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_ok()); // second submission let submitter = Address::new_id(1); - let checkpoint = CronCheckpoint { + let checkpoint = TopDownCheckpoint { epoch, top_down_msgs: vec![storable_msg(1)], }; - let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_ok()); // third submission let submitter = Address::new_id(2); - let checkpoint = CronCheckpoint { + let checkpoint = TopDownCheckpoint { epoch, top_down_msgs: vec![storable_msg(1), storable_msg(2)], }; - let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_ok()); // fourth submission, aborted let submitter = Address::new_id(3); - let checkpoint = CronCheckpoint { + let checkpoint = TopDownCheckpoint { epoch, top_down_msgs: vec![storable_msg(1), storable_msg(2), storable_msg(3)], }; - let r = h.submit_cron(&mut rt, submitter, checkpoint.clone()); + let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_ok()); // check aborted let st: State = rt.get_state(); assert_eq!( - st.cron_checkpoint_voting.last_voting_executed_epoch(), + st.topdown_checkpoint_voting.last_voting_executed_epoch(), *DEFAULT_GENESIS_EPOCH ); // not executed yet let submission = get_epoch_submissions(&mut rt, epoch).unwrap(); @@ -1513,65 +1513,65 @@ fn test_submit_cron_abort() { } #[test] -fn test_submit_cron_sequential_execution() { +fn test_submit_topdown_check_sequential_execution() { let (h, mut rt) = setup_root(); setup_membership(&h, &mut rt); - let pending_epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD * 2; - let checkpoint = CronCheckpoint { + let pending_epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_TOPDOWN_PERIOD * 2; + let checkpoint = TopDownCheckpoint { epoch: pending_epoch, top_down_msgs: vec![], }; // first submission let submitter = Address::new_id(0); - h.submit_cron(&mut rt, submitter, checkpoint.clone()) + h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()) .unwrap(); // second submission let submitter = Address::new_id(1); - h.submit_cron(&mut rt, submitter, checkpoint.clone()) + h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()) .unwrap(); // third submission let submitter = Address::new_id(2); - h.submit_cron(&mut rt, submitter, checkpoint.clone()) + h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()) .unwrap(); // fourth submission, not executed let submitter = Address::new_id(3); - h.submit_cron(&mut rt, submitter, checkpoint.clone()) + h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()) .unwrap(); let st: State = rt.get_state(); assert_eq!( - st.cron_checkpoint_voting.last_voting_executed_epoch(), + st.topdown_checkpoint_voting.last_voting_executed_epoch(), *DEFAULT_GENESIS_EPOCH ); // not executed yet assert_eq!( - *st.cron_checkpoint_voting.executable_epoch_queue(), + *st.topdown_checkpoint_voting.executable_epoch_queue(), Some(BTreeSet::from([pending_epoch])) ); // not executed yet // now we execute the previous epoch let msg = storable_msg(0); - let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD; - let checkpoint = CronCheckpoint { + let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_TOPDOWN_PERIOD; + let checkpoint = TopDownCheckpoint { epoch, top_down_msgs: vec![msg.clone()], }; // first submission let submitter = Address::new_id(0); - h.submit_cron(&mut rt, submitter, checkpoint.clone()) + h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()) .unwrap(); // second submission let submitter = Address::new_id(1); - h.submit_cron(&mut rt, submitter, checkpoint.clone()) + h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()) .unwrap(); // third submission let submitter = Address::new_id(2); - h.submit_cron(&mut rt, submitter, checkpoint.clone()) + h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()) .unwrap(); // fourth submission, executed let submitter = Address::new_id(3); @@ -1584,28 +1584,28 @@ fn test_submit_cron_sequential_execution() { None, ExitCode::OK, ); - h.submit_cron(&mut rt, submitter, checkpoint.clone()) + h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()) .unwrap(); let submission = get_epoch_submissions(&mut rt, epoch); assert!(submission.is_none()); let st: State = rt.get_state(); assert_eq!( - st.cron_checkpoint_voting.last_voting_executed_epoch(), + st.topdown_checkpoint_voting.last_voting_executed_epoch(), epoch ); // now we submit to the next epoch - let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_CRON_PERIOD * 3; - let checkpoint = CronCheckpoint { + let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_TOPDOWN_PERIOD * 3; + let checkpoint = TopDownCheckpoint { epoch, top_down_msgs: vec![], }; - h.submit_cron(&mut rt, submitter, checkpoint.clone()) + h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()) .unwrap(); let st: State = rt.get_state(); assert_eq!( - st.cron_checkpoint_voting.last_voting_executed_epoch(), + st.topdown_checkpoint_voting.last_voting_executed_epoch(), pending_epoch ); - assert_eq!(*st.cron_checkpoint_voting.executable_epoch_queue(), None); + assert_eq!(*st.topdown_checkpoint_voting.executable_epoch_queue(), None); } diff --git a/gateway/tests/harness.rs b/gateway/tests/harness.rs index 77c0da4..3c90183 100644 --- a/gateway/tests/harness.rs +++ b/gateway/tests/harness.rs @@ -23,11 +23,11 @@ use fvm_shared::MethodNum; use fvm_shared::METHOD_SEND; use ipc_gateway::checkpoint::ChildCheck; use ipc_gateway::{ - ext, get_topdown_msg, is_bottomup, Actor, Checkpoint, ConstructorParams, CrossMsg, + ext, get_topdown_msg, is_bottomup, Actor, BottomUpCheckpoint, ConstructorParams, CrossMsg, CrossMsgParams, FundParams, IPCAddress, Method, PropagateParams, State, StorableMsg, Subnet, - SubnetID, CROSS_MSG_FEE, DEFAULT_CHECKPOINT_PERIOD, MIN_COLLATERAL_AMOUNT, + SubnetID, TopDownCheckpoint, CROSS_MSG_FEE, DEFAULT_CHECKPOINT_PERIOD, MIN_COLLATERAL_AMOUNT, + SUBNET_ACTOR_REWARD_METHOD, }; -use ipc_gateway::{CronCheckpoint, SUBNET_ACTOR_REWARD_METHOD}; use ipc_sdk::ValidatorSet; use lazy_static::lazy_static; use primitives::{TCid, TCidContent}; @@ -43,7 +43,7 @@ lazy_static! { Address::new_bls(&[1; fvm_shared::address::BLS_PUB_LEN]).unwrap(); pub static ref ACTOR: Address = Address::new_actor("actor".as_bytes()); pub static ref SIG_TYPES: Vec = vec![*ACCOUNT_ACTOR_CODE_ID, *MULTISIG_ACTOR_CODE_ID]; - pub static ref DEFAULT_CRON_PERIOD: ChainEpoch = 20; + pub static ref DEFAULT_TOPDOWN_PERIOD: ChainEpoch = 20; pub static ref DEFAULT_GENESIS_EPOCH: ChainEpoch = 1; } @@ -81,8 +81,8 @@ impl Harness { rt.expect_validate_caller_addr(vec![INIT_ACTOR_ADDR]); let params = ConstructorParams { network_name: self.net_name.to_string(), - checkpoint_period: 10, - cron_period: *DEFAULT_CRON_PERIOD, + bottomup_check_period: 10, + topdown_check_period: *DEFAULT_TOPDOWN_PERIOD, genesis_epoch: *DEFAULT_GENESIS_EPOCH, }; rt.set_caller(*INIT_ACTOR_CODE_ID, INIT_ACTOR_ADDR); @@ -100,18 +100,18 @@ impl Harness { assert_eq!(st.network_name, self.net_name); assert_eq!(st.min_stake, TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT)); - assert_eq!(st.check_period, DEFAULT_CHECKPOINT_PERIOD); + assert_eq!(st.bottomup_check_period, DEFAULT_CHECKPOINT_PERIOD); assert_eq!(st.applied_bottomup_nonce, 0); assert_eq!( - st.cron_checkpoint_voting.submission_period(), - *DEFAULT_CRON_PERIOD + st.topdown_checkpoint_voting.submission_period(), + *DEFAULT_TOPDOWN_PERIOD ); assert_eq!( - st.cron_checkpoint_voting.genesis_epoch(), + st.topdown_checkpoint_voting.genesis_epoch(), *DEFAULT_GENESIS_EPOCH ); verify_empty_map(rt, st.subnets.cid()); - verify_empty_map(rt, st.checkpoints.cid()); + verify_empty_map(rt, st.bottomup_checkpoints.cid()); } pub fn register( @@ -244,7 +244,7 @@ impl Harness { &self, rt: &mut MockRuntime, id: &SubnetID, - ch: &Checkpoint, + ch: &BottomUpCheckpoint, code: ExitCode, ) -> Result<(), ActorError> { rt.set_caller(*SUBNET_ACTOR_CODE_ID, id.subnet_actor()); @@ -554,17 +554,17 @@ impl Harness { Ok(()) } - pub fn submit_cron( + pub fn submit_topdown_check( &self, rt: &mut MockRuntime, submitter: Address, - checkpoint: CronCheckpoint, + checkpoint: TopDownCheckpoint, ) -> Result<(), ActorError> { rt.set_caller(*ACCOUNT_ACTOR_CODE_ID, submitter); rt.expect_validate_caller_type(SIG_TYPES.clone()); rt.call::( - Method::SubmitCron as MethodNum, + Method::SubmitTopDownCheckpoint as MethodNum, IpldBlock::serialize_cbor(&checkpoint).unwrap(), )?; diff --git a/subnet-actor/src/lib.rs b/subnet-actor/src/lib.rs index aa029bc..4aa1aff 100644 --- a/subnet-actor/src/lib.rs +++ b/subnet-actor/src/lib.rs @@ -14,7 +14,7 @@ use fvm_ipld_encoding::RawBytes; use fvm_shared::econ::TokenAmount; use fvm_shared::error::ExitCode; use fvm_shared::{MethodNum, METHOD_CONSTRUCTOR, METHOD_SEND}; -use ipc_gateway::{Checkpoint, FundParams, MIN_COLLATERAL_AMOUNT}; +use ipc_gateway::{BottomUpCheckpoint, FundParams, MIN_COLLATERAL_AMOUNT}; use num::BigInt; use num_derive::FromPrimitive; use num_traits::{FromPrimitive, Zero}; @@ -60,7 +60,7 @@ pub trait SubnetActor { /// Submits a new checkpoint for the subnet. fn submit_checkpoint( rt: &mut impl Runtime, - ch: Checkpoint, + ch: BottomUpCheckpoint, ) -> Result, ActorError>; /// Distributes the rewards for the subnet to validators. @@ -249,7 +249,7 @@ impl SubnetActor for Actor { /// votes from 2/3 of miners with collateral. fn submit_checkpoint( rt: &mut impl Runtime, - ch: Checkpoint, + ch: BottomUpCheckpoint, ) -> Result, ActorError> { rt.validate_immediate_caller_type(CALLER_TYPES_SIGNABLE.iter())?; @@ -395,7 +395,7 @@ impl ActorCode for Actor { fn commit_checkpoint( st: &mut State, store: &impl Blockstore, - ch: &Checkpoint, + ch: &BottomUpCheckpoint, ) -> Result, ActorError> { match st.ensure_checkpoint_chained(store, ch) { Ok(is_chained) => { diff --git a/subnet-actor/src/state.rs b/subnet-actor/src/state.rs index 6ff094c..b96b9f0 100644 --- a/subnet-actor/src/state.rs +++ b/subnet-actor/src/state.rs @@ -11,7 +11,8 @@ use fvm_shared::clock::ChainEpoch; use fvm_shared::econ::TokenAmount; use ipc_actor_common::vote::Voting; use ipc_gateway::{ - Checkpoint, SubnetID, CHECKPOINT_GENESIS_CID, DEFAULT_CHECKPOINT_PERIOD, MIN_COLLATERAL_AMOUNT, + BottomUpCheckpoint, SubnetID, CHECKPOINT_GENESIS_CID, DEFAULT_CHECKPOINT_PERIOD, + MIN_COLLATERAL_AMOUNT, }; use ipc_sdk::epoch_key; use ipc_sdk::{Validator, ValidatorSet}; @@ -45,19 +46,19 @@ pub struct State { pub status: Status, #[serde(with = "serde_bytes")] pub genesis: Vec, - pub finality_threshold: ChainEpoch, // duplicated definition for easier data access in client applications - pub check_period: ChainEpoch, + pub bottomup_check_period: ChainEpoch, + pub topdown_check_period: ChainEpoch, pub genesis_epoch: ChainEpoch, // FIXME: Consider making checkpoints a HAMT instead of an AMT so we use // the AMT index instead of and epoch k for object indexing. - pub committed_checkpoints: TCid>, + pub committed_checkpoints: TCid>, pub validator_set: ValidatorSet, pub min_validators: u64, pub previous_executed_checkpoint_cid: Cid, - pub epoch_checkpoint_voting: Voting, + pub epoch_checkpoint_voting: Voting, } /// We should probably have a derive macro to mark an object as a state object, @@ -70,10 +71,15 @@ impl State { current_epoch: ChainEpoch, ) -> anyhow::Result { let min_stake = TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT); - let check_period = if params.check_period < DEFAULT_CHECKPOINT_PERIOD { + let bottomup_check_period = if params.bottomup_check_period < DEFAULT_CHECKPOINT_PERIOD { DEFAULT_CHECKPOINT_PERIOD } else { - params.check_period + params.bottomup_check_period + }; + let topdown_check_period = if params.topdown_check_period < DEFAULT_CHECKPOINT_PERIOD { + DEFAULT_CHECKPOINT_PERIOD + } else { + params.topdown_check_period }; let state = State { name: params.name, @@ -87,8 +93,8 @@ impl State { params.min_validator_stake }, min_validators: params.min_validators, - finality_threshold: params.finality_threshold, - check_period, + bottomup_check_period, + topdown_check_period, committed_checkpoints: TCid::new_hamt(store)?, genesis: params.genesis, status: Status::Instantiated, @@ -96,12 +102,12 @@ impl State { validator_set: ValidatorSet::default(), genesis_epoch: current_epoch, previous_executed_checkpoint_cid: *CHECKPOINT_GENESIS_CID, - epoch_checkpoint_voting: Voting::::new_with_ratio( + epoch_checkpoint_voting: Voting::::new_with_ratio( store, current_epoch, - check_period, - 1, + bottomup_check_period, 2, + 3, )?, }; @@ -256,7 +262,11 @@ impl State { } /// Do not call this function in transaction - pub fn verify_checkpoint(&self, rt: &mut impl Runtime, ch: &Checkpoint) -> anyhow::Result<()> { + pub fn verify_checkpoint( + &self, + rt: &mut impl Runtime, + ch: &BottomUpCheckpoint, + ) -> anyhow::Result<()> { // check that subnet is active if self.status != Status::Active { return Err(anyhow!( @@ -301,7 +311,7 @@ impl State { pub fn ensure_checkpoint_chained( &mut self, store: &impl Blockstore, - ch: &Checkpoint, + ch: &BottomUpCheckpoint, ) -> anyhow::Result { Ok( if self.previous_executed_checkpoint_cid != ch.prev_check().cid() { @@ -317,7 +327,7 @@ impl State { pub fn flush_checkpoint( &mut self, store: &BS, - ch: &Checkpoint, + ch: &BottomUpCheckpoint, ) -> anyhow::Result<()> { let epoch = ch.epoch(); self.committed_checkpoints.modify(store, |hamt| { @@ -338,8 +348,8 @@ impl Default for State { consensus: ConsensusType::Delegated, min_validator_stake: TokenAmount::from_atto(MIN_COLLATERAL_AMOUNT), total_stake: TokenAmount::zero(), - finality_threshold: 5, - check_period: 0, + bottomup_check_period: 0, + topdown_check_period: 0, genesis: Vec::new(), status: Status::Instantiated, stake: TCid::default(), diff --git a/subnet-actor/src/types.rs b/subnet-actor/src/types.rs index ffddbb9..fca077b 100644 --- a/subnet-actor/src/types.rs +++ b/subnet-actor/src/types.rs @@ -115,8 +115,8 @@ pub struct ConstructParams { pub consensus: ConsensusType, pub min_validator_stake: TokenAmount, pub min_validators: u64, - pub finality_threshold: ChainEpoch, - pub check_period: ChainEpoch, + pub bottomup_check_period: ChainEpoch, + pub topdown_check_period: ChainEpoch, // genesis is no longer generated by the actor // on-the-fly, but it is accepted as a construct // param diff --git a/subnet-actor/tests/actor_test.rs b/subnet-actor/tests/actor_test.rs index 6ac822a..f8c300b 100644 --- a/subnet-actor/tests/actor_test.rs +++ b/subnet-actor/tests/actor_test.rs @@ -16,7 +16,7 @@ mod test { use fvm_shared::error::ExitCode; use fvm_shared::METHOD_SEND; use ipc_gateway::{ - Checkpoint, FundParams, SubnetID, CHECKPOINT_GENESIS_CID, MIN_COLLATERAL_AMOUNT, + BottomUpCheckpoint, FundParams, SubnetID, CHECKPOINT_GENESIS_CID, MIN_COLLATERAL_AMOUNT, }; use ipc_subnet_actor::{ Actor, ConsensusType, ConstructParams, JoinParams, Method, State, Status, @@ -45,8 +45,8 @@ mod test { consensus: ConsensusType::Dummy, min_validator_stake: Default::default(), min_validators: 0, - finality_threshold: 0, - check_period: 0, + topdown_check_period: 0, + bottomup_check_period: 0, genesis: vec![], } } @@ -707,7 +707,7 @@ mod test { let subnet = SubnetID::new_from_parent(&root_subnet, test_actor_address); // we are targeting the next submission period let epoch = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period; - let mut checkpoint_0 = Checkpoint::new(subnet.clone(), epoch); + let mut checkpoint_0 = BottomUpCheckpoint::new(subnet.clone(), epoch); checkpoint_0.set_signature( RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) .unwrap() @@ -760,7 +760,7 @@ mod test { // If the epoch is wrong in the next checkpoint, it should be rejected. Not multiple of the // execution period. let prev_cid = checkpoint_0.cid(); - let mut checkpoint_1 = Checkpoint::new(subnet.clone(), epoch + 1); + let mut checkpoint_1 = BottomUpCheckpoint::new(subnet.clone(), epoch + 1); checkpoint_1.data.prev_check = TCid::from(prev_cid.clone()); runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, sender.clone()); runtime.expect_validate_caller_type(SIG_TYPES.clone()); @@ -775,7 +775,7 @@ mod test { // Start the voting for a new epoch, checking we can proceed with new epoch number. let epoch = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2; let prev_cid = checkpoint_0.cid(); - let mut checkpoint_4 = Checkpoint::new(subnet.clone(), epoch); + let mut checkpoint_4 = BottomUpCheckpoint::new(subnet.clone(), epoch); checkpoint_4.data.prev_check = TCid::from(prev_cid); checkpoint_4.set_signature( RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) @@ -852,7 +852,7 @@ mod test { let subnet = SubnetID::new_from_parent(&root_subnet, test_actor_address); // we are targeting the next submission period let epoch = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period; - let mut checkpoint_0 = Checkpoint::new(subnet.clone(), epoch); + let mut checkpoint_0 = BottomUpCheckpoint::new(subnet.clone(), epoch); checkpoint_0.data.prev_check = TCid::default(); checkpoint_0.set_signature( RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) @@ -963,7 +963,7 @@ mod test { let st: State = runtime.get_state(); let epoch_2 = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2; let prev_cid = Cid::default(); - let mut checkpoint_2 = Checkpoint::new(subnet.clone(), epoch_2); + let mut checkpoint_2 = BottomUpCheckpoint::new(subnet.clone(), epoch_2); checkpoint_2.data.prev_check = TCid::from(prev_cid); checkpoint_2.set_signature( RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) @@ -1004,7 +1004,7 @@ mod test { // Step 3 let epoch_1 = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 1; - let mut checkpoint_1 = Checkpoint::new(subnet.clone(), epoch_1); + let mut checkpoint_1 = BottomUpCheckpoint::new(subnet.clone(), epoch_1); checkpoint_1.set_signature( RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) .unwrap() @@ -1058,7 +1058,7 @@ mod test { fn send_checkpoint( runtime: &mut MockRuntime, sender: Address, - checkpoint: &Checkpoint, + checkpoint: &BottomUpCheckpoint, is_commit: bool, ) -> Result, ActorError> { runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, sender.clone()); From e2bf214d00dce4e5d836fcfd095fd1e43666387c Mon Sep 17 00:00:00 2001 From: cryptoAtwill Date: Wed, 5 Apr 2023 19:00:25 +0800 Subject: [PATCH 22/27] fix tests --- subnet-actor/tests/actor_test.rs | 38 ++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/subnet-actor/tests/actor_test.rs b/subnet-actor/tests/actor_test.rs index f8c300b..987771f 100644 --- a/subnet-actor/tests/actor_test.rs +++ b/subnet-actor/tests/actor_test.rs @@ -652,6 +652,7 @@ mod test { Address::new_id(10), Address::new_id(20), Address::new_id(30), + Address::new_id(40), ]; // first miner joins the subnet @@ -697,9 +698,9 @@ mod test { i += 1; } - // verify that we have an active subnet with 3 validators. + // verify that we have an active subnet with 4 validators. let st: State = runtime.get_state(); - assert_eq!(st.validator_set.validators().len(), 3); + assert_eq!(st.validator_set.validators().len(), 4); assert_eq!(st.status, Status::Active); // Generate the check point @@ -716,7 +717,7 @@ mod test { ); // Only validators should be entitled to submit checkpoints. - let non_miner = Address::new_id(40); + let non_miner = Address::new_id(50); runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, non_miner.clone()); runtime.expect_validate_caller_type(SIG_TYPES.clone()); expect_abort( @@ -742,11 +743,18 @@ mod test { let sender2 = miners.get(1).cloned().unwrap(); // This should have triggered commit - send_checkpoint(&mut runtime, sender2.clone(), &checkpoint_0, true).unwrap(); + send_checkpoint(&mut runtime, sender2.clone(), &checkpoint_0, false).unwrap(); + send_checkpoint( + &mut runtime, + miners.get(2).cloned().unwrap(), + &checkpoint_0, + true, + ) + .unwrap(); // Trying to submit an already committed checkpoint should fail, i.e. if the epoch is already // committed, then we should not allow voting again - let sender2 = miners.get(2).cloned().unwrap(); + let sender2 = miners.get(3).cloned().unwrap(); runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, sender2.clone()); runtime.expect_validate_caller_type(SIG_TYPES.clone()); expect_abort( @@ -797,6 +805,7 @@ mod test { Address::new_id(10), Address::new_id(20), Address::new_id(30), + Address::new_id(40), ]; // first miner joins the subnet @@ -844,8 +853,6 @@ mod test { // verify that we have an active subnet with 3 validators. let st: State = runtime.get_state(); - assert_eq!(st.validator_set.validators().len(), 3); - assert_eq!(st.status, Status::Active); // Generate the check point let root_subnet = SubnetID::from_str("/root").unwrap(); @@ -862,8 +869,8 @@ mod test { ); // Reject the submission as checkpoints are not chained - let non_miner = Address::new_id(10); - runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, non_miner.clone()); + let s = Address::new_id(10); + runtime.set_caller(*ACCOUNT_ACTOR_CODE_ID, s.clone()); runtime.expect_validate_caller_type(SIG_TYPES.clone()); expect_abort_contains_message( ExitCode::USR_ILLEGAL_STATE, @@ -879,7 +886,7 @@ mod test { /// next executable epoch, we need to reset the epoch. /// /// Test flows like the below: - /// 1. Create 3 validators and register them to the subnet with equal weight + /// 1. Create 4 validators and register them to the subnet with equal weight /// /// 2. Submit to epoch `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2`, we are skipping /// the first epoch and ensure this is executable. The previous checkpoint cid is set to some value `cid_a`. @@ -911,6 +918,7 @@ mod test { Address::new_id(10), Address::new_id(20), Address::new_id(30), + Address::new_id(40), ]; // first miner joins the subnet @@ -974,6 +982,7 @@ mod test { send_checkpoint(&mut runtime, miners[0].clone(), &checkpoint_2, false).unwrap(); send_checkpoint(&mut runtime, miners[1].clone(), &checkpoint_2, false).unwrap(); + send_checkpoint(&mut runtime, miners[2].clone(), &checkpoint_2, false).unwrap(); // performing checks let st: State = runtime.get_state(); @@ -999,7 +1008,7 @@ mod test { st.epoch_checkpoint_voting .load_most_voted_weight(runtime.store(), epoch_2) .unwrap(), - Some(TokenAmount::from_whole(2)) + Some(TokenAmount::from_whole(3)) ); // Step 3 @@ -1013,7 +1022,8 @@ mod test { ); send_checkpoint(&mut runtime, miners[0].clone(), &checkpoint_1, false).unwrap(); - send_checkpoint(&mut runtime, miners[1].clone(), &checkpoint_1, true).unwrap(); + send_checkpoint(&mut runtime, miners[1].clone(), &checkpoint_1, false).unwrap(); + send_checkpoint(&mut runtime, miners[2].clone(), &checkpoint_1, true).unwrap(); // performing checks let st: State = runtime.get_state(); @@ -1032,7 +1042,7 @@ mod test { st.epoch_checkpoint_voting .load_most_voted_weight(runtime.store(), epoch_2) .unwrap(), - Some(TokenAmount::from_whole(2)) + Some(TokenAmount::from_whole(3)) ); assert_eq!( st.epoch_checkpoint_voting @@ -1043,7 +1053,7 @@ mod test { // Step 4 checkpoint_2.data.prev_check = TCid::from(checkpoint_1.cid()); - send_checkpoint(&mut runtime, miners[2].clone(), &checkpoint_2, false).unwrap(); + send_checkpoint(&mut runtime, miners[3].clone(), &checkpoint_2, false).unwrap(); // perform checks let st: State = runtime.get_state(); From 3f1774d8a9fd92e045fa70f7df9add0fabb4d789 Mon Sep 17 00:00:00 2001 From: Alfonso de la Rocha Date: Wed, 5 Apr 2023 15:21:38 +0200 Subject: [PATCH 23/27] use CrossMsg for topdown checkpoint --- gateway/src/checkpoint.rs | 4 +-- gateway/src/lib.rs | 16 ++-------- gateway/tests/gateway_test.rs | 41 +++++++++++++------------ subnet-actor/src/lib.rs | 6 ++-- subnet-actor/src/state.rs | 10 +++--- subnet-actor/tests/actor_test.rs | 52 ++++++++++++++++---------------- 6 files changed, 60 insertions(+), 69 deletions(-) diff --git a/gateway/src/checkpoint.rs b/gateway/src/checkpoint.rs index 1666181..2496c09 100644 --- a/gateway/src/checkpoint.rs +++ b/gateway/src/checkpoint.rs @@ -1,4 +1,4 @@ -use crate::{ensure_message_sorted, CrossMsg, StorableMsg}; +use crate::{ensure_message_sorted, CrossMsg}; use anyhow::anyhow; use cid::multihash::Code; use cid::multihash::MultihashDigest; @@ -248,7 +248,7 @@ impl Validators { #[derive(Clone, Debug, Serialize_tuple, Deserialize_tuple, PartialEq, Eq)] pub struct TopDownCheckpoint { pub epoch: ChainEpoch, - pub top_down_msgs: Vec, + pub top_down_msgs: Vec, } impl UniqueVote for TopDownCheckpoint { diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index f13c541..7ac9663 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -737,13 +737,7 @@ impl Actor { Self::execute_next_topdown_epoch(rt)?; } for m in checkpoint.top_down_msgs { - Self::apply_msg_inner( - rt, - CrossMsg { - msg: m, - wrapped: false, - }, - )?; + Self::apply_msg_inner(rt, m)?; } } else { Self::execute_next_topdown_epoch(rt)?; @@ -980,13 +974,7 @@ impl Actor { if let Some(checkpoint) = checkpoint { for m in checkpoint.top_down_msgs { - Self::apply_msg_inner( - rt, - CrossMsg { - msg: m, - wrapped: false, - }, - )?; + Self::apply_msg_inner(rt, m)?; } } Ok(()) diff --git a/gateway/tests/gateway_test.rs b/gateway/tests/gateway_test.rs index 3c565f1..5cfc60a 100644 --- a/gateway/tests/gateway_test.rs +++ b/gateway/tests/gateway_test.rs @@ -1353,7 +1353,7 @@ fn test_submit_topdown_check_works_with_execution() { setup_membership(&h, &mut rt); let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_TOPDOWN_PERIOD; - let msg = storable_msg(0); + let msg = cross_msg(0); let checkpoint = TopDownCheckpoint { epoch, top_down_msgs: vec![msg.clone()], @@ -1422,10 +1422,10 @@ fn test_submit_topdown_check_works_with_execution() { // fourth submission, executed let submitter = Address::new_id(3); rt.expect_send( - msg.to.raw_addr().unwrap(), - msg.method, + msg.msg.to.raw_addr().unwrap(), + msg.msg.method, None, - msg.value, + msg.msg.value, None, ExitCode::OK, ); @@ -1440,14 +1440,17 @@ fn test_submit_topdown_check_works_with_execution() { ); } -fn storable_msg(nonce: u64) -> StorableMsg { - StorableMsg { - from: IPCAddress::new(&ROOTNET_ID, &Address::new_id(10)).unwrap(), - to: IPCAddress::new(&ROOTNET_ID, &Address::new_id(20)).unwrap(), - method: 0, - params: Default::default(), - value: Default::default(), - nonce, +fn cross_msg(nonce: u64) -> CrossMsg { + CrossMsg { + msg: StorableMsg { + from: IPCAddress::new(&ROOTNET_ID, &Address::new_id(10)).unwrap(), + to: IPCAddress::new(&ROOTNET_ID, &Address::new_id(20)).unwrap(), + method: 0, + params: Default::default(), + value: Default::default(), + nonce, + }, + wrapped: false, } } @@ -1472,7 +1475,7 @@ fn test_submit_topdown_check_abort() { let submitter = Address::new_id(1); let checkpoint = TopDownCheckpoint { epoch, - top_down_msgs: vec![storable_msg(1)], + top_down_msgs: vec![cross_msg(1)], }; let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_ok()); @@ -1481,7 +1484,7 @@ fn test_submit_topdown_check_abort() { let submitter = Address::new_id(2); let checkpoint = TopDownCheckpoint { epoch, - top_down_msgs: vec![storable_msg(1), storable_msg(2)], + top_down_msgs: vec![cross_msg(1), cross_msg(2)], }; let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_ok()); @@ -1490,7 +1493,7 @@ fn test_submit_topdown_check_abort() { let submitter = Address::new_id(3); let checkpoint = TopDownCheckpoint { epoch, - top_down_msgs: vec![storable_msg(1), storable_msg(2), storable_msg(3)], + top_down_msgs: vec![cross_msg(1), cross_msg(2), cross_msg(3)], }; let r = h.submit_topdown_check(&mut rt, submitter, checkpoint.clone()); assert!(r.is_ok()); @@ -1554,7 +1557,7 @@ fn test_submit_topdown_check_sequential_execution() { ); // not executed yet // now we execute the previous epoch - let msg = storable_msg(0); + let msg = cross_msg(0); let epoch = *DEFAULT_GENESIS_EPOCH + *DEFAULT_TOPDOWN_PERIOD; let checkpoint = TopDownCheckpoint { epoch, @@ -1577,10 +1580,10 @@ fn test_submit_topdown_check_sequential_execution() { let submitter = Address::new_id(3); // define expected send rt.expect_send( - msg.to.raw_addr().unwrap(), - msg.method, + msg.msg.to.raw_addr().unwrap(), + msg.msg.method, None, - msg.value, + msg.msg.value, None, ExitCode::OK, ); diff --git a/subnet-actor/src/lib.rs b/subnet-actor/src/lib.rs index 4aa1aff..3f47702 100644 --- a/subnet-actor/src/lib.rs +++ b/subnet-actor/src/lib.rs @@ -275,7 +275,7 @@ impl SubnetActor for Actor { let submission_epoch = ch.epoch(); let some_checkpoint = st - .epoch_checkpoint_voting + .bottomup_checkpoint_voting .submit_vote( rt.store(), ch, @@ -292,7 +292,7 @@ impl SubnetActor for Actor { if let Some(ch) = some_checkpoint { commit_checkpoint(st, store, &ch) } else if let Some(ch) = st - .epoch_checkpoint_voting + .bottomup_checkpoint_voting .get_next_executable_vote(store) .map_err(|_| actor_error!(illegal_state, "cannot check previous checkpoint"))? { @@ -409,7 +409,7 @@ fn commit_checkpoint( } }; - st.epoch_checkpoint_voting + st.bottomup_checkpoint_voting .mark_epoch_executed(store, ch.epoch()) .map_err(|e| { log::error!("encountered error marking epoch executed: {:?}", e); diff --git a/subnet-actor/src/state.rs b/subnet-actor/src/state.rs index b96b9f0..b4ce368 100644 --- a/subnet-actor/src/state.rs +++ b/subnet-actor/src/state.rs @@ -58,7 +58,7 @@ pub struct State { pub validator_set: ValidatorSet, pub min_validators: u64, pub previous_executed_checkpoint_cid: Cid, - pub epoch_checkpoint_voting: Voting, + pub bottomup_checkpoint_voting: Voting, } /// We should probably have a derive macro to mark an object as a state object, @@ -102,7 +102,7 @@ impl State { validator_set: ValidatorSet::default(), genesis_epoch: current_epoch, previous_executed_checkpoint_cid: *CHECKPOINT_GENESIS_CID, - epoch_checkpoint_voting: Voting::::new_with_ratio( + bottomup_checkpoint_voting: Voting::::new_with_ratio( store, current_epoch, bottomup_check_period, @@ -283,7 +283,7 @@ impl State { // the checkpoints are chained. This is an early termination check to ensure the checkpoints // are actually chained. if self - .epoch_checkpoint_voting + .bottomup_checkpoint_voting .is_next_executable_epoch(ch.epoch()) && self.previous_executed_checkpoint_cid != ch.prev_check().cid() { @@ -315,7 +315,7 @@ impl State { ) -> anyhow::Result { Ok( if self.previous_executed_checkpoint_cid != ch.prev_check().cid() { - self.epoch_checkpoint_voting + self.bottomup_checkpoint_voting .abort_epoch(store, ch.data.epoch)?; false } else { @@ -357,7 +357,7 @@ impl Default for State { min_validators: 0, genesis_epoch: 0, previous_executed_checkpoint_cid: *CHECKPOINT_GENESIS_CID, - epoch_checkpoint_voting: Voting { + bottomup_checkpoint_voting: Voting { genesis_epoch: 0, submission_period: 0, last_voting_executed_epoch: 0, diff --git a/subnet-actor/tests/actor_test.rs b/subnet-actor/tests/actor_test.rs index 987771f..51b188c 100644 --- a/subnet-actor/tests/actor_test.rs +++ b/subnet-actor/tests/actor_test.rs @@ -707,7 +707,7 @@ mod test { let root_subnet = SubnetID::from_str("/root").unwrap(); let subnet = SubnetID::new_from_parent(&root_subnet, test_actor_address); // we are targeting the next submission period - let epoch = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period; + let epoch = DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period; let mut checkpoint_0 = BottomUpCheckpoint::new(subnet.clone(), epoch); checkpoint_0.set_signature( RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) @@ -781,7 +781,7 @@ mod test { ); // Start the voting for a new epoch, checking we can proceed with new epoch number. - let epoch = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2; + let epoch = DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2; let prev_cid = checkpoint_0.cid(); let mut checkpoint_4 = BottomUpCheckpoint::new(subnet.clone(), epoch); checkpoint_4.data.prev_check = TCid::from(prev_cid); @@ -858,7 +858,7 @@ mod test { let root_subnet = SubnetID::from_str("/root").unwrap(); let subnet = SubnetID::new_from_parent(&root_subnet, test_actor_address); // we are targeting the next submission period - let epoch = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period; + let epoch = DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period; let mut checkpoint_0 = BottomUpCheckpoint::new(subnet.clone(), epoch); checkpoint_0.data.prev_check = TCid::default(); checkpoint_0.set_signature( @@ -888,26 +888,26 @@ mod test { /// Test flows like the below: /// 1. Create 4 validators and register them to the subnet with equal weight /// - /// 2. Submit to epoch `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2`, we are skipping + /// 2. Submit to epoch `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2`, we are skipping /// the first epoch and ensure this is executable. The previous checkpoint cid is set to some value `cid_a`. /// We should see the epoch number being stored in the next executable queue. /// Checks at step 2: - /// - executable_queue should contain `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2` + /// - executable_queue should contain `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` /// - last_executed_epoch is still `DEFAULT_CHAIN_EPOCH` /// - /// 3. Submit to epoch `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 1`, i.e. the previous + /// 3. Submit to epoch `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1`, i.e. the previous /// epoch in step 2. This would lead to the epoch being committed. The key is the checkpoint cid of the current /// epoch should be different from that in step 2, i.e. any value other than `cid_a` /// Checks at step 3: - /// - executable_queue should contain `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2` - /// - last_executed_epoch is still `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 1` + /// - executable_queue should contain `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` + /// - last_executed_epoch is still `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1` /// - /// 4. Submit to any epoch after `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 1`, should - /// trigger a reset in submission of epoch `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2`. + /// 4. Submit to any epoch after `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1`, should + /// trigger a reset in submission of epoch `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2`. /// Checks at step 4: - /// - executable_queue should have removed `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2` - /// - last_executed_epoch is still `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 1` - /// - submission at `DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2` is cleared + /// - executable_queue should have removed `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` + /// - last_executed_epoch is still `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1` + /// - submission at `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` is cleared #[test] fn test_submit_checkpoint_aborts_not_chained_reset_epoch() { let test_actor_address = Address::new_id(9999); @@ -969,7 +969,7 @@ mod test { // Step 2 let st: State = runtime.get_state(); - let epoch_2 = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2; + let epoch_2 = DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2; let prev_cid = Cid::default(); let mut checkpoint_2 = BottomUpCheckpoint::new(subnet.clone(), epoch_2); checkpoint_2.data.prev_check = TCid::from(prev_cid); @@ -991,28 +991,28 @@ mod test { CHECKPOINT_GENESIS_CID.clone() ); assert_eq!( - st.epoch_checkpoint_voting.last_voting_executed_epoch, + st.bottomup_checkpoint_voting.last_voting_executed_epoch, DEFAULT_CHAIN_EPOCH ); assert_eq!( - st.epoch_checkpoint_voting.executable_epoch_queue, + st.bottomup_checkpoint_voting.executable_epoch_queue, Some(BTreeSet::from([epoch_2])) ); assert_eq!( - st.epoch_checkpoint_voting + st.bottomup_checkpoint_voting .load_most_voted_submission(runtime.store(), epoch_2) .unwrap(), Some(checkpoint_2.clone()) ); assert_eq!( - st.epoch_checkpoint_voting + st.bottomup_checkpoint_voting .load_most_voted_weight(runtime.store(), epoch_2) .unwrap(), Some(TokenAmount::from_whole(3)) ); // Step 3 - let epoch_1 = DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 1; + let epoch_1 = DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1; let mut checkpoint_1 = BottomUpCheckpoint::new(subnet.clone(), epoch_1); checkpoint_1.set_signature( RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) @@ -1029,23 +1029,23 @@ mod test { let st: State = runtime.get_state(); assert_eq!(st.previous_executed_checkpoint_cid, checkpoint_1.cid()); assert_eq!( - st.epoch_checkpoint_voting.last_voting_executed_epoch, + st.bottomup_checkpoint_voting.last_voting_executed_epoch, epoch_1 ); assert_eq!( - st.epoch_checkpoint_voting.executable_epoch_queue, + st.bottomup_checkpoint_voting.executable_epoch_queue, Some(BTreeSet::from([ - DEFAULT_CHAIN_EPOCH + st.epoch_checkpoint_voting.submission_period * 2 + DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2 ])) ); assert_eq!( - st.epoch_checkpoint_voting + st.bottomup_checkpoint_voting .load_most_voted_weight(runtime.store(), epoch_2) .unwrap(), Some(TokenAmount::from_whole(3)) ); assert_eq!( - st.epoch_checkpoint_voting + st.bottomup_checkpoint_voting .load_most_voted_weight(runtime.store(), epoch_1) .unwrap(), None @@ -1059,10 +1059,10 @@ mod test { let st: State = runtime.get_state(); assert_eq!(st.previous_executed_checkpoint_cid, checkpoint_1.cid()); assert_eq!( - st.epoch_checkpoint_voting.last_voting_executed_epoch, + st.bottomup_checkpoint_voting.last_voting_executed_epoch, epoch_1 ); - assert_eq!(st.epoch_checkpoint_voting.executable_epoch_queue, None); + assert_eq!(st.bottomup_checkpoint_voting.executable_epoch_queue, None); } fn send_checkpoint( From 75455bfbb317b69658ee1653886f829bff0198d5 Mon Sep 17 00:00:00 2001 From: cryptoAtwill <108330426+cryptoAtwill@users.noreply.github.com> Date: Thu, 6 Apr 2023 18:30:39 +0800 Subject: [PATCH 24/27] fix checkpoint (#85) --- gateway/src/checkpoint.rs | 59 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/gateway/src/checkpoint.rs b/gateway/src/checkpoint.rs index 2496c09..b37160e 100644 --- a/gateway/src/checkpoint.rs +++ b/gateway/src/checkpoint.rs @@ -14,6 +14,7 @@ use ipc_sdk::ValidatorSet; use lazy_static::lazy_static; use num_traits::Zero; use primitives::{TCid, TLink}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; lazy_static! { @@ -168,7 +169,7 @@ pub struct CheckData { pub cross_msgs: BatchCrossMsgs, } -#[derive(Default, PartialEq, Eq, Clone, Debug, Serialize_tuple, Deserialize_tuple)] +#[derive(Default, PartialEq, Eq, Clone, Debug)] pub struct BatchCrossMsgs { pub cross_msgs: Option>, pub fee: TokenAmount, @@ -273,3 +274,59 @@ impl UniqueVote for TopDownCheckpoint { Ok(mh_code.digest(&to_vec(self).unwrap()).to_bytes()) } } + +impl Serialize for BatchCrossMsgs { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + if let Some(v) = self.cross_msgs.as_ref() { + let inner = (v, &self.fee); + serde::Serialize::serialize(&inner, serde_tuple::Serializer(serializer)) + } else { + let inner: (&Vec, &TokenAmount) = (&vec![], &self.fee); + serde::Serialize::serialize(&inner, serde_tuple::Serializer(serializer)) + } + } +} + +impl<'de> Deserialize<'de> for BatchCrossMsgs { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + type Inner = (Vec, TokenAmount); + let inner = Inner::deserialize(serde_tuple::Deserializer(deserializer))?; + Ok(BatchCrossMsgs { + cross_msgs: if inner.0.is_empty() { + None + } else { + Some(inner.0) + }, + fee: inner.1, + }) + } +} + +#[cfg(test)] +mod tests { + use crate::BottomUpCheckpoint; + use cid::Cid; + use fil_actors_runtime::cbor; + use ipc_sdk::subnet_id::SubnetID; + use primitives::TCid; + use std::str::FromStr; + + #[test] + fn test_serialization() { + let mut checkpoint = BottomUpCheckpoint::new(SubnetID::from_str("/root").unwrap(), 10); + checkpoint.data.prev_check = TCid::from( + Cid::from_str("bafy2bzacecnamqgqmifpluoeldx7zzglxcljo6oja4vrmtj7432rphldpdmm2") + .unwrap(), + ); + + let raw_bytes = cbor::serialize(&checkpoint, "").unwrap(); + let de = cbor::deserialize(&raw_bytes, "").unwrap(); + assert_eq!(checkpoint, de); + } +} From de180598fea6378fd2e9e129bc7199c8aa3d1ad6 Mon Sep 17 00:00:00 2001 From: cryptoAtwill <108330426+cryptoAtwill@users.noreply.github.com> Date: Mon, 10 Apr 2023 15:14:21 +0800 Subject: [PATCH 25/27] update queue serialization (#86) * update queue serialization * remove println * fix fmt --- common/Cargo.toml | 3 ++- common/src/vote/voting.rs | 45 ++++++++++++++++++++++++++++----------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/common/Cargo.toml b/common/Cargo.toml index aaee9f6..d5cb403 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -23,4 +23,5 @@ fil_actors_runtime = { git = "https://github.com/consensus-shipyard/fvm-utils", integer-encoding = { version = "3.0.3", default-features = false } [dev-dependencies] -serde_json = "1.0.95" \ No newline at end of file +serde_json = "1.0.95" +cid = "0.8.6" \ No newline at end of file diff --git a/common/src/vote/voting.rs b/common/src/vote/voting.rs index 2548c1d..83c3c6e 100644 --- a/common/src/vote/voting.rs +++ b/common/src/vote/voting.rs @@ -335,15 +335,27 @@ impl Serialize for Voting { where S: Serializer, { - let inner = ( - &self.genesis_epoch, - &self.submission_period, - &self.last_voting_executed_epoch, - &self.executable_epoch_queue, - &self.epoch_vote_submissions, - &self.threshold_ratio, - ); - inner.serialize(serde_tuple::Serializer(serializer)) + if let Some(queue) = &self.executable_epoch_queue { + let inner = ( + &self.genesis_epoch, + &self.submission_period, + &self.last_voting_executed_epoch, + &queue.iter().collect::>(), + &self.epoch_vote_submissions, + &self.threshold_ratio, + ); + inner.serialize(serde_tuple::Serializer(serializer)) + } else { + let inner: (_, _, _, &Vec, _, _) = ( + &self.genesis_epoch, + &self.submission_period, + &self.last_voting_executed_epoch, + &vec![], + &self.epoch_vote_submissions, + &self.threshold_ratio, + ); + inner.serialize(serde_tuple::Serializer(serializer)) + } } } @@ -356,16 +368,23 @@ impl<'de, T: DeserializeOwned> Deserialize<'de> for Voting { ChainEpoch, ChainEpoch, ChainEpoch, - Option>, + Vec, TCid>>, Ratio, ); let inner = >::deserialize(serde_tuple::Deserializer(deserializer))?; + + let queue = if inner.3.is_empty() { + None + } else { + let set = inner.3.into_iter().collect::>(); + Some(set) + }; Ok(Voting { genesis_epoch: inner.0, submission_period: inner.1, last_voting_executed_epoch: inner.2, - executable_epoch_queue: inner.3, + executable_epoch_queue: queue, epoch_vote_submissions: inner.4, threshold_ratio: inner.5, }) @@ -376,14 +395,16 @@ impl<'de, T: DeserializeOwned> Deserialize<'de> for Voting { mod tests { use crate::vote::submission::Ratio; use crate::vote::{EpochVoteSubmissions, UniqueBytesKey, UniqueVote, Voting}; + use cid::Cid; use fil_actors_runtime::builtin::HAMT_BIT_WIDTH; use fil_actors_runtime::fvm_ipld_hamt::BytesKey; - use fil_actors_runtime::make_empty_map; + use fil_actors_runtime::{cbor, make_empty_map}; use fvm_ipld_blockstore::MemoryBlockstore; use fvm_shared::clock::ChainEpoch; use primitives::{TCid, THamt}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; use std::collections::BTreeSet; + use std::str::FromStr; #[derive(PartialEq, Clone, Deserialize_tuple, Serialize_tuple, Debug)] struct DummyVote { From 54a62bb0ee632fc5810b24c878f3a09ae18a9789 Mon Sep 17 00:00:00 2001 From: Alfonso de la Rocha Date: Mon, 10 Apr 2023 12:47:14 +0200 Subject: [PATCH 26/27] genesis_epoch bottomup checkpoints to zero --- subnet-actor/src/state.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/subnet-actor/src/state.rs b/subnet-actor/src/state.rs index b4ce368..d6a3f91 100644 --- a/subnet-actor/src/state.rs +++ b/subnet-actor/src/state.rs @@ -100,11 +100,18 @@ impl State { status: Status::Instantiated, stake: TCid::new_hamt(store)?, validator_set: ValidatorSet::default(), + // genesis epoch determines the epoch from the parent when the + // subnet was spawned. genesis_epoch: current_epoch, previous_executed_checkpoint_cid: *CHECKPOINT_GENESIS_CID, bottomup_checkpoint_voting: Voting::::new_with_ratio( store, - current_epoch, + // NOTE: we currently use 0 as the genesis_epoch for subnets. We want + // checkpoints to be committed from genesis. In the future, we may want + // to make the logic a bit more complex and only start committing checkpoints + // from a specific epoch. This may be the case when an existing subnet + // docks to a parent. + 0, bottomup_check_period, 2, 3, From 21e794c4ba2693e7f7b81063af2d27cd37538155 Mon Sep 17 00:00:00 2001 From: Alfonso de la Rocha Date: Mon, 10 Apr 2023 13:25:51 +0200 Subject: [PATCH 27/27] fix test with new genesis checkpoint --- common/src/vote/voting.rs | 4 +-- subnet-actor/src/lib.rs | 8 +++++- subnet-actor/src/state.rs | 7 +----- subnet-actor/tests/actor_test.rs | 42 ++++++++++++++++---------------- 4 files changed, 30 insertions(+), 31 deletions(-) diff --git a/common/src/vote/voting.rs b/common/src/vote/voting.rs index 83c3c6e..a8b7fa7 100644 --- a/common/src/vote/voting.rs +++ b/common/src/vote/voting.rs @@ -395,16 +395,14 @@ impl<'de, T: DeserializeOwned> Deserialize<'de> for Voting { mod tests { use crate::vote::submission::Ratio; use crate::vote::{EpochVoteSubmissions, UniqueBytesKey, UniqueVote, Voting}; - use cid::Cid; use fil_actors_runtime::builtin::HAMT_BIT_WIDTH; use fil_actors_runtime::fvm_ipld_hamt::BytesKey; - use fil_actors_runtime::{cbor, make_empty_map}; + use fil_actors_runtime::make_empty_map; use fvm_ipld_blockstore::MemoryBlockstore; use fvm_shared::clock::ChainEpoch; use primitives::{TCid, THamt}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; use std::collections::BTreeSet; - use std::str::FromStr; #[derive(PartialEq, Clone, Deserialize_tuple, Serialize_tuple, Debug)] struct DummyVote { diff --git a/subnet-actor/src/lib.rs b/subnet-actor/src/lib.rs index 3f47702..4f854a5 100644 --- a/subnet-actor/src/lib.rs +++ b/subnet-actor/src/lib.rs @@ -81,7 +81,13 @@ impl SubnetActor for Actor { fn constructor(rt: &mut impl Runtime, params: ConstructParams) -> Result<(), ActorError> { rt.validate_immediate_caller_is(std::iter::once(&INIT_ACTOR_ADDR))?; - let st = State::new(rt.store(), params, rt.curr_epoch()).map_err(|e| { + // NOTE: we currently use 0 as the genesis_epoch for subnets so checkpoints + // are submitted directly from epoch 0. + // In the future we can use the current epoch. This will be really + // useful once we support the docking of subnets to new parents, etc. + // let genesis_epoch = rt.curr_epoch(); + let genesis_epoch = 0; + let st = State::new(rt.store(), params, genesis_epoch).map_err(|e| { e.downcast_default(ExitCode::USR_ILLEGAL_STATE, "Failed to create actor state") })?; diff --git a/subnet-actor/src/state.rs b/subnet-actor/src/state.rs index d6a3f91..3c729dc 100644 --- a/subnet-actor/src/state.rs +++ b/subnet-actor/src/state.rs @@ -106,12 +106,7 @@ impl State { previous_executed_checkpoint_cid: *CHECKPOINT_GENESIS_CID, bottomup_checkpoint_voting: Voting::::new_with_ratio( store, - // NOTE: we currently use 0 as the genesis_epoch for subnets. We want - // checkpoints to be committed from genesis. In the future, we may want - // to make the logic a bit more complex and only start committing checkpoints - // from a specific epoch. This may be the case when an existing subnet - // docks to a parent. - 0, + current_epoch, bottomup_check_period, 2, 3, diff --git a/subnet-actor/tests/actor_test.rs b/subnet-actor/tests/actor_test.rs index 51b188c..810ee01 100644 --- a/subnet-actor/tests/actor_test.rs +++ b/subnet-actor/tests/actor_test.rs @@ -32,7 +32,7 @@ mod test { // just a test address const IPC_GATEWAY_ADDR: u64 = 1024; const NETWORK_NAME: &'static str = "test"; - const DEFAULT_CHAIN_EPOCH: ChainEpoch = 10; + const DEFAULT_GENESIS_EPOCH: ChainEpoch = 0; lazy_static! { pub static ref SIG_TYPES: Vec = vec![*ACCOUNT_ACTOR_CODE_ID, *MULTISIG_ACTOR_CODE_ID]; @@ -68,7 +68,7 @@ mod test { runtime.expect_validate_caller_addr(vec![INIT_ACTOR_ADDR]); - runtime.set_epoch(DEFAULT_CHAIN_EPOCH); + runtime.set_epoch(DEFAULT_GENESIS_EPOCH); runtime .call::( @@ -95,7 +95,7 @@ mod test { assert_eq!(state.ipc_gateway_addr, Address::new_id(IPC_GATEWAY_ADDR)); assert_eq!(state.total_stake, TokenAmount::zero()); assert_eq!(state.validator_set.validators().is_empty(), true); - assert_eq!(state.genesis_epoch, DEFAULT_CHAIN_EPOCH); + assert_eq!(state.genesis_epoch, DEFAULT_GENESIS_EPOCH); } #[test] @@ -707,7 +707,7 @@ mod test { let root_subnet = SubnetID::from_str("/root").unwrap(); let subnet = SubnetID::new_from_parent(&root_subnet, test_actor_address); // we are targeting the next submission period - let epoch = DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period; + let epoch = DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period; let mut checkpoint_0 = BottomUpCheckpoint::new(subnet.clone(), epoch); checkpoint_0.set_signature( RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) @@ -781,7 +781,7 @@ mod test { ); // Start the voting for a new epoch, checking we can proceed with new epoch number. - let epoch = DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2; + let epoch = DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2; let prev_cid = checkpoint_0.cid(); let mut checkpoint_4 = BottomUpCheckpoint::new(subnet.clone(), epoch); checkpoint_4.data.prev_check = TCid::from(prev_cid); @@ -858,7 +858,7 @@ mod test { let root_subnet = SubnetID::from_str("/root").unwrap(); let subnet = SubnetID::new_from_parent(&root_subnet, test_actor_address); // we are targeting the next submission period - let epoch = DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period; + let epoch = DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period; let mut checkpoint_0 = BottomUpCheckpoint::new(subnet.clone(), epoch); checkpoint_0.data.prev_check = TCid::default(); checkpoint_0.set_signature( @@ -888,26 +888,26 @@ mod test { /// Test flows like the below: /// 1. Create 4 validators and register them to the subnet with equal weight /// - /// 2. Submit to epoch `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2`, we are skipping + /// 2. Submit to epoch `DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2`, we are skipping /// the first epoch and ensure this is executable. The previous checkpoint cid is set to some value `cid_a`. /// We should see the epoch number being stored in the next executable queue. /// Checks at step 2: - /// - executable_queue should contain `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` - /// - last_executed_epoch is still `DEFAULT_CHAIN_EPOCH` + /// - executable_queue should contain `DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` + /// - last_executed_epoch is still `DEFAULT_GENESIS_EPOCH` /// - /// 3. Submit to epoch `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1`, i.e. the previous + /// 3. Submit to epoch `DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1`, i.e. the previous /// epoch in step 2. This would lead to the epoch being committed. The key is the checkpoint cid of the current /// epoch should be different from that in step 2, i.e. any value other than `cid_a` /// Checks at step 3: - /// - executable_queue should contain `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` - /// - last_executed_epoch is still `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1` + /// - executable_queue should contain `DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` + /// - last_executed_epoch is still `DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1` /// - /// 4. Submit to any epoch after `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1`, should - /// trigger a reset in submission of epoch `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2`. + /// 4. Submit to any epoch after `DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1`, should + /// trigger a reset in submission of epoch `DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2`. /// Checks at step 4: - /// - executable_queue should have removed `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` - /// - last_executed_epoch is still `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1` - /// - submission at `DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` is cleared + /// - executable_queue should have removed `DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` + /// - last_executed_epoch is still `DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1` + /// - submission at `DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2` is cleared #[test] fn test_submit_checkpoint_aborts_not_chained_reset_epoch() { let test_actor_address = Address::new_id(9999); @@ -969,7 +969,7 @@ mod test { // Step 2 let st: State = runtime.get_state(); - let epoch_2 = DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2; + let epoch_2 = DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2; let prev_cid = Cid::default(); let mut checkpoint_2 = BottomUpCheckpoint::new(subnet.clone(), epoch_2); checkpoint_2.data.prev_check = TCid::from(prev_cid); @@ -992,7 +992,7 @@ mod test { ); assert_eq!( st.bottomup_checkpoint_voting.last_voting_executed_epoch, - DEFAULT_CHAIN_EPOCH + DEFAULT_GENESIS_EPOCH ); assert_eq!( st.bottomup_checkpoint_voting.executable_epoch_queue, @@ -1012,7 +1012,7 @@ mod test { ); // Step 3 - let epoch_1 = DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1; + let epoch_1 = DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 1; let mut checkpoint_1 = BottomUpCheckpoint::new(subnet.clone(), epoch_1); checkpoint_1.set_signature( RawBytes::serialize(Signature::new_secp256k1(vec![1, 2, 3, 4])) @@ -1035,7 +1035,7 @@ mod test { assert_eq!( st.bottomup_checkpoint_voting.executable_epoch_queue, Some(BTreeSet::from([ - DEFAULT_CHAIN_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2 + DEFAULT_GENESIS_EPOCH + st.bottomup_checkpoint_voting.submission_period * 2 ])) ); assert_eq!(