From 653a699a96bbfbfc168c099fca2f68f60ac15f6e Mon Sep 17 00:00:00 2001 From: simzzz Date: Wed, 8 Apr 2020 20:45:27 +0300 Subject: [PATCH 01/17] isDateRecent() + tests + broken isInitializing() --- src/status.rs | 74 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/src/status.rs b/src/status.rs index 26425ab..9ee2fcb 100644 --- a/src/status.rs +++ b/src/status.rs @@ -2,8 +2,8 @@ use crate::sentry_api::SentryApi; use chrono::{DateTime, Duration, Utc}; use primitives::{ market::Campaign, - sentry::{HeartbeatValidatorMessage, LastApproved, LastApprovedResponse}, - validator::{Heartbeat, MessageTypes}, + sentry::{HeartbeatValidatorMessage, LastApproved, LastApprovedResponse }, + validator::{Heartbeat, MessageTypes, NewState, ApproveState}, BalancesMap, BigNum, }; use reqwest::Error; @@ -35,6 +35,8 @@ struct Messages { last_approved: Option, leader_heartbeats: Vec, follower_heartbeats: Vec, + leader_new_state: Vec, + follower_approve_state: Vec, } pub enum IsFinalized { @@ -124,10 +126,12 @@ pub async fn get_status(sentry: &SentryApi, campaign: &Campaign) -> Result>) -> Vec bool { - todo!() +fn get_new_state(last_approved: Option) -> Vec { + + match last_approved { + Some(last_approved) => last_approved.new_state + .into_iter() + .filter_map(|new_state| match new_state.msg { + MessageTypes::NewState(new_state) => Some(new_state), + _ => None, + }) + .collect(), + None => Default::default(), + } +} + +fn get_approve_state(last_approved: Option) -> Vec { + match last_approved { + Some(last_approved) => last_approved.approve_state + .into_iter() + .filter_map(|approve_state| match approve_state.msg { + MessageTypes::ApproveState(approve_state) => Some(approve_state), + _ => None, + }) + .collect(), + None => Default::default(), + } +} + +fn is_initializing(leader: &[Heartbeat], follower: &[Heartbeat], new_state: &[NewState], approve_state: &[ApproveState]) -> bool { + (leader.len() as i32 == 0 && new_state.len() as i32 == 0) || (follower.len() as i32 == 0 && approve_state.len() as i32 == 0) } // at least one validator doesn't have a recent Heartbeat message @@ -209,7 +240,8 @@ fn is_offline(leader: &[Heartbeat], follower: &[Heartbeat]) -> bool { } fn is_date_recent(recency: &Duration, date: &DateTime) -> bool { - todo!() + let time_passed = Utc::now().signed_duration_since(date.clone()); + time_passed <= recency.clone() } fn is_disconnected() -> bool { @@ -244,5 +276,35 @@ mod test { ) } + #[test] + fn now_date_is_recent() { + let now = Utc::now(); + let recency = Duration::minutes(4); + assert!( + is_date_recent(&recency, &now), + "The present moment is a recent date!" + ) + } + + #[test] + fn slightly_past_is_recent() { + let recency = Duration::minutes(4); + let on_the_edge = Utc::now().checked_sub_signed(Duration::minutes(3)).unwrap(); + assert!( + is_date_recent(&recency, &on_the_edge), + "When date is just as old as the recency limit, it still counts as recent" + ) + } + + #[test] + fn old_date_is_not_recent() { + let recency = Duration::minutes(4); + let past = Utc::now().checked_sub_signed(Duration::minutes(10)).unwrap(); + assert_eq!( + is_date_recent(&recency, &past), + false, + "Date older than the recency limit is not recent" + ) + } // @TODO: test is_offline() } From 7fad9c9ec1137ca9dbe6d7645be978378e77ad9d Mon Sep 17 00:00:00 2001 From: simzzz Date: Thu, 9 Apr 2020 19:10:38 +0300 Subject: [PATCH 02/17] Progress with is_disconnected() --- src/status.rs | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/status.rs b/src/status.rs index 9ee2fcb..4954cc1 100644 --- a/src/status.rs +++ b/src/status.rs @@ -4,7 +4,7 @@ use primitives::{ market::Campaign, sentry::{HeartbeatValidatorMessage, LastApproved, LastApprovedResponse }, validator::{Heartbeat, MessageTypes, NewState, ApproveState}, - BalancesMap, BigNum, + BalancesMap, BigNum, ValidatorDesc }; use reqwest::Error; @@ -113,6 +113,7 @@ pub async fn is_finalized(sentry: &SentryApi, campaign: &Campaign) -> Result Result { // continue only if Campaign is not Finalized + let leader = campaign.channel.spec.validators.leader(); let leader_la = match is_finalized(sentry, campaign).await? { IsFinalized::Yes { reason, balances } => return Ok(Status::Finalized(reason, balances)), IsFinalized::No { leader } => leader, @@ -139,7 +140,7 @@ pub async fn get_status(sentry: &SentryApi, campaign: &Campaign) -> Result>) -> Vec) -> Vec { +fn hb_by_validator(validator: &ValidatorDesc, heartbeat: &Heartbeat) -> bool { + heartbeat.from == validator.id +} +fn get_new_state(last_approved: Option) -> Vec { match last_approved { Some(last_approved) => last_approved.new_state .into_iter() @@ -222,6 +226,7 @@ fn get_approve_state(last_approved: Option) -> Vec { } } +// there are no messages at all for at least one validator fn is_initializing(leader: &[Heartbeat], follower: &[Heartbeat], new_state: &[NewState], approve_state: &[ApproveState]) -> bool { (leader.len() as i32 == 0 && new_state.len() as i32 == 0) || (follower.len() as i32 == 0 && approve_state.len() as i32 == 0) } @@ -244,8 +249,31 @@ fn is_date_recent(recency: &Duration, date: &DateTime) -> bool { time_passed <= recency.clone() } -fn is_disconnected() -> bool { - todo!() +// validators have recent Heartbeat messages, but they don't seem to be propagating messages between one another (the majority of Heartbeats are not found on both validators) +fn is_disconnected(leader_hb: &[Heartbeat], follower_hb: &[Heartbeat], leader_validator: &ValidatorDesc) -> bool { + let recency = Duration::minutes(4); + + let follower_recent: Vec<&Heartbeat> = follower_hb + .iter() + .filter_map(|h| { + if is_date_recent(&recency, &h.timestamp) { + Some(h) + } else { + None + }}) + .collect(); + let follower_hb_from_leader_val: Vec<&Heartbeat> = leader_hb + .iter() + .filter_map(|h| { + if is_date_recent(&recency, &h.timestamp) && hb_by_validator(leader_validator, h) { + Some(h) + } else { + None + } + }) + .collect(); + + follower_recent.len() > 0 && follower_hb_from_leader_val.len() > 0 } fn is_rejected_state() -> bool { From 763ddd97daa6e16318ca41bb131d4f446b58e7d0 Mon Sep 17 00:00:00 2001 From: simzzz Date: Fri, 10 Apr 2020 13:51:33 +0300 Subject: [PATCH 03/17] disconnected function ready --- src/status.rs | 55 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/src/status.rs b/src/status.rs index 4954cc1..4de374b 100644 --- a/src/status.rs +++ b/src/status.rs @@ -37,6 +37,7 @@ struct Messages { follower_heartbeats: Vec, leader_new_state: Vec, follower_approve_state: Vec, + follower_hb_from_leader: Vec, } pub enum IsFinalized { @@ -129,6 +130,7 @@ pub async fn get_status(sentry: &SentryApi, campaign: &Campaign) -> Result Result>) -> Vec bool { +fn is_hb_by_validator(validator: &ValidatorDesc, heartbeat: &HeartbeatValidatorMessage) -> bool { heartbeat.from == validator.id } @@ -226,6 +228,25 @@ fn get_approve_state(last_approved: Option) -> Vec { } } +fn get_hb_by_validator(validator: &ValidatorDesc, heartbeats: Option>) -> Vec { + match heartbeats { + Some(heartbeats) => heartbeats + .into_iter() + .filter_map(|h| match h.msg { + MessageTypes::Heartbeat(h_msg) => { + if is_hb_by_validator(&validator, &h) { + Some(h_msg) + } else { + None + } + }, + _ => None, + }) + .collect(), + None => Default::default(), + } +} + // there are no messages at all for at least one validator fn is_initializing(leader: &[Heartbeat], follower: &[Heartbeat], new_state: &[NewState], approve_state: &[ApproveState]) -> bool { (leader.len() as i32 == 0 && new_state.len() as i32 == 0) || (follower.len() as i32 == 0 && approve_state.len() as i32 == 0) @@ -250,30 +271,32 @@ fn is_date_recent(recency: &Duration, date: &DateTime) -> bool { } // validators have recent Heartbeat messages, but they don't seem to be propagating messages between one another (the majority of Heartbeats are not found on both validators) -fn is_disconnected(leader_hb: &[Heartbeat], follower_hb: &[Heartbeat], leader_validator: &ValidatorDesc) -> bool { +fn is_disconnected(leader_hb: &[Heartbeat], follower_hb: &[Heartbeat], follower_hb_from_leader: &[Heartbeat]) -> bool { let recency = Duration::minutes(4); - let follower_recent: Vec<&Heartbeat> = follower_hb - .iter() + let follower_hb = follower_hb + .into_iter() .filter_map(|h| { if is_date_recent(&recency, &h.timestamp) { Some(h) } else { None - }}) - .collect(); - let follower_hb_from_leader_val: Vec<&Heartbeat> = leader_hb - .iter() + } + }) + .count(); + + let follower_hb_from_leader = follower_hb_from_leader + .into_iter() .filter_map(|h| { - if is_date_recent(&recency, &h.timestamp) && hb_by_validator(leader_validator, h) { - Some(h) - } else { - None - } + if is_date_recent(&recency, &h.timestamp) { + Some(h) + } else { + None + } }) - .collect(); + .count(); - follower_recent.len() > 0 && follower_hb_from_leader_val.len() > 0 + follower_hb > 0 && follower_hb_from_leader > 0 } fn is_rejected_state() -> bool { From 0cd05394c5c05a1c41fa708d27ae7f02a0cb67c1 Mon Sep 17 00:00:00 2001 From: simzzz Date: Fri, 10 Apr 2020 14:58:12 +0300 Subject: [PATCH 04/17] code is messy but it builds --- src/status.rs | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/status.rs b/src/status.rs index 4de374b..b89c616 100644 --- a/src/status.rs +++ b/src/status.rs @@ -119,18 +119,23 @@ pub async fn get_status(sentry: &SentryApi, campaign: &Campaign) -> Result return Ok(Status::Finalized(reason, balances)), IsFinalized::No { leader } => leader, }; + let leader_la_clone = match is_finalized(sentry, campaign).await? { + IsFinalized::Yes { reason, balances } => return Ok(Status::Finalized(reason, balances)), + IsFinalized::No { leader } => leader, + }; let follower = campaign.channel.spec.validators.follower(); let follower_la = sentry.get_last_approved(&follower).await?; + let follower_la_clone = sentry.get_last_approved(&follower).await?; // setup the messages for the checks let messages = Messages { last_approved: leader_la.last_approved, leader_heartbeats: get_heartbeats(leader_la.heartbeats), follower_heartbeats: get_heartbeats(follower_la.heartbeats), - leader_new_state: get_new_state(leader_la.last_approved), + leader_new_state: get_new_state(leader_la_clone.last_approved), follower_approve_state: get_approve_state(follower_la.last_approved), - follower_hb_from_leader: get_hb_by_validator(&leader, follower_la.heartbeats), + follower_hb_from_leader: get_hb_by_validator(&leader, follower_la_clone.heartbeats), }; // impl: isInitializing @@ -142,7 +147,7 @@ pub async fn get_status(sentry: &SentryApi, campaign: &Campaign) -> Result heartbeats .into_iter() - .filter_map(|h| match h.msg { - MessageTypes::Heartbeat(h_msg) => { - if is_hb_by_validator(&validator, &h) { - Some(h_msg) - } else { - None + .filter_map(|h| { + if is_hb_by_validator(&validator, &h) { + match h.msg { + MessageTypes::Heartbeat(h) => Some(h), + _ => None, } - }, - _ => None, + } else { + None + } }) .collect(), None => Default::default(), @@ -271,7 +276,7 @@ fn is_date_recent(recency: &Duration, date: &DateTime) -> bool { } // validators have recent Heartbeat messages, but they don't seem to be propagating messages between one another (the majority of Heartbeats are not found on both validators) -fn is_disconnected(leader_hb: &[Heartbeat], follower_hb: &[Heartbeat], follower_hb_from_leader: &[Heartbeat]) -> bool { +fn is_disconnected(follower_hb: &[Heartbeat], follower_hb_from_leader: &[Heartbeat]) -> bool { let recency = Duration::minutes(4); let follower_hb = follower_hb From 7afb8efe365e832e12749826be4b228d973c9183 Mon Sep 17 00:00:00 2001 From: simzzz Date: Fri, 10 Apr 2020 20:18:20 +0300 Subject: [PATCH 05/17] test templates for all cases --- src/status.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/status.rs b/src/status.rs index b89c616..79c6e54 100644 --- a/src/status.rs +++ b/src/status.rs @@ -332,6 +332,7 @@ mod test { ) } + // is_date_recent() #[test] fn now_date_is_recent() { let now = Utc::now(); @@ -362,5 +363,48 @@ mod test { "Date older than the recency limit is not recent" ) } + + // is_initializing() + #[test] + fn two_empty_message_arrays() { + todo!() + } + + #[test] + fn first_message_arr_is_empty() { + todo!() + } + + #[test] + fn second_message_arr_is_empty() { + todo!() + } + + #[test] + fn both_arrays_have_messages() { + todo!() + } + + // is_disconnected() + #[test] + fn no_recent_hbs_on_both_sides() { + todo!() + } + + #[test] + fn no_recent_follower_hbs() { + todo!() + } + + #[test] + fn no_recent_leader_hb_on_follower_validator() { + todo!() + } + + #[test] + fn recent_hbs_on_both_arrays() { + todo!() + } + // @TODO: test is_offline() } From 4f8483b2ff0f221cdb3020cfa52ea63fc202b1d6 Mon Sep 17 00:00:00 2001 From: simzzz Date: Mon, 13 Apr 2020 15:47:41 +0300 Subject: [PATCH 06/17] refactored, did requested changes in the code review --- src/status.rs | 82 ++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/src/status.rs b/src/status.rs index 79c6e54..01e5607 100644 --- a/src/status.rs +++ b/src/status.rs @@ -32,12 +32,10 @@ pub enum Finalized { } struct Messages { - last_approved: Option, + leader_last_approved: Option, + follower_last_approved: Option, leader_heartbeats: Vec, follower_heartbeats: Vec, - leader_new_state: Vec, - follower_approve_state: Vec, - follower_hb_from_leader: Vec, } pub enum IsFinalized { @@ -119,27 +117,20 @@ pub async fn get_status(sentry: &SentryApi, campaign: &Campaign) -> Result return Ok(Status::Finalized(reason, balances)), IsFinalized::No { leader } => leader, }; - let leader_la_clone = match is_finalized(sentry, campaign).await? { - IsFinalized::Yes { reason, balances } => return Ok(Status::Finalized(reason, balances)), - IsFinalized::No { leader } => leader, - }; let follower = campaign.channel.spec.validators.follower(); let follower_la = sentry.get_last_approved(&follower).await?; - let follower_la_clone = sentry.get_last_approved(&follower).await?; // setup the messages for the checks let messages = Messages { - last_approved: leader_la.last_approved, + leader_last_approved: leader_la.last_approved, + follower_last_approved: follower_la.last_approved, leader_heartbeats: get_heartbeats(leader_la.heartbeats), follower_heartbeats: get_heartbeats(follower_la.heartbeats), - leader_new_state: get_new_state(leader_la_clone.last_approved), - follower_approve_state: get_approve_state(follower_la.last_approved), - follower_hb_from_leader: get_hb_by_validator(&leader, follower_la_clone.heartbeats), }; // impl: isInitializing - if is_initializing(&messages.leader_heartbeats, &messages.follower_heartbeats, &messages.leader_new_state, &messages.follower_approve_state) { + if is_initializing(&messages.leader_heartbeats, &messages.follower_heartbeats, &messages.leader_last_approved, &messages.follower_last_approved) { return Ok(Status::Initializing); } @@ -147,7 +138,7 @@ pub async fn get_status(sentry: &SentryApi, campaign: &Campaign) -> Result) -> Vec { +fn get_new_state(last_approved: &Option) -> Vec { match last_approved { Some(last_approved) => last_approved.new_state .into_iter() @@ -220,7 +211,7 @@ fn get_new_state(last_approved: Option) -> Vec { } } -fn get_approve_state(last_approved: Option) -> Vec { +fn get_approve_state(last_approved: &Option) -> Vec { match last_approved { Some(last_approved) => last_approved.approve_state .into_iter() @@ -233,28 +224,25 @@ fn get_approve_state(last_approved: Option) -> Vec { } } -fn get_hb_by_validator(validator: &ValidatorDesc, heartbeats: Option>) -> Vec { - match heartbeats { - Some(heartbeats) => heartbeats - .into_iter() - .filter_map(|h| { - if is_hb_by_validator(&validator, &h) { - match h.msg { - MessageTypes::Heartbeat(h) => Some(h), - _ => None, - } - } else { - None +fn get_hb_by_validator(validator: &ValidatorDesc, heartbeats: &[Heartbeat]) -> Vec { + heartbeats + .into_iter() + .filter_map(|h| { + if is_hb_by_validator(&validator, &h) { + match h.msg { + MessageTypes::Heartbeat(h) => Some(h), + _ => None, } - }) - .collect(), - None => Default::default(), - } + } else { + None + } + }) + .collect() } // there are no messages at all for at least one validator -fn is_initializing(leader: &[Heartbeat], follower: &[Heartbeat], new_state: &[NewState], approve_state: &[ApproveState]) -> bool { - (leader.len() as i32 == 0 && new_state.len() as i32 == 0) || (follower.len() as i32 == 0 && approve_state.len() as i32 == 0) +fn is_initializing(leader: &[Heartbeat], follower: &[Heartbeat], leader_la: &Option, follower_la: &Option) -> bool { + (leader.is_empty() && get_new_state(leader_la).is_empty()) || (follower.is_empty() && get_approve_state(follower_la).is_empty()) } // at least one validator doesn't have a recent Heartbeat message @@ -271,15 +259,14 @@ fn is_offline(leader: &[Heartbeat], follower: &[Heartbeat]) -> bool { } fn is_date_recent(recency: &Duration, date: &DateTime) -> bool { - let time_passed = Utc::now().signed_duration_since(date.clone()); - time_passed <= recency.clone() + date >= &(Utc::now() - *recency) } // validators have recent Heartbeat messages, but they don't seem to be propagating messages between one another (the majority of Heartbeats are not found on both validators) -fn is_disconnected(follower_hb: &[Heartbeat], follower_hb_from_leader: &[Heartbeat]) -> bool { +fn is_disconnected(follower_hb: &[Heartbeat], leader: &ValidatorDesc) -> bool { let recency = Duration::minutes(4); - let follower_hb = follower_hb + let recent_follower_hb = follower_hb .into_iter() .filter_map(|h| { if is_date_recent(&recency, &h.timestamp) { @@ -290,7 +277,7 @@ fn is_disconnected(follower_hb: &[Heartbeat], follower_hb_from_leader: &[Heartbe }) .count(); - let follower_hb_from_leader = follower_hb_from_leader + let recent_follower_hb_from_leader = get_hb_by_validator(&leader, follower_hb) .into_iter() .filter_map(|h| { if is_date_recent(&recency, &h.timestamp) { @@ -301,7 +288,7 @@ fn is_disconnected(follower_hb: &[Heartbeat], follower_hb_from_leader: &[Heartbe }) .count(); - follower_hb > 0 && follower_hb_from_leader > 0 + recent_follower_hb > 0 && recent_follower_hb_from_leader > 0 } fn is_rejected_state() -> bool { @@ -356,7 +343,7 @@ mod test { #[test] fn old_date_is_not_recent() { let recency = Duration::minutes(4); - let past = Utc::now().checked_sub_signed(Duration::minutes(10)).unwrap(); + let past = Utc::now() - Duration::minutes(10); assert_eq!( is_date_recent(&recency, &past), false, @@ -367,7 +354,16 @@ mod test { // is_initializing() #[test] fn two_empty_message_arrays() { - todo!() + // let leader_hbs: [Heartbeat; 0] = []; + // let follower_hbs: [Heartbeat; 0] = []; + // let new_state_msgs: [NewState; 0] = []; + // let approve_state_msgs: [ApproveState; 0] = []; + + // assert_eq!( + // is_initializing(&leader_hbs, &follower_hbs, &new_state_msgs, &approve_state_msgs), + // true, + // "Both leader heartbeats + newstate and follower heatbeats + approvestate pairs are empty arrays" + // ) } #[test] From 41530bb3dfa367400d3547afac82aac92ad8f84e Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 13 Apr 2020 18:36:23 +0300 Subject: [PATCH 07/17] status - Message - impl helper functions --- src/status.rs | 364 +++++++++++++++++++++++++++++--------------------- 1 file changed, 212 insertions(+), 152 deletions(-) diff --git a/src/status.rs b/src/status.rs index 1f28ead..4d7d786 100644 --- a/src/status.rs +++ b/src/status.rs @@ -1,9 +1,9 @@ use crate::sentry_api::SentryApi; use chrono::{DateTime, Duration, Utc}; use primitives::{ - sentry::{HeartbeatValidatorMessage, LastApproved, LastApprovedResponse}, - validator::{Heartbeat, MessageTypes}, - BalancesMap, BigNum, Channel, + sentry::{HeartbeatValidatorMessage, LastApprovedResponse}, + validator::MessageTypes, + BalancesMap, BigNum, Channel, ValidatorId, }; use reqwest::Error; @@ -31,10 +31,95 @@ pub enum Finalized { } struct Messages { - leader_last_approved: Option, - follower_last_approved: Option, - leader_heartbeats: Vec, - follower_heartbeats: Vec, + leader: LastApprovedResponse, + follower: LastApprovedResponse, + recency: Duration, +} + +impl Messages { + fn has_leader_hb(&self) -> bool { + self.leader + .heartbeats + .as_ref() + .map(|heartbeats| heartbeats.len() > 0) + .unwrap_or(false) + } + + fn has_follower_hb(&self) -> bool { + self.follower + .heartbeats + .as_ref() + .map(|heartbeats| heartbeats.len() > 0) + .unwrap_or(false) + } + + fn has_follower_approve_state(&self) -> bool { + self.follower + .last_approved + .as_ref() + .map(|last_approved| last_approved.approve_state.is_some()) + .unwrap_or(false) + } + + fn has_recent_follower_hb(&self) -> bool { + self.follower + .heartbeats + .as_ref() + .map(|heartbeats| self.has_recent_heartbeat(&heartbeats)) + .unwrap_or(false) + } + + fn has_recent_leader_hb_from(&self, validator: &ValidatorId) -> bool { + self.leader + .heartbeats + .as_ref() + .map(|heartbeats| self.has_recent_heartbeat_from(heartbeats, Some(validator))) + .unwrap_or(false) + } + + fn has_recent_leader_hb(&self) -> bool { + self.leader + .heartbeats + .as_ref() + .map(|heartbeats| self.has_recent_heartbeat(&heartbeats)) + .unwrap_or(false) + } + + fn has_leader_new_state(&self) -> bool { + self.leader + .last_approved + .as_ref() + .map(|last_approved| last_approved.new_state.is_some()) + .unwrap_or(false) + } + + /// `from`: If `None` it will just check for a recent Heartbeat + fn has_recent_heartbeat_from( + &self, + heartbeats: &[HeartbeatValidatorMessage], + from: Option<&ValidatorId>, + ) -> bool { + heartbeats + .iter() + .any(|heartbeat_msg| match (from, &heartbeat_msg.msg) { + (Some(from), MessageTypes::Heartbeat(heartbeat)) + if &heartbeat_msg.from == from + && is_date_recent(self.recency, &heartbeat.timestamp) => + { + true + } + (None, MessageTypes::Heartbeat(heartbeat)) + if is_date_recent(self.recency, &heartbeat.timestamp) => + { + true + } + _ => false, + }) + } + + fn has_recent_heartbeat(&self, heartbeats: &[HeartbeatValidatorMessage]) -> bool { + self.has_recent_heartbeat_from(heartbeats, None) + } } #[derive(Debug, PartialEq, Eq)] @@ -107,7 +192,9 @@ pub async fn is_finalized(sentry: &SentryApi, channel: &Channel) -> Result Result { @@ -122,22 +209,21 @@ pub async fn get_status(sentry: &SentryApi, channel: &Channel) -> Result Result>) -> Vec { - match heartbeats { - Some(heartbeats) => heartbeats - .into_iter() - .filter_map(|heartbeat| match heartbeat.msg { - MessageTypes::Heartbeat(heartbeat) => Some(heartbeat), - _ => None, - }) - .collect(), - None => Default::default(), - } -} - -fn is_hb_by_validator(validator: &ValidatorDesc, heartbeat: &HeartbeatValidatorMessage) -> bool { - heartbeat.from == validator.id -} - -fn get_new_state(last_approved: &Option) -> Vec { - match last_approved { - Some(last_approved) => last_approved.new_state - .into_iter() - .filter_map(|new_state| match new_state.msg { - MessageTypes::NewState(new_state) => Some(new_state), - _ => None, - }) - .collect(), - None => Default::default(), - } -} - -fn get_approve_state(last_approved: &Option) -> Vec { - match last_approved { - Some(last_approved) => last_approved.approve_state - .into_iter() - .filter_map(|approve_state| match approve_state.msg { - MessageTypes::ApproveState(approve_state) => Some(approve_state), - _ => None, - }) - .collect(), - None => Default::default(), - } -} - -fn get_hb_by_validator(validator: &ValidatorDesc, heartbeats: &[Heartbeat]) -> Vec { - heartbeats - .into_iter() - .filter_map(|h| { - if is_hb_by_validator(&validator, &h) { - match h.msg { - MessageTypes::Heartbeat(h) => Some(h), - _ => None, - } - } else { - None - } - }) - .collect() -} +// fn is_hb_by_validator(validator: &ValidatorDesc, heartbeat: &HeartbeatValidatorMessage) -> bool { +// heartbeat.from == validator.id +// } + +// fn get_approve_state(last_approved: &Option) -> Vec { +// match last_approved { +// Some(last_approved) => last_approved.approve_state +// .into_iter() +// .filter_map(|approve_state| match approve_state.msg { +// MessageTypes::ApproveState(approve_state) => Some(approve_state), +// _ => None, +// }) +// .collect(), +// None => Default::default(), +// } +// } + +// fn get_hb_by_validator(validator: &ValidatorDesc, heartbeats: &[Heartbeat]) -> Vec { +// heartbeats +// .into_iter() +// .filter_map(|h| { +// if is_hb_by_validator(&validator, &h) { +// match h.msg { +// MessageTypes::Heartbeat(h) => Some(h), +// _ => None, +// } +// } else { +// None +// } +// }) +// .collect() +// } // there are no messages at all for at least one validator -fn is_initializing(leader: &[Heartbeat], follower: &[Heartbeat], leader_la: &Option, follower_la: &Option) -> bool { - (leader.is_empty() && get_new_state(leader_la).is_empty()) || (follower.is_empty() && get_approve_state(follower_la).is_empty()) +fn is_initializing(messages: &Messages) -> bool { + (messages.has_leader_hb() && messages.has_leader_new_state()) + || (messages.has_follower_hb() && messages.has_follower_approve_state()) } // at least one validator doesn't have a recent Heartbeat message -fn is_offline(leader: &[Heartbeat], follower: &[Heartbeat]) -> bool { - // @TODO: Move to configuration - let recency = Duration::minutes(4); - - !leader - .iter() - .any(|h| is_date_recent(&recency, &h.timestamp)) - || !follower - .iter() - .any(|h| is_date_recent(&recency, &h.timestamp)) +fn is_offline(messages: &Messages) -> bool { + !messages.has_recent_leader_hb() || !messages.has_recent_follower_hb() } -fn is_date_recent(recency: &Duration, date: &DateTime) -> bool { - date >= &(Utc::now() - *recency) +fn is_date_recent(recency: Duration, date: &DateTime) -> bool { + date >= &(Utc::now() - recency) } // validators have recent Heartbeat messages, but they don't seem to be propagating messages between one another (the majority of Heartbeats are not found on both validators) -fn is_disconnected(follower_hb: &[Heartbeat], leader: &ValidatorDesc) -> bool { - let recency = Duration::minutes(4); - - let recent_follower_hb = follower_hb - .into_iter() - .filter_map(|h| { - if is_date_recent(&recency, &h.timestamp) { - Some(h) - } else { - None - } - }) - .count(); - - let recent_follower_hb_from_leader = get_hb_by_validator(&leader, follower_hb) - .into_iter() - .filter_map(|h| { - if is_date_recent(&recency, &h.timestamp) { - Some(h) - } else { - None - } - }) - .count(); +fn is_disconnected(channel: &Channel, messages: &Messages) -> bool { + let follower = &channel.spec.validators.follower().id; - recent_follower_hb > 0 && recent_follower_hb_from_leader > 0 + messages.has_recent_follower_hb() && messages.has_recent_leader_hb_from(follower) } fn is_rejected_state() -> bool { @@ -383,10 +414,39 @@ mod test { #[test] fn is_offline_no_heartbeats() { + let messages = Messages { + leader: LastApprovedResponse { + last_approved: None, + heartbeats: Some(vec![]), + }, + follower: LastApprovedResponse { + last_approved: None, + heartbeats: Some(vec![]), + }, + recency: Duration::minutes(4), + }; + assert!( - is_offline(&[], &[]), - "On empty heartbeast it should be offline!" - ) + is_offline(&messages), + "On empty heartbeat it should be offline!" + ); + + let messages = Messages { + leader: LastApprovedResponse { + last_approved: None, + heartbeats: None, + }, + follower: LastApprovedResponse { + last_approved: None, + heartbeats: Some(vec![]), + }, + recency: Duration::minutes(4), + }; + + assert!( + is_offline(&messages), + "On empty heartbeat it should be offline!" + ); } // is_date_recent() @@ -395,7 +455,7 @@ mod test { let now = Utc::now(); let recency = Duration::minutes(4); assert!( - is_date_recent(&recency, &now), + is_date_recent(recency, &now), "The present moment is a recent date!" ) } @@ -405,7 +465,7 @@ mod test { let recency = Duration::minutes(4); let on_the_edge = Utc::now().checked_sub_signed(Duration::minutes(3)).unwrap(); assert!( - is_date_recent(&recency, &on_the_edge), + is_date_recent(recency, &on_the_edge), "When date is just as old as the recency limit, it still counts as recent" ) } @@ -415,15 +475,15 @@ mod test { let recency = Duration::minutes(4); let past = Utc::now() - Duration::minutes(10); assert_eq!( - is_date_recent(&recency, &past), + is_date_recent(recency, &past), false, "Date older than the recency limit is not recent" ) } // is_initializing() - #[test] - fn two_empty_message_arrays() { + // #[test] + // fn two_empty_message_arrays() { // let leader_hbs: [Heartbeat; 0] = []; // let follower_hbs: [Heartbeat; 0] = []; // let new_state_msgs: [NewState; 0] = []; @@ -434,43 +494,43 @@ mod test { // true, // "Both leader heartbeats + newstate and follower heatbeats + approvestate pairs are empty arrays" // ) - } + // } - #[test] - fn first_message_arr_is_empty() { - todo!() - } + // #[test] + // fn first_message_arr_is_empty() { + // todo!() + // } - #[test] - fn second_message_arr_is_empty() { - todo!() - } + // #[test] + // fn second_message_arr_is_empty() { + // todo!() + // } - #[test] - fn both_arrays_have_messages() { - todo!() - } + // #[test] + // fn both_arrays_have_messages() { + // todo!() + // } // is_disconnected() - #[test] - fn no_recent_hbs_on_both_sides() { - todo!() - } - - #[test] - fn no_recent_follower_hbs() { - todo!() - } - - #[test] - fn no_recent_leader_hb_on_follower_validator() { - todo!() - } - - #[test] - fn recent_hbs_on_both_arrays() { - todo!() - } + // #[test] + // fn no_recent_hbs_on_both_sides() { + // todo!() + // } + + // #[test] + // fn no_recent_follower_hbs() { + // todo!() + // } + + // #[test] + // fn no_recent_leader_hb_on_follower_validator() { + // todo!() + // } + + // #[test] + // fn recent_hbs_on_both_arrays() { + // todo!() + // } // @TODO: test is_offline() } From 1a82d93cbb35ee150048c36710cbc9d6b6568b67 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 13 Apr 2020 18:37:27 +0300 Subject: [PATCH 08/17] status - clean-up the commented code --- src/status.rs | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/src/status.rs b/src/status.rs index 4d7d786..892ea73 100644 --- a/src/status.rs +++ b/src/status.rs @@ -266,39 +266,6 @@ async fn fetch_balances(sentry: &SentryApi, channel: &Channel) -> Result bool { -// heartbeat.from == validator.id -// } - -// fn get_approve_state(last_approved: &Option) -> Vec { -// match last_approved { -// Some(last_approved) => last_approved.approve_state -// .into_iter() -// .filter_map(|approve_state| match approve_state.msg { -// MessageTypes::ApproveState(approve_state) => Some(approve_state), -// _ => None, -// }) -// .collect(), -// None => Default::default(), -// } -// } - -// fn get_hb_by_validator(validator: &ValidatorDesc, heartbeats: &[Heartbeat]) -> Vec { -// heartbeats -// .into_iter() -// .filter_map(|h| { -// if is_hb_by_validator(&validator, &h) { -// match h.msg { -// MessageTypes::Heartbeat(h) => Some(h), -// _ => None, -// } -// } else { -// None -// } -// }) -// .collect() -// } - // there are no messages at all for at least one validator fn is_initializing(messages: &Messages) -> bool { (messages.has_leader_hb() && messages.has_leader_new_state()) From b2dadec764d52c4b15d8d9f7d73baed14d26e193 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 13 Apr 2020 18:38:55 +0300 Subject: [PATCH 09/17] status - remove confusing method --- src/status.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/status.rs b/src/status.rs index 892ea73..448520f 100644 --- a/src/status.rs +++ b/src/status.rs @@ -65,7 +65,7 @@ impl Messages { self.follower .heartbeats .as_ref() - .map(|heartbeats| self.has_recent_heartbeat(&heartbeats)) + .map(|heartbeats| self.has_recent_heartbeat_from(&heartbeats, None)) .unwrap_or(false) } @@ -81,7 +81,7 @@ impl Messages { self.leader .heartbeats .as_ref() - .map(|heartbeats| self.has_recent_heartbeat(&heartbeats)) + .map(|heartbeats| self.has_recent_heartbeat_from(&heartbeats, None)) .unwrap_or(false) } @@ -116,10 +116,6 @@ impl Messages { _ => false, }) } - - fn has_recent_heartbeat(&self, heartbeats: &[HeartbeatValidatorMessage]) -> bool { - self.has_recent_heartbeat_from(heartbeats, None) - } } #[derive(Debug, PartialEq, Eq)] From 1175f242c84fb3839a567fa984cb027bef9f8605 Mon Sep 17 00:00:00 2001 From: Lachezar Lechev Date: Mon, 13 Apr 2020 18:53:23 +0300 Subject: [PATCH 10/17] status - fix bug with is_disconnected --- src/status.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/status.rs b/src/status.rs index 448520f..ab92e88 100644 --- a/src/status.rs +++ b/src/status.rs @@ -281,7 +281,7 @@ fn is_date_recent(recency: Duration, date: &DateTime) -> bool { fn is_disconnected(channel: &Channel, messages: &Messages) -> bool { let follower = &channel.spec.validators.follower().id; - messages.has_recent_follower_hb() && messages.has_recent_leader_hb_from(follower) + !(messages.has_recent_follower_hb() && messages.has_recent_leader_hb_from(follower)) } fn is_rejected_state() -> bool { From ea11af7b18128f9d23468080e1523c50d9ea7cf2 Mon Sep 17 00:00:00 2001 From: simzzz Date: Mon, 13 Apr 2020 19:24:00 +0300 Subject: [PATCH 11/17] added documentation comments --- src/status.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/status.rs b/src/status.rs index ab92e88..61972c7 100644 --- a/src/status.rs +++ b/src/status.rs @@ -262,13 +262,13 @@ async fn fetch_balances(sentry: &SentryApi, channel: &Channel) -> Result bool { (messages.has_leader_hb() && messages.has_leader_new_state()) || (messages.has_follower_hb() && messages.has_follower_approve_state()) } -// at least one validator doesn't have a recent Heartbeat message +/// at least one validator doesn't have a recent Heartbeat message fn is_offline(messages: &Messages) -> bool { !messages.has_recent_leader_hb() || !messages.has_recent_follower_hb() } @@ -277,7 +277,7 @@ fn is_date_recent(recency: Duration, date: &DateTime) -> bool { date >= &(Utc::now() - recency) } -// validators have recent Heartbeat messages, but they don't seem to be propagating messages between one another (the majority of Heartbeats are not found on both validators) +/// validators have recent Heartbeat messages, but they don't seem to be propagating messages between one another (the majority of Heartbeats are not found on both validators) fn is_disconnected(channel: &Channel, messages: &Messages) -> bool { let follower = &channel.spec.validators.follower().id; From b83ba545a32086310507a295bce0e6669eb60db3 Mon Sep 17 00:00:00 2001 From: simzzz Date: Tue, 14 Apr 2020 17:49:56 +0300 Subject: [PATCH 12/17] fixed is_initializing and added one test for it --- src/status.rs | 79 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/src/status.rs b/src/status.rs index 61972c7..9bfaa24 100644 --- a/src/status.rs +++ b/src/status.rs @@ -1,7 +1,7 @@ use crate::sentry_api::SentryApi; use chrono::{DateTime, Duration, Utc}; use primitives::{ - sentry::{HeartbeatValidatorMessage, LastApprovedResponse}, + sentry::{HeartbeatValidatorMessage, LastApprovedResponse, LastApproved}, validator::MessageTypes, BalancesMap, BigNum, Channel, ValidatorId, }; @@ -264,8 +264,8 @@ async fn fetch_balances(sentry: &SentryApi, channel: &Channel) -> Result bool { - (messages.has_leader_hb() && messages.has_leader_new_state()) - || (messages.has_follower_hb() && messages.has_follower_approve_state()) + (!messages.has_leader_hb() && !messages.has_leader_new_state()) + || (!messages.has_follower_hb() && !messages.has_follower_approve_state()) } /// at least one validator doesn't have a recent Heartbeat message @@ -445,28 +445,67 @@ mod test { } // is_initializing() - // #[test] - // fn two_empty_message_arrays() { - // let leader_hbs: [Heartbeat; 0] = []; - // let follower_hbs: [Heartbeat; 0] = []; - // let new_state_msgs: [NewState; 0] = []; - // let approve_state_msgs: [ApproveState; 0] = []; - - // assert_eq!( - // is_initializing(&leader_hbs, &follower_hbs, &new_state_msgs, &approve_state_msgs), - // true, - // "Both leader heartbeats + newstate and follower heatbeats + approvestate pairs are empty arrays" - // ) - // } + #[test] + fn two_empty_message_arrays() { + let messages = Messages { + leader: LastApprovedResponse { + last_approved: None, + heartbeats: Some(vec![]), + }, + follower: LastApprovedResponse { + last_approved: None, + heartbeats: Some(vec![]), + }, + recency: Duration::minutes(4), + }; + + assert_eq!( + is_initializing(&messages), + true, + "Both leader heartbeats + newstate and follower heatbeats + approvestate pairs are empty arrays" + ) + } // #[test] - // fn first_message_arr_is_empty() { - // todo!() + // fn leader_has_no_messages() { + // let messages = Messages { + // leader: LastApprovedResponse { + // last_approved: Some(vec![]), + // heartbeats: Some(vec![]), + // }, + // follower: LastApprovedResponse { + // last_approved: Some(vec![]), + // heartbeats: Some(vec![]), + // }, + // recency: Duration::minutes(4), + // }; + + // assert_eq!( + // is_initializing(&messages), + // true, + // "Both leader heartbeats + newstate and follower heatbeats + approvestate pairs are empty arrays" + // ) // } // #[test] - // fn second_message_arr_is_empty() { - // todo!() + // fn follower_has_no_messages() { + // let messages = Messages { + // leader: LastApprovedResponse { + // last_approved: Some(vec![]), + // heartbeats: Some(vec![]), + // }, + // follower: LastApprovedResponse { + // last_approved: Some(vec![]), + // heartbeats: Some(vec![]), + // }, + // recency: Duration::minutes(4), + // }; + + // assert_eq!( + // is_initializing(&messages), + // true, + // "Both leader heartbeats + newstate and follower heatbeats + approvestate pairs are empty arrays" + // ) // } // #[test] From 82b8ec4cd0f5b3cf463aa32625b20876f1f1e046 Mon Sep 17 00:00:00 2001 From: simzzz Date: Tue, 14 Apr 2020 20:48:50 +0300 Subject: [PATCH 13/17] tests for is_initializing --- src/status.rs | 160 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 114 insertions(+), 46 deletions(-) diff --git a/src/status.rs b/src/status.rs index 9bfaa24..171a9c3 100644 --- a/src/status.rs +++ b/src/status.rs @@ -304,8 +304,12 @@ fn is_ready() -> bool { mod test { use super::*; use httptest::{mappers::*, responders::*, Expectation, Server, ServerPool}; - use primitives::util::tests::prep_db::{ - DUMMY_CHANNEL, DUMMY_VALIDATOR_FOLLOWER, DUMMY_VALIDATOR_LEADER, + use primitives::{ + util::tests::prep_db::{ + DUMMY_CHANNEL, DUMMY_VALIDATOR_FOLLOWER, DUMMY_VALIDATOR_LEADER, + }, + sentry::{ApproveStateValidatorMessage, NewStateValidatorMessage}, + validator::{ApproveState, NewState, Heartbeat} }; static SERVER_POOL: ServerPool = ServerPool::new(4); @@ -323,6 +327,42 @@ mod test { channel } + fn get_approve_state_msg() -> Option { + Some(ApproveStateValidatorMessage { + from: DUMMY_VALIDATOR_LEADER.id.clone(), + received: Utc::now(), + msg: MessageTypes::ApproveState(ApproveState { + state_root: String::from("0x0"), + signature: String::from("0x0"), + is_healthy: true, + }), + }) + } + + fn get_heartbeat_msgs() -> Option> { + Some(vec![HeartbeatValidatorMessage { + from: DUMMY_VALIDATOR_LEADER.id.clone(), + received: Utc::now(), + msg: MessageTypes::Heartbeat(Heartbeat { + signature: String::from("0x0"), + state_root: String::from("0x0"), + timestamp: Utc::now(), + }), + }]) + } + + fn get_newstate_msg() -> Option { + Some(NewStateValidatorMessage { + from: DUMMY_VALIDATOR_LEADER.id.clone(), + received: Utc::now(), + msg: MessageTypes::NewState(NewState { + signature: String::from("0x0"), + state_root: String::from("0x0"), + balances: Default::default(), + }), + }) + } + #[tokio::test] async fn test_is_finalized_when_expired() { let server = SERVER_POOL.get_server(); @@ -466,52 +506,80 @@ mod test { ) } - // #[test] - // fn leader_has_no_messages() { - // let messages = Messages { - // leader: LastApprovedResponse { - // last_approved: Some(vec![]), - // heartbeats: Some(vec![]), - // }, - // follower: LastApprovedResponse { - // last_approved: Some(vec![]), - // heartbeats: Some(vec![]), - // }, - // recency: Duration::minutes(4), - // }; - - // assert_eq!( - // is_initializing(&messages), - // true, - // "Both leader heartbeats + newstate and follower heatbeats + approvestate pairs are empty arrays" - // ) - // } + #[test] + fn leader_has_no_messages() { + let messages = Messages { + leader: LastApprovedResponse { + last_approved: None, + heartbeats: Some(vec![]), + }, + follower: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: None, + approve_state: get_approve_state_msg(), + }), + heartbeats: get_heartbeat_msgs(), + }, + recency: Duration::minutes(4), + }; - // #[test] - // fn follower_has_no_messages() { - // let messages = Messages { - // leader: LastApprovedResponse { - // last_approved: Some(vec![]), - // heartbeats: Some(vec![]), - // }, - // follower: LastApprovedResponse { - // last_approved: Some(vec![]), - // heartbeats: Some(vec![]), - // }, - // recency: Duration::minutes(4), - // }; - - // assert_eq!( - // is_initializing(&messages), - // true, - // "Both leader heartbeats + newstate and follower heatbeats + approvestate pairs are empty arrays" - // ) - // } + assert_eq!( + is_initializing(&messages), + true, + "Leader has no new messages but the follower has heartbeats and approvestate" + ) + } - // #[test] - // fn both_arrays_have_messages() { - // todo!() - // } + #[test] + fn follower_has_no_messages() { + let messages = Messages { + leader: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: get_newstate_msg(), + approve_state: None, + }), + heartbeats: get_heartbeat_msgs(), + }, + follower: LastApprovedResponse { + last_approved: None, + heartbeats: Some(vec![]), + }, + recency: Duration::minutes(4), + }; + + assert_eq!( + is_initializing(&messages), + true, + "Follower has no new messages but leader has heartbeats and newstate" + ) + } + + #[test] + fn both_arrays_have_messages() { + let messages = Messages { + leader: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: get_newstate_msg(), + approve_state: None, + }), + heartbeats: get_heartbeat_msgs(), + }, + follower: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: None, + approve_state: get_approve_state_msg(), + }), + heartbeats: get_heartbeat_msgs(), + }, + recency: Duration::minutes(4), + }; + + assert_eq!( + is_initializing(&messages), + false, + "Both arrays have messages" + ) + } // is_disconnected() // #[test] From 68e60d1d8ebc04b0349a122851dc153440714162 Mon Sep 17 00:00:00 2001 From: simzzz Date: Wed, 15 Apr 2020 19:15:09 +0300 Subject: [PATCH 14/17] changed test functions for msgs, added is_disconnected tests --- src/status.rs | 208 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 170 insertions(+), 38 deletions(-) diff --git a/src/status.rs b/src/status.rs index 171a9c3..f7d21c0 100644 --- a/src/status.rs +++ b/src/status.rs @@ -327,8 +327,8 @@ mod test { channel } - fn get_approve_state_msg() -> Option { - Some(ApproveStateValidatorMessage { + fn get_approve_state_msg() -> ApproveStateValidatorMessage { + ApproveStateValidatorMessage { from: DUMMY_VALIDATOR_LEADER.id.clone(), received: Utc::now(), msg: MessageTypes::ApproveState(ApproveState { @@ -336,23 +336,23 @@ mod test { signature: String::from("0x0"), is_healthy: true, }), - }) + } } - fn get_heartbeat_msgs() -> Option> { - Some(vec![HeartbeatValidatorMessage { + fn get_heartbeat_msgs(recency: Duration) -> HeartbeatValidatorMessage { + HeartbeatValidatorMessage { from: DUMMY_VALIDATOR_LEADER.id.clone(), - received: Utc::now(), + received: Utc::now() - recency, msg: MessageTypes::Heartbeat(Heartbeat { signature: String::from("0x0"), state_root: String::from("0x0"), - timestamp: Utc::now(), + timestamp: Utc::now() - recency, }), - }]) + } } - fn get_newstate_msg() -> Option { - Some(NewStateValidatorMessage { + fn get_newstate_msg() -> NewStateValidatorMessage { + NewStateValidatorMessage { from: DUMMY_VALIDATOR_LEADER.id.clone(), received: Utc::now(), msg: MessageTypes::NewState(NewState { @@ -360,7 +360,7 @@ mod test { state_root: String::from("0x0"), balances: Default::default(), }), - }) + } } #[tokio::test] @@ -508,6 +508,8 @@ mod test { #[test] fn leader_has_no_messages() { + let approve_state = get_approve_state_msg(); + let heartbeat = get_heartbeat_msgs(Duration::minutes(0)); let messages = Messages { leader: LastApprovedResponse { last_approved: None, @@ -516,9 +518,9 @@ mod test { follower: LastApprovedResponse { last_approved: Some(LastApproved { new_state: None, - approve_state: get_approve_state_msg(), + approve_state: Some(approve_state), }), - heartbeats: get_heartbeat_msgs(), + heartbeats: Some(vec![heartbeat]), }, recency: Duration::minutes(4), }; @@ -532,13 +534,16 @@ mod test { #[test] fn follower_has_no_messages() { + let heartbeat = get_heartbeat_msgs(Duration::minutes(0)); + let new_state = get_newstate_msg(); + let messages = Messages { leader: LastApprovedResponse { last_approved: Some(LastApproved { - new_state: get_newstate_msg(), + new_state: Some(new_state), approve_state: None, }), - heartbeats: get_heartbeat_msgs(), + heartbeats: Some(vec![heartbeat]), }, follower: LastApprovedResponse { last_approved: None, @@ -556,20 +561,25 @@ mod test { #[test] fn both_arrays_have_messages() { + let heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); + let heartbeat_2 = get_heartbeat_msgs(Duration::minutes(0)); + let new_state = get_newstate_msg(); + let approve_state = get_approve_state_msg(); + let messages = Messages { leader: LastApprovedResponse { last_approved: Some(LastApproved { - new_state: get_newstate_msg(), + new_state: Some(new_state), approve_state: None, }), - heartbeats: get_heartbeat_msgs(), + heartbeats: Some(vec![heartbeat_1]), }, follower: LastApprovedResponse { last_approved: Some(LastApproved { new_state: None, - approve_state: get_approve_state_msg(), + approve_state: Some(approve_state), }), - heartbeats: get_heartbeat_msgs(), + heartbeats: Some(vec![heartbeat_2]), }, recency: Duration::minutes(4), }; @@ -582,25 +592,147 @@ mod test { } // is_disconnected() - // #[test] - // fn no_recent_hbs_on_both_sides() { - // todo!() - // } - - // #[test] - // fn no_recent_follower_hbs() { - // todo!() - // } - - // #[test] - // fn no_recent_leader_hb_on_follower_validator() { - // todo!() - // } - - // #[test] - // fn recent_hbs_on_both_arrays() { - // todo!() - // } + #[test] + fn no_recent_hbs_on_both_sides() { + let server = SERVER_POOL.get_server(); + let channel = get_test_channel(&server); + let heartbeat_1 = get_heartbeat_msgs(Duration::minutes(10)); + let heartbeat_2 = get_heartbeat_msgs(Duration::minutes(10)); + let new_state = get_newstate_msg(); + let approve_state = get_approve_state_msg(); + + let messages = Messages { + leader: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: Some(new_state), + approve_state: None, + }), + heartbeats: Some(vec![heartbeat_1]), + }, + follower: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: None, + approve_state: Some(approve_state), + }), + heartbeats: Some(vec![heartbeat_2]), + }, + recency: Duration::minutes(4), + }; + + assert_eq!( + is_disconnected(&channel, &messages), + true, + "Both leader and follower heartbeats have no recent messages" + ) + } + + #[test] + fn no_recent_follower_hbs() { + let server = SERVER_POOL.get_server(); + let channel = get_test_channel(&server); + let heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); + let heartbeat_2 = get_heartbeat_msgs(Duration::minutes(10)); + let new_state = get_newstate_msg(); + let approve_state = get_approve_state_msg(); + + let messages = Messages { + leader: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: Some(new_state), + approve_state: None, + }), + heartbeats: Some(vec![heartbeat_1]), + }, + follower: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: None, + approve_state: Some(approve_state), + }), + heartbeats: Some(vec![heartbeat_2]), + }, + recency: Duration::minutes(4), + }; + + assert_eq!( + is_disconnected(&channel, &messages), + true, + "Follower heartbeats have no recent messages" + ) + } + + #[test] + fn no_recent_leader_hb_on_follower_validator() { + let server = SERVER_POOL.get_server(); + let channel = get_test_channel(&server); + let follower_id = &channel.spec.validators.follower().id; + + let mut heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); + heartbeat_1.from = follower_id.clone(); + let mut heartbeat_2 = get_heartbeat_msgs(Duration::minutes(0)); + heartbeat_2.from = follower_id.clone(); + let new_state = get_newstate_msg(); + let approve_state = get_approve_state_msg(); + + let messages = Messages { + leader: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: Some(new_state), + approve_state: None, + }), + heartbeats: Some(vec![]), + }, + follower: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: None, + approve_state: Some(approve_state), + }), + heartbeats: Some(vec![heartbeat_1, heartbeat_2]), + }, + recency: Duration::minutes(4), + }; + + assert_eq!( + is_disconnected(&channel, &messages), + true, + "Follower validator has recent HB messages but none have come from the leader validator" + ) + } + + #[test] + fn recent_hbs_on_both_arrays() { + let server = SERVER_POOL.get_server(); + let channel = get_test_channel(&server); + let mut heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); + heartbeat_1.from = channel.spec.validators.leader().id; + let mut heartbeat_2 = get_heartbeat_msgs(Duration::minutes(0)); + heartbeat_2.from = channel.spec.validators.follower().id; + let new_state = get_newstate_msg(); + let approve_state = get_approve_state_msg(); + + let messages = Messages { + leader: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: Some(new_state), + approve_state: None, + }), + heartbeats: Some(vec![]), + }, + follower: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: None, + approve_state: Some(approve_state), + }), + heartbeats: Some(vec![heartbeat_1, heartbeat_2]), + }, + recency: Duration::minutes(4), + }; + + assert_eq!( + is_disconnected(&channel, &messages), + false, + "Follower validator has recent HB messages including ones coming from the leader validator" + ) + } // @TODO: test is_offline() } From a0fb3f3518b8444a33f690376a2a8958cdaf8e6f Mon Sep 17 00:00:00 2001 From: simzzz Date: Wed, 15 Apr 2020 20:25:02 +0300 Subject: [PATCH 15/17] tests now run successfully but one doesnt pass --- src/status.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/status.rs b/src/status.rs index f7d21c0..57a1925 100644 --- a/src/status.rs +++ b/src/status.rs @@ -703,9 +703,9 @@ mod test { let server = SERVER_POOL.get_server(); let channel = get_test_channel(&server); let mut heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_1.from = channel.spec.validators.leader().id; + heartbeat_1.from = channel.spec.validators.leader().clone().id; let mut heartbeat_2 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_2.from = channel.spec.validators.follower().id; + heartbeat_2.from = channel.spec.validators.follower().clone().id; let new_state = get_newstate_msg(); let approve_state = get_approve_state_msg(); From 9f02e2a10f5bfb6e5b7c2a34d54c0d838500c518 Mon Sep 17 00:00:00 2001 From: simzzz Date: Thu, 16 Apr 2020 20:22:54 +0300 Subject: [PATCH 16/17] Revamped is_disconnected() + the tests that belong to it --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/status.rs | 106 +++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 85 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 784ca01..b7ee568 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -987,7 +987,7 @@ checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" [[package]] name = "primitives" version = "0.1.0" -source = "git+https://github.com/AdExNetwork/adex-validator-stack-rust?branch=clean-up-marketing-campaign#e568c8447f84681174567b2392e477462eb3c96b" +source = "git+https://github.com/AdExNetwork/adex-validator-stack-rust?branch=dev#74618d47a4b170a128732dc6b92f3eedcf2bf02d" dependencies = [ "chrono", "fake", diff --git a/Cargo.toml b/Cargo.toml index 78f7e5f..5e7a593 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Lachezar Lechev "] edition = "2018" [dependencies] -primitives = { git = "https://github.com/AdExNetwork/adex-validator-stack-rust", branch = "clean-up-marketing-campaign" } +primitives = { git = "https://github.com/AdExNetwork/adex-validator-stack-rust", branch = "dev" } chrono = { version = "0.4" } # Server tokio = { version = "0.2", features = ["macros", "rt-threaded", "sync"] } diff --git a/src/status.rs b/src/status.rs index 57a1925..3beb9b7 100644 --- a/src/status.rs +++ b/src/status.rs @@ -1,7 +1,7 @@ use crate::sentry_api::SentryApi; use chrono::{DateTime, Duration, Utc}; use primitives::{ - sentry::{HeartbeatValidatorMessage, LastApprovedResponse, LastApproved}, + sentry::{HeartbeatValidatorMessage, LastApprovedResponse}, validator::MessageTypes, BalancesMap, BigNum, Channel, ValidatorId, }; @@ -77,6 +77,15 @@ impl Messages { .unwrap_or(false) } + + fn has_recent_follower_hb_from(&self, validator: &ValidatorId) -> bool { + self.follower + .heartbeats + .as_ref() + .map(|heartbeats| self.has_recent_heartbeat_from(heartbeats, Some(validator))) + .unwrap_or(false) + } + fn has_recent_leader_hb(&self) -> bool { self.leader .heartbeats @@ -279,9 +288,10 @@ fn is_date_recent(recency: Duration, date: &DateTime) -> bool { /// validators have recent Heartbeat messages, but they don't seem to be propagating messages between one another (the majority of Heartbeats are not found on both validators) fn is_disconnected(channel: &Channel, messages: &Messages) -> bool { + let leader = &channel.spec.validators.leader().id; let follower = &channel.spec.validators.follower().id; - !(messages.has_recent_follower_hb() && messages.has_recent_leader_hb_from(follower)) + !(messages.has_recent_leader_hb_from(follower) && messages.has_recent_follower_hb_from(leader)) } fn is_rejected_state() -> bool { @@ -308,7 +318,7 @@ mod test { util::tests::prep_db::{ DUMMY_CHANNEL, DUMMY_VALIDATOR_FOLLOWER, DUMMY_VALIDATOR_LEADER, }, - sentry::{ApproveStateValidatorMessage, NewStateValidatorMessage}, + sentry::{ApproveStateValidatorMessage, NewStateValidatorMessage, LastApproved}, validator::{ApproveState, NewState, Heartbeat} }; @@ -329,7 +339,7 @@ mod test { fn get_approve_state_msg() -> ApproveStateValidatorMessage { ApproveStateValidatorMessage { - from: DUMMY_VALIDATOR_LEADER.id.clone(), + from: DUMMY_VALIDATOR_LEADER.id, received: Utc::now(), msg: MessageTypes::ApproveState(ApproveState { state_root: String::from("0x0"), @@ -341,7 +351,7 @@ mod test { fn get_heartbeat_msgs(recency: Duration) -> HeartbeatValidatorMessage { HeartbeatValidatorMessage { - from: DUMMY_VALIDATOR_LEADER.id.clone(), + from: DUMMY_VALIDATOR_LEADER.id, received: Utc::now() - recency, msg: MessageTypes::Heartbeat(Heartbeat { signature: String::from("0x0"), @@ -353,7 +363,7 @@ mod test { fn get_newstate_msg() -> NewStateValidatorMessage { NewStateValidatorMessage { - from: DUMMY_VALIDATOR_LEADER.id.clone(), + from: DUMMY_VALIDATOR_LEADER.id, received: Utc::now(), msg: MessageTypes::NewState(NewState { signature: String::from("0x0"), @@ -630,8 +640,52 @@ mod test { fn no_recent_follower_hbs() { let server = SERVER_POOL.get_server(); let channel = get_test_channel(&server); - let heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); - let heartbeat_2 = get_heartbeat_msgs(Duration::minutes(10)); + let mut heartbeat_1 = get_heartbeat_msgs(Duration::minutes(10)); // not recent + heartbeat_1.from = channel.spec.validators.leader().id; + let mut heartbeat_2 = get_heartbeat_msgs(Duration::minutes(10)); + heartbeat_2.from = channel.spec.validators.follower().id; + let mut heartbeat_3 = get_heartbeat_msgs(Duration::minutes(10)); + heartbeat_3.from = channel.spec.validators.leader().id; + let mut heartbeat_4 = get_heartbeat_msgs(Duration::minutes(10)); + heartbeat_4.from = channel.spec.validators.follower().id; + let new_state = get_newstate_msg(); + let approve_state = get_approve_state_msg(); + + let messages = Messages { + leader: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: Some(new_state), + approve_state: None, + }), + heartbeats: Some(vec![heartbeat_1, heartbeat_2]), + }, + follower: LastApprovedResponse { + last_approved: Some(LastApproved { + new_state: None, + approve_state: Some(approve_state), + }), + heartbeats: Some(vec![heartbeat_3, heartbeat_4]), + }, + recency: Duration::minutes(4), + }; + + assert_eq!( + is_disconnected(&channel, &messages), + true, + "No recent heartbeats on both validators" + ) + } + + #[test] + fn no_recent_leader_hb_coming_from_follower() { + let server = SERVER_POOL.get_server(); + let channel = get_test_channel(&server); + let mut heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); + heartbeat_1.from = channel.spec.validators.leader().id; + let mut heartbeat_2 = get_heartbeat_msgs(Duration::minutes(0)); + heartbeat_2.from = channel.spec.validators.leader().id; + let mut heartbeat_3 = get_heartbeat_msgs(Duration::minutes(0)); + heartbeat_3.from = channel.spec.validators.follower().id; let new_state = get_newstate_msg(); let approve_state = get_approve_state_msg(); @@ -648,7 +702,7 @@ mod test { new_state: None, approve_state: Some(approve_state), }), - heartbeats: Some(vec![heartbeat_2]), + heartbeats: Some(vec![heartbeat_2, heartbeat_3]), }, recency: Duration::minutes(4), }; @@ -656,20 +710,22 @@ mod test { assert_eq!( is_disconnected(&channel, &messages), true, - "Follower heartbeats have no recent messages" + "Leader validator heartbeats have no recent messages that came from the follower" ) } #[test] - fn no_recent_leader_hb_on_follower_validator() { + fn no_recent_follower_hb_coming_from_leader() { let server = SERVER_POOL.get_server(); let channel = get_test_channel(&server); - let follower_id = &channel.spec.validators.follower().id; let mut heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_1.from = follower_id.clone(); + heartbeat_1.from = channel.spec.validators.leader().id; let mut heartbeat_2 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_2.from = follower_id.clone(); + heartbeat_2.from = channel.spec.validators.follower().id; + let mut heartbeat_3 = get_heartbeat_msgs(Duration::minutes(0)); + heartbeat_3.from = channel.spec.validators.follower().id; + let new_state = get_newstate_msg(); let approve_state = get_approve_state_msg(); @@ -679,14 +735,14 @@ mod test { new_state: Some(new_state), approve_state: None, }), - heartbeats: Some(vec![]), + heartbeats: Some(vec![heartbeat_1, heartbeat_2]), }, follower: LastApprovedResponse { last_approved: Some(LastApproved { new_state: None, approve_state: Some(approve_state), }), - heartbeats: Some(vec![heartbeat_1, heartbeat_2]), + heartbeats: Some(vec![heartbeat_3]), }, recency: Duration::minutes(4), }; @@ -694,18 +750,22 @@ mod test { assert_eq!( is_disconnected(&channel, &messages), true, - "Follower validator has recent HB messages but none have come from the leader validator" + "Follower validator heartbeats have no recent messages that came from the leader" ) } #[test] - fn recent_hbs_on_both_arrays() { + fn recent_hbs_coming_from_both_validators() { let server = SERVER_POOL.get_server(); let channel = get_test_channel(&server); let mut heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_1.from = channel.spec.validators.leader().clone().id; + heartbeat_1.from = channel.spec.validators.leader().id; let mut heartbeat_2 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_2.from = channel.spec.validators.follower().clone().id; + heartbeat_2.from = channel.spec.validators.follower().id; + let mut heartbeat_3 = get_heartbeat_msgs(Duration::minutes(0)); + heartbeat_3.from = channel.spec.validators.leader().id; + let mut heartbeat_4 = get_heartbeat_msgs(Duration::minutes(0)); + heartbeat_4.from = channel.spec.validators.follower().id; let new_state = get_newstate_msg(); let approve_state = get_approve_state_msg(); @@ -715,14 +775,14 @@ mod test { new_state: Some(new_state), approve_state: None, }), - heartbeats: Some(vec![]), + heartbeats: Some(vec![heartbeat_1, heartbeat_2]), }, follower: LastApprovedResponse { last_approved: Some(LastApproved { new_state: None, approve_state: Some(approve_state), }), - heartbeats: Some(vec![heartbeat_1, heartbeat_2]), + heartbeats: Some(vec![heartbeat_3, heartbeat_4]), }, recency: Duration::minutes(4), }; @@ -730,7 +790,7 @@ mod test { assert_eq!( is_disconnected(&channel, &messages), false, - "Follower validator has recent HB messages including ones coming from the leader validator" + "Leader hb has recent messages that came from the follower, and the follower has recent messages that came from the leader" ) } From 11b97ee15d30a4d148daf7d1547ede27061afcd4 Mon Sep 17 00:00:00 2001 From: simzzz Date: Thu, 23 Apr 2020 22:44:28 +0300 Subject: [PATCH 17/17] Fixed issues found in the code review --- src/status.rs | 119 ++++++++++++++++++++------------------------------ 1 file changed, 47 insertions(+), 72 deletions(-) diff --git a/src/status.rs b/src/status.rs index 3beb9b7..92733f9 100644 --- a/src/status.rs +++ b/src/status.rs @@ -349,9 +349,9 @@ mod test { } } - fn get_heartbeat_msgs(recency: Duration) -> HeartbeatValidatorMessage { + fn get_heartbeat_msg(recency: Duration, validator_id: ValidatorId) -> HeartbeatValidatorMessage { HeartbeatValidatorMessage { - from: DUMMY_VALIDATOR_LEADER.id, + from: validator_id, received: Utc::now() - recency, msg: MessageTypes::Heartbeat(Heartbeat { signature: String::from("0x0"), @@ -361,7 +361,7 @@ mod test { } } - fn get_newstate_msg() -> NewStateValidatorMessage { + fn get_new_state_msg() -> NewStateValidatorMessage { NewStateValidatorMessage { from: DUMMY_VALIDATOR_LEADER.id, received: Utc::now(), @@ -518,8 +518,9 @@ mod test { #[test] fn leader_has_no_messages() { + let channel = DUMMY_CHANNEL.clone(); let approve_state = get_approve_state_msg(); - let heartbeat = get_heartbeat_msgs(Duration::minutes(0)); + let heartbeat = get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.leader().id); let messages = Messages { leader: LastApprovedResponse { last_approved: None, @@ -544,8 +545,9 @@ mod test { #[test] fn follower_has_no_messages() { - let heartbeat = get_heartbeat_msgs(Duration::minutes(0)); - let new_state = get_newstate_msg(); + let channel = DUMMY_CHANNEL.clone(); + let heartbeat = get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.leader().id); + let new_state = get_new_state_msg(); let messages = Messages { leader: LastApprovedResponse { @@ -571,9 +573,10 @@ mod test { #[test] fn both_arrays_have_messages() { - let heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); - let heartbeat_2 = get_heartbeat_msgs(Duration::minutes(0)); - let new_state = get_newstate_msg(); + let channel = DUMMY_CHANNEL.clone(); + let leader_heartbeats = vec![get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.leader().id)]; + let follower_heartbeats = vec![get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.leader().id)]; + let new_state = get_new_state_msg(); let approve_state = get_approve_state_msg(); let messages = Messages { @@ -582,14 +585,14 @@ mod test { new_state: Some(new_state), approve_state: None, }), - heartbeats: Some(vec![heartbeat_1]), + heartbeats: Some(leader_heartbeats), }, follower: LastApprovedResponse { last_approved: Some(LastApproved { new_state: None, approve_state: Some(approve_state), }), - heartbeats: Some(vec![heartbeat_2]), + heartbeats: Some(follower_heartbeats), }, recency: Duration::minutes(4), }; @@ -604,27 +607,24 @@ mod test { // is_disconnected() #[test] fn no_recent_hbs_on_both_sides() { - let server = SERVER_POOL.get_server(); - let channel = get_test_channel(&server); - let heartbeat_1 = get_heartbeat_msgs(Duration::minutes(10)); - let heartbeat_2 = get_heartbeat_msgs(Duration::minutes(10)); - let new_state = get_newstate_msg(); - let approve_state = get_approve_state_msg(); + let channel = DUMMY_CHANNEL.clone(); + let leader_heartbeats = vec![get_heartbeat_msg(Duration::minutes(10), channel.spec.validators.leader().id)]; + let follower_heartbeats = vec![get_heartbeat_msg(Duration::minutes(10), channel.spec.validators.leader().id)]; let messages = Messages { leader: LastApprovedResponse { last_approved: Some(LastApproved { - new_state: Some(new_state), + new_state: None, approve_state: None, }), - heartbeats: Some(vec![heartbeat_1]), + heartbeats: Some(leader_heartbeats), }, follower: LastApprovedResponse { last_approved: Some(LastApproved { new_state: None, - approve_state: Some(approve_state), + approve_state: None, }), - heartbeats: Some(vec![heartbeat_2]), + heartbeats: Some(follower_heartbeats), }, recency: Duration::minutes(4), }; @@ -638,17 +638,10 @@ mod test { #[test] fn no_recent_follower_hbs() { - let server = SERVER_POOL.get_server(); - let channel = get_test_channel(&server); - let mut heartbeat_1 = get_heartbeat_msgs(Duration::minutes(10)); // not recent - heartbeat_1.from = channel.spec.validators.leader().id; - let mut heartbeat_2 = get_heartbeat_msgs(Duration::minutes(10)); - heartbeat_2.from = channel.spec.validators.follower().id; - let mut heartbeat_3 = get_heartbeat_msgs(Duration::minutes(10)); - heartbeat_3.from = channel.spec.validators.leader().id; - let mut heartbeat_4 = get_heartbeat_msgs(Duration::minutes(10)); - heartbeat_4.from = channel.spec.validators.follower().id; - let new_state = get_newstate_msg(); + let channel = DUMMY_CHANNEL.clone(); + let leader_heartbeats = vec![get_heartbeat_msg(Duration::minutes(1), channel.spec.validators.leader().id), get_heartbeat_msg(Duration::minutes(1), channel.spec.validators.follower().id)]; + let follower_heartbeats = vec![get_heartbeat_msg(Duration::minutes(10), channel.spec.validators.leader().id), get_heartbeat_msg(Duration::minutes(10), channel.spec.validators.follower().id)]; + let new_state = get_new_state_msg(); let approve_state = get_approve_state_msg(); let messages = Messages { @@ -657,14 +650,14 @@ mod test { new_state: Some(new_state), approve_state: None, }), - heartbeats: Some(vec![heartbeat_1, heartbeat_2]), + heartbeats: Some(leader_heartbeats), }, follower: LastApprovedResponse { last_approved: Some(LastApproved { new_state: None, approve_state: Some(approve_state), }), - heartbeats: Some(vec![heartbeat_3, heartbeat_4]), + heartbeats: Some(follower_heartbeats), }, recency: Duration::minutes(4), }; @@ -677,16 +670,11 @@ mod test { } #[test] - fn no_recent_leader_hb_coming_from_follower() { - let server = SERVER_POOL.get_server(); - let channel = get_test_channel(&server); - let mut heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_1.from = channel.spec.validators.leader().id; - let mut heartbeat_2 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_2.from = channel.spec.validators.leader().id; - let mut heartbeat_3 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_3.from = channel.spec.validators.follower().id; - let new_state = get_newstate_msg(); + fn no_hb_in_leader_where_from_points_to_follower() { + let channel = DUMMY_CHANNEL.clone(); + let leader_heartbeats = vec![get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.leader().id)]; + let follower_heartbeats = vec![get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.leader().id), get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.follower().id)]; + let new_state = get_new_state_msg(); let approve_state = get_approve_state_msg(); let messages = Messages { @@ -695,14 +683,14 @@ mod test { new_state: Some(new_state), approve_state: None, }), - heartbeats: Some(vec![heartbeat_1]), + heartbeats: Some(leader_heartbeats), }, follower: LastApprovedResponse { last_approved: Some(LastApproved { new_state: None, approve_state: Some(approve_state), }), - heartbeats: Some(vec![heartbeat_2, heartbeat_3]), + heartbeats: Some(follower_heartbeats), }, recency: Duration::minutes(4), }; @@ -715,18 +703,12 @@ mod test { } #[test] - fn no_recent_follower_hb_coming_from_leader() { - let server = SERVER_POOL.get_server(); - let channel = get_test_channel(&server); - - let mut heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_1.from = channel.spec.validators.leader().id; - let mut heartbeat_2 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_2.from = channel.spec.validators.follower().id; - let mut heartbeat_3 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_3.from = channel.spec.validators.follower().id; + fn no_hb_in_follower_where_from_points_to_leader() { + let channel = DUMMY_CHANNEL.clone(); + let leader_heartbeats = vec![get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.leader().id), get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.leader().id)]; + let follower_heartbeats = vec![get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.follower().id)]; - let new_state = get_newstate_msg(); + let new_state = get_new_state_msg(); let approve_state = get_approve_state_msg(); let messages = Messages { @@ -735,14 +717,14 @@ mod test { new_state: Some(new_state), approve_state: None, }), - heartbeats: Some(vec![heartbeat_1, heartbeat_2]), + heartbeats: Some(leader_heartbeats), }, follower: LastApprovedResponse { last_approved: Some(LastApproved { new_state: None, approve_state: Some(approve_state), }), - heartbeats: Some(vec![heartbeat_3]), + heartbeats: Some(follower_heartbeats), }, recency: Duration::minutes(4), }; @@ -756,17 +738,10 @@ mod test { #[test] fn recent_hbs_coming_from_both_validators() { - let server = SERVER_POOL.get_server(); - let channel = get_test_channel(&server); - let mut heartbeat_1 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_1.from = channel.spec.validators.leader().id; - let mut heartbeat_2 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_2.from = channel.spec.validators.follower().id; - let mut heartbeat_3 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_3.from = channel.spec.validators.leader().id; - let mut heartbeat_4 = get_heartbeat_msgs(Duration::minutes(0)); - heartbeat_4.from = channel.spec.validators.follower().id; - let new_state = get_newstate_msg(); + let channel = DUMMY_CHANNEL.clone(); + let leader_heartbeats = vec![get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.leader().id), get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.follower().id)]; + let follower_heartbeats = vec![get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.leader().id), get_heartbeat_msg(Duration::minutes(0), channel.spec.validators.follower().id)]; + let new_state = get_new_state_msg(); let approve_state = get_approve_state_msg(); let messages = Messages { @@ -775,14 +750,14 @@ mod test { new_state: Some(new_state), approve_state: None, }), - heartbeats: Some(vec![heartbeat_1, heartbeat_2]), + heartbeats: Some(leader_heartbeats), }, follower: LastApprovedResponse { last_approved: Some(LastApproved { new_state: None, approve_state: Some(approve_state), }), - heartbeats: Some(vec![heartbeat_3, heartbeat_4]), + heartbeats: Some(follower_heartbeats), }, recency: Duration::minutes(4), };