From 21baf8bf2ba4226ba6f7c9db014039dcc09bad34 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Thu, 4 Jul 2024 20:15:27 +0200 Subject: [PATCH 01/40] feat: introduce metrics crate --- Cargo.lock | 10 +++ Cargo.toml | 2 +- fendermint/app/Cargo.toml | 2 + fendermint/ipc-metrics/Cargo.toml | 12 ++++ fendermint/ipc-metrics/src/lib.rs | 38 ++++++++++ fendermint/ipc-metrics/src/register.rs | 96 ++++++++++++++++++++++++++ 6 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 fendermint/ipc-metrics/Cargo.toml create mode 100644 fendermint/ipc-metrics/src/lib.rs create mode 100644 fendermint/ipc-metrics/src/register.rs diff --git a/Cargo.lock b/Cargo.lock index 07f591bd0..56c466aec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2886,7 +2886,11 @@ dependencies = [ "fvm_shared", "hex", "ipc-api", +<<<<<<< HEAD "ipc-observability", +======= + "ipc-metrics", +>>>>>>> 06eb54a0 (feat: introduce metrics crate) "ipc-provider", "ipc_ipld_resolver", "k256 0.11.6", @@ -5053,12 +5057,18 @@ dependencies = [ ] [[package]] +<<<<<<< HEAD name = "ipc-observability" version = "0.1.0" dependencies = [ "hex", "lazy_static", "prometheus", +======= +name = "ipc-metrics" +version = "0.1.0" +dependencies = [ +>>>>>>> 06eb54a0 (feat: introduce metrics crate) "tracing", "tracing-appender", "tracing-subscriber", diff --git a/Cargo.toml b/Cargo.toml index 63dfcf3e5..b449ca177 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ members = [ "fendermint/tracing", "fendermint/vm/*", "fendermint/actors", - "fendermint/actors/chainmetadata", + "fendermint/actors/chainmetadata", "fendermint/ipc-metrics", ] [workspace.package] diff --git a/fendermint/app/Cargo.toml b/fendermint/app/Cargo.toml index b524d15b8..1bc91ff17 100644 --- a/fendermint/app/Cargo.toml +++ b/fendermint/app/Cargo.toml @@ -63,6 +63,8 @@ fendermint_vm_message = { path = "../vm/message" } fendermint_vm_resolver = { path = "../vm/resolver" } fendermint_vm_snapshot = { path = "../vm/snapshot" } fendermint_vm_topdown = { path = "../vm/topdown" } +ipc-metrics = { path = "../ipc-metrics" } + fvm = { workspace = true } diff --git a/fendermint/ipc-metrics/Cargo.toml b/fendermint/ipc-metrics/Cargo.toml new file mode 100644 index 000000000..df239490b --- /dev/null +++ b/fendermint/ipc-metrics/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ipc-metrics" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +license-file.workspace = true + +[dependencies] +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +tracing-appender = { workspace = true } diff --git a/fendermint/ipc-metrics/src/lib.rs b/fendermint/ipc-metrics/src/lib.rs new file mode 100644 index 000000000..d2329d0da --- /dev/null +++ b/fendermint/ipc-metrics/src/lib.rs @@ -0,0 +1,38 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +pub mod register; + +use std::fmt::Display; +use tracing::{debug, error, info, trace, warn}; + +pub trait Recordable { + fn record_metrics(&self); +} + +pub enum Level { + Trace, + Debug, + Info, + Warn, + Error, +} + +pub trait Traceable { + fn level(&self) -> Level; +} + +pub fn emit(trace: T) +where + T: Recordable + Traceable + Display, +{ + match trace.level() { + Level::Trace => trace!(event = %trace), + Level::Debug => debug!(event = %trace), + Level::Info => info!(event = %trace), + Level::Warn => warn!(event = %trace), + Level::Error => error!(event = %trace), + } + + trace.record_metrics(); +} diff --git a/fendermint/ipc-metrics/src/register.rs b/fendermint/ipc-metrics/src/register.rs new file mode 100644 index 000000000..f4334ad21 --- /dev/null +++ b/fendermint/ipc-metrics/src/register.rs @@ -0,0 +1,96 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use std::num::NonZeroUsize; +pub use tracing_appender::non_blocking::WorkerGuard; +use tracing_appender::rolling::{RollingFileAppender, Rotation}; +use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Layer}; + +#[derive(Default)] +pub struct JournalOpts<'a> { + pub enabled: bool, + pub directory: Option<&'a str>, + pub max_log_files: Option, + pub rotation: Option<&'a str>, + pub filters: Option>, +} + +fn appender_from_opts(opts: &JournalOpts<'_>) -> RollingFileAppender { + let directory = opts.directory.expect("journal directory must be set"); + let mut appender = RollingFileAppender::builder().filename_suffix("journal"); + + if let Some(max_log_files) = opts.max_log_files { + appender = appender.max_log_files( + NonZeroUsize::new(max_log_files) + .expect("max_log_files must be greater than 0") + .into(), + ); + }; + + if let Some(rotation_str) = opts.rotation { + let rotation = match rotation_str { + "minutely" => Rotation::DAILY, + "hourly" => Rotation::HOURLY, + "daily" => Rotation::DAILY, + "never" => Rotation::NEVER, + _ => panic!("invalid rotation: {}", rotation_str), + }; + + appender = appender.rotation(rotation); + }; + + appender + .build(directory) + .expect("failed to create journal appender") +} + +// register a tracing subscriber with the given options +// returns a guard that must be kept alive for the duration of the program (because it's non-blocking and needs to flush) +pub fn register_tracing_subscriber( + console_filter: EnvFilter, + opts: &JournalOpts<'_>, +) -> Option { + // log all traces to stderr (reserving stdout for any actual output such as from the CLI commands) + // TODO Karel - do we want to always allow it or should we make it configurable? + let console_layer = fmt::layer() + .with_writer(std::io::stderr) + .with_target(false) + .with_file(true) + .with_line_number(true) + .with_filter(console_filter); + + // add a file layer if log_dir is set + let (file_layer, file_guard) = if opts.enabled { + let (non_blocking, file_guard) = tracing_appender::non_blocking(appender_from_opts(opts)); + + let mut file_filter = EnvFilter::from_default_env(); + + if let Some(domain_filter) = &opts.filters { + for domain in domain_filter { + file_filter = file_filter.add_directive(domain.parse().unwrap()); + } + } + + let file_layer = fmt::layer() + .json() + .with_writer(non_blocking) + .with_span_events(fmt::format::FmtSpan::CLOSE) + .with_target(false) + .with_file(true) + .with_line_number(true) + .with_filter(file_filter); + + (Some(file_layer), Some(file_guard)) + } else { + (None, None) + }; + + let registry = tracing_subscriber::registry() + .with(console_layer) + .with(file_layer); + + tracing::subscriber::set_global_default(registry) + .expect("Unable to set a global tracing subscriber"); + + file_guard +} From 4d0905ee22ea6d91a15f2d876aedae39077a6d70 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 5 Jul 2024 17:08:12 +0200 Subject: [PATCH 02/40] feat: improve library with macros, add top down events --- Cargo.lock | 10 --- Cargo.toml | 4 +- fendermint/app/Cargo.toml | 1 - fendermint/ipc-metrics/Cargo.toml | 12 ---- fendermint/ipc-metrics/src/lib.rs | 38 ---------- fendermint/ipc-metrics/src/register.rs | 96 -------------------------- 6 files changed, 2 insertions(+), 159 deletions(-) delete mode 100644 fendermint/ipc-metrics/Cargo.toml delete mode 100644 fendermint/ipc-metrics/src/lib.rs delete mode 100644 fendermint/ipc-metrics/src/register.rs diff --git a/Cargo.lock b/Cargo.lock index 56c466aec..07f591bd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2886,11 +2886,7 @@ dependencies = [ "fvm_shared", "hex", "ipc-api", -<<<<<<< HEAD "ipc-observability", -======= - "ipc-metrics", ->>>>>>> 06eb54a0 (feat: introduce metrics crate) "ipc-provider", "ipc_ipld_resolver", "k256 0.11.6", @@ -5057,18 +5053,12 @@ dependencies = [ ] [[package]] -<<<<<<< HEAD name = "ipc-observability" version = "0.1.0" dependencies = [ "hex", "lazy_static", "prometheus", -======= -name = "ipc-metrics" -version = "0.1.0" -dependencies = [ ->>>>>>> 06eb54a0 (feat: introduce metrics crate) "tracing", "tracing-appender", "tracing-subscriber", diff --git a/Cargo.toml b/Cargo.toml index b449ca177..4727a5643 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,4 @@ [workspace] -resolver = "2" members = [ # contracts "contracts/binding", @@ -34,7 +33,7 @@ members = [ "fendermint/tracing", "fendermint/vm/*", "fendermint/actors", - "fendermint/actors/chainmetadata", "fendermint/ipc-metrics", + "fendermint/actors/chainmetadata", ] [workspace.package] @@ -246,3 +245,4 @@ opt-level = "z" strip = true codegen-units = 1 incremental = false +resolver = "2" diff --git a/fendermint/app/Cargo.toml b/fendermint/app/Cargo.toml index 1bc91ff17..8e884e69f 100644 --- a/fendermint/app/Cargo.toml +++ b/fendermint/app/Cargo.toml @@ -63,7 +63,6 @@ fendermint_vm_message = { path = "../vm/message" } fendermint_vm_resolver = { path = "../vm/resolver" } fendermint_vm_snapshot = { path = "../vm/snapshot" } fendermint_vm_topdown = { path = "../vm/topdown" } -ipc-metrics = { path = "../ipc-metrics" } diff --git a/fendermint/ipc-metrics/Cargo.toml b/fendermint/ipc-metrics/Cargo.toml deleted file mode 100644 index df239490b..000000000 --- a/fendermint/ipc-metrics/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "ipc-metrics" -version = "0.1.0" -authors.workspace = true -edition.workspace = true -license.workspace = true -license-file.workspace = true - -[dependencies] -tracing = { workspace = true } -tracing-subscriber = { workspace = true } -tracing-appender = { workspace = true } diff --git a/fendermint/ipc-metrics/src/lib.rs b/fendermint/ipc-metrics/src/lib.rs deleted file mode 100644 index d2329d0da..000000000 --- a/fendermint/ipc-metrics/src/lib.rs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -pub mod register; - -use std::fmt::Display; -use tracing::{debug, error, info, trace, warn}; - -pub trait Recordable { - fn record_metrics(&self); -} - -pub enum Level { - Trace, - Debug, - Info, - Warn, - Error, -} - -pub trait Traceable { - fn level(&self) -> Level; -} - -pub fn emit(trace: T) -where - T: Recordable + Traceable + Display, -{ - match trace.level() { - Level::Trace => trace!(event = %trace), - Level::Debug => debug!(event = %trace), - Level::Info => info!(event = %trace), - Level::Warn => warn!(event = %trace), - Level::Error => error!(event = %trace), - } - - trace.record_metrics(); -} diff --git a/fendermint/ipc-metrics/src/register.rs b/fendermint/ipc-metrics/src/register.rs deleted file mode 100644 index f4334ad21..000000000 --- a/fendermint/ipc-metrics/src/register.rs +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -use std::num::NonZeroUsize; -pub use tracing_appender::non_blocking::WorkerGuard; -use tracing_appender::rolling::{RollingFileAppender, Rotation}; -use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter, Layer}; - -#[derive(Default)] -pub struct JournalOpts<'a> { - pub enabled: bool, - pub directory: Option<&'a str>, - pub max_log_files: Option, - pub rotation: Option<&'a str>, - pub filters: Option>, -} - -fn appender_from_opts(opts: &JournalOpts<'_>) -> RollingFileAppender { - let directory = opts.directory.expect("journal directory must be set"); - let mut appender = RollingFileAppender::builder().filename_suffix("journal"); - - if let Some(max_log_files) = opts.max_log_files { - appender = appender.max_log_files( - NonZeroUsize::new(max_log_files) - .expect("max_log_files must be greater than 0") - .into(), - ); - }; - - if let Some(rotation_str) = opts.rotation { - let rotation = match rotation_str { - "minutely" => Rotation::DAILY, - "hourly" => Rotation::HOURLY, - "daily" => Rotation::DAILY, - "never" => Rotation::NEVER, - _ => panic!("invalid rotation: {}", rotation_str), - }; - - appender = appender.rotation(rotation); - }; - - appender - .build(directory) - .expect("failed to create journal appender") -} - -// register a tracing subscriber with the given options -// returns a guard that must be kept alive for the duration of the program (because it's non-blocking and needs to flush) -pub fn register_tracing_subscriber( - console_filter: EnvFilter, - opts: &JournalOpts<'_>, -) -> Option { - // log all traces to stderr (reserving stdout for any actual output such as from the CLI commands) - // TODO Karel - do we want to always allow it or should we make it configurable? - let console_layer = fmt::layer() - .with_writer(std::io::stderr) - .with_target(false) - .with_file(true) - .with_line_number(true) - .with_filter(console_filter); - - // add a file layer if log_dir is set - let (file_layer, file_guard) = if opts.enabled { - let (non_blocking, file_guard) = tracing_appender::non_blocking(appender_from_opts(opts)); - - let mut file_filter = EnvFilter::from_default_env(); - - if let Some(domain_filter) = &opts.filters { - for domain in domain_filter { - file_filter = file_filter.add_directive(domain.parse().unwrap()); - } - } - - let file_layer = fmt::layer() - .json() - .with_writer(non_blocking) - .with_span_events(fmt::format::FmtSpan::CLOSE) - .with_target(false) - .with_file(true) - .with_line_number(true) - .with_filter(file_filter); - - (Some(file_layer), Some(file_guard)) - } else { - (None, None) - }; - - let registry = tracing_subscriber::registry() - .with(console_layer) - .with(file_layer); - - tracing::subscriber::set_global_default(registry) - .expect("Unable to set a global tracing subscriber"); - - file_guard -} From b20f4872634cccf99ecd512f8418982ea7638d0f Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Mon, 8 Jul 2024 15:53:27 +0200 Subject: [PATCH 03/40] feat: emit metrics --- fendermint/vm/topdown/src/proxy.rs | 75 ++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/fendermint/vm/topdown/src/proxy.rs b/fendermint/vm/topdown/src/proxy.rs index 94a8e3177..86f6cb7b0 100644 --- a/fendermint/vm/topdown/src/proxy.rs +++ b/fendermint/vm/topdown/src/proxy.rs @@ -68,22 +68,35 @@ impl IPCProviderProxy { #[async_trait] impl ParentQueryProxy for IPCProviderProxy { async fn get_chain_head_height(&self) -> anyhow::Result { - let height = self.ipc_provider.chain_head(&self.parent_subnet).await?; - Ok(height as BlockHeight) + emit_event_with_latency(&self.parent_subnet.to_string(), "chain_head", || async { + let height = self.ipc_provider.chain_head(&self.parent_subnet).await?; + Ok(height as BlockHeight) + }) + .await } /// Get the genesis epoch of the child subnet, i.e. the epoch that the subnet was created in /// the parent subnet. async fn get_genesis_epoch(&self) -> anyhow::Result { - let height = self.ipc_provider.genesis_epoch(&self.child_subnet).await?; - Ok(height as BlockHeight) + emit_event_with_latency(&self.parent_subnet.to_string(), "genesis_epoch", || async { + let height = self.ipc_provider.genesis_epoch(&self.child_subnet).await?; + Ok(height as BlockHeight) + }) + .await } /// Getting the block hash at the target height. async fn get_block_hash(&self, height: BlockHeight) -> anyhow::Result { - self.ipc_provider - .get_block_hash(&self.parent_subnet, height as ChainEpoch) - .await + emit_event_with_latency( + &self.parent_subnet.to_string(), + "get_block_hash", + || async { + self.ipc_provider + .get_block_hash(&self.parent_subnet, height as ChainEpoch) + .await + }, + ) + .await } /// Get the top down messages from the starting to the ending height. @@ -91,14 +104,21 @@ impl ParentQueryProxy for IPCProviderProxy { &self, height: BlockHeight, ) -> anyhow::Result>> { - self.ipc_provider - .get_top_down_msgs(&self.child_subnet, height as ChainEpoch) - .await - .map(|mut v| { - // sort ascending, we dont assume the changes are ordered - v.value.sort_by(|a, b| a.nonce.cmp(&b.nonce)); - v - }) + emit_event_with_latency( + &self.parent_subnet.to_string(), + "get_top_down_msgs", + || async { + self.ipc_provider + .get_top_down_msgs(&self.child_subnet, height as ChainEpoch) + .await + .map(|mut v| { + // sort ascending, we dont assume the changes are ordered + v.value.sort_by(|a, b| a.nonce.cmp(&b.nonce)); + v + }) + }, + ) + .await } /// Get the validator set at the specified height. @@ -106,15 +126,22 @@ impl ParentQueryProxy for IPCProviderProxy { &self, height: BlockHeight, ) -> anyhow::Result>> { - self.ipc_provider - .get_validator_changeset(&self.child_subnet, height as ChainEpoch) - .await - .map(|mut v| { - // sort ascending, we dont assume the changes are ordered - v.value - .sort_by(|a, b| a.configuration_number.cmp(&b.configuration_number)); - v - }) + emit_event_with_latency( + &self.parent_subnet.to_string(), + "get_validator_changeset", + || async { + self.ipc_provider + .get_validator_changeset(&self.child_subnet, height as ChainEpoch) + .await + .map(|mut v| { + // sort ascending, we dont assume the changes are ordered + v.value + .sort_by(|a, b| a.configuration_number.cmp(&b.configuration_number)); + v + }) + }, + ) + .await } } From 52f195c0bfc25b3b1f77fe589f25cf73be87c693 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Tue, 9 Jul 2024 13:04:24 +0200 Subject: [PATCH 04/40] feat: add remaining top down events --- fendermint/vm/topdown/src/observe.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fendermint/vm/topdown/src/observe.rs b/fendermint/vm/topdown/src/observe.rs index 5a0caa66e..e2d9582fb 100644 --- a/fendermint/vm/topdown/src/observe.rs +++ b/fendermint/vm/topdown/src/observe.rs @@ -1,10 +1,12 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use ethers::core::k256::elliptic_curve::rand_core::block; use ipc_observability::{ impl_traceable, impl_traceables, lazy_static, register_metrics, serde::HexEncodableBlockHash, Recordable, TraceLevel, Traceable, }; +use ipc_provider::lotus::message::chain::Block; use prometheus::{ register_histogram_vec, register_int_counter_vec, register_int_gauge, register_int_gauge_vec, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec, Registry, From e0cb9fc14ec4c8d4e85463a6a5394c8f7751b1d6 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Tue, 9 Jul 2024 16:41:29 +0200 Subject: [PATCH 05/40] feat: remove unused code & fix build --- Cargo.toml | 2 +- fendermint/app/src/cmd/run.rs | 4 +- fendermint/app/src/events.rs | 26 ++++ fendermint/app/src/metrics/tracing.rs | 208 ++++++++++++++++++++++++++ fendermint/vm/topdown/src/observe.rs | 2 - 5 files changed, 237 insertions(+), 5 deletions(-) create mode 100644 fendermint/app/src/events.rs create mode 100644 fendermint/app/src/metrics/tracing.rs diff --git a/Cargo.toml b/Cargo.toml index 4727a5643..63dfcf3e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] +resolver = "2" members = [ # contracts "contracts/binding", @@ -245,4 +246,3 @@ opt-level = "z" strip = true codegen-units = 1 incremental = false -resolver = "2" diff --git a/fendermint/app/src/cmd/run.rs b/fendermint/app/src/cmd/run.rs index 87c1e11cd..adde7dccf 100644 --- a/fendermint/app/src/cmd/run.rs +++ b/fendermint/app/src/cmd/run.rs @@ -23,7 +23,7 @@ use fendermint_vm_snapshot::{SnapshotManager, SnapshotParams}; use fendermint_vm_topdown::observe::register_metrics as register_topdown_metrics; use fendermint_vm_topdown::proxy::{IPCProviderProxy, IPCProviderProxyWithLatency}; use fendermint_vm_topdown::sync::launch_polling_syncer; -use fendermint_vm_topdown::voting::{publish_vote_loop, Error as VoteError, VoteTally}; +use fendermint_vm_topdown::voting::{publish_vote_loop, VoteTally}; use fendermint_vm_topdown::{CachedFinalityProvider, IPCParentFinality, Toggle}; use fvm_shared::address::{current_network, Address, Network}; use ipc_ipld_resolver::{Event as ResolverEvent, VoteRecord}; @@ -539,7 +539,7 @@ async fn dispatch_vote( tracing::debug!("ignoring vote; topdown disabled"); return; } - let res = atomically_or_err(|| { + let _res = atomically_or_err(|| { parent_finality_votes.add_vote( vote.public_key.clone(), f.height, diff --git a/fendermint/app/src/events.rs b/fendermint/app/src/events.rs new file mode 100644 index 000000000..84fc2c8cd --- /dev/null +++ b/fendermint/app/src/events.rs @@ -0,0 +1,26 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::BlockHeight; + +/// Re-export other events, just to provide the visibility of where they are. +pub use fendermint_vm_event::{ + NewBottomUpCheckpoint, ParentFinalityCommitted, ParentFinalityMissingQuorum, +}; + +/// Hex encoded block hash. +pub type BlockHashHex<'a> = &'a str; + +#[derive(Debug, Default)] +pub struct ProposalProcessed<'a> { + pub is_accepted: bool, + pub block_height: BlockHeight, + pub block_hash: BlockHashHex<'a>, + pub num_txs: usize, + pub proposer: &'a str, +} + +#[derive(Debug, Default)] +pub struct NewBlock { + pub block_height: BlockHeight, +} diff --git a/fendermint/app/src/metrics/tracing.rs b/fendermint/app/src/metrics/tracing.rs new file mode 100644 index 000000000..3dfca6949 --- /dev/null +++ b/fendermint/app/src/metrics/tracing.rs @@ -0,0 +1,208 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT +//! Subscribing to tracing events and turning them into metrics. + +use std::marker::PhantomData; + +use tracing::{Event, Subscriber}; +use tracing_subscriber::{filter, layer, registry::LookupSpan, Layer}; + +use super::prometheus::app as am; +use crate::events::*; + +/// Create a layer that handles events by incrementing metrics. +pub fn layer() -> impl Layer +where + S: Subscriber, + for<'a> S: LookupSpan<'a>, +{ + MetricsLayer::new().with_filter(filter::filter_fn(|md| md.name().starts_with("event::"))) +} + +struct MetricsLayer { + _subscriber: PhantomData, +} + +impl MetricsLayer { + pub fn new() -> Self { + Self { + _subscriber: PhantomData, + } + } +} + +/// Check that the field exist on a type; if it doesn't this won't compile. +/// This ensures that we're mapping fields with the correct name. +macro_rules! check_field { + ($event_ty:ident :: $field:ident) => {{ + if false { + #[allow(clippy::needless_update)] + let _event = $event_ty { + $field: Default::default(), + ..Default::default() + }; + } + }}; +} + +/// Set a gague to an absolute value based on a field in an event. +macro_rules! set_gauge { + ($event:ident, $event_ty:ident :: $field:ident, $gauge:expr) => { + check_field!($event_ty::$field); + let mut fld = visitors::FindU64::new(stringify!($field)); + $event.record(&mut fld); + $gauge.set(fld.value as i64); + }; +} + +/// Increment a counter by the value of a field in the event. +macro_rules! inc_counter { + ($event:ident, $event_ty:ident :: $field:ident, $counter:expr) => { + check_field!($event_ty::$field); + let mut fld = visitors::FindU64::new(stringify!($field)); + $event.record(&mut fld); + $counter.inc_by(fld.value); + }; +} + +/// Increment a counter by 1. +/// +/// The field is ignored, it's only here because of how the macros look like. +macro_rules! inc1_counter { + ($event:ident, $event_ty:ident :: $field:ident, $counter:expr) => { + check_field!($event_ty::$field); + $counter.inc(); + }; +} + +/// Produce the prefixed event name from the type name. +macro_rules! event_name { + ($event_ty:ident) => { + concat!("event::", stringify!($event_ty)) + }; +} + +/// Call one of the macros that set values on a metric. +macro_rules! event_mapping { + ($op:ident, $event:ident, $event_ty:ident :: $field:ident, $metric:expr) => { + $op!($event, $event_ty::$field, $metric); + }; +} + +/// Match the event name to event DTO types and within the map fields to metrics. +macro_rules! event_match { + ($event:ident { $( $event_ty:ident { $( $field:ident => $op:ident ! $metric:expr ),* $(,)? } ),* } ) => { + match $event.metadata().name() { + $( + event_name!($event_ty) => { + $( + event_mapping!($op, $event, $event_ty :: $field, $metric); + )* + } + )* + _ => {} + } + }; +} + +impl Layer for MetricsLayer { + fn on_event(&self, event: &Event<'_>, _ctx: layer::Context<'_, S>) { + event_match!(event { + ParentFinalityCommitted { + block_height => set_gauge ! &am::TOPDOWN_FINALIZED_BLOCK_HEIGHT, + }, + ParentFinalityMissingQuorum { + block_hash => inc1_counter ! &am::TOPDOWN_FINALITY_MISSING_QUORUM, + }, + NewBottomUpCheckpoint { + block_height => set_gauge ! &am::BOTTOMUP_CKPT_BLOCK_HEIGHT, + next_configuration_number => set_gauge ! &am::BOTTOMUP_CKPT_CONFIG_NUM, + num_msgs => inc_counter ! &am::BOTTOMUP_CKPT_NUM_MSGS, + }, + NewBlock { + block_height => set_gauge ! &am::ABCI_COMMITTED_BLOCK_HEIGHT + } + }); + } +} + +mod visitors { + use tracing::field::{Field, Visit}; + + pub struct FindU64<'a> { + pub name: &'a str, + pub value: u64, + } + + impl<'a> FindU64<'a> { + pub fn new(name: &'a str) -> Self { + Self { name, value: 0 } + } + } + + // Looking for multiple values because the callsite might be passed as a literal which turns into an i64 for example. + impl<'a> Visit for FindU64<'a> { + fn record_u64(&mut self, field: &Field, value: u64) { + if field.name() == self.name { + self.value = value; + } + } + + fn record_i64(&mut self, field: &Field, value: i64) { + if field.name() == self.name { + self.value = value as u64; + } + } + + fn record_i128(&mut self, field: &Field, value: i128) { + if field.name() == self.name { + self.value = value as u64; + } + } + + fn record_u128(&mut self, field: &Field, value: u128) { + if field.name() == self.name { + self.value = value as u64; + } + } + + fn record_debug(&mut self, _field: &Field, _value: &dyn std::fmt::Debug) {} + } +} + +#[cfg(test)] +mod tests { + use fendermint_tracing::emit; + use fendermint_vm_event::ParentFinalityCommitted; + use prometheus::IntGauge; + use tracing_subscriber::layer::SubscriberExt; + + #[test] + fn test_metrics_layer() { + let gauge: &IntGauge = &super::super::prometheus::app::TOPDOWN_FINALIZED_BLOCK_HEIGHT; + + let v0 = gauge.get(); + gauge.inc(); + let v1 = gauge.get(); + assert!(v1 > v0, "gague should change without being registered"); + + let block_height = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + + let subscriber = tracing_subscriber::registry().with(super::layer()); + + tracing::subscriber::with_default(subscriber, || { + emit! { + ParentFinalityCommitted { block_height, block_hash: "metrics-test-block" } + } + }); + + assert_eq!( + gauge.get() as u64, + block_height, + "metrics should be captured" + ); + } +} diff --git a/fendermint/vm/topdown/src/observe.rs b/fendermint/vm/topdown/src/observe.rs index e2d9582fb..5a0caa66e 100644 --- a/fendermint/vm/topdown/src/observe.rs +++ b/fendermint/vm/topdown/src/observe.rs @@ -1,12 +1,10 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use ethers::core::k256::elliptic_curve::rand_core::block; use ipc_observability::{ impl_traceable, impl_traceables, lazy_static, register_metrics, serde::HexEncodableBlockHash, Recordable, TraceLevel, Traceable, }; -use ipc_provider::lotus::message::chain::Block; use prometheus::{ register_histogram_vec, register_int_counter_vec, register_int_gauge, register_int_gauge_vec, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec, Registry, From 5ca78d7b3b21ec090ba3fe4876329c0c82fdc55d Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Wed, 10 Jul 2024 13:44:36 +0200 Subject: [PATCH 06/40] feat: address comments --- fendermint/vm/topdown/src/observe.rs | 3 +++ fendermint/vm/topdown/src/voting.rs | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/fendermint/vm/topdown/src/observe.rs b/fendermint/vm/topdown/src/observe.rs index 5a0caa66e..cce0b3b2e 100644 --- a/fendermint/vm/topdown/src/observe.rs +++ b/fendermint/vm/topdown/src/observe.rs @@ -1,6 +1,9 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use hex; +use std::fmt; + use ipc_observability::{ impl_traceable, impl_traceables, lazy_static, register_metrics, serde::HexEncodableBlockHash, Recordable, TraceLevel, Traceable, diff --git a/fendermint/vm/topdown/src/voting.rs b/fendermint/vm/topdown/src/voting.rs index 793c2ab24..f8e0a6522 100644 --- a/fendermint/vm/topdown/src/voting.rs +++ b/fendermint/vm/topdown/src/voting.rs @@ -8,8 +8,8 @@ use std::hash::Hash; use std::{fmt::Debug, time::Duration}; use crate::observe::{ - ParentFinalityCommitted, ParentFinalityPeerQuorumReached, ParentFinalityPeerVoteReceived, - ParentFinalityPeerVoteSent, + HexEncodableBlockHash, ParentFinalityCommitted, ParentFinalityPeerQuorumReached, + ParentFinalityPeerVoteReceived, ParentFinalityPeerVoteSent, }; use crate::{BlockHash, BlockHeight}; use ipc_observability::{emit, serde::HexEncodableBlockHash}; From 9500325ccd623f39dfa64c30e9240fb92eab630b Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Wed, 10 Jul 2024 16:04:42 +0200 Subject: [PATCH 07/40] feat: use latency wrapper instead --- fendermint/app/src/cmd/run.rs | 2 +- fendermint/vm/topdown/src/proxy.rs | 120 +++++++++++++++++++---------- 2 files changed, 82 insertions(+), 40 deletions(-) diff --git a/fendermint/app/src/cmd/run.rs b/fendermint/app/src/cmd/run.rs index adde7dccf..be4f07719 100644 --- a/fendermint/app/src/cmd/run.rs +++ b/fendermint/app/src/cmd/run.rs @@ -259,7 +259,7 @@ async fn run(settings: Settings) -> anyhow::Result<()> { CachedFinalityProvider::uninitialized(config.clone(), ipc_provider.clone()).await?; let p = Arc::new(Toggle::enabled(finality_provider)); - (p, Some((ipc_provider, config))) + (p, Some((ipc_provider_with_latency, config))) } else { info!("topdown finality disabled"); (Arc::new(Toggle::disabled()), None) diff --git a/fendermint/vm/topdown/src/proxy.rs b/fendermint/vm/topdown/src/proxy.rs index 86f6cb7b0..abbd3812f 100644 --- a/fendermint/vm/topdown/src/proxy.rs +++ b/fendermint/vm/topdown/src/proxy.rs @@ -68,33 +68,94 @@ impl IPCProviderProxy { #[async_trait] impl ParentQueryProxy for IPCProviderProxy { async fn get_chain_head_height(&self) -> anyhow::Result { - emit_event_with_latency(&self.parent_subnet.to_string(), "chain_head", || async { - let height = self.ipc_provider.chain_head(&self.parent_subnet).await?; - Ok(height as BlockHeight) - }) - .await + let height = self.ipc_provider.chain_head(&self.parent_subnet).await?; + Ok(height as BlockHeight) } /// Get the genesis epoch of the child subnet, i.e. the epoch that the subnet was created in /// the parent subnet. async fn get_genesis_epoch(&self) -> anyhow::Result { - emit_event_with_latency(&self.parent_subnet.to_string(), "genesis_epoch", || async { - let height = self.ipc_provider.genesis_epoch(&self.child_subnet).await?; - Ok(height as BlockHeight) - }) - .await + let height = self.ipc_provider.genesis_epoch(&self.child_subnet).await?; + Ok(height as BlockHeight) } /// Getting the block hash at the target height. + async fn get_block_hash(&self, height: BlockHeight) -> anyhow::Result { + self.ipc_provider + .get_block_hash(&self.parent_subnet, height as ChainEpoch) + .await + } + + /// Get the top down messages from the starting to the ending height. + async fn get_top_down_msgs( + &self, + height: BlockHeight, + ) -> anyhow::Result>> { + self.ipc_provider + .get_top_down_msgs(&self.child_subnet, height as ChainEpoch) + .await + .map(|mut v| { + // sort ascending, we dont assume the changes are ordered + v.value.sort_by(|a, b| a.nonce.cmp(&b.nonce)); + v + }) + } + + /// Get the validator set at the specified height. + async fn get_validator_changes( + &self, + height: BlockHeight, + ) -> anyhow::Result>> { + self.ipc_provider + .get_validator_changeset(&self.child_subnet, height as ChainEpoch) + .await + .map(|mut v| { + // sort ascending, we dont assume the changes are ordered + v.value + .sort_by(|a, b| a.configuration_number.cmp(&b.configuration_number)); + v + }) + } +} + +pub struct IPCProviderProxyWithLatency { + inner: IPCProviderProxy, +} + +impl IPCProviderProxyWithLatency { + pub fn new(inner: IPCProviderProxy) -> Self { + Self { inner } + } +} + +#[async_trait] +impl ParentQueryProxy for IPCProviderProxyWithLatency { + #[instrument(skip(self))] + async fn get_chain_head_height(&self) -> anyhow::Result { + emit_event_with_latency( + &self.inner.parent_subnet.to_string(), + "chain_head", + || async { self.inner.get_chain_head_height().await }, + ) + .await + } + + #[instrument(skip(self))] + async fn get_genesis_epoch(&self) -> anyhow::Result { + emit_event_with_latency( + &self.inner.parent_subnet.to_string(), + "genesis_epoch", + || async { self.inner.get_genesis_epoch().await }, + ) + .await + } + + #[instrument(skip(self))] async fn get_block_hash(&self, height: BlockHeight) -> anyhow::Result { emit_event_with_latency( - &self.parent_subnet.to_string(), + &self.inner.parent_subnet.to_string(), "get_block_hash", - || async { - self.ipc_provider - .get_block_hash(&self.parent_subnet, height as ChainEpoch) - .await - }, + || async { self.inner.get_block_hash(height).await }, ) .await } @@ -105,18 +166,9 @@ impl ParentQueryProxy for IPCProviderProxy { height: BlockHeight, ) -> anyhow::Result>> { emit_event_with_latency( - &self.parent_subnet.to_string(), + &self.inner.parent_subnet.to_string(), "get_top_down_msgs", - || async { - self.ipc_provider - .get_top_down_msgs(&self.child_subnet, height as ChainEpoch) - .await - .map(|mut v| { - // sort ascending, we dont assume the changes are ordered - v.value.sort_by(|a, b| a.nonce.cmp(&b.nonce)); - v - }) - }, + || async { self.inner.get_top_down_msgs(height).await }, ) .await } @@ -127,19 +179,9 @@ impl ParentQueryProxy for IPCProviderProxy { height: BlockHeight, ) -> anyhow::Result>> { emit_event_with_latency( - &self.parent_subnet.to_string(), + &self.inner.parent_subnet.to_string(), "get_validator_changeset", - || async { - self.ipc_provider - .get_validator_changeset(&self.child_subnet, height as ChainEpoch) - .await - .map(|mut v| { - // sort ascending, we dont assume the changes are ordered - v.value - .sort_by(|a, b| a.configuration_number.cmp(&b.configuration_number)); - v - }) - }, + || async { self.inner.get_validator_changes(height).await }, ) .await } From ff3c09c1d558fecb0b6c31fbac97d29a108e5c77 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Thu, 11 Jul 2024 22:22:43 +0200 Subject: [PATCH 08/40] feat: add config filters --- fendermint/app/src/main.rs | 3 ++ fendermint/app/src/metrics/tracing.rs | 53 --------------------------- fendermint/vm/topdown/src/observe.rs | 53 +++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 53 deletions(-) diff --git a/fendermint/app/src/main.rs b/fendermint/app/src/main.rs index 117fa3121..13a86b74b 100644 --- a/fendermint/app/src/main.rs +++ b/fendermint/app/src/main.rs @@ -48,6 +48,9 @@ async fn main() { init_panic_handler(); + use fendermint_vm_topdown::observe::emit_all; + emit_all(); + if let Err(e) = cmd::exec(&opts).await { tracing::error!("failed to execute {:?}: {e:?}", opts); std::process::exit(fendermint_app::AppExitCode::UnknownError as i32); diff --git a/fendermint/app/src/metrics/tracing.rs b/fendermint/app/src/metrics/tracing.rs index 3dfca6949..397092396 100644 --- a/fendermint/app/src/metrics/tracing.rs +++ b/fendermint/app/src/metrics/tracing.rs @@ -65,16 +65,6 @@ macro_rules! inc_counter { }; } -/// Increment a counter by 1. -/// -/// The field is ignored, it's only here because of how the macros look like. -macro_rules! inc1_counter { - ($event:ident, $event_ty:ident :: $field:ident, $counter:expr) => { - check_field!($event_ty::$field); - $counter.inc(); - }; -} - /// Produce the prefixed event name from the type name. macro_rules! event_name { ($event_ty:ident) => { @@ -108,12 +98,6 @@ macro_rules! event_match { impl Layer for MetricsLayer { fn on_event(&self, event: &Event<'_>, _ctx: layer::Context<'_, S>) { event_match!(event { - ParentFinalityCommitted { - block_height => set_gauge ! &am::TOPDOWN_FINALIZED_BLOCK_HEIGHT, - }, - ParentFinalityMissingQuorum { - block_hash => inc1_counter ! &am::TOPDOWN_FINALITY_MISSING_QUORUM, - }, NewBottomUpCheckpoint { block_height => set_gauge ! &am::BOTTOMUP_CKPT_BLOCK_HEIGHT, next_configuration_number => set_gauge ! &am::BOTTOMUP_CKPT_CONFIG_NUM, @@ -169,40 +153,3 @@ mod visitors { fn record_debug(&mut self, _field: &Field, _value: &dyn std::fmt::Debug) {} } } - -#[cfg(test)] -mod tests { - use fendermint_tracing::emit; - use fendermint_vm_event::ParentFinalityCommitted; - use prometheus::IntGauge; - use tracing_subscriber::layer::SubscriberExt; - - #[test] - fn test_metrics_layer() { - let gauge: &IntGauge = &super::super::prometheus::app::TOPDOWN_FINALIZED_BLOCK_HEIGHT; - - let v0 = gauge.get(); - gauge.inc(); - let v1 = gauge.get(); - assert!(v1 > v0, "gague should change without being registered"); - - let block_height = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(); - - let subscriber = tracing_subscriber::registry().with(super::layer()); - - tracing::subscriber::with_default(subscriber, || { - emit! { - ParentFinalityCommitted { block_height, block_hash: "metrics-test-block" } - } - }); - - assert_eq!( - gauge.get() as u64, - block_height, - "metrics should be captured" - ); - } -} diff --git a/fendermint/vm/topdown/src/observe.rs b/fendermint/vm/topdown/src/observe.rs index cce0b3b2e..613864f26 100644 --- a/fendermint/vm/topdown/src/observe.rs +++ b/fendermint/vm/topdown/src/observe.rs @@ -151,6 +151,59 @@ impl Recordable for ParentFinalityCommitted<'_> { } } +pub fn emit_all() { + use ipc_observability::emit; + + emit(ParentRpcCalled { + source: "source", + json_rpc: "json_rpc", + method: "method", + status: "status", + latency: 1.0, + }); + + let hash = vec![0u8; 32]; + + emit(ParentFinalityAcquired { + source: "parent-finality", + is_null: false, + block_height: 10, + block_hash: Some(HexEncodableBlockHash(hash.clone())), + commitment_hash: Some(HexEncodableBlockHash(hash.clone())), + num_msgs: 5, + num_validator_changes: 6, + }); + + emit(ParentFinalityPeerVoteReceived { + validator: "abcd", + block_height: 10, + block_hash: HexEncodableBlockHash(hash.clone()), + commitment_hash: HexEncodableBlockHash(hash.clone()), + }); + + emit(ParentFinalityPeerVoteSent { + block_height: 10, + block_hash: HexEncodableBlockHash(hash.clone()), + commitment_hash: HexEncodableBlockHash(hash.clone()), + }); + + emit(ParentFinalityPeerQuorumReached { + block_height: 10, + block_hash: HexEncodableBlockHash(hash.clone()), + commitment_hash: HexEncodableBlockHash(hash.clone()), + weight: 5, + }); + + emit(ParentFinalityCommitted { + parent_height: 10, + block_hash: HexEncodableBlockHash(hash.clone()), + local_height: Some(3), + proposer: Some("proposer-validator"), + }); + + println!("Emitted all events"); +} + #[cfg(test)] mod tests { use super::*; From 5111bb4a79cd7f6434d1afa2208ce2e5482d5770 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 12 Jul 2024 13:49:43 +0200 Subject: [PATCH 09/40] feat: add config as a cmd flags --- fendermint/app/src/main.rs | 3 -- fendermint/vm/topdown/src/observe.rs | 53 ------------------------- ipc/observability/src/tracing_layers.rs | 8 ++++ 3 files changed, 8 insertions(+), 56 deletions(-) diff --git a/fendermint/app/src/main.rs b/fendermint/app/src/main.rs index 13a86b74b..117fa3121 100644 --- a/fendermint/app/src/main.rs +++ b/fendermint/app/src/main.rs @@ -48,9 +48,6 @@ async fn main() { init_panic_handler(); - use fendermint_vm_topdown::observe::emit_all; - emit_all(); - if let Err(e) = cmd::exec(&opts).await { tracing::error!("failed to execute {:?}: {e:?}", opts); std::process::exit(fendermint_app::AppExitCode::UnknownError as i32); diff --git a/fendermint/vm/topdown/src/observe.rs b/fendermint/vm/topdown/src/observe.rs index 613864f26..cce0b3b2e 100644 --- a/fendermint/vm/topdown/src/observe.rs +++ b/fendermint/vm/topdown/src/observe.rs @@ -151,59 +151,6 @@ impl Recordable for ParentFinalityCommitted<'_> { } } -pub fn emit_all() { - use ipc_observability::emit; - - emit(ParentRpcCalled { - source: "source", - json_rpc: "json_rpc", - method: "method", - status: "status", - latency: 1.0, - }); - - let hash = vec![0u8; 32]; - - emit(ParentFinalityAcquired { - source: "parent-finality", - is_null: false, - block_height: 10, - block_hash: Some(HexEncodableBlockHash(hash.clone())), - commitment_hash: Some(HexEncodableBlockHash(hash.clone())), - num_msgs: 5, - num_validator_changes: 6, - }); - - emit(ParentFinalityPeerVoteReceived { - validator: "abcd", - block_height: 10, - block_hash: HexEncodableBlockHash(hash.clone()), - commitment_hash: HexEncodableBlockHash(hash.clone()), - }); - - emit(ParentFinalityPeerVoteSent { - block_height: 10, - block_hash: HexEncodableBlockHash(hash.clone()), - commitment_hash: HexEncodableBlockHash(hash.clone()), - }); - - emit(ParentFinalityPeerQuorumReached { - block_height: 10, - block_hash: HexEncodableBlockHash(hash.clone()), - commitment_hash: HexEncodableBlockHash(hash.clone()), - weight: 5, - }); - - emit(ParentFinalityCommitted { - parent_height: 10, - block_hash: HexEncodableBlockHash(hash.clone()), - local_height: Some(3), - proposer: Some("proposer-validator"), - }); - - println!("Emitted all events"); -} - #[cfg(test)] mod tests { use super::*; diff --git a/ipc/observability/src/tracing_layers.rs b/ipc/observability/src/tracing_layers.rs index f680fbdcb..15ab07e97 100644 --- a/ipc/observability/src/tracing_layers.rs +++ b/ipc/observability/src/tracing_layers.rs @@ -13,6 +13,14 @@ pub struct DomainEventFilterLayer { inner: L, } +impl Deref for DomainEventFilterLayer { + type Target = Registry; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + impl DomainEventFilterLayer { pub fn new(domains: Option>, events: Option>, inner: L) -> Self { DomainEventFilterLayer { From 8ca34f1dd5a86e523bd1d2cbc69901c4218b6348 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Mon, 15 Jul 2024 11:59:42 +0200 Subject: [PATCH 10/40] feat: fix comments --- ipc/observability/src/tracing_layers.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ipc/observability/src/tracing_layers.rs b/ipc/observability/src/tracing_layers.rs index 15ab07e97..f680fbdcb 100644 --- a/ipc/observability/src/tracing_layers.rs +++ b/ipc/observability/src/tracing_layers.rs @@ -13,14 +13,6 @@ pub struct DomainEventFilterLayer { inner: L, } -impl Deref for DomainEventFilterLayer { - type Target = Registry; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - impl DomainEventFilterLayer { pub fn new(domains: Option>, events: Option>, inner: L) -> Self { DomainEventFilterLayer { From 6a8f94957bfbeae3059d5bb5f255bc5df230f4a9 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 12 Jul 2024 17:30:43 +0200 Subject: [PATCH 11/40] feat: add consensus traces --- fendermint/app/src/app.rs | 14 +++ fendermint/app/src/events.rs | 26 ----- fendermint/app/src/metrics/tracing.rs | 155 -------------------------- test-network/genesis.json | 97 ++++++++++++++++ test-network/keys/alice.pk | 1 + test-network/keys/alice.sk | 1 + test-network/keys/bob.pk | 1 + test-network/keys/bob.sk | 1 + test-network/keys/charlie.pk | 1 + test-network/keys/charlie.sk | 1 + test-network/keys/dave.pk | 1 + test-network/keys/dave.sk | 1 + test-network/traces/2024-07-12.traces | 2 + 13 files changed, 121 insertions(+), 181 deletions(-) delete mode 100644 fendermint/app/src/events.rs delete mode 100644 fendermint/app/src/metrics/tracing.rs create mode 100644 test-network/genesis.json create mode 100644 test-network/keys/alice.pk create mode 100644 test-network/keys/alice.sk create mode 100644 test-network/keys/bob.pk create mode 100644 test-network/keys/bob.sk create mode 100644 test-network/keys/charlie.pk create mode 100644 test-network/keys/charlie.sk create mode 100644 test-network/keys/dave.pk create mode 100644 test-network/keys/dave.sk create mode 100644 test-network/traces/2024-07-12.traces diff --git a/fendermint/app/src/app.rs b/fendermint/app/src/app.rs index e87677483..e01733fc0 100644 --- a/fendermint/app/src/app.rs +++ b/fendermint/app/src/app.rs @@ -715,8 +715,22 @@ where }); if accept { + emit(BlockProposalAccepted { + height: request.height.value(), + hash: HexEncodableBlockHash(request.hash.into()), + size: size_txs, + tx_count: num_txs, + validator: request.proposer_address.to_string().as_str(), + }); Ok(response::ProcessProposal::Accept) } else { + emit(BlockProposalRejected { + height: request.height.value(), + size: size_txs, + tx_count: num_txs, + validator: request.proposer_address.to_string().as_str(), + reason: "rejected", + }); Ok(response::ProcessProposal::Reject) } } diff --git a/fendermint/app/src/events.rs b/fendermint/app/src/events.rs deleted file mode 100644 index 84fc2c8cd..000000000 --- a/fendermint/app/src/events.rs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -use crate::BlockHeight; - -/// Re-export other events, just to provide the visibility of where they are. -pub use fendermint_vm_event::{ - NewBottomUpCheckpoint, ParentFinalityCommitted, ParentFinalityMissingQuorum, -}; - -/// Hex encoded block hash. -pub type BlockHashHex<'a> = &'a str; - -#[derive(Debug, Default)] -pub struct ProposalProcessed<'a> { - pub is_accepted: bool, - pub block_height: BlockHeight, - pub block_hash: BlockHashHex<'a>, - pub num_txs: usize, - pub proposer: &'a str, -} - -#[derive(Debug, Default)] -pub struct NewBlock { - pub block_height: BlockHeight, -} diff --git a/fendermint/app/src/metrics/tracing.rs b/fendermint/app/src/metrics/tracing.rs deleted file mode 100644 index 397092396..000000000 --- a/fendermint/app/src/metrics/tracing.rs +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT -//! Subscribing to tracing events and turning them into metrics. - -use std::marker::PhantomData; - -use tracing::{Event, Subscriber}; -use tracing_subscriber::{filter, layer, registry::LookupSpan, Layer}; - -use super::prometheus::app as am; -use crate::events::*; - -/// Create a layer that handles events by incrementing metrics. -pub fn layer() -> impl Layer -where - S: Subscriber, - for<'a> S: LookupSpan<'a>, -{ - MetricsLayer::new().with_filter(filter::filter_fn(|md| md.name().starts_with("event::"))) -} - -struct MetricsLayer { - _subscriber: PhantomData, -} - -impl MetricsLayer { - pub fn new() -> Self { - Self { - _subscriber: PhantomData, - } - } -} - -/// Check that the field exist on a type; if it doesn't this won't compile. -/// This ensures that we're mapping fields with the correct name. -macro_rules! check_field { - ($event_ty:ident :: $field:ident) => {{ - if false { - #[allow(clippy::needless_update)] - let _event = $event_ty { - $field: Default::default(), - ..Default::default() - }; - } - }}; -} - -/// Set a gague to an absolute value based on a field in an event. -macro_rules! set_gauge { - ($event:ident, $event_ty:ident :: $field:ident, $gauge:expr) => { - check_field!($event_ty::$field); - let mut fld = visitors::FindU64::new(stringify!($field)); - $event.record(&mut fld); - $gauge.set(fld.value as i64); - }; -} - -/// Increment a counter by the value of a field in the event. -macro_rules! inc_counter { - ($event:ident, $event_ty:ident :: $field:ident, $counter:expr) => { - check_field!($event_ty::$field); - let mut fld = visitors::FindU64::new(stringify!($field)); - $event.record(&mut fld); - $counter.inc_by(fld.value); - }; -} - -/// Produce the prefixed event name from the type name. -macro_rules! event_name { - ($event_ty:ident) => { - concat!("event::", stringify!($event_ty)) - }; -} - -/// Call one of the macros that set values on a metric. -macro_rules! event_mapping { - ($op:ident, $event:ident, $event_ty:ident :: $field:ident, $metric:expr) => { - $op!($event, $event_ty::$field, $metric); - }; -} - -/// Match the event name to event DTO types and within the map fields to metrics. -macro_rules! event_match { - ($event:ident { $( $event_ty:ident { $( $field:ident => $op:ident ! $metric:expr ),* $(,)? } ),* } ) => { - match $event.metadata().name() { - $( - event_name!($event_ty) => { - $( - event_mapping!($op, $event, $event_ty :: $field, $metric); - )* - } - )* - _ => {} - } - }; -} - -impl Layer for MetricsLayer { - fn on_event(&self, event: &Event<'_>, _ctx: layer::Context<'_, S>) { - event_match!(event { - NewBottomUpCheckpoint { - block_height => set_gauge ! &am::BOTTOMUP_CKPT_BLOCK_HEIGHT, - next_configuration_number => set_gauge ! &am::BOTTOMUP_CKPT_CONFIG_NUM, - num_msgs => inc_counter ! &am::BOTTOMUP_CKPT_NUM_MSGS, - }, - NewBlock { - block_height => set_gauge ! &am::ABCI_COMMITTED_BLOCK_HEIGHT - } - }); - } -} - -mod visitors { - use tracing::field::{Field, Visit}; - - pub struct FindU64<'a> { - pub name: &'a str, - pub value: u64, - } - - impl<'a> FindU64<'a> { - pub fn new(name: &'a str) -> Self { - Self { name, value: 0 } - } - } - - // Looking for multiple values because the callsite might be passed as a literal which turns into an i64 for example. - impl<'a> Visit for FindU64<'a> { - fn record_u64(&mut self, field: &Field, value: u64) { - if field.name() == self.name { - self.value = value; - } - } - - fn record_i64(&mut self, field: &Field, value: i64) { - if field.name() == self.name { - self.value = value as u64; - } - } - - fn record_i128(&mut self, field: &Field, value: i128) { - if field.name() == self.name { - self.value = value as u64; - } - } - - fn record_u128(&mut self, field: &Field, value: u128) { - if field.name() == self.name { - self.value = value as u64; - } - } - - fn record_debug(&mut self, _field: &Field, _value: &dyn std::fmt::Debug) {} - } -} diff --git a/test-network/genesis.json b/test-network/genesis.json new file mode 100644 index 000000000..b2f5501ae --- /dev/null +++ b/test-network/genesis.json @@ -0,0 +1,97 @@ +{ + "chain_name": "test", + "timestamp": 1680101412, + "network_version": 21, + "base_fee": "1000", + "power_scale": 0, + "validators": [ + { + "public_key": "BPMMj79Dc5slgNl1v29V0mn/ixHt7CO/MbsNLQP0ZpSEd5FuW2UQ5hzNsKhauI97UZ9jQN3t+kRJQaWlcRjcJdI=", + "power": "1000000000000000000" + } + ], + "accounts": [ + { + "meta": { + "Account": { + "owner": "t1vfs6z4k6cyoevukvhqomsgwqrb75i2jodwa4jxa" + } + }, + "balance": "10000000000000000000" + }, + { + "meta": { + "Account": { + "owner": "t410frr6xjymicyh4iltk5v27c4y4misr73mcqjzh45a" + } + }, + "balance": "10000000000000000000" + }, + { + "meta": { + "Account": { + "owner": "t1zrk7jjngc3uzobjkmlzjynjfgpxvxctxbxuvs5a" + } + }, + "balance": "10000000000000000000" + }, + { + "meta": { + "Account": { + "owner": "t410f5qlw62wdgzwdwfb3rjoheibxrhke3vmzxtcdzrq" + } + }, + "balance": "10000000000000000000" + }, + { + "meta": { + "Account": { + "owner": "t1kwdghtw7j72qb5dwf4gskyp4be4cter73nlojuy" + } + }, + "balance": "10000000000000000000" + }, + { + "meta": { + "Account": { + "owner": "t410flcjfiuotgwadabcoftuuo26ooaxrovjvwdccoji" + } + }, + "balance": "10000000000000000000" + }, + { + "meta": { + "Account": { + "owner": "t1e3acffdre4egtul5kp3c2hamm522dg6aqfgzioy" + } + }, + "balance": "10000000000000000000" + }, + { + "meta": { + "Account": { + "owner": "t410fst6awyokeeevmqiwcnlzwt3g2uazcsavr2u2iwi" + } + }, + "balance": "10000000000000000000" + }, + { + "meta": { + "Multisig": { + "signers": [ + "t1zrk7jjngc3uzobjkmlzjynjfgpxvxctxbxuvs5a", + "t1kwdghtw7j72qb5dwf4gskyp4be4cter73nlojuy", + "t1e3acffdre4egtul5kp3c2hamm522dg6aqfgzioy" + ], + "threshold": 2, + "vesting_duration": 1000000, + "vesting_start": 0 + } + }, + "balance": "30000000000000000000" + } + ], + "eam_permission_mode": { + "mode": "unrestricted" + } +} \ No newline at end of file diff --git a/test-network/keys/alice.pk b/test-network/keys/alice.pk new file mode 100644 index 000000000..794dee8f6 --- /dev/null +++ b/test-network/keys/alice.pk @@ -0,0 +1 @@ +AhIQ+C+xL8C9v45KcdczZj5H4sPxROx08lT+seQCEM6c \ No newline at end of file diff --git a/test-network/keys/alice.sk b/test-network/keys/alice.sk new file mode 100644 index 000000000..a8c648503 --- /dev/null +++ b/test-network/keys/alice.sk @@ -0,0 +1 @@ +UWJv1Gj1tmnxlKUQizJTxobyJuB7u4PbkO3J2pfd+PI= \ No newline at end of file diff --git a/test-network/keys/bob.pk b/test-network/keys/bob.pk new file mode 100644 index 000000000..20aed1d21 --- /dev/null +++ b/test-network/keys/bob.pk @@ -0,0 +1 @@ +AvMMj79Dc5slgNl1v29V0mn/ixHt7CO/MbsNLQP0ZpSE \ No newline at end of file diff --git a/test-network/keys/bob.sk b/test-network/keys/bob.sk new file mode 100644 index 000000000..3c63da5b8 --- /dev/null +++ b/test-network/keys/bob.sk @@ -0,0 +1 @@ +EWYQ7LHh7eLty4OUv1IL3Yjyke/P0fyqOUNv4TBqJaM= \ No newline at end of file diff --git a/test-network/keys/charlie.pk b/test-network/keys/charlie.pk new file mode 100644 index 000000000..087aae502 --- /dev/null +++ b/test-network/keys/charlie.pk @@ -0,0 +1 @@ +AhgmflhhhFLNq+xZ1BGY9b0v8o/D3VEcSkPtGdhpv/vR \ No newline at end of file diff --git a/test-network/keys/charlie.sk b/test-network/keys/charlie.sk new file mode 100644 index 000000000..67f08daca --- /dev/null +++ b/test-network/keys/charlie.sk @@ -0,0 +1 @@ +P2qcIpdwNblXYIeakFAnsNkRQKl+JpmAX4mmmPJvluI= \ No newline at end of file diff --git a/test-network/keys/dave.pk b/test-network/keys/dave.pk new file mode 100644 index 000000000..630dc08b0 --- /dev/null +++ b/test-network/keys/dave.pk @@ -0,0 +1 @@ +AsmyG+kLuoplF8iv4A8xD2VaR86tgVyWh7t0aqbRfpmg \ No newline at end of file diff --git a/test-network/keys/dave.sk b/test-network/keys/dave.sk new file mode 100644 index 000000000..5efce2ac0 --- /dev/null +++ b/test-network/keys/dave.sk @@ -0,0 +1 @@ +u/8eCA7tnRFriVCsWm9qpVktOrSr0t2gPgBJtszdC4M= \ No newline at end of file diff --git a/test-network/traces/2024-07-12.traces b/test-network/traces/2024-07-12.traces new file mode 100644 index 000000000..7efff3021 --- /dev/null +++ b/test-network/traces/2024-07-12.traces @@ -0,0 +1,2 @@ +{"timestamp":"2024-07-12T11:47:57.300184Z","level":"INFO","fields":{"domain":"Topdown","event":"ParentFinalityAcquired { source: \"parent-finality\", is_null: false, block_height: 10, block_hash: Some(0000000000000000000000000000000000000000000000000000000000000000), commitment_hash: Some(0000000000000000000000000000000000000000000000000000000000000000), num_msgs: 5, num_validator_changes: 6 }"},"filename":"ipc/observability/src/lib.rs","line_number":36} +{"timestamp":"2024-07-12T11:47:57.300423Z","level":"INFO","fields":{"domain":"Topdown","event":"ParentFinalityPeerVoteSent { block_height: 10, block_hash: 0000000000000000000000000000000000000000000000000000000000000000, commitment_hash: 0000000000000000000000000000000000000000000000000000000000000000 }"},"filename":"ipc/observability/src/lib.rs","line_number":36} From 6ee97b544d01d468cb290bd33cedbc7e892bea64 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 12 Jul 2024 22:35:47 +0200 Subject: [PATCH 12/40] feat: add execution metrics --- fendermint/vm/interpreter/src/fvm/check.rs | 2 ++ fendermint/vm/interpreter/src/fvm/exec.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/fendermint/vm/interpreter/src/fvm/check.rs b/fendermint/vm/interpreter/src/fvm/check.rs index 5e627007c..89c70a617 100644 --- a/fendermint/vm/interpreter/src/fvm/check.rs +++ b/fendermint/vm/interpreter/src/fvm/check.rs @@ -1,6 +1,8 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use std::time::Instant; + use async_trait::async_trait; use fvm_ipld_blockstore::Blockstore; diff --git a/fendermint/vm/interpreter/src/fvm/exec.rs b/fendermint/vm/interpreter/src/fvm/exec.rs index 150cb218a..f07be92af 100644 --- a/fendermint/vm/interpreter/src/fvm/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/exec.rs @@ -1,6 +1,8 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use std::time::Instant; + use anyhow::Context; use async_trait::async_trait; use std::collections::HashMap; From 40855319dc621012c658e34d3b986fa60af108c8 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Mon, 15 Jul 2024 16:37:23 +0200 Subject: [PATCH 13/40] feat: add proposals metrics & reason --- fendermint/app/src/app.rs | 4 ++-- fendermint/vm/interpreter/src/bytes.rs | 16 +++++++++++++--- fendermint/vm/interpreter/src/chain.rs | 12 ++++++++---- fendermint/vm/interpreter/src/lib.rs | 6 +++++- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/fendermint/app/src/app.rs b/fendermint/app/src/app.rs index e01733fc0..dcf0d6e47 100644 --- a/fendermint/app/src/app.rs +++ b/fendermint/app/src/app.rs @@ -690,7 +690,7 @@ where let size_txs = txs.iter().map(|tx| tx.len()).sum::(); let num_txs = txs.len(); - let accept = self + let (accept, reason) = self .interpreter .process(self.chain_env.clone(), txs) .await @@ -729,7 +729,7 @@ where size: size_txs, tx_count: num_txs, validator: request.proposer_address.to_string().as_str(), - reason: "rejected", + reason: reason.unwrap_or_default().as_str(), }); Ok(response::ProcessProposal::Reject) } diff --git a/fendermint/vm/interpreter/src/bytes.rs b/fendermint/vm/interpreter/src/bytes.rs index 1af84dd85..59ad88b41 100644 --- a/fendermint/vm/interpreter/src/bytes.rs +++ b/fendermint/vm/interpreter/src/bytes.rs @@ -125,13 +125,17 @@ where } /// Parse messages in the block, reject if unknown format. Pass the rest to the inner `ChainMessage` interpreter. - async fn process(&self, state: Self::State, msgs: Vec) -> anyhow::Result { + async fn process( + &self, + state: Self::State, + msgs: Vec, + ) -> anyhow::Result<(bool, Option)> { if msgs.len() > self.max_msgs { tracing::warn!( block_msgs = msgs.len(), "rejecting block: too many messages" ); - return Ok(false); + return Ok((false, Some(format!("too many messages: {}", msgs.len())))); } let mut chain_msgs = Vec::new(); @@ -152,7 +156,13 @@ where "failed to decode message in proposal as ChainMessage" ); if self.reject_malformed_proposal { - return Ok(false); + return Ok(( + false, + Some(format!( + "failed to decode message in proposal as ChainMessage: {}", + e + )), + )); } } Ok(msg) => chain_msgs.push(msg), diff --git a/fendermint/vm/interpreter/src/chain.rs b/fendermint/vm/interpreter/src/chain.rs index 79136138f..5c0fe8489 100644 --- a/fendermint/vm/interpreter/src/chain.rs +++ b/fendermint/vm/interpreter/src/chain.rs @@ -175,7 +175,11 @@ where } /// Perform finality checks on top-down transactions and availability checks on bottom-up transactions. - async fn process(&self, env: Self::State, msgs: Vec) -> anyhow::Result { + async fn process( + &self, + env: Self::State, + msgs: Vec, + ) -> anyhow::Result<(bool, Option)> { for msg in msgs { match msg { ChainMessage::Ipc(IpcMessage::BottomUpExec(msg)) => { @@ -194,7 +198,7 @@ where .await; if !is_resolved { - return Ok(false); + return Ok((false, Some("checkpoint not resolved".to_string()))); } } ChainMessage::Ipc(IpcMessage::TopDownExec(ParentFinality { @@ -208,13 +212,13 @@ where let is_final = atomically(|| env.parent_finality_provider.check_proposal(&prop)).await; if !is_final { - return Ok(false); + return Ok((false, Some("parent finality not available".to_string()))); } } _ => {} }; } - Ok(true) + Ok((true, None)) } } diff --git a/fendermint/vm/interpreter/src/lib.rs b/fendermint/vm/interpreter/src/lib.rs index 42dab76b8..a12ccfcc0 100644 --- a/fendermint/vm/interpreter/src/lib.rs +++ b/fendermint/vm/interpreter/src/lib.rs @@ -54,7 +54,11 @@ pub trait ProposalInterpreter: Sync + Send { /// This is our chance check whether CIDs proposed for execution are available. /// /// Return `true` if we can accept this block, `false` to reject it. - async fn process(&self, state: Self::State, msgs: Vec) -> anyhow::Result; + async fn process( + &self, + state: Self::State, + msgs: Vec, + ) -> anyhow::Result<(bool, Option)>; } /// The `ExecInterpreter` applies messages on some state, which is From 313ff71daa9173a03b9f172687560065f6533cba Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Tue, 16 Jul 2024 10:40:41 +0200 Subject: [PATCH 14/40] feat: add mempool event --- fendermint/app/src/app.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fendermint/app/src/app.rs b/fendermint/app/src/app.rs index dcf0d6e47..d9958cd72 100644 --- a/fendermint/app/src/app.rs +++ b/fendermint/app/src/app.rs @@ -625,7 +625,13 @@ where let mut mpool_received_trace = MpoolReceived::default(); let response = match result { - Err(e) => invalid_check_tx(AppError::InvalidEncoding, e.description), + Err(e) => { + emit(MpoolReceivedInvalidMessage { + reason: "InvalidEncoding", + description: e.description.as_ref(), + }); + invalid_check_tx(AppError::InvalidEncoding, e.description) + } Ok(result) => match result { Err(IllegalMessage) => invalid_check_tx(AppError::IllegalMessage, "".to_owned()), Ok(Err(InvalidSignature(d))) => invalid_check_tx(AppError::InvalidSignature, d), From f91bfa1a4e9dc23c563f4ff7861364e7d2a75bf1 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Tue, 16 Jul 2024 10:44:19 +0200 Subject: [PATCH 15/40] feat: remove --- test-network/genesis.json | 97 --------------------------- test-network/keys/alice.pk | 1 - test-network/keys/alice.sk | 1 - test-network/keys/bob.pk | 1 - test-network/keys/bob.sk | 1 - test-network/keys/charlie.pk | 1 - test-network/keys/charlie.sk | 1 - test-network/keys/dave.pk | 1 - test-network/keys/dave.sk | 1 - test-network/traces/2024-07-12.traces | 2 - 10 files changed, 107 deletions(-) delete mode 100644 test-network/genesis.json delete mode 100644 test-network/keys/alice.pk delete mode 100644 test-network/keys/alice.sk delete mode 100644 test-network/keys/bob.pk delete mode 100644 test-network/keys/bob.sk delete mode 100644 test-network/keys/charlie.pk delete mode 100644 test-network/keys/charlie.sk delete mode 100644 test-network/keys/dave.pk delete mode 100644 test-network/keys/dave.sk delete mode 100644 test-network/traces/2024-07-12.traces diff --git a/test-network/genesis.json b/test-network/genesis.json deleted file mode 100644 index b2f5501ae..000000000 --- a/test-network/genesis.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "chain_name": "test", - "timestamp": 1680101412, - "network_version": 21, - "base_fee": "1000", - "power_scale": 0, - "validators": [ - { - "public_key": "BPMMj79Dc5slgNl1v29V0mn/ixHt7CO/MbsNLQP0ZpSEd5FuW2UQ5hzNsKhauI97UZ9jQN3t+kRJQaWlcRjcJdI=", - "power": "1000000000000000000" - } - ], - "accounts": [ - { - "meta": { - "Account": { - "owner": "t1vfs6z4k6cyoevukvhqomsgwqrb75i2jodwa4jxa" - } - }, - "balance": "10000000000000000000" - }, - { - "meta": { - "Account": { - "owner": "t410frr6xjymicyh4iltk5v27c4y4misr73mcqjzh45a" - } - }, - "balance": "10000000000000000000" - }, - { - "meta": { - "Account": { - "owner": "t1zrk7jjngc3uzobjkmlzjynjfgpxvxctxbxuvs5a" - } - }, - "balance": "10000000000000000000" - }, - { - "meta": { - "Account": { - "owner": "t410f5qlw62wdgzwdwfb3rjoheibxrhke3vmzxtcdzrq" - } - }, - "balance": "10000000000000000000" - }, - { - "meta": { - "Account": { - "owner": "t1kwdghtw7j72qb5dwf4gskyp4be4cter73nlojuy" - } - }, - "balance": "10000000000000000000" - }, - { - "meta": { - "Account": { - "owner": "t410flcjfiuotgwadabcoftuuo26ooaxrovjvwdccoji" - } - }, - "balance": "10000000000000000000" - }, - { - "meta": { - "Account": { - "owner": "t1e3acffdre4egtul5kp3c2hamm522dg6aqfgzioy" - } - }, - "balance": "10000000000000000000" - }, - { - "meta": { - "Account": { - "owner": "t410fst6awyokeeevmqiwcnlzwt3g2uazcsavr2u2iwi" - } - }, - "balance": "10000000000000000000" - }, - { - "meta": { - "Multisig": { - "signers": [ - "t1zrk7jjngc3uzobjkmlzjynjfgpxvxctxbxuvs5a", - "t1kwdghtw7j72qb5dwf4gskyp4be4cter73nlojuy", - "t1e3acffdre4egtul5kp3c2hamm522dg6aqfgzioy" - ], - "threshold": 2, - "vesting_duration": 1000000, - "vesting_start": 0 - } - }, - "balance": "30000000000000000000" - } - ], - "eam_permission_mode": { - "mode": "unrestricted" - } -} \ No newline at end of file diff --git a/test-network/keys/alice.pk b/test-network/keys/alice.pk deleted file mode 100644 index 794dee8f6..000000000 --- a/test-network/keys/alice.pk +++ /dev/null @@ -1 +0,0 @@ -AhIQ+C+xL8C9v45KcdczZj5H4sPxROx08lT+seQCEM6c \ No newline at end of file diff --git a/test-network/keys/alice.sk b/test-network/keys/alice.sk deleted file mode 100644 index a8c648503..000000000 --- a/test-network/keys/alice.sk +++ /dev/null @@ -1 +0,0 @@ -UWJv1Gj1tmnxlKUQizJTxobyJuB7u4PbkO3J2pfd+PI= \ No newline at end of file diff --git a/test-network/keys/bob.pk b/test-network/keys/bob.pk deleted file mode 100644 index 20aed1d21..000000000 --- a/test-network/keys/bob.pk +++ /dev/null @@ -1 +0,0 @@ -AvMMj79Dc5slgNl1v29V0mn/ixHt7CO/MbsNLQP0ZpSE \ No newline at end of file diff --git a/test-network/keys/bob.sk b/test-network/keys/bob.sk deleted file mode 100644 index 3c63da5b8..000000000 --- a/test-network/keys/bob.sk +++ /dev/null @@ -1 +0,0 @@ -EWYQ7LHh7eLty4OUv1IL3Yjyke/P0fyqOUNv4TBqJaM= \ No newline at end of file diff --git a/test-network/keys/charlie.pk b/test-network/keys/charlie.pk deleted file mode 100644 index 087aae502..000000000 --- a/test-network/keys/charlie.pk +++ /dev/null @@ -1 +0,0 @@ -AhgmflhhhFLNq+xZ1BGY9b0v8o/D3VEcSkPtGdhpv/vR \ No newline at end of file diff --git a/test-network/keys/charlie.sk b/test-network/keys/charlie.sk deleted file mode 100644 index 67f08daca..000000000 --- a/test-network/keys/charlie.sk +++ /dev/null @@ -1 +0,0 @@ -P2qcIpdwNblXYIeakFAnsNkRQKl+JpmAX4mmmPJvluI= \ No newline at end of file diff --git a/test-network/keys/dave.pk b/test-network/keys/dave.pk deleted file mode 100644 index 630dc08b0..000000000 --- a/test-network/keys/dave.pk +++ /dev/null @@ -1 +0,0 @@ -AsmyG+kLuoplF8iv4A8xD2VaR86tgVyWh7t0aqbRfpmg \ No newline at end of file diff --git a/test-network/keys/dave.sk b/test-network/keys/dave.sk deleted file mode 100644 index 5efce2ac0..000000000 --- a/test-network/keys/dave.sk +++ /dev/null @@ -1 +0,0 @@ -u/8eCA7tnRFriVCsWm9qpVktOrSr0t2gPgBJtszdC4M= \ No newline at end of file diff --git a/test-network/traces/2024-07-12.traces b/test-network/traces/2024-07-12.traces deleted file mode 100644 index 7efff3021..000000000 --- a/test-network/traces/2024-07-12.traces +++ /dev/null @@ -1,2 +0,0 @@ -{"timestamp":"2024-07-12T11:47:57.300184Z","level":"INFO","fields":{"domain":"Topdown","event":"ParentFinalityAcquired { source: \"parent-finality\", is_null: false, block_height: 10, block_hash: Some(0000000000000000000000000000000000000000000000000000000000000000), commitment_hash: Some(0000000000000000000000000000000000000000000000000000000000000000), num_msgs: 5, num_validator_changes: 6 }"},"filename":"ipc/observability/src/lib.rs","line_number":36} -{"timestamp":"2024-07-12T11:47:57.300423Z","level":"INFO","fields":{"domain":"Topdown","event":"ParentFinalityPeerVoteSent { block_height: 10, block_hash: 0000000000000000000000000000000000000000000000000000000000000000, commitment_hash: 0000000000000000000000000000000000000000000000000000000000000000 }"},"filename":"ipc/observability/src/lib.rs","line_number":36} From e801cdcb82bde2ead9dde90c36a2f3b90d5073c4 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Tue, 16 Jul 2024 14:49:56 +0200 Subject: [PATCH 16/40] feat: address comments --- fendermint/app/src/app.rs | 45 +++++++++---------- fendermint/vm/interpreter/src/bytes.rs | 13 +++--- fendermint/vm/interpreter/src/chain.rs | 8 ++-- fendermint/vm/interpreter/src/fvm/check.rs | 2 - fendermint/vm/interpreter/src/fvm/exec.rs | 2 - .../vm/interpreter/src/fvm/state/exec.rs | 37 +++++++++++++++ .../vm/interpreter/src/fvm/state/mod.rs | 2 +- fendermint/vm/interpreter/src/lib.rs | 2 +- 8 files changed, 70 insertions(+), 41 deletions(-) diff --git a/fendermint/app/src/app.rs b/fendermint/app/src/app.rs index d9958cd72..be2190799 100644 --- a/fendermint/app/src/app.rs +++ b/fendermint/app/src/app.rs @@ -696,11 +696,7 @@ where let size_txs = txs.iter().map(|tx| tx.len()).sum::(); let num_txs = txs.len(); - let (accept, reason) = self - .interpreter - .process(self.chain_env.clone(), txs) - .await - .context("failed to process proposal")?; + let process_result = self.interpreter.process(self.chain_env.clone(), txs).await; emit(BlockProposalReceived { height: request.height.value(), @@ -720,24 +716,27 @@ where reason: None, }); - if accept { - emit(BlockProposalAccepted { - height: request.height.value(), - hash: HexEncodableBlockHash(request.hash.into()), - size: size_txs, - tx_count: num_txs, - validator: request.proposer_address.to_string().as_str(), - }); - Ok(response::ProcessProposal::Accept) - } else { - emit(BlockProposalRejected { - height: request.height.value(), - size: size_txs, - tx_count: num_txs, - validator: request.proposer_address.to_string().as_str(), - reason: reason.unwrap_or_default().as_str(), - }); - Ok(response::ProcessProposal::Reject) + match process_result { + Ok(_) => { + emit(BlockProposalAccepted { + height: request.height.value(), + hash: HexEncodableBlockHash(request.hash.into()), + size: size_txs, + tx_count: num_txs, + validator: request.proposer_address.to_string().as_str(), + }); + Ok(response::ProcessProposal::Accept) + } + Err(e) => { + emit(BlockProposalRejected { + height: request.height.value(), + size: size_txs, + tx_count: num_txs, + validator: request.proposer_address.to_string().as_str(), + reason: e.as_str(), + }); + Ok(response::ProcessProposal::Reject) + } } } diff --git a/fendermint/vm/interpreter/src/bytes.rs b/fendermint/vm/interpreter/src/bytes.rs index 59ad88b41..b1afaad66 100644 --- a/fendermint/vm/interpreter/src/bytes.rs +++ b/fendermint/vm/interpreter/src/bytes.rs @@ -129,13 +129,13 @@ where &self, state: Self::State, msgs: Vec, - ) -> anyhow::Result<(bool, Option)> { + ) -> anyhow::Result { if msgs.len() > self.max_msgs { tracing::warn!( block_msgs = msgs.len(), "rejecting block: too many messages" ); - return Ok((false, Some(format!("too many messages: {}", msgs.len())))); + return Err(format!("too many messages: {}", msgs.len())); } let mut chain_msgs = Vec::new(); @@ -156,12 +156,9 @@ where "failed to decode message in proposal as ChainMessage" ); if self.reject_malformed_proposal { - return Ok(( - false, - Some(format!( - "failed to decode message in proposal as ChainMessage: {}", - e - )), + return Err(format!( + "failed to decode message in proposal as ChainMessage: {}", + e )); } } diff --git a/fendermint/vm/interpreter/src/chain.rs b/fendermint/vm/interpreter/src/chain.rs index 5c0fe8489..05860bfec 100644 --- a/fendermint/vm/interpreter/src/chain.rs +++ b/fendermint/vm/interpreter/src/chain.rs @@ -179,7 +179,7 @@ where &self, env: Self::State, msgs: Vec, - ) -> anyhow::Result<(bool, Option)> { + ) -> anyhow::Result { for msg in msgs { match msg { ChainMessage::Ipc(IpcMessage::BottomUpExec(msg)) => { @@ -198,7 +198,7 @@ where .await; if !is_resolved { - return Ok((false, Some("checkpoint not resolved".to_string()))); + return Err("checkpoint not resolved".to_string()); } } ChainMessage::Ipc(IpcMessage::TopDownExec(ParentFinality { @@ -212,13 +212,13 @@ where let is_final = atomically(|| env.parent_finality_provider.check_proposal(&prop)).await; if !is_final { - return Ok((false, Some("parent finality not available".to_string()))); + return Err("parent finality not available".to_string()); } } _ => {} }; } - Ok((true, None)) + Ok(true) } } diff --git a/fendermint/vm/interpreter/src/fvm/check.rs b/fendermint/vm/interpreter/src/fvm/check.rs index 89c70a617..5e627007c 100644 --- a/fendermint/vm/interpreter/src/fvm/check.rs +++ b/fendermint/vm/interpreter/src/fvm/check.rs @@ -1,8 +1,6 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use std::time::Instant; - use async_trait::async_trait; use fvm_ipld_blockstore::Blockstore; diff --git a/fendermint/vm/interpreter/src/fvm/exec.rs b/fendermint/vm/interpreter/src/fvm/exec.rs index f07be92af..150cb218a 100644 --- a/fendermint/vm/interpreter/src/fvm/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/exec.rs @@ -1,8 +1,6 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use std::time::Instant; - use anyhow::Context; use async_trait::async_trait; use std::collections::HashMap; diff --git a/fendermint/vm/interpreter/src/fvm/state/exec.rs b/fendermint/vm/interpreter/src/fvm/state/exec.rs index 7d5f10dda..46e291eae 100644 --- a/fendermint/vm/interpreter/src/fvm/state/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/state/exec.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0, MIT use std::collections::{HashMap, HashSet}; +use std::time::{Duration, Instant}; use anyhow::Ok; use cid::Cid; @@ -350,3 +351,39 @@ fn check_error(e: anyhow::Error) -> (ApplyRet, ActorAddressMap) { }; (ret, Default::default()) } + +pub struct ElapsedExecution<'a, DB> +where + DB: Blockstore + Clone + 'static, +{ + state: &'a mut FvmExecState, +} + +impl<'a, DB> ElapsedExecution<'a, DB> +where + DB: Blockstore + Clone + 'static, +{ + pub fn new(state: &'a mut FvmExecState) -> Self { + Self { state } + } + + pub fn execute_implicit( + &mut self, + msg: Message, + ) -> anyhow::Result<(ApplyRet, ActorAddressMap, Duration)> { + let start = Instant::now(); + let result = self.state.execute_implicit(msg)?; + let duration = start.elapsed(); + Ok((result.0, result.1, duration)) + } + + pub fn execute_explicit( + &mut self, + msg: Message, + ) -> anyhow::Result<(ApplyRet, ActorAddressMap, Duration)> { + let start = Instant::now(); + let result = self.state.execute_explicit(msg)?; + let duration = start.elapsed(); + Ok((result.0, result.1, duration)) + } +} diff --git a/fendermint/vm/interpreter/src/fvm/state/mod.rs b/fendermint/vm/interpreter/src/fvm/state/mod.rs index 22a1504e4..0547ce9d8 100644 --- a/fendermint/vm/interpreter/src/fvm/state/mod.rs +++ b/fendermint/vm/interpreter/src/fvm/state/mod.rs @@ -12,7 +12,7 @@ pub mod snapshot; use std::sync::Arc; pub use check::FvmCheckState; -pub use exec::{BlockHash, FvmExecState, FvmStateParams, FvmUpdatableParams}; +pub use exec::{BlockHash, ElapsedExecution, FvmExecState, FvmStateParams, FvmUpdatableParams}; pub use genesis::{empty_state_tree, FvmGenesisState}; pub use query::FvmQueryState; diff --git a/fendermint/vm/interpreter/src/lib.rs b/fendermint/vm/interpreter/src/lib.rs index a12ccfcc0..f8a40e6d0 100644 --- a/fendermint/vm/interpreter/src/lib.rs +++ b/fendermint/vm/interpreter/src/lib.rs @@ -58,7 +58,7 @@ pub trait ProposalInterpreter: Sync + Send { &self, state: Self::State, msgs: Vec, - ) -> anyhow::Result<(bool, Option)>; + ) -> anyhow::Result; } /// The `ExecInterpreter` applies messages on some state, which is From 97ee818d037553b303914eef996dc894479a0a79 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Tue, 16 Jul 2024 15:59:21 +0200 Subject: [PATCH 17/40] feat: address comments --- fendermint/app/src/app.rs | 2 +- fendermint/vm/interpreter/src/bytes.rs | 10 ++++------ fendermint/vm/interpreter/src/chain.rs | 7 ++++--- fendermint/vm/interpreter/src/errors.rs | 16 ++++++++++++++++ fendermint/vm/interpreter/src/fvm/state/exec.rs | 2 ++ fendermint/vm/interpreter/src/lib.rs | 3 ++- 6 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 fendermint/vm/interpreter/src/errors.rs diff --git a/fendermint/app/src/app.rs b/fendermint/app/src/app.rs index be2190799..61ebadb68 100644 --- a/fendermint/app/src/app.rs +++ b/fendermint/app/src/app.rs @@ -733,7 +733,7 @@ where size: size_txs, tx_count: num_txs, validator: request.proposer_address.to_string().as_str(), - reason: e.as_str(), + reason: e.to_string().as_str(), }); Ok(response::ProcessProposal::Reject) } diff --git a/fendermint/vm/interpreter/src/bytes.rs b/fendermint/vm/interpreter/src/bytes.rs index b1afaad66..e3a45ca3e 100644 --- a/fendermint/vm/interpreter/src/bytes.rs +++ b/fendermint/vm/interpreter/src/bytes.rs @@ -9,6 +9,7 @@ use fvm_ipld_encoding::Error as IpldError; use crate::{ chain::{ChainMessageApplyRet, ChainMessageCheckRes}, + errors::ProcessError, fvm::{FvmQuery, FvmQueryRet}, CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProposalInterpreter, QueryInterpreter, }; @@ -129,13 +130,13 @@ where &self, state: Self::State, msgs: Vec, - ) -> anyhow::Result { + ) -> anyhow::Result { if msgs.len() > self.max_msgs { tracing::warn!( block_msgs = msgs.len(), "rejecting block: too many messages" ); - return Err(format!("too many messages: {}", msgs.len())); + return Err(ProcessError::TooManyMessages(msgs.len())); } let mut chain_msgs = Vec::new(); @@ -156,10 +157,7 @@ where "failed to decode message in proposal as ChainMessage" ); if self.reject_malformed_proposal { - return Err(format!( - "failed to decode message in proposal as ChainMessage: {}", - e - )); + return Err(ProcessError::FailedToDecodeMessage(e.to_string())); } } Ok(msg) => chain_msgs.push(msg), diff --git a/fendermint/vm/interpreter/src/chain.rs b/fendermint/vm/interpreter/src/chain.rs index 05860bfec..dae3dc951 100644 --- a/fendermint/vm/interpreter/src/chain.rs +++ b/fendermint/vm/interpreter/src/chain.rs @@ -1,5 +1,6 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use crate::errors::ProcessError; use crate::fvm::state::ipc::GatewayCaller; use crate::fvm::{topdown, FvmApplyRet, PowerUpdates}; use crate::{ @@ -179,7 +180,7 @@ where &self, env: Self::State, msgs: Vec, - ) -> anyhow::Result { + ) -> anyhow::Result { for msg in msgs { match msg { ChainMessage::Ipc(IpcMessage::BottomUpExec(msg)) => { @@ -198,7 +199,7 @@ where .await; if !is_resolved { - return Err("checkpoint not resolved".to_string()); + return Err(ProcessError::CheckpointNotResolved); } } ChainMessage::Ipc(IpcMessage::TopDownExec(ParentFinality { @@ -212,7 +213,7 @@ where let is_final = atomically(|| env.parent_finality_provider.check_proposal(&prop)).await; if !is_final { - return Err("parent finality not available".to_string()); + return Err(ProcessError::ParentFinalityNotAvailable); } } _ => {} diff --git a/fendermint/vm/interpreter/src/errors.rs b/fendermint/vm/interpreter/src/errors.rs new file mode 100644 index 000000000..62e736395 --- /dev/null +++ b/fendermint/vm/interpreter/src/errors.rs @@ -0,0 +1,16 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum ProcessError { + #[error("checkpoint not resolved")] + CheckpointNotResolved, + #[error("parent finality not available")] + ParentFinalityNotAvailable, + #[error("too many messages: {0}")] + TooManyMessages(usize), + #[error("failed to decode message in proposal as ChainMessage: {0}")] + FailedToDecodeMessage(String), +} diff --git a/fendermint/vm/interpreter/src/fvm/state/exec.rs b/fendermint/vm/interpreter/src/fvm/state/exec.rs index 46e291eae..3ab8b4fe0 100644 --- a/fendermint/vm/interpreter/src/fvm/state/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/state/exec.rs @@ -38,6 +38,8 @@ pub type ActorAddressMap = HashMap; /// The result of the message application bundled with any delegated addresses of event emitters. pub type ExecResult = anyhow::Result<(ApplyRet, ActorAddressMap)>; +use crate::fvm::observe::MsgExecCheck; + /// Parts of the state which evolve during the lifetime of the chain. #[serde_as] #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] diff --git a/fendermint/vm/interpreter/src/lib.rs b/fendermint/vm/interpreter/src/lib.rs index f8a40e6d0..49ac252ca 100644 --- a/fendermint/vm/interpreter/src/lib.rs +++ b/fendermint/vm/interpreter/src/lib.rs @@ -4,6 +4,7 @@ use async_trait::async_trait; pub mod bytes; pub mod chain; +pub mod errors; pub mod fvm; pub mod signed; @@ -58,7 +59,7 @@ pub trait ProposalInterpreter: Sync + Send { &self, state: Self::State, msgs: Vec, - ) -> anyhow::Result; + ) -> anyhow::Result; } /// The `ExecInterpreter` applies messages on some state, which is From 6b12da45005ee0213f0a165d8df4555cd19e0763 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Tue, 16 Jul 2024 16:11:24 +0200 Subject: [PATCH 18/40] feat: fix clippy --- fendermint/vm/interpreter/src/fvm/state/exec.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/fendermint/vm/interpreter/src/fvm/state/exec.rs b/fendermint/vm/interpreter/src/fvm/state/exec.rs index 3ab8b4fe0..46e291eae 100644 --- a/fendermint/vm/interpreter/src/fvm/state/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/state/exec.rs @@ -38,8 +38,6 @@ pub type ActorAddressMap = HashMap; /// The result of the message application bundled with any delegated addresses of event emitters. pub type ExecResult = anyhow::Result<(ApplyRet, ActorAddressMap)>; -use crate::fvm::observe::MsgExecCheck; - /// Parts of the state which evolve during the lifetime of the chain. #[serde_as] #[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] From 2300c6eca4907d5492e6c3bfb8751f61a8c0b93a Mon Sep 17 00:00:00 2001 From: raulk Date: Tue, 16 Jul 2024 17:42:49 +0100 Subject: [PATCH 19/40] minor cleanup. --- fendermint/app/src/cmd/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fendermint/app/src/cmd/run.rs b/fendermint/app/src/cmd/run.rs index be4f07719..adde7dccf 100644 --- a/fendermint/app/src/cmd/run.rs +++ b/fendermint/app/src/cmd/run.rs @@ -259,7 +259,7 @@ async fn run(settings: Settings) -> anyhow::Result<()> { CachedFinalityProvider::uninitialized(config.clone(), ipc_provider.clone()).await?; let p = Arc::new(Toggle::enabled(finality_provider)); - (p, Some((ipc_provider_with_latency, config))) + (p, Some((ipc_provider, config))) } else { info!("topdown finality disabled"); (Arc::new(Toggle::disabled()), None) From 66d8dafa64a0338fcbfd530389a36735f5fe03a4 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Tue, 16 Jul 2024 20:38:52 +0200 Subject: [PATCH 20/40] feat: fix typo --- fendermint/app/src/app.rs | 39 ++++++++++++++-------------- fendermint/app/src/observe.rs | 2 ++ fendermint/vm/topdown/src/observe.rs | 3 --- fendermint/vm/topdown/src/voting.rs | 4 +-- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/fendermint/app/src/app.rs b/fendermint/app/src/app.rs index 61ebadb68..5c402969b 100644 --- a/fendermint/app/src/app.rs +++ b/fendermint/app/src/app.rs @@ -716,28 +716,27 @@ where reason: None, }); - match process_result { - Ok(_) => { - emit(BlockProposalAccepted { - height: request.height.value(), - hash: HexEncodableBlockHash(request.hash.into()), - size: size_txs, - tx_count: num_txs, - validator: request.proposer_address.to_string().as_str(), - }); - Ok(response::ProcessProposal::Accept) - } + let mut proposal_evaluated = BlockProposalEvaluated { + height: request.height.value(), + hash: HexEncodableBlockHash(request.hash.into()), + size: size_txs, + tx_count: num_txs, + validator: &request.proposer_address, + accept: true, + reason: None, + }; + + let process_proposal = match process_result { + Ok(_) => response::ProcessProposal::Accept, Err(e) => { - emit(BlockProposalRejected { - height: request.height.value(), - size: size_txs, - tx_count: num_txs, - validator: request.proposer_address.to_string().as_str(), - reason: e.to_string().as_str(), - }); - Ok(response::ProcessProposal::Reject) + proposal_evaluated.accept = false; + proposal_evaluated.reason = Some(e); + response::ProcessProposal::Reject } - } + }; + + emit(proposal_evaluated); + Ok(process_proposal) } /// Signals the beginning of a new block, prior to any `DeliverTx` calls. diff --git a/fendermint/app/src/observe.rs b/fendermint/app/src/observe.rs index 793abe2fa..7f434827c 100644 --- a/fendermint/app/src/observe.rs +++ b/fendermint/app/src/observe.rs @@ -170,6 +170,8 @@ mod tests { validator: &id, }); + let id = Id::new([0x01; 20]); + emit(BlockProposalSent { height: 1, size: 100, diff --git a/fendermint/vm/topdown/src/observe.rs b/fendermint/vm/topdown/src/observe.rs index cce0b3b2e..5a0caa66e 100644 --- a/fendermint/vm/topdown/src/observe.rs +++ b/fendermint/vm/topdown/src/observe.rs @@ -1,9 +1,6 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use hex; -use std::fmt; - use ipc_observability::{ impl_traceable, impl_traceables, lazy_static, register_metrics, serde::HexEncodableBlockHash, Recordable, TraceLevel, Traceable, diff --git a/fendermint/vm/topdown/src/voting.rs b/fendermint/vm/topdown/src/voting.rs index f8e0a6522..793c2ab24 100644 --- a/fendermint/vm/topdown/src/voting.rs +++ b/fendermint/vm/topdown/src/voting.rs @@ -8,8 +8,8 @@ use std::hash::Hash; use std::{fmt::Debug, time::Duration}; use crate::observe::{ - HexEncodableBlockHash, ParentFinalityCommitted, ParentFinalityPeerQuorumReached, - ParentFinalityPeerVoteReceived, ParentFinalityPeerVoteSent, + ParentFinalityCommitted, ParentFinalityPeerQuorumReached, ParentFinalityPeerVoteReceived, + ParentFinalityPeerVoteSent, }; use crate::{BlockHash, BlockHeight}; use ipc_observability::{emit, serde::HexEncodableBlockHash}; From 48b4a08512ad1ec1bd3e1e4c23fad17cf5c662ee Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Tue, 16 Jul 2024 21:36:08 +0200 Subject: [PATCH 21/40] feat: standartize mpool trace --- fendermint/app/src/observe.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/fendermint/app/src/observe.rs b/fendermint/app/src/observe.rs index 7f434827c..793abe2fa 100644 --- a/fendermint/app/src/observe.rs +++ b/fendermint/app/src/observe.rs @@ -170,8 +170,6 @@ mod tests { validator: &id, }); - let id = Id::new([0x01; 20]); - emit(BlockProposalSent { height: 1, size: 100, From d1248b771333c88f9f7946ecabfe06fbb3413e86 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 19 Jul 2024 00:08:46 +0200 Subject: [PATCH 22/40] feat: address comments --- fendermint/app/src/app.rs | 14 ++++--- fendermint/app/src/cmd/run.rs | 4 +- fendermint/app/src/observe.rs | 15 ++++++++ fendermint/vm/interpreter/src/bytes.rs | 13 +++++-- fendermint/vm/interpreter/src/chain.rs | 13 ++++--- fendermint/vm/interpreter/src/errors.rs | 2 + .../vm/interpreter/src/fvm/state/exec.rs | 38 +------------------ .../vm/interpreter/src/fvm/state/mod.rs | 2 +- fendermint/vm/interpreter/src/lib.rs | 13 ++++++- fendermint/vm/topdown/src/proxy.rs | 1 + 10 files changed, 61 insertions(+), 54 deletions(-) diff --git a/fendermint/app/src/app.rs b/fendermint/app/src/app.rs index 5c402969b..897df8b97 100644 --- a/fendermint/app/src/app.rs +++ b/fendermint/app/src/app.rs @@ -26,7 +26,8 @@ use fendermint_vm_interpreter::fvm::store::ReadOnlyBlockstore; use fendermint_vm_interpreter::fvm::{FvmApplyRet, FvmGenesisOutput, PowerUpdates}; use fendermint_vm_interpreter::signed::InvalidSignature; use fendermint_vm_interpreter::{ - CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProposalInterpreter, QueryInterpreter, + CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProcessResult, ProposalInterpreter, + QueryInterpreter, }; use fendermint_vm_message::query::FvmQueryHeight; use fendermint_vm_snapshot::{SnapshotClient, SnapshotError}; @@ -696,7 +697,10 @@ where let size_txs = txs.iter().map(|tx| tx.len()).sum::(); let num_txs = txs.len(); - let process_result = self.interpreter.process(self.chain_env.clone(), txs).await; + let process_result = self + .interpreter + .process(self.chain_env.clone(), txs) + .await?; emit(BlockProposalReceived { height: request.height.value(), @@ -727,10 +731,10 @@ where }; let process_proposal = match process_result { - Ok(_) => response::ProcessProposal::Accept, - Err(e) => { + ProcessResult::Accepted => response::ProcessProposal::Accept, + ProcessResult::Rejected(reason) => { proposal_evaluated.accept = false; - proposal_evaluated.reason = Some(e); + proposal_evaluated.reason = Some(reason); response::ProcessProposal::Reject } }; diff --git a/fendermint/app/src/cmd/run.rs b/fendermint/app/src/cmd/run.rs index adde7dccf..87c1e11cd 100644 --- a/fendermint/app/src/cmd/run.rs +++ b/fendermint/app/src/cmd/run.rs @@ -23,7 +23,7 @@ use fendermint_vm_snapshot::{SnapshotManager, SnapshotParams}; use fendermint_vm_topdown::observe::register_metrics as register_topdown_metrics; use fendermint_vm_topdown::proxy::{IPCProviderProxy, IPCProviderProxyWithLatency}; use fendermint_vm_topdown::sync::launch_polling_syncer; -use fendermint_vm_topdown::voting::{publish_vote_loop, VoteTally}; +use fendermint_vm_topdown::voting::{publish_vote_loop, Error as VoteError, VoteTally}; use fendermint_vm_topdown::{CachedFinalityProvider, IPCParentFinality, Toggle}; use fvm_shared::address::{current_network, Address, Network}; use ipc_ipld_resolver::{Event as ResolverEvent, VoteRecord}; @@ -539,7 +539,7 @@ async fn dispatch_vote( tracing::debug!("ignoring vote; topdown disabled"); return; } - let _res = atomically_or_err(|| { + let res = atomically_or_err(|| { parent_finality_votes.add_vote( vote.public_key.clone(), f.height, diff --git a/fendermint/app/src/observe.rs b/fendermint/app/src/observe.rs index 793abe2fa..a37a6930a 100644 --- a/fendermint/app/src/observe.rs +++ b/fendermint/app/src/observe.rs @@ -1,9 +1,14 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use fendermint_vm_message::conv::from_eth; use fvm_shared::address::Address; use fvm_shared::econ::TokenAmount; +<<<<<<< HEAD +======= +use fendermint_vm_interpreter::errors::ProcessError; +>>>>>>> e7e7cb5c (feat: address comments) use fendermint_vm_interpreter::fvm::FvmMessage; use tendermint::account::Id; @@ -147,8 +152,18 @@ pub struct MpoolReceived { impl Recordable for MpoolReceived { fn record_metrics(&self) { + let from = self + .message + .as_ref() + .map(|m| m.from.to_string()) + .unwrap_or("".to_string()); + MPOOL_RECEIVED +<<<<<<< HEAD .with_label_values(&[&self.accept.to_string()]) +======= + .with_label_values(&[&self.accept.to_string(), &from]) +>>>>>>> e7e7cb5c (feat: address comments) .inc(); } } diff --git a/fendermint/vm/interpreter/src/bytes.rs b/fendermint/vm/interpreter/src/bytes.rs index e3a45ca3e..af8273fb7 100644 --- a/fendermint/vm/interpreter/src/bytes.rs +++ b/fendermint/vm/interpreter/src/bytes.rs @@ -11,7 +11,8 @@ use crate::{ chain::{ChainMessageApplyRet, ChainMessageCheckRes}, errors::ProcessError, fvm::{FvmQuery, FvmQueryRet}, - CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProposalInterpreter, QueryInterpreter, + CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProcessResult, ProposalInterpreter, + QueryInterpreter, }; pub type BytesMessageApplyRes = Result; @@ -130,13 +131,15 @@ where &self, state: Self::State, msgs: Vec, - ) -> anyhow::Result { + ) -> anyhow::Result { if msgs.len() > self.max_msgs { tracing::warn!( block_msgs = msgs.len(), "rejecting block: too many messages" ); - return Err(ProcessError::TooManyMessages(msgs.len())); + return Ok(ProcessResult::Rejected(ProcessError::TooManyMessages( + msgs.len(), + ))); } let mut chain_msgs = Vec::new(); @@ -157,7 +160,9 @@ where "failed to decode message in proposal as ChainMessage" ); if self.reject_malformed_proposal { - return Err(ProcessError::FailedToDecodeMessage(e.to_string())); + return Ok(ProcessResult::Rejected( + ProcessError::FailedToDecodeMessage(e.to_string()), + )); } } Ok(msg) => chain_msgs.push(msg), diff --git a/fendermint/vm/interpreter/src/chain.rs b/fendermint/vm/interpreter/src/chain.rs index dae3dc951..00a3ccbdb 100644 --- a/fendermint/vm/interpreter/src/chain.rs +++ b/fendermint/vm/interpreter/src/chain.rs @@ -7,7 +7,8 @@ use crate::{ fvm::state::FvmExecState, fvm::FvmMessage, signed::{SignedMessageApplyRes, SignedMessageCheckRes, SyntheticMessage, VerifiableMessage}, - CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProposalInterpreter, QueryInterpreter, + CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProcessResult, ProposalInterpreter, + QueryInterpreter, }; use anyhow::{bail, Context}; use async_stm::atomically; @@ -180,7 +181,7 @@ where &self, env: Self::State, msgs: Vec, - ) -> anyhow::Result { + ) -> anyhow::Result { for msg in msgs { match msg { ChainMessage::Ipc(IpcMessage::BottomUpExec(msg)) => { @@ -199,7 +200,7 @@ where .await; if !is_resolved { - return Err(ProcessError::CheckpointNotResolved); + return Ok(ProcessResult::Rejected(ProcessError::CheckpointNotResolved)); } } ChainMessage::Ipc(IpcMessage::TopDownExec(ParentFinality { @@ -213,13 +214,15 @@ where let is_final = atomically(|| env.parent_finality_provider.check_proposal(&prop)).await; if !is_final { - return Err(ProcessError::ParentFinalityNotAvailable); + return Ok(ProcessResult::Rejected( + ProcessError::ParentFinalityNotAvailable, + )); } } _ => {} }; } - Ok(true) + Ok(ProcessResult::Accepted) } } diff --git a/fendermint/vm/interpreter/src/errors.rs b/fendermint/vm/interpreter/src/errors.rs index 62e736395..8cdf28a40 100644 --- a/fendermint/vm/interpreter/src/errors.rs +++ b/fendermint/vm/interpreter/src/errors.rs @@ -13,4 +13,6 @@ pub enum ProcessError { TooManyMessages(usize), #[error("failed to decode message in proposal as ChainMessage: {0}")] FailedToDecodeMessage(String), + #[error("")] + Empty, } diff --git a/fendermint/vm/interpreter/src/fvm/state/exec.rs b/fendermint/vm/interpreter/src/fvm/state/exec.rs index 46e291eae..def96ce14 100644 --- a/fendermint/vm/interpreter/src/fvm/state/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/state/exec.rs @@ -177,11 +177,13 @@ where /// Execute message implicitly. pub fn execute_implicit(&mut self, msg: Message) -> ExecResult { + // TODO Karel - measure here self.execute_message(msg, ApplyKind::Implicit) } /// Execute message explicitly. pub fn execute_explicit(&mut self, msg: Message) -> ExecResult { + // TODO Karel - measure here self.execute_message(msg, ApplyKind::Explicit) } @@ -351,39 +353,3 @@ fn check_error(e: anyhow::Error) -> (ApplyRet, ActorAddressMap) { }; (ret, Default::default()) } - -pub struct ElapsedExecution<'a, DB> -where - DB: Blockstore + Clone + 'static, -{ - state: &'a mut FvmExecState, -} - -impl<'a, DB> ElapsedExecution<'a, DB> -where - DB: Blockstore + Clone + 'static, -{ - pub fn new(state: &'a mut FvmExecState) -> Self { - Self { state } - } - - pub fn execute_implicit( - &mut self, - msg: Message, - ) -> anyhow::Result<(ApplyRet, ActorAddressMap, Duration)> { - let start = Instant::now(); - let result = self.state.execute_implicit(msg)?; - let duration = start.elapsed(); - Ok((result.0, result.1, duration)) - } - - pub fn execute_explicit( - &mut self, - msg: Message, - ) -> anyhow::Result<(ApplyRet, ActorAddressMap, Duration)> { - let start = Instant::now(); - let result = self.state.execute_explicit(msg)?; - let duration = start.elapsed(); - Ok((result.0, result.1, duration)) - } -} diff --git a/fendermint/vm/interpreter/src/fvm/state/mod.rs b/fendermint/vm/interpreter/src/fvm/state/mod.rs index 0547ce9d8..22a1504e4 100644 --- a/fendermint/vm/interpreter/src/fvm/state/mod.rs +++ b/fendermint/vm/interpreter/src/fvm/state/mod.rs @@ -12,7 +12,7 @@ pub mod snapshot; use std::sync::Arc; pub use check::FvmCheckState; -pub use exec::{BlockHash, ElapsedExecution, FvmExecState, FvmStateParams, FvmUpdatableParams}; +pub use exec::{BlockHash, FvmExecState, FvmStateParams, FvmUpdatableParams}; pub use genesis::{empty_state_tree, FvmGenesisState}; pub use query::FvmQueryState; diff --git a/fendermint/vm/interpreter/src/lib.rs b/fendermint/vm/interpreter/src/lib.rs index 49ac252ca..011249b02 100644 --- a/fendermint/vm/interpreter/src/lib.rs +++ b/fendermint/vm/interpreter/src/lib.rs @@ -28,6 +28,17 @@ pub trait GenesisInterpreter: Sync + Send { ) -> anyhow::Result<(Self::State, Self::Output)>; } +pub enum ProcessResult { + Accepted, + Rejected(errors::ProcessError), +} + +impl ProcessResult { + pub fn is_accepted(&self) -> bool { + matches!(self, ProcessResult::Accepted) + } +} + /// Prepare and process transaction proposals. #[async_trait] pub trait ProposalInterpreter: Sync + Send { @@ -59,7 +70,7 @@ pub trait ProposalInterpreter: Sync + Send { &self, state: Self::State, msgs: Vec, - ) -> anyhow::Result; + ) -> anyhow::Result; } /// The `ExecInterpreter` applies messages on some state, which is diff --git a/fendermint/vm/topdown/src/proxy.rs b/fendermint/vm/topdown/src/proxy.rs index abbd3812f..d52c3b70d 100644 --- a/fendermint/vm/topdown/src/proxy.rs +++ b/fendermint/vm/topdown/src/proxy.rs @@ -118,6 +118,7 @@ impl ParentQueryProxy for IPCProviderProxy { } } +// TODO - create a macro for this pub struct IPCProviderProxyWithLatency { inner: IPCProviderProxy, } From 1c77f3acf27de6d979d923018ecca3509002e20b Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 19 Jul 2024 00:15:10 +0200 Subject: [PATCH 23/40] lint: clippy --- fendermint/app/src/observe.rs | 1 - fendermint/vm/interpreter/src/fvm/state/exec.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/fendermint/app/src/observe.rs b/fendermint/app/src/observe.rs index a37a6930a..792f4c9be 100644 --- a/fendermint/app/src/observe.rs +++ b/fendermint/app/src/observe.rs @@ -1,7 +1,6 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use fendermint_vm_message::conv::from_eth; use fvm_shared::address::Address; use fvm_shared::econ::TokenAmount; diff --git a/fendermint/vm/interpreter/src/fvm/state/exec.rs b/fendermint/vm/interpreter/src/fvm/state/exec.rs index def96ce14..34c4dc765 100644 --- a/fendermint/vm/interpreter/src/fvm/state/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/state/exec.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0, MIT use std::collections::{HashMap, HashSet}; -use std::time::{Duration, Instant}; use anyhow::Ok; use cid::Cid; From e94d7900f97998f3ab9306a07de53c746b006819 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Tue, 9 Jul 2024 19:09:25 +0200 Subject: [PATCH 24/40] feat: add observe events & metrics to checkpoint --- .../vm/interpreter/src/fvm/checkpoint.rs | 20 ++-- fendermint/vm/interpreter/src/fvm/mod.rs | 2 +- fendermint/vm/interpreter/src/fvm/observe.rs | 92 ++++++++++++++++++- 3 files changed, 105 insertions(+), 9 deletions(-) diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index 8efd04cb7..786f84320 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -15,16 +15,16 @@ use fvm_shared::{address::Address, chainid::ChainID}; use fendermint_crypto::PublicKey; use fendermint_crypto::SecretKey; -use fendermint_tracing::emit; use fendermint_vm_actor_interface::eam::EthAddress; use fendermint_vm_actor_interface::ipc::BottomUpCheckpoint; -use fendermint_vm_event::NewBottomUpCheckpoint; use fendermint_vm_genesis::{Power, Validator, ValidatorKey}; use ipc_actors_abis::checkpointing_facet as checkpoint; use ipc_actors_abis::gateway_getter_facet as getter; use ipc_api::staking::ConfigurationNumber; +use ipc_observability::emit; +use super::observe::{CheckpointCreated, CheckpointSigned}; use super::state::ipc::tokens_to_burn; use super::{ broadcast::Broadcaster, @@ -121,11 +121,11 @@ where power_diff(curr_power_table, next_power_table) }; - emit!(NewBottomUpCheckpoint { - block_height: height.value(), - block_hash: &hex::encode(block_hash), - num_msgs, - next_configuration_number, + emit(CheckpointCreated { + height: height.value(), + hash: &hex::encode(block_hash), + msg_count: num_msgs, + config_number: next_configuration_number, }); Ok(Some((checkpoint, power_updates))) @@ -255,6 +255,12 @@ where .await .context("failed to broadcast checkpoint signature")?; + emit(CheckpointSigned { + height: height.value(), + hash: &hex::encode(cp.block_hash), + validator: &hex::encode(validator_ctx.public_key.serialize()), + }); + tracing::debug!(?height, "submitted checkpoint signature"); } } diff --git a/fendermint/vm/interpreter/src/fvm/mod.rs b/fendermint/vm/interpreter/src/fvm/mod.rs index 83cb934d9..0aefb4b2e 100644 --- a/fendermint/vm/interpreter/src/fvm/mod.rs +++ b/fendermint/vm/interpreter/src/fvm/mod.rs @@ -8,7 +8,7 @@ mod checkpoint; mod exec; mod externs; mod genesis; -mod observe; +pub mod observe; mod query; pub mod state; pub mod store; diff --git a/fendermint/vm/interpreter/src/fvm/observe.rs b/fendermint/vm/interpreter/src/fvm/observe.rs index 02f1b86d0..55583c7ca 100644 --- a/fendermint/vm/interpreter/src/fvm/observe.rs +++ b/fendermint/vm/interpreter/src/fvm/observe.rs @@ -5,7 +5,11 @@ use ipc_observability::{ impl_traceable, impl_traceables, lazy_static, register_metrics, Recordable, TraceLevel, Traceable, }; -use prometheus::{register_histogram, Histogram, Registry}; + +use prometheus::{ + register_histogram, register_int_counter, register_int_gauge, register_int_gauge_vec, + Histogram, IntCounter, IntGauge, IntGaugeVec, Registry, +}; use fvm_shared::message::Message; @@ -18,6 +22,21 @@ register_metrics! { = register_histogram!("exec_fvm_apply_execution_time_secs", "Execution time of FVM apply in seconds"); EXEC_FVM_CALL_EXECUTION_TIME_SECS: Histogram = register_histogram!("exec_fvm_call_execution_time_secs", "Execution time of FVM call in seconds"); + BOTTOMUP_CHECKPOINT_CREATED_TOTAL: IntCounter + = register_int_counter!("bottomup_checkpoint_created_total", "Bottom-up checkpoint produced"); + BOTTOMUP_CHECKPOINT_CREATED_HEIGHT: IntGauge + = register_int_gauge!("bottomup_checkpoint_created_height", "Height of the checkpoint created"); + BOTTOMUP_CHECKPOINT_CREATED_MSGCOUNT: IntGauge + = register_int_gauge!("bottomup_checkpoint_created_msgcount", "Number of messages in the checkpoint created"); + BOTTOMUP_CHECKPOINT_CREATED_CONFIGNUM: IntGauge + = register_int_gauge!("bottomup_checkpoint_created_confignum", "Configuration number of the checkpoint created"); + BOTTOMUP_CHECKPOINT_SIGNED_HEIGHT: IntGaugeVec = register_int_gauge_vec!( + "bottomup_checkpoint_signed_height", + "Height of the checkpoint signed", + &["validator"] + ); + BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT: IntGauge + = register_int_gauge!("bottomup_checkpoint_finalized_height", "Height of the checkpoint finalized"); } impl_traceables!(TraceLevel::Info, "Execution", MsgExec); @@ -54,6 +73,61 @@ impl Recordable for MsgExec { } } +impl_traceables!( + TraceLevel::Info, + "Bottom-up", + CheckpointCreated<'a>, + CheckpointSigned<'a>, + CheckpointFinalized<'a> +); + +/// Hex encoded hash. +pub type HashHex<'a> = &'a str; + +#[derive(Debug)] +pub struct CheckpointCreated<'a> { + pub height: u64, + pub hash: HashHex<'a>, + pub msg_count: usize, + pub config_number: u64, +} + +impl Recordable for CheckpointCreated<'_> { + fn record_metrics(&self) { + BOTTOMUP_CHECKPOINT_CREATED_TOTAL.inc(); + BOTTOMUP_CHECKPOINT_CREATED_HEIGHT.set(self.height as i64); + BOTTOMUP_CHECKPOINT_CREATED_MSGCOUNT.set(self.msg_count as i64); + BOTTOMUP_CHECKPOINT_CREATED_CONFIGNUM.set(self.config_number as i64); + } +} + +#[derive(Debug)] +pub struct CheckpointSigned<'a> { + pub height: u64, + pub hash: HashHex<'a>, + pub validator: &'a str, +} + +impl Recordable for CheckpointSigned<'_> { + fn record_metrics(&self) { + BOTTOMUP_CHECKPOINT_SIGNED_HEIGHT + .with_label_values(&[self.validator]) + .set(self.height as i64); + } +} + +#[derive(Debug)] +pub struct CheckpointFinalized<'a> { + pub height: usize, + pub hash: HashHex<'a>, +} + +impl Recordable for CheckpointFinalized<'_> { + fn record_metrics(&self) { + BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT.set(self.height as i64); + } +} + #[cfg(test)] mod tests { use super::*; @@ -91,5 +165,21 @@ mod tests { exit_code: 1, message: message.clone(), }); + + emit(CheckpointCreated { + height: 1, + hash: "hash", + msg_count: 2, + config_number: 3, + }); + emit(CheckpointSigned { + height: 1, + hash: "hash", + validator: "validator", + }); + emit(CheckpointFinalized { + height: 1, + hash: "hash", + }); } } From a91ff7abd39cd1f707abe4b9257d4b46eb2fc5e5 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Wed, 10 Jul 2024 15:21:17 +0200 Subject: [PATCH 25/40] feat: improve hash --- .../vm/interpreter/src/fvm/checkpoint.rs | 6 ++-- fendermint/vm/interpreter/src/fvm/observe.rs | 36 ++++++++++++------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index 786f84320..5042b7844 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -24,7 +24,7 @@ use ipc_actors_abis::gateway_getter_facet as getter; use ipc_api::staking::ConfigurationNumber; use ipc_observability::emit; -use super::observe::{CheckpointCreated, CheckpointSigned}; +use super::observe::{CheckpointCreated, CheckpointSigned, HexEncodableBlockHash}; use super::state::ipc::tokens_to_burn; use super::{ broadcast::Broadcaster, @@ -123,7 +123,7 @@ where emit(CheckpointCreated { height: height.value(), - hash: &hex::encode(block_hash), + hash: HexEncodableBlockHash(block_hash.to_vec()), msg_count: num_msgs, config_number: next_configuration_number, }); @@ -257,7 +257,7 @@ where emit(CheckpointSigned { height: height.value(), - hash: &hex::encode(cp.block_hash), + hash: HexEncodableBlockHash(cp.block_hash.to_vec()), validator: &hex::encode(validator_ctx.public_key.serialize()), }); diff --git a/fendermint/vm/interpreter/src/fvm/observe.rs b/fendermint/vm/interpreter/src/fvm/observe.rs index 55583c7ca..ce032efd2 100644 --- a/fendermint/vm/interpreter/src/fvm/observe.rs +++ b/fendermint/vm/interpreter/src/fvm/observe.rs @@ -1,6 +1,9 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT +use hex; +use std::fmt; + use ipc_observability::{ impl_traceable, impl_traceables, lazy_static, register_metrics, Recordable, TraceLevel, Traceable, @@ -76,23 +79,31 @@ impl Recordable for MsgExec { impl_traceables!( TraceLevel::Info, "Bottom-up", - CheckpointCreated<'a>, + CheckpointCreated, CheckpointSigned<'a>, - CheckpointFinalized<'a> + CheckpointFinalized ); /// Hex encoded hash. pub type HashHex<'a> = &'a str; +// Hex encodable block hash. +pub struct HexEncodableBlockHash(pub Vec); + +impl fmt::Debug for HexEncodableBlockHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", hex::encode(&self.0)) + } +} #[derive(Debug)] -pub struct CheckpointCreated<'a> { +pub struct CheckpointCreated { pub height: u64, - pub hash: HashHex<'a>, + pub hash: HexEncodableBlockHash, pub msg_count: usize, pub config_number: u64, } -impl Recordable for CheckpointCreated<'_> { +impl Recordable for CheckpointCreated { fn record_metrics(&self) { BOTTOMUP_CHECKPOINT_CREATED_TOTAL.inc(); BOTTOMUP_CHECKPOINT_CREATED_HEIGHT.set(self.height as i64); @@ -104,7 +115,7 @@ impl Recordable for CheckpointCreated<'_> { #[derive(Debug)] pub struct CheckpointSigned<'a> { pub height: u64, - pub hash: HashHex<'a>, + pub hash: HexEncodableBlockHash, pub validator: &'a str, } @@ -117,12 +128,12 @@ impl Recordable for CheckpointSigned<'_> { } #[derive(Debug)] -pub struct CheckpointFinalized<'a> { +pub struct CheckpointFinalized { pub height: usize, - pub hash: HashHex<'a>, + pub hash: HexEncodableBlockHash, } -impl Recordable for CheckpointFinalized<'_> { +impl Recordable for CheckpointFinalized { fn record_metrics(&self) { BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT.set(self.height as i64); } @@ -165,21 +176,22 @@ mod tests { exit_code: 1, message: message.clone(), }); + let hash = vec![0x01, 0x02, 0x03]; emit(CheckpointCreated { height: 1, - hash: "hash", + hash: HexEncodableBlockHash(hash.clone()), msg_count: 2, config_number: 3, }); emit(CheckpointSigned { height: 1, - hash: "hash", + hash: HexEncodableBlockHash(hash.clone()), validator: "validator", }); emit(CheckpointFinalized { height: 1, - hash: "hash", + hash: HexEncodableBlockHash(hash.clone()), }); } } From 5e8588582378ec709e107b098d9902d1ca4964be Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 12 Jul 2024 14:07:21 +0200 Subject: [PATCH 26/40] feat: trigger last event --- fendermint/vm/interpreter/src/fvm/checkpoint.rs | 9 ++++++++- fendermint/vm/interpreter/src/fvm/observe.rs | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index 5042b7844..bdcea9c64 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -24,7 +24,9 @@ use ipc_actors_abis::gateway_getter_facet as getter; use ipc_api::staking::ConfigurationNumber; use ipc_observability::emit; -use super::observe::{CheckpointCreated, CheckpointSigned, HexEncodableBlockHash}; +use super::observe::{ + CheckpointCreated, CheckpointFinalized, CheckpointSigned, HexEncodableBlockHash, +}; use super::state::ipc::tokens_to_burn; use super::{ broadcast::Broadcaster, @@ -128,6 +130,11 @@ where config_number: next_configuration_number, }); + emit(CheckpointFinalized { + height: height.value(), + hash: HexEncodableBlockHash(block_hash.to_vec()), + }); + Ok(Some((checkpoint, power_updates))) } diff --git a/fendermint/vm/interpreter/src/fvm/observe.rs b/fendermint/vm/interpreter/src/fvm/observe.rs index ce032efd2..1f3b5056a 100644 --- a/fendermint/vm/interpreter/src/fvm/observe.rs +++ b/fendermint/vm/interpreter/src/fvm/observe.rs @@ -78,7 +78,7 @@ impl Recordable for MsgExec { impl_traceables!( TraceLevel::Info, - "Bottom-up", + "Bottomup", CheckpointCreated, CheckpointSigned<'a>, CheckpointFinalized @@ -129,7 +129,7 @@ impl Recordable for CheckpointSigned<'_> { #[derive(Debug)] pub struct CheckpointFinalized { - pub height: usize, + pub height: u64, pub hash: HexEncodableBlockHash, } From 60ecd3bebafc4525ed890f9677bcda9a1c8dd6d6 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Mon, 15 Jul 2024 12:24:03 +0200 Subject: [PATCH 27/40] feat: add log to releayer --- ipc/provider/src/checkpoint.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index 0cb77efe1..ad60d4064 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -203,11 +203,15 @@ impl BottomUpCheckpointMan let result = Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event) .await + .inspect(|_| { + tracing::info!("submitted bottom up checkpoint at height {height}"); + }) .inspect_err(|err| { tracing::error!( "Fail to submit checkpoint at height {height}: {err}" ); }); + drop(submission_permit); result })); From 5a2c6f88009a57116b2e7edaf163d32c3100c3c8 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Mon, 15 Jul 2024 13:19:13 +0200 Subject: [PATCH 28/40] feat: add to relayer --- Cargo.lock | 4 ++ ipc/cli/Cargo.toml | 2 + ipc/cli/src/commands/checkpoint/relayer.rs | 31 +++++++++++ ipc/provider/Cargo.toml | 2 + ipc/provider/src/checkpoint.rs | 9 +++- ipc/provider/src/lib.rs | 1 + ipc/provider/src/observe.rs | 60 ++++++++++++++++++++++ 7 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 ipc/provider/src/observe.rs diff --git a/Cargo.lock b/Cargo.lock index 07f591bd0..c55c1291f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5037,6 +5037,8 @@ dependencies = [ "num-derive 0.3.3", "num-traits", "openssl", + "prometheus", + "prometheus_exporter", "reqwest", "serde", "serde_bytes", @@ -5084,6 +5086,7 @@ dependencies = [ "hex", "indoc", "ipc-api", + "ipc-observability", "ipc-types", "ipc-wallet", "ipc_actors_abis", @@ -5091,6 +5094,7 @@ dependencies = [ "log", "num-derive 0.3.3", "num-traits", + "prometheus", "reqwest", "serde", "serde_bytes", diff --git a/ipc/cli/Cargo.toml b/ipc/cli/Cargo.toml index 6dc4e71d5..493a0d420 100644 --- a/ipc/cli/Cargo.toml +++ b/ipc/cli/Cargo.toml @@ -41,6 +41,8 @@ tokio-tungstenite = { workspace = true } toml = "0.7.2" url = { workspace = true } zeroize = "1.6.0" +prometheus = { workspace = true } +prometheus_exporter = { workspace = true } ipc-wallet = { workspace = true } ipc-provider = { workspace = true } diff --git a/ipc/cli/src/commands/checkpoint/relayer.rs b/ipc/cli/src/commands/checkpoint/relayer.rs index 43d927b40..e89de5252 100644 --- a/ipc/cli/src/commands/checkpoint/relayer.rs +++ b/ipc/cli/src/commands/checkpoint/relayer.rs @@ -4,6 +4,7 @@ use crate::commands::get_subnet_config; use crate::{require_fil_addr_from_str, CommandLineHandler, GlobalArguments}; use anyhow::anyhow; +use anyhow::Context; use async_trait::async_trait; use clap::Args; use fvm_shared::address::Address; @@ -12,11 +13,16 @@ use ipc_api::subnet_id::SubnetID; use ipc_provider::checkpoint::BottomUpCheckpointManager; use ipc_provider::config::Config; use ipc_provider::new_evm_keystore_from_config; +use ipc_provider::observe::register_metrics as register_checkpoint_metrics; use ipc_wallet::EvmKeyStore; +use std::net::SocketAddr; use std::str::FromStr; use std::sync::{Arc, RwLock}; use std::time::Duration; +use prometheus; +use prometheus_exporter; + const DEFAULT_POLLING_INTERVAL: u64 = 15; /// The command to run the bottom up relayer in the background. @@ -29,6 +35,25 @@ impl CommandLineHandler for BottomUpRelayer { async fn handle(global: &GlobalArguments, arguments: &Self::Arguments) -> anyhow::Result<()> { log::debug!("start bottom up relayer with args: {:?}", arguments); + // Prometheus metrics + match &arguments.metrics_address { + Some(addr) => { + let addr = SocketAddr::from_str(addr)?; + + let registry = prometheus::Registry::new(); + register_checkpoint_metrics(®istry)?; + + let mut builder = prometheus_exporter::Builder::new(addr); + builder.with_registry(registry); + let _ = builder.start().context("failed to start metrics server")?; + + log::info!("serving metrics on: {addr}"); + } + None => { + log::info!("metrics disabled"); + } + } + let config_path = global.config_path(); let config = Arc::new(Config::from_file(&config_path)?); let mut keystore = new_evm_keystore_from_config(config)?; @@ -95,4 +120,10 @@ pub(crate) struct BottomUpRelayerArgs { help = "The max parallelism for submitting checkpoints" )] pub max_parallelism: usize, + + #[arg( + long, + help = "Metrics address to listen on. Enables Prometheus metrics if set" + )] + pub metrics_address: Option, } diff --git a/ipc/provider/Cargo.toml b/ipc/provider/Cargo.toml index cacfe9aeb..0d56f7f9f 100644 --- a/ipc/provider/Cargo.toml +++ b/ipc/provider/Cargo.toml @@ -47,6 +47,8 @@ ipc-types = { workspace = true } ipc-wallet = { workspace = true, features = ["with-ethers"] } ipc-api = { workspace = true } ipc_actors_abis = { workspace = true } +ipc-observability = { workspace = true } +prometheus = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index ad60d4064..d01572de5 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -4,11 +4,13 @@ use crate::config::Subnet; use crate::manager::{BottomUpCheckpointRelayer, EthSubnetManager}; +use crate::observe::{CheckpointFinalized, HexEncodableBlockHash}; use anyhow::{anyhow, Result}; use futures_util::future::try_join_all; use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; use ipc_api::checkpoint::{BottomUpCheckpointBundle, QuorumReachedEvent}; +use ipc_observability::emit; use ipc_wallet::{EthKeyAddress, PersistentKeyStore}; use std::cmp::max; use std::fmt::{Display, Formatter}; @@ -200,11 +202,16 @@ impl BottomUpCheckpointMan .unwrap(); all_submit_tasks.push(tokio::task::spawn(async move { let height = event.height; + let hash = bundle.checkpoint.block_hash.clone(); + let result = Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event) .await .inspect(|_| { - tracing::info!("submitted bottom up checkpoint at height {height}"); + emit(CheckpointFinalized { + height, + hash: HexEncodableBlockHash(hash), + }); }) .inspect_err(|err| { tracing::error!( diff --git a/ipc/provider/src/lib.rs b/ipc/provider/src/lib.rs index 4cf58c1ee..c9cf0efc1 100644 --- a/ipc/provider/src/lib.rs +++ b/ipc/provider/src/lib.rs @@ -38,6 +38,7 @@ pub mod config; pub mod jsonrpc; pub mod lotus; pub mod manager; +pub mod observe; const DEFAULT_REPO_PATH: &str = ".ipc"; const DEFAULT_CONFIG_NAME: &str = "config.toml"; diff --git a/ipc/provider/src/observe.rs b/ipc/provider/src/observe.rs new file mode 100644 index 000000000..187f0d13e --- /dev/null +++ b/ipc/provider/src/observe.rs @@ -0,0 +1,60 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use std::fmt; + +use ipc_observability::{ + impl_traceable, impl_traceables, lazy_static, register_metrics, Recordable, TraceLevel, + Traceable, +}; +use prometheus::{register_int_gauge, IntGauge, Registry}; + +register_metrics! { + BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT: IntGauge + = register_int_gauge!("bottomup_checkpoint_finalized_height", "Height of the checkpoint finalized"); +} + +impl_traceables!(TraceLevel::Info, "Bottomup", CheckpointFinalized); + +// Hex encodable block hash. +pub struct HexEncodableBlockHash(pub Vec); + +impl fmt::Debug for HexEncodableBlockHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", hex::encode(&self.0)) + } +} + +#[derive(Debug)] +pub struct CheckpointFinalized { + pub height: i64, + pub hash: HexEncodableBlockHash, +} + +impl Recordable for CheckpointFinalized { + fn record_metrics(&self) { + BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT.set(self.height); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ipc_observability::emit; + + #[test] + fn test_metrics() { + let registry = Registry::new(); + register_metrics(®istry).unwrap(); + } + + #[test] + fn test_emit() { + let hash = vec![0x01, 0x02, 0x03]; + + emit(CheckpointFinalized { + height: 1, + hash: HexEncodableBlockHash(hash.clone()), + }); + } +} From 7faf32863fbef2a3ff2d3f21f2781df2fb7fc6ce Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Mon, 15 Jul 2024 14:26:00 +0200 Subject: [PATCH 29/40] feat: fix clippy --- ipc/cli/src/commands/checkpoint/relayer.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ipc/cli/src/commands/checkpoint/relayer.rs b/ipc/cli/src/commands/checkpoint/relayer.rs index e89de5252..12a121880 100644 --- a/ipc/cli/src/commands/checkpoint/relayer.rs +++ b/ipc/cli/src/commands/checkpoint/relayer.rs @@ -20,9 +20,6 @@ use std::str::FromStr; use std::sync::{Arc, RwLock}; use std::time::Duration; -use prometheus; -use prometheus_exporter; - const DEFAULT_POLLING_INTERVAL: u64 = 15; /// The command to run the bottom up relayer in the background. @@ -38,6 +35,9 @@ impl CommandLineHandler for BottomUpRelayer { // Prometheus metrics match &arguments.metrics_address { Some(addr) => { + use prometheus; + use prometheus_exporter; + let addr = SocketAddr::from_str(addr)?; let registry = prometheus::Registry::new(); From 92fa8b2cba65ba3e889be77d06ab5744407115bc Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Mon, 15 Jul 2024 14:53:54 +0200 Subject: [PATCH 30/40] feat: remove unused code --- .../vm/interpreter/src/fvm/checkpoint.rs | 9 +------- fendermint/vm/interpreter/src/fvm/observe.rs | 21 +------------------ 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index bdcea9c64..5042b7844 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -24,9 +24,7 @@ use ipc_actors_abis::gateway_getter_facet as getter; use ipc_api::staking::ConfigurationNumber; use ipc_observability::emit; -use super::observe::{ - CheckpointCreated, CheckpointFinalized, CheckpointSigned, HexEncodableBlockHash, -}; +use super::observe::{CheckpointCreated, CheckpointSigned, HexEncodableBlockHash}; use super::state::ipc::tokens_to_burn; use super::{ broadcast::Broadcaster, @@ -130,11 +128,6 @@ where config_number: next_configuration_number, }); - emit(CheckpointFinalized { - height: height.value(), - hash: HexEncodableBlockHash(block_hash.to_vec()), - }); - Ok(Some((checkpoint, power_updates))) } diff --git a/fendermint/vm/interpreter/src/fvm/observe.rs b/fendermint/vm/interpreter/src/fvm/observe.rs index 1f3b5056a..ce16f50ae 100644 --- a/fendermint/vm/interpreter/src/fvm/observe.rs +++ b/fendermint/vm/interpreter/src/fvm/observe.rs @@ -38,8 +38,6 @@ register_metrics! { "Height of the checkpoint signed", &["validator"] ); - BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT: IntGauge - = register_int_gauge!("bottomup_checkpoint_finalized_height", "Height of the checkpoint finalized"); } impl_traceables!(TraceLevel::Info, "Execution", MsgExec); @@ -80,8 +78,7 @@ impl_traceables!( TraceLevel::Info, "Bottomup", CheckpointCreated, - CheckpointSigned<'a>, - CheckpointFinalized + CheckpointSigned<'a> ); /// Hex encoded hash. @@ -127,18 +124,6 @@ impl Recordable for CheckpointSigned<'_> { } } -#[derive(Debug)] -pub struct CheckpointFinalized { - pub height: u64, - pub hash: HexEncodableBlockHash, -} - -impl Recordable for CheckpointFinalized { - fn record_metrics(&self) { - BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT.set(self.height as i64); - } -} - #[cfg(test)] mod tests { use super::*; @@ -189,9 +174,5 @@ mod tests { hash: HexEncodableBlockHash(hash.clone()), validator: "validator", }); - emit(CheckpointFinalized { - height: 1, - hash: HexEncodableBlockHash(hash.clone()), - }); } } From af0ff70a70875a0ba481778ba054c63fb8c4ef00 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Wed, 17 Jul 2024 14:07:19 +0200 Subject: [PATCH 31/40] feat: emit checkpoint finalized from fvm & cleanup after merge --- .../vm/interpreter/src/fvm/checkpoint.rs | 34 +++++++++++++++++-- fendermint/vm/interpreter/src/fvm/exec.rs | 2 ++ fendermint/vm/interpreter/src/fvm/observe.rs | 24 +++++++++---- .../vm/interpreter/src/fvm/state/ipc.rs | 13 ++++++- ipc/provider/src/checkpoint.rs | 4 +-- ipc/provider/src/observe.rs | 15 ++------ 6 files changed, 68 insertions(+), 24 deletions(-) diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index 5042b7844..cb4b1b465 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -22,9 +22,9 @@ use fendermint_vm_genesis::{Power, Validator, ValidatorKey}; use ipc_actors_abis::checkpointing_facet as checkpoint; use ipc_actors_abis::gateway_getter_facet as getter; use ipc_api::staking::ConfigurationNumber; -use ipc_observability::emit; +use ipc_observability::{emit, serde::HexEncodableBlockHash}; -use super::observe::{CheckpointCreated, CheckpointSigned, HexEncodableBlockHash}; +use super::observe::{CheckpointCreated, CheckpointFinalized, CheckpointSigned}; use super::state::ipc::tokens_to_burn; use super::{ broadcast::Broadcaster, @@ -296,6 +296,36 @@ where Ok(()) } +// Emit a CheckpointFinalized trace event if a checkpoint has been finalized on the current block. +pub fn emit_trace_if_check_checkpoint_finalized( + gateway: &GatewayCaller, + state: &mut FvmExecState, +) -> anyhow::Result<()> +where + DB: Blockstore + Clone, +{ + if !gateway.enabled(state)? { + return Ok(()); + } + + let block_height = state.block_height(); + let block_hash = state + .block_hash() + .ok_or_else(|| anyhow!("block hash not set"))?; + + // Check if the checkpoint has been finalized. + let checkpoint_quorum = gateway.checkpoint_info(state, block_height)?; + + if checkpoint_quorum.reached { + emit(CheckpointFinalized { + height: block_height, + hash: HexEncodableBlockHash(block_hash.to_vec()), + }) + } + + Ok(()) +} + fn convert_tokenizables( tokenizables: Vec, ) -> anyhow::Result> { diff --git a/fendermint/vm/interpreter/src/fvm/exec.rs b/fendermint/vm/interpreter/src/fvm/exec.rs index 150cb218a..77235229b 100644 --- a/fendermint/vm/interpreter/src/fvm/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/exec.rs @@ -186,6 +186,8 @@ where } async fn end(&self, mut state: Self::State) -> anyhow::Result<(Self::State, Self::EndOutput)> { + checkpoint::emit_trace_if_check_checkpoint_finalized(&self.gateway, &mut state)?; + let updates = if let Some((checkpoint, updates)) = checkpoint::maybe_create_checkpoint(&self.gateway, &mut state) .context("failed to create checkpoint")? diff --git a/fendermint/vm/interpreter/src/fvm/observe.rs b/fendermint/vm/interpreter/src/fvm/observe.rs index ce16f50ae..d7844ce82 100644 --- a/fendermint/vm/interpreter/src/fvm/observe.rs +++ b/fendermint/vm/interpreter/src/fvm/observe.rs @@ -1,12 +1,9 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use hex; -use std::fmt; - use ipc_observability::{ - impl_traceable, impl_traceables, lazy_static, register_metrics, Recordable, TraceLevel, - Traceable, + impl_traceable, impl_traceables, lazy_static, register_metrics, serde::HexEncodableBlockHash, + Recordable, TraceLevel, Traceable, }; use prometheus::{ @@ -38,6 +35,8 @@ register_metrics! { "Height of the checkpoint signed", &["validator"] ); + BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT: IntGauge + = register_int_gauge!("bottomup_checkpoint_finalized_height", "Height of the checkpoint finalized"); } impl_traceables!(TraceLevel::Info, "Execution", MsgExec); @@ -78,7 +77,8 @@ impl_traceables!( TraceLevel::Info, "Bottomup", CheckpointCreated, - CheckpointSigned<'a> + CheckpointSigned<'a>, + CheckpointFinalized ); /// Hex encoded hash. @@ -124,6 +124,18 @@ impl Recordable for CheckpointSigned<'_> { } } +#[derive(Debug)] +pub struct CheckpointFinalized { + pub height: i64, + pub hash: HexEncodableBlockHash, +} + +impl Recordable for CheckpointFinalized { + fn record_metrics(&self) { + BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT.set(self.height); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/fendermint/vm/interpreter/src/fvm/state/ipc.rs b/fendermint/vm/interpreter/src/fvm/state/ipc.rs index 4cd74d5fc..97c510f23 100644 --- a/fendermint/vm/interpreter/src/fvm/state/ipc.rs +++ b/fendermint/vm/interpreter/src/fvm/state/ipc.rs @@ -146,6 +146,17 @@ impl GatewayCaller { self.getter.call(state, |c| c.get_incomplete_checkpoints()) } + /// Retrieve checkpoint info by block height. + pub fn checkpoint_info( + &self, + state: &mut FvmExecState, + height: i64, + ) -> anyhow::Result { + self.getter.call(state, |c| { + c.get_checkpoint_info(ethers::types::U256::from(height)) + }) + } + /// Apply all pending validator changes, returning the newly adopted configuration number, or 0 if there were no changes. pub fn apply_validator_changes(&self, state: &mut FvmExecState) -> anyhow::Result { self.topdown.call(state, |c| c.apply_finality_changes()) @@ -304,7 +315,7 @@ impl GatewayCaller { height: u64, ) -> anyhow::Result> { let (_, _, addrs, _) = self.getter.call(state, |c| { - c.get_checkpoint_signature_bundle(ethers::types::U256::from(height)) + c.get_checkpoint_signature_bundle(et::U256::from(height)) })?; let addrs = addrs.into_iter().map(|a| a.into()).collect(); diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index d01572de5..81edd5234 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -4,13 +4,13 @@ use crate::config::Subnet; use crate::manager::{BottomUpCheckpointRelayer, EthSubnetManager}; -use crate::observe::{CheckpointFinalized, HexEncodableBlockHash}; +use crate::observe::CheckpointFinalized; use anyhow::{anyhow, Result}; use futures_util::future::try_join_all; use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; use ipc_api::checkpoint::{BottomUpCheckpointBundle, QuorumReachedEvent}; -use ipc_observability::emit; +use ipc_observability::{emit, serde::HexEncodableBlockHash}; use ipc_wallet::{EthKeyAddress, PersistentKeyStore}; use std::cmp::max; use std::fmt::{Display, Formatter}; diff --git a/ipc/provider/src/observe.rs b/ipc/provider/src/observe.rs index 187f0d13e..604c8c954 100644 --- a/ipc/provider/src/observe.rs +++ b/ipc/provider/src/observe.rs @@ -1,11 +1,9 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use std::fmt; - use ipc_observability::{ - impl_traceable, impl_traceables, lazy_static, register_metrics, Recordable, TraceLevel, - Traceable, + impl_traceable, impl_traceables, lazy_static, register_metrics, serde::HexEncodableBlockHash, + Recordable, TraceLevel, Traceable, }; use prometheus::{register_int_gauge, IntGauge, Registry}; @@ -16,15 +14,6 @@ register_metrics! { impl_traceables!(TraceLevel::Info, "Bottomup", CheckpointFinalized); -// Hex encodable block hash. -pub struct HexEncodableBlockHash(pub Vec); - -impl fmt::Debug for HexEncodableBlockHash { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", hex::encode(&self.0)) - } -} - #[derive(Debug)] pub struct CheckpointFinalized { pub height: i64, From 6ddaf9b4b0570db403cd8a055676912c70363dc5 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 19 Jul 2024 00:20:41 +0200 Subject: [PATCH 32/40] fix: remove dead code --- fendermint/vm/interpreter/src/fvm/observe.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/fendermint/vm/interpreter/src/fvm/observe.rs b/fendermint/vm/interpreter/src/fvm/observe.rs index d7844ce82..52ad9dd7c 100644 --- a/fendermint/vm/interpreter/src/fvm/observe.rs +++ b/fendermint/vm/interpreter/src/fvm/observe.rs @@ -83,14 +83,6 @@ impl_traceables!( /// Hex encoded hash. pub type HashHex<'a> = &'a str; -// Hex encodable block hash. -pub struct HexEncodableBlockHash(pub Vec); - -impl fmt::Debug for HexEncodableBlockHash { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", hex::encode(&self.0)) - } -} #[derive(Debug)] pub struct CheckpointCreated { From 07949394b876e1d722d453d7e414726ac62a6116 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 19 Jul 2024 00:37:50 +0200 Subject: [PATCH 33/40] feat: change public key --- .../vm/interpreter/src/fvm/checkpoint.rs | 2 +- fendermint/vm/interpreter/src/fvm/observe.rs | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index cb4b1b465..bf52eff94 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -258,7 +258,7 @@ where emit(CheckpointSigned { height: height.value(), hash: HexEncodableBlockHash(cp.block_hash.to_vec()), - validator: &hex::encode(validator_ctx.public_key.serialize()), + validator: validator_ctx.public_key, }); tracing::debug!(?height, "submitted checkpoint signature"); diff --git a/fendermint/vm/interpreter/src/fvm/observe.rs b/fendermint/vm/interpreter/src/fvm/observe.rs index 52ad9dd7c..687819836 100644 --- a/fendermint/vm/interpreter/src/fvm/observe.rs +++ b/fendermint/vm/interpreter/src/fvm/observe.rs @@ -11,6 +11,7 @@ use prometheus::{ Histogram, IntCounter, IntGauge, IntGaugeVec, Registry, }; +use fendermint_crypto::PublicKey; use fvm_shared::message::Message; register_metrics! { @@ -77,7 +78,7 @@ impl_traceables!( TraceLevel::Info, "Bottomup", CheckpointCreated, - CheckpointSigned<'a>, + CheckpointSigned, CheckpointFinalized ); @@ -102,16 +103,16 @@ impl Recordable for CheckpointCreated { } #[derive(Debug)] -pub struct CheckpointSigned<'a> { +pub struct CheckpointSigned { pub height: u64, pub hash: HexEncodableBlockHash, - pub validator: &'a str, + pub validator: PublicKey, } -impl Recordable for CheckpointSigned<'_> { +impl Recordable for CheckpointSigned { fn record_metrics(&self) { BOTTOMUP_CHECKPOINT_SIGNED_HEIGHT - .with_label_values(&[self.validator]) + .with_label_values(&[format!("{:?}", self.validator).as_str()]) .set(self.height as i64); } } @@ -141,9 +142,11 @@ mod tests { #[test] fn test_emit() { + use fendermint_crypto::SecretKey; use fvm_ipld_encoding::RawBytes; use fvm_shared::address::Address; use fvm_shared::econ::TokenAmount; + use rand::thread_rng; let message = Message { version: 1, @@ -173,10 +176,14 @@ mod tests { msg_count: 2, config_number: 3, }); + + let mut r = thread_rng(); + let secret_key = SecretKey::random(&mut r); + emit(CheckpointSigned { height: 1, hash: HexEncodableBlockHash(hash.clone()), - validator: "validator", + validator: secret_key.public_key(), }); } } From 8b938d4ba3ccc5344fa8e99fe8da3ebcb6a596f0 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 19 Jul 2024 16:50:10 +0200 Subject: [PATCH 34/40] feat: update main --- fendermint/app/src/app.rs | 39 ++++---------- fendermint/app/src/observe.rs | 8 --- fendermint/vm/interpreter/src/bytes.rs | 18 ++----- fendermint/vm/interpreter/src/chain.rs | 18 ++----- fendermint/vm/interpreter/src/errors.rs | 18 ------- fendermint/vm/interpreter/src/lib.rs | 18 +------ fendermint/vm/topdown/src/proxy.rs | 70 ------------------------- 7 files changed, 19 insertions(+), 170 deletions(-) delete mode 100644 fendermint/vm/interpreter/src/errors.rs diff --git a/fendermint/app/src/app.rs b/fendermint/app/src/app.rs index 897df8b97..ad0788e8a 100644 --- a/fendermint/app/src/app.rs +++ b/fendermint/app/src/app.rs @@ -626,13 +626,7 @@ where let mut mpool_received_trace = MpoolReceived::default(); let response = match result { - Err(e) => { - emit(MpoolReceivedInvalidMessage { - reason: "InvalidEncoding", - description: e.description.as_ref(), - }); - invalid_check_tx(AppError::InvalidEncoding, e.description) - } + Err(e) => invalid_check_tx(AppError::InvalidEncoding, e.description), Ok(result) => match result { Err(IllegalMessage) => invalid_check_tx(AppError::IllegalMessage, "".to_owned()), Ok(Err(InvalidSignature(d))) => invalid_check_tx(AppError::InvalidSignature, d), @@ -697,10 +691,11 @@ where let size_txs = txs.iter().map(|tx| tx.len()).sum::(); let num_txs = txs.len(); - let process_result = self + let accept = self .interpreter .process(self.chain_env.clone(), txs) - .await?; + .await + .context("failed to process proposal")?; emit(BlockProposalReceived { height: request.height.value(), @@ -720,27 +715,11 @@ where reason: None, }); - let mut proposal_evaluated = BlockProposalEvaluated { - height: request.height.value(), - hash: HexEncodableBlockHash(request.hash.into()), - size: size_txs, - tx_count: num_txs, - validator: &request.proposer_address, - accept: true, - reason: None, - }; - - let process_proposal = match process_result { - ProcessResult::Accepted => response::ProcessProposal::Accept, - ProcessResult::Rejected(reason) => { - proposal_evaluated.accept = false; - proposal_evaluated.reason = Some(reason); - response::ProcessProposal::Reject - } - }; - - emit(proposal_evaluated); - Ok(process_proposal) + if accept { + Ok(response::ProcessProposal::Accept) + } else { + Ok(response::ProcessProposal::Reject) + } } /// Signals the beginning of a new block, prior to any `DeliverTx` calls. diff --git a/fendermint/app/src/observe.rs b/fendermint/app/src/observe.rs index 792f4c9be..2521d20f5 100644 --- a/fendermint/app/src/observe.rs +++ b/fendermint/app/src/observe.rs @@ -4,10 +4,6 @@ use fvm_shared::address::Address; use fvm_shared::econ::TokenAmount; -<<<<<<< HEAD -======= -use fendermint_vm_interpreter::errors::ProcessError; ->>>>>>> e7e7cb5c (feat: address comments) use fendermint_vm_interpreter::fvm::FvmMessage; use tendermint::account::Id; @@ -158,11 +154,7 @@ impl Recordable for MpoolReceived { .unwrap_or("".to_string()); MPOOL_RECEIVED -<<<<<<< HEAD .with_label_values(&[&self.accept.to_string()]) -======= - .with_label_values(&[&self.accept.to_string(), &from]) ->>>>>>> e7e7cb5c (feat: address comments) .inc(); } } diff --git a/fendermint/vm/interpreter/src/bytes.rs b/fendermint/vm/interpreter/src/bytes.rs index af8273fb7..1af84dd85 100644 --- a/fendermint/vm/interpreter/src/bytes.rs +++ b/fendermint/vm/interpreter/src/bytes.rs @@ -9,10 +9,8 @@ use fvm_ipld_encoding::Error as IpldError; use crate::{ chain::{ChainMessageApplyRet, ChainMessageCheckRes}, - errors::ProcessError, fvm::{FvmQuery, FvmQueryRet}, - CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProcessResult, ProposalInterpreter, - QueryInterpreter, + CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProposalInterpreter, QueryInterpreter, }; pub type BytesMessageApplyRes = Result; @@ -127,19 +125,13 @@ where } /// Parse messages in the block, reject if unknown format. Pass the rest to the inner `ChainMessage` interpreter. - async fn process( - &self, - state: Self::State, - msgs: Vec, - ) -> anyhow::Result { + async fn process(&self, state: Self::State, msgs: Vec) -> anyhow::Result { if msgs.len() > self.max_msgs { tracing::warn!( block_msgs = msgs.len(), "rejecting block: too many messages" ); - return Ok(ProcessResult::Rejected(ProcessError::TooManyMessages( - msgs.len(), - ))); + return Ok(false); } let mut chain_msgs = Vec::new(); @@ -160,9 +152,7 @@ where "failed to decode message in proposal as ChainMessage" ); if self.reject_malformed_proposal { - return Ok(ProcessResult::Rejected( - ProcessError::FailedToDecodeMessage(e.to_string()), - )); + return Ok(false); } } Ok(msg) => chain_msgs.push(msg), diff --git a/fendermint/vm/interpreter/src/chain.rs b/fendermint/vm/interpreter/src/chain.rs index 00a3ccbdb..79136138f 100644 --- a/fendermint/vm/interpreter/src/chain.rs +++ b/fendermint/vm/interpreter/src/chain.rs @@ -1,14 +1,12 @@ // Copyright 2022-2024 Protocol Labs // SPDX-License-Identifier: Apache-2.0, MIT -use crate::errors::ProcessError; use crate::fvm::state::ipc::GatewayCaller; use crate::fvm::{topdown, FvmApplyRet, PowerUpdates}; use crate::{ fvm::state::FvmExecState, fvm::FvmMessage, signed::{SignedMessageApplyRes, SignedMessageCheckRes, SyntheticMessage, VerifiableMessage}, - CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProcessResult, ProposalInterpreter, - QueryInterpreter, + CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProposalInterpreter, QueryInterpreter, }; use anyhow::{bail, Context}; use async_stm::atomically; @@ -177,11 +175,7 @@ where } /// Perform finality checks on top-down transactions and availability checks on bottom-up transactions. - async fn process( - &self, - env: Self::State, - msgs: Vec, - ) -> anyhow::Result { + async fn process(&self, env: Self::State, msgs: Vec) -> anyhow::Result { for msg in msgs { match msg { ChainMessage::Ipc(IpcMessage::BottomUpExec(msg)) => { @@ -200,7 +194,7 @@ where .await; if !is_resolved { - return Ok(ProcessResult::Rejected(ProcessError::CheckpointNotResolved)); + return Ok(false); } } ChainMessage::Ipc(IpcMessage::TopDownExec(ParentFinality { @@ -214,15 +208,13 @@ where let is_final = atomically(|| env.parent_finality_provider.check_proposal(&prop)).await; if !is_final { - return Ok(ProcessResult::Rejected( - ProcessError::ParentFinalityNotAvailable, - )); + return Ok(false); } } _ => {} }; } - Ok(ProcessResult::Accepted) + Ok(true) } } diff --git a/fendermint/vm/interpreter/src/errors.rs b/fendermint/vm/interpreter/src/errors.rs deleted file mode 100644 index 8cdf28a40..000000000 --- a/fendermint/vm/interpreter/src/errors.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2022-2024 Protocol Labs -// SPDX-License-Identifier: Apache-2.0, MIT - -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum ProcessError { - #[error("checkpoint not resolved")] - CheckpointNotResolved, - #[error("parent finality not available")] - ParentFinalityNotAvailable, - #[error("too many messages: {0}")] - TooManyMessages(usize), - #[error("failed to decode message in proposal as ChainMessage: {0}")] - FailedToDecodeMessage(String), - #[error("")] - Empty, -} diff --git a/fendermint/vm/interpreter/src/lib.rs b/fendermint/vm/interpreter/src/lib.rs index 011249b02..42dab76b8 100644 --- a/fendermint/vm/interpreter/src/lib.rs +++ b/fendermint/vm/interpreter/src/lib.rs @@ -4,7 +4,6 @@ use async_trait::async_trait; pub mod bytes; pub mod chain; -pub mod errors; pub mod fvm; pub mod signed; @@ -28,17 +27,6 @@ pub trait GenesisInterpreter: Sync + Send { ) -> anyhow::Result<(Self::State, Self::Output)>; } -pub enum ProcessResult { - Accepted, - Rejected(errors::ProcessError), -} - -impl ProcessResult { - pub fn is_accepted(&self) -> bool { - matches!(self, ProcessResult::Accepted) - } -} - /// Prepare and process transaction proposals. #[async_trait] pub trait ProposalInterpreter: Sync + Send { @@ -66,11 +54,7 @@ pub trait ProposalInterpreter: Sync + Send { /// This is our chance check whether CIDs proposed for execution are available. /// /// Return `true` if we can accept this block, `false` to reject it. - async fn process( - &self, - state: Self::State, - msgs: Vec, - ) -> anyhow::Result; + async fn process(&self, state: Self::State, msgs: Vec) -> anyhow::Result; } /// The `ExecInterpreter` applies messages on some state, which is diff --git a/fendermint/vm/topdown/src/proxy.rs b/fendermint/vm/topdown/src/proxy.rs index d52c3b70d..94a8e3177 100644 --- a/fendermint/vm/topdown/src/proxy.rs +++ b/fendermint/vm/topdown/src/proxy.rs @@ -129,76 +129,6 @@ impl IPCProviderProxyWithLatency { } } -#[async_trait] -impl ParentQueryProxy for IPCProviderProxyWithLatency { - #[instrument(skip(self))] - async fn get_chain_head_height(&self) -> anyhow::Result { - emit_event_with_latency( - &self.inner.parent_subnet.to_string(), - "chain_head", - || async { self.inner.get_chain_head_height().await }, - ) - .await - } - - #[instrument(skip(self))] - async fn get_genesis_epoch(&self) -> anyhow::Result { - emit_event_with_latency( - &self.inner.parent_subnet.to_string(), - "genesis_epoch", - || async { self.inner.get_genesis_epoch().await }, - ) - .await - } - - #[instrument(skip(self))] - async fn get_block_hash(&self, height: BlockHeight) -> anyhow::Result { - emit_event_with_latency( - &self.inner.parent_subnet.to_string(), - "get_block_hash", - || async { self.inner.get_block_hash(height).await }, - ) - .await - } - - /// Get the top down messages from the starting to the ending height. - async fn get_top_down_msgs( - &self, - height: BlockHeight, - ) -> anyhow::Result>> { - emit_event_with_latency( - &self.inner.parent_subnet.to_string(), - "get_top_down_msgs", - || async { self.inner.get_top_down_msgs(height).await }, - ) - .await - } - - /// Get the validator set at the specified height. - async fn get_validator_changes( - &self, - height: BlockHeight, - ) -> anyhow::Result>> { - emit_event_with_latency( - &self.inner.parent_subnet.to_string(), - "get_validator_changeset", - || async { self.inner.get_validator_changes(height).await }, - ) - .await - } -} - -// TODO - create a macro for this -pub struct IPCProviderProxyWithLatency { - inner: IPCProviderProxy, -} - -impl IPCProviderProxyWithLatency { - pub fn new(inner: IPCProviderProxy) -> Self { - Self { inner } - } -} - #[async_trait] impl ParentQueryProxy for IPCProviderProxyWithLatency { #[instrument(skip(self))] From 7431138cf5e17765951d3bdd4a2045179118bf85 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 19 Jul 2024 16:54:52 +0200 Subject: [PATCH 35/40] fix: finish rebase --- fendermint/app/Cargo.toml | 1 - fendermint/app/src/app.rs | 3 +-- fendermint/app/src/observe.rs | 6 ------ fendermint/vm/interpreter/src/fvm/state/exec.rs | 2 -- fendermint/vm/interpreter/src/fvm/state/ipc.rs | 2 +- 5 files changed, 2 insertions(+), 12 deletions(-) diff --git a/fendermint/app/Cargo.toml b/fendermint/app/Cargo.toml index 8e884e69f..b524d15b8 100644 --- a/fendermint/app/Cargo.toml +++ b/fendermint/app/Cargo.toml @@ -65,7 +65,6 @@ fendermint_vm_snapshot = { path = "../vm/snapshot" } fendermint_vm_topdown = { path = "../vm/topdown" } - fvm = { workspace = true } fvm_ipld_blockstore = { workspace = true } fvm_ipld_car = { workspace = true } diff --git a/fendermint/app/src/app.rs b/fendermint/app/src/app.rs index ad0788e8a..e87677483 100644 --- a/fendermint/app/src/app.rs +++ b/fendermint/app/src/app.rs @@ -26,8 +26,7 @@ use fendermint_vm_interpreter::fvm::store::ReadOnlyBlockstore; use fendermint_vm_interpreter::fvm::{FvmApplyRet, FvmGenesisOutput, PowerUpdates}; use fendermint_vm_interpreter::signed::InvalidSignature; use fendermint_vm_interpreter::{ - CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProcessResult, ProposalInterpreter, - QueryInterpreter, + CheckInterpreter, ExecInterpreter, GenesisInterpreter, ProposalInterpreter, QueryInterpreter, }; use fendermint_vm_message::query::FvmQueryHeight; use fendermint_vm_snapshot::{SnapshotClient, SnapshotError}; diff --git a/fendermint/app/src/observe.rs b/fendermint/app/src/observe.rs index 2521d20f5..793abe2fa 100644 --- a/fendermint/app/src/observe.rs +++ b/fendermint/app/src/observe.rs @@ -147,12 +147,6 @@ pub struct MpoolReceived { impl Recordable for MpoolReceived { fn record_metrics(&self) { - let from = self - .message - .as_ref() - .map(|m| m.from.to_string()) - .unwrap_or("".to_string()); - MPOOL_RECEIVED .with_label_values(&[&self.accept.to_string()]) .inc(); diff --git a/fendermint/vm/interpreter/src/fvm/state/exec.rs b/fendermint/vm/interpreter/src/fvm/state/exec.rs index 34c4dc765..7d5f10dda 100644 --- a/fendermint/vm/interpreter/src/fvm/state/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/state/exec.rs @@ -176,13 +176,11 @@ where /// Execute message implicitly. pub fn execute_implicit(&mut self, msg: Message) -> ExecResult { - // TODO Karel - measure here self.execute_message(msg, ApplyKind::Implicit) } /// Execute message explicitly. pub fn execute_explicit(&mut self, msg: Message) -> ExecResult { - // TODO Karel - measure here self.execute_message(msg, ApplyKind::Explicit) } diff --git a/fendermint/vm/interpreter/src/fvm/state/ipc.rs b/fendermint/vm/interpreter/src/fvm/state/ipc.rs index 97c510f23..12caa26c6 100644 --- a/fendermint/vm/interpreter/src/fvm/state/ipc.rs +++ b/fendermint/vm/interpreter/src/fvm/state/ipc.rs @@ -315,7 +315,7 @@ impl GatewayCaller { height: u64, ) -> anyhow::Result> { let (_, _, addrs, _) = self.getter.call(state, |c| { - c.get_checkpoint_signature_bundle(et::U256::from(height)) + c.get_checkpoint_signature_bundle(ethers::types::U256::from(height)) })?; let addrs = addrs.into_iter().map(|a| a.into()).collect(); From 02f68efe71ab1ab48ac6e6e59ade846e16b9e044 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 19 Jul 2024 18:11:40 +0200 Subject: [PATCH 36/40] feat: address PR commnets --- Cargo.lock | 1 + fendermint/app/src/cmd/run.rs | 10 +++-- fendermint/app/src/metrics/mod.rs | 1 - fendermint/app/src/metrics/prometheus.rs | 41 +------------------ .../vm/interpreter/src/fvm/checkpoint.rs | 5 ++- fendermint/vm/interpreter/src/fvm/exec.rs | 12 ++++-- fendermint/vm/interpreter/src/fvm/observe.rs | 11 +++-- ipc/observability/Cargo.toml | 1 + ipc/observability/src/lib.rs | 2 + ipc/observability/src/macros.rs | 8 ++++ ipc/observability/src/observe.rs | 30 ++++++++++++++ 11 files changed, 71 insertions(+), 51 deletions(-) create mode 100644 ipc/observability/src/observe.rs diff --git a/Cargo.lock b/Cargo.lock index c55c1291f..b056f037c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5058,6 +5058,7 @@ dependencies = [ name = "ipc-observability" version = "0.1.0" dependencies = [ + "anyhow", "hex", "lazy_static", "prometheus", diff --git a/fendermint/app/src/cmd/run.rs b/fendermint/app/src/cmd/run.rs index 87c1e11cd..2dac3ef6a 100644 --- a/fendermint/app/src/cmd/run.rs +++ b/fendermint/app/src/cmd/run.rs @@ -11,6 +11,7 @@ use fendermint_crypto::SecretKey; use fendermint_rocksdb::{blockstore::NamespaceBlockstore, namespaces, RocksDb, RocksDbConfig}; use fendermint_vm_actor_interface::eam::EthAddress; use fendermint_vm_interpreter::chain::ChainEnv; +use fendermint_vm_interpreter::fvm::observe::register_metrics as register_interpreter_metrics; use fendermint_vm_interpreter::fvm::upgrades::UpgradeScheduler; use fendermint_vm_interpreter::{ bytes::{BytesMessageInterpreter, ProposalPrepareMode}, @@ -27,6 +28,7 @@ use fendermint_vm_topdown::voting::{publish_vote_loop, Error as VoteError, VoteT use fendermint_vm_topdown::{CachedFinalityProvider, IPCParentFinality, Toggle}; use fvm_shared::address::{current_network, Address, Network}; use ipc_ipld_resolver::{Event as ResolverEvent, VoteRecord}; +use ipc_observability::observe::register_metrics as register_default_metrics; use ipc_provider::config::subnet::{EVMSubnet, SubnetConfig}; use ipc_provider::IpcProvider; use libp2p::identity::secp256k1; @@ -38,6 +40,7 @@ use tracing::info; use crate::cmd::key::read_secret_key; use crate::{cmd, options::run::RunArgs, settings::Settings}; +use fendermint_app::observe::register_metrics as register_consensus_metrics; cmd! { RunArgs(self, settings) { @@ -70,10 +73,11 @@ async fn run(settings: Settings) -> anyhow::Result<()> { let metrics_registry = if settings.metrics.enabled { let registry = prometheus::Registry::new(); + register_default_metrics(®istry).context("failed to register default metrics")?; register_topdown_metrics(®istry).context("failed to register topdown metrics")?; - - fendermint_app::metrics::register_app_metrics(®istry) - .context("failed to register metrics")?; + register_interpreter_metrics(®istry) + .context("failed to register interpreter metrics")?; + register_consensus_metrics(®istry).context("failed to register consensus metrics")?; Some(registry) } else { diff --git a/fendermint/app/src/metrics/mod.rs b/fendermint/app/src/metrics/mod.rs index 34459e81c..6ba7a4af6 100644 --- a/fendermint/app/src/metrics/mod.rs +++ b/fendermint/app/src/metrics/mod.rs @@ -3,5 +3,4 @@ mod prometheus; -pub use prometheus::app::register_metrics as register_app_metrics; pub use prometheus::eth::register_metrics as register_eth_metrics; diff --git a/fendermint/app/src/metrics/prometheus.rs b/fendermint/app/src/metrics/prometheus.rs index 8be633924..9594a012c 100644 --- a/fendermint/app/src/metrics/prometheus.rs +++ b/fendermint/app/src/metrics/prometheus.rs @@ -2,42 +2,9 @@ // SPDX-License-Identifier: Apache-2.0, MIT //! Prometheus metrics -macro_rules! metrics { - ($($name:ident : $type:ty = $desc:literal);* $(;)?) => { - $( - paste! { - lazy_static! { - pub static ref $name: $type = $type::new(stringify!([< $name:lower >]), $desc).unwrap(); - } - } - )* - - pub fn register_metrics(registry: &Registry) -> anyhow::Result<()> { - $(registry.register(Box::new($name.clone()))?;)* - Ok(()) - } - }; - } - -/// Metrics emitted by endermint. -pub mod app { - use lazy_static::lazy_static; - use paste::paste; - use prometheus::{IntCounter, IntGauge, Registry}; - - metrics! { - BOTTOMUP_CKPT_BLOCK_HEIGHT: IntGauge = "Highest bottom-up checkpoint created"; - BOTTOMUP_CKPT_CONFIG_NUM: IntGauge = "Highest configuration number checkpointed"; - BOTTOMUP_CKPT_NUM_MSGS: IntCounter = "Number of bottom-up messages observed since start"; - - // This metrics is available in CometBFT as well, but it's something that should increase even without subnets, - // which can be a useful way to check if metrics work at all. - ABCI_COMMITTED_BLOCK_HEIGHT: IntGauge = "Highest committed block"; - } -} - /// Metrics emitted by the Ethereum API facade. pub mod eth { + // TODO - migrate these metrics to new observability architecture use fendermint_eth_api::apis::RPC_METHOD_CALL_LATENCY_SECONDS; pub fn register_metrics(registry: &prometheus::Registry) -> anyhow::Result<()> { @@ -48,12 +15,6 @@ pub mod eth { #[cfg(test)] mod tests { - #[test] - fn can_register_app_metrics() { - let r = prometheus::Registry::new(); - super::app::register_metrics(&r).unwrap(); - } - #[test] fn can_register_eth_metrics() { let r = prometheus::Registry::new(); diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index bf52eff94..00a07aaa5 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -24,7 +24,9 @@ use ipc_actors_abis::gateway_getter_facet as getter; use ipc_api::staking::ConfigurationNumber; use ipc_observability::{emit, serde::HexEncodableBlockHash}; -use super::observe::{CheckpointCreated, CheckpointFinalized, CheckpointSigned}; +use super::observe::{ + CheckpointCreated, CheckpointFinalized, CheckpointSigned, CheckpointSignedRole, +}; use super::state::ipc::tokens_to_burn; use super::{ broadcast::Broadcaster, @@ -256,6 +258,7 @@ where .context("failed to broadcast checkpoint signature")?; emit(CheckpointSigned { + role: CheckpointSignedRole::Own, height: height.value(), hash: HexEncodableBlockHash(cp.block_hash.to_vec()), validator: validator_ctx.public_key, diff --git a/fendermint/vm/interpreter/src/fvm/exec.rs b/fendermint/vm/interpreter/src/fvm/exec.rs index 77235229b..902f75be5 100644 --- a/fendermint/vm/interpreter/src/fvm/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/exec.rs @@ -9,14 +9,14 @@ use fendermint_vm_actor_interface::{chainmetadata, cron, system}; use fvm::executor::ApplyRet; use fvm_ipld_blockstore::Blockstore; use fvm_shared::{address::Address, ActorID, MethodNum, BLOCK_GAS_LIMIT}; -use ipc_observability::{emit, measure_time}; +use ipc_observability::{emit, measure_time, observe::TracingError, Traceable}; use tendermint_rpc::Client; use crate::ExecInterpreter; use super::{ checkpoint::{self, PowerUpdates}, - observe::{MsgExec, MsgExecPurpose}, + observe::{CheckpointFinalized, MsgExec, MsgExecPurpose}, state::FvmExecState, FvmMessage, FvmMessageInterpreter, }; @@ -186,7 +186,13 @@ where } async fn end(&self, mut state: Self::State) -> anyhow::Result<(Self::State, Self::EndOutput)> { - checkpoint::emit_trace_if_check_checkpoint_finalized(&self.gateway, &mut state)?; + let _ = checkpoint::emit_trace_if_check_checkpoint_finalized(&self.gateway, &mut state) + .inspect_err(|e| { + emit(TracingError { + affected_event: CheckpointFinalized::name(), + reason: e.to_string(), + }); + }); let updates = if let Some((checkpoint, updates)) = checkpoint::maybe_create_checkpoint(&self.gateway, &mut state) diff --git a/fendermint/vm/interpreter/src/fvm/observe.rs b/fendermint/vm/interpreter/src/fvm/observe.rs index 687819836..d0304b879 100644 --- a/fendermint/vm/interpreter/src/fvm/observe.rs +++ b/fendermint/vm/interpreter/src/fvm/observe.rs @@ -82,9 +82,6 @@ impl_traceables!( CheckpointFinalized ); -/// Hex encoded hash. -pub type HashHex<'a> = &'a str; - #[derive(Debug)] pub struct CheckpointCreated { pub height: u64, @@ -102,8 +99,15 @@ impl Recordable for CheckpointCreated { } } +#[derive(Debug)] +pub enum CheckpointSignedRole { + Own, + Peer, +} + #[derive(Debug)] pub struct CheckpointSigned { + pub role: CheckpointSignedRole, pub height: u64, pub hash: HexEncodableBlockHash, pub validator: PublicKey, @@ -181,6 +185,7 @@ mod tests { let secret_key = SecretKey::random(&mut r); emit(CheckpointSigned { + role: CheckpointSignedRole::Own, height: 1, hash: HexEncodableBlockHash(hash.clone()), validator: secret_key.public_key(), diff --git a/ipc/observability/Cargo.toml b/ipc/observability/Cargo.toml index de39eba22..67679a858 100644 --- a/ipc/observability/Cargo.toml +++ b/ipc/observability/Cargo.toml @@ -13,3 +13,4 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } tracing-appender = { workspace = true } hex = { workspace = true } +anyhow = { workspace = true } diff --git a/ipc/observability/src/lib.rs b/ipc/observability/src/lib.rs index 504c256a9..57fe44d7a 100644 --- a/ipc/observability/src/lib.rs +++ b/ipc/observability/src/lib.rs @@ -5,6 +5,7 @@ pub mod macros; pub mod traces; mod tracing_layers; pub use lazy_static::lazy_static; +pub mod observe; pub mod serde; use std::fmt::Debug; @@ -19,6 +20,7 @@ pub trait Recordable { pub trait Traceable { fn trace_level(&self) -> TraceLevel; fn domain(&self) -> &'static str; + fn name() -> &'static str; } pub enum TraceLevel { diff --git a/ipc/observability/src/macros.rs b/ipc/observability/src/macros.rs index 5f87f1d23..d0100b098 100644 --- a/ipc/observability/src/macros.rs +++ b/ipc/observability/src/macros.rs @@ -22,6 +22,10 @@ macro_rules! register_metrics { macro_rules! impl_traceable { ($struct_name:ident<$lifetime:tt>, $trace_level:expr, $domain:expr) => { impl<$lifetime> Traceable for $struct_name<$lifetime> { + fn name() -> &'static str { + stringify!($struct_name) + } + fn trace_level(&self) -> TraceLevel { $trace_level } @@ -33,6 +37,10 @@ macro_rules! impl_traceable { }; ($struct_name:ident, $trace_level:expr, $domain:expr) => { impl Traceable for $struct_name { + fn name() -> &'static str { + stringify!($struct_name) + } + fn trace_level(&self) -> TraceLevel { $trace_level } diff --git a/ipc/observability/src/observe.rs b/ipc/observability/src/observe.rs new file mode 100644 index 000000000..80c04f49e --- /dev/null +++ b/ipc/observability/src/observe.rs @@ -0,0 +1,30 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::{ + impl_traceable, impl_traceables, lazy_static, register_metrics, Recordable, TraceLevel, + Traceable, +}; +use anyhow; +use prometheus::{register_int_counter_vec, IntCounterVec, Registry}; + +register_metrics! { + TRACING_ERRORS: IntCounterVec + = register_int_counter_vec!("tracing_errors", "Number of tracing errors", &["event"]); +} + +impl_traceables!(TraceLevel::Error, "TracingError", TracingError<'a>); + +#[derive(Debug)] +pub struct TracingError<'a> { + pub affected_event: &'a str, + pub reason: String, +} + +impl Recordable for TracingError<'_> { + fn record_metrics(&self) { + TRACING_ERRORS + .with_label_values(&[self.affected_event]) + .inc(); + } +} From 7803cc2c64fb8a71342da8359d94462950405048 Mon Sep 17 00:00:00 2001 From: Karel Moravec Date: Fri, 19 Jul 2024 20:01:46 +0200 Subject: [PATCH 37/40] feat: rename event --- fendermint/vm/interpreter/src/fvm/checkpoint.rs | 4 ++-- fendermint/vm/interpreter/src/fvm/exec.rs | 4 ++-- fendermint/vm/interpreter/src/fvm/observe.rs | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index 00a07aaa5..447c31f9c 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -25,7 +25,7 @@ use ipc_api::staking::ConfigurationNumber; use ipc_observability::{emit, serde::HexEncodableBlockHash}; use super::observe::{ - CheckpointCreated, CheckpointFinalized, CheckpointSigned, CheckpointSignedRole, + CheckpointCommitted, CheckpointCreated, CheckpointSigned, CheckpointSignedRole, }; use super::state::ipc::tokens_to_burn; use super::{ @@ -320,7 +320,7 @@ where let checkpoint_quorum = gateway.checkpoint_info(state, block_height)?; if checkpoint_quorum.reached { - emit(CheckpointFinalized { + emit(CheckpointCommitted { height: block_height, hash: HexEncodableBlockHash(block_hash.to_vec()), }) diff --git a/fendermint/vm/interpreter/src/fvm/exec.rs b/fendermint/vm/interpreter/src/fvm/exec.rs index 902f75be5..5fd432e70 100644 --- a/fendermint/vm/interpreter/src/fvm/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/exec.rs @@ -16,7 +16,7 @@ use crate::ExecInterpreter; use super::{ checkpoint::{self, PowerUpdates}, - observe::{CheckpointFinalized, MsgExec, MsgExecPurpose}, + observe::{CheckpointCommitted, MsgExec, MsgExecPurpose}, state::FvmExecState, FvmMessage, FvmMessageInterpreter, }; @@ -189,7 +189,7 @@ where let _ = checkpoint::emit_trace_if_check_checkpoint_finalized(&self.gateway, &mut state) .inspect_err(|e| { emit(TracingError { - affected_event: CheckpointFinalized::name(), + affected_event: CheckpointCommitted::name(), reason: e.to_string(), }); }); diff --git a/fendermint/vm/interpreter/src/fvm/observe.rs b/fendermint/vm/interpreter/src/fvm/observe.rs index d0304b879..131324522 100644 --- a/fendermint/vm/interpreter/src/fvm/observe.rs +++ b/fendermint/vm/interpreter/src/fvm/observe.rs @@ -79,7 +79,7 @@ impl_traceables!( "Bottomup", CheckpointCreated, CheckpointSigned, - CheckpointFinalized + CheckpointCommitted ); #[derive(Debug)] @@ -122,12 +122,12 @@ impl Recordable for CheckpointSigned { } #[derive(Debug)] -pub struct CheckpointFinalized { +pub struct CheckpointCommitted { pub height: i64, pub hash: HexEncodableBlockHash, } -impl Recordable for CheckpointFinalized { +impl Recordable for CheckpointCommitted { fn record_metrics(&self) { BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT.set(self.height); } From 27b1a672c75ed87172980b4348b4568c34fccc7d Mon Sep 17 00:00:00 2001 From: raulk Date: Mon, 22 Jul 2024 11:11:55 +0100 Subject: [PATCH 38/40] rename events: CheckpointFinalized + CheckpointSubmitted. --- fendermint/vm/interpreter/src/fvm/checkpoint.rs | 4 ++-- fendermint/vm/interpreter/src/fvm/exec.rs | 4 ++-- fendermint/vm/interpreter/src/fvm/observe.rs | 6 +++--- ipc/provider/src/checkpoint.rs | 4 ++-- ipc/provider/src/observe.rs | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index 447c31f9c..d5133d02c 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -25,7 +25,7 @@ use ipc_api::staking::ConfigurationNumber; use ipc_observability::{emit, serde::HexEncodableBlockHash}; use super::observe::{ - CheckpointCommitted, CheckpointCreated, CheckpointSigned, CheckpointSignedRole, + CheckpointFinalized, CheckpointCreated, CheckpointSigned, CheckpointSignedRole, }; use super::state::ipc::tokens_to_burn; use super::{ @@ -320,7 +320,7 @@ where let checkpoint_quorum = gateway.checkpoint_info(state, block_height)?; if checkpoint_quorum.reached { - emit(CheckpointCommitted { + emit(CheckpointFinalized { height: block_height, hash: HexEncodableBlockHash(block_hash.to_vec()), }) diff --git a/fendermint/vm/interpreter/src/fvm/exec.rs b/fendermint/vm/interpreter/src/fvm/exec.rs index 5fd432e70..902f75be5 100644 --- a/fendermint/vm/interpreter/src/fvm/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/exec.rs @@ -16,7 +16,7 @@ use crate::ExecInterpreter; use super::{ checkpoint::{self, PowerUpdates}, - observe::{CheckpointCommitted, MsgExec, MsgExecPurpose}, + observe::{CheckpointFinalized, MsgExec, MsgExecPurpose}, state::FvmExecState, FvmMessage, FvmMessageInterpreter, }; @@ -189,7 +189,7 @@ where let _ = checkpoint::emit_trace_if_check_checkpoint_finalized(&self.gateway, &mut state) .inspect_err(|e| { emit(TracingError { - affected_event: CheckpointCommitted::name(), + affected_event: CheckpointFinalized::name(), reason: e.to_string(), }); }); diff --git a/fendermint/vm/interpreter/src/fvm/observe.rs b/fendermint/vm/interpreter/src/fvm/observe.rs index 131324522..d0304b879 100644 --- a/fendermint/vm/interpreter/src/fvm/observe.rs +++ b/fendermint/vm/interpreter/src/fvm/observe.rs @@ -79,7 +79,7 @@ impl_traceables!( "Bottomup", CheckpointCreated, CheckpointSigned, - CheckpointCommitted + CheckpointFinalized ); #[derive(Debug)] @@ -122,12 +122,12 @@ impl Recordable for CheckpointSigned { } #[derive(Debug)] -pub struct CheckpointCommitted { +pub struct CheckpointFinalized { pub height: i64, pub hash: HexEncodableBlockHash, } -impl Recordable for CheckpointCommitted { +impl Recordable for CheckpointFinalized { fn record_metrics(&self) { BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT.set(self.height); } diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index 81edd5234..9a7ab54dd 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -4,7 +4,7 @@ use crate::config::Subnet; use crate::manager::{BottomUpCheckpointRelayer, EthSubnetManager}; -use crate::observe::CheckpointFinalized; +use crate::observe::CheckpointSubmitted; use anyhow::{anyhow, Result}; use futures_util::future::try_join_all; use fvm_shared::address::Address; @@ -208,7 +208,7 @@ impl BottomUpCheckpointMan Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event) .await .inspect(|_| { - emit(CheckpointFinalized { + emit(CheckpointSubmitted { height, hash: HexEncodableBlockHash(hash), }); diff --git a/ipc/provider/src/observe.rs b/ipc/provider/src/observe.rs index 604c8c954..e134ef277 100644 --- a/ipc/provider/src/observe.rs +++ b/ipc/provider/src/observe.rs @@ -12,15 +12,15 @@ register_metrics! { = register_int_gauge!("bottomup_checkpoint_finalized_height", "Height of the checkpoint finalized"); } -impl_traceables!(TraceLevel::Info, "Bottomup", CheckpointFinalized); +impl_traceables!(TraceLevel::Info, "Bottomup", CheckpointSubmitted); #[derive(Debug)] -pub struct CheckpointFinalized { +pub struct CheckpointSubmitted { pub height: i64, pub hash: HexEncodableBlockHash, } -impl Recordable for CheckpointFinalized { +impl Recordable for CheckpointSubmitted { fn record_metrics(&self) { BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT.set(self.height); } @@ -41,7 +41,7 @@ mod tests { fn test_emit() { let hash = vec![0x01, 0x02, 0x03]; - emit(CheckpointFinalized { + emit(CheckpointSubmitted { height: 1, hash: HexEncodableBlockHash(hash.clone()), }); From e66910b35c048e98d981ba201ad1c07ba1724ebe Mon Sep 17 00:00:00 2001 From: raulk Date: Mon, 22 Jul 2024 11:16:16 +0100 Subject: [PATCH 39/40] add comments. --- fendermint/vm/interpreter/src/fvm/checkpoint.rs | 2 ++ fendermint/vm/interpreter/src/fvm/exec.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index d5133d02c..74f55910a 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -317,6 +317,8 @@ where .ok_or_else(|| anyhow!("block hash not set"))?; // Check if the checkpoint has been finalized. + // If no checkpoint was emitted at this height, the QuorumInfo struct will carry zero values, + // including reached=false. let checkpoint_quorum = gateway.checkpoint_info(state, block_height)?; if checkpoint_quorum.reached { diff --git a/fendermint/vm/interpreter/src/fvm/exec.rs b/fendermint/vm/interpreter/src/fvm/exec.rs index 902f75be5..f53e630a9 100644 --- a/fendermint/vm/interpreter/src/fvm/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/exec.rs @@ -186,6 +186,7 @@ where } async fn end(&self, mut state: Self::State) -> anyhow::Result<(Self::State, Self::EndOutput)> { + // TODO: Consider doing this async, since it's purely informational and not consensus-critical. let _ = checkpoint::emit_trace_if_check_checkpoint_finalized(&self.gateway, &mut state) .inspect_err(|e| { emit(TracingError { From 43df0a955bcc0016f82f619576bbfeb71dd3ec5d Mon Sep 17 00:00:00 2001 From: raulk Date: Mon, 22 Jul 2024 11:20:11 +0100 Subject: [PATCH 40/40] cargo fmt. --- fendermint/vm/interpreter/src/fvm/checkpoint.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index 74f55910a..ab9b2c0d5 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -25,7 +25,7 @@ use ipc_api::staking::ConfigurationNumber; use ipc_observability::{emit, serde::HexEncodableBlockHash}; use super::observe::{ - CheckpointFinalized, CheckpointCreated, CheckpointSigned, CheckpointSignedRole, + CheckpointCreated, CheckpointFinalized, CheckpointSigned, CheckpointSignedRole, }; use super::state::ipc::tokens_to_burn; use super::{