diff --git a/Cargo.lock b/Cargo.lock index 07f591bd0..b056f037c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5037,6 +5037,8 @@ dependencies = [ "num-derive 0.3.3", "num-traits", "openssl", + "prometheus", + "prometheus_exporter", "reqwest", "serde", "serde_bytes", @@ -5056,6 +5058,7 @@ dependencies = [ name = "ipc-observability" version = "0.1.0" dependencies = [ + "anyhow", "hex", "lazy_static", "prometheus", @@ -5084,6 +5087,7 @@ dependencies = [ "hex", "indoc", "ipc-api", + "ipc-observability", "ipc-types", "ipc-wallet", "ipc_actors_abis", @@ -5091,6 +5095,7 @@ dependencies = [ "log", "num-derive 0.3.3", "num-traits", + "prometheus", "reqwest", "serde", "serde_bytes", diff --git a/fendermint/app/src/cmd/run.rs b/fendermint/app/src/cmd/run.rs index 87c1e11cd..2dac3ef6a 100644 --- a/fendermint/app/src/cmd/run.rs +++ b/fendermint/app/src/cmd/run.rs @@ -11,6 +11,7 @@ use fendermint_crypto::SecretKey; use fendermint_rocksdb::{blockstore::NamespaceBlockstore, namespaces, RocksDb, RocksDbConfig}; use fendermint_vm_actor_interface::eam::EthAddress; use fendermint_vm_interpreter::chain::ChainEnv; +use fendermint_vm_interpreter::fvm::observe::register_metrics as register_interpreter_metrics; use fendermint_vm_interpreter::fvm::upgrades::UpgradeScheduler; use fendermint_vm_interpreter::{ bytes::{BytesMessageInterpreter, ProposalPrepareMode}, @@ -27,6 +28,7 @@ use fendermint_vm_topdown::voting::{publish_vote_loop, Error as VoteError, VoteT use fendermint_vm_topdown::{CachedFinalityProvider, IPCParentFinality, Toggle}; use fvm_shared::address::{current_network, Address, Network}; use ipc_ipld_resolver::{Event as ResolverEvent, VoteRecord}; +use ipc_observability::observe::register_metrics as register_default_metrics; use ipc_provider::config::subnet::{EVMSubnet, SubnetConfig}; use ipc_provider::IpcProvider; use libp2p::identity::secp256k1; @@ -38,6 +40,7 @@ use tracing::info; use crate::cmd::key::read_secret_key; use crate::{cmd, options::run::RunArgs, settings::Settings}; +use fendermint_app::observe::register_metrics as register_consensus_metrics; cmd! { RunArgs(self, settings) { @@ -70,10 +73,11 @@ async fn run(settings: Settings) -> anyhow::Result<()> { let metrics_registry = if settings.metrics.enabled { let registry = prometheus::Registry::new(); + register_default_metrics(®istry).context("failed to register default metrics")?; register_topdown_metrics(®istry).context("failed to register topdown metrics")?; - - fendermint_app::metrics::register_app_metrics(®istry) - .context("failed to register metrics")?; + register_interpreter_metrics(®istry) + .context("failed to register interpreter metrics")?; + register_consensus_metrics(®istry).context("failed to register consensus metrics")?; Some(registry) } else { diff --git a/fendermint/app/src/metrics/mod.rs b/fendermint/app/src/metrics/mod.rs index 34459e81c..6ba7a4af6 100644 --- a/fendermint/app/src/metrics/mod.rs +++ b/fendermint/app/src/metrics/mod.rs @@ -3,5 +3,4 @@ mod prometheus; -pub use prometheus::app::register_metrics as register_app_metrics; pub use prometheus::eth::register_metrics as register_eth_metrics; diff --git a/fendermint/app/src/metrics/prometheus.rs b/fendermint/app/src/metrics/prometheus.rs index 8be633924..9594a012c 100644 --- a/fendermint/app/src/metrics/prometheus.rs +++ b/fendermint/app/src/metrics/prometheus.rs @@ -2,42 +2,9 @@ // SPDX-License-Identifier: Apache-2.0, MIT //! Prometheus metrics -macro_rules! metrics { - ($($name:ident : $type:ty = $desc:literal);* $(;)?) => { - $( - paste! { - lazy_static! { - pub static ref $name: $type = $type::new(stringify!([< $name:lower >]), $desc).unwrap(); - } - } - )* - - pub fn register_metrics(registry: &Registry) -> anyhow::Result<()> { - $(registry.register(Box::new($name.clone()))?;)* - Ok(()) - } - }; - } - -/// Metrics emitted by endermint. -pub mod app { - use lazy_static::lazy_static; - use paste::paste; - use prometheus::{IntCounter, IntGauge, Registry}; - - metrics! { - BOTTOMUP_CKPT_BLOCK_HEIGHT: IntGauge = "Highest bottom-up checkpoint created"; - BOTTOMUP_CKPT_CONFIG_NUM: IntGauge = "Highest configuration number checkpointed"; - BOTTOMUP_CKPT_NUM_MSGS: IntCounter = "Number of bottom-up messages observed since start"; - - // This metrics is available in CometBFT as well, but it's something that should increase even without subnets, - // which can be a useful way to check if metrics work at all. - ABCI_COMMITTED_BLOCK_HEIGHT: IntGauge = "Highest committed block"; - } -} - /// Metrics emitted by the Ethereum API facade. pub mod eth { + // TODO - migrate these metrics to new observability architecture use fendermint_eth_api::apis::RPC_METHOD_CALL_LATENCY_SECONDS; pub fn register_metrics(registry: &prometheus::Registry) -> anyhow::Result<()> { @@ -48,12 +15,6 @@ pub mod eth { #[cfg(test)] mod tests { - #[test] - fn can_register_app_metrics() { - let r = prometheus::Registry::new(); - super::app::register_metrics(&r).unwrap(); - } - #[test] fn can_register_eth_metrics() { let r = prometheus::Registry::new(); diff --git a/fendermint/vm/interpreter/src/fvm/checkpoint.rs b/fendermint/vm/interpreter/src/fvm/checkpoint.rs index 8efd04cb7..ab9b2c0d5 100644 --- a/fendermint/vm/interpreter/src/fvm/checkpoint.rs +++ b/fendermint/vm/interpreter/src/fvm/checkpoint.rs @@ -15,16 +15,18 @@ use fvm_shared::{address::Address, chainid::ChainID}; use fendermint_crypto::PublicKey; use fendermint_crypto::SecretKey; -use fendermint_tracing::emit; use fendermint_vm_actor_interface::eam::EthAddress; use fendermint_vm_actor_interface::ipc::BottomUpCheckpoint; -use fendermint_vm_event::NewBottomUpCheckpoint; use fendermint_vm_genesis::{Power, Validator, ValidatorKey}; use ipc_actors_abis::checkpointing_facet as checkpoint; use ipc_actors_abis::gateway_getter_facet as getter; use ipc_api::staking::ConfigurationNumber; +use ipc_observability::{emit, serde::HexEncodableBlockHash}; +use super::observe::{ + CheckpointCreated, CheckpointFinalized, CheckpointSigned, CheckpointSignedRole, +}; use super::state::ipc::tokens_to_burn; use super::{ broadcast::Broadcaster, @@ -121,11 +123,11 @@ where power_diff(curr_power_table, next_power_table) }; - emit!(NewBottomUpCheckpoint { - block_height: height.value(), - block_hash: &hex::encode(block_hash), - num_msgs, - next_configuration_number, + emit(CheckpointCreated { + height: height.value(), + hash: HexEncodableBlockHash(block_hash.to_vec()), + msg_count: num_msgs, + config_number: next_configuration_number, }); Ok(Some((checkpoint, power_updates))) @@ -255,6 +257,13 @@ where .await .context("failed to broadcast checkpoint signature")?; + emit(CheckpointSigned { + role: CheckpointSignedRole::Own, + height: height.value(), + hash: HexEncodableBlockHash(cp.block_hash.to_vec()), + validator: validator_ctx.public_key, + }); + tracing::debug!(?height, "submitted checkpoint signature"); } } @@ -290,6 +299,38 @@ where Ok(()) } +// Emit a CheckpointFinalized trace event if a checkpoint has been finalized on the current block. +pub fn emit_trace_if_check_checkpoint_finalized( + gateway: &GatewayCaller, + state: &mut FvmExecState, +) -> anyhow::Result<()> +where + DB: Blockstore + Clone, +{ + if !gateway.enabled(state)? { + return Ok(()); + } + + let block_height = state.block_height(); + let block_hash = state + .block_hash() + .ok_or_else(|| anyhow!("block hash not set"))?; + + // Check if the checkpoint has been finalized. + // If no checkpoint was emitted at this height, the QuorumInfo struct will carry zero values, + // including reached=false. + let checkpoint_quorum = gateway.checkpoint_info(state, block_height)?; + + if checkpoint_quorum.reached { + emit(CheckpointFinalized { + height: block_height, + hash: HexEncodableBlockHash(block_hash.to_vec()), + }) + } + + Ok(()) +} + fn convert_tokenizables( tokenizables: Vec, ) -> anyhow::Result> { diff --git a/fendermint/vm/interpreter/src/fvm/exec.rs b/fendermint/vm/interpreter/src/fvm/exec.rs index 150cb218a..f53e630a9 100644 --- a/fendermint/vm/interpreter/src/fvm/exec.rs +++ b/fendermint/vm/interpreter/src/fvm/exec.rs @@ -9,14 +9,14 @@ use fendermint_vm_actor_interface::{chainmetadata, cron, system}; use fvm::executor::ApplyRet; use fvm_ipld_blockstore::Blockstore; use fvm_shared::{address::Address, ActorID, MethodNum, BLOCK_GAS_LIMIT}; -use ipc_observability::{emit, measure_time}; +use ipc_observability::{emit, measure_time, observe::TracingError, Traceable}; use tendermint_rpc::Client; use crate::ExecInterpreter; use super::{ checkpoint::{self, PowerUpdates}, - observe::{MsgExec, MsgExecPurpose}, + observe::{CheckpointFinalized, MsgExec, MsgExecPurpose}, state::FvmExecState, FvmMessage, FvmMessageInterpreter, }; @@ -186,6 +186,15 @@ where } async fn end(&self, mut state: Self::State) -> anyhow::Result<(Self::State, Self::EndOutput)> { + // TODO: Consider doing this async, since it's purely informational and not consensus-critical. + let _ = checkpoint::emit_trace_if_check_checkpoint_finalized(&self.gateway, &mut state) + .inspect_err(|e| { + emit(TracingError { + affected_event: CheckpointFinalized::name(), + reason: e.to_string(), + }); + }); + let updates = if let Some((checkpoint, updates)) = checkpoint::maybe_create_checkpoint(&self.gateway, &mut state) .context("failed to create checkpoint")? diff --git a/fendermint/vm/interpreter/src/fvm/mod.rs b/fendermint/vm/interpreter/src/fvm/mod.rs index 83cb934d9..0aefb4b2e 100644 --- a/fendermint/vm/interpreter/src/fvm/mod.rs +++ b/fendermint/vm/interpreter/src/fvm/mod.rs @@ -8,7 +8,7 @@ mod checkpoint; mod exec; mod externs; mod genesis; -mod observe; +pub mod observe; mod query; pub mod state; pub mod store; diff --git a/fendermint/vm/interpreter/src/fvm/observe.rs b/fendermint/vm/interpreter/src/fvm/observe.rs index 02f1b86d0..d0304b879 100644 --- a/fendermint/vm/interpreter/src/fvm/observe.rs +++ b/fendermint/vm/interpreter/src/fvm/observe.rs @@ -2,11 +2,16 @@ // SPDX-License-Identifier: Apache-2.0, MIT use ipc_observability::{ - impl_traceable, impl_traceables, lazy_static, register_metrics, Recordable, TraceLevel, - Traceable, + impl_traceable, impl_traceables, lazy_static, register_metrics, serde::HexEncodableBlockHash, + Recordable, TraceLevel, Traceable, }; -use prometheus::{register_histogram, Histogram, Registry}; +use prometheus::{ + register_histogram, register_int_counter, register_int_gauge, register_int_gauge_vec, + Histogram, IntCounter, IntGauge, IntGaugeVec, Registry, +}; + +use fendermint_crypto::PublicKey; use fvm_shared::message::Message; register_metrics! { @@ -18,6 +23,21 @@ register_metrics! { = register_histogram!("exec_fvm_apply_execution_time_secs", "Execution time of FVM apply in seconds"); EXEC_FVM_CALL_EXECUTION_TIME_SECS: Histogram = register_histogram!("exec_fvm_call_execution_time_secs", "Execution time of FVM call in seconds"); + BOTTOMUP_CHECKPOINT_CREATED_TOTAL: IntCounter + = register_int_counter!("bottomup_checkpoint_created_total", "Bottom-up checkpoint produced"); + BOTTOMUP_CHECKPOINT_CREATED_HEIGHT: IntGauge + = register_int_gauge!("bottomup_checkpoint_created_height", "Height of the checkpoint created"); + BOTTOMUP_CHECKPOINT_CREATED_MSGCOUNT: IntGauge + = register_int_gauge!("bottomup_checkpoint_created_msgcount", "Number of messages in the checkpoint created"); + BOTTOMUP_CHECKPOINT_CREATED_CONFIGNUM: IntGauge + = register_int_gauge!("bottomup_checkpoint_created_confignum", "Configuration number of the checkpoint created"); + BOTTOMUP_CHECKPOINT_SIGNED_HEIGHT: IntGaugeVec = register_int_gauge_vec!( + "bottomup_checkpoint_signed_height", + "Height of the checkpoint signed", + &["validator"] + ); + BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT: IntGauge + = register_int_gauge!("bottomup_checkpoint_finalized_height", "Height of the checkpoint finalized"); } impl_traceables!(TraceLevel::Info, "Execution", MsgExec); @@ -54,6 +74,65 @@ impl Recordable for MsgExec { } } +impl_traceables!( + TraceLevel::Info, + "Bottomup", + CheckpointCreated, + CheckpointSigned, + CheckpointFinalized +); + +#[derive(Debug)] +pub struct CheckpointCreated { + pub height: u64, + pub hash: HexEncodableBlockHash, + pub msg_count: usize, + pub config_number: u64, +} + +impl Recordable for CheckpointCreated { + fn record_metrics(&self) { + BOTTOMUP_CHECKPOINT_CREATED_TOTAL.inc(); + BOTTOMUP_CHECKPOINT_CREATED_HEIGHT.set(self.height as i64); + BOTTOMUP_CHECKPOINT_CREATED_MSGCOUNT.set(self.msg_count as i64); + BOTTOMUP_CHECKPOINT_CREATED_CONFIGNUM.set(self.config_number as i64); + } +} + +#[derive(Debug)] +pub enum CheckpointSignedRole { + Own, + Peer, +} + +#[derive(Debug)] +pub struct CheckpointSigned { + pub role: CheckpointSignedRole, + pub height: u64, + pub hash: HexEncodableBlockHash, + pub validator: PublicKey, +} + +impl Recordable for CheckpointSigned { + fn record_metrics(&self) { + BOTTOMUP_CHECKPOINT_SIGNED_HEIGHT + .with_label_values(&[format!("{:?}", self.validator).as_str()]) + .set(self.height as i64); + } +} + +#[derive(Debug)] +pub struct CheckpointFinalized { + pub height: i64, + pub hash: HexEncodableBlockHash, +} + +impl Recordable for CheckpointFinalized { + fn record_metrics(&self) { + BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT.set(self.height); + } +} + #[cfg(test)] mod tests { use super::*; @@ -67,9 +146,11 @@ mod tests { #[test] fn test_emit() { + use fendermint_crypto::SecretKey; use fvm_ipld_encoding::RawBytes; use fvm_shared::address::Address; use fvm_shared::econ::TokenAmount; + use rand::thread_rng; let message = Message { version: 1, @@ -91,5 +172,23 @@ mod tests { exit_code: 1, message: message.clone(), }); + let hash = vec![0x01, 0x02, 0x03]; + + emit(CheckpointCreated { + height: 1, + hash: HexEncodableBlockHash(hash.clone()), + msg_count: 2, + config_number: 3, + }); + + let mut r = thread_rng(); + let secret_key = SecretKey::random(&mut r); + + emit(CheckpointSigned { + role: CheckpointSignedRole::Own, + height: 1, + hash: HexEncodableBlockHash(hash.clone()), + validator: secret_key.public_key(), + }); } } diff --git a/fendermint/vm/interpreter/src/fvm/state/ipc.rs b/fendermint/vm/interpreter/src/fvm/state/ipc.rs index 4cd74d5fc..12caa26c6 100644 --- a/fendermint/vm/interpreter/src/fvm/state/ipc.rs +++ b/fendermint/vm/interpreter/src/fvm/state/ipc.rs @@ -146,6 +146,17 @@ impl GatewayCaller { self.getter.call(state, |c| c.get_incomplete_checkpoints()) } + /// Retrieve checkpoint info by block height. + pub fn checkpoint_info( + &self, + state: &mut FvmExecState, + height: i64, + ) -> anyhow::Result { + self.getter.call(state, |c| { + c.get_checkpoint_info(ethers::types::U256::from(height)) + }) + } + /// Apply all pending validator changes, returning the newly adopted configuration number, or 0 if there were no changes. pub fn apply_validator_changes(&self, state: &mut FvmExecState) -> anyhow::Result { self.topdown.call(state, |c| c.apply_finality_changes()) diff --git a/ipc/cli/Cargo.toml b/ipc/cli/Cargo.toml index 6dc4e71d5..493a0d420 100644 --- a/ipc/cli/Cargo.toml +++ b/ipc/cli/Cargo.toml @@ -41,6 +41,8 @@ tokio-tungstenite = { workspace = true } toml = "0.7.2" url = { workspace = true } zeroize = "1.6.0" +prometheus = { workspace = true } +prometheus_exporter = { workspace = true } ipc-wallet = { workspace = true } ipc-provider = { workspace = true } diff --git a/ipc/cli/src/commands/checkpoint/relayer.rs b/ipc/cli/src/commands/checkpoint/relayer.rs index 43d927b40..12a121880 100644 --- a/ipc/cli/src/commands/checkpoint/relayer.rs +++ b/ipc/cli/src/commands/checkpoint/relayer.rs @@ -4,6 +4,7 @@ use crate::commands::get_subnet_config; use crate::{require_fil_addr_from_str, CommandLineHandler, GlobalArguments}; use anyhow::anyhow; +use anyhow::Context; use async_trait::async_trait; use clap::Args; use fvm_shared::address::Address; @@ -12,7 +13,9 @@ use ipc_api::subnet_id::SubnetID; use ipc_provider::checkpoint::BottomUpCheckpointManager; use ipc_provider::config::Config; use ipc_provider::new_evm_keystore_from_config; +use ipc_provider::observe::register_metrics as register_checkpoint_metrics; use ipc_wallet::EvmKeyStore; +use std::net::SocketAddr; use std::str::FromStr; use std::sync::{Arc, RwLock}; use std::time::Duration; @@ -29,6 +32,28 @@ impl CommandLineHandler for BottomUpRelayer { async fn handle(global: &GlobalArguments, arguments: &Self::Arguments) -> anyhow::Result<()> { log::debug!("start bottom up relayer with args: {:?}", arguments); + // Prometheus metrics + match &arguments.metrics_address { + Some(addr) => { + use prometheus; + use prometheus_exporter; + + let addr = SocketAddr::from_str(addr)?; + + let registry = prometheus::Registry::new(); + register_checkpoint_metrics(®istry)?; + + let mut builder = prometheus_exporter::Builder::new(addr); + builder.with_registry(registry); + let _ = builder.start().context("failed to start metrics server")?; + + log::info!("serving metrics on: {addr}"); + } + None => { + log::info!("metrics disabled"); + } + } + let config_path = global.config_path(); let config = Arc::new(Config::from_file(&config_path)?); let mut keystore = new_evm_keystore_from_config(config)?; @@ -95,4 +120,10 @@ pub(crate) struct BottomUpRelayerArgs { help = "The max parallelism for submitting checkpoints" )] pub max_parallelism: usize, + + #[arg( + long, + help = "Metrics address to listen on. Enables Prometheus metrics if set" + )] + pub metrics_address: Option, } diff --git a/ipc/observability/Cargo.toml b/ipc/observability/Cargo.toml index de39eba22..67679a858 100644 --- a/ipc/observability/Cargo.toml +++ b/ipc/observability/Cargo.toml @@ -13,3 +13,4 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } tracing-appender = { workspace = true } hex = { workspace = true } +anyhow = { workspace = true } diff --git a/ipc/observability/src/lib.rs b/ipc/observability/src/lib.rs index 504c256a9..57fe44d7a 100644 --- a/ipc/observability/src/lib.rs +++ b/ipc/observability/src/lib.rs @@ -5,6 +5,7 @@ pub mod macros; pub mod traces; mod tracing_layers; pub use lazy_static::lazy_static; +pub mod observe; pub mod serde; use std::fmt::Debug; @@ -19,6 +20,7 @@ pub trait Recordable { pub trait Traceable { fn trace_level(&self) -> TraceLevel; fn domain(&self) -> &'static str; + fn name() -> &'static str; } pub enum TraceLevel { diff --git a/ipc/observability/src/macros.rs b/ipc/observability/src/macros.rs index 5f87f1d23..d0100b098 100644 --- a/ipc/observability/src/macros.rs +++ b/ipc/observability/src/macros.rs @@ -22,6 +22,10 @@ macro_rules! register_metrics { macro_rules! impl_traceable { ($struct_name:ident<$lifetime:tt>, $trace_level:expr, $domain:expr) => { impl<$lifetime> Traceable for $struct_name<$lifetime> { + fn name() -> &'static str { + stringify!($struct_name) + } + fn trace_level(&self) -> TraceLevel { $trace_level } @@ -33,6 +37,10 @@ macro_rules! impl_traceable { }; ($struct_name:ident, $trace_level:expr, $domain:expr) => { impl Traceable for $struct_name { + fn name() -> &'static str { + stringify!($struct_name) + } + fn trace_level(&self) -> TraceLevel { $trace_level } diff --git a/ipc/observability/src/observe.rs b/ipc/observability/src/observe.rs new file mode 100644 index 000000000..80c04f49e --- /dev/null +++ b/ipc/observability/src/observe.rs @@ -0,0 +1,30 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use crate::{ + impl_traceable, impl_traceables, lazy_static, register_metrics, Recordable, TraceLevel, + Traceable, +}; +use anyhow; +use prometheus::{register_int_counter_vec, IntCounterVec, Registry}; + +register_metrics! { + TRACING_ERRORS: IntCounterVec + = register_int_counter_vec!("tracing_errors", "Number of tracing errors", &["event"]); +} + +impl_traceables!(TraceLevel::Error, "TracingError", TracingError<'a>); + +#[derive(Debug)] +pub struct TracingError<'a> { + pub affected_event: &'a str, + pub reason: String, +} + +impl Recordable for TracingError<'_> { + fn record_metrics(&self) { + TRACING_ERRORS + .with_label_values(&[self.affected_event]) + .inc(); + } +} diff --git a/ipc/provider/Cargo.toml b/ipc/provider/Cargo.toml index cacfe9aeb..0d56f7f9f 100644 --- a/ipc/provider/Cargo.toml +++ b/ipc/provider/Cargo.toml @@ -47,6 +47,8 @@ ipc-types = { workspace = true } ipc-wallet = { workspace = true, features = ["with-ethers"] } ipc-api = { workspace = true } ipc_actors_abis = { workspace = true } +ipc-observability = { workspace = true } +prometheus = { workspace = true } [dev-dependencies] tempfile = { workspace = true } diff --git a/ipc/provider/src/checkpoint.rs b/ipc/provider/src/checkpoint.rs index 0cb77efe1..9a7ab54dd 100644 --- a/ipc/provider/src/checkpoint.rs +++ b/ipc/provider/src/checkpoint.rs @@ -4,11 +4,13 @@ use crate::config::Subnet; use crate::manager::{BottomUpCheckpointRelayer, EthSubnetManager}; +use crate::observe::CheckpointSubmitted; use anyhow::{anyhow, Result}; use futures_util::future::try_join_all; use fvm_shared::address::Address; use fvm_shared::clock::ChainEpoch; use ipc_api::checkpoint::{BottomUpCheckpointBundle, QuorumReachedEvent}; +use ipc_observability::{emit, serde::HexEncodableBlockHash}; use ipc_wallet::{EthKeyAddress, PersistentKeyStore}; use std::cmp::max; use std::fmt::{Display, Formatter}; @@ -200,14 +202,23 @@ impl BottomUpCheckpointMan .unwrap(); all_submit_tasks.push(tokio::task::spawn(async move { let height = event.height; + let hash = bundle.checkpoint.block_hash.clone(); + let result = Self::submit_checkpoint(parent_handler_clone, submitter, bundle, event) .await + .inspect(|_| { + emit(CheckpointSubmitted { + height, + hash: HexEncodableBlockHash(hash), + }); + }) .inspect_err(|err| { tracing::error!( "Fail to submit checkpoint at height {height}: {err}" ); }); + drop(submission_permit); result })); diff --git a/ipc/provider/src/lib.rs b/ipc/provider/src/lib.rs index 4cf58c1ee..c9cf0efc1 100644 --- a/ipc/provider/src/lib.rs +++ b/ipc/provider/src/lib.rs @@ -38,6 +38,7 @@ pub mod config; pub mod jsonrpc; pub mod lotus; pub mod manager; +pub mod observe; const DEFAULT_REPO_PATH: &str = ".ipc"; const DEFAULT_CONFIG_NAME: &str = "config.toml"; diff --git a/ipc/provider/src/observe.rs b/ipc/provider/src/observe.rs new file mode 100644 index 000000000..e134ef277 --- /dev/null +++ b/ipc/provider/src/observe.rs @@ -0,0 +1,49 @@ +// Copyright 2022-2024 Protocol Labs +// SPDX-License-Identifier: Apache-2.0, MIT + +use ipc_observability::{ + impl_traceable, impl_traceables, lazy_static, register_metrics, serde::HexEncodableBlockHash, + Recordable, TraceLevel, Traceable, +}; +use prometheus::{register_int_gauge, IntGauge, Registry}; + +register_metrics! { + BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT: IntGauge + = register_int_gauge!("bottomup_checkpoint_finalized_height", "Height of the checkpoint finalized"); +} + +impl_traceables!(TraceLevel::Info, "Bottomup", CheckpointSubmitted); + +#[derive(Debug)] +pub struct CheckpointSubmitted { + pub height: i64, + pub hash: HexEncodableBlockHash, +} + +impl Recordable for CheckpointSubmitted { + fn record_metrics(&self) { + BOTTOMUP_CHECKPOINT_FINALIZED_HEIGHT.set(self.height); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ipc_observability::emit; + + #[test] + fn test_metrics() { + let registry = Registry::new(); + register_metrics(®istry).unwrap(); + } + + #[test] + fn test_emit() { + let hash = vec![0x01, 0x02, 0x03]; + + emit(CheckpointSubmitted { + height: 1, + hash: HexEncodableBlockHash(hash.clone()), + }); + } +}