From 136dbef5d7290b9d37ccc1a6c9b29bdc060a4a2c Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 9 May 2023 10:40:32 +0200 Subject: [PATCH 01/26] Prepare for different types of event sources --- crates/relayer-cli/src/commands/listen.rs | 12 +- crates/relayer/src/chain/cosmos.rs | 24 ++-- crates/relayer/src/chain/handle.rs | 2 +- crates/relayer/src/chain/runtime.rs | 2 +- crates/relayer/src/error.rs | 8 +- crates/relayer/src/event.rs | 8 +- .../relayer/src/event/{monitor => }/error.rs | 0 crates/relayer/src/event/rpc.rs | 2 +- crates/relayer/src/event/source.rs | 127 +++++++++++++++++ crates/relayer/src/event/source/pull.rs | 0 .../src/event/{monitor.rs => source/push.rs} | 130 +++--------------- crates/relayer/src/link/relay_path.rs | 2 +- crates/relayer/src/supervisor.rs | 4 +- crates/relayer/src/worker/cmd.rs | 2 +- crates/relayer/src/worker/handle.rs | 2 +- crates/relayer/src/worker/packet.rs | 2 +- 16 files changed, 184 insertions(+), 143 deletions(-) rename crates/relayer/src/event/{monitor => }/error.rs (100%) create mode 100644 crates/relayer/src/event/source.rs create mode 100644 crates/relayer/src/event/source/pull.rs rename crates/relayer/src/event/{monitor.rs => source/push.rs} (80%) diff --git a/crates/relayer-cli/src/commands/listen.rs b/crates/relayer-cli/src/commands/listen.rs index da3bf3dc8d..174fefbaaf 100644 --- a/crates/relayer-cli/src/commands/listen.rs +++ b/crates/relayer-cli/src/commands/listen.rs @@ -14,7 +14,9 @@ use tendermint_rpc::{client::CompatMode, Client, HttpClient}; use tokio::runtime::Runtime as TokioRuntime; use tracing::{error, info, instrument}; -use ibc_relayer::{chain::handle::Subscription, config::ChainConfig, event::monitor::EventMonitor}; +use ibc_relayer::{ + chain::handle::Subscription, config::ChainConfig, event::source::push::EventSource, +}; use ibc_relayer_types::{core::ics24_host::identifier::ChainId, events::IbcEvent}; use crate::prelude::*; @@ -140,7 +142,7 @@ fn subscribe( compat_mode: CompatMode, rt: Arc, ) -> eyre::Result { - let (mut event_monitor, tx_cmd) = EventMonitor::new( + let (mut event_source, tx_cmd) = EventSource::new( chain_config.id.clone(), chain_config.websocket_addr.clone(), compat_mode, @@ -148,14 +150,14 @@ fn subscribe( ) .map_err(|e| eyre!("could not initialize event monitor: {}", e))?; - event_monitor + event_source .init_subscriptions() .map_err(|e| eyre!("could not initialize subscriptions: {}", e))?; - let queries = event_monitor.queries(); + let queries = event_source.queries(); info!("listening for queries: {}", queries.iter().format(", "),); - thread::spawn(|| event_monitor.run()); + thread::spawn(|| event_source.run()); let subscription = tx_cmd.subscribe()?; Ok(subscription) diff --git a/crates/relayer/src/chain/cosmos.rs b/crates/relayer/src/chain/cosmos.rs index 13b9fb9282..6d2e832b5a 100644 --- a/crates/relayer/src/chain/cosmos.rs +++ b/crates/relayer/src/chain/cosmos.rs @@ -95,7 +95,7 @@ use crate::config::{parse_gas_prices, ChainConfig, GasPrice}; use crate::consensus_state::AnyConsensusState; use crate::denom::DenomTrace; use crate::error::Error; -use crate::event::monitor::{EventMonitor, TxMonitorCmd}; +use crate::event::source::{EventSource, TxEventSourceCmd}; use crate::event::IbcEventWithHeight; use crate::keyring::{KeyRing, Secp256k1KeyPair, SigningKeyPair}; use crate::light_client::tendermint::LightClient as TmLightClient; @@ -148,7 +148,7 @@ pub struct CosmosSdkChain { /// A cached copy of the account information account: Option, - tx_monitor_cmd: Option, + tx_monitor_cmd: Option, } impl CosmosSdkChain { @@ -285,22 +285,18 @@ impl CosmosSdkChain { Ok(()) } - fn init_event_monitor(&mut self) -> Result { - crate::time!("init_event_monitor"); + fn init_event_source(&mut self) -> Result { + crate::time!("init_event_source"); - let (mut event_monitor, monitor_tx) = EventMonitor::new( + let (event_source, monitor_tx) = EventSource::push( self.config.id.clone(), self.config.websocket_addr.clone(), self.compat_mode, self.rt.clone(), ) - .map_err(Error::event_monitor)?; + .map_err(Error::event_source)?; - event_monitor - .init_subscriptions() - .map_err(Error::event_monitor)?; - - thread::spawn(move || event_monitor.run()); + thread::spawn(move || event_source.run()); Ok(monitor_tx) } @@ -809,7 +805,7 @@ impl ChainEndpoint for CosmosSdkChain { fn shutdown(self) -> Result<(), Error> { if let Some(monitor_tx) = self.tx_monitor_cmd { - monitor_tx.shutdown().map_err(Error::event_monitor)?; + monitor_tx.shutdown().map_err(Error::event_source)?; } Ok(()) @@ -827,13 +823,13 @@ impl ChainEndpoint for CosmosSdkChain { let tx_monitor_cmd = match &self.tx_monitor_cmd { Some(tx_monitor_cmd) => tx_monitor_cmd, None => { - let tx_monitor_cmd = self.init_event_monitor()?; + let tx_monitor_cmd = self.init_event_source()?; self.tx_monitor_cmd = Some(tx_monitor_cmd); self.tx_monitor_cmd.as_ref().unwrap() } }; - let subscription = tx_monitor_cmd.subscribe().map_err(Error::event_monitor)?; + let subscription = tx_monitor_cmd.subscribe().map_err(Error::event_source)?; Ok(subscription) } diff --git a/crates/relayer/src/chain/handle.rs b/crates/relayer/src/chain/handle.rs index 93aa773f65..be905173cb 100644 --- a/crates/relayer/src/chain/handle.rs +++ b/crates/relayer/src/chain/handle.rs @@ -36,7 +36,7 @@ use crate::{ denom::DenomTrace, error::Error, event::{ - monitor::{EventBatch, Result as MonitorResult}, + source::{EventBatch, Result as MonitorResult}, IbcEventWithHeight, }, keyring::AnySigningKeyPair, diff --git a/crates/relayer/src/chain/runtime.rs b/crates/relayer/src/chain/runtime.rs index 2c97d55bda..0426f6377d 100644 --- a/crates/relayer/src/chain/runtime.rs +++ b/crates/relayer/src/chain/runtime.rs @@ -53,7 +53,7 @@ use super::{ pub struct Threads { pub chain_runtime: thread::JoinHandle<()>, - pub event_monitor: Option>, + pub event_source: Option>, } pub struct ChainRuntime { diff --git a/crates/relayer/src/error.rs b/crates/relayer/src/error.rs index c64a56f4a1..07ad300fea 100644 --- a/crates/relayer/src/error.rs +++ b/crates/relayer/src/error.rs @@ -40,7 +40,7 @@ use ibc_relayer_types::{ use crate::chain::cosmos::version; use crate::chain::cosmos::BLOCK_MAX_BYTES_MAX_FRACTION; use crate::config::Error as ConfigError; -use crate::event::monitor; +use crate::event::source; use crate::keyring::{errors::Error as KeyringError, KeyType}; use crate::sdk_error::SdkError; @@ -86,9 +86,9 @@ define_error! { { url: tendermint_rpc::Url } |e| { format!("Websocket error to endpoint {}", e.url) }, - EventMonitor - [ monitor::Error ] - |_| { "event monitor error" }, + EventSource + [ source::Error ] + |_| { "event source error" }, Grpc |_| { "gRPC error" }, diff --git a/crates/relayer/src/event.rs b/crates/relayer/src/event.rs index 0ed811c66e..d50bc9b1db 100644 --- a/crates/relayer/src/event.rs +++ b/crates/relayer/src/event.rs @@ -1,4 +1,7 @@ use core::fmt::{Display, Error as FmtError, Formatter}; +use serde::Serialize; +use tendermint::abci::Event as AbciEvent; + use ibc_relayer_types::{ applications::ics29_fee::events::{DistributeFeePacket, IncentivizedPacket}, applications::ics31_icq::events::CrossChainQueryPacket, @@ -21,14 +24,13 @@ use ibc_relayer_types::{ events::{Error as IbcEventError, IbcEvent, IbcEventType}, Height, }; -use serde::Serialize; -use tendermint::abci::Event as AbciEvent; use crate::light_client::decode_header; pub mod bus; -pub mod monitor; +pub mod error; pub mod rpc; +pub mod source; #[derive(Clone, Debug, Serialize)] pub struct IbcEventWithHeight { diff --git a/crates/relayer/src/event/monitor/error.rs b/crates/relayer/src/event/error.rs similarity index 100% rename from crates/relayer/src/event/monitor/error.rs rename to crates/relayer/src/event/error.rs diff --git a/crates/relayer/src/event/rpc.rs b/crates/relayer/src/event/rpc.rs index b1bad42fad..6be2febc91 100644 --- a/crates/relayer/src/event/rpc.rs +++ b/crates/relayer/src/event/rpc.rs @@ -11,7 +11,7 @@ use ibc_relayer_types::core::ics24_host::identifier::ChainId; use ibc_relayer_types::events::IbcEvent; use crate::chain::cosmos::types::events::channel::RawObject; -use crate::event::monitor::queries; +use crate::event::source::queries; use crate::telemetry; use super::{ibc_event_try_from_abci_event, IbcEventWithHeight}; diff --git a/crates/relayer/src/event/source.rs b/crates/relayer/src/event/source.rs new file mode 100644 index 0000000000..45115fdecd --- /dev/null +++ b/crates/relayer/src/event/source.rs @@ -0,0 +1,127 @@ +pub mod pull; +pub mod push; + +use std::sync::Arc; + +use crossbeam_channel as channel; + +use futures::Stream; +use tendermint_rpc::{ + client::CompatMode, event::Event as RpcEvent, Error as RpcError, WebSocketClientUrl, +}; +use tokio::runtime::Runtime as TokioRuntime; + +use ibc_relayer_types::{ + core::ics02_client::height::Height, core::ics24_host::identifier::ChainId, +}; + +pub use super::error::{Error, ErrorDetail}; + +use super::IbcEventWithHeight; +use crate::chain::{handle::Subscription, tracking::TrackingId}; + +pub type Result = core::result::Result; + +pub enum EventSource { + Push(push::EventSource), +} + +impl EventSource { + pub fn push( + chain_id: ChainId, + ws_url: WebSocketClientUrl, + rpc_compat: CompatMode, + rt: Arc, + ) -> Result<(Self, TxEventSourceCmd)> { + let (mut source, tx) = push::EventSource::new(chain_id, ws_url, rpc_compat, rt)?; + source.init_subscriptions()?; + Ok((Self::Push(source), tx)) + } + + pub fn run(self) { + match self { + Self::Push(source) => source.run(), + } + } +} + +/// A batch of events from a chain at a specific height +#[derive(Clone, Debug)] +pub struct EventBatch { + pub chain_id: ChainId, + pub tracking_id: TrackingId, + pub height: Height, + pub events: Vec, +} + +type SubscriptionResult = core::result::Result; +type SubscriptionStream = dyn Stream + Send + Sync + Unpin; + +pub type EventSender = channel::Sender>; +pub type EventReceiver = channel::Receiver>; + +#[derive(Clone, Debug)] +pub struct TxEventSourceCmd(channel::Sender); + +impl TxEventSourceCmd { + pub fn shutdown(&self) -> Result<()> { + self.0 + .send(EventSourceCmd::Shutdown) + .map_err(|_| Error::channel_send_failed()) + } + + pub fn subscribe(&self) -> Result { + let (tx, rx) = crossbeam_channel::bounded(1); + + self.0 + .send(EventSourceCmd::Subscribe(tx)) + .map_err(|_| Error::channel_send_failed())?; + + let subscription = rx.recv().map_err(|_| Error::channel_recv_failed())?; + Ok(subscription) + } +} + +#[derive(Debug)] +pub enum EventSourceCmd { + Shutdown, + Subscribe(channel::Sender), +} + +// TODO: These are SDK specific, should be eventually moved. +pub mod queries { + use tendermint_rpc::query::{EventType, Query}; + + pub fn all() -> Vec { + // Note: Tendermint-go supports max 5 query specifiers! + vec![ + new_block(), + ibc_client(), + ibc_connection(), + ibc_channel(), + ibc_query(), + // This will be needed when we send misbehavior evidence to full node + // Query::eq("message.module", "evidence"), + ] + } + + pub fn new_block() -> Query { + Query::from(EventType::NewBlock) + } + + pub fn ibc_client() -> Query { + Query::eq("message.module", "ibc_client") + } + + pub fn ibc_connection() -> Query { + Query::eq("message.module", "ibc_connection") + } + + pub fn ibc_channel() -> Query { + Query::eq("message.module", "ibc_channel") + } + + pub fn ibc_query() -> Query { + Query::eq("message.module", "interchainquery") + } +} diff --git a/crates/relayer/src/event/source/pull.rs b/crates/relayer/src/event/source/pull.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/relayer/src/event/monitor.rs b/crates/relayer/src/event/source/push.rs similarity index 80% rename from crates/relayer/src/event/monitor.rs rename to crates/relayer/src/event/source/push.rs index 339d8aeb49..c3aebdb766 100644 --- a/crates/relayer/src/event/monitor.rs +++ b/crates/relayer/src/event/source/push.rs @@ -12,16 +12,15 @@ use tokio::{runtime::Runtime as TokioRuntime, sync::mpsc}; use tracing::{debug, error, info, instrument, trace}; use tendermint_rpc::{ - client::CompatMode, event::Event as RpcEvent, query::Query, Error as RpcError, - SubscriptionClient, WebSocketClient, WebSocketClientDriver, WebSocketClientUrl, + client::CompatMode, event::Event as RpcEvent, query::Query, SubscriptionClient, + WebSocketClient, WebSocketClientDriver, WebSocketClientUrl, }; -use ibc_relayer_types::{ - core::ics02_client::height::Height, core::ics24_host::identifier::ChainId, events::IbcEvent, -}; +use ibc_relayer_types::{core::ics24_host::identifier::ChainId, events::IbcEvent}; use crate::{ - chain::{handle::Subscription, tracking::TrackingId}, + chain::tracking::TrackingId, + event::{bus::EventBus, error::*, IbcEventWithHeight}, telemetry, util::{ retry::{retry_with_index, RetryResult}, @@ -29,12 +28,7 @@ use crate::{ }, }; -mod error; -pub use error::*; - -use super::{bus::EventBus, IbcEventWithHeight}; - -pub type Result = core::result::Result; +use super::{EventBatch, EventSourceCmd, Result, SubscriptionStream, TxEventSourceCmd}; mod retry_strategy { use crate::util::retry::clamp_total; @@ -52,48 +46,6 @@ mod retry_strategy { } /// A batch of events from a chain at a specific height -#[derive(Clone, Debug)] -pub struct EventBatch { - pub chain_id: ChainId, - pub tracking_id: TrackingId, - pub height: Height, - pub events: Vec, -} - -type SubscriptionResult = core::result::Result; -type SubscriptionStream = dyn Stream + Send + Sync + Unpin; - -pub type EventSender = channel::Sender>; -pub type EventReceiver = channel::Receiver>; - -#[derive(Clone, Debug)] -pub struct TxMonitorCmd(channel::Sender); - -impl TxMonitorCmd { - pub fn shutdown(&self) -> Result<()> { - self.0 - .send(MonitorCmd::Shutdown) - .map_err(|_| Error::channel_send_failed()) - } - - pub fn subscribe(&self) -> Result { - let (tx, rx) = crossbeam_channel::bounded(1); - - self.0 - .send(MonitorCmd::Subscribe(tx)) - .map_err(|_| Error::channel_send_failed())?; - - let subscription = rx.recv().map_err(|_| Error::channel_recv_failed())?; - Ok(subscription) - } -} - -#[derive(Debug)] -pub enum MonitorCmd { - Shutdown, - Subscribe(channel::Sender), -} - /// Connect to a Tendermint node, subscribe to a set of queries, /// receive push events over a websocket, and filter them for the /// event handler. @@ -101,7 +53,7 @@ pub enum MonitorCmd { /// The default events that are queried are: /// - [`EventType::NewBlock`](tendermint_rpc::query::EventType::NewBlock) /// - [`EventType::Tx`](tendermint_rpc::query::EventType::Tx) -pub struct EventMonitor { +pub struct EventSource { chain_id: ChainId, /// WebSocket to collect events from client: WebSocketClient, @@ -114,7 +66,7 @@ pub struct EventMonitor { /// Channel where to send client driver errors tx_err: mpsc::UnboundedSender, /// Channel where to receive commands - rx_cmd: channel::Receiver, + rx_cmd: channel::Receiver, /// Node Address ws_url: WebSocketClientUrl, /// RPC compatibility mode @@ -127,48 +79,10 @@ pub struct EventMonitor { rt: Arc, } -// TODO: These are SDK specific, should be eventually moved. -pub mod queries { - use tendermint_rpc::query::{EventType, Query}; - - pub fn all() -> Vec { - // Note: Tendermint-go supports max 5 query specifiers! - vec![ - new_block(), - ibc_client(), - ibc_connection(), - ibc_channel(), - ibc_query(), - // This will be needed when we send misbehavior evidence to full node - // Query::eq("message.module", "evidence"), - ] - } - - pub fn new_block() -> Query { - Query::from(EventType::NewBlock) - } - - pub fn ibc_client() -> Query { - Query::eq("message.module", "ibc_client") - } - - pub fn ibc_connection() -> Query { - Query::eq("message.module", "ibc_connection") - } - - pub fn ibc_channel() -> Query { - Query::eq("message.module", "ibc_channel") - } - - pub fn ibc_query() -> Query { - Query::eq("message.module", "interchainquery") - } -} - -impl EventMonitor { +impl EventSource { /// Create an event monitor, and connect to a node #[instrument( - name = "event_monitor.create", + name = "event_source.create", level = "error", skip_all, fields(chain = %chain_id, url = %ws_url) @@ -178,7 +92,7 @@ impl EventMonitor { ws_url: WebSocketClientUrl, rpc_compat: CompatMode, rt: Arc, - ) -> Result<(Self, TxMonitorCmd)> { + ) -> Result<(Self, TxEventSourceCmd)> { let event_bus = EventBus::new(); let (tx_cmd, rx_cmd) = channel::unbounded(); @@ -192,7 +106,7 @@ impl EventMonitor { let driver_handle = rt.spawn(run_driver(driver, tx_err.clone())); // TODO: move them to config file(?) - let event_queries = queries::all(); + let event_queries = super::queries::all(); let monitor = Self { rt, @@ -209,7 +123,7 @@ impl EventMonitor { subscriptions: Box::new(futures::stream::empty()), }; - Ok((monitor, TxMonitorCmd(tx_cmd))) + Ok((monitor, TxEventSourceCmd(tx_cmd))) } /// The list of [`Query`] that this event monitor is subscribing for. @@ -218,7 +132,7 @@ impl EventMonitor { } /// Clear the current subscriptions, and subscribe again to all queries. - #[instrument(name = "event_monitor.init_subscriptions", skip_all, fields(chain = %self.chain_id))] + #[instrument(name = "event_source.init_subscriptions", skip_all, fields(chain = %self.chain_id))] pub fn init_subscriptions(&mut self) -> Result<()> { let mut subscriptions = vec![]; @@ -241,7 +155,7 @@ impl EventMonitor { } #[instrument( - name = "event_monitor.try_reconnect", + name = "event_source.try_reconnect", level = "error", skip_all, fields(chain = %self.chain_id) @@ -281,7 +195,7 @@ impl EventMonitor { /// Try to resubscribe to events #[instrument( - name = "event_monitor.try_resubscribe", + name = "event_source.try_resubscribe", level = "error", skip_all, fields(chain = %self.chain_id) @@ -296,7 +210,7 @@ impl EventMonitor { /// See the [`retry`](https://docs.rs/retry) crate and the /// [`crate::util::retry`] module for more information. #[instrument( - name = "event_monitor.reconnect", + name = "event_source.reconnect", level = "error", skip_all, fields(chain = %self.chain_id) @@ -333,7 +247,7 @@ impl EventMonitor { /// Event monitor loop #[allow(clippy::while_let_loop)] #[instrument( - name = "event_monitor", + name = "event_source", level = "error", skip_all, fields(chain = %self.chain_id) @@ -379,8 +293,8 @@ impl EventMonitor { // Process any shutdown or subscription commands if let Ok(cmd) = self.rx_cmd.try_recv() { match cmd { - MonitorCmd::Shutdown => return Next::Abort, - MonitorCmd::Subscribe(tx) => { + EventSourceCmd::Shutdown => return Next::Abort, + EventSourceCmd::Subscribe(tx) => { if let Err(e) = tx.send(self.event_bus.subscribe()) { error!("failed to send back subscription: {e}"); } @@ -398,8 +312,8 @@ impl EventMonitor { // Before handling the batch, check if there are any pending shutdown or subscribe commands. if let Ok(cmd) = self.rx_cmd.try_recv() { match cmd { - MonitorCmd::Shutdown => return Next::Abort, - MonitorCmd::Subscribe(tx) => { + EventSourceCmd::Shutdown => return Next::Abort, + EventSourceCmd::Subscribe(tx) => { if let Err(e) = tx.send(self.event_bus.subscribe()) { error!("failed to send back subscription: {e}"); } diff --git a/crates/relayer/src/link/relay_path.rs b/crates/relayer/src/link/relay_path.rs index de51f21908..cc72e660d3 100644 --- a/crates/relayer/src/link/relay_path.rs +++ b/crates/relayer/src/link/relay_path.rs @@ -42,7 +42,7 @@ use crate::chain::tracking::TrackedMsgs; use crate::chain::tracking::TrackingId; use crate::channel::error::ChannelError; use crate::channel::Channel; -use crate::event::monitor::EventBatch; +use crate::event::source::EventBatch; use crate::event::IbcEventWithHeight; use crate::foreign_client::{ForeignClient, ForeignClientError}; use crate::link::error::{self, LinkError}; diff --git a/crates/relayer/src/supervisor.rs b/crates/relayer/src/supervisor.rs index 3a6029c4dd..87f8f0a055 100644 --- a/crates/relayer/src/supervisor.rs +++ b/crates/relayer/src/supervisor.rs @@ -19,7 +19,7 @@ use crate::{ chain::{endpoint::HealthCheck, handle::ChainHandle, tracking::TrackingId}, config::Config, event::{ - monitor::{self, Error as EventError, ErrorDetail as EventErrorDetail, EventBatch}, + source::{self, Error as EventError, ErrorDetail as EventErrorDetail, EventBatch}, IbcEventWithHeight, }, object::Object, @@ -51,7 +51,7 @@ use cmd::SupervisorCmd; use self::{scan::ChainScanner, spawn::SpawnContext}; -type ArcBatch = Arc>; +type ArcBatch = Arc>; type Subscription = Receiver; /** diff --git a/crates/relayer/src/worker/cmd.rs b/crates/relayer/src/worker/cmd.rs index b7d16c647c..5b2ea5709f 100644 --- a/crates/relayer/src/worker/cmd.rs +++ b/crates/relayer/src/worker/cmd.rs @@ -2,7 +2,7 @@ use core::fmt::{Display, Error as FmtError, Formatter}; use ibc_relayer_types::{core::ics02_client::events::NewBlock, Height}; -use crate::event::monitor::EventBatch; +use crate::event::source::EventBatch; /// A command for a [`WorkerHandle`](crate::worker::WorkerHandle). #[derive(Debug, Clone)] diff --git a/crates/relayer/src/worker/handle.rs b/crates/relayer/src/worker/handle.rs index 0f01787795..3b21391558 100644 --- a/crates/relayer/src/worker/handle.rs +++ b/crates/relayer/src/worker/handle.rs @@ -15,7 +15,7 @@ use crate::chain::tracking::TrackingId; use crate::event::IbcEventWithHeight; use crate::util::lock::{LockExt, RwArc}; use crate::util::task::TaskHandle; -use crate::{event::monitor::EventBatch, object::Object}; +use crate::{event::source::EventBatch, object::Object}; use super::{WorkerCmd, WorkerId}; diff --git a/crates/relayer/src/worker/packet.rs b/crates/relayer/src/worker/packet.rs index ffae0b24ca..acd6b9c90a 100644 --- a/crates/relayer/src/worker/packet.rs +++ b/crates/relayer/src/worker/packet.rs @@ -19,7 +19,7 @@ use ibc_relayer_types::Height; use crate::chain::handle::ChainHandle; use crate::config::filter::FeePolicy; -use crate::event::monitor::EventBatch; +use crate::event::source::EventBatch; use crate::foreign_client::HasExpiredOrFrozenError; use crate::link::Resubmit; use crate::link::{error::LinkError, Link}; From babf3e79a2633a6c466b4bf5a6694673d1e9d0be Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 9 May 2023 11:12:10 +0200 Subject: [PATCH 02/26] Add pull-based event source --- .../src/core/ics02_client/height.rs | 9 +- crates/relayer/src/event.rs | 1 - crates/relayer/src/event/source.rs | 13 +- crates/relayer/src/event/source/pull.rs | 383 ++++++++++++++++++ .../relayer/src/event/source/pull/extract.rs | 101 +++++ crates/relayer/src/event/source/push.rs | 6 +- .../event/{rpc.rs => source/push/extract.rs} | 4 +- 7 files changed, 511 insertions(+), 6 deletions(-) create mode 100644 crates/relayer/src/event/source/pull/extract.rs rename crates/relayer/src/event/{rpc.rs => source/push/extract.rs} (99%) diff --git a/crates/relayer-types/src/core/ics02_client/height.rs b/crates/relayer-types/src/core/ics02_client/height.rs index 09409bcb4a..6b0622854a 100644 --- a/crates/relayer-types/src/core/ics02_client/height.rs +++ b/crates/relayer-types/src/core/ics02_client/height.rs @@ -1,4 +1,4 @@ -use crate::prelude::*; +use crate::{core::ics24_host::identifier::ChainId, prelude::*}; use core::cmp::Ordering; use core::num::ParseIntError; @@ -33,6 +33,13 @@ impl Height { }) } + pub fn from_tm(height: tendermint::block::Height, chain_id: &ChainId) -> Self { + Self { + revision_number: chain_id.version(), + revision_height: height.value(), + } + } + pub fn revision_number(&self) -> u64 { self.revision_number } diff --git a/crates/relayer/src/event.rs b/crates/relayer/src/event.rs index d50bc9b1db..7d3e958505 100644 --- a/crates/relayer/src/event.rs +++ b/crates/relayer/src/event.rs @@ -29,7 +29,6 @@ use crate::light_client::decode_header; pub mod bus; pub mod error; -pub mod rpc; pub mod source; #[derive(Clone, Debug, Serialize)] diff --git a/crates/relayer/src/event/source.rs b/crates/relayer/src/event/source.rs index 45115fdecd..0aaae48144 100644 --- a/crates/relayer/src/event/source.rs +++ b/crates/relayer/src/event/source.rs @@ -7,7 +7,7 @@ use crossbeam_channel as channel; use futures::Stream; use tendermint_rpc::{ - client::CompatMode, event::Event as RpcEvent, Error as RpcError, WebSocketClientUrl, + client::CompatMode, event::Event as RpcEvent, Error as RpcError, HttpClient, WebSocketClientUrl, }; use tokio::runtime::Runtime as TokioRuntime; @@ -24,6 +24,7 @@ pub type Result = core::result::Result; pub enum EventSource { Push(push::EventSource), + Pull(pull::EventSource), } impl EventSource { @@ -38,9 +39,19 @@ impl EventSource { Ok((Self::Push(source), tx)) } + pub fn pull( + chain_id: ChainId, + rpc_client: HttpClient, + rt: Arc, + ) -> Result<(Self, TxEventSourceCmd)> { + let (source, tx) = pull::EventSource::new(chain_id, rpc_client, rt)?; + Ok((Self::Pull(source), tx)) + } + pub fn run(self) { match self { Self::Push(source) => source.run(), + Self::Pull(source) => source.run(), } } } diff --git a/crates/relayer/src/event/source/pull.rs b/crates/relayer/src/event/source/pull.rs index e69de29bb2..b2e30f5d78 100644 --- a/crates/relayer/src/event/source/pull.rs +++ b/crates/relayer/src/event/source/pull.rs @@ -0,0 +1,383 @@ +pub mod extract; + +use std::sync::Arc; + +use crossbeam_channel as channel; +use tokio::{ + runtime::Runtime as TokioRuntime, + time::{sleep, Duration, Instant}, +}; +use tracing::{debug, error, error_span, trace}; + +use tendermint::abci; +use tendermint::block::Height as BlockHeight; +use tendermint_rpc::{Client, HttpClient}; + +use ibc_relayer_types::{ + core::{ + ics02_client::{events::NewBlock, height::Height}, + ics24_host::identifier::ChainId, + }, + events::IbcEvent, +}; + +use crate::{ + chain::tracking::TrackingId, + event::{bus::EventBus, source::Error, IbcEventWithHeight}, + telemetry, + util::retry::ConstantGrowth, +}; + +use super::{EventBatch, EventSourceCmd, TxEventSourceCmd}; + +use self::extract::extract_events; + +pub type Result = core::result::Result; + +const QUERY_INTERVAL: Duration = Duration::from_secs(1); +const MAX_QUERY_INTERVAL: Duration = Duration::from_secs(2); + +pub struct EventSource { + /// Chain identifier + chain_id: ChainId, + + /// RPC client + rpc_client: HttpClient, + + /// Event bus for broadcasting events + event_bus: EventBus>>, + + /// Channel where to receive commands + rx_cmd: channel::Receiver, + + /// Tokio runtime + rt: Arc, + + /// Last fetched block height + last_fetched_height: BlockHeight, +} + +impl EventSource { + pub fn new( + chain_id: ChainId, + rpc_client: HttpClient, + rt: Arc, + ) -> Result<(Self, TxEventSourceCmd)> { + let event_bus = EventBus::new(); + let (tx_cmd, rx_cmd) = channel::unbounded(); + + let monitor = Self { + rt, + chain_id, + rpc_client, + event_bus, + rx_cmd, + last_fetched_height: BlockHeight::from(0_u32), + }; + + Ok((monitor, TxEventSourceCmd(tx_cmd))) + } + + pub fn run(mut self) { + let _span = error_span!("event_monitor", chain.id = %self.chain_id).entered(); + + debug!("starting event monitor"); + + let rt = self.rt.clone(); + + rt.block_on(async { + let mut backoff = monitor_backoff(); + + // Initialize the latest fetched height + if let Ok(latest_height) = latest_height(&self.rpc_client).await { + self.last_fetched_height = latest_height; + } + + // Continuously run the event loop, so that when it aborts + // because of WebSocket client restart, we pick up the work again. + loop { + let before_step = Instant::now(); + + match self.step().await { + Ok(Next::Abort) => break, + + Ok(Next::Continue) => { + // Reset the backoff + backoff = monitor_backoff(); + + // Check if we need to wait some more before the next iteration. + let delay = QUERY_INTERVAL.checked_sub(before_step.elapsed()); + + if let Some(delay_remaining) = delay { + sleep(delay_remaining).await; + } + + continue; + } + + Err(e) => { + error!("event monitor encountered an error: {e}"); + + // Let's backoff the little bit to give the chain some time to recover. + let delay = backoff.next().expect("backoff is an infinite iterator"); + + error!("retrying in {delay:?}..."); + sleep(delay).await; + } + } + } + }); + + debug!("shutting down event monitor"); + } + + async fn step(&mut self) -> Result { + // Process any shutdown or subscription commands before we start doing any work + if let Some(next) = self.try_process_cmd() { + return Ok(next); + } + + let latest_height = latest_height(&self.rpc_client).await?; + + let batches = if latest_height > self.last_fetched_height { + trace!( + "latest height ({latest_height}) > latest fetched height ({})", + self.last_fetched_height + ); + + self.fetch_batches(latest_height).await.map(Some)? + } else { + trace!( + "latest height ({latest_height}) <= latest fetched height ({})", + self.last_fetched_height + ); + + None + }; + + // Before handling the batch, check if there are any pending shutdown or subscribe commands. + if let Some(next) = self.try_process_cmd() { + return Ok(next); + } + + for batch in batches.unwrap_or_default() { + self.broadcast_batch(batch); + } + + Ok(Next::Continue) + } + + fn try_process_cmd(&mut self) -> Option { + if let Ok(cmd) = self.rx_cmd.try_recv() { + match cmd { + EventSourceCmd::Shutdown => return Some(Next::Abort), + EventSourceCmd::Subscribe(tx) => { + if let Err(e) = tx.send(self.event_bus.subscribe()) { + error!("failed to send back subscription: {e}"); + } + } + } + } + + None + } + + async fn fetch_batches(&mut self, latest_height: BlockHeight) -> Result> { + let start_height = self.last_fetched_height.increment(); + + trace!("fetching blocks from {start_height} to {latest_height}"); + + let heights = HeightRangeInclusive::new(start_height, latest_height); + let mut batches = Vec::with_capacity(heights.len()); + + for height in heights { + trace!("collecting events at height {height}"); + + let result = collect_events(&self.rpc_client, &self.chain_id, height).await; + + match result { + Ok(batch) => { + self.last_fetched_height = height; + + if let Some(batch) = batch { + batches.push(batch); + } + } + Err(e) => { + error!(%height, "failed to collect events: {e}"); + break; + } + } + } + + Ok(batches) + } + + /// Collect the IBC events from the subscriptions + fn broadcast_batch(&mut self, batch: EventBatch) { + telemetry!(ws_events, &batch.chain_id, batch.events.len() as u64); + self.event_bus.broadcast(Arc::new(Ok(batch))); + } +} + +fn monitor_backoff() -> impl Iterator { + ConstantGrowth::new(QUERY_INTERVAL, Duration::from_millis(500)) + .clamp(MAX_QUERY_INTERVAL, usize::MAX) +} + +fn dedupe(events: Vec) -> Vec { + use itertools::Itertools; + use std::hash::{Hash, Hasher}; + + #[derive(Clone)] + struct HashEvent(abci::Event); + + impl PartialEq for HashEvent { + fn eq(&self, other: &Self) -> bool { + // NOTE: We don't compare on the index because it is not deterministic + // NOTE: We need to check the length of the attributes in order + // to not miss any attribute + self.0.kind == other.0.kind + && self.0.attributes.len() == other.0.attributes.len() + && self + .0 + .attributes + .iter() + .zip(other.0.attributes.iter()) + .all(|(a, b)| a.key == b.key && a.value == b.value) + } + } + + impl Eq for HashEvent {} + + impl Hash for HashEvent { + fn hash(&self, state: &mut H) { + self.0.kind.hash(state); + + for attr in &self.0.attributes { + // NOTE: We don't hash the index because it is not deterministic + attr.key.hash(state); + attr.value.hash(state); + } + } + } + + events + .into_iter() + .map(HashEvent) + .unique() + .map(|e| e.0) + .collect() +} + +/// Collect the IBC events from an RPC event +async fn collect_events( + rpc_client: &HttpClient, + chain_id: &ChainId, + latest_block_height: BlockHeight, +) -> Result> { + let abci_events = fetch_all_events(rpc_client, latest_block_height).await?; + trace!("Found {} ABCI events before dedupe", abci_events.len()); + + let abci_events = dedupe(abci_events); + trace!("Found {} ABCI events after dedupe", abci_events.len()); + + let height = Height::from_tm(latest_block_height, chain_id); + let new_block_event = + IbcEventWithHeight::new(IbcEvent::NewBlock(NewBlock::new(height)), height); + + let mut block_events = extract_events(chain_id, height, &abci_events).unwrap_or_default(); + let mut events = Vec::with_capacity(block_events.len() + 1); + events.push(new_block_event); + events.append(&mut block_events); + + trace!( + "collected {events_len} events at height {height}: {events:#?}", + events_len = events.len(), + height = height, + ); + + Ok(Some(EventBatch { + chain_id: chain_id.clone(), + tracking_id: TrackingId::new_uuid(), + height, + events, + })) +} + +async fn fetch_all_events( + rpc_client: &HttpClient, + height: BlockHeight, +) -> Result> { + let mut response = rpc_client.block_results(height).await.map_err(Error::rpc)?; + let mut events = vec![]; + + if let Some(begin_block_events) = &mut response.begin_block_events { + events.append(begin_block_events); + } + + if let Some(txs_results) = &mut response.txs_results { + for tx_result in txs_results { + if tx_result.code != abci::Code::Ok { + // Transaction failed, skip it + continue; + } + + events.append(&mut tx_result.events); + } + } + + if let Some(end_block_events) = &mut response.end_block_events { + events.append(end_block_events); + } + + Ok(events) +} + +async fn latest_height(rpc_client: &HttpClient) -> Result { + rpc_client + .abci_info() + .await + .map(|status| status.last_block_height) + .map_err(Error::rpc) +} + +pub enum Next { + Abort, + Continue, +} + +pub struct HeightRangeInclusive { + current: BlockHeight, + end: BlockHeight, +} + +impl HeightRangeInclusive { + pub fn new(start: BlockHeight, end: BlockHeight) -> Self { + Self { + current: start, + end, + } + } +} + +impl Iterator for HeightRangeInclusive { + type Item = BlockHeight; + + fn next(&mut self) -> Option { + if self.current > self.end { + None + } else { + let current = self.current; + self.current = self.current.increment(); + Some(current) + } + } + + fn size_hint(&self) -> (usize, Option) { + let size = self.end.value() - self.current.value() + 1; + (size as usize, Some(size as usize)) + } +} + +impl ExactSizeIterator for HeightRangeInclusive {} diff --git a/crates/relayer/src/event/source/pull/extract.rs b/crates/relayer/src/event/source/pull/extract.rs new file mode 100644 index 0000000000..8ff993e030 --- /dev/null +++ b/crates/relayer/src/event/source/pull/extract.rs @@ -0,0 +1,101 @@ +use ibc_relayer_types::applications::ics29_fee::events::DistributionType; +use tendermint::abci; + +use ibc_relayer_types::core::ics02_client::height::Height; +use ibc_relayer_types::core::ics24_host::identifier::ChainId; +use ibc_relayer_types::events::IbcEvent; + +use crate::telemetry; + +use crate::event::{ibc_event_try_from_abci_event, IbcEventWithHeight}; + +pub fn extract_events( + chain_id: &ChainId, + height: Height, + events: &[abci::Event], +) -> Result, String> { + let mut events_with_height = vec![]; + + for abci_event in events { + match ibc_event_try_from_abci_event(abci_event) { + Ok(event) if should_collect_event(&event) => { + if let IbcEvent::DistributeFeePacket(dist) = &event { + // Only record rewarded fees + if let DistributionType::Reward = dist.distribution_type { + telemetry!(fees_amount, chain_id, &dist.receiver, dist.fee.clone()); + } + } + + events_with_height.push(IbcEventWithHeight { height, event }); + } + + _ => {} + } + } + + Ok(events_with_height) +} + +fn should_collect_event(e: &IbcEvent) -> bool { + event_is_type_packet(e) + || event_is_type_channel(e) + || event_is_type_connection(e) + || event_is_type_client(e) + || event_is_type_fee(e) + || event_is_type_cross_chain_query(e) +} + +fn event_is_type_packet(ev: &IbcEvent) -> bool { + matches!( + ev, + IbcEvent::SendPacket(_) + | IbcEvent::ReceivePacket(_) + | IbcEvent::WriteAcknowledgement(_) + | IbcEvent::AcknowledgePacket(_) + | IbcEvent::TimeoutPacket(_) + | IbcEvent::TimeoutOnClosePacket(_) + ) +} + +fn event_is_type_client(ev: &IbcEvent) -> bool { + matches!( + ev, + IbcEvent::CreateClient(_) + | IbcEvent::UpdateClient(_) + | IbcEvent::UpgradeClient(_) + | IbcEvent::ClientMisbehaviour(_) + ) +} + +fn event_is_type_connection(ev: &IbcEvent) -> bool { + matches!( + ev, + IbcEvent::OpenInitConnection(_) + | IbcEvent::OpenTryConnection(_) + | IbcEvent::OpenAckConnection(_) + | IbcEvent::OpenConfirmConnection(_) + ) +} + +fn event_is_type_channel(ev: &IbcEvent) -> bool { + matches!( + ev, + IbcEvent::OpenInitChannel(_) + | IbcEvent::OpenTryChannel(_) + | IbcEvent::OpenAckChannel(_) + | IbcEvent::OpenConfirmChannel(_) + | IbcEvent::CloseInitChannel(_) + | IbcEvent::CloseConfirmChannel(_) + ) +} + +fn event_is_type_cross_chain_query(ev: &IbcEvent) -> bool { + matches!(ev, IbcEvent::CrossChainQueryPacket(_)) +} + +fn event_is_type_fee(ev: &IbcEvent) -> bool { + matches!( + ev, + IbcEvent::IncentivizedPacket(_) | IbcEvent::DistributeFeePacket(_) + ) +} diff --git a/crates/relayer/src/event/source/push.rs b/crates/relayer/src/event/source/push.rs index c3aebdb766..604293e049 100644 --- a/crates/relayer/src/event/source/push.rs +++ b/crates/relayer/src/event/source/push.rs @@ -1,3 +1,5 @@ +pub mod extract; + use alloc::sync::Arc; use core::cmp::Ordering; @@ -30,6 +32,8 @@ use crate::{ use super::{EventBatch, EventSourceCmd, Result, SubscriptionStream, TxEventSourceCmd}; +use self::extract::extract_events; + mod retry_strategy { use crate::util::retry::clamp_total; use core::time::Duration; @@ -382,7 +386,7 @@ fn collect_events( chain_id: &ChainId, event: RpcEvent, ) -> impl Stream> { - let events = crate::event::rpc::get_all_events(chain_id, event).unwrap_or_default(); + let events = extract_events(chain_id, event).unwrap_or_default(); stream::iter(events).map(Ok) } diff --git a/crates/relayer/src/event/rpc.rs b/crates/relayer/src/event/source/push/extract.rs similarity index 99% rename from crates/relayer/src/event/rpc.rs rename to crates/relayer/src/event/source/push/extract.rs index 6be2febc91..97ff858a98 100644 --- a/crates/relayer/src/event/rpc.rs +++ b/crates/relayer/src/event/source/push/extract.rs @@ -14,7 +14,7 @@ use crate::chain::cosmos::types::events::channel::RawObject; use crate::event::source::queries; use crate::telemetry; -use super::{ibc_event_try_from_abci_event, IbcEventWithHeight}; +use crate::event::{ibc_event_try_from_abci_event, IbcEventWithHeight}; /// Extract IBC events from Tendermint RPC events /// @@ -116,7 +116,7 @@ use super::{ibc_event_try_from_abci_event, IbcEventWithHeight}; /// {Begin,End}Block events however do not have any such `message.action` associated with them, so /// this doesn't work. For this reason, we extract block events in the following order -> /// OpenInit -> OpenTry -> OpenAck -> OpenConfirm -> SendPacket -> CloseInit -> CloseConfirm. -pub fn get_all_events( +pub fn extract_events( chain_id: &ChainId, result: RpcEvent, ) -> Result, String> { From 32f5c04d8dacca1f3517f6205256861a1d4a3eb6 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 9 May 2023 11:18:26 +0200 Subject: [PATCH 03/26] Add `event_source` chain config option (`push` or `pull`) --- crates/relayer-cli/src/chain_registry.rs | 1 + crates/relayer/src/chain/cosmos.rs | 22 +++++++++++++------ crates/relayer/src/config.rs | 13 +++++++++++ tools/test-framework/src/types/single/node.rs | 2 ++ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/crates/relayer-cli/src/chain_registry.rs b/crates/relayer-cli/src/chain_registry.rs index 79454cef65..df04076acf 100644 --- a/crates/relayer-cli/src/chain_registry.rs +++ b/crates/relayer-cli/src/chain_registry.rs @@ -120,6 +120,7 @@ where rpc_addr: rpc_data.rpc_address, websocket_addr: websocket_address, grpc_addr: grpc_address, + event_source: Default::default(), rpc_timeout: default::rpc_timeout(), genesis_restart: None, account_prefix: chain_data.bech32_prefix, diff --git a/crates/relayer/src/chain/cosmos.rs b/crates/relayer/src/chain/cosmos.rs index 6d2e832b5a..b51c6f11e2 100644 --- a/crates/relayer/src/chain/cosmos.rs +++ b/crates/relayer/src/chain/cosmos.rs @@ -287,13 +287,21 @@ impl CosmosSdkChain { fn init_event_source(&mut self) -> Result { crate::time!("init_event_source"); - - let (event_source, monitor_tx) = EventSource::push( - self.config.id.clone(), - self.config.websocket_addr.clone(), - self.compat_mode, - self.rt.clone(), - ) + use crate::config::EventSource as Mode; + + let (event_source, monitor_tx) = match self.config.event_source { + Mode::Push => EventSource::push( + self.config.id.clone(), + self.config.websocket_addr.clone(), + self.compat_mode, + self.rt.clone(), + ), + Mode::Pull => EventSource::pull( + self.config.id.clone(), + self.rpc_client.clone(), + self.rt.clone(), + ), + } .map_err(Error::event_source)?; thread::spawn(move || event_source.run()); diff --git a/crates/relayer/src/config.rs b/crates/relayer/src/config.rs index 2d9674d8d9..8ae0a26159 100644 --- a/crates/relayer/src/config.rs +++ b/crates/relayer/src/config.rs @@ -438,6 +438,17 @@ pub struct GenesisRestart { pub archive_addr: Url, } +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum EventSource { + /// Push-based event source, via WebSocket + #[default] + Push, + + /// Pull-based event source, via RPC /block_results + Pull, +} + #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct ChainConfig { @@ -447,6 +458,8 @@ pub struct ChainConfig { pub rpc_addr: Url, pub websocket_addr: WebSocketClientUrl, pub grpc_addr: Url, + #[serde(default)] + pub event_source: EventSource, #[serde(default = "default::rpc_timeout", with = "humantime_serde")] pub rpc_timeout: Duration, pub account_prefix: String, diff --git a/tools/test-framework/src/types/single/node.rs b/tools/test-framework/src/types/single/node.rs index 65a366c5d1..25a1926f7a 100644 --- a/tools/test-framework/src/types/single/node.rs +++ b/tools/test-framework/src/types/single/node.rs @@ -133,12 +133,14 @@ impl FullNode { .as_path() .display() .to_string(); + Ok(config::ChainConfig { id: self.chain_driver.chain_id.clone(), r#type: ChainType::CosmosSdk, rpc_addr: Url::from_str(&self.chain_driver.rpc_address())?, websocket_addr: WebSocketClientUrl::from_str(&self.chain_driver.websocket_address())?, grpc_addr: Url::from_str(&self.chain_driver.grpc_address())?, + event_source: config::EventSource::Push, rpc_timeout: Duration::from_secs(10), genesis_restart: None, account_prefix: self.chain_driver.account_prefix.clone(), From d683700eb958dfa6841e0474517e7c674c2448a6 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 9 May 2023 12:09:42 +0200 Subject: [PATCH 04/26] Use poll as default --- crates/relayer/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/relayer/src/config.rs b/crates/relayer/src/config.rs index 8ae0a26159..b94397e006 100644 --- a/crates/relayer/src/config.rs +++ b/crates/relayer/src/config.rs @@ -442,10 +442,10 @@ pub struct GenesisRestart { #[serde(rename_all = "lowercase")] pub enum EventSource { /// Push-based event source, via WebSocket - #[default] Push, /// Pull-based event source, via RPC /block_results + #[default] Pull, } From 0b3ae013904d1aa91a11bf1c533bcb7be1ffb939 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 9 May 2023 13:43:29 +0200 Subject: [PATCH 05/26] Change `push` to `websocket` and `pull` to `rpc` --- config.toml | 6 +++++ crates/relayer-cli/src/commands/listen.rs | 2 +- crates/relayer/src/chain/cosmos.rs | 4 ++-- crates/relayer/src/config.rs | 4 ++-- crates/relayer/src/event/source.rs | 24 +++++++++---------- .../src/event/source/{pull.rs => rpc.rs} | 0 .../src/event/source/{pull => rpc}/extract.rs | 0 .../event/source/{push.rs => websocket.rs} | 0 .../source/{push => websocket}/extract.rs | 0 tools/test-framework/src/types/single/node.rs | 2 +- 10 files changed, 24 insertions(+), 18 deletions(-) rename crates/relayer/src/event/source/{pull.rs => rpc.rs} (100%) rename crates/relayer/src/event/source/{pull => rpc}/extract.rs (100%) rename crates/relayer/src/event/source/{push.rs => websocket.rs} (100%) rename crates/relayer/src/event/source/{push => websocket}/extract.rs (100%) diff --git a/config.toml b/config.toml index 2ee3a28947..b3f89662fa 100644 --- a/config.toml +++ b/config.toml @@ -126,6 +126,11 @@ grpc_addr = 'http://127.0.0.1:9090' # listens on. Required websocket_addr = 'ws://127.0.0.1:26657/websocket' +# The type of event source to use for getting events from the chain. +# Either `websocket` for WebSocket or `rpc` for RPC via the `/block_results` endpoint. +# Default: 'websocket' +event_source = 'websocket' + # Specify the maximum amount of time (duration) that the RPC requests should # take before timing out. Default: 10s (10 seconds) # Note: Hermes uses this parameter _only_ in `start` mode; for all other CLIs, @@ -310,6 +315,7 @@ id = 'ibc-1' rpc_addr = 'http://127.0.0.1:26557' grpc_addr = 'http://127.0.0.1:9091' websocket_addr = 'ws://127.0.0.1:26557/websocket' +event_source = 'websocket' rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' diff --git a/crates/relayer-cli/src/commands/listen.rs b/crates/relayer-cli/src/commands/listen.rs index 174fefbaaf..16eb3ecf5c 100644 --- a/crates/relayer-cli/src/commands/listen.rs +++ b/crates/relayer-cli/src/commands/listen.rs @@ -15,7 +15,7 @@ use tokio::runtime::Runtime as TokioRuntime; use tracing::{error, info, instrument}; use ibc_relayer::{ - chain::handle::Subscription, config::ChainConfig, event::source::push::EventSource, + chain::handle::Subscription, config::ChainConfig, event::source::websocket::EventSource, }; use ibc_relayer_types::{core::ics24_host::identifier::ChainId, events::IbcEvent}; diff --git a/crates/relayer/src/chain/cosmos.rs b/crates/relayer/src/chain/cosmos.rs index b51c6f11e2..68c877d0e3 100644 --- a/crates/relayer/src/chain/cosmos.rs +++ b/crates/relayer/src/chain/cosmos.rs @@ -290,13 +290,13 @@ impl CosmosSdkChain { use crate::config::EventSource as Mode; let (event_source, monitor_tx) = match self.config.event_source { - Mode::Push => EventSource::push( + Mode::WebSocket => EventSource::websocket( self.config.id.clone(), self.config.websocket_addr.clone(), self.compat_mode, self.rt.clone(), ), - Mode::Pull => EventSource::pull( + Mode::Rpc => EventSource::rpc( self.config.id.clone(), self.rpc_client.clone(), self.rt.clone(), diff --git a/crates/relayer/src/config.rs b/crates/relayer/src/config.rs index b94397e006..8f2c0b9007 100644 --- a/crates/relayer/src/config.rs +++ b/crates/relayer/src/config.rs @@ -442,11 +442,11 @@ pub struct GenesisRestart { #[serde(rename_all = "lowercase")] pub enum EventSource { /// Push-based event source, via WebSocket - Push, + WebSocket, /// Pull-based event source, via RPC /block_results #[default] - Pull, + Rpc, } #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] diff --git a/crates/relayer/src/event/source.rs b/crates/relayer/src/event/source.rs index 0aaae48144..4b92b75626 100644 --- a/crates/relayer/src/event/source.rs +++ b/crates/relayer/src/event/source.rs @@ -1,5 +1,5 @@ -pub mod pull; -pub mod push; +pub mod rpc; +pub mod websocket; use std::sync::Arc; @@ -23,35 +23,35 @@ use crate::chain::{handle::Subscription, tracking::TrackingId}; pub type Result = core::result::Result; pub enum EventSource { - Push(push::EventSource), - Pull(pull::EventSource), + WebSocket(websocket::EventSource), + Rpc(rpc::EventSource), } impl EventSource { - pub fn push( + pub fn websocket( chain_id: ChainId, ws_url: WebSocketClientUrl, rpc_compat: CompatMode, rt: Arc, ) -> Result<(Self, TxEventSourceCmd)> { - let (mut source, tx) = push::EventSource::new(chain_id, ws_url, rpc_compat, rt)?; + let (mut source, tx) = websocket::EventSource::new(chain_id, ws_url, rpc_compat, rt)?; source.init_subscriptions()?; - Ok((Self::Push(source), tx)) + Ok((Self::WebSocket(source), tx)) } - pub fn pull( + pub fn rpc( chain_id: ChainId, rpc_client: HttpClient, rt: Arc, ) -> Result<(Self, TxEventSourceCmd)> { - let (source, tx) = pull::EventSource::new(chain_id, rpc_client, rt)?; - Ok((Self::Pull(source), tx)) + let (source, tx) = rpc::EventSource::new(chain_id, rpc_client, rt)?; + Ok((Self::Rpc(source), tx)) } pub fn run(self) { match self { - Self::Push(source) => source.run(), - Self::Pull(source) => source.run(), + Self::WebSocket(source) => source.run(), + Self::Rpc(source) => source.run(), } } } diff --git a/crates/relayer/src/event/source/pull.rs b/crates/relayer/src/event/source/rpc.rs similarity index 100% rename from crates/relayer/src/event/source/pull.rs rename to crates/relayer/src/event/source/rpc.rs diff --git a/crates/relayer/src/event/source/pull/extract.rs b/crates/relayer/src/event/source/rpc/extract.rs similarity index 100% rename from crates/relayer/src/event/source/pull/extract.rs rename to crates/relayer/src/event/source/rpc/extract.rs diff --git a/crates/relayer/src/event/source/push.rs b/crates/relayer/src/event/source/websocket.rs similarity index 100% rename from crates/relayer/src/event/source/push.rs rename to crates/relayer/src/event/source/websocket.rs diff --git a/crates/relayer/src/event/source/push/extract.rs b/crates/relayer/src/event/source/websocket/extract.rs similarity index 100% rename from crates/relayer/src/event/source/push/extract.rs rename to crates/relayer/src/event/source/websocket/extract.rs diff --git a/tools/test-framework/src/types/single/node.rs b/tools/test-framework/src/types/single/node.rs index 25a1926f7a..9a672baff5 100644 --- a/tools/test-framework/src/types/single/node.rs +++ b/tools/test-framework/src/types/single/node.rs @@ -140,7 +140,7 @@ impl FullNode { rpc_addr: Url::from_str(&self.chain_driver.rpc_address())?, websocket_addr: WebSocketClientUrl::from_str(&self.chain_driver.websocket_address())?, grpc_addr: Url::from_str(&self.chain_driver.grpc_address())?, - event_source: config::EventSource::Push, + event_source: config::EventSource::WebSocket, rpc_timeout: Duration::from_secs(10), genesis_restart: None, account_prefix: self.chain_driver.account_prefix.clone(), From 396efc545f4ebdb3543edc7c292a83f93ea16454 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 9 May 2023 13:53:16 +0200 Subject: [PATCH 06/26] Add `poll_interval` setting to the chain config --- config.toml | 6 ++++++ crates/relayer-cli/src/chain_registry.rs | 5 +++-- crates/relayer/src/chain/cosmos.rs | 1 + crates/relayer/src/config.rs | 6 ++++++ crates/relayer/src/event/source.rs | 5 +++-- crates/relayer/src/event/source/rpc.rs | 20 ++++++++++--------- tools/test-framework/src/types/single/node.rs | 1 + 7 files changed, 31 insertions(+), 13 deletions(-) diff --git a/config.toml b/config.toml index b3f89662fa..7ade8cc792 100644 --- a/config.toml +++ b/config.toml @@ -131,6 +131,11 @@ websocket_addr = 'ws://127.0.0.1:26657/websocket' # Default: 'websocket' event_source = 'websocket' +# The interval at which to poll for blocks when the `event_source` is `rpc`. +# Has no effect when `event_source` is set to `websocket`. +# Default: 1 second +poll_interval = '1s' + # Specify the maximum amount of time (duration) that the RPC requests should # take before timing out. Default: 10s (10 seconds) # Note: Hermes uses this parameter _only_ in `start` mode; for all other CLIs, @@ -316,6 +321,7 @@ rpc_addr = 'http://127.0.0.1:26557' grpc_addr = 'http://127.0.0.1:9091' websocket_addr = 'ws://127.0.0.1:26557/websocket' event_source = 'websocket' +poll_interval = '1s' rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' diff --git a/crates/relayer-cli/src/chain_registry.rs b/crates/relayer-cli/src/chain_registry.rs index df04076acf..9ee5f3cf0e 100644 --- a/crates/relayer-cli/src/chain_registry.rs +++ b/crates/relayer-cli/src/chain_registry.rs @@ -24,7 +24,7 @@ use ibc_chain_registry::querier::SimpleHermesRpcQuerier; use ibc_relayer::config::filter::{FilterPattern, PacketFilter}; use ibc_relayer::config::gas_multiplier::GasMultiplier; use ibc_relayer::config::types::{MaxMsgNum, MaxTxSize, Memo}; -use ibc_relayer::config::{default, AddressType, ChainConfig, GasPrice}; +use ibc_relayer::config::{default, AddressType, ChainConfig, EventSource, GasPrice}; use ibc_relayer::keyring::Store; const MAX_HEALTHY_QUERY_RETRIES: u8 = 5; @@ -120,7 +120,8 @@ where rpc_addr: rpc_data.rpc_address, websocket_addr: websocket_address, grpc_addr: grpc_address, - event_source: Default::default(), + event_source: EventSource::Rpc, + poll_interval: default::poll_interval(), rpc_timeout: default::rpc_timeout(), genesis_restart: None, account_prefix: chain_data.bech32_prefix, diff --git a/crates/relayer/src/chain/cosmos.rs b/crates/relayer/src/chain/cosmos.rs index 68c877d0e3..da69fd96b5 100644 --- a/crates/relayer/src/chain/cosmos.rs +++ b/crates/relayer/src/chain/cosmos.rs @@ -299,6 +299,7 @@ impl CosmosSdkChain { Mode::Rpc => EventSource::rpc( self.config.id.clone(), self.rpc_client.clone(), + self.config.poll_interval, self.rt.clone(), ), } diff --git a/crates/relayer/src/config.rs b/crates/relayer/src/config.rs index 8f2c0b9007..cc90ad71bf 100644 --- a/crates/relayer/src/config.rs +++ b/crates/relayer/src/config.rs @@ -173,6 +173,10 @@ pub mod default { Duration::from_secs(10) } + pub fn poll_interval() -> Duration { + Duration::from_secs(1) + } + pub fn clock_drift() -> Duration { Duration::from_secs(5) } @@ -460,6 +464,8 @@ pub struct ChainConfig { pub grpc_addr: Url, #[serde(default)] pub event_source: EventSource, + #[serde(default = "default::poll_interval", with = "humantime_serde")] + pub poll_interval: Duration, #[serde(default = "default::rpc_timeout", with = "humantime_serde")] pub rpc_timeout: Duration, pub account_prefix: String, diff --git a/crates/relayer/src/event/source.rs b/crates/relayer/src/event/source.rs index 4b92b75626..50d379eb95 100644 --- a/crates/relayer/src/event/source.rs +++ b/crates/relayer/src/event/source.rs @@ -1,7 +1,7 @@ pub mod rpc; pub mod websocket; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use crossbeam_channel as channel; @@ -42,9 +42,10 @@ impl EventSource { pub fn rpc( chain_id: ChainId, rpc_client: HttpClient, + poll_interval: Duration, rt: Arc, ) -> Result<(Self, TxEventSourceCmd)> { - let (source, tx) = rpc::EventSource::new(chain_id, rpc_client, rt)?; + let (source, tx) = rpc::EventSource::new(chain_id, rpc_client, poll_interval, rt)?; Ok((Self::Rpc(source), tx)) } diff --git a/crates/relayer/src/event/source/rpc.rs b/crates/relayer/src/event/source/rpc.rs index b2e30f5d78..af02a3a0bf 100644 --- a/crates/relayer/src/event/source/rpc.rs +++ b/crates/relayer/src/event/source/rpc.rs @@ -34,9 +34,6 @@ use self::extract::extract_events; pub type Result = core::result::Result; -const QUERY_INTERVAL: Duration = Duration::from_secs(1); -const MAX_QUERY_INTERVAL: Duration = Duration::from_secs(2); - pub struct EventSource { /// Chain identifier chain_id: ChainId, @@ -44,6 +41,9 @@ pub struct EventSource { /// RPC client rpc_client: HttpClient, + /// Poll interval + poll_interval: Duration, + /// Event bus for broadcasting events event_bus: EventBus>>, @@ -61,6 +61,7 @@ impl EventSource { pub fn new( chain_id: ChainId, rpc_client: HttpClient, + poll_interval: Duration, rt: Arc, ) -> Result<(Self, TxEventSourceCmd)> { let event_bus = EventBus::new(); @@ -70,6 +71,7 @@ impl EventSource { rt, chain_id, rpc_client, + poll_interval, event_bus, rx_cmd, last_fetched_height: BlockHeight::from(0_u32), @@ -86,7 +88,7 @@ impl EventSource { let rt = self.rt.clone(); rt.block_on(async { - let mut backoff = monitor_backoff(); + let mut backoff = monitor_backoff(self.poll_interval); // Initialize the latest fetched height if let Ok(latest_height) = latest_height(&self.rpc_client).await { @@ -103,10 +105,10 @@ impl EventSource { Ok(Next::Continue) => { // Reset the backoff - backoff = monitor_backoff(); + backoff = monitor_backoff(self.poll_interval); // Check if we need to wait some more before the next iteration. - let delay = QUERY_INTERVAL.checked_sub(before_step.elapsed()); + let delay = self.poll_interval.checked_sub(before_step.elapsed()); if let Some(delay_remaining) = delay { sleep(delay_remaining).await; @@ -220,9 +222,9 @@ impl EventSource { } } -fn monitor_backoff() -> impl Iterator { - ConstantGrowth::new(QUERY_INTERVAL, Duration::from_millis(500)) - .clamp(MAX_QUERY_INTERVAL, usize::MAX) +fn monitor_backoff(poll_interval: Duration) -> impl Iterator { + ConstantGrowth::new(poll_interval, Duration::from_millis(500)) + .clamp(poll_interval * 5, usize::MAX) } fn dedupe(events: Vec) -> Vec { diff --git a/tools/test-framework/src/types/single/node.rs b/tools/test-framework/src/types/single/node.rs index 9a672baff5..bd2d076bcc 100644 --- a/tools/test-framework/src/types/single/node.rs +++ b/tools/test-framework/src/types/single/node.rs @@ -141,6 +141,7 @@ impl FullNode { websocket_addr: WebSocketClientUrl::from_str(&self.chain_driver.websocket_address())?, grpc_addr: Url::from_str(&self.chain_driver.grpc_address())?, event_source: config::EventSource::WebSocket, + poll_interval: config::default::poll_interval(), rpc_timeout: Duration::from_secs(10), genesis_restart: None, account_prefix: self.chain_driver.account_prefix.clone(), From 068d5ab6dfebf6d15cd951f14af7cc47e10fe4ab Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 16 May 2023 12:13:48 +0200 Subject: [PATCH 07/26] Add changelog entry --- .../features/ibc-relayer/2850-poll-event-source.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .changelog/unreleased/features/ibc-relayer/2850-poll-event-source.md diff --git a/.changelog/unreleased/features/ibc-relayer/2850-poll-event-source.md b/.changelog/unreleased/features/ibc-relayer/2850-poll-event-source.md new file mode 100644 index 0000000000..8c8a675af3 --- /dev/null +++ b/.changelog/unreleased/features/ibc-relayer/2850-poll-event-source.md @@ -0,0 +1,12 @@ +- Add a poll-based event source which fetches events from the chain using + the `/block_results` RPC endpoint instead of getting them over WebSocket. + + To use the poll-based event source, set `event_source = 'rpc'` in the per-chain configuration. + + **Warning** + Only use this if you think Hermes is not getting all + the events it should, eg. when relaying for a CosmWasm-enabled blockchain + which emits IBC events in a smart contract where the events lack the + `message` attribute key. See #3190 and #2809 for more background information. + + ([\#2850](https://github.com/informalsystems/hermes/issues/2850)) From a5d62da14278855ceac392efb703ab426eb196a1 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 16 May 2023 12:17:37 +0200 Subject: [PATCH 08/26] Revert d683700 and go back to using the WebSocket event source by default --- crates/relayer-cli/src/chain_registry.rs | 3 ++- crates/relayer/src/config.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/relayer-cli/src/chain_registry.rs b/crates/relayer-cli/src/chain_registry.rs index be015bff90..075746b552 100644 --- a/crates/relayer-cli/src/chain_registry.rs +++ b/crates/relayer-cli/src/chain_registry.rs @@ -109,6 +109,7 @@ where MAX_HEALTHY_QUERY_RETRIES, ) .await?; + let websocket_address = rpc_data.websocket.clone().try_into().map_err(|e| { RegistryError::websocket_url_parse_error(rpc_data.websocket.to_string(), e) @@ -120,7 +121,7 @@ where rpc_addr: rpc_data.rpc_address, websocket_addr: websocket_address, grpc_addr: grpc_address, - event_source: EventSource::Rpc, + event_source: EventSource::WebSocket, poll_interval: default::poll_interval(), rpc_timeout: default::rpc_timeout(), genesis_restart: None, diff --git a/crates/relayer/src/config.rs b/crates/relayer/src/config.rs index 2c62368439..1f64071d1b 100644 --- a/crates/relayer/src/config.rs +++ b/crates/relayer/src/config.rs @@ -451,10 +451,10 @@ pub struct GenesisRestart { #[serde(rename_all = "lowercase")] pub enum EventSource { /// Push-based event source, via WebSocket + #[default] WebSocket, /// Pull-based event source, via RPC /block_results - #[default] Rpc, } From 2a0110169671d037efc6b38c872b8b975fd56c2d Mon Sep 17 00:00:00 2001 From: Sean Chen Date: Tue, 16 May 2023 10:44:43 -0500 Subject: [PATCH 09/26] Improve WebSocket::EventSource doc comment --- crates/relayer/src/event/source/websocket.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/relayer/src/event/source/websocket.rs b/crates/relayer/src/event/source/websocket.rs index 6b9b8c1e54..90b7116866 100644 --- a/crates/relayer/src/event/source/websocket.rs +++ b/crates/relayer/src/event/source/websocket.rs @@ -49,7 +49,9 @@ mod retry_strategy { } } -/// A batch of events from a chain at a specific height +/// A batch of events received from a WebSocket endpoint from a +/// chain at a specific height. +/// /// Connect to a Tendermint node, subscribe to a set of queries, /// receive push events over a websocket, and filter them for the /// event handler. From ceb210a1694cf8066bd5e1d1c4a5cfaebf097aa4 Mon Sep 17 00:00:00 2001 From: Sean Chen Date: Tue, 16 May 2023 10:56:39 -0500 Subject: [PATCH 10/26] Cargo fmt --- crates/relayer/src/event/source/websocket.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/relayer/src/event/source/websocket.rs b/crates/relayer/src/event/source/websocket.rs index 90b7116866..48e5efe29c 100644 --- a/crates/relayer/src/event/source/websocket.rs +++ b/crates/relayer/src/event/source/websocket.rs @@ -51,7 +51,7 @@ mod retry_strategy { /// A batch of events received from a WebSocket endpoint from a /// chain at a specific height. -/// +/// /// Connect to a Tendermint node, subscribe to a set of queries, /// receive push events over a websocket, and filter them for the /// event handler. From 586c217a31c5f360de547ac247414209a0eba9c2 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 16 May 2023 18:36:02 +0200 Subject: [PATCH 11/26] Add more tracing to event sources --- crates/relayer/src/event/source/rpc.rs | 9 +++++++++ crates/relayer/src/event/source/websocket.rs | 14 ++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/crates/relayer/src/event/source/rpc.rs b/crates/relayer/src/event/source/rpc.rs index af02a3a0bf..7618f27fb4 100644 --- a/crates/relayer/src/event/source/rpc.rs +++ b/crates/relayer/src/event/source/rpc.rs @@ -218,6 +218,15 @@ impl EventSource { /// Collect the IBC events from the subscriptions fn broadcast_batch(&mut self, batch: EventBatch) { telemetry!(ws_events, &batch.chain_id, batch.events.len() as u64); + + debug!( + chain = %batch.chain_id, + count = %batch.events.len(), + height = %batch.height, + "broadcasting batch of {} events", + batch.events.len() + ); + self.event_bus.broadcast(Arc::new(Ok(batch))); } } diff --git a/crates/relayer/src/event/source/websocket.rs b/crates/relayer/src/event/source/websocket.rs index 48e5efe29c..f57d9706ce 100644 --- a/crates/relayer/src/event/source/websocket.rs +++ b/crates/relayer/src/event/source/websocket.rs @@ -328,7 +328,7 @@ impl EventSource { } match result { - Ok(batch) => self.process_batch(batch), + Ok(batch) => self.broadcast_batch(batch), Err(e) => { if let ErrorDetail::SubscriptionCancelled(reason) = e.detail() { error!("subscription cancelled, reason: {}", reason); @@ -375,11 +375,17 @@ impl EventSource { self.event_bus.broadcast(Arc::new(Err(error))); } - /// Collect the IBC events from the subscriptions - fn process_batch(&mut self, batch: EventBatch) { + /// Broadcast a batch of events to all subscribers. + fn broadcast_batch(&mut self, batch: EventBatch) { telemetry!(ws_events, &batch.chain_id, batch.events.len() as u64); - debug!(chain = %batch.chain_id, len = %batch.events.len(), "emitting batch"); + debug!( + chain = %batch.chain_id, + count = %batch.events.len(), + height = %batch.height, + "broadcasting batch of {} events", + batch.events.len() + ); self.event_bus.broadcast(Arc::new(Ok(batch))); } From d9a691b91deb080ceda997fc1235ab830ec013da Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 16 May 2023 18:40:52 +0200 Subject: [PATCH 12/26] Apply suggestions from code review Co-authored-by: Sean Chen Signed-off-by: Romain Ruetschi --- crates/relayer/src/chain/cosmos.rs | 2 +- crates/relayer/src/event/source/rpc.rs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/relayer/src/chain/cosmos.rs b/crates/relayer/src/chain/cosmos.rs index 0fb973bb41..1ae11aa05d 100644 --- a/crates/relayer/src/chain/cosmos.rs +++ b/crates/relayer/src/chain/cosmos.rs @@ -288,7 +288,7 @@ impl CosmosSdkChain { fn init_event_source(&mut self) -> Result { crate::time!( - "init_event_monitor", + "init_event_source", { "src_chain": self.config().id.to_string(), } diff --git a/crates/relayer/src/event/source/rpc.rs b/crates/relayer/src/event/source/rpc.rs index 7618f27fb4..82d9094363 100644 --- a/crates/relayer/src/event/source/rpc.rs +++ b/crates/relayer/src/event/source/rpc.rs @@ -34,6 +34,8 @@ use self::extract::extract_events; pub type Result = core::result::Result; +/// Captures the mode in which events are fetched from a chain, either via WebSocket +/// or via RPC endpoint. pub struct EventSource { /// Chain identifier chain_id: ChainId, From 1d40aed03b77b3089d548e3346f9982518743ac3 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Wed, 17 May 2023 09:41:30 +0200 Subject: [PATCH 13/26] Change uses of event monitor to event source --- crates/relayer-cli/src/commands/listen.rs | 2 +- crates/relayer/src/event/error.rs | 4 ++-- crates/relayer/src/event/source/rpc.rs | 18 +++++++++--------- crates/relayer/src/event/source/websocket.rs | 20 ++++++++++---------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/crates/relayer-cli/src/commands/listen.rs b/crates/relayer-cli/src/commands/listen.rs index 16eb3ecf5c..409eb04fb4 100644 --- a/crates/relayer-cli/src/commands/listen.rs +++ b/crates/relayer-cli/src/commands/listen.rs @@ -148,7 +148,7 @@ fn subscribe( compat_mode, rt, ) - .map_err(|e| eyre!("could not initialize event monitor: {}", e))?; + .map_err(|e| eyre!("could not initialize event source: {}", e))?; event_source .init_subscriptions() diff --git a/crates/relayer/src/event/error.rs b/crates/relayer/src/event/error.rs index 5bd12337dd..daf1e0a01b 100644 --- a/crates/relayer/src/event/error.rs +++ b/crates/relayer/src/event/error.rs @@ -36,10 +36,10 @@ define_error! { |e| { format!("failed to extract IBC events: {0}", e.reason) }, ChannelSendFailed - |_| { "event monitor: internal message-passing failure: could not send message" }, + |_| { "event source: internal message-passing failure: could not send message" }, ChannelRecvFailed - |_| { "event monitor: internal message-passing failure: could not receive message" }, + |_| { "event source: internal message-passing failure: could not receive message" }, SubscriptionCancelled [ TraceError ] diff --git a/crates/relayer/src/event/source/rpc.rs b/crates/relayer/src/event/source/rpc.rs index 82d9094363..1bd80f0340 100644 --- a/crates/relayer/src/event/source/rpc.rs +++ b/crates/relayer/src/event/source/rpc.rs @@ -69,7 +69,7 @@ impl EventSource { let event_bus = EventBus::new(); let (tx_cmd, rx_cmd) = channel::unbounded(); - let monitor = Self { + let source = Self { rt, chain_id, rpc_client, @@ -79,18 +79,18 @@ impl EventSource { last_fetched_height: BlockHeight::from(0_u32), }; - Ok((monitor, TxEventSourceCmd(tx_cmd))) + Ok((source, TxEventSourceCmd(tx_cmd))) } pub fn run(mut self) { - let _span = error_span!("event_monitor", chain.id = %self.chain_id).entered(); + let _span = error_span!("event_source.rpc", chain.id = %self.chain_id).entered(); - debug!("starting event monitor"); + debug!("collecting events"); let rt = self.rt.clone(); rt.block_on(async { - let mut backoff = monitor_backoff(self.poll_interval); + let mut backoff = poll_backoff(self.poll_interval); // Initialize the latest fetched height if let Ok(latest_height) = latest_height(&self.rpc_client).await { @@ -107,7 +107,7 @@ impl EventSource { Ok(Next::Continue) => { // Reset the backoff - backoff = monitor_backoff(self.poll_interval); + backoff = poll_backoff(self.poll_interval); // Check if we need to wait some more before the next iteration. let delay = self.poll_interval.checked_sub(before_step.elapsed()); @@ -120,7 +120,7 @@ impl EventSource { } Err(e) => { - error!("event monitor encountered an error: {e}"); + error!("event source encountered an error: {e}"); // Let's backoff the little bit to give the chain some time to recover. let delay = backoff.next().expect("backoff is an infinite iterator"); @@ -132,7 +132,7 @@ impl EventSource { } }); - debug!("shutting down event monitor"); + debug!("shutting down event source"); } async fn step(&mut self) -> Result { @@ -233,7 +233,7 @@ impl EventSource { } } -fn monitor_backoff(poll_interval: Duration) -> impl Iterator { +fn poll_backoff(poll_interval: Duration) -> impl Iterator { ConstantGrowth::new(poll_interval, Duration::from_millis(500)) .clamp(poll_interval * 5, usize::MAX) } diff --git a/crates/relayer/src/event/source/websocket.rs b/crates/relayer/src/event/source/websocket.rs index f57d9706ce..219266bd6e 100644 --- a/crates/relayer/src/event/source/websocket.rs +++ b/crates/relayer/src/event/source/websocket.rs @@ -86,7 +86,7 @@ pub struct EventSource { } impl EventSource { - /// Create an event monitor, and connect to a node + /// Create an event source, and connect to a node #[instrument( name = "event_source.create", level = "error", @@ -114,7 +114,7 @@ impl EventSource { // TODO: move them to config file(?) let event_queries = super::queries::all(); - let monitor = Self { + let source = Self { rt, chain_id, client, @@ -129,10 +129,10 @@ impl EventSource { subscriptions: Box::new(futures::stream::empty()), }; - Ok((monitor, TxEventSourceCmd(tx_cmd))) + Ok((source, TxEventSourceCmd(tx_cmd))) } - /// The list of [`Query`] that this event monitor is subscribing for. + /// The list of [`Query`] that this event source is subscribing for. pub fn queries(&self) -> &[Query] { &self.event_queries } @@ -250,16 +250,16 @@ impl EventSource { } } - /// Event monitor loop + /// Event source loop #[allow(clippy::while_let_loop)] #[instrument( - name = "event_source", + name = "event_source.websocket", level = "error", skip_all, fields(chain = %self.chain_id) )] pub fn run(mut self) { - debug!("starting event monitor"); + debug!("collecting events"); // Continuously run the event loop, so that when it aborts // because of WebSocket client restart, we pick up the work again. @@ -270,7 +270,7 @@ impl EventSource { } } - debug!("event monitor is shutting down"); + debug!("event source is shutting down"); // Close the WebSocket connection let _ = self.client.close(); @@ -278,7 +278,7 @@ impl EventSource { // Wait for the WebSocket driver to finish let _ = self.rt.block_on(self.driver_handle); - trace!("event monitor has successfully shut down"); + trace!("event source has successfully shut down"); } fn run_loop(&mut self) -> Next { @@ -454,7 +454,7 @@ async fn run_driver( ) { if let Err(e) = driver.run().await { if tx.send(e).is_err() { - error!("failed to relay driver error to event monitor"); + error!("failed to relay driver error to event source"); } } } From 53b0a0794f24b34e9018d2dcaa3963049bc40a9c Mon Sep 17 00:00:00 2001 From: Sean Chen Date: Wed, 17 May 2023 07:50:21 -0500 Subject: [PATCH 14/26] Fix `rpc::EventSource` doc comment --- crates/relayer/src/event/source/rpc.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/relayer/src/event/source/rpc.rs b/crates/relayer/src/event/source/rpc.rs index 1bd80f0340..de7bbe840f 100644 --- a/crates/relayer/src/event/source/rpc.rs +++ b/crates/relayer/src/event/source/rpc.rs @@ -34,8 +34,7 @@ use self::extract::extract_events; pub type Result = core::result::Result; -/// Captures the mode in which events are fetched from a chain, either via WebSocket -/// or via RPC endpoint. +/// An RPC endpoint that serves as a source of events for a given chain. pub struct EventSource { /// Chain identifier chain_id: ChainId, From 829348d7aa16231b159289f7f26d687ff3c763eb Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 23 May 2023 16:55:27 +0200 Subject: [PATCH 15/26] Group source-specific settings together and rename types of event sources to `push` and `pull` --- ci/misbehaviour/config.toml | 4 +- ci/misbehaviour/config_fork.toml | 7 +- config.toml | 41 ++-- crates/relayer-cli/example | 199 ------------------ crates/relayer-cli/src/chain_registry.rs | 10 +- crates/relayer-cli/src/commands/listen.rs | 12 +- .../tests/fixtures/two_chains.toml | 8 +- crates/relayer-rest/tests/mock.rs | 2 +- crates/relayer/src/chain/cosmos.rs | 14 +- crates/relayer/src/config.rs | 40 ++-- .../config/fixtures/relayer_conf_example.toml | 4 +- .../relayer_conf_example_decoding_size.toml | 4 +- .../relayer_conf_example_fee_filter.toml | 6 +- .../configuration/configure-hermes.md | 8 +- .../files/hermes/local-chains/config.toml | 4 +- .../more-chains/config_with_filters.toml | 8 +- .../more-chains/config_without_filters.toml | 8 +- .../more-chains/hermes_second_instance.toml | 8 +- .../files/hermes/production/config.toml | 4 +- tools/test-framework/src/types/single/node.rs | 10 +- 20 files changed, 108 insertions(+), 293 deletions(-) delete mode 100644 crates/relayer-cli/example diff --git a/ci/misbehaviour/config.toml b/ci/misbehaviour/config.toml index 16b0e4f8d0..f5531d3f81 100644 --- a/ci/misbehaviour/config.toml +++ b/ci/misbehaviour/config.toml @@ -120,7 +120,7 @@ grpc_addr = 'http://127.0.0.1:9090' # Specify the WebSocket address and port where the chain WebSocket server # listens on. Required -websocket_addr = 'ws://127.0.0.1:26657/websocket' +event_source = { mode = 'push', url = 'ws://127.0.0.1:26657/websocket', batch_delay = '500ms' } # Specify the maximum amount of time (duration) that the RPC requests should # take before timing out. Default: 10s (10 seconds) @@ -301,7 +301,7 @@ memo_prefix = '' id = 'ibc-1' rpc_addr = 'http://127.0.0.1:26557' grpc_addr = 'http://127.0.0.1:9091' -websocket_addr = 'ws://127.0.0.1:26557/websocket' +event_source = { mode = 'push', url = 'ws://127.0.0.1:26557/websocket', batch_delay = '500ms' } rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' diff --git a/ci/misbehaviour/config_fork.toml b/ci/misbehaviour/config_fork.toml index 2996f08f99..d911b14798 100644 --- a/ci/misbehaviour/config_fork.toml +++ b/ci/misbehaviour/config_fork.toml @@ -118,9 +118,8 @@ rpc_addr = 'http://127.0.0.1:26657' # Specify the GRPC address and port where the chain GRPC server listens on. Required grpc_addr = 'http://127.0.0.1:9090' -# Specify the WebSocket address and port where the chain WebSocket server -# listens on. Required -websocket_addr = 'ws://127.0.0.1:26657/websocket' +# The type of event source to use for getting events from the chain. +event_source = { mode = 'push', url = 'ws://127.0.0.1:26557/websocket', batch_delay = '500ms' } # Specify the maximum amount of time (duration) that the RPC requests should # take before timing out. Default: 10s (10 seconds) @@ -301,7 +300,7 @@ memo_prefix = '' id = 'ibc-1' rpc_addr = 'http://127.0.0.1:26457' grpc_addr = 'http://127.0.0.1:9092' -websocket_addr = 'ws://127.0.0.1:26457/websocket' +event_source = { mode = 'push', url = 'ws://127.0.0.1:26457/websocket', batch_delay = '500ms' } rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' diff --git a/config.toml b/config.toml index 1152b578f3..f46df84e8e 100644 --- a/config.toml +++ b/config.toml @@ -122,31 +122,29 @@ rpc_addr = 'http://127.0.0.1:26657' # Specify the GRPC address and port where the chain GRPC server listens on. Required grpc_addr = 'http://127.0.0.1:9090' -# Specify the WebSocket address and port where the chain WebSocket server -# listens on. Required -websocket_addr = 'ws://127.0.0.1:26657/websocket' - # The type of event source to use for getting events from the chain. -# This setting can take one of two values: +# +# This setting can take two types of values, as an inline table: # -# - `websocket` for WebSocket -# - `rpc` for RPC via the `/block_results` endpoint +# a) Push: for receiving IBC events over WebSocket # -# Default: 'websocket' -event_source = 'websocket' - -# Delay until event batch is emitted if no NewBlock events have come yet. +# `{ mode = 'push', batch_delay = '500ms' }` # -# Note: Only applies when `event_source` is set to `websocket`. +# where # -# Default: 500ms -batch_delay = '500ms' - -# The interval at which to poll for blocks when the `event_source` is `rpc`. -# Note: Only applies when `event_source` is set to `rpc`. +# - `url` is the WebSocket URL to connect to. Required +# - `batch_delay` is the delay until event batch is +# emitted if no NewBlock events have come yet. Default: 500ms +# +# b) Poll: for polling for IBC events via the `/block_results` RPC endpoint +# +# `{ mode = 'poll', poll_interval = '1s' }` +# +# where +# +# - `poll_interval` is the interval at which to poll for blocks. Default: 1s # -# Default: 1 second -poll_interval = '1s' +event_source = { mode = 'push', url = 'ws://127.0.0.1:26657/websocket', batch_delay = '500ms' } # Specify the maximum amount of time (duration) that the RPC requests should # take before timing out. Default: 10s (10 seconds) @@ -331,11 +329,8 @@ memo_prefix = '' id = 'ibc-1' rpc_addr = 'http://127.0.0.1:26557' grpc_addr = 'http://127.0.0.1:9091' -websocket_addr = 'ws://127.0.0.1:26557/websocket' -event_source = 'websocket' -poll_interval = '1s' +event_source = { mode = 'push', url = 'ws://127.0.0.1:26557/websocket', batch_delay = '500ms' } rpc_timeout = '10s' -batch_delay = '500ms' account_prefix = 'cosmos' key_name = 'testkey' store_prefix = 'ibc' diff --git a/crates/relayer-cli/example b/crates/relayer-cli/example deleted file mode 100644 index 6323f50bc3..0000000000 --- a/crates/relayer-cli/example +++ /dev/null @@ -1,199 +0,0 @@ -[global] -log_level = 'info' -[mode.clients] -enabled = true -refresh = true -misbehaviour = true - -[mode.connections] -enabled = false - -[mode.channels] -enabled = false - -[mode.packets] -enabled = true -clear_interval = 100 -clear_on_start = true -tx_confirmation = false - -[rest] -enabled = false -host = '127.0.0.1' -port = 3000 - -[telemetry] -enabled = false -host = '127.0.0.1' -port = 3001 - -[[chains]] -id = 'cosmoshub-4' -type = 'CosmosSdk' -rpc_addr = 'https://rpc-cosmoshub.ecostake.com/' -websocket_addr = 'wss://rpc-cosmoshub.ecostake.com/websocket' -grpc_addr = 'https://grpc-cosmoshub-ia.notional.ventures/' -rpc_timeout = '10s' -account_prefix = 'cosmos' -key_name = 'a' -key_store_type = 'Test' -store_prefix = 'ibc' -default_gas = 100000 -max_gas = 400000 -gas_multiplier = 1.1 -max_msg_num = 30 -max_tx_size = 2097152 -clock_drift = '5s' -max_block_time = '30s' -memo_prefix = '' -proof_specs = ''' -[ - { - "leaf_spec": { - "hash": 1, - "prehash_key": 0, - "prehash_value": 1, - "length": 1, - "prefix": "AA==" - }, - "inner_spec": { - "child_order": [ - 0, - 1 - ], - "child_size": 33, - "min_prefix_length": 4, - "max_prefix_length": 12, - "empty_child": "", - "hash": 1 - }, - "max_depth": 0, - "min_depth": 0 - }, - { - "leaf_spec": { - "hash": 1, - "prehash_key": 0, - "prehash_value": 1, - "length": 1, - "prefix": "AA==" - }, - "inner_spec": { - "child_order": [ - 0, - 1 - ], - "child_size": 32, - "min_prefix_length": 1, - "max_prefix_length": 1, - "empty_child": "", - "hash": 1 - }, - "max_depth": 0, - "min_depth": 0 - } -]''' - -[chains.trust_threshold] -numerator = '1' -denominator = '3' - -[chains.gas_price] -price = 0.1 -denom = 'uatom' - -[chains.packet_filter] -policy = 'allow' -list = [[ - 'transfer', - 'channel-141', -]] - -[chains.address_type] -derivation = 'cosmos' - -[[chains]] -id = 'osmosis-1' -type = 'CosmosSdk' -rpc_addr = 'https://rpc-osmosis.ecostake.com/' -websocket_addr = 'wss://rpc-osmosis.ecostake.com/websocket' -grpc_addr = 'https://grpc-osmosis-ia.notional.ventures/' -rpc_timeout = '10s' -account_prefix = 'osmo' -key_name = 'b' -key_store_type = 'Test' -store_prefix = 'ibc' -default_gas = 100000 -max_gas = 400000 -gas_multiplier = 1.1 -max_msg_num = 30 -max_tx_size = 2097152 -clock_drift = '5s' -max_block_time = '30s' -memo_prefix = '' -proof_specs = ''' -[ - { - "leaf_spec": { - "hash": 1, - "prehash_key": 0, - "prehash_value": 1, - "length": 1, - "prefix": "AA==" - }, - "inner_spec": { - "child_order": [ - 0, - 1 - ], - "child_size": 33, - "min_prefix_length": 4, - "max_prefix_length": 12, - "empty_child": "", - "hash": 1 - }, - "max_depth": 0, - "min_depth": 0 - }, - { - "leaf_spec": { - "hash": 1, - "prehash_key": 0, - "prehash_value": 1, - "length": 1, - "prefix": "AA==" - }, - "inner_spec": { - "child_order": [ - 0, - 1 - ], - "child_size": 32, - "min_prefix_length": 1, - "max_prefix_length": 1, - "empty_child": "", - "hash": 1 - }, - "max_depth": 0, - "min_depth": 0 - } -]''' - -[chains.trust_threshold] -numerator = '1' -denominator = '3' - -[chains.gas_price] -price = 0.1 -denom = 'uosmo' - -[chains.packet_filter] -policy = 'allow' -list = [[ - 'transfer', - 'channel-0', -]] - -[chains.address_type] -derivation = 'cosmos' - diff --git a/crates/relayer-cli/src/chain_registry.rs b/crates/relayer-cli/src/chain_registry.rs index 458fb05521..2b3d9b8bc1 100644 --- a/crates/relayer-cli/src/chain_registry.rs +++ b/crates/relayer-cli/src/chain_registry.rs @@ -24,7 +24,7 @@ use ibc_chain_registry::querier::SimpleHermesRpcQuerier; use ibc_relayer::config::filter::{FilterPattern, PacketFilter}; use ibc_relayer::config::gas_multiplier::GasMultiplier; use ibc_relayer::config::types::{MaxMsgNum, MaxTxSize, Memo}; -use ibc_relayer::config::{default, AddressType, ChainConfig, EventSource, GasPrice}; +use ibc_relayer::config::{default, AddressType, ChainConfig, EventSourceMode, GasPrice}; use ibc_relayer::keyring::Store; const MAX_HEALTHY_QUERY_RETRIES: u8 = 5; @@ -119,12 +119,12 @@ where id: chain_data.chain_id, r#type: default::chain_type(), rpc_addr: rpc_data.rpc_address, - websocket_addr: websocket_address, grpc_addr: grpc_address, - event_source: EventSource::WebSocket, - poll_interval: default::poll_interval(), + event_source: EventSourceMode::Push { + url: websocket_address, + batch_delay: default::batch_delay(), + }, rpc_timeout: default::rpc_timeout(), - batch_delay: default::batch_delay(), genesis_restart: None, account_prefix: chain_data.bech32_prefix, key_name: String::new(), diff --git a/crates/relayer-cli/src/commands/listen.rs b/crates/relayer-cli/src/commands/listen.rs index d65ab4b22a..84a0c383e8 100644 --- a/crates/relayer-cli/src/commands/listen.rs +++ b/crates/relayer-cli/src/commands/listen.rs @@ -15,7 +15,9 @@ use tokio::runtime::Runtime as TokioRuntime; use tracing::{error, info, instrument}; use ibc_relayer::{ - chain::handle::Subscription, config::ChainConfig, event::source::websocket::EventSource, + chain::handle::Subscription, + config::{ChainConfig, EventSourceMode}, + event::source::websocket::EventSource, }; use ibc_relayer_types::{core::ics24_host::identifier::ChainId, events::IbcEvent}; @@ -142,11 +144,15 @@ fn subscribe( compat_mode: CompatMode, rt: Arc, ) -> eyre::Result { + let EventSourceMode::Push { url, batch_delay } = &chain_config.event_source else { + return Err(eyre!("unsupported event source mode, only 'push' is supported for listening to events")); + }; + let (mut event_source, tx_cmd) = EventSource::new( chain_config.id.clone(), - chain_config.websocket_addr.clone(), + url.clone(), compat_mode, - chain_config.batch_delay, + *batch_delay, rt, ) .map_err(|e| eyre!("could not initialize event source: {}", e))?; diff --git a/crates/relayer-cli/tests/fixtures/two_chains.toml b/crates/relayer-cli/tests/fixtures/two_chains.toml index 6ce815451c..db9358261d 100644 --- a/crates/relayer-cli/tests/fixtures/two_chains.toml +++ b/crates/relayer-cli/tests/fixtures/two_chains.toml @@ -24,7 +24,7 @@ tx_confirmation = true id = 'ibc-0' rpc_addr = 'http://127.0.0.1:26657' grpc_addr = 'http://127.0.0.1:9090' -websocket_addr = 'ws://127.0.0.1:26657/websocket' +event_source = { mode = 'push', url = 'ws://127.0.0.1:26657/websocket', batch_delay = '500ms' } rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' @@ -39,9 +39,9 @@ denominator = '3' [[chains]] id = 'ibc-1' -rpc_addr = 'http://127.0.0.1:26657' -grpc_addr = 'http://127.0.0.1:9090' -websocket_addr = 'ws://127.0.0.1:26657/websocket' +rpc_addr = 'http://127.0.0.1:26457' +grpc_addr = 'http://127.0.0.1:9091' +event_source = { mode = 'push', url = 'ws://127.0.0.1:26457/websocket', batch_delay = '500ms' } rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' diff --git a/crates/relayer-rest/tests/mock.rs b/crates/relayer-rest/tests/mock.rs index fa8db5e4a4..42e1c41113 100644 --- a/crates/relayer-rest/tests/mock.rs +++ b/crates/relayer-rest/tests/mock.rs @@ -98,7 +98,7 @@ const MOCK_CHAIN_CONFIG: &str = r#" id = 'mock-0' rpc_addr = 'http://127.0.0.1:26557' grpc_addr = 'http://127.0.0.1:9091' -websocket_addr = 'ws://127.0.0.1:26557/websocket' +event_source = { mode = 'push', url = 'ws://127.0.0.1:26557/websocket', batch_delay = '500ms' } rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' diff --git a/crates/relayer/src/chain/cosmos.rs b/crates/relayer/src/chain/cosmos.rs index 858bbd6dae..4a449cd8bf 100644 --- a/crates/relayer/src/chain/cosmos.rs +++ b/crates/relayer/src/chain/cosmos.rs @@ -294,20 +294,20 @@ impl CosmosSdkChain { } ); - use crate::config::EventSource as Mode; + use crate::config::EventSourceMode as Mode; - let (event_source, monitor_tx) = match self.config.event_source { - Mode::WebSocket => EventSource::websocket( + let (event_source, monitor_tx) = match &self.config.event_source { + Mode::Push { url, batch_delay } => EventSource::websocket( self.config.id.clone(), - self.config.websocket_addr.clone(), + url.clone(), self.compat_mode, - self.config.batch_delay, + *batch_delay, self.rt.clone(), ), - Mode::Rpc => EventSource::rpc( + Mode::Pull { poll_interval } => EventSource::rpc( self.config.id.clone(), self.rpc_client.clone(), - self.config.poll_interval, + *poll_interval, self.rt.clone(), ), } diff --git a/crates/relayer/src/config.rs b/crates/relayer/src/config.rs index 7ca73d41e0..929294d71b 100644 --- a/crates/relayer/src/config.rs +++ b/crates/relayer/src/config.rs @@ -450,34 +450,48 @@ pub struct GenesisRestart { pub archive_addr: Url, } -#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, Default)] -#[serde(rename_all = "lowercase")] -pub enum EventSource { +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +#[serde(tag = "mode", rename_all = "lowercase")] +pub enum EventSourceMode { /// Push-based event source, via WebSocket - #[default] - WebSocket, + Push { + /// The WebSocket URL to connect to + url: WebSocketClientUrl, + + /// Maximum amount of time to wait for a NewBlock event before emitting the event batch + #[serde(default = "default::batch_delay", with = "humantime_serde")] + batch_delay: Duration, + }, /// Pull-based event source, via RPC /block_results - Rpc, + Pull { + /// The polling interval + #[serde(default = "default::poll_interval", with = "humantime_serde")] + poll_interval: Duration, + }, } #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct ChainConfig { + /// The chain's network identifier pub id: ChainId, + + /// The chain type #[serde(default = "default::chain_type")] pub r#type: ChainType, + + /// The RPC URL to connect to pub rpc_addr: Url, - pub websocket_addr: WebSocketClientUrl, + + /// The gRPC URL to connect to pub grpc_addr: Url, - #[serde(default)] - pub event_source: EventSource, - #[serde(default = "default::poll_interval", with = "humantime_serde")] - pub poll_interval: Duration, + + /// The type of event source and associated settings + pub event_source: EventSourceMode, + #[serde(default = "default::rpc_timeout", with = "humantime_serde")] pub rpc_timeout: Duration, - #[serde(default = "default::batch_delay", with = "humantime_serde")] - pub batch_delay: Duration, pub account_prefix: String, pub key_name: String, #[serde(default)] diff --git a/crates/relayer/tests/config/fixtures/relayer_conf_example.toml b/crates/relayer/tests/config/fixtures/relayer_conf_example.toml index 8d27b1357d..ea41a2888f 100644 --- a/crates/relayer/tests/config/fixtures/relayer_conf_example.toml +++ b/crates/relayer/tests/config/fixtures/relayer_conf_example.toml @@ -24,7 +24,7 @@ tx_confirmation = true id = 'chain_A' rpc_addr = 'http://127.0.0.1:26657' grpc_addr = 'http://127.0.0.1:9090' -websocket_addr = 'ws://localhost:26657/websocket' +event_source = { mode = 'push', url = 'ws://localhost:26657/websocket', batch_delay = '500ms' } rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' @@ -49,7 +49,7 @@ list = [ id = 'chain_B' rpc_addr = 'http://127.0.0.1:26557' grpc_addr = 'http://127.0.0.1:9090' -websocket_addr = 'ws://localhost:26557/websocket' +event_source = { mode = 'push', url = 'ws://localhost:26557/websocket', batch_delay = '500ms' } rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' diff --git a/crates/relayer/tests/config/fixtures/relayer_conf_example_decoding_size.toml b/crates/relayer/tests/config/fixtures/relayer_conf_example_decoding_size.toml index e6c51b578a..6df007f2f7 100644 --- a/crates/relayer/tests/config/fixtures/relayer_conf_example_decoding_size.toml +++ b/crates/relayer/tests/config/fixtures/relayer_conf_example_decoding_size.toml @@ -24,7 +24,7 @@ tx_confirmation = true id = 'chain_A' rpc_addr = 'http://127.0.0.1:26657' grpc_addr = 'http://127.0.0.1:9090' -websocket_addr = 'ws://localhost:26657/websocket' +event_source = { mode = 'push', url = 'ws://localhost:26657/websocket', batch_delay = '500ms' } rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' @@ -43,7 +43,7 @@ address_type = { derivation = 'cosmos' } id = 'chain_B' rpc_addr = 'http://127.0.0.1:26557' grpc_addr = 'http://127.0.0.1:9090' -websocket_addr = 'ws://localhost:26557/websocket' +event_source = { mode = 'push', url = 'ws://localhost:26557/websocket', batch_delay = '500ms' } rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' diff --git a/crates/relayer/tests/config/fixtures/relayer_conf_example_fee_filter.toml b/crates/relayer/tests/config/fixtures/relayer_conf_example_fee_filter.toml index ab4359c27f..60a0cc7e0c 100644 --- a/crates/relayer/tests/config/fixtures/relayer_conf_example_fee_filter.toml +++ b/crates/relayer/tests/config/fixtures/relayer_conf_example_fee_filter.toml @@ -24,7 +24,7 @@ tx_confirmation = true id = 'chain_A' rpc_addr = 'http://127.0.0.1:26657' grpc_addr = 'http://127.0.0.1:9090' -websocket_addr = 'ws://localhost:26657/websocket' +event_source = { mode = 'push', url = 'ws://127.0.0.1:26657/websocket', batch_delay = '500ms' } rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' @@ -55,7 +55,7 @@ recv = [ { amount = 0 }] id = 'chain_B' rpc_addr = 'http://127.0.0.1:26557' grpc_addr = 'http://127.0.0.1:9090' -websocket_addr = 'ws://localhost:26557/websocket' +event_source = { mode = 'push', url = 'ws://127.0.0.1:26557/websocket', batch_delay = '500ms' } rpc_timeout = '10s' account_prefix = 'cosmos' key_name = 'testkey' @@ -64,4 +64,4 @@ gas_price = { price = 0.001, denom = 'stake' } clock_drift = '5s' trusting_period = '14days' trust_threshold = { numerator = '1', denominator = '3' } -address_type = { derivation = 'ethermint', proto_type = { pk_type = '/injective.crypto.v1beta1.ethsecp256k1.PubKey' } } \ No newline at end of file +address_type = { derivation = 'ethermint', proto_type = { pk_type = '/injective.crypto.v1beta1.ethsecp256k1.PubKey' } } diff --git a/guide/src/documentation/configuration/configure-hermes.md b/guide/src/documentation/configuration/configure-hermes.md index 078214986d..eeba55feb7 100644 --- a/guide/src/documentation/configuration/configure-hermes.md +++ b/guide/src/documentation/configuration/configure-hermes.md @@ -49,13 +49,13 @@ please refer to the [Keys](../commands/keys/index.md) sections in order to learn Hermes supports connection via TLS for use-cases such as connecting from behind a proxy or a load balancer. In order to enable this, you'll want to set the -`rpc_addr`, `grpc_addr`, or `websocket_addr` parameters to specify a TLS +`rpc_addr`, `grpc_addr`, or `event_source` parameters to specify a TLS connection via HTTPS using the following scheme (note that the port number 443 is just used for example): ``` rpc_addr = 'https://domain.com:443' grpc_addr = 'https://domain.com:443' -websocket_addr = 'wss://domain.com:443/websocket' +event_source = { mode = 'push', url = 'wss://domain.com:443/websocket', batch_delay = '500ms' } ``` ## Support for Interchain Accounts @@ -102,7 +102,7 @@ list = [ ## Connecting to a full node protected by HTTP Basic Authentication To connect to a full node protected by [HTTP Basic Authentication][http-basic-auth], -specify the username and password in the `rpc_addr`, `grpc_addr` and `websocket_addr` settings +specify the username and password in the `rpc_addr`, `grpc_addr` and `event_source` settings under the chain configuration in `config.toml`. Here is an example with username `hello` and password `world`, assuming the RPC, WebSocket and gRPC servers @@ -116,7 +116,7 @@ id = 'my-chain-0' rpc_addr = 'https://hello:world@mydomain.com:26657' grpc_addr = 'https://hello:world@mydomain.com:9090' -websocket_addr = 'wss://hello:world@mydomain.com:26657/websocket' +event_source = { mode = 'push', url = 'wss://hello:world@mydomain.com:26657/websocket', batch_delay = '500ms' } # ... ``` diff --git a/guide/src/templates/files/hermes/local-chains/config.toml b/guide/src/templates/files/hermes/local-chains/config.toml index 57f8b2f5df..383cdf4467 100644 --- a/guide/src/templates/files/hermes/local-chains/config.toml +++ b/guide/src/templates/files/hermes/local-chains/config.toml @@ -29,7 +29,7 @@ port = 3001 id = 'ibc-0' rpc_addr = 'http://localhost:27030' grpc_addr = 'http://localhost:27032' -websocket_addr = 'ws://localhost:27030/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27030/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet' @@ -44,7 +44,7 @@ trust_threshold = { numerator = '1', denominator = '3' } id = 'ibc-1' rpc_addr = 'http://localhost:27040' grpc_addr = 'http://localhost:27042' -websocket_addr = 'ws://localhost:27040/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27040/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet' diff --git a/guide/src/templates/files/hermes/more-chains/config_with_filters.toml b/guide/src/templates/files/hermes/more-chains/config_with_filters.toml index a1adc2f134..59899f8b91 100644 --- a/guide/src/templates/files/hermes/more-chains/config_with_filters.toml +++ b/guide/src/templates/files/hermes/more-chains/config_with_filters.toml @@ -29,7 +29,7 @@ port = 3001 id = 'ibc-0' rpc_addr = 'http://localhost:27050' grpc_addr = 'http://localhost:27052' -websocket_addr = 'ws://localhost:27050/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27050/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet' @@ -51,7 +51,7 @@ list = [ id = 'ibc-1' rpc_addr = 'http://localhost:27060' grpc_addr = 'http://localhost:27062' -websocket_addr = 'ws://localhost:27060/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27060/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet' @@ -74,7 +74,7 @@ list = [ id = 'ibc-2' rpc_addr = 'http://localhost:27070' grpc_addr = 'http://localhost:27072' -websocket_addr = 'ws://localhost:27070/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27070/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet' @@ -96,7 +96,7 @@ list = [ id = 'ibc-3' rpc_addr = 'http://localhost:27080' grpc_addr = 'http://localhost:27082' -websocket_addr = 'ws://localhost:27080/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27080/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet' diff --git a/guide/src/templates/files/hermes/more-chains/config_without_filters.toml b/guide/src/templates/files/hermes/more-chains/config_without_filters.toml index 11a74570de..fff4a5100d 100644 --- a/guide/src/templates/files/hermes/more-chains/config_without_filters.toml +++ b/guide/src/templates/files/hermes/more-chains/config_without_filters.toml @@ -29,7 +29,7 @@ port = 3001 id = 'ibc-0' rpc_addr = 'http://localhost:27050' grpc_addr = 'http://localhost:27052' -websocket_addr = 'ws://localhost:27050/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27050/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet' @@ -44,7 +44,7 @@ trust_threshold = { numerator = '1', denominator = '3' } id = 'ibc-1' rpc_addr = 'http://localhost:27060' grpc_addr = 'http://localhost:27062' -websocket_addr = 'ws://localhost:27060/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27060/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet' @@ -59,7 +59,7 @@ trust_threshold = { numerator = '1', denominator = '3' } id = 'ibc-2' rpc_addr = 'http://localhost:27070' grpc_addr = 'http://localhost:27072' -websocket_addr = 'ws://localhost:27070/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27070/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet' @@ -74,7 +74,7 @@ trust_threshold = { numerator = '1', denominator = '3' } id = 'ibc-3' rpc_addr = 'http://localhost:27080' grpc_addr = 'http://localhost:27082' -websocket_addr = 'ws://localhost:27080/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27080/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet' diff --git a/guide/src/templates/files/hermes/more-chains/hermes_second_instance.toml b/guide/src/templates/files/hermes/more-chains/hermes_second_instance.toml index be1f559908..3d50e4836a 100644 --- a/guide/src/templates/files/hermes/more-chains/hermes_second_instance.toml +++ b/guide/src/templates/files/hermes/more-chains/hermes_second_instance.toml @@ -29,7 +29,7 @@ port = 3002 id = 'ibc-0' rpc_addr = 'http://localhost:27050' grpc_addr = 'http://localhost:27052' -websocket_addr = 'ws://localhost:27050/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27050/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet1' @@ -50,7 +50,7 @@ list = [ id = 'ibc-1' rpc_addr = 'http://localhost:27060' grpc_addr = 'http://localhost:27062' -websocket_addr = 'ws://localhost:27060/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27060/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet1' @@ -72,7 +72,7 @@ list = [ id = 'ibc-2' rpc_addr = 'http://localhost:27070' grpc_addr = 'http://localhost:27072' -websocket_addr = 'ws://localhost:27070/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27070/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet1' @@ -93,7 +93,7 @@ list = [ id = 'ibc-3' rpc_addr = 'http://localhost:27080' grpc_addr = 'http://localhost:27082' -websocket_addr = 'ws://localhost:27080/websocket' +event_source = { mode = 'push', url = 'ws://localhost:27080/websocket', batch_delay = '500ms' } rpc_timeout = '15s' account_prefix = 'cosmos' key_name = 'wallet1' diff --git a/guide/src/templates/files/hermes/production/config.toml b/guide/src/templates/files/hermes/production/config.toml index bc6ad8265b..90f4a1cca8 100644 --- a/guide/src/templates/files/hermes/production/config.toml +++ b/guide/src/templates/files/hermes/production/config.toml @@ -31,7 +31,7 @@ port = 3001 id = 'cosmoshub-4' type = 'CosmosSdk' rpc_addr = 'https://rpc.cosmoshub.strange.love/' -websocket_addr = 'wss://rpc.cosmoshub.strange.love/websocket' +event_source = { mode = 'push', url = 'wss://rpc.cosmoshub.strange.love/websocket', batch_delay = '500ms' } grpc_addr = 'https://grpc-cosmoshub-ia.notional.ventures/' rpc_timeout = '10s' account_prefix = 'cosmos' @@ -70,7 +70,7 @@ derivation = 'cosmos' id = 'osmosis-1' type = 'CosmosSdk' rpc_addr = 'https://rpc.osmosis.interbloc.org/' -websocket_addr = 'wss://rpc.osmosis.interbloc.org/websocket' +event_source = { mode = 'push', url = 'wss://rpc.osmosis.interbloc.org/websocket', batch_delay = '500ms' } grpc_addr = 'https://grpc-osmosis-ia.notional.ventures/' rpc_timeout = '10s' account_prefix = 'osmo' diff --git a/tools/test-framework/src/types/single/node.rs b/tools/test-framework/src/types/single/node.rs index a2beaebd6e..a53fa1a014 100644 --- a/tools/test-framework/src/types/single/node.rs +++ b/tools/test-framework/src/types/single/node.rs @@ -138,12 +138,12 @@ impl FullNode { id: self.chain_driver.chain_id.clone(), r#type: ChainType::CosmosSdk, rpc_addr: Url::from_str(&self.chain_driver.rpc_address())?, - websocket_addr: WebSocketClientUrl::from_str(&self.chain_driver.websocket_address())?, grpc_addr: Url::from_str(&self.chain_driver.grpc_address())?, - event_source: config::EventSource::WebSocket, - poll_interval: config::default::poll_interval(), - batch_delay: ibc_relayer::config::default::batch_delay(), - rpc_timeout: ibc_relayer::config::default::rpc_timeout(), + event_source: config::EventSourceMode::Push { + url: WebSocketClientUrl::from_str(&self.chain_driver.websocket_address())?, + batch_delay: config::default::batch_delay(), + }, + rpc_timeout: config::default::rpc_timeout(), genesis_restart: None, account_prefix: self.chain_driver.account_prefix.clone(), key_name: self.wallets.relayer.id.0.clone(), From cadbcb9babf7b2165bccc15e385e55c9e02eb9d6 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 23 May 2023 17:01:04 +0200 Subject: [PATCH 16/26] Formatting --- crates/relayer/src/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/relayer/src/config.rs b/crates/relayer/src/config.rs index 56f2bc742c..2093fff715 100644 --- a/crates/relayer/src/config.rs +++ b/crates/relayer/src/config.rs @@ -723,4 +723,3 @@ mod tests { assert_eq!(expected, parsed); } } - From 081678e5fa175bea262ba635b1e08a2abfcf2a75 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 23 May 2023 17:57:48 +0200 Subject: [PATCH 17/26] Fix leftover merge conflict --- config.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config.toml b/config.toml index 2e6696dca7..d45584cd57 100644 --- a/config.toml +++ b/config.toml @@ -348,11 +348,7 @@ rpc_addr = 'http://127.0.0.1:26557' grpc_addr = 'http://127.0.0.1:9091' event_source = { mode = 'push', url = 'ws://127.0.0.1:26557/websocket', batch_delay = '500ms' } rpc_timeout = '10s' -<<<<<<< HEAD -======= -batch_delay = '500ms' trusted_node = false ->>>>>>> master account_prefix = 'cosmos' key_name = 'testkey' store_prefix = 'ibc' From 2f6890653245bdcd1b1a9b098222e3aee358a1dd Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Mon, 29 May 2023 17:05:30 +0200 Subject: [PATCH 18/26] Delete config_fork.toml Signed-off-by: Romain Ruetschi --- ci/misbehaviour/config_fork.toml | 318 ------------------------------- 1 file changed, 318 deletions(-) delete mode 100644 ci/misbehaviour/config_fork.toml diff --git a/ci/misbehaviour/config_fork.toml b/ci/misbehaviour/config_fork.toml deleted file mode 100644 index d911b14798..0000000000 --- a/ci/misbehaviour/config_fork.toml +++ /dev/null @@ -1,318 +0,0 @@ -# The global section has parameters that apply globally to the relayer operation. -[global] - -# Specify the verbosity for the relayer logging output. Default: 'info' -# Valid options are 'error', 'warn', 'info', 'debug', 'trace'. -log_level = 'debug' - - -# Specify the mode to be used by the relayer. [Required] -[mode] - -# Specify the client mode. -[mode.clients] - -# Whether or not to enable the client workers. [Required] -enabled = true - -# Whether or not to enable periodic refresh of clients. [Default: true] -# This feature only applies to clients that underlie an open channel. -# For Tendermint clients, the frequency at which Hermes refreshes them is 2/3 of their -# trusting period (e.g., refresh every ~9 days if the trusting period is 14 days). -# Note: Even if this is disabled, clients will be refreshed automatically if -# there is activity on a connection or channel they are involved with. -refresh = true - -# Whether or not to enable misbehaviour detection for clients. [Default: false] -misbehaviour = true - -# Specify the connections mode. -[mode.connections] - -# Whether or not to enable the connection workers for handshake completion. [Required] -enabled = true - -# Specify the channels mode. -[mode.channels] - -# Whether or not to enable the channel workers for handshake completion. [Required] -enabled = true - -# Specify the packets mode. -[mode.packets] - -# Whether or not to enable the packet workers. [Required] -enabled = true - -# Parametrize the periodic packet clearing feature. -# Interval (in number of blocks) at which pending packets -# should be periodically cleared. A value of '0' will disable -# periodic packet clearing. [Default: 100] -clear_interval = 100 - -# Whether or not to clear packets on start. [Default: true] -clear_on_start = true - -# Toggle the transaction confirmation mechanism. -# The tx confirmation mechanism periodically queries the `/tx_search` RPC -# endpoint to check that previously-submitted transactions -# (to any chain in this config file) have been successfully delivered. -# If they have not been, and `clear_interval = 0`, then those packets are -# queued up for re-submission. -# If set to `false`, the following telemetry metrics will be disabled: -# `acknowledgment_packets_confirmed`, `receive_packets_confirmed` and `timeout_packets_confirmed`. -# [Default: false] -tx_confirmation = false - -# Auto register the counterparty payee on a destination chain to -# the relayer's address on the source chain. This can be used -# for simple configuration of the relayer to receive fees for -# relaying RecvPacket on fee-enabled channels. -# For more complex configuration, turn this off and use the CLI -# to manually register the payee addresses. -# [Default: false] -auto_register_counterparty_payee = false - -# The REST section defines parameters for Hermes' built-in RESTful API. -# https://hermes.informal.systems/rest.html -[rest] - -# Whether or not to enable the REST service. Default: false -enabled = false - -# Specify the IPv4/6 host over which the built-in HTTP server will serve the RESTful -# API requests. Default: 127.0.0.1 -host = '127.0.0.1' - -# Specify the port over which the built-in HTTP server will serve the restful API -# requests. Default: 3000 -port = 3000 - - -# The telemetry section defines parameters for Hermes' built-in telemetry capabilities. -# https://hermes.informal.systems/telemetry.html -[telemetry] - -# Whether or not to enable the telemetry service. Default: false -enabled = false - -# Specify the IPv4/6 host over which the built-in HTTP server will serve the metrics -# gathered by the telemetry service. Default: 127.0.0.1 -host = '127.0.0.1' - -# Specify the port over which the built-in HTTP server will serve the metrics gathered -# by the telemetry service. Default: 3001 -port = 3001 - - -# A chains section includes parameters related to a chain and the full node to which -# the relayer can send transactions and queries. -[[chains]] - -# Specify the chain ID. Required -id = 'ibc-0' - -# Specify the RPC address and port where the chain RPC server listens on. Required -rpc_addr = 'http://127.0.0.1:26657' - -# Specify the GRPC address and port where the chain GRPC server listens on. Required -grpc_addr = 'http://127.0.0.1:9090' - -# The type of event source to use for getting events from the chain. -event_source = { mode = 'push', url = 'ws://127.0.0.1:26557/websocket', batch_delay = '500ms' } - -# Specify the maximum amount of time (duration) that the RPC requests should -# take before timing out. Default: 10s (10 seconds) -# Note: Hermes uses this parameter _only_ in `start` mode; for all other CLIs, -# Hermes uses a large preconfigured timeout (on the order of minutes). -rpc_timeout = '10s' - -# Specify the prefix used by the chain. Required -account_prefix = 'cosmos' - -# Specify the name of the private key to use for signing transactions. Required -# See the Adding Keys chapter for more information about managing signing keys: -# https://hermes.informal.systems/commands/keys/index.html#adding-keys -key_name = 'testkey' - -# Specify the address type which determines: -# 1) address derivation; -# 2) how to retrieve and decode accounts and pubkeys; -# 3) the message signing method. -# The current configuration options are for Cosmos SDK and Ethermint. -# -# Example configuration for chains based on Ethermint library: -# -# address_type = { derivation = 'ethermint', proto_type = { pk_type = '/ethermint.crypto.v1.ethsecp256k1.PubKey' } } -# -# Default: { derivation = 'cosmos' }, i.e. address derivation as in Cosmos SDK. -# Warning: This is an advanced feature! Modify with caution. -address_type = { derivation = 'cosmos' } - -# Specify the store prefix used by the on-chain IBC modules. Required -# Recommended value for Cosmos SDK: 'ibc' -store_prefix = 'ibc' - -# Gas Parameters -# -# The term 'gas' is used to denote the amount of computation needed to execute -# and validate a transaction on-chain. It can be thought of as fuel that gets -# spent in order to power the on-chain execution of a transaction. -# -# Hermes attempts to simulate how much gas a transaction will expend on its -# target chain. From that, it calculates the cost of that gas by multiplying the -# amount of estimated gas by the `gas_multiplier` and the `gas_price` -# (estimated gas * `gas_multiplier` * `gas_price`) in order to compute the -# total fee to be deducted from the relayer's wallet. -# -# The `simulate_tx` operation does not always correctly estimate the appropriate -# amount of gas that a transaction requires. In those cases when the operation -# fails, Hermes will attempt to submit the transaction using the specified -# `default_gas` and `max_gas` parameters. In the case that a transaction would -# require more than `max_gas`, it doesn't get submitted and a -# `TxSimulateGasEstimateExceeded` error is returned. - -# Specify the default amount of gas to be used in case the tx simulation fails, -# and Hermes cannot estimate the amount of gas needed. -# Default: 100 000 -default_gas = 100000 - -# Specify the maximum amount of gas to be used as the gas limit for a transaction. -# If `default_gas` is unspecified, then `max_gas` will be used as `default_gas`. -# Default: 400 000 -max_gas = 400000 - -# Specify the price per gas used of the fee to submit a transaction and -# the denomination of the fee. -# -# The specified gas price should always be greater or equal to the `min-gas-price` -# configured on the chain. This is to ensure that at least some minimal price is -# paid for each unit of gas per transaction. -# -# Required -gas_price = { price = 0.001, denom = 'stake' } - -# Multiply this amount with the gas estimate, used to compute the fee -# and account for potential estimation error. -# -# The purpose of multiplying by `gas_multiplier` is to provide a bit of a buffer -# to catch some of the cases when the gas estimation calculation is on the low -# end. -# -# Example: With this setting set to 1.1, then if the estimated gas -# is 80_000, then gas used to compute the fee will be adjusted to -# 80_000 * 1.1 = 88_000. -# -# Default: 1.1, ie. the gas is increased by 10% -# Minimum value: 1.0 -gas_multiplier = 1.1 - -# Specify how many IBC messages at most to include in a single transaction. -# Default: 30 -max_msg_num = 30 - -# Specify the maximum size, in bytes, of each transaction that Hermes will submit. -# Default: 2097152 (2 MiB) -max_tx_size = 2097152 - -# Specify the maximum amount of time to tolerate a clock drift. -# The clock drift parameter defines how much new (untrusted) header's time -# can drift into the future. Default: 5s -clock_drift = '5s' - -# Specify the maximum time per block for this chain. -# The block time together with the clock drift are added to the source drift to estimate -# the maximum clock drift when creating a client on this chain. Default: 30s -# For cosmos-SDK chains a good approximation is `timeout_propose` + `timeout_commit` -# Note: This MUST be the same as the `max_expected_time_per_block` genesis parameter for Tendermint chains. -max_block_time = '30s' - -# Specify the amount of time to be used as the light client trusting period. -# It should be significantly less than the unbonding period -# (e.g. unbonding period = 3 weeks, trusting period = 2 weeks). -# Default: 2/3 of the `unbonding period` for Cosmos SDK chains -trusting_period = '14days' - -# Specify the trust threshold for the light client, ie. the minimum fraction of validators -# which must overlap across two blocks during light client verification. -# Default: { numerator = '2', denominator = '3' }, ie. 2/3. -# Warning: This is an advanced feature! Modify with caution. -trust_threshold = { numerator = '2', denominator = '3' } - -# Specify a string that Hermes will use as a memo for each transaction it submits -# to this chain. The string is limited to 50 characters. Default: '' (empty). -# Note: Hermes will append to the string defined here additional -# operational debugging information, e.g., relayer build version. -memo_prefix = '' - -# This section specifies the filters for policy based relaying. -# -# Default: no policy / filters, allow all packets on all channels. -# -# Only packet filtering based on channel identifier can be specified. -# A channel filter has two fields: -# 1. `policy` - one of two types are supported: -# - 'allow': permit relaying _only on_ the port/channel id in the list below, -# - 'deny': permit relaying on any channel _except for_ the list below. -# 2. `list` - the list of channels specified by the port and channel identifiers. -# Optionally, each element may also contains wildcards, for eg. 'ica*' -# to match all identifiers starting with 'ica' or '*' to match all identifiers. -# -# Example configuration of a channel filter, only allowing packet relaying on -# channel with port ID 'transfer' and channel ID 'channel-0', as well as on -# all ICA channels. -# -# [chains.packet_filter] -# policy = 'allow' -# list = [ -# ['ica*', '*'], -# ['transfer', 'channel-0'], -# ] - -# This section specifies the filters for incentivized packet relaying. -# Default: no filters, will relay all packets even if they -# are not incentivized. -# -# It is possible to specify the channel or use wildcards for the -# channels. -# The only fee which can be parametrized is the `recv_fee`. -# -# Example configuration of a filter which will only relay incentivized -# packets, with no regards for channel and amount. -# -# [chains.packet_filter.min_fees.'*'] -# recv = [ { amount = 0 } ] -# -# Example configuration of a filter which will only relay packets if they are -# from the channel 'channel-0', and they have a `recv_fee` of at least 20 stake -# or 10 uatom. -# -# [chains.packet_filter.min_fees.'channel-0'] -# recv = [ { amount = 20, denom = 'stake' }, { amount = 10, denom = 'uatom' } ] - -# Specify that the transaction fees should be payed from this fee granter's account. -# Optional. If unspecified (the default behavior), then no fee granter is used, and -# the account specified in `key_name` will pay the tx fees for all transactions -# submitted to this chain. -# fee_granter = '' - -[[chains]] -id = 'ibc-1' -rpc_addr = 'http://127.0.0.1:26457' -grpc_addr = 'http://127.0.0.1:9092' -event_source = { mode = 'push', url = 'ws://127.0.0.1:26457/websocket', batch_delay = '500ms' } -rpc_timeout = '10s' -account_prefix = 'cosmos' -key_name = 'testkey' -store_prefix = 'ibc' -default_gas = 100000 -max_gas = 400000 -gas_price = { price = 0.001, denom = 'stake' } -gas_multiplier = 1.1 -max_msg_num = 30 -max_tx_size = 2097152 -clock_drift = '5s' -max_block_time = '30s' -trusting_period = '14days' -trust_threshold = { numerator = '1', denominator = '3' } -address_type = { derivation = 'cosmos' } From 59107b091721c33967989e88d9d1eea8a2cae018 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Mon, 29 May 2023 17:05:52 +0200 Subject: [PATCH 19/26] Revert "Delete config_fork.toml" This reverts commit 2f6890653245bdcd1b1a9b098222e3aee358a1dd. --- ci/misbehaviour/config_fork.toml | 318 +++++++++++++++++++++++++++++++ 1 file changed, 318 insertions(+) create mode 100644 ci/misbehaviour/config_fork.toml diff --git a/ci/misbehaviour/config_fork.toml b/ci/misbehaviour/config_fork.toml new file mode 100644 index 0000000000..d911b14798 --- /dev/null +++ b/ci/misbehaviour/config_fork.toml @@ -0,0 +1,318 @@ +# The global section has parameters that apply globally to the relayer operation. +[global] + +# Specify the verbosity for the relayer logging output. Default: 'info' +# Valid options are 'error', 'warn', 'info', 'debug', 'trace'. +log_level = 'debug' + + +# Specify the mode to be used by the relayer. [Required] +[mode] + +# Specify the client mode. +[mode.clients] + +# Whether or not to enable the client workers. [Required] +enabled = true + +# Whether or not to enable periodic refresh of clients. [Default: true] +# This feature only applies to clients that underlie an open channel. +# For Tendermint clients, the frequency at which Hermes refreshes them is 2/3 of their +# trusting period (e.g., refresh every ~9 days if the trusting period is 14 days). +# Note: Even if this is disabled, clients will be refreshed automatically if +# there is activity on a connection or channel they are involved with. +refresh = true + +# Whether or not to enable misbehaviour detection for clients. [Default: false] +misbehaviour = true + +# Specify the connections mode. +[mode.connections] + +# Whether or not to enable the connection workers for handshake completion. [Required] +enabled = true + +# Specify the channels mode. +[mode.channels] + +# Whether or not to enable the channel workers for handshake completion. [Required] +enabled = true + +# Specify the packets mode. +[mode.packets] + +# Whether or not to enable the packet workers. [Required] +enabled = true + +# Parametrize the periodic packet clearing feature. +# Interval (in number of blocks) at which pending packets +# should be periodically cleared. A value of '0' will disable +# periodic packet clearing. [Default: 100] +clear_interval = 100 + +# Whether or not to clear packets on start. [Default: true] +clear_on_start = true + +# Toggle the transaction confirmation mechanism. +# The tx confirmation mechanism periodically queries the `/tx_search` RPC +# endpoint to check that previously-submitted transactions +# (to any chain in this config file) have been successfully delivered. +# If they have not been, and `clear_interval = 0`, then those packets are +# queued up for re-submission. +# If set to `false`, the following telemetry metrics will be disabled: +# `acknowledgment_packets_confirmed`, `receive_packets_confirmed` and `timeout_packets_confirmed`. +# [Default: false] +tx_confirmation = false + +# Auto register the counterparty payee on a destination chain to +# the relayer's address on the source chain. This can be used +# for simple configuration of the relayer to receive fees for +# relaying RecvPacket on fee-enabled channels. +# For more complex configuration, turn this off and use the CLI +# to manually register the payee addresses. +# [Default: false] +auto_register_counterparty_payee = false + +# The REST section defines parameters for Hermes' built-in RESTful API. +# https://hermes.informal.systems/rest.html +[rest] + +# Whether or not to enable the REST service. Default: false +enabled = false + +# Specify the IPv4/6 host over which the built-in HTTP server will serve the RESTful +# API requests. Default: 127.0.0.1 +host = '127.0.0.1' + +# Specify the port over which the built-in HTTP server will serve the restful API +# requests. Default: 3000 +port = 3000 + + +# The telemetry section defines parameters for Hermes' built-in telemetry capabilities. +# https://hermes.informal.systems/telemetry.html +[telemetry] + +# Whether or not to enable the telemetry service. Default: false +enabled = false + +# Specify the IPv4/6 host over which the built-in HTTP server will serve the metrics +# gathered by the telemetry service. Default: 127.0.0.1 +host = '127.0.0.1' + +# Specify the port over which the built-in HTTP server will serve the metrics gathered +# by the telemetry service. Default: 3001 +port = 3001 + + +# A chains section includes parameters related to a chain and the full node to which +# the relayer can send transactions and queries. +[[chains]] + +# Specify the chain ID. Required +id = 'ibc-0' + +# Specify the RPC address and port where the chain RPC server listens on. Required +rpc_addr = 'http://127.0.0.1:26657' + +# Specify the GRPC address and port where the chain GRPC server listens on. Required +grpc_addr = 'http://127.0.0.1:9090' + +# The type of event source to use for getting events from the chain. +event_source = { mode = 'push', url = 'ws://127.0.0.1:26557/websocket', batch_delay = '500ms' } + +# Specify the maximum amount of time (duration) that the RPC requests should +# take before timing out. Default: 10s (10 seconds) +# Note: Hermes uses this parameter _only_ in `start` mode; for all other CLIs, +# Hermes uses a large preconfigured timeout (on the order of minutes). +rpc_timeout = '10s' + +# Specify the prefix used by the chain. Required +account_prefix = 'cosmos' + +# Specify the name of the private key to use for signing transactions. Required +# See the Adding Keys chapter for more information about managing signing keys: +# https://hermes.informal.systems/commands/keys/index.html#adding-keys +key_name = 'testkey' + +# Specify the address type which determines: +# 1) address derivation; +# 2) how to retrieve and decode accounts and pubkeys; +# 3) the message signing method. +# The current configuration options are for Cosmos SDK and Ethermint. +# +# Example configuration for chains based on Ethermint library: +# +# address_type = { derivation = 'ethermint', proto_type = { pk_type = '/ethermint.crypto.v1.ethsecp256k1.PubKey' } } +# +# Default: { derivation = 'cosmos' }, i.e. address derivation as in Cosmos SDK. +# Warning: This is an advanced feature! Modify with caution. +address_type = { derivation = 'cosmos' } + +# Specify the store prefix used by the on-chain IBC modules. Required +# Recommended value for Cosmos SDK: 'ibc' +store_prefix = 'ibc' + +# Gas Parameters +# +# The term 'gas' is used to denote the amount of computation needed to execute +# and validate a transaction on-chain. It can be thought of as fuel that gets +# spent in order to power the on-chain execution of a transaction. +# +# Hermes attempts to simulate how much gas a transaction will expend on its +# target chain. From that, it calculates the cost of that gas by multiplying the +# amount of estimated gas by the `gas_multiplier` and the `gas_price` +# (estimated gas * `gas_multiplier` * `gas_price`) in order to compute the +# total fee to be deducted from the relayer's wallet. +# +# The `simulate_tx` operation does not always correctly estimate the appropriate +# amount of gas that a transaction requires. In those cases when the operation +# fails, Hermes will attempt to submit the transaction using the specified +# `default_gas` and `max_gas` parameters. In the case that a transaction would +# require more than `max_gas`, it doesn't get submitted and a +# `TxSimulateGasEstimateExceeded` error is returned. + +# Specify the default amount of gas to be used in case the tx simulation fails, +# and Hermes cannot estimate the amount of gas needed. +# Default: 100 000 +default_gas = 100000 + +# Specify the maximum amount of gas to be used as the gas limit for a transaction. +# If `default_gas` is unspecified, then `max_gas` will be used as `default_gas`. +# Default: 400 000 +max_gas = 400000 + +# Specify the price per gas used of the fee to submit a transaction and +# the denomination of the fee. +# +# The specified gas price should always be greater or equal to the `min-gas-price` +# configured on the chain. This is to ensure that at least some minimal price is +# paid for each unit of gas per transaction. +# +# Required +gas_price = { price = 0.001, denom = 'stake' } + +# Multiply this amount with the gas estimate, used to compute the fee +# and account for potential estimation error. +# +# The purpose of multiplying by `gas_multiplier` is to provide a bit of a buffer +# to catch some of the cases when the gas estimation calculation is on the low +# end. +# +# Example: With this setting set to 1.1, then if the estimated gas +# is 80_000, then gas used to compute the fee will be adjusted to +# 80_000 * 1.1 = 88_000. +# +# Default: 1.1, ie. the gas is increased by 10% +# Minimum value: 1.0 +gas_multiplier = 1.1 + +# Specify how many IBC messages at most to include in a single transaction. +# Default: 30 +max_msg_num = 30 + +# Specify the maximum size, in bytes, of each transaction that Hermes will submit. +# Default: 2097152 (2 MiB) +max_tx_size = 2097152 + +# Specify the maximum amount of time to tolerate a clock drift. +# The clock drift parameter defines how much new (untrusted) header's time +# can drift into the future. Default: 5s +clock_drift = '5s' + +# Specify the maximum time per block for this chain. +# The block time together with the clock drift are added to the source drift to estimate +# the maximum clock drift when creating a client on this chain. Default: 30s +# For cosmos-SDK chains a good approximation is `timeout_propose` + `timeout_commit` +# Note: This MUST be the same as the `max_expected_time_per_block` genesis parameter for Tendermint chains. +max_block_time = '30s' + +# Specify the amount of time to be used as the light client trusting period. +# It should be significantly less than the unbonding period +# (e.g. unbonding period = 3 weeks, trusting period = 2 weeks). +# Default: 2/3 of the `unbonding period` for Cosmos SDK chains +trusting_period = '14days' + +# Specify the trust threshold for the light client, ie. the minimum fraction of validators +# which must overlap across two blocks during light client verification. +# Default: { numerator = '2', denominator = '3' }, ie. 2/3. +# Warning: This is an advanced feature! Modify with caution. +trust_threshold = { numerator = '2', denominator = '3' } + +# Specify a string that Hermes will use as a memo for each transaction it submits +# to this chain. The string is limited to 50 characters. Default: '' (empty). +# Note: Hermes will append to the string defined here additional +# operational debugging information, e.g., relayer build version. +memo_prefix = '' + +# This section specifies the filters for policy based relaying. +# +# Default: no policy / filters, allow all packets on all channels. +# +# Only packet filtering based on channel identifier can be specified. +# A channel filter has two fields: +# 1. `policy` - one of two types are supported: +# - 'allow': permit relaying _only on_ the port/channel id in the list below, +# - 'deny': permit relaying on any channel _except for_ the list below. +# 2. `list` - the list of channels specified by the port and channel identifiers. +# Optionally, each element may also contains wildcards, for eg. 'ica*' +# to match all identifiers starting with 'ica' or '*' to match all identifiers. +# +# Example configuration of a channel filter, only allowing packet relaying on +# channel with port ID 'transfer' and channel ID 'channel-0', as well as on +# all ICA channels. +# +# [chains.packet_filter] +# policy = 'allow' +# list = [ +# ['ica*', '*'], +# ['transfer', 'channel-0'], +# ] + +# This section specifies the filters for incentivized packet relaying. +# Default: no filters, will relay all packets even if they +# are not incentivized. +# +# It is possible to specify the channel or use wildcards for the +# channels. +# The only fee which can be parametrized is the `recv_fee`. +# +# Example configuration of a filter which will only relay incentivized +# packets, with no regards for channel and amount. +# +# [chains.packet_filter.min_fees.'*'] +# recv = [ { amount = 0 } ] +# +# Example configuration of a filter which will only relay packets if they are +# from the channel 'channel-0', and they have a `recv_fee` of at least 20 stake +# or 10 uatom. +# +# [chains.packet_filter.min_fees.'channel-0'] +# recv = [ { amount = 20, denom = 'stake' }, { amount = 10, denom = 'uatom' } ] + +# Specify that the transaction fees should be payed from this fee granter's account. +# Optional. If unspecified (the default behavior), then no fee granter is used, and +# the account specified in `key_name` will pay the tx fees for all transactions +# submitted to this chain. +# fee_granter = '' + +[[chains]] +id = 'ibc-1' +rpc_addr = 'http://127.0.0.1:26457' +grpc_addr = 'http://127.0.0.1:9092' +event_source = { mode = 'push', url = 'ws://127.0.0.1:26457/websocket', batch_delay = '500ms' } +rpc_timeout = '10s' +account_prefix = 'cosmos' +key_name = 'testkey' +store_prefix = 'ibc' +default_gas = 100000 +max_gas = 400000 +gas_price = { price = 0.001, denom = 'stake' } +gas_multiplier = 1.1 +max_msg_num = 30 +max_tx_size = 2097152 +clock_drift = '5s' +max_block_time = '30s' +trusting_period = '14days' +trust_threshold = { numerator = '1', denominator = '3' } +address_type = { derivation = 'cosmos' } From 164c05e1a332440df7e9c24902b713b7d27a0046 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Mon, 29 May 2023 17:06:47 +0200 Subject: [PATCH 20/26] Fix typo Signed-off-by: Romain Ruetschi --- .../unreleased/features/ibc-relayer/2850-poll-event-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog/unreleased/features/ibc-relayer/2850-poll-event-source.md b/.changelog/unreleased/features/ibc-relayer/2850-poll-event-source.md index 8c8a675af3..933a7edede 100644 --- a/.changelog/unreleased/features/ibc-relayer/2850-poll-event-source.md +++ b/.changelog/unreleased/features/ibc-relayer/2850-poll-event-source.md @@ -1,7 +1,7 @@ - Add a poll-based event source which fetches events from the chain using the `/block_results` RPC endpoint instead of getting them over WebSocket. - To use the poll-based event source, set `event_source = 'rpc'` in the per-chain configuration. + To use the poll-based event source, set `event_source = 'poll'` in the per-chain configuration. **Warning** Only use this if you think Hermes is not getting all From 682efd2b8a9cd6ee069b09b8dc4dd203c045c195 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 30 May 2023 17:17:14 +0200 Subject: [PATCH 21/26] Fix typo in config and rename `poll_interval` to `interval` --- config.toml | 6 +++--- crates/relayer/src/chain/cosmos.rs | 4 ++-- crates/relayer/src/config.rs | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/config.toml b/config.toml index 25d55f2954..0331d6e677 100644 --- a/config.toml +++ b/config.toml @@ -147,13 +147,13 @@ grpc_addr = 'http://127.0.0.1:9090' # processing, increasing the latency of Hermes, but are more likely to batch events together. # The default value provides good latency while minimizing the number of client updates needed. -# b) Poll: for polling for IBC events via the `/block_results` RPC endpoint +# b) Pull: for polling for IBC events via the `/block_results` RPC endpoint # -# `{ mode = 'poll', poll_interval = '1s' }` +# `{ mode = 'pull', interval = '1s' }` # # where # -# - `poll_interval` is the interval at which to poll for blocks. Default: 1s +# - `interval` is the interval at which to poll for blocks. Default: 1s # event_source = { mode = 'push', url = 'ws://127.0.0.1:26657/websocket', batch_delay = '500ms' } diff --git a/crates/relayer/src/chain/cosmos.rs b/crates/relayer/src/chain/cosmos.rs index 726faf5eb3..f782e886c9 100644 --- a/crates/relayer/src/chain/cosmos.rs +++ b/crates/relayer/src/chain/cosmos.rs @@ -304,10 +304,10 @@ impl CosmosSdkChain { *batch_delay, self.rt.clone(), ), - Mode::Pull { poll_interval } => EventSource::rpc( + Mode::Pull { interval } => EventSource::rpc( self.config.id.clone(), self.rpc_client.clone(), - *poll_interval, + *interval, self.rt.clone(), ), } diff --git a/crates/relayer/src/config.rs b/crates/relayer/src/config.rs index 2093fff715..f6754c34e5 100644 --- a/crates/relayer/src/config.rs +++ b/crates/relayer/src/config.rs @@ -468,10 +468,11 @@ pub enum EventSourceMode { }, /// Pull-based event source, via RPC /block_results + #[serde(alias = "poll")] Pull { /// The polling interval #[serde(default = "default::poll_interval", with = "humantime_serde")] - poll_interval: Duration, + interval: Duration, }, } From ebee52c7b92e4b23fb4bf88c0f2db39c4efe6bf0 Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 30 May 2023 17:27:17 +0200 Subject: [PATCH 22/26] Clarify logic and doc for processing commands after fetching batch --- crates/relayer/src/event/source/rpc.rs | 21 ++++++--- crates/relayer/src/event/source/websocket.rs | 48 ++++++++++++-------- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/crates/relayer/src/event/source/rpc.rs b/crates/relayer/src/event/source/rpc.rs index de7bbe840f..e5cef7aa0a 100644 --- a/crates/relayer/src/event/source/rpc.rs +++ b/crates/relayer/src/event/source/rpc.rs @@ -136,8 +136,8 @@ impl EventSource { async fn step(&mut self) -> Result { // Process any shutdown or subscription commands before we start doing any work - if let Some(next) = self.try_process_cmd() { - return Ok(next); + if let Next::Abort = self.try_process_cmd() { + return Ok(Next::Abort); } let latest_height = latest_height(&self.rpc_client).await?; @@ -159,8 +159,13 @@ impl EventSource { }; // Before handling the batch, check if there are any pending shutdown or subscribe commands. - if let Some(next) = self.try_process_cmd() { - return Ok(next); + // + // This avoids having the supervisor process an event batch after the event source has been shutdown. + // + // It also allows subscribers to receive the latest event batch even if they + // subscribe while the batch being fetched. + if let Next::Abort = self.try_process_cmd() { + return Ok(Next::Abort); } for batch in batches.unwrap_or_default() { @@ -170,10 +175,12 @@ impl EventSource { Ok(Next::Continue) } - fn try_process_cmd(&mut self) -> Option { + /// Process any pending commands, if any. + fn try_process_cmd(&mut self) -> Next { if let Ok(cmd) = self.rx_cmd.try_recv() { match cmd { - EventSourceCmd::Shutdown => return Some(Next::Abort), + EventSourceCmd::Shutdown => return Next::Abort, + EventSourceCmd::Subscribe(tx) => { if let Err(e) = tx.send(self.event_bus.subscribe()) { error!("failed to send back subscription: {e}"); @@ -182,7 +189,7 @@ impl EventSource { } } - None + Next::Continue } async fn fetch_batches(&mut self, latest_height: BlockHeight) -> Result> { diff --git a/crates/relayer/src/event/source/websocket.rs b/crates/relayer/src/event/source/websocket.rs index 119e2ccfcb..6a7c29b5a3 100644 --- a/crates/relayer/src/event/source/websocket.rs +++ b/crates/relayer/src/event/source/websocket.rs @@ -307,16 +307,9 @@ impl EventSource { pin_mut!(batches); loop { - // Process any shutdown or subscription commands - if let Ok(cmd) = self.rx_cmd.try_recv() { - match cmd { - EventSourceCmd::Shutdown => return Next::Abort, - EventSourceCmd::Subscribe(tx) => { - if let Err(e) = tx.send(self.event_bus.subscribe()) { - error!("failed to send back subscription: {e}"); - } - } - } + // Process any shutdown or subscription commands before we start doing any work. + if let Next::Abort = self.try_process_cmd() { + return Next::Abort; } let result = tokio::select! { @@ -325,15 +318,15 @@ impl EventSource { }; // Before handling the batch, check if there are any pending shutdown or subscribe commands. - if let Ok(cmd) = self.rx_cmd.try_recv() { - match cmd { - EventSourceCmd::Shutdown => return Next::Abort, - EventSourceCmd::Subscribe(tx) => { - if let Err(e) = tx.send(self.event_bus.subscribe()) { - error!("failed to send back subscription: {e}"); - } - } - } + // + // This avoids having the supervisor process an event batch after the event source has been shutdown, + // and issues during testing where the WebSocket connection might get closed before the event + // source has been shutdown. + // + // It also allows subscribers to receive the latest event batch even if they + // subscribe while the batch being fetched. + if let Next::Abort = self.try_process_cmd() { + return Next::Abort; } match result { @@ -382,6 +375,23 @@ impl EventSource { self.event_bus.broadcast(Arc::new(Ok(batch))); } + + /// Process a pending command, if any. + fn try_process_cmd(&mut self) -> Next { + if let Ok(cmd) = self.rx_cmd.try_recv() { + match cmd { + EventSourceCmd::Shutdown => return Next::Abort, + + EventSourceCmd::Subscribe(tx) => { + if let Err(e) = tx.send(self.event_bus.subscribe()) { + error!("failed to send back subscription: {e}"); + } + } + } + } + + Next::Continue + } } /// Collect the IBC events from an RPC event From b9410b212b48b1b624ab6a61ffaa7d3063e9eaba Mon Sep 17 00:00:00 2001 From: Romain Ruetschi Date: Tue, 30 May 2023 17:28:08 +0200 Subject: [PATCH 23/26] Do not collect `DistributeFee` events --- crates/relayer/src/event/source/rpc/extract.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/relayer/src/event/source/rpc/extract.rs b/crates/relayer/src/event/source/rpc/extract.rs index 8ff993e030..4d394e71d7 100644 --- a/crates/relayer/src/event/source/rpc/extract.rs +++ b/crates/relayer/src/event/source/rpc/extract.rs @@ -24,9 +24,9 @@ pub fn extract_events( if let DistributionType::Reward = dist.distribution_type { telemetry!(fees_amount, chain_id, &dist.receiver, dist.fee.clone()); } + } else { + events_with_height.push(IbcEventWithHeight { height, event }); } - - events_with_height.push(IbcEventWithHeight { height, event }); } _ => {} From 7bc3e2cf8e0c281882fa984c8013280c5109b1af Mon Sep 17 00:00:00 2001 From: Sean Chen Date: Mon, 5 Jun 2023 11:34:56 -0500 Subject: [PATCH 24/26] Add section to hermes guide on configuring for wasm relaying --- .../configuration/configure-hermes.md | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/guide/src/documentation/configuration/configure-hermes.md b/guide/src/documentation/configuration/configure-hermes.md index cb8d8f7f2e..6cecc743c2 100644 --- a/guide/src/documentation/configuration/configure-hermes.md +++ b/guide/src/documentation/configuration/configure-hermes.md @@ -140,6 +140,29 @@ event_source = { mode = 'push', url = 'wss://hello:world@mydomain.com:26657/webs > **Caution:** Warning: The "Basic" authentication scheme sends the credentials encoded but not encrypted. > This would be completely insecure unless the exchange was over a secure connection (HTTPS/TLS). +## Configuring Support for Wasm Relaying + +As of version 1.6.0, Hermes supports the relaying of wasm messages natively. This is facilitated by configuring +Hermes to use pull-based relaying by polling for IBC events via the `/block_results` RPC endpoint. Set +the `event_source` parameter to pull mode in `config.toml`: + +```toml +event_source = 'poll' +``` + +The default interval at which Hermes polls the RPC endpoint is 1 second. If you need to change the interval, +you can do so like this: + +```toml +event_source = { mode = 'pull', interval = '2s' } +``` + +The pull model of relaying is in contrast with Hermes' default push model, where IBC events are received +over WebSocket. This mode should only be used in situations where Hermes misses events that it should +be receiving, such as when relaying for a CosmWasm-enabled blockchain which emits IBC events without the +`message` attribute. Without this attribute, the WebSocket is not able to catch these events to stream +to Hermes, so the `/block_results` RPC endpoint must be used instead. + [http-basic-auth]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication [ica]: https://github.com/cosmos/ibc/blob/master/spec/app/ics-027-interchain-accounts/README.md [chain-registry]: https://github.com/cosmos/chain-registry From 35055ba8e6fad99c3a2e3e496a79d33c33c3631d Mon Sep 17 00:00:00 2001 From: Sean Chen Date: Mon, 5 Jun 2023 11:38:54 -0500 Subject: [PATCH 25/26] Remove an extra word --- guide/src/documentation/configuration/configure-hermes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/documentation/configuration/configure-hermes.md b/guide/src/documentation/configuration/configure-hermes.md index 6cecc743c2..f77cdc3647 100644 --- a/guide/src/documentation/configuration/configure-hermes.md +++ b/guide/src/documentation/configuration/configure-hermes.md @@ -137,7 +137,7 @@ event_source = { mode = 'push', url = 'wss://hello:world@mydomain.com:26657/webs # ... ``` -> **Caution:** Warning: The "Basic" authentication scheme sends the credentials encoded but not encrypted. +> **Caution:** The "Basic" authentication scheme sends the credentials encoded but not encrypted. > This would be completely insecure unless the exchange was over a secure connection (HTTPS/TLS). ## Configuring Support for Wasm Relaying From 27965e40fe8e67d3f39c59ae4ed899f639d8e0fa Mon Sep 17 00:00:00 2001 From: Sean Chen Date: Mon, 5 Jun 2023 11:43:30 -0500 Subject: [PATCH 26/26] Guide formatting --- guide/src/documentation/configuration/configure-hermes.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/guide/src/documentation/configuration/configure-hermes.md b/guide/src/documentation/configuration/configure-hermes.md index f77cdc3647..832c63fc75 100644 --- a/guide/src/documentation/configuration/configure-hermes.md +++ b/guide/src/documentation/configuration/configure-hermes.md @@ -158,8 +158,10 @@ event_source = { mode = 'pull', interval = '2s' } ``` The pull model of relaying is in contrast with Hermes' default push model, where IBC events are received -over WebSocket. This mode should only be used in situations where Hermes misses events that it should -be receiving, such as when relaying for a CosmWasm-enabled blockchain which emits IBC events without the +over WebSocket. + +> **Note:** This mode should only be used in situations where Hermes misses events that it should +be receiving, such as when relaying for CosmWasm-enabled blockchains which emit IBC events without the `message` attribute. Without this attribute, the WebSocket is not able to catch these events to stream to Hermes, so the `/block_results` RPC endpoint must be used instead.