Split out test framework from integration tests (#1961)

* Split out test framework from integration tests

* Fix integration test not triggering

* Add changelog

* Fix doc tests

.changelog/unreleased/improvements/ibc-integration-test/1961-test-framework.md

@@ -0,0 +1 @@
+-  Split out test framework as new crate `ibc-test-framework` from `ibc-integration-test`. ([#1961](https://github.com/informalsystems/ibc-rs/pull/1961))

Cargo.toml

@@ -10,6 +10,7 @@ members = [
     "telemetry",
     "proto",
     "tools/integration-test",
+    "tools/test-framework",
 ]
 
 exclude = [

tools/integration-test/Cargo.toml

@@ -18,29 +18,11 @@ ibc             = { path = "../../modules" }
 ibc-relayer     = { path = "../../relayer" }
 ibc-relayer-cli = { path = "../../relayer-cli" }
 ibc-proto       = { path = "../../proto" }
+ibc-test-framework = { path = "../test-framework" }
 tendermint      = { version = "=0.23.5" }
 tendermint-rpc  = { version = "=0.23.5", features = ["http-client", "websocket-client"] }
 
-tokio = { version = "1.0", features = ["full"] }
-tracing = "0.1.32"
-tracing-subscriber = "0.3.9"
-eyre = "0.6.7"
-color-eyre = "0.6"
-rand = "0.8.5"
-env_logger = "0.9.0"
-hex = "0.4.3"
-serde = "1.0"
 serde_json = "1"
-serde_yaml = "0.8.23"
-itertools = "0.10"
-toml = "0.5"
-subtle-encoding = "0.5.1"
-sha2 = "0.10.2"
-crossbeam-channel = "0.5.2"
-prost = { version = "0.9", default-features = false }
-prost-types = { version = "0.9", default-features = false }
-semver = "1.0.6"
-flex-error = "0.4.4"
 
 [features]
 default = []

tools/integration-test/src/bin/test_setup_with_binary_channel.rs

@@ -25,8 +25,8 @@
     ```
 */
 
-use ibc_integration_test::prelude::*;
 use ibc_relayer::keyring::Store;
+use ibc_test_framework::prelude::*;
 
 struct Test;
 

tools/integration-test/src/lib.rs

@@ -1,21 +1,2 @@
-// #![deny(warnings)]
-#![allow(clippy::too_many_arguments)]
-#![allow(clippy::type_complexity)]
-#![allow(clippy::ptr_arg)]
-#![doc = include_str!("../README.md")]
-
-extern crate alloc;
-
-pub mod bootstrap;
-pub mod chain;
-pub mod error;
-pub mod framework;
-pub mod ibc;
-pub mod prelude;
-pub mod relayer;
-pub mod types;
-pub mod util;
-
-#[cfg(any(test, doc))]
-#[macro_use]
+#[cfg(test)]
 pub mod tests;

tools/integration-test/src/tests/clear_packet.rs

@@ -1,6 +1,6 @@
-use crate::ibc::denom::derive_ibc_denom;
-use crate::prelude::*;
-use crate::util::random::random_u64_range;
+use ibc_test_framework::ibc::denom::derive_ibc_denom;
+use ibc_test_framework::prelude::*;
+use ibc_test_framework::util::random::random_u64_range;
 
 #[test]
 fn test_clear_packet() -> Result<(), Error> {

tools/integration-test/src/tests/client_expiration.rs

@@ -5,20 +5,20 @@ use ibc_relayer::config::default::connection_delay as default_connection_delay;
 use ibc_relayer::config::{self, Config, ModeConfig};
 use std::thread::sleep;
 
-use crate::bootstrap::binary::chain::bootstrap_foreign_client_pair;
-use crate::bootstrap::binary::channel::{
+use ibc_test_framework::bootstrap::binary::chain::bootstrap_foreign_client_pair;
+use ibc_test_framework::bootstrap::binary::channel::{
     bootstrap_channel_with_chains, bootstrap_channel_with_connection,
 };
-use crate::bootstrap::binary::connection::bootstrap_connection;
-use crate::ibc::denom::derive_ibc_denom;
-use crate::prelude::*;
-use crate::relayer::channel::{
+use ibc_test_framework::bootstrap::binary::connection::bootstrap_connection;
+use ibc_test_framework::ibc::denom::derive_ibc_denom;
+use ibc_test_framework::prelude::*;
+use ibc_test_framework::relayer::channel::{
     assert_eventually_channel_established, init_channel, query_channel_end,
 };
-use crate::relayer::connection::{
+use ibc_test_framework::relayer::connection::{
     assert_eventually_connection_established, init_connection, query_connection_end,
 };
-use crate::relayer::refresh::spawn_refresh_client_tasks;
+use ibc_test_framework::relayer::refresh::spawn_refresh_client_tasks;
 
 // The cosmos ChainHandle handles requests in serial, and a refresh client
 // request may get blocked by other operations and cause the refresh to fail

tools/integration-test/src/tests/example.rs

@@ -28,7 +28,7 @@
 
     We first define an empty struct [`ExampleTest`] to represent our test case.
     We then implement the
-    [`BinaryChannelTest`](crate::framework::binary::channel::BinaryChannelTest)
+    [`BinaryChannelTest`](ibc_test_framework::framework::binary::channel::BinaryChannelTest)
     trait so that the test framework sets up the relayer with two chains
     running together with connected channels.
 
@@ -44,12 +44,12 @@
 
     Finally, we define the `example_test` function with the `#[test]` pragma
     as the entry point for Rust to execute the test. We call the runner function
-    [`run_binary_channel_test`](crate::framework::binary::channel::run_binary_channel_test),
+    [`run_binary_channel_test`](ibc_test_framework::framework::binary::channel::run_binary_channel_test),
     which accepts a reference to any struct implementing
-    [`BinaryChannelTest`](crate::framework::binary::channel::BinaryChannelTest)
+    [`BinaryChannelTest`](ibc_test_framework::framework::binary::channel::BinaryChannelTest)
     and run the test for us.
 
-    By convention, the tests written are placed in the [`tests`](crate::tests)
+    By convention, the tests written are placed in the [`tests`](ibc_test_framework::tests)
     module. We can then run the test on the command line such as follows:
 
     ```bash
@@ -62,7 +62,7 @@
     and `RUST_BACKTRACE` to display backtrace when errors occurred.
     The test flag `--test-threads=1` is set so that Rust do not run multiple
     tests in parallel, as it can make it difficult to follow the logs.
-    See [TestConfig](crate::types::config::TestConfig) for more information
+    See [TestConfig](ibc_test_framework::types::config::TestConfig) for more information
     about configuring how the tests should be run.
 
     For this example, we disable the test from running by default, since
@@ -112,7 +112,7 @@
     same directory.
 */
 
-use crate::prelude::*;
+use ibc_test_framework::prelude::*;
 
 #[test]
 pub fn example_test() -> Result<(), Error> {

tools/integration-test/src/tests/manual/mod.rs

@@ -7,7 +7,7 @@
     are only enabled when the `"manual"` feature flag is enabled manually.
 
     Any tests that require manual verification should be placed here.
-    It is also fine to use [`suspend`](crate::util::suspend::suspend)
+    It is also fine to use [`suspend`](ibc_test_framework::util::suspend::suspend)
     inside the manual test, as the CI is not going to run the test.
 */
 

tools/integration-test/src/tests/manual/simulation.rs

@@ -14,8 +14,8 @@
 use core::time::Duration;
 use ibc_relayer::config::{types::MaxMsgNum, Config};
 
-use crate::prelude::*;
-use crate::relayer::transfer::tx_raw_ft_transfer;
+use ibc_test_framework::prelude::*;
+use ibc_test_framework::relayer::transfer::tx_raw_ft_transfer;
 
 #[test]
 fn test_simulation() -> Result<(), Error> {

tools/integration-test/src/tests/memo.rs

@@ -1,9 +1,9 @@
 use ibc_relayer::config::{types::Memo, Config};
 use serde_json as json;
 
-use crate::ibc::denom::derive_ibc_denom;
-use crate::prelude::*;
-use crate::util::random::{random_string, random_u64_range};
+use ibc_test_framework::ibc::denom::derive_ibc_denom;
+use ibc_test_framework::prelude::*;
+use ibc_test_framework::util::random::{random_string, random_u64_range};
 
 #[test]
 fn test_memo() -> Result<(), Error> {

tools/integration-test/src/tests/ordered_channel.rs

@@ -1,6 +1,6 @@
-use crate::ibc::denom::derive_ibc_denom;
-use crate::prelude::*;
-use crate::util::random::random_u64_range;
+use ibc_test_framework::ibc::denom::derive_ibc_denom;
+use ibc_test_framework::prelude::*;
+use ibc_test_framework::util::random::random_u64_range;
 
 #[test]
 fn test_ordered_channel() -> Result<(), Error> {

tools/integration-test/src/tests/supervisor.rs

@@ -1,9 +1,11 @@
-use crate::ibc::denom::derive_ibc_denom;
 use ibc_relayer::config::{self, Config, ModeConfig};
+use ibc_test_framework::ibc::denom::derive_ibc_denom;
 
-use crate::prelude::*;
-use crate::relayer::channel::{assert_eventually_channel_established, init_channel};
-use crate::relayer::connection::{assert_eventually_connection_established, init_connection};
+use ibc_test_framework::prelude::*;
+use ibc_test_framework::relayer::channel::{assert_eventually_channel_established, init_channel};
+use ibc_test_framework::relayer::connection::{
+    assert_eventually_connection_established, init_connection,
+};
 
 #[test]
 fn test_supervisor() -> Result<(), Error> {

tools/integration-test/src/tests/ternary_transfer.rs

@@ -1,5 +1,5 @@
-use crate::ibc::denom::derive_ibc_denom;
-use crate::prelude::*;
+use ibc_test_framework::ibc::denom::derive_ibc_denom;
+use ibc_test_framework::prelude::*;
 
 #[test]
 fn test_ternary_ibc_transfer() -> Result<(), Error> {

tools/integration-test/src/tests/transfer.rs

@@ -1,6 +1,6 @@
-use crate::ibc::denom::derive_ibc_denom;
-use crate::prelude::*;
-use crate::util::random::random_u64_range;
+use ibc_test_framework::ibc::denom::derive_ibc_denom;
+use ibc_test_framework::prelude::*;
+use ibc_test_framework::util::random::random_u64_range;
 
 #[test]
 fn test_ibc_transfer() -> Result<(), Error> {

tools/test-framework/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name       = "ibc-test-framework"
+version    = "0.13.0-rc.0"
+edition    = "2021"
+license    = "Apache-2.0"
+readme     = "README.md"
+keywords   = ["blockchain", "consensus", "cosmos", "ibc", "tendermint"]
+homepage   = "https://hermes.informal.systems/"
+repository = "https://github.com/informalsystems/ibc-rs"
+authors    = ["Informal Systems <hello@informal.systems>"]
+
+description = """
+  Integration tests for IBC Relayer
+"""
+
+[dependencies]
+ibc             = { path = "../../modules" }
+ibc-relayer     = { path = "../../relayer" }
+ibc-relayer-cli = { path = "../../relayer-cli" }
+ibc-proto       = { path = "../../proto" }
+tendermint      = { version = "=0.23.5" }
+tendermint-rpc  = { version = "=0.23.5", features = ["http-client", "websocket-client"] }
+
+tokio = { version = "1.0", features = ["full"] }
+tracing = "0.1.32"
+tracing-subscriber = "0.3.9"
+eyre = "0.6.7"
+color-eyre = "0.6"
+rand = "0.8.5"
+env_logger = "0.9.0"
+hex = "0.4.3"
+serde = "1.0"
+serde_json = "1"
+serde_yaml = "0.8.23"
+itertools = "0.10"
+toml = "0.5"
+subtle-encoding = "0.5.1"
+sha2 = "0.10.2"
+crossbeam-channel = "0.5.2"
+prost = { version = "0.9", default-features = false }
+prost-types = { version = "0.9", default-features = false }
+semver = "1.0.6"
+flex-error = "0.4.4"

tools/integration-test/README.md → tools/test-framework/README.md

@@ -20,11 +20,6 @@ Other than Rust, the test suite assumes the `gaiad` binary is present in `$PATH`
 nix shell github:informalsystems/cosmos.nix#gaia5
 ```
 
-## Quick Start
-
-See [`tests::example`](crate::tests::example) for an example test with
-access to a full test setup.
-
 ## Diagrams
 
 Some diagrams have been prepared to ease the understanding of the test framework:

tools/integration-test/src/bootstrap/binary/chain.rs → tools/test-framework/src/bootstrap/binary/chain.rs

@@ -189,7 +189,7 @@ pub fn bootstrap_foreign_client<ChainA: ChainHandle, ChainB: ChainHandle>(
    For example, the following test should fail to compile:
 
    ```rust,compile_fail
-   # use ibc_integration_test::bootstrap::binary::chain::spawn_chain_handle;
+   # use ibc_test_framework::bootstrap::binary::chain::spawn_chain_handle;
    fn same<T>(_: T, _: T) {}
 
    let chain_a = spawn_chain_handle(|| {}, todo!(), todo!()).unwrap();

tools/test-framework/src/lib.rs

@@ -0,0 +1,17 @@
+// #![deny(warnings)]
+#![allow(clippy::too_many_arguments)]
+#![allow(clippy::type_complexity)]
+#![allow(clippy::ptr_arg)]
+#![doc = include_str!("../README.md")]
+
+extern crate alloc;
+
+pub mod bootstrap;
+pub mod chain;
+pub mod error;
+pub mod framework;
+pub mod ibc;
+pub mod prelude;
+pub mod relayer;
+pub mod types;
+pub mod util;