diff --git a/Cargo.lock b/Cargo.lock index f224636219..60f97ebcf6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2744,12 +2744,13 @@ version = "0.1.0" dependencies = [ "clap", "color-eyre 0.6.2", + "futures", "graphql_client", "reqwest", "serde", "serde_json", - "tendermint 0.32.0", - "tendermint-rpc", + "tendermint 0.33.0", + "tendermint-rpc 0.33.0", "tokio", "tracing", "tracing-subscriber 0.3.17", @@ -4994,6 +4995,34 @@ dependencies = [ "zeroize", ] +[[package]] +name = "tendermint" +version = "0.33.0" +source = "git+https://github.com/unionlabs/tendermint-rs?branch=add-proto-bn254#2dcb71b8c88d13d0a40811d36a6f13731510b4c5" +dependencies = [ + "bytes", + "digest 0.10.7", + "ed25519 2.2.1", + "ed25519-consensus", + "flex-error", + "futures", + "num-traits", + "once_cell", + "prost", + "prost-types", + "serde", + "serde_bytes", + "serde_json", + "serde_repr", + "sha2 0.10.7", + "signature 2.1.0", + "subtle", + "subtle-encoding", + "tendermint-proto 0.33.0", + "time", + "zeroize", +] + [[package]] name = "tendermint-config" version = "0.32.0" @@ -5007,6 +5036,19 @@ dependencies = [ "url", ] +[[package]] +name = "tendermint-config" +version = "0.33.0" +source = "git+https://github.com/unionlabs/tendermint-rs?branch=add-proto-bn254#2dcb71b8c88d13d0a40811d36a6f13731510b4c5" +dependencies = [ + "flex-error", + "serde", + "serde_json", + "tendermint 0.33.0", + "toml 0.5.11", + "url", +] + [[package]] name = "tendermint-light-client-verifier" version = "0.29.1" @@ -5055,6 +5097,23 @@ dependencies = [ "time", ] +[[package]] +name = "tendermint-proto" +version = "0.33.0" +source = "git+https://github.com/unionlabs/tendermint-rs?branch=add-proto-bn254#2dcb71b8c88d13d0a40811d36a6f13731510b4c5" +dependencies = [ + "bytes", + "flex-error", + "num-derive", + "num-traits", + "prost", + "prost-types", + "serde", + "serde_bytes", + "subtle-encoding", + "time", +] + [[package]] name = "tendermint-rpc" version = "0.32.0" @@ -5079,7 +5138,7 @@ dependencies = [ "subtle", "subtle-encoding", "tendermint 0.32.0", - "tendermint-config", + "tendermint-config 0.32.0", "tendermint-proto 0.32.0", "thiserror", "time", @@ -5090,6 +5149,40 @@ dependencies = [ "walkdir", ] +[[package]] +name = "tendermint-rpc" +version = "0.33.0" +source = "git+https://github.com/unionlabs/tendermint-rs?branch=add-proto-bn254#2dcb71b8c88d13d0a40811d36a6f13731510b4c5" +dependencies = [ + "async-trait", + "bytes", + "flex-error", + "futures", + "getrandom", + "http", + "hyper", + "hyper-proxy", + "hyper-rustls 0.22.1", + "peg", + "pin-project", + "semver", + "serde", + "serde_bytes", + "serde_json", + "subtle", + "subtle-encoding", + "tendermint 0.33.0", + "tendermint-config 0.33.0", + "tendermint-proto 0.33.0", + "thiserror", + "time", + "tokio", + "tracing", + "url", + "uuid", + "walkdir", +] + [[package]] name = "term" version = "0.7.0" @@ -5876,7 +5969,7 @@ dependencies = [ "subtle-encoding", "tendermint 0.32.0", "tendermint-proto 0.32.0", - "tendermint-rpc", + "tendermint-rpc 0.32.0", "tokio", "tokio-stream", "tonic", diff --git a/dictionary.txt b/dictionary.txt index adfb83f141..3eaf10c376 100644 --- a/dictionary.txt +++ b/dictionary.txt @@ -601,3 +601,4 @@ zedxv zerolog zkps λpxrx +omnichain \ No newline at end of file diff --git a/hubble/Cargo.toml b/hubble/Cargo.toml index 7da99268b5..fb012e93a4 100644 --- a/hubble/Cargo.toml +++ b/hubble/Cargo.toml @@ -13,7 +13,8 @@ serde = { version = "1.0", features = ["derive"] } reqwest = { version = "^0.11", features = ["json", "blocking"] } serde_json = { version = "1.0" } color-eyre = "0.6.2" -clap = { version = "4.4.0", features = ["derive"] } +clap = { version = "4.4.0", features = ["derive", "env"] } url = { version = "2.4.1", features = ["serde"] } tracing = "0.1.37" tracing-subscriber = "0.3.17" +futures = "0.3.28" diff --git a/hubble/README.md b/hubble/README.md new file mode 100644 index 0000000000..9fccd26330 --- /dev/null +++ b/hubble/README.md @@ -0,0 +1,4 @@ +Hubble is an omnichain indexer. + +> **Warning** +> This is not ready for public usage. It can index tendermint chains, but the schema is not public yet. diff --git a/hubble/hubble.nix b/hubble/hubble.nix index 3ef3277576..c686f3cf2f 100644 --- a/hubble/hubble.nix +++ b/hubble/hubble.nix @@ -11,20 +11,16 @@ }; in { - inherit (hubble) checks ; + inherit (hubble) checks; packages = { hubble = hubble.packages.hubble; - hubble-image = pkgs.dockerTools.buildImage { + hubble-image = pkgs.dockerTools.buildLayeredImage { name = "hubble"; - - copyToRoot = pkgs.buildEnv { - name = "image-root"; - paths = [ pkgs.coreutils-full hubble ]; - pathsToLink = [ "/bin" ]; - }; + contents = [ pkgs.coreutils-full pkgs.cacert self'.packages.hubble ]; config = { - Entrypoint = [ (pkgs.lib.getExe self'.packages.uniond) ]; + Entrypoint = [ (pkgs.lib.getExe self'.packages.hubble) ]; + Env = [ "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" ]; }; }; }; diff --git a/hubble/src/cli.rs b/hubble/src/cli.rs index c26b7d80c1..51a930f57f 100644 --- a/hubble/src/cli.rs +++ b/hubble/src/cli.rs @@ -8,12 +8,13 @@ use url::Url; #[command(author, version, about, long_about = None)] pub struct Args { /// The url to the hasura graphql endpoint. - #[arg(short, long)] - pub hasura: Url, + #[arg(short, long, env = "HUBBLE_URL")] + pub url: Url, /// The admin secret used to authenticate with hasura. - #[arg(short, long)] + #[arg(short, long, env = "HUBBLE_SECRET")] pub secret: String, /// Indexer configurations to start. + #[arg(short, long, env = "HUBBLE_INDEXERS")] pub indexers: Vec, } diff --git a/hubble/src/main.rs b/hubble/src/main.rs index 02776e9d3a..ee1d91ff1a 100644 --- a/hubble/src/main.rs +++ b/hubble/src/main.rs @@ -1,18 +1,28 @@ use clap::Parser; -use tracing::info; +use tracing::{error, info}; mod cli; mod hasura; mod tm; -#[tokio::main(flavor = "current_thread")] +#[tokio::main] async fn main() { + color_eyre::install().unwrap(); + let args = crate::cli::Args::parse(); tracing_subscriber::fmt::init(); + let mut handles = vec![]; + for indexer in args.indexers.into_iter() { info!("starting indexer {:?}", indexer); - let url = args.hasura.clone(); + let url = args.url.clone(); let secret = args.secret.clone(); - tokio::task::spawn_local(async move { indexer.index(&url, &secret).await.unwrap() }); + handles.push(tokio::task::spawn(async move { + // indexer should never return with Ok, thus we log the error. + let result = indexer.index(&url, &secret).await; + error!("indexer {:?} exited with: {:?}", &indexer, result); + })); } + + futures::future::join_all(handles).await; } diff --git a/hubble/src/tm.rs b/hubble/src/tm.rs index 4c4c329b3a..9d59e24e5d 100644 --- a/hubble/src/tm.rs +++ b/hubble/src/tm.rs @@ -52,13 +52,15 @@ impl Config { let mut height: u32 = if data.blocks.is_empty() { 0 } else { - TryInto::::try_into(data.blocks[0].height).unwrap() + 1_u32 + TryInto::::try_into(data.blocks[0].height).unwrap() }; debug!("latest stored block height is: {}", &height); let chain_db_id = data.chains[0].id; loop { + height += 1; + info!("indexing block {}", &height); // if we're caught up indexing to the latest height, this will error. In that case, // we retry until we obtain the next header. @@ -140,7 +142,6 @@ impl Config { }; do_post::(secret, &url, &db, v).await?; - height += 1; } } }