From f640e05e997b449b6cff826fbb911e0637315255 Mon Sep 17 00:00:00 2001 From: karel Date: Fri, 1 Sep 2023 16:26:14 +0100 Subject: [PATCH] feat(hubble): add indexing to devnets latest fix hasura-cli feat(hubble): add indexing to devnets --- dictionary.txt | 13 ++++ flake.nix | 74 ++++++++++------------- hubble/hubble.nix | 86 +++++++++++++++++++++++---- hubble/src/graphql/operations.graphql | 8 ++- hubble/src/hasura.rs | 10 ++++ hubble/src/main.rs | 34 ++++++++--- hubble/src/tm.rs | 71 +++++++++++++++------- networks/devnet.nix | 15 ++--- networks/services/hasura.nix | 17 +++--- networks/services/hubble.nix | 21 +++++++ networks/services/postgres.nix | 1 + networks/services/uniond.nix | 1 + 12 files changed, 247 insertions(+), 104 deletions(-) create mode 100644 networks/services/hubble.nix diff --git a/dictionary.txt b/dictionary.txt index 9d384ff59b..d83901109b 100644 --- a/dictionary.txt +++ b/dictionary.txt @@ -21,6 +21,7 @@ DATADIR DYLD Denoms Feegrant +GOARCH GOPATH GOPRIVATE Getenv @@ -151,6 +152,7 @@ codegen codespace coeff cofactor +coinbases cometbft cometbls cometbn @@ -163,6 +165,7 @@ consensusparamkeeper consensusparamtypes consensys consts +conv converstion coreutils corsdomain @@ -179,6 +182,7 @@ cratedir crisiskeeper crisistypes ctypes +dataconnector datadir datetime delegators @@ -215,6 +219,7 @@ feegrant feegrantkeeper feegrantmodule fetchurl +fkey fksmg floorlog foldl @@ -292,6 +297,7 @@ icahostkeeper icahosttypes icatypes identifing +iflag ilog imap impls @@ -359,6 +365,7 @@ neofetch netutil networkid nextest +nextval nixbuildnet nixfmt nixos @@ -368,6 +375,7 @@ nodetime nodiscover noexecstack nolint +notrunc numtide omit omnichain @@ -395,11 +403,13 @@ pflag phang pingpong pipefail +pkey pkgconfig pkgs pname poisonphang polkadot +postgrespassword posthandler precommit prehash @@ -434,12 +444,14 @@ randao rawfile rclone readarray +receiveds redelegate redelegating redelegation redelegations reentrancy reflectionv +regclass regen repr reqwest @@ -476,6 +488,7 @@ slashingtypes snapshotter snapshottypes solomachine +spents splitn srcs stakingkeeper diff --git a/flake.nix b/flake.nix index 7d0e572290..25dbafdcc4 100644 --- a/flake.nix +++ b/flake.nix @@ -225,50 +225,36 @@ }; }; - devShells = - let - baseShell = { - buildInputs = [ rust.toolchains.dev ] ++ (with pkgs; [ - bacon - cargo-nextest - go_1_20 - gopls - go-tools - gotools - jq - marksman - nil - nixfmt - nodejs - openssl - pkg-config - protobuf - solc - yarn - yq - foundry-bin - go-ethereum - hasura-cli - ]); - nativeBuildInputs = [ - config.treefmt.build.wrapper - ] ++ lib.attrsets.attrValues config.treefmt.build.programs; - GOPRIVATE = "github.com/unionlabs/*"; - }; - in - { - default = pkgs.mkShell baseShell; - githook = pkgs.mkShell (baseShell // { - inherit (self'.checks.pre-commit-check) shellHook; - }); - evm = pkgs.mkShell (baseShell // { - buildInputs = baseShell.buildInputs ++ [ - pkgs.solc - pkgs.foundry-bin - pkgs.go-ethereum - ]; - }); - }; + devShells.default = pkgs.mkShell { + name = "union-devShell"; + buildInputs = [ rust.toolchains.dev ] ++ (with pkgs; [ + bacon + cargo-nextest + go_1_20 + gopls + go-tools + gotools + jq + marksman + nil + nixfmt + nodejs + openssl + pkg-config + protobuf + solc + yarn + yq + foundry-bin + go-ethereum + hasura-cli + ]); + nativeBuildInputs = [ + config.treefmt.build.wrapper + ] ++ lib.attrsets.attrValues config.treefmt.build.programs; + GOPRIVATE = "github.com/unionlabs/*"; + }; + treefmt = let diff --git a/hubble/hubble.nix b/hubble/hubble.nix index 22a4b43e98..d3da01d31d 100644 --- a/hubble/hubble.nix +++ b/hubble/hubble.nix @@ -24,16 +24,82 @@ }; }; - hubble-migrations = pkgs.stdenv.mkDerivation { - name = "hubble-migrations"; - src = [./hasura ]; - installPhase = '' - mkdir -p $out - cp -r $src/metadata $out - cp -r $src/migrations $out - cp -r $src/seeds $out - ''; - }; + hasura-cli = + let + cli-ext-hashes = { + linux-amd64 = "sha256-4XDLKIO/nT6LVaM6aSBpvKOMGhW5ca04iA1PMGBTWI8="; + linux-arm64 = "sha256-GJ9Xtx1g0nIbirS9SOb/B8AeNhAMq6PDwx2HMiqXW9Q="; + darwin-amd64 = "sha256-9qoW9SlGVafNP/t8xh3JMkpwWZ4BDR0QuEJO8g2sLHg="; + darwin-arm64 = "sha256-wwWk/mOdnCeIan18Mj6qmwbmBUmtA8eqtR4g0UHrNBo="; + }; + cli-ext = pkgs.stdenv.mkDerivation rec { + pname = "hasura-cli-ext"; + version = "2.4.0-beta.3"; + src = pkgs.fetchurl { + url = "https://graphql-engine-cdn.hasura.io/cli-ext/releases/versioned/v${version}/cli-ext-${pkgs.go.GOOS}-${pkgs.go.GOARCH}"; + sha256 = cli-ext-hashes."${pkgs.go.GOOS}-${pkgs.go.GOARCH}"; + }; + dontUnpack = true; + phases = [ "installPhase" "fixupPhase" ]; + installPhase = '' + mkdir -p $out/bin + cp $src $out/bin/cli-ext + chmod +x $out/bin/cli-ext + ''; + preFixup = + let + libPath = pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc ]; + in + '' + orig_size=$(stat --printf=%s $out/bin/cli-ext) + + patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" $out/bin/cli-ext + patchelf --set-rpath ${libPath} $out/bin/cli-ext + new_size=$(stat --printf=%s $out/bin/cli-ext) + + ###### zeit-pkg fixing starts here. + # we're replacing plaintext js code that looks like + # PAYLOAD_POSITION = '1234 ' | 0 + # [...] + # PRELUDE_POSITION = '1234 ' | 0 + # ^-----20-chars-----^^------22-chars------^ + # ^-- grep points here + # + # var_* are as described above + # shift_by seems to be safe so long as all patchelf adjustments occur + # before any locations pointed to by hardcoded offsets + + var_skip=20 + var_select=22 + shift_by=$(expr $new_size - $orig_size) + + function fix_offset { + # $1 = name of variable to adjust + location=$(grep -obUam1 "$1" $out/bin/cli-ext | cut -d: -f1) + location=$(expr $location + $var_skip) + + value=$(dd if=$out/bin/cli-ext iflag=count_bytes,skip_bytes skip=$location \ + bs=1 count=$var_select status=none) + value=$(expr $shift_by + $value) + + echo -n $value | dd of=$out/bin/cli-ext bs=1 seek=$location conv=notrunc + } + + fix_offset PAYLOAD_POSITION + fix_offset PRELUDE_POSITION + ''; + dontStrip = true; + }; + in + pkgs.symlinkJoin { + name = "hasura"; + paths = [ pkgs.hasura-cli ]; + buildInputs = [ pkgs.makeWrapper ]; + postBuild = '' + wrapProgram $out/bin/hasura \ + --add-flags "--cli-ext-path ${cli-ext}/bin/cli-ext" + ''; + }; }; }; } diff --git a/hubble/src/graphql/operations.graphql b/hubble/src/graphql/operations.graphql index 69c471da59..f73a336542 100644 --- a/hubble/src/graphql/operations.graphql +++ b/hubble/src/graphql/operations.graphql @@ -12,4 +12,10 @@ query GetLatestBlock($chain_id: String!) { chains(where: {chain_id: {_eq: $chain_id}}) { id } -} \ No newline at end of file +} + +mutation InsertChain($chain_id: String!) { + insert_chains_one(object: {chain_id: $chain_id}) { + id + } +} diff --git a/hubble/src/hasura.rs b/hubble/src/hasura.rs index b94e275a5d..d5a147c576 100644 --- a/hubble/src/hasura.rs +++ b/hubble/src/hasura.rs @@ -75,3 +75,13 @@ pub struct InsertBlock; skip_serializing_none )] pub struct GetLatestBlock; + +#[derive(GraphQLQuery)] +#[graphql( + schema_path = "src/graphql/schema.graphql", + query_path = "src/graphql/operations.graphql", + response_derives = "Debug", + normalization = "rust", + skip_serializing_none +)] +pub struct InsertChain; diff --git a/hubble/src/main.rs b/hubble/src/main.rs index 7acbdf21f2..43f6b5e20a 100644 --- a/hubble/src/main.rs +++ b/hubble/src/main.rs @@ -1,15 +1,18 @@ #![feature(return_position_impl_trait_in_trait)] +#![feature(result_option_inspect)] use clap::Parser; use hasura::HasuraDataStore; use reqwest::Client; -use tracing::{error, info}; +use tokio::task::JoinSet; +use tracing::{error, info, warn}; + mod cli; mod hasura; mod tm; #[tokio::main] -async fn main() { +async fn main() -> color_eyre::eyre::Result<()> { color_eyre::install().unwrap(); let args = crate::cli::Args::parse(); @@ -19,17 +22,30 @@ async fn main() { let secret = args.secret.clone(); let client = Client::new(); let db = HasuraDataStore::new(client, url, secret); + let mut set = JoinSet::new(); - let handles = args.indexers.into_iter().map(|indexer| { + args.indexers.into_iter().for_each(|indexer| { let db = db.clone(); - async move { + set.spawn(async move { info!("starting indexer {:?}", indexer); - // indexer should never return with Ok, thus we log the error. - let result = indexer.index(db).await; - error!("indexer {:?} exited with: {:?}", &indexer, result); - } + let result = indexer.index(db).await.inspect_err(|err| { + warn!("indexer {:?} exited with: {:?}", &indexer, err); + }); + result + }); }); - futures::future::join_all(handles).await; + while let Some(res) = set.join_next().await { + if let Err(err) = res { + error!( + "encountered error while indexing: {:?}. shutting down.", + err + ); + set.shutdown().await; + return Err(err.into()); + } + info!("indexer exited gracefully"); + } + Ok(()) } diff --git a/hubble/src/tm.rs b/hubble/src/tm.rs index 812721e707..67c958b6c3 100644 --- a/hubble/src/tm.rs +++ b/hubble/src/tm.rs @@ -1,3 +1,5 @@ +use std::ops::Range; + use tendermint::genesis::Genesis; use tendermint_rpc::{error::ErrorDetail, response_error::Code, Client, Error, HttpClient}; use tokio::time::{sleep, Duration}; @@ -13,11 +15,13 @@ use crate::{ pub struct Config { pub url: Url, pub chain_id: Option, + pub range: Option>, } impl Config { pub async fn index(&self, db: D) -> Result<(), color_eyre::eyre::Report> { let client = HttpClient::new(self.url.as_str()).unwrap(); + let range = self.range.as_ref().unwrap_or(&(0..u32::MAX)); // If there is no chain_id override, we query it from the node. This // is the expected default. @@ -32,27 +36,12 @@ impl Config { } }; - // We query for the last indexed block to not waste resources re-indexing. - debug!("fetching latest stored block"); - let latest_stored = db - .do_post::(get_latest_block::Variables { chain_id }) - .await?; - - let data = latest_stored - .data - .expect("db should be prepared for indexing"); - - let mut height: u32 = if data.blocks.is_empty() { - 0 - } else { - TryInto::::try_into(data.blocks[0].height).unwrap() - }; - debug!("latest stored block height is: {}", &height); - - let chain_db_id = data.chains[0].id; - + let (mut height, chain_db_id) = get_current_data(&db, chain_id).await?; + height += 1; loop { - height += 1; + if !range.contains(&height) { + return Ok(()); + } info!("indexing block {}", &height); // if we're caught up indexing to the latest height, this will error. In that case, @@ -61,7 +50,7 @@ impl Config { let header = match client.block(height).await { Err(err) => { if is_height_exceeded_error(&err) { - debug!("caught up indexing, sleeping for 1 second"); + debug!("caught up indexing, sleeping for 1 second: {:?}", err); sleep(Duration::from_millis(1000)).await; continue; } else { @@ -135,12 +124,13 @@ impl Config { }; db.do_post::(v).await?; + height += 1; } } } /// The RPC will return an internal error on queries for blocks exceeding the current height. -/// `is_height_exceeded_error` unwrangles the error and checks for this case. +/// `is_height_exceeded_error` untangles the error and checks for this case. pub fn is_height_exceeded_error(err: &Error) -> bool { let detail = err.detail(); if let ErrorDetail::Response(err) = detail { @@ -152,3 +142,40 @@ pub fn is_height_exceeded_error(err: &Error) -> bool { } false } + +/// Obtains the current height and chain_db_id for the chain_id. If the chain_id is not stored yet, an entry is created. +async fn get_current_data( + db: &D, + chain_id: String, +) -> Result<(u32, i64), color_eyre::eyre::Report> { + // We query for the last indexed block to not waste resources re-indexing. + debug!("fetching latest stored block"); + let latest_stored = db + .do_post::(get_latest_block::Variables { + chain_id: chain_id.clone(), + }) + .await?; + + let data = latest_stored + .data + .expect("db should be prepared for indexing"); + + let height: u32 = if data.blocks.is_empty() { + 0 + } else { + TryInto::::try_into(data.blocks[0].height).unwrap() + }; + debug!("latest stored block height is: {}", &height); + + let chain_db_id = if let Some(chains) = data.chains.get(0) { + chains.id + } else { + let created = db + .do_post::(insert_chain::Variables { chain_id }) + .await?; + + created.data.unwrap().insert_chains_one.unwrap().id + }; + + Ok((height, chain_db_id)) +} diff --git a/networks/devnet.nix b/networks/devnet.nix index 30993125f5..ad22098dc7 100644 --- a/networks/devnet.nix +++ b/networks/devnet.nix @@ -1,5 +1,5 @@ { ... }: { - perSystem = { devnetConfig, pkgs, self', inputs', ... }: + perSystem = { devnetConfig, pkgs, lib, self', inputs', ... }: let arion = inputs'.arion.packages.default; @@ -33,16 +33,17 @@ postgres = import ./services/postgres.nix { }; }; - hasura-services = import ./services/hasura.nix { migrations = self'.packages.hubble-migrations }; + hasura-services = import ./services/hasura.nix { migrations = self'.packages.hubble-migrations; }; + hubble-services = { hubble = import ./services/hubble.nix { inherit lib; image = self'.packages.hubble-image; }; }; devnet = { project.name = "devnet"; - services = postgres-services // hasura-services; + services = sepolia-services // uniond-services // postgres-services // hasura-services // hubble-services; }; union = { project.name = "union"; - services = uniond-services; + services = uniond-services // postgres-services // hasura-services // hubble-services; }; sepolia = { @@ -55,11 +56,7 @@ }; spec-cosmos = { - modules = [{ - project.name = "union-devnet-cosmos"; - networks.union-devnet = { }; - services = uniond-services; - }]; + modules = [ (union // { networks.union-devnet = { }; }) ]; }; spec-evm = { diff --git a/networks/services/hasura.nix b/networks/services/hasura.nix index 50583d7c5d..731b963924 100644 --- a/networks/services/hasura.nix +++ b/networks/services/hasura.nix @@ -1,23 +1,23 @@ -{ migrations, ... }: +{ ... }: { hasura.service = { - image = "hasura/graphql-engine:v2.33.0.cli-migrations-v3"; + image = "hasura/graphql-engine:v2.33.1.cli-migrations-v3"; tty = true; stop_signal = "SIGINT"; ports = [ "8080:8080" ]; environment = { - HASURA_GRAPHQL_METADATA_DATABASE_URL = "postgres://postgres:postgrespassword@postgres:5432/postgres"; - PG_DATABASE_URL = "postgres://postgres:postgrespassword@postgres:5432/postgres"; + HASURA_GRAPHQL_METADATA_DATABASE_URL = "postgres://postgres:postgrespassword@postgres:5432/default"; + PG_DATABASE_URL = "postgres://postgres:postgrespassword@postgres:5432/default"; HASURA_GRAPHQL_ENABLE_CONSOLE = "true"; HASURA_GRAPHQL_DEV_MODE = "true"; HASURA_GRAPHQL_ADMIN_SECRET = "secret"; HASURA_GRAPHQL_METADATA_DEFAULTS = ''{"backend_configs":{"dataconnector":{"athena":{"uri":"http://data-connector-agent:8081/api/v1/athena"},"mariadb":{"uri":"http://data-connector-agent:8081/api/v1/mariadb"},"mysql8":{"uri":"http://data-connector-agent:8081/api/v1/mysql"},"oracle":{"uri":"http://data-connector-agent:8081/api/v1/oracle"},"snowflake":{"uri":"http://data-connector-agent:8081/api/v1/snowflake"}}}}''; }; - volumes = [ - "${migrations}/metatdata:/hasura-metatdata" - "${migrations}/migrations:/hasura-migrations" + volumes = [ + "${../../hubble/hasura/metadata}:/hasura-metadata" + "${../../hubble/hasura/migrations}:/hasura-migrations" ]; depends_on = { @@ -39,10 +39,9 @@ test = [ "CMD-SHELL" '' - curl -f http://localhost:8081/api/v1/athena/health + curl -f http://localhost:8081/api/v1/athena/health '' ]; }; }; } - diff --git a/networks/services/hubble.nix b/networks/services/hubble.nix new file mode 100644 index 0000000000..ced5f538cd --- /dev/null +++ b/networks/services/hubble.nix @@ -0,0 +1,21 @@ +{ lib, image, ... }: +{ + build.image = lib.mkForce image; + service = { + tty = true; + stop_signal = "SIGINT"; + network_mode = "host"; + environment = { + HUBBLE_URL = "http://localhost:8080/v1/graphql"; + HUBBLE_INDEXERS = ''{"type": "Tm", "url": "http://localhost:26657"}''; + HUBBLE_SECRET = "secret"; + RUST_LOG = "DEBUG"; + }; + depends_on = { + hasura = { + condition = "service_healthy"; + }; + uniond-0 = { }; + }; + }; +} diff --git a/networks/services/postgres.nix b/networks/services/postgres.nix index 439dc45102..71bb7b38dd 100644 --- a/networks/services/postgres.nix +++ b/networks/services/postgres.nix @@ -9,6 +9,7 @@ ]; environment = { POSTGRES_PASSWORD = "postgrespassword"; + POSTGRES_DB = "default"; }; }; } diff --git a/networks/services/uniond.nix b/networks/services/uniond.nix index 4ac21ddd4c..e7424e590c 100644 --- a/networks/services/uniond.nix +++ b/networks/services/uniond.nix @@ -47,6 +47,7 @@ in ]; healthcheck = { interval = "5s"; + start_period = "20s"; retries = 8; test = [ "CMD-SHELL"