feat(hubble): add indexing to devnets

latest

fix

hasura-cli

feat(hubble): add indexing to devnets

dictionary.txt

@@ -21,6 +21,7 @@ DATADIR
 DYLD
 Denoms
 Feegrant
+GOARCH
 GOPATH
 GOPRIVATE
 Getenv
@@ -151,6 +152,7 @@ codegen
 codespace
 coeff
 cofactor
+coinbases
 cometbft
 cometbls
 cometbn
@@ -163,6 +165,7 @@ consensusparamkeeper
 consensusparamtypes
 consensys
 consts
+conv
 converstion
 coreutils
 corsdomain
@@ -179,6 +182,7 @@ cratedir
 crisiskeeper
 crisistypes
 ctypes
+dataconnector
 datadir
 datetime
 delegators
@@ -215,6 +219,7 @@ feegrant
 feegrantkeeper
 feegrantmodule
 fetchurl
+fkey
 fksmg
 floorlog
 foldl
@@ -292,6 +297,7 @@ icahostkeeper
 icahosttypes
 icatypes
 identifing
+iflag
 ilog
 imap
 impls
@@ -359,6 +365,7 @@ neofetch
 netutil
 networkid
 nextest
+nextval
 nixbuildnet
 nixfmt
 nixos
@@ -368,6 +375,7 @@ nodetime
 nodiscover
 noexecstack
 nolint
+notrunc
 numtide
 omit
 omnichain
@@ -395,11 +403,13 @@ pflag
 phang
 pingpong
 pipefail
+pkey
 pkgconfig
 pkgs
 pname
 poisonphang
 polkadot
+postgrespassword
 posthandler
 precommit
 prehash
@@ -434,12 +444,14 @@ randao
 rawfile
 rclone
 readarray
+receiveds
 redelegate
 redelegating
 redelegation
 redelegations
 reentrancy
 reflectionv
+regclass
 regen
 repr
 reqwest
@@ -476,6 +488,7 @@ slashingtypes
 snapshotter
 snapshottypes
 solomachine
+spents
 splitn
 srcs
 stakingkeeper

flake.nix

@@ -225,50 +225,36 @@
             };
           };
 
-          devShells =
-            let
-              baseShell = {
-                buildInputs = [ rust.toolchains.dev ] ++ (with pkgs; [
-                  bacon
-                  cargo-nextest
-                  go_1_20
-                  gopls
-                  go-tools
-                  gotools
-                  jq
-                  marksman
-                  nil
-                  nixfmt
-                  nodejs
-                  openssl
-                  pkg-config
-                  protobuf
-                  solc
-                  yarn
-                  yq
-                  foundry-bin
-                  go-ethereum
-                  hasura-cli
-                ]);
-                nativeBuildInputs = [
-                  config.treefmt.build.wrapper
-                ] ++ lib.attrsets.attrValues config.treefmt.build.programs;
-                GOPRIVATE = "github.com/unionlabs/*";
-              };
-            in
-            {
-              default = pkgs.mkShell baseShell;
-              githook = pkgs.mkShell (baseShell // {
-                inherit (self'.checks.pre-commit-check) shellHook;
-              });
-              evm = pkgs.mkShell (baseShell // {
-                buildInputs = baseShell.buildInputs ++ [
-                  pkgs.solc
-                  pkgs.foundry-bin
-                  pkgs.go-ethereum
-                ];
-              });
-            };
+          devShells.default = pkgs.mkShell {
+            name = "union-devShell";
+            buildInputs = [ rust.toolchains.dev ] ++ (with pkgs; [
+              bacon
+              cargo-nextest
+              go_1_20
+              gopls
+              go-tools
+              gotools
+              jq
+              marksman
+              nil
+              nixfmt
+              nodejs
+              openssl
+              pkg-config
+              protobuf
+              solc
+              yarn
+              yq
+              foundry-bin
+              go-ethereum
+              hasura-cli
+            ]);
+            nativeBuildInputs = [
+              config.treefmt.build.wrapper
+            ] ++ lib.attrsets.attrValues config.treefmt.build.programs;
+            GOPRIVATE = "github.com/unionlabs/*";
+          };
+
 
           treefmt =
             let

hubble/hubble.nix

@@ -24,16 +24,82 @@
           };
         };
 
-        hubble-migrations = pkgs.stdenv.mkDerivation {
-          name = "hubble-migrations";
-          src = [./hasura ];
-          installPhase = ''
-            mkdir -p $out
-            cp -r $src/metadata $out
-            cp -r $src/migrations $out
-            cp -r $src/seeds $out
-          '';
-        };
+        hasura-cli =
+          let
+            cli-ext-hashes = {
+              linux-amd64 = "sha256-4XDLKIO/nT6LVaM6aSBpvKOMGhW5ca04iA1PMGBTWI8=";
+              linux-arm64 = "sha256-GJ9Xtx1g0nIbirS9SOb/B8AeNhAMq6PDwx2HMiqXW9Q=";
+              darwin-amd64 = "sha256-9qoW9SlGVafNP/t8xh3JMkpwWZ4BDR0QuEJO8g2sLHg=";
+              darwin-arm64 = "sha256-wwWk/mOdnCeIan18Mj6qmwbmBUmtA8eqtR4g0UHrNBo=";
+            };
+            cli-ext = pkgs.stdenv.mkDerivation rec {
+              pname = "hasura-cli-ext";
+              version = "2.4.0-beta.3";
+              src = pkgs.fetchurl {
+                url = "https://graphql-engine-cdn.hasura.io/cli-ext/releases/versioned/v${version}/cli-ext-${pkgs.go.GOOS}-${pkgs.go.GOARCH}";
+                sha256 = cli-ext-hashes."${pkgs.go.GOOS}-${pkgs.go.GOARCH}";
+              };
+              dontUnpack = true;
+              phases = [ "installPhase" "fixupPhase" ];
+              installPhase = ''
+                mkdir -p $out/bin
+                cp $src $out/bin/cli-ext
+                chmod +x $out/bin/cli-ext
+              '';
+              preFixup =
+                let
+                  libPath = pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc ];
+                in
+                ''
+                  orig_size=$(stat --printf=%s $out/bin/cli-ext)
+
+                  patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" $out/bin/cli-ext
+                  patchelf --set-rpath ${libPath} $out/bin/cli-ext
+                  new_size=$(stat --printf=%s $out/bin/cli-ext)
+
+                  ###### zeit-pkg fixing starts here.
+                  # we're replacing plaintext js code that looks like
+                  # PAYLOAD_POSITION = '1234                  ' | 0
+                  # [...]
+                  # PRELUDE_POSITION = '1234                  ' | 0
+                  # ^-----20-chars-----^^------22-chars------^
+                  # ^-- grep points here
+                  #
+                  # var_* are as described above
+                  # shift_by seems to be safe so long as all patchelf adjustments occur
+                  # before any locations pointed to by hardcoded offsets
+
+                  var_skip=20
+                  var_select=22
+                  shift_by=$(expr $new_size - $orig_size)
+
+                  function fix_offset {
+                    # $1 = name of variable to adjust
+                    location=$(grep -obUam1 "$1" $out/bin/cli-ext | cut -d: -f1)
+                    location=$(expr $location + $var_skip)
+
+                    value=$(dd if=$out/bin/cli-ext iflag=count_bytes,skip_bytes skip=$location \
+                               bs=1 count=$var_select status=none)
+                    value=$(expr $shift_by + $value)
+
+                    echo -n $value | dd of=$out/bin/cli-ext bs=1 seek=$location conv=notrunc
+                  }
+
+                  fix_offset PAYLOAD_POSITION
+                  fix_offset PRELUDE_POSITION
+                '';
+              dontStrip = true;
+            };
+          in
+          pkgs.symlinkJoin {
+            name = "hasura";
+            paths = [ pkgs.hasura-cli ];
+            buildInputs = [ pkgs.makeWrapper ];
+            postBuild = ''
+              wrapProgram $out/bin/hasura \
+                --add-flags "--cli-ext-path ${cli-ext}/bin/cli-ext"
+            '';
+          };
       };
     };
 }

hubble/src/graphql/operations.graphql

@@ -12,4 +12,10 @@ query GetLatestBlock($chain_id: String!) {
   chains(where: {chain_id: {_eq: $chain_id}}) {
     id
   }
-}
+}
+
+mutation InsertChain($chain_id: String!) {
+  insert_chains_one(object: {chain_id: $chain_id}) {
+    id
+  }
+}

hubble/src/hasura.rs

@@ -75,3 +75,13 @@ pub struct InsertBlock;
     skip_serializing_none
 )]
 pub struct GetLatestBlock;
+
+#[derive(GraphQLQuery)]
+#[graphql(
+    schema_path = "src/graphql/schema.graphql",
+    query_path = "src/graphql/operations.graphql",
+    response_derives = "Debug",
+    normalization = "rust",
+    skip_serializing_none
+)]
+pub struct InsertChain;

hubble/src/main.rs

@@ -1,15 +1,18 @@
 #![feature(return_position_impl_trait_in_trait)]
+#![feature(result_option_inspect)]
 
 use clap::Parser;
 use hasura::HasuraDataStore;
 use reqwest::Client;
-use tracing::{error, info};
+use tokio::task::JoinSet;
+use tracing::{error, info, warn};
+
 mod cli;
 mod hasura;
 mod tm;
 
 #[tokio::main]
-async fn main() {
+async fn main() -> color_eyre::eyre::Result<()> {
     color_eyre::install().unwrap();
 
     let args = crate::cli::Args::parse();
@@ -19,17 +22,30 @@ async fn main() {
     let secret = args.secret.clone();
     let client = Client::new();
     let db = HasuraDataStore::new(client, url, secret);
+    let mut set = JoinSet::new();
 
-    let handles = args.indexers.into_iter().map(|indexer| {
+    args.indexers.into_iter().for_each(|indexer| {
         let db = db.clone();
-        async move {
+        set.spawn(async move {
             info!("starting indexer {:?}", indexer);
-
             // indexer should never return with Ok, thus we log the error.
-            let result = indexer.index(db).await;
-            error!("indexer {:?} exited with: {:?}", &indexer, result);
-        }
+            let result = indexer.index(db).await.inspect_err(|err| {
+                warn!("indexer {:?} exited with: {:?}", &indexer, err);
+            });
+            result
+        });
     });
 
-    futures::future::join_all(handles).await;
+    while let Some(res) = set.join_next().await {
+        if let Err(err) = res {
+            error!(
+                "encountered error while indexing: {:?}. shutting down.",
+                err
+            );
+            set.shutdown().await;
+            return Err(err.into());
+        }
+        info!("indexer exited gracefully");
+    }
+    Ok(())
 }