feat(hubble): switch to timescaledb (#794)

cspell.json

@@ -28,6 +28,7 @@
     "**/*.pb.go",
     "**/testdata",
     "networks/genesis/union-testnet-2/**",
-    "networks/genesis/union-testnet-3/**"
+    "networks/genesis/union-testnet-3/**",
+    "hubble/hasura/**"
   ]
 }

dictionary.txt

@@ -31,6 +31,7 @@ Getenv
 HKTs
 HMAC
 Hasher
+Hhvf
 Hyperlane
 IAVL
 IIBC
@@ -400,6 +401,7 @@ nasm
 neighbouring
 neoeinstein
 neofetch
+neondb
 netutil
 networkid
 nextest
@@ -534,6 +536,7 @@ spents
 splitn
 sqlx
 srcs
+sslmode
 stakingkeeper
 stakingtypes
 stargate
@@ -599,6 +602,8 @@ tonumber
 treefmt
 trustlessly
 tryopen
+tsdb
+tsdbadmin
 txes
 txpool
 typecheck
@@ -628,6 +633,7 @@ unionvisorbn
 unjail
 unjailed
 unmarshal
+unopinionated
 unsetting
 unvalidated
 unwithdrawn

hubble/README.md

@@ -1,4 +1,38 @@
-Hubble is an omnichain indexer.
+# Overview
 
-> **Warning**
-> This is not ready for public usage. It can index tendermint chains, but the schema is not public yet.
+Hubble spawns multiple indexers inside [tokio](https://tokio.rs/) tasks. Each indexer is configured through a JSON config passed through the CLI:
+
+```
+--indexers '[{"type": "tendermint", "url": "http://localhost:26657"}]'
+```
+
+Each indexer inserts data into a Postgres DB. The migrations are included in the `hasura` directory.
+
+```mermaid
+erDiagram
+    Hubble ||--|{ Hasura : posts
+    Hasura ||--|{ Timescale : stores
+    User ||--|{ Hasura : queries
+```
+
+## Database Schema
+
+Hubble is unopinionated regarding database schema. It has three conceptual models, chains, blocks, and events. Events are stored as JSONB, which means that further destructuring and aggregation occurs inside the DB itself. For this, we recommend [Timescale](https://github.com/timescale/timescaledb).
+
+As an example, Tendermint's `coin_received` event can be destructured like so:
+
+```sql
+CREATE OR REPLACE VIEW "v0"."coin_receiveds" AS
+ SELECT events.block_id,
+    events.index,
+    events."time",
+    (((events.data -> 'attributes'::text) -> 0) ->> 'value'::text) AS receiver,
+    ("substring"((((events.data -> 'attributes'::text) -> 1) ->> 'value'::text), '^\d+'::text))::numeric AS amount,
+    "substring"((((events.data -> 'attributes'::text) -> 1) ->> 'value'::text), '[a-zA-Z].*'::text) AS denom
+   FROM v0.events
+  WHERE ((events.data ->> 'type'::text) = 'coin_received'::text);
+```
+
+DBAs should choose between using views or materialized views.
+
+Data can be aggregated using time_buckets, and subsequently summed into totals, i.e. aggregate transfers into 1h buckets, and then sum over these buckets to obtain the current balances. Use [real-time aggregates](https://docs.timescale.com/use-timescale/latest/continuous-aggregates/real-time-aggregates/) for this.

hubble/hasura/config.yaml

@@ -1,6 +1,5 @@
 version: 3
-endpoint: https://localhost:8080
-admin_secret: secret
+endpoint: https://noble-pika-27.hasura.app
 metadata_directory: metadata
 actions:
   kind: synchronous

hubble/hasura/metadata/databases/databases.yaml

@@ -6,15 +6,16 @@
         from_env: PG_DATABASE_URL
       isolation_level: read-committed
       use_prepared_statements: false
-  logical_models:
-    - fields:
-        - name: receiver
-          type:
-            nullable: false
-            scalar: text
-        - name: amount
-          type:
-            nullable: true
-            scalar: text
-      name: coin_received
   tables: "!include default/tables/tables.yaml"
+- name: timescale
+  kind: postgres
+  configuration:
+    connection_info:
+      database_url:
+        from_env: TIMESCALE_DATABASE_URL
+      isolation_level: read-committed
+      pool_settings:
+        connection_lifetime: 600
+        total_max_connections: 30
+      use_prepared_statements: false
+  tables: "!include timescale/tables/tables.yaml"

hubble/hasura/metadata/databases/default/tables/demo_faucet_claims.yaml

@@ -0,0 +1,3 @@
+table:
+  name: faucet_claims
+  schema: demo

hubble/hasura/metadata/databases/default/tables/demo_queue.yaml

@@ -0,0 +1,3 @@
+table:
+  name: queue
+  schema: demo

hubble/hasura/metadata/databases/default/tables/demo_txes.yaml

@@ -0,0 +1,3 @@
+table:
+  name: txes
+  schema: demo

hubble/hasura/metadata/databases/default/tables/public_blocks.yaml

@@ -1,28 +1,3 @@
 table:
   name: blocks
   schema: public
-object_relationships:
-  - name: chain
-    using:
-      foreign_key_constraint_on: chain_id
-array_relationships:
-  - name: events
-    using:
-      foreign_key_constraint_on:
-        column: block_id
-        table:
-          name: events
-          schema: public
-select_permissions:
-  - role: user
-    permission:
-      columns:
-        - is_finalized
-        - chain_id
-        - height
-        - id
-        - hash
-      filter: {}
-      limit: 50
-      allow_aggregations: true
-    comment: ""

hubble/hasura/metadata/databases/default/tables/public_chains.yaml

@@ -1,11 +1,3 @@
 table:
   name: chains
   schema: public
-array_relationships:
-  - name: blocks
-    using:
-      foreign_key_constraint_on:
-        column: chain_id
-        table:
-          name: blocks
-          schema: public

hubble/hasura/metadata/databases/default/tables/public_events.yaml

@@ -1,7 +1,3 @@
 table:
   name: events
   schema: public
-object_relationships:
-  - name: block
-    using:
-      foreign_key_constraint_on: block_id

hubble/hasura/metadata/databases/default/tables/tables.yaml

@@ -1,12 +1,6 @@
-- "!include public_addresses.yaml"
+- "!include demo_faucet_claims.yaml"
+- "!include demo_queue.yaml"
+- "!include demo_txes.yaml"
 - "!include public_blocks.yaml"
 - "!include public_chains.yaml"
-- "!include public_coin_receiveds.yaml"
-- "!include public_coin_spents.yaml"
-- "!include public_coinbases.yaml"
-- "!include public_commissions.yaml"
 - "!include public_events.yaml"
-- "!include public_messages.yaml"
-- "!include public_mints.yaml"
-- "!include public_rewards.yaml"
-- "!include public_transfers.yaml"

hubble/hasura/metadata/databases/timescale/tables/tables.yaml

@@ -0,0 +1,6 @@
+- "!include v0_blocks.yaml"
+- "!include v0_chains.yaml"
+- "!include v0_coin_receiveds.yaml"
+- "!include v0_coin_receiveds_30m.yaml"
+- "!include v0_coin_receiveds_totals.yaml"
+- "!include v0_events.yaml"

hubble/hasura/metadata/databases/timescale/tables/v0_blocks.yaml

@@ -0,0 +1,3 @@
+table:
+  name: blocks
+  schema: v0

hubble/hasura/metadata/databases/timescale/tables/v0_chains.yaml

@@ -0,0 +1,3 @@
+table:
+  name: chains
+  schema: v0

hubble/hasura/metadata/databases/timescale/tables/v0_coin_receiveds.yaml

@@ -0,0 +1,3 @@
+table:
+  name: coin_receiveds
+  schema: v0

hubble/hasura/metadata/databases/timescale/tables/v0_coin_receiveds_30m.yaml

@@ -0,0 +1,3 @@
+table:
+  name: coin_receiveds_30m
+  schema: v0

hubble/hasura/metadata/databases/timescale/tables/v0_coin_receiveds_totals.yaml

@@ -0,0 +1,3 @@
+table:
+  name: coin_receiveds_totals
+  schema: v0

hubble/hasura/metadata/databases/timescale/tables/v0_events.yaml

@@ -0,0 +1,3 @@
+table:
+  name: events
+  schema: v0

hubble/hasura/metadata/remote_schemas.yaml

@@ -1 +1,8 @@
-[]
+- name: faucet
+  definition:
+    url: https://faucet.0xc0dejug.uno/graphql
+    timeout_seconds: 60
+    customization:
+      root_fields_namespace: union
+    forward_client_headers: true
+  comment: ""