From 79861f23bdfc5b9bc8cbef5f74c9628175fe02e8 Mon Sep 17 00:00:00 2001 From: "Karel L. Kubat" Date: Fri, 1 Sep 2023 15:03:45 +0100 Subject: [PATCH] feat(hubble): add migrations --- hubble/hasura/config.yaml | 3 +- hubble/hasura/metadata/api_limits.yaml | 1 + hubble/hasura/metadata/backend_configs.yaml | 13 ++ .../hasura/metadata/databases/databases.yaml | 21 ++- .../default/tables/public_addresses.yaml | 3 + .../default/tables/public_blocks.yaml | 28 ++++ .../default/tables/public_chains.yaml | 11 ++ .../default/tables/public_coin_receiveds.yaml | 13 ++ .../default/tables/public_coin_spents.yaml | 3 + .../default/tables/public_coinbases.yaml | 3 + .../default/tables/public_commissions.yaml | 3 + .../default/tables/public_events.yaml | 7 + .../default/tables/public_messages.yaml | 3 + .../default/tables/public_mints.yaml | 3 + .../default/tables/public_rewards.yaml | 3 + .../default/tables/public_transfers.yaml | 3 + .../databases/default/tables/tables.yaml | 12 ++ .../graphql_schema_introspection.yaml | 1 + hubble/hasura/metadata/inherited_roles.yaml | 1 + hubble/hasura/metadata/metrics_config.yaml | 2 + hubble/hasura/metadata/network.yaml | 1 + hubble/hasura/metadata/opentelemetry.yaml | 1 + hubble/hasura/metadata/rest_endpoints.yaml | 1 + .../default/1693576035737_init/up.sql | 138 ++++++++++++++++++ hubble/hasura/seeds/.gitkeep | 0 hubble/hubble.nix | 11 ++ networks/devnet.nix | 2 +- networks/services/hasura.nix | 9 +- 28 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 hubble/hasura/metadata/api_limits.yaml create mode 100644 hubble/hasura/metadata/backend_configs.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_addresses.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_blocks.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_chains.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_coin_receiveds.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_coin_spents.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_coinbases.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_commissions.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_events.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_messages.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_mints.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_rewards.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/public_transfers.yaml create mode 100644 hubble/hasura/metadata/databases/default/tables/tables.yaml create mode 100644 hubble/hasura/metadata/graphql_schema_introspection.yaml create mode 100644 hubble/hasura/metadata/inherited_roles.yaml create mode 100644 hubble/hasura/metadata/metrics_config.yaml create mode 100644 hubble/hasura/metadata/network.yaml create mode 100644 hubble/hasura/metadata/opentelemetry.yaml create mode 100644 hubble/hasura/metadata/rest_endpoints.yaml create mode 100644 hubble/hasura/migrations/default/1693576035737_init/up.sql create mode 100644 hubble/hasura/seeds/.gitkeep diff --git a/hubble/hasura/config.yaml b/hubble/hasura/config.yaml index 725c8005e8..168ae40c0a 100644 --- a/hubble/hasura/config.yaml +++ b/hubble/hasura/config.yaml @@ -1,5 +1,6 @@ version: 3 -endpoint: http://localhost:8080 +endpoint: https://noble-pika-27.hasura.app +admin_secret: 3N5Mt2f4Y1AC7dE663AsGqRy66yiHBuZ3RMgUjM6X4Q3Ma8G2yKmVoasdasdsadasda metadata_directory: metadata actions: kind: synchronous diff --git a/hubble/hasura/metadata/api_limits.yaml b/hubble/hasura/metadata/api_limits.yaml new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/hubble/hasura/metadata/api_limits.yaml @@ -0,0 +1 @@ +{} diff --git a/hubble/hasura/metadata/backend_configs.yaml b/hubble/hasura/metadata/backend_configs.yaml new file mode 100644 index 0000000000..3a181b59bb --- /dev/null +++ b/hubble/hasura/metadata/backend_configs.yaml @@ -0,0 +1,13 @@ +dataconnector: + athena: + uri: http://localhost:8081/api/v1/athena + mariadb: + uri: http://localhost:8081/api/v1/mariadb + mongodb: + uri: http://localhost:8082 + mysql8: + uri: http://localhost:8081/api/v1/mysql + oracle: + uri: http://localhost:8081/api/v1/oracle + snowflake: + uri: http://localhost:8081/api/v1/snowflake diff --git a/hubble/hasura/metadata/databases/databases.yaml b/hubble/hasura/metadata/databases/databases.yaml index fe51488c70..529da07e2e 100644 --- a/hubble/hasura/metadata/databases/databases.yaml +++ b/hubble/hasura/metadata/databases/databases.yaml @@ -1 +1,20 @@ -[] +- name: default + kind: postgres + configuration: + connection_info: + database_url: + from_env: PG_DATABASE_URL + isolation_level: read-committed + use_prepared_statements: false + logical_models: + - fields: + - name: receiver + type: + nullable: false + scalar: text + - name: amount + type: + nullable: true + scalar: text + name: coin_received + tables: "!include default/tables/tables.yaml" diff --git a/hubble/hasura/metadata/databases/default/tables/public_addresses.yaml b/hubble/hasura/metadata/databases/default/tables/public_addresses.yaml new file mode 100644 index 0000000000..d1ab2f0a1e --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_addresses.yaml @@ -0,0 +1,3 @@ +table: + name: addresses + schema: public diff --git a/hubble/hasura/metadata/databases/default/tables/public_blocks.yaml b/hubble/hasura/metadata/databases/default/tables/public_blocks.yaml new file mode 100644 index 0000000000..48aa022221 --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_blocks.yaml @@ -0,0 +1,28 @@ +table: + name: blocks + schema: public +object_relationships: + - name: chain + using: + foreign_key_constraint_on: chain_id +array_relationships: + - name: events + using: + foreign_key_constraint_on: + column: block_id + table: + name: events + schema: public +select_permissions: + - role: user + permission: + columns: + - is_finalized + - chain_id + - height + - id + - hash + filter: {} + limit: 50 + allow_aggregations: true + comment: "" diff --git a/hubble/hasura/metadata/databases/default/tables/public_chains.yaml b/hubble/hasura/metadata/databases/default/tables/public_chains.yaml new file mode 100644 index 0000000000..955eda3d28 --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_chains.yaml @@ -0,0 +1,11 @@ +table: + name: chains + schema: public +array_relationships: + - name: blocks + using: + foreign_key_constraint_on: + column: chain_id + table: + name: blocks + schema: public diff --git a/hubble/hasura/metadata/databases/default/tables/public_coin_receiveds.yaml b/hubble/hasura/metadata/databases/default/tables/public_coin_receiveds.yaml new file mode 100644 index 0000000000..8d34373a4a --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_coin_receiveds.yaml @@ -0,0 +1,13 @@ +table: + name: coin_receiveds + schema: public +object_relationships: + - name: event + using: + manual_configuration: + column_mapping: + id: id + insertion_order: null + remote_table: + name: events + schema: public diff --git a/hubble/hasura/metadata/databases/default/tables/public_coin_spents.yaml b/hubble/hasura/metadata/databases/default/tables/public_coin_spents.yaml new file mode 100644 index 0000000000..45caa1e9ff --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_coin_spents.yaml @@ -0,0 +1,3 @@ +table: + name: coin_spents + schema: public diff --git a/hubble/hasura/metadata/databases/default/tables/public_coinbases.yaml b/hubble/hasura/metadata/databases/default/tables/public_coinbases.yaml new file mode 100644 index 0000000000..5475368642 --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_coinbases.yaml @@ -0,0 +1,3 @@ +table: + name: coinbases + schema: public diff --git a/hubble/hasura/metadata/databases/default/tables/public_commissions.yaml b/hubble/hasura/metadata/databases/default/tables/public_commissions.yaml new file mode 100644 index 0000000000..8cd682769b --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_commissions.yaml @@ -0,0 +1,3 @@ +table: + name: commissions + schema: public diff --git a/hubble/hasura/metadata/databases/default/tables/public_events.yaml b/hubble/hasura/metadata/databases/default/tables/public_events.yaml new file mode 100644 index 0000000000..bb01c3f5d9 --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_events.yaml @@ -0,0 +1,7 @@ +table: + name: events + schema: public +object_relationships: + - name: block + using: + foreign_key_constraint_on: block_id diff --git a/hubble/hasura/metadata/databases/default/tables/public_messages.yaml b/hubble/hasura/metadata/databases/default/tables/public_messages.yaml new file mode 100644 index 0000000000..b5b80939f7 --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_messages.yaml @@ -0,0 +1,3 @@ +table: + name: messages + schema: public diff --git a/hubble/hasura/metadata/databases/default/tables/public_mints.yaml b/hubble/hasura/metadata/databases/default/tables/public_mints.yaml new file mode 100644 index 0000000000..d8ae1a8bd4 --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_mints.yaml @@ -0,0 +1,3 @@ +table: + name: mints + schema: public diff --git a/hubble/hasura/metadata/databases/default/tables/public_rewards.yaml b/hubble/hasura/metadata/databases/default/tables/public_rewards.yaml new file mode 100644 index 0000000000..814e1fe9c6 --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_rewards.yaml @@ -0,0 +1,3 @@ +table: + name: rewards + schema: public diff --git a/hubble/hasura/metadata/databases/default/tables/public_transfers.yaml b/hubble/hasura/metadata/databases/default/tables/public_transfers.yaml new file mode 100644 index 0000000000..4426a1ba1b --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/public_transfers.yaml @@ -0,0 +1,3 @@ +table: + name: transfers + schema: public diff --git a/hubble/hasura/metadata/databases/default/tables/tables.yaml b/hubble/hasura/metadata/databases/default/tables/tables.yaml new file mode 100644 index 0000000000..98e03afc62 --- /dev/null +++ b/hubble/hasura/metadata/databases/default/tables/tables.yaml @@ -0,0 +1,12 @@ +- "!include public_addresses.yaml" +- "!include public_blocks.yaml" +- "!include public_chains.yaml" +- "!include public_coin_receiveds.yaml" +- "!include public_coin_spents.yaml" +- "!include public_coinbases.yaml" +- "!include public_commissions.yaml" +- "!include public_events.yaml" +- "!include public_messages.yaml" +- "!include public_mints.yaml" +- "!include public_rewards.yaml" +- "!include public_transfers.yaml" diff --git a/hubble/hasura/metadata/graphql_schema_introspection.yaml b/hubble/hasura/metadata/graphql_schema_introspection.yaml new file mode 100644 index 0000000000..61a4dcac29 --- /dev/null +++ b/hubble/hasura/metadata/graphql_schema_introspection.yaml @@ -0,0 +1 @@ +disabled_for_roles: [] diff --git a/hubble/hasura/metadata/inherited_roles.yaml b/hubble/hasura/metadata/inherited_roles.yaml new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/hubble/hasura/metadata/inherited_roles.yaml @@ -0,0 +1 @@ +[] diff --git a/hubble/hasura/metadata/metrics_config.yaml b/hubble/hasura/metadata/metrics_config.yaml new file mode 100644 index 0000000000..33b18c3b3b --- /dev/null +++ b/hubble/hasura/metadata/metrics_config.yaml @@ -0,0 +1,2 @@ +analyze_query_variables: true +analyze_response_body: true diff --git a/hubble/hasura/metadata/network.yaml b/hubble/hasura/metadata/network.yaml new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/hubble/hasura/metadata/network.yaml @@ -0,0 +1 @@ +{} diff --git a/hubble/hasura/metadata/opentelemetry.yaml b/hubble/hasura/metadata/opentelemetry.yaml new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/hubble/hasura/metadata/opentelemetry.yaml @@ -0,0 +1 @@ +{} diff --git a/hubble/hasura/metadata/rest_endpoints.yaml b/hubble/hasura/metadata/rest_endpoints.yaml new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/hubble/hasura/metadata/rest_endpoints.yaml @@ -0,0 +1 @@ +[] diff --git a/hubble/hasura/migrations/default/1693576035737_init/up.sql b/hubble/hasura/migrations/default/1693576035737_init/up.sql new file mode 100644 index 0000000000..ad6c0b2d2a --- /dev/null +++ b/hubble/hasura/migrations/default/1693576035737_init/up.sql @@ -0,0 +1,138 @@ +SET check_function_bodies = false; +CREATE TABLE public.addresses ( + address text NOT NULL, + chain_id integer NOT NULL, + id integer NOT NULL +); +CREATE SEQUENCE public.addresses_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE public.addresses_id_seq OWNED BY public.addresses.id; +CREATE TABLE public.blocks ( + hash text NOT NULL, + chain_id integer NOT NULL, + height integer NOT NULL, + id integer NOT NULL, + is_finalized boolean DEFAULT false +); +CREATE SEQUENCE public.blocks_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE public.blocks_id_seq OWNED BY public.blocks.id; +CREATE TABLE public.chains ( + id integer NOT NULL, + chain_id text NOT NULL +); +CREATE SEQUENCE public.chains_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE public.chains_id_seq OWNED BY public.chains.id; +CREATE TABLE public.events ( + id integer NOT NULL, + index integer NOT NULL, + data jsonb NOT NULL, + block_id integer NOT NULL +); +CREATE VIEW public.coin_receiveds AS + SELECT events.id, + (((events.data -> 'attributes'::text) -> 0) ->> 'value'::text) AS receiver, + "substring"((((events.data -> 'attributes'::text) -> 1) ->> 'value'::text), '^\d+'::text) AS amount, + "substring"((((events.data -> 'attributes'::text) -> 1) ->> 'value'::text), '[a-zA-Z].*'::text) AS denom + FROM public.events + WHERE ((events.data ->> 'type'::text) = 'coin_received'::text); +CREATE VIEW public.coin_spents AS + SELECT events.id, + (((events.data -> 'attributes'::text) -> 0) ->> 'value'::text) AS spender, + "substring"((((events.data -> 'attributes'::text) -> 1) ->> 'value'::text), '^\d+'::text) AS amount, + "substring"((((events.data -> 'attributes'::text) -> 1) ->> 'value'::text), '[a-zA-Z].*'::text) AS denom + FROM public.events + WHERE ((events.data ->> 'type'::text) = 'coin_spent'::text); +CREATE VIEW public.coinbases AS + SELECT events.id, + (((events.data -> 'attributes'::text) -> 0) ->> 'value'::text) AS minter, + "substring"((((events.data -> 'attributes'::text) -> 1) ->> 'value'::text), '^\d+'::text) AS amount, + "substring"((((events.data -> 'attributes'::text) -> 1) ->> 'value'::text), '[a-zA-Z].*'::text) AS denom + FROM public.events + WHERE ((events.data ->> 'type'::text) = 'coinbase'::text); +CREATE VIEW public.commissions AS + SELECT events.id, + (((events.data -> 'attributes'::text) -> 1) ->> 'value'::text) AS validator, + "substring"((((events.data -> 'attributes'::text) -> 0) ->> 'value'::text), '^\d+'::text) AS amount, + "substring"((((events.data -> 'attributes'::text) -> 0) ->> 'value'::text), '[a-zA-Z].*'::text) AS denom + FROM public.events + WHERE ((events.data ->> 'type'::text) = 'commission'::text); +CREATE SEQUENCE public.events_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER SEQUENCE public.events_id_seq OWNED BY public.events.id; +CREATE VIEW public.messages AS + SELECT events.id, + (((events.data -> 'attributes'::text) -> 0) ->> 'value'::text) AS sender + FROM public.events + WHERE ((events.data ->> 'type'::text) = 'message'::text); +CREATE VIEW public.mints AS + SELECT events.id, + (((events.data -> 'attributes'::text) -> 0) ->> 'value'::text) AS bonded_ratio, + (((events.data -> 'attributes'::text) -> 1) ->> 'value'::text) AS inflation, + (((events.data -> 'attributes'::text) -> 2) ->> 'value'::text) AS annual_provisions, + (((events.data -> 'attributes'::text) -> 3) ->> 'value'::text) AS amount + FROM public.events + WHERE ((events.data ->> 'type'::text) = 'mint'::text); +CREATE VIEW public.rewards AS + SELECT events.id, + (((events.data -> 'attributes'::text) -> 1) ->> 'value'::text) AS validator, + "substring"((((events.data -> 'attributes'::text) -> 0) ->> 'value'::text), '^\d+'::text) AS amount, + "substring"((((events.data -> 'attributes'::text) -> 0) ->> 'value'::text), '[a-zA-Z].*'::text) AS denom + FROM public.events + WHERE ((events.data ->> 'type'::text) = 'rewards'::text); +CREATE VIEW public.transfers AS + SELECT events.id, + (((events.data -> 'attributes'::text) -> 0) ->> 'value'::text) AS recipient, + (((events.data -> 'attributes'::text) -> 1) ->> 'value'::text) AS sender, + "substring"((((events.data -> 'attributes'::text) -> 2) ->> 'value'::text), '^\d+'::text) AS amount, + "substring"((((events.data -> 'attributes'::text) -> 2) ->> 'value'::text), '[a-zA-Z].*'::text) AS denom + FROM public.events + WHERE ((events.data ->> 'type'::text) = 'transfer'::text); +ALTER TABLE ONLY public.addresses ALTER COLUMN id SET DEFAULT nextval('public.addresses_id_seq'::regclass); +ALTER TABLE ONLY public.blocks ALTER COLUMN id SET DEFAULT nextval('public.blocks_id_seq'::regclass); +ALTER TABLE ONLY public.chains ALTER COLUMN id SET DEFAULT nextval('public.chains_id_seq'::regclass); +ALTER TABLE ONLY public.events ALTER COLUMN id SET DEFAULT nextval('public.events_id_seq'::regclass); +ALTER TABLE ONLY public.addresses + ADD CONSTRAINT addresses_address_chain_id_key UNIQUE (address, chain_id); +ALTER TABLE ONLY public.addresses + ADD CONSTRAINT addresses_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.blocks + ADD CONSTRAINT blocks_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.chains + ADD CONSTRAINT chains_id_key UNIQUE (id); +ALTER TABLE ONLY public.chains + ADD CONSTRAINT chains_pkey PRIMARY KEY (id, chain_id); +ALTER TABLE ONLY public.events + ADD CONSTRAINT events_pkey PRIMARY KEY (id); +CREATE INDEX coin_received_idx ON public.events USING btree (((data ->> 'type'::text))); +CREATE INDEX idx_blocks_chain_id ON public.blocks USING btree (chain_id); +CREATE INDEX idx_blocks_chain_id_height ON public.blocks USING btree (chain_id, height); +CREATE INDEX idx_blocks_height_desc ON public.blocks USING btree (height DESC); +CREATE INDEX idx_chains_chain_id ON public.chains USING btree (chain_id); +CREATE INDEX idx_events_block_id ON public.events USING btree (block_id); +CREATE INDEX receiver ON public.events USING gin (((((data -> 'attributes'::text) -> 0) -> 'value'::text))) WHERE ((data ->> 'type'::text) = 'coin_received'::text); +ALTER TABLE ONLY public.blocks + ADD CONSTRAINT blocks_chain_id_fkey FOREIGN KEY (chain_id) REFERENCES public.chains(id) ON UPDATE CASCADE ON DELETE CASCADE; +ALTER TABLE ONLY public.events + ADD CONSTRAINT events_block_id_fkey FOREIGN KEY (block_id) REFERENCES public.blocks(id) ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/hubble/hasura/seeds/.gitkeep b/hubble/hasura/seeds/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hubble/hubble.nix b/hubble/hubble.nix index c686f3cf2f..22a4b43e98 100644 --- a/hubble/hubble.nix +++ b/hubble/hubble.nix @@ -23,6 +23,17 @@ Env = [ "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" ]; }; }; + + hubble-migrations = pkgs.stdenv.mkDerivation { + name = "hubble-migrations"; + src = [./hasura ]; + installPhase = '' + mkdir -p $out + cp -r $src/metadata $out + cp -r $src/migrations $out + cp -r $src/seeds $out + ''; + }; }; }; } diff --git a/networks/devnet.nix b/networks/devnet.nix index 09e731e8e0..30993125f5 100644 --- a/networks/devnet.nix +++ b/networks/devnet.nix @@ -33,7 +33,7 @@ postgres = import ./services/postgres.nix { }; }; - hasura-services = import ./services/hasura.nix { }; + hasura-services = import ./services/hasura.nix { migrations = self'.packages.hubble-migrations }; devnet = { project.name = "devnet"; diff --git a/networks/services/hasura.nix b/networks/services/hasura.nix index 5a4c820f8d..50583d7c5d 100644 --- a/networks/services/hasura.nix +++ b/networks/services/hasura.nix @@ -1,7 +1,7 @@ -{ ... }: +{ migrations, ... }: { hasura.service = { - image = "hasura/graphql-engine:v2.33.0"; + image = "hasura/graphql-engine:v2.33.0.cli-migrations-v3"; tty = true; stop_signal = "SIGINT"; ports = [ @@ -15,6 +15,11 @@ HASURA_GRAPHQL_ADMIN_SECRET = "secret"; HASURA_GRAPHQL_METADATA_DEFAULTS = ''{"backend_configs":{"dataconnector":{"athena":{"uri":"http://data-connector-agent:8081/api/v1/athena"},"mariadb":{"uri":"http://data-connector-agent:8081/api/v1/mariadb"},"mysql8":{"uri":"http://data-connector-agent:8081/api/v1/mysql"},"oracle":{"uri":"http://data-connector-agent:8081/api/v1/oracle"},"snowflake":{"uri":"http://data-connector-agent:8081/api/v1/snowflake"}}}}''; }; + volumes = [ + "${migrations}/metatdata:/hasura-metatdata" + "${migrations}/migrations:/hasura-migrations" + ]; + depends_on = { data-connector-agent = { condition = "service_healthy";