4145 pg cron

.github/workflows/build-test-deploy.yml

@@ -30,7 +30,7 @@ jobs:
       # Label used to access the service container
       postgres:
         # Docker Hub image
-        image: postgres
+        image: ghcr.io/connext/database:sha-fa66dca
         # Provide the password for postgres
         env:
           POSTGRES_PASSWORD: qwerty
@@ -519,9 +519,8 @@ jobs:
     if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/testnet-prod' || github.ref == 'refs/heads/prod'
     runs-on: ubuntu-latest
     env:
-      REGISTRY: 679752396206.dkr.ecr.us-east-1.amazonaws.com
-      IMAGE_TAG: ${{ github.sha }}
-      REPOSITORY: nxtp-cartographer
+      REGISTRY: ghcr.io
+      IMAGE_NAME: connext/sequencer-subscriber
 
     steps:
       - name: Checkout repository

build-docker-images.sh

@@ -22,3 +22,6 @@ docker build --tag lighthouse:latest --file ./docker/lighthouse/Dockerfile .
 echo "Docker: building cartographer-transfers and cartographer-routers images..."
 docker build --tag cartographer-transfers:latest --file ./docker/cartographer/transfers/Dockerfile .
 docker build --tag cartographer-routers:latest --file ./docker/cartographer/routers/Dockerfile .
+
+echo "Docker: building db image..."
+docker build --tag db:latest --file ./docker/db/Dockerfile .

docker/db/Dockerfile

@@ -0,0 +1,7 @@
+FROM postgres:14
+
+# Update and install the pg_cron extension
+RUN apt-get update && apt-get install -y postgresql-14-cron
+
+RUN echo "shared_preload_libraries='pg_cron'" >> /usr/share/postgresql/postgresql.conf.sample
+RUN echo "cron.database_name='connext'" >> /usr/share/postgresql/postgresql.conf.sample

ops/mainnet/prod/backend/main.tf

@@ -43,7 +43,7 @@ module "cartographer_db" {
     Domain      = var.domain
   }
 
-  parameter_group_name = "default.postgres14"
+  parameter_group_name = "rds-postgres"
   vpc_id               = module.network.vpc_id
 
   hosted_zone_id             = data.aws_route53_zone.primary.zone_id

ops/modules/db/main.tf

@@ -64,3 +64,17 @@ resource "aws_route53_record" "db" {
   records = [aws_db_instance.db.address]
 }
 
+resource "aws_db_parameter_group" "rds_postgres" {
+  name   = "rds-postgres"
+  family = "postgres14"
+
+  parameter {
+    name  = "shared_preload_libraries"
+    value = "pg_cron"
+  }
+
+  parameter {
+    name  = "cron.database_name"
+    value = var.name
+  }
+}

ops/testnet/prod/backend/main.tf

@@ -43,7 +43,7 @@ module "cartographer_db" {
     Domain      = var.domain
   }
 
-  parameter_group_name = "default.postgres14"
+  parameter_group_name = "rds-postgres"
   vpc_id               = module.network.vpc_id
 
   hosted_zone_id             = data.aws_route53_zone.primary.zone_id

packages/adapters/database/README.md

@@ -1,9 +1,15 @@
 # Local Setup
 
+- Build Postgres image:
+
+```sh
+docker build --tag db:latest --file ./docker/db/Dockerfile .
+```
+
 - Run Postgres locally:
 
 ```sh
-docker run -p 5432:5432 -e POSTGRES_PASSWORD=qwerty postgres
+docker run --name cartographer-db --rm -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=qwerty db
 ```
 
 - Run database migrations:
@@ -43,3 +49,11 @@ Create the Typescript schema using [Zapatos](https://jawj.github.io/zapatos/):
 ```sh
 yarn workspace @connext/nxtp-adapters-database zapatos
 ```
+
+# Materialized Views
+
+Checking `pg_cron` scheduled runs:
+
+```
+select * from cron.job_run_details
+```

packages/adapters/database/db/migrations/20230510210620_pg_cron_for_mat_view.sql

@@ -0,0 +1,9 @@
+-- migrate:up
+CREATE EXTENSION IF NOT EXISTS pg_cron;
+
+-- Create a cron job to refresh the materialized views every 30 mins
+SELECT cron.schedule('30 * * * *', $$REFRESH MATERIALIZED VIEW CONCURRENTLY public.daily_transfer_volume$$);
+SELECT cron.schedule('30 * * * *', $$REFRESH MATERIALIZED VIEW CONCURRENTLY public.hourly_transfer_volume$$);
+
+-- migrate:down
+DROP EXTENSION IF EXISTS pg_cron;

packages/adapters/database/db/schema.sql

@@ -9,6 +9,27 @@ SET xmloption = content;
 SET client_min_messages = warning;
 SET row_security = off;
 
+--
+-- Name: pg_cron; Type: EXTENSION; Schema: -; Owner: -
+--
+
+CREATE EXTENSION IF NOT EXISTS pg_cron WITH SCHEMA pg_catalog;
+
+
+--
+-- Name: EXTENSION pg_cron; Type: COMMENT; Schema: -; Owner: -
+--
+
+COMMENT ON EXTENSION pg_cron IS 'Job scheduler for PostgreSQL';
+
+
+--
+-- Name: public; Type: SCHEMA; Schema: -; Owner: -
+--
+
+-- *not* creating schema, since initdb creates it
+
+
 --
 -- Name: action_type; Type: TYPE; Schema: public; Owner: -
 --
@@ -453,10 +474,10 @@ CREATE VIEW public.transfers_with_price AS
 
 
 --
--- Name: daily_transfer_volume; Type: VIEW; Schema: public; Owner: -
+-- Name: daily_transfer_volume; Type: MATERIALIZED VIEW; Schema: public; Owner: -
 --
 
-CREATE VIEW public.daily_transfer_volume AS
+CREATE MATERIALIZED VIEW public.daily_transfer_volume AS
  SELECT tf.status,
     (date_trunc('day'::text, to_timestamp((tf.xcall_timestamp)::double precision)))::date AS transfer_date,
     tf.origin_domain AS origin_chain,
@@ -465,9 +486,11 @@ CREATE VIEW public.daily_transfer_volume AS
     tf.origin_transacting_asset AS asset,
     sum((tf.origin_transacting_amount)::numeric) AS volume,
     avg(tf.asset_usd_price) AS avg_price,
-    sum(tf.usd_amount) AS usd_volume
+    sum(tf.usd_amount) AS usd_volume,
+    row_number() OVER () AS id
    FROM public.transfers_with_price tf
-  GROUP BY tf.status, ((date_trunc('day'::text, to_timestamp((tf.xcall_timestamp)::double precision)))::date), tf.origin_domain, tf.destination_domain, (regexp_replace((tf.routers)::text, '[\{\}]'::text, ''::text, 'g'::text)), tf.origin_transacting_asset;
+  GROUP BY tf.status, ((date_trunc('day'::text, to_timestamp((tf.xcall_timestamp)::double precision)))::date), tf.origin_domain, tf.destination_domain, (regexp_replace((tf.routers)::text, '[\{\}]'::text, ''::text, 'g'::text)), tf.origin_transacting_asset
+  WITH NO DATA;
 
 
 --
@@ -550,10 +573,10 @@ CREATE VIEW public.hourly_transfer_metrics AS
 
 
 --
--- Name: hourly_transfer_volume; Type: VIEW; Schema: public; Owner: -
+-- Name: hourly_transfer_volume; Type: MATERIALIZED VIEW; Schema: public; Owner: -
 --
 
-CREATE VIEW public.hourly_transfer_volume AS
+CREATE MATERIALIZED VIEW public.hourly_transfer_volume AS
  SELECT tf.status,
     date_trunc('hour'::text, to_timestamp((tf.xcall_timestamp)::double precision)) AS transfer_hour,
     tf.origin_domain AS origin_chain,
@@ -562,9 +585,11 @@ CREATE VIEW public.hourly_transfer_volume AS
     tf.origin_transacting_asset AS asset,
     sum((tf.origin_transacting_amount)::numeric) AS volume,
     avg(tf.asset_usd_price) AS avg_price,
-    sum(tf.usd_amount) AS usd_volume
+    sum(tf.usd_amount) AS usd_volume,
+    row_number() OVER () AS id
    FROM public.transfers_with_price tf
-  GROUP BY tf.status, (date_trunc('hour'::text, to_timestamp((tf.xcall_timestamp)::double precision))), tf.origin_domain, tf.destination_domain, (regexp_replace((tf.routers)::text, '[\{\}]'::text, ''::text, 'g'::text)), tf.origin_transacting_asset;
+  GROUP BY tf.status, (date_trunc('hour'::text, to_timestamp((tf.xcall_timestamp)::double precision))), tf.origin_domain, tf.destination_domain, (regexp_replace((tf.routers)::text, '[\{\}]'::text, ''::text, 'g'::text)), tf.origin_transacting_asset
+  WITH NO DATA;
 
 
 --
@@ -804,22 +829,6 @@ CREATE VIEW public.transfer_count AS
   GROUP BY tf.status, ((date_trunc('day'::text, to_timestamp((tf.xcall_timestamp)::double precision)))::date), tf.origin_domain, tf.origin_transacting_asset;
 
 
---
--- Name: transfer_volume; Type: VIEW; Schema: public; Owner: -
---
-
-CREATE VIEW public.transfer_volume AS
- SELECT tf.status,
-    (date_trunc('day'::text, to_timestamp((tf.xcall_timestamp)::double precision)))::date AS transfer_day,
-    tf.origin_domain AS origin_chain,
-    tf.origin_transacting_asset AS asset,
-    sum((tf.origin_transacting_amount)::numeric) AS volume,
-    avg(tf.asset_usd_price) AS avg_price,
-    sum(tf.usd_amount) AS usd_volume
-   FROM public.transfers_with_price tf
-  GROUP BY tf.status, ((date_trunc('day'::text, to_timestamp((tf.xcall_timestamp)::double precision)))::date), tf.origin_domain, tf.origin_transacting_asset;
-
-
 --
 -- Name: transfers_with_ttr_ttv; Type: VIEW; Schema: public; Owner: -
 --
@@ -1157,6 +1166,34 @@ ALTER TABLE ONLY public.transfers
 CREATE INDEX asset_prices_timestamp ON public.asset_prices USING btree ("timestamp");
 
 
+--
+-- Name: daily_transfer_volume_id_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX daily_transfer_volume_id_idx ON public.daily_transfer_volume USING btree (id);
+
+
+--
+-- Name: hourly_transfer_volume_id_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX hourly_transfer_volume_id_idx ON public.hourly_transfer_volume USING btree (id);
+
+
+--
+-- Name: idx_daily_transfer_volume_transfer_date; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_daily_transfer_volume_transfer_date ON public.daily_transfer_volume USING btree (transfer_date);
+
+
+--
+-- Name: idx_hourly_transfer_volume_transfer_hour; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX idx_hourly_transfer_volume_transfer_hour ON public.hourly_transfer_volume USING btree (transfer_hour);
+
+
 --
 -- Name: messages_processed_index_idx; Type: INDEX; Schema: public; Owner: -
 --
@@ -1192,6 +1229,20 @@ CREATE INDEX transfers_status_xcall_timestamp_idx ON public.transfers USING btre
 CREATE INDEX transfers_xcall_timestamp ON public.transfers USING btree (xcall_timestamp);
 
 
+--
+-- Name: transfers_xcall_transaction_hash_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX transfers_xcall_transaction_hash_idx ON public.transfers USING btree (xcall_transaction_hash);
+
+
+--
+-- Name: transfers_xcall_tx_origin_idx; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX transfers_xcall_tx_origin_idx ON public.transfers USING btree (xcall_tx_origin);
+
+
 --
 -- Name: transfers update_time_on_transfers; Type: TRIGGER; Schema: public; Owner: -
 --
@@ -1215,6 +1266,32 @@ ALTER TABLE ONLY public.asset_balances
     ADD CONSTRAINT fk_router FOREIGN KEY (router_address) REFERENCES public.routers(address);
 
 
+--
+-- Name: job cron_job_policy; Type: POLICY; Schema: cron; Owner: -
+--
+
+CREATE POLICY cron_job_policy ON cron.job USING ((username = CURRENT_USER));
+
+
+--
+-- Name: job_run_details cron_job_run_details_policy; Type: POLICY; Schema: cron; Owner: -
+--
+
+CREATE POLICY cron_job_run_details_policy ON cron.job_run_details USING ((username = CURRENT_USER));
+
+
+--
+-- Name: job; Type: ROW SECURITY; Schema: cron; Owner: -
+--
+
+ALTER TABLE cron.job ENABLE ROW LEVEL SECURITY;
+
+--
+-- Name: job_run_details; Type: ROW SECURITY; Schema: cron; Owner: -
+--
+
+ALTER TABLE cron.job_run_details ENABLE ROW LEVEL SECURITY;
+
 --
 -- PostgreSQL database dump complete
 --
@@ -1291,4 +1368,5 @@ INSERT INTO public.schema_migrations (version) VALUES
     ('20230508151158'),
     ('20230509112648'),
     ('20230509123037'),
-    ('20230509165732');
+    ('20230509165732'),
+    ('20230510210620');

packages/adapters/database/package.json

@@ -18,7 +18,7 @@
     "verify": "yarn test && yarn clean && yarn build && yarn lint --max-warnings 0",
     "purge": "yarn clean && rimraf ./coverage ./node_modules",
     "tsp": "yarn run tsp:root \"$@\" --cwd $(pwd)",
-    "docker:start:postgres": "docker run --name cartographer-db --rm -p 5432:5432 -e POSTGRES_PASSWORD=qwerty postgres",
+    "docker:start:postgres": "docker run --name cartographer-db --rm -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=qwerty ghcr.io/connext/database:sha-fa66dca",
     "zapatos": "zapatos",
     "dbmate": "dbmate",
     "version": "yarn version"