diff --git a/govtool/frontend/Makefile b/govtool/frontend/Makefile index 4c5c964bd..2721792b7 100644 --- a/govtool/frontend/Makefile +++ b/govtool/frontend/Makefile @@ -6,25 +6,19 @@ endif .DEFAULT_GOAL := push-frontend -# environment variables -gtm_id := $(shell echo $${GTM_ID}) -sentry_dsn := $(shell echo $${SENTRY_DSN}) - # image tags frontend_image_tag := $(shell git log -n 1 --format="%H" -- $(root_dir)/govtool/frontend) .PHONY: build-frontend build-frontend: docker-login @:$(call check_defined, cardano_network) - @:$(call check_defined, gtm_id) - @:$(call check_defined, sentry_dsn) if [[ "$(cardano_network)" = "mainnet" ]]; then NETWORK_FLAG=1; else NETWORK_FLAG=0; fi; \ $(call check_image_on_ecr,frontend,$(frontend_image_tag)) || \ $(docker) build --tag "$(repo_url)/frontend:$(frontend_image_tag)" \ --build-arg VITE_BASE_URL="https://$(domain)/api" \ - --build-arg VITE_GTM_ID="$(gtm_id)" \ + --build-arg VITE_GTM_ID="$${GTM_ID}" \ --build-arg VITE_NETWORK_FLAG="$$NETWORK_FLAG" \ - --build-arg VITE_SENTRY_DSN="$(sentry_dsn)" \ + --build-arg VITE_SENTRY_DSN="$${SENTRY_DSN}" \ $(root_dir)/govtool/frontend .PHONY: push-frontend diff --git a/scripts/govtool/Makefile b/scripts/govtool/Makefile index 619d73d6b..e20289a86 100644 --- a/scripts/govtool/Makefile +++ b/scripts/govtool/Makefile @@ -2,24 +2,10 @@ include ../../govtool/backend/Makefile include ../../govtool/frontend/Makefile include utils.mk include info.mk +include config.mk .DEFAULT_GOAL := info -# tools -curl ?= curl -rsync ?= rsync -ssh-keyscan ?= ssh-keyscan - -# environment variables -grafana_admin_password := $(shell echo $${GRAFANA_ADMIN_PASSWORD}) -grafana_slack_recipient := $(shell echo $${GRAFANA_SLACK_RECIPIENT}) -grafana_slack_oauth_token := $(shell echo $${GRAFANA_SLACK_OAUTH_TOKEN}) - -# target addresses -docker_host := $(domain) -docker_user := ubuntu -ssh_url := $(docker_user)@$(docker_host) - # stack configuration docker_compose_file := docker-compose.$(env).yml @@ -28,33 +14,16 @@ cardano_node_image_tag := 8.8.0-pre cardano_db_sync_image_tag := sancho-4-0-0-fix-config .PHONY: all -all: deploy-stack info #notify - -.PHONY: prepare-config -prepare-config: - @:$(call check_defined, cardano_network) - @:$(call check_defined, grafana_slack_recipient) - @:$(call check_defined, grafana_slack_oauth_token) - export CARDANO_NETWORK=$(cardano_network); \ - export GRAFANA_SLACK_RECIPIENT=$(grafana_slack_recipient); \ - export GRAFANA_SLACK_OAUTH_TOKEN=$(grafana_slack_oauth_token); \ - ./prepare-config.sh - -.PHONY: upload-config -upload-config: check-env-defined prepare-config - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) - $(rsync) -av -e 'ssh -o StrictHostKeyChecking=no' config/target/. $(ssh_url):config +all: deploy-stack info notify .PHONY: deploy-stack deploy-stack: upload-config push-backend push-frontend @:$(call check_defined, cardano_network) @:$(call check_defined, env) - @:$(call check_defined, grafana_admin_password) export CARDANO_NETWORK=$(cardano_network); \ export DOCKER_HOST=ssh://$(ssh_url); \ export ENVIRONMENT=$(env); \ - export GRAFANA_ADMIN_PASSWORD=$(grafana_admin_password); \ + export GRAFANA_ADMIN_PASSWORD=$${GRAFANA_ADMIN_PASSWORD}; \ export BACKEND_TAG=$(backend_image_tag); \ export FRONTEND_TAG=$(frontend_image_tag); \ export CARDANO_NODE_TAG=$(cardano_node_image_tag); \ diff --git a/scripts/govtool/common.mk b/scripts/govtool/common.mk index b80940a08..79d71b77a 100644 --- a/scripts/govtool/common.mk +++ b/scripts/govtool/common.mk @@ -4,6 +4,9 @@ ONESHELL: # tools docker ?= docker +curl ?= curl +rsync ?= rsync +ssh-keyscan ?= ssh-keyscan # environment variables env := $(shell echo $${ENVIRONMENT}) @@ -17,6 +20,9 @@ root_dir := $(shell git rev-parse --show-toplevel) # target addresses domain := $(shell echo $${DOMAIN}) repo_url ?= 733019650473.dkr.ecr.eu-west-1.amazonaws.com +docker_host := $(domain) +docker_user := ubuntu +ssh_url := $(docker_user)@$(docker_host) # stack configuration compose_stack_name := govtool-$(env)-$(cardano_network) diff --git a/scripts/govtool/config.mk b/scripts/govtool/config.mk new file mode 100644 index 000000000..45fb00880 --- /dev/null +++ b/scripts/govtool/config.mk @@ -0,0 +1,82 @@ +common_mk := ../../scripts/govtool/common.mk +ifeq ($(origin $(common_mk)), undefined) + $(eval $(common_mk) := included) + include $(common_mk) +endif + +# directory paths +config_dir := $(root_dir)/scripts/govtool/config +target_config_dir := $(config_dir)/target +template_config_dir := $(config_dir)/templates +cardano_node_config_dir := $(target_config_dir)/cardano-node +dbsync_secrets_dir := $(target_config_dir)/dbsync-secrets +grafana_provisioning_dir := $(target_config_dir)/grafana-provisioning +nginx_config_dir := $(target_config_dir)/nginx + +# metadata +cardano_config_provider := https://book.world.dev.cardano.org + +.PHONY: prepare-config +prepare-config: clear enable-prometheus prepare-dbsync-secrets prepare-backend-config prepare-prometheus-config prepare-grafana-provisioning prepare-nginx-config + +.PHONY: clear +clear: + rm -rf $(target_config_dir) + +.PHONY: fetch-cardano-node-config +fetch-cardano-node-config: + @:$(call check_defined, cardano_network) + mkdir -p $(cardano_node_config_dir) + $(curl) -s "$(cardano_config_provider)/env-$(cardano_network).html" | \ + grep -E -o '[a-z-]+\.json' | \ + sort -u | \ + xargs -I"{}" $(curl) -s "$(cardano_config_provider)/environments/$(cardano_network)/{}" -o "$(cardano_node_config_dir)/{}" + +.PHONY: enable-prometheus +enable-prometheus: fetch-cardano-node-config + sed -i '/"hasPrometheus"/ { N; s/"127\.0\.0\.1"/"0.0.0.0"/ }' "$(cardano_node_config_dir)/config.json" + +.PHONY: prepare-dbsync-secrets +prepare-dbsync-secrets: + mkdir -p $(dbsync_secrets_dir) + echo "$${DBSYNC_POSTGRES_USER}" > "$(dbsync_secrets_dir)/postgres_user"; \ + echo "$${DBSYNC_POSTGRES_PASSWORD}" > "$(dbsync_secrets_dir)/postgres_password"; \ + echo "$${DBSYNC_POSTGRES_DB}" > "$(dbsync_secrets_dir)/postgres_db" + +.PHONY: prepare-backend-config +prepare-backend-config: + sed -e "s/DBSYNC_POSTGRES_DB/$${DBSYNC_POSTGRES_DB}/" \ + -e "s/DBSYNC_POSTGRES_USER/$${DBSYNC_POSTGRES_USER}/" \ + -e "s/DBSYNC_POSTGRES_PASSWORD/$${DBSYNC_POSTGRES_PASSWORD}/" \ + -e "s|SENTRY_DSN|$${SENTRY_DSN_BACKEND}|" \ + "$(config_dir)/templates/backend-config.json.tpl" \ + > "$(target_config_dir)/backend-config.json" + +.PHONY: prepare-prometheus-config +prepare-prometheus-config: + cp -a "$(template_config_dir)/prometheus.yml" "$(target_config_dir)/prometheus.yml" + + PHONY: prepare-grafana-provisioning +prepare-grafana-provisioning: + mkdir -p $(grafana_provisioning_dir) + cp -a $(template_config_dir)/grafana-provisioning/* $(grafana_provisioning_dir) + sed -e "s/GRAFANA_SLACK_RECIPIENT/$${GRAFANA_SLACK_RECIPIENT}/" \ + -e "s|GRAFANA_SLACK_OAUTH_TOKEN|$${GRAFANA_SLACK_OAUTH_TOKEN}|" \ + -i $(grafana_provisioning_dir)/alerting/alerting.yml + +.PHONY: prepare-nginx-config +prepare-nginx-config: + @:$(call check_defined, domain) + mkdir -p $(nginx_config_dir) + touch "$(nginx_config_dir)/auth.conf" + touch "$(nginx_config_dir)/govtool.htpasswd" + if [[ "$(domain)" == *"sanchonet.govtool.byron.network"* ]]; then \ + echo "$${NGINX_BASIC_AUTH}" > "$(nginx_config_dir)/govtool.htpasswd"; \ + echo "auth_basic \"Restricted\";" > "$(nginx_config_dir)/auth.conf"; \ + echo "auth_basic_user_file /etc/nginx/conf.d/govtool.htpasswd;" >> "$(nginx_config_dir)/auth.conf"; \ + fi + +.PHONY: upload-config +upload-config: check-env-defined prepare-config + @:$(call check_defined, ssh_url) + $(rsync) -av -e 'ssh -o StrictHostKeyChecking=no' config/target/. $(ssh_url):config diff --git a/scripts/govtool/config/secrets/backend-config.json b/scripts/govtool/config/secrets/backend-config.json deleted file mode 100644 index d60da33f1..000000000 --- a/scripts/govtool/config/secrets/backend-config.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "dbsyncconfig" : { - "host" : "postgres", - "dbname" : "cexplorer", - "user" : "postgres", - "password" : "pSa8JCpQOACMUdGb", - "port" : 5432 - }, - "fakedbsyncconfig" : { - "host" : "postgres-fakedbsync", - "dbname" : "govtool", - "user" : "test", - "password" : "test", - "port" : 5432 - }, - "port" : 9876, - "host" : "0.0.0.0" -} diff --git a/scripts/govtool/config/secrets/postgres_db b/scripts/govtool/config/secrets/postgres_db deleted file mode 100644 index cdc4e3a7e..000000000 --- a/scripts/govtool/config/secrets/postgres_db +++ /dev/null @@ -1 +0,0 @@ -cexplorer \ No newline at end of file diff --git a/scripts/govtool/config/secrets/postgres_password b/scripts/govtool/config/secrets/postgres_password deleted file mode 100644 index 1a27864b8..000000000 --- a/scripts/govtool/config/secrets/postgres_password +++ /dev/null @@ -1 +0,0 @@ -pSa8JCpQOACMUdGb diff --git a/scripts/govtool/config/secrets/postgres_user b/scripts/govtool/config/secrets/postgres_user deleted file mode 100644 index 7d72bd782..000000000 --- a/scripts/govtool/config/secrets/postgres_user +++ /dev/null @@ -1 +0,0 @@ -postgres \ No newline at end of file diff --git a/scripts/govtool/config/secrets/backend-config.json.tpl b/scripts/govtool/config/templates/backend-config.json.tpl similarity index 100% rename from scripts/govtool/config/secrets/backend-config.json.tpl rename to scripts/govtool/config/templates/backend-config.json.tpl diff --git a/scripts/govtool/config/grafana-provisioning/alerting/alerting.yml b/scripts/govtool/config/templates/grafana-provisioning/alerting/alerting.yml similarity index 100% rename from scripts/govtool/config/grafana-provisioning/alerting/alerting.yml rename to scripts/govtool/config/templates/grafana-provisioning/alerting/alerting.yml diff --git a/scripts/govtool/config/grafana-provisioning/dashboards/dashboard.yml b/scripts/govtool/config/templates/grafana-provisioning/dashboards/dashboard.yml similarity index 100% rename from scripts/govtool/config/grafana-provisioning/dashboards/dashboard.yml rename to scripts/govtool/config/templates/grafana-provisioning/dashboards/dashboard.yml diff --git a/scripts/govtool/config/grafana-provisioning/dashboards/govtool.json b/scripts/govtool/config/templates/grafana-provisioning/dashboards/govtool.json similarity index 100% rename from scripts/govtool/config/grafana-provisioning/dashboards/govtool.json rename to scripts/govtool/config/templates/grafana-provisioning/dashboards/govtool.json diff --git a/scripts/govtool/config/grafana-provisioning/dashboards/traefik_rev4.json b/scripts/govtool/config/templates/grafana-provisioning/dashboards/traefik_rev4.json similarity index 100% rename from scripts/govtool/config/grafana-provisioning/dashboards/traefik_rev4.json rename to scripts/govtool/config/templates/grafana-provisioning/dashboards/traefik_rev4.json diff --git a/scripts/govtool/config/grafana-provisioning/datasources/datasource.yml b/scripts/govtool/config/templates/grafana-provisioning/datasources/datasource.yml similarity index 100% rename from scripts/govtool/config/grafana-provisioning/datasources/datasource.yml rename to scripts/govtool/config/templates/grafana-provisioning/datasources/datasource.yml diff --git a/scripts/govtool/config/templates/prometheus.yml b/scripts/govtool/config/templates/prometheus.yml new file mode 100644 index 000000000..0d637f3f1 --- /dev/null +++ b/scripts/govtool/config/templates/prometheus.yml @@ -0,0 +1,23 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + monitor: 'govtool' +scrape_configs: + - job_name: 'traefik' + scrape_interval: 5s + static_configs: + - targets: ['traefik:8082'] + - job_name: 'cardano' + scrape_interval: 5s + static_configs: + - targets: ['cardano-node:12798'] + - job_name: 'cardano_db_sync' + scrape_interval: 5s + metrics_path: / + static_configs: + - targets: ['cardano-db-sync:8080'] + - job_name: 'host' + scrape_interval: 5s + static_configs: + - targets: ['host.docker.internal:9100'] diff --git a/scripts/govtool/info.mk b/scripts/govtool/info.mk index d3ac1d552..719651c6e 100644 --- a/scripts/govtool/info.mk +++ b/scripts/govtool/info.mk @@ -28,7 +28,7 @@ notify: @:$(call check_defined, cardano_network) @:$(call check_defined, env) $(curl) -X POST https://slack.com/api/chat.postMessage\ - -H "Authorization: Bearer $(grafana_slack_oauth_token)" \ + -H "Authorization: Bearer $${GRAFANA_SLACK_OAUTH_TOKEN}" \ -H "Content-Type: application/json; charset=utf-8" \ --data "{ \"channel\":\"$(grafana_slack_recipient)\", \"text\":\":rocket: *Deploy performed on \`$(env)\`*\n- from *branch* \`$(branch)\` (\`$(commit)\`),\n- using *Cardano Node* version \`$(cardano_node_image_tag)\`,\n- using *Cardano DB Sync* version \`$(cardano_db_sync_image_tag)\`,\n- using *GovTool backend* version \`$(backend_image_tag)\`,\n- using *Govtool frontend* version \`$(frontend_image_tag)\`.\n$(pipeline_info)\" }" diff --git a/scripts/govtool/prepare-config.sh b/scripts/govtool/prepare-config.sh deleted file mode 100755 index 78ce75a9d..000000000 --- a/scripts/govtool/prepare-config.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -set -eu pipefail - -repo_root_dir="$(git rev-parse --show-toplevel)" -config_dir="$repo_root_dir/scripts/govtool/config" -target_config_dir="$repo_root_dir/scripts/govtool/config/target" -mkdir -p "$target_config_dir" - -# cardano node config -cardano_node_config_dir="$target_config_dir/cardano-node" -mkdir -p "$cardano_node_config_dir" -for file in $(curl -s https://book.world.dev.cardano.org/env-"$CARDANO_NETWORK".html | grep -E -o '[a-z-]+\.json' | sort -u); do - curl -s "https://book.world.dev.cardano.org/environments/$CARDANO_NETWORK/$file" -o "$cardano_node_config_dir/$file" -done -# enable prometheus in node config -sed -i '/"hasPrometheus"/ { N; s/"127\.0\.0\.1"/"0.0.0.0"/ }' "$cardano_node_config_dir/config.json" - -# dbsync secret files -dbsync_secrets_dir="$target_config_dir/dbsync-secrets" -mkdir -p "$dbsync_secrets_dir" -echo "$DBSYNC_POSTGRES_USER" > "$dbsync_secrets_dir/postgres_user" -echo "$DBSYNC_POSTGRES_PASSWORD" > "$dbsync_secrets_dir/postgres_password" -echo "$DBSYNC_POSTGRES_DB" > "$dbsync_secrets_dir/postgres_db" - -# backend config file -sed -e "s/DBSYNC_POSTGRES_DB/$DBSYNC_POSTGRES_DB/" \ - -e "s/DBSYNC_POSTGRES_USER/$DBSYNC_POSTGRES_USER/" \ - -e "s/DBSYNC_POSTGRES_PASSWORD/$DBSYNC_POSTGRES_PASSWORD/" \ - -e "s|SENTRY_DSN|$SENTRY_DSN_BACKEND|" \ - "$config_dir/secrets/backend-config.json.tpl" \ - > "$target_config_dir/backend-config.json" - -# prometheus config file -cat >"$target_config_dir/prometheus.yml" <<_EOF_ -global: - scrape_interval: 15s - evaluation_interval: 15s - external_labels: - monitor: 'govtool' -scrape_configs: - - job_name: 'traefik' - scrape_interval: 5s - static_configs: - - targets: ['traefik:8082'] - - job_name: 'cardano' - scrape_interval: 5s - static_configs: - - targets: ['cardano-node:12798'] - - job_name: 'cardano_db_sync' - scrape_interval: 5s - metrics_path: / - static_configs: - - targets: ['cardano-db-sync:8080'] - - job_name: 'host' - scrape_interval: 5s - static_configs: - - targets: ['host.docker.internal:9100'] -_EOF_ - -# grafana provisioning dir -grafana_provisioning_dir="$target_config_dir/grafana-provisioning" -mkdir -p "$grafana_provisioning_dir" -cp -a "$config_dir/grafana-provisioning/"* "$grafana_provisioning_dir" -sed -e "s/GRAFANA_SLACK_RECIPIENT/$GRAFANA_SLACK_RECIPIENT/" \ - -e "s|GRAFANA_SLACK_OAUTH_TOKEN|$GRAFANA_SLACK_OAUTH_TOKEN|" \ - -i "$grafana_provisioning_dir/alerting/alerting.yml" - -# nginx config for frontend optional basic auth -nginx_config_dir="$target_config_dir/nginx" -rm -rf "$nginx_config_dir" -mkdir -p "$nginx_config_dir" -if [[ "$DOMAIN" == *"sanchonet.govtool.byron.network"* ]]; then - cat >"$nginx_config_dir/auth.conf" <<_EOF_ -auth_basic "Restricted"; -auth_basic_user_file /etc/nginx/conf.d/govtool.htpasswd; -_EOF_ - cat >"$nginx_config_dir/govtool.htpasswd" <<_EOF_ -$NGINX_BASIC_AUTH -_EOF_ -else - # create empty files if no basic auth is needed - touch "$nginx_config_dir/auth.conf" - touch "$nginx_config_dir/govtool.htpasswd" -fi diff --git a/scripts/govtool/utils.mk b/scripts/govtool/utils.mk index 37ccda6e6..3cb2bcc27 100644 --- a/scripts/govtool/utils.mk +++ b/scripts/govtool/utils.mk @@ -1,7 +1,5 @@ .PHONY: ssh ssh: - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) export TERM=xterm-256color; \ ssh $(ssh_url) @@ -9,12 +7,11 @@ ssh: docker: @:$(call check_defined, cardano_network) @:$(call check_defined, env) - @:$(call check_defined, grafana_admin_password) @:$(call check_defined, cmd) export CARDANO_NETWORK=$(cardano_network); \ export DOCKER_HOST=ssh://$(ssh_url); \ export ENVIRONMENT=$(env); \ - export GRAFANA_ADMIN_PASSWORD=$(grafana_admin_password); \ + export GRAFANA_ADMIN_PASSWORD=$${GRAFANA_ADMIN_PASSWORD}; \ export BACKEND_TAG=$(backend_image_tag); \ export FRONTEND_TAG=$(frontend_image_tag); \ export CARDANO_NODE_TAG=$(cardano_node_image_tag); \