From 31f7b8cebaedfc1cf4ace3d326f18b6625e27718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Placzy=C5=84ski?= Date: Tue, 5 Mar 2024 07:42:55 +0100 Subject: [PATCH] [#385] Integrate prepare-config script functionality into Makefile system This commit eliminates the prepare-config.sh script, incorporating its configuration preparation logic into the Makefile workflow with a new config.mk file. This adjustment enhances the consistency of the deployment process, aligning it with the modular Makefile approach. It simplifies the management of configurations by dynamically generating necessary files from templates, thereby improving the security and maintainability of configuration secrets. Adjustments to related Makefiles files have been made to accommodate these changes, streamlining the entire configuration and deployment pipeline. --- govtool/frontend/Makefile | 10 +-- scripts/govtool/Makefile | 37 +------- scripts/govtool/common.mk | 6 ++ scripts/govtool/config.mk | 82 ++++++++++++++++++ .../config/secrets/backend-config.json | 18 ---- scripts/govtool/config/secrets/postgres_db | 1 - .../govtool/config/secrets/postgres_password | 1 - scripts/govtool/config/secrets/postgres_user | 1 - .../backend-config.json.tpl | 0 .../alerting/alerting.yml | 0 .../dashboards/dashboard.yml | 0 .../dashboards/govtool.json | 0 .../dashboards/traefik_rev4.json | 0 .../datasources/datasource.yml | 0 .../govtool/config/templates/prometheus.yml | 23 +++++ scripts/govtool/info.mk | 2 +- scripts/govtool/prepare-config.sh | 85 ------------------- scripts/govtool/utils.mk | 5 +- 18 files changed, 118 insertions(+), 153 deletions(-) create mode 100644 scripts/govtool/config.mk delete mode 100644 scripts/govtool/config/secrets/backend-config.json delete mode 100644 scripts/govtool/config/secrets/postgres_db delete mode 100644 scripts/govtool/config/secrets/postgres_password delete mode 100644 scripts/govtool/config/secrets/postgres_user rename scripts/govtool/config/{secrets => templates}/backend-config.json.tpl (100%) rename scripts/govtool/config/{ => templates}/grafana-provisioning/alerting/alerting.yml (100%) rename scripts/govtool/config/{ => templates}/grafana-provisioning/dashboards/dashboard.yml (100%) rename scripts/govtool/config/{ => templates}/grafana-provisioning/dashboards/govtool.json (100%) rename scripts/govtool/config/{ => templates}/grafana-provisioning/dashboards/traefik_rev4.json (100%) rename scripts/govtool/config/{ => templates}/grafana-provisioning/datasources/datasource.yml (100%) create mode 100644 scripts/govtool/config/templates/prometheus.yml delete mode 100755 scripts/govtool/prepare-config.sh diff --git a/govtool/frontend/Makefile b/govtool/frontend/Makefile index 4c5c964bd..2721792b7 100644 --- a/govtool/frontend/Makefile +++ b/govtool/frontend/Makefile @@ -6,25 +6,19 @@ endif .DEFAULT_GOAL := push-frontend -# environment variables -gtm_id := $(shell echo $${GTM_ID}) -sentry_dsn := $(shell echo $${SENTRY_DSN}) - # image tags frontend_image_tag := $(shell git log -n 1 --format="%H" -- $(root_dir)/govtool/frontend) .PHONY: build-frontend build-frontend: docker-login @:$(call check_defined, cardano_network) - @:$(call check_defined, gtm_id) - @:$(call check_defined, sentry_dsn) if [[ "$(cardano_network)" = "mainnet" ]]; then NETWORK_FLAG=1; else NETWORK_FLAG=0; fi; \ $(call check_image_on_ecr,frontend,$(frontend_image_tag)) || \ $(docker) build --tag "$(repo_url)/frontend:$(frontend_image_tag)" \ --build-arg VITE_BASE_URL="https://$(domain)/api" \ - --build-arg VITE_GTM_ID="$(gtm_id)" \ + --build-arg VITE_GTM_ID="$${GTM_ID}" \ --build-arg VITE_NETWORK_FLAG="$$NETWORK_FLAG" \ - --build-arg VITE_SENTRY_DSN="$(sentry_dsn)" \ + --build-arg VITE_SENTRY_DSN="$${SENTRY_DSN}" \ $(root_dir)/govtool/frontend .PHONY: push-frontend diff --git a/scripts/govtool/Makefile b/scripts/govtool/Makefile index 619d73d6b..e20289a86 100644 --- a/scripts/govtool/Makefile +++ b/scripts/govtool/Makefile @@ -2,24 +2,10 @@ include ../../govtool/backend/Makefile include ../../govtool/frontend/Makefile include utils.mk include info.mk +include config.mk .DEFAULT_GOAL := info -# tools -curl ?= curl -rsync ?= rsync -ssh-keyscan ?= ssh-keyscan - -# environment variables -grafana_admin_password := $(shell echo $${GRAFANA_ADMIN_PASSWORD}) -grafana_slack_recipient := $(shell echo $${GRAFANA_SLACK_RECIPIENT}) -grafana_slack_oauth_token := $(shell echo $${GRAFANA_SLACK_OAUTH_TOKEN}) - -# target addresses -docker_host := $(domain) -docker_user := ubuntu -ssh_url := $(docker_user)@$(docker_host) - # stack configuration docker_compose_file := docker-compose.$(env).yml @@ -28,33 +14,16 @@ cardano_node_image_tag := 8.8.0-pre cardano_db_sync_image_tag := sancho-4-0-0-fix-config .PHONY: all -all: deploy-stack info #notify - -.PHONY: prepare-config -prepare-config: - @:$(call check_defined, cardano_network) - @:$(call check_defined, grafana_slack_recipient) - @:$(call check_defined, grafana_slack_oauth_token) - export CARDANO_NETWORK=$(cardano_network); \ - export GRAFANA_SLACK_RECIPIENT=$(grafana_slack_recipient); \ - export GRAFANA_SLACK_OAUTH_TOKEN=$(grafana_slack_oauth_token); \ - ./prepare-config.sh - -.PHONY: upload-config -upload-config: check-env-defined prepare-config - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) - $(rsync) -av -e 'ssh -o StrictHostKeyChecking=no' config/target/. $(ssh_url):config +all: deploy-stack info notify .PHONY: deploy-stack deploy-stack: upload-config push-backend push-frontend @:$(call check_defined, cardano_network) @:$(call check_defined, env) - @:$(call check_defined, grafana_admin_password) export CARDANO_NETWORK=$(cardano_network); \ export DOCKER_HOST=ssh://$(ssh_url); \ export ENVIRONMENT=$(env); \ - export GRAFANA_ADMIN_PASSWORD=$(grafana_admin_password); \ + export GRAFANA_ADMIN_PASSWORD=$${GRAFANA_ADMIN_PASSWORD}; \ export BACKEND_TAG=$(backend_image_tag); \ export FRONTEND_TAG=$(frontend_image_tag); \ export CARDANO_NODE_TAG=$(cardano_node_image_tag); \ diff --git a/scripts/govtool/common.mk b/scripts/govtool/common.mk index b80940a08..79d71b77a 100644 --- a/scripts/govtool/common.mk +++ b/scripts/govtool/common.mk @@ -4,6 +4,9 @@ ONESHELL: # tools docker ?= docker +curl ?= curl +rsync ?= rsync +ssh-keyscan ?= ssh-keyscan # environment variables env := $(shell echo $${ENVIRONMENT}) @@ -17,6 +20,9 @@ root_dir := $(shell git rev-parse --show-toplevel) # target addresses domain := $(shell echo $${DOMAIN}) repo_url ?= 733019650473.dkr.ecr.eu-west-1.amazonaws.com +docker_host := $(domain) +docker_user := ubuntu +ssh_url := $(docker_user)@$(docker_host) # stack configuration compose_stack_name := govtool-$(env)-$(cardano_network) diff --git a/scripts/govtool/config.mk b/scripts/govtool/config.mk new file mode 100644 index 000000000..45fb00880 --- /dev/null +++ b/scripts/govtool/config.mk @@ -0,0 +1,82 @@ +common_mk := ../../scripts/govtool/common.mk +ifeq ($(origin $(common_mk)), undefined) + $(eval $(common_mk) := included) + include $(common_mk) +endif + +# directory paths +config_dir := $(root_dir)/scripts/govtool/config +target_config_dir := $(config_dir)/target +template_config_dir := $(config_dir)/templates +cardano_node_config_dir := $(target_config_dir)/cardano-node +dbsync_secrets_dir := $(target_config_dir)/dbsync-secrets +grafana_provisioning_dir := $(target_config_dir)/grafana-provisioning +nginx_config_dir := $(target_config_dir)/nginx + +# metadata +cardano_config_provider := https://book.world.dev.cardano.org + +.PHONY: prepare-config +prepare-config: clear enable-prometheus prepare-dbsync-secrets prepare-backend-config prepare-prometheus-config prepare-grafana-provisioning prepare-nginx-config + +.PHONY: clear +clear: + rm -rf $(target_config_dir) + +.PHONY: fetch-cardano-node-config +fetch-cardano-node-config: + @:$(call check_defined, cardano_network) + mkdir -p $(cardano_node_config_dir) + $(curl) -s "$(cardano_config_provider)/env-$(cardano_network).html" | \ + grep -E -o '[a-z-]+\.json' | \ + sort -u | \ + xargs -I"{}" $(curl) -s "$(cardano_config_provider)/environments/$(cardano_network)/{}" -o "$(cardano_node_config_dir)/{}" + +.PHONY: enable-prometheus +enable-prometheus: fetch-cardano-node-config + sed -i '/"hasPrometheus"/ { N; s/"127\.0\.0\.1"/"0.0.0.0"/ }' "$(cardano_node_config_dir)/config.json" + +.PHONY: prepare-dbsync-secrets +prepare-dbsync-secrets: + mkdir -p $(dbsync_secrets_dir) + echo "$${DBSYNC_POSTGRES_USER}" > "$(dbsync_secrets_dir)/postgres_user"; \ + echo "$${DBSYNC_POSTGRES_PASSWORD}" > "$(dbsync_secrets_dir)/postgres_password"; \ + echo "$${DBSYNC_POSTGRES_DB}" > "$(dbsync_secrets_dir)/postgres_db" + +.PHONY: prepare-backend-config +prepare-backend-config: + sed -e "s/DBSYNC_POSTGRES_DB/$${DBSYNC_POSTGRES_DB}/" \ + -e "s/DBSYNC_POSTGRES_USER/$${DBSYNC_POSTGRES_USER}/" \ + -e "s/DBSYNC_POSTGRES_PASSWORD/$${DBSYNC_POSTGRES_PASSWORD}/" \ + -e "s|SENTRY_DSN|$${SENTRY_DSN_BACKEND}|" \ + "$(config_dir)/templates/backend-config.json.tpl" \ + > "$(target_config_dir)/backend-config.json" + +.PHONY: prepare-prometheus-config +prepare-prometheus-config: + cp -a "$(template_config_dir)/prometheus.yml" "$(target_config_dir)/prometheus.yml" + + PHONY: prepare-grafana-provisioning +prepare-grafana-provisioning: + mkdir -p $(grafana_provisioning_dir) + cp -a $(template_config_dir)/grafana-provisioning/* $(grafana_provisioning_dir) + sed -e "s/GRAFANA_SLACK_RECIPIENT/$${GRAFANA_SLACK_RECIPIENT}/" \ + -e "s|GRAFANA_SLACK_OAUTH_TOKEN|$${GRAFANA_SLACK_OAUTH_TOKEN}|" \ + -i $(grafana_provisioning_dir)/alerting/alerting.yml + +.PHONY: prepare-nginx-config +prepare-nginx-config: + @:$(call check_defined, domain) + mkdir -p $(nginx_config_dir) + touch "$(nginx_config_dir)/auth.conf" + touch "$(nginx_config_dir)/govtool.htpasswd" + if [[ "$(domain)" == *"sanchonet.govtool.byron.network"* ]]; then \ + echo "$${NGINX_BASIC_AUTH}" > "$(nginx_config_dir)/govtool.htpasswd"; \ + echo "auth_basic \"Restricted\";" > "$(nginx_config_dir)/auth.conf"; \ + echo "auth_basic_user_file /etc/nginx/conf.d/govtool.htpasswd;" >> "$(nginx_config_dir)/auth.conf"; \ + fi + +.PHONY: upload-config +upload-config: check-env-defined prepare-config + @:$(call check_defined, ssh_url) + $(rsync) -av -e 'ssh -o StrictHostKeyChecking=no' config/target/. $(ssh_url):config diff --git a/scripts/govtool/config/secrets/backend-config.json b/scripts/govtool/config/secrets/backend-config.json deleted file mode 100644 index d60da33f1..000000000 --- a/scripts/govtool/config/secrets/backend-config.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "dbsyncconfig" : { - "host" : "postgres", - "dbname" : "cexplorer", - "user" : "postgres", - "password" : "pSa8JCpQOACMUdGb", - "port" : 5432 - }, - "fakedbsyncconfig" : { - "host" : "postgres-fakedbsync", - "dbname" : "govtool", - "user" : "test", - "password" : "test", - "port" : 5432 - }, - "port" : 9876, - "host" : "0.0.0.0" -} diff --git a/scripts/govtool/config/secrets/postgres_db b/scripts/govtool/config/secrets/postgres_db deleted file mode 100644 index cdc4e3a7e..000000000 --- a/scripts/govtool/config/secrets/postgres_db +++ /dev/null @@ -1 +0,0 @@ -cexplorer \ No newline at end of file diff --git a/scripts/govtool/config/secrets/postgres_password b/scripts/govtool/config/secrets/postgres_password deleted file mode 100644 index 1a27864b8..000000000 --- a/scripts/govtool/config/secrets/postgres_password +++ /dev/null @@ -1 +0,0 @@ -pSa8JCpQOACMUdGb diff --git a/scripts/govtool/config/secrets/postgres_user b/scripts/govtool/config/secrets/postgres_user deleted file mode 100644 index 7d72bd782..000000000 --- a/scripts/govtool/config/secrets/postgres_user +++ /dev/null @@ -1 +0,0 @@ -postgres \ No newline at end of file diff --git a/scripts/govtool/config/secrets/backend-config.json.tpl b/scripts/govtool/config/templates/backend-config.json.tpl similarity index 100% rename from scripts/govtool/config/secrets/backend-config.json.tpl rename to scripts/govtool/config/templates/backend-config.json.tpl diff --git a/scripts/govtool/config/grafana-provisioning/alerting/alerting.yml b/scripts/govtool/config/templates/grafana-provisioning/alerting/alerting.yml similarity index 100% rename from scripts/govtool/config/grafana-provisioning/alerting/alerting.yml rename to scripts/govtool/config/templates/grafana-provisioning/alerting/alerting.yml diff --git a/scripts/govtool/config/grafana-provisioning/dashboards/dashboard.yml b/scripts/govtool/config/templates/grafana-provisioning/dashboards/dashboard.yml similarity index 100% rename from scripts/govtool/config/grafana-provisioning/dashboards/dashboard.yml rename to scripts/govtool/config/templates/grafana-provisioning/dashboards/dashboard.yml diff --git a/scripts/govtool/config/grafana-provisioning/dashboards/govtool.json b/scripts/govtool/config/templates/grafana-provisioning/dashboards/govtool.json similarity index 100% rename from scripts/govtool/config/grafana-provisioning/dashboards/govtool.json rename to scripts/govtool/config/templates/grafana-provisioning/dashboards/govtool.json diff --git a/scripts/govtool/config/grafana-provisioning/dashboards/traefik_rev4.json b/scripts/govtool/config/templates/grafana-provisioning/dashboards/traefik_rev4.json similarity index 100% rename from scripts/govtool/config/grafana-provisioning/dashboards/traefik_rev4.json rename to scripts/govtool/config/templates/grafana-provisioning/dashboards/traefik_rev4.json diff --git a/scripts/govtool/config/grafana-provisioning/datasources/datasource.yml b/scripts/govtool/config/templates/grafana-provisioning/datasources/datasource.yml similarity index 100% rename from scripts/govtool/config/grafana-provisioning/datasources/datasource.yml rename to scripts/govtool/config/templates/grafana-provisioning/datasources/datasource.yml diff --git a/scripts/govtool/config/templates/prometheus.yml b/scripts/govtool/config/templates/prometheus.yml new file mode 100644 index 000000000..0d637f3f1 --- /dev/null +++ b/scripts/govtool/config/templates/prometheus.yml @@ -0,0 +1,23 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + monitor: 'govtool' +scrape_configs: + - job_name: 'traefik' + scrape_interval: 5s + static_configs: + - targets: ['traefik:8082'] + - job_name: 'cardano' + scrape_interval: 5s + static_configs: + - targets: ['cardano-node:12798'] + - job_name: 'cardano_db_sync' + scrape_interval: 5s + metrics_path: / + static_configs: + - targets: ['cardano-db-sync:8080'] + - job_name: 'host' + scrape_interval: 5s + static_configs: + - targets: ['host.docker.internal:9100'] diff --git a/scripts/govtool/info.mk b/scripts/govtool/info.mk index da245e32f..741e457c4 100644 --- a/scripts/govtool/info.mk +++ b/scripts/govtool/info.mk @@ -28,7 +28,7 @@ notify: @:$(call check_defined, cardano_network) @:$(call check_defined, env) $(curl) -X POST https://slack.com/api/chat.postMessage\ - -H "Authorization: Bearer $(grafana_slack_oauth_token)" \ + -H "Authorization: Bearer $${GRAFANA_SLACK_OAUTH_TOKEN}" \ -H "Content-Type: application/json; charset=utf-8" \ --data "{ \"channel\":\"$${GRAFANA_SLACK_RECIPIENT}\", \"text\":\":rocket: *Deploy performed on \`$(env)\`*\n- from *branch* \`$(branch)\` (\`$(commit)\`),\n- using *Cardano Node* version \`$(cardano_node_image_tag)\`,\n- using *Cardano DB Sync* version \`$(cardano_db_sync_image_tag)\`,\n- using *GovTool backend* version \`$(backend_image_tag)\`,\n- using *Govtool frontend* version \`$(frontend_image_tag)\`.\n$(pipeline_info)\" }" diff --git a/scripts/govtool/prepare-config.sh b/scripts/govtool/prepare-config.sh deleted file mode 100755 index 78ce75a9d..000000000 --- a/scripts/govtool/prepare-config.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -set -eu pipefail - -repo_root_dir="$(git rev-parse --show-toplevel)" -config_dir="$repo_root_dir/scripts/govtool/config" -target_config_dir="$repo_root_dir/scripts/govtool/config/target" -mkdir -p "$target_config_dir" - -# cardano node config -cardano_node_config_dir="$target_config_dir/cardano-node" -mkdir -p "$cardano_node_config_dir" -for file in $(curl -s https://book.world.dev.cardano.org/env-"$CARDANO_NETWORK".html | grep -E -o '[a-z-]+\.json' | sort -u); do - curl -s "https://book.world.dev.cardano.org/environments/$CARDANO_NETWORK/$file" -o "$cardano_node_config_dir/$file" -done -# enable prometheus in node config -sed -i '/"hasPrometheus"/ { N; s/"127\.0\.0\.1"/"0.0.0.0"/ }' "$cardano_node_config_dir/config.json" - -# dbsync secret files -dbsync_secrets_dir="$target_config_dir/dbsync-secrets" -mkdir -p "$dbsync_secrets_dir" -echo "$DBSYNC_POSTGRES_USER" > "$dbsync_secrets_dir/postgres_user" -echo "$DBSYNC_POSTGRES_PASSWORD" > "$dbsync_secrets_dir/postgres_password" -echo "$DBSYNC_POSTGRES_DB" > "$dbsync_secrets_dir/postgres_db" - -# backend config file -sed -e "s/DBSYNC_POSTGRES_DB/$DBSYNC_POSTGRES_DB/" \ - -e "s/DBSYNC_POSTGRES_USER/$DBSYNC_POSTGRES_USER/" \ - -e "s/DBSYNC_POSTGRES_PASSWORD/$DBSYNC_POSTGRES_PASSWORD/" \ - -e "s|SENTRY_DSN|$SENTRY_DSN_BACKEND|" \ - "$config_dir/secrets/backend-config.json.tpl" \ - > "$target_config_dir/backend-config.json" - -# prometheus config file -cat >"$target_config_dir/prometheus.yml" <<_EOF_ -global: - scrape_interval: 15s - evaluation_interval: 15s - external_labels: - monitor: 'govtool' -scrape_configs: - - job_name: 'traefik' - scrape_interval: 5s - static_configs: - - targets: ['traefik:8082'] - - job_name: 'cardano' - scrape_interval: 5s - static_configs: - - targets: ['cardano-node:12798'] - - job_name: 'cardano_db_sync' - scrape_interval: 5s - metrics_path: / - static_configs: - - targets: ['cardano-db-sync:8080'] - - job_name: 'host' - scrape_interval: 5s - static_configs: - - targets: ['host.docker.internal:9100'] -_EOF_ - -# grafana provisioning dir -grafana_provisioning_dir="$target_config_dir/grafana-provisioning" -mkdir -p "$grafana_provisioning_dir" -cp -a "$config_dir/grafana-provisioning/"* "$grafana_provisioning_dir" -sed -e "s/GRAFANA_SLACK_RECIPIENT/$GRAFANA_SLACK_RECIPIENT/" \ - -e "s|GRAFANA_SLACK_OAUTH_TOKEN|$GRAFANA_SLACK_OAUTH_TOKEN|" \ - -i "$grafana_provisioning_dir/alerting/alerting.yml" - -# nginx config for frontend optional basic auth -nginx_config_dir="$target_config_dir/nginx" -rm -rf "$nginx_config_dir" -mkdir -p "$nginx_config_dir" -if [[ "$DOMAIN" == *"sanchonet.govtool.byron.network"* ]]; then - cat >"$nginx_config_dir/auth.conf" <<_EOF_ -auth_basic "Restricted"; -auth_basic_user_file /etc/nginx/conf.d/govtool.htpasswd; -_EOF_ - cat >"$nginx_config_dir/govtool.htpasswd" <<_EOF_ -$NGINX_BASIC_AUTH -_EOF_ -else - # create empty files if no basic auth is needed - touch "$nginx_config_dir/auth.conf" - touch "$nginx_config_dir/govtool.htpasswd" -fi diff --git a/scripts/govtool/utils.mk b/scripts/govtool/utils.mk index 37ccda6e6..3cb2bcc27 100644 --- a/scripts/govtool/utils.mk +++ b/scripts/govtool/utils.mk @@ -1,7 +1,5 @@ .PHONY: ssh ssh: - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) export TERM=xterm-256color; \ ssh $(ssh_url) @@ -9,12 +7,11 @@ ssh: docker: @:$(call check_defined, cardano_network) @:$(call check_defined, env) - @:$(call check_defined, grafana_admin_password) @:$(call check_defined, cmd) export CARDANO_NETWORK=$(cardano_network); \ export DOCKER_HOST=ssh://$(ssh_url); \ export ENVIRONMENT=$(env); \ - export GRAFANA_ADMIN_PASSWORD=$(grafana_admin_password); \ + export GRAFANA_ADMIN_PASSWORD=$${GRAFANA_ADMIN_PASSWORD}; \ export BACKEND_TAG=$(backend_image_tag); \ export FRONTEND_TAG=$(frontend_image_tag); \ export CARDANO_NODE_TAG=$(cardano_node_image_tag); \