chore: move db migration to helm hooks (#243)

.github/workflows/run-tests-and-sonar-scan.yml

@@ -40,6 +40,9 @@ jobs:
       - name: Install project dependencies
         run: make install-deps mocks docs
 
+      - name: Run database migrations
+        run: make ci-migrate
+
       - name: Run tests
         env:
           DB_HOST: localhost

Makefile

@@ -15,6 +15,7 @@ install-deps: ## Install dependencies
 	@echo "===> Installing dependencies"
 	@go install github.com/swaggo/swag/cmd/swag@latest
 	@go install go.uber.org/mock/mockgen@latest
+	@go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest
 	@echo "===> Done"
 
 .PHONY: test
@@ -76,3 +77,8 @@ bootstrap-minimal: ## Bootstrap docker compose setup with mock and postgres only
 .PHONY: teardown
 teardown: ## Teardown docker compose setup
 	@docker compose down
+
+.PHONY: ci-migrate
+ci-migrate: ## Run migrations in CI
+	@echo "===> Running migrations"
+	@migrate -path db/migrations -database "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable" up

cmd/argo-watcher/config/config.go

@@ -19,6 +19,14 @@ type KeycloakConfig struct {
 	PrivilegedGroups        []string `env:"KEYCLOAK_PRIVILEGED_GROUPS" json:"privileged_groups,omitempty"`
 }
 
+type DatabaseConfig struct {
+	Host     string `env:"DB_HOST" json:"db_host,omitempty"`
+	Port     int    `env:"DB_PORT" json:"db_port,omitempty"`
+	Name     string `env:"DB_NAME" json:"db_name,omitempty"`
+	User     string `env:"DB_USER" json:"db_user,omitempty"`
+	Password string `env:"DB_PASSWORD" json:"-"`
+}
+
 type ServerConfig struct {
 	ArgoUrl           url.URL        `env:"ARGO_URL,required" json:"argo_cd_url"`
 	ArgoUrlAlias      string         `env:"ARGO_URL_ALIAS" json:"argo_cd_url_alias,omitempty"` // Used to generate App URL. Can be omitted if ArgoUrl is reachable from outside.
@@ -34,13 +42,8 @@ type ServerConfig struct {
 	LogFormat         string         `env:"LOG_FORMAT" envDefault:"json" json:"-"`
 	Host              string         `env:"HOST" envDefault:"0.0.0.0" json:"-"`
 	Port              string         `env:"PORT" envDefault:"8080" json:"-"`
-	DbHost            string         `env:"DB_HOST" json:"db_host,omitempty"`
-	DbPort            int            `env:"DB_PORT" json:"db_port,omitempty"`
-	DbName            string         `env:"DB_NAME" json:"db_name,omitempty"`
-	DbUser            string         `env:"DB_USER" json:"db_user,omitempty"`
-	DbPassword        string         `env:"DB_PASSWORD" json:"-"`
-	DbMigrationsPath  string         `env:"DB_MIGRATIONS_PATH" envDefault:"db/migrations" json:"-"`
 	DeployToken       string         `env:"ARGO_WATCHER_DEPLOY_TOKEN" json:"-"`
+	Db                DatabaseConfig `json:"db,omitempty"`
 	Keycloak          KeycloakConfig `json:"keycloak,omitempty"`
 }
 

cmd/argo-watcher/config/config_test.go

@@ -64,11 +64,14 @@ func TestServerConfig_GetRetryAttempts(t *testing.T) {
 }
 
 func TestServerConfig_JSONExcludesSensitiveFields(t *testing.T) {
+	databaseConfig := DatabaseConfig{
+		Password: "db-password",
+	}
 	// Create a ServerConfig instance with some dummy data
 	config := &ServerConfig{
 		ArgoToken:   "secret-token",
-		DbPassword:  "db-password",
 		DeployToken: "deploy-token",
+		Db:          databaseConfig,
 	}
 
 	// Marshal the ServerConfig instance to JSON

cmd/argo-watcher/state/postgres_state.go

@@ -2,7 +2,6 @@ package state
 
 import (
 	"database/sql"
-	"errors"
 	"fmt"
 	"time"
 
@@ -16,10 +15,6 @@ import (
 	"github.com/avast/retry-go/v4"
 	_ "github.com/lib/pq"
 
-	"github.com/golang-migrate/migrate/v4"
-	migratePostgres "github.com/golang-migrate/migrate/v4/database/postgres"
-	_ "github.com/golang-migrate/migrate/v4/source/file"
-
 	"github.com/shini4i/argo-watcher/cmd/argo-watcher/config"
 	"github.com/shini4i/argo-watcher/cmd/argo-watcher/state/state_models"
 	"github.com/shini4i/argo-watcher/internal/models"
@@ -37,41 +32,6 @@ func (state *PostgresState) Connect(serverConfig *config.ServerConfig) error {
 	} else {
 		state.orm = orm
 	}
-
-	// run migrations
-	if err := state.runMigrations(serverConfig.DbMigrationsPath); err != nil {
-		return fmt.Errorf("failed running migrations: %s", err.Error())
-	}
-
-	return nil
-}
-
-// Runs migrations
-// TODO: migrate to gorm supported migrations library - https://gorm.io/docs/migration.html#Atlas-Integration
-func (state *PostgresState) runMigrations(dbMigrationPath string) error {
-	migrationsPath := fmt.Sprintf("file://%s", dbMigrationPath)
-
-	connection, err := state.orm.DB()
-	if err != nil {
-		return err
-	}
-
-	driver, err := migratePostgres.WithInstance(connection, &migratePostgres.Config{})
-	if err != nil {
-		return err
-	}
-
-	migrations, err := migrate.NewWithDatabaseInstance(migrationsPath, "postgres", driver)
-	if err != nil {
-		return err
-	}
-
-	log.Debug().Msg("Running database migrations...")
-	if err := migrations.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
-		return err
-	}
-
-	log.Debug().Msg("Database schema is up to date.")
 	return nil
 }
 
@@ -235,7 +195,7 @@ func (state *PostgresState) doProcessPostgresObsoleteTasks() error {
 
 func getDsn(serverConfig *config.ServerConfig) string {
 	dsnTemplate := "host=%s port=%d user=%s password=%s dbname=%s sslmode=disable TimeZone=UTC"
-	return fmt.Sprintf(dsnTemplate, serverConfig.DbHost, serverConfig.DbPort, serverConfig.DbUser, serverConfig.DbPassword, serverConfig.DbName)
+	return fmt.Sprintf(dsnTemplate, serverConfig.Db.Host, serverConfig.Db.Port, serverConfig.Db.User, serverConfig.Db.Password, serverConfig.Db.Name)
 }
 
 func getOrmLogger(serverConfig *config.ServerConfig) *gorm.Config {

cmd/argo-watcher/state/postgres_state_test.go

@@ -59,14 +59,16 @@ var (
 )
 
 func TestPostgresState_Add(t *testing.T) {
+	databaseConfig := config.DatabaseConfig{
+		Host:     os.Getenv("DB_HOST"),
+		Port:     5432,
+		Name:     os.Getenv("DB_NAME"),
+		User:     os.Getenv("DB_USER"),
+		Password: os.Getenv("DB_PASSWORD"),
+	}
 	testConfig := &config.ServerConfig{
-		StateType:        "postgres",
-		DbHost:           os.Getenv("DB_HOST"),
-		DbPort:           5432,
-		DbUser:           os.Getenv("DB_USER"),
-		DbName:           os.Getenv("DB_NAME"),
-		DbPassword:       os.Getenv("DB_PASSWORD"),
-		DbMigrationsPath: "../../../db/migrations",
+		StateType: "postgres",
+		Db:        databaseConfig,
 	}
 	err := postgresState.Connect(testConfig)
 	if err != nil {
@@ -188,12 +190,15 @@ func TestPostgresState_Check(t *testing.T) {
 }
 
 func TestGetDsn(t *testing.T) {
+	databaseConfig := config.DatabaseConfig{
+		Host:     "localhost",
+		Port:     5432,
+		Name:     "testdb",
+		User:     "admin",
+		Password: "password123",
+	}
 	testConfig := &config.ServerConfig{
-		DbHost:     "localhost",
-		DbPort:     5432,
-		DbUser:     "admin",
-		DbPassword: "password123",
-		DbName:     "testdb",
+		Db: databaseConfig,
 	}
 
 	expectedDsn := "host=localhost port=5432 user=admin password=password123 dbname=testdb sslmode=disable TimeZone=UTC"

docker-compose.yml

@@ -7,10 +7,23 @@ services:
       POSTGRES_USER: watcher
       POSTGRES_PASSWORD: watcher
       POSTGRES_DB: watcher
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U watcher" ]
+      interval: 30s
+      timeout: 30s
+      retries: 3
     volumes:
       - ./db/data:/var/lib/postgresql/data
     ports:
       - "5432:5432"
+  migrations:
+    image: migrate/migrate:v4.17.0
+    command: -path=/migrations/ -database postgres://watcher:watcher@postgres:5432/watcher?sslmode=disable up
+    volumes:
+      - ./db/migrations:/migrations
+    depends_on:
+      postgres:
+        condition: service_healthy
   backend:
     image: golang:1.21-alpine3.18
     volumes:
@@ -29,8 +42,8 @@ services:
       DB_PASSWORD: watcher
       DB_MIGRATIONS_PATH: /app/db/migrations
     depends_on:
-      - postgres
-      - mock
+      migrations:
+        condition: service_completed_successfully
     ports:
       - "8080:8080"
   frontend:

docs/development.md

@@ -44,6 +44,8 @@ make docs
 
 ## Back-End Development
 
+> The information below is needed only if you don't want to use docker-compose for some reason.
+
 To start developing argo-watcher you will need golang `1.21+`
 
 Start mock of the argo-cd server
@@ -75,6 +77,12 @@ Start database
 docker compose up postgres
 ```
 
+Run migrations
+
+```shell
+migrate -path file://db/migrations -database "postgresql://watcher:watcher@localhost:5432/watcher?sslmode=disable" up
+```
+
 Start server
 
 ```shell
@@ -83,7 +91,7 @@ cd cmd/argo-watcher
 # install dependencies
 go mod tidy
 # OR start argo-watcher (postgres)
-LOG_LEVEL=debug LOG_FORMAT=text ARGO_URL=http://localhost:8081 ARGO_TOKEN=example STATE_TYPE=postgres DB_USER=watcher DB_PASSWORD=watcher DB_NAME=watcher DB_MIGRATIONS_PATH=../../db/migrations go run . -server
+LOG_LEVEL=debug LOG_FORMAT=text ARGO_URL=http://localhost:8081 ARGO_TOKEN=example STATE_TYPE=postgres DB_USER=watcher DB_PASSWORD=watcher DB_NAME=watcher go run . -server
 ```
 
 #### Logs in simple text

go.mod

@@ -10,7 +10,6 @@ require (
 	github.com/go-git/go-billy/v5 v5.5.0
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/go-playground/validator/v10 v10.16.0
-	github.com/golang-migrate/migrate/v4 v4.16.2
 	github.com/google/uuid v1.4.0
 	github.com/lib/pq v1.10.9
 	github.com/prometheus/client_golang v1.17.0
@@ -52,8 +51,6 @@ require (
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx/v5 v5.4.3 // indirect
@@ -68,7 +65,9 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-sqlite3 v1.14.16 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/microsoft/go-mssqldb v1.0.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
@@ -82,7 +81,6 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect