From c28035fa85cd94813a5d6b6837c850e0842f5323 Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 11:58:01 -0700 Subject: [PATCH 01/17] return database error in logs --- cmd/argo-watcher/state/in_memory_state.go | 5 ++-- cmd/argo-watcher/state/postgres_state.go | 32 ++++++++++++++--------- cmd/argo-watcher/state/state.go | 10 ++++--- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/cmd/argo-watcher/state/in_memory_state.go b/cmd/argo-watcher/state/in_memory_state.go index f177ea6d..1b41b883 100644 --- a/cmd/argo-watcher/state/in_memory_state.go +++ b/cmd/argo-watcher/state/in_memory_state.go @@ -19,8 +19,9 @@ type InMemoryState struct { // Connect is a placeholder method that does not establish any connection. // It logs a debug message indicating that the InMemoryState does not connect to anything and skips the connection process. // This method exists to fulfill the State interface requirement and has no functional value. -func (state *InMemoryState) Connect(serverConfig *config.ServerConfig) { +func (state *InMemoryState) Connect(serverConfig *config.ServerConfig) error { log.Debug().Msg("InMemoryState does not connect to anything. Skipping.") + return nil } // Add adds a new task to the in-memory state. @@ -126,7 +127,7 @@ func (state *InMemoryState) ProcessObsoleteTasks(retryTimes uint) { err := retry.Do( func() error { state.tasks = processInMemoryObsoleteTasks(state.tasks) - return desiredRetryError + return errDesiredRetry }, retry.DelayType(retry.FixedDelay), retry.Delay(60*time.Minute), diff --git a/cmd/argo-watcher/state/postgres_state.go b/cmd/argo-watcher/state/postgres_state.go index c9d48501..86af6a8b 100644 --- a/cmd/argo-watcher/state/postgres_state.go +++ b/cmd/argo-watcher/state/postgres_state.go @@ -26,31 +26,37 @@ type PostgresState struct { // It takes a pointer to a config.ServerConfig parameter and initializes the database connection. // The method also performs database migrations using the specified migrations path. // If any errors occur during connection establishment or migrations, they are logged and may cause a panic. -func (state *PostgresState) Connect(serverConfig *config.ServerConfig) { +func (state *PostgresState) Connect(serverConfig *config.ServerConfig) error { c := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", serverConfig.DbHost, serverConfig.DbPort, serverConfig.DbUser, serverConfig.DbPassword, serverConfig.DbName) + log.Debug().Msg(c) + db, err := sql.Open("postgres", c) if err != nil { - panic(err) + return err } migrationsPath := fmt.Sprintf("file://%s", serverConfig.DbMigrationsPath) - driver, _ := postgres.WithInstance(db, &postgres.Config{}) - migrations, _ := migrate.NewWithDatabaseInstance( - migrationsPath, - "postgres", driver) + driver, err := postgres.WithInstance(db, &postgres.Config{}) + if err != nil { + return err + } - log.Debug().Msg("Running database migrations...") + migrations, err := migrate.NewWithDatabaseInstance(migrationsPath, "postgres", driver) + if err != nil { + return err + } - switch err = migrations.Up(); err { - case migrate.ErrNoChange, nil: - log.Debug().Msg("Database schema is up to date.") - default: - panic(err) + log.Debug().Msg("Running database migrations...") + err = migrations.Up() + if err != nil { + return err } + log.Debug().Msg("Database schema is up to date.") state.db = db + return nil } // Add inserts a new task into the PostgreSQL database with the provided details. @@ -263,7 +269,7 @@ func (state *PostgresState) ProcessObsoleteTasks(retryTimes uint) { log.Error().Msgf("Couldn't process obsolete tasks. Got the following error: %s", err) return err } - return desiredRetryError + return errDesiredRetry }, retry.DelayType(retry.FixedDelay), retry.Delay(60*time.Minute), diff --git a/cmd/argo-watcher/state/state.go b/cmd/argo-watcher/state/state.go index bde1e36d..8d46fc72 100644 --- a/cmd/argo-watcher/state/state.go +++ b/cmd/argo-watcher/state/state.go @@ -9,10 +9,10 @@ import ( "github.com/shini4i/argo-watcher/internal/models" ) -var desiredRetryError = errors.New("desired retry error") +var errDesiredRetry = errors.New("desired retry error") type State interface { - Connect(serverConfig *config.ServerConfig) + Connect(serverConfig *config.ServerConfig) error Add(task models.Task) error GetTasks(startTime float64, endTime float64, app string) []models.Task GetTask(id string) (*models.Task, error) @@ -41,6 +41,10 @@ func NewState(serverConfig *config.ServerConfig) (State, error) { return nil, fmt.Errorf("unexpected state type received: %s", name) } - state.Connect(serverConfig) + err := state.Connect(serverConfig) + if err != nil { + return nil, err + } + return state, nil } From cf4325a3d853174bc4e1bfea080eb4f134abec50 Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 11:58:13 -0700 Subject: [PATCH 02/17] add documentation on how to start server with DB --- docs/development.md | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/development.md b/docs/development.md index 43575968..9944b0f3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -55,15 +55,34 @@ cd cmd/mock go run . ``` -Start the argo-watcher server +### Start the argo-watcher server (in-memory) ```shell # go to backend directory cd cmd/argo-watcher # install dependencies go mod tidy -# start argo-watcher -ARGO_URL=http://localhost:8081 STATE_TYPE=in-memory go run . -server +# start argo-watcher (in-memory) +LOG_LEVEL=debug ARGO_URL=http://localhost:8081 ARGO_TOKEN=example STATE_TYPE=in-memory go run . -server +``` + + +### Start the argo-watcher server (postgres) + +Start database +```shell +# start the database in a separate terminal window +docker compose up postgres +``` + +Start server +```shell +# go to backend directory +cd cmd/argo-watcher +# install dependencies +go mod tidy +# OR start argo-watcher (postgres) +LOG_LEVEL=debug ARGO_URL=http://localhost:8081 ARGO_TOKEN=example STATE_TYPE=postgres DB_USER=watcher DB_PASSWORD=watcher DB_NAME=watcher DB_MIGRATIONS_PATH="./../../db/migrations" go run . -server ``` ### Running the unit tests From d66bfcf3a4f13bd33ffabc1453acaf30c741a441 Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 12:53:57 -0700 Subject: [PATCH 03/17] create connection using ORM --- cmd/argo-watcher/state/postgres_state.go | 66 ++++++++++++++++++------ go.mod | 10 +++- go.sum | 18 +++++++ 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/cmd/argo-watcher/state/postgres_state.go b/cmd/argo-watcher/state/postgres_state.go index 86af6a8b..be6d4af9 100644 --- a/cmd/argo-watcher/state/postgres_state.go +++ b/cmd/argo-watcher/state/postgres_state.go @@ -6,11 +6,15 @@ import ( "fmt" "time" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/pgdialect" + "github.com/uptrace/bun/driver/pgdriver" + "github.com/avast/retry-go/v4" - "github.com/golang-migrate/migrate/v4" _ "github.com/lib/pq" "github.com/rs/zerolog/log" + "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database/postgres" _ "github.com/golang-migrate/migrate/v4/source/file" @@ -19,43 +23,75 @@ import ( ) type PostgresState struct { - db *sql.DB + db *sql.DB // for backwards compatibility. NOTE: note save when using multiple connections in ORM (connection POOL or reconnecting) + orm *bun.DB } // Connect establishes a connection to the PostgreSQL database using the provided server configuration. -// It takes a pointer to a config.ServerConfig parameter and initializes the database connection. -// The method also performs database migrations using the specified migrations path. -// If any errors occur during connection establishment or migrations, they are logged and may cause a panic. func (state *PostgresState) Connect(serverConfig *config.ServerConfig) error { - c := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", serverConfig.DbHost, serverConfig.DbPort, serverConfig.DbUser, serverConfig.DbPassword, serverConfig.DbName) + // create options + options := []pgdriver.Option{ + // connection options + pgdriver.WithNetwork("tcp"), + pgdriver.WithAddr(fmt.Sprintf("%s:%s", serverConfig.DbHost, serverConfig.DbPort)), // localhost:5432 + pgdriver.WithTLSConfig(nil), // sslmode=disable + // DB timeout configrations + pgdriver.WithTimeout(5 * time.Second), + pgdriver.WithDialTimeout(5 * time.Second), + pgdriver.WithReadTimeout(5 * time.Second), + pgdriver.WithWriteTimeout(5 * time.Second), + } + if serverConfig.DbName != "" { + options = append(options, pgdriver.WithDatabase(serverConfig.DbName)) + } + if serverConfig.DbUser != "" { + options = append(options, pgdriver.WithUser(serverConfig.DbUser)) + } + if serverConfig.DbPassword != "" { + options = append(options, pgdriver.WithPassword(serverConfig.DbPassword)) + } - log.Debug().Msg(c) + // create driver + connector := pgdriver.NewConnector(options...) + driver := sql.OpenDB(connector) - db, err := sql.Open("postgres", c) - if err != nil { + // Confirm a successful connection. + if err := driver.Ping(); err != nil { return err } - migrationsPath := fmt.Sprintf("file://%s", serverConfig.DbMigrationsPath) + // create ORM database connection + state.orm = bun.NewDB(driver, pgdialect.New()) + state.db = state.orm.DB - driver, err := postgres.WithInstance(db, &postgres.Config{}) + // do migrations (temporary version) + // TODO: change to use ORM migrations + err := runMigrations(serverConfig, state.db) if err != nil { return err } - migrations, err := migrate.NewWithDatabaseInstance(migrationsPath, "postgres", driver) + return nil +} + +func runMigrations(serverConfig *config.ServerConfig, db *sql.DB) error { + migrationsPath := fmt.Sprintf("file://%s", serverConfig.DbMigrationsPath) + migrationDriver, err := postgres.WithInstance(db, &postgres.Config{}) + if err != nil { + return err + } + migrations, err := migrate.NewWithDatabaseInstance(migrationsPath, "postgres", migrationDriver) if err != nil { return err } log.Debug().Msg("Running database migrations...") err = migrations.Up() - if err != nil { + if err != nil && err != migrate.ErrNoChange { return err } - log.Debug().Msg("Database schema is up to date.") - state.db = db + log.Debug().Msg("Database schema is up to date.") return nil } diff --git a/go.mod b/go.mod index 03756321..454ab2d1 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 github.com/lib/pq v1.10.6 github.com/prometheus/client_golang v1.12.2 - github.com/rs/zerolog v1.29.0 + github.com/rs/zerolog v1.29.1 github.com/stretchr/testify v1.8.3 github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a github.com/swaggo/gin-swagger v1.5.2 @@ -39,6 +39,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect @@ -54,8 +55,14 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect + github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect + github.com/uptrace/bun v1.1.14 // indirect + github.com/uptrace/bun/dialect/pgdialect v1.1.14 // indirect + github.com/uptrace/bun/driver/pgdriver v1.1.14 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.uber.org/atomic v1.7.0 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.10.0 // indirect @@ -65,4 +72,5 @@ require ( golang.org/x/tools v0.10.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + mellium.im/sasl v0.3.1 // indirect ) diff --git a/go.sum b/go.sum index 6020ec59..93421811 100644 --- a/go.sum +++ b/go.sum @@ -338,6 +338,7 @@ github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+ github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -744,6 +745,7 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -1056,6 +1058,8 @@ github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OK github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= +github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -1143,6 +1147,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= @@ -1151,6 +1157,12 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/uptrace/bun v1.1.14 h1:S5vvNnjEynJ0CvnrBOD7MIRW7q/WbtvFXrdfy0lddAM= +github.com/uptrace/bun v1.1.14/go.mod h1:RHk6DrIisO62dv10pUOJCz5MphXThuOTpVNYEYv7NI8= +github.com/uptrace/bun/dialect/pgdialect v1.1.14 h1:b7+V1KDJPQSFYgkG/6YLXCl2uvwEY3kf/GSM7hTHRDY= +github.com/uptrace/bun/dialect/pgdialect v1.1.14/go.mod h1:v6YiaXmnKQ2FlhRD2c0ZfKd+QXH09pYn4H8ojaavkKk= +github.com/uptrace/bun/driver/pgdriver v1.1.14 h1:V2Etm7mLGS3mhx8ddxZcUnwZLX02Jmq9JTlo0sNVDhA= +github.com/uptrace/bun/driver/pgdriver v1.1.14/go.mod h1:D4FjWV9arDYct6sjMJhFoyU71SpllZRHXFRRP2Kd0Kw= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -1164,6 +1176,10 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmF github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= @@ -1931,6 +1947,8 @@ k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= +mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= modernc.org/cc/v3 v3.32.4/go.mod h1:0R6jl1aZlIl2avnYfbfHBS1QB6/f+16mihBObaBC878= modernc.org/ccgo/v3 v3.9.2/go.mod h1:gnJpy6NIVqkETT+L5zPsQFj7L2kkhfPMzOghRNv/CFo= From d8d6b30f966734ba626e7b116648eeb3d0f005e6 Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 13:12:43 -0700 Subject: [PATCH 04/17] add log format for development --- cmd/argo-watcher/config/config.go | 4 +++- cmd/argo-watcher/server.go | 17 +++++++++++++++-- docs/development.md | 7 +++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/cmd/argo-watcher/config/config.go b/cmd/argo-watcher/config/config.go index 985ff84a..49aaf63f 100644 --- a/cmd/argo-watcher/config/config.go +++ b/cmd/argo-watcher/config/config.go @@ -2,9 +2,10 @@ package config import ( "errors" - "github.com/shini4i/argo-watcher/internal/helpers" "strconv" + "github.com/shini4i/argo-watcher/internal/helpers" + envConfig "github.com/kelseyhightower/envconfig" ) @@ -18,6 +19,7 @@ type ServerConfig struct { StateType string `required:"false" envconfig:"STATE_TYPE"` StaticFilePath string `required:"false" envconfig:"STATIC_FILES_PATH" default:"static"` LogLevel string `required:"false" envconfig:"LOG_LEVEL" default:"info"` + LogFormat string `required:"false" envconfig:"LOG_FORMAT" default:"json"` Host string `required:"false" envconfig:"HOST" default:"0.0.0.0"` Port string `required:"false" envconfig:"PORT" default:"8080"` DbHost string `required:"false" envconfig:"DB_HOST" default:"localhost"` diff --git a/cmd/argo-watcher/server.go b/cmd/argo-watcher/server.go index 2aa58b22..293d3e00 100644 --- a/cmd/argo-watcher/server.go +++ b/cmd/argo-watcher/server.go @@ -1,16 +1,29 @@ package main import ( + "os" + "time" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/shini4i/argo-watcher/cmd/argo-watcher/config" "github.com/shini4i/argo-watcher/cmd/argo-watcher/state" ) +const ( + LOG_FORMAT_TEXT = "text" +) + // initLogs initializes the logging configuration based on the provided log level. // It parses the log level string and sets the global log level accordingly using the zerolog library. // If the log level string is invalid, it falls back to the default InfoLevel. -func initLogs(logLevel string) { +func initLogs(logLevel string, logFormat string) { + // set log format + if logFormat == LOG_FORMAT_TEXT { + output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339} + log.Logger = zerolog.New(output).With().Timestamp().Logger() + } + // set log level if logLevel, err := zerolog.ParseLevel(logLevel); err != nil { log.Warn().Msgf("Couldn't parse log level. Got the following error: %s", err) } else { @@ -27,7 +40,7 @@ func serverWatcher() { } // initialize logs - initLogs(serverConfig.LogLevel) + initLogs(serverConfig.LogLevel, serverConfig.LogFormat) // initialize metrics metrics := &Metrics{} diff --git a/docs/development.md b/docs/development.md index 9944b0f3..4547ce43 100644 --- a/docs/development.md +++ b/docs/development.md @@ -85,6 +85,13 @@ go mod tidy LOG_LEVEL=debug ARGO_URL=http://localhost:8081 ARGO_TOKEN=example STATE_TYPE=postgres DB_USER=watcher DB_PASSWORD=watcher DB_NAME=watcher DB_MIGRATIONS_PATH="./../../db/migrations" go run . -server ``` +#### Logs in simple text + +```shell +# add LOG_FORMAT=text for simple text logs +LOG_LEVEL=debug LOG_FORMAT=text go run . -server +``` + ### Running the unit tests Use the following snippets to run argo-watcher unit tests From 1ee0b75671fd6649a126e5f4ff23c68aec03b790 Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 16:28:18 -0700 Subject: [PATCH 05/17] move plain text logs type to config package --- cmd/argo-watcher/config/config.go | 4 ++++ cmd/argo-watcher/server.go | 6 +----- cmd/argo-watcher/server_test.go | 7 ++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cmd/argo-watcher/config/config.go b/cmd/argo-watcher/config/config.go index 49aaf63f..141f3716 100644 --- a/cmd/argo-watcher/config/config.go +++ b/cmd/argo-watcher/config/config.go @@ -9,6 +9,10 @@ import ( envConfig "github.com/kelseyhightower/envconfig" ) +const ( + LOG_FORMAT_TEXT = "text" +) + type ServerConfig struct { ArgoUrl string `required:"true" envconfig:"ARGO_URL"` ArgoToken string `required:"true" envconfig:"ARGO_TOKEN"` diff --git a/cmd/argo-watcher/server.go b/cmd/argo-watcher/server.go index 293d3e00..a59d5367 100644 --- a/cmd/argo-watcher/server.go +++ b/cmd/argo-watcher/server.go @@ -10,16 +10,12 @@ import ( "github.com/shini4i/argo-watcher/cmd/argo-watcher/state" ) -const ( - LOG_FORMAT_TEXT = "text" -) - // initLogs initializes the logging configuration based on the provided log level. // It parses the log level string and sets the global log level accordingly using the zerolog library. // If the log level string is invalid, it falls back to the default InfoLevel. func initLogs(logLevel string, logFormat string) { // set log format - if logFormat == LOG_FORMAT_TEXT { + if logFormat == config.LOG_FORMAT_TEXT { output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339} log.Logger = zerolog.New(output).With().Timestamp().Logger() } diff --git a/cmd/argo-watcher/server_test.go b/cmd/argo-watcher/server_test.go index 65ac4e42..48bf9385 100644 --- a/cmd/argo-watcher/server_test.go +++ b/cmd/argo-watcher/server_test.go @@ -1,14 +1,15 @@ package main import ( + "testing" + "github.com/rs/zerolog" "github.com/stretchr/testify/assert" - "testing" ) func TestServer_initLogs_correct(t *testing.T) { // Invoke the function being tested - initLogs("fatal") + initLogs("fatal", "json") // Assert that the global log level is set to the expected value assert.Equal(t, zerolog.FatalLevel, zerolog.GlobalLevel()) @@ -19,7 +20,7 @@ func TestServer_initLogs_correct(t *testing.T) { func TestServer_initLogs_invalid(t *testing.T) { // Invoke the function being tested - initLogs("invalid") + initLogs("invalid", "json") // Assert that the global log level is set to info level when the log level is invalid assert.Equal(t, zerolog.InfoLevel, zerolog.GlobalLevel()) From a11f78eb97cd1276aa4478770c3ccb7d410b8a06 Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 16:28:49 -0700 Subject: [PATCH 06/17] creating tasks with orm --- cmd/argo-watcher/argo.go | 21 ++-- cmd/argo-watcher/router.go | 2 +- cmd/argo-watcher/state/in_memory_state.go | 12 +- .../state/in_memory_state_test.go | 11 +- cmd/argo-watcher/state/postgres_state.go | 116 +++++++----------- cmd/argo-watcher/state/postgres_state_test.go | 11 +- cmd/argo-watcher/state/state.go | 2 +- .../state/state_models/task_model.go | 26 ++++ go.mod | 9 ++ go.sum | 19 +++ internal/models/constants.go | 1 + pkg/client/client_test.go | 4 +- 12 files changed, 127 insertions(+), 107 deletions(-) create mode 100644 cmd/argo-watcher/state/state_models/task_model.go diff --git a/cmd/argo-watcher/argo.go b/cmd/argo-watcher/argo.go index 6a4cc61d..136a53e4 100644 --- a/cmd/argo-watcher/argo.go +++ b/cmd/argo-watcher/argo.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/google/uuid" "github.com/rs/zerolog/log" "github.com/shini4i/argo-watcher/cmd/argo-watcher/state" @@ -72,9 +71,6 @@ func (argo *Argo) AddTask(task models.Task) (*models.Task, error) { return nil, errors.New(err.Error()) } - task.Id = uuid.New().String() - log.Info().Str("id", task.Id).Msg("Starting new task creation") - if task.Images == nil || len(task.Images) == 0 { return nil, fmt.Errorf("trying to create task without images") } @@ -83,23 +79,22 @@ func (argo *Argo) AddTask(task models.Task) (*models.Task, error) { return nil, fmt.Errorf("trying to create task without app name") } - log.Info().Str("id", task.Id).Msgf("A new task was triggered") + newTask, err := argo.state.Add(task) + if err != nil { + return nil, err + } - for index, value := range task.Images { - log.Info().Str("id", task.Id).Msgf("Task image [%d] expecting tag %s in app %s.", + log.Info().Str("id", newTask.Id).Msgf("A new task was triggered") + for index, value := range newTask.Images { + log.Info().Str("id", newTask.Id).Msgf("Task image [%d] expecting tag %s in app %s.", index, value.Tag, task.App, ) } - err = argo.state.Add(task) - if err != nil { - return nil, err - } - argo.metrics.AddProcessedDeployment() - return &task, nil + return newTask, nil } func (argo *Argo) GetTasks(startTime float64, endTime float64, app string) models.TasksResponse { diff --git a/cmd/argo-watcher/router.go b/cmd/argo-watcher/router.go index b316f6a7..2b0a0fa4 100644 --- a/cmd/argo-watcher/router.go +++ b/cmd/argo-watcher/router.go @@ -127,7 +127,7 @@ func (env *Env) addTask(c *gin.Context) { // return information about created task c.JSON(http.StatusAccepted, models.TaskStatus{ Id: newTask.Id, - Status: "accepted", + Status: models.StatusAccepted, }) } diff --git a/cmd/argo-watcher/state/in_memory_state.go b/cmd/argo-watcher/state/in_memory_state.go index 1b41b883..718bafc1 100644 --- a/cmd/argo-watcher/state/in_memory_state.go +++ b/cmd/argo-watcher/state/in_memory_state.go @@ -28,11 +28,11 @@ func (state *InMemoryState) Connect(serverConfig *config.ServerConfig) error { // It takes a models.Task parameter and updates the task's created timestamp and status. // The method appends the task to the list of tasks in the in-memory state. // It always returns nil as there is no error handling in the in-memory implementation. -func (state *InMemoryState) Add(task models.Task) error { +func (state *InMemoryState) Add(task models.Task) (*models.Task, error) { task.Created = float64(time.Now().Unix()) - task.Status = "in progress" + task.Status = models.StatusInProgressMessage state.tasks = append(state.tasks, task) - return nil + return &task, nil } // GetTasks retrieves tasks from the in-memory state based on the provided time range and app filter. @@ -148,11 +148,11 @@ func (state *InMemoryState) ProcessObsoleteTasks(retryTimes uint) { func processInMemoryObsoleteTasks(tasks []models.Task) []models.Task { var updatedTasks []models.Task for _, task := range tasks { - if task.Status == "app not found" { + if task.Status == models.StatusAppNotFoundMessage { continue } - if task.Status == "in progress" && task.Updated+3600 < float64(time.Now().Unix()) { - task.Status = "aborted" + if task.Status == models.StatusInProgressMessage && task.Updated+3600 < float64(time.Now().Unix()) { + task.Status = models.StatusAborted } updatedTasks = append(updatedTasks, task) } diff --git a/cmd/argo-watcher/state/in_memory_state_test.go b/cmd/argo-watcher/state/in_memory_state_test.go index 1275c57a..39ec210c 100644 --- a/cmd/argo-watcher/state/in_memory_state_test.go +++ b/cmd/argo-watcher/state/in_memory_state_test.go @@ -1,10 +1,11 @@ package state import ( - "github.com/stretchr/testify/assert" "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/google/uuid" "github.com/shini4i/argo-watcher/internal/models" @@ -27,7 +28,7 @@ var ( Tag: "v0.0.1", }, }, - Status: "in progress", + Status: models.StatusInProgressMessage, }, { Id: uuid.New().String(), @@ -41,14 +42,14 @@ var ( Tag: "v0.0.1", }, }, - Status: "in progress", + Status: models.StatusInProgressMessage, }, } ) func TestInMemoryState_Add(t *testing.T) { for _, task := range tasks { - if err := state.Add(task); err != nil { + if _, err := state.Add(task); err != nil { t.Errorf("Unexpected error: %s", err) } } @@ -57,7 +58,7 @@ func TestInMemoryState_Add(t *testing.T) { func TestInMemoryState_GetTask(t *testing.T) { task, _ := state.GetTask(taskId) - assert.Equal(t, task.Status, "in progress") + assert.Equal(t, task.Status, models.StatusInProgressMessage) } func TestInMemoryState_GetTasks(t *testing.T) { diff --git a/cmd/argo-watcher/state/postgres_state.go b/cmd/argo-watcher/state/postgres_state.go index be6d4af9..6763651d 100644 --- a/cmd/argo-watcher/state/postgres_state.go +++ b/cmd/argo-watcher/state/postgres_state.go @@ -6,119 +6,87 @@ import ( "fmt" "time" - "github.com/uptrace/bun" - "github.com/uptrace/bun/dialect/pgdialect" - "github.com/uptrace/bun/driver/pgdriver" + "github.com/rs/zerolog/log" + "gorm.io/datatypes" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/logger" "github.com/avast/retry-go/v4" _ "github.com/lib/pq" - "github.com/rs/zerolog/log" - "github.com/golang-migrate/migrate/v4" - "github.com/golang-migrate/migrate/v4/database/postgres" _ "github.com/golang-migrate/migrate/v4/source/file" "github.com/shini4i/argo-watcher/cmd/argo-watcher/config" + "github.com/shini4i/argo-watcher/cmd/argo-watcher/state/state_models" "github.com/shini4i/argo-watcher/internal/models" ) type PostgresState struct { db *sql.DB // for backwards compatibility. NOTE: note save when using multiple connections in ORM (connection POOL or reconnecting) - orm *bun.DB + orm *gorm.DB } // Connect establishes a connection to the PostgreSQL database using the provided server configuration. func (state *PostgresState) Connect(serverConfig *config.ServerConfig) error { - // create options - options := []pgdriver.Option{ - // connection options - pgdriver.WithNetwork("tcp"), - pgdriver.WithAddr(fmt.Sprintf("%s:%s", serverConfig.DbHost, serverConfig.DbPort)), // localhost:5432 - pgdriver.WithTLSConfig(nil), // sslmode=disable - // DB timeout configrations - pgdriver.WithTimeout(5 * time.Second), - pgdriver.WithDialTimeout(5 * time.Second), - pgdriver.WithReadTimeout(5 * time.Second), - pgdriver.WithWriteTimeout(5 * time.Second), - } - if serverConfig.DbName != "" { - options = append(options, pgdriver.WithDatabase(serverConfig.DbName)) - } - if serverConfig.DbUser != "" { - options = append(options, pgdriver.WithUser(serverConfig.DbUser)) - } - if serverConfig.DbPassword != "" { - options = append(options, pgdriver.WithPassword(serverConfig.DbPassword)) - } - - // create driver - connector := pgdriver.NewConnector(options...) - driver := sql.OpenDB(connector) - - // Confirm a successful connection. - if err := driver.Ping(); err != nil { - return err + dsnTemplate := "host=%s port=%s user=%s password=%s dbname=%s sslmode=disable TimeZone=UTC" + dsn := fmt.Sprintf(dsnTemplate, serverConfig.DbHost, serverConfig.DbPort, serverConfig.DbUser, serverConfig.DbPassword, serverConfig.DbName) + + // create connection + ormConfig := &gorm.Config{} + // we can leave logger enabled only for text format + if serverConfig.LogFormat != config.LOG_FORMAT_TEXT { + // disable logging until we implement zerolog logger for ORM + ormConfig.Logger = logger.Default.LogMode(logger.Silent) } - - // create ORM database connection - state.orm = bun.NewDB(driver, pgdialect.New()) - state.db = state.orm.DB - - // do migrations (temporary version) - // TODO: change to use ORM migrations - err := runMigrations(serverConfig, state.db) + orm, err := gorm.Open(postgres.Open(dsn), ormConfig) if err != nil { return err } - return nil -} + // save orm object + state.orm = orm -func runMigrations(serverConfig *config.ServerConfig, db *sql.DB) error { - migrationsPath := fmt.Sprintf("file://%s", serverConfig.DbMigrationsPath) - migrationDriver, err := postgres.WithInstance(db, &postgres.Config{}) - if err != nil { - return err - } - migrations, err := migrate.NewWithDatabaseInstance(migrationsPath, "postgres", migrationDriver) + // run migrations + err = orm.AutoMigrate(&state_models.TaskModel{}) if err != nil { return err } - log.Debug().Msg("Running database migrations...") - err = migrations.Up() - if err != nil && err != migrate.ErrNoChange { + // save connection for backwards compatibility + state.db, err = orm.DB() + if err != nil { return err } - log.Debug().Msg("Database schema is up to date.") return nil } // Add inserts a new task into the PostgreSQL database with the provided details. // It takes a models.Task parameter and returns an error if the insertion fails. // The method executes an INSERT query to add a new record with the task details, including the current UTC time. -func (state *PostgresState) Add(task models.Task) error { - images, err := json.Marshal(task.Images) - if err != nil { - return fmt.Errorf("could not marshal images into json") +func (state *PostgresState) Add(task models.Task) (*models.Task, error) { + ormTask := state_models.TaskModel{ + Images: datatypes.NewJSONSlice(task.Images), + Status: models.StatusInProgressMessage, + ApplicationName: sql.NullString{String: task.App, Valid: true}, + Author: sql.NullString{String: task.Author, Valid: true}, + Project: sql.NullString{String: task.Project, Valid: true}, } - _, err = state.db.Exec("INSERT INTO tasks(id, created, images, status, app, author, project) VALUES ($1, $2, $3, $4, $5, $6, $7)", - task.Id, - time.Now().UTC(), - images, - "in progress", - task.App, - task.Author, - task.Project, - ) - if err != nil { - log.Error().Str("id", task.Id).Msgf("Failed to create task database record with error: %s", err) - return fmt.Errorf("failed to create task in database") + result := state.orm.Create(&ormTask) + if result.Error != nil { + log.Error().Msgf("Failed to create task database record with error: %s", result.Error) + return nil, fmt.Errorf("failed to create task in database") } - return nil + log.Info().Msg(ormTask.Id.String()) + + // pass new values to the task object + task.Id = ormTask.Id.String() + task.Created = float64(ormTask.Created.UnixMilli()) + + return &task, nil } // GetTasks retrieves a list of tasks from the PostgreSQL database based on the provided time range and optional app filter. diff --git a/cmd/argo-watcher/state/postgres_state_test.go b/cmd/argo-watcher/state/postgres_state_test.go index eefd1064..c4e24efd 100644 --- a/cmd/argo-watcher/state/postgres_state_test.go +++ b/cmd/argo-watcher/state/postgres_state_test.go @@ -1,11 +1,12 @@ package state import ( - "github.com/stretchr/testify/assert" "os" "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/shini4i/argo-watcher/cmd/argo-watcher/config" "github.com/shini4i/argo-watcher/internal/helpers" @@ -34,7 +35,7 @@ var ( Tag: "v0.0.1", }, }, - Status: "in progress", + Status: models.StatusInProgressMessage, }, { Id: abortedTaskId, @@ -48,7 +49,7 @@ var ( Tag: "v0.0.1", }, }, - Status: "in progress", + Status: models.StatusInProgressMessage, }, { Id: appNotFoundTaskId, @@ -82,7 +83,7 @@ func TestPostgresState_Add(t *testing.T) { panic(err) } for _, task := range postgresTasks { - err := postgresState.Add(task) + _, err := postgresState.Add(task) if err != nil { t.Errorf("got error %s, expected nil", err.Error()) } @@ -110,7 +111,7 @@ func TestPostgresState_GetTask(t *testing.T) { t.Errorf("got error %s, expected nil", err.Error()) } - assert.Equal(t, "in progress", task.Status) + assert.Equal(t, task.Status, models.StatusInProgressMessage) } func TestPostgresState_SetTaskStatus(t *testing.T) { diff --git a/cmd/argo-watcher/state/state.go b/cmd/argo-watcher/state/state.go index 8d46fc72..0fd8e505 100644 --- a/cmd/argo-watcher/state/state.go +++ b/cmd/argo-watcher/state/state.go @@ -13,7 +13,7 @@ var errDesiredRetry = errors.New("desired retry error") type State interface { Connect(serverConfig *config.ServerConfig) error - Add(task models.Task) error + Add(task models.Task) (*models.Task, error) GetTasks(startTime float64, endTime float64, app string) []models.Task GetTask(id string) (*models.Task, error) SetTaskStatus(id string, status string, reason string) diff --git a/cmd/argo-watcher/state/state_models/task_model.go b/cmd/argo-watcher/state/state_models/task_model.go new file mode 100644 index 00000000..e800aac3 --- /dev/null +++ b/cmd/argo-watcher/state/state_models/task_model.go @@ -0,0 +1,26 @@ +package state_models + +import ( + "database/sql" + "time" + + "github.com/google/uuid" + "github.com/shini4i/argo-watcher/internal/models" + "gorm.io/datatypes" +) + +type TaskModel struct { + Id uuid.UUID `gorm:"column:id;type:uuid;default:gen_random_uuid()"` + Created time.Time `gorm:"column:created;autoCreateTime;not null;index;"` + Updated time.Time `gorm:"column:updated;autoUpdateTime;not null;"` + Images datatypes.JSONSlice[models.Image] `gorm:"column:images;not null;"` + Status string `gorm:"column:status;type:VARCHAR(20);not null;index;"` + ApplicationName sql.NullString `gorm:"column:app;type:VARCHAR(255);"` + Author sql.NullString `gorm:"column:author;type:VARCHAR(255);"` + Project sql.NullString `gorm:"column:project;type:VARCHAR(255);"` + StatusReason sql.NullString `gorm:"column:status_reason;default:''"` +} + +func (TaskModel) TableName() string { + return "tasks" +} diff --git a/go.mod b/go.mod index 454ab2d1..0c03a939 100644 --- a/go.mod +++ b/go.mod @@ -35,11 +35,16 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.3.1 // indirect github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect @@ -72,5 +77,9 @@ require ( golang.org/x/tools v0.10.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/datatypes v1.2.0 // indirect + gorm.io/driver/mysql v1.4.7 // indirect + gorm.io/driver/postgres v1.5.2 // indirect + gorm.io/gorm v1.25.2 // indirect mellium.im/sasl v0.3.1 // indirect ) diff --git a/go.sum b/go.sum index 93421811..aee6cb9a 100644 --- a/go.sum +++ b/go.sum @@ -496,6 +496,8 @@ github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -715,6 +717,7 @@ github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfG github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= @@ -726,6 +729,8 @@ github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= @@ -740,6 +745,8 @@ github.com/jackc/pgx/v4 v4.5.0/go.mod h1:EpAKPLdnTorwmPUUsqrPxy5fphV18j9q3wrfRXg github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6fOLDxqtlyhe9UWgfIi9R8+8v8GKV5TRA/o= github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA= +github.com/jackc/pgx/v5 v5.3.1 h1:Fcr8QJ1ZeLi5zsPZqQeUZhNhxfkkKBOgJuYkJHoBOtU= +github.com/jackc/pgx/v5 v5.3.1/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -748,6 +755,9 @@ github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -1890,9 +1900,18 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/datatypes v1.2.0 h1:5YT+eokWdIxhJgWHdrb2zYUimyk0+TaFth+7a0ybzco= +gorm.io/datatypes v1.2.0/go.mod h1:o1dh0ZvjIjhH/bngTpypG6lVRJ5chTBxE09FH/71k04= +gorm.io/driver/mysql v1.4.7 h1:rY46lkCspzGHn7+IYsNpSfEv9tA+SU4SkkB+GFX125Y= +gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= +gorm.io/driver/postgres v1.5.2 h1:ytTDxxEv+MplXOfFe3Lzm7SjG09fcdb3Z/c056DTBx0= +gorm.io/driver/postgres v1.5.2/go.mod h1:fmpX0m2I1PKuR7mKZiEluwrP3hbs+ps7JIGMUBpCgl8= gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= +gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/internal/models/constants.go b/internal/models/constants.go index 0e0ae5c9..436407ff 100644 --- a/internal/models/constants.go +++ b/internal/models/constants.go @@ -9,4 +9,5 @@ const ( StatusConnectionUnavailable = "cannot connect to database" StatusArgoCDFailedLogin = "failed to login to argocd" StatusDeployedMessage = "deployed" + StatusAccepted = "accepted" ) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 5c0994f3..da6d44ed 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -36,7 +36,7 @@ func addTaskHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) err := json.NewEncoder(w).Encode(models.TaskStatus{ - Status: "accepted", + Status: models.StatusAccepted, Id: taskId, }) if err != nil { @@ -90,7 +90,7 @@ func init() { func TestAddTask(t *testing.T) { expected := models.TaskStatus{ - Status: "accepted", + Status: models.StatusAccepted, Id: taskId, } From 577ab03c24b67c38364d1e298f21df46447c5eae Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 17:19:17 -0700 Subject: [PATCH 07/17] use orm for postgresql --- cmd/argo-watcher/state/in_memory_state.go | 3 +- cmd/argo-watcher/state/postgres_state.go | 183 ++++++------------ cmd/argo-watcher/state/state.go | 2 +- .../state/state_models/task_model.go | 20 +- 4 files changed, 75 insertions(+), 133 deletions(-) diff --git a/cmd/argo-watcher/state/in_memory_state.go b/cmd/argo-watcher/state/in_memory_state.go index 718bafc1..3cbe6542 100644 --- a/cmd/argo-watcher/state/in_memory_state.go +++ b/cmd/argo-watcher/state/in_memory_state.go @@ -81,7 +81,7 @@ func (state *InMemoryState) GetTask(id string) (*models.Task, error) { // It takes a string parameter for the task ID, status, and reason. // The method iterates over the tasks in the in-memory state and updates the task with the matching ID. // Note that this method does not perform any error handling if the task ID is not found. -func (state *InMemoryState) SetTaskStatus(id string, status string, reason string) { +func (state *InMemoryState) SetTaskStatus(id string, status string, reason string) error { for idx, task := range state.tasks { if task.Id == id { state.tasks[idx].Status = status @@ -89,6 +89,7 @@ func (state *InMemoryState) SetTaskStatus(id string, status string, reason strin state.tasks[idx].Updated = float64(time.Now().Unix()) } } + return nil } // GetAppList retrieves a list of unique app names from the tasks in the in-memory state. diff --git a/cmd/argo-watcher/state/postgres_state.go b/cmd/argo-watcher/state/postgres_state.go index 6763651d..d6e416ea 100644 --- a/cmd/argo-watcher/state/postgres_state.go +++ b/cmd/argo-watcher/state/postgres_state.go @@ -2,10 +2,10 @@ package state import ( "database/sql" - "encoding/json" "fmt" "time" + "github.com/google/uuid" "github.com/rs/zerolog/log" "gorm.io/datatypes" "gorm.io/driver/postgres" @@ -38,7 +38,12 @@ func (state *PostgresState) Connect(serverConfig *config.ServerConfig) error { if serverConfig.LogFormat != config.LOG_FORMAT_TEXT { // disable logging until we implement zerolog logger for ORM ormConfig.Logger = logger.Default.LogMode(logger.Silent) + } else { + // output all the SQL queries + ormConfig.Logger = logger.Default.LogMode(logger.Info) } + + // create ORM driver orm, err := gorm.Open(postgres.Open(dsn), ormConfig) if err != nil { return err @@ -97,67 +102,25 @@ func (state *PostgresState) GetTasks(startTime float64, endTime float64, app str startTimeUTC := time.Unix(int64(startTime), 0).UTC() endTimeUTC := time.Unix(int64(endTime), 0).UTC() - var rows *sql.Rows - var err error - - if app == "" { - rows, err = state.db.Query( - "select id, extract(epoch from created) AS created, "+ - "extract(epoch from updated) AS updated, "+ - "images, status, status_reason, app, author, "+ - "project from tasks where created >= $1 AND created <= $2", - startTimeUTC, endTimeUTC) - } else { - rows, err = state.db.Query( - "select id, extract(epoch from created) AS created, "+ - "extract(epoch from updated) AS updated, "+ - "images, status, status_reason, app, author, "+ - "project from tasks where created >= $1 AND created <= $2 AND app = $3", - startTimeUTC, endTimeUTC, app) + query := state.orm.Model(&state_models.TaskModel{}).Where("created > ?", startTimeUTC).Where("created <= ?", endTimeUTC) + if app != "" { + query.Where("app = ?", app) } - - if err != nil { - log.Error().Msg(err.Error()) + var ormTasks []state_models.TaskModel + result := query.Find(&ormTasks) + if result.Error != nil { + log.Error().Msg(result.Error.Error()) return nil } - defer func(rows *sql.Rows) { - err := rows.Close() - if err != nil { - log.Error().Msg(err.Error()) - } - }(rows) - - var tasks []models.Task - - // This is required to handle potential null values in updated column - var updated sql.NullFloat64 - // A temporary variable to store images column content - var images []uint8 - - for rows.Next() { - var task models.Task - - if err := rows.Scan(&task.Id, &task.Created, &updated, &images, &task.Status, &task.StatusReason, &task.App, &task.Author, &task.Project); err != nil { - panic(err) - } - - if updated.Valid { - task.Updated = updated.Float64 - } - - if err := json.Unmarshal(images, &task.Images); err != nil { - panic(err) - } - - tasks = append(tasks, task) + var tasks []models.Task = []models.Task{} + for index := 0; index < len(ormTasks); index++ { + ormTask := ormTasks[index] + task := ormTask.ConvertToExternalTask() + tasks = append(tasks, *task) } - if tasks == nil { - return []models.Task{} - } else { - return tasks - } + return tasks } // GetTask retrieves a task from the PostgreSQL database based on the provided task ID. @@ -165,55 +128,30 @@ func (state *PostgresState) GetTasks(startTime float64, endTime float64, app str // The method executes a SELECT query with the given task ID and scans the result into the task struct. // It handles converting the created and updated timestamps to float64 values and unmarshalling the images from the database. func (state *PostgresState) GetTask(id string) (*models.Task, error) { - var ( - task models.Task - imagesBytes []uint8 - images []models.Image - createdStr string - updatedNull sql.NullTime - created time.Time - err error - ) - - query := ` - SELECT id, status, status_reason, app, author, project, images, created, updated - FROM tasks - WHERE id=$1 - ` - - row := state.db.QueryRow(query, id) - - if err := row.Scan(&task.Id, &task.Status, &task.StatusReason, &task.App, &task.Author, &task.Project, &imagesBytes, &createdStr, &updatedNull); err != nil { - return nil, err - } - - if err := json.Unmarshal(imagesBytes, &images); err != nil { - return nil, err - } - - task.Images = images - - if created, err = time.Parse(time.RFC3339, createdStr); err != nil { - return nil, err - } - task.Created = float64(created.Unix()) - - if updatedNull.Valid { - updatedFloat := updatedNull.Time.Unix() - task.Updated = float64(updatedFloat) + var ormTask state_models.TaskModel + result := state.orm.Take(&ormTask, "id = ?", id) + if result.Error != nil { + return nil, result.Error } - - return &task, nil + task := ormTask.ConvertToExternalTask() + return task, nil } // SetTaskStatus updates the status, status_reason, and updated fields of a task in the PostgreSQL database. // It takes the task ID, new status, and status reason as input parameters. // The updated field is set to the current UTC time. -func (state *PostgresState) SetTaskStatus(id string, status string, reason string) { - _, err := state.db.Exec("UPDATE tasks SET status=$1, status_reason=$2, updated=$3 WHERE id=$4", status, reason, time.Now().UTC(), id) +func (state *PostgresState) SetTaskStatus(id string, status string, reason string) error { + uuidv4, err := uuid.Parse(id) if err != nil { - log.Error().Msg(err.Error()) + return err } + var ormTask state_models.TaskModel = state_models.TaskModel{Id: uuidv4} + result := state.orm.Model(ormTask).Updates(state_models.TaskModel{Status: status, StatusReason: sql.NullString{String: reason, Valid: true}}) + if result.Error != nil { + return result.Error + } + + return nil } // GetAppList retrieves a list of distinct application names from the tasks table in the PostgreSQL database. @@ -222,30 +160,11 @@ func (state *PostgresState) SetTaskStatus(id string, status string, reason strin func (state *PostgresState) GetAppList() []string { var apps []string - rows, err := state.db.Query("SELECT DISTINCT app FROM tasks") - if err != nil { - log.Error().Msg(err.Error()) - } - - defer func(rows *sql.Rows) { - err := rows.Close() - if err != nil { - log.Error().Msg(err.Error()) - } - }(rows) - - for rows.Next() { - var app string - if err := rows.Scan(&app); err != nil { - panic(err) - } - apps = append(apps, app) - } - - if apps == nil { + result := state.orm.Model(&state_models.TaskModel{}).Distinct().Pluck("ApplicationName", &apps) + if result.Error != nil { + log.Error().Msg(result.Error.Error()) return []string{} } - return apps } @@ -253,12 +172,14 @@ func (state *PostgresState) GetAppList() []string { // It returns true if the database connection is successful and the test query is executed without errors. // It returns false if there is an error in the database connection or the test query execution. func (state *PostgresState) Check() bool { - _, err := state.db.Exec("SELECT 1") + connection, err := state.orm.DB() if err != nil { log.Error().Msg(err.Error()) return false } - return true + + err = connection.Ping() + return err == nil } // ProcessObsoleteTasks monitors and handles obsolete tasks in the PostgreSQL state. @@ -269,7 +190,7 @@ func (state *PostgresState) ProcessObsoleteTasks(retryTimes uint) { log.Debug().Msg("Starting watching for obsolete tasks...") err := retry.Do( func() error { - if err := processPostgresObsoleteTasks(state.db); err != nil { + if err := state.doProcessPostgresObsoleteTasks(); err != nil { log.Error().Msgf("Couldn't process obsolete tasks. Got the following error: %s", err) return err } @@ -289,15 +210,21 @@ func (state *PostgresState) ProcessObsoleteTasks(retryTimes uint) { // It removes tasks with a status of 'app not found' and marks tasks older than 1 hour as 'aborted'. // The function expects a valid *sql.DB connection to the PostgreSQL database. // It returns an error if any database operation encounters an error; otherwise, it returns nil. -func processPostgresObsoleteTasks(db *sql.DB) error { +func (state *PostgresState) doProcessPostgresObsoleteTasks() error { log.Debug().Msg("Removing obsolete tasks...") - if _, err := db.Exec("DELETE FROM tasks WHERE status = 'app not found'"); err != nil { - return err + + var result *gorm.DB + + log.Debug().Msg("Marking app not found tasks older than 1 hour as aborted...") + result = state.orm.Where("status = ?", models.StatusAppNotFoundMessage).Where("created < now() - interval '1 hour'").Delete(&state_models.TaskModel{}) + if result.Error != nil { + return result.Error } - log.Debug().Msg("Marking tasks older than 1 hour as aborted...") - if _, err := db.Exec("UPDATE tasks SET status='aborted' WHERE status = 'in progress' AND created < now() - interval '1 hour'"); err != nil { - return err + log.Debug().Msg("Marking in progress tasks older than 1 hour as aborted...") + result = state.orm.Where("status = ?", models.StatusInProgressMessage).Where("created < now() - interval '1 hour'").Delete(&state_models.TaskModel{}) + if result.Error != nil { + return result.Error } return nil diff --git a/cmd/argo-watcher/state/state.go b/cmd/argo-watcher/state/state.go index 0fd8e505..55333d4b 100644 --- a/cmd/argo-watcher/state/state.go +++ b/cmd/argo-watcher/state/state.go @@ -16,7 +16,7 @@ type State interface { Add(task models.Task) (*models.Task, error) GetTasks(startTime float64, endTime float64, app string) []models.Task GetTask(id string) (*models.Task, error) - SetTaskStatus(id string, status string, reason string) + SetTaskStatus(id string, status string, reason string) error GetAppList() []string Check() bool ProcessObsoleteTasks(retryTimes uint) diff --git a/cmd/argo-watcher/state/state_models/task_model.go b/cmd/argo-watcher/state/state_models/task_model.go index e800aac3..27b3ee59 100644 --- a/cmd/argo-watcher/state/state_models/task_model.go +++ b/cmd/argo-watcher/state/state_models/task_model.go @@ -15,12 +15,26 @@ type TaskModel struct { Updated time.Time `gorm:"column:updated;autoUpdateTime;not null;"` Images datatypes.JSONSlice[models.Image] `gorm:"column:images;not null;"` Status string `gorm:"column:status;type:VARCHAR(20);not null;index;"` - ApplicationName sql.NullString `gorm:"column:app;type:VARCHAR(255);"` - Author sql.NullString `gorm:"column:author;type:VARCHAR(255);"` - Project sql.NullString `gorm:"column:project;type:VARCHAR(255);"` + ApplicationName sql.NullString `gorm:"column:app;type:VARCHAR(255);not null;"` + Author sql.NullString `gorm:"column:author;type:VARCHAR(255);not null;"` + Project sql.NullString `gorm:"column:project;type:VARCHAR(255);not null;"` StatusReason sql.NullString `gorm:"column:status_reason;default:''"` } func (TaskModel) TableName() string { return "tasks" } + +func (ormTask *TaskModel) ConvertToExternalTask() *models.Task { + return &models.Task{ + Id: ormTask.Id.String(), + Created: float64(ormTask.Created.Unix()), + Updated: float64(ormTask.Updated.Unix()), + App: ormTask.ApplicationName.String, + Author: ormTask.Author.String, + Project: ormTask.Project.String, + Images: ormTask.Images, + Status: ormTask.Status, + StatusReason: ormTask.StatusReason.String, + } +} From d06fc4b6bf9c3e96cccb7447da4a95b332ce52f4 Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 17:19:58 -0700 Subject: [PATCH 08/17] remove db fallback --- cmd/argo-watcher/state/postgres_state.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cmd/argo-watcher/state/postgres_state.go b/cmd/argo-watcher/state/postgres_state.go index d6e416ea..638e1d89 100644 --- a/cmd/argo-watcher/state/postgres_state.go +++ b/cmd/argo-watcher/state/postgres_state.go @@ -23,7 +23,6 @@ import ( ) type PostgresState struct { - db *sql.DB // for backwards compatibility. NOTE: note save when using multiple connections in ORM (connection POOL or reconnecting) orm *gorm.DB } @@ -53,17 +52,12 @@ func (state *PostgresState) Connect(serverConfig *config.ServerConfig) error { state.orm = orm // run migrations + // note: this doesn't delete existing columns. only adds new ones err = orm.AutoMigrate(&state_models.TaskModel{}) if err != nil { return err } - // save connection for backwards compatibility - state.db, err = orm.DB() - if err != nil { - return err - } - return nil } From 17aed237d3d3fd1a3e27112d4fc2b6680d47314d Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 17:21:12 -0700 Subject: [PATCH 09/17] fix pgsql tests --- cmd/argo-watcher/state/postgres_state_test.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/argo-watcher/state/postgres_state_test.go b/cmd/argo-watcher/state/postgres_state_test.go index c4e24efd..248f4102 100644 --- a/cmd/argo-watcher/state/postgres_state_test.go +++ b/cmd/argo-watcher/state/postgres_state_test.go @@ -78,7 +78,11 @@ func TestPostgresState_Add(t *testing.T) { DbMigrationsPath: "../../../db/migrations", } postgresState.Connect(config) - _, err := postgresState.db.Exec("TRUNCATE TABLE tasks") + db, err := postgresState.orm.DB() + if err != nil { + panic(err) + } + _, err = db.Exec("TRUNCATE TABLE tasks") if err != nil { panic(err) } @@ -140,7 +144,11 @@ func TestPostgresState_GetAppList(t *testing.T) { func TestPostgresState_ProcessObsoleteTasks(t *testing.T) { // set updated time to 2 hour ago for obsolete task updatedTime := time.Now().UTC().Add(-2 * time.Hour) - if _, err := postgresState.db.Exec("UPDATE tasks SET created = $1 WHERE id = $2", updatedTime, abortedTaskId); err != nil { + db, err := postgresState.orm.DB() + if err != nil { + panic(err) + } + if _, err := db.Exec("UPDATE tasks SET created = $1 WHERE id = $2", updatedTime, abortedTaskId); err != nil { t.Errorf("got error %s, expected nil", err.Error()) } From 940d41c9f5912006b09a2009459ff6250e4af738 Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 17:21:55 -0700 Subject: [PATCH 10/17] remove deprecated migrations --- db/migrations/000001_init.down.sql | 0 db/migrations/000001_init.up.sql | 12 ------------ db/migrations/000002_status_reason.down.sql | 1 - db/migrations/000002_status_reason.up.sql | 1 - 4 files changed, 14 deletions(-) delete mode 100644 db/migrations/000001_init.down.sql delete mode 100644 db/migrations/000001_init.up.sql delete mode 100644 db/migrations/000002_status_reason.down.sql delete mode 100644 db/migrations/000002_status_reason.up.sql diff --git a/db/migrations/000001_init.down.sql b/db/migrations/000001_init.down.sql deleted file mode 100644 index e69de29b..00000000 diff --git a/db/migrations/000001_init.up.sql b/db/migrations/000001_init.up.sql deleted file mode 100644 index aa07e854..00000000 --- a/db/migrations/000001_init.up.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE TABLE IF NOT EXISTS tasks -( - id VARCHAR(36) NOT NULL PRIMARY KEY, - created TIMESTAMP NOT NULL, - updated TIMESTAMP DEFAULT NULL, - images JSON NOT NULL, - status VARCHAR(20) NOT NULL, - app VARCHAR(255) DEFAULT NULL, - author VARCHAR(255) DEFAULT NULL, - project VARCHAR(255) DEFAULT NULL -); -CREATE INDEX IF NOT EXISTS tasks_idx_created ON tasks (created); diff --git a/db/migrations/000002_status_reason.down.sql b/db/migrations/000002_status_reason.down.sql deleted file mode 100644 index b63cd578..00000000 --- a/db/migrations/000002_status_reason.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE tasks DROP COLUMN status_reason; diff --git a/db/migrations/000002_status_reason.up.sql b/db/migrations/000002_status_reason.up.sql deleted file mode 100644 index 57fd03df..00000000 --- a/db/migrations/000002_status_reason.up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE tasks ADD COLUMN status_reason TEXT DEFAULT ''; From 825ff9172aeb31f62bd4020df4e7c484235adb17 Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 17:35:32 -0700 Subject: [PATCH 11/17] improve documentation for server config --- cmd/argo-watcher/config/config.go | 3 +- cmd/argo-watcher/state/postgres_state_test.go | 13 ++++--- docs/development.md | 4 +-- docs/installation.md | 36 ++++++++++--------- 4 files changed, 29 insertions(+), 27 deletions(-) diff --git a/cmd/argo-watcher/config/config.go b/cmd/argo-watcher/config/config.go index 141f3716..9934c3f6 100644 --- a/cmd/argo-watcher/config/config.go +++ b/cmd/argo-watcher/config/config.go @@ -22,6 +22,7 @@ type ServerConfig struct { RegistryProxyUrl string `required:"false" envconfig:"DOCKER_IMAGES_PROXY"` StateType string `required:"false" envconfig:"STATE_TYPE"` StaticFilePath string `required:"false" envconfig:"STATIC_FILES_PATH" default:"static"` + SkipTlsVerify string `required:"false" envconfig:"SKIP_TLS_VERIFY" default:"false"` LogLevel string `required:"false" envconfig:"LOG_LEVEL" default:"info"` LogFormat string `required:"false" envconfig:"LOG_FORMAT" default:"json"` Host string `required:"false" envconfig:"HOST" default:"0.0.0.0"` @@ -31,8 +32,6 @@ type ServerConfig struct { DbName string `required:"false" envconfig:"DB_NAME"` DbUser string `required:"false" envconfig:"DB_USER"` DbPassword string `required:"false" envconfig:"DB_PASSWORD"` - DbMigrationsPath string `required:"false" envconfig:"DB_MIGRATIONS_PATH" default:"db/migrations"` - SkipTlsVerify string `required:"false" envconfig:"SKIP_TLS_VERIFY" default:"false"` } // NewServerConfig parses the server configuration from environment variables using the envconfig package. diff --git a/cmd/argo-watcher/state/postgres_state_test.go b/cmd/argo-watcher/state/postgres_state_test.go index 248f4102..6e612f16 100644 --- a/cmd/argo-watcher/state/postgres_state_test.go +++ b/cmd/argo-watcher/state/postgres_state_test.go @@ -69,13 +69,12 @@ var ( func TestPostgresState_Add(t *testing.T) { config := &config.ServerConfig{ - StateType: "postgresql", - DbHost: os.Getenv("DB_HOST"), - DbPort: "5432", - DbUser: os.Getenv("DB_USER"), - DbName: os.Getenv("DB_NAME"), - DbPassword: os.Getenv("DB_PASSWORD"), - DbMigrationsPath: "../../../db/migrations", + StateType: "postgresql", + DbHost: os.Getenv("DB_HOST"), + DbPort: "5432", + DbUser: os.Getenv("DB_USER"), + DbName: os.Getenv("DB_NAME"), + DbPassword: os.Getenv("DB_PASSWORD"), } postgresState.Connect(config) db, err := postgresState.orm.DB() diff --git a/docs/development.md b/docs/development.md index 4547ce43..060cf2bb 100644 --- a/docs/development.md +++ b/docs/development.md @@ -63,7 +63,7 @@ cd cmd/argo-watcher # install dependencies go mod tidy # start argo-watcher (in-memory) -LOG_LEVEL=debug ARGO_URL=http://localhost:8081 ARGO_TOKEN=example STATE_TYPE=in-memory go run . -server +LOG_LEVEL=debug LOG_FORMAT=text ARGO_URL=http://localhost:8081 ARGO_TOKEN=example STATE_TYPE=in-memory go run . -server ``` @@ -82,7 +82,7 @@ cd cmd/argo-watcher # install dependencies go mod tidy # OR start argo-watcher (postgres) -LOG_LEVEL=debug ARGO_URL=http://localhost:8081 ARGO_TOKEN=example STATE_TYPE=postgres DB_USER=watcher DB_PASSWORD=watcher DB_NAME=watcher DB_MIGRATIONS_PATH="./../../db/migrations" go run . -server +LOG_LEVEL=debug LOG_FORMAT=text ARGO_URL=http://localhost:8081 ARGO_TOKEN=example STATE_TYPE=postgres DB_USER=watcher DB_PASSWORD=watcher DB_NAME=watcher DB_MIGRATIONS_PATH="./../../db/migrations" go run . -server ``` #### Logs in simple text diff --git a/docs/installation.md b/docs/installation.md index b14b4d44..0c4199a8 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -43,22 +43,26 @@ ingress: Argo Watcher Server supports the following environment variables -| Variable | Description | Mandatory | -|-------------------|-----------------------------------------------------------------|-----------| -| STATE_TYPE | Accepts "in-memory" (non-HA option) and "postgres" (HA option). | Yes | -| STATIC_FILES_PATH | Path to the UI website of Argo Watcher | Yes | -| ARGO_URL | ArgoCD URL | Yes | -| ARGO_TOKEN | ArgoCD API token | Yes | -| ARGO_TIMEOUT | Time that Argo Watcher is allowed to wait for deployment. | No | -| ARGO_API_TIMEOUT | Timeout for ArgoCD API calls. Defaults to 60 seconds | No | -| SKIP_TLS_VERIFY | Skip SSL verification during API calls | No | -| HOST | Host for Argo Watcher server. Defaults to 0.0.0.0 | No | -| PORT | Port for Argo Watcher server. Defaults to 8080 | No | -| DB_HOST | Database host (Required for STATE_TYPE=postgres) | No | -| DB_PORT | Database port (Required for STATE_TYPE=postgres) | No | -| DB_NAME | Database name (Required for STATE_TYPE=postgres) | No | -| DB_USER | Database username(Required for STATE_TYPE=postgres) | No | -| DB_PASSWORD | Database password (Required for STATE_TYPE=postgres) | No | +| Variable | Description | Default | Mandatory | +|---------------------|-----------------------------------------------------------------------------|-----------|---------------| +| ARGO_URL | ArgoCD URL | | Yes | +| ARGO_TOKEN | ArgoCD API token | | Yes | +| ARGO_API_TIMEOUT | Timeout for ArgoCD API calls. Defaults to 60 seconds | 60 | No | +| ARGO_TIMEOUT | Time that Argo Watcher is allowed to wait for deployment. | 0 | No | +| ARGO_REFRESH_APP | Refresh application during status check | true | No | +| DOCKER_IMAGES_PROXY | Define registry proxy url for image checks | | No | +| STATE_TYPE | Accepts "in-memory" (non-HA option) and "postgres" (HA option). | | Yes | +| STATIC_FILES_PATH | Path to the UI website of Argo Watcher | static | No | +| SKIP_TLS_VERIFY | Skip SSL verification during API calls | false | No | +| LOG_LEVEL | Severity for logging (trace,debug,info,warn,error,fatal, panic) | info | No | +| LOG_FORMAT | json (used for production by default) or text (used for development) | json | No | +| HOST | Host for Argo Watcher server. | 0.0.0.0 | No | +| PORT | Port for Argo Watcher server. | 8080 | No | +| DB_HOST | Database host (Required for STATE_TYPE=postgres) | localhost | Conditional | +| DB_PORT | Database port (Required for STATE_TYPE=postgres) | 5432 | Conditional | +| DB_NAME | Database name (Required for STATE_TYPE=postgres) | | Conditional | +| DB_USER | Database username(Required for STATE_TYPE=postgres) | | Conditional | +| DB_PASSWORD | Database password (Required for STATE_TYPE=postgres) | | Conditional | # Client setup From a07dbe5a34ccd75389173a54148ec20448bcf105 Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 17:46:10 -0700 Subject: [PATCH 12/17] fix argo test tasks --- cmd/argo-watcher/argo_test.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/cmd/argo-watcher/argo_test.go b/cmd/argo-watcher/argo_test.go index 6b7af126..f3ef1b70 100644 --- a/cmd/argo-watcher/argo_test.go +++ b/cmd/argo-watcher/argo_test.go @@ -5,6 +5,7 @@ import ( "regexp" "testing" + "github.com/google/uuid" "github.com/shini4i/argo-watcher/cmd/argo-watcher/mock" "github.com/shini4i/argo-watcher/internal/models" "github.com/stretchr/testify/assert" @@ -211,7 +212,7 @@ func TestArgoAddTask(t *testing.T) { // mock calls to add task stateError := fmt.Errorf("database error") - state.EXPECT().Add(gomock.Any()).Return(stateError) + state.EXPECT().Add(gomock.Any()).Return(nil, stateError) // argo manager argo := &Argo{} @@ -244,24 +245,33 @@ func TestArgoAddTask(t *testing.T) { metrics.EXPECT().SetArgoUnavailable(false) metrics.EXPECT().AddProcessedDeployment() - // mock calls to add task - state.EXPECT().Add(gomock.Any()).Return(nil) - - // argo manager - argo := &Argo{} - argo.Init(state, api, metrics) + // tasks task := models.Task{ App: "test-app", Images: []models.Image{ {Tag: taskImageTag}, }, } - newTask, err := argo.AddTask(task) + newTask := models.Task{ + Id: uuid.NewString(), + App: "test-app", + Images: []models.Image{ + {Tag: taskImageTag}, + }, + } + + // mock calls to add task + state.EXPECT().Add(gomock.Any()).Return(&newTask, nil) + + // argo manager + argo := &Argo{} + argo.Init(state, api, metrics) + newTaskReturned, err := argo.AddTask(task) // assertions assert.Nil(t, err) - assert.NotNil(t, newTask) + assert.NotNil(t, newTaskReturned) uuidRegexp := regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$") - assert.Regexp(t, uuidRegexp, newTask.Id, "Must match Regexp for uuid v4") + assert.Regexp(t, uuidRegexp, newTaskReturned.Id, "Must match Regexp for uuid v4") }) } From 4336d83c7f8dd7641df39482f2a178d7f8b2c40c Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 18:07:12 -0700 Subject: [PATCH 13/17] fix postgres tests --- cmd/argo-watcher/state/postgres_state.go | 2 +- cmd/argo-watcher/state/postgres_state_test.go | 115 ++++++++++-------- 2 files changed, 66 insertions(+), 51 deletions(-) diff --git a/cmd/argo-watcher/state/postgres_state.go b/cmd/argo-watcher/state/postgres_state.go index 638e1d89..f99d4fcc 100644 --- a/cmd/argo-watcher/state/postgres_state.go +++ b/cmd/argo-watcher/state/postgres_state.go @@ -216,7 +216,7 @@ func (state *PostgresState) doProcessPostgresObsoleteTasks() error { } log.Debug().Msg("Marking in progress tasks older than 1 hour as aborted...") - result = state.orm.Where("status = ?", models.StatusInProgressMessage).Where("created < now() - interval '1 hour'").Delete(&state_models.TaskModel{}) + result = state.orm.Where("status = ?", models.StatusInProgressMessage).Where("created < now() - interval '1 hour'").Updates(&state_models.TaskModel{Status: models.StatusAborted}) if result.Error != nil { return result.Error } diff --git a/cmd/argo-watcher/state/postgres_state_test.go b/cmd/argo-watcher/state/postgres_state_test.go index 6e612f16..fbb34317 100644 --- a/cmd/argo-watcher/state/postgres_state_test.go +++ b/cmd/argo-watcher/state/postgres_state_test.go @@ -13,55 +13,46 @@ import ( "github.com/shini4i/argo-watcher/internal/models" ) -const ( - deployedTaskId = "782e6e84-e67d-11ec-9f2f-8a68373f0f50" - appNotFoundTaskId = "5fa2d291-506a-42ab-804a-8bd75dba53e1" - abortedTaskId = "1c35d840-41d1-4b4f-a393-b8b71145686b" -) - var ( created = float64(time.Now().Unix()) postgresState = PostgresState{} - postgresTasks = []models.Task{ - { - Id: deployedTaskId, - Created: created, - App: "Test", - Author: "Test Author", - Project: "Test Project", - Images: []models.Image{ - { - Image: "test", - Tag: "v0.0.1", - }, + + deployedTaskId string + deployedTask = models.Task{ + Created: created, + App: "Test", + Author: "Test Author", + Project: "Test Project", + Images: []models.Image{ + { + Image: "test", + Tag: "v0.0.1", }, - Status: models.StatusInProgressMessage, }, - { - Id: abortedTaskId, - Created: created, - App: "Test2", - Author: "Test Author", - Project: "Test Project", - Images: []models.Image{ - { - Image: "test2", - Tag: "v0.0.1", - }, + } + appNotFoundTaskId string + appNotFoundTask = models.Task{ + Created: created, + App: "Test2", + Author: "Test Author", + Project: "Test Project", + Images: []models.Image{ + { + Image: "test2", + Tag: "v0.0.1", }, - Status: models.StatusInProgressMessage, }, - { - Id: appNotFoundTaskId, - Created: created, - App: "ObsoleteApp", - Author: "Test Author", - Project: "Test Project", - Images: []models.Image{ - { - Image: "test", - Tag: "v0.0.1", - }, + } + abortedTaskId string + abortedTask = models.Task{ + Created: created, + App: "ObsoleteApp", + Author: "Test Author", + Project: "Test Project", + Images: []models.Image{ + { + Image: "test", + Tag: "v0.0.1", }, }, } @@ -69,14 +60,18 @@ var ( func TestPostgresState_Add(t *testing.T) { config := &config.ServerConfig{ - StateType: "postgresql", + StateType: "postgres", DbHost: os.Getenv("DB_HOST"), DbPort: "5432", DbUser: os.Getenv("DB_USER"), DbName: os.Getenv("DB_NAME"), DbPassword: os.Getenv("DB_PASSWORD"), + LogFormat: config.LOG_FORMAT_TEXT, + } + err := postgresState.Connect(config) + if err != nil { + panic(err) } - postgresState.Connect(config) db, err := postgresState.orm.DB() if err != nil { panic(err) @@ -85,12 +80,23 @@ func TestPostgresState_Add(t *testing.T) { if err != nil { panic(err) } - for _, task := range postgresTasks { - _, err := postgresState.Add(task) - if err != nil { - t.Errorf("got error %s, expected nil", err.Error()) - } + deployedTaskResult, err := postgresState.Add(deployedTask) + if err != nil { + t.Errorf("got error %s, expected nil", err.Error()) } + deployedTaskId = deployedTaskResult.Id + + appNotFoundTaskResult, err := postgresState.Add(appNotFoundTask) + if err != nil { + t.Errorf("got error %s, expected nil", err.Error()) + } + appNotFoundTaskId = appNotFoundTaskResult.Id + + abortedTaskResult, err := postgresState.Add(abortedTask) + if err != nil { + t.Errorf("got error %s, expected nil", err.Error()) + } + abortedTaskId = abortedTaskResult.Id } func TestPostgresState_GetTasks(t *testing.T) { @@ -143,14 +149,23 @@ func TestPostgresState_GetAppList(t *testing.T) { func TestPostgresState_ProcessObsoleteTasks(t *testing.T) { // set updated time to 2 hour ago for obsolete task updatedTime := time.Now().UTC().Add(-2 * time.Hour) + + // get connections db, err := postgresState.orm.DB() if err != nil { panic(err) } + + // update obsolete task if _, err := db.Exec("UPDATE tasks SET created = $1 WHERE id = $2", updatedTime, abortedTaskId); err != nil { t.Errorf("got error %s, expected nil", err.Error()) } + // update not found task + if _, err := db.Exec("UPDATE tasks SET created = $1 WHERE id = $2", updatedTime, appNotFoundTaskId); err != nil { + t.Errorf("got error %s, expected nil", err.Error()) + } + postgresState.ProcessObsoleteTasks(1) // Check that obsolete task was deleted @@ -160,7 +175,7 @@ func TestPostgresState_ProcessObsoleteTasks(t *testing.T) { if task, err := postgresState.GetTask(abortedTaskId); err != nil { t.Errorf("got error %s, expected nil", err.Error()) } else { - assert.Equal(t, "aborted", task.Status) + assert.Equal(t, models.StatusAborted, task.Status) } } From c5fca43755d9a7f7496243fc7ba6440d1d047b6d Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 18:16:17 -0700 Subject: [PATCH 14/17] fix golang-lint errors --- cmd/argo-watcher/argo_status_updater.go | 40 +++++++++++++++---- .../state/in_memory_state_test.go | 5 ++- cmd/argo-watcher/state/postgres_state_test.go | 10 +++-- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/cmd/argo-watcher/argo_status_updater.go b/cmd/argo-watcher/argo_status_updater.go index bc96ee87..3ed5659f 100644 --- a/cmd/argo-watcher/argo_status_updater.go +++ b/cmd/argo-watcher/argo_status_updater.go @@ -185,47 +185,68 @@ func (updater *ArgoStatusUpdater) handleArgoAPIFailure(task models.Task, err err func (updater *ArgoStatusUpdater) handleAppNotFound(task models.Task, err error) { log.Info().Str("id", task.Id).Msgf("Application %s does not exist.", task.App) reason := fmt.Sprintf(ArgoAPIErrorTemplate, err.Error()) - updater.argo.state.SetTaskStatus(task.Id, models.StatusAppNotFoundMessage, reason) + errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusAppNotFoundMessage, reason) + if errStatusChange != nil { + log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + } } func (updater *ArgoStatusUpdater) handleArgoUnavailable(task models.Task, err error) { log.Error().Str("id", task.Id).Msg("ArgoCD is not available. Aborting.") reason := fmt.Sprintf(ArgoAPIErrorTemplate, err.Error()) - updater.argo.state.SetTaskStatus(task.Id, models.StatusAborted, reason) + errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusAborted, reason) + if errStatusChange != nil { + log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + } } func (updater *ArgoStatusUpdater) handleDeploymentFailed(task models.Task, err error) { log.Warn().Str("id", task.Id).Msgf("Deployment failed. Aborting with error: %s", err) updater.argo.metrics.AddFailedDeployment(task.App) reason := fmt.Sprintf(ArgoAPIErrorTemplate, err.Error()) - updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) + errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) + if errStatusChange != nil { + log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + } } func (updater *ArgoStatusUpdater) handleDeploymentSuccess(task models.Task) { log.Info().Str("id", task.Id).Msg("App is running on the excepted version.") updater.argo.metrics.ResetFailedDeployment(task.App) - updater.argo.state.SetTaskStatus(task.Id, models.StatusDeployedMessage, "") + errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusDeployedMessage, "") + if errStatusChange != nil { + log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + } } func (updater *ArgoStatusUpdater) handleAppNotAvailable(task models.Task, err error) { log.Warn().Str("id", task.Id).Msgf("Deployment failed. Application not available\n%s", err.Error()) updater.argo.metrics.AddFailedDeployment(task.App) reason := fmt.Sprintf("Application not available\n\n%s", err.Error()) - updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) + errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) + if errStatusChange != nil { + log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + } } func (updater *ArgoStatusUpdater) handleAppNotHealthy(task models.Task, err error) { log.Warn().Str("id", task.Id).Msgf("Deployment failed. Application not healthy\n%s", err.Error()) updater.argo.metrics.AddFailedDeployment(task.App) reason := fmt.Sprintf("Application not healthy\n\n%s", err.Error()) - updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) + errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) + if errStatusChange != nil { + log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + } } func (updater *ArgoStatusUpdater) handleAppOutOfSync(task models.Task, err error) { log.Warn().Str("id", task.Id).Msgf("Deployment failed. Application out of sync\n%s", err.Error()) updater.argo.metrics.AddFailedDeployment(task.App) reason := fmt.Sprintf("Application out of sync\n\n%s", err.Error()) - updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) + errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) + if errStatusChange != nil { + log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + } } func (updater *ArgoStatusUpdater) handleDeploymentUnexpectedStatus(task models.Task, err error) { @@ -233,5 +254,8 @@ func (updater *ArgoStatusUpdater) handleDeploymentUnexpectedStatus(task models.T log.Error().Str("id", task.Id).Msgf("Deployment error\n%s", err.Error()) updater.argo.metrics.AddFailedDeployment(task.App) reason := fmt.Sprintf("Deployment timeout\n\n%s", err.Error()) - updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) + errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) + if errStatusChange != nil { + log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + } } diff --git a/cmd/argo-watcher/state/in_memory_state_test.go b/cmd/argo-watcher/state/in_memory_state_test.go index 39ec210c..41d08865 100644 --- a/cmd/argo-watcher/state/in_memory_state_test.go +++ b/cmd/argo-watcher/state/in_memory_state_test.go @@ -70,7 +70,10 @@ func TestInMemoryState_GetTasks(t *testing.T) { } func TestInMemoryState_SetTaskStatus(t *testing.T) { - state.SetTaskStatus(taskId, "deployed", "") + err := state.SetTaskStatus(taskId, "deployed", "") + if err != nil { + t.Errorf("got %s, expected %s", err, "nil") + } if taskInfo, _ := state.GetTask(taskId); taskInfo.Status != "deployed" { t.Errorf("got %s, expected %s", taskInfo.Status, "deployed") diff --git a/cmd/argo-watcher/state/postgres_state_test.go b/cmd/argo-watcher/state/postgres_state_test.go index fbb34317..5a8e0f1c 100644 --- a/cmd/argo-watcher/state/postgres_state_test.go +++ b/cmd/argo-watcher/state/postgres_state_test.go @@ -66,7 +66,6 @@ func TestPostgresState_Add(t *testing.T) { DbUser: os.Getenv("DB_USER"), DbName: os.Getenv("DB_NAME"), DbPassword: os.Getenv("DB_PASSWORD"), - LogFormat: config.LOG_FORMAT_TEXT, } err := postgresState.Connect(config) if err != nil { @@ -124,8 +123,13 @@ func TestPostgresState_GetTask(t *testing.T) { } func TestPostgresState_SetTaskStatus(t *testing.T) { - postgresState.SetTaskStatus(deployedTaskId, "deployed", "") - postgresState.SetTaskStatus(appNotFoundTaskId, "app not found", "") + if err := postgresState.SetTaskStatus(deployedTaskId, "deployed", ""); err != nil { + t.Errorf("got error %s, expected nil", err.Error()) + } + + if err := postgresState.SetTaskStatus(appNotFoundTaskId, "app not found", ""); err != nil { + t.Errorf("got error %s, expected nil", err.Error()) + } if taskInfo, _ := postgresState.GetTask(deployedTaskId); taskInfo.Status != "deployed" { t.Errorf("got %s, expected %s", taskInfo.Status, "deployed") From 53252401b893c53eb3b9c54721af1cf8af63a3b1 Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Mon, 7 Aug 2023 18:20:19 -0700 Subject: [PATCH 15/17] remove code smell for duplicate error message --- cmd/argo-watcher/argo_status_updater.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cmd/argo-watcher/argo_status_updater.go b/cmd/argo-watcher/argo_status_updater.go index 3ed5659f..e2ce857a 100644 --- a/cmd/argo-watcher/argo_status_updater.go +++ b/cmd/argo-watcher/argo_status_updater.go @@ -13,6 +13,7 @@ import ( ) const defaultErrorMessage string = "could not retrieve details" +const failedToUpdateTaskStatusTemplate string = "Failed to change task status: %s" type ArgoStatusUpdater struct { argo Argo @@ -187,7 +188,7 @@ func (updater *ArgoStatusUpdater) handleAppNotFound(task models.Task, err error) reason := fmt.Sprintf(ArgoAPIErrorTemplate, err.Error()) errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusAppNotFoundMessage, reason) if errStatusChange != nil { - log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + log.Error().Str("id", task.Id).Msgf(failedToUpdateTaskStatusTemplate, errStatusChange) } } @@ -196,7 +197,7 @@ func (updater *ArgoStatusUpdater) handleArgoUnavailable(task models.Task, err er reason := fmt.Sprintf(ArgoAPIErrorTemplate, err.Error()) errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusAborted, reason) if errStatusChange != nil { - log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + log.Error().Str("id", task.Id).Msgf(failedToUpdateTaskStatusTemplate, errStatusChange) } } @@ -206,7 +207,7 @@ func (updater *ArgoStatusUpdater) handleDeploymentFailed(task models.Task, err e reason := fmt.Sprintf(ArgoAPIErrorTemplate, err.Error()) errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) if errStatusChange != nil { - log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + log.Error().Str("id", task.Id).Msgf(failedToUpdateTaskStatusTemplate, errStatusChange) } } @@ -215,7 +216,7 @@ func (updater *ArgoStatusUpdater) handleDeploymentSuccess(task models.Task) { updater.argo.metrics.ResetFailedDeployment(task.App) errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusDeployedMessage, "") if errStatusChange != nil { - log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + log.Error().Str("id", task.Id).Msgf(failedToUpdateTaskStatusTemplate, errStatusChange) } } @@ -225,7 +226,7 @@ func (updater *ArgoStatusUpdater) handleAppNotAvailable(task models.Task, err er reason := fmt.Sprintf("Application not available\n\n%s", err.Error()) errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) if errStatusChange != nil { - log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + log.Error().Str("id", task.Id).Msgf(failedToUpdateTaskStatusTemplate, errStatusChange) } } @@ -235,7 +236,7 @@ func (updater *ArgoStatusUpdater) handleAppNotHealthy(task models.Task, err erro reason := fmt.Sprintf("Application not healthy\n\n%s", err.Error()) errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) if errStatusChange != nil { - log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + log.Error().Str("id", task.Id).Msgf(failedToUpdateTaskStatusTemplate, errStatusChange) } } @@ -245,7 +246,7 @@ func (updater *ArgoStatusUpdater) handleAppOutOfSync(task models.Task, err error reason := fmt.Sprintf("Application out of sync\n\n%s", err.Error()) errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) if errStatusChange != nil { - log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + log.Error().Str("id", task.Id).Msgf(failedToUpdateTaskStatusTemplate, errStatusChange) } } @@ -256,6 +257,6 @@ func (updater *ArgoStatusUpdater) handleDeploymentUnexpectedStatus(task models.T reason := fmt.Sprintf("Deployment timeout\n\n%s", err.Error()) errStatusChange := updater.argo.state.SetTaskStatus(task.Id, models.StatusFailedMessage, reason) if errStatusChange != nil { - log.Error().Str("id", task.Id).Msgf("Failed to change task status: %s", errStatusChange) + log.Error().Str("id", task.Id).Msgf(failedToUpdateTaskStatusTemplate, errStatusChange) } } From 5bb13de433045e4c3fe0dd0942bed315827b408a Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Wed, 9 Aug 2023 23:00:33 -0700 Subject: [PATCH 16/17] improvements to unit tests --- cmd/argo-watcher/argo_api_test.go | 11 +++---- .../state/in_memory_state_test.go | 19 +++++------- cmd/argo-watcher/state/postgres_state_test.go | 30 +++++++++---------- pkg/client/client_test.go | 8 ++--- 4 files changed, 32 insertions(+), 36 deletions(-) diff --git a/cmd/argo-watcher/argo_api_test.go b/cmd/argo-watcher/argo_api_test.go index b3c54972..760e69f1 100644 --- a/cmd/argo-watcher/argo_api_test.go +++ b/cmd/argo-watcher/argo_api_test.go @@ -3,11 +3,12 @@ package main import ( "encoding/json" "fmt" - "github.com/rs/zerolog/log" "net/http" "net/http/httptest" "testing" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/assert" "github.com/shini4i/argo-watcher/internal/models" @@ -78,7 +79,7 @@ func TestArgoApi_GetUserInfo(t *testing.T) { if receivedUserinfo, err := api.GetUserInfo(); err != nil { t.Error(err) } else { - assert.Equal(t, *receivedUserinfo, userinfo) + assert.Equal(t, userinfo, *receivedUserinfo) } } @@ -86,8 +87,8 @@ func TestArgoApi_GetApplication(t *testing.T) { if app, err := api.GetApplication("test"); err != nil { t.Error(err) } else { - assert.Equal(t, app.Status.Health.Status, "Healthy") - assert.Equal(t, app.Status.Sync.Status, "Synced") - assert.Equal(t, app.Status.Summary.Images, []string{"example.com/image:v0.1.0", "example.com/image:v0.1.1"}) + assert.Equal(t, "Healthy", app.Status.Health.Status) + assert.Equal(t, "Synced", app.Status.Sync.Status) + assert.Equal(t, []string{"example.com/image:v0.1.0", "example.com/image:v0.1.1"}, app.Status.Summary.Images) } } diff --git a/cmd/argo-watcher/state/in_memory_state_test.go b/cmd/argo-watcher/state/in_memory_state_test.go index 41d08865..558ea146 100644 --- a/cmd/argo-watcher/state/in_memory_state_test.go +++ b/cmd/argo-watcher/state/in_memory_state_test.go @@ -58,7 +58,7 @@ func TestInMemoryState_Add(t *testing.T) { func TestInMemoryState_GetTask(t *testing.T) { task, _ := state.GetTask(taskId) - assert.Equal(t, task.Status, models.StatusInProgressMessage) + assert.Equal(t, models.StatusInProgressMessage, task.Status) } func TestInMemoryState_GetTasks(t *testing.T) { @@ -70,24 +70,21 @@ func TestInMemoryState_GetTasks(t *testing.T) { } func TestInMemoryState_SetTaskStatus(t *testing.T) { - err := state.SetTaskStatus(taskId, "deployed", "") - if err != nil { - t.Errorf("got %s, expected %s", err, "nil") - } + err := state.SetTaskStatus(taskId, models.StatusDeployedMessage, "") + assert.NoError(t, err) - if taskInfo, _ := state.GetTask(taskId); taskInfo.Status != "deployed" { - t.Errorf("got %s, expected %s", taskInfo.Status, "deployed") - } + taskInfo, _ := state.GetTask(taskId) + assert.Equal(t, models.StatusDeployedMessage, taskInfo.Status) } func TestInMemoryState_GetAppList(t *testing.T) { - assert.Equal(t, state.GetAppList(), []string{"Test", "Test2"}) + assert.Equal(t, []string{"Test", "Test2"}, state.GetAppList()) } func TestInMemoryState_GetAppListEmpty(t *testing.T) { state := InMemoryState{} // We must make sure that we are returning an empty slice and not nil - assert.Equal(t, state.GetAppList(), []string{}) + assert.Equal(t, []string{}, state.GetAppList()) } func TestInMemoryState_ProcessObsoleteTasks(t *testing.T) { @@ -100,7 +97,7 @@ func TestInMemoryState_ProcessObsoleteTasks(t *testing.T) { assert.Len(t, tasks, 2) // Only non-obsolete tasks should remain // Check that the status of the obsolete task has been updated - assert.Equal(t, "aborted", tasks[1].Status) + assert.Equal(t, models.StatusAborted, tasks[1].Status) } func TestInMemoryState_Check(t *testing.T) { diff --git a/cmd/argo-watcher/state/postgres_state_test.go b/cmd/argo-watcher/state/postgres_state_test.go index 5a8e0f1c..1786aa7e 100644 --- a/cmd/argo-watcher/state/postgres_state_test.go +++ b/cmd/argo-watcher/state/postgres_state_test.go @@ -115,29 +115,27 @@ func TestPostgresState_GetTask(t *testing.T) { var task *models.Task var err error - if task, err = postgresState.GetTask(deployedTaskId); err != nil { - t.Errorf("got error %s, expected nil", err.Error()) - } + task, err = postgresState.GetTask(deployedTaskId) + assert.NoError(t, err) - assert.Equal(t, task.Status, models.StatusInProgressMessage) + assert.Equal(t, models.StatusInProgressMessage, task.Status) } func TestPostgresState_SetTaskStatus(t *testing.T) { - if err := postgresState.SetTaskStatus(deployedTaskId, "deployed", ""); err != nil { - t.Errorf("got error %s, expected nil", err.Error()) - } + var err error - if err := postgresState.SetTaskStatus(appNotFoundTaskId, "app not found", ""); err != nil { - t.Errorf("got error %s, expected nil", err.Error()) - } + err = postgresState.SetTaskStatus(deployedTaskId, models.StatusDeployedMessage, "") + assert.NoError(t, err) - if taskInfo, _ := postgresState.GetTask(deployedTaskId); taskInfo.Status != "deployed" { - t.Errorf("got %s, expected %s", taskInfo.Status, "deployed") - } + err = postgresState.SetTaskStatus(appNotFoundTaskId, models.StatusAppNotFoundMessage, "") + assert.NoError(t, err) - if taskInfo, _ := postgresState.GetTask(appNotFoundTaskId); taskInfo.Status != "app not found" { - t.Errorf("got %s, expected %s", taskInfo.Status, "app not found") - } + var taskInfo *models.Task + taskInfo, _ = postgresState.GetTask(deployedTaskId) + assert.Equal(t, models.StatusDeployedMessage, taskInfo.Status) + + taskInfo, _ = postgresState.GetTask(appNotFoundTaskId) + assert.Equal(t, models.StatusAppNotFoundMessage, taskInfo.Status) } func TestPostgresState_GetAppList(t *testing.T) { diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index da6d44ed..bc131902 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -118,22 +118,22 @@ func TestGetTaskStatus(t *testing.T) { status := client.getTaskStatus(taskId).Status if status != models.StatusDeployedMessage { - t.Errorf(messageTemplate, "deployed", status) + t.Errorf(messageTemplate, models.StatusDeployedMessage, status) } status = client.getTaskStatus(appNotFoundId).Status if status != models.StatusAppNotFoundMessage { - t.Errorf(messageTemplate, "app not found", status) + t.Errorf(messageTemplate, models.StatusAppNotFoundMessage, status) } status = client.getTaskStatus(argocdUnavailableId).Status if status != models.StatusArgoCDUnavailableMessage { - t.Errorf(messageTemplate, "ArgoCD is unavailable", status) + t.Errorf(messageTemplate, models.StatusArgoCDUnavailableMessage, status) } status = client.getTaskStatus(failedTaskId).Status if status != models.StatusFailedMessage { - t.Errorf(messageTemplate, "failed", status) + t.Errorf(messageTemplate, models.StatusFailedMessage, status) } } From d7860580baf55e83cdd9f38a37fd86d47acd669c Mon Sep 17 00:00:00 2001 From: Bogdans Ozerkins Date: Wed, 9 Aug 2023 23:00:47 -0700 Subject: [PATCH 17/17] remove migration paths. now migrations are based on orm models --- docker-compose.yml | 1 - docs/development.md | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 05d6ca7b..10963949 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,6 @@ services: DB_USER: watcher DB_NAME: watcher DB_PASSWORD: watcher - DB_MIGRATIONS_PATH: /app/db/migrations depends_on: - postgres - mock diff --git a/docs/development.md b/docs/development.md index 060cf2bb..5363d771 100644 --- a/docs/development.md +++ b/docs/development.md @@ -82,7 +82,7 @@ cd cmd/argo-watcher # install dependencies go mod tidy # OR start argo-watcher (postgres) -LOG_LEVEL=debug LOG_FORMAT=text ARGO_URL=http://localhost:8081 ARGO_TOKEN=example STATE_TYPE=postgres DB_USER=watcher DB_PASSWORD=watcher DB_NAME=watcher DB_MIGRATIONS_PATH="./../../db/migrations" go run . -server +LOG_LEVEL=debug LOG_FORMAT=text ARGO_URL=http://localhost:8081 ARGO_TOKEN=example STATE_TYPE=postgres DB_USER=watcher DB_PASSWORD=watcher DB_NAME=watcher go run . -server ``` #### Logs in simple text