From 5638345d60ea30c47dba15fb0a07c96d17bfee83 Mon Sep 17 00:00:00 2001 From: spoukke Date: Fri, 3 Feb 2023 14:07:03 +0100 Subject: [PATCH 01/14] feat: start implementing webhook WIP --- go.mod | 3 ++- go.sum | 3 +++ internal/burrito/config/config.go | 7 +++++-- internal/webhook/webhook.go | 21 +++++++++++++++------ 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 63ff8604..80b56b18 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/padok-team/burrito go 1.19 require ( + github.com/go-playground/webhooks/v6 v6.0.1 github.com/onsi/ginkgo/v2 v2.6.0 github.com/onsi/gomega v1.24.1 k8s.io/apimachinery v0.26.0 @@ -56,7 +57,7 @@ require ( github.com/hashicorp/hc-install v0.4.0 github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/terraform-exec v0.17.3 - github.com/imdario/mergo v0.3.13 + github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index a4f91fd3..ec70f702 100644 --- a/go.sum +++ b/go.sum @@ -145,9 +145,12 @@ github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXym github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-playground/webhooks/v6 v6.0.1 h1:ssqgU7vZ+xK+/Uwx4zkf5tfmzOHnLBpzSp5bJ4cX3rg= +github.com/go-playground/webhooks/v6 v6.0.1/go.mod h1:GCocmfMtpJdkEOM1uG9p2nXzg1kY5X/LtvQgtPHUaaA= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogits/go-gogs-client v0.0.0-20200905025246-8bb8a50cb355/go.mod h1:cY2AIrMgHm6oOHmR7jY+9TtjzSjQ3iG7tURJG3Y6XH0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= diff --git a/internal/burrito/config/config.go b/internal/burrito/config/config.go index c905c4b4..03369241 100644 --- a/internal/burrito/config/config.go +++ b/internal/burrito/config/config.go @@ -18,8 +18,11 @@ type Config struct { } type WebhookConfig struct { - RepositoryProvider string `yaml:"provider"` - Secret string `yaml:"secret"` + Github WebhookGithubConfig `yaml:"github"` +} + +type WebhookGithubConfig struct { + Secret string `yaml:"secret"` } type ControllerConfig struct { diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index 27fd0bdc..b6a388b3 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -1,25 +1,34 @@ package webhook import ( + "github.com/go-playground/webhooks/v6/github" "github.com/padok-team/burrito/internal/burrito/config" - "sigs.k8s.io/controller-runtime/pkg/client" ) type Handler interface { Handle() } -type Webhook struct { +type WebhookHandler struct { config *config.Config - client client.Client + github *github.Webhook } -func New(c *config.Config) *Webhook { - return &Webhook{ +func New(c *config.Config) *WebhookHandler { + return &WebhookHandler{ config: c, } } -func (w *Webhook) Exec() { +func (w *WebhookHandler) Init() error { + githubWebhook, err := github.New(github.Options.Secret(w.config.Webhook.Github.Secret)) + if err != nil { + return err + } + w.github = githubWebhook + return nil +} + +func (w *WebhookHandler) Handle() { return } From d3d13b1f48205969a9c83b677827ba76abb439d4 Mon Sep 17 00:00:00 2001 From: spoukke Date: Tue, 7 Feb 2023 16:52:31 +0100 Subject: [PATCH 02/14] feat: start implementing web server which will receive webhooks --- internal/burrito/config/config.go | 5 ++++ internal/server/server.go | 38 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 internal/server/server.go diff --git a/internal/burrito/config/config.go b/internal/burrito/config/config.go index 03369241..d574f164 100644 --- a/internal/burrito/config/config.go +++ b/internal/burrito/config/config.go @@ -15,6 +15,7 @@ type Config struct { Controller ControllerConfig `yaml:"controller"` Webhook WebhookConfig `yaml:"webhook"` Redis Redis `yaml:"redis"` + Server Server `yaml:"server"` } type WebhookConfig struct { @@ -65,6 +66,10 @@ type Redis struct { Database int `yaml:"database"` } +type Server struct { + Port string `yaml:"port"` +} + func (c *Config) Load(flags *pflag.FlagSet) error { v := viper.New() diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 00000000..77b0fbc2 --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,38 @@ +package server + +import ( + "errors" + "fmt" + "log" + "net/http" + "os" + + "github.com/padok-team/burrito/internal/burrito/config" +) + +type Server struct { + config *config.Config +} + +func (s *Server) Exec() { + http.HandleFunc("/healthz", handleHealthz) + http.HandleFunc("/webhook", handleWebhook) + + err := http.ListenAndServe(fmt.Sprintf(":%s", s.config.Server.Port), nil) + if errors.Is(err, http.ErrServerClosed) { + log.Println("server is closed") + } + if err != nil { + log.Println("error starting server, exiting: %s", err) + os.Exit(1) + } +} + +func handleWebhook(w http.ResponseWriter, r *http.Request) { + +} + +func handleHealthz(w http.ResponseWriter, r *http.Request) { + // The HTTP server is always healthy. + // TODO: check it can get terraformlayers and/or repositories +} From 19ae5d900d1662f5fd963a1b8786b12a25cd65f3 Mon Sep 17 00:00:00 2001 From: spoukke Date: Tue, 7 Feb 2023 17:43:35 +0100 Subject: [PATCH 03/14] feat: WIP implement webhook handler --- internal/burrito/config/config.go | 8 +- internal/webhook/webhook.go | 121 +++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 2 deletions(-) diff --git a/internal/burrito/config/config.go b/internal/burrito/config/config.go index d574f164..021851b8 100644 --- a/internal/burrito/config/config.go +++ b/internal/burrito/config/config.go @@ -19,13 +19,19 @@ type Config struct { } type WebhookConfig struct { - Github WebhookGithubConfig `yaml:"github"` + Github WebhookGithubConfig `yaml:"github"` + Gitlab WebhookGitlabConfig `yaml:"gitlab"` + Namespace string `yaml:"namespace"` } type WebhookGithubConfig struct { Secret string `yaml:"secret"` } +type WebhookGitlabConfig struct { + Secret string `yaml:"secret"` +} + type ControllerConfig struct { WatchedNamespaces []string `yaml:"namespaces"` Timers ControllerTimers `yaml:"timers"` diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index b6a388b3..3c1bbb3e 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -1,8 +1,19 @@ package webhook import ( + "context" + "fmt" + "path/filepath" + "strings" + "github.com/go-playground/webhooks/v6/github" + "github.com/go-playground/webhooks/v6/gitlab" + "github.com/padok-team/burrito/internal/annotations" "github.com/padok-team/burrito/internal/burrito/config" + + "sigs.k8s.io/controller-runtime/pkg/client" + + configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" ) type Handler interface { @@ -10,8 +21,10 @@ type Handler interface { } type WebhookHandler struct { + client.Client config *config.Config github *github.Webhook + gitlab *gitlab.Webhook } func New(c *config.Config) *WebhookHandler { @@ -26,9 +39,115 @@ func (w *WebhookHandler) Init() error { return err } w.github = githubWebhook + gitlabWebhook, err := gitlab.New(gitlab.Options.Secret(w.config.Webhook.Gitlab.Secret)) + if err != nil { + return err + } + w.gitlab = gitlabWebhook return nil } -func (w *WebhookHandler) Handle() { +func (w *WebhookHandler) Handle(payload interface{}) { + webUrls, revision, change, touchedHead, changedFiles := affectedRevisionInfo(payload) + if len(webUrls) == 0 { + fmt.Println("Ignoring webhook event") + return + } + for _, webURL := range webUrls { + fmt.Println("Received push event repo: %s, revision: %s, touchedHead: %v", webURL, revision, touchedHead) + } + // The next 2 lines probably dont work, waiting for chat GPT to be up \o/ + repositories := &configv1alpha1.TerraformRepositoryList{} + err := w.Client.List(context.TODO(), repositories) + if err != nil { + fmt.Println("could not get repositories") + } + + for _, url := range webUrls { + for _, repo := range repositories.Items { + if repo.Spec.Repository.Url != url { + continue + } + // The next 2 lines probably dont work, waiting for chat GPT to be up \o/ + // Should we link lkayer and repositories by label to make this list easier? + layers := &configv1alpha1.TerraformLayerList{} + err := w.Client.List(context.TODO(), layers) + if err != nil { + fmt.Println("could not get layers") + } + for _, layer := range layers.Items { + if layerFilesHaveChanged(&layer, changedFiles) { + ann := map[string]string{} + ann[annotations.LastBranchCommit] = change.shaAfter + err = annotations.AddAnnotations(context.TODO(), w.Client, layer, ann) + } + } + } + } return } + +type changeInfo struct { + shaBefore string + shaAfter string +} + +func parseRevision(ref string) string { + refParts := strings.SplitN(ref, "/", 3) + return refParts[len(refParts)-1] +} + +func affectedRevisionInfo(payloadIf interface{}) (webUrls []string, revision string, change changeInfo, touchedHead bool, changedFiles []string) { + switch payload := payloadIf.(type) { + case github.PushPayload: + webUrls = append(webUrls, payload.Repository.HTMLURL) + revision = parseRevision(payload.Ref) + change.shaAfter = parseRevision(payload.After) + change.shaBefore = parseRevision(payload.Before) + touchedHead = bool(payload.Repository.DefaultBranch == revision) + for _, commit := range payload.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Modified...) + changedFiles = append(changedFiles, commit.Removed...) + } + case gitlab.PushEventPayload: + webUrls = append(webUrls, payload.Project.WebURL) + revision = parseRevision(payload.Ref) + change.shaAfter = parseRevision(payload.After) + change.shaBefore = parseRevision(payload.Before) + touchedHead = bool(payload.Project.DefaultBranch == revision) + for _, commit := range payload.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Modified...) + changedFiles = append(changedFiles, commit.Removed...) + } + default: + fmt.Println("event not handled") + } + return webUrls, revision, change, touchedHead, changedFiles +} + +func layerFilesHaveChanged(layer *configv1alpha1.TerraformLayer, changedFiles []string) bool { + // an empty slice of changed files means that the payload didn't include a list + // of changed files and we have to assume that a refresh is required + if len(changedFiles) == 0 { + return true + } + + // At last one changed file must be under refresh path + for _, f := range changedFiles { + f = ensureAbsPath(f) + if strings.Contains(f, layer.Spec.Path) { + return true + } + } + + return false +} + +func ensureAbsPath(input string) string { + if !filepath.IsAbs(input) { + return string(filepath.Separator) + input + } + return input +} From 17254cc0300f1232f7e0f8af5418809f4fb24cfc Mon Sep 17 00:00:00 2001 From: spoukke Date: Tue, 7 Feb 2023 17:48:11 +0100 Subject: [PATCH 04/14] feat: make cmd start server instead of webhook --- cmd/root.go | 4 ++-- cmd/server/server.go | 15 +++++++++++++++ cmd/server/start.go | 21 +++++++++++++++++++++ cmd/webhook/start.go | 18 ------------------ cmd/webhook/webhook.go | 15 --------------- internal/burrito/burrito.go | 7 +++++++ internal/burrito/server.go | 5 +++++ internal/burrito/webhook.go | 5 ----- internal/server/server.go | 6 ++++++ internal/webhook/webhook.go | 10 +++++----- 10 files changed, 61 insertions(+), 45 deletions(-) create mode 100644 cmd/server/server.go create mode 100644 cmd/server/start.go delete mode 100644 cmd/webhook/start.go delete mode 100644 cmd/webhook/webhook.go create mode 100644 internal/burrito/server.go delete mode 100644 internal/burrito/webhook.go diff --git a/cmd/root.go b/cmd/root.go index 8d1ccdfd..dccc9668 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,7 +6,7 @@ package cmd import ( "github.com/padok-team/burrito/cmd/controllers" "github.com/padok-team/burrito/cmd/runner" - "github.com/padok-team/burrito/cmd/webhook" + "github.com/padok-team/burrito/cmd/server" "github.com/padok-team/burrito/internal/burrito" "github.com/spf13/cobra" @@ -27,6 +27,6 @@ func buildBurritoCmd(app *burrito.App) *cobra.Command { cmd.AddCommand(controllers.BuildControllersCmd(app)) cmd.AddCommand(runner.BuildRunnerCmd(app)) - cmd.AddCommand(webhook.BuildWebhookCmd(app)) + cmd.AddCommand(server.BuildServerCmd(app)) return cmd } diff --git a/cmd/server/server.go b/cmd/server/server.go new file mode 100644 index 00000000..7b1e01a3 --- /dev/null +++ b/cmd/server/server.go @@ -0,0 +1,15 @@ +package server + +import ( + "github.com/padok-team/burrito/internal/burrito" + "github.com/spf13/cobra" +) + +func BuildServerCmd(app *burrito.App) *cobra.Command { + cmd := &cobra.Command{ + Use: "server", + Short: "cmd to use burrito's server", + } + cmd.AddCommand(buildServerStartCmd(app)) + return cmd +} diff --git a/cmd/server/start.go b/cmd/server/start.go new file mode 100644 index 00000000..e735f264 --- /dev/null +++ b/cmd/server/start.go @@ -0,0 +1,21 @@ +package server + +import ( + "github.com/padok-team/burrito/internal/burrito" + "github.com/spf13/cobra" +) + +func buildServerStartCmd(app *burrito.App) *cobra.Command { + cmd := &cobra.Command{ + Use: "start", + Short: "Start burrito's server", + RunE: func(cmd *cobra.Command, args []string) error { + app.StartServer() + return nil + }, + } + + cmd.Flags().StringVar(&app.Config.Server.Port, "port", "80", "port the server listens on") + + return cmd +} diff --git a/cmd/webhook/start.go b/cmd/webhook/start.go deleted file mode 100644 index fc2221a6..00000000 --- a/cmd/webhook/start.go +++ /dev/null @@ -1,18 +0,0 @@ -package webhook - -import ( - "github.com/padok-team/burrito/internal/burrito" - "github.com/spf13/cobra" -) - -func buildWebhookStartCmd(app *burrito.App) *cobra.Command { - cmd := &cobra.Command{ - Use: "start", - Short: "Start Burrito webhook", - RunE: func(cmd *cobra.Command, args []string) error { - app.StartWebhook() - return nil - }, - } - return cmd -} diff --git a/cmd/webhook/webhook.go b/cmd/webhook/webhook.go deleted file mode 100644 index 5733b4d3..00000000 --- a/cmd/webhook/webhook.go +++ /dev/null @@ -1,15 +0,0 @@ -package webhook - -import ( - "github.com/padok-team/burrito/internal/burrito" - "github.com/spf13/cobra" -) - -func BuildWebhookCmd(app *burrito.App) *cobra.Command { - cmd := &cobra.Command{ - Use: "webhook", - Short: "cmd to use burrito's webhook", - } - cmd.AddCommand(buildWebhookStartCmd(app)) - return cmd -} diff --git a/internal/burrito/burrito.go b/internal/burrito/burrito.go index ed37ab9c..f87f246d 100644 --- a/internal/burrito/burrito.go +++ b/internal/burrito/burrito.go @@ -7,6 +7,7 @@ import ( "github.com/padok-team/burrito/internal/burrito/config" "github.com/padok-team/burrito/internal/controllers" "github.com/padok-team/burrito/internal/runner" + "github.com/padok-team/burrito/internal/server" "github.com/padok-team/burrito/internal/webhook" ) @@ -16,6 +17,7 @@ type App struct { Runner Runner Controllers Controllers Webhook Webhook + Server Server Out io.Writer Err io.Writer @@ -25,6 +27,10 @@ type Webhook interface { Exec() } +type Server interface { + Exec() +} + type Runner interface { Exec() } @@ -40,6 +46,7 @@ func New() (*App, error) { Runner: runner.New(c), Controllers: controllers.New(c), Webhook: webhook.New(c), + Server: server.New(c), Out: os.Stdout, Err: os.Stderr, } diff --git a/internal/burrito/server.go b/internal/burrito/server.go new file mode 100644 index 00000000..42f18ea5 --- /dev/null +++ b/internal/burrito/server.go @@ -0,0 +1,5 @@ +package burrito + +func (app *App) StartServer() { + app.Server.Exec() +} diff --git a/internal/burrito/webhook.go b/internal/burrito/webhook.go deleted file mode 100644 index 4f3a27da..00000000 --- a/internal/burrito/webhook.go +++ /dev/null @@ -1,5 +0,0 @@ -package burrito - -func (app *App) StartWebhook() { - app.Runner.Exec() -} diff --git a/internal/server/server.go b/internal/server/server.go index 77b0fbc2..f730504d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -14,6 +14,12 @@ type Server struct { config *config.Config } +func New(c *config.Config) *Server { + return &Server{ + config: c, + } +} + func (s *Server) Exec() { http.HandleFunc("/healthz", handleHealthz) http.HandleFunc("/webhook", handleWebhook) diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index 3c1bbb3e..ec63ad21 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -20,20 +20,20 @@ type Handler interface { Handle() } -type WebhookHandler struct { +type Webhook struct { client.Client config *config.Config github *github.Webhook gitlab *gitlab.Webhook } -func New(c *config.Config) *WebhookHandler { - return &WebhookHandler{ +func New(c *config.Config) *Webhook { + return &Webhook{ config: c, } } -func (w *WebhookHandler) Init() error { +func (w *Webhook) Init() error { githubWebhook, err := github.New(github.Options.Secret(w.config.Webhook.Github.Secret)) if err != nil { return err @@ -47,7 +47,7 @@ func (w *WebhookHandler) Init() error { return nil } -func (w *WebhookHandler) Handle(payload interface{}) { +func (w *Webhook) Handle(payload interface{}) { webUrls, revision, change, touchedHead, changedFiles := affectedRevisionInfo(payload) if len(webUrls) == 0 { fmt.Println("Ignoring webhook event") From 0dde9db48b5db6dd1840464481d1822a1aa5c854 Mon Sep 17 00:00:00 2001 From: Alan Date: Wed, 8 Feb 2023 18:15:19 +0100 Subject: [PATCH 05/14] improvement(webhook): webhook is now handled by the server --- internal/burrito/burrito.go | 7 ----- internal/server/server.go | 15 +++++----- internal/webhook/webhook.go | 58 +++++++++++++++++++++++++++++++++++-- 3 files changed, 64 insertions(+), 16 deletions(-) diff --git a/internal/burrito/burrito.go b/internal/burrito/burrito.go index f87f246d..e8a76b93 100644 --- a/internal/burrito/burrito.go +++ b/internal/burrito/burrito.go @@ -8,7 +8,6 @@ import ( "github.com/padok-team/burrito/internal/controllers" "github.com/padok-team/burrito/internal/runner" "github.com/padok-team/burrito/internal/server" - "github.com/padok-team/burrito/internal/webhook" ) type App struct { @@ -16,17 +15,12 @@ type App struct { Runner Runner Controllers Controllers - Webhook Webhook Server Server Out io.Writer Err io.Writer } -type Webhook interface { - Exec() -} - type Server interface { Exec() } @@ -45,7 +39,6 @@ func New() (*App, error) { Config: c, Runner: runner.New(c), Controllers: controllers.New(c), - Webhook: webhook.New(c), Server: server.New(c), Out: os.Stdout, Err: os.Stderr, diff --git a/internal/server/server.go b/internal/server/server.go index f730504d..0b8acdae 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -8,21 +8,26 @@ import ( "os" "github.com/padok-team/burrito/internal/burrito/config" + "github.com/padok-team/burrito/internal/webhook" ) type Server struct { - config *config.Config + config *config.Config + Webhook *webhook.Webhook } func New(c *config.Config) *Server { + webhook := webhook.New(c) + webhook.Init() return &Server{ - config: c, + config: c, + Webhook: webhook, } } func (s *Server) Exec() { http.HandleFunc("/healthz", handleHealthz) - http.HandleFunc("/webhook", handleWebhook) + http.HandleFunc("/webhook", s.Webhook.GetHttpHandler()) err := http.ListenAndServe(fmt.Sprintf(":%s", s.config.Server.Port), nil) if errors.Is(err, http.ErrServerClosed) { @@ -34,10 +39,6 @@ func (s *Server) Exec() { } } -func handleWebhook(w http.ResponseWriter, r *http.Request) { - -} - func handleHealthz(w http.ResponseWriter, r *http.Request) { // The HTTP server is always healthy. // TODO: check it can get terraformlayers and/or repositories diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index ec63ad21..89ced958 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -2,7 +2,11 @@ package webhook import ( "context" + "errors" "fmt" + "html" + "log" + "net/http" "path/filepath" "strings" @@ -10,10 +14,14 @@ import ( "github.com/go-playground/webhooks/v6/gitlab" "github.com/padok-team/burrito/internal/annotations" "github.com/padok-team/burrito/internal/burrito/config" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" ) type Handler interface { @@ -34,6 +42,16 @@ func New(c *config.Config) *Webhook { } func (w *Webhook) Init() error { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(configv1alpha1.AddToScheme(scheme)) + cl, err := client.New(ctrl.GetConfigOrDie(), client.Options{ + Scheme: scheme, + }) + if err != nil { + return err + } + w.Client = cl githubWebhook, err := github.New(github.Options.Secret(w.config.Webhook.Github.Secret)) if err != nil { return err @@ -47,6 +65,42 @@ func (w *Webhook) Init() error { return nil } +func (w *Webhook) GetHttpHandler() func(http.ResponseWriter, *http.Request) { + return func(writer http.ResponseWriter, r *http.Request) { + var payload interface{} + var err error + + switch { + case r.Header.Get("X-GitHub-Event") != "": + payload, err = w.github.Parse(r, github.PushEvent, github.PingEvent) + if errors.Is(err, github.ErrHMACVerificationFailed) { + log.Println("GitHub webhook HMAC verification failed") + } + case r.Header.Get("X-Gitlab-Event") != "": + payload, err = w.gitlab.Parse(r, gitlab.PushEvents, gitlab.TagEvents) + if errors.Is(err, gitlab.ErrGitLabTokenVerificationFailed) { + log.Println("GitLab webhook token verification failed") + } + default: + log.Println("Ignoring unknown webhook event") + http.Error(writer, "Unknown webhook event", http.StatusBadRequest) + return + } + + if err != nil { + log.Printf("Webhook processing failed: %s", err) + status := http.StatusBadRequest + if r.Method != "POST" { + status = http.StatusMethodNotAllowed + } + http.Error(writer, fmt.Sprintf("Webhook processing failed: %s", html.EscapeString(err.Error())), status) + return + } + + w.Handle(payload) + } +} + func (w *Webhook) Handle(payload interface{}) { webUrls, revision, change, touchedHead, changedFiles := affectedRevisionInfo(payload) if len(webUrls) == 0 { @@ -54,7 +108,7 @@ func (w *Webhook) Handle(payload interface{}) { return } for _, webURL := range webUrls { - fmt.Println("Received push event repo: %s, revision: %s, touchedHead: %v", webURL, revision, touchedHead) + fmt.Printf("Received push event repo: %s, revision: %s, touchedHead: %v", webURL, revision, touchedHead) } // The next 2 lines probably dont work, waiting for chat GPT to be up \o/ repositories := &configv1alpha1.TerraformRepositoryList{} @@ -71,7 +125,7 @@ func (w *Webhook) Handle(payload interface{}) { // The next 2 lines probably dont work, waiting for chat GPT to be up \o/ // Should we link lkayer and repositories by label to make this list easier? layers := &configv1alpha1.TerraformLayerList{} - err := w.Client.List(context.TODO(), layers) + err := w.Client.List(context.TODO(), layers, &client.ListOptions{}) if err != nil { fmt.Println("could not get layers") } From e5d00f488309668a0c2ce93e24c1267b6df694b4 Mon Sep 17 00:00:00 2001 From: spoukke Date: Thu, 9 Feb 2023 11:34:28 +0100 Subject: [PATCH 06/14] chore: catch webhook init error --- internal/server/server.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/server/server.go b/internal/server/server.go index 0b8acdae..c56b2a9d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -18,7 +18,10 @@ type Server struct { func New(c *config.Config) *Server { webhook := webhook.New(c) - webhook.Init() + err := webhook.Init() + if err != nil { + log.Println(fmt.Sprintf("error initializing webhook: %s", err)) + } return &Server{ config: c, Webhook: webhook, @@ -34,7 +37,7 @@ func (s *Server) Exec() { log.Println("server is closed") } if err != nil { - log.Println("error starting server, exiting: %s", err) + log.Println(fmt.Sprintf("error starting server, exiting: %s", err)) os.Exit(1) } } From ec4e7640fd4d211603609ffc2225074469bdf1b6 Mon Sep 17 00:00:00 2001 From: spoukke Date: Thu, 9 Feb 2023 11:34:49 +0100 Subject: [PATCH 07/14] chore: update default server port --- cmd/server/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/server/start.go b/cmd/server/start.go index e735f264..bc1ca281 100644 --- a/cmd/server/start.go +++ b/cmd/server/start.go @@ -15,7 +15,7 @@ func buildServerStartCmd(app *burrito.App) *cobra.Command { }, } - cmd.Flags().StringVar(&app.Config.Server.Port, "port", "80", "port the server listens on") + cmd.Flags().StringVar(&app.Config.Server.Port, "port", "8080", "port the server listens on") return cmd } From a7eda7bb1255136f523a00e22c708375555421cf Mon Sep 17 00:00:00 2001 From: spoukke Date: Thu, 9 Feb 2023 11:36:08 +0100 Subject: [PATCH 08/14] feat: rename annotations methods --- internal/annotations/annotations.go | 4 ++-- internal/runner/runner.go | 2 +- internal/webhook/webhook.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/annotations/annotations.go b/internal/annotations/annotations.go index cd547046..a6f52ca8 100644 --- a/internal/annotations/annotations.go +++ b/internal/annotations/annotations.go @@ -22,7 +22,7 @@ const ( ForceApply string = "notifications.terraform.padok.cloud/force-apply" ) -func AddAnnotations(ctx context.Context, c client.Client, obj configv1alpha1.TerraformLayer, annotations map[string]string) error { +func Add(ctx context.Context, c client.Client, obj configv1alpha1.TerraformLayer, annotations map[string]string) error { patch := client.MergeFrom(obj.DeepCopy()) currentAnnotations := obj.GetAnnotations() for k, v := range annotations { @@ -32,7 +32,7 @@ func AddAnnotations(ctx context.Context, c client.Client, obj configv1alpha1.Ter return c.Patch(ctx, &obj, patch) } -func RemoveAnnotation(ctx context.Context, c client.Client, obj configv1alpha1.TerraformLayer, annotation string) error { +func Remove(ctx context.Context, c client.Client, obj configv1alpha1.TerraformLayer, annotation string) error { patch := client.MergeFrom(obj.DeepCopy()) annotations := obj.GetAnnotations() delete(annotations, annotation) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 40a7eef8..caa79371 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -98,7 +98,7 @@ func (r *Runner) Exec() { number++ ann[annotations.Failure] = strconv.Itoa(number) } - err = annotations.AddAnnotations(context.TODO(), r.client, *r.layer, ann) + err = annotations.Add(context.TODO(), r.client, *r.layer, ann) if err != nil { log.Printf("Could not update layer annotations: %s", err) } diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index 89ced958..cdd4f721 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -133,7 +133,7 @@ func (w *Webhook) Handle(payload interface{}) { if layerFilesHaveChanged(&layer, changedFiles) { ann := map[string]string{} ann[annotations.LastBranchCommit] = change.shaAfter - err = annotations.AddAnnotations(context.TODO(), w.Client, layer, ann) + err = annotations.Add(context.TODO(), w.Client, layer, ann) } } } From 11e8663fef9909570f42fd57f03d5e9299e25a52 Mon Sep 17 00:00:00 2001 From: spoukke Date: Thu, 9 Feb 2023 11:39:01 +0100 Subject: [PATCH 09/14] chore: use log instead of logs --- internal/webhook/webhook.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index cdd4f721..eb1ec457 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -104,17 +104,17 @@ func (w *Webhook) GetHttpHandler() func(http.ResponseWriter, *http.Request) { func (w *Webhook) Handle(payload interface{}) { webUrls, revision, change, touchedHead, changedFiles := affectedRevisionInfo(payload) if len(webUrls) == 0 { - fmt.Println("Ignoring webhook event") + log.Println("Ignoring webhook event") return } for _, webURL := range webUrls { - fmt.Printf("Received push event repo: %s, revision: %s, touchedHead: %v", webURL, revision, touchedHead) + log.Printf("Received push event repo: %s, revision: %s, touchedHead: %v", webURL, revision, touchedHead) } // The next 2 lines probably dont work, waiting for chat GPT to be up \o/ repositories := &configv1alpha1.TerraformRepositoryList{} err := w.Client.List(context.TODO(), repositories) if err != nil { - fmt.Println("could not get repositories") + log.Println("could not get repositories") } for _, url := range webUrls { @@ -127,7 +127,7 @@ func (w *Webhook) Handle(payload interface{}) { layers := &configv1alpha1.TerraformLayerList{} err := w.Client.List(context.TODO(), layers, &client.ListOptions{}) if err != nil { - fmt.Println("could not get layers") + log.Println("could not get layers") } for _, layer := range layers.Items { if layerFilesHaveChanged(&layer, changedFiles) { From 4b3280870d2b52161918b82f4f8a5315b811b0ff Mon Sep 17 00:00:00 2001 From: Alan Date: Thu, 9 Feb 2023 11:57:10 +0100 Subject: [PATCH 10/14] chore(webhook): add some logs just to see where's it crashing --- internal/webhook/webhook.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index eb1ec457..abce4ce7 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -72,6 +72,7 @@ func (w *Webhook) GetHttpHandler() func(http.ResponseWriter, *http.Request) { switch { case r.Header.Get("X-GitHub-Event") != "": + log.Println("Detected a Github event") payload, err = w.github.Parse(r, github.PushEvent, github.PingEvent) if errors.Is(err, github.ErrHMACVerificationFailed) { log.Println("GitHub webhook HMAC verification failed") @@ -130,10 +131,14 @@ func (w *Webhook) Handle(payload interface{}) { log.Println("could not get layers") } for _, layer := range layers.Items { + log.Printf("Evaluating %s", layer.Name) if layerFilesHaveChanged(&layer, changedFiles) { ann := map[string]string{} ann[annotations.LastBranchCommit] = change.shaAfter err = annotations.Add(context.TODO(), w.Client, layer, ann) + if err != nil { + log.Printf("Error adding annotation to layer %s", err) + } } } } From a44db5e5da59ce5a32fccec52d41209694ec1c44 Mon Sep 17 00:00:00 2001 From: spoukke Date: Thu, 9 Feb 2023 11:59:52 +0100 Subject: [PATCH 11/14] chore: update log method --- internal/server/server.go | 4 ++-- internal/webhook/webhook.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/server/server.go b/internal/server/server.go index c56b2a9d..6a340b5d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -20,7 +20,7 @@ func New(c *config.Config) *Server { webhook := webhook.New(c) err := webhook.Init() if err != nil { - log.Println(fmt.Sprintf("error initializing webhook: %s", err)) + log.Printf("error initializing webhook: %s", err) } return &Server{ config: c, @@ -37,7 +37,7 @@ func (s *Server) Exec() { log.Println("server is closed") } if err != nil { - log.Println(fmt.Sprintf("error starting server, exiting: %s", err)) + log.Printf("error starting server, exiting: %s", err) os.Exit(1) } } diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index abce4ce7..efc602ca 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -181,7 +181,7 @@ func affectedRevisionInfo(payloadIf interface{}) (webUrls []string, revision str changedFiles = append(changedFiles, commit.Removed...) } default: - fmt.Println("event not handled") + log.Println("event not handled") } return webUrls, revision, change, touchedHead, changedFiles } From bd35e7c931670b26cdcb00eb8fa42e5f23abb025 Mon Sep 17 00:00:00 2001 From: spoukke Date: Thu, 9 Feb 2023 12:07:30 +0100 Subject: [PATCH 12/14] chore: add sshUrls listing --- internal/webhook/webhook.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index efc602ca..66e549f3 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -103,7 +103,7 @@ func (w *Webhook) GetHttpHandler() func(http.ResponseWriter, *http.Request) { } func (w *Webhook) Handle(payload interface{}) { - webUrls, revision, change, touchedHead, changedFiles := affectedRevisionInfo(payload) + webUrls, sshUrls, revision, change, touchedHead, changedFiles := affectedRevisionInfo(payload) if len(webUrls) == 0 { log.Println("Ignoring webhook event") return @@ -118,7 +118,9 @@ func (w *Webhook) Handle(payload interface{}) { log.Println("could not get repositories") } - for _, url := range webUrls { + allUrls := append(webUrls, sshUrls...) + for _, url := range allUrls { + log.Printf("%s", url) for _, repo := range repositories.Items { if repo.Spec.Repository.Url != url { continue @@ -156,10 +158,11 @@ func parseRevision(ref string) string { return refParts[len(refParts)-1] } -func affectedRevisionInfo(payloadIf interface{}) (webUrls []string, revision string, change changeInfo, touchedHead bool, changedFiles []string) { +func affectedRevisionInfo(payloadIf interface{}) (webUrls []string, sshUrls []string, revision string, change changeInfo, touchedHead bool, changedFiles []string) { switch payload := payloadIf.(type) { case github.PushPayload: webUrls = append(webUrls, payload.Repository.HTMLURL) + sshUrls = append(sshUrls, payload.Repository.SSHURL) revision = parseRevision(payload.Ref) change.shaAfter = parseRevision(payload.After) change.shaBefore = parseRevision(payload.Before) @@ -183,7 +186,7 @@ func affectedRevisionInfo(payloadIf interface{}) (webUrls []string, revision str default: log.Println("event not handled") } - return webUrls, revision, change, touchedHead, changedFiles + return webUrls, sshUrls, revision, change, touchedHead, changedFiles } func layerFilesHaveChanged(layer *configv1alpha1.TerraformLayer, changedFiles []string) bool { From 81f1e5d4cca5955502c8917511ea359d97aea6cd Mon Sep 17 00:00:00 2001 From: spoukke Date: Thu, 9 Feb 2023 12:25:02 +0100 Subject: [PATCH 13/14] feat: handle webhook annotation in state machine --- .../controllers/terraformlayer/conditions.go | 33 +++++++++++++++++++ internal/controllers/terraformlayer/states.go | 7 ++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/internal/controllers/terraformlayer/conditions.go b/internal/controllers/terraformlayer/conditions.go index 32eb7a0c..0dea5392 100644 --- a/internal/controllers/terraformlayer/conditions.go +++ b/internal/controllers/terraformlayer/conditions.go @@ -51,6 +51,39 @@ func (r *Reconciler) IsPlanArtifactUpToDate(t *configv1alpha1.TerraformLayer) (m return condition, false } +func (r *Reconciler) IsLastCommitPlanned(t *configv1alpha1.TerraformLayer) (metav1.Condition, bool) { + condition := metav1.Condition{ + Type: "IsLastCommitPlanned", + ObservedGeneration: t.GetObjectMeta().GetGeneration(), + Status: metav1.ConditionUnknown, + LastTransitionTime: metav1.NewTime(time.Now()), + } + lastPlannedCommit, ok := t.Annotations[annotations.LastPlanCommit] + if !ok { + condition.Reason = "NoPlanHasRunYet" + condition.Message = "No plan has run on this layer yet" + condition.Status = metav1.ConditionTrue + return condition, true + } + lastBranchCommit, ok := t.Annotations[annotations.LastBranchCommit] + if !ok { + condition.Reason = "NoCommitReceived" + condition.Message = "No commit has been received from webhook" + condition.Status = metav1.ConditionTrue + return condition, true + } + if lastPlannedCommit == lastBranchCommit { + condition.Reason = "LastCommitPlanned" + condition.Message = "The last commit has already been planned" + condition.Status = metav1.ConditionFalse + return condition, false + } + condition.Reason = "LastCommitNotPlanned" + condition.Message = "The last received commit has not been planned yet" + condition.Status = metav1.ConditionTrue + return condition, true +} + func (r *Reconciler) IsApplyUpToDate(t *configv1alpha1.TerraformLayer) (metav1.Condition, bool) { condition := metav1.Condition{ Type: "IsApplyUpToDate", diff --git a/internal/controllers/terraformlayer/states.go b/internal/controllers/terraformlayer/states.go index ae7c2a23..46a551ee 100644 --- a/internal/controllers/terraformlayer/states.go +++ b/internal/controllers/terraformlayer/states.go @@ -19,16 +19,17 @@ func (r *Reconciler) GetState(ctx context.Context, l *configv1alpha1.TerraformLa log := log.FromContext(ctx) c1, isPlanArtifactUpToDate := r.IsPlanArtifactUpToDate(l) c2, isApplyUpToDate := r.IsApplyUpToDate(l) + c3, isLastCommitPlanned := r.IsLastCommitPlanned(l) // c3, hasFailed := HasFailed(r) - conditions := []metav1.Condition{c1, c2} + conditions := []metav1.Condition{c1, c2, c3} switch { case isPlanArtifactUpToDate && isApplyUpToDate: log.Info("Layer is up to date, waiting for a new drift detection cycle") return &IdleState{}, conditions - case isPlanArtifactUpToDate && !isApplyUpToDate: + case isPlanArtifactUpToDate && !isApplyUpToDate && !isLastCommitPlanned: log.Info("Layer needs to be applied, acquiring lock and creating a new runner") return &ApplyNeededState{}, conditions - case !isPlanArtifactUpToDate: + case !isPlanArtifactUpToDate || !isLastCommitPlanned: log.Info("Layer needs to be planned, acquiring lock and creating a new runner") return &PlanNeededState{}, conditions default: From 317403e3b5fb093643420a0a11e53b82589deb1f Mon Sep 17 00:00:00 2001 From: spoukke Date: Thu, 9 Feb 2023 12:30:39 +0100 Subject: [PATCH 14/14] fix: change condition values --- internal/controllers/terraformlayer/conditions.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controllers/terraformlayer/conditions.go b/internal/controllers/terraformlayer/conditions.go index 0dea5392..61b279e9 100644 --- a/internal/controllers/terraformlayer/conditions.go +++ b/internal/controllers/terraformlayer/conditions.go @@ -75,12 +75,12 @@ func (r *Reconciler) IsLastCommitPlanned(t *configv1alpha1.TerraformLayer) (meta if lastPlannedCommit == lastBranchCommit { condition.Reason = "LastCommitPlanned" condition.Message = "The last commit has already been planned" - condition.Status = metav1.ConditionFalse + condition.Status = metav1.ConditionTrue return condition, false } condition.Reason = "LastCommitNotPlanned" condition.Message = "The last received commit has not been planned yet" - condition.Status = metav1.ConditionTrue + condition.Status = metav1.ConditionFalse return condition, true }