From 2bf2f085a468e7824f00c9d5724086190680531a Mon Sep 17 00:00:00 2001 From: Lucas Marques Date: Fri, 20 Dec 2024 12:08:42 +0100 Subject: [PATCH] feat: centralize git clients in one util package --- .../terraformpullrequest/controller.go | 120 +++---- .../terraformpullrequest/controller_test.go | 10 +- .../terraformpullrequest/github/provider.go | 159 --------- .../terraformpullrequest/gitlab/provider.go | 100 ------ .../terraformpullrequest/mock/provider.go | 36 --- internal/runner/repository.go | 146 --------- internal/runner/runner.go | 38 ++- internal/runner/runner_test.go | 8 +- internal/utils/gitprovider/github/github.go | 303 ++++++++++++++++++ internal/utils/gitprovider/gitlab/gitlab.go | 241 ++++++++++++++ internal/utils/gitprovider/gitprovider.go | 71 ++++ internal/utils/gitprovider/mock/mock.go | 77 +++++ .../utils/gitprovider/standard/standard.go | 83 +++++ internal/utils/gitprovider/types/types.go | 61 ++++ internal/webhook/github/provider.go | 97 ------ internal/webhook/gitlab/provider.go | 91 ------ internal/webhook/webhook.go | 56 ++-- 17 files changed, 968 insertions(+), 729 deletions(-) delete mode 100644 internal/controllers/terraformpullrequest/github/provider.go delete mode 100644 internal/controllers/terraformpullrequest/gitlab/provider.go delete mode 100644 internal/controllers/terraformpullrequest/mock/provider.go delete mode 100644 internal/runner/repository.go create mode 100644 internal/utils/gitprovider/github/github.go create mode 100644 internal/utils/gitprovider/gitlab/gitlab.go create mode 100644 internal/utils/gitprovider/gitprovider.go create mode 100644 internal/utils/gitprovider/mock/mock.go create mode 100644 internal/utils/gitprovider/standard/standard.go create mode 100644 internal/utils/gitprovider/types/types.go delete mode 100644 internal/webhook/github/provider.go delete mode 100644 internal/webhook/gitlab/provider.go diff --git a/internal/controllers/terraformpullrequest/controller.go b/internal/controllers/terraformpullrequest/controller.go index 284235fc..be1f4943 100644 --- a/internal/controllers/terraformpullrequest/controller.go +++ b/internal/controllers/terraformpullrequest/controller.go @@ -3,13 +3,10 @@ package terraformpullrequest import ( "context" "fmt" + "strconv" "github.com/google/go-cmp/cmp" "github.com/padok-team/burrito/internal/burrito/config" - "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" - "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/github" - "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/gitlab" - "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/mock" datastore "github.com/padok-team/burrito/internal/datastore/client" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -25,20 +22,16 @@ import ( log "github.com/sirupsen/logrus" configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" + "github.com/padok-team/burrito/internal/utils/gitprovider" + gt "github.com/padok-team/burrito/internal/utils/gitprovider/types" ) -type Provider interface { - Init() error - GetChanges(*configv1alpha1.TerraformRepository, *configv1alpha1.TerraformPullRequest) ([]string, error) - Comment(*configv1alpha1.TerraformRepository, *configv1alpha1.TerraformPullRequest, comment.Comment) error -} - // Reconciler reconciles a TerraformPullRequest object type Reconciler struct { client.Client Scheme *runtime.Scheme Config *config.Config - Providers map[string]Provider + Providers map[string]gitprovider.Provider Recorder record.EventRecorder Datastore datastore.Client } @@ -107,7 +100,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu // SetupWithManager sets up the controller with the Manager. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Providers = make(map[string]Provider) + r.Providers = make(map[string]gitprovider.Provider) err := r.initializeDefaultProviders() if err != nil { log.Errorf("Some legacy configuration was found, but could not initialize default providers: %s", err) @@ -119,7 +112,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func GetProviderForPullRequest(pr *configv1alpha1.TerraformPullRequest, r *Reconciler) (Provider, error) { +func GetProviderForPullRequest(pr *configv1alpha1.TerraformPullRequest, r *Reconciler) (gitprovider.Provider, error) { for key, p := range r.Providers { if fmt.Sprintf("%s/%s", pr.Spec.Repository.Namespace, pr.Spec.Repository.Name) == key { return p, nil @@ -146,11 +139,15 @@ func ignorePredicate() predicate.Predicate { } } -func (r *Reconciler) initializeProvider(ctx context.Context, repository *configv1alpha1.TerraformRepository) (Provider, error) { +func (r *Reconciler) initializeProvider(ctx context.Context, repository *configv1alpha1.TerraformRepository) (gitprovider.Provider, error) { + if repository.Spec.Repository.Url == "" { + return nil, fmt.Errorf("no repository URL found in TerraformRepository.spec.repository.url for repository %s. Skipping provider initialization", repository.Name) + } if repository.Spec.Repository.SecretName == "" { - log.Debugf("no webhook secret configured for repository %s/%s, skipping provider initialization", repository.Namespace, repository.Name) + log.Debugf("no secret configured for repository %s/%s, skipping provider initialization", repository.Namespace, repository.Name) return nil, nil } + log.Infof("KUBE API REQUEST: getting secret %s/%s", repository.Namespace, repository.Spec.Repository.SecretName) secret := &corev1.Secret{} err := r.Client.Get(ctx, types.NamespacedName{ Name: repository.Spec.Repository.SecretName, @@ -160,32 +157,20 @@ func (r *Reconciler) initializeProvider(ctx context.Context, repository *configv log.Errorf("failed to get credentials secret for repository %s: %s", repository.Name, err) return nil, err } - var provider Provider + config := gitprovider.Config{ + AppID: parseSecretInt64(secret.Data["githubAppId"]), + URL: repository.Spec.Repository.Url, + AppInstallationID: parseSecretInt64(secret.Data["githubAppInstallationId"]), + AppPrivateKey: string(secret.Data["githubAppPrivateKey"]), + GitHubToken: string(secret.Data["githubToken"]), + GitLabToken: string(secret.Data["gitlabToken"]), + EnableMock: secret.Data["enableMock"] != nil && string(secret.Data["enableMock"]) == "true", + } - if repository.Spec.Repository.Url == "" { - return nil, fmt.Errorf("no repository URL found in TerraformRepository.spec.repository.url, %s", repository.Name) - } - if secret.Data["enableMock"] != nil && string(secret.Data["enableMock"]) == "true" { - provider = &mock.Mock{} - } else if secret.Data["githubAppId"] != nil && secret.Data["githubAppInstallationId"] != nil && secret.Data["githubAppPrivateKey"] != nil { - provider = &github.Github{ - AppId: string(secret.Data["githubAppId"]), - AppInstallationId: string(secret.Data["githubAppInstallationId"]), - AppPrivateKey: string(secret.Data["githubAppPrivateKey"]), - Url: repository.Spec.Repository.Url, - } - } else if secret.Data["githubToken"] != nil { - provider = &github.Github{ - ApiToken: string(secret.Data["githubToken"]), - Url: repository.Spec.Repository.Url, - } - } else if secret.Data["gitlabToken"] != nil { - provider = &gitlab.Gitlab{ - ApiToken: string(secret.Data["gitlabToken"]), - Url: repository.Spec.Repository.Url, - } - } else { - return nil, fmt.Errorf("no valid provider credentials found in secret %s. Please provide at least one of the following: , , in the secret referenced in TerraformRepository.spec.repository.secretName", repository.Spec.Repository.SecretName) + provider, err := gitprovider.New(config, []string{gt.Capabilities.Comment, gt.Capabilities.Changes}) + if err != nil { + log.Errorf("failed to create provider for repository %s: %s", repository.Name, err) + return nil, err } err = provider.Init() @@ -198,47 +183,30 @@ func (r *Reconciler) initializeProvider(ctx context.Context, repository *configv func (r *Reconciler) initializeDefaultProviders() error { // This initializes default providers for the controller if user has provided legacy configuration - if r.Config.Controller.GithubConfig.AppId != 0 && r.Config.Controller.GithubConfig.InstallationId != 0 && r.Config.Controller.GithubConfig.PrivateKey != "" { - provider := &github.Github{ - AppId: fmt.Sprintf("%d", r.Config.Controller.GithubConfig.AppId), - AppInstallationId: fmt.Sprintf("%d", r.Config.Controller.GithubConfig.InstallationId), - AppPrivateKey: r.Config.Controller.GithubConfig.PrivateKey, - Url: "https://github.com", - } - err := provider.Init() - if err != nil { - return err - } - r.Providers["defaultGitHubApp"] = provider - log.Infof("initialized default GitHub provider (GitHub App)") + var config = gitprovider.Config{ + AppID: r.Config.Controller.GithubConfig.AppId, + AppInstallationID: r.Config.Controller.GithubConfig.InstallationId, + AppPrivateKey: r.Config.Controller.GithubConfig.PrivateKey, + GitHubToken: r.Config.Controller.GithubConfig.APIToken, + GitLabToken: r.Config.Controller.GitlabConfig.APIToken, + URL: "https://github.com", } - if r.Config.Controller.GithubConfig.APIToken != "" { - provider := &github.Github{ - ApiToken: r.Config.Controller.GithubConfig.APIToken, - Url: "https://github.com", - } - err := provider.Init() - if err != nil { - return err - } - r.Providers["defaultGitHubApiToken"] = provider - log.Infof("initialized default GitHub provider (API Token)") + + providers, err := gitprovider.ListAvailable(config, []string{gt.Capabilities.Changes, gt.Capabilities.Comment}) + if err != nil { + return err } - if r.Config.Controller.GitlabConfig.APIToken != "" { - gitlabURL := "https://gitlab.com" - if r.Config.Controller.GitlabConfig.URL == "" { - gitlabURL = r.Config.Controller.GitlabConfig.URL - } - provider := &gitlab.Gitlab{ - ApiToken: r.Config.Controller.GitlabConfig.APIToken, - Url: gitlabURL, - } - err := provider.Init() + for _, provider := range providers { + providerInstance, err := gitprovider.NewWithName(config, provider) if err != nil { return err } - r.Providers["defaultGitLabApiToken"] = provider - log.Infof("initialized default GitLab provider (API Token)") + r.Providers["default_"+provider] = providerInstance } return nil } + +func parseSecretInt64(data []byte) int64 { + v, _ := strconv.ParseInt(string(data), 10, 64) + return v +} diff --git a/internal/controllers/terraformpullrequest/controller_test.go b/internal/controllers/terraformpullrequest/controller_test.go index 1bf0ab67..8280af3d 100644 --- a/internal/controllers/terraformpullrequest/controller_test.go +++ b/internal/controllers/terraformpullrequest/controller_test.go @@ -41,9 +41,9 @@ import ( "github.com/padok-team/burrito/internal/annotations" "github.com/padok-team/burrito/internal/burrito/config" controller "github.com/padok-team/burrito/internal/controllers/terraformpullrequest" - provider "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/mock" datastore "github.com/padok-team/burrito/internal/datastore/client" utils "github.com/padok-team/burrito/internal/testing" + "github.com/padok-team/burrito/internal/utils/gitprovider" //+kubebuilder:scaffold:imports ) @@ -108,8 +108,12 @@ var _ = BeforeSuite(func() { Config: config.TestConfig(), Scheme: scheme.Scheme, Datastore: datastore.NewMockClient(), - Providers: map[string]controller.Provider{ - "mock": &provider.Mock{}, + Providers: map[string]gitprovider.Provider{ + "mock": func() gitprovider.Provider { + provider, err := gitprovider.NewWithName(gitprovider.Config{EnableMock: true}, "mock") + Expect(err).NotTo(HaveOccurred()) + return provider + }(), }, Recorder: record.NewBroadcasterForTests(1*time.Second).NewRecorder(scheme.Scheme, corev1.EventSource{ Component: "burrito", diff --git a/internal/controllers/terraformpullrequest/github/provider.go b/internal/controllers/terraformpullrequest/github/provider.go deleted file mode 100644 index 33fec279..00000000 --- a/internal/controllers/terraformpullrequest/github/provider.go +++ /dev/null @@ -1,159 +0,0 @@ -package github - -import ( - "context" - "errors" - "fmt" - "net/http" - "net/url" - "strconv" - "strings" - - "github.com/bradleyfalzon/ghinstallation/v2" - "github.com/google/go-github/v66/github" - configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" - "github.com/padok-team/burrito/internal/annotations" - "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" - utils "github.com/padok-team/burrito/internal/utils/url" - log "github.com/sirupsen/logrus" - "golang.org/x/oauth2" -) - -type Github struct { - *github.Client - AppId string - AppInstallationId string - AppPrivateKey string - ApiToken string - Url string -} - -type GitHubSubscription string - -const ( - GitHubEnterprise GitHubSubscription = "enterprise" - GitHubClassic GitHubSubscription = "classic" -) - -func (g *Github) IsAppConfigPresent() bool { - return g.AppId != "" && g.AppInstallationId != "" && g.AppPrivateKey != "" -} - -func (g *Github) IsAPITokenConfigPresent() bool { - return g.ApiToken != "" -} - -func (g *Github) Init() error { - apiUrl, subscription, err := inferBaseURL(utils.NormalizeUrl(g.Url)) - if err != nil { - return err - } - httpClient := &http.Client{} - if g.IsAppConfigPresent() { - appId, err := strconv.ParseInt(g.AppId, 10, 64) - if err != nil { - return errors.New("error while parsing github app id: " + err.Error()) - } - appInstallationId, err := strconv.ParseInt(g.AppInstallationId, 10, 64) - if err != nil { - return errors.New("error while parsing github app installation id: " + err.Error()) - } - itr, err := ghinstallation.New(http.DefaultTransport, appId, appInstallationId, []byte(g.AppPrivateKey)) - if err != nil { - return errors.New("error while creating github installation client: " + err.Error()) - } - httpClient.Transport = itr - } else if g.IsAPITokenConfigPresent() { - ctx := context.Background() - - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: g.ApiToken}, - ) - httpClient = oauth2.NewClient(ctx, ts) - } else { - return errors.New("github config is not present") - } - - if subscription == GitHubEnterprise { - g.Client, err = github.NewClient(httpClient).WithEnterpriseURLs(apiUrl, apiUrl) - if err != nil { - return errors.New("error while creating github enterprise client: " + err.Error()) - } - } else if subscription == GitHubClassic { - g.Client = github.NewClient(httpClient) - } - return nil -} - -func (g *Github) GetChanges(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest) ([]string, error) { - owner, repoName := parseGithubUrl(repository.Spec.Repository.Url) - id, err := strconv.Atoi(pr.Spec.ID) - if err != nil { - log.Errorf("Error while parsing Github pull request ID: %s", err) - return []string{}, err - } - // Per page is 30 by default, max is 100 - opts := &github.ListOptions{ - PerPage: 100, - } - // Get all pull request files from Github - var allChangedFiles []string - for { - changedFiles, resp, err := g.Client.PullRequests.ListFiles(context.TODO(), owner, repoName, id, opts) - if err != nil { - return []string{}, err - } - for _, file := range changedFiles { - if *file.Status != "unchanged" { - allChangedFiles = append(allChangedFiles, *file.Filename) - } - } - if resp.NextPage == 0 { - break - } - opts.Page = resp.NextPage - } - return allChangedFiles, nil -} - -func (g *Github) Comment(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest, comment comment.Comment) error { - body, err := comment.Generate(pr.Annotations[annotations.LastBranchCommit]) - if err != nil { - log.Errorf("Error while generating comment: %s", err) - return err - } - owner, repoName := parseGithubUrl(repository.Spec.Repository.Url) - id, err := strconv.Atoi(pr.Spec.ID) - if err != nil { - log.Errorf("Error while parsing Github pull request ID: %s", err) - return err - } - _, _, err = g.Client.Issues.CreateComment(context.TODO(), owner, repoName, id, &github.IssueComment{ - Body: &body, - }) - return err -} - -func parseGithubUrl(url string) (string, string) { - normalizedUrl := utils.NormalizeUrl(url) - // nomalized url are "https://padok.github.com/owner/repo" - // we remove "https://" then split on "/" - split := strings.Split(normalizedUrl[8:], "/") - return split[1], split[2] -} - -func inferBaseURL(repoURL string) (string, GitHubSubscription, error) { - parsedURL, err := url.Parse(repoURL) - if err != nil { - return "", "", fmt.Errorf("invalid repository URL: %w", err) - } - - host := parsedURL.Host - host = strings.TrimPrefix(host, "www.") - - if host != "github.com" { - return fmt.Sprintf("https://%s/api/v3", host), GitHubEnterprise, nil - } else { - return "", GitHubClassic, nil - } -} diff --git a/internal/controllers/terraformpullrequest/gitlab/provider.go b/internal/controllers/terraformpullrequest/gitlab/provider.go deleted file mode 100644 index 19b30f29..00000000 --- a/internal/controllers/terraformpullrequest/gitlab/provider.go +++ /dev/null @@ -1,100 +0,0 @@ -package gitlab - -import ( - "fmt" - "net/url" - "strconv" - "strings" - - configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" - "github.com/padok-team/burrito/internal/annotations" - "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" - utils "github.com/padok-team/burrito/internal/utils/url" - log "github.com/sirupsen/logrus" - "github.com/xanzy/go-gitlab" -) - -type Gitlab struct { - *gitlab.Client - ApiToken string - Url string -} - -func (g *Gitlab) Init() error { - apiUrl, err := inferBaseURL(utils.NormalizeUrl(g.Url)) - if err != nil { - return err - } - client, err := gitlab.NewClient(g.ApiToken, gitlab.WithBaseURL(apiUrl)) - if err != nil { - return err - } - g.Client = client - return nil -} - -func (g *Gitlab) GetChanges(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest) ([]string, error) { - id, err := strconv.Atoi(pr.Spec.ID) - if err != nil { - log.Errorf("Error while parsing Gitlab merge request ID: %s", err) - return []string{}, err - } - listOpts := gitlab.ListMergeRequestDiffsOptions{ - ListOptions: gitlab.ListOptions{ - PerPage: 20, - }, - } - var changes []string - for { - diffs, resp, err := g.Client.MergeRequests.ListMergeRequestDiffs(getGitlabNamespacedName(repository.Spec.Repository.Url), id, &listOpts) - if err != nil { - log.Errorf("Error while getting merge request changes: %s", err) - return []string{}, err - } - for _, change := range diffs { - changes = append(changes, change.NewPath) - } - if resp.NextPage == 0 { - break - } - listOpts.Page = resp.NextPage - } - return changes, nil -} - -func (g *Gitlab) Comment(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest, comment comment.Comment) error { - body, err := comment.Generate(pr.Annotations[annotations.LastBranchCommit]) - if err != nil { - log.Errorf("Error while generating comment: %s", err) - return err - } - id, err := strconv.Atoi(pr.Spec.ID) - if err != nil { - log.Errorf("Error while parsing Gitlab merge request ID: %s", err) - return err - } - _, _, err = g.Client.Notes.CreateMergeRequestNote(getGitlabNamespacedName(repository.Spec.Repository.Url), id, &gitlab.CreateMergeRequestNoteOptions{ - Body: gitlab.Ptr(body), - }) - if err != nil { - log.Errorf("Error while creating merge request note: %s", err) - return err - } - return nil -} - -func getGitlabNamespacedName(url string) string { - normalizedUrl := utils.NormalizeUrl(url) - return strings.Join(strings.Split(normalizedUrl[8:], "/")[1:], "/") -} - -func inferBaseURL(repoURL string) (string, error) { - parsedURL, err := url.Parse(repoURL) - if err != nil { - return "", fmt.Errorf("invalid repository URL: %w", err) - } - - host := parsedURL.Host - host = strings.TrimPrefix(host, "www.") - return fmt.Sprintf("https://%s/api/v4", host), nil -} diff --git a/internal/controllers/terraformpullrequest/mock/provider.go b/internal/controllers/terraformpullrequest/mock/provider.go deleted file mode 100644 index 8239aace..00000000 --- a/internal/controllers/terraformpullrequest/mock/provider.go +++ /dev/null @@ -1,36 +0,0 @@ -package mock - -import ( - configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" - "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" - log "github.com/sirupsen/logrus" -) - -type Mock struct{} - -func (m *Mock) Init() error { - log.Infof("Mock provider initialized") - return nil -} - -func (m *Mock) GetChanges(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest) ([]string, error) { - log.Infof("Mock provider all changed files") - var allChangedFiles []string - // Handle not useful PR - if pr.Spec.ID == "100" { - allChangedFiles = []string{ - "README.md", - } - return allChangedFiles, nil - } - allChangedFiles = []string{ - "terraform/main.tf", - "terragrunt/inputs.hcl", - } - return allChangedFiles, nil -} - -func (m *Mock) Comment(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest, comment comment.Comment) error { - log.Infof("Mock provider comment posted") - return nil -} diff --git a/internal/runner/repository.go b/internal/runner/repository.go deleted file mode 100644 index b193fc76..00000000 --- a/internal/runner/repository.go +++ /dev/null @@ -1,146 +0,0 @@ -package runner - -import ( - "context" - nethttp "net/http" - "strings" - - "github.com/bradleyfalzon/ghinstallation/v2" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" - "github.com/padok-team/burrito/internal/burrito/config" - log "github.com/sirupsen/logrus" -) - -type CloneOptions struct { - AuthenticationType string - CloneOptions *git.CloneOptions -} - -// Fetch the content of the specified repository on the specified branch with git clone -// -// TODO: Fetch repo from datastore when repository controller is implemented -func FetchRepositoryContent(repo *configv1alpha1.TerraformRepository, branch string, config *config.Config) (*git.Repository, error) { - log.Infof("fetching repository %s on %s branch with git clone", repo.Spec.Repository.Url, branch) - cloneOptionsList, err := getCloneOptionsList(config.Runner.Repository, repo.Spec.Repository.Url, branch) - if err != nil { - return &git.Repository{}, err - } - - var lastErr error - for _, cloneOptions := range cloneOptionsList { - options := cloneOptions.CloneOptions - log.Infof("trying to clone with %s authentication", cloneOptions.AuthenticationType) - repo, err := git.PlainClone(config.Runner.RepositoryPath, false, options) - if err == nil { - return repo, nil - } - lastErr = err - log.Warnf("clone attempt failed: %v", err) - } - - return &git.Repository{}, lastErr -} - -func getCloneOptionsList(repository config.RepositoryConfig, URL, branch string) ([]*CloneOptions, error) { - cloneOptions := &git.CloneOptions{ - ReferenceName: plumbing.NewBranchReferenceName(branch), - URL: URL, - } - - var cloneOptionsList []*CloneOptions - - if strings.Contains(URL, "https://") { - // HTTPS cloning methods - if repository.Username != "" && repository.Password != "" { - log.Infof("Git username and password found in repository secret") - cloneOptionsList = append(cloneOptionsList, &CloneOptions{ - AuthenticationType: "username-password", - CloneOptions: &git.CloneOptions{ - ReferenceName: cloneOptions.ReferenceName, - URL: cloneOptions.URL, - Auth: &http.BasicAuth{ - Username: repository.Username, - Password: repository.Password, - }, - }, - }) - } - if repository.GithubAppId != 0 && repository.GithubAppInstallationId != 0 && repository.GithubAppPrivateKey != "" { - log.Infof("Github app credentials found in repository secret") - tr, err := ghinstallation.New( - nethttp.DefaultTransport, - repository.GithubAppId, - repository.GithubAppInstallationId, - []byte(repository.GithubAppPrivateKey), - ) - if err == nil { - token, err := tr.Token(context.Background()) - if err == nil { - cloneOptionsList = append(cloneOptionsList, &CloneOptions{ - AuthenticationType: "github-app", - CloneOptions: &git.CloneOptions{ - ReferenceName: cloneOptions.ReferenceName, - URL: cloneOptions.URL, - Auth: &http.BasicAuth{ - Username: "x-access-token", - Password: token, - }, - }, - }) - } else { - log.Warnf("failed to create Github app token from credentials: %v", err) - } - } - } - if repository.GithubToken != "" { - log.Infof("Github token found in repository secret") - cloneOptionsList = append(cloneOptionsList, &CloneOptions{ - AuthenticationType: "github-token", - CloneOptions: &git.CloneOptions{ - ReferenceName: cloneOptions.ReferenceName, - URL: cloneOptions.URL, - Auth: &http.BasicAuth{ - Username: "x-access-token", - Password: repository.GithubToken, - }, - }, - }) - } - if repository.GitlabToken != "" { - log.Infof("Gitlab token found in repository secret") - cloneOptionsList = append(cloneOptionsList, &CloneOptions{ - AuthenticationType: "gitlab-token", - CloneOptions: &git.CloneOptions{ - ReferenceName: cloneOptions.ReferenceName, - URL: cloneOptions.URL, - Auth: &http.BasicAuth{ - Password: repository.GitlabToken, - }, - }, - }) - } - cloneOptionsList = append(cloneOptionsList, &CloneOptions{ - AuthenticationType: "passwordless", - CloneOptions: cloneOptions, - }) - } else { - // SSH cloning method - if repository.SSHPrivateKey != "" { - log.Infof("adding SSH private key authentication") - publicKeys, err := ssh.NewPublicKeys("git", []byte(repository.SSHPrivateKey), "") - if err == nil { - cloneOptions.Auth = publicKeys - } - } - cloneOptionsList = append(cloneOptionsList, &CloneOptions{ - AuthenticationType: "ssh", - CloneOptions: cloneOptions, - }) - } - - return cloneOptionsList, nil -} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index add69d12..bf846a3e 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -10,6 +10,8 @@ import ( datastore "github.com/padok-team/burrito/internal/datastore/client" "github.com/padok-team/burrito/internal/runner/tools" "github.com/padok-team/burrito/internal/utils" + "github.com/padok-team/burrito/internal/utils/gitprovider" + gt "github.com/padok-team/burrito/internal/utils/gitprovider/types" runnerutils "github.com/padok-team/burrito/internal/utils/runner" log "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/types" @@ -21,6 +23,7 @@ type Runner struct { exec tools.BaseExec Datastore datastore.Client Client client.Client + GitProvider gitprovider.Provider Layer *configv1alpha1.TerraformLayer Run *configv1alpha1.TerraformRun Repository *configv1alpha1.TerraformRepository @@ -68,6 +71,7 @@ func (r *Runner) initClients() error { datastoreClient := datastore.NewDefaultClient(r.config.Datastore) r.Datastore = datastoreClient + return nil } @@ -83,7 +87,12 @@ func (r *Runner) Init() error { r.workingDir = filepath.Join(r.config.Runner.RepositoryPath, r.Layer.Spec.Path) - r.gitRepository, err = FetchRepositoryContent(r.Repository, r.Layer.Spec.Branch, r.config) + err = r.initGitProvider() + if err != nil { + log.Errorf("error initializing git provider: %s", err) + } + log.Info("successfully initialized git provider") + r.gitRepository, err = r.GitProvider.Clone(r.Repository, r.Layer.Spec.Branch, r.config.Runner.RepositoryPath) if err != nil { log.Errorf("error fetching repository: %s", err) return err @@ -152,7 +161,32 @@ func (r *Runner) GetResources() error { } log.Infof("successfully retrieved repo") r.Repository = repository - log.Infof("kubernetes resources successfully retrieved") return nil } + +func (r *Runner) initGitProvider() error { + config := gitprovider.Config{ + URL: r.Repository.Spec.Repository.Url, + AppID: r.config.Runner.Repository.GithubAppId, + AppInstallationID: r.config.Runner.Repository.GithubAppInstallationId, + AppPrivateKey: r.config.Runner.Repository.GithubAppPrivateKey, + GitHubToken: r.config.Runner.Repository.GithubToken, + GitLabToken: r.config.Runner.Repository.GitlabToken, + Username: r.config.Runner.Repository.Username, + Password: r.config.Runner.Repository.Password, + SSHPrivateKey: r.config.Runner.Repository.SSHPrivateKey, + } + provider, err := gitprovider.New(config, []string{gt.Capabilities.Clone}) + if err != nil { + log.Errorf("error initializing git provider: %s", err) + return err + } + r.GitProvider = provider + err = r.GitProvider.Init() + if err != nil { + log.Errorf("error initializing git provider: %s", err) + return err + } + return nil +} diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index cfa9d2e6..38d1b7cd 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -15,6 +15,7 @@ import ( datastore "github.com/padok-team/burrito/internal/datastore/client" "github.com/padok-team/burrito/internal/runner" utils "github.com/padok-team/burrito/internal/testing" + "github.com/padok-team/burrito/internal/utils/gitprovider" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -96,7 +97,12 @@ func cleanup(conf *config.Config) { func executeRunner(r *runner.Runner) error { r.Datastore = datastore.NewMockClient() r.Client = k8sClient - err := r.Init() + var err error + r.GitProvider, err = gitprovider.NewWithName(gitprovider.Config{}, "standard") + if err != nil { + return err + } + err = r.Init() if err != nil { return err } diff --git a/internal/utils/gitprovider/github/github.go b/internal/utils/gitprovider/github/github.go new file mode 100644 index 00000000..3c4f8755 --- /dev/null +++ b/internal/utils/gitprovider/github/github.go @@ -0,0 +1,303 @@ +package github + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + nethttp "net/http" + "net/url" + "slices" + "strconv" + "strings" + + "github.com/bradleyfalzon/ghinstallation/v2" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport/http" + wh "github.com/go-playground/webhooks/github" + "github.com/google/go-github/v66/github" + configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" + "github.com/padok-team/burrito/internal/annotations" + "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" + "github.com/padok-team/burrito/internal/utils/gitprovider/types" + utils "github.com/padok-team/burrito/internal/utils/url" + "github.com/padok-team/burrito/internal/webhook/event" + log "github.com/sirupsen/logrus" + "golang.org/x/oauth2" +) + +type Github struct { + *github.Client + Config types.Config + HttpClient *nethttp.Client + WebhookHandler *wh.Webhook + GitHubClientType string + itr *ghinstallation.Transport +} + +type GitHubSubscription string + +const ( + GitHubEnterprise GitHubSubscription = "enterprise" + GitHubClassic GitHubSubscription = "classic" +) + +func IsAvailable(config types.Config, capabilities []string) bool { + var allCapabilities = []string{types.Capabilities.Clone, types.Capabilities.Comment, types.Capabilities.Changes, types.Capabilities.Webhook} + // Check that the configuration is valid + // For webhook handling, the GitHub token is not required + webhookOnlyRequested := len(capabilities) == 1 && capabilities[0] == types.Capabilities.Webhook + hasGitHubToken := config.GitHubToken != "" + hasAppCredentials := config.AppID != 0 && config.AppInstallationID != 0 && config.AppPrivateKey != "" + hasWebhookSecret := config.WebhookSecret != "" + + if !hasGitHubToken && !hasAppCredentials && !(webhookOnlyRequested && hasWebhookSecret) { + return false + } + // Check that the requested capabilities are supported + for _, c := range capabilities { + if !slices.Contains(allCapabilities, c) { + return false + } + } + return true +} + +func (g *Github) Init() error { + apiUrl, subscription, err := inferBaseURL(utils.NormalizeUrl(g.Config.URL)) + if err != nil { + return err + } + + var httpClient *nethttp.Client + + // GitHub App authentication first + if g.Config.AppID != 0 && g.Config.AppInstallationID != 0 && g.Config.AppPrivateKey != "" { + itr, err := ghinstallation.New( + nethttp.DefaultTransport, + g.Config.AppID, + g.Config.AppInstallationID, + []byte(g.Config.AppPrivateKey), + ) + if err != nil { + return fmt.Errorf("error creating GitHub App client: %w", err) + } + g.GitHubClientType = "app" + httpClient = &nethttp.Client{Transport: itr} + g.itr = itr + } else if g.Config.GitHubToken != "" { + // Try GitHub Token authentication + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: g.Config.GitHubToken}) + g.GitHubClientType = "token" + httpClient = oauth2.NewClient(context.Background(), ts) + } else if g.Config.Username != "" && g.Config.Password != "" { + // Try basic authentication + ts := oauth2.StaticTokenSource(&oauth2.Token{ + AccessToken: base64.StdEncoding.EncodeToString( + []byte(fmt.Sprintf("%s:%s", g.Config.Username, g.Config.Password)), + ), + }) + g.GitHubClientType = "basic" + httpClient = oauth2.NewClient(context.Background(), ts) + } else { + return errors.New("no valid authentication method provided") + } + + // Create the appropriate client based on GitHub type + if subscription == GitHubEnterprise { + g.Client, err = github.NewClient(httpClient).WithEnterpriseURLs(apiUrl, apiUrl) + } else { + g.Client = github.NewClient(httpClient) + } + return nil +} + +func (g *Github) InitWebhookHandler() error { + handler, err := wh.New(wh.Options.Secret(g.Config.WebhookSecret)) + if err != nil { + return err + } + g.WebhookHandler = handler + return nil +} + +func (g *Github) GetChanges(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest) ([]string, error) { + owner, repoName := parseGithubUrl(repository.Spec.Repository.Url) + id, err := strconv.Atoi(pr.Spec.ID) + if err != nil { + log.Errorf("Error while parsing Github pull request ID: %s", err) + return []string{}, err + } + // Per page is 30 by default, max is 100 + opts := &github.ListOptions{ + PerPage: 100, + } + // Get all pull request files from Github + var allChangedFiles []string + for { + changedFiles, resp, err := g.Client.PullRequests.ListFiles(context.TODO(), owner, repoName, id, opts) + if err != nil { + return []string{}, err + } + for _, file := range changedFiles { + if *file.Status != "unchanged" { + allChangedFiles = append(allChangedFiles, *file.Filename) + } + } + if resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + return allChangedFiles, nil +} + +func (g *Github) Comment(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest, comment comment.Comment) error { + body, err := comment.Generate(pr.Annotations[annotations.LastBranchCommit]) + if err != nil { + log.Errorf("Error while generating comment: %s", err) + return err + } + owner, repoName := parseGithubUrl(repository.Spec.Repository.Url) + id, err := strconv.Atoi(pr.Spec.ID) + if err != nil { + log.Errorf("Error while parsing Github pull request ID: %s", err) + return err + } + _, _, err = g.Client.Issues.CreateComment(context.TODO(), owner, repoName, id, &github.IssueComment{ + Body: &body, + }) + return err +} + +func (g *Github) Clone(repository *configv1alpha1.TerraformRepository, branch string, repositoryPath string) (*git.Repository, error) { + cloneOptions := &git.CloneOptions{ + ReferenceName: plumbing.NewBranchReferenceName(branch), + URL: repository.Spec.Repository.Url, + } + + if g.GitHubClientType == "app" { + token, err := g.itr.Token(context.Background()) + if err != nil { + return nil, fmt.Errorf("error getting GitHub App token: %w", err) + } + cloneOptions.Auth = &http.BasicAuth{ + Username: "x-access-token", + Password: token, + } + cloneOptions.URL = repository.Spec.Repository.Url + } else if g.GitHubClientType == "token" { + cloneOptions.Auth = &http.BasicAuth{ + Username: "x-access-token", + Password: g.Config.GitHubToken, + } + } else if g.GitHubClientType == "basic" { + cloneOptions.Auth = &http.BasicAuth{ + Username: g.Config.Username, + Password: g.Config.Password, + } + } else { + log.Info("No authentication method provided, falling back to unauthenticated clone") + } + + log.Infof("Cloning github repository %s on %s branch with github %s authentication", repository.Spec.Repository.Url, branch, g.GitHubClientType) + repo, err := git.PlainClone(repositoryPath, false, cloneOptions) + if err != nil { + return nil, err + } + return repo, nil +} + +func (g *Github) ParseWebhookPayload(r *nethttp.Request) (interface{}, bool) { + // if the request is not a GitHub event, return false + if r.Header.Get("X-GitHub-Event") == "" { + return nil, false + } else { + // check if the request can be verified with the secret of this provider + p, err := g.WebhookHandler.Parse(r, wh.PushEvent, wh.PingEvent, wh.PullRequestEvent) + if errors.Is(err, wh.ErrHMACVerificationFailed) { + return nil, false + } else if err != nil { + log.Errorf("an error occurred during request parsing : %s", err) + return nil, false + } + return p, true + } +} + +func (g *Github) GetEventFromWebhookPayload(p interface{}) (event.Event, error) { + var e event.Event + var err error + switch payload := p.(type) { + case wh.PushPayload: + log.Infof("parsing Github push event payload") + changedFiles := []string{} + for _, commit := range payload.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Modified...) + changedFiles = append(changedFiles, commit.Removed...) + } + e = &event.PushEvent{ + URL: utils.NormalizeUrl(payload.Repository.HTMLURL), + Revision: event.ParseRevision(payload.Ref), + ChangeInfo: event.ChangeInfo{ + ShaBefore: payload.Before, + ShaAfter: payload.After, + }, + Changes: changedFiles, + } + case wh.PullRequestPayload: + log.Infof("parsing Github pull request event payload") + if err != nil { + log.Warnf("could not retrieve pull request from Github API: %s", err) + return nil, err + } + e = &event.PullRequestEvent{ + ID: strconv.FormatInt(payload.PullRequest.Number, 10), + URL: utils.NormalizeUrl(payload.Repository.HTMLURL), + Revision: payload.PullRequest.Head.Ref, + Action: getNormalizedAction(payload.Action), + Base: payload.PullRequest.Base.Ref, + Commit: payload.PullRequest.Head.Sha, + } + default: + return nil, errors.New("unsupported Event") + } + return e, nil +} + +func getNormalizedAction(action string) string { + switch action { + case "opened", "reopened": + return event.PullRequestOpened + case "closed": + return event.PullRequestClosed + default: + return action + } +} + +func parseGithubUrl(url string) (string, string) { + normalizedUrl := utils.NormalizeUrl(url) + // nomalized url are "https://padok.github.com/owner/repo" + // we remove "https://" then split on "/" + split := strings.Split(normalizedUrl[8:], "/") + return split[1], split[2] +} + +func inferBaseURL(repoURL string) (string, GitHubSubscription, error) { + parsedURL, err := url.Parse(repoURL) + if err != nil { + return "", "", fmt.Errorf("invalid repository URL: %w", err) + } + + host := parsedURL.Host + host = strings.TrimPrefix(host, "www.") + + if host != "github.com" { + return fmt.Sprintf("https://%s/api/v3", host), GitHubEnterprise, nil + } else { + return "", GitHubClassic, nil + } +} diff --git a/internal/utils/gitprovider/gitlab/gitlab.go b/internal/utils/gitprovider/gitlab/gitlab.go new file mode 100644 index 00000000..4a618f99 --- /dev/null +++ b/internal/utils/gitprovider/gitlab/gitlab.go @@ -0,0 +1,241 @@ +package gitlab + +import ( + "errors" + "fmt" + nethttp "net/http" + "net/url" + "slices" + "strconv" + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport/http" + wh "github.com/go-playground/webhooks/gitlab" + configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" + "github.com/padok-team/burrito/internal/annotations" + "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" + "github.com/padok-team/burrito/internal/utils/gitprovider/types" + utils "github.com/padok-team/burrito/internal/utils/url" + "github.com/padok-team/burrito/internal/webhook/event" + log "github.com/sirupsen/logrus" + "github.com/xanzy/go-gitlab" +) + +type Gitlab struct { + *gitlab.Client + WebhookHandler *wh.Webhook + Config types.Config +} + +func IsAvailable(config types.Config, capabilities []string) bool { + var allCapabilities = []string{types.Capabilities.Clone, types.Capabilities.Comment, types.Capabilities.Changes, types.Capabilities.Webhook} + // Check that the configuration is valid + // For webhook handling, the GitLab token is not required + webhookOnlyRequested := len(capabilities) == 1 && capabilities[0] == types.Capabilities.Webhook + hasGitLabToken := config.GitLabToken != "" + hasWebhookSecret := config.WebhookSecret != "" + + if !(hasGitLabToken || (webhookOnlyRequested && hasWebhookSecret)) { + return false + } + // Check that the requested capabilities are supported + for _, c := range capabilities { + if !slices.Contains(allCapabilities, c) { + return false + } + } + return true +} + +func (g *Gitlab) Init() error { + apiUrl, err := inferBaseURL(utils.NormalizeUrl(g.Config.URL)) + if err != nil { + return err + } + + var token string + if g.Config.GitLabToken != "" { + token = g.Config.GitLabToken + } else if g.Config.Username != "" && g.Config.Password != "" { + token = g.Config.Password + } else { + log.Info("No authentication method provided, falling back to unauthenticated clone") + } + + client, err := gitlab.NewClient(token, gitlab.WithBaseURL(apiUrl)) + if err != nil { + return fmt.Errorf("failed to create GitLab client: %w", err) + } + + g.Client = client + return nil +} + +func (g *Gitlab) InitWebhookHandler() error { + handler, err := wh.New(wh.Options.Secret(g.Config.WebhookSecret)) + if err != nil { + return err + } + g.WebhookHandler = handler + return nil +} + +func (g *Gitlab) GetChanges(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest) ([]string, error) { + id, err := strconv.Atoi(pr.Spec.ID) + if err != nil { + log.Errorf("Error while parsing Gitlab merge request ID: %s", err) + return []string{}, err + } + listOpts := gitlab.ListMergeRequestDiffsOptions{ + ListOptions: gitlab.ListOptions{ + PerPage: 20, + }, + } + var changes []string + for { + diffs, resp, err := g.Client.MergeRequests.ListMergeRequestDiffs(getGitlabNamespacedName(repository.Spec.Repository.Url), id, &listOpts) + if err != nil { + log.Errorf("Error while getting merge request changes: %s", err) + return []string{}, err + } + for _, change := range diffs { + changes = append(changes, change.NewPath) + } + if resp.NextPage == 0 { + break + } + listOpts.Page = resp.NextPage + } + return changes, nil +} + +func (g *Gitlab) Comment(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest, comment comment.Comment) error { + body, err := comment.Generate(pr.Annotations[annotations.LastBranchCommit]) + if err != nil { + log.Errorf("Error while generating comment: %s", err) + return err + } + id, err := strconv.Atoi(pr.Spec.ID) + if err != nil { + log.Errorf("Error while parsing Gitlab merge request ID: %s", err) + return err + } + _, _, err = g.Client.Notes.CreateMergeRequestNote(getGitlabNamespacedName(repository.Spec.Repository.Url), id, &gitlab.CreateMergeRequestNoteOptions{ + Body: gitlab.Ptr(body), + }) + if err != nil { + log.Errorf("Error while creating merge request note: %s", err) + return err + } + return nil +} + +func (g *Gitlab) Clone(repository *configv1alpha1.TerraformRepository, branch string, repositoryPath string) (*git.Repository, error) { + cloneOptions := &git.CloneOptions{ + ReferenceName: plumbing.NewBranchReferenceName(branch), + URL: repository.Spec.Repository.Url, + } + + if g.Config.GitLabToken != "" { + cloneOptions.Auth = &http.BasicAuth{ + Password: g.Config.GitLabToken, + } + } else if g.Config.Username != "" && g.Config.Password != "" { + cloneOptions.Auth = &http.BasicAuth{ + Username: g.Config.Username, + Password: g.Config.Password, + } + } else { + return nil, errors.New("no valid authentication method provided") + } + + log.Infof("Cloning gitlab repository %s on %s branch", repository.Spec.Repository.Url, branch) + repo, err := git.PlainClone(repositoryPath, false, cloneOptions) + if err != nil { + return nil, err + } + return repo, nil +} + +func (g *Gitlab) ParseWebhookPayload(r *nethttp.Request) (interface{}, bool) { + // if the request is not a GitLab event, return false + if r.Header.Get("X-Gitlab-Event") == "" { + return nil, false + } else { + // check if the request can be verified with the secret of this provider + p, err := g.WebhookHandler.Parse(r, wh.PushEvents, wh.TagEvents, wh.MergeRequestEvents) + if errors.Is(err, wh.ErrGitLabTokenVerificationFailed) { + return nil, false + } else if err != nil { + log.Errorf("an error occurred during request parsing: %s", err) + return nil, false + } + return p, true + } +} + +func (g *Gitlab) GetEventFromWebhookPayload(p interface{}) (event.Event, error) { + var e event.Event + + switch payload := p.(type) { + case wh.PushEventPayload: + log.Infof("parsing Gitlab push event payload") + changedFiles := []string{} + for _, commit := range payload.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Modified...) + changedFiles = append(changedFiles, commit.Removed...) + } + e = &event.PushEvent{ + URL: utils.NormalizeUrl(payload.Project.WebURL), + Revision: event.ParseRevision(payload.Ref), + ChangeInfo: event.ChangeInfo{ + ShaBefore: payload.Before, + ShaAfter: payload.After, + }, + Changes: changedFiles, + } + case wh.MergeRequestEventPayload: + log.Infof("parsing Gitlab merge request event payload") + e = &event.PullRequestEvent{ + ID: strconv.Itoa(int(payload.ObjectAttributes.IID)), + URL: utils.NormalizeUrl(payload.Project.WebURL), + Revision: payload.ObjectAttributes.SourceBranch, + Action: getNormalizedAction(payload.ObjectAttributes.Action), + Base: payload.ObjectAttributes.TargetBranch, + Commit: payload.ObjectAttributes.LastCommit.ID, + } + default: + return nil, errors.New("unsupported event") + } + return e, nil +} + +func getNormalizedAction(action string) string { + switch action { + case "open", "reopen": + return event.PullRequestOpened + case "close", "merge": + return event.PullRequestClosed + default: + return action + } +} + +func getGitlabNamespacedName(url string) string { + normalizedUrl := utils.NormalizeUrl(url) + return strings.Join(strings.Split(normalizedUrl[8:], "/")[1:], "/") +} + +func inferBaseURL(repoURL string) (string, error) { + parsedURL, err := url.Parse(repoURL) + if err != nil { + return "", fmt.Errorf("invalid repository URL: %w", err) + } + + host := parsedURL.Host + host = strings.TrimPrefix(host, "www.") + return fmt.Sprintf("https://%s/api/v4", host), nil +} diff --git a/internal/utils/gitprovider/gitprovider.go b/internal/utils/gitprovider/gitprovider.go new file mode 100644 index 00000000..9f9e6d4a --- /dev/null +++ b/internal/utils/gitprovider/gitprovider.go @@ -0,0 +1,71 @@ +package gitprovider + +import ( + "fmt" + "slices" + + "github.com/padok-team/burrito/internal/utils/gitprovider/github" + "github.com/padok-team/burrito/internal/utils/gitprovider/gitlab" + "github.com/padok-team/burrito/internal/utils/gitprovider/mock" + "github.com/padok-team/burrito/internal/utils/gitprovider/standard" + "github.com/padok-team/burrito/internal/utils/gitprovider/types" + log "github.com/sirupsen/logrus" +) + +type Provider = types.Provider +type Config = types.Config + +var providers = map[string]struct { + IsAvailable func(types.Config, []string) bool + create func(types.Config) types.Provider + priority int64 +}{ + "github": {github.IsAvailable, func(config types.Config) types.Provider { return &github.Github{Config: config} }, 0}, + "gitlab": {gitlab.IsAvailable, func(config types.Config) types.Provider { return &gitlab.Gitlab{Config: config} }, 1}, + "mock": {mock.IsAvailable, func(types.Config) types.Provider { return &mock.Mock{} }, 99}, + "standard": {standard.IsAvailable, func(config types.Config) types.Provider { return &standard.Standard{Config: config} }, 100}, +} + +// New creates a new git provider based on the given configuration and capabilities. +// The provider is selected based on the given capabilities and the provider's priority. +func New(config types.Config, capabilities []string) (types.Provider, error) { + available, err := ListAvailable(config, capabilities) + if err != nil || len(available) == 0 { + return nil, fmt.Errorf("No git provider available with the given configuration") + } + log.Infof("Creating git provider of type %s", available[0]) + return NewWithName(config, available[0]) +} + +// NewWithName creates a new git provider based on the given configuration and provider name. +// Caution: this function does not check if the provider is available with the given configuration or capabilities. +// It is the caller's responsibility to ensure that the provider is available. Use ListAvailable for that purpose. +func NewWithName(config types.Config, providerName string) (types.Provider, error) { + if provider, exists := providers[providerName]; exists { + return provider.create(config), nil + } + return nil, fmt.Errorf("unknown provider %s", providerName) +} + +// ListAvailable returns a list of providers that: +// - match the requested capabilities +// - can be initialized with the given configuration +func ListAvailable(config types.Config, capabilities []string) ([]string, error) { + var availableProviders []string + for name, provider := range providers { + if provider.IsAvailable(config, capabilities) { + availableProviders = append(availableProviders, name) + } + } + + slices.SortFunc(availableProviders, func(a, b string) int { + if providers[a].priority < providers[b].priority { + return -1 + } else if providers[a].priority > providers[b].priority { + return 1 + } + return 0 + }) + + return availableProviders, nil +} diff --git a/internal/utils/gitprovider/mock/mock.go b/internal/utils/gitprovider/mock/mock.go new file mode 100644 index 00000000..d30c2615 --- /dev/null +++ b/internal/utils/gitprovider/mock/mock.go @@ -0,0 +1,77 @@ +package mock + +import ( + "net/http" + "slices" + + "github.com/go-git/go-git/v5" + configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" + "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" + "github.com/padok-team/burrito/internal/utils/gitprovider/types" + "github.com/padok-team/burrito/internal/webhook/event" + log "github.com/sirupsen/logrus" +) + +type Mock struct { + Config types.Config +} + +func IsAvailable(config types.Config, capabilities []string) bool { + allCapabilities := []string{types.Capabilities.Clone, types.Capabilities.Comment, types.Capabilities.Changes, types.Capabilities.Webhook} + if !config.EnableMock { + return false + } + for _, c := range capabilities { + if !slices.Contains(allCapabilities, c) { + return false + } + } + return true +} + +func (m *Mock) Init() error { + log.Infof("Mock provider initialized") + return nil +} + +func (m *Mock) InitWebhookHandler() error { + log.Infof("Mock provider webhook handler initialized") + return nil +} + +func (m *Mock) GetChanges(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest) ([]string, error) { + log.Infof("Mock provider all changed files") + var allChangedFiles []string + // Handle not useful PR + if pr.Spec.ID == "100" { + allChangedFiles = []string{ + "README.md", + } + return allChangedFiles, nil + } + allChangedFiles = []string{ + "terraform/main.tf", + "terragrunt/inputs.hcl", + } + return allChangedFiles, nil +} + +func (m *Mock) Comment(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest, comment comment.Comment) error { + log.Infof("Mock provider comment posted") + return nil +} + +func (g *Mock) Clone(repository *configv1alpha1.TerraformRepository, branch string, repositoryPath string) (*git.Repository, error) { + log.Infof("Mock provider repository cloned") + return nil, nil +} + +func (m *Mock) ParseWebhookPayload(payload *http.Request) (interface{}, bool) { + log.Infof("Mock provider webhook payload parsed") + return nil, true +} + +func (m *Mock) GetEventFromWebhookPayload(payload interface{}) (event.Event, error) { + log.Infof("Mock provider webhook event parsed") + return nil, nil +} diff --git a/internal/utils/gitprovider/standard/standard.go b/internal/utils/gitprovider/standard/standard.go new file mode 100644 index 00000000..d52d2173 --- /dev/null +++ b/internal/utils/gitprovider/standard/standard.go @@ -0,0 +1,83 @@ +package standard + +import ( + "fmt" + nethttp "net/http" + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" + configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" + "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" + "github.com/padok-team/burrito/internal/utils/gitprovider/types" + "github.com/padok-team/burrito/internal/webhook/event" + log "github.com/sirupsen/logrus" +) + +type Standard struct { + Config types.Config +} + +func IsAvailable(config types.Config, capabilities []string) bool { + return len(capabilities) == 1 && capabilities[0] == types.Capabilities.Clone +} +func (s *Standard) Init() error { + return nil +} + +func (s *Standard) InitWebhookHandler() error { + return fmt.Errorf("InitWebhookHandler not supported for standard git provider. Provide a specific credentials for providers such as GitHub or GitLab") +} + +func (s *Standard) GetChanges(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest) ([]string, error) { + return nil, fmt.Errorf("GetChanges not supported for standard git provider. Provide a specific credentials for providers such as GitHub or GitLab") +} + +func (s *Standard) Comment(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest, comment comment.Comment) error { + return fmt.Errorf("Comment not supported for standard git provider. Provide a specific credentials for providers such as GitHub or GitLab") +} + +func (s *Standard) CreatePullRequest(repository *configv1alpha1.TerraformRepository, pr *configv1alpha1.TerraformPullRequest) error { + return fmt.Errorf("CreatePullRequest not supported for standard git provider. Provide a specific credentials for providers such as GitHub or GitLab") +} + +func (g *Standard) Clone(repository *configv1alpha1.TerraformRepository, branch string, repositoryPath string) (*git.Repository, error) { + cloneOptions := &git.CloneOptions{ + ReferenceName: plumbing.NewBranchReferenceName(branch), + URL: repository.Spec.Repository.Url, + } + isSSH := strings.HasPrefix(repository.Spec.Repository.Url, "git@") || strings.Contains(repository.Spec.Repository.Url, "ssh://") + + if isSSH && g.Config.SSHPrivateKey != "" { + publicKeys, err := ssh.NewPublicKeys("git", []byte(g.Config.SSHPrivateKey), "") + if err != nil { + return nil, err + } + cloneOptions.Auth = publicKeys + } else if g.Config.Username != "" && g.Config.Password != "" { + cloneOptions.Auth = &http.BasicAuth{ + Username: g.Config.Username, + Password: g.Config.Password, + } + } else { + log.Info("No authentication method provided, falling back to unauthenticated clone") + } + + log.Infof("Cloning remote repository %s on %s branch with git", repository.Spec.Repository.Url, branch) + repo, err := git.PlainClone(repositoryPath, false, cloneOptions) + if err != nil { + return nil, err + } + return repo, nil +} + +func (m *Standard) ParseWebhookPayload(payload *nethttp.Request) (interface{}, bool) { + log.Errorf("ParseWebhookPayload not supported for standard git provider. Provide a specific credentials for providers such as GitHub or GitLab") + return nil, false +} + +func (m *Standard) GetEventFromWebhookPayload(payload interface{}) (event.Event, error) { + return nil, fmt.Errorf("GetEventFromWebhookPayload not supported for standard git provider. Provide a specific credentials for providers such as GitHub or GitLab") +} diff --git a/internal/utils/gitprovider/types/types.go b/internal/utils/gitprovider/types/types.go new file mode 100644 index 00000000..e978ca1c --- /dev/null +++ b/internal/utils/gitprovider/types/types.go @@ -0,0 +1,61 @@ +package types + +import ( + "net/http" + + "github.com/go-git/go-git/v5" + configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" + "github.com/padok-team/burrito/internal/controllers/terraformpullrequest/comment" + "github.com/padok-team/burrito/internal/webhook/event" +) + +// Config holds all possible authentication configurations +type Config struct { + // Basic auth + Username string + Password string + + // SSH auth + SSHPrivateKey string + + // GitHub App auth + AppID int64 + AppInstallationID int64 + AppPrivateKey string + + // Token auth + GitHubToken string + GitLabToken string + + // Repository URL + URL string + + // Mock provider + EnableMock bool + + // Secret for webhook handling + WebhookSecret string +} + +// Provider interface defines methods that must be implemented by git providers +type Provider interface { + Init() error + InitWebhookHandler() error + GetChanges(*configv1alpha1.TerraformRepository, *configv1alpha1.TerraformPullRequest) ([]string, error) + Comment(*configv1alpha1.TerraformRepository, *configv1alpha1.TerraformPullRequest, comment.Comment) error + Clone(*configv1alpha1.TerraformRepository, string, string) (*git.Repository, error) + ParseWebhookPayload(r *http.Request) (interface{}, bool) + GetEventFromWebhookPayload(interface{}) (event.Event, error) +} + +var Capabilities = struct { + Clone string + Comment string + Changes string + Webhook string +}{ + Clone: "clone", + Comment: "comment", + Changes: "changes", + Webhook: "webhook", +} diff --git a/internal/webhook/github/provider.go b/internal/webhook/github/provider.go deleted file mode 100644 index e829cb98..00000000 --- a/internal/webhook/github/provider.go +++ /dev/null @@ -1,97 +0,0 @@ -package github - -import ( - "errors" - "net/http" - "strconv" - - utils "github.com/padok-team/burrito/internal/utils/url" - - "github.com/go-playground/webhooks/github" - "github.com/padok-team/burrito/internal/webhook/event" - - log "github.com/sirupsen/logrus" -) - -type Github struct { - github *github.Webhook - Secret string -} - -func (g *Github) Init() error { - githubWebhook, err := github.New(github.Options.Secret(g.Secret)) - if err != nil { - return err - } - g.github = githubWebhook - return nil -} - -func (g *Github) ParseFromProvider(r *http.Request) (interface{}, bool) { - // if the request is not a GitHub event, return false - if r.Header.Get("X-GitHub-Event") == "" { - return nil, false - } else { - // check if the request can be verified with the secret of this provider - p, err := g.github.Parse(r, github.PushEvent, github.PingEvent, github.PullRequestEvent) - if errors.Is(err, github.ErrHMACVerificationFailed) { - return nil, false - } else if err != nil { - log.Errorf("an error occurred during request parsing : %s", err) - return nil, false - } - return p, true - } -} - -func (g *Github) GetEvent(p interface{}) (event.Event, error) { - var e event.Event - var err error - switch payload := p.(type) { - case github.PushPayload: - log.Infof("parsing Github push event payload") - changedFiles := []string{} - for _, commit := range payload.Commits { - changedFiles = append(changedFiles, commit.Added...) - changedFiles = append(changedFiles, commit.Modified...) - changedFiles = append(changedFiles, commit.Removed...) - } - e = &event.PushEvent{ - URL: utils.NormalizeUrl(payload.Repository.HTMLURL), - Revision: event.ParseRevision(payload.Ref), - ChangeInfo: event.ChangeInfo{ - ShaBefore: payload.Before, - ShaAfter: payload.After, - }, - Changes: changedFiles, - } - case github.PullRequestPayload: - log.Infof("parsing Github pull request event payload") - if err != nil { - log.Warnf("could not retrieve pull request from Github API: %s", err) - return nil, err - } - e = &event.PullRequestEvent{ - ID: strconv.FormatInt(payload.PullRequest.Number, 10), - URL: utils.NormalizeUrl(payload.Repository.HTMLURL), - Revision: payload.PullRequest.Head.Ref, - Action: getNormalizedAction(payload.Action), - Base: payload.PullRequest.Base.Ref, - Commit: payload.PullRequest.Head.Sha, - } - default: - return nil, errors.New("unsupported Event") - } - return e, nil -} - -func getNormalizedAction(action string) string { - switch action { - case "opened", "reopened": - return event.PullRequestOpened - case "closed": - return event.PullRequestClosed - default: - return action - } -} diff --git a/internal/webhook/gitlab/provider.go b/internal/webhook/gitlab/provider.go deleted file mode 100644 index 8dc407d6..00000000 --- a/internal/webhook/gitlab/provider.go +++ /dev/null @@ -1,91 +0,0 @@ -package gitlab - -import ( - "errors" - "net/http" - "strconv" - - "github.com/go-playground/webhooks/gitlab" - utils "github.com/padok-team/burrito/internal/utils/url" - "github.com/padok-team/burrito/internal/webhook/event" - log "github.com/sirupsen/logrus" -) - -type Gitlab struct { - gitlab *gitlab.Webhook - Secret string -} - -func (g *Gitlab) Init() error { - gitlabWebhook, err := gitlab.New(gitlab.Options.Secret(g.Secret)) - if err != nil { - return err - } - g.gitlab = gitlabWebhook - return nil -} - -func (g *Gitlab) ParseFromProvider(r *http.Request) (interface{}, bool) { - // if the request is not a GitLab event, return false - if r.Header.Get("X-Gitlab-Event") == "" { - return nil, false - } else { - // check if the request can be verified with the secret of this provider - p, err := g.gitlab.Parse(r, gitlab.PushEvents, gitlab.TagEvents, gitlab.MergeRequestEvents) - if errors.Is(err, gitlab.ErrGitLabTokenVerificationFailed) { - return nil, false - } else if err != nil { - log.Errorf("an error occurred during request parsing: %s", err) - return nil, false - } - return p, true - } -} - -func (g *Gitlab) GetEvent(p interface{}) (event.Event, error) { - var e event.Event - - switch payload := p.(type) { - case gitlab.PushEventPayload: - log.Infof("parsing Gitlab push event payload") - changedFiles := []string{} - for _, commit := range payload.Commits { - changedFiles = append(changedFiles, commit.Added...) - changedFiles = append(changedFiles, commit.Modified...) - changedFiles = append(changedFiles, commit.Removed...) - } - e = &event.PushEvent{ - URL: utils.NormalizeUrl(payload.Project.WebURL), - Revision: event.ParseRevision(payload.Ref), - ChangeInfo: event.ChangeInfo{ - ShaBefore: payload.Before, - ShaAfter: payload.After, - }, - Changes: changedFiles, - } - case gitlab.MergeRequestEventPayload: - log.Infof("parsing Gitlab merge request event payload") - e = &event.PullRequestEvent{ - ID: strconv.Itoa(int(payload.ObjectAttributes.IID)), - URL: utils.NormalizeUrl(payload.Project.WebURL), - Revision: payload.ObjectAttributes.SourceBranch, - Action: getNormalizedAction(payload.ObjectAttributes.Action), - Base: payload.ObjectAttributes.TargetBranch, - Commit: payload.ObjectAttributes.LastCommit.ID, - } - default: - return nil, errors.New("unsupported event") - } - return e, nil -} - -func getNormalizedAction(action string) string { - switch action { - case "open", "reopen": - return event.PullRequestOpened - case "close", "merge": - return event.PullRequestClosed - default: - return action - } -} diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index a3dd1cc4..733a4f03 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -9,9 +9,8 @@ import ( "github.com/labstack/echo/v4" configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" "github.com/padok-team/burrito/internal/burrito/config" + "github.com/padok-team/burrito/internal/utils/gitprovider" "github.com/padok-team/burrito/internal/webhook/event" - "github.com/padok-team/burrito/internal/webhook/github" - "github.com/padok-team/burrito/internal/webhook/gitlab" log "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -26,13 +25,13 @@ type Handler interface { type Webhook struct { client.Client Config *config.Config - Providers map[string][]Provider + Providers map[string][]gitprovider.Provider } func New(c *config.Config) *Webhook { return &Webhook{ Config: c, - Providers: make(map[string][]Provider), + Providers: make(map[string][]gitprovider.Provider), } } @@ -75,9 +74,9 @@ func (w *Webhook) GetHttpHandler() func(c echo.Context) error { var event event.Event for _, ps := range w.Providers { for _, p := range ps { - parsed, ok := p.ParseFromProvider(r) + parsed, ok := p.ParseWebhookPayload(r) if ok { - event, err = p.GetEvent(parsed) + event, err = p.GetEventFromWebhookPayload(parsed) break } } @@ -103,7 +102,7 @@ func (w *Webhook) GetHttpHandler() func(c echo.Context) error { } } -func (w *Webhook) initializeProviders(r configv1alpha1.TerraformRepository) ([]Provider, error) { +func (w *Webhook) initializeProviders(r configv1alpha1.TerraformRepository) ([]gitprovider.Provider, error) { if r.Spec.Repository.SecretName == "" { log.Debugf("Tried to initialize default providers, but no webhook secret configured for repository %s/%s", r.Namespace, r.Name) return nil, nil @@ -122,13 +121,28 @@ func (w *Webhook) initializeProviders(r configv1alpha1.TerraformRepository) ([]P } webhookSecret := string(value) - providers := []Provider{ - &github.Github{Secret: webhookSecret}, - &gitlab.Gitlab{Secret: webhookSecret}, + availableProviders, err := gitprovider.ListAvailable(gitprovider.Config{WebhookSecret: webhookSecret}, []string{"webhook"}) + if err != nil { + return nil, fmt.Errorf("failed to list available providers: %w", err) + } + + providers := make([]gitprovider.Provider, 0) + for _, availableProvider := range availableProviders { + provider, err := gitprovider.NewWithName(gitprovider.Config{WebhookSecret: webhookSecret}, availableProvider) + if err != nil { + log.Errorf("failed to create provider %s: %s", availableProvider, err) + continue + } + err = provider.InitWebhookHandler() + if err != nil { + log.Errorf("failed to initialize provider %s: %s", availableProvider, err) + continue + } + providers = append(providers, provider) } for _, p := range providers { - err := p.Init() + err := p.InitWebhookHandler() if err != nil { return nil, fmt.Errorf("failed to initialize provider: %w", err) } @@ -139,22 +153,28 @@ func (w *Webhook) initializeProviders(r configv1alpha1.TerraformRepository) ([]P func (w *Webhook) initializeDefaultProvider() error { if w.Providers["githubDefault"] != nil && w.Config.Server.Webhook.Github.Secret != "" { - provider := &github.Github{Secret: w.Config.Server.Webhook.Github.Secret} - err := provider.Init() + provider, err := gitprovider.NewWithName(gitprovider.Config{WebhookSecret: w.Config.Server.Webhook.Github.Secret}, "github") + if err != nil { + return fmt.Errorf("failed to create default provider: %w", err) + } + err = provider.InitWebhookHandler() if err != nil { return fmt.Errorf("failed to initialize default provider: %w", err) } - w.Providers["githubDefault"] = []Provider{provider} + w.Providers["githubDefault"] = []gitprovider.Provider{provider} log.Info("initialized default GitHub webhook handler") } if w.Providers["gitlabDefault"] != nil && w.Config.Server.Webhook.Gitlab.Secret != "" { - provider := &gitlab.Gitlab{Secret: w.Config.Server.Webhook.Gitlab.Secret} - err := provider.Init() + provider, err := gitprovider.NewWithName(gitprovider.Config{WebhookSecret: w.Config.Server.Webhook.Gitlab.Secret}, "gitlab") + if err != nil { + return fmt.Errorf("failed to create default provider: %w", err) + } + err = provider.InitWebhookHandler() if err != nil { return fmt.Errorf("failed to initialize default provider: %w", err) } - w.Providers["gitlabDefault"] = []Provider{provider} - log.Info("initialized default Gitlab webhook handler") + w.Providers["gitlabDefault"] = []gitprovider.Provider{provider} + log.Info("initialized default GitLab webhook handler") } return nil }