From 82e1ce937c9f299a93f3edef0e53b3d05f388238 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Sun, 11 Feb 2024 18:42:33 +0100 Subject: [PATCH] Refactor internal services (#915) --- cmd/server/server.go | 42 +-- cmd/server/setup.go | 96 ------- docs/docs/20-usage/15-terminiology/index.md | 1 + .../75-addons/00-overview.md | 4 - .../75-addons/20-creating-addons.md | 10 +- server/api/global_secret.go | 19 +- server/api/org_secret.go | 19 +- server/api/pipeline.go | 8 +- server/api/registry.go | 32 ++- server/api/repo_secret.go | 32 ++- server/api/signature_public_key.go | 2 +- server/config.go | 32 +-- server/forge/configFetcher.go | 190 -------------- server/model/environ.go | 5 - server/model/registry.go | 15 -- server/model/secret.go | 23 -- server/pipeline/create.go | 7 +- server/pipeline/items.go | 12 +- server/pipeline/restart.go | 27 +- server/plugins/config/http.go | 83 ------ server/plugins/secrets/builtin.go | 136 ---------- server/services/config/combined.go | 40 +++ server/services/config/combined_test.go | 248 ++++++++++++++++++ server/services/config/forge.go | 180 +++++++++++++ .../config/forge_test.go} | 229 +--------------- server/services/config/http.go | 88 +++++++ .../config/service.go} | 9 +- .../{plugins => services}/encryption/aes.go | 4 +- .../encryption/aes_builder.go | 10 +- .../encryption/aes_encryption.go | 0 .../encryption/aes_state.go | 0 .../encryption/aes_test.go | 0 .../encryption/constants.go | 6 +- .../encryption/encryption.go | 8 +- .../encryption/encryption_builder.go | 10 +- .../encryption/no_encryption.go | 8 +- .../{plugins => services}/encryption/tink.go | 4 +- .../encryption/tink_builder.go | 10 +- .../encryption/tink_keyset.go | 0 .../encryption/tink_keyset_watcher.go | 0 .../encryption/tink_state.go | 0 .../encryption/types}/encryption.go | 2 +- .../encryption/wrapper/store/constants.go | 0 .../encryption/wrapper/store/secret_store.go | 0 .../wrapper/store/secret_store_wrapper.go | 7 +- server/services/environment/extension.go | 22 ++ .../environment}/parse.go | 6 +- .../environment}/parse_test.go | 16 +- server/services/manager.go | 82 ++++++ .../permissions/admins.go | 0 .../permissions/admins_test.go | 0 .../{plugins => services}/permissions/orgs.go | 0 .../permissions/orgs_test.go | 0 .../permissions/repo_owners.go | 0 .../permissions/repo_owners_test.go | 0 .../registry/combined.go} | 16 +- server/{plugins => services}/registry/db.go | 22 +- .../registry/filesystem.go | 8 +- server/services/registry/service.go | 32 +++ server/services/secret/db.go | 133 ++++++++++ .../secret/db_test.go} | 12 +- server/services/secret/service.go | 40 +++ server/services/setup.go | 95 +++++++ server/{plugins => services}/utils/http.go | 0 .../{plugins => services}/utils/http_test.go | 2 +- shared/addon/types/types.go | 6 +- 66 files changed, 1160 insertions(+), 990 deletions(-) delete mode 100644 server/forge/configFetcher.go delete mode 100644 server/plugins/config/http.go delete mode 100644 server/plugins/secrets/builtin.go create mode 100644 server/services/config/combined.go create mode 100644 server/services/config/combined_test.go create mode 100644 server/services/config/forge.go rename server/{forge/configFetcher_test.go => services/config/forge_test.go} (57%) create mode 100644 server/services/config/http.go rename server/{plugins/config/extension.go => services/config/service.go} (63%) rename server/{plugins => services}/encryption/aes.go (94%) rename server/{plugins => services}/encryption/aes_builder.go (81%) rename server/{plugins => services}/encryption/aes_encryption.go (100%) rename server/{plugins => services}/encryption/aes_state.go (100%) rename server/{plugins => services}/encryption/aes_test.go (100%) rename server/{plugins => services}/encryption/constants.go (98%) rename server/{plugins => services}/encryption/encryption.go (86%) rename server/{plugins => services}/encryption/encryption_builder.go (84%) rename server/{plugins => services}/encryption/no_encryption.go (80%) rename server/{plugins => services}/encryption/tink.go (93%) rename server/{plugins => services}/encryption/tink_builder.go (84%) rename server/{plugins => services}/encryption/tink_keyset.go (100%) rename server/{plugins => services}/encryption/tink_keyset_watcher.go (100%) rename server/{plugins => services}/encryption/tink_state.go (100%) rename server/{model => services/encryption/types}/encryption.go (99%) rename server/{plugins => services}/encryption/wrapper/store/constants.go (100%) rename server/{plugins => services}/encryption/wrapper/store/secret_store.go (100%) rename server/{plugins => services}/encryption/wrapper/store/secret_store_wrapper.go (94%) create mode 100644 server/services/environment/extension.go rename server/{plugins/environments => services/environment}/parse.go (86%) rename server/{plugins/environments => services/environment}/parse_test.go (53%) create mode 100644 server/services/manager.go rename server/{plugins => services}/permissions/admins.go (100%) rename server/{plugins => services}/permissions/admins_test.go (100%) rename server/{plugins => services}/permissions/orgs.go (100%) rename server/{plugins => services}/permissions/orgs_test.go (100%) rename server/{plugins => services}/permissions/repo_owners.go (100%) rename server/{plugins => services}/permissions/repo_owners_test.go (100%) rename server/{plugins/registry/combine.go => services/registry/combined.go} (70%) rename server/{plugins => services}/registry/db.go (60%) rename server/{plugins => services}/registry/filesystem.go (92%) create mode 100644 server/services/registry/service.go create mode 100644 server/services/secret/db.go rename server/{plugins/secrets/builtin_test.go => services/secret/db_test.go} (82%) create mode 100644 server/services/secret/service.go create mode 100644 server/services/setup.go rename server/{plugins => services}/utils/http.go (100%) rename server/{plugins => services}/utils/http_test.go (96%) diff --git a/cmd/server/server.go b/cmd/server/server.go index eb5b9f0350..3eb3b4bbcf 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -42,12 +42,11 @@ import ( woodpeckerGrpcServer "go.woodpecker-ci.org/woodpecker/v2/server/grpc" "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" - // "go.woodpecker-ci.org/woodpecker/v2/server/plugins/encryption" - // encryptedStore "go.woodpecker-ci.org/woodpecker/v2/server/plugins/encryption/wrapper/store" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/router" "go.woodpecker-ci.org/woodpecker/v2/server/router/middleware" + "go.woodpecker-ci.org/woodpecker/v2/server/services" + "go.woodpecker-ci.org/woodpecker/v2/server/services/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/web" "go.woodpecker-ci.org/woodpecker/v2/shared/constant" @@ -271,48 +270,21 @@ func run(c *cli.Context) error { return g.Wait() } -func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) error { +func setupEvilGlobals(c *cli.Context, s store.Store, f forge.Forge) error { // forge server.Config.Services.Forge = f - server.Config.Services.Timeout = c.Duration("forge-timeout") // services - server.Config.Services.Queue = setupQueue(c, v) + server.Config.Services.Queue = setupQueue(c, s) server.Config.Services.Logs = logging.New() server.Config.Services.Pubsub = pubsub.New() - var err error - server.Config.Services.Registries, err = setupRegistryService(c, v) - if err != nil { - return err - } - - // TODO(1544): fix encrypted store - // // encryption - // encryptedSecretStore := encryptedStore.NewSecretStore(v) - // err := encryption.Encryption(c, v).WithClient(encryptedSecretStore).Build() - // if err != nil { - // log.Fatal().Err(err).Msg("could not create encryption service") - // } - // server.Config.Services.Secrets = setupSecretService(c, encryptedSecretStore) - server.Config.Services.Secrets, err = setupSecretService(c, v) - if err != nil { - return err - } - server.Config.Services.Environ, err = setupEnvironService(c, v) - if err != nil { - return err - } server.Config.Services.Membership = setupMembershipService(c, f) - server.Config.Services.SignaturePrivateKey, server.Config.Services.SignaturePublicKey, err = setupSignatureKeys(v) + serviceMangager, err := services.NewManager(c, s) if err != nil { - return err - } - - server.Config.Services.ConfigService, err = setupConfigService(c) - if err != nil { - return err + return fmt.Errorf("could not setup service manager: %w", err) } + server.Config.Services.Manager = serviceMangager // authentication server.Config.Pipeline.AuthenticatePublicRepos = c.Bool("authenticate-public-repos") diff --git a/cmd/server/setup.go b/cmd/server/setup.go index b7ebb8aeb4..63479c901f 100644 --- a/cmd/server/setup.go +++ b/cmd/server/setup.go @@ -17,11 +17,6 @@ package main import ( "context" - "crypto" - "crypto/ed25519" - "crypto/rand" - "encoding/hex" - "errors" "fmt" "net/url" "os" @@ -41,15 +36,9 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/forge/gitea" "go.woodpecker-ci.org/woodpecker/v2/server/forge/github" "go.woodpecker-ci.org/woodpecker/v2/server/forge/gitlab" - "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/environments" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/registry" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secrets" "go.woodpecker-ci.org/woodpecker/v2/server/queue" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/store/datastore" - "go.woodpecker-ci.org/woodpecker/v2/server/store/types" "go.woodpecker-ci.org/woodpecker/v2/shared/addon" addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types" ) @@ -111,48 +100,6 @@ func setupQueue(c *cli.Context, s store.Store) queue.Queue { return queue.WithTaskStore(queue.New(c.Context), s) } -func setupSecretService(c *cli.Context, s model.SecretStore) (model.SecretService, error) { - addonService, err := addon.Load[model.SecretService](c.StringSlice("addons"), addonTypes.TypeSecretService) - if err != nil { - return nil, err - } - if addonService != nil { - return addonService.Value, nil - } - - return secrets.New(c.Context, s), nil -} - -func setupRegistryService(c *cli.Context, s store.Store) (model.RegistryService, error) { - addonService, err := addon.Load[model.RegistryService](c.StringSlice("addons"), addonTypes.TypeRegistryService) - if err != nil { - return nil, err - } - if addonService != nil { - return addonService.Value, nil - } - - if c.String("docker-config") != "" { - return registry.Combined( - registry.New(s), - registry.Filesystem(c.String("docker-config")), - ), nil - } - return registry.New(s), nil -} - -func setupEnvironService(c *cli.Context, _ store.Store) (model.EnvironService, error) { - addonService, err := addon.Load[model.EnvironService](c.StringSlice("addons"), addonTypes.TypeEnvironmentService) - if err != nil { - return nil, err - } - if addonService != nil { - return addonService.Value, nil - } - - return environments.Parse(c.StringSlice("environment")), nil -} - func setupMembershipService(_ *cli.Context, r forge.Forge) cache.MembershipService { return cache.NewMembershipService(r) } @@ -292,46 +239,3 @@ func setupMetrics(g *errgroup.Group, _store store.Store) { } }) } - -// setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for extensions) -func setupSignatureKeys(_store store.Store) (crypto.PrivateKey, crypto.PublicKey, error) { - privKeyID := "signature-private-key" - - privKey, err := _store.ServerConfigGet(privKeyID) - if errors.Is(err, types.RecordNotExist) { - _, privKey, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - return nil, nil, fmt.Errorf("failed to generate private key: %w", err) - } - err = _store.ServerConfigSet(privKeyID, hex.EncodeToString(privKey)) - if err != nil { - return nil, nil, fmt.Errorf("failed to store private key: %w", err) - } - log.Debug().Msg("created private key") - return privKey, privKey.Public(), nil - } else if err != nil { - return nil, nil, fmt.Errorf("failed to load private key: %w", err) - } - privKeyStr, err := hex.DecodeString(privKey) - if err != nil { - return nil, nil, fmt.Errorf("failed to decode private key: %w", err) - } - privateKey := ed25519.PrivateKey(privKeyStr) - return privateKey, privateKey.Public(), nil -} - -func setupConfigService(c *cli.Context) (config.Extension, error) { - addonExt, err := addon.Load[config.Extension](c.StringSlice("addons"), addonTypes.TypeConfigService) - if err != nil { - return nil, err - } - if addonExt != nil { - return addonExt.Value, nil - } - - if endpoint := c.String("config-service-endpoint"); endpoint != "" { - return config.NewHTTP(endpoint, server.Config.Services.SignaturePrivateKey), nil - } - - return nil, nil -} diff --git a/docs/docs/20-usage/15-terminiology/index.md b/docs/docs/20-usage/15-terminiology/index.md index 4e4f614894..3d8f127725 100644 --- a/docs/docs/20-usage/15-terminiology/index.md +++ b/docs/docs/20-usage/15-terminiology/index.md @@ -31,6 +31,7 @@ - **YAML File**: A file format used to define and configure [workflows][Workflow]. - **Dependency**: [Workflows][Workflow] can depend on each other, and if possible, they are executed in parallel. - **Status**: Status refers to the outcome of a step or [workflow][Workflow] after it has been executed, determined by the internal command exit code. At the end of a [workflow][Workflow], its status is sent to the [forge][Forge]. +- **Service extension**: Some parts of woodpecker internal services like secrets storage or config fetcher can be replaced through service extensions. ## Pipeline events diff --git a/docs/docs/30-administration/75-addons/00-overview.md b/docs/docs/30-administration/75-addons/00-overview.md index 31c93e7824..747dc4b366 100644 --- a/docs/docs/30-administration/75-addons/00-overview.md +++ b/docs/docs/30-administration/75-addons/00-overview.md @@ -13,10 +13,6 @@ To adapt Woodpecker to your needs beyond the [configuration](../10-server-config Addons can be used for: - Forges -- Config services -- Secret services -- Environment services -- Registry services ## Restrictions diff --git a/docs/docs/30-administration/75-addons/20-creating-addons.md b/docs/docs/30-administration/75-addons/20-creating-addons.md index 8ad4fe3e9e..283c456f45 100644 --- a/docs/docs/30-administration/75-addons/20-creating-addons.md +++ b/docs/docs/30-administration/75-addons/20-creating-addons.md @@ -19,13 +19,9 @@ Directly import Woodpecker's Go package (`go.woodpecker-ci.org/woodpecker/woodpe ### Return types -| Addon type | Return type | -| -------------------- | ---------------------------------------------------------------------- | -| `Forge` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge` | -| `ConfigService` | `"go.woodpecker-ci.org/woodpecker/v2/server/plugins/config".Extension` | -| `SecretService` | `"go.woodpecker-ci.org/woodpecker/v2/server/model".SecretService` | -| `EnvironmentService` | `"go.woodpecker-ci.org/woodpecker/v2/server/model".EnvironmentService` | -| `RegistryService` | `"go.woodpecker-ci.org/woodpecker/v2/server/model".RegistryService` | +| Addon type | Return type | +| ---------- | -------------------------------------------------------------------- | +| `Forge` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge` | ### Using configurations diff --git a/server/api/global_secret.go b/server/api/global_secret.go index e2322035b7..b01de72e5b 100644 --- a/server/api/global_secret.go +++ b/server/api/global_secret.go @@ -35,7 +35,8 @@ import ( // @Param page query int false "for response pagination, page offset number" default(1) // @Param perPage query int false "for response pagination, max items per page" default(50) func GetGlobalSecretList(c *gin.Context) { - list, err := server.Config.Services.Secrets.GlobalSecretList(session.Pagination(c)) + secretService := server.Config.Services.Manager.SecretService() + list, err := secretService.GlobalSecretList(session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting global secret list. %s", err) return @@ -59,7 +60,8 @@ func GetGlobalSecretList(c *gin.Context) { // @Param secret path string true "the secret's name" func GetGlobalSecret(c *gin.Context) { name := c.Param("secret") - secret, err := server.Config.Services.Secrets.GlobalSecretFind(name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.GlobalSecretFind(name) if err != nil { handleDBError(c, err) return @@ -92,7 +94,9 @@ func PostGlobalSecret(c *gin.Context) { c.String(http.StatusBadRequest, "Error inserting global secret. %s", err) return } - if err := server.Config.Services.Secrets.GlobalSecretCreate(secret); err != nil { + + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.GlobalSecretCreate(secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting global secret %q. %s", in.Name, err) return } @@ -119,7 +123,8 @@ func PatchGlobalSecret(c *gin.Context) { return } - secret, err := server.Config.Services.Secrets.GlobalSecretFind(name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.GlobalSecretFind(name) if err != nil { handleDBError(c, err) return @@ -138,7 +143,8 @@ func PatchGlobalSecret(c *gin.Context) { c.String(http.StatusBadRequest, "Error updating global secret. %s", err) return } - if err := server.Config.Services.Secrets.GlobalSecretUpdate(secret); err != nil { + + if err := secretService.GlobalSecretUpdate(secret); err != nil { c.String(http.StatusInternalServerError, "Error updating global secret %q. %s", in.Name, err) return } @@ -156,7 +162,8 @@ func PatchGlobalSecret(c *gin.Context) { // @Param secret path string true "the secret's name" func DeleteGlobalSecret(c *gin.Context) { name := c.Param("secret") - if err := server.Config.Services.Secrets.GlobalSecretDelete(name); err != nil { + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.GlobalSecretDelete(name); err != nil { handleDBError(c, err) return } diff --git a/server/api/org_secret.go b/server/api/org_secret.go index b598ef378b..579820bc98 100644 --- a/server/api/org_secret.go +++ b/server/api/org_secret.go @@ -44,7 +44,8 @@ func GetOrgSecret(c *gin.Context) { return } - secret, err := server.Config.Services.Secrets.OrgSecretFind(orgID, name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.OrgSecretFind(orgID, name) if err != nil { handleDBError(c, err) return @@ -70,7 +71,8 @@ func GetOrgSecretList(c *gin.Context) { return } - list, err := server.Config.Services.Secrets.OrgSecretList(orgID, session.Pagination(c)) + secretService := server.Config.Services.Manager.SecretService() + list, err := secretService.OrgSecretList(orgID, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list for %q. %s", orgID, err) return @@ -116,7 +118,9 @@ func PostOrgSecret(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error inserting org %q secret. %s", orgID, err) return } - if err := server.Config.Services.Secrets.OrgSecretCreate(orgID, secret); err != nil { + + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.OrgSecretCreate(orgID, secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting org %q secret %q. %s", orgID, in.Name, err) return } @@ -149,7 +153,8 @@ func PatchOrgSecret(c *gin.Context) { return } - secret, err := server.Config.Services.Secrets.OrgSecretFind(orgID, name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.OrgSecretFind(orgID, name) if err != nil { handleDBError(c, err) return @@ -168,7 +173,8 @@ func PatchOrgSecret(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error updating org %q secret. %s", orgID, err) return } - if err := server.Config.Services.Secrets.OrgSecretUpdate(orgID, secret); err != nil { + + if err := secretService.OrgSecretUpdate(orgID, secret); err != nil { c.String(http.StatusInternalServerError, "Error updating org %q secret %q. %s", orgID, in.Name, err) return } @@ -193,7 +199,8 @@ func DeleteOrgSecret(c *gin.Context) { return } - if err := server.Config.Services.Secrets.OrgSecretDelete(orgID, name); err != nil { + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.OrgSecretDelete(orgID, name); err != nil { handleDBError(c, err) return } diff --git a/server/api/pipeline.go b/server/api/pipeline.go index 5fbfaa09b3..c151e1f264 100644 --- a/server/api/pipeline.go +++ b/server/api/pipeline.go @@ -434,13 +434,7 @@ func PostPipeline(c *gin.Context) { } } - netrc, err := server.Config.Services.Forge.Netrc(user, repo) - if err != nil { - handlePipelineErr(c, err) - return - } - - newpipeline, err := pipeline.Restart(c, _store, pl, user, repo, envs, netrc) + newpipeline, err := pipeline.Restart(c, _store, pl, user, repo, envs) if err != nil { handlePipelineErr(c, err) } else { diff --git a/server/api/registry.go b/server/api/registry.go index 27b1c9e047..60e6648ff8 100644 --- a/server/api/registry.go +++ b/server/api/registry.go @@ -35,11 +35,11 @@ import ( // @Param repo_id path int true "the repository id" // @Param registry path string true "the registry name" func GetRegistry(c *gin.Context) { - var ( - repo = session.Repo(c) - name = c.Param("registry") - ) - registry, err := server.Config.Services.Registries.RegistryFind(repo, name) + repo := session.Repo(c) + name := c.Param("registry") + + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + registry, err := registryService.RegistryFind(repo, name) if err != nil { handleDBError(c, err) return @@ -75,7 +75,9 @@ func PostRegistry(c *gin.Context) { c.String(http.StatusBadRequest, "Error inserting registry. %s", err) return } - if err := server.Config.Services.Registries.RegistryCreate(repo, registry); err != nil { + + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + if err := registryService.RegistryCreate(repo, registry); err != nil { c.String(http.StatusInternalServerError, "Error inserting registry %q. %s", in.Address, err) return } @@ -106,7 +108,8 @@ func PatchRegistry(c *gin.Context) { return } - registry, err := server.Config.Services.Registries.RegistryFind(repo, name) + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + registry, err := registryService.RegistryFind(repo, name) if err != nil { handleDBError(c, err) return @@ -122,7 +125,7 @@ func PatchRegistry(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error updating registry. %s", err) return } - if err := server.Config.Services.Registries.RegistryUpdate(repo, registry); err != nil { + if err := registryService.RegistryUpdate(repo, registry); err != nil { c.String(http.StatusInternalServerError, "Error updating registry %q. %s", in.Address, err) return } @@ -142,7 +145,8 @@ func PatchRegistry(c *gin.Context) { // @Param perPage query int false "for response pagination, max items per page" default(50) func GetRegistryList(c *gin.Context) { repo := session.Repo(c) - list, err := server.Config.Services.Registries.RegistryList(repo, session.Pagination(c)) + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + list, err := registryService.RegistryList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting registry list. %s", err) return @@ -166,11 +170,11 @@ func GetRegistryList(c *gin.Context) { // @Param repo_id path int true "the repository id" // @Param registry path string true "the registry name" func DeleteRegistry(c *gin.Context) { - var ( - repo = session.Repo(c) - name = c.Param("registry") - ) - err := server.Config.Services.Registries.RegistryDelete(repo, name) + repo := session.Repo(c) + name := c.Param("registry") + + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + err := registryService.RegistryDelete(repo, name) if err != nil { handleDBError(c, err) return diff --git a/server/api/repo_secret.go b/server/api/repo_secret.go index c7ef9eb72d..2a1b963483 100644 --- a/server/api/repo_secret.go +++ b/server/api/repo_secret.go @@ -36,11 +36,11 @@ import ( // @Param repo_id path int true "the repository id" // @Param secretName path string true "the secret name" func GetSecret(c *gin.Context) { - var ( - repo = session.Repo(c) - name = c.Param("secret") - ) - secret, err := server.Config.Services.Secrets.SecretFind(repo, name) + repo := session.Repo(c) + name := c.Param("secret") + + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + secret, err := secretService.SecretFind(repo, name) if err != nil { handleDBError(c, err) return @@ -77,7 +77,9 @@ func PostSecret(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error inserting secret. %s", err) return } - if err := server.Config.Services.Secrets.SecretCreate(repo, secret); err != nil { + + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + if err := secretService.SecretCreate(repo, secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting secret %q. %s", in.Name, err) return } @@ -108,7 +110,8 @@ func PatchSecret(c *gin.Context) { return } - secret, err := server.Config.Services.Secrets.SecretFind(repo, name) + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + secret, err := secretService.SecretFind(repo, name) if err != nil { handleDBError(c, err) return @@ -127,7 +130,7 @@ func PatchSecret(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error updating secret. %s", err) return } - if err := server.Config.Services.Secrets.SecretUpdate(repo, secret); err != nil { + if err := secretService.SecretUpdate(repo, secret); err != nil { c.String(http.StatusInternalServerError, "Error updating secret %q. %s", in.Name, err) return } @@ -147,7 +150,8 @@ func PatchSecret(c *gin.Context) { // @Param perPage query int false "for response pagination, max items per page" default(50) func GetSecretList(c *gin.Context) { repo := session.Repo(c) - list, err := server.Config.Services.Secrets.SecretList(repo, session.Pagination(c)) + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + list, err := secretService.SecretList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list. %s", err) return @@ -171,11 +175,11 @@ func GetSecretList(c *gin.Context) { // @Param repo_id path int true "the repository id" // @Param secretName path string true "the secret name" func DeleteSecret(c *gin.Context) { - var ( - repo = session.Repo(c) - name = c.Param("secret") - ) - if err := server.Config.Services.Secrets.SecretDelete(repo, name); err != nil { + repo := session.Repo(c) + name := c.Param("secret") + + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + if err := secretService.SecretDelete(repo, name); err != nil { handleDBError(c, err) return } diff --git a/server/api/signature_public_key.go b/server/api/signature_public_key.go index 937f900247..b9364166b4 100644 --- a/server/api/signature_public_key.go +++ b/server/api/signature_public_key.go @@ -34,7 +34,7 @@ import ( // @Tags System // @Param Authorization header string true "Insert your personal access token" default(Bearer ) func GetSignaturePublicKey(c *gin.Context) { - b, err := x509.MarshalPKIXPublicKey(server.Config.Services.SignaturePublicKey) + b, err := x509.MarshalPKIXPublicKey(server.Config.Services.Manager.SignaturePublicKey()) if err != nil { log.Error().Err(err).Msg("can't marshal public key") c.AbortWithStatus(http.StatusInternalServerError) diff --git a/server/config.go b/server/config.go index a06f42b164..84b56c29ad 100644 --- a/server/config.go +++ b/server/config.go @@ -18,42 +18,26 @@ package server import ( - "crypto" "time" "go.woodpecker-ci.org/woodpecker/v2/server/cache" "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/queue" + "go.woodpecker-ci.org/woodpecker/v2/server/services" + "go.woodpecker-ci.org/woodpecker/v2/server/services/permissions" ) var Config = struct { Services struct { - Pubsub *pubsub.Publisher - Queue queue.Queue - Logs logging.Log - Secrets model.SecretService - Registries model.RegistryService - Environ model.EnvironService - Forge forge.Forge - Timeout time.Duration - Membership cache.MembershipService - ConfigService config.Extension - SignaturePrivateKey crypto.PrivateKey - SignaturePublicKey crypto.PublicKey - } - Storage struct { - // Users model.UserStore - // Repos model.RepoStore - // Builds model.BuildStore - // Logs model.LogStore - Steps model.StepStore - // Registries model.RegistryStore - // Secrets model.SecretStore + Pubsub *pubsub.Publisher + Queue queue.Queue + Logs logging.Log + Forge forge.Forge + Membership cache.MembershipService + Manager *services.Manager } Server struct { Key string diff --git a/server/forge/configFetcher.go b/server/forge/configFetcher.go deleted file mode 100644 index ef989b34c5..0000000000 --- a/server/forge/configFetcher.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package forge - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "github.com/rs/zerolog/log" - - "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" - "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" - "go.woodpecker-ci.org/woodpecker/v2/shared/constant" -) - -type ConfigFetcher struct { - forge Forge - user *model.User - repo *model.Repo - pipeline *model.Pipeline - configExtension config.Extension - timeout time.Duration -} - -func NewConfigFetcher(forge Forge, timeout time.Duration, configExtension config.Extension, user *model.User, repo *model.Repo, pipeline *model.Pipeline) *ConfigFetcher { - return &ConfigFetcher{ - forge: forge, - user: user, - repo: repo, - pipeline: pipeline, - configExtension: configExtension, - timeout: timeout, - } -} - -// Fetch pipeline config from source forge -func (cf *ConfigFetcher) Fetch(ctx context.Context) (files []*types.FileMeta, err error) { - log.Trace().Msgf("start fetching config for '%s'", cf.repo.FullName) - - // try to fetch 3 times - for i := 0; i < 3; i++ { - files, err = cf.fetch(ctx, strings.TrimSpace(cf.repo.Config)) - if err != nil { - log.Trace().Err(err).Msgf("%d. try failed", i+1) - } - if errors.Is(err, context.DeadlineExceeded) { - continue - } - - if cf.configExtension != nil { - fetchCtx, cancel := context.WithTimeout(ctx, cf.timeout) - defer cancel() // ok here as we only try http fetching once, returning on fail and success - - log.Trace().Msgf("configFetcher[%s]: getting config from external http service", cf.repo.FullName) - netrc, err := cf.forge.Netrc(cf.user, cf.repo) - if err != nil { - return nil, fmt.Errorf("could not get Netrc data from forge: %w", err) - } - - newConfigs, useOld, err := cf.configExtension.FetchConfig(fetchCtx, cf.repo, cf.pipeline, files, netrc) - if err != nil { - log.Error().Err(err).Msg("could not fetch config via http") - return nil, fmt.Errorf("could not fetch config via http: %w", err) - } - - if !useOld { - return newConfigs, nil - } - } - - return - } - return -} - -// fetch config by timeout -func (cf *ConfigFetcher) fetch(c context.Context, config string) ([]*types.FileMeta, error) { - ctx, cancel := context.WithTimeout(c, cf.timeout) - defer cancel() - - if len(config) > 0 { - log.Trace().Msgf("configFetcher[%s]: use user config '%s'", cf.repo.FullName, config) - - // could be adapted to allow the user to supply a list like we do in the defaults - configs := []string{config} - - fileMeta, err := cf.getFirstAvailableConfig(ctx, configs) - if err == nil { - return fileMeta, err - } - - return nil, fmt.Errorf("user defined config '%s' not found: %w", config, err) - } - - log.Trace().Msgf("configFetcher[%s]: user did not define own config, following default procedure", cf.repo.FullName) - // for the order see shared/constants/constants.go - fileMeta, err := cf.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:]) - if err == nil { - return fileMeta, err - } - - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - return []*types.FileMeta{}, fmt.Errorf("configFetcher: fallback did not find config: %w", err) - } -} - -func filterPipelineFiles(files []*types.FileMeta) []*types.FileMeta { - var res []*types.FileMeta - - for _, file := range files { - if strings.HasSuffix(file.Name, ".yml") || strings.HasSuffix(file.Name, ".yaml") { - res = append(res, file) - } - } - - return res -} - -func (cf *ConfigFetcher) checkPipelineFile(c context.Context, config string) ([]*types.FileMeta, error) { - file, err := cf.forge.File(c, cf.user, cf.repo, cf.pipeline, config) - - if err == nil && len(file) != 0 { - log.Trace().Msgf("configFetcher[%s]: found file '%s'", cf.repo.FullName, config) - - return []*types.FileMeta{{ - Name: config, - Data: file, - }}, nil - } - - return nil, err -} - -func (cf *ConfigFetcher) getFirstAvailableConfig(c context.Context, configs []string) ([]*types.FileMeta, error) { - var forgeErr []error - for _, fileOrFolder := range configs { - if strings.HasSuffix(fileOrFolder, "/") { - // config is a folder - files, err := cf.forge.Dir(c, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(fileOrFolder, "/")) - // if folder is not supported we will get a "Not implemented" error and continue - if err != nil { - if !(errors.Is(err, types.ErrNotImplemented) || errors.Is(err, &types.ErrConfigNotFound{})) { - log.Error().Err(err).Str("repo", cf.repo.FullName).Str("user", cf.user.Login).Msg("could not get folder from forge") - forgeErr = append(forgeErr, err) - } - continue - } - files = filterPipelineFiles(files) - if len(files) != 0 { - log.Trace().Msgf("configFetcher[%s]: found %d files in '%s'", cf.repo.FullName, len(files), fileOrFolder) - return files, nil - } - } - - // config is a file - if fileMeta, err := cf.checkPipelineFile(c, fileOrFolder); err == nil { - log.Trace().Msgf("configFetcher[%s]: found file: '%s'", cf.repo.FullName, fileOrFolder) - return fileMeta, nil - } else if !errors.Is(err, &types.ErrConfigNotFound{}) { - forgeErr = append(forgeErr, err) - } - } - - // got unexpected errors - if len(forgeErr) != 0 { - return nil, errors.Join(forgeErr...) - } - - // nothing found - return nil, &types.ErrConfigNotFound{Configs: configs} -} diff --git a/server/model/environ.go b/server/model/environ.go index 4c70db4c61..87581ecb41 100644 --- a/server/model/environ.go +++ b/server/model/environ.go @@ -24,11 +24,6 @@ var ( errEnvironValueInvalid = errors.New("invalid Environment Variable Value") ) -// EnvironService defines a service for managing environment variables. -type EnvironService interface { - EnvironList(*Repo) ([]*Environ, error) -} - // EnvironStore persists environment information to storage. type EnvironStore interface { EnvironList(*Repo) ([]*Environ, error) diff --git a/server/model/registry.go b/server/model/registry.go index 0c270756e1..e2bfe9ee9c 100644 --- a/server/model/registry.go +++ b/server/model/registry.go @@ -26,21 +26,6 @@ var ( errRegistryPasswordInvalid = errors.New("invalid registry password") ) -// RegistryService defines a service for managing registries. -type RegistryService interface { - RegistryFind(*Repo, string) (*Registry, error) - RegistryList(*Repo, *ListOptions) ([]*Registry, error) - RegistryCreate(*Repo, *Registry) error - RegistryUpdate(*Repo, *Registry) error - RegistryDelete(*Repo, string) error -} - -// ReadOnlyRegistryService defines a service for managing registries. -type ReadOnlyRegistryService interface { - RegistryFind(*Repo, string) (*Registry, error) - RegistryList(*Repo, *ListOptions) ([]*Registry, error) -} - // RegistryStore persists registry information to storage. type RegistryStore interface { RegistryFind(*Repo, string) (*Registry, error) diff --git a/server/model/secret.go b/server/model/secret.go index d5d97ca8a5..cde0f49b4a 100644 --- a/server/model/secret.go +++ b/server/model/secret.go @@ -29,29 +29,6 @@ var ( ErrSecretEventInvalid = errors.New("invalid secret event") ) -// SecretService defines a service for managing secrets. -type SecretService interface { - SecretListPipeline(*Repo, *Pipeline, *ListOptions) ([]*Secret, error) - // Repository secrets - SecretFind(*Repo, string) (*Secret, error) - SecretList(*Repo, *ListOptions) ([]*Secret, error) - SecretCreate(*Repo, *Secret) error - SecretUpdate(*Repo, *Secret) error - SecretDelete(*Repo, string) error - // Organization secrets - OrgSecretFind(int64, string) (*Secret, error) - OrgSecretList(int64, *ListOptions) ([]*Secret, error) - OrgSecretCreate(int64, *Secret) error - OrgSecretUpdate(int64, *Secret) error - OrgSecretDelete(int64, string) error - // Global secrets - GlobalSecretFind(string) (*Secret, error) - GlobalSecretList(*ListOptions) ([]*Secret, error) - GlobalSecretCreate(*Secret) error - GlobalSecretUpdate(*Secret) error - GlobalSecretDelete(string) error -} - // SecretStore persists secret information to storage. type SecretStore interface { SecretFind(*Repo, string) (*Secret, error) diff --git a/server/pipeline/create.go b/server/pipeline/create.go index da467425c5..3d25744474 100644 --- a/server/pipeline/create.go +++ b/server/pipeline/create.go @@ -34,6 +34,7 @@ var skipPipelineRegex = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`) // Create a new pipeline and start it func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline *model.Pipeline) (*model.Pipeline, error) { + _forge := server.Config.Services.Forge repoUser, err := _store.GetUser(repo.UserID) if err != nil { msg := fmt.Sprintf("failure to find repo owner via id '%d'", repo.UserID) @@ -54,7 +55,7 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline // If the forge has a refresh token, the current access token // may be stale. Therefore, we should refresh prior to dispatching // the pipeline. - forge.Refresh(ctx, server.Config.Services.Forge, _store, repoUser) + forge.Refresh(ctx, _forge, _store, repoUser) // update some pipeline fields pipeline.RepoID = repo.ID @@ -68,8 +69,8 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline } // fetch the pipeline file from the forge - configFetcher := forge.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.Timeout, server.Config.Services.ConfigService, repoUser, repo, pipeline) - forgeYamlConfigs, configFetchErr := configFetcher.Fetch(ctx) + configService := server.Config.Services.Manager.ConfigServiceFromRepo(repo) + forgeYamlConfigs, configFetchErr := configService.Fetch(ctx, _forge, repoUser, repo, pipeline, nil, false) if errors.Is(configFetchErr, &forge_types.ErrConfigNotFound{}) { log.Debug().Str("repo", repo.FullName).Err(configFetchErr).Msgf("cannot find config '%s' in '%s' with user: '%s'", repo.Config, pipeline.Ref, repoUser.Login) if err := _store.DeletePipeline(pipeline); err != nil { diff --git a/server/pipeline/items.go b/server/pipeline/items.go index 3e24a28add..35206cd117 100644 --- a/server/pipeline/items.go +++ b/server/pipeline/items.go @@ -42,12 +42,14 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod log.Error().Err(err).Str("repo", repo.FullName).Msgf("error getting last pipeline before pipeline number '%d'", currentPipeline.Number) } - secs, err := server.Config.Services.Secrets.SecretListPipeline(repo, currentPipeline, &model.ListOptions{All: true}) + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + secs, err := secretService.SecretListPipeline(repo, currentPipeline, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("error getting secrets for %s#%d", repo.FullName, currentPipeline.Number) } - regs, err := server.Config.Services.Registries.RegistryList(repo, &model.ListOptions{All: true}) + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + regs, err := registryService.RegistryList(repo, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("error getting registry credentials for %s#%d", repo.FullName, currentPipeline.Number) } @@ -55,8 +57,10 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod if envs == nil { envs = map[string]string{} } - if server.Config.Services.Environ != nil { - globals, _ := server.Config.Services.Environ.EnvironList(repo) + + environmentService := server.Config.Services.Manager.EnvironmentService() + if environmentService != nil { + globals, _ := environmentService.EnvironList(repo) for _, global := range globals { envs[global.Name] = global.Value } diff --git a/server/pipeline/restart.go b/server/pipeline/restart.go index 73933ceeb3..4cab9ae928 100644 --- a/server/pipeline/restart.go +++ b/server/pipeline/restart.go @@ -28,15 +28,14 @@ import ( ) // Restart a pipeline by creating a new one out of the old and start it -func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipeline, user *model.User, repo *model.Repo, envs map[string]string, netrc *model.Netrc) (*model.Pipeline, error) { +func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipeline, user *model.User, repo *model.Repo, envs map[string]string) (*model.Pipeline, error) { + forge := server.Config.Services.Forge switch lastPipeline.Status { case model.StatusDeclined, model.StatusBlocked: return nil, &ErrBadRequest{Msg: fmt.Sprintf("cannot restart a pipeline with status %s", lastPipeline.Status)} } - var pipelineFiles []*forge_types.FileMeta - // fetch the old pipeline config from the database configs, err := store.ConfigsForPipeline(lastPipeline.ID) if err != nil { @@ -44,25 +43,17 @@ func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipelin return nil, &ErrNotFound{Msg: fmt.Sprintf("failure to get pipeline config for %s. %s", repo.FullName, err)} } + var pipelineFiles []*forge_types.FileMeta for _, y := range configs { pipelineFiles = append(pipelineFiles, &forge_types.FileMeta{Data: y.Data, Name: y.Name}) } - // If the config extension is active we should refetch the config in case something changed - if server.Config.Services.ConfigService != nil { - currentFileMeta := make([]*forge_types.FileMeta, len(configs)) - for i, cfg := range configs { - currentFileMeta[i] = &forge_types.FileMeta{Name: cfg.Name, Data: cfg.Data} - } - - newConfig, useOld, err := server.Config.Services.ConfigService.FetchConfig(ctx, repo, lastPipeline, currentFileMeta, netrc) - if err != nil { - return nil, &ErrBadRequest{ - Msg: fmt.Sprintf("On fetching external pipeline config: %s", err), - } - } - if !useOld { - pipelineFiles = newConfig + // If the config service is active we should refetch the config in case something changed + configService := server.Config.Services.Manager.ConfigServiceFromRepo(repo) + pipelineFiles, err = configService.Fetch(ctx, forge, user, repo, lastPipeline, pipelineFiles, true) + if err != nil { + return nil, &ErrBadRequest{ + Msg: fmt.Sprintf("On fetching external pipeline config: %s", err), } } diff --git a/server/plugins/config/http.go b/server/plugins/config/http.go deleted file mode 100644 index 5dbc2909bc..0000000000 --- a/server/plugins/config/http.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "context" - "crypto" - "fmt" - - forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" - "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/utils" -) - -type http struct { - endpoint string - privateKey crypto.PrivateKey -} - -// Same as forge.FileMeta but with json tags and string data -type config struct { - Name string `json:"name"` - Data string `json:"data"` -} - -type requestStructure struct { - Repo *model.Repo `json:"repo"` - Pipeline *model.Pipeline `json:"pipeline"` - Configuration []*config `json:"configs"` - Netrc *model.Netrc `json:"netrc"` -} - -type responseStructure struct { - Configs []config `json:"configs"` -} - -func NewHTTP(endpoint string, privateKey crypto.PrivateKey) Extension { - return &http{endpoint, privateKey} -} - -func (cp *http) FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge_types.FileMeta, netrc *model.Netrc) (configData []*forge_types.FileMeta, useOld bool, err error) { - currentConfigs := make([]*config, len(currentFileMeta)) - for i, pipe := range currentFileMeta { - currentConfigs[i] = &config{Name: pipe.Name, Data: string(pipe.Data)} - } - - response := new(responseStructure) - body := requestStructure{ - Repo: repo, - Pipeline: pipeline, - Configuration: currentConfigs, - Netrc: netrc, - } - - status, err := utils.Send(ctx, "POST", cp.endpoint, cp.privateKey, body, response) - if err != nil && status != 204 { - return nil, false, fmt.Errorf("failed to fetch config via http (%d) %w", status, err) - } - - var newFileMeta []*forge_types.FileMeta - if status != 200 { - newFileMeta = make([]*forge_types.FileMeta, 0) - } else { - newFileMeta = make([]*forge_types.FileMeta, len(response.Configs)) - for i, pipe := range response.Configs { - newFileMeta[i] = &forge_types.FileMeta{Name: pipe.Name, Data: []byte(pipe.Data)} - } - } - - return newFileMeta, status == 204, nil -} diff --git a/server/plugins/secrets/builtin.go b/server/plugins/secrets/builtin.go deleted file mode 100644 index bb2e40ffae..0000000000 --- a/server/plugins/secrets/builtin.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package secrets - -import ( - "context" - - "go.woodpecker-ci.org/woodpecker/v2/server/model" -) - -type builtin struct { - context.Context - store model.SecretStore -} - -// New returns a new local secret service. -func New(ctx context.Context, store model.SecretStore) model.SecretService { - return &builtin{store: store, Context: ctx} -} - -func (b *builtin) SecretFind(repo *model.Repo, name string) (*model.Secret, error) { - return b.store.SecretFind(repo, name) -} - -func (b *builtin) SecretList(repo *model.Repo, p *model.ListOptions) ([]*model.Secret, error) { - return b.store.SecretList(repo, false, p) -} - -func (b *builtin) SecretListPipeline(repo *model.Repo, _ *model.Pipeline, p *model.ListOptions) ([]*model.Secret, error) { - s, err := b.store.SecretList(repo, true, p) - if err != nil { - return nil, err - } - - // Return only secrets with unique name - // Priority order in case of duplicate names are repository, user/organization, global - secrets := make([]*model.Secret, 0, len(s)) - uniq := make(map[string]struct{}) - for _, condition := range []struct { - IsRepository bool - IsOrganization bool - IsGlobal bool - }{ - {IsRepository: true}, - {IsOrganization: true}, - {IsGlobal: true}, - } { - for _, secret := range s { - if secret.IsRepository() != condition.IsRepository || secret.IsOrganization() != condition.IsOrganization || secret.IsGlobal() != condition.IsGlobal { - continue - } - if _, ok := uniq[secret.Name]; ok { - continue - } - uniq[secret.Name] = struct{}{} - secrets = append(secrets, secret) - } - } - return secrets, nil -} - -func (b *builtin) SecretCreate(_ *model.Repo, in *model.Secret) error { - return b.store.SecretCreate(in) -} - -func (b *builtin) SecretUpdate(_ *model.Repo, in *model.Secret) error { - return b.store.SecretUpdate(in) -} - -func (b *builtin) SecretDelete(repo *model.Repo, name string) error { - secret, err := b.store.SecretFind(repo, name) - if err != nil { - return err - } - return b.store.SecretDelete(secret) -} - -func (b *builtin) OrgSecretFind(owner int64, name string) (*model.Secret, error) { - return b.store.OrgSecretFind(owner, name) -} - -func (b *builtin) OrgSecretList(owner int64, p *model.ListOptions) ([]*model.Secret, error) { - return b.store.OrgSecretList(owner, p) -} - -func (b *builtin) OrgSecretCreate(_ int64, in *model.Secret) error { - return b.store.SecretCreate(in) -} - -func (b *builtin) OrgSecretUpdate(_ int64, in *model.Secret) error { - return b.store.SecretUpdate(in) -} - -func (b *builtin) OrgSecretDelete(owner int64, name string) error { - secret, err := b.store.OrgSecretFind(owner, name) - if err != nil { - return err - } - return b.store.SecretDelete(secret) -} - -func (b *builtin) GlobalSecretFind(owner string) (*model.Secret, error) { - return b.store.GlobalSecretFind(owner) -} - -func (b *builtin) GlobalSecretList(p *model.ListOptions) ([]*model.Secret, error) { - return b.store.GlobalSecretList(p) -} - -func (b *builtin) GlobalSecretCreate(in *model.Secret) error { - return b.store.SecretCreate(in) -} - -func (b *builtin) GlobalSecretUpdate(in *model.Secret) error { - return b.store.SecretUpdate(in) -} - -func (b *builtin) GlobalSecretDelete(name string) error { - secret, err := b.store.GlobalSecretFind(name) - if err != nil { - return err - } - return b.store.SecretDelete(secret) -} diff --git a/server/services/config/combined.go b/server/services/config/combined.go new file mode 100644 index 0000000000..5daebb2812 --- /dev/null +++ b/server/services/config/combined.go @@ -0,0 +1,40 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + + "go.woodpecker-ci.org/woodpecker/v2/server/forge" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" + "go.woodpecker-ci.org/woodpecker/v2/server/model" +) + +type combined struct { + services []Service +} + +func NewCombined(services ...Service) Service { + return &combined{services: services} +} + +func (c *combined) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline, oldConfigData []*types.FileMeta, restart bool) (files []*types.FileMeta, err error) { + files = oldConfigData + for _, s := range c.services { + files, err = s.Fetch(ctx, forge, user, repo, pipeline, files, restart) + } + + return files, err +} diff --git a/server/services/config/combined_test.go b/server/services/config/combined_test.go new file mode 100644 index 0000000000..4bccd6d507 --- /dev/null +++ b/server/services/config/combined_test.go @@ -0,0 +1,248 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config_test + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/go-ap/httpsig" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks" + forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" + "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" +) + +func TestFetchFromConfigService(t *testing.T) { + t.Parallel() + + type file struct { + name string + data []byte + } + + dummyData := []byte("TEST") + + testTable := []struct { + name string + repoConfig string + files []file + expectedFileNames []string + expectedError bool + }{ + { + name: "External Fetch empty repo", + repoConfig: "", + files: []file{}, + expectedFileNames: []string{"override1", "override2", "override3"}, + expectedError: false, + }, + { + name: "Default config - Additional sub-folders", + repoConfig: "", + files: []file{{ + name: ".woodpecker/test.yml", + data: dummyData, + }, { + name: ".woodpecker/sub-folder/config.yml", + data: dummyData, + }}, + expectedFileNames: []string{"override1", "override2", "override3"}, + expectedError: false, + }, + { + name: "Fetch empty", + repoConfig: " ", + files: []file{{ + name: ".woodpecker/.keep", + data: dummyData, + }, { + name: ".woodpecker.yml", + data: nil, + }, { + name: ".woodpecker.yaml", + data: dummyData, + }}, + expectedFileNames: []string{}, + expectedError: true, + }, + { + name: "Use old config", + repoConfig: ".my-ci-folder/", + files: []file{{ + name: ".woodpecker/test.yml", + data: dummyData, + }, { + name: ".woodpecker.yml", + data: dummyData, + }, { + name: ".woodpecker.yaml", + data: dummyData, + }, { + name: ".my-ci-folder/test.yml", + data: dummyData, + }}, + expectedFileNames: []string{ + ".my-ci-folder/test.yml", + }, + expectedError: false, + }, + } + + pubEd25519Key, privEd25519Key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal("can't generate ed25519 key pair") + } + + fixtureHandler := func(w http.ResponseWriter, r *http.Request) { + // check signature + pubKeyID := "woodpecker-ci-plugins" + + keystore := httpsig.NewMemoryKeyStore() + keystore.SetKey(pubKeyID, pubEd25519Key) + + verifier := httpsig.NewVerifier(keystore) + verifier.SetRequiredHeaders([]string{"(request-target)", "date"}) + + keyID, err := verifier.Verify(r) + if err != nil { + http.Error(w, "Invalid signature", http.StatusBadRequest) + return + } + + if keyID != pubKeyID { + http.Error(w, "Used wrong key", http.StatusBadRequest) + return + } + + type config struct { + Name string `json:"name"` + Data string `json:"data"` + } + + type incoming struct { + Repo *model.Repo `json:"repo"` + Build *model.Pipeline `json:"pipeline"` + Configuration []*config `json:"config"` + } + + var req incoming + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "can't read body", http.StatusBadRequest) + return + } + err = json.Unmarshal(body, &req) + if err != nil { + http.Error(w, "Failed to parse JSON"+err.Error(), http.StatusBadRequest) + return + } + + if req.Repo.Name == "Fetch empty" { + w.WriteHeader(404) + return + } + + if req.Repo.Name == "Use old config" { + w.WriteHeader(204) + return + } + + fmt.Fprint(w, `{ + "configs": [ + { + "name": "override1", + "data": "some new pipelineconfig \n pipe, pipe, pipe" + }, + { + "name": "override2", + "data": "some new pipelineconfig \n pipe, pipe, pipe" + }, + { + "name": "override3", + "data": "some new pipelineconfig \n pipe, pipe, pipe" + } + ] +}`) + } + + ts := httptest.NewServer(http.HandlerFunc(fixtureHandler)) + defer ts.Close() + httpFetcher := config.NewHTTP(ts.URL, privEd25519Key) + + for _, tt := range testTable { + t.Run(tt.name, func(t *testing.T) { + repo := &model.Repo{Owner: "laszlocph", Name: tt.name, Config: tt.repoConfig} // Using test name as repo name to provide different responses in mock server + + f := new(mocks.Forge) + dirs := map[string][]*forge_types.FileMeta{} + for _, file := range tt.files { + f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil) + path := filepath.Dir(file.name) + if path != "." { + dirs[path] = append(dirs[path], &forge_types.FileMeta{ + Name: file.name, + Data: file.data, + }) + } + } + + for path, files := range dirs { + f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, path).Return(files, nil) + } + + // if the previous mocks do not match return not found errors + f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("file not found")) + f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("directory not found")) + + f.On("Netrc", mock.Anything, mock.Anything).Return(&model.Netrc{Machine: "mock", Login: "mock", Password: "mock"}, nil) + + forgeFetcher := config.NewForge(time.Second * 3) + configFetcher := config.NewCombined(forgeFetcher, httpFetcher) + files, err := configFetcher.Fetch( + context.Background(), + f, + &model.User{Token: "xxx"}, + repo, + &model.Pipeline{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"}, + []*forge_types.FileMeta{}, + false, + ) + if tt.expectedError && err == nil { + t.Fatal("expected an error") + } else if !tt.expectedError && err != nil { + t.Fatal("error fetching config:", err) + } + + matchingFiles := make([]string, len(files)) + for i := range files { + matchingFiles[i] = files[i].Name + } + assert.ElementsMatch(t, tt.expectedFileNames, matchingFiles, "expected some other pipeline files") + }) + } +} diff --git a/server/services/config/forge.go b/server/services/config/forge.go new file mode 100644 index 0000000000..fc794bad8a --- /dev/null +++ b/server/services/config/forge.go @@ -0,0 +1,180 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/rs/zerolog/log" + + "go.woodpecker-ci.org/woodpecker/v2/server/forge" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" + "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/shared/constant" +) + +const ( + forgeFetchingRetryCount = 3 +) + +type forgeFetcher struct { + timeout time.Duration +} + +func NewForge(timeout time.Duration) Service { + return &forgeFetcher{ + timeout: timeout, + } +} + +func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline, oldConfigData []*types.FileMeta, restart bool) (files []*types.FileMeta, err error) { + // skip fetching if we are restarting and have the old config + if restart && len(oldConfigData) > 0 { + return + } + + ffc := &forgeFetcherContext{ + forge: forge, + user: user, + repo: repo, + pipeline: pipeline, + timeout: f.timeout, + } + + // try to fetch multiple times + for i := 0; i < forgeFetchingRetryCount; i++ { + files, err = ffc.fetch(ctx, strings.TrimSpace(repo.Config)) + if err != nil { + log.Trace().Err(err).Msgf("%d. try failed", i+1) + } + if errors.Is(err, context.DeadlineExceeded) { + continue + } + } + + return +} + +type forgeFetcherContext struct { + forge forge.Forge + user *model.User + repo *model.Repo + pipeline *model.Pipeline + timeout time.Duration +} + +// fetch config by timeout +func (f *forgeFetcherContext) fetch(c context.Context, config string) ([]*types.FileMeta, error) { + ctx, cancel := context.WithTimeout(c, f.timeout) + defer cancel() + + if len(config) > 0 { + log.Trace().Msgf("configFetcher[%s]: use user config '%s'", f.repo.FullName, config) + + // could be adapted to allow the user to supply a list like we do in the defaults + configs := []string{config} + + fileMetas, err := f.getFirstAvailableConfig(ctx, configs) + if err == nil { + return fileMetas, err + } + + return nil, fmt.Errorf("user defined config '%s' not found: %w", config, err) + } + + log.Trace().Msgf("configFetcher[%s]: user did not define own config, following default procedure", f.repo.FullName) + // for the order see shared/constants/constants.go + fileMetas, err := f.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:]) + if err == nil { + return fileMetas, err + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + return nil, fmt.Errorf("configFetcher: fallback did not find config: %w", err) + } +} + +func filterPipelineFiles(files []*types.FileMeta) []*types.FileMeta { + var res []*types.FileMeta + + for _, file := range files { + if strings.HasSuffix(file.Name, ".yml") || strings.HasSuffix(file.Name, ".yaml") { + res = append(res, file) + } + } + + return res +} + +func (f *forgeFetcherContext) checkPipelineFile(c context.Context, config string) ([]*types.FileMeta, error) { + file, err := f.forge.File(c, f.user, f.repo, f.pipeline, config) + + if err == nil && len(file) != 0 { + log.Trace().Msgf("configFetcher[%s]: found file '%s'", f.repo.FullName, config) + + return []*types.FileMeta{{ + Name: config, + Data: file, + }}, nil + } + + return nil, err +} + +func (f *forgeFetcherContext) getFirstAvailableConfig(c context.Context, configs []string) ([]*types.FileMeta, error) { + var forgeErr []error + for _, fileOrFolder := range configs { + if strings.HasSuffix(fileOrFolder, "/") { + // config is a folder + files, err := f.forge.Dir(c, f.user, f.repo, f.pipeline, strings.TrimSuffix(fileOrFolder, "/")) + // if folder is not supported we will get a "Not implemented" error and continue + if err != nil { + if !(errors.Is(err, types.ErrNotImplemented) || errors.Is(err, &types.ErrConfigNotFound{})) { + log.Error().Err(err).Str("repo", f.repo.FullName).Str("user", f.user.Login).Msg("could not get folder from forge") + forgeErr = append(forgeErr, err) + } + continue + } + files = filterPipelineFiles(files) + if len(files) != 0 { + log.Trace().Msgf("configFetcher[%s]: found %d files in '%s'", f.repo.FullName, len(files), fileOrFolder) + return files, nil + } + } + + // config is a file + if fileMeta, err := f.checkPipelineFile(c, fileOrFolder); err == nil { + log.Trace().Msgf("configFetcher[%s]: found file: '%s'", f.repo.FullName, fileOrFolder) + return fileMeta, nil + } else if !errors.Is(err, &types.ErrConfigNotFound{}) { + forgeErr = append(forgeErr, err) + } + } + + // got unexpected errors + if len(forgeErr) != 0 { + return nil, errors.Join(forgeErr...) + } + + // nothing found + return nil, &types.ErrConfigNotFound{Configs: configs} +} diff --git a/server/forge/configFetcher_test.go b/server/services/config/forge_test.go similarity index 57% rename from server/forge/configFetcher_test.go rename to server/services/config/forge_test.go index 36b25d9c58..c442e78fd2 100644 --- a/server/forge/configFetcher_test.go +++ b/server/services/config/forge_test.go @@ -12,30 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package forge_test +package config_test import ( "context" - "crypto/ed25519" - "crypto/rand" - "encoding/json" "fmt" - "io" - "net/http" - "net/http/httptest" "path/filepath" "testing" "time" - "github.com/go-ap/httpsig" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" ) func TestFetch(t *testing.T) { @@ -313,223 +305,18 @@ func TestFetch(t *testing.T) { f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("file not found")) f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("directory not found")) - configFetcher := forge.NewConfigFetcher( - f, - time.Second*3, - nil, - &model.User{Token: "xxx"}, - repo, - &model.Pipeline{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"}, + configFetcher := config.NewForge( + time.Second * 3, ) - files, err := configFetcher.Fetch(context.Background()) - if tt.expectedError && err == nil { - t.Fatal("expected an error") - } else if !tt.expectedError && err != nil { - t.Fatal("error fetching config:", err) - } - - matchingFiles := make([]string, len(files)) - for i := range files { - matchingFiles[i] = files[i].Name - } - assert.ElementsMatch(t, tt.expectedFileNames, matchingFiles, "expected some other pipeline files") - }) - } -} - -func TestFetchFromConfigService(t *testing.T) { - t.Parallel() - - type file struct { - name string - data []byte - } - - dummyData := []byte("TEST") - - testTable := []struct { - name string - repoConfig string - files []file - expectedFileNames []string - expectedError bool - }{ - { - name: "External Fetch empty repo", - repoConfig: "", - files: []file{}, - expectedFileNames: []string{"override1", "override2", "override3"}, - expectedError: false, - }, - { - name: "Default config - Additional sub-folders", - repoConfig: "", - files: []file{{ - name: ".woodpecker/test.yml", - data: dummyData, - }, { - name: ".woodpecker/sub-folder/config.yml", - data: dummyData, - }}, - expectedFileNames: []string{"override1", "override2", "override3"}, - expectedError: false, - }, - { - name: "Fetch empty", - repoConfig: " ", - files: []file{{ - name: ".woodpecker/.keep", - data: dummyData, - }, { - name: ".woodpecker.yml", - data: nil, - }, { - name: ".woodpecker.yaml", - data: dummyData, - }}, - expectedFileNames: []string{}, - expectedError: true, - }, - { - name: "Use old config", - repoConfig: ".my-ci-folder/", - files: []file{{ - name: ".woodpecker/test.yml", - data: dummyData, - }, { - name: ".woodpecker.yml", - data: dummyData, - }, { - name: ".woodpecker.yaml", - data: dummyData, - }, { - name: ".my-ci-folder/test.yml", - data: dummyData, - }}, - expectedFileNames: []string{ - ".my-ci-folder/test.yml", - }, - expectedError: false, - }, - } - - pubEd25519Key, privEd25519Key, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - t.Fatal("can't generate ed25519 key pair") - } - - fixtureHandler := func(w http.ResponseWriter, r *http.Request) { - // check signature - pubKeyID := "woodpecker-ci-plugins" - - keystore := httpsig.NewMemoryKeyStore() - keystore.SetKey(pubKeyID, pubEd25519Key) - - verifier := httpsig.NewVerifier(keystore) - verifier.SetRequiredHeaders([]string{"(request-target)", "date"}) - - keyID, err := verifier.Verify(r) - if err != nil { - http.Error(w, "Invalid signature", http.StatusBadRequest) - return - } - - if keyID != pubKeyID { - http.Error(w, "Used wrong key", http.StatusBadRequest) - return - } - - type config struct { - Name string `json:"name"` - Data string `json:"data"` - } - - type incoming struct { - Repo *model.Repo `json:"repo"` - Build *model.Pipeline `json:"pipeline"` - Configuration []*config `json:"config"` - } - - var req incoming - body, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "can't read body", http.StatusBadRequest) - return - } - err = json.Unmarshal(body, &req) - if err != nil { - http.Error(w, "Failed to parse JSON"+err.Error(), http.StatusBadRequest) - return - } - - if req.Repo.Name == "Fetch empty" { - w.WriteHeader(404) - return - } - - if req.Repo.Name == "Use old config" { - w.WriteHeader(204) - return - } - - fmt.Fprint(w, `{ - "configs": [ - { - "name": "override1", - "data": "some new pipelineconfig \n pipe, pipe, pipe" - }, - { - "name": "override2", - "data": "some new pipelineconfig \n pipe, pipe, pipe" - }, - { - "name": "override3", - "data": "some new pipelineconfig \n pipe, pipe, pipe" - } - ] -}`) - } - - ts := httptest.NewServer(http.HandlerFunc(fixtureHandler)) - defer ts.Close() - configAPI := config.NewHTTP(ts.URL, privEd25519Key) - - for _, tt := range testTable { - t.Run(tt.name, func(t *testing.T) { - repo := &model.Repo{Owner: "laszlocph", Name: tt.name, Config: tt.repoConfig} // Using test name as repo name to provide different responses in mock server - - f := new(mocks.Forge) - dirs := map[string][]*forge_types.FileMeta{} - for _, file := range tt.files { - f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil) - path := filepath.Dir(file.name) - if path != "." { - dirs[path] = append(dirs[path], &forge_types.FileMeta{ - Name: file.name, - Data: file.data, - }) - } - } - - for path, files := range dirs { - f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, path).Return(files, nil) - } - - // if the previous mocks do not match return not found errors - f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("file not found")) - f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("directory not found")) - - f.On("Netrc", mock.Anything, mock.Anything).Return(&model.Netrc{Machine: "mock", Login: "mock", Password: "mock"}, nil) - - configFetcher := forge.NewConfigFetcher( + files, err := configFetcher.Fetch( + context.Background(), f, - time.Second*3, - configAPI, &model.User{Token: "xxx"}, repo, &model.Pipeline{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"}, + nil, + false, ) - files, err := configFetcher.Fetch(context.Background()) if tt.expectedError && err == nil { t.Fatal("expected an error") } else if !tt.expectedError && err != nil { diff --git a/server/services/config/http.go b/server/services/config/http.go new file mode 100644 index 0000000000..dc51a47349 --- /dev/null +++ b/server/services/config/http.go @@ -0,0 +1,88 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + "crypto" + "fmt" + + "go.woodpecker-ci.org/woodpecker/v2/server/forge" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" + "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" +) + +type http struct { + endpoint string + privateKey crypto.PrivateKey +} + +// configData same as forge.FileMeta but with json tags and string data +type configData struct { + Name string `json:"name"` + Data string `json:"data"` +} + +type requestStructure struct { + Repo *model.Repo `json:"repo"` + Pipeline *model.Pipeline `json:"pipeline"` + Netrc *model.Netrc `json:"netrc"` + Configuration []*configData `json:"configs"` // TODO: deprecate in favor of netrc and remove in next major release +} + +type responseStructure struct { + Configs []*configData `json:"configs"` +} + +func NewHTTP(endpoint string, privateKey crypto.PrivateKey) Service { + return &http{endpoint, privateKey} +} + +func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline, oldConfigData []*types.FileMeta, _ bool) ([]*types.FileMeta, error) { + currentConfigs := make([]*configData, len(oldConfigData)) + for i, pipe := range oldConfigData { + currentConfigs[i] = &configData{Name: pipe.Name, Data: string(pipe.Data)} + } + + netrc, err := forge.Netrc(user, repo) + if err != nil { + return nil, fmt.Errorf("could not get Netrc data from forge: %w", err) + } + + response := new(responseStructure) + body := requestStructure{ + Repo: repo, + Pipeline: pipeline, + Configuration: currentConfigs, + Netrc: netrc, + } + + status, err := utils.Send(ctx, "POST", h.endpoint, h.privateKey, body, response) + if err != nil && status != 204 { + return nil, fmt.Errorf("failed to fetch config via http (%d) %w", status, err) + } + + if status != 200 { + return oldConfigData, nil + } + + fileMetas := make([]*types.FileMeta, len(response.Configs)) + for i, config := range response.Configs { + fileMetas[i] = &types.FileMeta{Name: config.Name, Data: []byte(config.Data)} + } + + return fileMetas, nil +} diff --git a/server/plugins/config/extension.go b/server/services/config/service.go similarity index 63% rename from server/plugins/config/extension.go rename to server/services/config/service.go index 9b9778af60..12ba46f21b 100644 --- a/server/plugins/config/extension.go +++ b/server/services/config/service.go @@ -1,4 +1,4 @@ -// Copyright 2022 Woodpecker Authors +// Copyright 2024 Woodpecker Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,10 +17,11 @@ package config import ( "context" - forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" + "go.woodpecker-ci.org/woodpecker/v2/server/forge" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" ) -type Extension interface { - FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge_types.FileMeta, netrc *model.Netrc) (configData []*forge_types.FileMeta, useOld bool, err error) +type Service interface { + Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline, oldConfigData []*types.FileMeta, restart bool) (configData []*types.FileMeta, err error) } diff --git a/server/plugins/encryption/aes.go b/server/services/encryption/aes.go similarity index 94% rename from server/plugins/encryption/aes.go rename to server/services/encryption/aes.go index a0766f4bee..56a784e27a 100644 --- a/server/plugins/encryption/aes.go +++ b/server/services/encryption/aes.go @@ -21,7 +21,7 @@ import ( "github.com/google/tink/go/subtle/random" - "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) @@ -29,7 +29,7 @@ type aesEncryptionService struct { cipher cipher.AEAD keyID string store store.Store - clients []model.EncryptionClient + clients []types.EncryptionClient } func (svc *aesEncryptionService) Encrypt(plaintext, associatedData string) (string, error) { diff --git a/server/plugins/encryption/aes_builder.go b/server/services/encryption/aes_builder.go similarity index 81% rename from server/plugins/encryption/aes_builder.go rename to server/services/encryption/aes_builder.go index bc0bbe9455..1da6df8a04 100644 --- a/server/plugins/encryption/aes_builder.go +++ b/server/services/encryption/aes_builder.go @@ -20,27 +20,27 @@ import ( "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) type aesConfiguration struct { password string store store.Store - clients []model.EncryptionClient + clients []types.EncryptionClient } -func newAES(ctx *cli.Context, s store.Store) model.EncryptionServiceBuilder { +func newAES(ctx *cli.Context, s store.Store) types.EncryptionServiceBuilder { key := ctx.String(rawKeyConfigFlag) return &aesConfiguration{key, s, nil} } -func (c aesConfiguration) WithClients(clients []model.EncryptionClient) model.EncryptionServiceBuilder { +func (c aesConfiguration) WithClients(clients []types.EncryptionClient) types.EncryptionServiceBuilder { c.clients = clients return c } -func (c aesConfiguration) Build() (model.EncryptionService, error) { +func (c aesConfiguration) Build() (types.EncryptionService, error) { svc := &aesEncryptionService{ cipher: nil, store: c.store, diff --git a/server/plugins/encryption/aes_encryption.go b/server/services/encryption/aes_encryption.go similarity index 100% rename from server/plugins/encryption/aes_encryption.go rename to server/services/encryption/aes_encryption.go diff --git a/server/plugins/encryption/aes_state.go b/server/services/encryption/aes_state.go similarity index 100% rename from server/plugins/encryption/aes_state.go rename to server/services/encryption/aes_state.go diff --git a/server/plugins/encryption/aes_test.go b/server/services/encryption/aes_test.go similarity index 100% rename from server/plugins/encryption/aes_test.go rename to server/services/encryption/aes_test.go diff --git a/server/plugins/encryption/constants.go b/server/services/encryption/constants.go similarity index 98% rename from server/plugins/encryption/constants.go rename to server/services/encryption/constants.go index 5a0f28500c..90feabd324 100644 --- a/server/plugins/encryption/constants.go +++ b/server/services/encryption/constants.go @@ -64,9 +64,9 @@ const ( logMessageEncryptionDisabled = "encryption disabled" logMessageEncryptionKeyRegistered = "registered new encryption key" logMessageClientsInitialized = "initialized encryption on registered clients" - logMessageClientsEnabled = "enabled encryption on registered services" - logMessageClientsRotated = "updated encryption key on registered services" - logMessageClientsDecrypted = "disabled encryption on registered services" + logMessageClientsEnabled = "enabled encryption on registered service" + logMessageClientsRotated = "updated encryption key on registered service" + logMessageClientsDecrypted = "disabled encryption on registered service" ) // tink diff --git a/server/plugins/encryption/encryption.go b/server/services/encryption/encryption.go similarity index 86% rename from server/plugins/encryption/encryption.go rename to server/services/encryption/encryption.go index 34e9189cd1..4548aa8a3d 100644 --- a/server/plugins/encryption/encryption.go +++ b/server/services/encryption/encryption.go @@ -19,21 +19,21 @@ import ( "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) type builder struct { store store.Store ctx *cli.Context - clients []model.EncryptionClient + clients []types.EncryptionClient } -func Encryption(ctx *cli.Context, s store.Store) model.EncryptionBuilder { +func Encryption(ctx *cli.Context, s store.Store) types.EncryptionBuilder { return &builder{store: s, ctx: ctx} } -func (b builder) WithClient(client model.EncryptionClient) model.EncryptionBuilder { +func (b builder) WithClient(client types.EncryptionClient) types.EncryptionBuilder { b.clients = append(b.clients, client) return b } diff --git a/server/plugins/encryption/encryption_builder.go b/server/services/encryption/encryption_builder.go similarity index 84% rename from server/plugins/encryption/encryption_builder.go rename to server/services/encryption/encryption_builder.go index f14a8824a6..700fbdb77e 100644 --- a/server/plugins/encryption/encryption_builder.go +++ b/server/services/encryption/encryption_builder.go @@ -18,11 +18,11 @@ import ( "errors" "fmt" - "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/store/types" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" + storeTypes "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) -func (b builder) getService(keyType string) (model.EncryptionService, error) { +func (b builder) getService(keyType string) (types.EncryptionService, error) { if keyType == keyTypeNone { return nil, errors.New(errMessageNoKeysProvided) } @@ -41,7 +41,7 @@ func (b builder) getService(keyType string) (model.EncryptionService, error) { func (b builder) isEnabled() (bool, error) { _, err := b.store.ServerConfigGet(ciphertextSampleConfigKey) - if err != nil && !errors.Is(err, types.RecordNotExist) { + if err != nil && !errors.Is(err, storeTypes.RecordNotExist) { return false, fmt.Errorf(errTemplateFailedLoadingServerConfig, err) } return err == nil, nil @@ -61,7 +61,7 @@ func (b builder) detectKeyType() (string, error) { return keyTypeNone, nil } -func (b builder) serviceBuilder(keyType string) (model.EncryptionServiceBuilder, error) { +func (b builder) serviceBuilder(keyType string) (types.EncryptionServiceBuilder, error) { switch { case keyType == keyTypeTink: return newTink(b.ctx, b.store), nil diff --git a/server/plugins/encryption/no_encryption.go b/server/services/encryption/no_encryption.go similarity index 80% rename from server/plugins/encryption/no_encryption.go rename to server/services/encryption/no_encryption.go index 0d13d5ad45..6bc9230d01 100644 --- a/server/plugins/encryption/no_encryption.go +++ b/server/services/encryption/no_encryption.go @@ -14,18 +14,18 @@ package encryption -import "go.woodpecker-ci.org/woodpecker/v2/server/model" +import "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" type noEncryptionBuilder struct { - clients []model.EncryptionClient + clients []types.EncryptionClient } -func (b noEncryptionBuilder) WithClients(clients []model.EncryptionClient) model.EncryptionServiceBuilder { +func (b noEncryptionBuilder) WithClients(clients []types.EncryptionClient) types.EncryptionServiceBuilder { b.clients = clients return b } -func (b noEncryptionBuilder) Build() (model.EncryptionService, error) { +func (b noEncryptionBuilder) Build() (types.EncryptionService, error) { svc := &noEncryption{} for _, client := range b.clients { err := client.SetEncryptionService(svc) diff --git a/server/plugins/encryption/tink.go b/server/services/encryption/tink.go similarity index 93% rename from server/plugins/encryption/tink.go rename to server/services/encryption/tink.go index 22111b802d..3c210ac7a7 100644 --- a/server/plugins/encryption/tink.go +++ b/server/services/encryption/tink.go @@ -21,7 +21,7 @@ import ( "github.com/fsnotify/fsnotify" "github.com/google/tink/go/tink" - "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) @@ -31,7 +31,7 @@ type tinkEncryptionService struct { encryption tink.AEAD store store.Store keysetFileWatcher *fsnotify.Watcher - clients []model.EncryptionClient + clients []types.EncryptionClient } func (svc *tinkEncryptionService) Encrypt(plaintext, associatedData string) (string, error) { diff --git a/server/plugins/encryption/tink_builder.go b/server/services/encryption/tink_builder.go similarity index 84% rename from server/plugins/encryption/tink_builder.go rename to server/services/encryption/tink_builder.go index a7fba536e1..9dd1ddd799 100644 --- a/server/plugins/encryption/tink_builder.go +++ b/server/services/encryption/tink_builder.go @@ -20,27 +20,27 @@ import ( "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) type tinkConfiguration struct { keysetFilePath string store store.Store - clients []model.EncryptionClient + clients []types.EncryptionClient } -func newTink(ctx *cli.Context, s store.Store) model.EncryptionServiceBuilder { +func newTink(ctx *cli.Context, s store.Store) types.EncryptionServiceBuilder { filepath := ctx.String(tinkKeysetFilepathConfigFlag) return &tinkConfiguration{filepath, s, nil} } -func (c tinkConfiguration) WithClients(clients []model.EncryptionClient) model.EncryptionServiceBuilder { +func (c tinkConfiguration) WithClients(clients []types.EncryptionClient) types.EncryptionServiceBuilder { c.clients = clients return c } -func (c tinkConfiguration) Build() (model.EncryptionService, error) { +func (c tinkConfiguration) Build() (types.EncryptionService, error) { svc := &tinkEncryptionService{ keysetFilePath: c.keysetFilePath, primaryKeyID: "", diff --git a/server/plugins/encryption/tink_keyset.go b/server/services/encryption/tink_keyset.go similarity index 100% rename from server/plugins/encryption/tink_keyset.go rename to server/services/encryption/tink_keyset.go diff --git a/server/plugins/encryption/tink_keyset_watcher.go b/server/services/encryption/tink_keyset_watcher.go similarity index 100% rename from server/plugins/encryption/tink_keyset_watcher.go rename to server/services/encryption/tink_keyset_watcher.go diff --git a/server/plugins/encryption/tink_state.go b/server/services/encryption/tink_state.go similarity index 100% rename from server/plugins/encryption/tink_state.go rename to server/services/encryption/tink_state.go diff --git a/server/model/encryption.go b/server/services/encryption/types/encryption.go similarity index 99% rename from server/model/encryption.go rename to server/services/encryption/types/encryption.go index d3d9878509..2d6b136fb8 100644 --- a/server/model/encryption.go +++ b/server/services/encryption/types/encryption.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package model +package types // EncryptionBuilder is user API to obtain correctly configured encryption type EncryptionBuilder interface { diff --git a/server/plugins/encryption/wrapper/store/constants.go b/server/services/encryption/wrapper/store/constants.go similarity index 100% rename from server/plugins/encryption/wrapper/store/constants.go rename to server/services/encryption/wrapper/store/constants.go diff --git a/server/plugins/encryption/wrapper/store/secret_store.go b/server/services/encryption/wrapper/store/secret_store.go similarity index 100% rename from server/plugins/encryption/wrapper/store/secret_store.go rename to server/services/encryption/wrapper/store/secret_store.go diff --git a/server/plugins/encryption/wrapper/store/secret_store_wrapper.go b/server/services/encryption/wrapper/store/secret_store_wrapper.go similarity index 94% rename from server/plugins/encryption/wrapper/store/secret_store_wrapper.go rename to server/services/encryption/wrapper/store/secret_store_wrapper.go index d1ce549b72..c776bcfa16 100644 --- a/server/plugins/encryption/wrapper/store/secret_store_wrapper.go +++ b/server/services/encryption/wrapper/store/secret_store_wrapper.go @@ -22,11 +22,12 @@ import ( "github.com/rs/zerolog/log" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" ) type EncryptedSecretStore struct { store model.SecretStore - encryption model.EncryptionService + encryption types.EncryptionService } // ensure wrapper match interface @@ -37,7 +38,7 @@ func NewSecretStore(secretStore model.SecretStore) *EncryptedSecretStore { return &wrapper } -func (wrapper *EncryptedSecretStore) SetEncryptionService(service model.EncryptionService) error { +func (wrapper *EncryptedSecretStore) SetEncryptionService(service types.EncryptionService) error { if wrapper.encryption != nil { return errors.New(errMessageInitSeveralTimes) } @@ -63,7 +64,7 @@ func (wrapper *EncryptedSecretStore) EnableEncryption() error { return nil } -func (wrapper *EncryptedSecretStore) MigrateEncryption(newEncryptionService model.EncryptionService) error { +func (wrapper *EncryptedSecretStore) MigrateEncryption(newEncryptionService types.EncryptionService) error { log.Warn().Msg(logMessageMigratingSecretsEncryption) secrets, err := wrapper.store.SecretListAll() if err != nil { diff --git a/server/services/environment/extension.go b/server/services/environment/extension.go new file mode 100644 index 0000000000..f8ad148215 --- /dev/null +++ b/server/services/environment/extension.go @@ -0,0 +1,22 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package environment + +import "go.woodpecker-ci.org/woodpecker/v2/server/model" + +// Service defines a service for managing environment variables. +type Service interface { + EnvironList(*model.Repo) ([]*model.Environ, error) +} diff --git a/server/plugins/environments/parse.go b/server/services/environment/parse.go similarity index 86% rename from server/plugins/environments/parse.go rename to server/services/environment/parse.go index b06294c2e6..c1a6b8c878 100644 --- a/server/plugins/environments/parse.go +++ b/server/services/environment/parse.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package environments +package environment import ( "strings" @@ -26,8 +26,8 @@ type builtin struct { globals []*model.Environ } -// Parse returns a model.EnvironService based on a string slice where key and value are separated by a ":" delimiter. -func Parse(params []string) model.EnvironService { +// Parse returns a Service based on a string slice where key and value are separated by a ":" delimiter. +func Parse(params []string) Service { var globals []*model.Environ for _, item := range params { diff --git a/server/plugins/environments/parse_test.go b/server/services/environment/parse_test.go similarity index 53% rename from server/plugins/environments/parse_test.go rename to server/services/environment/parse_test.go index 9e1feacc49..97e7575698 100644 --- a/server/plugins/environments/parse_test.go +++ b/server/services/environment/parse_test.go @@ -1,4 +1,18 @@ -package environments +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package environment import ( "testing" diff --git a/server/services/manager.go b/server/services/manager.go new file mode 100644 index 0000000000..d5669df2b6 --- /dev/null +++ b/server/services/manager.go @@ -0,0 +1,82 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package services + +import ( + "crypto" + + "github.com/urfave/cli/v2" + + "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" + "go.woodpecker-ci.org/woodpecker/v2/server/services/environment" + "go.woodpecker-ci.org/woodpecker/v2/server/services/registry" + "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/store" +) + +type Manager struct { + secret secret.Service + registry registry.Service + config config.Service + environment environment.Service + signaturePrivateKey crypto.PrivateKey + signaturePublicKey crypto.PublicKey +} + +func NewManager(c *cli.Context, store store.Store) (*Manager, error) { + signaturePrivateKey, signaturePublicKey, err := setupSignatureKeys(store) + if err != nil { + return nil, err + } + + return &Manager{ + signaturePrivateKey: signaturePrivateKey, + signaturePublicKey: signaturePublicKey, + secret: setupSecretService(store), + registry: setupRegistryService(store, c.String("docker-config")), + config: setupConfigService(c, signaturePrivateKey), + environment: environment.Parse(c.StringSlice("environment")), + }, nil +} + +func (e *Manager) SignaturePublicKey() crypto.PublicKey { + return e.signaturePublicKey +} + +func (e *Manager) SecretServiceFromRepo(_ *model.Repo) secret.Service { + return e.SecretService() +} + +func (e *Manager) SecretService() secret.Service { + return e.secret +} + +func (e *Manager) RegistryServiceFromRepo(_ *model.Repo) registry.Service { + return e.RegistryService() +} + +func (e *Manager) RegistryService() registry.Service { + return e.registry +} + +func (e *Manager) ConfigServiceFromRepo(_ *model.Repo) config.Service { + // TODO: decied based on repo property which config service to use + return e.config +} + +func (e *Manager) EnvironmentService() environment.Service { + return e.environment +} diff --git a/server/plugins/permissions/admins.go b/server/services/permissions/admins.go similarity index 100% rename from server/plugins/permissions/admins.go rename to server/services/permissions/admins.go diff --git a/server/plugins/permissions/admins_test.go b/server/services/permissions/admins_test.go similarity index 100% rename from server/plugins/permissions/admins_test.go rename to server/services/permissions/admins_test.go diff --git a/server/plugins/permissions/orgs.go b/server/services/permissions/orgs.go similarity index 100% rename from server/plugins/permissions/orgs.go rename to server/services/permissions/orgs.go diff --git a/server/plugins/permissions/orgs_test.go b/server/services/permissions/orgs_test.go similarity index 100% rename from server/plugins/permissions/orgs_test.go rename to server/services/permissions/orgs_test.go diff --git a/server/plugins/permissions/repo_owners.go b/server/services/permissions/repo_owners.go similarity index 100% rename from server/plugins/permissions/repo_owners.go rename to server/services/permissions/repo_owners.go diff --git a/server/plugins/permissions/repo_owners_test.go b/server/services/permissions/repo_owners_test.go similarity index 100% rename from server/plugins/permissions/repo_owners_test.go rename to server/services/permissions/repo_owners_test.go diff --git a/server/plugins/registry/combine.go b/server/services/registry/combined.go similarity index 70% rename from server/plugins/registry/combine.go rename to server/services/registry/combined.go index d4f6c7e45c..4f52bca1a5 100644 --- a/server/plugins/registry/combine.go +++ b/server/services/registry/combined.go @@ -19,11 +19,11 @@ import ( ) type combined struct { - registries []model.ReadOnlyRegistryService - dbRegistry model.RegistryService + registries []ReadOnlyService + dbRegistry Service } -func Combined(dbRegistry model.RegistryService, registries ...model.ReadOnlyRegistryService) model.RegistryService { +func NewCombined(dbRegistry Service, registries ...ReadOnlyService) Service { registries = append(registries, dbRegistry) return &combined{ registries: registries, @@ -31,7 +31,7 @@ func Combined(dbRegistry model.RegistryService, registries ...model.ReadOnlyRegi } } -func (c combined) RegistryFind(repo *model.Repo, name string) (*model.Registry, error) { +func (c *combined) RegistryFind(repo *model.Repo, name string) (*model.Registry, error) { for _, registry := range c.registries { res, err := registry.RegistryFind(repo, name) if err != nil { @@ -44,7 +44,7 @@ func (c combined) RegistryFind(repo *model.Repo, name string) (*model.Registry, return nil, nil } -func (c combined) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { +func (c *combined) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { var registries []*model.Registry for _, registry := range c.registries { list, err := registry.RegistryList(repo, &model.ListOptions{All: true}) @@ -56,14 +56,14 @@ func (c combined) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model return model.ApplyPagination(p, registries), nil } -func (c combined) RegistryCreate(repo *model.Repo, registry *model.Registry) error { +func (c *combined) RegistryCreate(repo *model.Repo, registry *model.Registry) error { return c.dbRegistry.RegistryCreate(repo, registry) } -func (c combined) RegistryUpdate(repo *model.Repo, registry *model.Registry) error { +func (c *combined) RegistryUpdate(repo *model.Repo, registry *model.Registry) error { return c.dbRegistry.RegistryUpdate(repo, registry) } -func (c combined) RegistryDelete(repo *model.Repo, name string) error { +func (c *combined) RegistryDelete(repo *model.Repo, name string) error { return c.dbRegistry.RegistryDelete(repo, name) } diff --git a/server/plugins/registry/db.go b/server/services/registry/db.go similarity index 60% rename from server/plugins/registry/db.go rename to server/services/registry/db.go index d36b3aa3e5..5e7339557f 100644 --- a/server/plugins/registry/db.go +++ b/server/services/registry/db.go @@ -23,26 +23,26 @@ type db struct { } // New returns a new local registry service. -func New(store model.RegistryStore) model.RegistryService { +func NewDB(store model.RegistryStore) Service { return &db{store} } -func (b *db) RegistryFind(repo *model.Repo, name string) (*model.Registry, error) { - return b.store.RegistryFind(repo, name) +func (d *db) RegistryFind(repo *model.Repo, name string) (*model.Registry, error) { + return d.store.RegistryFind(repo, name) } -func (b *db) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { - return b.store.RegistryList(repo, p) +func (d *db) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { + return d.store.RegistryList(repo, p) } -func (b *db) RegistryCreate(_ *model.Repo, in *model.Registry) error { - return b.store.RegistryCreate(in) +func (d *db) RegistryCreate(_ *model.Repo, in *model.Registry) error { + return d.store.RegistryCreate(in) } -func (b *db) RegistryUpdate(_ *model.Repo, in *model.Registry) error { - return b.store.RegistryUpdate(in) +func (d *db) RegistryUpdate(_ *model.Repo, in *model.Registry) error { + return d.store.RegistryUpdate(in) } -func (b *db) RegistryDelete(repo *model.Repo, addr string) error { - return b.store.RegistryDelete(repo, addr) +func (d *db) RegistryDelete(repo *model.Repo, addr string) error { + return d.store.RegistryDelete(repo, addr) } diff --git a/server/plugins/registry/filesystem.go b/server/services/registry/filesystem.go similarity index 92% rename from server/plugins/registry/filesystem.go rename to server/services/registry/filesystem.go index db9e24a0d3..4d98818cc0 100644 --- a/server/plugins/registry/filesystem.go +++ b/server/services/registry/filesystem.go @@ -31,7 +31,7 @@ type filesystem struct { path string } -func Filesystem(path string) model.ReadOnlyRegistryService { +func NewFilesystem(path string) ReadOnlyService { return &filesystem{path} } @@ -85,12 +85,12 @@ func parseDockerConfig(path string) ([]*model.Registry, error) { return auths, nil } -func (b *filesystem) RegistryFind(*model.Repo, string) (*model.Registry, error) { +func (f *filesystem) RegistryFind(*model.Repo, string) (*model.Registry, error) { return nil, nil } -func (b *filesystem) RegistryList(_ *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { - regs, err := parseDockerConfig(b.path) +func (f *filesystem) RegistryList(_ *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { + regs, err := parseDockerConfig(f.path) if err != nil { return nil, err } diff --git a/server/services/registry/service.go b/server/services/registry/service.go new file mode 100644 index 0000000000..252ce9d9a9 --- /dev/null +++ b/server/services/registry/service.go @@ -0,0 +1,32 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import "go.woodpecker-ci.org/woodpecker/v2/server/model" + +// Service defines a service for managing registries. +type Service interface { + RegistryFind(*model.Repo, string) (*model.Registry, error) + RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error) + RegistryCreate(*model.Repo, *model.Registry) error + RegistryUpdate(*model.Repo, *model.Registry) error + RegistryDelete(*model.Repo, string) error +} + +// ReadOnlyService defines a service for managing registries. +type ReadOnlyService interface { + RegistryFind(*model.Repo, string) (*model.Registry, error) + RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error) +} diff --git a/server/services/secret/db.go b/server/services/secret/db.go new file mode 100644 index 0000000000..b3d3202618 --- /dev/null +++ b/server/services/secret/db.go @@ -0,0 +1,133 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secret + +import ( + "go.woodpecker-ci.org/woodpecker/v2/server/model" +) + +type db struct { + store model.SecretStore +} + +// NewDB returns a new local secret service. +func NewDB(store model.SecretStore) Service { + return &db{store: store} +} + +func (d *db) SecretFind(repo *model.Repo, name string) (*model.Secret, error) { + return d.store.SecretFind(repo, name) +} + +func (d *db) SecretList(repo *model.Repo, p *model.ListOptions) ([]*model.Secret, error) { + return d.store.SecretList(repo, false, p) +} + +func (d *db) SecretListPipeline(repo *model.Repo, _ *model.Pipeline, p *model.ListOptions) ([]*model.Secret, error) { + s, err := d.store.SecretList(repo, true, p) + if err != nil { + return nil, err + } + + // Return only secrets with unique name + // Priority order in case of duplicate names are repository, user/organization, global + secrets := make([]*model.Secret, 0, len(s)) + uniq := make(map[string]struct{}) + for _, condition := range []struct { + IsRepository bool + IsOrganization bool + IsGlobal bool + }{ + {IsRepository: true}, + {IsOrganization: true}, + {IsGlobal: true}, + } { + for _, secret := range s { + if secret.IsRepository() != condition.IsRepository || secret.IsOrganization() != condition.IsOrganization || secret.IsGlobal() != condition.IsGlobal { + continue + } + if _, ok := uniq[secret.Name]; ok { + continue + } + uniq[secret.Name] = struct{}{} + secrets = append(secrets, secret) + } + } + return secrets, nil +} + +func (d *db) SecretCreate(_ *model.Repo, in *model.Secret) error { + return d.store.SecretCreate(in) +} + +func (d *db) SecretUpdate(_ *model.Repo, in *model.Secret) error { + return d.store.SecretUpdate(in) +} + +func (d *db) SecretDelete(repo *model.Repo, name string) error { + secret, err := d.store.SecretFind(repo, name) + if err != nil { + return err + } + return d.store.SecretDelete(secret) +} + +func (d *db) OrgSecretFind(owner int64, name string) (*model.Secret, error) { + return d.store.OrgSecretFind(owner, name) +} + +func (d *db) OrgSecretList(owner int64, p *model.ListOptions) ([]*model.Secret, error) { + return d.store.OrgSecretList(owner, p) +} + +func (d *db) OrgSecretCreate(_ int64, in *model.Secret) error { + return d.store.SecretCreate(in) +} + +func (d *db) OrgSecretUpdate(_ int64, in *model.Secret) error { + return d.store.SecretUpdate(in) +} + +func (d *db) OrgSecretDelete(owner int64, name string) error { + secret, err := d.store.OrgSecretFind(owner, name) + if err != nil { + return err + } + return d.store.SecretDelete(secret) +} + +func (d *db) GlobalSecretFind(owner string) (*model.Secret, error) { + return d.store.GlobalSecretFind(owner) +} + +func (d *db) GlobalSecretList(p *model.ListOptions) ([]*model.Secret, error) { + return d.store.GlobalSecretList(p) +} + +func (d *db) GlobalSecretCreate(in *model.Secret) error { + return d.store.SecretCreate(in) +} + +func (d *db) GlobalSecretUpdate(in *model.Secret) error { + return d.store.SecretUpdate(in) +} + +func (d *db) GlobalSecretDelete(name string) error { + secret, err := d.store.GlobalSecretFind(name) + if err != nil { + return err + } + return d.store.SecretDelete(secret) +} diff --git a/server/plugins/secrets/builtin_test.go b/server/services/secret/db_test.go similarity index 82% rename from server/plugins/secrets/builtin_test.go rename to server/services/secret/db_test.go index 6c35aebfd7..6f6099f57f 100644 --- a/server/plugins/secrets/builtin_test.go +++ b/server/services/secret/db_test.go @@ -12,23 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package secrets_test +package secret_test import ( - "context" "testing" "github.com/franela/goblin" "github.com/stretchr/testify/mock" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secrets" + "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" mocks_store "go.woodpecker-ci.org/woodpecker/v2/server/store/mocks" ) func TestSecretListPipeline(t *testing.T) { g := goblin.Goblin(t) - ctx := context.Background() mockStore := mocks_store.NewStore(t) // global secret @@ -66,7 +64,7 @@ func TestSecretListPipeline(t *testing.T) { repoSecret, }, nil) - s, err := secrets.New(ctx, mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) + s, err := secret.NewDB(mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) g.Assert(err).IsNil() g.Assert(len(s)).Equal(1) @@ -79,7 +77,7 @@ func TestSecretListPipeline(t *testing.T) { orgSecret, }, nil) - s, err := secrets.New(ctx, mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) + s, err := secret.NewDB(mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) g.Assert(err).IsNil() g.Assert(len(s)).Equal(1) @@ -91,7 +89,7 @@ func TestSecretListPipeline(t *testing.T) { globalSecret, }, nil) - s, err := secrets.New(ctx, mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) + s, err := secret.NewDB(mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) g.Assert(err).IsNil() g.Assert(len(s)).Equal(1) diff --git a/server/services/secret/service.go b/server/services/secret/service.go new file mode 100644 index 0000000000..24568c195b --- /dev/null +++ b/server/services/secret/service.go @@ -0,0 +1,40 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secret + +import "go.woodpecker-ci.org/woodpecker/v2/server/model" + +// Service defines a service for managing secrets. +type Service interface { + SecretListPipeline(*model.Repo, *model.Pipeline, *model.ListOptions) ([]*model.Secret, error) + // Repository secrets + SecretFind(*model.Repo, string) (*model.Secret, error) + SecretList(*model.Repo, *model.ListOptions) ([]*model.Secret, error) + SecretCreate(*model.Repo, *model.Secret) error + SecretUpdate(*model.Repo, *model.Secret) error + SecretDelete(*model.Repo, string) error + // Organization secrets + OrgSecretFind(int64, string) (*model.Secret, error) + OrgSecretList(int64, *model.ListOptions) ([]*model.Secret, error) + OrgSecretCreate(int64, *model.Secret) error + OrgSecretUpdate(int64, *model.Secret) error + OrgSecretDelete(int64, string) error + // Global secrets + GlobalSecretFind(string) (*model.Secret, error) + GlobalSecretList(*model.ListOptions) ([]*model.Secret, error) + GlobalSecretCreate(*model.Secret) error + GlobalSecretUpdate(*model.Secret) error + GlobalSecretDelete(string) error +} diff --git a/server/services/setup.go b/server/services/setup.go new file mode 100644 index 0000000000..b6ca4506fa --- /dev/null +++ b/server/services/setup.go @@ -0,0 +1,95 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package services + +import ( + "crypto" + "crypto/ed25519" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" + + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" + "go.woodpecker-ci.org/woodpecker/v2/server/services/registry" + "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/store" + "go.woodpecker-ci.org/woodpecker/v2/server/store/types" +) + +func setupRegistryService(store store.Store, dockerConfig string) registry.Service { + if dockerConfig != "" { + return registry.NewCombined( + registry.NewDB(store), + registry.NewFilesystem(dockerConfig), + ) + } + + return registry.NewDB(store) +} + +func setupSecretService(store store.Store) secret.Service { + // TODO(1544): fix encrypted store + // // encryption + // encryptedSecretStore := encryptedStore.NewSecretStore(v) + // err := encryption.Encryption(c, v).WithClient(encryptedSecretStore).Build() + // if err != nil { + // log.Fatal().Err(err).Msg("could not create encryption service") + // } + + return secret.NewDB(store) +} + +func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) config.Service { + timeout := c.Duration("forge-timeout") + configFetcher := config.NewForge(timeout) + + if endpoint := c.String("config-extension-endpoint"); endpoint != "" { + httpFetcher := config.NewHTTP(endpoint, privateSignatureKey) + return config.NewCombined(configFetcher, httpFetcher) + } + + return configFetcher +} + +// setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for service extensions) +func setupSignatureKeys(_store store.Store) (crypto.PrivateKey, crypto.PublicKey, error) { + privKeyID := "signature-private-key" + + privKey, err := _store.ServerConfigGet(privKeyID) + if errors.Is(err, types.RecordNotExist) { + _, privKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate private key: %w", err) + } + err = _store.ServerConfigSet(privKeyID, hex.EncodeToString(privKey)) + if err != nil { + return nil, nil, fmt.Errorf("failed to store private key: %w", err) + } + log.Debug().Msg("created private key") + return privKey, privKey.Public(), nil + } else if err != nil { + return nil, nil, fmt.Errorf("failed to load private key: %w", err) + } + privKeyStr, err := hex.DecodeString(privKey) + if err != nil { + return nil, nil, fmt.Errorf("failed to decode private key: %w", err) + } + privateKey := ed25519.PrivateKey(privKeyStr) + return privateKey, privateKey.Public(), nil +} diff --git a/server/plugins/utils/http.go b/server/services/utils/http.go similarity index 100% rename from server/plugins/utils/http.go rename to server/services/utils/http.go diff --git a/server/plugins/utils/http_test.go b/server/services/utils/http_test.go similarity index 96% rename from server/plugins/utils/http_test.go rename to server/services/utils/http_test.go index 723a20a6a4..272be5203d 100644 --- a/server/plugins/utils/http_test.go +++ b/server/services/utils/http_test.go @@ -25,7 +25,7 @@ import ( "github.com/go-ap/httpsig" "github.com/stretchr/testify/assert" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/utils" + "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" ) func TestSign(t *testing.T) { diff --git a/shared/addon/types/types.go b/shared/addon/types/types.go index 3287efc575..96331d280c 100644 --- a/shared/addon/types/types.go +++ b/shared/addon/types/types.go @@ -3,9 +3,5 @@ package types type Type string const ( - TypeForge Type = "forge" - TypeConfigService Type = "config_service" - TypeSecretService Type = "secret_service" - TypeEnvironmentService Type = "environment_service" - TypeRegistryService Type = "registry_service" + TypeForge Type = "forge" )