From 5345947268444f6d77509c9d9b60507be95e0be3 Mon Sep 17 00:00:00 2001 From: Southclaws Date: Fri, 20 Mar 2020 18:21:16 +0000 Subject: [PATCH 1/7] fix #30 --- main.go | 4 ++-- service/service.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 8270f1c..ec360e9 100644 --- a/main.go +++ b/main.go @@ -40,7 +40,7 @@ this repository has new commits, Pico will automatically reconfigure.`, Flags: []cli.Flag{ cli.StringFlag{Name: "hostname", EnvVar: "HOSTNAME"}, cli.StringFlag{Name: "directory", EnvVar: "DIRECTORY", Value: "./cache/"}, - cli.BoolFlag{Name: "no-ssh", EnvVar: "NO_SSH"}, + cli.BoolFlag{Name: "ssh", EnvVar: "SSH"}, cli.DurationFlag{Name: "check-interval", EnvVar: "CHECK_INTERVAL", Value: time.Second * 10}, cli.StringFlag{Name: "vault-addr", EnvVar: "VAULT_ADDR"}, cli.StringFlag{Name: "vault-token", EnvVar: "VAULT_TOKEN"}, @@ -71,7 +71,7 @@ this repository has new commits, Pico will automatically reconfigure.`, Target: c.Args().First(), Hostname: hostname, Directory: c.String("directory"), - NoSSH: c.Bool("no-ssh"), + SSH: c.Bool("ssh"), CheckInterval: c.Duration("check-interval"), VaultAddress: c.String("vault-addr"), VaultToken: c.String("vault-token"), diff --git a/service/service.go b/service/service.go index c69650e..355d870 100644 --- a/service/service.go +++ b/service/service.go @@ -27,7 +27,7 @@ import ( type Config struct { Target string Hostname string - NoSSH bool + SSH bool Directory string CheckInterval time.Duration VaultAddress string @@ -52,7 +52,7 @@ func Initialise(c Config) (app *App, err error) { app.config = c var authMethod transport.AuthMethod - if !c.NoSSH { + if c.SSH { authMethod, err = ssh.NewSSHAgentAuth("git") if err != nil { return nil, errors.Wrap(err, "failed to set up SSH authentication") From d8ddf065263bc205986f60bae0abafc00ee8bf7c Mon Sep 17 00:00:00 2001 From: Southclaws Date: Fri, 20 Mar 2020 19:07:18 +0000 Subject: [PATCH 2/7] resolve #22 --- main.go | 9 ++++++++- reconfigurer/git.go | 8 ++++---- service/service.go | 10 ++++++++-- task/target.go | 7 +++++++ 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index ec360e9..5e96276 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( _ "github.com/picostack/pico/logger" "github.com/picostack/pico/service" + "github.com/picostack/pico/task" ) var version = "master" @@ -38,6 +39,8 @@ this repository has new commits, Pico will automatically reconfigure.`, Usage: "argument `target` specifies Git repository for configuration.", ArgsUsage: "target", Flags: []cli.Flag{ + cli.StringFlag{Name: "git-username", EnvVar: "GIT_USERNAME"}, + cli.StringFlag{Name: "git-password", EnvVar: "GIT_PASSWORD"}, cli.StringFlag{Name: "hostname", EnvVar: "HOSTNAME"}, cli.StringFlag{Name: "directory", EnvVar: "DIRECTORY", Value: "./cache/"}, cli.BoolFlag{Name: "ssh", EnvVar: "SSH"}, @@ -68,7 +71,11 @@ this repository has new commits, Pico will automatically reconfigure.`, zap.L().Debug("initialising service") svc, err := service.Initialise(service.Config{ - Target: c.Args().First(), + Target: task.Repo{ + URL: c.Args().First(), + User: c.String("git-username"), + Pass: c.String("git-password"), + }, Hostname: hostname, Directory: c.String("directory"), SSH: c.Bool("ssh"), diff --git a/reconfigurer/git.go b/reconfigurer/git.go index 62b50da..b8ef296 100644 --- a/reconfigurer/git.go +++ b/reconfigurer/git.go @@ -23,7 +23,7 @@ type GitProvider struct { hostname string configRepo string checkInterval time.Duration - ssh transport.AuthMethod + authMethod transport.AuthMethod configWatcher *gitwatch.Session } @@ -34,14 +34,14 @@ func New( hostname string, configRepo string, checkInterval time.Duration, - ssh transport.AuthMethod, + authMethod transport.AuthMethod, ) *GitProvider { return &GitProvider{ directory: directory, hostname: hostname, configRepo: configRepo, checkInterval: checkInterval, - ssh: ssh, + authMethod: authMethod, } } @@ -104,7 +104,7 @@ func (p *GitProvider) watchConfig() (err error) { []gitwatch.Repository{{URL: p.configRepo}}, p.checkInterval, p.directory, - p.ssh, + p.authMethod, false) if err != nil { return errors.Wrap(err, "failed to watch config target") diff --git a/service/service.go b/service/service.go index 355d870..20b0f3e 100644 --- a/service/service.go +++ b/service/service.go @@ -12,6 +12,7 @@ import ( "go.uber.org/zap" "golang.org/x/sync/errgroup" "gopkg.in/src-d/go-git.v4/plumbing/transport" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" "gopkg.in/src-d/go-git.v4/plumbing/transport/ssh" "github.com/picostack/pico/executor" @@ -25,7 +26,7 @@ import ( // Config specifies static configuration parameters (from CLI or environment) type Config struct { - Target string + Target task.Repo Hostname string SSH bool Directory string @@ -57,6 +58,11 @@ func Initialise(c Config) (app *App, err error) { if err != nil { return nil, errors.Wrap(err, "failed to set up SSH authentication") } + } else if c.Target.User != "" { + authMethod = &http.BasicAuth{ + Username: c.Target.User, + Password: c.Target.Pass, + } } var secretStore secret.Store @@ -85,7 +91,7 @@ func Initialise(c Config) (app *App, err error) { app.reconfigurer = reconfigurer.New( c.Directory, c.Hostname, - c.Target, + c.Target.URL, c.CheckInterval, authMethod, ) diff --git a/task/target.go b/task/target.go index d5ea9f1..474321b 100644 --- a/task/target.go +++ b/task/target.go @@ -15,6 +15,13 @@ type ExecutionTask struct { Env map[string]string } +// Repo represents a Git repo with credentials +type Repo struct { + URL string + User string + Pass string +} + // Targets is just a list of target objects, to implement the Sort interface type Targets []Target From 6fdd81297807895a304e8f16eef69fbb5fd444af Mon Sep 17 00:00:00 2001 From: Southclaws Date: Mon, 23 Mar 2020 16:09:01 +0000 Subject: [PATCH 3/7] run tests for staging PRs --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a6fc9f8..2dc00a2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: push: branches: [master] pull_request: - branches: [master] + branches: [master, staging] jobs: test: From 42d7c8d20ab9f4ecc7305980b0964d4467885e36 Mon Sep 17 00:00:00 2001 From: Barnaby Keene Date: Mon, 23 Mar 2020 16:17:34 +0000 Subject: [PATCH 4/7] resolve #24 (#47) --- main.go | 2 ++ service/service.go | 62 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/main.go b/main.go index 5e96276..0de84fa 100644 --- a/main.go +++ b/main.go @@ -49,6 +49,7 @@ this repository has new commits, Pico will automatically reconfigure.`, cli.StringFlag{Name: "vault-token", EnvVar: "VAULT_TOKEN"}, cli.StringFlag{Name: "vault-path", EnvVar: "VAULT_PATH", Value: "/secret"}, cli.DurationFlag{Name: "vault-renew-interval", EnvVar: "VAULT_RENEW_INTERVAL", Value: time.Hour * 24}, + cli.StringFlag{Name: "vault-config-path", EnvVar: "VAULT_CONFIG_PATH", Value: "pico"}, }, Action: func(c *cli.Context) (err error) { if !c.Args().Present() { @@ -84,6 +85,7 @@ this repository has new commits, Pico will automatically reconfigure.`, VaultToken: c.String("vault-token"), VaultPath: c.String("vault-path"), VaultRenewal: c.Duration("vault-renew-interval"), + VaultConfig: c.String("vault-config-path"), }) if err != nil { return errors.Wrap(err, "failed to initialise") diff --git a/service/service.go b/service/service.go index 20b0f3e..2db7c54 100644 --- a/service/service.go +++ b/service/service.go @@ -35,6 +35,7 @@ type Config struct { VaultToken string VaultPath string VaultRenewal time.Duration + VaultConfig string } // App stores application state @@ -52,19 +53,6 @@ func Initialise(c Config) (app *App, err error) { app.config = c - var authMethod transport.AuthMethod - if c.SSH { - authMethod, err = ssh.NewSSHAgentAuth("git") - if err != nil { - return nil, errors.Wrap(err, "failed to set up SSH authentication") - } - } else if c.Target.User != "" { - authMethod = &http.BasicAuth{ - Username: c.Target.User, - Password: c.Target.Pass, - } - } - var secretStore secret.Store if c.VaultAddress != "" { zap.L().Debug("connecting to vault", @@ -83,6 +71,18 @@ func Initialise(c Config) (app *App, err error) { } } + secretConfig, err := secretStore.GetSecretsForTarget(c.VaultConfig) + if err != nil { + zap.L().Info("could not read additional config from vault", zap.String("path", c.VaultConfig)) + err = nil + } + zap.L().Debug("read configuration secrets from secret store", zap.Strings("keys", getKeys(secretConfig))) + + authMethod, err := getAuthMethod(c, secretConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to create an authentication method from the given config") + } + app.secrets = secretStore app.bus = make(chan task.ExecutionTask, 100) @@ -143,3 +143,39 @@ func (app *App) Start(ctx context.Context) error { return g.Wait() } + +func getAuthMethod(c Config, secretConfig map[string]string) (transport.AuthMethod, error) { + if c.SSH { + authMethod, err := ssh.NewSSHAgentAuth("git") + if err != nil { + return nil, errors.Wrap(err, "failed to set up SSH authentication") + } + return authMethod, nil + } + + if c.Target.User != "" && c.Target.Pass != "" { + return &http.BasicAuth{ + Username: c.Target.User, + Password: c.Target.Pass, + }, nil + } + + user, userok := secretConfig["GIT_USERNAME"] + pass, passok := secretConfig["GIT_PASSWORD"] + if userok && passok { + return &http.BasicAuth{ + Username: user, + Password: pass, + }, nil + } + + return nil, nil +} + +func getKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + return keys +} From c97044b84f7225dec1af4f22fd63da56f19d500d Mon Sep 17 00:00:00 2001 From: Barnaby Keene Date: Tue, 24 Mar 2020 23:26:30 +0000 Subject: [PATCH 5/7] Vault arbitrary env passthrough (#52) * resolve #46 Adds a new option: pass-env, which when true will pass the pico process environment to children. Defaults to false to promote separation of environments. Adds support for passing prefixed variables from the global Pico. The prefix is GLOBAL_ and is not configurable because I felt the config flags are growing. Adds some better unit tests for execution config and environment merging. * correctly strip the prefix for global configuration values pulled from the secret store --- executor/cmd.go | 67 +++++++++++++++++++++++++++++++++-------- executor/cmd_test.go | 61 +++++++++++++++++++++++++++++++++---- main.go | 20 ++++++------ secret/memory/memory.go | 8 +++-- secret/secret.go | 17 +++++++++++ service/service.go | 23 +++++++------- task/target.go | 14 ++++++--- 7 files changed, 165 insertions(+), 45 deletions(-) diff --git a/executor/cmd.go b/executor/cmd.go index f740bb1..4da8bf4 100644 --- a/executor/cmd.go +++ b/executor/cmd.go @@ -12,13 +12,24 @@ var _ Executor = &CommandExecutor{} // CommandExecutor handles command invocation targets type CommandExecutor struct { - secrets secret.Store + secrets secret.Store + passEnvironment bool // pass the Pico process environment to children + configSecretPath string // path to global secrets to pass to children + configSecretPrefix string // only pass secrets with this prefix, usually GLOBAL_ } // NewCommandExecutor creates a new CommandExecutor -func NewCommandExecutor(secrets secret.Store) CommandExecutor { +func NewCommandExecutor( + secrets secret.Store, + passEnvironment bool, + configSecretPath string, + configSecretPrefix string, +) CommandExecutor { return CommandExecutor{ - secrets: secrets, + secrets: secrets, + passEnvironment: passEnvironment, + configSecretPath: configSecretPath, + configSecretPrefix: configSecretPrefix, } } @@ -34,20 +45,38 @@ func (e *CommandExecutor) Subscribe(bus chan task.ExecutionTask) { } } -func (e *CommandExecutor) execute( - target task.Target, +type exec struct { + path string + env map[string]string + shutdown bool + passEnvironment bool +} + +func (e *CommandExecutor) prepare( + name string, path string, shutdown bool, execEnv map[string]string, -) (err error) { - secrets, err := e.secrets.GetSecretsForTarget(target.Name) +) (exec, error) { + // get global secrets from the Pico config path in the secret store. + // only secrets with the prefix are retrieved. + global, err := secret.GetPrefixedSecrets(e.secrets, e.configSecretPath, e.configSecretPrefix) if err != nil { - return errors.Wrap(err, "failed to get secrets for target") + return exec{}, errors.Wrap(err, "failed to get global secrets for target") + } + + secrets, err := e.secrets.GetSecretsForTarget(name) + if err != nil { + return exec{}, errors.Wrap(err, "failed to get secrets for target") } env := make(map[string]string) - // merge execution environment with secrets + // merge execution environment with secrets in the following order: + // globals first, then execution environment, then per-target secrets + for k, v := range global { + env[k] = v + } for k, v := range execEnv { env[k] = v } @@ -55,13 +84,27 @@ func (e *CommandExecutor) execute( env[k] = v } + return exec{path, env, shutdown, e.passEnvironment}, nil +} + +func (e *CommandExecutor) execute( + target task.Target, + path string, + shutdown bool, + execEnv map[string]string, +) (err error) { + ex, err := e.prepare(target.Name, path, shutdown, execEnv) + if err != nil { + return err + } + zap.L().Debug("executing with secrets", zap.String("target", target.Name), zap.Strings("cmd", target.Up), zap.String("url", target.RepoURL), zap.String("dir", path), - zap.Int("env", len(env)), - zap.Int("secrets", len(secrets))) + zap.Int("env", len(ex.env)), + zap.Bool("passthrough", e.passEnvironment)) - return target.Execute(path, env, shutdown) + return target.Execute(ex.path, ex.env, ex.shutdown, ex.passEnvironment) } diff --git a/executor/cmd_test.go b/executor/cmd_test.go index fe00db2..db8661a 100644 --- a/executor/cmd_test.go +++ b/executor/cmd_test.go @@ -1,4 +1,4 @@ -package executor_test +package executor import ( "os" @@ -7,9 +7,9 @@ import ( "golang.org/x/sync/errgroup" - "github.com/picostack/pico/executor" "github.com/picostack/pico/secret/memory" "github.com/picostack/pico/task" + "github.com/stretchr/testify/assert" _ "github.com/picostack/pico/logger" ) @@ -20,11 +20,13 @@ func TestMain(m *testing.M) { } func TestCommandExecutor(t *testing.T) { - ce := executor.NewCommandExecutor(&memory.MemorySecrets{ - Secrets: map[string]string{ - "SOME_SECRET": "123", + ce := NewCommandExecutor(&memory.MemorySecrets{ + Secrets: map[string]map[string]string{ + "test": map[string]string{ + "SOME_SECRET": "123", + }, }, - }) + }, false, "pico", "GLOBAL_") bus := make(chan task.ExecutionTask) g := errgroup.Group{} @@ -55,3 +57,50 @@ func TestCommandExecutor(t *testing.T) { os.RemoveAll(".test/.git") } + +func TestCommandPrepareWithoutPassthrough(t *testing.T) { + ce := NewCommandExecutor(&memory.MemorySecrets{ + Secrets: map[string]map[string]string{ + "test": map[string]string{ + "SOME_SECRET": "123", + }, + }, + }, false, "pico", "GLOBAL_") + + ex, err := ce.prepare("test", "./", false, nil) + assert.NoError(t, err) + assert.Equal(t, exec{ + path: "./", + env: map[string]string{ + "SOME_SECRET": "123", + }, + shutdown: false, + passEnvironment: false, + }, ex) +} + +func TestCommandPrepareWithGlobal(t *testing.T) { + ce := NewCommandExecutor(&memory.MemorySecrets{ + Secrets: map[string]map[string]string{ + "test": map[string]string{ + "SOME_SECRET": "123", + }, + "pico": map[string]string{ + "GLOBAL_SECRET": "456", + "IGNORE": "this", + }, + }, + }, false, "pico", "GLOBAL_") + + ex, err := ce.prepare("test", "./", false, nil) + assert.NoError(t, err) + assert.Equal(t, exec{ + path: "./", + env: map[string]string{ + "SOME_SECRET": "123", + "SECRET": "456", + }, + shutdown: false, + passEnvironment: false, + }, ex) +} diff --git a/main.go b/main.go index 0de84fa..e88cc11 100644 --- a/main.go +++ b/main.go @@ -43,6 +43,7 @@ this repository has new commits, Pico will automatically reconfigure.`, cli.StringFlag{Name: "git-password", EnvVar: "GIT_PASSWORD"}, cli.StringFlag{Name: "hostname", EnvVar: "HOSTNAME"}, cli.StringFlag{Name: "directory", EnvVar: "DIRECTORY", Value: "./cache/"}, + cli.DurationFlag{Name: "pass-env", EnvVar: "PASS_ENV"}, cli.BoolFlag{Name: "ssh", EnvVar: "SSH"}, cli.DurationFlag{Name: "check-interval", EnvVar: "CHECK_INTERVAL", Value: time.Second * 10}, cli.StringFlag{Name: "vault-addr", EnvVar: "VAULT_ADDR"}, @@ -77,15 +78,16 @@ this repository has new commits, Pico will automatically reconfigure.`, User: c.String("git-username"), Pass: c.String("git-password"), }, - Hostname: hostname, - Directory: c.String("directory"), - SSH: c.Bool("ssh"), - CheckInterval: c.Duration("check-interval"), - VaultAddress: c.String("vault-addr"), - VaultToken: c.String("vault-token"), - VaultPath: c.String("vault-path"), - VaultRenewal: c.Duration("vault-renew-interval"), - VaultConfig: c.String("vault-config-path"), + Hostname: hostname, + Directory: c.String("directory"), + PassEnvironment: c.Bool("pass-env"), + SSH: c.Bool("ssh"), + CheckInterval: c.Duration("check-interval"), + VaultAddress: c.String("vault-addr"), + VaultToken: c.String("vault-token"), + VaultPath: c.String("vault-path"), + VaultRenewal: c.Duration("vault-renew-interval"), + VaultConfig: c.String("vault-config-path"), }) if err != nil { return errors.Wrap(err, "failed to initialise") diff --git a/secret/memory/memory.go b/secret/memory/memory.go index c9050ee..b871a59 100644 --- a/secret/memory/memory.go +++ b/secret/memory/memory.go @@ -6,12 +6,16 @@ import ( // MemorySecrets implements a simple in-memory secret.Store for testing type MemorySecrets struct { - Secrets map[string]string + Secrets map[string]map[string]string } var _ secret.Store = &MemorySecrets{} // GetSecretsForTarget implements secret.Store func (v *MemorySecrets) GetSecretsForTarget(name string) (map[string]string, error) { - return v.Secrets, nil + table, ok := v.Secrets[name] + if !ok { + return nil, nil + } + return table, nil } diff --git a/secret/secret.go b/secret/secret.go index 22df301..6da4cf4 100644 --- a/secret/secret.go +++ b/secret/secret.go @@ -3,7 +3,24 @@ // any secrets that match it. package secret +import "strings" + // Store describes a type that can securely obtain secrets for services. type Store interface { GetSecretsForTarget(name string) (map[string]string, error) } + +// GetPrefixedSecrets uses a Store to get a set of secrets that use a prefix. +func GetPrefixedSecrets(s Store, path, prefix string) (map[string]string, error) { + all, err := s.GetSecretsForTarget(path) + if err != nil { + return nil, err + } + pass := make(map[string]string) + for k, v := range all { + if strings.HasPrefix(k, prefix) { + pass[strings.TrimPrefix(k, prefix)] = v + } + } + return pass, nil +} diff --git a/service/service.go b/service/service.go index 2db7c54..02c1be0 100644 --- a/service/service.go +++ b/service/service.go @@ -26,16 +26,17 @@ import ( // Config specifies static configuration parameters (from CLI or environment) type Config struct { - Target task.Repo - Hostname string - SSH bool - Directory string - CheckInterval time.Duration - VaultAddress string - VaultToken string - VaultPath string - VaultRenewal time.Duration - VaultConfig string + Target task.Repo + Hostname string + SSH bool + Directory string + PassEnvironment bool + CheckInterval time.Duration + VaultAddress string + VaultToken string + VaultPath string + VaultRenewal time.Duration + VaultConfig string } // App stores application state @@ -119,7 +120,7 @@ func (app *App) Start(ctx context.Context) error { // states and potentially retry in some circumstances. Pico should be the // kind of service that barely goes down, only when absolutely necessary. - ce := executor.NewCommandExecutor(app.secrets) + ce := executor.NewCommandExecutor(app.secrets, app.config.PassEnvironment, app.config.VaultConfig, "GLOBAL_") g.Go(func() error { ce.Subscribe(app.bus) return nil diff --git a/task/target.go b/task/target.go index 474321b..982eb05 100644 --- a/task/target.go +++ b/task/target.go @@ -52,7 +52,7 @@ type Target struct { // Execute runs the target's command in the specified directory with the // specified environment variables -func (t *Target) Execute(dir string, env map[string]string, shutdown bool) (err error) { +func (t *Target) Execute(dir string, env map[string]string, shutdown bool, inheritEnv bool) (err error) { if env == nil { env = make(map[string]string) } @@ -67,10 +67,10 @@ func (t *Target) Execute(dir string, env map[string]string, shutdown bool) (err command = t.Up } - return execute(dir, env, command) + return execute(dir, env, command, inheritEnv) } -func execute(dir string, env map[string]string, command []string) (err error) { +func execute(dir string, env map[string]string, command []string, inheritEnv bool) (err error) { if len(command) == 0 { return errors.New("attempt to execute target with empty command") } @@ -83,10 +83,14 @@ func execute(dir string, env map[string]string, command []string) (err error) { cmd.Stdout = os.Stdout cmd.Stderr = os.Stdout - cmd.Env = os.Environ() + var cmdEnv []string + if inheritEnv { + cmdEnv = os.Environ() + } for k, v := range env { - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + cmdEnv = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) } + cmd.Env = cmdEnv return cmd.Run() } From cb33c8d247a28f982453c1507f2938ecba273d9e Mon Sep 17 00:00:00 2001 From: Barnaby Keene Date: Tue, 24 Mar 2020 23:26:46 +0000 Subject: [PATCH 6/7] Per target auth secrets (#51) * started work on per-target auth secrets for #24 Adds an `A` configuration primitive that declares a set of options for getting Git credentials for targets. * implement per-target authentication --- config/config.go | 23 +++++++++++++++++++++-- go.mod | 2 +- go.sum | 2 ++ service/service.go | 2 +- task/target.go | 3 +++ watcher/git.go | 40 ++++++++++++++++++++++++++++++++++++---- 6 files changed, 64 insertions(+), 8 deletions(-) diff --git a/config/config.go b/config/config.go index dea4b75..30dd0c3 100644 --- a/config/config.go +++ b/config/config.go @@ -21,8 +21,17 @@ import ( // State represents a desired system state type State struct { - Targets task.Targets `json:"targets"` - Env map[string]string `json:"env"` + Targets task.Targets `json:"targets"` + AuthMethods []AuthMethod `json:"auths"` + Env map[string]string `json:"env"` +} + +// AuthMethod represents a method of authentication for a target +type AuthMethod struct { + Name string `json:"name"` // name of the auth method + Path string `json:"path"` // path within the secret store + UserKey string `json:"user_key"` // key for username + PassKey string `json:"pass_key"` // key for password } // ConfigFromDirectory searches a directory for configuration files and @@ -72,6 +81,7 @@ func (cb *configBuilder) construct(hostname string) (err error) { cb.vm.Run(`'use strict'; var STATE = { targets: [], + auths: [], env: {} }; @@ -90,6 +100,15 @@ function T(t) { function E(k, v) { STATE.env[k] = v } + +function A(a) { + if(a.name === undefined) { throw "auth name undefined"; } + if(a.path === undefined) { throw "auth path undefined"; } + if(a.user_key === undefined) { throw "auth user_key undefined"; } + if(a.pass_key === undefined) { throw "auth pass_key undefined"; } + + STATE.auths.push(a); +} `) cb.vm.Set("HOSTNAME", hostname) //nolint:errcheck diff --git a/go.mod b/go.mod index 1116440..10d7426 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/picostack/pico go 1.13 require ( - github.com/Southclaws/gitwatch v1.3.2 + github.com/Southclaws/gitwatch v1.3.3 github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/eapache/go-resiliency v1.2.0 github.com/frankban/quicktest v1.4.1 // indirect diff --git a/go.sum b/go.sum index 65687cb..f457427 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/Southclaws/gitwatch v1.3.1 h1:4XtiujsnxHKSKze3Tb5sWwTdBxSVW/JLbK54ruJ github.com/Southclaws/gitwatch v1.3.1/go.mod h1:xCudUiwWxkDYZ69cEhlTwAKIzbG1OpnA/s/pjPIW6gU= github.com/Southclaws/gitwatch v1.3.2 h1:zmt571n8ItXgkRJPyCFtFjcymvsFOGcm7JnHNpFDP+8= github.com/Southclaws/gitwatch v1.3.2/go.mod h1:xCudUiwWxkDYZ69cEhlTwAKIzbG1OpnA/s/pjPIW6gU= +github.com/Southclaws/gitwatch v1.3.3 h1:w5AI9IcMEVqb6cPyDjM9tvOI4r26m4UHAl5BVEvgKac= +github.com/Southclaws/gitwatch v1.3.3/go.mod h1:xCudUiwWxkDYZ69cEhlTwAKIzbG1OpnA/s/pjPIW6gU= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= diff --git a/service/service.go b/service/service.go index 02c1be0..5fa0504 100644 --- a/service/service.go +++ b/service/service.go @@ -102,7 +102,7 @@ func Initialise(c Config) (app *App, err error) { app.config.Directory, app.bus, app.config.CheckInterval, - authMethod, + secretStore, ) return diff --git a/task/target.go b/task/target.go index 982eb05..15d5585 100644 --- a/task/target.go +++ b/task/target.go @@ -48,6 +48,9 @@ type Target struct { // Whether or not to run `Command` on first run, useful if the command is `docker-compose up` InitialRun bool `json:"initial_run"` + + // Auth method to use from the auth store + Auth string `json:"auth"` } // Execute runs the target's command in the specified directory with the diff --git a/watcher/git.go b/watcher/git.go index 5ae001d..0da26a7 100644 --- a/watcher/git.go +++ b/watcher/git.go @@ -11,8 +11,10 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" "gopkg.in/src-d/go-git.v4/plumbing/transport" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" "github.com/picostack/pico/config" + "github.com/picostack/pico/secret" "github.com/picostack/pico/task" ) @@ -24,7 +26,7 @@ type GitWatcher struct { directory string bus chan task.ExecutionTask checkInterval time.Duration - ssh transport.AuthMethod + secrets secret.Store targetsWatcher *gitwatch.Session state config.State @@ -42,13 +44,13 @@ func NewGitWatcher( directory string, bus chan task.ExecutionTask, checkInterval time.Duration, - ssh transport.AuthMethod, + secrets secret.Store, ) *GitWatcher { return &GitWatcher{ directory: directory, bus: bus, checkInterval: checkInterval, - ssh: ssh, + secrets: secrets, initialise: make(chan bool), newState: make(chan config.State, 16), @@ -161,11 +163,16 @@ func (w *GitWatcher) watchTargets() (err error) { if t.Branch != "" { dir = fmt.Sprintf("%s_%s", t.Name, t.Branch) } + auth, err := w.getAuthForTarget(t) + if err != nil { + return err + } zap.L().Debug("assigned target", zap.String("url", t.RepoURL), zap.String("directory", dir)) targetRepos[i] = gitwatch.Repository{ URL: t.RepoURL, Branch: t.Branch, Directory: dir, + Auth: auth, } } @@ -177,7 +184,7 @@ func (w *GitWatcher) watchTargets() (err error) { targetRepos, w.checkInterval, w.directory, - w.ssh, + nil, false) if err != nil { return errors.Wrap(err, "failed to watch targets") @@ -211,6 +218,31 @@ func (w *GitWatcher) handle(e gitwatch.Event) (err error) { return nil } +func (w GitWatcher) getAuthForTarget(t task.Target) (transport.AuthMethod, error) { + for _, a := range w.state.AuthMethods { + if a.Name == t.Auth { + s, err := w.secrets.GetSecretsForTarget(a.Path) + if err != nil { + return nil, err + } + username, ok := s[a.UserKey] + if !ok { + return nil, errors.Errorf("auth object 'user_key' did not point to a valid element in the specified secret at '%s'", a.Path) + } + password, ok := s[a.PassKey] + if !ok { + return nil, errors.Errorf("auth object 'pass_key' did not point to a valid element in the specified secret at '%s'", a.Path) + } + zap.L().Debug("using auth method for target", zap.String("name", a.Name)) + return &http.BasicAuth{ + Username: username, + Password: password, + }, nil + } + } + return nil, nil +} + func (w GitWatcher) executeTargets(targets []task.Target, shutdown bool) { zap.L().Debug("executing all targets", zap.Bool("shutdown", shutdown), From 1c4462a1da55145a5d8e9da5dd81eb2270437d27 Mon Sep 17 00:00:00 2001 From: Barnaby Keene Date: Wed, 25 Mar 2020 00:05:32 +0000 Subject: [PATCH 7/7] fixed a bug with environment vars being passed to targets (#54) --- executor/cmd.go | 2 +- executor/cmd_test.go | 10 ++++++++-- task/target.go | 20 +++++++++++++------- task/target_test.go | 31 +++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 10 deletions(-) create mode 100644 task/target_test.go diff --git a/executor/cmd.go b/executor/cmd.go index 4da8bf4..9b81bf9 100644 --- a/executor/cmd.go +++ b/executor/cmd.go @@ -103,7 +103,7 @@ func (e *CommandExecutor) execute( zap.Strings("cmd", target.Up), zap.String("url", target.RepoURL), zap.String("dir", path), - zap.Int("env", len(ex.env)), + zap.Any("env", ex.env), zap.Bool("passthrough", e.passEnvironment)) return target.Execute(ex.path, ex.env, ex.shutdown, ex.passEnvironment) diff --git a/executor/cmd_test.go b/executor/cmd_test.go index db8661a..d9e77f7 100644 --- a/executor/cmd_test.go +++ b/executor/cmd_test.go @@ -67,12 +67,15 @@ func TestCommandPrepareWithoutPassthrough(t *testing.T) { }, }, false, "pico", "GLOBAL_") - ex, err := ce.prepare("test", "./", false, nil) + ex, err := ce.prepare("test", "./", false, map[string]string{ + "DATA_DIR": "/data/shared", + }) assert.NoError(t, err) assert.Equal(t, exec{ path: "./", env: map[string]string{ "SOME_SECRET": "123", + "DATA_DIR": "/data/shared", }, shutdown: false, passEnvironment: false, @@ -92,13 +95,16 @@ func TestCommandPrepareWithGlobal(t *testing.T) { }, }, false, "pico", "GLOBAL_") - ex, err := ce.prepare("test", "./", false, nil) + ex, err := ce.prepare("test", "./", false, map[string]string{ + "DATA_DIR": "/data/shared", + }) assert.NoError(t, err) assert.Equal(t, exec{ path: "./", env: map[string]string{ "SOME_SECRET": "123", "SECRET": "456", + "DATA_DIR": "/data/shared", }, shutdown: false, passEnvironment: false, diff --git a/task/target.go b/task/target.go index 15d5585..bfefe40 100644 --- a/task/target.go +++ b/task/target.go @@ -1,10 +1,11 @@ package task import ( - "errors" "fmt" "os" "os/exec" + + "github.com/pkg/errors" ) // ExecutionTask encodes a Target with additional execution-time information. @@ -70,15 +71,20 @@ func (t *Target) Execute(dir string, env map[string]string, shutdown bool, inher command = t.Up } - return execute(dir, env, command, inheritEnv) + c, err := prepare(dir, env, command, inheritEnv) + if err != nil { + return errors.Wrap(err, "failed to prepare command for execution") + } + + return c.Run() } -func execute(dir string, env map[string]string, command []string, inheritEnv bool) (err error) { +func prepare(dir string, env map[string]string, command []string, inheritEnv bool) (cmd *exec.Cmd, err error) { if len(command) == 0 { - return errors.New("attempt to execute target with empty command") + return nil, errors.New("attempt to execute target with empty command") } - cmd := exec.Command(command[0]) + cmd = exec.Command(command[0]) if len(command) > 1 { cmd.Args = append(cmd.Args, command[1:]...) } @@ -91,9 +97,9 @@ func execute(dir string, env map[string]string, command []string, inheritEnv boo cmdEnv = os.Environ() } for k, v := range env { - cmdEnv = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + cmdEnv = append(cmdEnv, fmt.Sprintf("%s=%s", k, v)) } cmd.Env = cmdEnv - return cmd.Run() + return cmd, nil } diff --git a/task/target_test.go b/task/target_test.go new file mode 100644 index 0000000..278b4ba --- /dev/null +++ b/task/target_test.go @@ -0,0 +1,31 @@ +package task + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPrepareTargetExecution(t *testing.T) { + c, err := prepare(".", map[string]string{ + "VAR_1": "one", + "VAR_2": "two", + "VAR_3": "three", + "VAR_4": "four", + }, []string{"docker-compose", "up", "-d"}, false) + assert.NoError(t, err) + + assert.Equal(t, []string{"docker-compose", "up", "-d"}, c.Args) + want := []string{ + "VAR_1=one", + "VAR_2=two", + "VAR_3=three", + "VAR_4=four", + } + got := c.Env + sort.Strings(want) + sort.Strings(got) + assert.Equal(t, want, got) + assert.Equal(t, ".", c.Dir) +}