diff --git a/api/jobs_test.go b/api/jobs_test.go index a0b214069eeb..ba227445ecba 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -275,6 +275,7 @@ func TestJobs_Canonicalize(t *testing.T) { EmbeddedTmpl: helper.StringToPtr("FOO=bar\n"), DestPath: helper.StringToPtr("local/file.env"), Envvars: helper.BoolToPtr(true), + VaultGrace: helper.TimeToPtr(3 * time.Second), }, }, }, @@ -389,6 +390,7 @@ func TestJobs_Canonicalize(t *testing.T) { LeftDelim: helper.StringToPtr("{{"), RightDelim: helper.StringToPtr("}}"), Envvars: helper.BoolToPtr(false), + VaultGrace: helper.TimeToPtr(5 * time.Minute), }, { SourcePath: helper.StringToPtr(""), @@ -401,6 +403,7 @@ func TestJobs_Canonicalize(t *testing.T) { LeftDelim: helper.StringToPtr("{{"), RightDelim: helper.StringToPtr("}}"), Envvars: helper.BoolToPtr(true), + VaultGrace: helper.TimeToPtr(3 * time.Second), }, }, }, diff --git a/api/tasks.go b/api/tasks.go index c859850c2a99..79ddbe6852f2 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -364,6 +364,7 @@ type Template struct { LeftDelim *string `mapstructure:"left_delimiter"` RightDelim *string `mapstructure:"right_delimiter"` Envvars *bool `mapstructure:"env"` + VaultGrace *time.Duration `mapstructure:"vault_grace"` } func (tmpl *Template) Canonicalize() { @@ -404,6 +405,9 @@ func (tmpl *Template) Canonicalize() { if tmpl.Envvars == nil { tmpl.Envvars = helper.BoolToPtr(false) } + if tmpl.VaultGrace == nil { + tmpl.VaultGrace = helper.TimeToPtr(5 * time.Minute) + } } type Vault struct { diff --git a/client/consul_template.go b/client/consul_template.go index 00b6020b3382..2a887c8770ff 100644 --- a/client/consul_template.go +++ b/client/consul_template.go @@ -17,6 +17,7 @@ import ( multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/env" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" ) @@ -341,11 +342,6 @@ func templateRunner(tmpls []*structs.Template, config *config.Config, return nil, nil, nil } - runnerConfig, err := runnerConfig(config, vaultToken) - if err != nil { - return nil, nil, err - } - // Parse the templates allowAbs := config.ReadBoolDefault(hostSrcOption, true) ctmplMapping, err := parseTemplateConfigs(tmpls, taskDir, taskEnv, allowAbs) @@ -353,13 +349,11 @@ func templateRunner(tmpls []*structs.Template, config *config.Config, return nil, nil, err } - // Set the config - flat := ctconf.TemplateConfigs(make([]*ctconf.TemplateConfig, 0, len(ctmplMapping))) - for ctmpl := range ctmplMapping { - local := ctmpl - flat = append(flat, &local) + // Create the runner configuration. + runnerConfig, err := newRunnerConfig(config, vaultToken, ctmplMapping) + if err != nil { + return nil, nil, err } - runnerConfig.Templates = &flat runner, err := manager.NewRunner(runnerConfig, false, false) if err != nil { @@ -429,12 +423,33 @@ func parseTemplateConfigs(tmpls []*structs.Template, taskDir string, return ctmpls, nil } -// runnerConfig returns a consul-template runner configuration, setting the -// Vault and Consul configurations based on the clients configs. -func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, error) { +// newRunnerConfig returns a consul-template runner configuration, setting the +// Vault and Consul configurations based on the clients configs. The parameters +// are the client config, Vault token if set and the mapping of consul-templates +// to Nomad templates. +func newRunnerConfig(config *config.Config, vaultToken string, + templateMapping map[ctconf.TemplateConfig]*structs.Template) (*ctconf.Config, error) { + conf := ctconf.DefaultConfig() - t, f := true, false + // Gather the consul-template tempates + flat := ctconf.TemplateConfigs(make([]*ctconf.TemplateConfig, 0, len(templateMapping))) + for ctmpl := range templateMapping { + local := ctmpl + flat = append(flat, &local) + } + conf.Templates = &flat + + // Go through the templates and determine the minimum Vault grace + vaultGrace := time.Duration(-1) + for _, tmpl := range templateMapping { + // Initial condition + if vaultGrace < 0 { + vaultGrace = tmpl.VaultGrace + } else if tmpl.VaultGrace < vaultGrace { + vaultGrace = tmpl.VaultGrace + } + } // Force faster retries if testRetryRate != 0 { @@ -450,7 +465,7 @@ func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, err if config.ConsulConfig.EnableSSL != nil && *config.ConsulConfig.EnableSSL { verify := config.ConsulConfig.VerifySSL != nil && *config.ConsulConfig.VerifySSL conf.Consul.SSL = &ctconf.SSLConfig{ - Enabled: &t, + Enabled: helper.BoolToPtr(true), Verify: &verify, Cert: &config.ConsulConfig.CertFile, Key: &config.ConsulConfig.KeyFile, @@ -465,7 +480,7 @@ func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, err } conf.Consul.Auth = &ctconf.AuthConfig{ - Enabled: &t, + Enabled: helper.BoolToPtr(true), Username: &parts[0], Password: &parts[1], } @@ -475,17 +490,18 @@ func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, err // Setup the Vault config // Always set these to ensure nothing is picked up from the environment emptyStr := "" - conf.Vault.RenewToken = &f + conf.Vault.RenewToken = helper.BoolToPtr(false) conf.Vault.Token = &emptyStr if config.VaultConfig != nil && config.VaultConfig.IsEnabled() { conf.Vault.Address = &config.VaultConfig.Addr conf.Vault.Token = &vaultToken + conf.Vault.Grace = helper.TimeToPtr(vaultGrace) if strings.HasPrefix(config.VaultConfig.Addr, "https") || config.VaultConfig.TLSCertFile != "" { skipVerify := config.VaultConfig.TLSSkipVerify != nil && *config.VaultConfig.TLSSkipVerify verify := !skipVerify conf.Vault.SSL = &ctconf.SSLConfig{ - Enabled: &t, + Enabled: helper.BoolToPtr(true), Verify: &verify, Cert: &config.VaultConfig.TLSCertFile, Key: &config.VaultConfig.TLSKeyFile, @@ -495,8 +511,8 @@ func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, err } } else { conf.Vault.SSL = &ctconf.SSLConfig{ - Enabled: &f, - Verify: &f, + Enabled: helper.BoolToPtr(false), + Verify: helper.BoolToPtr(false), Cert: &emptyStr, Key: &emptyStr, CaCert: &emptyStr, diff --git a/client/consul_template_test.go b/client/consul_template_test.go index fcc3665a9bab..d3ef17b735cf 100644 --- a/client/consul_template_test.go +++ b/client/consul_template_test.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/nomad/nomad/structs" sconfig "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/nomad/testutil" + "github.com/stretchr/testify/assert" ) const ( @@ -1062,7 +1063,7 @@ func TestTaskTemplateManager_Config_ServerName(t *testing.T) { Addr: "https://localhost/", TLSServerName: "notlocalhost", } - ctconf, err := runnerConfig(c, "token") + ctconf, err := newRunnerConfig(c, "token", nil) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -1071,3 +1072,41 @@ func TestTaskTemplateManager_Config_ServerName(t *testing.T) { t.Fatalf("expected %q but found %q", c.VaultConfig.TLSServerName, *ctconf.Vault.SSL.ServerName) } } + +// TestTaskTemplateManager_Config_VaultGrace asserts the vault_grace setting is +// propogated to consul-template's configuration. +func TestTaskTemplateManager_Config_VaultGrace(t *testing.T) { + t.Parallel() + assert := assert.New(t) + c := config.DefaultConfig() + c.VaultConfig = &sconfig.VaultConfig{ + Enabled: helper.BoolToPtr(true), + Addr: "https://localhost/", + TLSServerName: "notlocalhost", + } + + // Make a template that will render immediately + templates := []*structs.Template{ + { + EmbeddedTmpl: "bar", + DestPath: "foo", + ChangeMode: structs.TemplateChangeModeNoop, + VaultGrace: 10 * time.Second, + }, + { + EmbeddedTmpl: "baz", + DestPath: "bam", + ChangeMode: structs.TemplateChangeModeNoop, + VaultGrace: 100 * time.Second, + }, + } + + taskEnv := env.NewTaskEnv(nil, nil) + ctmplMapping, err := parseTemplateConfigs(templates, "/fake/dir", taskEnv, false) + assert.Nil(err, "Parsing Templates") + + ctconf, err := newRunnerConfig(c, "token", ctmplMapping) + assert.Nil(err, "Building Runner Config") + assert.NotNil(ctconf.Vault.Grace, "Vault Grace Pointer") + assert.Equal(10*time.Second, *ctconf.Vault.Grace, "Vault Grace Value") +} diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 6cf9b03a40eb..b7da88a7955d 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -781,6 +781,7 @@ func ApiTaskToStructsTask(apiTask *api.Task, structsTask *structs.Task) { LeftDelim: *template.LeftDelim, RightDelim: *template.RightDelim, Envvars: *template.Envvars, + VaultGrace: *template.VaultGrace, } } } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 4ff4b0e3658c..7e085721672b 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -1170,6 +1170,8 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Perms: helper.StringToPtr("666"), LeftDelim: helper.StringToPtr("abc"), RightDelim: helper.StringToPtr("def"), + Envvars: helper.BoolToPtr(true), + VaultGrace: helper.TimeToPtr(3 * time.Second), }, }, DispatchPayload: &api.DispatchPayloadConfig{ @@ -1357,6 +1359,8 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Perms: "666", LeftDelim: "abc", RightDelim: "def", + Envvars: true, + VaultGrace: 3 * time.Second, }, }, DispatchPayload: &structs.DispatchPayloadConfig{ diff --git a/jobspec/parse.go b/jobspec/parse.go index 34b6f32f4e97..a97fb9ee271c 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -863,6 +863,7 @@ func parseTemplates(result *[]*api.Template, list *ast.ObjectList) error { "source", "splay", "env", + "vault_grace", } if err := checkHCLKeys(o.Val, valid); err != nil { return err diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 340ed76e2de0..a9215493d12f 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -183,6 +183,7 @@ func TestParse(t *testing.T) { Splay: helper.TimeToPtr(10 * time.Second), Perms: helper.StringToPtr("0644"), Envvars: helper.BoolToPtr(true), + VaultGrace: helper.TimeToPtr(33 * time.Second), }, { SourcePath: helper.StringToPtr("bar"), diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index cfcfc5a1d814..d77596075e71 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -158,6 +158,7 @@ job "binstore-storagelocker" { change_signal = "foo" splay = "10s" env = true + vault_grace = "33s" } template { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index c40a5718e651..a0f9df3c9f19 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -3287,6 +3287,11 @@ type Template struct { // Empty lines and lines starting with # will be ignored, but to avoid // escaping issues #s within lines will not be treated as comments. Envvars bool + + // VaultGrace is the grace duration between lease renewal and reacquiring a + // secret. If the lease of a secret is less than the grace, a new secret is + // acquired. + VaultGrace time.Duration } // DefaultTemplate returns a default template. @@ -3360,6 +3365,10 @@ func (t *Template) Validate() error { } } + if t.VaultGrace.Nanoseconds() < 0 { + multierror.Append(&mErr, fmt.Errorf("Vault grace must be greater than zero: %v < 0", t.VaultGrace)) + } + return mErr.ErrorOrNil() } diff --git a/vendor/github.com/hashicorp/consul-template/config/config.go b/vendor/github.com/hashicorp/consul-template/config/config.go index d8e91b40e1d7..2b1d95b673d0 100644 --- a/vendor/github.com/hashicorp/consul-template/config/config.go +++ b/vendor/github.com/hashicorp/consul-template/config/config.go @@ -230,7 +230,7 @@ func Parse(s string) (*Config, error) { }) // FlattenFlatten keys belonging to the templates. We cannot do this above - // because it is an array of tmeplates. + // because it is an array of templates. if templates, ok := parsed["template"].([]map[string]interface{}); ok { for _, template := range templates { flattenKeys(template, []string{ diff --git a/vendor/github.com/hashicorp/consul-template/config/vault.go b/vendor/github.com/hashicorp/consul-template/config/vault.go index 9a0cb7c11a92..6a051c0a41b7 100644 --- a/vendor/github.com/hashicorp/consul-template/config/vault.go +++ b/vendor/github.com/hashicorp/consul-template/config/vault.go @@ -11,7 +11,7 @@ const ( // DefaultVaultGrace is the default grace period before which to read a new // secret from Vault. If a lease is due to expire in 5 minutes, Consul // Template will read a new secret at that time minus this value. - DefaultVaultGrace = 15 * time.Second + DefaultVaultGrace = 5 * time.Minute // DefaultVaultRenewToken is the default value for if the Vault token should // be renewed. diff --git a/vendor/github.com/hashicorp/consul-template/dependency/errors.go b/vendor/github.com/hashicorp/consul-template/dependency/errors.go index 2857ac9b685a..dd03ac877157 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/errors.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/errors.go @@ -9,3 +9,5 @@ var ErrStopped = errors.New("dependency stopped") // ErrContinue is a special error which says to continue (retry) on error. var ErrContinue = errors.New("dependency continue") + +var ErrLeaseExpired = errors.New("lease expired or is not renewable") diff --git a/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go b/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go index 5f30556138b0..6bd0a95260ff 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/vault_common.go @@ -1,15 +1,23 @@ package dependency -import "time" +import ( + "log" + "math/rand" + "time" + + "github.com/hashicorp/vault/api" +) var ( // VaultDefaultLeaseDuration is the default lease duration in seconds. VaultDefaultLeaseDuration = 5 * time.Minute ) -// Secret is a vault secret. +// Secret is the structure returned for every secret within Vault. type Secret struct { - RequestID string + // The request ID that generated this response + RequestID string + LeaseID string LeaseDuration int Renewable bool @@ -17,23 +25,171 @@ type Secret struct { // Data is the actual contents of the secret. The format of the data // is arbitrary and up to the secret backend. Data map[string]interface{} + + // Warnings contains any warnings related to the operation. These + // are not issues that caused the command to fail, but that the + // client should be aware of. + Warnings []string + + // Auth, if non-nil, means that there was authentication information + // attached to this response. + Auth *SecretAuth + + // WrapInfo, if non-nil, means that the initial response was wrapped in the + // cubbyhole of the given token (which has a TTL of the given number of + // seconds) + WrapInfo *SecretWrapInfo +} + +// SecretAuth is the structure containing auth information if we have it. +type SecretAuth struct { + ClientToken string + Accessor string + Policies []string + Metadata map[string]string + + LeaseDuration int + Renewable bool +} + +// SecretWrapInfo contains wrapping information if we have it. If what is +// contained is an authentication token, the accessor for the token will be +// available in WrappedAccessor. +type SecretWrapInfo struct { + Token string + TTL int + CreationTime time.Time + WrappedAccessor string } -// leaseDurationOrDefault returns a value or the default lease duration. -func leaseDurationOrDefault(d int) int { - if d == 0 { - return int(VaultDefaultLeaseDuration.Nanoseconds() / 1000000000) +// vaultRenewDuration accepts a secret and returns the recommended amount of +// time to sleep. +func vaultRenewDuration(s *Secret) time.Duration { + // Handle whether this is an auth or a regular secret. + base := s.LeaseDuration + if s.Auth != nil && s.Auth.LeaseDuration > 0 { + base = s.Auth.LeaseDuration } - return d + + // Ensure we have a lease duration, since sometimes this can be zero. + if base <= 0 { + base = int(VaultDefaultLeaseDuration.Seconds()) + } + + // Convert to float seconds. + sleep := float64(time.Duration(base) * time.Second) + + // Renew at 1/3 the remaining lease. This will give us an opportunity to retry + // at least one more time should the first renewal fail. + sleep = sleep / 3.0 + + // Use a randomness so many clients do not hit Vault simultaneously. + sleep = sleep * (rand.Float64() + 1) / 2.0 + + return time.Duration(sleep) } -// vaultRenewDuration accepts a given renew duration (lease duration) and -// returns the cooresponding time.Duration. If the duration is 0 (not provided), -// this falls back to the VaultDefaultLeaseDuration. -func vaultRenewDuration(d int) time.Duration { - dur := time.Duration(d/2.0) * time.Second - if dur == 0 { - dur = VaultDefaultLeaseDuration +// printVaultWarnings prints warnings for a given dependency. +func printVaultWarnings(d Dependency, warnings []string) { + for _, w := range warnings { + log.Printf("[WARN] %s: %s", d, w) + } +} + +// vaultSecretRenewable determines if the given secret is renewable. +func vaultSecretRenewable(s *Secret) bool { + if s.Auth != nil { + return s.Auth.Renewable + } + return s.Renewable +} + +// transformSecret transforms an api secret into our secret. This does not deep +// copy underlying deep data structures, so it's not safe to modify the vault +// secret as that may modify the data in the transformed secret. +func transformSecret(theirs *api.Secret) *Secret { + var ours Secret + updateSecret(&ours, theirs) + return &ours +} + +// updateSecret updates our secret with the new data from the api, careful to +// not overwrite missing data. Renewals don't include the original secret, and +// we don't want to delete that data accidentially. +func updateSecret(ours *Secret, theirs *api.Secret) { + if theirs.RequestID != "" { + ours.RequestID = theirs.RequestID + } + + if theirs.LeaseID != "" { + ours.LeaseID = theirs.LeaseID + } + + if theirs.LeaseDuration != 0 { + ours.LeaseDuration = theirs.LeaseDuration + } + + if theirs.Renewable { + ours.Renewable = theirs.Renewable + } + + if len(theirs.Data) != 0 { + ours.Data = theirs.Data + } + + if len(theirs.Warnings) != 0 { + ours.Warnings = theirs.Warnings + } + + if theirs.Auth != nil { + if ours.Auth == nil { + ours.Auth = &SecretAuth{} + } + + if theirs.Auth.ClientToken != "" { + ours.Auth.ClientToken = theirs.Auth.ClientToken + } + + if theirs.Auth.Accessor != "" { + ours.Auth.Accessor = theirs.Auth.Accessor + } + + if len(theirs.Auth.Policies) != 0 { + ours.Auth.Policies = theirs.Auth.Policies + } + + if len(theirs.Auth.Metadata) != 0 { + ours.Auth.Metadata = theirs.Auth.Metadata + } + + if theirs.Auth.LeaseDuration != 0 { + ours.Auth.LeaseDuration = theirs.Auth.LeaseDuration + } + + if theirs.Auth.Renewable { + ours.Auth.Renewable = theirs.Auth.Renewable + } + } + + if theirs.WrapInfo != nil { + if ours.WrapInfo == nil { + ours.WrapInfo = &SecretWrapInfo{} + } + + if theirs.WrapInfo.Token != "" { + ours.WrapInfo.Token = theirs.WrapInfo.Token + } + + if theirs.WrapInfo.TTL != 0 { + ours.WrapInfo.TTL = theirs.WrapInfo.TTL + } + + if !theirs.WrapInfo.CreationTime.IsZero() { + ours.WrapInfo.CreationTime = theirs.WrapInfo.CreationTime + } + + if theirs.WrapInfo.WrappedAccessor != "" { + ours.WrapInfo.WrappedAccessor = theirs.WrapInfo.WrappedAccessor + } } - return dur } diff --git a/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go b/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go index a903bac51b5e..2a6e948a5763 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/vault_read.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/hashicorp/vault/api" "github.com/pkg/errors" ) @@ -21,6 +22,9 @@ type VaultReadQuery struct { path string secret *Secret + + // vaultSecret is the actual Vault secret which we are renewing + vaultSecret *api.Secret } // NewVaultReadQuery creates a new datacenter dependency. @@ -47,71 +51,68 @@ func (d *VaultReadQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interfac opts = opts.Merge(&QueryOptions{}) - // If this is not the first query and we have a lease duration, sleep until we - // try to renew. - if opts.WaitIndex != 0 && d.secret != nil && d.secret.LeaseDuration != 0 { - dur := vaultRenewDuration(d.secret.LeaseDuration) - - log.Printf("[TRACE] %s: long polling for %s", d, dur) - - select { - case <-d.stopCh: - return nil, nil, ErrStopped - case <-time.After(dur): - } - } + if d.secret != nil { + if vaultSecretRenewable(d.secret) { + log.Printf("[TRACE] %s: starting renewer", d) - // Attempt to renew the secret. If we do not have a secret or if that secret - // is not renewable, we will attempt a (re-)read later. - if d.secret != nil && d.secret.LeaseID != "" && d.secret.Renewable { - log.Printf("[TRACE] %s: PUT %s", d, &url.URL{ - Path: "/v1/sys/renew/" + d.secret.LeaseID, - RawQuery: opts.String(), - }) - - renewal, err := clients.Vault().Sys().Renew(d.secret.LeaseID, 0) - if err == nil { - log.Printf("[TRACE] %s: successfully renewed %s", d, d.secret.LeaseID) - - // Print any warnings - d.printWarnings(renewal.Warnings) - - secret := &Secret{ - RequestID: renewal.RequestID, - LeaseID: renewal.LeaseID, - LeaseDuration: d.secret.LeaseDuration, - Renewable: renewal.Renewable, - Data: d.secret.Data, + renewer, err := clients.Vault().NewRenewer(&api.RenewerInput{ + Grace: opts.VaultGrace, + Secret: d.vaultSecret, + }) + if err != nil { + return nil, nil, errors.Wrap(err, d.String()) } - // For some older versions of Vault, the renewal did not include the - // remaining lease duration, so just use the original lease duration, - // because it's the best we can do. - if renewal.LeaseDuration != 0 { - secret.LeaseDuration = renewal.LeaseDuration + go renewer.Renew() + defer renewer.Stop() + + RENEW: + for { + select { + case err := <-renewer.DoneCh(): + if err != nil { + log.Printf("[WARN] %s: failed to renew: %s", d, err) + } + log.Printf("[WARN] %s: renewer returned (maybe the lease expired)", d) + break RENEW + case renewal := <-renewer.RenewCh(): + log.Printf("[TRACE] %s: successfully renewed", d) + printVaultWarnings(d, renewal.Secret.Warnings) + updateSecret(d.secret, renewal.Secret) + case <-d.stopCh: + return nil, nil, ErrStopped + } } - d.secret = secret - - // If the remaining time on the lease is less than or equal to our - // configured grace period, generate a new credential now. This will help - // minimize downtime, since Vault will revoke credentials immediately - // when their maximum TTL expires. - remaining := time.Duration(d.secret.LeaseDuration) * time.Second - if remaining <= opts.VaultGrace { - log.Printf("[DEBUG] %s: remaining lease (%s) < grace (%s), acquiring new", - d, remaining, opts.VaultGrace) - return d.readSecret(clients, opts) + } else { + // The secret isn't renewable, probably the generic secret backend. + dur := vaultRenewDuration(d.secret) + if dur < opts.VaultGrace { + log.Printf("[TRACE] %s: remaining lease %s is less than grace, skipping sleep", d, dur) + } else { + log.Printf("[TRACE] %s: secret is not renewable, sleeping for %s", d, dur) + select { + case <-time.After(dur): + // The lease is almost expired, it's time to request a new one. + case <-d.stopCh: + return nil, nil, ErrStopped + } } - - return respWithMetadata(secret) } + } - // The renewal failed for some reason. - log.Printf("[WARN] %s: failed to renew %s: %s", d, d.secret.LeaseID, err) + // We don't have a secret, or the prior renewal failed + vaultSecret, err := d.readSecret(clients, opts) + if err != nil { + return nil, nil, errors.Wrap(err, d.String()) } - // If we got this far, we either didn't have a secret to renew, the secret was - // not renewable, or the renewal failed, so attempt a fresh read. - return d.readSecret(clients, opts) + // Print any warnings + printVaultWarnings(d, vaultSecret.Warnings) + + // Create the cloned secret which will be exposed to the template. + d.vaultSecret = vaultSecret + d.secret = transformSecret(vaultSecret) + + return respWithMetadata(d.secret) } // CanShare returns if this dependency is shareable. @@ -134,38 +135,17 @@ func (d *VaultReadQuery) Type() Type { return TypeVault } -func (d *VaultReadQuery) printWarnings(warnings []string) { - for _, w := range warnings { - log.Printf("[WARN] %s: %s", d, w) - } -} - -func (d *VaultReadQuery) readSecret(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) { +func (d *VaultReadQuery) readSecret(clients *ClientSet, opts *QueryOptions) (*api.Secret, error) { log.Printf("[TRACE] %s: GET %s", d, &url.URL{ Path: "/v1/" + d.path, RawQuery: opts.String(), }) vaultSecret, err := clients.Vault().Logical().Read(d.path) if err != nil { - return nil, nil, errors.Wrap(err, d.String()) + return nil, errors.Wrap(err, d.String()) } - - // The secret could be nil if it does not exist. if vaultSecret == nil { - return nil, nil, fmt.Errorf("%s: no secret exists at %s", d, d.path) + return nil, fmt.Errorf("no secret exists at %s", d.path) } - - // Print any warnings. - d.printWarnings(vaultSecret.Warnings) - - // Create our cloned secret. - secret := &Secret{ - LeaseID: vaultSecret.LeaseID, - LeaseDuration: leaseDurationOrDefault(vaultSecret.LeaseDuration), - Renewable: vaultSecret.Renewable, - Data: vaultSecret.Data, - } - d.secret = secret - - return respWithMetadata(secret) + return vaultSecret, nil } diff --git a/vendor/github.com/hashicorp/consul-template/dependency/vault_token.go b/vendor/github.com/hashicorp/consul-template/dependency/vault_token.go index 30e198a693b2..2c81c67f6da2 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/vault_token.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/vault_token.go @@ -2,9 +2,9 @@ package dependency import ( "log" - "net/url" "time" + "github.com/hashicorp/vault/api" "github.com/pkg/errors" ) @@ -15,16 +15,24 @@ var ( // VaultTokenQuery is the dependency to Vault for a secret type VaultTokenQuery struct { - stopCh chan struct{} - - leaseID string - leaseDuration int + stopCh chan struct{} + secret *Secret + vaultSecret *api.Secret } // NewVaultTokenQuery creates a new dependency. -func NewVaultTokenQuery() (*VaultTokenQuery, error) { +func NewVaultTokenQuery(token string) (*VaultTokenQuery, error) { + vaultSecret := &api.Secret{ + Auth: &api.SecretAuth{ + ClientToken: token, + Renewable: true, + LeaseDuration: 1, + }, + } return &VaultTokenQuery{ - stopCh: make(chan struct{}, 1), + stopCh: make(chan struct{}, 1), + vaultSecret: vaultSecret, + secret: transformSecret(vaultSecret), }, nil } @@ -38,44 +46,53 @@ func (d *VaultTokenQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interfa opts = opts.Merge(&QueryOptions{}) - log.Printf("[TRACE] %s: GET %s", d, &url.URL{ - Path: "/v1/auth/token/renew-self", - RawQuery: opts.String(), - }) + if vaultSecretRenewable(d.secret) { + log.Printf("[TRACE] %s: starting renewer", d) - // If this is not the first query and we have a lease duration, sleep until we - // try to renew. - if opts.WaitIndex != 0 && d.leaseDuration != 0 { - dur := vaultRenewDuration(d.leaseDuration) - - log.Printf("[TRACE] %s: long polling for %s", d, dur) + renewer, err := clients.Vault().NewRenewer(&api.RenewerInput{ + Grace: opts.VaultGrace, + Secret: d.vaultSecret, + }) + if err != nil { + return nil, nil, errors.Wrap(err, d.String()) + } + go renewer.Renew() + defer renewer.Stop() + + RENEW: + for { + select { + case err := <-renewer.DoneCh(): + if err != nil { + log.Printf("[WARN] %s: failed to renew: %s", d, err) + } + log.Printf("[WARN] %s: renewer returned (maybe the lease expired)", d) + break RENEW + case renewal := <-renewer.RenewCh(): + log.Printf("[TRACE] %s: successfully renewed", d) + printVaultWarnings(d, renewal.Secret.Warnings) + updateSecret(d.secret, renewal.Secret) + case <-d.stopCh: + return nil, nil, ErrStopped + } + } + } + // The secret isn't renewable, probably the generic secret backend. + dur := vaultRenewDuration(d.secret) + if dur < opts.VaultGrace { + log.Printf("[TRACE] %s: remaining lease %s is less than grace, skipping sleep", d, dur) + } else { + log.Printf("[TRACE] %s: token is not renewable, sleeping for %s", d, dur) select { + case <-time.After(dur): + // The lease is almost expired, it's time to request a new one. case <-d.stopCh: return nil, nil, ErrStopped - case <-time.After(dur): } } - token, err := clients.Vault().Auth().Token().RenewSelf(0) - if err != nil { - return nil, nil, errors.Wrap(err, d.String()) - } - - // Create our cloned secret - secret := &Secret{ - LeaseID: token.LeaseID, - LeaseDuration: token.Auth.LeaseDuration, - Renewable: token.Auth.Renewable, - Data: token.Data, - } - - d.leaseID = secret.LeaseID - d.leaseDuration = secret.LeaseDuration - - log.Printf("[DEBUG] %s: renewed token", d) - - return respWithMetadata(secret) + return nil, nil, ErrLeaseExpired } // CanShare returns if this dependency is shareable. diff --git a/vendor/github.com/hashicorp/consul-template/dependency/vault_write.go b/vendor/github.com/hashicorp/consul-template/dependency/vault_write.go index f0e825a19756..e3aa01544448 100644 --- a/vendor/github.com/hashicorp/consul-template/dependency/vault_write.go +++ b/vendor/github.com/hashicorp/consul-template/dependency/vault_write.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/hashicorp/vault/api" "github.com/pkg/errors" ) @@ -26,6 +27,9 @@ type VaultWriteQuery struct { data map[string]interface{} dataHash string secret *Secret + + // vaultSecret is the actual Vault secret which we are renewing + vaultSecret *api.Secret } // NewVaultWriteQuery creates a new datacenter dependency. @@ -54,70 +58,68 @@ func (d *VaultWriteQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interfa opts = opts.Merge(&QueryOptions{}) - // If this is not the first query and we have a lease duration, sleep until we - // try to renew. - if opts.WaitIndex != 0 && d.secret != nil && d.secret.LeaseDuration != 0 { - dur := vaultRenewDuration(d.secret.LeaseDuration) - - log.Printf("[TRACE] %s: long polling for %s", d, dur) - - select { - case <-d.stopCh: - return nil, nil, ErrStopped - case <-time.After(dur): - } - } + if d.secret != nil { + if vaultSecretRenewable(d.secret) { + log.Printf("[TRACE] %s: starting renewer", d) - // Attempt to renew the secret. If we do not have a secret or if that secret - // is not renewable, we will attempt a (re-)write later. - if d.secret != nil && d.secret.LeaseID != "" && d.secret.Renewable { - log.Printf("[TRACE] %s: PUT %s", d, &url.URL{ - Path: "/v1/sys/renew/" + d.secret.LeaseID, - RawQuery: opts.String(), - }) - - renewal, err := clients.Vault().Sys().Renew(d.secret.LeaseID, 0) - if err == nil { - log.Printf("[TRACE] %s: successfully renewed %s", d, d.secret.LeaseID) - - // Print any warnings - d.printWarnings(renewal.Warnings) - - secret := &Secret{ - RequestID: renewal.RequestID, - LeaseID: renewal.LeaseID, - Renewable: renewal.Renewable, - Data: d.secret.Data, + renewer, err := clients.Vault().NewRenewer(&api.RenewerInput{ + Grace: opts.VaultGrace, + Secret: d.vaultSecret, + }) + if err != nil { + return nil, nil, errors.Wrap(err, d.String()) } - // For some older versions of Vault, the renewal did not include the - // remaining lease duration, so just use the original lease duration, - // because it's the best we can do. - if renewal.LeaseDuration != 0 { - secret.LeaseDuration = renewal.LeaseDuration + go renewer.Renew() + defer renewer.Stop() + + RENEW: + for { + select { + case err := <-renewer.DoneCh(): + if err != nil { + log.Printf("[WARN] %s: failed to renew: %s", d, err) + } + log.Printf("[WARN] %s: renewer returned (maybe the lease expired)", d) + break RENEW + case renewal := <-renewer.RenewCh(): + log.Printf("[TRACE] %s: successfully renewed", d) + printVaultWarnings(d, renewal.Secret.Warnings) + updateSecret(d.secret, renewal.Secret) + case <-d.stopCh: + return nil, nil, ErrStopped + } } - d.secret = secret - - // If the remaining time on the lease is less than or equal to our - // configured grace period, generate a new credential now. This will help - // minimize downtime, since Vault will revoke credentials immediately - // when their maximum TTL expires. - remaining := time.Duration(d.secret.LeaseDuration) * time.Second - if remaining <= opts.VaultGrace { - log.Printf("[DEBUG] %s: remaining lease (%s) < grace (%s), acquiring new", - d, remaining, opts.VaultGrace) - return d.writeSecret(clients, opts) + } else { + // The secret isn't renewable, probably the generic secret backend. + dur := vaultRenewDuration(d.secret) + if dur < opts.VaultGrace { + log.Printf("[TRACE] %s: remaining lease %s is less than grace, skipping sleep", d, dur) + } else { + log.Printf("[TRACE] %s: secret is not renewable, sleeping for %s", d, dur) + select { + case <-time.After(dur): + // The lease is almost expired, it's time to request a new one. + case <-d.stopCh: + return nil, nil, ErrStopped + } } - - return respWithMetadata(secret) } + } - // The renewal failed for some reason. - log.Printf("[WARN] %s: failed to renew %s: %s", d, d.secret.LeaseID, err) + // We don't have a secret, or the prior renewal failed + vaultSecret, err := d.writeSecret(clients, opts) + if err != nil { + return nil, nil, errors.Wrap(err, d.String()) } - // If we got this far, we either didn't have a secret to renew, the secret was - // not renewable, or the renewal failed, so attempt a fresh write. - return d.writeSecret(clients, opts) + // Print any warnings + printVaultWarnings(d, vaultSecret.Warnings) + + // Create the cloned secret which will be exposed to the template. + d.vaultSecret = vaultSecret + d.secret = transformSecret(vaultSecret) + + return respWithMetadata(d.secret) } // CanShare returns if this dependency is shareable. @@ -164,7 +166,7 @@ func (d *VaultWriteQuery) printWarnings(warnings []string) { } } -func (d *VaultWriteQuery) writeSecret(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) { +func (d *VaultWriteQuery) writeSecret(clients *ClientSet, opts *QueryOptions) (*api.Secret, error) { log.Printf("[TRACE] %s: PUT %s", d, &url.URL{ Path: "/v1/" + d.path, RawQuery: opts.String(), @@ -172,27 +174,11 @@ func (d *VaultWriteQuery) writeSecret(clients *ClientSet, opts *QueryOptions) (i vaultSecret, err := clients.Vault().Logical().Write(d.path, d.data) if err != nil { - return nil, nil, errors.Wrap(err, d.String()) + return nil, errors.Wrap(err, d.String()) } - - // The secret could be nil if it does not exist. if vaultSecret == nil { - return nil, nil, fmt.Errorf("%s: no secret exists at %s", d, d.path) - } - - // Print any warnings. - for _, w := range vaultSecret.Warnings { - log.Printf("[WARN] %s: %s", d, w) - } - - // Create our cloned secret. - secret := &Secret{ - LeaseID: vaultSecret.LeaseID, - LeaseDuration: leaseDurationOrDefault(vaultSecret.LeaseDuration), - Renewable: vaultSecret.Renewable, - Data: vaultSecret.Data, + return nil, fmt.Errorf("no secret exists at %s", d.path) } - d.secret = secret - return respWithMetadata(secret) + return vaultSecret, nil } diff --git a/vendor/github.com/hashicorp/consul-template/manager/renderer.go b/vendor/github.com/hashicorp/consul-template/manager/renderer.go index 24655f4e9d98..ea5d0bcebefa 100644 --- a/vendor/github.com/hashicorp/consul-template/manager/renderer.go +++ b/vendor/github.com/hashicorp/consul-template/manager/renderer.go @@ -11,6 +11,7 @@ import ( "github.com/pkg/errors" ) +// RenderInput is used as input to the render function. type RenderInput struct { Backup bool Contents []byte @@ -20,9 +21,22 @@ type RenderInput struct { Perms os.FileMode } +// RenderResult is returned and stored. It contains the status of the render +// operationg. type RenderResult struct { - DidRender bool + // DidRender indicates if the template rendered to disk. This will be false in + // the event of an error, but it will also be false in dry mode or when the + // template on disk matches the new result. + DidRender bool + + // WouldRender indicates if the template would have rendered to disk. This + // will return false in the event of an error, but will return true in dry + // mode or when the template on disk matches the new result. WouldRender bool + + // Contents are the actual contents of the resulting template from the render + // operation. + Contents []byte } // Render atomically renders a file contents to disk, returning a result of @@ -37,6 +51,7 @@ func Render(i *RenderInput) (*RenderResult, error) { return &RenderResult{ DidRender: false, WouldRender: true, + Contents: existing, }, nil } @@ -51,6 +66,7 @@ func Render(i *RenderInput) (*RenderResult, error) { return &RenderResult{ DidRender: true, WouldRender: true, + Contents: existing, }, nil } diff --git a/vendor/github.com/hashicorp/consul-template/manager/runner.go b/vendor/github.com/hashicorp/consul-template/manager/runner.go index 8a6da94b3356..770e72200944 100644 --- a/vendor/github.com/hashicorp/consul-template/manager/runner.go +++ b/vendor/github.com/hashicorp/consul-template/manager/runner.go @@ -42,8 +42,7 @@ type Runner struct { dry, once bool // outStream and errStream are the io.Writer streams where the runner will - // write information. These streams can be set using the SetOutStream() - // and SetErrStream() functions. + // write information. // inStream is the ioReader where the runner will read information. outStream, errStream io.Writer @@ -118,6 +117,9 @@ type RenderEvent struct { // Template is the template attempting to be rendered. Template *template.Template + // Contents is the raw, rendered contents from the template. + Contents []byte + // TemplateConfigs is the list of template configs that correspond to this // template. TemplateConfigs []*config.TemplateConfig @@ -643,6 +645,9 @@ func (r *Runner) Run() error { event.DidRender = true event.LastDidRender = renderTime + // Update the contents + event.Contents = result.Contents + // Record that at least one template was rendered. renderedAny = true @@ -1175,6 +1180,7 @@ func newWatcher(c *config.Config, clients *dep.ClientSet, once bool) (*watch.Wat RetryFuncDefault: nil, RetryFuncVault: watch.RetryFunc(c.Vault.Retry.RetryFunc()), VaultGrace: config.TimeDurationVal(c.Vault.Grace), + VaultToken: config.StringVal(c.Vault.Token), }) if err != nil { return nil, errors.Wrap(err, "runner") diff --git a/vendor/github.com/hashicorp/consul-template/watch/view.go b/vendor/github.com/hashicorp/consul-template/watch/view.go index b495ac4c7b7d..d8e42add938d 100644 --- a/vendor/github.com/hashicorp/consul-template/watch/view.go +++ b/vendor/github.com/hashicorp/consul-template/watch/view.go @@ -150,7 +150,7 @@ func (v *View) poll(viewCh chan<- *View, errCh chan<- error) { // example, Consul make have an outage, but when it returns, the view // is unchanged. We have to reset the counter retries, but not update the // actual template. - log.Printf("[TRACE] view %s successful contact, resetting retries", v.dependency) + log.Printf("[TRACE] (view) %s successful contact, resetting retries", v.dependency) retries = 0 goto WAIT case err := <-fetchErrCh: diff --git a/vendor/github.com/hashicorp/consul-template/watch/watcher.go b/vendor/github.com/hashicorp/consul-template/watch/watcher.go index a4ee4938c9e4..89fe53b8dae5 100644 --- a/vendor/github.com/hashicorp/consul-template/watch/watcher.go +++ b/vendor/github.com/hashicorp/consul-template/watch/watcher.go @@ -62,6 +62,9 @@ type NewWatcherInput struct { // RenewVault indicates if this watcher should renew Vault tokens. RenewVault bool + // VaultToken is the vault token to renew. + VaultToken string + // RetryFuncs specify the different ways to retry based on the upstream. RetryFuncConsul RetryFunc RetryFuncDefault RetryFunc @@ -90,7 +93,7 @@ func NewWatcher(i *NewWatcherInput) (*Watcher, error) { // Start a watcher for the Vault renew if that config was specified if i.RenewVault { - vt, err := dep.NewVaultTokenQuery() + vt, err := dep.NewVaultTokenQuery(i.VaultToken) if err != nil { return nil, errors.Wrap(err, "watcher") } diff --git a/vendor/github.com/hashicorp/vault/api/auth_token.go b/vendor/github.com/hashicorp/vault/api/auth_token.go index aff10f4109cb..4f74f61fe5f2 100644 --- a/vendor/github.com/hashicorp/vault/api/auth_token.go +++ b/vendor/github.com/hashicorp/vault/api/auth_token.go @@ -135,6 +135,26 @@ func (c *TokenAuth) RenewSelf(increment int) (*Secret, error) { return ParseSecret(resp.Body) } +// RenewTokenAsSelf behaves like renew-self, but authenticates using a provided +// token instead of the token attached to the client. +func (c *TokenAuth) RenewTokenAsSelf(token string, increment int) (*Secret, error) { + r := c.c.NewRequest("PUT", "/v1/auth/token/renew-self") + r.ClientToken = token + + body := map[string]interface{}{"increment": increment} + if err := r.SetJSONBody(body); err != nil { + return nil, err + } + + resp, err := c.c.RawRequest(r) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + return ParseSecret(resp.Body) +} + // RevokeAccessor revokes a token associated with the given accessor // along with all the child tokens. func (c *TokenAuth) RevokeAccessor(accessor string) error { diff --git a/vendor/github.com/hashicorp/vault/api/client.go b/vendor/github.com/hashicorp/vault/api/client.go index 88a8ea4f9542..b35ba49d509b 100644 --- a/vendor/github.com/hashicorp/vault/api/client.go +++ b/vendor/github.com/hashicorp/vault/api/client.go @@ -6,13 +6,17 @@ import ( "net/http" "net/url" "os" + "path" "strconv" "strings" "sync" "time" + "golang.org/x/net/http2" + "github.com/hashicorp/go-cleanhttp" "github.com/hashicorp/go-rootcerts" + "github.com/hashicorp/vault/helper/parseutil" "github.com/sethgrid/pester" ) @@ -21,10 +25,12 @@ const EnvVaultCACert = "VAULT_CACERT" const EnvVaultCAPath = "VAULT_CAPATH" const EnvVaultClientCert = "VAULT_CLIENT_CERT" const EnvVaultClientKey = "VAULT_CLIENT_KEY" +const EnvVaultClientTimeout = "VAULT_CLIENT_TIMEOUT" const EnvVaultInsecure = "VAULT_SKIP_VERIFY" const EnvVaultTLSServerName = "VAULT_TLS_SERVER_NAME" const EnvVaultWrapTTL = "VAULT_WRAP_TTL" const EnvVaultMaxRetries = "VAULT_MAX_RETRIES" +const EnvVaultToken = "VAULT_TOKEN" // WrappingLookupFunc is a function that, given an HTTP verb and a path, // returns an optional string duration to be used for response wrapping (e.g. @@ -50,6 +56,9 @@ type Config struct { // MaxRetries controls the maximum number of times to retry when a 5xx error // occurs. Set to 0 or less to disable retrying. Defaults to 0. MaxRetries int + + // Timeout is for setting custom timeout parameter in the HttpClient + Timeout time.Duration } // TLSConfig contains the parameters needed to configure TLS on the HTTP client @@ -84,8 +93,7 @@ type TLSConfig struct { // setting the `VAULT_ADDR` environment variable. func DefaultConfig() *Config { config := &Config{ - Address: "https://127.0.0.1:8200", - + Address: "https://127.0.0.1:8200", HttpClient: cleanhttp.DefaultClient(), } config.HttpClient.Timeout = time.Second * 60 @@ -104,9 +112,8 @@ func DefaultConfig() *Config { // ConfigureTLS takes a set of TLS configurations and applies those to the the HTTP client. func (c *Config) ConfigureTLS(t *TLSConfig) error { - if c.HttpClient == nil { - return fmt.Errorf("config HTTP Client must be set") + c.HttpClient = DefaultConfig().HttpClient } var clientCert tls.Certificate @@ -154,6 +161,7 @@ func (c *Config) ReadEnvironment() error { var envCAPath string var envClientCert string var envClientKey string + var envClientTimeout time.Duration var envInsecure bool var envTLSServerName string var envMaxRetries *uint64 @@ -181,6 +189,13 @@ func (c *Config) ReadEnvironment() error { if v := os.Getenv(EnvVaultClientKey); v != "" { envClientKey = v } + if t := os.Getenv(EnvVaultClientTimeout); t != "" { + clientTimeout, err := parseutil.ParseDurationSecond(t) + if err != nil { + return fmt.Errorf("Could not parse %s", EnvVaultClientTimeout) + } + envClientTimeout = clientTimeout + } if v := os.Getenv(EnvVaultInsecure); v != "" { var err error envInsecure, err = strconv.ParseBool(v) @@ -213,6 +228,10 @@ func (c *Config) ReadEnvironment() error { c.MaxRetries = int(*envMaxRetries) + 1 } + if envClientTimeout != 0 { + c.Timeout = envClientTimeout + } + return nil } @@ -247,6 +266,11 @@ func NewClient(c *Config) (*Client, error) { c.HttpClient = DefaultConfig().HttpClient } + tp := c.HttpClient.Transport.(*http.Transport) + if err := http2.ConfigureTransport(tp); err != nil { + return nil, err + } + redirFunc := func() { // Ensure redirects are not automatically followed // Note that this is sane for the API client as it has its own @@ -254,9 +278,9 @@ func NewClient(c *Config) (*Client, error) { // but in e.g. http_test actual redirect handling is necessary c.HttpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { // Returning this value causes the Go net library to not close the - // response body and nil out the error. Otherwise pester tries + // response body and to nil out the error. Otherwise pester tries // three times on every redirect because it sees an error from this - // function being passed through. + // function (to prevent redirects) passing through to it. return http.ErrUseLastResponse } } @@ -268,7 +292,7 @@ func NewClient(c *Config) (*Client, error) { config: c, } - if token := os.Getenv("VAULT_TOKEN"); token != "" { + if token := os.Getenv(EnvVaultToken); token != "" { client.SetToken(token) } @@ -292,6 +316,16 @@ func (c *Client) Address() string { return c.addr.String() } +// SetMaxRetries sets the number of retries that will be used in the case of certain errors +func (c *Client) SetMaxRetries(retries int) { + c.config.MaxRetries = retries +} + +// SetClientTimeout sets the client request timeout +func (c *Client) SetClientTimeout(timeout time.Duration) { + c.config.Timeout = timeout +} + // SetWrappingLookupFunc sets a lookup function that returns desired wrap TTLs // for a given operation and path func (c *Client) SetWrappingLookupFunc(lookupFunc WrappingLookupFunc) { @@ -315,16 +349,22 @@ func (c *Client) ClearToken() { c.token = "" } +// Clone creates a copy of this client. +func (c *Client) Clone() (*Client, error) { + return NewClient(c.config) +} + // NewRequest creates a new raw request object to query the Vault server // configured for this client. This is an advanced method and generally // doesn't need to be called externally. -func (c *Client) NewRequest(method, path string) *Request { +func (c *Client) NewRequest(method, requestPath string) *Request { req := &Request{ Method: method, URL: &url.URL{ + User: c.addr.User, Scheme: c.addr.Scheme, Host: c.addr.Host, - Path: path, + Path: path.Join(c.addr.Path, requestPath), }, ClientToken: c.token, Params: make(map[string][]string), @@ -332,18 +372,21 @@ func (c *Client) NewRequest(method, path string) *Request { var lookupPath string switch { - case strings.HasPrefix(path, "/v1/"): - lookupPath = strings.TrimPrefix(path, "/v1/") - case strings.HasPrefix(path, "v1/"): - lookupPath = strings.TrimPrefix(path, "v1/") + case strings.HasPrefix(requestPath, "/v1/"): + lookupPath = strings.TrimPrefix(requestPath, "/v1/") + case strings.HasPrefix(requestPath, "v1/"): + lookupPath = strings.TrimPrefix(requestPath, "v1/") default: - lookupPath = path + lookupPath = requestPath } if c.wrappingLookupFunc != nil { req.WrapTTL = c.wrappingLookupFunc(method, lookupPath) } else { req.WrapTTL = DefaultWrappingLookupFunc(method, lookupPath) } + if c.config.Timeout != 0 { + c.config.HttpClient.Timeout = c.config.Timeout + } return req } diff --git a/vendor/github.com/hashicorp/vault/api/renewer.go b/vendor/github.com/hashicorp/vault/api/renewer.go new file mode 100644 index 000000000000..a2a4b66d5870 --- /dev/null +++ b/vendor/github.com/hashicorp/vault/api/renewer.go @@ -0,0 +1,302 @@ +package api + +import ( + "errors" + "math/rand" + "sync" + "time" +) + +var ( + ErrRenewerMissingInput = errors.New("missing input to renewer") + ErrRenewerMissingSecret = errors.New("missing secret to renew") + ErrRenewerNotRenewable = errors.New("secret is not renewable") + ErrRenewerNoSecretData = errors.New("returned empty secret data") + + // DefaultRenewerGrace is the default grace period + DefaultRenewerGrace = 15 * time.Second + + // DefaultRenewerRenewBuffer is the default size of the buffer for renew + // messages on the channel. + DefaultRenewerRenewBuffer = 5 +) + +// Renewer is a process for renewing a secret. +// +// renewer, err := client.NewRenewer(&RenewerInput{ +// Secret: mySecret, +// }) +// go renewer.Renew() +// defer renewer.Stop() +// +// for { +// select { +// case err := <-renewer.DoneCh(): +// if err != nil { +// log.Fatal(err) +// } +// +// // Renewal is now over +// case renewal := <-renewer.RenewCh(): +// log.Printf("Successfully renewed: %#v", renewal) +// } +// } +// +// +// The `DoneCh` will return if renewal fails or if the remaining lease duration +// after a renewal is less than or equal to the grace (in number of seconds). In +// both cases, the caller should attempt a re-read of the secret. Clients should +// check the return value of the channel to see if renewal was successful. +type Renewer struct { + l sync.Mutex + + client *Client + secret *Secret + grace time.Duration + random *rand.Rand + doneCh chan error + renewCh chan *RenewOutput + + stopped bool + stopCh chan struct{} +} + +// RenewerInput is used as input to the renew function. +type RenewerInput struct { + // Secret is the secret to renew + Secret *Secret + + // Grace is a minimum renewal before returning so the upstream client + // can do a re-read. This can be used to prevent clients from waiting + // too long to read a new credential and incur downtime. + Grace time.Duration + + // Rand is the randomizer to use for underlying randomization. If not + // provided, one will be generated and seeded automatically. If provided, it + // is assumed to have already been seeded. + Rand *rand.Rand + + // RenewBuffer is the size of the buffered channel where renew messages are + // dispatched. + RenewBuffer int +} + +// RenewOutput is the metadata returned to the client (if it's listening) to +// renew messages. +type RenewOutput struct { + // RenewedAt is the timestamp when the renewal took place (UTC). + RenewedAt time.Time + + // Secret is the underlying renewal data. It's the same struct as all data + // that is returned from Vault, but since this is renewal data, it will not + // usually include the secret itself. + Secret *Secret +} + +// NewRenewer creates a new renewer from the given input. +func (c *Client) NewRenewer(i *RenewerInput) (*Renewer, error) { + if i == nil { + return nil, ErrRenewerMissingInput + } + + secret := i.Secret + if secret == nil { + return nil, ErrRenewerMissingSecret + } + + grace := i.Grace + if grace == 0 { + grace = DefaultRenewerGrace + } + + random := i.Rand + if random == nil { + random = rand.New(rand.NewSource(int64(time.Now().Nanosecond()))) + } + + renewBuffer := i.RenewBuffer + if renewBuffer == 0 { + renewBuffer = DefaultRenewerRenewBuffer + } + + return &Renewer{ + client: c, + secret: secret, + grace: grace, + random: random, + doneCh: make(chan error, 1), + renewCh: make(chan *RenewOutput, renewBuffer), + + stopped: false, + stopCh: make(chan struct{}), + }, nil +} + +// DoneCh returns the channel where the renewer will publish when renewal stops. +// If there is an error, this will be an error. +func (r *Renewer) DoneCh() <-chan error { + return r.doneCh +} + +// RenewCh is a channel that receives a message when a successful renewal takes +// place and includes metadata about the renewal. +func (r *Renewer) RenewCh() <-chan *RenewOutput { + return r.renewCh +} + +// Stop stops the renewer. +func (r *Renewer) Stop() { + r.l.Lock() + if !r.stopped { + close(r.stopCh) + r.stopped = true + } + r.l.Unlock() +} + +// Renew starts a background process for renewing this secret. When the secret +// is has auth data, this attempts to renew the auth (token). When the secret +// has a lease, this attempts to renew the lease. +func (r *Renewer) Renew() { + var result error + if r.secret.Auth != nil { + result = r.renewAuth() + } else { + result = r.renewLease() + } + + select { + case r.doneCh <- result: + case <-r.stopCh: + } +} + +// renewAuth is a helper for renewing authentication. +func (r *Renewer) renewAuth() error { + if !r.secret.Auth.Renewable || r.secret.Auth.ClientToken == "" { + return ErrRenewerNotRenewable + } + + client, token := r.client, r.secret.Auth.ClientToken + + for { + // Check if we are stopped. + select { + case <-r.stopCh: + return nil + default: + } + + // Renew the auth. + renewal, err := client.Auth().Token().RenewTokenAsSelf(token, 0) + if err != nil { + return err + } + + // Push a message that a renewal took place. + select { + case r.renewCh <- &RenewOutput{time.Now().UTC(), renewal}: + default: + } + + // Somehow, sometimes, this happens. + if renewal == nil || renewal.Auth == nil { + return ErrRenewerNoSecretData + } + + // Do nothing if we are not renewable + if !renewal.Auth.Renewable { + return ErrRenewerNotRenewable + } + + // Grab the lease duration and sleep duration - note that we grab the auth + // lease duration, not the secret lease duration. + leaseDuration := time.Duration(renewal.Auth.LeaseDuration) * time.Second + sleepDuration := r.sleepDuration(leaseDuration) + + // If we are within grace, return now. + if leaseDuration <= r.grace || sleepDuration <= r.grace { + return nil + } + + select { + case <-r.stopCh: + return nil + case <-time.After(sleepDuration): + continue + } + } +} + +// renewLease is a helper for renewing a lease. +func (r *Renewer) renewLease() error { + if !r.secret.Renewable || r.secret.LeaseID == "" { + return ErrRenewerNotRenewable + } + + client, leaseID := r.client, r.secret.LeaseID + + for { + // Check if we are stopped. + select { + case <-r.stopCh: + return nil + default: + } + + // Renew the lease. + renewal, err := client.Sys().Renew(leaseID, 0) + if err != nil { + return err + } + + // Push a message that a renewal took place. + select { + case r.renewCh <- &RenewOutput{time.Now().UTC(), renewal}: + default: + } + + // Somehow, sometimes, this happens. + if renewal == nil { + return ErrRenewerNoSecretData + } + + // Do nothing if we are not renewable + if !renewal.Renewable { + return ErrRenewerNotRenewable + } + + // Grab the lease duration and sleep duration + leaseDuration := time.Duration(renewal.LeaseDuration) * time.Second + sleepDuration := r.sleepDuration(leaseDuration) + + // If we are within grace, return now. + if leaseDuration <= r.grace || sleepDuration <= r.grace { + return nil + } + + select { + case <-r.stopCh: + return nil + case <-time.After(sleepDuration): + continue + } + } +} + +// sleepDuration calculates the time to sleep given the base lease duration. The +// base is the resulting lease duration. It will be reduced to 1/3 and +// multiplied by a random float between 0.0 and 1.0. This extra randomness +// prevents multiple clients from all trying to renew simultaneously. +func (r *Renewer) sleepDuration(base time.Duration) time.Duration { + sleep := float64(base) + + // Renew at 1/3 the remaining lease. This will give us an opportunity to retry + // at least one more time should the first renewal fail. + sleep = sleep / 3.0 + + // Use a randomness so many clients do not hit Vault simultaneously. + sleep = sleep * (r.random.Float64() + 1) / 2.0 + + return time.Duration(sleep) +} diff --git a/vendor/github.com/hashicorp/vault/api/request.go b/vendor/github.com/hashicorp/vault/api/request.go index 8f22dd572565..83a28bd9f653 100644 --- a/vendor/github.com/hashicorp/vault/api/request.go +++ b/vendor/github.com/hashicorp/vault/api/request.go @@ -14,6 +14,7 @@ type Request struct { Method string URL *url.URL Params url.Values + Headers http.Header ClientToken string WrapTTL string Obj interface{} @@ -55,10 +56,19 @@ func (r *Request) ToHTTP() (*http.Request, error) { return nil, err } + req.URL.User = r.URL.User req.URL.Scheme = r.URL.Scheme req.URL.Host = r.URL.Host req.Host = r.URL.Host + if r.Headers != nil { + for header, vals := range r.Headers { + for _, val := range vals { + req.Header.Add(header, val) + } + } + } + if len(r.ClientToken) != 0 { req.Header.Set("X-Vault-Token", r.ClientToken) } diff --git a/vendor/github.com/hashicorp/vault/api/response.go b/vendor/github.com/hashicorp/vault/api/response.go index 7c8ac9f97211..05502e1b0f62 100644 --- a/vendor/github.com/hashicorp/vault/api/response.go +++ b/vendor/github.com/hashicorp/vault/api/response.go @@ -25,8 +25,9 @@ func (r *Response) DecodeJSON(out interface{}) error { // this will fully consume the response body, but will not close it. The // body must still be closed manually. func (r *Response) Error() error { - // 200 to 399 are okay status codes - if r.StatusCode >= 200 && r.StatusCode < 400 { + // 200 to 399 are okay status codes. 429 is the code for health status of + // standby nodes. + if (r.StatusCode >= 200 && r.StatusCode < 400) || r.StatusCode == 429 { return nil } diff --git a/vendor/github.com/hashicorp/vault/api/sys_audit.go b/vendor/github.com/hashicorp/vault/api/sys_audit.go index 1ffdef880fe8..89f2141664af 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_audit.go +++ b/vendor/github.com/hashicorp/vault/api/sys_audit.go @@ -3,6 +3,7 @@ package api import ( "fmt" + "github.com/fatih/structs" "github.com/mitchellh/mapstructure" ) @@ -71,13 +72,18 @@ func (c *Sys) ListAudit() (map[string]*Audit, error) { return mounts, nil } +// DEPRECATED: Use EnableAuditWithOptions instead func (c *Sys) EnableAudit( path string, auditType string, desc string, opts map[string]string) error { - body := map[string]interface{}{ - "type": auditType, - "description": desc, - "options": opts, - } + return c.EnableAuditWithOptions(path, &EnableAuditOptions{ + Type: auditType, + Description: desc, + Options: opts, + }) +} + +func (c *Sys) EnableAuditWithOptions(path string, options *EnableAuditOptions) error { + body := structs.Map(options) r := c.c.NewRequest("PUT", fmt.Sprintf("/v1/sys/audit/%s", path)) if err := r.SetJSONBody(body); err != nil { @@ -106,9 +112,17 @@ func (c *Sys) DisableAudit(path string) error { // individually documented because the map almost directly to the raw HTTP API // documentation. Please refer to that documentation for more details. +type EnableAuditOptions struct { + Type string `json:"type" structs:"type"` + Description string `json:"description" structs:"description"` + Options map[string]string `json:"options" structs:"options"` + Local bool `json:"local" structs:"local"` +} + type Audit struct { Path string Type string Description string Options map[string]string + Local bool } diff --git a/vendor/github.com/hashicorp/vault/api/sys_auth.go b/vendor/github.com/hashicorp/vault/api/sys_auth.go index 1940e84172c2..fd55e429e620 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_auth.go +++ b/vendor/github.com/hashicorp/vault/api/sys_auth.go @@ -3,6 +3,7 @@ package api import ( "fmt" + "github.com/fatih/structs" "github.com/mitchellh/mapstructure" ) @@ -42,11 +43,16 @@ func (c *Sys) ListAuth() (map[string]*AuthMount, error) { return mounts, nil } +// DEPRECATED: Use EnableAuthWithOptions instead func (c *Sys) EnableAuth(path, authType, desc string) error { - body := map[string]string{ - "type": authType, - "description": desc, - } + return c.EnableAuthWithOptions(path, &EnableAuthOptions{ + Type: authType, + Description: desc, + }) +} + +func (c *Sys) EnableAuthWithOptions(path string, options *EnableAuthOptions) error { + body := structs.Map(options) r := c.c.NewRequest("POST", fmt.Sprintf("/v1/sys/auth/%s", path)) if err := r.SetJSONBody(body); err != nil { @@ -75,13 +81,23 @@ func (c *Sys) DisableAuth(path string) error { // individually documentd because the map almost directly to the raw HTTP API // documentation. Please refer to that documentation for more details. +type EnableAuthOptions struct { + Type string `json:"type" structs:"type"` + Description string `json:"description" structs:"description"` + Local bool `json:"local" structs:"local"` + PluginName string `json:"plugin_name,omitempty" structs:"plugin_name,omitempty" mapstructure:"plugin_name"` +} + type AuthMount struct { Type string `json:"type" structs:"type" mapstructure:"type"` Description string `json:"description" structs:"description" mapstructure:"description"` + Accessor string `json:"accessor" structs:"accessor" mapstructure:"accessor"` Config AuthConfigOutput `json:"config" structs:"config" mapstructure:"config"` + Local bool `json:"local" structs:"local" mapstructure:"local"` } type AuthConfigOutput struct { - DefaultLeaseTTL int `json:"default_lease_ttl" structs:"default_lease_ttl" mapstructure:"default_lease_ttl"` - MaxLeaseTTL int `json:"max_lease_ttl" structs:"max_lease_ttl" mapstructure:"max_lease_ttl"` + DefaultLeaseTTL int `json:"default_lease_ttl" structs:"default_lease_ttl" mapstructure:"default_lease_ttl"` + MaxLeaseTTL int `json:"max_lease_ttl" structs:"max_lease_ttl" mapstructure:"max_lease_ttl"` + PluginName string `json:"plugin_name,omitempty" structs:"plugin_name,omitempty" mapstructure:"plugin_name"` } diff --git a/vendor/github.com/hashicorp/vault/api/sys_config_cors.go b/vendor/github.com/hashicorp/vault/api/sys_config_cors.go new file mode 100644 index 000000000000..e7f2a59453c7 --- /dev/null +++ b/vendor/github.com/hashicorp/vault/api/sys_config_cors.go @@ -0,0 +1,56 @@ +package api + +func (c *Sys) CORSStatus() (*CORSResponse, error) { + r := c.c.NewRequest("GET", "/v1/sys/config/cors") + resp, err := c.c.RawRequest(r) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result CORSResponse + err = resp.DecodeJSON(&result) + return &result, err +} + +func (c *Sys) ConfigureCORS(req *CORSRequest) (*CORSResponse, error) { + r := c.c.NewRequest("PUT", "/v1/sys/config/cors") + if err := r.SetJSONBody(req); err != nil { + return nil, err + } + + resp, err := c.c.RawRequest(r) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result CORSResponse + err = resp.DecodeJSON(&result) + return &result, err +} + +func (c *Sys) DisableCORS() (*CORSResponse, error) { + r := c.c.NewRequest("DELETE", "/v1/sys/config/cors") + + resp, err := c.c.RawRequest(r) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result CORSResponse + err = resp.DecodeJSON(&result) + return &result, err + +} + +type CORSRequest struct { + AllowedOrigins string `json:"allowed_origins"` + Enabled bool `json:"enabled"` +} + +type CORSResponse struct { + AllowedOrigins string `json:"allowed_origins"` + Enabled bool `json:"enabled"` +} diff --git a/vendor/github.com/hashicorp/vault/api/sys_health.go b/vendor/github.com/hashicorp/vault/api/sys_health.go new file mode 100644 index 000000000000..5c0884a4153b --- /dev/null +++ b/vendor/github.com/hashicorp/vault/api/sys_health.go @@ -0,0 +1,24 @@ +package api + +func (c *Sys) Health() (*HealthResponse, error) { + r := c.c.NewRequest("GET", "/v1/sys/health") + resp, err := c.c.RawRequest(r) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var result HealthResponse + err = resp.DecodeJSON(&result) + return &result, err +} + +type HealthResponse struct { + Initialized bool `json:"initialized"` + Sealed bool `json:"sealed"` + Standby bool `json:"standby"` + ServerTimeUTC int64 `json:"server_time_utc"` + Version string `json:"version"` + ClusterName string `json:"cluster_name,omitempty"` + ClusterID string `json:"cluster_id,omitempty"` +} diff --git a/vendor/github.com/hashicorp/vault/api/sys_leader.go b/vendor/github.com/hashicorp/vault/api/sys_leader.go index 201ac732e2a3..4951c46e1809 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_leader.go +++ b/vendor/github.com/hashicorp/vault/api/sys_leader.go @@ -14,7 +14,8 @@ func (c *Sys) Leader() (*LeaderResponse, error) { } type LeaderResponse struct { - HAEnabled bool `json:"ha_enabled"` - IsSelf bool `json:"is_self"` - LeaderAddress string `json:"leader_address"` + HAEnabled bool `json:"ha_enabled"` + IsSelf bool `json:"is_self"` + LeaderAddress string `json:"leader_address"` + LeaderClusterAddress string `json:"leader_cluster_address"` } diff --git a/vendor/github.com/hashicorp/vault/api/sys_lease.go b/vendor/github.com/hashicorp/vault/api/sys_leases.go similarity index 76% rename from vendor/github.com/hashicorp/vault/api/sys_lease.go rename to vendor/github.com/hashicorp/vault/api/sys_leases.go index e5c19c42cda9..34bd99e65223 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_lease.go +++ b/vendor/github.com/hashicorp/vault/api/sys_leases.go @@ -1,7 +1,7 @@ package api func (c *Sys) Renew(id string, increment int) (*Secret, error) { - r := c.c.NewRequest("PUT", "/v1/sys/renew") + r := c.c.NewRequest("PUT", "/v1/sys/leases/renew") body := map[string]interface{}{ "increment": increment, @@ -21,7 +21,7 @@ func (c *Sys) Renew(id string, increment int) (*Secret, error) { } func (c *Sys) Revoke(id string) error { - r := c.c.NewRequest("PUT", "/v1/sys/revoke/"+id) + r := c.c.NewRequest("PUT", "/v1/sys/leases/revoke/"+id) resp, err := c.c.RawRequest(r) if err == nil { defer resp.Body.Close() @@ -30,7 +30,7 @@ func (c *Sys) Revoke(id string) error { } func (c *Sys) RevokePrefix(id string) error { - r := c.c.NewRequest("PUT", "/v1/sys/revoke-prefix/"+id) + r := c.c.NewRequest("PUT", "/v1/sys/leases/revoke-prefix/"+id) resp, err := c.c.RawRequest(r) if err == nil { defer resp.Body.Close() @@ -39,7 +39,7 @@ func (c *Sys) RevokePrefix(id string) error { } func (c *Sys) RevokeForce(id string) error { - r := c.c.NewRequest("PUT", "/v1/sys/revoke-force/"+id) + r := c.c.NewRequest("PUT", "/v1/sys/leases/revoke-force/"+id) resp, err := c.c.RawRequest(r) if err == nil { defer resp.Body.Close() diff --git a/vendor/github.com/hashicorp/vault/api/sys_mounts.go b/vendor/github.com/hashicorp/vault/api/sys_mounts.go index ca5e42707a2c..e0bb9ff0d538 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_mounts.go +++ b/vendor/github.com/hashicorp/vault/api/sys_mounts.go @@ -123,20 +123,27 @@ type MountInput struct { Type string `json:"type" structs:"type"` Description string `json:"description" structs:"description"` Config MountConfigInput `json:"config" structs:"config"` + Local bool `json:"local" structs:"local"` } type MountConfigInput struct { DefaultLeaseTTL string `json:"default_lease_ttl" structs:"default_lease_ttl" mapstructure:"default_lease_ttl"` MaxLeaseTTL string `json:"max_lease_ttl" structs:"max_lease_ttl" mapstructure:"max_lease_ttl"` + ForceNoCache bool `json:"force_no_cache" structs:"force_no_cache" mapstructure:"force_no_cache"` + PluginName string `json:"plugin_name,omitempty" structs:"plugin_name,omitempty" mapstructure:"plugin_name"` } type MountOutput struct { Type string `json:"type" structs:"type"` Description string `json:"description" structs:"description"` + Accessor string `json:"accessor" structs:"accessor"` Config MountConfigOutput `json:"config" structs:"config"` + Local bool `json:"local" structs:"local"` } type MountConfigOutput struct { - DefaultLeaseTTL int `json:"default_lease_ttl" structs:"default_lease_ttl" mapstructure:"default_lease_ttl"` - MaxLeaseTTL int `json:"max_lease_ttl" structs:"max_lease_ttl" mapstructure:"max_lease_ttl"` + DefaultLeaseTTL int `json:"default_lease_ttl" structs:"default_lease_ttl" mapstructure:"default_lease_ttl"` + MaxLeaseTTL int `json:"max_lease_ttl" structs:"max_lease_ttl" mapstructure:"max_lease_ttl"` + ForceNoCache bool `json:"force_no_cache" structs:"force_no_cache" mapstructure:"force_no_cache"` + PluginName string `json:"plugin_name,omitempty" structs:"plugin_name,omitempty" mapstructure:"plugin_name"` } diff --git a/vendor/github.com/hashicorp/vault/api/sys_seal.go b/vendor/github.com/hashicorp/vault/api/sys_seal.go index b80e33a940c1..97a49aeb4401 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_seal.go +++ b/vendor/github.com/hashicorp/vault/api/sys_seal.go @@ -53,6 +53,7 @@ type SealStatusResponse struct { T int `json:"t"` N int `json:"n"` Progress int `json:"progress"` + Nonce string `json:"nonce"` Version string `json:"version"` ClusterName string `json:"cluster_name,omitempty"` ClusterID string `json:"cluster_id,omitempty"` diff --git a/vendor/github.com/hashicorp/vault/helper/jsonutil/json.go b/vendor/github.com/hashicorp/vault/helper/jsonutil/json.go index c237db0f64dd..a96745be8f61 100644 --- a/vendor/github.com/hashicorp/vault/helper/jsonutil/json.go +++ b/vendor/github.com/hashicorp/vault/helper/jsonutil/json.go @@ -2,13 +2,19 @@ package jsonutil import ( "bytes" + "compress/gzip" "encoding/json" "fmt" "io" + + "github.com/hashicorp/vault/helper/compressutil" ) // Encodes/Marshals the given object into JSON func EncodeJSON(in interface{}) ([]byte, error) { + if in == nil { + return nil, fmt.Errorf("input for encoding is nil") + } var buf bytes.Buffer enc := json.NewEncoder(&buf) if err := enc.Encode(in); err != nil { @@ -17,15 +23,60 @@ func EncodeJSON(in interface{}) ([]byte, error) { return buf.Bytes(), nil } -// Decodes/Unmarshals the given JSON into a desired object +// EncodeJSONAndCompress encodes the given input into JSON and compresses the +// encoded value (using Gzip format BestCompression level, by default). A +// canary byte is placed at the beginning of the returned bytes for the logic +// in decompression method to identify compressed input. +func EncodeJSONAndCompress(in interface{}, config *compressutil.CompressionConfig) ([]byte, error) { + if in == nil { + return nil, fmt.Errorf("input for encoding is nil") + } + + // First JSON encode the given input + encodedBytes, err := EncodeJSON(in) + if err != nil { + return nil, err + } + + if config == nil { + config = &compressutil.CompressionConfig{ + Type: compressutil.CompressionTypeGzip, + GzipCompressionLevel: gzip.BestCompression, + } + } + + return compressutil.Compress(encodedBytes, config) +} + +// DecodeJSON tries to decompress the given data. The call to decompress, fails +// if the content was not compressed in the first place, which is identified by +// a canary byte before the compressed data. If the data is not compressed, it +// is JSON decoded directly. Otherwise the decompressed data will be JSON +// decoded. func DecodeJSON(data []byte, out interface{}) error { - if data == nil { + if data == nil || len(data) == 0 { return fmt.Errorf("'data' being decoded is nil") } if out == nil { return fmt.Errorf("output parameter 'out' is nil") } + // Decompress the data if it was compressed in the first place + decompressedBytes, uncompressed, err := compressutil.Decompress(data) + if err != nil { + return fmt.Errorf("failed to decompress JSON: err: %v", err) + } + if !uncompressed && (decompressedBytes == nil || len(decompressedBytes) == 0) { + return fmt.Errorf("decompressed data being decoded is invalid") + } + + // If the input supplied failed to contain the compression canary, it + // will be notified by the compression utility. Decode the decompressed + // input. + if !uncompressed { + data = decompressedBytes + } + return DecodeJSONFromReader(bytes.NewReader(data), out) } diff --git a/vendor/vendor.json b/vendor/vendor.json index 2c91615bcfa7..d180973f5987 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -611,44 +611,44 @@ { "checksumSHA1": "Nu2j1GusM7ZH0uYrGzqr1K7yH7I=", "path": "github.com/hashicorp/consul-template/child", - "revision": "ecbc27c1922fed2f562e7fb63e1ad24e818fa60e", - "revisionTime": "2017-07-05T14:04:00Z" + "revision": "7b3f45039cf3ad1a758683fd3eebb1cc72affa06", + "revisionTime": "2017-08-01T00:58:49Z" }, { - "checksumSHA1": "QWcGW3wELSp/YsOVzCW02oEYR7c=", + "checksumSHA1": "lemUzh6uQDMxuvTT/BREYdGcS0U=", "path": "github.com/hashicorp/consul-template/config", - "revision": "ecbc27c1922fed2f562e7fb63e1ad24e818fa60e", - "revisionTime": "2017-07-05T14:04:00Z" + "revision": "7b3f45039cf3ad1a758683fd3eebb1cc72affa06", + "revisionTime": "2017-08-01T00:58:49Z" }, { - "checksumSHA1": "mV7yjHpIfO4yRAdQaBlAqdGDKO8=", + "checksumSHA1": "WVZ+pqn/HLLXjj+Tj5ZZvD7w6r0=", "path": "github.com/hashicorp/consul-template/dependency", - "revision": "ecbc27c1922fed2f562e7fb63e1ad24e818fa60e", - "revisionTime": "2017-07-05T14:04:00Z" + "revision": "7b3f45039cf3ad1a758683fd3eebb1cc72affa06", + "revisionTime": "2017-08-01T00:58:49Z" }, { - "checksumSHA1": "ZTlPhrxNzME75A4ydXM88TFt3Qs=", + "checksumSHA1": "Cu8hIII8Z6FAuunFI/jXPLl0nQA=", "path": "github.com/hashicorp/consul-template/manager", - "revision": "ecbc27c1922fed2f562e7fb63e1ad24e818fa60e", - "revisionTime": "2017-07-05T14:04:00Z" + "revision": "7b3f45039cf3ad1a758683fd3eebb1cc72affa06", + "revisionTime": "2017-08-01T00:58:49Z" }, { "checksumSHA1": "oskgb0WteBKOItG8NNDduM7E/D0=", "path": "github.com/hashicorp/consul-template/signals", - "revision": "ecbc27c1922fed2f562e7fb63e1ad24e818fa60e", - "revisionTime": "2017-07-05T14:04:00Z" + "revision": "7b3f45039cf3ad1a758683fd3eebb1cc72affa06", + "revisionTime": "2017-08-01T00:58:49Z" }, { "checksumSHA1": "zSvJlNfZS3fCRlFaZ7r9Q+N17T8=", "path": "github.com/hashicorp/consul-template/template", - "revision": "ecbc27c1922fed2f562e7fb63e1ad24e818fa60e", - "revisionTime": "2017-07-05T14:04:00Z" + "revision": "7b3f45039cf3ad1a758683fd3eebb1cc72affa06", + "revisionTime": "2017-08-01T00:58:49Z" }, { - "checksumSHA1": "85W96Fo50FmrMaba7Dk12aDfwWs=", + "checksumSHA1": "b4+Y+02pY2Y5620F9ALzKg8Zmdw=", "path": "github.com/hashicorp/consul-template/watch", - "revision": "ecbc27c1922fed2f562e7fb63e1ad24e818fa60e", - "revisionTime": "2017-07-05T14:04:00Z" + "revision": "7b3f45039cf3ad1a758683fd3eebb1cc72affa06", + "revisionTime": "2017-08-01T00:58:49Z" }, { "checksumSHA1": "jfELEMRhiTcppZmRH+ZwtkVS5Uw=", @@ -911,16 +911,16 @@ "revisionTime": "2016-08-21T23:40:57Z" }, { - "checksumSHA1": "31yBeS6U3xm7VJ7ZvDxRgBxXP0A=", + "checksumSHA1": "hLIXn9iQhPcjY+/G64j3mIlLlK8=", "path": "github.com/hashicorp/vault/api", - "revision": "f4adc7fa960ed8e828f94bc6785bcdbae8d1b263", - "revisionTime": "2016-12-16T21:07:16Z" + "revision": "0c3e14f047aede0a70256e1e8b321610910b246e", + "revisionTime": "2017-08-01T15:50:41Z" }, { - "checksumSHA1": "5lR6EdY0ARRdKAq3hZcL38STD8Q=", + "checksumSHA1": "yUiSTPf0QUuL2r/81sjuytqBoeQ=", "path": "github.com/hashicorp/vault/helper/jsonutil", - "revision": "bcf98fa8d61d1870c3af689f1b090b29a9c12d8c", - "revisionTime": "2016-08-02T20:35:37Z" + "revision": "0c3e14f047aede0a70256e1e8b321610910b246e", + "revisionTime": "2017-08-01T15:50:41Z" }, { "checksumSHA1": "VMaF3Q7RIrRzvbnPbqxuSLryOvc=", diff --git a/website/source/api/json-jobs.html.md b/website/source/api/json-jobs.html.md index c83f58b4367d..fad418b8427b 100644 --- a/website/source/api/json-jobs.html.md +++ b/website/source/api/json-jobs.html.md @@ -734,6 +734,14 @@ README][ct]. splay value before invoking the change mode. Should be specified in nanoseconds. +- `VaultGrace` - Specifies the grace period between lease renewal and secret + re-acquisition. When renewing a secret, if the remaining lease is less than or + equal to the configured grace, the template will request a new credential. + This prevents Vault from revoking the secret at its expiration and the task + having a stale secret. If the grace is set to a value that is higher than your + default TTL or max TTL, the template will always read a new secret. If the + task defines several templates, the `vault_grace` will be set to the lowest + value across all the templates. ```json { diff --git a/website/source/docs/job-specification/template.html.md b/website/source/docs/job-specification/template.html.md index e83a20416395..c9924a6f5909 100644 --- a/website/source/docs/job-specification/template.html.md +++ b/website/source/docs/job-specification/template.html.md @@ -94,6 +94,15 @@ README][ct]. Since Nomad v0.6.0, templates can be read as environment variables. prevent a thundering herd problem where all task instances restart at the same time. +- `vault_grace` `(string: "5m")` - Specifies the grace period between lease + renewal and secret re-acquisition. When renewing a secret, if the remaining + lease is less than or equal to the configured grace, the template will request + a new credential. This prevents Vault from revoking the secret at its + expiration and the task having a stale secret. If the grace is set to a value + that is higher than your default TTL or max TTL, the template will always read + a new secret. If the task defines several templates, the `vault_grace` will be + set to the lowest value across all the templates. + ## `template` Examples The following examples only show the `template` stanzas. Remember that the