Merge branch 'master-oss' into ts-revoke-salted-opt

CHANGELOG.md

@@ -1,9 +1,39 @@
 ## 0.10.2 (Unreleased)
 
+DEPRECATIONS/CHANGES:
+
+ * As of this release, the Vault CLI (via `vault unwrap`) and Go API (via
+   `Logical().Unwrap()`) can no longer unwrap response-wrapped tokens produced
+   by Vault prior to 0.6.2. These can still be read manually by performing a
+   read on `cubbyhole/response` and decoding the JSON-encoded value.
+ * PKI duration return types: The PKI backend now returns durations (e.g. when
+   reading a role) as an integer number of seconds instead of a Go-style
+   string, in line with how the rest of Vault's API returns durations.
+
 IMPROVEMENTS:
 
+ * api: Close renewer's doneCh when the renewer is stopped, so that programs
+   expecting a final value through doneCh behave correctly [GH-4472]
  * cli: `vault login` now supports a `-no-print` flag to suppress printing
    token information but still allow storing into the token helper [GH-4454]
+ * core/pkcs11 (enterprise): Add support for CKM_AES_CBS_PAD, CKM_RSA_PKCS, and 
+   CKM_RSA_PKCS_OAEP mechanisms
+ * core/pkcs11 (enterprise): HSM slots can now be selected by token label instead
+   of just slot number
+ * core/seal (enterprise): Lazily rewrap data when seal keys are rotated
+ * expiration: Allow revoke-prefix and revoke-force to work on single leases as
+   well as prefixes [GH-4450]
+
+BUG FIXES:
+
+ * auth/approle: Make invalid role_id a 400 error instead of 500 [GH-4470]
+ * auth/cert: Fix Identity alias using serial number instead of common name
+   [GH-4475]
+ * core: When using the `use_always` option with PROXY protocol support, do not
+   require `authorized_addrs` to be set [GH-4065]
+ * secret/kv: Fix response wrapping for KV v2 [GH-4511]
+ * secret/pki: Fix path length parameter being ignored when using
+   `use_csr_values` and signing an intermediate CA cert [GH-4459]
 
 ## 0.10.1/0.9.7 (April 25th, 2018)
 

Makefile

@@ -102,17 +102,16 @@ static-assets:
 
 test-ember:
 	@echo "--> Installing JavaScript assets"
-	@cd ui && yarn install && bower install && yarn install phantomjs-prebuilt
+	@cd ui && yarn
 	@echo "--> Running ember tests"
-	@cd ui && node_modules/phantomjs-prebuilt/bin/phantomjs --version
-	@cd ui && npm test
+	@cd ui && yarn run test-oss
 
 ember-dist:
 	@echo "--> Installing JavaScript assets"
-	@cd ui && yarn install && bower install --allow-root
+	@cd ui && yarn
 	@cd ui && npm rebuild node-sass
 	@echo "--> Building Ember application"
-	@cd ui && npm run build
+	@cd ui && yarn run build
 	@rm -rf ui/if-you-need-to-delete-this-open-an-issue-async-disk-cache
 
 static-dist: ember-dist static-assets

api/client.go

@@ -16,9 +16,9 @@ import (
 
 	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/go-cleanhttp"
+	retryablehttp "github.com/hashicorp/go-retryablehttp"
 	"github.com/hashicorp/go-rootcerts"
 	"github.com/hashicorp/vault/helper/parseutil"
-	"github.com/sethgrid/pester"
 	"golang.org/x/net/http2"
 )
 
@@ -59,8 +59,9 @@ type Config struct {
 	// (or http.DefaultClient).
 	HttpClient *http.Client
 
-	// MaxRetries controls the maximum number of times to retry when a 5xx error
-	// occurs. Set to 0 or less to disable retrying. Defaults to 0.
+	// MaxRetries controls the maximum number of times to retry when a 5xx
+	// error occurs. Set to 0 to disable retrying. Defaults to 2 (for a total
+	// of three tries).
 	MaxRetries int
 
 	// Timeout is for setting custom timeout parameter in the HttpClient
@@ -69,6 +70,9 @@ type Config struct {
 	// If there is an error when creating the configuration, this will be the
 	// error
 	Error error
+
+	// The Backoff function to use; a default is used if not provided
+	Backoff retryablehttp.Backoff
 }
 
 // TLSConfig contains the parameters needed to configure TLS on the HTTP client
@@ -131,12 +135,15 @@ func DefaultConfig() *Config {
 	// but in e.g. http_test actual redirect handling is necessary
 	config.HttpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
 		// Returning this value causes the Go net library to not close the
-		// response body and to nil out the error. Otherwise pester tries
-		// three times on every redirect because it sees an error from this
+		// response body and to nil out the error. Otherwise retry clients may
+		// try three times on every redirect because it sees an error from this
 		// function (to prevent redirects) passing through to it.
 		return http.ErrUseLastResponse
 	}
 
+	config.Backoff = retryablehttp.LinearJitterBackoff
+	config.MaxRetries = 2
+
 	return config
 }
 
@@ -269,7 +276,7 @@ func (c *Config) ReadEnvironment() error {
 	}
 
 	if envMaxRetries != nil {
-		c.MaxRetries = int(*envMaxRetries) + 1
+		c.MaxRetries = int(*envMaxRetries)
 	}
 
 	if envClientTimeout != 0 {
@@ -382,6 +389,15 @@ func (c *Client) SetClientTimeout(timeout time.Duration) {
 	c.config.Timeout = timeout
 }
 
+// CurrentWrappingLookupFunc sets a lookup function that returns desired wrap TTLs
+// for a given operation and path
+func (c *Client) CurrentWrappingLookupFunc() WrappingLookupFunc {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+
+	return c.wrappingLookupFunc
+}
+
 // SetWrappingLookupFunc sets a lookup function that returns desired wrap TTLs
 // for a given operation and path
 func (c *Client) SetWrappingLookupFunc(lookupFunc WrappingLookupFunc) {
@@ -434,6 +450,16 @@ func (c *Client) SetHeaders(headers http.Header) {
 	c.headers = headers
 }
 
+// SetBackoff sets the backoff function to be used for future requests.
+func (c *Client) SetBackoff(backoff retryablehttp.Backoff) {
+	c.modifyLock.RLock()
+	c.config.modifyLock.Lock()
+	defer c.config.modifyLock.Unlock()
+	c.modifyLock.RUnlock()
+
+	c.config.Backoff = backoff
+}
+
 // Clone creates a new client with the same configuration. Note that the same
 // underlying http.Client is used; modifying the client from more than one
 // goroutine at once may not be safe, so modify the client as needed and then
@@ -449,6 +475,7 @@ func (c *Client) Clone() (*Client, error) {
 		HttpClient: config.HttpClient,
 		MaxRetries: config.MaxRetries,
 		Timeout:    config.Timeout,
+		Backoff:    config.Backoff,
 	}
 	config.modifyLock.RUnlock()
 
@@ -544,14 +571,28 @@ func (c *Client) RawRequest(r *Request) (*Response, error) {
 
 	redirectCount := 0
 START:
-	req, err := r.ToHTTP()
+	req, err := r.toRetryableHTTP(false)
 	if err != nil {
 		return nil, err
 	}
+	if req == nil {
+		return nil, fmt.Errorf("nil request created")
+	}
+
+	backoff := c.config.Backoff
+	if backoff == nil {
+		backoff = retryablehttp.LinearJitterBackoff
+	}
 
-	client := pester.NewExtendedClient(c.config.HttpClient)
-	client.Backoff = pester.LinearJitterBackoff
-	client.MaxRetries = c.config.MaxRetries
+	client := &retryablehttp.Client{
+		HTTPClient:   c.config.HttpClient,
+		RetryWaitMin: 1000 * time.Millisecond,
+		RetryWaitMax: 1500 * time.Millisecond,
+		RetryMax:     c.config.MaxRetries,
+		CheckRetry:   retryablehttp.DefaultRetryPolicy,
+		Backoff:      backoff,
+		ErrorHandler: retryablehttp.PassthroughErrorHandler,
+	}
 
 	var result *Response
 	resp, err := client.Do(req)

api/logical.go

@@ -1,14 +1,8 @@
 package api
 
 import (
-	"bytes"
-	"fmt"
 	"io"
-	"net/http"
 	"os"
-
-	"github.com/hashicorp/errwrap"
-	"github.com/hashicorp/vault/helper/jsonutil"
 )
 
 const (
@@ -188,49 +182,23 @@ func (c *Logical) Unwrap(wrappingToken string) (*Secret, error) {
 	if resp != nil {
 		defer resp.Body.Close()
 	}
-
-	// Return all errors except those that are from a 404 as we handle the not
-	// found error as a special case.
-	if err != nil && (resp == nil || resp.StatusCode != 404) {
-		return nil, err
-	}
-	if resp == nil {
-		return nil, nil
-	}
-
-	switch resp.StatusCode {
-	case http.StatusOK: // New method is supported
-		return ParseSecret(resp.Body)
-	case http.StatusNotFound: // Fall back to old method
-	default:
+	if resp != nil && resp.StatusCode == 404 {
+		secret, parseErr := ParseSecret(resp.Body)
+		switch parseErr {
+		case nil:
+		case io.EOF:
+			return nil, nil
+		default:
+			return nil, err
+		}
+		if secret != nil && (len(secret.Warnings) > 0 || len(secret.Data) > 0) {
+			return secret, nil
+		}
 		return nil, nil
 	}
-
-	if wrappingToken != "" {
-		origToken := c.c.Token()
-		defer c.c.SetToken(origToken)
-		c.c.SetToken(wrappingToken)
-	}
-
-	secret, err := c.Read(wrappedResponseLocation)
 	if err != nil {
-		return nil, errwrap.Wrapf(fmt.Sprintf("error reading %q: {{err}}", wrappedResponseLocation), err)
-	}
-	if secret == nil {
-		return nil, fmt.Errorf("no value found at %q", wrappedResponseLocation)
-	}
-	if secret.Data == nil {
-		return nil, fmt.Errorf("\"data\" not found in wrapping response")
-	}
-	if _, ok := secret.Data["response"]; !ok {
-		return nil, fmt.Errorf("\"response\" not found in wrapping response \"data\" map")
-	}
-
-	wrappedSecret := new(Secret)
-	buf := bytes.NewBufferString(secret.Data["response"].(string))
-	if err := jsonutil.DecodeJSONFromReader(buf, wrappedSecret); err != nil {
-		return nil, errwrap.Wrapf("error unmarshalling wrapped secret: {{err}}", err)
+		return nil, err
 	}
 
-	return wrappedSecret, nil
+	return ParseSecret(resp.Body)
 }

api/renewer.go

@@ -64,9 +64,7 @@ type RenewerInput struct {
 	// Secret is the secret to renew
 	Secret *Secret
 
-	// Grace is a minimum renewal before returning so the upstream client
-	// can do a re-read. This can be used to prevent clients from waiting
-	// too long to read a new credential and incur downtime.
+	// DEPRECATED: this does not do anything.
 	Grace time.Duration
 
 	// Rand is the randomizer to use for underlying randomization. If not
@@ -107,8 +105,6 @@ func (c *Client) NewRenewer(i *RenewerInput) (*Renewer, error) {
 		return nil, ErrRenewerMissingSecret
 	}
 
-	grace := i.Grace
-
 	random := i.Rand
 	if random == nil {
 		random = rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
@@ -122,7 +118,6 @@ func (c *Client) NewRenewer(i *RenewerInput) (*Renewer, error) {
 	return &Renewer{
 		client:    c,
 		secret:    secret,
-		grace:     grace,
 		increment: i.Increment,
 		random:    random,
 		doneCh:    make(chan error, 1),
@@ -166,10 +161,7 @@ func (r *Renewer) Renew() {
 		result = r.renewLease()
 	}
 
-	select {
-	case r.doneCh <- result:
-	case <-r.stopCh:
-	}
+	r.doneCh <- result
 }
 
 // renewAuth is a helper for renewing authentication.

api/renewer_integration_test.go

@@ -130,7 +130,9 @@ func TestRenewer_Renew(t *testing.T) {
 			defer v.Stop()
 
 			done, renewed := false, false
+			timeout := time.After(5 * time.Second)
 			for {
+
 				if done {
 					break
 				}
@@ -141,6 +143,8 @@ func TestRenewer_Renew(t *testing.T) {
 						if err != nil {
 							t.Fatalf("renewal failed with an error: %v", err)
 						}
+						// We can break out early here
+						done = true
 					} else {
 						t.Errorf("should have renewed once before returning: %s", err)
 					}
@@ -155,7 +159,7 @@ func TestRenewer_Renew(t *testing.T) {
 						t.Errorf("expected lease to <= 5s: %#v", renew)
 					}
 					renewed = true
-				case <-time.After(5 * time.Second):
+				case <-timeout:
 					if !renewed {
 						t.Errorf("no renewal")
 					}
@@ -186,6 +190,7 @@ func TestRenewer_Renew(t *testing.T) {
 			defer v.Stop()
 
 			renewed, done := false, false
+			timeout := time.After(10 * time.Second)
 			for {
 				if done {
 					break
@@ -197,6 +202,8 @@ func TestRenewer_Renew(t *testing.T) {
 						if err != nil {
 							t.Fatalf("renewal failed with an error: %v", err)
 						}
+						// We can break out early here
+						done = true
 					} else {
 						t.Errorf("should have renewed once before returning: %s", err)
 					}
@@ -221,7 +228,7 @@ func TestRenewer_Renew(t *testing.T) {
 						t.Error("expected an accessor")
 					}
 					renewed = true
-				case <-time.After(10 * time.Second):
+				case <-timeout:
 					if !renewed {
 						t.Errorf("no renewal")
 					}

api/renewer_test.go

@@ -3,7 +3,6 @@ package api
 import (
 	"reflect"
 	"testing"
-	"time"
 )
 
 func TestRenewer_NewRenewer(t *testing.T) {
@@ -44,18 +43,6 @@ func TestRenewer_NewRenewer(t *testing.T) {
 			},
 			false,
 		},
-		{
-			"custom_grace",
-			&RenewerInput{
-				Secret: &Secret{},
-				Grace:  30 * time.Second,
-			},
-			&Renewer{
-				secret: &Secret{},
-				grace:  30 * time.Second,
-			},
-			false,
-		},
 	}
 
 	for _, tc := range cases {