Merge remote-tracking branch 'origin/master' into invalid_aws_wal

.hooks/pre-push

@@ -6,6 +6,10 @@ if [ "$remote" = "enterprise" ]; then
     exit 0
 fi
 
+if [ "$remote" = "ent" ]; then
+    exit 0
+fi
+
 if [ -f version/version_ent.go ]; then
     echo "Found enterprise version file while pushing to oss remote"
     exit 1

CHANGELOG.md

@@ -1,13 +1,35 @@
 ## 0.11.2 (Unreleased)
 
+BUG FIXES:
+
+ * core: Re-add deprecated capabilities information for now [GH-5360]
+ * core: Fix handling of cyclic token relationships [GH-4803]
+ * storage/mysql: Fix locking on MariaDB [GH-5343]
+
+IMPROVEMENTS:
+
+ * plugins: Add `env` parameter when registering plugins to the catalog to allow
+   operators to include environment variables during plugin execution. [GH-5359]
+
+## 0.11.1.1 (September 17th, 2018) (Enterprise Only)
+
 BUG FIXES:
 
  * agent: Fix auth handler-based wrapping of output tokens [GH-5316]
+ * core: Properly store the replication checkpoint file if it's larger than the
+   storage engine's per-item limit
+ * core: Improve WAL deletion rate
+ * core: Fix token creation on performance standby nodes
+ * core: Fix unwrapping inside a namespace
+ * core: Always forward tidy operations from performance standby nodes
 
 IMPROVEMENTS:
 
- * auth/aws: add support for key/value pairs or JSON values for 
+ * auth/aws: add support for key/value pairs or JSON values for
    `iam_request_headers` with IAM auth method [GH-5320]
+ * auth/aws, secret/aws: Throttling errors from the AWS API will now be
+   reported as 502 errors by Vault, along with the original error [GH-5270]
+ * replication: Start fetching during a sync from where it previously errored
 
 ## 0.11.1 (September 6th, 2018)
 
@@ -40,8 +62,6 @@ BUG FIXES:
  * core: Ensure we use a background context when stepping down [GH-5290]
  * core: Properly check error return from random byte reading [GH-5277]
  * core: Re-add `sys/` top-route injection for now [GH-5241]
- * core: Properly store the replication checkpoint file if it's larger than the
-   storage engine's per-item limit
  * core: Policies stored in minified JSON would return an error [GH-5229]
  * core: Evaluate templated policies in capabilities check [GH-5250]
  * identity: Update MemDB with identity group alias while loading groups [GH-5289]

Makefile

@@ -92,7 +92,7 @@ vet:
 # source files.
 prep: fmtcheck
 	@sh -c "'$(CURDIR)/scripts/goversioncheck.sh' '$(GO_VERSION_MIN)'"
-	go generate $(go list ./... | grep -v /vendor/)
+	@go generate $(go list ./... | grep -v /vendor/)
 	@if [ -d .git/hooks ]; then cp .hooks/* .git/hooks/; fi
 
 # bootstrap the build by downloading additional tools
@@ -143,18 +143,20 @@ proto:
 	protoc helper/forwarding/types.proto --go_out=plugins=grpc:../../..
 	protoc logical/*.proto --go_out=plugins=grpc:../../..
 	protoc physical/types.proto --go_out=plugins=grpc:../../..
+	protoc helper/identity/mfa/types.proto --go_out=plugins=grpc:../../..
 	protoc helper/identity/types.proto --go_out=plugins=grpc:../../..
 	protoc builtin/logical/database/dbplugin/*.proto --go_out=plugins=grpc:../../..
 	protoc logical/plugin/pb/*.proto --go_out=plugins=grpc:../../..
-	sed -i -e 's/Idp/IDP/' -e 's/Url/URL/' -e 's/Id/ID/' -e 's/EntityId/EntityID/' -e 's/Api/API/' -e 's/Qr/QR/' -e 's/protobuf:"/sentinel:"" protobuf:"/' helper/identity/types.pb.go helper/storagepacker/types.pb.go logical/plugin/pb/backend.pb.go
+	sed -i '1s;^;// +build !enterprise\n;' physical/types.pb.go
+	sed -i '1s;^;// +build !enterprise\n;' helper/identity/mfa/types.pb.go
+	sed -i -e 's/Idp/IDP/' -e 's/Url/URL/' -e 's/Id/ID/' -e 's/IDentity/Identity/' -e 's/EntityId/EntityID/' -e 's/Api/API/' -e 's/Qr/QR/' -e 's/Totp/TOTP/' -e 's/Mfa/MFA/' -e 's/Pingid/PingID/' -e 's/protobuf:"/sentinel:"" protobuf:"/' -e 's/namespaceId/namespaceID/' -e 's/Ttl/TTL/' -e 's/BoundCidrs/BoundCIDRs/' helper/identity/types.pb.go helper/storagepacker/types.pb.go logical/plugin/pb/backend.pb.go logical/identity.pb.go
 
 fmtcheck:
 	@true
 #@sh -c "'$(CURDIR)/scripts/gofmtcheck.sh'"
 
 fmt:
-	@true
-#gofmt -w $(GOFMT_FILES)
+	gofmt -w $(GOFMT_FILES)
 
 spellcheck:
 	@echo "==> Spell checking website..."

api/sys_capabilities.go

@@ -50,5 +50,15 @@ func (c *Sys) Capabilities(token, path string) ([]string, error) {
 		return nil, err
 	}
 
+	if len(res) == 0 {
+		_, ok := secret.Data["capabilities"]
+		if ok {
+			err = mapstructure.Decode(secret.Data["capabilities"], &res)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
 	return res, nil
 }

audit/format.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/SermoDigital/jose/jws"
 	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/helper/salt"
 	"github.com/hashicorp/vault/logical"
 	"github.com/mitchellh/copystructure"
@@ -113,33 +114,43 @@ func (f *AuditFormatter) FormatRequest(ctx context.Context, w io.Writer, config
 		errString = in.OuterErr.Error()
 	}
 
+	ns, err := namespace.FromContext(ctx)
+	if err != nil {
+		return err
+	}
+
 	reqEntry := &AuditRequestEntry{
 		Type:  "request",
 		Error: errString,
 
 		Auth: AuditAuth{
-			ClientToken:      auth.ClientToken,
-			Accessor:         auth.Accessor,
-			DisplayName:      auth.DisplayName,
-			Policies:         auth.Policies,
-			TokenPolicies:    auth.TokenPolicies,
-			IdentityPolicies: auth.IdentityPolicies,
-			Metadata:         auth.Metadata,
-			EntityID:         auth.EntityID,
-			RemainingUses:    req.ClientTokenRemainingUses,
+			ClientToken:               auth.ClientToken,
+			Accessor:                  auth.Accessor,
+			DisplayName:               auth.DisplayName,
+			Policies:                  auth.Policies,
+			TokenPolicies:             auth.TokenPolicies,
+			IdentityPolicies:          auth.IdentityPolicies,
+			ExternalNamespacePolicies: auth.ExternalNamespacePolicies,
+			Metadata:                  auth.Metadata,
+			EntityID:                  auth.EntityID,
+			RemainingUses:             req.ClientTokenRemainingUses,
 		},
 
 		Request: AuditRequest{
 			ID:                  req.ID,
 			ClientToken:         req.ClientToken,
 			ClientTokenAccessor: req.ClientTokenAccessor,
 			Operation:           req.Operation,
-			Path:                req.Path,
-			Data:                req.Data,
-			PolicyOverride:      req.PolicyOverride,
-			RemoteAddr:          getRemoteAddr(req),
-			ReplicationCluster:  req.ReplicationCluster,
-			Headers:             req.Headers,
+			Namespace: AuditNamespace{
+				ID:   ns.ID,
+				Path: ns.Path,
+			},
+			Path:               req.Path,
+			Data:               req.Data,
+			PolicyOverride:     req.PolicyOverride,
+			RemoteAddr:         getRemoteAddr(req),
+			ReplicationCluster: req.ReplicationCluster,
+			Headers:            req.Headers,
 		},
 	}
 
@@ -276,17 +287,23 @@ func (f *AuditFormatter) FormatResponse(ctx context.Context, w io.Writer, config
 		errString = in.OuterErr.Error()
 	}
 
+	ns, err := namespace.FromContext(ctx)
+	if err != nil {
+		return err
+	}
+
 	var respAuth *AuditAuth
 	if resp.Auth != nil {
 		respAuth = &AuditAuth{
-			ClientToken:      resp.Auth.ClientToken,
-			Accessor:         resp.Auth.Accessor,
-			DisplayName:      resp.Auth.DisplayName,
-			Policies:         resp.Auth.Policies,
-			TokenPolicies:    resp.Auth.TokenPolicies,
-			IdentityPolicies: resp.Auth.IdentityPolicies,
-			Metadata:         resp.Auth.Metadata,
-			NumUses:          resp.Auth.NumUses,
+			ClientToken:               resp.Auth.ClientToken,
+			Accessor:                  resp.Auth.Accessor,
+			DisplayName:               resp.Auth.DisplayName,
+			Policies:                  resp.Auth.Policies,
+			TokenPolicies:             resp.Auth.TokenPolicies,
+			IdentityPolicies:          resp.Auth.IdentityPolicies,
+			ExternalNamespacePolicies: resp.Auth.ExternalNamespacePolicies,
+			Metadata:                  resp.Auth.Metadata,
+			NumUses:                   resp.Auth.NumUses,
 		}
 	}
 
@@ -317,28 +334,33 @@ func (f *AuditFormatter) FormatResponse(ctx context.Context, w io.Writer, config
 		Type:  "response",
 		Error: errString,
 		Auth: AuditAuth{
-			DisplayName:      auth.DisplayName,
-			Policies:         auth.Policies,
-			TokenPolicies:    auth.TokenPolicies,
-			IdentityPolicies: auth.IdentityPolicies,
-			Metadata:         auth.Metadata,
-			ClientToken:      auth.ClientToken,
-			Accessor:         auth.Accessor,
-			RemainingUses:    req.ClientTokenRemainingUses,
-			EntityID:         auth.EntityID,
+			DisplayName:               auth.DisplayName,
+			Policies:                  auth.Policies,
+			TokenPolicies:             auth.TokenPolicies,
+			IdentityPolicies:          auth.IdentityPolicies,
+			ExternalNamespacePolicies: auth.ExternalNamespacePolicies,
+			Metadata:                  auth.Metadata,
+			ClientToken:               auth.ClientToken,
+			Accessor:                  auth.Accessor,
+			RemainingUses:             req.ClientTokenRemainingUses,
+			EntityID:                  auth.EntityID,
 		},
 
 		Request: AuditRequest{
 			ID:                  req.ID,
 			ClientToken:         req.ClientToken,
 			ClientTokenAccessor: req.ClientTokenAccessor,
 			Operation:           req.Operation,
-			Path:                req.Path,
-			Data:                req.Data,
-			PolicyOverride:      req.PolicyOverride,
-			RemoteAddr:          getRemoteAddr(req),
-			ReplicationCluster:  req.ReplicationCluster,
-			Headers:             req.Headers,
+			Namespace: AuditNamespace{
+				ID:   ns.ID,
+				Path: ns.Path,
+			},
+			Path:               req.Path,
+			Data:               req.Data,
+			PolicyOverride:     req.PolicyOverride,
+			RemoteAddr:         getRemoteAddr(req),
+			ReplicationCluster: req.ReplicationCluster,
+			Headers:            req.Headers,
 		},
 
 		Response: AuditResponse{
@@ -386,6 +408,7 @@ type AuditRequest struct {
 	Operation           logical.Operation      `json:"operation"`
 	ClientToken         string                 `json:"client_token"`
 	ClientTokenAccessor string                 `json:"client_token_accessor"`
+	Namespace           AuditNamespace         `json:"namespace"`
 	Path                string                 `json:"path"`
 	Data                map[string]interface{} `json:"data"`
 	PolicyOverride      bool                   `json:"policy_override"`
@@ -403,16 +426,17 @@ type AuditResponse struct {
 }
 
 type AuditAuth struct {
-	ClientToken      string            `json:"client_token"`
-	Accessor         string            `json:"accessor"`
-	DisplayName      string            `json:"display_name"`
-	Policies         []string          `json:"policies"`
-	TokenPolicies    []string          `json:"token_policies,omitempty"`
-	IdentityPolicies []string          `json:"identity_policies,omitempty"`
-	Metadata         map[string]string `json:"metadata"`
-	NumUses          int               `json:"num_uses,omitempty"`
-	RemainingUses    int               `json:"remaining_uses,omitempty"`
-	EntityID         string            `json:"entity_id"`
+	ClientToken               string              `json:"client_token"`
+	Accessor                  string              `json:"accessor"`
+	DisplayName               string              `json:"display_name"`
+	Policies                  []string            `json:"policies"`
+	TokenPolicies             []string            `json:"token_policies,omitempty"`
+	IdentityPolicies          []string            `json:"identity_policies,omitempty"`
+	ExternalNamespacePolicies map[string][]string `json:"external_namespace_policies,omitempty"`
+	Metadata                  map[string]string   `json:"metadata"`
+	NumUses                   int                 `json:"num_uses,omitempty"`
+	RemainingUses             int                 `json:"remaining_uses,omitempty"`
+	EntityID                  string              `json:"entity_id"`
 }
 
 type AuditSecret struct {
@@ -428,6 +452,11 @@ type AuditResponseWrapInfo struct {
 	WrappedAccessor string `json:"wrapped_accessor,omitempty"`
 }
 
+type AuditNamespace struct {
+	ID   string `json:"id"`
+	Path string `json:"path"`
+}
+
 // getRemoteAddr safely gets the remote address avoiding a nil pointer
 func getRemoteAddr(req *logical.Request) string {
 	if req != nil && req.Connection != nil {

audit/format_json_test.go

@@ -13,6 +13,7 @@ import (
 	"fmt"
 
 	"github.com/hashicorp/vault/helper/jsonutil"
+	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/helper/salt"
 	"github.com/hashicorp/vault/logical"
 )
@@ -91,7 +92,7 @@ func TestFormatJSON_formatRequest(t *testing.T) {
 			Request:  tc.Req,
 			OuterErr: tc.Err,
 		}
-		if err := formatter.FormatRequest(context.Background(), &buf, config, in); err != nil {
+		if err := formatter.FormatRequest(namespace.RootContext(nil), &buf, config, in); err != nil {
 			t.Fatalf("bad: %s\nerr: %s", name, err)
 		}
 
@@ -104,6 +105,7 @@ func TestFormatJSON_formatRequest(t *testing.T) {
 		if err := jsonutil.DecodeJSON([]byte(expectedResultStr), &expectedjson); err != nil {
 			t.Fatalf("bad json: %s", err)
 		}
+		expectedjson.Request.Namespace = AuditNamespace{ID: "root"}
 
 		var actualjson = new(AuditRequestEntry)
 		if err := jsonutil.DecodeJSON([]byte(buf.String())[len(tc.Prefix):], &actualjson); err != nil {

audit/format_jsonx_test.go

@@ -11,6 +11,7 @@ import (
 
 	"fmt"
 
+	"github.com/hashicorp/vault/helper/namespace"
 	"github.com/hashicorp/vault/helper/salt"
 	"github.com/hashicorp/vault/logical"
 )
@@ -52,7 +53,7 @@ func TestFormatJSONx_formatRequest(t *testing.T) {
 			errors.New("this is an error"),
 			"",
 			"",
-			fmt.Sprintf(`<json:object name="auth"><json:string name="accessor">bar</json:string><json:string name="client_token">%s</json:string><json:string name="display_name">testtoken</json:string><json:string name="entity_id"></json:string><json:null name="metadata" /><json:array name="policies"><json:string>root</json:string></json:array></json:object><json:string name="error">this is an error</json:string><json:object name="request"><json:string name="client_token"></json:string><json:string name="client_token_accessor"></json:string><json:null name="data" /><json:object name="headers"><json:array name="foo"><json:string>bar</json:string></json:array></json:object><json:string name="id"></json:string><json:string name="operation">update</json:string><json:string name="path">/foo</json:string><json:boolean name="policy_override">false</json:boolean><json:string name="remote_address">127.0.0.1</json:string><json:number name="wrap_ttl">60</json:number></json:object><json:string name="type">request</json:string>`,
+			fmt.Sprintf(`<json:object name="auth"><json:string name="accessor">bar</json:string><json:string name="client_token">%s</json:string><json:string name="display_name">testtoken</json:string><json:string name="entity_id"></json:string><json:null name="metadata" /><json:array name="policies"><json:string>root</json:string></json:array></json:object><json:string name="error">this is an error</json:string><json:object name="request"><json:string name="client_token"></json:string><json:string name="client_token_accessor"></json:string><json:null name="data" /><json:object name="headers"><json:array name="foo"><json:string>bar</json:string></json:array></json:object><json:string name="id"></json:string><json:object name="namespace"><json:string name="id">root</json:string><json:string name="path"></json:string></json:object><json:string name="operation">update</json:string><json:string name="path">/foo</json:string><json:boolean name="policy_override">false</json:boolean><json:string name="remote_address">127.0.0.1</json:string><json:number name="wrap_ttl">60</json:number></json:object><json:string name="type">request</json:string>`,
 				fooSalted),
 		},
 		"auth, request with prefix": {
@@ -73,7 +74,7 @@ func TestFormatJSONx_formatRequest(t *testing.T) {
 			errors.New("this is an error"),
 			"",
 			"@cee: ",
-			fmt.Sprintf(`<json:object name="auth"><json:string name="accessor">bar</json:string><json:string name="client_token">%s</json:string><json:string name="display_name">testtoken</json:string><json:string name="entity_id"></json:string><json:null name="metadata" /><json:array name="policies"><json:string>root</json:string></json:array></json:object><json:string name="error">this is an error</json:string><json:object name="request"><json:string name="client_token"></json:string><json:string name="client_token_accessor"></json:string><json:null name="data" /><json:object name="headers"><json:array name="foo"><json:string>bar</json:string></json:array></json:object><json:string name="id"></json:string><json:string name="operation">update</json:string><json:string name="path">/foo</json:string><json:boolean name="policy_override">false</json:boolean><json:string name="remote_address">127.0.0.1</json:string><json:number name="wrap_ttl">60</json:number></json:object><json:string name="type">request</json:string>`,
+			fmt.Sprintf(`<json:object name="auth"><json:string name="accessor">bar</json:string><json:string name="client_token">%s</json:string><json:string name="display_name">testtoken</json:string><json:string name="entity_id"></json:string><json:null name="metadata" /><json:array name="policies"><json:string>root</json:string></json:array></json:object><json:string name="error">this is an error</json:string><json:object name="request"><json:string name="client_token"></json:string><json:string name="client_token_accessor"></json:string><json:null name="data" /><json:object name="headers"><json:array name="foo"><json:string>bar</json:string></json:array></json:object><json:string name="id"></json:string><json:object name="namespace"><json:string name="id">root</json:string><json:string name="path"></json:string></json:object><json:string name="operation">update</json:string><json:string name="path">/foo</json:string><json:boolean name="policy_override">false</json:boolean><json:string name="remote_address">127.0.0.1</json:string><json:number name="wrap_ttl">60</json:number></json:object><json:string name="type">request</json:string>`,
 				fooSalted),
 		},
 	}
@@ -95,7 +96,7 @@ func TestFormatJSONx_formatRequest(t *testing.T) {
 			Request:  tc.Req,
 			OuterErr: tc.Err,
 		}
-		if err := formatter.FormatRequest(context.Background(), &buf, config, in); err != nil {
+		if err := formatter.FormatRequest(namespace.RootContext(nil), &buf, config, in); err != nil {
 			t.Fatalf("bad: %s\nerr: %s", name, err)
 		}
 

builtin/credential/approle/path_tidy_user_id.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/vault/helper/consts"
 	"github.com/hashicorp/vault/logical"
 	"github.com/hashicorp/vault/logical/framework"
 )
@@ -27,6 +28,11 @@ func pathTidySecretID(b *backend) *framework.Path {
 
 // tidySecretID is used to delete entries in the whitelist that are expired.
 func (b *backend) tidySecretID(ctx context.Context, req *logical.Request) (*logical.Response, error) {
+	// If we are a performance standby forward the request to the active node
+	if b.System().ReplicationState().HasState(consts.ReplicationPerformanceStandby) {
+		return nil, logical.ErrReadOnly
+	}
+
 	if !atomic.CompareAndSwapUint32(b.tidySecretIDCASGuard, 0, 1) {
 		resp := &logical.Response{}
 		resp.AddWarning("Tidy operation already in progress.")