Merge remote-tracking branch 'oss/master' into type-keypairs

* oss/master: (30 commits)
  Handle 'not supplied' case for field type TypeNameString (#3546)
  Fix deprecated cassandra backend tests (#3543)
  changelog++
  auth/aws: Make disallow_reauthentication and allow_instance_migration mutually exclusive (#3291)
  changelog++
  More Mount Conflict Detection (#2919)
  Fix swallowed errors in TestRollbackManager_Join() (#3327)
  changelog++
  added AWS enpoint handling (#3416)
  Seal wrap all root tokens and their leases (#3540)
  Return group memberships of entity during read (#3526)
  Add note on support for using rec keys on /sys/rekey (#3517)
  Add third party tools list to website (#3488)
  Minor client refactoring (#3539)
  changelog++
  Add PKCS8 marshaling to PKI (#3518)
  Update SSH list roles docs (#3536)
  Update gocql dep
  changelog++
  Return role info for each role on pathRoleList (#3532)
  ...

CHANGELOG.md

@@ -13,19 +13,33 @@ DEPRECATIONS/CHANGES:
    nonce to the login endpoint. The custom nonce set by the client will from
    now on, not be returned back with the authentication response, and hence not
    audit logged.
+ * AWS Auth role options: The API will now error when trying to create or
+   update a role with the mutually-exclusive options
+   `disallow_reauthentication` and `allow_instance_migration`.
  * SSH CA role read changes: When reading back a role from the `ssh` backend,
    the TTL/max TTL values will now be an integer number of seconds rather than
    a string. This better matches the API elsewhere in Vault.
+ * SSH role list changes: When listing roles from the `ssh` backend via the API,
+   the response data will additionally return a `key_info` map that will contain
+   a map of each key with a corresponding object containing the `key_type`.
+
+FEATURES:
+
+ * ** RSA Support for Transit Backend**: Transit backend can now generate RSA
+   keys which can be used for encryption and signing. [GH-3489]
 
 IMPROVEMENTS:
 
  * api: Add ability to set custom headers on each call [GH-3394]
  * command/server: Add config option to disable requesting client certificates
    [GH-3373]
+ * core: Disallow mounting underneath an existing path, not just over [GH-2919]
  * physical/file: Use `700` as permissions when creating directories. The files
    themselves were `600` and are all encrypted, but this doesn't hurt.
+ * secret/aws: Add ability to use custom IAM/STS endpoints [GH-3416]
  * secret/cassandra: Work around Cassandra ignoring consistency levels for a
    user listing query [GH-3469]
+ * secret/pki: Private keys can now be marshalled as PKCS#8 [GH-3518]
  * secret/pki: Allow entering URLs for `pki` as both comma-separated strings and JSON
    arrays [GH-3409]
  * secret/ssh: Role TTL/max TTL can now be specified as either a string or an
@@ -38,6 +52,7 @@ BUG FIXES:
 
  * api: Fix panic when setting a custom HTTP client but with a nil transport
    [GH-3435] [GH-3437]
+ * auth/aws: Don't allow mutually exclusive options [GH-3291]
  * auth/radius: Fix logging in in some situations [GH-3461]
  * core: Fix memleak when a connection would connect to the cluster port and
    then go away [GH-3513]
@@ -46,6 +61,8 @@ BUG FIXES:
    responses when requests were forwarded to the active node [GH-3485]
  * physical/etcd3: Fix some listing issues due to how etcd3 does prefix
    matching [GH-3406]
+ * physical/etcd3: Fix case where standbys can lose their etcd client lease
+   [GH-3031]
  * physical/file: Fix listing when underscores are the first component of a
    path [GH-3476]
  * plugins: Allow response errors to be returned from backend plugins [GH-3412]
@@ -146,7 +163,7 @@ IMPROVEMENTS:
 
  * audit/file: Allow specifying `stdout` as the `file_path` to log to standard
    output [GH-3235]
- * auth/aws: Allow wildcards in `bound_iam_principal_id` [GH-3213]
+ * auth/aws: Allow wildcards in `bound_iam_principal_arn` [GH-3213]
  * auth/okta: Compare groups case-insensitively since Okta is only
    case-preserving [GH-3240]
  * auth/okta: Standardize Okta configuration APIs across backends [GH-3245]

api/client.go

@@ -13,6 +13,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/go-rootcerts"
 	"github.com/hashicorp/vault/helper/parseutil"
@@ -63,6 +64,10 @@ type Config struct {
 
 	// Timeout is for setting custom timeout parameter in the HttpClient
 	Timeout time.Duration
+
+	// If there is an error when creating the configuration, this will be the
+	// error
+	Error error
 }
 
 // TLSConfig contains the parameters needed to configure TLS on the HTTP client
@@ -110,17 +115,15 @@ func DefaultConfig() *Config {
 		MinVersion: tls.VersionTLS12,
 	}
 	if err := http2.ConfigureTransport(transport); err != nil {
+		config.Error = err
 		return nil
 	}
 
 	if err := config.ReadEnvironment(); err != nil {
+		config.Error = err
 		return nil
 	}
 
-	if v := os.Getenv(EnvVaultAddress); v != "" {
-		config.Address = v
-	}
-
 	// Ensure redirects are not automatically followed
 	// Note that this is sane for the API client as it has its own
 	// redirect handling logic (and thus also for command/meta),
@@ -142,36 +145,41 @@ func (c *Config) ConfigureTLS(t *TLSConfig) error {
 	if c.HttpClient == nil {
 		c.HttpClient = DefaultConfig().HttpClient
 	}
+	clientTLSConfig := c.HttpClient.Transport.(*http.Transport).TLSClientConfig
 
 	var clientCert tls.Certificate
 	foundClientCert := false
-	if t.CACert != "" || t.CAPath != "" || t.ClientCert != "" || t.ClientKey != "" || t.Insecure {
-		if t.ClientCert != "" && t.ClientKey != "" {
-			var err error
-			clientCert, err = tls.LoadX509KeyPair(t.ClientCert, t.ClientKey)
-			if err != nil {
-				return err
-			}
-			foundClientCert = true
-		} else if t.ClientCert != "" || t.ClientKey != "" {
-			return fmt.Errorf("Both client cert and client key must be provided")
+
+	switch {
+	case t.ClientCert != "" && t.ClientKey != "":
+		var err error
+		clientCert, err = tls.LoadX509KeyPair(t.ClientCert, t.ClientKey)
+		if err != nil {
+			return err
 		}
+		foundClientCert = true
+	case t.ClientCert != "" || t.ClientKey != "":
+		return fmt.Errorf("Both client cert and client key must be provided")
 	}
 
-	clientTLSConfig := c.HttpClient.Transport.(*http.Transport).TLSClientConfig
-	rootConfig := &rootcerts.Config{
-		CAFile: t.CACert,
-		CAPath: t.CAPath,
-	}
-	if err := rootcerts.ConfigureTLS(clientTLSConfig, rootConfig); err != nil {
-		return err
+	if t.CACert != "" || t.CAPath != "" {
+		rootConfig := &rootcerts.Config{
+			CAFile: t.CACert,
+			CAPath: t.CAPath,
+		}
+		if err := rootcerts.ConfigureTLS(clientTLSConfig, rootConfig); err != nil {
+			return err
+		}
 	}
 
-	clientTLSConfig.InsecureSkipVerify = t.Insecure
+	if t.Insecure {
+		clientTLSConfig.InsecureSkipVerify = true
+	}
 
 	if foundClientCert {
 		clientTLSConfig.Certificates = []tls.Certificate{clientCert}
 	}
+
 	if t.TLSServerName != "" {
 		clientTLSConfig.ServerName = t.TLSServerName
 	}
@@ -290,6 +298,9 @@ func NewClient(c *Config) (*Client, error) {
 	if def == nil {
 		return nil, fmt.Errorf("could not create/read default configuration")
 	}
+	if def.Error != nil {
+		return nil, errwrap.Wrapf("error encountered setting up default configuration: {{err}}", def.Error)
+	}
 
 	if c == nil {
 		c = def

builtin/credential/aws/backend.go

@@ -99,6 +99,9 @@ func Backend(conf *logical.BackendConfig) (*backend, error) {
 			LocalStorage: []string{
 				"whitelist/identity/",
 			},
+			SealWrapStorage: []string{
+				"config/client",
+			},
 		},
 		Paths: []*framework.Path{
 			pathLogin(b),

builtin/credential/aws/path_role.go

@@ -663,6 +663,10 @@ func (b *backend) pathRoleCreateUpdate(
 		roleEntry.AllowInstanceMigration = data.Get("allow_instance_migration").(bool)
 	}
 
+	if roleEntry.AllowInstanceMigration && roleEntry.DisallowReauthentication {
+		return logical.ErrorResponse("cannot specify both disallow_reauthentication=true and allow_instance_migration=true"), nil
+	}
+
 	var resp logical.Response
 
 	ttlRaw, ok := data.GetOk("ttl")

builtin/credential/aws/path_role_tag.go

@@ -124,6 +124,10 @@ func (b *backend) pathRoleTagUpdate(
 		resp.AddWarning("Role does not allow instance migration. Login will not be allowed with this tag unless the role value is updated.")
 	}
 
+	if disallowReauthentication && allowInstanceMigration {
+		return logical.ErrorResponse("cannot set both disallow_reauthentication and allow_instance_migration"), nil
+	}
+
 	// max_ttl for the role tag should be less than the max_ttl set on the role.
 	maxTTL := time.Duration(data.Get("max_ttl").(int)) * time.Second
 

builtin/credential/aws/path_role_test.go

@@ -66,12 +66,25 @@ func TestBackend_pathRoleEc2(t *testing.T) {
 		Data:      data,
 		Storage:   storage,
 	})
-	if resp != nil && resp.IsError() {
-		t.Fatalf("failed to create role: %s", resp.Data["error"])
+	if err != nil {
+		t.Fatal(err)
 	}
+	if resp == nil || !resp.IsError() {
+		t.Fatalf("expected failure to create role with both allow_instance_migration true and disallow_reauthentication true")
+	}
+	data["disallow_reauthentication"] = false
+	resp, err = b.HandleRequest(&logical.Request{
+		Operation: logical.UpdateOperation,
+		Path:      "role/ami-abcd123",
+		Data:      data,
+		Storage:   storage,
+	})
 	if err != nil {
 		t.Fatal(err)
 	}
+	if resp != nil && resp.IsError() {
+		t.Fatalf("failure to update role: %v", resp.Data["error"])
+	}
 	resp, err = b.HandleRequest(&logical.Request{
 		Operation: logical.ReadOperation,
 		Path:      "role/ami-abcd123",
@@ -80,8 +93,12 @@ func TestBackend_pathRoleEc2(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !resp.Data["allow_instance_migration"].(bool) || !resp.Data["disallow_reauthentication"].(bool) {
-		t.Fatal("bad: expected:true got:false\n")
+	if !resp.Data["allow_instance_migration"].(bool) {
+		t.Fatal("bad: expected allow_instance_migration:true got:false\n")
+	}
+
+	if resp.Data["disallow_reauthentication"].(bool) {
+		t.Fatal("bad: expected disallow_reauthentication: false got:true\n")
 	}
 
 	// add another entry, to test listing of role entries
@@ -529,7 +546,7 @@ func TestAwsEc2_RoleCrud(t *testing.T) {
 		"ttl":                       "10m",
 		"max_ttl":                   "20m",
 		"policies":                  "testpolicy1,testpolicy2",
-		"disallow_reauthentication": true,
+		"disallow_reauthentication": false,
 		"hmac_key":                  "testhmackey",
 		"period":                    "1m",
 	}
@@ -567,7 +584,7 @@ func TestAwsEc2_RoleCrud(t *testing.T) {
 		"ttl":                       time.Duration(600),
 		"max_ttl":                   time.Duration(1200),
 		"policies":                  []string{"testpolicy1", "testpolicy2"},
-		"disallow_reauthentication": true,
+		"disallow_reauthentication": false,
 		"period":                    time.Duration(60),
 	}
 

builtin/credential/ldap/backend.go

@@ -31,6 +31,10 @@ func Backend() *backend {
 			Unauthenticated: []string{
 				"login/*",
 			},
+
+			SealWrapStorage: []string{
+				"config",
+			},
 		},
 
 		Paths: append([]*framework.Path{

builtin/credential/okta/backend.go

@@ -25,6 +25,9 @@ func Backend() *backend {
 			Unauthenticated: []string{
 				"login/*",
 			},
+			SealWrapStorage: []string{
+				"config",
+			},
 		},
 
 		Paths: append([]*framework.Path{

builtin/credential/radius/backend.go

@@ -26,6 +26,10 @@ func Backend() *backend {
 				"login",
 				"login/*",
 			},
+
+			SealWrapStorage: []string{
+				"config",
+			},
 		},
 
 		Paths: append([]*framework.Path{

builtin/logical/aws/backend.go

@@ -25,6 +25,9 @@ func Backend() *backend {
 			LocalStorage: []string{
 				framework.WALPrefix,
 			},
+			SealWrapStorage: []string{
+				"config/root",
+			},
 		},
 
 		Paths: []*framework.Path{

builtin/logical/aws/client.go

@@ -13,8 +13,9 @@ import (
 	"github.com/hashicorp/vault/logical"
 )
 
-func getRootConfig(s logical.Storage) (*aws.Config, error) {
+func getRootConfig(s logical.Storage, clientType string) (*aws.Config, error) {
 	credsConfig := &awsutil.CredentialsConfig{}
+	var endpoint string
 
 	entry, err := s.Get("config/root")
 	if err != nil {
@@ -29,6 +30,12 @@ func getRootConfig(s logical.Storage) (*aws.Config, error) {
 		credsConfig.AccessKey = config.AccessKey
 		credsConfig.SecretKey = config.SecretKey
 		credsConfig.Region = config.Region
+		switch {
+		case clientType == "iam" && config.IAMEndpoint != "":
+			endpoint = *aws.String(config.IAMEndpoint)
+		case clientType == "sts" && config.STSEndpoint != "":
+			endpoint = *aws.String(config.STSEndpoint)
+		}
 	}
 
 	if credsConfig.Region == "" {
@@ -51,28 +58,32 @@ func getRootConfig(s logical.Storage) (*aws.Config, error) {
 	return &aws.Config{
 		Credentials: creds,
 		Region:      aws.String(credsConfig.Region),
+		Endpoint:    &endpoint,
 		HTTPClient:  cleanhttp.DefaultClient(),
 	}, nil
 }
 
 func clientIAM(s logical.Storage) (*iam.IAM, error) {
-	awsConfig, err := getRootConfig(s)
+	awsConfig, err := getRootConfig(s, "iam")
 	if err != nil {
 		return nil, err
 	}
+
 	client := iam.New(session.New(awsConfig))
+
 	if client == nil {
 		return nil, fmt.Errorf("could not obtain iam client")
 	}
 	return client, nil
 }
 
 func clientSTS(s logical.Storage) (*sts.STS, error) {
-	awsConfig, err := getRootConfig(s)
+	awsConfig, err := getRootConfig(s, "sts")
 	if err != nil {
 		return nil, err
 	}
 	client := sts.New(session.New(awsConfig))
+
 	if client == nil {
 		return nil, fmt.Errorf("could not obtain sts client")
 	}