auth/aws: Allow wildcard in bound_iam_principal_id

This allows specifying a trailing wildcard in the bound_iam_principal_id
in a semantically similar way to AWS. If a user specifies a
bound_iam_principal_arn with a trailing wildcard, then Vault will NOT
resolve the unqiue ID (analogous to the way AWS handles wildcards) and
instead will just store the ARN.

At login time, if a wildcard is specified in the bound ARN, Vault will
first resolve the full ARN of the authenticating principal because the
path component of roles is not visible from the GetCallerIdentity
response. A cache has been included to speed up performance of these
lookups and should be append only. To prevent unbounded growth of the
cache, old entries are cleaned up if not used within a period of time.

Also, as the complexity of scenarios of when Vault might make AWS API
calls increases, including a recommended IAM policy to give the Vault
server that can be more simply referenced in the future.

Fixes hashicorp#3179

builtin/credential/aws/backend.go

@@ -60,6 +60,11 @@ type backend struct {
 	// will be flushed. The empty STS role signifies the master account
 	IAMClientsMap map[string]map[string]*iam.IAM
 
+	// Map of AWS unique IDs to the full ARN corresponding to that unique ID
+	// This avoids the overhead of an AWS API hit for every login request
+	// using the IAM auth method when bound_iam_principal_arn contains a wildcard
+	iamUserIdToArn map[string]*awsUniqueIdMapEntry
+
 	// AWS Account ID of the "default" AWS credentials
 	// This cache avoids the need to call GetCallerIdentity repeatedly to learn it
 	// We can't store this because, in certain pathological cases, it could change
@@ -77,6 +82,7 @@ func Backend(conf *logical.BackendConfig) (*backend, error) {
 		tidyCooldownPeriod: time.Hour,
 		EC2ClientsMap:      make(map[string]map[string]*ec2.EC2),
 		IAMClientsMap:      make(map[string]map[string]*iam.IAM),
+		iamUserIdToArn:     make(map[string]*awsUniqueIdMapEntry),
 	}
 
 	b.resolveArnToUniqueIDFunc = b.resolveArnToRealUniqueId
@@ -124,7 +130,8 @@ func Backend(conf *logical.BackendConfig) (*backend, error) {
 // Currently this will be triggered once in a minute by the RollbackManager.
 //
 // The tasks being done currently by this function are to cleanup the expired
-// entries of both blacklist role tags and whitelist identities. Tidying is done
+// entries of both blacklist role tags and whitelist identities as well as stale
+// entries in the iamUserIdToArn cache. Tidying is done
 // not once in a minute, but once in an hour, controlled by 'tidyCooldownPeriod'.
 // Tidying of blacklist and whitelist are by default enabled. This can be
 // changed using `config/tidy/roletags` and `config/tidy/identities` endpoints.
@@ -174,6 +181,9 @@ func (b *backend) periodicFunc(req *logical.Request) error {
 			b.tidyWhitelistIdentity(req.Storage, safety_buffer)
 		}
 
+		// get rid of old unique ID entries
+		b.cleanOldCachedUniqueIdMapping()
+
 		// Update the time at which to run the tidy functions again.
 		b.nextTidyTime = time.Now().Add(b.tidyCooldownPeriod)
 	}

builtin/credential/aws/backend_test.go

@@ -1330,6 +1330,10 @@ func TestBackendAcc_LoginWithCallerIdentity(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// Calling this once here to ensure it won't raise any unexpected errors on an empty
+	// cache
+	b.cleanOldCachedUniqueIdMapping()
+
 	// Override the default AWS env vars (if set) with our test creds
 	// so that the credential provider chain will pick them up
 	// NOTE that I'm not bothing to override the shared config file location,
@@ -1522,22 +1526,14 @@ func TestBackendAcc_LoginWithCallerIdentity(t *testing.T) {
 		t.Fatalf("bad: expected valid login: resp:%#v", resp)
 	}
 
-	renewReq := &logical.Request{
-		Storage: storage,
-		Auth:    &logical.Auth{},
-	}
+	renewReq := generateRenewRequest(storage, resp.Auth)
+	// dump a fake ARN into the metadata to ensure that we ONLY look
+	// at the unique ID that has been generated
+	renewReq.Auth.Metadata["canonical_arn"] = "fake_arn"
 	empty_login_fd := &framework.FieldData{
 		Raw:    map[string]interface{}{},
 		Schema: pathLogin(b).Fields,
 	}
-	renewReq.Auth.InternalData = resp.Auth.InternalData
-	renewReq.Auth.Metadata = resp.Auth.Metadata
-	renewReq.Auth.LeaseOptions = resp.Auth.LeaseOptions
-	renewReq.Auth.Policies = resp.Auth.Policies
-	renewReq.Auth.IssueTime = time.Now()
-	// dump a fake ARN into the metadata to ensure that we ONLY look
-	// at the unique ID that has been generated
-	renewReq.Auth.Metadata["canonical_arn"] = "fake_arn"
 	// ensure we can renew
 	resp, err = b.pathLoginRenew(renewReq, empty_login_fd)
 	if err != nil {
@@ -1571,5 +1567,60 @@ func TestBackendAcc_LoginWithCallerIdentity(t *testing.T) {
 	if err == nil || (resp != nil && !resp.IsError()) {
 		t.Errorf("bad: expected failed renew due to changed AWS role ID: resp: %#v", resp, err)
 	}
+	// Undo the fake resolver...
+	b.resolveArnToUniqueIDFunc = b.resolveArnToRealUniqueId
+
+	// Now test that wildcard matching works
+	wildcardRoleName := "valid_wildcard"
+	wildcardEntity := *entity
+	wildcardEntity.FriendlyName = "*"
+	roleData["bound_iam_principal_arn"] = wildcardEntity.canonicalArn()
+	roleRequest.Path = "role/" + wildcardRoleName
+	resp, err = b.HandleRequest(roleRequest)
+	if err != nil || (resp != nil && resp.IsError()) {
+		t.Fatalf("bad: failed to create wildcard role: resp:%#v\nerr:%v", resp, err)
+	}
+
+	loginData["role"] = wildcardRoleName
+	resp, err = b.HandleRequest(loginRequest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil || resp.Auth == nil || resp.IsError() {
+		t.Fatalf("bad: expected valid login: resp:%#v", resp)
+	}
+	// and ensure we can renew
+	renewReq = generateRenewRequest(storage, resp.Auth)
+	resp, err = b.pathLoginRenew(renewReq, empty_login_fd)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("got nil response from renew")
+	}
+	if resp.IsError() {
+		t.Fatalf("got error when renewing: %#v", *resp)
+	}
+	// ensure the cache is populated
+	cachedArn := b.getCachedUserId(resp.Auth.Metadata["client_user_id"])
+	if cachedArn == "" {
+		t.Errorf("got empty ARN back from user ID cache; expected full arn")
+	}
+	// Calling this again to ensure it won't panic or raise unexpected errors when there
+	// are cached values
+	b.cleanOldCachedUniqueIdMapping()
+}
+
+func generateRenewRequest(s logical.Storage, auth *logical.Auth) *logical.Request {
+	renewReq := &logical.Request{
+		Storage: s,
+		Auth:    &logical.Auth{},
+	}
+	renewReq.Auth.InternalData = auth.InternalData
+	renewReq.Auth.Metadata = auth.Metadata
+	renewReq.Auth.LeaseOptions = auth.LeaseOptions
+	renewReq.Auth.Policies = auth.Policies
+	renewReq.Auth.IssueTime = time.Now()
 
+	return renewReq
 }

builtin/credential/aws/client.go

@@ -2,6 +2,7 @@ package awsauth
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
@@ -141,6 +142,48 @@ func (b *backend) flushCachedIAMClients() {
 	}
 }
 
+// Deletes old cached mappings of unique IDs to ARNs
+// Write locks configMutex
+func (b *backend) cleanOldCachedUniqueIdMapping() {
+	b.configMutex.Lock()
+	defer b.configMutex.Unlock()
+	for id, mapEntry := range b.iamUserIdToArn {
+		// clean all entries last accessed more than MaxLeaseTTL ago
+		if time.Since(mapEntry.LastAccessed) > b.System().MaxLeaseTTL() {
+			delete(b.iamUserIdToArn, id)
+		}
+	}
+}
+
+// Gets an entry out of the user ID cache
+// Write locks configMutex
+func (b *backend) getCachedUserId(userId string) string {
+	if userId == "" {
+		return ""
+	}
+	b.configMutex.Lock()
+	defer b.configMutex.Unlock()
+	if entry, ok := b.iamUserIdToArn[userId]; ok {
+		entry.LastAccessed = time.Now()
+		return entry.Arn
+	}
+	return ""
+}
+
+// Sets an entry in the user ID cache
+// Write locks configMutex
+func (b *backend) setCachedUserId(userId, arn string) {
+	if userId != "" {
+		b.configMutex.Lock()
+		defer b.configMutex.Unlock()
+		entry := awsUniqueIdMapEntry{
+			LastAccessed: time.Now(),
+			Arn:          arn,
+		}
+		b.iamUserIdToArn[userId] = &entry
+	}
+}
+
 func (b *backend) stsRoleForAccount(s logical.Storage, accountID string) (string, error) {
 	// Check if an STS configuration exists for the AWS account
 	sts, err := b.lockedAwsStsEntry(s, accountID)
@@ -250,3 +293,8 @@ func (b *backend) clientIAM(s logical.Storage, region, accountID string) (*iam.I
 	}
 	return b.IAMClientsMap[region][stsRole], nil
 }
+
+type awsUniqueIdMapEntry struct {
+	LastAccessed time.Time // This is primarily so we can clean up the cache so it doesn't monotonically grow
+	Arn          string
+}

builtin/credential/aws/path_login.go

@@ -928,6 +928,12 @@ func (b *backend) pathLoginRenewIam(
 		}
 	}
 
+	// Note that the error messages below can leak a little bit of information about the role information
+	// For example, if on renew, the client gets the "error parsing ARN..." error message, the client
+	// will konw that it's a wildcard bind (but not the actual bind), even if the client can't actually
+	// read the role directly to know what the bind is. It's a relatively small amount of leakage, in
+	// some fairly corner cases, and in the most likely error case (role has been changed to a new ARN),
+	// the error message is identical.
 	if roleEntry.BoundIamPrincipalARN != "" {
 		// We might not get here if all bindings were on the inferred entity, which we've already validated
 		// above
@@ -936,10 +942,31 @@ func (b *backend) pathLoginRenewIam(
 			// Resolving unique IDs is enabled and the auth metadata contains the unique ID, so checking the
 			// unique ID is authoritative at this stage
 			if roleEntry.BoundIamPrincipalID != clientUserId {
-				return nil, fmt.Errorf("role no longer bound to ID %q", clientUserId)
+				return nil, fmt.Errorf("role no longer bound to ARN %q", canonicalArn)
+			}
+		} else if strings.HasSuffix(roleEntry.BoundIamPrincipalARN, "*") {
+			fullArn := b.getCachedUserId(clientUserId)
+			if fullArn == "" {
+				entity, err := parseIamArn(canonicalArn)
+				if err != nil {
+					return nil, fmt.Errorf("error parsing ARN %q: %v", canonicalArn, err)
+				}
+				fullArn, err = b.fullArn(entity, req.Storage)
+				if err != nil {
+					return nil, fmt.Errorf("error looking up full ARN of entity %v: %v", entity, err)
+				}
+				if fullArn == "" {
+					return nil, fmt.Errorf("got empty string back when looking up full ARN of entity %v", entity)
+				}
+				if clientUserId != "" {
+					b.setCachedUserId(clientUserId, fullArn)
+				}
+			}
+			if !strutil.GlobbedStringsMatch(roleEntry.BoundIamPrincipalARN, fullArn) {
+				return nil, fmt.Errorf("role no longer bound to ARN %q", canonicalArn)
 			}
 		} else if roleEntry.BoundIamPrincipalARN != canonicalArn {
-			return nil, fmt.Errorf("role no longer bound to arn %q", canonicalArn)
+			return nil, fmt.Errorf("role no longer bound to ARN %q", canonicalArn)
 		}
 	}
 
@@ -1123,7 +1150,7 @@ func (b *backend) pathLoginUpdateIam(
 	callerUniqueId := strings.Split(callerID.UserId, ":")[0]
 	entity, err := parseIamArn(callerID.Arn)
 	if err != nil {
-		return logical.ErrorResponse(fmt.Sprintf("Error parsing arn: %v", err)), nil
+		return logical.ErrorResponse(fmt.Sprintf("error parsing arn %q: %v", callerID.Arn, err)), nil
 	}
 
 	roleName := data.Get("role").(string)
@@ -1152,8 +1179,27 @@ func (b *backend) pathLoginUpdateIam(
 		if callerUniqueId != roleEntry.BoundIamPrincipalID {
 			return logical.ErrorResponse(fmt.Sprintf("expected IAM %s %s to resolve to unique AWS ID %q but got %q instead", entity.Type, entity.FriendlyName, roleEntry.BoundIamPrincipalID, callerUniqueId)), nil
 		}
-	} else if roleEntry.BoundIamPrincipalARN != "" && roleEntry.BoundIamPrincipalARN != entity.canonicalArn() {
-		return logical.ErrorResponse(fmt.Sprintf("IAM Principal %q does not belong to the role %q", callerID.Arn, roleName)), nil
+	} else if roleEntry.BoundIamPrincipalARN != "" {
+		if strings.HasSuffix(roleEntry.BoundIamPrincipalARN, "*") {
+			fullArn := b.getCachedUserId(callerUniqueId)
+			if fullArn == "" {
+				fullArn, err = b.fullArn(entity, req.Storage)
+				if err != nil {
+					return logical.ErrorResponse(fmt.Sprintf("error looking up full ARN of entity %v: %v", entity, err)), nil
+				}
+				if fullArn == "" {
+					return logical.ErrorResponse(fmt.Sprintf("got empty string back when looking up full ARN of entity %v", entity)), nil
+				}
+				b.setCachedUserId(callerUniqueId, fullArn)
+			}
+			if !strutil.GlobbedStringsMatch(roleEntry.BoundIamPrincipalARN, fullArn) {
+				// Note: Intentionally giving the exact same error message as a few lines below. Otherwise, we might leak information
+				// about whether the bound IAM principal ARN is a wildcard or not, and what that wildcard is.
+				return logical.ErrorResponse(fmt.Sprintf("IAM Principal %q does not belong to the role %q", callerID.Arn, roleName)), nil
+			}
+		} else if roleEntry.BoundIamPrincipalARN != entity.canonicalArn() {
+			return logical.ErrorResponse(fmt.Sprintf("IAM Principal %q does not belong to the role %q", callerID.Arn, roleName)), nil
+		}
 	}
 
 	policies := roleEntry.Policies
@@ -1502,6 +1548,7 @@ type iamEntity struct {
 	SessionInfo   string
 }
 
+// Returns a Vault-internal canonical ARN for referring to an IAM entity
 func (e *iamEntity) canonicalArn() string {
 	entityType := e.Type
 	// canonicalize "assumed-role" into "role"
@@ -1516,6 +1563,46 @@ func (e *iamEntity) canonicalArn() string {
 	return fmt.Sprintf("arn:%s:iam::%s:%s/%s", e.Partition, e.AccountNumber, entityType, e.FriendlyName)
 }
 
+// This returns the "full" ARN of an iamEntity, how it would be referred to in AWS proper
+func (b *backend) fullArn(e *iamEntity, s logical.Storage) (string, error) {
+	// Not assuming path is reliable for any entity types
+	client, err := b.clientIAM(s, getAnyRegionForAwsPartition(e.Partition).ID(), e.AccountNumber)
+	if err != nil {
+		return "", fmt.Errorf("error creating IAM client: %v", err)
+	}
+
+	switch e.Type {
+	case "user":
+		input := iam.GetUserInput{
+			UserName: aws.String(e.FriendlyName),
+		}
+		resp, err := client.GetUser(&input)
+		if err != nil {
+			return "", fmt.Errorf("error fetching user %q: %v", e.FriendlyName, err)
+		}
+		if resp == nil {
+			return "", fmt.Errorf("nil response from GetUser")
+		}
+		return *(resp.User.Arn), nil
+	case "assumed-role":
+		fallthrough
+	case "role":
+		input := iam.GetRoleInput{
+			RoleName: aws.String(e.FriendlyName),
+		}
+		resp, err := client.GetRole(&input)
+		if err != nil {
+			return "", fmt.Errorf("error fetching role %q: %v", e.FriendlyName, err)
+		}
+		if resp == nil {
+			return "", fmt.Errorf("nil response form GetRole")
+		}
+		return *(resp.Role.Arn), nil
+	default:
+		return "", fmt.Errorf("unrecognized entity type: %s", e.Type)
+	}
+}
+
 const iamServerIdHeader = "X-Vault-AWS-IAM-Server-ID"
 
 const pathLoginSyn = `

builtin/credential/aws/path_role.go

@@ -318,7 +318,8 @@ func (b *backend) upgradeRoleEntry(s logical.Storage, roleEntry *awsRoleEntry) (
 	if roleEntry.AuthType == iamAuthType &&
 		roleEntry.ResolveAWSUniqueIDs &&
 		roleEntry.BoundIamPrincipalARN != "" &&
-		roleEntry.BoundIamPrincipalID == "" {
+		roleEntry.BoundIamPrincipalID == "" &&
+		!strings.HasSuffix(roleEntry.BoundIamPrincipalARN, "*") {
 		principalId, err := b.resolveArnToUniqueIDFunc(s, roleEntry.BoundIamPrincipalARN)
 		if err != nil {
 			return false, err
@@ -493,14 +494,17 @@ func (b *backend) pathRoleCreateUpdate(
 		// This allows the user to sumbit an update with the same ARN to force Vault
 		// to re-resolve the ARN to the unique ID, in case an entity was deleted and
 		// recreated
-		if roleEntry.ResolveAWSUniqueIDs {
+		if roleEntry.ResolveAWSUniqueIDs && !strings.HasSuffix(roleEntry.BoundIamPrincipalARN, "*") {
 			principalID, err := b.resolveArnToUniqueIDFunc(req.Storage, principalARN)
 			if err != nil {
 				return logical.ErrorResponse(fmt.Sprintf("failed updating the unique ID of ARN %#v: %#v", principalARN, err)), nil
 			}
 			roleEntry.BoundIamPrincipalID = principalID
+		} else {
+			// Need to handle the case where we're switching from a non-wildcard principal to a wildcard principal
+			roleEntry.BoundIamPrincipalID = ""
 		}
-	} else if roleEntry.ResolveAWSUniqueIDs && roleEntry.BoundIamPrincipalARN != "" {
+	} else if roleEntry.ResolveAWSUniqueIDs && roleEntry.BoundIamPrincipalARN != "" && !strings.HasSuffix(roleEntry.BoundIamPrincipalARN, "*") {
 		// we're turning on resolution on this role, so ensure we update it
 		principalID, err := b.resolveArnToUniqueIDFunc(req.Storage, roleEntry.BoundIamPrincipalARN)
 		if err != nil {