Skip to content

Commit

Permalink
[v14] e2e test redshift cluster (#43984)
Browse files Browse the repository at this point in the history 
* add redshift cluster e2e tests

* test redshift cluster iam role and existing db user iam auth
* test redshift cluster auto user provisioning
* add error checking in test cleanup

* fix redshift auto user deadlocking
  • Loading branch information
@GavinFrazar
GavinFrazar authored Jul 10, 2024
1 parent 3116a53 commit c47d32a
Show file tree
Hide file tree
Showing 12 changed files with 592 additions and 177 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/aws-e2e-tests-non-root.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ env:
RDS_POSTGRES_INSTANCE_NAME: ci-database-e2e-tests-rds-postgres-instance-us-west-2-307493967395
RDS_MYSQL_INSTANCE_NAME: ci-database-e2e-tests-rds-mysql-instance-us-west-2-307493967395
RDS_MARIADB_INSTANCE_NAME: ci-database-e2e-tests-rds-mariadb-instance-us-west-2-307493967395
REDSHIFT_ACCESS_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-redshift-access
REDSHIFT_DISCOVERY_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-redshift-discovery
REDSHIFT_CLUSTER_NAME: ci-database-e2e-tests-redshift-cluster-us-west-2-307493967395
REDSHIFT_SERVERLESS_ACCESS_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-redshift-serverless-access
REDSHIFT_SERVERLESS_DISCOVERY_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-redshift-serverless-discovery
REDSHIFT_SERVERLESS_ENDPOINT_NAME: ci-database-e2e-tests-redshift-serverless-workgroup-rss-access-us-west-2-307493967395
Expand Down
86 changes: 81 additions & 5 deletions e2e/aws/databases_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,19 @@ package e2e
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"net"
"os"
"strconv"
"testing"
"time"

"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
mysqlclient "github.com/go-mysql-org/go-mysql/client"
"github.com/jackc/pgconn"
"github.com/jackc/pgx/v4"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

Expand Down Expand Up @@ -59,6 +63,7 @@ func TestDatabases(t *testing.T) {
t.Run("unmatched discovery", awsDBDiscoveryUnmatched)
t.Run("rds", testRDS)
t.Run("redshift serverless", testRedshiftServerless)
t.Run("redshift cluster", testRedshiftCluster)
}

func awsDBDiscoveryUnmatched(t *testing.T) {
Expand All @@ -71,8 +76,9 @@ func awsDBDiscoveryUnmatched(t *testing.T) {
for matcherType, assumeRoleARN := range map[string]string{
// add a new matcher/role here to test that discovery properly
// does *not* that kind of database for some unmatched tag.
types.AWSMatcherRDS: mustGetEnv(t, rdsDiscoveryRoleEnv),
types.AWSMatcherRedshiftServerless: mustGetEnv(t, rssDiscoveryRoleEnv),
types.AWSMatcherRDS: mustGetEnv(t, rdsDiscoveryRoleARNEnv),
types.AWSMatcherRedshiftServerless: mustGetEnv(t, rssDiscoveryRoleARNEnv),
types.AWSMatcherRedshift: mustGetEnv(t, redshiftDiscoveryRoleARNEnv),
} {
matchers = append(matchers, types.AWSMatcher{
Types: []string{matcherType},
Expand Down Expand Up @@ -157,16 +163,18 @@ func postgresLocalProxyConnTest(t *testing.T, cluster *helpers.TeleInstance, use
defer cancel()
lp := startLocalALPNProxy(t, ctx, user, cluster, route)

connString := fmt.Sprintf("postgres://%s@%v/%s",
route.Username, lp.GetAddr(), route.Database)
pgconnConfig, err := pgconn.ParseConfig(fmt.Sprintf("postgres://%v/", lp.GetAddr()))
require.NoError(t, err)
pgconnConfig.User = route.Username
pgconnConfig.Database = route.Database
var pgConn *pgconn.PgConn
// retry for a while, the database service might need time to give
// itself IAM rds:connect permissions.
require.EventuallyWithT(t, func(t *assert.CollectT) {
var err error
ctx, cancel := context.WithTimeout(context.Background(), connRetryTick)
defer cancel()
pgConn, err = pgconn.Connect(ctx, connString)
pgConn, err = pgconn.ConnectConfig(ctx, pgconnConfig)
assert.NoError(t, err)
assert.NotNil(t, pgConn)
}, waitForConnTimeout, connRetryTick, "connecting to postgres")
Expand Down Expand Up @@ -315,3 +323,71 @@ func waitForDatabases(t *testing.T, auth *service.TeleportProcess, wantNames ...
}
}, 1*time.Minute, time.Second, "waiting for the database service to heartbeat the databases")
}

// dbUserLogin contains common info needed to connect as a db user via
// password auth.
type dbUserLogin struct {
username string
password string
address string
port int
}

func connectPostgres(t *testing.T, ctx context.Context, info dbUserLogin, dbName string) *pgx.Conn {
pgCfg, err := pgx.ParseConfig(fmt.Sprintf("postgres://%s:%d/?sslmode=verify-full", info.address, info.port))
require.NoError(t, err)
pgCfg.User = info.username
pgCfg.Password = info.password
pgCfg.Database = dbName
pgCfg.TLSConfig = &tls.Config{
ServerName: info.address,
RootCAs: awsCertPool.Clone(),
}

conn, err := pgx.ConnectConfig(ctx, pgCfg)
require.NoError(t, err)
t.Cleanup(func() {
_ = conn.Close(ctx)
})
return conn
}

// secretPassword is used to unmarshal an AWS Secrets Manager
// user password secret.
type secretPassword struct {
Password string `json:"password"`
}

// getMasterUserPassword is a helper that fetches a db master user and password
// from AWS Secrets Manager.
func getMasterUserPassword(t *testing.T, ctx context.Context, secretID string) string {
t.Helper()
secretVal := getSecretValue(t, ctx, secretID)
require.NotNil(t, secretVal.SecretString)
var secret secretPassword
if err := json.Unmarshal([]byte(*secretVal.SecretString), &secret); err != nil {
// being paranoid. I don't want to leak the secret string in test error
// logs.
require.FailNow(t, "error unmarshaling secret string")
}
if len(secret.Password) == 0 {
require.FailNow(t, "empty master user secret string")
}
return secret.Password
}

func getSecretValue(t *testing.T, ctx context.Context, secretID string) secretsmanager.GetSecretValueOutput {
t.Helper()
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(mustGetEnv(t, awsRegionEnv)),
)
require.NoError(t, err)

secretsClt := secretsmanager.NewFromConfig(cfg)
secretVal, err := secretsClt.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{
SecretId: &secretID,
})
require.NoError(t, err)
require.NotNil(t, secretVal)
return *secretVal
}
36 changes: 22 additions & 14 deletions e2e/aws/fixtures_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,28 @@ func init() {
}

func getAWSGlobalCertBundlePool() (*x509.CertPool, error) {
// AWS global certificate bundle
const certBundleURL = "https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem"
certPool := x509.NewCertPool()

// AWS global certificate bundles
for _, url := range []string{
"https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem",
"https://s3.amazonaws.com/redshift-downloads/amazon-trust-ca-bundle.crt",
} {
certBytes, err := getAWSCertBundle(url)
if err != nil {
return nil, trace.Wrap(err)
}
ok := certPool.AppendCertsFromPEM(certBytes)
if !ok {
return nil, trace.BadParameter("failed to parse AWS cert bundle %v", url)
}
}

resp, err := http.Get(certBundleURL)
return certPool, nil
}

func getAWSCertBundle(url string) ([]byte, error) {
resp, err := http.Get(url)
if err != nil {
return nil, trace.Wrap(err)
}
Expand All @@ -71,17 +89,7 @@ func getAWSGlobalCertBundlePool() (*x509.CertPool, error) {
}

certBytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, trace.Wrap(err)
}

certPool := x509.NewCertPool()
ok := certPool.AppendCertsFromPEM(certBytes)
if !ok {
return nil, trace.Errorf("error parsing AWS cert bundle")
}

return certPool, nil
return certBytes, trace.Wrap(err)
}

// mustGetEnv is a test helper that fetches an env variable or fails with an
Expand Down
41 changes: 28 additions & 13 deletions e2e/aws/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,16 @@ const (
// discoveryMatcherLabelsEnv is the env variable that specifies the matcher
// labels to use in test discovery services.
discoveryMatcherLabelsEnv = "DISCOVERY_MATCHER_LABELS"
// rdsAccessRoleEnv is the environment variable that specifies the IAM role
// that Teleport Database Service will assume to access RDS databases.
// rdsAccessRoleARNEnv is the environment variable that specifies the IAM
// role ARN that Teleport Database Service will assume to access RDS
// databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
rdsAccessRoleEnv = "RDS_ACCESS_ROLE"
// rdsDiscoveryRoleEnv is the environment variable that specifies the
// IAM role that Teleport Discovery Service will assume to discover
rdsAccessRoleARNEnv = "RDS_ACCESS_ROLE"
// rdsDiscoveryRoleARNEnv is the environment variable that specifies the
// IAM role ARN that Teleport Discovery Service will assume to discover
// RDS databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
rdsDiscoveryRoleEnv = "RDS_DISCOVERY_ROLE"
rdsDiscoveryRoleARNEnv = "RDS_DISCOVERY_ROLE"
// rdsPostgresInstanceNameEnv is the environment variable that specifies the
// name of the RDS Postgres DB instance that will be created by the Teleport
// Discovery Service.
Expand All @@ -50,16 +51,16 @@ const (
// name of the RDS MariaDB instance that will be created by the Teleport
// Discovery Service.
rdsMariaDBInstanceNameEnv = "RDS_MARIADB_INSTANCE_NAME"
// rssAccessRoleEnv is the environment variable that specifies the IAM role
// that Teleport Database Service will assume to access Redshift Serverless
// databases.
// rssAccessRoleARNEnv is the environment variable that specifies the IAM
// role ARN that Teleport Database Service will assume to access Redshift
// Serverless databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
rssAccessRoleEnv = "REDSHIFT_SERVERLESS_ACCESS_ROLE"
// rssDiscoveryRoleEnv is the environment variable that specifies the
// IAM role that Teleport Discovery Service will assume to discover
rssAccessRoleARNEnv = "REDSHIFT_SERVERLESS_ACCESS_ROLE"
// rssDiscoveryRoleARNEnv is the environment variable that specifies the
// IAM role ARN that Teleport Discovery Service will assume to discover
// Redshift Serverless databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
rssDiscoveryRoleEnv = "REDSHIFT_SERVERLESS_DISCOVERY_ROLE"
rssDiscoveryRoleARNEnv = "REDSHIFT_SERVERLESS_DISCOVERY_ROLE"
// rssNameEnv is the environment variable that specifies the
// name of the Redshift Serverless workgroup that will be created by the
// Teleport Discovery Service.
Expand All @@ -71,6 +72,20 @@ const (
// rssDBUserEnv is the name of the IAM role that tests will use as a
// database user to connect to Redshift Serverless.
rssDBUserEnv = "REDSHIFT_SERVERLESS_IAM_DB_USER"
// redshiftAccessRoleARNEnv is the environment variable that specifies the
// IAM role ARN that Teleport Database Service will assume to access Redshift
// cluster databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
redshiftAccessRoleARNEnv = "REDSHIFT_ACCESS_ROLE"
// redshiftDiscoveryRoleARNEnv is the environment variable that specifies the
// IAM role ARN that Teleport Discovery Service will assume to discover
// Redshift cluster databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
redshiftDiscoveryRoleARNEnv = "REDSHIFT_DISCOVERY_ROLE"
// redshiftNameEnv is the environment variable that specifies the
// name of the Redshift cluster db that will be created by the
// Teleport Discovery Service.
redshiftNameEnv = "REDSHIFT_CLUSTER_NAME"
// kubeSvcRoleARNEnv is the environment variable that specifies
// the IAM role that Teleport Kubernetes Service will assume to access the EKS cluster.
// This role needs to have the following permissions:
Expand Down
Loading

0 comments on commit c47d32a

Please sign in to comment.