Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[v14] e2e test redshift cluster #43984

Merged
merged 2 commits into from
Jul 10, 2024
Merged

[v14] e2e test redshift cluster #43984

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8580de2
add redshift cluster e2e tests
GavinFrazar May 13, 2024
139be2b
fix redshift auto user deadlocking
GavinFrazar Jul 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/aws-e2e-tests-non-root.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ env:
RDS_POSTGRES_INSTANCE_NAME: ci-database-e2e-tests-rds-postgres-instance-us-west-2-307493967395
RDS_MYSQL_INSTANCE_NAME: ci-database-e2e-tests-rds-mysql-instance-us-west-2-307493967395
RDS_MARIADB_INSTANCE_NAME: ci-database-e2e-tests-rds-mariadb-instance-us-west-2-307493967395
REDSHIFT_ACCESS_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-redshift-access
REDSHIFT_DISCOVERY_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-redshift-discovery
REDSHIFT_CLUSTER_NAME: ci-database-e2e-tests-redshift-cluster-us-west-2-307493967395
REDSHIFT_SERVERLESS_ACCESS_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-redshift-serverless-access
REDSHIFT_SERVERLESS_DISCOVERY_ROLE: arn:aws:iam::307493967395:role/ci-database-e2e-tests-redshift-serverless-discovery
REDSHIFT_SERVERLESS_ENDPOINT_NAME: ci-database-e2e-tests-redshift-serverless-workgroup-rss-access-us-west-2-307493967395
Expand Down
86 changes: 81 additions & 5 deletions e2e/aws/databases_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,19 @@ package e2e
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"net"
"os"
"strconv"
"testing"
"time"

"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
mysqlclient "github.com/go-mysql-org/go-mysql/client"
"github.com/jackc/pgconn"
"github.com/jackc/pgx/v4"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

Expand Down Expand Up @@ -59,6 +63,7 @@ func TestDatabases(t *testing.T) {
t.Run("unmatched discovery", awsDBDiscoveryUnmatched)
t.Run("rds", testRDS)
t.Run("redshift serverless", testRedshiftServerless)
t.Run("redshift cluster", testRedshiftCluster)
}

func awsDBDiscoveryUnmatched(t *testing.T) {
Expand All @@ -71,8 +76,9 @@ func awsDBDiscoveryUnmatched(t *testing.T) {
for matcherType, assumeRoleARN := range map[string]string{
// add a new matcher/role here to test that discovery properly
// does *not* that kind of database for some unmatched tag.
types.AWSMatcherRDS: mustGetEnv(t, rdsDiscoveryRoleEnv),
types.AWSMatcherRedshiftServerless: mustGetEnv(t, rssDiscoveryRoleEnv),
types.AWSMatcherRDS: mustGetEnv(t, rdsDiscoveryRoleARNEnv),
types.AWSMatcherRedshiftServerless: mustGetEnv(t, rssDiscoveryRoleARNEnv),
types.AWSMatcherRedshift: mustGetEnv(t, redshiftDiscoveryRoleARNEnv),
} {
matchers = append(matchers, types.AWSMatcher{
Types: []string{matcherType},
Expand Down Expand Up @@ -157,16 +163,18 @@ func postgresLocalProxyConnTest(t *testing.T, cluster *helpers.TeleInstance, use
defer cancel()
lp := startLocalALPNProxy(t, ctx, user, cluster, route)

connString := fmt.Sprintf("postgres://%s@%v/%s",
route.Username, lp.GetAddr(), route.Database)
pgconnConfig, err := pgconn.ParseConfig(fmt.Sprintf("postgres://%v/", lp.GetAddr()))
require.NoError(t, err)
pgconnConfig.User = route.Username
pgconnConfig.Database = route.Database
var pgConn *pgconn.PgConn
// retry for a while, the database service might need time to give
// itself IAM rds:connect permissions.
require.EventuallyWithT(t, func(t *assert.CollectT) {
var err error
ctx, cancel := context.WithTimeout(context.Background(), connRetryTick)
defer cancel()
pgConn, err = pgconn.Connect(ctx, connString)
pgConn, err = pgconn.ConnectConfig(ctx, pgconnConfig)
assert.NoError(t, err)
assert.NotNil(t, pgConn)
}, waitForConnTimeout, connRetryTick, "connecting to postgres")
Expand Down Expand Up @@ -315,3 +323,71 @@ func waitForDatabases(t *testing.T, auth *service.TeleportProcess, wantNames ...
}
}, 1*time.Minute, time.Second, "waiting for the database service to heartbeat the databases")
}

// dbUserLogin contains common info needed to connect as a db user via
// password auth.
type dbUserLogin struct {
username string
password string
address string
port int
}

func connectPostgres(t *testing.T, ctx context.Context, info dbUserLogin, dbName string) *pgx.Conn {
pgCfg, err := pgx.ParseConfig(fmt.Sprintf("postgres://%s:%d/?sslmode=verify-full", info.address, info.port))
require.NoError(t, err)
pgCfg.User = info.username
pgCfg.Password = info.password
pgCfg.Database = dbName
pgCfg.TLSConfig = &tls.Config{
ServerName: info.address,
RootCAs: awsCertPool.Clone(),
}

conn, err := pgx.ConnectConfig(ctx, pgCfg)
require.NoError(t, err)
t.Cleanup(func() {
_ = conn.Close(ctx)
})
return conn
}

// secretPassword is used to unmarshal an AWS Secrets Manager
// user password secret.
type secretPassword struct {
Password string `json:"password"`
}

// getMasterUserPassword is a helper that fetches a db master user and password
// from AWS Secrets Manager.
func getMasterUserPassword(t *testing.T, ctx context.Context, secretID string) string {
t.Helper()
secretVal := getSecretValue(t, ctx, secretID)
require.NotNil(t, secretVal.SecretString)
var secret secretPassword
if err := json.Unmarshal([]byte(*secretVal.SecretString), &secret); err != nil {
// being paranoid. I don't want to leak the secret string in test error
// logs.
require.FailNow(t, "error unmarshaling secret string")
}
if len(secret.Password) == 0 {
require.FailNow(t, "empty master user secret string")
}
return secret.Password
}

func getSecretValue(t *testing.T, ctx context.Context, secretID string) secretsmanager.GetSecretValueOutput {
t.Helper()
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(mustGetEnv(t, awsRegionEnv)),
)
require.NoError(t, err)

secretsClt := secretsmanager.NewFromConfig(cfg)
secretVal, err := secretsClt.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{
SecretId: &secretID,
})
require.NoError(t, err)
require.NotNil(t, secretVal)
return *secretVal
}
36 changes: 22 additions & 14 deletions e2e/aws/fixtures_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,28 @@ func init() {
}

func getAWSGlobalCertBundlePool() (*x509.CertPool, error) {
// AWS global certificate bundle
const certBundleURL = "https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem"
certPool := x509.NewCertPool()

// AWS global certificate bundles
for _, url := range []string{
"https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem",
"https://s3.amazonaws.com/redshift-downloads/amazon-trust-ca-bundle.crt",
} {
certBytes, err := getAWSCertBundle(url)
if err != nil {
return nil, trace.Wrap(err)
}
ok := certPool.AppendCertsFromPEM(certBytes)
if !ok {
return nil, trace.BadParameter("failed to parse AWS cert bundle %v", url)
}
}

resp, err := http.Get(certBundleURL)
return certPool, nil
}

func getAWSCertBundle(url string) ([]byte, error) {
resp, err := http.Get(url)
if err != nil {
return nil, trace.Wrap(err)
}
Expand All @@ -71,17 +89,7 @@ func getAWSGlobalCertBundlePool() (*x509.CertPool, error) {
}

certBytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, trace.Wrap(err)
}

certPool := x509.NewCertPool()
ok := certPool.AppendCertsFromPEM(certBytes)
if !ok {
return nil, trace.Errorf("error parsing AWS cert bundle")
}

return certPool, nil
return certBytes, trace.Wrap(err)
}

// mustGetEnv is a test helper that fetches an env variable or fails with an
Expand Down
41 changes: 28 additions & 13 deletions e2e/aws/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,16 @@ const (
// discoveryMatcherLabelsEnv is the env variable that specifies the matcher
// labels to use in test discovery services.
discoveryMatcherLabelsEnv = "DISCOVERY_MATCHER_LABELS"
// rdsAccessRoleEnv is the environment variable that specifies the IAM role
// that Teleport Database Service will assume to access RDS databases.
// rdsAccessRoleARNEnv is the environment variable that specifies the IAM
// role ARN that Teleport Database Service will assume to access RDS
// databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
rdsAccessRoleEnv = "RDS_ACCESS_ROLE"
// rdsDiscoveryRoleEnv is the environment variable that specifies the
// IAM role that Teleport Discovery Service will assume to discover
rdsAccessRoleARNEnv = "RDS_ACCESS_ROLE"
// rdsDiscoveryRoleARNEnv is the environment variable that specifies the
// IAM role ARN that Teleport Discovery Service will assume to discover
// RDS databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
rdsDiscoveryRoleEnv = "RDS_DISCOVERY_ROLE"
rdsDiscoveryRoleARNEnv = "RDS_DISCOVERY_ROLE"
// rdsPostgresInstanceNameEnv is the environment variable that specifies the
// name of the RDS Postgres DB instance that will be created by the Teleport
// Discovery Service.
Expand All @@ -50,16 +51,16 @@ const (
// name of the RDS MariaDB instance that will be created by the Teleport
// Discovery Service.
rdsMariaDBInstanceNameEnv = "RDS_MARIADB_INSTANCE_NAME"
// rssAccessRoleEnv is the environment variable that specifies the IAM role
// that Teleport Database Service will assume to access Redshift Serverless
// databases.
// rssAccessRoleARNEnv is the environment variable that specifies the IAM
// role ARN that Teleport Database Service will assume to access Redshift
// Serverless databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
rssAccessRoleEnv = "REDSHIFT_SERVERLESS_ACCESS_ROLE"
// rssDiscoveryRoleEnv is the environment variable that specifies the
// IAM role that Teleport Discovery Service will assume to discover
rssAccessRoleARNEnv = "REDSHIFT_SERVERLESS_ACCESS_ROLE"
// rssDiscoveryRoleARNEnv is the environment variable that specifies the
// IAM role ARN that Teleport Discovery Service will assume to discover
// Redshift Serverless databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
rssDiscoveryRoleEnv = "REDSHIFT_SERVERLESS_DISCOVERY_ROLE"
rssDiscoveryRoleARNEnv = "REDSHIFT_SERVERLESS_DISCOVERY_ROLE"
// rssNameEnv is the environment variable that specifies the
// name of the Redshift Serverless workgroup that will be created by the
// Teleport Discovery Service.
Expand All @@ -71,6 +72,20 @@ const (
// rssDBUserEnv is the name of the IAM role that tests will use as a
// database user to connect to Redshift Serverless.
rssDBUserEnv = "REDSHIFT_SERVERLESS_IAM_DB_USER"
// redshiftAccessRoleARNEnv is the environment variable that specifies the
// IAM role ARN that Teleport Database Service will assume to access Redshift
// cluster databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
redshiftAccessRoleARNEnv = "REDSHIFT_ACCESS_ROLE"
// redshiftDiscoveryRoleARNEnv is the environment variable that specifies the
// IAM role ARN that Teleport Discovery Service will assume to discover
// Redshift cluster databases.
// See modules/databases-ci/ from cloud-terraform repo for more details.
redshiftDiscoveryRoleARNEnv = "REDSHIFT_DISCOVERY_ROLE"
// redshiftNameEnv is the environment variable that specifies the
// name of the Redshift cluster db that will be created by the
// Teleport Discovery Service.
redshiftNameEnv = "REDSHIFT_CLUSTER_NAME"
// kubeSvcRoleARNEnv is the environment variable that specifies
// the IAM role that Teleport Kubernetes Service will assume to access the EKS cluster.
// This role needs to have the following permissions:
Expand Down
Loading
Loading