From 37f73ea6e99062e75cec480244e2e4a461a39125 Mon Sep 17 00:00:00 2001 From: fuelvin1 Date: Mon, 5 Aug 2024 19:24:33 -0700 Subject: [PATCH 1/9] chore: initial adds --- packages/amplify-e2e-core/src/utils/rds.ts | 15 +++++++++++++-- .../src/__tests__/sql-models-2.test.ts | 2 +- .../src/sql-datatabase-controller.ts | 4 +++- scripts/e2e-test-local-cluster-config.json | 8 ++++++++ .../e2e-test-local-cluster-config.sample.json | 16 ++++++++++++++++ 5 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 scripts/e2e-test-local-cluster-config.json create mode 100644 scripts/e2e-test-local-cluster-config.sample.json diff --git a/packages/amplify-e2e-core/src/utils/rds.ts b/packages/amplify-e2e-core/src/utils/rds.ts index 3dcaf69ab2..80812e5087 100644 --- a/packages/amplify-e2e-core/src/utils/rds.ts +++ b/packages/amplify-e2e-core/src/utils/rds.ts @@ -26,6 +26,8 @@ import { SecretsManagerClient, CreateSecretCommand, DeleteSecretCommand, GetSecr import { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand } from '@aws-sdk/client-kms'; import { knex } from 'knex'; import axios from 'axios'; +import fs from 'fs-extra'; +import path from 'path'; import { sleep } from './sleep'; const DEFAULT_DB_INSTANCE_TYPE = 'db.m5.large'; @@ -308,10 +310,19 @@ export const setupRDSInstanceAndData = async ( * @returns Endpoint address, port and database name of the created RDS cluster. */ -export const setupRDSClusterAndData = async (config: RDSConfig, queries?: string[]): Promise => { +export const setupRDSClusterAndData = async (localTesting: boolean, config: RDSConfig, queries?: string[]): Promise => { console.log(`Creating RDS ${config.engine} DB cluster with identifier ${config.identifier}`); - const dbCluster = await createRDSCluster(config); + let dbCluster; + localTesting = false; + if (!localTesting) { + dbCluster = await createRDSCluster(config); + } /*else { + const repoRoot = path.join(__dirname, '..', '..', '..', '..'); + const localClusterPath = path.join(repoRoot, 'scripts', 'e2e-test-local-cluster-config.json'); + const localCluster: ClusterInfo = JSON.parse(fs.readFileSync(localClusterPath, 'utf-8')); + dbCluster = + }*/ if (!dbCluster.secretArn) { throw new Error('Failed to store db connection config in secrets manager'); diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-models-2.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-models-2.test.ts index ba005bd011..f0e3ee77ee 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-models-2.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-models-2.test.ts @@ -17,7 +17,7 @@ describe('CDK GraphQL Transformer deployments with SQL datasources', () => { const region = process.env.CLI_REGION ?? 'us-west-2'; const dbname = 'default_db'; - const engine = 'mysql'; + const engine = 'postgres'; const databaseController: SqlDatatabaseController = new SqlDatatabaseController( ['CREATE TABLE todos (id VARCHAR(40) PRIMARY KEY, description VARCHAR(256))'], diff --git a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts index 0e724b2464..73ff266277 100644 --- a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts +++ b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts @@ -16,6 +16,7 @@ import { deleteDBCluster, isOptInRegion, isDataAPISupported, + isCI, } from 'amplify-category-api-e2e-core'; import { SecretsManagerClient, CreateSecretCommand, DeleteSecretCommand, GetSecretValueCommand } from '@aws-sdk/client-secrets-manager'; import { @@ -64,7 +65,8 @@ export class SqlDatatabaseController { setupDatabase = async (): Promise => { let dbConfig; if (this.useDataAPI) { - dbConfig = await setupRDSClusterAndData(this.options, this.setupQueries); + const enableLocalTesting = isCI(); + dbConfig = await setupRDSClusterAndData(enableLocalTesting, this.options, this.setupQueries); } else { dbConfig = await setupRDSInstanceAndData(this.options, this.setupQueries); } diff --git a/scripts/e2e-test-local-cluster-config.json b/scripts/e2e-test-local-cluster-config.json new file mode 100644 index 0000000000..7349ce8984 --- /dev/null +++ b/scripts/e2e-test-local-cluster-config.json @@ -0,0 +1,8 @@ +{ + "clusterArn": "arn:aws:rds:us-west-2:637423428135:cluster:dydylglqks", + "endpoint": "dydylglqks.cluster-ct8aeycis380.us-west-2.rds.amazonaws.com", + "port": 5432, + "dbName": "default_db", + "secretArn": "", + "dbInstance": null +} diff --git a/scripts/e2e-test-local-cluster-config.sample.json b/scripts/e2e-test-local-cluster-config.sample.json new file mode 100644 index 0000000000..a80df17ff7 --- /dev/null +++ b/scripts/e2e-test-local-cluster-config.sample.json @@ -0,0 +1,16 @@ +{ + "us-east-1": [ + { + "dbConfig": { + "identifier": "IDENTIFIER_VALUE" + } + } + ], + "us-west-2": [ + { + "dbConfig": { + "identifier": "IDENTIFIER_VALUE" + } + } + ] +} From bf518fb093316513ed7de4f5edb9e1d7d53eab29 Mon Sep 17 00:00:00 2001 From: fuelvin1 Date: Tue, 6 Aug 2024 10:49:49 -0700 Subject: [PATCH 2/9] chore: in progress --- packages/amplify-e2e-core/src/utils/rds.ts | 32 +++++++++++++++++----- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/packages/amplify-e2e-core/src/utils/rds.ts b/packages/amplify-e2e-core/src/utils/rds.ts index 80812e5087..9f29ba87a0 100644 --- a/packages/amplify-e2e-core/src/utils/rds.ts +++ b/packages/amplify-e2e-core/src/utils/rds.ts @@ -21,6 +21,8 @@ import { PutParameterCommand, PutParameterCommandInput, PutParameterCommandOutput, + GetParameterCommand, + GetParametersByPathCommand, } from '@aws-sdk/client-ssm'; import { SecretsManagerClient, CreateSecretCommand, DeleteSecretCommand, GetSecretValueCommand } from '@aws-sdk/client-secrets-manager'; import { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand } from '@aws-sdk/client-kms'; @@ -314,15 +316,31 @@ export const setupRDSClusterAndData = async (localTesting: boolean, config: RDSC console.log(`Creating RDS ${config.engine} DB cluster with identifier ${config.identifier}`); let dbCluster; - localTesting = false; + localTesting = true; if (!localTesting) { dbCluster = await createRDSCluster(config); - } /*else { - const repoRoot = path.join(__dirname, '..', '..', '..', '..'); - const localClusterPath = path.join(repoRoot, 'scripts', 'e2e-test-local-cluster-config.json'); - const localCluster: ClusterInfo = JSON.parse(fs.readFileSync(localClusterPath, 'utf-8')); - dbCluster = - }*/ + } else { + try { + const repoRoot = path.join(__dirname, '..', '..', '..', '..'); + const localClusterPath = path.join(repoRoot, 'scripts', 'e2e-test-local-cluster-config.json'); + const configInfo = JSON.parse(fs.readFileSync(localClusterPath, 'utf-8')); + const connectionUri = configInfo.connectionConfigs.connectionUri; + + const ssmClient = new SSMClient({ region: config.region }); + const getParameterCommand = new GetParametersByPathCommand({ Path: connectionUri }); + const getParameterResponse = await ssmClient.send(getParameterCommand); + + if (getParameterResponse.Parameters) { + getParameterResponse.Parameters.forEach((parameter) => { + console.log(parameter); + }); + } else { + console.log('NO PARAMETERS FOUND'); + } + } catch (error) { + console.log('Error: ', error); + } + } if (!dbCluster.secretArn) { throw new Error('Failed to store db connection config in secrets manager'); From 0bf9ae70bc70d8bd06f617dc02366a3bd4b5c1ce Mon Sep 17 00:00:00 2001 From: fuelvin1 Date: Tue, 6 Aug 2024 10:51:48 -0700 Subject: [PATCH 3/9] chore: in progress --- scripts/e2e-test-local-cluster-config.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/e2e-test-local-cluster-config.json b/scripts/e2e-test-local-cluster-config.json index 7349ce8984..36d698a3d4 100644 --- a/scripts/e2e-test-local-cluster-config.json +++ b/scripts/e2e-test-local-cluster-config.json @@ -1,8 +1,8 @@ { - "clusterArn": "arn:aws:rds:us-west-2:637423428135:cluster:dydylglqks", - "endpoint": "dydylglqks.cluster-ct8aeycis380.us-west-2.rds.amazonaws.com", - "port": 5432, - "dbName": "default_db", - "secretArn": "", - "dbInstance": null + "dbConfig": { + "identifier": "" + }, + "connectionConfigs": { + "connectionUri": "/BfgiBDcCQYM/test" + } } From fcb8c5d3a7e511d701d896c2c0830d16e162420d Mon Sep 17 00:00:00 2001 From: fuelvin1 Date: Tue, 13 Aug 2024 09:46:28 -0700 Subject: [PATCH 4/9] chore: fixing lambda issue --- packages/amplify-e2e-core/src/utils/rds.ts | 99 +++-- .../src/__tests__/sql-models-2.test.ts | 2 +- .../src/__tests__/sql-pg-canary.test.ts | 4 +- .../src/sql-datatabase-controller.ts | 366 +++++++++++++----- scripts/e2e-test-local-cluster-config.json | 23 +- 5 files changed, 360 insertions(+), 134 deletions(-) diff --git a/packages/amplify-e2e-core/src/utils/rds.ts b/packages/amplify-e2e-core/src/utils/rds.ts index 9f29ba87a0..456f1acfeb 100644 --- a/packages/amplify-e2e-core/src/utils/rds.ts +++ b/packages/amplify-e2e-core/src/utils/rds.ts @@ -5,11 +5,16 @@ import { CreateDBInstanceCommandInput, DBInstance, DeleteDBInstanceCommand, + DescribeDBClustersCommand, waitUntilDBInstanceAvailable, CreateDBClusterMessage, waitUntilDBClusterAvailable, DeleteDBClusterCommand, DeleteDBClusterCommandInput, + $Command, + DescribeDBInstancesCommand, + DescribeDBClustersCommandOutput, + DescribeDBInstancesCommandOutput, } from '@aws-sdk/client-rds'; import { RDSDataClient, ExecuteStatementCommand, ExecuteStatementCommandInput, Field } from '@aws-sdk/client-rds-data'; import generator from 'generate-password'; @@ -23,6 +28,7 @@ import { PutParameterCommandOutput, GetParameterCommand, GetParametersByPathCommand, + GetParameterResult, } from '@aws-sdk/client-ssm'; import { SecretsManagerClient, CreateSecretCommand, DeleteSecretCommand, GetSecretValueCommand } from '@aws-sdk/client-secrets-manager'; import { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand } from '@aws-sdk/client-kms'; @@ -43,7 +49,7 @@ export type SqlEngine = 'mysql' | 'postgres'; export type RDSConfig = { identifier: string; engine: SqlEngine; - dbname: string; + dbname?: string; username: string; password?: string; region: string; @@ -245,6 +251,57 @@ export const createRDSCluster = async (config: RDSConfig): Promise } }; +/** + * Accesses the local RDS Aurora serverless V2 cluster with one DB instance using the given input configuration. + * @param config Configuration of the database cluster. + * @returns EndPoint address, port and database name of the accessed RDS cluster. + */ +export const useLocalCluster = async (config: RDSConfig): Promise => { + try { + // Send requests to get the cluster and instance config info + const databaseName = 'defaultdb'; + const identifier = config.identifier; + const instanceIdentifier = createInstanceIdentifier(config.identifier); + const client = new RDSClient({ region: config.region }); + const describeClusterCommand = new DescribeDBClustersCommand({ Filters: [{ Name: 'db-cluster-id', Values: [identifier] }] }); + const describeInstanceCommand = new DescribeDBInstancesCommand({ DBInstanceIdentifier: instanceIdentifier }); + + let describeClusterResponse: DescribeDBClustersCommandOutput; + let describeInstanceResponse: DescribeDBInstancesCommandOutput; + try { + describeClusterResponse = await client.send(describeClusterCommand); + if (describeClusterResponse == null) { + throw Error('Specified cluster is null.'); + } + } catch (error) { + throw Error(`Error in getting ${identifier} cluster info: ${error}`); + } + try { + describeInstanceResponse = await client.send(describeInstanceCommand); + if (describeInstanceResponse == null) { + throw Error('Specified instance is null.'); + } + } catch (error) { + throw Error(`Error in getting ${instanceIdentifier} cluster info: ${error}`); + } + + // Extract the cluster and instance from the responses + const dbClusterObj = describeClusterResponse.DBClusters[0]; + const dbInstance = describeInstanceResponse.DBInstances[0]; + + return { + clusterArn: dbClusterObj.DBClusterArn, + endpoint: dbClusterObj.Endpoint, + port: dbClusterObj.Port, + dbName: databaseName, + dbInstance: dbInstance, + secretArn: dbClusterObj.MasterUserSecret.SecretArn, + }; + } catch (error) { + console.log('Error: ', error); + } +}; + /** * Creates a new RDS instance using the given input configuration, runs the given queries and returns the details of the created RDS * instance. @@ -315,31 +372,12 @@ export const setupRDSInstanceAndData = async ( export const setupRDSClusterAndData = async (localTesting: boolean, config: RDSConfig, queries?: string[]): Promise => { console.log(`Creating RDS ${config.engine} DB cluster with identifier ${config.identifier}`); - let dbCluster; - localTesting = true; + let dbCluster: ClusterInfo; + if (!localTesting) { dbCluster = await createRDSCluster(config); } else { - try { - const repoRoot = path.join(__dirname, '..', '..', '..', '..'); - const localClusterPath = path.join(repoRoot, 'scripts', 'e2e-test-local-cluster-config.json'); - const configInfo = JSON.parse(fs.readFileSync(localClusterPath, 'utf-8')); - const connectionUri = configInfo.connectionConfigs.connectionUri; - - const ssmClient = new SSMClient({ region: config.region }); - const getParameterCommand = new GetParametersByPathCommand({ Path: connectionUri }); - const getParameterResponse = await ssmClient.send(getParameterCommand); - - if (getParameterResponse.Parameters) { - getParameterResponse.Parameters.forEach((parameter) => { - console.log(parameter); - }); - } else { - console.log('NO PARAMETERS FOUND'); - } - } catch (error) { - console.log('Error: ', error); - } + dbCluster = await useLocalCluster(config); } if (!dbCluster.secretArn) { @@ -348,13 +386,10 @@ export const setupRDSClusterAndData = async (localTesting: boolean, config: RDSC const client = new RDSDataClient({ region: config.region }); - // create a new test database with given name - const sanitizedDbName = config.dbname.replace(/[^a-zA-Z0-9_]/g, ''); - const createDBInput: ExecuteStatementCommandInput = { resourceArn: dbCluster.clusterArn, secretArn: dbCluster.secretArn, - sql: `create database ${sanitizedDbName}`, + sql: `create database ${config.dbname}`, database: dbCluster.dbName, }; @@ -373,10 +408,10 @@ export const setupRDSClusterAndData = async (localTesting: boolean, config: RDSC resourceArn: dbCluster.clusterArn, secretArn: dbCluster.secretArn, sql: query, - database: sanitizedDbName, + database: config.dbname, }; const executeStatementResponse = await client.send(new ExecuteStatementCommand(executeStatementInput)); - console.log('Create table response: ' + JSON.stringify(executeStatementResponse)); + console.log('Run query response: ' + JSON.stringify(executeStatementResponse)); } catch (err) { throw new Error(`Error in creating tables in test database: ${JSON.stringify(err, null, 4)}`); } @@ -386,7 +421,7 @@ export const setupRDSClusterAndData = async (localTesting: boolean, config: RDSC clusterArn: dbCluster.clusterArn, endpoint: dbCluster.endpoint, port: dbCluster.port, - dbName: sanitizedDbName, + dbName: config.dbname, dbInstance: dbCluster.dbInstance, secretArn: dbCluster.secretArn, }; @@ -424,7 +459,7 @@ export const deleteDBInstance = async (identifier: string, region: string): Prom // ); } catch (error) { console.log(error); - throw new Error(`Error in deleting RDS instance: ${error.response.json}`); + throw new Error(`Error in deleting RDS instance: ${error.json}`); } }; @@ -451,7 +486,7 @@ export const deleteDBCluster = async (identifier: string, region: string): Promi await client.send(command); } catch (error) { console.log(error); - throw new Error(`Error in deleting RDS cluster ${identifier}: ${error.response.json}`); + throw new Error(`Error in deleting RDS cluster ${identifier}: ${error.json}`); } }; diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-models-2.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-models-2.test.ts index f0e3ee77ee..ba005bd011 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-models-2.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-models-2.test.ts @@ -17,7 +17,7 @@ describe('CDK GraphQL Transformer deployments with SQL datasources', () => { const region = process.env.CLI_REGION ?? 'us-west-2'; const dbname = 'default_db'; - const engine = 'postgres'; + const engine = 'mysql'; const databaseController: SqlDatatabaseController = new SqlDatatabaseController( ['CREATE TABLE todos (id VARCHAR(40) PRIMARY KEY, description VARCHAR(256))'], diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts index 8453be4e06..a6998f2441 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts @@ -1,4 +1,4 @@ -import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core'; +import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core'; import generator from 'generate-password'; import { getResourceNamesForStrategyName } from '@aws-amplify/graphql-transformer-core'; import { SqlDatatabaseController } from '../sql-datatabase-controller'; @@ -14,7 +14,6 @@ describe('Canary using Postgres lambda model datasource strategy', () => { // sufficient password length that meets the requirements for RDS cluster/instance const [username, password, identifier] = generator.generateMultiple(3, { length: 11 }); const region = process.env.CLI_REGION ?? 'us-west-2'; - const dbname = generateDBName(); const engine = 'postgres'; const databaseController: SqlDatatabaseController = new SqlDatatabaseController( @@ -22,7 +21,6 @@ describe('Canary using Postgres lambda model datasource strategy', () => { { identifier, engine, - dbname, username, password, region, diff --git a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts index 73ff266277..929ae72c33 100644 --- a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts +++ b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts @@ -1,5 +1,6 @@ import path from 'path'; import * as fs from 'fs-extra'; +import generator from 'generate-password'; import { SqlModelDataSourceDbConnectionConfig, ModelDataSourceStrategySqlDbType } from '@aws-amplify/graphql-api-construct'; import { deleteSSMParameters, @@ -17,6 +18,8 @@ import { isOptInRegion, isDataAPISupported, isCI, + generateDBName, + isDataAPISupportedRegion, } from 'amplify-category-api-e2e-core'; import { SecretsManagerClient, CreateSecretCommand, DeleteSecretCommand, GetSecretValueCommand } from '@aws-sdk/client-secrets-manager'; import { @@ -24,6 +27,14 @@ import { isSqlModelDataSourceSsmDbConnectionConfig, isSqlModelDataSourceSsmDbConnectionStringConfig, } from '@aws-amplify/graphql-transformer-interfaces'; +import { + GetParameterCommand, + GetParameterResult, + GetParametersByPathCommand, + GetParametersByPathResult, + PutParameterCommand, + SSMClient, +} from '@aws-sdk/client-ssm'; export interface SqlDatabaseDetails { dbConfig: { @@ -52,21 +63,72 @@ export interface SqlDatabaseDetails { export class SqlDatatabaseController { private databaseDetails: SqlDatabaseDetails | undefined; private useDataAPI: boolean; + private enableLocalTesting: boolean; - constructor(private readonly setupQueries: Array, private readonly options: RDSConfig) { + constructor(private readonly setupQueries: Array, private options: RDSConfig) { // Data API is not supported in opted-in regions if (options.engine === 'postgres' && isDataAPISupported(options.region)) { this.useDataAPI = true; } else { this.useDataAPI = false; } + this.enableLocalTesting = /*isCI()*/ true; + + // If database name not manually set, provide and sanitize the config dbname + if (!options.dbname || options.dbname.length == 0 || this.enableLocalTesting) { + this.options.dbname = generateDBName().replace(/[^a-zA-Z0-9_]/g, ''); + } } + setNewConfig = async (): Promise => { + // Access the specific config from the local cluster configs JSON file + const repoRoot = path.join(__dirname, '..', '..', '..'); + const localClusterPath = path.join(repoRoot, 'scripts', 'e2e-test-local-cluster-config.json'); + const localClustersObject = JSON.parse(fs.readFileSync(localClusterPath, 'utf-8')); + const cluster = localClustersObject[this.options.region][0]; + + // Get the config identifier and connection URI + const identifier = cluster.dbConfig.identifier; + const connectionUri = `/${identifier}/test`; + + const ssmClient = new SSMClient({ region: this.options.region }); + const getParameterCommand = new GetParametersByPathCommand({ Path: connectionUri, WithDecryption: true }); + const getParameterResponse: GetParametersByPathResult = await ssmClient.send(getParameterCommand); + + const setParameterCommand = new PutParameterCommand({ + Name: `${connectionUri}/databaseName`, + Value: this.options.dbname, + Overwrite: true, + }); + const putParameterResponse = await ssmClient.send(setParameterCommand); + + //const dbname = getParameterResponse.Parameters.find(obj => obj.Name === `${connectionUri}/databaseName`).Value; + const username = getParameterResponse.Parameters.find((obj) => obj.Name === `${connectionUri}/username`).Value; + const password = getParameterResponse.Parameters.find((obj) => obj.Name === `${connectionUri}/password`).Value; + + const setParameterCommand2 = new PutParameterCommand({ Name: `${connectionUri}/password`, Value: password, Overwrite: true }); + const putParameterResponse2 = await ssmClient.send(setParameterCommand); + + const config: RDSConfig = { + identifier, + engine: this.options.engine, + dbname: this.options.dbname, + username, + password, + region: this.options.region, + }; + + return config; + }; + setupDatabase = async (): Promise => { let dbConfig; + if (this.useDataAPI) { - const enableLocalTesting = isCI(); - dbConfig = await setupRDSClusterAndData(enableLocalTesting, this.options, this.setupQueries); + if (this.enableLocalTesting) { + this.options = await this.setNewConfig(); + } + dbConfig = await setupRDSClusterAndData(this.enableLocalTesting, this.options, this.setupQueries); } else { dbConfig = await setupRDSInstanceAndData(this.options, this.setupQueries); } @@ -83,110 +145,230 @@ export class SqlDatatabaseController { }; console.log(`Stored db connection config in Secrets manager: ${JSON.stringify(dbConnectionConfigSecretsManager)}`); - if (this.useDataAPI || !this.options.password) { + if (!this.enableLocalTesting) { + if (this.useDataAPI || !this.options.password) { + const secretArn = dbConfig.secretArn; + const secretsManagerClient = new SecretsManagerClient({ region: this.options.region }); + const secretManagerCommand = new GetSecretValueCommand({ + SecretId: secretArn, + }); + const secretsManagerResponse = await secretsManagerClient.send(secretManagerCommand); + const { password: managedPassword } = JSON.parse(secretsManagerResponse.SecretString); + if (!managedPassword) { + throw new Error('Unable to get RDS cluster master user password'); + } + this.options.password = managedPassword; + } + const { secretArn: secretArnWithCustomKey, keyArn: keyArn } = await storeDbConnectionConfigWithSecretsManager({ + region: this.options.region, + username: this.options.username, + password: this.options.password, + secretName: `${this.options.identifier}-secret-custom-key`, + useCustomEncryptionKey: true, + }); + + if (!secretArnWithCustomKey) { + throw new Error('Failed to store db connection config for secrets manager'); + } + const dbConnectionConfigSecretsManagerCustomKey = { + databaseName: this.options.dbname, + hostname: dbConfig.endpoint, + port: dbConfig.port, + secretArn: secretArnWithCustomKey, + keyArn, + }; + console.log(`Stored db connection config in Secrets manager: ${JSON.stringify(dbConnectionConfigSecretsManagerCustomKey)}`); + + const pathPrefix = `/${this.options.identifier}/test`; + const engine = this.options.engine; + const dbConnectionConfigSSM = await storeDbConnectionConfig({ + region: this.options.region, + pathPrefix, + hostname: dbConfig.endpoint, + port: dbConfig.port, + databaseName: this.options.dbname, + username: this.options.username, + password: this.options.password, + }); + const dbConnectionStringConfigSSM = await storeDbConnectionStringConfig({ + region: this.options.region, + pathPrefix, + connectionUri: this.getConnectionUri( + engine, + this.options.username, + this.options.password, + dbConfig.endpoint, + dbConfig.port, + this.options.dbname, + ), + }); + const dbConnectionStringConfigMultiple = await storeDbConnectionStringConfig({ + region: this.options.region, + pathPrefix, + connectionUri: [ + 'mysql://username:password@host:3306/dbname', + this.getConnectionUri( + engine, + this.options.username, + this.options.password, + dbConfig.endpoint, + dbConfig.port, + this.options.dbname, + ), + ], + }); + const parameters = { + ...dbConnectionConfigSSM, + ...dbConnectionStringConfigSSM, + ...dbConnectionStringConfigMultiple, + }; + if (!dbConnectionConfigSSM) { + throw new Error('Failed to store db connection config for SSM'); + } + console.log(`Stored db connection config in SSM: ${JSON.stringify(Object.keys(parameters))}`); + + this.databaseDetails = { + dbConfig: { + endpoint: dbConfig.endpoint, + port: dbConfig.port, + dbName: this.options.dbname, + strategyName: `${engine}DBStrategy`, + dbType: engine === 'postgres' ? 'POSTGRES' : 'MYSQL', + vpcConfig: extractVpcConfigFromDbInstance(dbConfig.dbInstance), + }, + connectionConfigs: { + ssm: dbConnectionConfigSSM, + secretsManager: dbConnectionConfigSecretsManager, + secretsManagerCustomKey: dbConnectionConfigSecretsManagerCustomKey, + secretsManagerManagedSecret: { + databaseName: this.options.dbname, + hostname: dbConfig.endpoint, + port: dbConfig.port, + secretArn: dbConfig.secretArn, + }, + connectionUri: dbConnectionStringConfigSSM, + connectionUriMultiple: dbConnectionStringConfigMultiple, + }, + }; + return this.databaseDetails; + } else { + // this.enableLocalTesting const secretArn = dbConfig.secretArn; const secretsManagerClient = new SecretsManagerClient({ region: this.options.region }); const secretManagerCommand = new GetSecretValueCommand({ SecretId: secretArn, }); const secretsManagerResponse = await secretsManagerClient.send(secretManagerCommand); - const { password: managedPassword } = JSON.parse(secretsManagerResponse.SecretString); - if (!managedPassword) { - throw new Error('Unable to get RDS cluster master user password'); + const secretArnWithCustomKey = secretsManagerResponse.ARN; // arn:aws:secretsmanager:us-west-2:637423428135:secret:yjLvhaVbVT-secret-custom-key-BTqGiz + const keyArn = 'arn:aws:kms:us-west-2:637423428135:key/a84fe5fe-ee01-44bf-b94e-57604a109c77'; // ELVIN - GET THE MASTER CREDENTIALS KMS KEY SOMEHOW (like "arn:aws:kms:us-west-2:637423428135:key/62ec591f-7188-49db-8819-87c15727908f") + + const pathPrefix = `/${this.options.identifier}/test`; + const engine = this.options.engine; + const dbConnectionConfigSSM = await storeDbConnectionConfig({ + region: this.options.region, + pathPrefix, + hostname: dbConfig.endpoint, + port: dbConfig.port, + databaseName: this.options.dbname, + username: this.options.username, + password: this.options.password, + }); + const dbConnectionStringConfigSSM = await storeDbConnectionStringConfig({ + region: this.options.region, + pathPrefix, + connectionUri: this.getConnectionUri( + engine, + this.options.username, + this.options.password, + dbConfig.endpoint, + dbConfig.port, + this.options.dbname, + ), + }); + const dbConnectionStringConfigMultiple = await storeDbConnectionStringConfig({ + region: this.options.region, + pathPrefix, + connectionUri: [ + 'mysql://username:password@host:3306/dbname', + this.getConnectionUri( + engine, + this.options.username, + this.options.password, + dbConfig.endpoint, + dbConfig.port, + this.options.dbname, + ), + ], + }); + + if (!secretArnWithCustomKey) { + throw new Error('Failed to store db connection config for secrets manager'); } - this.options.password = managedPassword; - } + const dbConnectionConfigSecretsManagerCustomKey = { + databaseName: this.options.dbname, + hostname: dbConfig.endpoint, + port: dbConfig.port, + secretArn: secretArnWithCustomKey, + keyArn, + }; + console.log(`Stored db connection config in Secrets manager: ${JSON.stringify(dbConnectionConfigSecretsManagerCustomKey)}`); - const { secretArn: secretArnWithCustomKey, keyArn } = await storeDbConnectionConfigWithSecretsManager({ - region: this.options.region, - username: this.options.username, - password: this.options.password, - secretName: `${this.options.identifier}-secret-custom-key`, - useCustomEncryptionKey: true, - }); - if (!secretArnWithCustomKey) { - throw new Error('Failed to store db connection config for secrets manager'); - } - const dbConnectionConfigSecretsManagerCustomKey = { - databaseName: this.options.dbname, - hostname: dbConfig.endpoint, - port: dbConfig.port, - secretArn: secretArnWithCustomKey, - keyArn, - }; - console.log(`Stored db connection config in Secrets manager: ${JSON.stringify(dbConnectionConfigSecretsManagerCustomKey)}`); + const identifier = this.options.identifier; + /*const pathPrefix = `/${identifier}/test`; + const engine = this.options.engine; + const dbConnectionConfigSSM = { + hostnameSsmPath: `${pathPrefix}/hostname`, + portSsmPath: `${pathPrefix}/port`, + usernameSsmPath: `${pathPrefix}/username`, + passwordSsmPath: `${pathPrefix}/password`, + databaseNameSsmPath: `${pathPrefix}/databaseName`, + };*/ - const pathPrefix = `/${this.options.identifier}/test`; - const engine = this.options.engine; - const dbConnectionConfigSSM = await storeDbConnectionConfig({ - region: this.options.region, - pathPrefix, - hostname: dbConfig.endpoint, - port: dbConfig.port, - databaseName: this.options.dbname, - username: this.options.username, - password: this.options.password, - }); - const dbConnectionStringConfigSSM = await storeDbConnectionStringConfig({ - region: this.options.region, - pathPrefix, - connectionUri: this.getConnectionUri( - engine, - this.options.username, - this.options.password, - dbConfig.endpoint, - dbConfig.port, - this.options.dbname, - ), - }); - const dbConnectionStringConfigMultiple = await storeDbConnectionStringConfig({ - region: this.options.region, - pathPrefix, - connectionUri: [ - 'mysql://username:password@host:3306/dbname', - this.getConnectionUri(engine, this.options.username, this.options.password, dbConfig.endpoint, dbConfig.port, this.options.dbname), - ], - }); - const parameters = { - ...dbConnectionConfigSSM, - ...dbConnectionStringConfigSSM, - ...dbConnectionStringConfigMultiple, - }; - if (!dbConnectionConfigSSM) { - throw new Error('Failed to store db connection config for SSM'); - } - console.log(`Stored db connection config in SSM: ${JSON.stringify(Object.keys(parameters))}`); + /*const dbConnectionStringConfigSSM = {connectionUriSsmPath: `${pathPrefix}/connectionUri`}; + const dbConnectionStringConfigMultiple = {connectionUriSsmPath: [ + `${pathPrefix}/connectionUri/doesnotexist`, + `${pathPrefix}/connectionUri`, + ]};*/ + const parameters = { + ...dbConnectionConfigSSM, + ...dbConnectionStringConfigSSM, + ...dbConnectionStringConfigMultiple, + }; + if (!dbConnectionConfigSSM) { + throw new Error('Failed to store db connection config for SSM'); + } + console.log(`Stored db connection config in SSM: ${JSON.stringify(Object.keys(parameters))}`); - this.databaseDetails = { - dbConfig: { - endpoint: dbConfig.endpoint, - port: dbConfig.port, - dbName: this.options.dbname, - strategyName: `${engine}DBStrategy`, - dbType: engine === 'postgres' ? 'POSTGRES' : 'MYSQL', - vpcConfig: extractVpcConfigFromDbInstance(dbConfig.dbInstance), - }, - connectionConfigs: { - ssm: dbConnectionConfigSSM, - secretsManager: dbConnectionConfigSecretsManager, - secretsManagerCustomKey: dbConnectionConfigSecretsManagerCustomKey, - secretsManagerManagedSecret: { - databaseName: this.options.dbname, - hostname: dbConfig.endpoint, + this.databaseDetails = { + dbConfig: { + endpoint: dbConfig.endpoint, port: dbConfig.port, - secretArn: dbConfig.secretArn, + dbName: this.options.dbname, + strategyName: `${engine}DBStrategy`, + dbType: engine === 'postgres' ? 'POSTGRES' : 'MYSQL', + vpcConfig: extractVpcConfigFromDbInstance(dbConfig.dbInstance), }, - connectionUri: dbConnectionStringConfigSSM, - connectionUriMultiple: dbConnectionStringConfigMultiple, - }, - }; - - return this.databaseDetails; + connectionConfigs: { + ssm: dbConnectionConfigSSM, + secretsManager: dbConnectionConfigSecretsManager, + secretsManagerCustomKey: dbConnectionConfigSecretsManagerCustomKey, + secretsManagerManagedSecret: { + databaseName: this.options.dbname, + hostname: dbConfig.endpoint, + port: dbConfig.port, + secretArn: dbConfig.secretArn, + }, + connectionUri: dbConnectionStringConfigSSM, + connectionUriMultiple: dbConnectionStringConfigMultiple, + }, + }; + return this.databaseDetails; + } }; cleanupDatabase = async (): Promise => { - if (!this.databaseDetails) { - // Database has not been set up. + if (!this.databaseDetails || this.enableLocalTesting) { + // Database has not been set up or using a local test cluster. return; } diff --git a/scripts/e2e-test-local-cluster-config.json b/scripts/e2e-test-local-cluster-config.json index 36d698a3d4..e9728d865b 100644 --- a/scripts/e2e-test-local-cluster-config.json +++ b/scripts/e2e-test-local-cluster-config.json @@ -1,8 +1,19 @@ { - "dbConfig": { - "identifier": "" - }, - "connectionConfigs": { - "connectionUri": "/BfgiBDcCQYM/test" - } + "us-east-1": [ + { + "dbConfig": { + "identifier": "testIdentifier" + }, + "connectionConfigs": { + "connectionUri": "" + } + } + ], + "us-west-2": [ + { + "dbConfig": { + "identifier": "YMmwoSDbAQ" + } + } + ] } From 642e1e58b8f044b8153046de1fecd5ea7d4f3a5e Mon Sep 17 00:00:00 2001 From: fuelvin1 Date: Tue, 13 Aug 2024 15:27:38 -0700 Subject: [PATCH 5/9] chore: working e2e local testing --- packages/amplify-e2e-core/src/utils/rds.ts | 46 ++++++++++++++++--- .../uuid-pk-ddbprimary-sqlrelated.test.ts | 4 +- .../uuid-pk-sqlprimary-ddbrelated.test.ts | 4 +- .../uuid-pk-sqlprimary-sqlrelated.test.ts | 4 +- .../src/__tests__/sql-pg-canary.test.ts | 2 + .../src/__tests__/sql-pg-models.test.ts | 4 +- .../src/sql-datatabase-controller.ts | 39 ++++++++-------- scripts/e2e-test-local-cluster-config.json | 5 +- 8 files changed, 65 insertions(+), 43 deletions(-) diff --git a/packages/amplify-e2e-core/src/utils/rds.ts b/packages/amplify-e2e-core/src/utils/rds.ts index 456f1acfeb..77e6ec09f2 100644 --- a/packages/amplify-e2e-core/src/utils/rds.ts +++ b/packages/amplify-e2e-core/src/utils/rds.ts @@ -45,6 +45,8 @@ const DEFAULT_SECURITY_GROUP = 'default'; const IPIFY_URL = 'https://api.ipify.org/'; const AWSCHECKIP_URL = 'https://checkip.amazonaws.com/'; +let clusterInfo: ClusterInfo; + export type SqlEngine = 'mysql' | 'postgres'; export type RDSConfig = { identifier: string; @@ -256,7 +258,7 @@ export const createRDSCluster = async (config: RDSConfig): Promise * @param config Configuration of the database cluster. * @returns EndPoint address, port and database name of the accessed RDS cluster. */ -export const useLocalCluster = async (config: RDSConfig): Promise => { +export const useRDSCluster = async (config: RDSConfig): Promise => { try { // Send requests to get the cluster and instance config info const databaseName = 'defaultdb'; @@ -369,21 +371,23 @@ export const setupRDSInstanceAndData = async ( * @returns Endpoint address, port and database name of the created RDS cluster. */ -export const setupRDSClusterAndData = async (localTesting: boolean, config: RDSConfig, queries?: string[]): Promise => { - console.log(`Creating RDS ${config.engine} DB cluster with identifier ${config.identifier}`); - +export const setupRDSClusterAndData = async (useLocalCluster: boolean, config: RDSConfig, queries?: string[]): Promise => { let dbCluster: ClusterInfo; - if (!localTesting) { + if (!useLocalCluster) { dbCluster = await createRDSCluster(config); + console.log(`Creating RDS ${config.engine} DB cluster with identifier ${config.identifier}`); } else { - dbCluster = await useLocalCluster(config); + dbCluster = await useRDSCluster(config); + console.log(`Using RDS ${config.engine} DB cluster with identifier ${config.identifier}`); } if (!dbCluster.secretArn) { throw new Error('Failed to store db connection config in secrets manager'); } + clusterInfo = dbCluster; + const client = new RDSDataClient({ region: config.region }); const createDBInput: ExecuteStatementCommandInput = { @@ -417,6 +421,19 @@ export const setupRDSClusterAndData = async (localTesting: boolean, config: RDSC } } + /*try { + const executeStatementInput: ExecuteStatementCommandInput = { + resourceArn: dbCluster.clusterArn, + secretArn: dbCluster.secretArn, + sql: `drop database ${config.dbname} with (FORCE)`, + database: dbCluster.dbName, + }; + const executeStatementResponse = await client.send(new ExecuteStatementCommand(executeStatementInput)); + console.log('Run query response: ' + JSON.stringify(executeStatementResponse)); + } catch (err) { + throw new Error(`Error in running queries in test database: ${err}`); + }*/ + return { clusterArn: dbCluster.clusterArn, endpoint: dbCluster.endpoint, @@ -427,6 +444,23 @@ export const setupRDSClusterAndData = async (localTesting: boolean, config: RDSC }; }; +export const dropDatabase = async (config: RDSConfig): Promise => { + const client = new RDSDataClient({ region: config.region }); + // Drop the test database + try { + const executeStatementInput: ExecuteStatementCommandInput = { + resourceArn: clusterInfo.clusterArn, + secretArn: clusterInfo.secretArn, + sql: `drop database ${config.dbname} with (FORCE)`, + database: clusterInfo.dbName, + }; + const executeStatementResponse = await client.send(new ExecuteStatementCommand(executeStatementInput)); + console.log('Run query response: ' + JSON.stringify(executeStatementResponse)); + } catch (err) { + throw new Error(`Error in running queries in test database: ${err}`); + } +}; + /** * Deletes the given RDS instance * @param identifier RDS Instance identifier to delete diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-ddbprimary-sqlrelated.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-ddbprimary-sqlrelated.test.ts index 720d7db167..d7e51b5e5f 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-ddbprimary-sqlrelated.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-ddbprimary-sqlrelated.test.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import * as fs from 'fs-extra'; import * as generator from 'generate-password'; -import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core'; +import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core'; import { DDB_AMPLIFY_MANAGED_DATASOURCE_STRATEGY } from '@aws-amplify/graphql-transformer-core'; import { cdkDeploy, cdkDestroy, initCDKProject } from '../../../commands'; import { SqlDatabaseDetails, SqlDatatabaseController } from '../../../sql-datatabase-controller'; @@ -23,7 +23,6 @@ describe('PostgreSQL tables with UUID primary keys', () => { const region = process.env.CLI_REGION ?? 'us-west-2'; const baseProjFolderName = path.basename(__filename, '.test.ts'); - const dbname = generateDBName(); const [dbUsername, dbIdentifier] = generator.generateMultiple(2); let dbDetails: SqlDatabaseDetails; @@ -46,7 +45,6 @@ describe('PostgreSQL tables with UUID primary keys', () => { { identifier: dbIdentifier, engine: 'postgres', - dbname, username: dbUsername, region, }, diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-ddbrelated.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-ddbrelated.test.ts index 32d6e9156c..205a860a7d 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-ddbrelated.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-ddbrelated.test.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import * as fs from 'fs-extra'; import * as generator from 'generate-password'; -import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core'; +import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core'; import { DDB_AMPLIFY_MANAGED_DATASOURCE_STRATEGY } from '@aws-amplify/graphql-transformer-core'; import { cdkDeploy, cdkDestroy, initCDKProject } from '../../../commands'; import { SqlDatabaseDetails, SqlDatatabaseController } from '../../../sql-datatabase-controller'; @@ -23,7 +23,6 @@ describe('PostgreSQL tables with UUID primary keys', () => { const region = process.env.CLI_REGION ?? 'us-west-2'; const baseProjFolderName = path.basename(__filename, '.test.ts'); - const dbname = generateDBName(); const [dbUsername, dbIdentifier] = generator.generateMultiple(2); let dbDetails: SqlDatabaseDetails; @@ -46,7 +45,6 @@ describe('PostgreSQL tables with UUID primary keys', () => { { identifier: dbIdentifier, engine: 'postgres', - dbname, username: dbUsername, region, }, diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-sqlrelated.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-sqlrelated.test.ts index 934201aaaf..2e8f61532d 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-sqlrelated.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-sqlrelated.test.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import * as fs from 'fs-extra'; import * as generator from 'generate-password'; -import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core'; +import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core'; import { cdkDeploy, cdkDestroy, initCDKProject } from '../../../commands'; import { SqlDatabaseDetails, SqlDatatabaseController } from '../../../sql-datatabase-controller'; import { TestDefinition, dbDetailsToModelDataSourceStrategy, writeStackConfig, writeTestDefinitions } from '../../../utils'; @@ -22,7 +22,6 @@ describe('PostgreSQL tables with UUID primary keys', () => { const region = process.env.CLI_REGION ?? 'us-west-2'; const baseProjFolderName = path.basename(__filename, '.test.ts'); - const dbname = generateDBName(); const [dbUsername, dbIdentifier] = generator.generateMultiple(2); let dbDetails: SqlDatabaseDetails; @@ -45,7 +44,6 @@ describe('PostgreSQL tables with UUID primary keys', () => { { identifier: dbIdentifier, engine: 'postgres', - dbname, username: dbUsername, region, }, diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts index a6998f2441..2ec6411c9e 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts @@ -31,7 +31,9 @@ describe('Canary using Postgres lambda model datasource strategy', () => { const resourceNames = getResourceNamesForStrategyName(strategyName); beforeAll(async () => { + console.time('sql-pg-canary test setup'); await databaseController.setupDatabase(); + console.timeEnd('sql-pg-canary test setup'); }); afterAll(async () => { diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-models.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-models.test.ts index 7034bd5896..32fbb00d49 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-models.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-models.test.ts @@ -1,4 +1,4 @@ -import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core'; +import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core'; import generator from 'generate-password'; import { getResourceNamesForStrategyName } from '@aws-amplify/graphql-transformer-core'; import { SqlDatatabaseController } from '../sql-datatabase-controller'; @@ -15,7 +15,6 @@ describe('CDK GraphQL Transformer deployments with Postgres SQL datasources', () // sufficient password length that meets the requirements for RDS cluster/instance const [username, password, identifier] = generator.generateMultiple(3, { length: 11 }); const region = process.env.CLI_REGION ?? 'us-west-2'; - const dbname = generateDBName(); const engine = 'postgres'; const databaseController: SqlDatatabaseController = new SqlDatatabaseController( @@ -23,7 +22,6 @@ describe('CDK GraphQL Transformer deployments with Postgres SQL datasources', () { identifier, engine, - dbname, username, password, region, diff --git a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts index 929ae72c33..defbc37608 100644 --- a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts +++ b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts @@ -20,6 +20,7 @@ import { isCI, generateDBName, isDataAPISupportedRegion, + dropDatabase, } from 'amplify-category-api-e2e-core'; import { SecretsManagerClient, CreateSecretCommand, DeleteSecretCommand, GetSecretValueCommand } from '@aws-sdk/client-secrets-manager'; import { @@ -72,7 +73,7 @@ export class SqlDatatabaseController { } else { this.useDataAPI = false; } - this.enableLocalTesting = /*isCI()*/ true; + this.enableLocalTesting = isCI(); // If database name not manually set, provide and sanitize the config dbname if (!options.dbname || options.dbname.length == 0 || this.enableLocalTesting) { @@ -89,6 +90,11 @@ export class SqlDatatabaseController { // Get the config identifier and connection URI const identifier = cluster.dbConfig.identifier; + + if (identifier === '' || identifier === null) { + return null; + } + const connectionUri = `/${identifier}/test`; const ssmClient = new SSMClient({ region: this.options.region }); @@ -102,13 +108,9 @@ export class SqlDatatabaseController { }); const putParameterResponse = await ssmClient.send(setParameterCommand); - //const dbname = getParameterResponse.Parameters.find(obj => obj.Name === `${connectionUri}/databaseName`).Value; const username = getParameterResponse.Parameters.find((obj) => obj.Name === `${connectionUri}/username`).Value; const password = getParameterResponse.Parameters.find((obj) => obj.Name === `${connectionUri}/password`).Value; - const setParameterCommand2 = new PutParameterCommand({ Name: `${connectionUri}/password`, Value: password, Overwrite: true }); - const putParameterResponse2 = await ssmClient.send(setParameterCommand); - const config: RDSConfig = { identifier, engine: this.options.engine, @@ -126,7 +128,13 @@ export class SqlDatatabaseController { if (this.useDataAPI) { if (this.enableLocalTesting) { - this.options = await this.setNewConfig(); + const newConfig = await this.setNewConfig(); + if (newConfig === null) { + // If local cluster identifier is empty or non-existent, disable local testing and create a new cluster + this.enableLocalTesting = false; + } else { + this.options = newConfig; + } } dbConfig = await setupRDSClusterAndData(this.enableLocalTesting, this.options, this.setupQueries); } else { @@ -314,21 +322,7 @@ export class SqlDatatabaseController { console.log(`Stored db connection config in Secrets manager: ${JSON.stringify(dbConnectionConfigSecretsManagerCustomKey)}`); const identifier = this.options.identifier; - /*const pathPrefix = `/${identifier}/test`; - const engine = this.options.engine; - const dbConnectionConfigSSM = { - hostnameSsmPath: `${pathPrefix}/hostname`, - portSsmPath: `${pathPrefix}/port`, - usernameSsmPath: `${pathPrefix}/username`, - passwordSsmPath: `${pathPrefix}/password`, - databaseNameSsmPath: `${pathPrefix}/databaseName`, - };*/ - - /*const dbConnectionStringConfigSSM = {connectionUriSsmPath: `${pathPrefix}/connectionUri`}; - const dbConnectionStringConfigMultiple = {connectionUriSsmPath: [ - `${pathPrefix}/connectionUri/doesnotexist`, - `${pathPrefix}/connectionUri`, - ]};*/ + const parameters = { ...dbConnectionConfigSSM, ...dbConnectionStringConfigSSM, @@ -369,6 +363,9 @@ export class SqlDatatabaseController { cleanupDatabase = async (): Promise => { if (!this.databaseDetails || this.enableLocalTesting) { // Database has not been set up or using a local test cluster. + if (this.enableLocalTesting) { + dropDatabase(this.options); + } return; } diff --git a/scripts/e2e-test-local-cluster-config.json b/scripts/e2e-test-local-cluster-config.json index e9728d865b..81bc605d95 100644 --- a/scripts/e2e-test-local-cluster-config.json +++ b/scripts/e2e-test-local-cluster-config.json @@ -2,10 +2,7 @@ "us-east-1": [ { "dbConfig": { - "identifier": "testIdentifier" - }, - "connectionConfigs": { - "connectionUri": "" + "identifier": "" } } ], From 14f6930ce141aaaafde4a6d3868a50267c3ec84d Mon Sep 17 00:00:00 2001 From: fuelvin1 Date: Wed, 14 Aug 2024 15:20:58 -0700 Subject: [PATCH 6/9] chore: update isCI --- .../src/sql-datatabase-controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts index defbc37608..28ae61a53f 100644 --- a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts +++ b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts @@ -73,7 +73,7 @@ export class SqlDatatabaseController { } else { this.useDataAPI = false; } - this.enableLocalTesting = isCI(); + this.enableLocalTesting = !isCI(); // If database name not manually set, provide and sanitize the config dbname if (!options.dbname || options.dbname.length == 0 || this.enableLocalTesting) { From c89dcf0b55026ea2b6ef40b3341b8191df619077 Mon Sep 17 00:00:00 2001 From: fuelvin1 Date: Tue, 20 Aug 2024 11:27:24 -0700 Subject: [PATCH 7/9] chore: refactored local testing code --- .gitignore | 1 + packages/amplify-e2e-core/src/utils/rds.ts | 148 +++---- .../src/sql-datatabase-controller.ts | 370 +++++------------- .../src/utils/sql-local-testing.ts | 29 ++ scripts/e2e-test-local-cluster-config.json | 6 +- .../e2e-test-local-cluster-config.sample.json | 6 +- yarn.lock | 7 +- 7 files changed, 209 insertions(+), 358 deletions(-) create mode 100644 packages/amplify-graphql-api-construct-tests/src/utils/sql-local-testing.ts diff --git a/.gitignore b/.gitignore index 255df3e152..d0b2ae737f 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,4 @@ codebuild_specs/debug_workflow.yml .npmrc verdaccio-logs.txt scripts/components/private_packages.ts +scripts/e2e-test-local-cluster-config.json \ No newline at end of file diff --git a/packages/amplify-e2e-core/src/utils/rds.ts b/packages/amplify-e2e-core/src/utils/rds.ts index 77e6ec09f2..1a3580086c 100644 --- a/packages/amplify-e2e-core/src/utils/rds.ts +++ b/packages/amplify-e2e-core/src/utils/rds.ts @@ -4,17 +4,14 @@ import { CreateDBInstanceCommand, CreateDBInstanceCommandInput, DBInstance, - DeleteDBInstanceCommand, - DescribeDBClustersCommand, - waitUntilDBInstanceAvailable, CreateDBClusterMessage, - waitUntilDBClusterAvailable, DeleteDBClusterCommand, DeleteDBClusterCommandInput, - $Command, + DeleteDBInstanceCommand, + DescribeDBClustersCommand, DescribeDBInstancesCommand, - DescribeDBClustersCommandOutput, - DescribeDBInstancesCommandOutput, + waitUntilDBClusterAvailable, + waitUntilDBInstanceAvailable, } from '@aws-sdk/client-rds'; import { RDSDataClient, ExecuteStatementCommand, ExecuteStatementCommandInput, Field } from '@aws-sdk/client-rds-data'; import generator from 'generate-password'; @@ -26,16 +23,11 @@ import { PutParameterCommand, PutParameterCommandInput, PutParameterCommandOutput, - GetParameterCommand, - GetParametersByPathCommand, - GetParameterResult, } from '@aws-sdk/client-ssm'; import { SecretsManagerClient, CreateSecretCommand, DeleteSecretCommand, GetSecretValueCommand } from '@aws-sdk/client-secrets-manager'; import { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand } from '@aws-sdk/client-kms'; import { knex } from 'knex'; import axios from 'axios'; -import fs from 'fs-extra'; -import path from 'path'; import { sleep } from './sleep'; const DEFAULT_DB_INSTANCE_TYPE = 'db.m5.large'; @@ -45,8 +37,6 @@ const DEFAULT_SECURITY_GROUP = 'default'; const IPIFY_URL = 'https://api.ipify.org/'; const AWSCHECKIP_URL = 'https://checkip.amazonaws.com/'; -let clusterInfo: ClusterInfo; - export type SqlEngine = 'mysql' | 'postgres'; export type RDSConfig = { identifier: string; @@ -67,6 +57,7 @@ export type ClusterInfo = { dbName: string; secretArn: string; dbInstance: DBInstance; + username?: string; }; const getRDSEngineType = (engine: SqlEngine): string => { @@ -258,49 +249,73 @@ export const createRDSCluster = async (config: RDSConfig): Promise * @param config Configuration of the database cluster. * @returns EndPoint address, port and database name of the accessed RDS cluster. */ -export const useRDSCluster = async (config: RDSConfig): Promise => { +export const setupDataInExistingCluster = async (identifier: string, config: RDSConfig, queries: string[]): Promise => { try { - // Send requests to get the cluster and instance config info - const databaseName = 'defaultdb'; - const identifier = config.identifier; - const instanceIdentifier = createInstanceIdentifier(config.identifier); const client = new RDSClient({ region: config.region }); - const describeClusterCommand = new DescribeDBClustersCommand({ Filters: [{ Name: 'db-cluster-id', Values: [identifier] }] }); - const describeInstanceCommand = new DescribeDBInstancesCommand({ DBInstanceIdentifier: instanceIdentifier }); + const describeClusterResponse = await client.send( + new DescribeDBClustersCommand({ Filters: [{ Name: 'db-cluster-id', Values: [identifier] }] }), + ); + if (!describeClusterResponse || !describeClusterResponse?.DBClusters[0]) { + throw Error('Specified cluster info cannot be fetched'); + } + const dbClusterObj = describeClusterResponse.DBClusters[0]; + const instances = dbClusterObj?.DBClusterMembers; + if (!instances || instances?.length === 0) { + throw new Error('No instances are present in the specified cluster'); + } + const instanceId = instances[0]?.DBInstanceIdentifier; + const describeInstanceCommand = new DescribeDBInstancesCommand({ DBInstanceIdentifier: instanceId }); + const describeInstanceResponse = await client.send(describeInstanceCommand); + if (!describeInstanceResponse || describeInstanceResponse?.DBInstances?.length === 0) { + throw Error('Specified cluster instance info cannot be fetched'); + } - let describeClusterResponse: DescribeDBClustersCommandOutput; - let describeInstanceResponse: DescribeDBInstancesCommandOutput; + const clusterArn = dbClusterObj.DBClusterArn; + const secretArn = dbClusterObj.MasterUserSecret.SecretArn; + const defaultDbName = dbClusterObj.DatabaseName; + const dataClient = new RDSDataClient({ region: config.region }); + const createDBInput: ExecuteStatementCommandInput = { + resourceArn: clusterArn, + secretArn, + sql: `create database ${config.dbname}`, + database: defaultDbName, + }; + + const createDBCommand = new ExecuteStatementCommand(createDBInput); try { - describeClusterResponse = await client.send(describeClusterCommand); - if (describeClusterResponse == null) { - throw Error('Specified cluster is null.'); - } - } catch (error) { - throw Error(`Error in getting ${identifier} cluster info: ${error}`); + const createDBResponse = await dataClient.send(createDBCommand); + console.log('Create database response: ' + JSON.stringify(createDBResponse)); + } catch (err) { + console.log(err); } - try { - describeInstanceResponse = await client.send(describeInstanceCommand); - if (describeInstanceResponse == null) { - throw Error('Specified instance is null.'); + + // create the test tables in the test database + for (const query of queries ?? []) { + try { + const executeStatementInput: ExecuteStatementCommandInput = { + resourceArn: clusterArn, + secretArn: secretArn, + sql: query, + database: config.dbname, + }; + const executeStatementResponse = await dataClient.send(new ExecuteStatementCommand(executeStatementInput)); + console.log('Run query response: ' + JSON.stringify(executeStatementResponse)); + } catch (err) { + throw new Error(`Error in creating tables in test database: ${JSON.stringify(err, null, 4)}`); } - } catch (error) { - throw Error(`Error in getting ${instanceIdentifier} cluster info: ${error}`); } - // Extract the cluster and instance from the responses - const dbClusterObj = describeClusterResponse.DBClusters[0]; - const dbInstance = describeInstanceResponse.DBInstances[0]; - return { - clusterArn: dbClusterObj.DBClusterArn, + clusterArn, endpoint: dbClusterObj.Endpoint, port: dbClusterObj.Port, - dbName: databaseName, - dbInstance: dbInstance, - secretArn: dbClusterObj.MasterUserSecret.SecretArn, + dbName: config.dbname, + dbInstance: describeInstanceResponse.DBInstances[0], + secretArn, + username: dbClusterObj.MasterUsername, }; } catch (error) { - console.log('Error: ', error); + console.log('Error while setting up the test data in existing cluster: ', JSON.stringify(error)); } }; @@ -371,23 +386,14 @@ export const setupRDSInstanceAndData = async ( * @returns Endpoint address, port and database name of the created RDS cluster. */ -export const setupRDSClusterAndData = async (useLocalCluster: boolean, config: RDSConfig, queries?: string[]): Promise => { - let dbCluster: ClusterInfo; - - if (!useLocalCluster) { - dbCluster = await createRDSCluster(config); - console.log(`Creating RDS ${config.engine} DB cluster with identifier ${config.identifier}`); - } else { - dbCluster = await useRDSCluster(config); - console.log(`Using RDS ${config.engine} DB cluster with identifier ${config.identifier}`); - } +export const setupRDSClusterAndData = async (config: RDSConfig, queries?: string[]): Promise => { + const dbCluster = await createRDSCluster(config); + console.log(`Creating RDS ${config.engine} DB cluster with identifier ${config.identifier}`); if (!dbCluster.secretArn) { throw new Error('Failed to store db connection config in secrets manager'); } - clusterInfo = dbCluster; - const client = new RDSDataClient({ region: config.region }); const createDBInput: ExecuteStatementCommandInput = { @@ -421,19 +427,6 @@ export const setupRDSClusterAndData = async (useLocalCluster: boolean, config: R } } - /*try { - const executeStatementInput: ExecuteStatementCommandInput = { - resourceArn: dbCluster.clusterArn, - secretArn: dbCluster.secretArn, - sql: `drop database ${config.dbname} with (FORCE)`, - database: dbCluster.dbName, - }; - const executeStatementResponse = await client.send(new ExecuteStatementCommand(executeStatementInput)); - console.log('Run query response: ' + JSON.stringify(executeStatementResponse)); - } catch (err) { - throw new Error(`Error in running queries in test database: ${err}`); - }*/ - return { clusterArn: dbCluster.clusterArn, endpoint: dbCluster.endpoint, @@ -444,23 +437,6 @@ export const setupRDSClusterAndData = async (useLocalCluster: boolean, config: R }; }; -export const dropDatabase = async (config: RDSConfig): Promise => { - const client = new RDSDataClient({ region: config.region }); - // Drop the test database - try { - const executeStatementInput: ExecuteStatementCommandInput = { - resourceArn: clusterInfo.clusterArn, - secretArn: clusterInfo.secretArn, - sql: `drop database ${config.dbname} with (FORCE)`, - database: clusterInfo.dbName, - }; - const executeStatementResponse = await client.send(new ExecuteStatementCommand(executeStatementInput)); - console.log('Run query response: ' + JSON.stringify(executeStatementResponse)); - } catch (err) { - throw new Error(`Error in running queries in test database: ${err}`); - } -}; - /** * Deletes the given RDS instance * @param identifier RDS Instance identifier to delete diff --git a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts index 28ae61a53f..8c589f27c1 100644 --- a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts +++ b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts @@ -1,10 +1,10 @@ import path from 'path'; import * as fs from 'fs-extra'; -import generator from 'generate-password'; import { SqlModelDataSourceDbConnectionConfig, ModelDataSourceStrategySqlDbType } from '@aws-amplify/graphql-api-construct'; import { deleteSSMParameters, deleteDbConnectionConfigWithSecretsManager, + deleteDBCluster, deleteDBInstance, extractVpcConfigFromDbInstance, RDSConfig, @@ -14,28 +14,18 @@ import { storeDbConnectionConfig, storeDbConnectionStringConfig, storeDbConnectionConfigWithSecretsManager, - deleteDBCluster, - isOptInRegion, isDataAPISupported, isCI, generateDBName, - isDataAPISupportedRegion, - dropDatabase, + setupDataInExistingCluster, } from 'amplify-category-api-e2e-core'; -import { SecretsManagerClient, CreateSecretCommand, DeleteSecretCommand, GetSecretValueCommand } from '@aws-sdk/client-secrets-manager'; +import { SecretsManagerClient, GetSecretValueCommand } from '@aws-sdk/client-secrets-manager'; import { isSqlModelDataSourceSecretsManagerDbConnectionConfig, isSqlModelDataSourceSsmDbConnectionConfig, isSqlModelDataSourceSsmDbConnectionStringConfig, } from '@aws-amplify/graphql-transformer-interfaces'; -import { - GetParameterCommand, - GetParameterResult, - GetParametersByPathCommand, - GetParametersByPathResult, - PutParameterCommand, - SSMClient, -} from '@aws-sdk/client-ssm'; +import { getClusterIdentifier } from './utils/sql-local-testing'; export interface SqlDatabaseDetails { dbConfig: { @@ -70,10 +60,10 @@ export class SqlDatatabaseController { // Data API is not supported in opted-in regions if (options.engine === 'postgres' && isDataAPISupported(options.region)) { this.useDataAPI = true; + this.enableLocalTesting = !isCI() && getClusterIdentifier(options.region, options.engine) !== undefined; } else { this.useDataAPI = false; } - this.enableLocalTesting = !isCI(); // If database name not manually set, provide and sanitize the config dbname if (!options.dbname || options.dbname.length == 0 || this.enableLocalTesting) { @@ -81,62 +71,17 @@ export class SqlDatatabaseController { } } - setNewConfig = async (): Promise => { - // Access the specific config from the local cluster configs JSON file - const repoRoot = path.join(__dirname, '..', '..', '..'); - const localClusterPath = path.join(repoRoot, 'scripts', 'e2e-test-local-cluster-config.json'); - const localClustersObject = JSON.parse(fs.readFileSync(localClusterPath, 'utf-8')); - const cluster = localClustersObject[this.options.region][0]; - - // Get the config identifier and connection URI - const identifier = cluster.dbConfig.identifier; - - if (identifier === '' || identifier === null) { - return null; - } - - const connectionUri = `/${identifier}/test`; - - const ssmClient = new SSMClient({ region: this.options.region }); - const getParameterCommand = new GetParametersByPathCommand({ Path: connectionUri, WithDecryption: true }); - const getParameterResponse: GetParametersByPathResult = await ssmClient.send(getParameterCommand); - - const setParameterCommand = new PutParameterCommand({ - Name: `${connectionUri}/databaseName`, - Value: this.options.dbname, - Overwrite: true, - }); - const putParameterResponse = await ssmClient.send(setParameterCommand); - - const username = getParameterResponse.Parameters.find((obj) => obj.Name === `${connectionUri}/username`).Value; - const password = getParameterResponse.Parameters.find((obj) => obj.Name === `${connectionUri}/password`).Value; - - const config: RDSConfig = { - identifier, - engine: this.options.engine, - dbname: this.options.dbname, - username, - password, - region: this.options.region, - }; - - return config; - }; - setupDatabase = async (): Promise => { let dbConfig; if (this.useDataAPI) { if (this.enableLocalTesting) { - const newConfig = await this.setNewConfig(); - if (newConfig === null) { - // If local cluster identifier is empty or non-existent, disable local testing and create a new cluster - this.enableLocalTesting = false; - } else { - this.options = newConfig; - } + const identifier = getClusterIdentifier(this.options.region, this.options.engine); + dbConfig = await setupDataInExistingCluster(identifier, this.options, this.setupQueries); + this.options.username = dbConfig.username; + } else { + dbConfig = await setupRDSClusterAndData(this.options, this.setupQueries); } - dbConfig = await setupRDSClusterAndData(this.enableLocalTesting, this.options, this.setupQueries); } else { dbConfig = await setupRDSInstanceAndData(this.options, this.setupQueries); } @@ -153,226 +98,117 @@ export class SqlDatatabaseController { }; console.log(`Stored db connection config in Secrets manager: ${JSON.stringify(dbConnectionConfigSecretsManager)}`); - if (!this.enableLocalTesting) { - if (this.useDataAPI || !this.options.password) { - const secretArn = dbConfig.secretArn; - const secretsManagerClient = new SecretsManagerClient({ region: this.options.region }); - const secretManagerCommand = new GetSecretValueCommand({ - SecretId: secretArn, - }); - const secretsManagerResponse = await secretsManagerClient.send(secretManagerCommand); - const { password: managedPassword } = JSON.parse(secretsManagerResponse.SecretString); - if (!managedPassword) { - throw new Error('Unable to get RDS cluster master user password'); - } - this.options.password = managedPassword; - } - const { secretArn: secretArnWithCustomKey, keyArn: keyArn } = await storeDbConnectionConfigWithSecretsManager({ - region: this.options.region, - username: this.options.username, - password: this.options.password, - secretName: `${this.options.identifier}-secret-custom-key`, - useCustomEncryptionKey: true, - }); - - if (!secretArnWithCustomKey) { - throw new Error('Failed to store db connection config for secrets manager'); - } - const dbConnectionConfigSecretsManagerCustomKey = { - databaseName: this.options.dbname, - hostname: dbConfig.endpoint, - port: dbConfig.port, - secretArn: secretArnWithCustomKey, - keyArn, - }; - console.log(`Stored db connection config in Secrets manager: ${JSON.stringify(dbConnectionConfigSecretsManagerCustomKey)}`); - - const pathPrefix = `/${this.options.identifier}/test`; - const engine = this.options.engine; - const dbConnectionConfigSSM = await storeDbConnectionConfig({ - region: this.options.region, - pathPrefix, - hostname: dbConfig.endpoint, - port: dbConfig.port, - databaseName: this.options.dbname, - username: this.options.username, - password: this.options.password, - }); - const dbConnectionStringConfigSSM = await storeDbConnectionStringConfig({ - region: this.options.region, - pathPrefix, - connectionUri: this.getConnectionUri( - engine, - this.options.username, - this.options.password, - dbConfig.endpoint, - dbConfig.port, - this.options.dbname, - ), - }); - const dbConnectionStringConfigMultiple = await storeDbConnectionStringConfig({ - region: this.options.region, - pathPrefix, - connectionUri: [ - 'mysql://username:password@host:3306/dbname', - this.getConnectionUri( - engine, - this.options.username, - this.options.password, - dbConfig.endpoint, - dbConfig.port, - this.options.dbname, - ), - ], - }); - const parameters = { - ...dbConnectionConfigSSM, - ...dbConnectionStringConfigSSM, - ...dbConnectionStringConfigMultiple, - }; - if (!dbConnectionConfigSSM) { - throw new Error('Failed to store db connection config for SSM'); - } - console.log(`Stored db connection config in SSM: ${JSON.stringify(Object.keys(parameters))}`); - - this.databaseDetails = { - dbConfig: { - endpoint: dbConfig.endpoint, - port: dbConfig.port, - dbName: this.options.dbname, - strategyName: `${engine}DBStrategy`, - dbType: engine === 'postgres' ? 'POSTGRES' : 'MYSQL', - vpcConfig: extractVpcConfigFromDbInstance(dbConfig.dbInstance), - }, - connectionConfigs: { - ssm: dbConnectionConfigSSM, - secretsManager: dbConnectionConfigSecretsManager, - secretsManagerCustomKey: dbConnectionConfigSecretsManagerCustomKey, - secretsManagerManagedSecret: { - databaseName: this.options.dbname, - hostname: dbConfig.endpoint, - port: dbConfig.port, - secretArn: dbConfig.secretArn, - }, - connectionUri: dbConnectionStringConfigSSM, - connectionUriMultiple: dbConnectionStringConfigMultiple, - }, - }; - return this.databaseDetails; - } else { - // this.enableLocalTesting + if (this.useDataAPI || !this.options.password || this.enableLocalTesting) { const secretArn = dbConfig.secretArn; const secretsManagerClient = new SecretsManagerClient({ region: this.options.region }); const secretManagerCommand = new GetSecretValueCommand({ SecretId: secretArn, }); const secretsManagerResponse = await secretsManagerClient.send(secretManagerCommand); - const secretArnWithCustomKey = secretsManagerResponse.ARN; // arn:aws:secretsmanager:us-west-2:637423428135:secret:yjLvhaVbVT-secret-custom-key-BTqGiz - const keyArn = 'arn:aws:kms:us-west-2:637423428135:key/a84fe5fe-ee01-44bf-b94e-57604a109c77'; // ELVIN - GET THE MASTER CREDENTIALS KMS KEY SOMEHOW (like "arn:aws:kms:us-west-2:637423428135:key/62ec591f-7188-49db-8819-87c15727908f") - - const pathPrefix = `/${this.options.identifier}/test`; - const engine = this.options.engine; - const dbConnectionConfigSSM = await storeDbConnectionConfig({ - region: this.options.region, - pathPrefix, - hostname: dbConfig.endpoint, - port: dbConfig.port, - databaseName: this.options.dbname, - username: this.options.username, - password: this.options.password, - }); - const dbConnectionStringConfigSSM = await storeDbConnectionStringConfig({ - region: this.options.region, - pathPrefix, - connectionUri: this.getConnectionUri( - engine, - this.options.username, - this.options.password, - dbConfig.endpoint, - dbConfig.port, - this.options.dbname, - ), - }); - const dbConnectionStringConfigMultiple = await storeDbConnectionStringConfig({ - region: this.options.region, - pathPrefix, - connectionUri: [ - 'mysql://username:password@host:3306/dbname', - this.getConnectionUri( - engine, - this.options.username, - this.options.password, - dbConfig.endpoint, - dbConfig.port, - this.options.dbname, - ), - ], - }); - - if (!secretArnWithCustomKey) { - throw new Error('Failed to store db connection config for secrets manager'); + const { password: managedPassword } = JSON.parse(secretsManagerResponse.SecretString); + if (!managedPassword) { + throw new Error('Unable to get RDS cluster master user password'); } - const dbConnectionConfigSecretsManagerCustomKey = { - databaseName: this.options.dbname, - hostname: dbConfig.endpoint, - port: dbConfig.port, - secretArn: secretArnWithCustomKey, - keyArn, - }; - console.log(`Stored db connection config in Secrets manager: ${JSON.stringify(dbConnectionConfigSecretsManagerCustomKey)}`); + this.options.password = managedPassword; + } + const { secretArn: secretArnWithCustomKey, keyArn: keyArn } = await storeDbConnectionConfigWithSecretsManager({ + region: this.options.region, + username: this.options.username, + password: this.options.password, + secretName: `${this.options.identifier}-${this.options.dbname}-secret-custom-key`, + useCustomEncryptionKey: true, + }); - const identifier = this.options.identifier; + if (!secretArnWithCustomKey) { + throw new Error('Failed to store db connection config for secrets manager'); + } + const dbConnectionConfigSecretsManagerCustomKey = { + databaseName: this.options.dbname, + hostname: dbConfig.endpoint, + port: dbConfig.port, + secretArn: secretArnWithCustomKey, + keyArn, + }; + console.log(`Stored db connection config in Secrets manager: ${JSON.stringify(dbConnectionConfigSecretsManagerCustomKey)}`); - const parameters = { - ...dbConnectionConfigSSM, - ...dbConnectionStringConfigSSM, - ...dbConnectionStringConfigMultiple, - }; - if (!dbConnectionConfigSSM) { - throw new Error('Failed to store db connection config for SSM'); - } - console.log(`Stored db connection config in SSM: ${JSON.stringify(Object.keys(parameters))}`); + const pathPrefix = `/${this.options.identifier}/${this.options.dbname}/test`; + const engine = this.options.engine; + const dbConnectionConfigSSM = await storeDbConnectionConfig({ + region: this.options.region, + pathPrefix, + hostname: dbConfig.endpoint, + port: dbConfig.port, + databaseName: this.options.dbname, + username: this.options.username, + password: this.options.password, + }); + const dbConnectionStringConfigSSM = await storeDbConnectionStringConfig({ + region: this.options.region, + pathPrefix, + connectionUri: this.getConnectionUri( + engine, + this.options.username, + this.options.password, + dbConfig.endpoint, + dbConfig.port, + this.options.dbname, + ), + }); + const dbConnectionStringConfigMultiple = await storeDbConnectionStringConfig({ + region: this.options.region, + pathPrefix, + connectionUri: [ + 'mysql://username:password@host:3306/dbname', + this.getConnectionUri(engine, this.options.username, this.options.password, dbConfig.endpoint, dbConfig.port, this.options.dbname), + ], + }); + const parameters = { + ...dbConnectionConfigSSM, + ...dbConnectionStringConfigSSM, + ...dbConnectionStringConfigMultiple, + }; + if (!dbConnectionConfigSSM) { + throw new Error('Failed to store db connection config for SSM'); + } + console.log(`Stored db connection config in SSM: ${JSON.stringify(Object.keys(parameters))}`); - this.databaseDetails = { - dbConfig: { - endpoint: dbConfig.endpoint, + this.databaseDetails = { + dbConfig: { + endpoint: dbConfig.endpoint, + port: dbConfig.port, + dbName: this.options.dbname, + strategyName: `${engine}DBStrategy`, + dbType: engine === 'postgres' ? 'POSTGRES' : 'MYSQL', + vpcConfig: extractVpcConfigFromDbInstance(dbConfig.dbInstance), + }, + connectionConfigs: { + ssm: dbConnectionConfigSSM, + secretsManager: dbConnectionConfigSecretsManager, + secretsManagerCustomKey: dbConnectionConfigSecretsManagerCustomKey, + secretsManagerManagedSecret: { + databaseName: this.options.dbname, + hostname: dbConfig.endpoint, port: dbConfig.port, - dbName: this.options.dbname, - strategyName: `${engine}DBStrategy`, - dbType: engine === 'postgres' ? 'POSTGRES' : 'MYSQL', - vpcConfig: extractVpcConfigFromDbInstance(dbConfig.dbInstance), - }, - connectionConfigs: { - ssm: dbConnectionConfigSSM, - secretsManager: dbConnectionConfigSecretsManager, - secretsManagerCustomKey: dbConnectionConfigSecretsManagerCustomKey, - secretsManagerManagedSecret: { - databaseName: this.options.dbname, - hostname: dbConfig.endpoint, - port: dbConfig.port, - secretArn: dbConfig.secretArn, - }, - connectionUri: dbConnectionStringConfigSSM, - connectionUriMultiple: dbConnectionStringConfigMultiple, + secretArn: dbConfig.secretArn, }, - }; - return this.databaseDetails; - } + connectionUri: dbConnectionStringConfigSSM, + connectionUriMultiple: dbConnectionStringConfigMultiple, + }, + }; + return this.databaseDetails; }; cleanupDatabase = async (): Promise => { - if (!this.databaseDetails || this.enableLocalTesting) { - // Database has not been set up or using a local test cluster. - if (this.enableLocalTesting) { - dropDatabase(this.options); - } + if (!this.databaseDetails) { return; } - if (this.useDataAPI) { - await deleteDBCluster(this.options.identifier, this.options.region); - } else { - await deleteDBInstance(this.options.identifier, this.options.region); + if (!this.enableLocalTesting) { + if (this.useDataAPI) { + await deleteDBCluster(this.options.identifier, this.options.region); + } else { + await deleteDBInstance(this.options.identifier, this.options.region); + } } const { connectionConfigs } = this.databaseDetails; diff --git a/packages/amplify-graphql-api-construct-tests/src/utils/sql-local-testing.ts b/packages/amplify-graphql-api-construct-tests/src/utils/sql-local-testing.ts new file mode 100644 index 0000000000..b8285a235c --- /dev/null +++ b/packages/amplify-graphql-api-construct-tests/src/utils/sql-local-testing.ts @@ -0,0 +1,29 @@ +import path from 'path'; +import * as fs from 'fs-extra'; +import { SqlEngine } from 'amplify-category-api-e2e-core'; + +export const getClusterIdentifier = (region: string, engine: SqlEngine): string | undefined => { + const repoRoot = path.join(__dirname, '..', '..', '..', '..'); + const localClusterPath = path.join(repoRoot, 'scripts', 'e2e-test-local-cluster-config.json'); + if (!fs.existsSync(localClusterPath)) { + return; + } + try { + const localClustersObject = JSON.parse(fs.readFileSync(localClusterPath, 'utf-8')); + const regionObject = localClustersObject[region]; + if (!regionObject || regionObject?.length === 0) { + return; + } + + const clusterConfig = regionObject[0]?.dbConfig; + if (!clusterConfig || !clusterConfig.engine) { + return; + } + // Get the config identifier and connection URI + const identifier = clusterConfig.identifier; + return identifier; + } catch (err) { + // cannot get local cluster information + return; + } +}; diff --git a/scripts/e2e-test-local-cluster-config.json b/scripts/e2e-test-local-cluster-config.json index 81bc605d95..54dc73120e 100644 --- a/scripts/e2e-test-local-cluster-config.json +++ b/scripts/e2e-test-local-cluster-config.json @@ -2,14 +2,16 @@ "us-east-1": [ { "dbConfig": { - "identifier": "" + "identifier": "", + "engine": "postgres" } } ], "us-west-2": [ { "dbConfig": { - "identifier": "YMmwoSDbAQ" + "identifier": "usEhhWfnYh", + "engine": "postgres" } } ] diff --git a/scripts/e2e-test-local-cluster-config.sample.json b/scripts/e2e-test-local-cluster-config.sample.json index a80df17ff7..6ec17aa657 100644 --- a/scripts/e2e-test-local-cluster-config.sample.json +++ b/scripts/e2e-test-local-cluster-config.sample.json @@ -2,14 +2,16 @@ "us-east-1": [ { "dbConfig": { - "identifier": "IDENTIFIER_VALUE" + "identifier": "IDENTIFIER_VALUE", + "engine": "ENGINE_TYPE" } } ], "us-west-2": [ { "dbConfig": { - "identifier": "IDENTIFIER_VALUE" + "identifier": "IDENTIFIER_VALUE", + "engine": "ENGINE_TYPE" } } ] diff --git a/yarn.lock b/yarn.lock index 556232fdcb..d5456e5e70 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8970,11 +8970,16 @@ async@^2.6.4: dependencies: lodash "^4.17.14" -async@^3.2.0, async@^3.2.3, async@^3.2.4: +async@^3.2.0, async@^3.2.4: version "3.2.5" resolved "https://registry.npmjs.org/async/-/async-3.2.5.tgz#ebd52a8fdaf7a2289a24df399f8d8485c8a46b66" integrity sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg== +async@^3.2.3: + version "3.2.6" + resolved "https://registry.npmjs.org/async/-/async-3.2.6.tgz#1b0728e14929d51b85b449b7f06e27c1145e38ce" + integrity sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA== + asynciterator.prototype@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/asynciterator.prototype/-/asynciterator.prototype-1.0.0.tgz#8c5df0514936cdd133604dfcc9d3fb93f09b2b62" From 591526b5209fce9248a90d7ed2f8efae66e9016d Mon Sep 17 00:00:00 2001 From: fuelvin1 Date: Tue, 20 Aug 2024 13:34:25 -0700 Subject: [PATCH 8/9] chore: incorporated Phani's comments --- packages/amplify-e2e-core/src/utils/rds.ts | 13 ++++++++----- scripts/e2e-test-local-cluster-config.json | 18 ------------------ 2 files changed, 8 insertions(+), 23 deletions(-) delete mode 100644 scripts/e2e-test-local-cluster-config.json diff --git a/packages/amplify-e2e-core/src/utils/rds.ts b/packages/amplify-e2e-core/src/utils/rds.ts index 1a3580086c..71684a05c9 100644 --- a/packages/amplify-e2e-core/src/utils/rds.ts +++ b/packages/amplify-e2e-core/src/utils/rds.ts @@ -245,9 +245,11 @@ export const createRDSCluster = async (config: RDSConfig): Promise }; /** - * Accesses the local RDS Aurora serverless V2 cluster with one DB instance using the given input configuration. + * Setup the test database and data in the pre-existing RDS Aurora serverless V2 cluster with one writer DB instance. Get the necessary configuration settings of the cluster and instance. + * @param identifier Cluster idenfitier. * @param config Configuration of the database cluster. - * @returns EndPoint address, port and database name of the accessed RDS cluster. + * @param queries Initial queries to be executed. + * @returns Cluster configuration information. */ export const setupDataInExistingCluster = async (identifier: string, config: RDSConfig, queries: string[]): Promise => { try { @@ -387,8 +389,8 @@ export const setupRDSInstanceAndData = async ( */ export const setupRDSClusterAndData = async (config: RDSConfig, queries?: string[]): Promise => { - const dbCluster = await createRDSCluster(config); console.log(`Creating RDS ${config.engine} DB cluster with identifier ${config.identifier}`); + const dbCluster = await createRDSCluster(config); if (!dbCluster.secretArn) { throw new Error('Failed to store db connection config in secrets manager'); @@ -399,6 +401,7 @@ export const setupRDSClusterAndData = async (config: RDSConfig, queries?: string const createDBInput: ExecuteStatementCommandInput = { resourceArn: dbCluster.clusterArn, secretArn: dbCluster.secretArn, + // database name is sanitized from when we generate it sql: `create database ${config.dbname}`, database: dbCluster.dbName, }; @@ -469,7 +472,7 @@ export const deleteDBInstance = async (identifier: string, region: string): Prom // ); } catch (error) { console.log(error); - throw new Error(`Error in deleting RDS instance: ${error.json}`); + throw new Error(`Error in deleting RDS instance: ${JSON.stringify(error)}`); } }; @@ -496,7 +499,7 @@ export const deleteDBCluster = async (identifier: string, region: string): Promi await client.send(command); } catch (error) { console.log(error); - throw new Error(`Error in deleting RDS cluster ${identifier}: ${error.json}`); + throw new Error(`Error in deleting RDS cluster ${identifier}: ${JSON.stringify(error)}`); } }; diff --git a/scripts/e2e-test-local-cluster-config.json b/scripts/e2e-test-local-cluster-config.json deleted file mode 100644 index 54dc73120e..0000000000 --- a/scripts/e2e-test-local-cluster-config.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "us-east-1": [ - { - "dbConfig": { - "identifier": "", - "engine": "postgres" - } - } - ], - "us-west-2": [ - { - "dbConfig": { - "identifier": "usEhhWfnYh", - "engine": "postgres" - } - } - ] -} From ac30eb579613f1df6472ec89cc2fa9367c34793d Mon Sep 17 00:00:00 2001 From: fuelvin1 Date: Tue, 20 Aug 2024 15:27:41 -0700 Subject: [PATCH 9/9] chore: addressed Tim's comments --- packages/amplify-e2e-core/src/utils/rds.ts | 17 ++++++++++------- .../src/sql-datatabase-controller.ts | 1 + 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/amplify-e2e-core/src/utils/rds.ts b/packages/amplify-e2e-core/src/utils/rds.ts index 71684a05c9..ab7a5a6005 100644 --- a/packages/amplify-e2e-core/src/utils/rds.ts +++ b/packages/amplify-e2e-core/src/utils/rds.ts @@ -276,10 +276,12 @@ export const setupDataInExistingCluster = async (identifier: string, config: RDS const secretArn = dbClusterObj.MasterUserSecret.SecretArn; const defaultDbName = dbClusterObj.DatabaseName; const dataClient = new RDSDataClient({ region: config.region }); + const sanitizedDbName = config.dbname?.replace(/[^a-zA-Z0-9_]/g, ''); + const createDBInput: ExecuteStatementCommandInput = { resourceArn: clusterArn, secretArn, - sql: `create database ${config.dbname}`, + sql: `create database ${sanitizedDbName}`, database: defaultDbName, }; @@ -298,7 +300,7 @@ export const setupDataInExistingCluster = async (identifier: string, config: RDS resourceArn: clusterArn, secretArn: secretArn, sql: query, - database: config.dbname, + database: sanitizedDbName, }; const executeStatementResponse = await dataClient.send(new ExecuteStatementCommand(executeStatementInput)); console.log('Run query response: ' + JSON.stringify(executeStatementResponse)); @@ -311,7 +313,7 @@ export const setupDataInExistingCluster = async (identifier: string, config: RDS clusterArn, endpoint: dbClusterObj.Endpoint, port: dbClusterObj.Port, - dbName: config.dbname, + dbName: sanitizedDbName, dbInstance: describeInstanceResponse.DBInstances[0], secretArn, username: dbClusterObj.MasterUsername, @@ -398,11 +400,12 @@ export const setupRDSClusterAndData = async (config: RDSConfig, queries?: string const client = new RDSDataClient({ region: config.region }); + const sanitizedDbName = config.dbname?.replace(/[^a-zA-Z0-9_]/g, ''); + const createDBInput: ExecuteStatementCommandInput = { resourceArn: dbCluster.clusterArn, secretArn: dbCluster.secretArn, - // database name is sanitized from when we generate it - sql: `create database ${config.dbname}`, + sql: `create database ${sanitizedDbName}`, database: dbCluster.dbName, }; @@ -421,7 +424,7 @@ export const setupRDSClusterAndData = async (config: RDSConfig, queries?: string resourceArn: dbCluster.clusterArn, secretArn: dbCluster.secretArn, sql: query, - database: config.dbname, + database: sanitizedDbName, }; const executeStatementResponse = await client.send(new ExecuteStatementCommand(executeStatementInput)); console.log('Run query response: ' + JSON.stringify(executeStatementResponse)); @@ -434,7 +437,7 @@ export const setupRDSClusterAndData = async (config: RDSConfig, queries?: string clusterArn: dbCluster.clusterArn, endpoint: dbCluster.endpoint, port: dbCluster.port, - dbName: config.dbname, + dbName: sanitizedDbName, dbInstance: dbCluster.dbInstance, secretArn: dbCluster.secretArn, }; diff --git a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts index 8c589f27c1..accfc2a938 100644 --- a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts +++ b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts @@ -79,6 +79,7 @@ export class SqlDatatabaseController { const identifier = getClusterIdentifier(this.options.region, this.options.engine); dbConfig = await setupDataInExistingCluster(identifier, this.options, this.setupQueries); this.options.username = dbConfig.username; + this.options.dbname = dbConfig.dbName; } else { dbConfig = await setupRDSClusterAndData(this.options, this.setupQueries); }