Merge pull request aws-amplify#2778 from aws-amplify/sql-local-testing

chore: add sql local testing setup

.gitignore

@@ -44,3 +44,4 @@ codebuild_specs/debug_workflow.yml
 .npmrc
 verdaccio-logs.txt
 scripts/components/private_packages.ts
+scripts/e2e-test-local-cluster-config.json

packages/amplify-e2e-core/src/utils/rds.ts

@@ -4,12 +4,14 @@ import {
   CreateDBInstanceCommand,
   CreateDBInstanceCommandInput,
   DBInstance,
-  DeleteDBInstanceCommand,
-  waitUntilDBInstanceAvailable,
   CreateDBClusterMessage,
-  waitUntilDBClusterAvailable,
   DeleteDBClusterCommand,
   DeleteDBClusterCommandInput,
+  DeleteDBInstanceCommand,
+  DescribeDBClustersCommand,
+  DescribeDBInstancesCommand,
+  waitUntilDBClusterAvailable,
+  waitUntilDBInstanceAvailable,
 } from '@aws-sdk/client-rds';
 import { RDSDataClient, ExecuteStatementCommand, ExecuteStatementCommandInput, Field } from '@aws-sdk/client-rds-data';
 import generator from 'generate-password';
@@ -39,7 +41,7 @@ export type SqlEngine = 'mysql' | 'postgres';
 export type RDSConfig = {
   identifier: string;
   engine: SqlEngine;
-  dbname: string;
+  dbname?: string;
   username: string;
   password?: string;
   region: string;
@@ -55,6 +57,7 @@ export type ClusterInfo = {
   dbName: string;
   secretArn: string;
   dbInstance: DBInstance;
+  username?: string;
 };
 
 const getRDSEngineType = (engine: SqlEngine): string => {
@@ -241,6 +244,85 @@ export const createRDSCluster = async (config: RDSConfig): Promise<ClusterInfo>
   }
 };
 
+/**
+ * Setup the test database and data in the pre-existing RDS Aurora serverless V2 cluster with one writer DB instance. Get the necessary configuration settings of the cluster and instance.
+ * @param identifier Cluster idenfitier.
+ * @param config Configuration of the database cluster.
+ * @param queries Initial queries to be executed.
+ * @returns Cluster configuration information.
+ */
+export const setupDataInExistingCluster = async (identifier: string, config: RDSConfig, queries: string[]): Promise<ClusterInfo> => {
+  try {
+    const client = new RDSClient({ region: config.region });
+    const describeClusterResponse = await client.send(
+      new DescribeDBClustersCommand({ Filters: [{ Name: 'db-cluster-id', Values: [identifier] }] }),
+    );
+    if (!describeClusterResponse || !describeClusterResponse?.DBClusters[0]) {
+      throw Error('Specified cluster info cannot be fetched');
+    }
+    const dbClusterObj = describeClusterResponse.DBClusters[0];
+    const instances = dbClusterObj?.DBClusterMembers;
+    if (!instances || instances?.length === 0) {
+      throw new Error('No instances are present in the specified cluster');
+    }
+    const instanceId = instances[0]?.DBInstanceIdentifier;
+    const describeInstanceCommand = new DescribeDBInstancesCommand({ DBInstanceIdentifier: instanceId });
+    const describeInstanceResponse = await client.send(describeInstanceCommand);
+    if (!describeInstanceResponse || describeInstanceResponse?.DBInstances?.length === 0) {
+      throw Error('Specified cluster instance info cannot be fetched');
+    }
+
+    const clusterArn = dbClusterObj.DBClusterArn;
+    const secretArn = dbClusterObj.MasterUserSecret.SecretArn;
+    const defaultDbName = dbClusterObj.DatabaseName;
+    const dataClient = new RDSDataClient({ region: config.region });
+    const sanitizedDbName = config.dbname?.replace(/[^a-zA-Z0-9_]/g, '');
+
+    const createDBInput: ExecuteStatementCommandInput = {
+      resourceArn: clusterArn,
+      secretArn,
+      sql: `create database ${sanitizedDbName}`,
+      database: defaultDbName,
+    };
+
+    const createDBCommand = new ExecuteStatementCommand(createDBInput);
+    try {
+      const createDBResponse = await dataClient.send(createDBCommand);
+      console.log('Create database response: ' + JSON.stringify(createDBResponse));
+    } catch (err) {
+      console.log(err);
+    }
+
+    // create the test tables in the test database
+    for (const query of queries ?? []) {
+      try {
+        const executeStatementInput: ExecuteStatementCommandInput = {
+          resourceArn: clusterArn,
+          secretArn: secretArn,
+          sql: query,
+          database: sanitizedDbName,
+        };
+        const executeStatementResponse = await dataClient.send(new ExecuteStatementCommand(executeStatementInput));
+        console.log('Run query response: ' + JSON.stringify(executeStatementResponse));
+      } catch (err) {
+        throw new Error(`Error in creating tables in test database: ${JSON.stringify(err, null, 4)}`);
+      }
+    }
+
+    return {
+      clusterArn,
+      endpoint: dbClusterObj.Endpoint,
+      port: dbClusterObj.Port,
+      dbName: sanitizedDbName,
+      dbInstance: describeInstanceResponse.DBInstances[0],
+      secretArn,
+      username: dbClusterObj.MasterUsername,
+    };
+  } catch (error) {
+    console.log('Error while setting up the test data in existing cluster: ', JSON.stringify(error));
+  }
+};
+
 /**
  * Creates a new RDS instance using the given input configuration, runs the given queries and returns the details of the created RDS
  * instance.
@@ -310,7 +392,6 @@ export const setupRDSInstanceAndData = async (
 
 export const setupRDSClusterAndData = async (config: RDSConfig, queries?: string[]): Promise<ClusterInfo> => {
   console.log(`Creating RDS ${config.engine} DB cluster with identifier ${config.identifier}`);
-
   const dbCluster = await createRDSCluster(config);
 
   if (!dbCluster.secretArn) {
@@ -319,8 +400,7 @@ export const setupRDSClusterAndData = async (config: RDSConfig, queries?: string
 
   const client = new RDSDataClient({ region: config.region });
 
-  // create a new test database with given name
-  const sanitizedDbName = config.dbname.replace(/[^a-zA-Z0-9_]/g, '');
+  const sanitizedDbName = config.dbname?.replace(/[^a-zA-Z0-9_]/g, '');
 
   const createDBInput: ExecuteStatementCommandInput = {
     resourceArn: dbCluster.clusterArn,
@@ -347,7 +427,7 @@ export const setupRDSClusterAndData = async (config: RDSConfig, queries?: string
         database: sanitizedDbName,
       };
       const executeStatementResponse = await client.send(new ExecuteStatementCommand(executeStatementInput));
-      console.log('Create table response: ' + JSON.stringify(executeStatementResponse));
+      console.log('Run query response: ' + JSON.stringify(executeStatementResponse));
     } catch (err) {
       throw new Error(`Error in creating tables in test database: ${JSON.stringify(err, null, 4)}`);
     }
@@ -395,7 +475,7 @@ export const deleteDBInstance = async (identifier: string, region: string): Prom
     // );
   } catch (error) {
     console.log(error);
-    throw new Error(`Error in deleting RDS instance: ${error.response.json}`);
+    throw new Error(`Error in deleting RDS instance: ${JSON.stringify(error)}`);
   }
 };
 
@@ -422,7 +502,7 @@ export const deleteDBCluster = async (identifier: string, region: string): Promi
     await client.send(command);
   } catch (error) {
     console.log(error);
-    throw new Error(`Error in deleting RDS cluster ${identifier}: ${error.response.json}`);
+    throw new Error(`Error in deleting RDS cluster ${identifier}: ${JSON.stringify(error)}`);
   }
 };
 

packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-ddbprimary-sqlrelated.test.ts

@@ -2,7 +2,7 @@
 import * as path from 'path';
 import * as fs from 'fs-extra';
 import * as generator from 'generate-password';
-import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core';
+import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core';
 import { DDB_AMPLIFY_MANAGED_DATASOURCE_STRATEGY } from '@aws-amplify/graphql-transformer-core';
 import { cdkDeploy, cdkDestroy, initCDKProject } from '../../../commands';
 import { SqlDatabaseDetails, SqlDatatabaseController } from '../../../sql-datatabase-controller';
@@ -23,7 +23,6 @@ describe('PostgreSQL tables with UUID primary keys', () => {
   const region = process.env.CLI_REGION ?? 'us-west-2';
   const baseProjFolderName = path.basename(__filename, '.test.ts');
 
-  const dbname = generateDBName();
   const [dbUsername, dbIdentifier] = generator.generateMultiple(2);
   let dbDetails: SqlDatabaseDetails;
 
@@ -46,7 +45,6 @@ describe('PostgreSQL tables with UUID primary keys', () => {
     {
       identifier: dbIdentifier,
       engine: 'postgres',
-      dbname,
       username: dbUsername,
       region,
     },

packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-ddbrelated.test.ts

@@ -2,7 +2,7 @@
 import * as path from 'path';
 import * as fs from 'fs-extra';
 import * as generator from 'generate-password';
-import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core';
+import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core';
 import { DDB_AMPLIFY_MANAGED_DATASOURCE_STRATEGY } from '@aws-amplify/graphql-transformer-core';
 import { cdkDeploy, cdkDestroy, initCDKProject } from '../../../commands';
 import { SqlDatabaseDetails, SqlDatatabaseController } from '../../../sql-datatabase-controller';
@@ -23,7 +23,6 @@ describe('PostgreSQL tables with UUID primary keys', () => {
   const region = process.env.CLI_REGION ?? 'us-west-2';
   const baseProjFolderName = path.basename(__filename, '.test.ts');
 
-  const dbname = generateDBName();
   const [dbUsername, dbIdentifier] = generator.generateMultiple(2);
   let dbDetails: SqlDatabaseDetails;
 
@@ -46,7 +45,6 @@ describe('PostgreSQL tables with UUID primary keys', () => {
     {
       identifier: dbIdentifier,
       engine: 'postgres',
-      dbname,
       username: dbUsername,
       region,
     },

packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-sqlrelated.test.ts

@@ -2,7 +2,7 @@
 import * as path from 'path';
 import * as fs from 'fs-extra';
 import * as generator from 'generate-password';
-import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core';
+import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core';
 import { cdkDeploy, cdkDestroy, initCDKProject } from '../../../commands';
 import { SqlDatabaseDetails, SqlDatatabaseController } from '../../../sql-datatabase-controller';
 import { TestDefinition, dbDetailsToModelDataSourceStrategy, writeStackConfig, writeTestDefinitions } from '../../../utils';
@@ -22,7 +22,6 @@ describe('PostgreSQL tables with UUID primary keys', () => {
   const region = process.env.CLI_REGION ?? 'us-west-2';
   const baseProjFolderName = path.basename(__filename, '.test.ts');
 
-  const dbname = generateDBName();
   const [dbUsername, dbIdentifier] = generator.generateMultiple(2);
   let dbDetails: SqlDatabaseDetails;
 
@@ -45,7 +44,6 @@ describe('PostgreSQL tables with UUID primary keys', () => {
     {
       identifier: dbIdentifier,
       engine: 'postgres',
-      dbname,
       username: dbUsername,
       region,
     },

packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts

@@ -1,4 +1,4 @@
-import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core';
+import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core';
 import generator from 'generate-password';
 import { getResourceNamesForStrategyName } from '@aws-amplify/graphql-transformer-core';
 import { SqlDatatabaseController } from '../sql-datatabase-controller';
@@ -14,15 +14,13 @@ describe('Canary using Postgres lambda model datasource strategy', () => {
   // sufficient password length that meets the requirements for RDS cluster/instance
   const [username, password, identifier] = generator.generateMultiple(3, { length: 11 });
   const region = process.env.CLI_REGION ?? 'us-west-2';
-  const dbname = generateDBName();
   const engine = 'postgres';
 
   const databaseController: SqlDatatabaseController = new SqlDatatabaseController(
     ['CREATE TABLE "todos" ("id" VARCHAR(40) PRIMARY KEY, "description" VARCHAR(256))'],
     {
       identifier,
       engine,
-      dbname,
       username,
       password,
       region,
@@ -33,7 +31,9 @@ describe('Canary using Postgres lambda model datasource strategy', () => {
   const resourceNames = getResourceNamesForStrategyName(strategyName);
 
   beforeAll(async () => {
+    console.time('sql-pg-canary test setup');
     await databaseController.setupDatabase();
+    console.timeEnd('sql-pg-canary test setup');
   });
 
   afterAll(async () => {

packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-models.test.ts

@@ -1,4 +1,4 @@
-import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core';
+import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core';
 import generator from 'generate-password';
 import { getResourceNamesForStrategyName } from '@aws-amplify/graphql-transformer-core';
 import { SqlDatatabaseController } from '../sql-datatabase-controller';
@@ -15,15 +15,13 @@ describe('CDK GraphQL Transformer deployments with Postgres SQL datasources', ()
   // sufficient password length that meets the requirements for RDS cluster/instance
   const [username, password, identifier] = generator.generateMultiple(3, { length: 11 });
   const region = process.env.CLI_REGION ?? 'us-west-2';
-  const dbname = generateDBName();
   const engine = 'postgres';
 
   const databaseController: SqlDatatabaseController = new SqlDatatabaseController(
     ['CREATE TABLE "todos" ("id" VARCHAR(40) PRIMARY KEY, "description" VARCHAR(256))'],
     {
       identifier,
       engine,
-      dbname,
       username,
       password,
       region,