feat(integ-runner): Support non-TypeScript tests

packages/@aws-cdk/aws-events-targets/test/aws-api/integ.aws-api.ts

@@ -1,5 +1,6 @@
 import * as events from '@aws-cdk/aws-events';
 import * as cdk from '@aws-cdk/core';
+import { ECS, RDS } from 'aws-sdk';
 import * as targets from '../../lib';
 
 const app = new cdk.App();
@@ -22,15 +23,15 @@ class AwsApi extends cdk.Stack {
       parameters: {
         service: 'cool-service',
         forceNewDeployment: true,
-      } as AWS.ECS.UpdateServiceRequest,
+      } as ECS.UpdateServiceRequest,
     }));
 
     scheduleRule.addTarget(new targets.AwsApi({
       service: 'RDS',
       action: 'stopDBInstance',
       parameters: {
         DBInstanceIdentifier: 'dev-instance',
-      } as AWS.RDS.StopDBInstanceMessage,
+      } as RDS.StopDBInstanceMessage,
     }));
 
     // Create snapshots when a DB instance restarts
@@ -48,7 +49,7 @@ class AwsApi extends cdk.Stack {
       action: 'createDBSnapshot',
       parameters: {
         DBInstanceIdentifier: events.EventField.fromPath('$.detail.SourceArn'),
-      } as AWS.RDS.CreateDBSnapshotMessage,
+      } as RDS.CreateDBSnapshotMessage,
     }));
   }
 }

packages/@aws-cdk/integ-runner/README.md

@@ -68,6 +68,12 @@ to be a self contained CDK app. The runner will execute the following for each f
   Read the list of tests from this file
 - `--disable-update-workflow` (default=`false`)
   If this is set to `true` then the [update workflow](#update-workflow) will be disabled
+- `--app`
+  The command used by the test runner to synth the test files. Uses default run commands based on the language the test is written in. You can use {filePath} in the command to specify where the test file name should be inserted in the command. Example: `--app "python3.8 {filePath}"`
+- `--language` (default=`['csharp', 'fsharp', 'go', 'java', 'javascript', 'python', 'typescript']`)
+  The list of languages to discover tests for. Defaults to all CDK-supported languages. Example: `--language python --language typescript`
+- `--test-regex`
+  A custom pattern in the JS RegExp format to match integration test file prefixes. Defaults are set based on naming conventions for the language the test is written in. Example: `--test-regex "^Integ\."`
 
 Example:
 

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -30,9 +30,14 @@ async function main() {
     .options('from-file', { type: 'string', desc: 'Read TEST names from a file (one TEST per line)' })
     .option('inspect-failures', { type: 'boolean', desc: 'Keep the integ test cloud assembly if a failure occurs for inspection', default: false })
     .option('disable-update-workflow', { type: 'boolean', default: false, desc: 'If this is "true" then the stack update workflow will be disabled' })
+    .option('app', { type: 'string', default: undefined, desc: 'The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file name should be inserted. Example: --app="python3.8 {filePath}"' })
+    .option('language', { type: 'array', default: ['csharp', 'fsharp', 'go', 'java', 'javascript', 'python', 'typescript'], desc: 'The languages that tests will search for. Defaults to all languages supported by CDK.' })
+    .option('test-regex', { type: 'string', default: undefined, desc: 'A custom pattern in the JS RegExp format to match integration test file prefixes.' })
     .strict()
     .argv;
 
+  const customRegex = argv['test-regex'] ? new RegExp(argv['test-regex']) : undefined;
+
   const pool = workerpool.pool(path.join(__dirname, '../lib/workers/extract/index.js'), {
     maxWorkers: argv['max-workers'],
   });
@@ -56,7 +61,7 @@ async function main() {
   let testsSucceeded = false;
   try {
     if (argv.list) {
-      const tests = await new IntegrationTests(argv.directory).fromCliArgs();
+      const tests = await new IntegrationTests(argv.directory, customRegex, argv.language).fromCliArgs();
       process.stdout.write(tests.map(t => t.discoveryRelativeFileName).join('\n') + '\n');
       return;
     }
@@ -68,14 +73,16 @@ async function main() {
       ? (await fs.readFile(fromFile, { encoding: 'utf8' })).split('\n').filter(x => x)
       : (argv._.length > 0 ? argv._ : undefined); // 'undefined' means no request
 
-    testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory)).fromCliArgs(requestedTests, exclude)));
+    testsFromArgs.push(...(await new IntegrationTests(path.resolve(argv.directory), customRegex, argv.language)
+      .fromCliArgs(requestedTests, exclude)));
 
     // always run snapshot tests, but if '--force' is passed then
     // run integration tests on all failed tests, not just those that
     // failed snapshot tests
     failedSnapshots = await runSnapshotTests(pool, testsFromArgs, {
       retain: argv['inspect-failures'],
       verbose: Boolean(argv.verbose),
+      appCommand: argv.app,
     });
     for (const failure of failedSnapshots) {
       destructiveChanges.push(...failure.destructiveChanges ?? []);
@@ -99,6 +106,7 @@ async function main() {
         dryRun: argv['dry-run'],
         verbosity: argv.verbose,
         updateWorkflow: !argv['disable-update-workflow'],
+        appCommand: argv.app,
       });
       testsSucceeded = success;
 

packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts

@@ -79,6 +79,11 @@ export class IntegTest {
    */
   public readonly temporaryOutputDir: string;
 
+  /**
+   * Language the test is written in
+   */
+  public readonly language: string;
+
   constructor(public readonly info: IntegTestInfo) {
     this.absoluteFileName = path.resolve(info.fileName);
     this.fileName = path.relative(process.cwd(), info.fileName);
@@ -97,10 +102,16 @@ export class IntegTest {
       ? parsed.name
       : path.join(path.relative(this.info.discoveryRoot, parsed.dir), parsed.name);
 
-    const nakedTestName = parsed.name.slice(6); // Leave name without 'integ.' and '.ts'
+    const nakedTestName = new IntegrationTests(this.directory).stripPrefixAndSuffix(parsed.base); // Leave name without 'integ.' and '.ts'
     this.normalizedTestName = parsed.name;
     this.snapshotDir = path.join(this.directory, `${nakedTestName}.integ.snapshot`);
     this.temporaryOutputDir = path.join(this.directory, `${CDK_OUTDIR_PREFIX}.${nakedTestName}`);
+
+    const language = new IntegrationTests(this.directory).getLanguage(parsed.base);
+    if (!language) {
+      throw new Error('Given file does not match any of the allowed languages for this test');
+    }
+    this.language = language;
   }
 
   /**
@@ -146,7 +157,75 @@ export interface IntegrationTestFileConfig {
  * Discover integration tests
  */
 export class IntegrationTests {
-  constructor(private readonly directory: string) {
+  /**
+   * Will discover integration tests with naming conventions typical to each language. Examples:
+   * - TypeScript/JavaScript: integ.test.ts or integ-test.ts or integ_test.ts
+   * - Python/Go: integ_test.py
+   * - Java/C#: IntegTest.cs
+   * @private
+   */
+  private readonly prefixMapping: Map<string, RegExp> = new Map<string, RegExp>([
+    ['csharp', new RegExp(/^Integ/)],
+    ['fsharp', new RegExp(/^Integ/)],
+    ['go', new RegExp(/^integ_/)],
+    ['java', new RegExp(/^Integ/)],
+    ['javascript', new RegExp(/^integ\./)],
+    ['python', new RegExp(/^integ_/)],
+    ['typescript', new RegExp(/^integ\./)],
+  ]);
+  private readonly suffixMapping: Map<string, RegExp> = new Map<string, RegExp>([
+    ['csharp', new RegExp(/\.cs$/)],
+    ['fsharp', new RegExp(/\.fs$/)],
+    ['go', new RegExp(/\.go$/)],
+    ['java', new RegExp(/\.java$/)],
+    ['javascript', new RegExp(/\.js$/)],
+    ['python', new RegExp(/\.py$/)],
+    // Allow files ending in .ts but not in .d.ts
+    ['typescript', new RegExp(/(?<!\.d)\.ts$/)],
+  ]);
+
+  constructor(private readonly directory: string, customPrefix?: RegExp, allowedLanguages?: Array<string>) {
+    if (customPrefix) {
+      for (const language of this.prefixMapping.keys()) {
+        this.prefixMapping.set(language, customPrefix);
+      }
+    }
+    if (allowedLanguages) {
+      const disallowedLanguages = Array.from(this.prefixMapping.keys()).filter((language) => !allowedLanguages.includes(language));
+      for (const disallowedLanguage of disallowedLanguages) {
+        this.prefixMapping.delete(disallowedLanguage);
+        this.suffixMapping.delete(disallowedLanguage);
+      }
+    }
+  }
+
+  public stripPrefixAndSuffix(fileName: string): string {
+    const language = this.getLanguage(fileName);
+    if (!language) {
+      return fileName;
+    }
+
+    const suffix = this.suffixMapping.get(language) ?? '';
+    const prefix = this.prefixMapping.get(language) ?? '';
+
+    return fileName.replace(prefix, '').replace(suffix, '');
+  }
+
+  private hasValidPrefixSuffix(fileName: string): boolean {
+    const language = this.getLanguage(fileName);
+    if (!language) {
+      return false;
+    }
+
+    const suffix = this.suffixMapping.get(language);
+    const prefix = this.prefixMapping.get(language);
+
+    return <boolean>(suffix?.test(fileName) && prefix?.test(fileName));
+  }
+
+  public getLanguage(fileName: string): string | undefined {
+    const [language] = Array.from(this.suffixMapping.entries()).find(([, regex]) => regex.test(fileName)) ?? [undefined, undefined];
+    return language;
   }
 
   /**
@@ -212,8 +291,24 @@ export class IntegrationTests {
 
   private async discover(): Promise<IntegTest[]> {
     const files = await this.readTree();
-    const integs = files.filter(fileName => path.basename(fileName).startsWith('integ.') && path.basename(fileName).endsWith('.js'));
-    return this.request(integs);
+    const integs = files.filter(fileName => this.hasValidPrefixSuffix(path.basename(fileName)));
+
+    const discoveredTestNames = new Set<string>();
+    const integsWithoutDuplicates = new Array<string>();
+
+    // Remove duplicate test names that would just overwrite each other's snapshots anyway.
+    // To make sure the precendence of files is deterministic, iterate the files in lexicographic order.
+    // Additionally, to give precedence to .ts files over their compiled .js version,
+    // use descending lexicographic ordering, so the .ts files are picked up first.
+    for (const integFileName of integs.sort().reverse()) {
+      const testName = this.stripPrefixAndSuffix(path.basename(integFileName));
+      if (!discoveredTestNames.has(testName)) {
+        integsWithoutDuplicates.push(integFileName);
+      }
+      discoveredTestNames.add(testName);
+    }
+
+    return this.request(integsWithoutDuplicates);
   }
 
   private request(files: string[]): IntegTest[] {

packages/@aws-cdk/integ-runner/lib/runner/runner-base.ts

@@ -1,6 +1,7 @@
 import * as path from 'path';
 import { TestCase, DefaultCdkOptions } from '@aws-cdk/cloud-assembly-schema';
 import { AVAILABILITY_ZONE_FALLBACK_CONTEXT_KEY, FUTURE_FLAGS, TARGET_PARTITIONS, FUTURE_FLAGS_EXPIRED } from '@aws-cdk/cx-api';
+import { pythonExecutable } from 'aws-cdk/lib/init';
 import { CdkCliWrapper, ICdk } from 'cdk-cli-wrapper';
 import * as fs from 'fs-extra';
 import { flatten } from '../utils';
@@ -49,6 +50,14 @@ export interface IntegRunnerOptions {
    */
   readonly cdk?: ICdk;
 
+  /**
+   * You can specify a custom run command, and it will be applied to all test files.
+   * If it contains {filePath}, the test file names will be substituted at that place in the command for each run.
+   *
+   * @default - test run command will be `node`
+   */
+  readonly appCommand?: string;
+
   /**
    * Show output from running integration tests
    *
@@ -64,6 +73,41 @@ export interface IntegRunnerOptions {
  * Represents an Integration test runner
  */
 export abstract class IntegRunner {
+
+  // Best-effort reconstruction of the classpath of the file in the project.
+  // Example: /absolute/file/path/src/main/java/com/myorg/IntegTest.java -> com.myorg.IntegTest
+  // Should work for standard Java project layouts.
+  private static getJavaClassPath(absoluteFilePath: string): string | undefined {
+    const packagePath = absoluteFilePath.split('/java/').slice(-1)[0];
+    if (!packagePath) {
+      return undefined;
+    }
+    // string.replaceAll isn't available in the TS version the project uses
+    return packagePath.split('/').join('.').replace('.java', '');
+  }
+  // Will find the closest pom.xml file in the directory structure for this file
+  private static getJavaPomPath(executionDirectoryPath: string): string | undefined {
+    // Will generate the full list of ancestors in the directory path
+    // Example: ['/', '/home', '/home/MyUser', '/home/MyUser/Desktop']
+    const dirHierarchy = executionDirectoryPath
+      .split(path.sep)
+      .filter(dirName => dirName !== '')
+      .reduce((hierarchy, segment) => {
+        hierarchy.push(path.join(hierarchy[hierarchy.length - 1], segment));
+        return hierarchy;
+      },
+      // For Windows support
+      [path.toNamespacedPath('/')]);
+
+    for (const parentDir of dirHierarchy.reverse()) {
+      const searchPath = path.join(parentDir, 'pom.xml');
+      if (fs.existsSync(searchPath)) {
+        return path.relative(executionDirectoryPath, searchPath);
+      }
+    }
+    return undefined;
+  }
+
   /**
    * The directory where the snapshot will be stored
    */
@@ -150,7 +194,24 @@ export abstract class IntegRunner {
       },
     });
     this.cdkOutDir = options.integOutDir ?? this.test.temporaryOutputDir;
-    this.cdkApp = `node ${path.relative(this.directory, this.test.fileName)}`;
+
+    const defaultAppCommands = new Map<string, string>([
+      ['javascript', 'node {filePath}'],
+      ['typescript', 'node -r ts-node/register {filePath}'],
+      ['python', `${pythonExecutable()} {filePath}`],
+      ['go', 'go mod download && go run {filePath}'],
+      ['csharp', 'dotnet run {filePath}'],
+      ['fsharp', 'dotnet run {filePath}'],
+      ['java', `mvn -f ${IntegRunner.getJavaPomPath(path.dirname(options.test.absoluteFileName))} -e -q compile exec:java clean -Dexec.mainClass=${IntegRunner.getJavaClassPath(options.test.absoluteFileName)}`],
+    ]);
+    const testRunCommand = options.appCommand ?? defaultAppCommands.get(this.test.language);
+
+    if (!testRunCommand) {
+      throw new Error('Could not find default run command for this file extension. Try specifying a custom one with the --app flag.');
+    }
+
+    this.cdkApp = testRunCommand.replace('{filePath}', path.relative(this.directory, this.test.fileName));
+
     this.profile = options.profile;
     if (this.hasSnapshot()) {
       this.expectedTestSuite = this.loadManifest();

packages/@aws-cdk/integ-runner/lib/workers/common.ts

@@ -103,6 +103,13 @@ export interface SnapshotVerificationOptions {
    * @default false
    */
   readonly verbose?: boolean;
+
+  /**
+   * The CLI command used to run the test files.
+   *
+   * @default - based on language of each file
+   */
+  readonly appCommand?: string;
 }
 
 /**
@@ -162,6 +169,13 @@ export interface IntegTestOptions {
    * @default true
    */
   readonly updateWorkflow?: boolean;
+
+  /**
+   * The CLI command used to run the test files.
+   *
+   * @default - based on language of each file
+   */
+  readonly appCommand?: string;
 }
 
 /**

packages/@aws-cdk/integ-runner/lib/workers/extract/extract_worker.ts

@@ -27,6 +27,7 @@ export function integTestWorker(request: IntegTestBatchRequest): IntegTestWorker
         env: {
           AWS_REGION: request.region,
         },
+        appCommand: request.appCommand,
         showOutput: verbosity >= 2,
       }, testInfo.destructiveChanges);
 
@@ -105,7 +106,7 @@ export function snapshotTestWorker(testInfo: IntegTestInfo, options: SnapshotVer
   }, 60_000);
 
   try {
-    const runner = new IntegSnapshotRunner({ test });
+    const runner = new IntegSnapshotRunner({ test, appCommand: options.appCommand });
     if (!runner.hasSnapshot()) {
       workerpool.workerEmit({
         reason: DiagnosticReason.NO_SNAPSHOT,

packages/@aws-cdk/integ-runner/lib/workers/integ-test-worker.ts

@@ -135,6 +135,7 @@ export async function runIntegrationTestsInParallel(
         dryRun: options.dryRun,
         verbosity: options.verbosity,
         updateWorkflow: options.updateWorkflow,
+        appCommand: options.appCommand,
       }], {
         on: printResults,
       });