feat(integ-runner): support --language presets

packages/@aws-cdk/integ-runner/README.md

@@ -53,7 +53,7 @@ to be a self contained CDK app. The runner will execute the following for each f
 - `--verbose` (default=`false`)
   verbose logging, including integration test metrics
   (specify multiple times to increase verbosity)
-- `--parallel-regions` (default=`us-east-1`,`us-east-2`, `us-west-2`)
+- `--parallel-regions` (default=`us-east-1`,`us-east-2`,`us-west-2`)
   List of regions to run tests in. If this is provided then all tests will
   be run in parallel across these regions
 - `--directory` (default=`test`)
@@ -68,6 +68,8 @@ to be a self contained CDK app. The runner will execute the following for each f
   Read the list of tests from this file
 - `--disable-update-workflow` (default=`false`)
   If this is set to `true` then the [update workflow](#update-workflow) will be disabled
+- `--language [javascript|typescript|python|java|csharp|fsharp|go]` (default=`javascript`)
+  Use a preset to run integration tests for the selected language
 - `--app`
   The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".
 - `--test-regex`

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -4,7 +4,7 @@ import * as path from 'path';
 import * as chalk from 'chalk';
 import * as workerpool from 'workerpool';
 import * as logger from './logger';
-import { IntegrationTests, IntegTestInfo } from './runner/integration-tests';
+import { IntegrationTests, IntegrationTestsDiscovery, IntegTestInfo } from './runner/integration-tests';
 import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics, IntegTestWorkerConfig, DestructiveChange } from './workers';
 
 // https://github.com/yargs/yargs/issues/1929
@@ -17,7 +17,7 @@ export function parseCliArgs(args: string[] = []) {
     .usage('Usage: integ-runner [TEST...]')
     .option('config', {
       config: true,
-      configParser: IntegrationTests.configFromFile,
+      configParser: configFromFile,
       default: 'integ.config.json',
       desc: 'Load options from a JSON config file. Options provided as CLI arguments take precedent.',
     })
@@ -35,6 +35,13 @@ export function parseCliArgs(args: string[] = []) {
     .options('from-file', { type: 'string', desc: 'Read TEST names from a file (one TEST per line)' })
     .option('inspect-failures', { type: 'boolean', desc: 'Keep the integ test cloud assembly if a failure occurs for inspection', default: false })
     .option('disable-update-workflow', { type: 'boolean', default: false, desc: 'If this is "true" then the stack update workflow will be disabled' })
+    .option('language', {
+      alias: 'l',
+      default: 'javascript',
+      choices: ['javascript', 'typescript', 'python'],
+      type: 'array',
+      desc: 'Use these presets to run integration tests for the selected languages',
+    })
     .option('app', { type: 'string', default: undefined, desc: 'The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".' })
     .option('test-regex', { type: 'array', desc: 'Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.', default: [] })
     .strict()
@@ -80,19 +87,16 @@ export function parseCliArgs(args: string[] = []) {
     force: argv.force as boolean,
     dryRun: argv['dry-run'] as boolean,
     disableUpdateWorkflow: argv['disable-update-workflow'] as boolean,
+    language: arrayFromYargs(argv.language),
   };
 }
 
 
 export async function main(args: string[]) {
   const options = parseCliArgs(args);
 
-  const testsFromArgs = await new IntegrationTests(path.resolve(options.directory)).fromCliArgs({
-    app: options.app,
-    testRegex: options.testRegex,
-    tests: options.tests,
-    exclude: options.exclude,
-  });
+  const testsFromArgs = await new IntegrationTests(path.resolve(options.directory))
+    .discover(IntegrationTestsDiscovery.fromCliOptions(options));
 
   // List only prints the discoverd tests
   if (options.list) {
@@ -227,3 +231,21 @@ export function cli(args: string[] = process.argv.slice(2)) {
     process.exitCode = 1;
   });
 }
+
+/**
+ * Read CLI options from a config file if provided.
+ *
+ * @param fileName
+ * @returns parsed CLI config options
+ */
+function configFromFile(fileName?: string): Record<string, any> {
+  if (!fileName) {
+    return {};
+  }
+
+  try {
+    return JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
+  } catch {
+    return {};
+  }
+}

packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts

@@ -139,6 +139,32 @@ export class IntegTest {
   }
 }
 
+/**
+ * Returns the name of the Python executable for this OS
+ */
+function pythonExecutable() {
+  let python = 'python3';
+  if (process.platform === 'win32') {
+    python = 'python';
+  }
+  return python;
+}
+
+class IntegrationTestPreset {
+  public constructor(readonly app: string, readonly testRegex: string[]) {}
+}
+
+const PRESETS = {
+  javascript: new IntegrationTestPreset('node {filePath}', ['^integ\..*\.js$/']),
+  typescript: new IntegrationTestPreset('node -r ts-node/register {filePath}', ['^integ\.(?!.*\.d\.ts$).*\.ts$']),
+  python: new IntegrationTestPreset(`${pythonExecutable()} {filePath}`, ['^integ_.*\.py$']),
+  csharp: new IntegrationTestPreset('dotnet run --project {filePath}', ['^Integ.*\.csproj$']),
+  fsharp: new IntegrationTestPreset('dotnet run --project {filePath}', ['^Integ.*\.fsproj$']),
+  // these are still unconfirmed and need testing
+  go: new IntegrationTestPreset('go mod download && go run {filePath}', ['^integ_.*\.go$']),
+  java: new IntegrationTestPreset('mvn -e -q compile exec:java', ['^Integ.*\.java$']),
+};
+
 /**
  * Configuration options how integration test files are discovered
  */
@@ -159,41 +185,67 @@ export interface IntegrationTestsDiscoveryOptions {
   readonly tests?: string[];
 
   /**
-    * Detect integration test files matching any of these JavaScript regex patterns.
-    *
-    * @default
-    */
-  readonly testRegex?: string[];
-
-  /**
-   * The CLI command used to run this test.
-   * If it contains {filePath}, the test file names will be substituted at that place in the command for each run.
+   * A map of of the app commands to run integration tests with,
+   * and the regex patterns matching the integration test files each app command.
    *
-   * @default - test run command will be `node {filePath}`
+   * If the app command contains {filePath}, the test file names will be substituted at that place in the command for each run.
    */
-  readonly app?: string;
+  readonly testCases: {
+    [app: string]: string[]
+  }
 }
 
-
-/**
- * Discover integration tests
- */
-export class IntegrationTests {
+export class IntegrationTestsDiscovery {
   /**
-   * Return configuration options from a file
+   * Get integration tests discovery options from CLI options
    */
-  public static configFromFile(fileName?: string): Record<string, any> {
-    if (!fileName) {
-      return {};
+  public static fromCliOptions(options: {
+    app?: string;
+    testRegex?: string[],
+    language?: string[],
+    tests?: string[],
+    exclude?: boolean,
+  }): IntegrationTestsDiscoveryOptions {
+    const baseOptions = {
+      tests: options.tests,
+      exclude: options.exclude,
+    };
+
+    // Explicitly set app and test regex
+    if (options.app && options.testRegex) {
+      return {
+        testCases: {
+          [options.app]: options.testRegex,
+        },
+        ...baseOptions,
+      };
     }
 
-    try {
-      return JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
-    } catch {
-      return {};
-    }
+    const presetTestCases = Object.fromEntries(options.language?.map(language => {
+      if (language && language in PRESETS) {
+        const preset = PRESETS[language as keyof typeof PRESETS];
+        return [preset.app, preset.testRegex];
+      }
+
+      return [];
+    }) ?? []);
+
+    // const patterns = options.testRegex ?? ['^integ\\..*\\.js$'];
+    return {
+      testCases: presetTestCases,
+      tests: options.tests,
+      exclude: options.exclude,
+    };
   }
 
+  private constructor() {}
+}
+
+
+/**
+ * Discover integration tests
+ */
+export class IntegrationTests {
   constructor(private readonly directory: string) {
   }
 
@@ -237,29 +289,21 @@ export class IntegrationTests {
    * @param tests Tests to include or exclude, undefined means include all tests.
    * @param exclude Whether the 'tests' list is inclusive or exclusive (inclusive by default).
    */
-  public async fromCliArgs(options: IntegrationTestsDiscoveryOptions = {}): Promise<IntegTest[]> {
-    return this.discover(options);
-  }
-
-  private async discover(options: IntegrationTestsDiscoveryOptions): Promise<IntegTest[]> {
-    const patterns = options.testRegex ?? ['^integ\\..*\\.js$'];
-
+  public async discover(options: IntegrationTestsDiscoveryOptions): Promise<IntegTest[]> {
     const files = await this.readTree();
-    const integs = files.filter(fileName => patterns.some((p) => {
-      const regex = new RegExp(p);
-      return regex.test(fileName) || regex.test(path.basename(fileName));
-    }));
-
-    return this.request(integs, options);
-  }
-
-  private request(files: string[], options: IntegrationTestsDiscoveryOptions): IntegTest[] {
-    const discoveredTests = files.map(fileName => new IntegTest({
-      discoveryRoot: this.directory,
-      fileName,
-      appCommand: options.app,
-    }));
 
+    const discoveredTests = Object.entries(options.testCases)
+      .flatMap(([appCommand, patterns] ) => files
+        .filter(fileName => patterns.some((pattern) => {
+          const regex = new RegExp(pattern);
+          return regex.test(fileName) || regex.test(path.basename(fileName));
+        }))
+        .map(fileName => new IntegTest({
+          discoveryRoot: this.directory,
+          fileName,
+          appCommand,
+        })),
+      );
 
     return this.filterTests(discoveredTests, options.tests, options.exclude);
   }