feat(integ-runner): support --language presets

packages/@aws-cdk/integ-runner/.eslintrc.js

@@ -1,3 +1,4 @@
 const baseConfig = require('@aws-cdk/cdk-build-tools/config/eslintrc');
 baseConfig.parserOptions.project = __dirname + '/tsconfig.json';
+baseConfig.ignorePatterns = [...baseConfig.ignorePatterns, "test/language-tests/**/integ.*.ts"];
 module.exports = baseConfig;

packages/@aws-cdk/integ-runner/README.md

@@ -70,16 +70,32 @@ to be a self contained CDK app. The runner will execute the following for each f
   If this is set to `true` then the [update workflow](#update-workflow) will be disabled
 - `--app`
   The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".
+
+  Use together with `--test-regex` to fully customize how tests are run, or use with a single `--language` preset to change the command used for this language.
 - `--test-regex`
   Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.
-
+
+  Use together with `--app` to fully customize how tests are run, or use with a single `--language` preset to change which files are detected for this language.
+- `--language`
+  The language presets to use. You can discover and run tests written in multiple languages by passing this flag multiple times (`--language javascript --language typescript`). Defaults to all supported languages. Currently supported language presets are:
+  - `javascript`:
+    - File RegExp: `^integ\..*\.js$`
+    - App run command: `node {filePath}`
+  - `typescript`:\
+    Note that for TypeScript files compiled to JavaScript, the JS tests will take precedence and the TS ones won't be evaluated.
+    - File RegExp: `^integ\..*(?<!\.d)\.ts$`
+    - App run command: `node -r ts-node/register {filePath}`
+  - `python`:
+    - File RegExp: `^integ_.*\.py$`
+    - App run command: `python {filePath}`
+
 Example:
 
 ```bash
-integ-runner --update-on-failed --parallel-regions us-east-1 --parallel-regions us-east-2 --parallel-regions us-west-2 --directory ./
+integ-runner --update-on-failed --parallel-regions us-east-1 --parallel-regions us-east-2 --parallel-regions us-west-2 --directory ./ --language python
 ```
 
-This will search for integration tests recursively from the current directory and then execute them in parallel across `us-east-1`, `us-east-2`, & `us-west-2`.
+This will search for python integration tests recursively from the current directory and then execute them in parallel across `us-east-1`, `us-east-2`, & `us-west-2`.
 
 If you are providing a list of tests to execute, either as CLI arguments or from a file, the name of the test needs to be relative to the `directory`.
 For example, if there is a test `aws-iam/test/integ.policy.js` and the current working directory is `aws-iam` you would provide `integ.policy.js`

packages/@aws-cdk/integ-runner/lib/cli.ts

@@ -4,7 +4,7 @@ import * as path from 'path';
 import * as chalk from 'chalk';
 import * as workerpool from 'workerpool';
 import * as logger from './logger';
-import { IntegrationTests, IntegTestInfo } from './runner/integration-tests';
+import { IntegrationTests, IntegrationTestsDiscovery, IntegTestInfo } from './runner/integration-tests';
 import { runSnapshotTests, runIntegrationTests, IntegRunnerMetrics, IntegTestWorkerConfig, DestructiveChange } from './workers';
 
 // https://github.com/yargs/yargs/issues/1929
@@ -17,7 +17,7 @@ export function parseCliArgs(args: string[] = []) {
     .usage('Usage: integ-runner [TEST...]')
     .option('config', {
       config: true,
-      configParser: IntegrationTests.configFromFile,
+      configParser: configFromFile,
       default: 'integ.config.json',
       desc: 'Load options from a JSON config file. Options provided as CLI arguments take precedent.',
     })
@@ -35,6 +35,13 @@ export function parseCliArgs(args: string[] = []) {
     .options('from-file', { type: 'string', desc: 'Read TEST names from a file (one TEST per line)' })
     .option('inspect-failures', { type: 'boolean', desc: 'Keep the integ test cloud assembly if a failure occurs for inspection', default: false })
     .option('disable-update-workflow', { type: 'boolean', default: false, desc: 'If this is "true" then the stack update workflow will be disabled' })
+    .option('language', {
+      alias: 'l',
+      default: ['javascript', 'typescript', 'python'],
+      choices: ['javascript', 'typescript', 'python'],
+      type: 'array',
+      desc: 'Use these presets to run integration tests for the selected languages',
+    })
     .option('app', { type: 'string', default: undefined, desc: 'The custom CLI command that will be used to run the test files. You can include {filePath} to specify where in the command the test file path should be inserted. Example: --app="python3.8 {filePath}".' })
     .option('test-regex', { type: 'array', desc: 'Detect integration test files matching this JavaScript regex pattern. If used multiple times, all files matching any one of the patterns are detected.', default: [] })
     .strict()
@@ -80,19 +87,16 @@ export function parseCliArgs(args: string[] = []) {
     force: argv.force as boolean,
     dryRun: argv['dry-run'] as boolean,
     disableUpdateWorkflow: argv['disable-update-workflow'] as boolean,
+    language: arrayFromYargs(argv.language),
   };
 }
 
 
 export async function main(args: string[]) {
   const options = parseCliArgs(args);
 
-  const testsFromArgs = await new IntegrationTests(path.resolve(options.directory)).fromCliArgs({
-    app: options.app,
-    testRegex: options.testRegex,
-    tests: options.tests,
-    exclude: options.exclude,
-  });
+  const testsFromArgs = await new IntegrationTests(path.resolve(options.directory))
+    .discover(IntegrationTestsDiscovery.fromCliOptions(options));
 
   // List only prints the discoverd tests
   if (options.list) {
@@ -227,3 +231,21 @@ export function cli(args: string[] = process.argv.slice(2)) {
     process.exitCode = 1;
   });
 }
+
+/**
+ * Read CLI options from a config file if provided.
+ *
+ * @param fileName
+ * @returns parsed CLI config options
+ */
+function configFromFile(fileName?: string): Record<string, any> {
+  if (!fileName) {
+    return {};
+  }
+
+  try {
+    return JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
+  } catch {
+    return {};
+  }
+}

packages/@aws-cdk/integ-runner/lib/runner/integration-tests.ts

@@ -159,41 +159,118 @@ export interface IntegrationTestsDiscoveryOptions {
   readonly tests?: string[];
 
   /**
-    * Detect integration test files matching any of these JavaScript regex patterns.
-    *
-    * @default
-    */
-  readonly testRegex?: string[];
-
-  /**
-   * The CLI command used to run this test.
-   * If it contains {filePath}, the test file names will be substituted at that place in the command for each run.
+   * A map of of the app commands to run integration tests with,
+   * and the regex patterns matching the integration test files each app command.
    *
-   * @default - test run command will be `node {filePath}`
+   * If the app command contains {filePath}, the test file names will be substituted at that place in the command for each run.
    */
-  readonly app?: string;
+  readonly testCases: {
+    [app: string]: string[]
+  }
 }
 
-
 /**
- * Discover integration tests
+ * Returns the name of the Python executable for the current OS
  */
-export class IntegrationTests {
+function pythonExecutable() {
+  let python = 'python3';
+  if (process.platform === 'win32') {
+    python = 'python';
+  }
+  return python;
+}
+
+export class IntegrationTestsDiscovery {
   /**
-   * Return configuration options from a file
+   * Get integration tests discovery options from CLI options
    */
-  public static configFromFile(fileName?: string): Record<string, any> {
-    if (!fileName) {
-      return {};
+  public static fromCliOptions(options: {
+    app?: string;
+    exclude?: boolean,
+    language?: string[],
+    testRegex?: string[],
+    tests?: string[],
+  }): IntegrationTestsDiscoveryOptions {
+    const baseOptions = {
+      tests: options.tests,
+      exclude: options.exclude,
+    };
+
+    // Explicitly set both, app and test-regex
+    if (options.app && options.testRegex) {
+      return {
+        testCases: {
+          [options.app]: options.testRegex,
+        },
+        ...baseOptions,
+      };
+    }
+
+    // Use the selected presets
+    if (!options.app && !options.testRegex) {
+      return {
+        testCases: this.getLanguagePresets(options.language),
+        ...baseOptions,
+      };
     }
 
-    try {
-      return JSON.parse(fs.readFileSync(fileName, { encoding: 'utf-8' }));
-    } catch {
-      return {};
+    // Only one of app or test-regex is set, with a single preset selected
+    // => override either app or test-regex
+    if (options.language?.length === 1) {
+      const [presetApp, presetTestRegex] = this.getLanguagePreset(options.language[0]);
+      return {
+        testCases: {
+          [options.app ?? presetApp]: options.testRegex ?? presetTestRegex,
+        },
+        ...baseOptions,
+      };
     }
+
+    // Only one of app or test-regex is set, with multiple presets
+    // => impossible to resolve
+    const option = options.app ? '--app' : '--test-regex';
+    throw new Error(`Only a single "--language" can be used with "${option}". Alternatively provide both "--app" and "--test-regex" to fully customize the configuration.`);
   }
 
+  /**
+   * Get the default configuration for a language
+   */
+  private static getLanguagePreset(language: string) {
+    const languagePresets: {
+      [language: string]: [string, string[]]
+    } = {
+      javascript: ['node {filePath}', ['^integ\\..*\\.js$']],
+      typescript: ['node -r ts-node/register {filePath}', ['^integ\\.(?!.*\\.d\\.ts$).*\\.ts$']],
+      python: [`${pythonExecutable()} {filePath}`, ['^integ_.*\\.py$']],
+      csharp: ['dotnet run --project {filePath}', ['^Integ.*\\.csproj$']],
+      fsharp: ['dotnet run --project {filePath}', ['^Integ.*\\.fsproj$']],
+      // these are still unconfirmed and need testing
+      go: ['go mod download && go run {filePath}', ['^integ_.*\\.go$']],
+      java: ['mvn -e -q compile exec:java', ['^Integ.*\\.java$']],
+    };
+
+    return languagePresets[language];
+  }
+
+  /**
+   * Get the config for all selected languages
+   */
+  private static getLanguagePresets(languages: string[] = []) {
+    return Object.fromEntries(
+      languages
+        .map(language => this.getLanguagePreset(language))
+        .filter(Boolean),
+    );
+  }
+
+  private constructor() {}
+}
+
+
+/**
+ * Discover integration tests
+ */
+export class IntegrationTests {
   constructor(private readonly directory: string) {
   }
 
@@ -209,7 +286,6 @@ export class IntegrationTests {
       return discoveredTests;
     }
 
-
     const allTests = discoveredTests.filter(t => {
       const matches = requestedTests.some(pattern => t.matches(pattern));
       return matches !== !!exclude; // Looks weird but is equal to (matches && !exclude) || (!matches && exclude)
@@ -237,29 +313,21 @@ export class IntegrationTests {
    * @param tests Tests to include or exclude, undefined means include all tests.
    * @param exclude Whether the 'tests' list is inclusive or exclusive (inclusive by default).
    */
-  public async fromCliArgs(options: IntegrationTestsDiscoveryOptions = {}): Promise<IntegTest[]> {
-    return this.discover(options);
-  }
-
-  private async discover(options: IntegrationTestsDiscoveryOptions): Promise<IntegTest[]> {
-    const patterns = options.testRegex ?? ['^integ\\..*\\.js$'];
-
+  public async discover(options: IntegrationTestsDiscoveryOptions): Promise<IntegTest[]> {
     const files = await this.readTree();
-    const integs = files.filter(fileName => patterns.some((p) => {
-      const regex = new RegExp(p);
-      return regex.test(fileName) || regex.test(path.basename(fileName));
-    }));
-
-    return this.request(integs, options);
-  }
-
-  private request(files: string[], options: IntegrationTestsDiscoveryOptions): IntegTest[] {
-    const discoveredTests = files.map(fileName => new IntegTest({
-      discoveryRoot: this.directory,
-      fileName,
-      appCommand: options.app,
-    }));
 
+    const discoveredTests = Object.entries(options.testCases)
+      .flatMap(([appCommand, patterns]) => files
+        .filter(fileName => patterns.some((pattern) => {
+          const regex = new RegExp(pattern);
+          return regex.test(fileName) || regex.test(path.basename(fileName));
+        }))
+        .map(fileName => new IntegTest({
+          discoveryRoot: this.directory,
+          fileName,
+          appCommand,
+        })),
+      );
 
     return this.filterTests(discoveredTests, options.tests, options.exclude);
   }

packages/@aws-cdk/integ-runner/package.json

@@ -14,6 +14,7 @@
     "awslint": "cdk-awslint",
     "pkglint": "pkglint -f",
     "test": "cdk-test",
+    "integ": "integ-runner",
     "watch": "cdk-watch",
     "build+test": "yarn build && yarn test",
     "build+test+package": "yarn build+test && yarn package",
@@ -52,15 +53,19 @@
   "license": "Apache-2.0",
   "devDependencies": {
     "@aws-cdk/cdk-build-tools": "0.0.0",
-    "@types/mock-fs": "^4.13.1",
-    "mock-fs": "^4.14.0",
+    "@aws-cdk/core": "0.0.0",
+    "@aws-cdk/integ-tests": "0.0.0",
     "@aws-cdk/pkglint": "0.0.0",
     "@types/fs-extra": "^8.1.2",
     "@types/jest": "^27.5.2",
+    "@types/mock-fs": "^4.13.1",
     "@types/node": "^14.18.34",
     "@types/workerpool": "^6.1.0",
     "@types/yargs": "^15.0.14",
-    "jest": "^27.5.1"
+    "constructs": "^10.0.0",
+    "mock-fs": "^4.14.0",
+    "jest": "^27.5.1",
+    "ts-node": "^10.9.1"
   },
   "dependencies": {
     "@aws-cdk/cloud-assembly-schema": "0.0.0",