chore(testrunner): introduce test result, reuse it in ipc (#3644)

microsoft · Aug 26, 2020 · a20bb94 · a20bb94
1 parent 9e2e870
commit a20bb94
Show file tree

Hide file tree

Showing 29 changed files with 433 additions and 422 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -1,8 +1,5 @@
 test/assets/modernizr.js
-third_party/*
-utils/browser/playwright-web.js
 utils/doclint/check_public_api/test/
-utils/testrunner/examples/
 lib/
 *.js
 src/generated/*
@@ -14,5 +11,7 @@ src/server/webkit/protocol.ts
 /electron-types.d.ts
 utils/generate_types/overrides.d.ts
 utils/generate_types/test/test.ts
-test/
-test-runner/
+/test/
+node_modules/
+browser_patches/*/checkout/
+packages/**/*.d.ts
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -2,7 +2,6 @@ module.exports = {
     parser: '@typescript-eslint/parser',
     plugins: ['@typescript-eslint', 'notice'],
     parserOptions: {
-      project: ['./tsconfig.json', './test/tsconfig.json'],
       ecmaVersion: 9,
       sourceType: 'module',
     },
@@ -113,8 +112,5 @@ module.exports = {
             "mustMatch": "Copyright",
             "templateFile": "./utils/copyright.js",
         }],
-
-        // type-aware rules
-        "@typescript-eslint/no-unnecessary-type-assertion": 2,
     }
 };
diff --git a/package.json b/package.json
@@ -13,7 +13,7 @@
     "ftest": "cross-env BROWSER=firefox node test-runner/cli test/",
     "wtest": "cross-env BROWSER=webkit node test-runner/cli test/",
     "test": "node test-runner/cli test/",
-    "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts ./src || eslint --ext js,ts ./src",
+    "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts . || eslint --ext js,ts .",
     "tsc": "tsc -p .",
     "tsc-installer": "tsc -p ./src/install/tsconfig.json",
     "doc": "node utils/doclint/cli.js",

diff --git a/src/utils/browserPaths.ts b/src/utils/browserPaths.ts
@@ -32,7 +32,9 @@ export type BrowserDescriptor = {
 export const hostPlatform = ((): BrowserPlatform => {
   const platform = os.platform();
   if (platform === 'darwin') {
-    const macVersion = execSync('sw_vers -productVersion').toString('utf8').trim().split('.').slice(0, 2).join('.');
+    const macVersion = execSync('sw_vers -productVersion', {
+      stdio: ['ignore', 'pipe', 'ignore']
+    }).toString('utf8').trim().split('.').slice(0, 2).join('.');
     return `mac${macVersion}` as BrowserPlatform;
   }
   if (platform === 'linux') {

diff --git a/test-runner/src/builtin.fixtures.ts b/test-runner/src/builtin.fixtures.ts
@@ -30,7 +30,7 @@ declare global {
     repeat(n: number): DescribeFunction;
   };
 
-  type ItFunction<STATE> = ((name: string, inner: (state: STATE) => Promise<void>) => void) & {
+  type ItFunction<STATE> = ((name: string, inner: (state: STATE) => Promise<void> | void) => void) & {
     fail(condition: boolean): ItFunction<STATE>;
     skip(condition: boolean): ItFunction<STATE>;
     slow(): ItFunction<STATE>;

diff --git a/test-runner/src/cli.ts b/test-runner/src/cli.ts
@@ -33,64 +33,64 @@ export const reporters = {
 };
 
 program
-  .version('Version ' + /** @type {any} */ (require)('../package.json').version)
-  .option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
-  .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
-  .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2) as any)
-  .option('--reporter <reporter>', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot')
-  .option('--trial-run', 'Only collect the matching tests and report them as passing')
-  .option('--quiet', 'Suppress stdio', false)
-  .option('--debug', 'Run tests in-process for debugging', false)
-  .option('--output <outputDir>', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results'))
-  .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000')
-  .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
-  .action(async (command) => {
-    const testDir = path.resolve(process.cwd(), command.args[0]);
-    const config: RunnerConfig = {
-      debug: command.debug,
-      forbidOnly: command.forbidOnly,
-      quiet: command.quiet,
-      grep: command.grep,
-      jobs: command.jobs,
-      outputDir: command.output,
-      snapshotDir: path.join(testDir, '__snapshots__'),
-      testDir,
-      timeout: command.timeout,
-      trialRun: command.trialRun,
-      updateSnapshots: command.updateSnapshots
-    };
+    .version('Version ' + /** @type {any} */ (require)('../package.json').version)
+    .option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
+    .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
+    .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2) as any)
+    .option('--reporter <reporter>', 'Specify reporter to use, comma-separated, can be "dot", "list", "json"', 'dot')
+    .option('--trial-run', 'Only collect the matching tests and report them as passing')
+    .option('--quiet', 'Suppress stdio', false)
+    .option('--debug', 'Run tests in-process for debugging', false)
+    .option('--output <outputDir>', 'Folder for output artifacts, default: test-results', path.join(process.cwd(), 'test-results'))
+    .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', '10000')
+    .option('-u, --update-snapshots', 'Use this flag to re-record every snapshot that fails during this test run')
+    .action(async command => {
+      const testDir = path.resolve(process.cwd(), command.args[0]);
+      const config: RunnerConfig = {
+        debug: command.debug,
+        forbidOnly: command.forbidOnly,
+        quiet: command.quiet,
+        grep: command.grep,
+        jobs: command.jobs,
+        outputDir: command.output,
+        snapshotDir: path.join(testDir, '__snapshots__'),
+        testDir,
+        timeout: command.timeout,
+        trialRun: command.trialRun,
+        updateSnapshots: command.updateSnapshots
+      };
 
-    const reporterList = command.reporter.split(',');
-    const reporterObjects: Reporter[] = reporterList.map(c => {
-      if (reporters[c])
-        return new reporters[c]();
-      try {
-        const p = path.resolve(process.cwd(), c);
-        return new (require(p).default);
-      } catch (e) {
-        console.error('Invalid reporter ' + c, e);
+      const reporterList = command.reporter.split(',');
+      const reporterObjects: Reporter[] = reporterList.map(c => {
+        if (reporters[c])
+          return new reporters[c]();
+        try {
+          const p = path.resolve(process.cwd(), c);
+          return new (require(p).default)();
+        } catch (e) {
+          console.error('Invalid reporter ' + c, e);
+          process.exit(1);
+        }
+      });
+
+      const files = collectFiles(testDir, '', command.args.slice(1));
+      const result = await run(config, files, new Multiplexer(reporterObjects));
+      if (result === 'forbid-only') {
+        console.error('=====================================');
+        console.error(' --forbid-only found a focused test.');
+        console.error('=====================================');
         process.exit(1);
       }
-    });
-
-    const files = collectFiles(testDir, '', command.args.slice(1));
-    const result = await run(config, files, new Multiplexer(reporterObjects));
-    if (result === 'forbid-only') {
-      console.error('=====================================');
-      console.error(' --forbid-only found a focused test.');
-      console.error('=====================================');
-      process.exit(1);
-    }
 
-    if (result === 'no-tests') {
-      console.error('=================');
-      console.error(' no tests found.');
-      console.error('=================');
-      process.exit(1);
-    }
+      if (result === 'no-tests') {
+        console.error('=================');
+        console.error(' no tests found.');
+        console.error('=================');
+        process.exit(1);
+      }
 
-    process.exit(result === 'failed' ? 1 : 0);
-  });
+      process.exit(result === 'failed' ? 1 : 0);
+    });
 
 program.parse(process.argv);
 

diff --git a/test-runner/src/expect.ts b/test-runner/src/expect.ts
@@ -35,7 +35,7 @@ export function initializeImageMatcher(config: RunnerConfig) {
   function toMatchImage(received: Buffer, name: string, options?: { threshold?: number }) {
     const { pass, message } = compare(received, name, config, testFile, options);
     return { pass, message: () => message };
-  };
+  }
   expect.extend({ toMatchImage });
 }