feat(testrunner): allow annotating tests as flaky (#3663)

test-runner/src/builtin.fixtures.ts

@@ -32,6 +32,7 @@ declare global {
 
   type ItFunction<STATE> = ((name: string, inner: (state: STATE) => Promise<void> | void) => void) & {
     fail(condition: boolean): ItFunction<STATE>;
+    flaky(condition: boolean): ItFunction<STATE>;
     skip(condition: boolean): ItFunction<STATE>;
     slow(): ItFunction<STATE>;
     repeat(n: number): ItFunction<STATE>;

test-runner/src/fixtures.ts

@@ -160,16 +160,14 @@ export class FixturePool {
         this.resolveParametersAndRun(fn, info.config, info).then(() => {
           info.result.status = 'passed';
           clearTimeout(timer);
+        }).catch(e => {
+          info.result.status = 'failed';
+          info.result.error = serializeError(e);
         }),
         timerPromise.then(() => {
           info.result.status = 'timedOut';
-          Promise.reject(new Error(`Timeout of ${timeout}ms exceeded`));
         })
       ]);
-    } catch (e) {
-      info.result.status = 'failed';
-      info.result.error = serializeError(e);
-      throw e;
     } finally {
       await this.teardownScope('test');
     }

test-runner/src/index.ts

@@ -106,7 +106,5 @@ export async function run(config: RunnerConfig, files: string[], reporter: Repor
     for (const f of afterFunctions)
       await f();
   }
-  return suite.findTest(test => {
-    return !!test.results.find(result => result.status === 'failed' || result.status === 'timedOut');
-  }) ? 'failed' : 'passed';
+  return suite.findTest(test => !test._ok()) ? 'failed' : 'passed';
 }

test-runner/src/runner.ts

@@ -167,6 +167,8 @@ export class Runner {
     const worker = this._config.debug ? new InProcessWorker(this) : new OopWorker(this);
     worker.on('testBegin', params => {
       const { test } = this._testById.get(params.id);
+      test._skipped = params.skipped;
+      test._flaky = params.flaky;
       this._reporter.onTestBegin(test);
     });
     worker.on('testEnd', params => {

test-runner/src/spec.ts

@@ -53,7 +53,7 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
   const suites = [suite];
   suite.file = file;
 
-  const it = specBuilder(['skip', 'fail', 'slow', 'only'], (specs, title, fn) => {
+  const it = specBuilder(['skip', 'fail', 'slow', 'only', 'flaky'], (specs, title, fn) => {
     const suite = suites[0];
     const test = new Test(title, fn);
     test.file = file;
@@ -67,11 +67,13 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
       test._skipped = true;
     if (!only && specs.fail && specs.fail[0])
       test._skipped = true;
+    if (specs.flaky && specs.flaky[0])
+      test._flaky = true;
     suite._addTest(test);
     return test;
   });
 
-  const describe = specBuilder(['skip', 'fail', 'only'], (specs, title, fn) => {
+  const describe = specBuilder(['skip', 'fixme', 'only'], (specs, title, fn) => {
     const child = new Suite(title, suites[0]);
     suites[0]._addSuite(child);
     child.file = file;
@@ -80,8 +82,6 @@ export function spec(suite: Suite, file: string, timeout: number): () => void {
       child.only = true;
     if (!only && specs.skip && specs.skip[0])
       child.skipped = true;
-    if (!only && specs.fail && specs.fail[0])
-      child.skipped = true;
     suites.unshift(child);
     fn();
     suites.shift();

test-runner/src/test.ts

@@ -21,13 +21,14 @@ export class Test {
   title: string;
   file: string;
   only = false;
-  _skipped = false;
   slow = false;
   timeout = 0;
   fn: Function;
   results: TestResult[] = [];
 
   _id: string;
+  _skipped = false;
+  _flaky = false;
   _overriddenFn: Function;
   _startTime: number;
 
@@ -56,12 +57,25 @@ export class Test {
     return result;
   }
 
+  _ok(): boolean {
+    if (this._skipped)
+      return true;
+    const hasFailedResults = !!this.results.find(r => r.status !== 'passed' && r.status !== 'skipped');
+    if (!hasFailedResults)
+      return true;
+    if (!this._flaky)
+      return false;
+    const hasPassedResults = !!this.results.find(r => r.status === 'passed');
+    return hasPassedResults;
+  }
+
   _clone(): Test {
     const test = new Test(this.title, this.fn);
     test.suite = this.suite;
     test.only = this.only;
     test.file = this.file;
     test.timeout = this.timeout;
+    test._flaky = this._flaky;
     test._overriddenFn = this._overriddenFn;
     return test;
   }

test-runner/src/testRunner.ts

@@ -151,11 +151,15 @@ export class TestRunner extends EventEmitter {
 
     const id = test._id;
     this._testId = id;
-    this.emit('testBegin', { id });
+    this.emit('testBegin', {
+      id,
+      skipped: test._skipped,
+      flaky: test._flaky,
+    });
 
     const result: TestResult = {
       duration: 0,
-      status: 'none',
+      status: 'passed',
       stdout: [],
       stderr: [],
       data: {}
@@ -179,15 +183,15 @@ export class TestRunner extends EventEmitter {
       } else {
         result.status = 'passed';
       }
-      result.duration = Date.now() - startTime;
-      this.emit('testEnd', { id, result });
     } catch (error) {
-      result.error = serializeError(error);
+      // Error in the test fixture teardown.
       result.status = 'failed';
-      result.duration = Date.now() - startTime;
-      this._failedTestId = this._testId;
-      this.emit('testEnd', { id, result });
+      result.error = serializeError(error);
     }
+    result.duration = Date.now() - startTime;
+    this.emit('testEnd', { id, result });
+    if (result.status !== 'passed')
+      this._failedTestId = this._testId;
     this._testResult = null;
     this._testId = null;
   }

test-runner/test/assets/allow-flaky.js

@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+it.flaky('flake', async ({}) => {
+	try {
+		fs.readFileSync(path.join(__dirname, '..', 'test-results', 'allow-flaky.txt'));
+	} catch (e) {
+		// First time this fails.
+		fs.writeFileSync(path.join(__dirname, '..', 'test-results', 'allow-flaky.txt'), 'TRUE');
+		expect(true).toBe(false);
+	}
+});

test-runner/test/exit-code.spec.ts

@@ -100,6 +100,12 @@ it('should report suite errors', async () => {
   expect(output).toContain('Suite error');
 });
 
+it('should allow flaky', async () => {
+  const result = await runTest('allow-flaky.js', { retries: 1 });
+  expect(result.exitCode).toBe(0);
+  expect(result.flaky).toBe(1);
+});
+
 async function runTest(filePath: string, params: any = {}) {
   const outputDir = path.join(__dirname, 'test-results');
   const reportFile = path.join(outputDir, 'results.json');