feat(testrunner): allow filtering by name and show all focused tests (#…

…1354)
microsoft · Mar 12, 2020 · 1cd00bd · 1cd00bd
1 parent b43f33f
commit 1cd00bd
Show file tree

Hide file tree

Showing 4 changed files with 120 additions and 78 deletions.
diff --git a/test/test.js b/test/test.js
@@ -124,9 +124,10 @@ for (const browserConfig of BROWSER_CONFIGS) {
   });
 }
 
-if (process.env.CI && testRunner.hasFocusedTestsOrSuites()) {
-  console.error('ERROR: "focused" tests/suites are prohibited on bots. Remove any "fit"/"fdescribe" declarations.');
-  process.exit(1);
+const filterArgIndex = process.argv.indexOf('--filter');
+if (filterArgIndex !== -1) {
+  const filter = process.argv[filterArgIndex + 1];
+  testRunner.focusMatchingTests(new RegExp(filter, 'i'));
 }
 
 new Reporter(testRunner, {

diff --git a/utils/testrunner/Reporter.js b/utils/testrunner/Reporter.js
@@ -47,7 +47,18 @@ class Reporter {
     if (allTests.length === runnableTests.length) {
       console.log(`Running all ${colors.yellow(runnableTests.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
     } else {
-      console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`);
+      console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`);
+      console.log('');
+      const focusedSuites = this._runner.focusedSuites();
+      const focusedTests = this._runner.focusedTests();
+      if (focusedSuites.length) {
+        console.log('Focused Suites and Tests:');
+        for (let i = 0; i < focusedSuites.length; ++i)
+          console.log(`  ${i + 1}) ${focusedSuites[i].fullName} (${formatLocation(focusedSuites[i].location)})`);
+        for (let i = 0; i < focusedTests.length; ++i)
+          console.log(`  ${i + 1 + focusedSuites.length}) ${focusedTests[i].fullName} (${formatLocation(focusedTests[i].location)})`);
+        console.log('');
+      }
     }
   }
 

diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js
@@ -116,6 +116,7 @@ class Suite {
     this.declaredMode = declaredMode;
     /** @type {!Array<(!Test|!Suite)>} */
     this.children = [];
+    this.location = getCallerLocation(__filename);
 
     this.beforeAll = null;
     this.beforeEach = null;
@@ -441,18 +442,19 @@ class TestRunner extends EventEmitter {
       timeout = 10 * 1000, // Default timeout is 10 seconds.
       parallel = 1,
       breakOnFailure = false,
+      crashIfTestsAreFocusedOnCI = true,
       disableTimeoutWhenInspectorIsEnabled = true,
     } = options;
+    this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI;
     this._sourceMapSupport = new SourceMapSupport();
     this._rootSuite = new Suite(null, '', TestMode.Run);
     this._currentSuite = this._rootSuite;
     this._tests = [];
+    this._suites = [];
     this._timeout = timeout === 0 ? INFINITE_TIMEOUT : timeout;
     this._parallel = parallel;
     this._breakOnFailure = breakOnFailure;
 
-    this._hasFocusedTestsOrSuites = false;
-
     if (MAJOR_NODEJS_VERSION >= 8 && disableTimeoutWhenInspectorIsEnabled) {
       if (inspector.url()) {
         console.log('TestRunner detected inspector; overriding certain properties to be debugger-friendly');
@@ -508,18 +510,17 @@ class TestRunner extends EventEmitter {
     const test = new Test(this._currentSuite, name, callback, mode, timeout);
     this._currentSuite.children.push(test);
     this._tests.push(test);
-    this._hasFocusedTestsOrSuites = this._hasFocusedTestsOrSuites || mode === TestMode.Focus;
     return test;
   }
 
   _addSuite(mode, name, callback, ...args) {
     const oldSuite = this._currentSuite;
     const suite = new Suite(this._currentSuite, name, mode);
+    this._suites.push(suite);
     this._currentSuite.children.push(suite);
     this._currentSuite = suite;
     callback(...args);
     this._currentSuite = oldSuite;
-    this._hasFocusedTestsOrSuites = this._hasFocusedTestsOrSuites || mode === TestMode.Focus;
   }
 
   _addHook(hookName, callback) {
@@ -530,27 +531,34 @@ class TestRunner extends EventEmitter {
 
   async run() {
     let session = this._debuggerLogBreakpointLines.size ? await setLogBreakpoints(this._debuggerLogBreakpointLines) : null;
-    const runnableTests = this._runnableTests();
+    const runnableTests = this.runnableTests();
     this.emit(TestRunner.Events.Started, runnableTests);
-    this._runningPass = new TestPass(this, this._parallel, this._breakOnFailure);
-    const termination = await this._runningPass.run(runnableTests).catch(e => {
-      console.error(e);
-      throw e;
-    });
-    this._runningPass = null;
+
     const result = {};
-    if (termination) {
-      result.result = termination.result;
-      result.exitCode = 130;
-      result.terminationMessage = termination.message;
-      result.terminationError = termination.error;
+    if (this._crashIfTestsAreFocusedOnCI && process.env.CI && this.hasFocusedTestsOrSuites()) {
+      result.result = TestResult.Crashed;
+      result.exitCode = 2;
+      result.terminationMessage = '"focused" tests or suites are probitted on CI';
     } else {
-      if (this.failedTests().length) {
-        result.result = TestResult.Failed;
-        result.exitCode = 1;
+      this._runningPass = new TestPass(this, this._parallel, this._breakOnFailure);
+      const termination = await this._runningPass.run(runnableTests).catch(e => {
+        console.error(e);
+        throw e;
+      });
+      this._runningPass = null;
+      if (termination) {
+        result.result = termination.result;
+        result.exitCode = 130;
+        result.terminationMessage = termination.message;
+        result.terminationError = termination.error;
       } else {
-        result.result = TestResult.Ok;
-        result.exitCode = 0;
+        if (this.failedTests().length) {
+          result.result = TestResult.Failed;
+          result.exitCode = 1;
+        } else {
+          result.result = TestResult.Ok;
+          result.exitCode = 0;
+        }
       }
     }
     this.emit(TestRunner.Events.Finished, result);
@@ -569,8 +577,8 @@ class TestRunner extends EventEmitter {
     return this._timeout;
   }
 
-  _runnableTests() {
-    if (!this._hasFocusedTestsOrSuites)
+  runnableTests() {
+    if (!this.hasFocusedTestsOrSuites())
       return this._tests;
 
     const tests = [];
@@ -601,8 +609,23 @@ class TestRunner extends EventEmitter {
     return tests.map(t => t.test);
   }
 
+  focusedSuites() {
+    return this._suites.filter(suite => suite.declaredMode === 'focus');
+  }
+
+  focusedTests() {
+    return this._tests.filter(test => test.declaredMode === 'focus');
+  }
+
   hasFocusedTestsOrSuites() {
-    return this._hasFocusedTestsOrSuites;
+    return !!this.focusedTests().length || !!this.focusedSuites().length;
+  }
+
+  focusMatchingTests(fullNameRegex) {
+    for (const test of this._tests) {
+      if (fullNameRegex.test(test.fullName))
+        test.declaredMode = 'focus';
+    }
   }
 
   tests() {