fix: mark test as failed when regression test does not fail

src/Tracker/index.ts

@@ -168,32 +168,55 @@ export class Tracker {
      */
     this.aggregates.total++
 
+    /**
+     * Test was skipped
+     */
     if (payload.isSkipped) {
       this.aggregates.skipped++
       return
     }
 
+    /**
+     * Test was a todo
+     */
     if (payload.isTodo) {
       this.aggregates.todo++
       return
     }
 
     /**
-     * Test completed successfully
+     * Regression test. Mark test as failed, when there is no error
+     * Because, we expect regression tests to have errors.
+     *
+     * However, there is no need to move anything to the failure
+     * tree, since there is no real error
      */
-    if (!payload.hasError) {
-      this.aggregates.passed++
+    if (payload.isFailing) {
+      if (!payload.hasError) {
+        this.aggregates.failed++
+        this.hasError = true
+      } else {
+        this.aggregates.regression++
+      }
+
       return
     }
 
     /**
-     * Test has error, but is regression test
+     * Test completed successfully
      */
-    if (payload.hasError && payload.isFailing) {
-      this.aggregates.regression++
+    if (!payload.hasError) {
+      this.aggregates.passed++
       return
     }
 
+    this.markTestAsFailed(payload)
+  }
+
+  /**
+   * Mark test as failed
+   */
+  private markTestAsFailed(payload: TestEndNode) {
     /**
      * Bump failed count
      */

test/tracker/index.spec.ts

@@ -357,4 +357,49 @@ test.group('Tracker', () => {
     ])
     assert.deepEqual(summary.failedTestsTitles, [])
   })
+
+  test('mark test as failed when regression test passes', async (assert) => {
+    const emitter = new Emitter()
+    const refiner = new Refiner({})
+    const tracker = new Tracker()
+
+    const runner = new Runner(emitter)
+    const unit = new Suite('unit', emitter)
+    const functional = new Suite('functional', emitter)
+
+    const group = new Group('arithmetic', emitter, refiner)
+    const testInstance = new Test('test', new TestContext(), emitter, refiner)
+    testInstance.run(() => {})
+
+    const testInstance1 = new Test('test 1', new TestContext(), emitter, refiner)
+    testInstance1.run(() => {}).fails()
+
+    runner.add(unit).add(functional)
+    unit.add(group)
+    group.add(testInstance1)
+
+    functional.add(testInstance)
+
+    emitter.on('runner:start', (payload) => tracker.processEvent('runner:start', payload))
+    emitter.on('runner:end', (payload) => tracker.processEvent('runner:end', payload))
+    emitter.on('suite:start', (payload) => tracker.processEvent('suite:start', payload))
+    emitter.on('suite:end', (payload) => tracker.processEvent('suite:end', payload))
+    emitter.on('group:start', (payload) => tracker.processEvent('group:start', payload))
+    emitter.on('group:end', (payload) => tracker.processEvent('group:end', payload))
+    emitter.on('test:start', (payload) => tracker.processEvent('test:start', payload))
+    emitter.on('test:end', (payload) => tracker.processEvent('test:end', payload))
+
+    await Promise.all([pEvent(emitter, 'runner:end'), runner.exec()])
+    const summary = tracker.getSummary()
+    assert.isTrue(summary.hasError)
+    assert.deepEqual(summary.runnerErrors, [])
+    assert.equal(summary.total, 2)
+    assert.equal(summary.passed, 1)
+    assert.equal(summary.skipped, 0)
+    assert.equal(summary.todo, 0)
+    assert.equal(summary.failed, 1)
+    assert.equal(summary.regression, 0)
+    assert.deepEqual(summary.failureTree, [])
+    assert.deepEqual(summary.failedTestsTitles, [])
+  })
 })