fix(runner): make the default value of retry and repeats 0 (#3638)

docs/api/index.md

@@ -17,12 +17,16 @@ interface TestOptions {
   timeout?: number
   /**
    * Will retry the test specific number of times if it fails
+   *
+   * @default 0
    */
   retry?: number
   /**
    * Will repeat the same test several times even if it fails each time
    * If you have "retry" option and it fails, it will use every retry in each cycle
    * Useful for debugging random failings
+   *
+   * @default 0
    */
   repeats?: number
 }

packages/runner/src/run.ts

@@ -127,22 +127,20 @@ export async function runTest(test: Test, runner: VitestRunner) {
   test.result = {
     state: 'run',
     startTime: start,
+    retryCount: 0,
   }
   updateTask(test, runner)
 
   setCurrentTest(test)
 
-  const repeats = typeof test.repeats === 'number' ? test.repeats : 1
-
-  for (let repeatCount = 0; repeatCount < repeats; repeatCount++) {
-    const retry = test.retry || 1
-
-    for (let retryCount = 0; retryCount < retry; retryCount++) {
+  const repeats = test.repeats ?? 0
+  for (let repeatCount = 0; repeatCount <= repeats; repeatCount++) {
+    const retry = test.retry ?? 0
+    for (let retryCount = 0; retryCount <= retry; retryCount++) {
       let beforeEachCleanups: HookCleanupCallback[] = []
       try {
         await runner.onBeforeTryTest?.(test, { retry: retryCount, repeats: repeatCount })
 
-        test.result.retryCount = retryCount
         test.result.repeatCount = repeatCount
 
         beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite])
@@ -188,9 +186,10 @@ export async function runTest(test: Test, runner: VitestRunner) {
       if (test.result.state === 'pass')
         break
 
-      if (retryCount < retry - 1) {
+      if (retryCount < retry) {
         // reset state when retry test
         test.result.state = 'run'
+        test.result.retryCount = (test.result.retryCount ?? 0) + 1
       }
 
       // update retry info

packages/runner/src/types/tasks.ts

@@ -166,15 +166,14 @@ export interface TestOptions {
    * Times to retry the test if fails. Useful for making flaky tests more stable.
    * When retries is up, the last test error will be thrown.
    *
-   * @default 1
+   * @default 0
    */
   retry?: number
   /**
    * How many times the test will run.
    * Only inner tests will repeat if set on `describe()`, nested `describe()` will inherit parent's repeat by default.
    *
-   * @default 1
-   *
+   * @default 0
    */
   repeats?: number
 }

packages/vitest/src/node/reporters/renderers/listRenderer.ts

@@ -100,7 +100,7 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level =
     if (level === 0 && task.type === 'suite' && task.projectName)
       prefix += formatProjectName(task.projectName)
 
-    if (task.type === 'test' && task.result?.retryCount && task.result.retryCount > 1)
+    if (task.type === 'test' && task.result?.retryCount && task.result.retryCount > 0)
       suffix += c.yellow(` (retry x${task.result.retryCount})`)
 
     if (task.type === 'suite' && !task.meta?.typecheck) {
@@ -111,7 +111,7 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level =
     if (task.mode === 'skip' || task.mode === 'todo')
       suffix += ` ${c.dim(c.gray('[skipped]'))}`
 
-    if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 1)
+    if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 0)
       suffix += c.yellow(` (repeat x${task.result.repeatCount})`)
 
     if (task.result?.duration != null) {

test/core/test/propagate-options-nested-suite.test.ts

@@ -19,5 +19,5 @@ describe(
       })
     })
   },
-  { retry: 10 },
+  { retry: 9 },
 )

test/core/test/repeats.test.ts

@@ -1,3 +1,4 @@
+import { getCurrentTest } from '@vitest/runner'
 import { afterAll, describe, expect, test } from 'vitest'
 
 const testNumbers: number[] = []
@@ -7,16 +8,16 @@ describe('testing it/test', () => {
 
   test('test 1', () => {
     testNumbers.push(1)
-  }, { repeats: 5 })
+  }, { repeats: 4 })
 
   test('test 2', () => {
     testNumbers.push(2)
-  }, { repeats: 3 })
+  }, { repeats: 2 })
 
   test.fails('test 3', () => {
     testNumbers.push(3)
     expect(testNumbers).toStrictEqual(result)
-  }, { repeats: 1 })
+  }, { repeats: 0 })
 
   afterAll(() => {
     result.push(3)
@@ -30,7 +31,7 @@ describe('testing describe', () => {
   test('test 1', () => {
     describeNumbers.push(1)
   })
-}, { repeats: 3 })
+}, { repeats: 2 })
 
 afterAll(() => {
   expect(describeNumbers).toStrictEqual([1, 1, 1])
@@ -39,14 +40,21 @@ afterAll(() => {
 const retryNumbers: number[] = []
 
 describe('testing repeats with retry', () => {
-  const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
-  test('test 1', () => {
-    retryNumbers.push(1)
-  }, { repeats: 5, retry: 2 })
-
-  afterAll(() => {
-    expect(retryNumbers).toStrictEqual(result)
+  describe('normal test', () => {
+    const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+    test.fails('test 1', () => {
+      retryNumbers.push(1)
+      expect(1).toBe(2)
+    }, { repeats: 4, retry: 1 })
+
+    afterAll(() => {
+      expect(retryNumbers).toStrictEqual(result)
+    })
   })
+
+  test('should not reset retry count', () => {
+    expect(getCurrentTest()!.result?.retryCount).toBe(3)
+  }, { repeats: 2, retry: 1 })
 })
 
 const nestedDescribeNumbers: number[] = []
@@ -71,10 +79,10 @@ describe('testing nested describe', () => {
           nestedDescribeNumbers.push(4)
         })
       }, 100)
-    }, { repeats: 3 })
+    }, { repeats: 2 })
   })
 
   afterAll(() => {
     expect(nestedDescribeNumbers).toStrictEqual([1, 1, 2, 2, 3, 3, 3, 4, 4, 4])
   })
-}, { repeats: 2 })
+}, { repeats: 1 })

test/core/test/retry-only.test.ts

@@ -11,5 +11,5 @@ describe.only('description.only retry', () => {
   it('test should not inherit options from the description block if exists', () => {
     count5 += 1
     expect(count5).toBe(5)
-  }, { retry: 5 })
-}, { retry: 2 })
+  }, { retry: 4 })
+}, { retry: 1 })

test/core/test/retry.test.ts

@@ -4,13 +4,13 @@ let count1 = 0
 it('retry test', () => {
   count1 += 1
   expect(count1).toBe(3)
-}, { retry: 3 })
+}, { retry: 2 })
 
 let count2 = 0
 it.fails('retry test fails', () => {
   count2 += 1
   expect(count2).toBe(3)
-}, { retry: 2 })
+}, { retry: 1 })
 
 let count3 = 0
 it('retry test fails', () => {

test/failing/fixtures/expects/soft.test.ts

@@ -73,13 +73,13 @@ test('retry will passed', () => {
   expect.soft(num += 1).toBe(3)
   expect.soft(num += 1).toBe(4)
 }, {
-  retry: 2,
+  retry: 1,
 })
 
 num = 0
 test('retry will failed', () => {
   expect.soft(num += 1).toBe(4)
   expect.soft(num += 1).toBe(5)
 }, {
-  retry: 2,
+  retry: 1,
 })