From 297922b2fbf0038fd590bed5e14ce6c4afc85fe5 Mon Sep 17 00:00:00 2001 From: Anthony Fu Date: Sun, 4 Sep 2022 15:53:43 +0800 Subject: [PATCH] feat: introduce `retry` option for tests (#1929) --- docs/api/index.md | 11 ++- .../node/reporters/renderers/listRenderer.ts | 3 + packages/vitest/src/runtime/run.ts | 92 +++++++++++-------- packages/vitest/src/runtime/suite.ts | 21 +++-- packages/vitest/src/types/tasks.ts | 26 +++++- test/core/test/retry.test.ts | 25 +++++ 6 files changed, 122 insertions(+), 56 deletions(-) create mode 100644 test/core/test/retry.test.ts diff --git a/docs/api/index.md b/docs/api/index.md index 256f206331ea..0dc492007da7 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -9,6 +9,11 @@ The following types are used in the type signatures below ```ts type Awaitable = T | PromiseLike type TestFunction = () => Awaitable + +interface TestOptions { + timeout?: number + retry?: number +} ``` When a test function returns a promise, the runner will wait until it is resolved to collect async expectations. If the promise is rejected, the test will fail. @@ -19,7 +24,7 @@ In Jest, `TestFunction` can also be of type `(done: DoneCallback) => void`. If t ## test -- **Type:** `(name: string, fn: TestFunction, timeout?: number) => void` +- **Type:** `(name: string, fn: TestFunction, timeout?: number | TestOptions) => void` - **Alias:** `it` `test` defines a set of related expectations. It receives the test name and a function that holds the expectations to test. @@ -36,7 +41,7 @@ In Jest, `TestFunction` can also be of type `(done: DoneCallback) => void`. If t ### test.skip -- **Type:** `(name: string, fn: TestFunction, timeout?: number) => void` +- **Type:** `(name: string, fn: TestFunction, timeout?: number | TestOptions) => void` - **Alias:** `it.skip` If you want to skip running certain tests, but you don't want to delete the code due to any reason, you can use `test.skip` to avoid running them. @@ -330,7 +335,7 @@ When you use `test` in the top level of file, they are collected as part of the ### describe.shuffle -- **Type:** `(name: string, fn: TestFunction, timeout?: number) => void` +- **Type:** `(name: string, fn: TestFunction, timeout?: number | TestOptions) => void` Vitest provides a way to run all tests in random order via CLI flag [`--sequence.shuffle`](/guide/cli) or config option [`sequence.shuffle`](/config/#sequence-shuffle), but if you want to have only part of your test suite to run tests in random order, you can mark it with this flag. diff --git a/packages/vitest/src/node/reporters/renderers/listRenderer.ts b/packages/vitest/src/node/reporters/renderers/listRenderer.ts index 97b7f43033a0..ec4c2593a9a0 100644 --- a/packages/vitest/src/node/reporters/renderers/listRenderer.ts +++ b/packages/vitest/src/node/reporters/renderers/listRenderer.ts @@ -92,6 +92,9 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level = let suffix = '' const prefix = ` ${getStateSymbol(task)} ` + if (task.type === 'test' && task.result?.retryCount && task.result.retryCount > 1) + suffix += c.yellow(` (retry x${task.result.retryCount})`) + if (task.type === 'suite') suffix += c.dim(` (${getTests(task).length})`) diff --git a/packages/vitest/src/runtime/run.ts b/packages/vitest/src/runtime/run.ts index 7c56a0d04db2..e55acb457292 100644 --- a/packages/vitest/src/runtime/run.ts +++ b/packages/vitest/src/runtime/run.ts @@ -108,48 +108,60 @@ export async function runTest(test: Test) { workerState.current = test - let beforeEachCleanups: HookCleanupCallback[] = [] - try { - beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', [test.context, test.suite]) - setState({ - assertionCalls: 0, - isExpectingAssertions: false, - isExpectingAssertionsError: null, - expectedAssertionsNumber: null, - expectedAssertionsNumberErrorGen: null, - testPath: test.suite.file?.filepath, - currentTestName: getFullName(test), - }, (globalThis as any)[GLOBAL_EXPECT]) - await getFn(test)() - const { - assertionCalls, - expectedAssertionsNumber, - expectedAssertionsNumberErrorGen, - isExpectingAssertions, - isExpectingAssertionsError, + const retry = test.retry || 1 + for (let retryCount = 0; retryCount < retry; retryCount++) { + let beforeEachCleanups: HookCleanupCallback[] = [] + try { + beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', [test.context, test.suite]) + setState({ + assertionCalls: 0, + isExpectingAssertions: false, + isExpectingAssertionsError: null, + expectedAssertionsNumber: null, + expectedAssertionsNumberErrorGen: null, + testPath: test.suite.file?.filepath, + currentTestName: getFullName(test), + }, (globalThis as any)[GLOBAL_EXPECT]) + + test.result.retryCount = retryCount + + await getFn(test)() + const { + assertionCalls, + expectedAssertionsNumber, + expectedAssertionsNumberErrorGen, + isExpectingAssertions, + isExpectingAssertionsError, // @ts-expect-error local is private - } = test.context._local - ? test.context.expect.getState() - : getState((globalThis as any)[GLOBAL_EXPECT]) - if (expectedAssertionsNumber !== null && assertionCalls !== expectedAssertionsNumber) - throw expectedAssertionsNumberErrorGen!() - if (isExpectingAssertions === true && assertionCalls === 0) - throw isExpectingAssertionsError - - test.result.state = 'pass' - } - catch (e) { - test.result.state = 'fail' - test.result.error = processError(e) - } + } = test.context._local + ? test.context.expect.getState() + : getState((globalThis as any)[GLOBAL_EXPECT]) + if (expectedAssertionsNumber !== null && assertionCalls !== expectedAssertionsNumber) + throw expectedAssertionsNumberErrorGen!() + if (isExpectingAssertions === true && assertionCalls === 0) + throw isExpectingAssertionsError - try { - await callSuiteHook(test.suite, test, 'afterEach', [test.context, test.suite]) - await Promise.all(beforeEachCleanups.map(i => i?.())) - } - catch (e) { - test.result.state = 'fail' - test.result.error = processError(e) + test.result.state = 'pass' + } + catch (e) { + test.result.state = 'fail' + test.result.error = processError(e) + } + + try { + await callSuiteHook(test.suite, test, 'afterEach', [test.context, test.suite]) + await Promise.all(beforeEachCleanups.map(i => i?.())) + } + catch (e) { + test.result.state = 'fail' + test.result.error = processError(e) + } + + if (test.result.state === 'pass') + break + + // update retry info + updateTask(test) } // if test is marked to be failed, flip the result diff --git a/packages/vitest/src/runtime/suite.ts b/packages/vitest/src/runtime/suite.ts index ae7d088f91b2..b41b855ea1dd 100644 --- a/packages/vitest/src/runtime/suite.ts +++ b/packages/vitest/src/runtime/suite.ts @@ -1,5 +1,5 @@ import util from 'util' -import type { BenchFunction, BenchOptions, Benchmark, BenchmarkAPI, File, RunMode, Suite, SuiteAPI, SuiteCollector, SuiteFactory, SuiteHooks, Task, Test, TestAPI, TestFunction } from '../types' +import type { BenchFunction, BenchOptions, Benchmark, BenchmarkAPI, File, RunMode, Suite, SuiteAPI, SuiteCollector, SuiteFactory, SuiteHooks, Task, Test, TestAPI, TestFunction, TestOptions } from '../types' import { getWorkerState, isObject, isRunningInBenchmark, isRunningInTest, noop } from '../utils' import { createChainable } from './chain' import { collectTask, collectorContext, createTestContext, runWithSuite, withTimeout } from './context' @@ -8,8 +8,8 @@ import { getHooks, setFn, setHooks } from './map' // apis export const suite = createSuite() export const test = createTest( - function (name: string, fn?: TestFunction, timeout?: number) { - getCurrentSuite().test.fn.call(this, name, fn, timeout) + function (name: string, fn?: TestFunction, options?: number | TestOptions) { + getCurrentSuite().test.fn.call(this, name, fn, options) }, ) @@ -76,12 +76,15 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m initSuite() - const test = createTest(function (name: string, fn = noop, timeout?: number) { + const test = createTest(function (name: string, fn = noop, options?: number | TestOptions) { if (!isRunningInTest()) throw new Error('`test()` and `it()` is only available in test mode.') const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' + if (typeof options === 'number') + options = { timeout: options } + const test: Test = { id: '', type: 'test', @@ -89,7 +92,9 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m mode, suite: undefined!, fails: this.fails, + retry: options?.retry, } as Omit as Test + if (this.concurrent || concurrent) test.concurrent = true if (shuffle) @@ -104,7 +109,7 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m setFn(test, withTimeout( () => fn(context), - timeout, + options?.timeout, )) tasks.push(test) @@ -220,7 +225,7 @@ function createTest(fn: ( this: Record<'concurrent' | 'skip' | 'only' | 'todo' | 'fails', boolean | undefined>, title: string, fn?: TestFunction, - timeout?: number + options?: number | TestOptions ) => void )) { const testFn = fn as any @@ -228,10 +233,10 @@ function createTest(fn: ( testFn.each = function(this: { withContext: () => TestAPI }, cases: ReadonlyArray) { const test = this.withContext() - return (name: string, fn: (...args: T[]) => void, timeout?: number) => { + return (name: string, fn: (...args: T[]) => void, options?: number | TestOptions) => { cases.forEach((i, idx) => { const items = Array.isArray(i) ? i : [i] - test(formatTitle(name, items, idx), () => fn(...items), timeout) + test(formatTitle(name, items, idx), () => fn(...items), options) }) } } diff --git a/packages/vitest/src/types/tasks.ts b/packages/vitest/src/types/tasks.ts index 6ba8f8673c0a..ba8673964a0d 100644 --- a/packages/vitest/src/types/tasks.ts +++ b/packages/vitest/src/types/tasks.ts @@ -15,6 +15,7 @@ export interface TaskBase { suite?: Suite file?: File result?: TaskResult + retry?: number logs?: UserConsoleLog[] } @@ -26,6 +27,7 @@ export interface TaskResult { error?: ErrorWithDiff htmlError?: string hooks?: Partial> + retryCount?: number benchmark?: BenchmarkResult } @@ -111,30 +113,44 @@ interface TestEachFunction { (cases: ReadonlyArray): ( name: string, fn: (...args: T) => Awaitable, - timeout?: number, + options?: number | TestOptions, ) => void >(cases: ReadonlyArray): ( name: string, fn: (...args: ExtractEachCallbackArgs) => Awaitable, - timeout?: number, + options?: number | TestOptions, ) => void (cases: ReadonlyArray): ( name: string, fn: (...args: T[]) => Awaitable, - timeout?: number, + options?: number | TestOptions, ) => void } type ChainableTestAPI = ChainableFunction< 'concurrent' | 'only' | 'skip' | 'todo' | 'fails', - [name: string, fn?: TestFunction, timeout?: number], + [name: string, fn?: TestFunction, options?: number | TestOptions], void, { each: TestEachFunction - (name: string, fn?: TestFunction, timeout?: number): void + (name: string, fn?: TestFunction, options?: number | TestOptions): void } > +export interface TestOptions { + /** + * Test timeout. + */ + timeout?: number + /** + * Times to retry the test if fails. Useful for making flaky tests more stable. + * When retries is up, the last test error will be thrown. + * + * @default 1 + */ + retry?: number +} + export type TestAPI = ChainableTestAPI & { each: TestEachFunction skipIf(condition: any): ChainableTestAPI diff --git a/test/core/test/retry.test.ts b/test/core/test/retry.test.ts new file mode 100644 index 000000000000..3d8da3670cce --- /dev/null +++ b/test/core/test/retry.test.ts @@ -0,0 +1,25 @@ +import { expect, it } from 'vitest' + +let count1 = 0 +it('retry test', () => { + count1 += 1 + expect(count1).toBe(3) +}, { retry: 3 }) + +let count2 = 0 +it.fails('retry test fails', () => { + count2 += 1 + expect(count2).toBe(3) +}, { retry: 2 }) + +let count3 = 0 +it('retry test fails', () => { + count3 += 1 + expect(count3).toBe(3) +}, { retry: 10 }) + +it('result', () => { + expect(count1).toEqual(3) + expect(count2).toEqual(2) + expect(count3).toEqual(3) +})