From c3292cf54c86e349d0a5a30db516a1d717d18263 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 2 Oct 2024 12:29:10 +0900 Subject: [PATCH 1/5] fix(runner): run `onTestFinished` and `onTestFailed` during `retry` and `repeats` --- packages/runner/src/run.ts | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index abcde35cc281..7b26c2d7d30e 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -271,6 +271,26 @@ export async function runTest(test: Test | Custom, runner: VitestRunner): Promis failTask(test.result, e, runner.config.diffOptions) } + try { + await callTaskHooks(test, test.onFinished || [], 'stack') + } + catch (e) { + failTask(test.result, e, runner.config.diffOptions) + } + + if (test.result.state === 'fail') { + try { + await callTaskHooks( + test, + test.onFailed || [], + runner.config.sequence.hooks, + ) + } + catch (e) { + failTask(test.result, e, runner.config.diffOptions) + } + } + if (test.result.state === 'pass') { break } @@ -286,26 +306,6 @@ export async function runTest(test: Test | Custom, runner: VitestRunner): Promis } } - try { - await callTaskHooks(test, test.onFinished || [], 'stack') - } - catch (e) { - failTask(test.result, e, runner.config.diffOptions) - } - - if (test.result.state === 'fail') { - try { - await callTaskHooks( - test, - test.onFailed || [], - runner.config.sequence.hooks, - ) - } - catch (e) { - failTask(test.result, e, runner.config.diffOptions) - } - } - // if test is marked to be failed, flip the result if (test.fails) { if (test.result.state === 'pass') { From 48bfd52f9c47793a23be5859cc7716cff84ae7d4 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 2 Oct 2024 13:44:12 +0900 Subject: [PATCH 2/5] test: wip --- test/core/test/on-finished.test.ts | 159 ++++++++++++++++++++++++++++- 1 file changed, 158 insertions(+), 1 deletion(-) diff --git a/test/core/test/on-finished.test.ts b/test/core/test/on-finished.test.ts index 83d049cf971f..486307b580fd 100644 --- a/test/core/test/on-finished.test.ts +++ b/test/core/test/on-finished.test.ts @@ -1,4 +1,4 @@ -import { expect, it, onTestFinished } from 'vitest' +import { describe, expect, it, onTestFailed, onTestFinished } from 'vitest' const collected: any[] = [] const multiple: any[] = [] @@ -59,3 +59,160 @@ it('after', () => { expect(collected).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) expect(multiple).toEqual([4, 3, 2, 1]) }) + +describe('repeats pass', () => { + const state = new Array() + + it('run', { repeats: 2 }, () => { + state.push(0) + + onTestFinished(() => { + state.push(1) + }) + + onTestFailed(() => { + state.push(2) + }) + }) + + it('assert', () => { + // TODO: 010101 + expect(state).toMatchInlineSnapshot(` + [ + 0, + 1, + 0, + 1, + 1, + 0, + 1, + 1, + 1, + ] + `) + }) +}) + +describe('repeats fail', () => { + const state = new Array() + + it.fails('run', { repeats: 2 }, (t) => { + state.push(0) + + onTestFinished(() => { + state.push(1) + }) + + onTestFailed(() => { + state.push(2) + }) + + if (t.task.result?.repeatCount === 1) { + throw new Error('fail') + } + }) + + it('assert', () => { + // TODO: 012012012 + expect(state).toMatchInlineSnapshot(` + [ + 0, + 1, + 0, + 1, + 1, + 2, + 2, + 0, + 1, + 1, + 1, + 2, + 2, + 2, + ] + `) + }) +}) + +describe('retry pass', () => { + const state = new Array() + + it('run', { retry: 2 }, (t) => { + state.push(0) + + onTestFinished(() => { + state.push(1) + }) + + onTestFailed(() => { + state.push(2) + }) + + if (t.task.result?.retryCount && t.task.result?.retryCount > 1) { + return + } + throw new Error('fail') + }) + + it('assert', () => { + // TODO: 01201201 + expect(state).toMatchInlineSnapshot(` + [ + 0, + 1, + 2, + 0, + 1, + 1, + 2, + 2, + 0, + 1, + 1, + 1, + ] + `) + }) +}) + +describe('retry fail', () => { + const state = new Array() + + it.fails('run', { retry: 2 }, () => { + state.push(0) + + onTestFinished(() => { + state.push(1) + }) + + onTestFailed(() => { + state.push(2) + }) + + throw new Error('fail') + }) + + it('assert', () => { + // TODO: 012012012 + expect(state).toMatchInlineSnapshot(` + [ + 0, + 1, + 2, + 0, + 1, + 1, + 2, + 2, + 0, + 1, + 1, + 1, + 2, + 2, + 2, + ] + `) + }) +}) From 41be7fc18903de9a2c500c78cb9050ea86d2084f Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 2 Oct 2024 13:51:57 +0900 Subject: [PATCH 3/5] test: tweak --- test/core/test/on-finished.test.ts | 132 ++++++++++++++--------------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/test/core/test/on-finished.test.ts b/test/core/test/on-finished.test.ts index 486307b580fd..02b3ba4893b5 100644 --- a/test/core/test/on-finished.test.ts +++ b/test/core/test/on-finished.test.ts @@ -61,17 +61,17 @@ it('after', () => { }) describe('repeats pass', () => { - const state = new Array() + const state: string[] = [] it('run', { repeats: 2 }, () => { - state.push(0) + state.push('run') onTestFinished(() => { - state.push(1) + state.push('finish') }) onTestFailed(() => { - state.push(2) + state.push('fail') }) }) @@ -79,32 +79,32 @@ describe('repeats pass', () => { // TODO: 010101 expect(state).toMatchInlineSnapshot(` [ - 0, - 1, - 0, - 1, - 1, - 0, - 1, - 1, - 1, + "run", + "finish", + "run", + "finish", + "finish", + "run", + "finish", + "finish", + "finish", ] `) }) }) describe('repeats fail', () => { - const state = new Array() + const state: string[] = [] it.fails('run', { repeats: 2 }, (t) => { - state.push(0) + state.push('run') onTestFinished(() => { - state.push(1) + state.push('finish') }) onTestFailed(() => { - state.push(2) + state.push('fail') }) if (t.task.result?.repeatCount === 1) { @@ -116,37 +116,37 @@ describe('repeats fail', () => { // TODO: 012012012 expect(state).toMatchInlineSnapshot(` [ - 0, - 1, - 0, - 1, - 1, - 2, - 2, - 0, - 1, - 1, - 1, - 2, - 2, - 2, + "run", + "finish", + "run", + "finish", + "finish", + "fail", + "fail", + "run", + "finish", + "finish", + "finish", + "fail", + "fail", + "fail", ] `) }) }) describe('retry pass', () => { - const state = new Array() + const state: string[] = [] it('run', { retry: 2 }, (t) => { - state.push(0) + state.push('run') onTestFinished(() => { - state.push(1) + state.push('finish') }) onTestFailed(() => { - state.push(2) + state.push('fail') }) if (t.task.result?.retryCount && t.task.result?.retryCount > 1) { @@ -159,35 +159,35 @@ describe('retry pass', () => { // TODO: 01201201 expect(state).toMatchInlineSnapshot(` [ - 0, - 1, - 2, - 0, - 1, - 1, - 2, - 2, - 0, - 1, - 1, - 1, + "run", + "finish", + "fail", + "run", + "finish", + "finish", + "fail", + "fail", + "run", + "finish", + "finish", + "finish", ] `) }) }) describe('retry fail', () => { - const state = new Array() + const state: string[] = [] it.fails('run', { retry: 2 }, () => { - state.push(0) + state.push('run') onTestFinished(() => { - state.push(1) + state.push('finish') }) onTestFailed(() => { - state.push(2) + state.push('fail') }) throw new Error('fail') @@ -197,21 +197,21 @@ describe('retry fail', () => { // TODO: 012012012 expect(state).toMatchInlineSnapshot(` [ - 0, - 1, - 2, - 0, - 1, - 1, - 2, - 2, - 0, - 1, - 1, - 1, - 2, - 2, - 2, + "run", + "finish", + "fail", + "run", + "finish", + "finish", + "fail", + "fail", + "run", + "finish", + "finish", + "finish", + "fail", + "fail", + "fail", ] `) }) From d4275d7117394e5c77aebdf629e0b7c87dbea502 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 2 Oct 2024 14:02:09 +0900 Subject: [PATCH 4/5] fix: clear hook state on rerun --- packages/runner/src/run.ts | 3 + test/core/test/on-finished.test.ts | 117 ++++++++++++----------------- 2 files changed, 52 insertions(+), 68 deletions(-) diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 7b26c2d7d30e..399931736cd2 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -291,6 +291,9 @@ export async function runTest(test: Test | Custom, runner: VitestRunner): Promis } } + delete test.onFailed + delete test.onFinished + if (test.result.state === 'pass') { break } diff --git a/test/core/test/on-finished.test.ts b/test/core/test/on-finished.test.ts index 02b3ba4893b5..9162bd8464f3 100644 --- a/test/core/test/on-finished.test.ts +++ b/test/core/test/on-finished.test.ts @@ -63,31 +63,28 @@ it('after', () => { describe('repeats pass', () => { const state: string[] = [] - it('run', { repeats: 2 }, () => { - state.push('run') + it('run', { repeats: 2 }, (t) => { + const tag = `(${t.task.result?.retryCount}, ${t.task.result?.repeatCount}) ` + state.push(tag + "run") onTestFinished(() => { - state.push('finish') + state.push(tag + 'finish') }) onTestFailed(() => { - state.push('fail') + state.push(tag + 'fail') }) }) it('assert', () => { - // TODO: 010101 expect(state).toMatchInlineSnapshot(` [ - "run", - "finish", - "run", - "finish", - "finish", - "run", - "finish", - "finish", - "finish", + "(0, 0) run", + "(0, 0) finish", + "(0, 1) run", + "(0, 1) finish", + "(0, 2) run", + "(0, 2) finish", ] `) }) @@ -97,14 +94,15 @@ describe('repeats fail', () => { const state: string[] = [] it.fails('run', { repeats: 2 }, (t) => { - state.push('run') + const tag = `(${t.task.result?.retryCount}, ${t.task.result?.repeatCount}) ` + state.push(tag + "run") onTestFinished(() => { - state.push('finish') + state.push(tag + 'finish') }) onTestFailed(() => { - state.push('fail') + state.push(tag + 'fail') }) if (t.task.result?.repeatCount === 1) { @@ -113,23 +111,16 @@ describe('repeats fail', () => { }) it('assert', () => { - // TODO: 012012012 expect(state).toMatchInlineSnapshot(` [ - "run", - "finish", - "run", - "finish", - "finish", - "fail", - "fail", - "run", - "finish", - "finish", - "finish", - "fail", - "fail", - "fail", + "(0, 0) run", + "(0, 0) finish", + "(0, 1) run", + "(0, 1) finish", + "(0, 1) fail", + "(0, 2) run", + "(0, 2) finish", + "(0, 2) fail", ] `) }) @@ -139,14 +130,15 @@ describe('retry pass', () => { const state: string[] = [] it('run', { retry: 2 }, (t) => { - state.push('run') + const tag = `(${t.task.result?.retryCount}, ${t.task.result?.repeatCount}) ` + state.push(tag + "run") onTestFinished(() => { - state.push('finish') + state.push(tag + 'finish') }) onTestFailed(() => { - state.push('fail') + state.push(tag + 'fail') }) if (t.task.result?.retryCount && t.task.result?.retryCount > 1) { @@ -156,21 +148,16 @@ describe('retry pass', () => { }) it('assert', () => { - // TODO: 01201201 expect(state).toMatchInlineSnapshot(` [ - "run", - "finish", - "fail", - "run", - "finish", - "finish", - "fail", - "fail", - "run", - "finish", - "finish", - "finish", + "(0, 0) run", + "(0, 0) finish", + "(0, 0) fail", + "(1, 0) run", + "(1, 0) finish", + "(1, 0) fail", + "(2, 0) run", + "(2, 0) finish", ] `) }) @@ -179,39 +166,33 @@ describe('retry pass', () => { describe('retry fail', () => { const state: string[] = [] - it.fails('run', { retry: 2 }, () => { - state.push('run') + it.fails('run', { retry: 2 }, (t) => { + const tag = `(${t.task.result?.retryCount}, ${t.task.result?.repeatCount}) ` + state.push(tag + "run") onTestFinished(() => { - state.push('finish') + state.push(tag + 'finish') }) onTestFailed(() => { - state.push('fail') + state.push(tag + 'fail') }) throw new Error('fail') }) it('assert', () => { - // TODO: 012012012 expect(state).toMatchInlineSnapshot(` [ - "run", - "finish", - "fail", - "run", - "finish", - "finish", - "fail", - "fail", - "run", - "finish", - "finish", - "finish", - "fail", - "fail", - "fail", + "(0, 0) run", + "(0, 0) finish", + "(0, 0) fail", + "(1, 0) run", + "(1, 0) finish", + "(1, 0) fail", + "(2, 0) run", + "(2, 0) finish", + "(2, 0) fail", ] `) }) From fd6987bde88536a4d29651546dffc0f20f55fed6 Mon Sep 17 00:00:00 2001 From: Hiroshi Ogawa Date: Wed, 2 Oct 2024 14:06:12 +0900 Subject: [PATCH 5/5] chore: lint --- test/core/test/on-finished.test.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/core/test/on-finished.test.ts b/test/core/test/on-finished.test.ts index 9162bd8464f3..b7ac84fc49f0 100644 --- a/test/core/test/on-finished.test.ts +++ b/test/core/test/on-finished.test.ts @@ -65,14 +65,14 @@ describe('repeats pass', () => { it('run', { repeats: 2 }, (t) => { const tag = `(${t.task.result?.retryCount}, ${t.task.result?.repeatCount}) ` - state.push(tag + "run") + state.push(`${tag}run`) onTestFinished(() => { - state.push(tag + 'finish') + state.push(`${tag}finish`) }) onTestFailed(() => { - state.push(tag + 'fail') + state.push(`${tag}fail`) }) }) @@ -95,14 +95,14 @@ describe('repeats fail', () => { it.fails('run', { repeats: 2 }, (t) => { const tag = `(${t.task.result?.retryCount}, ${t.task.result?.repeatCount}) ` - state.push(tag + "run") + state.push(`${tag}run`) onTestFinished(() => { - state.push(tag + 'finish') + state.push(`${tag}finish`) }) onTestFailed(() => { - state.push(tag + 'fail') + state.push(`${tag}fail`) }) if (t.task.result?.repeatCount === 1) { @@ -131,14 +131,14 @@ describe('retry pass', () => { it('run', { retry: 2 }, (t) => { const tag = `(${t.task.result?.retryCount}, ${t.task.result?.repeatCount}) ` - state.push(tag + "run") + state.push(`${tag}run`) onTestFinished(() => { - state.push(tag + 'finish') + state.push(`${tag}finish`) }) onTestFailed(() => { - state.push(tag + 'fail') + state.push(`${tag}fail`) }) if (t.task.result?.retryCount && t.task.result?.retryCount > 1) { @@ -168,14 +168,14 @@ describe('retry fail', () => { it.fails('run', { retry: 2 }, (t) => { const tag = `(${t.task.result?.retryCount}, ${t.task.result?.repeatCount}) ` - state.push(tag + "run") + state.push(`${tag}run`) onTestFinished(() => { - state.push(tag + 'finish') + state.push(`${tag}finish`) }) onTestFailed(() => { - state.push(tag + 'fail') + state.push(`${tag}fail`) }) throw new Error('fail')