From fc53f56344065cd6bb4c45425484aefc03adff89 Mon Sep 17 00:00:00 2001 From: Vladimir Date: Fri, 31 May 2024 15:09:11 +0200 Subject: [PATCH] feat: allow configuring expect options in the config (#5729) --- docs/config/index.md | 35 +++++++++++++++++++ docs/guide/cli-table.md | 3 ++ packages/vitest/src/integrations/chai/poll.ts | 9 ++++- packages/vitest/src/node/cli/cli-config.ts | 33 +++++++++++++++++ packages/vitest/src/node/config.ts | 2 ++ packages/vitest/src/runtime/runners/test.ts | 7 ++++ packages/vitest/src/types/config.ts | 26 ++++++++++++++ .../fails/concurrent-suite-deadlock.test.ts | 5 ++- .../fails/concurrent-test-deadlock.test.ts | 5 ++- test/cli/fixtures/fails/no-assertions.test.ts | 3 ++ .../fails/test-extend/fixture-error.test.ts | 5 ++- test/cli/fixtures/fails/test-timeout.test.ts | 6 ++-- test/cli/fixtures/fails/unhandled.test.ts | 3 +- test/cli/fixtures/fails/vite.config.ts | 3 ++ .../stacktraces/require-assertions.test.js | 5 +++ test/cli/fixtures/stacktraces/vite.config.ts | 3 ++ .../cli/test/__snapshots__/fails.test.ts.snap | 2 ++ .../__snapshots__/stacktraces.test.ts.snap | 11 ++++++ test/core/test/cli-test.test.ts | 18 ++++++++++ test/core/test/web-worker-node.test.ts | 2 +- 20 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 test/cli/fixtures/fails/no-assertions.test.ts create mode 100644 test/cli/fixtures/stacktraces/require-assertions.test.js diff --git a/docs/config/index.md b/docs/config/index.md index 044547f49821..38c311ebf21a 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -2287,3 +2287,38 @@ If you just need to configure snapshots feature, use [`snapshotFormat`](#snapsho - **Type:** `Partial` Environment variables available on `process.env` and `import.meta.env` during tests. These variables will not be available in the main process (in `globalSetup`, for example). + +### expect + +- **Type:** `ExpectOptions` + +#### expect.requireAssertions + +- **Type:** `boolean` +- **Default:** `false` + +The same as calling [`expect.hasAssertions()`](/api/expect#expect-hasassertions) at the start of every test. This makes sure that no test will pass accidentally. + +::: tip +This only works with Vitest's `expect`. If you use `assert` ot `.should` assertions, they will not count, and your test will fail due to the lack of expect assertions. + +You can change the value of this by calling `vi.setConfig({ expect: { requireAssertions: false } })`. The config will be applied to every subsequent `expect` call until the `vi.resetConfig` is called manually. +::: + +#### expect.poll + +Global configuration options for [`expect.poll`](/api/expect#poll). These are the same options you can pass down to `expect.poll(condition, options)`. + +##### expect.poll.interval + +- **Type:** `number` +- **Default:** `50` + +Polling interval in milliseconds + +##### expect.poll.timeout + +- **Type:** `number` +- **Default:** `1000` + +Polling timeout in milliseconds diff --git a/docs/guide/cli-table.md b/docs/guide/cli-table.md index 0f5f56de05ae..08701222d585 100644 --- a/docs/guide/cli-table.md +++ b/docs/guide/cli-table.md @@ -114,6 +114,9 @@ | `--slowTestThreshold ` | Threshold in milliseconds for a test to be considered slow (default: `300`) | | `--teardownTimeout ` | Default timeout of a teardown function in milliseconds (default: `10000`) | | `--maxConcurrency ` | Maximum number of concurrent tests in a suite (default: `5`) | +| `--expect.requireAssertions` | Require that all tests have at least one assertion | +| `--expect.poll.interval ` | Poll interval in milliseconds for `expect.poll()` assertions (default: `50`) | +| `--expect.poll.timeout ` | Poll timeout in milliseconds for `expect.poll()` assertions (default: `1000`) | | `--run` | Disable watch mode | | `--no-color` | Removes colors from the console output | | `--clearScreen` | Clear terminal screen when re-running tests during watch mode (default: `true`) | diff --git a/packages/vitest/src/integrations/chai/poll.ts b/packages/vitest/src/integrations/chai/poll.ts index 8627e132dcdf..ad0168ee6bcc 100644 --- a/packages/vitest/src/integrations/chai/poll.ts +++ b/packages/vitest/src/integrations/chai/poll.ts @@ -1,6 +1,7 @@ import * as chai from 'chai' import type { ExpectStatic } from '@vitest/expect' import { getSafeTimers } from '@vitest/utils' +import { getWorkerState } from '../../utils' // these matchers are not supported because they don't make sense with poll const unsupported = [ @@ -26,7 +27,13 @@ const unsupported = [ export function createExpectPoll(expect: ExpectStatic): ExpectStatic['poll'] { return function poll(fn, options = {}) { - const { interval = 50, timeout = 1000, message } = options + const state = getWorkerState() + const defaults = state.config.expect?.poll ?? {} + const { + interval = defaults.interval ?? 50, + timeout = defaults.timeout ?? 1000, + message, + } = options // @ts-expect-error private poll access const assertion = expect(null, message).withContext({ poll: true }) as Assertion const proxy: any = new Proxy(assertion, { diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index 6b6e6f28a5ee..e92924629e31 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -585,6 +585,39 @@ export const cliOptionsConfig: VitestCLIOptions = { description: 'Maximum number of concurrent tests in a suite (default: `5`)', argument: '', }, + expect: { + description: 'Configuration options for `expect()` matches', + argument: '', // no displayed + subcommands: { + requireAssertions: { + description: 'Require that all tests have at least one assertion', + }, + poll: { + description: 'Default options for `expect.poll()`', + argument: '', + subcommands: { + interval: { + description: 'Poll interval in milliseconds for `expect.poll()` assertions (default: `50`)', + argument: '', + }, + timeout: { + description: 'Poll timeout in milliseconds for `expect.poll()` assertions (default: `1000`)', + argument: '', + }, + }, + transform(value) { + if (typeof value !== 'object') + throw new Error(`Unexpected value for --expect.poll: ${value}. If you need to configure timeout, use --expect.poll.timeout=`) + return value + }, + }, + }, + transform(value) { + if (typeof value !== 'object') + throw new Error(`Unexpected value for --expect: ${value}. If you need to configure expect options, use --expect.{name}= syntax`) + return value + }, + }, // CLI only options run: { diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index ee5704bd4280..fd3dd0119c45 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -179,6 +179,8 @@ export function resolveConfig( throw new Error(`You cannot set "coverage.reportsDirectory" as ${reportsDirectory}. Vitest needs to be able to remove this directory before test run`) } + resolved.expect ??= {} + resolved.deps ??= {} resolved.deps.moduleDirectories ??= [] resolved.deps.moduleDirectories = resolved.deps.moduleDirectories.map((dir) => { diff --git a/packages/vitest/src/runtime/runners/test.ts b/packages/vitest/src/runtime/runners/test.ts index 73c9f0c7d8e7..70cbb06c78da 100644 --- a/packages/vitest/src/runtime/runners/test.ts +++ b/packages/vitest/src/runtime/runners/test.ts @@ -15,6 +15,8 @@ export class VitestTestRunner implements VitestRunner { private __vitest_executor!: VitestExecutor private cancelRun = false + private assertionsErrors = new WeakMap, Error>() + constructor(public config: ResolvedConfig) {} importFile(filepath: string, source: VitestRunnerImportSource): unknown { @@ -123,9 +125,14 @@ export class VitestTestRunner implements VitestRunner { throw expectedAssertionsNumberErrorGen!() if (isExpectingAssertions === true && assertionCalls === 0) throw isExpectingAssertionsError + if (this.config.expect.requireAssertions && assertionCalls === 0) + throw this.assertionsErrors.get(test) } extendTaskContext(context: TaskContext): ExtendedContext { + // create error during the test initialization so we have a nice stack trace + if (this.config.expect.requireAssertions) + this.assertionsErrors.set(context.task, new Error('expected any number of assertion, but got none')) let _expect: ExpectStatic | undefined Object.defineProperty(context, 'expect', { get() { diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index f3929bfe43c4..7a2cee668927 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -706,6 +706,31 @@ export interface InlineConfig { waitForDebugger?: boolean } + /** + * Configuration options for expect() matches. + */ + expect?: { + /** + * Throw an error if tests don't have any expect() assertions. + */ + requireAssertions?: boolean + /** + * Default options for expect.poll() + */ + poll?: { + /** + * Timeout in milliseconds + * @default 1000 + */ + timeout?: number + /** + * Polling interval in milliseconds + * @default 50 + */ + interval?: number + } + } + /** * Modify default Chai config. Vitest uses Chai for `expect` and `assert` matches. * https://github.com/chaijs/chai/blob/4.x.x/lib/chai/config.js @@ -974,6 +999,7 @@ export type RuntimeConfig = Pick< | 'restoreMocks' | 'fakeTimers' | 'maxConcurrency' + | 'expect' > & { sequence?: { concurrent?: boolean diff --git a/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts b/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts index 0c57d449dde4..4ab2a9001ccc 100644 --- a/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts +++ b/test/cli/fixtures/fails/concurrent-suite-deadlock.test.ts @@ -1,5 +1,5 @@ import { createDefer } from '@vitest/utils' -import { describe, test, vi } from 'vitest' +import { describe, test, vi, expect } from 'vitest' // 3 tests depend on each other, // so they will deadlock when maxConcurrency < 3 @@ -21,11 +21,13 @@ describe('wrapper', { concurrent: true, timeout: 500 }, () => { describe('1st suite', () => { test('a', async () => { + expect(1).toBe(1) defers[0].resolve() await defers[2] }) test('b', async () => { + expect(1).toBe(1) await defers[0] defers[1].resolve() await defers[2] @@ -34,6 +36,7 @@ describe('wrapper', { concurrent: true, timeout: 500 }, () => { describe('2nd suite', () => { test('c', async () => { + expect(1).toBe(1) await defers[1] defers[2].resolve() }) diff --git a/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts b/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts index b56fd442dcdc..aa1dc763eaef 100644 --- a/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts +++ b/test/cli/fixtures/fails/concurrent-test-deadlock.test.ts @@ -1,4 +1,4 @@ -import { describe, test, vi } from 'vitest' +import { describe, expect, test, vi } from 'vitest' import { createDefer } from '@vitest/utils' // 3 tests depend on each other, @@ -20,17 +20,20 @@ describe('wrapper', { concurrent: true, timeout: 500 }, () => { ] test('a', async () => { + expect(1).toBe(1) defers[0].resolve() await defers[2] }) test('b', async () => { + expect(1).toBe(1) await defers[0] defers[1].resolve() await defers[2] }) test('c', async () => { + expect(1).toBe(1) await defers[1] defers[2].resolve() }) diff --git a/test/cli/fixtures/fails/no-assertions.test.ts b/test/cli/fixtures/fails/no-assertions.test.ts new file mode 100644 index 000000000000..2536624ce713 --- /dev/null +++ b/test/cli/fixtures/fails/no-assertions.test.ts @@ -0,0 +1,3 @@ +import { it } from 'vitest' + +it('test without assertions') \ No newline at end of file diff --git a/test/cli/fixtures/fails/test-extend/fixture-error.test.ts b/test/cli/fixtures/fails/test-extend/fixture-error.test.ts index 2ec15cf73b8f..1ad54b53907c 100644 --- a/test/cli/fixtures/fails/test-extend/fixture-error.test.ts +++ b/test/cli/fixtures/fails/test-extend/fixture-error.test.ts @@ -10,7 +10,9 @@ describe('error thrown in beforeEach fixtures', () => { // eslint-disable-next-line unused-imports/no-unused-vars beforeEach<{ a: never }>(({ a }) => {}) - myTest('error is handled', () => {}) + myTest('error is handled', () => { + expect(1).toBe(1) + }) }) describe('error thrown in afterEach fixtures', () => { @@ -24,6 +26,7 @@ describe('error thrown in afterEach fixtures', () => { afterEach<{ a: never }>(({ a }) => {}) myTest('fixture errors', () => { + expect(1).toBe(1) expectTypeOf(1).toEqualTypeOf() }) }) diff --git a/test/cli/fixtures/fails/test-timeout.test.ts b/test/cli/fixtures/fails/test-timeout.test.ts index f67add79b8b4..d06123b55778 100644 --- a/test/cli/fixtures/fails/test-timeout.test.ts +++ b/test/cli/fixtures/fails/test-timeout.test.ts @@ -4,12 +4,12 @@ test('hi', async () => { await new Promise(resolve => setTimeout(resolve, 1000)) }, 10) -suite('suite timeout', () => { +suite('suite timeout', { + timeout: 100, +}, () => { test('hi', async () => { await new Promise(resolve => setTimeout(resolve, 500)) }) -}, { - timeout: 100, }) suite('suite timeout simple input', () => { diff --git a/test/cli/fixtures/fails/unhandled.test.ts b/test/cli/fixtures/fails/unhandled.test.ts index 00276381eceb..c517adb9fc88 100644 --- a/test/cli/fixtures/fails/unhandled.test.ts +++ b/test/cli/fixtures/fails/unhandled.test.ts @@ -1,8 +1,9 @@ // @vitest-environment jsdom -import { test } from 'vitest' +import { expect, test } from 'vitest' test('unhandled exception', () => { + expect(1).toBe(1) addEventListener('custom', () => { throw new Error('some error') }) diff --git a/test/cli/fixtures/fails/vite.config.ts b/test/cli/fixtures/fails/vite.config.ts index c67d3c311423..0c76012368bb 100644 --- a/test/cli/fixtures/fails/vite.config.ts +++ b/test/cli/fixtures/fails/vite.config.ts @@ -8,5 +8,8 @@ export default defineConfig({ isolate: false, }, }, + expect: { + requireAssertions: true, + } }, }) diff --git a/test/cli/fixtures/stacktraces/require-assertions.test.js b/test/cli/fixtures/stacktraces/require-assertions.test.js new file mode 100644 index 000000000000..c30da6a8dfea --- /dev/null +++ b/test/cli/fixtures/stacktraces/require-assertions.test.js @@ -0,0 +1,5 @@ +import { test } from 'vitest' + +test('assertion is not called', () => { + // no expect +}) diff --git a/test/cli/fixtures/stacktraces/vite.config.ts b/test/cli/fixtures/stacktraces/vite.config.ts index 64d31c38f6d1..7938ce782b6c 100644 --- a/test/cli/fixtures/stacktraces/vite.config.ts +++ b/test/cli/fixtures/stacktraces/vite.config.ts @@ -45,5 +45,8 @@ export default defineConfig({ pool: 'forks', include: ['**/*.{test,spec}.{imba,?(c|m)[jt]s?(x)}'], setupFiles: ['./setup.js'], + expect: { + requireAssertions: true, + }, }, }) diff --git a/test/cli/test/__snapshots__/fails.test.ts.snap b/test/cli/test/__snapshots__/fails.test.ts.snap index 9dbbe710ceb9..5ee6a3c42c5a 100644 --- a/test/cli/test/__snapshots__/fails.test.ts.snap +++ b/test/cli/test/__snapshots__/fails.test.ts.snap @@ -35,6 +35,8 @@ exports[`should fail mock-import-proxy-module.test.ts > mock-import-proxy-module exports[`should fail nested-suite.test.ts > nested-suite.test.ts 1`] = `"AssertionError: expected true to be false // Object.is equality"`; +exports[`should fail no-assertions.test.ts > no-assertions.test.ts 1`] = `"Error: expected any number of assertion, but got none"`; + exports[`should fail primitive-error.test.ts > primitive-error.test.ts 1`] = `"Unknown Error: 42"`; exports[`should fail snapshot-with-not.test.ts > snapshot-with-not.test.ts 1`] = ` diff --git a/test/cli/test/__snapshots__/stacktraces.test.ts.snap b/test/cli/test/__snapshots__/stacktraces.test.ts.snap index 91a73a22ac5c..fc131c7c44eb 100644 --- a/test/cli/test/__snapshots__/stacktraces.test.ts.snap +++ b/test/cli/test/__snapshots__/stacktraces.test.ts.snap @@ -158,6 +158,17 @@ exports[`stacktraces should respect sourcemaps > mocked-imported.test.ts > mocke " `; +exports[`stacktraces should respect sourcemaps > require-assertions.test.js > require-assertions.test.js 1`] = ` +" ❯ require-assertions.test.js:3:1 + 1| import { test } from 'vitest' + 2| + 3| test('assertion is not called', () => { + | ^ + 4| // no expect + 5| }) +" +`; + exports[`stacktraces should respect sourcemaps > reset-modules.test.ts > reset-modules.test.ts 1`] = ` " ❯ reset-modules.test.ts:16:26 14| expect(2 + 1).eq(3) diff --git a/test/core/test/cli-test.test.ts b/test/core/test/cli-test.test.ts index 0de6b9437469..bf2bcbdaf727 100644 --- a/test/core/test/cli-test.test.ts +++ b/test/core/test/cli-test.test.ts @@ -316,6 +316,24 @@ test('merge-reports', () => { expect(getCLIOptions('--merge-reports different-folder')).toEqual({ mergeReports: 'different-folder' }) }) +test('configure expect', () => { + expect(() => getCLIOptions('vitest --expect.poll=1000')).toThrowErrorMatchingInlineSnapshot(`[Error: Unexpected value for --expect.poll: true. If you need to configure timeout, use --expect.poll.timeout=]`) + expect(() => getCLIOptions('vitest --expect=1000')).toThrowErrorMatchingInlineSnapshot(`[Error: Unexpected value for --expect: true. If you need to configure expect options, use --expect.{name}= syntax]`) + expect(getCLIOptions('vitest --expect.poll.interval=100 --expect.poll.timeout=300')).toEqual({ + expect: { + poll: { + interval: 100, + timeout: 300, + }, + }, + }) + expect(getCLIOptions('vitest --expect.requireAssertions')).toEqual({ + expect: { + requireAssertions: true, + }, + }) +}) + test('public parseCLI works correctly', () => { expect(parseCLI('vitest dev')).toEqual({ filter: [], diff --git a/test/core/test/web-worker-node.test.ts b/test/core/test/web-worker-node.test.ts index 03b498c94852..403bd9a7bbcd 100644 --- a/test/core/test/web-worker-node.test.ts +++ b/test/core/test/web-worker-node.test.ts @@ -263,7 +263,7 @@ it('doesn\'t trigger events, if closed', async () => { worker.port.close() await new Promise((resolve) => { worker.port.addEventListener('message', () => { - expect.fail('should not trigger message') + expect.unreachable('should not trigger message') }) worker.port.postMessage('event') setTimeout(resolve, 100)