Skip to content

Commit

Permalink
feat(config): add shuffle.files and shuffle.tests options (#5281)
Browse files Browse the repository at this point in the history
  • Loading branch information
fenghan34 authored Mar 14, 2024
1 parent 201bd06 commit 356db87
Show file tree
Hide file tree
Showing 6 changed files with 104 additions and 6 deletions.
22 changes: 19 additions & 3 deletions docs/config/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -1768,13 +1768,29 @@ Sharding is happening before sorting, and only if `--shard` option is provided.
#### sequence.shuffle
- **Type**: `boolean`
- **Type**: `boolean | { files?, tests? }`
- **Default**: `false`
- **CLI**: `--sequence.shuffle`, `--sequence.shuffle=false`
If you want tests to run randomly, you can enable it with this option, or CLI argument [`--sequence.shuffle`](/guide/cli).
If you want files and tests to run randomly, you can enable it with this option, or CLI argument [`--sequence.shuffle`](/guide/cli).
Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your files and tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously.
#### sequence.shuffle.files <Badge type="info">1.4.0+</Badge> {#sequence-shuffle-files}
- **Type**: `boolean`
- **Default**: `false`
- **CLI**: `--sequence.shuffle.files`, `--sequence.shuffle.files=false`
Whether to randomize files, be aware that long running tests will not start earlier if you enable this option.
#### sequence.shuffle.tests <Badge type="info">1.4.0+</Badge> {#sequence-shuffle-tests}
- **Type**: `boolean`
- **Default**: `false`
- **CLI**: `--sequence.shuffle.tests`, `--sequence.shuffle.tests=false`
Vitest usually uses cache to sort tests, so long running tests start earlier - this makes tests run faster. If your tests will run in random order you will lose this performance improvement, but it may be useful to track tests that accidentally depend on another run previously.
Whether to randomize tests.
#### sequence.concurrent <Badge type="info">0.32.2+</Badge> {#sequence-concurrent}
Expand Down
11 changes: 10 additions & 1 deletion packages/vitest/src/node/cli/cli-config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,16 @@ export const cliOptionsConfig: VitestCLIOptions = {
argument: '<options>',
subcommands: {
shuffle: {
description: 'Run tests in a random order. Enabling this option will impact Vitest\'s cache and have a performance impact. May be useful to find tests that accidentally depend on another run previously (default: false)',
description: 'Run files and tests in a random order. Enabling this option will impact Vitest\'s cache and have a performance impact. May be useful to find tests that accidentally depend on another run previously (default: false)',
argument: '',
subcommands: {
files: {
description: 'Run files in a random order. Long running tests will not start earlier if you enable this option. (default: false)',
},
tests: {
description: 'Run tests in a random oder (default: false)',
},
},
},
concurrent: {
description: 'Make tests run in parallel (default: false)',
Expand Down
5 changes: 5 additions & 0 deletions packages/vitest/src/node/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,11 @@ export function resolveConfig(
resolved.cache.dir = VitestCache.resolveCacheDir(resolved.root, resolved.cache.dir, resolved.name)

resolved.sequence ??= {} as any
if (resolved.sequence.shuffle && typeof resolved.sequence.shuffle === 'object') {
const { files, tests } = resolved.sequence.shuffle
resolved.sequence.sequencer ??= files ? RandomSequencer : BaseSequencer
resolved.sequence.shuffle = tests
}
if (!resolved.sequence?.sequencer) {
// CLI flag has higher priority
resolved.sequence.sequencer = resolved.sequence.shuffle
Expand Down
16 changes: 14 additions & 2 deletions packages/vitest/src/types/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,22 @@ interface SequenceOptions {
*/
sequencer?: TestSequencerConstructor
/**
* Should tests run in random order.
* Should files and tests run in random order.
* @default false
*/
shuffle?: boolean
shuffle?: boolean | {
/**
* Should files run in random order. Long running tests will not start
* earlier if you enable this option.
* @default false
*/
files?: boolean
/**
* Should tests run in random order.
* @default false
*/
tests?: boolean
}
/**
* Should tests run in parallel.
* @default false
Expand Down
49 changes: 49 additions & 0 deletions test/config/test/shuffle-options.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import type { InlineConfig } from 'vitest'
import { expect, test } from 'vitest'
import { runVitest } from '../../test-utils'

function run(sequence: InlineConfig['sequence']) {
return runVitest({
sequence,
include: [],
})
}

class CustomSequencer {
sort() {
return []
}

shard() {
return []
}
}

test.each([
false,
{ files: false, tests: false },
{ files: false, tests: true },
],
)('should use BaseSequencer if shuffle is %o', async (shuffle) => {
const { vitest } = await run({ shuffle })
expect(vitest?.config.sequence.sequencer.name).toBe('BaseSequencer')
})

test.each([
true,
{ files: true, tests: false },
{ files: true, tests: true },
])('should use RandomSequencer if shuffle is %o', async (shuffle) => {
const { vitest } = await run({ shuffle })
expect(vitest?.config.sequence.sequencer.name).toBe('RandomSequencer')
})

test.each([
false,
true,
{ files: true, tests: false },
{ files: true, tests: true },
])('should always use CustomSequencer if passed', async (shuffle) => {
const { vitest } = await run({ shuffle, sequencer: CustomSequencer })
expect(vitest?.config.sequence.sequencer.name).toBe('CustomSequencer')
})
7 changes: 7 additions & 0 deletions test/core/test/cli-test.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,13 @@ test('cache is parsed correctly', () => {
})
})

test('shuffle is parsed correctly', () => {
expect(getCLIOptions('--sequence.shuffle')).toEqual({ sequence: { shuffle: true } })
expect(getCLIOptions('--sequence.shuffle=false')).toEqual({ sequence: { shuffle: false } })
expect(getCLIOptions('--sequence.shuffle.files --sequence.shuffle.tests')).toEqual({ sequence: { shuffle: { files: true, tests: true } } })
expect(getCLIOptions('--sequence.shuffle.files=false --sequence.shuffle.tests=false')).toEqual({ sequence: { shuffle: { files: false, tests: false } } })
})

test('typecheck correctly passes down arguments', () => {
const { options, args } = parseArguments('--typecheck some.name.ts')
expect(options).toEqual({ typecheck: { enabled: true } })
Expand Down

0 comments on commit 356db87

Please sign in to comment.