feat(runner): add "queued" state (#6931)

Co-authored-by: Ari Perkkiö <ari.perkkio@gmail.com>

packages/browser/src/client/tester/runner.ts

@@ -104,6 +104,10 @@ export function createBrowserRunner(
       }
     }
 
+    onCollectStart = (file: File) => {
+      return rpc().onQueued(file)
+    }
+
     onCollected = async (files: File[]): Promise<unknown> => {
       files.forEach((file) => {
         file.prepareDuration = state.durations.prepare

packages/browser/src/node/rpc.ts

@@ -1,5 +1,5 @@
 import type { ErrorWithDiff } from 'vitest'
-import type { BrowserCommandContext, ResolveSnapshotPathHandlerContext } from 'vitest/node'
+import type { BrowserCommandContext, ResolveSnapshotPathHandlerContext, TestModule } from 'vitest/node'
 import type { WebSocket } from 'ws'
 import type { BrowserServer } from './server'
 import type { WebSocketBrowserEvents, WebSocketBrowserHandlers } from './types'
@@ -75,6 +75,11 @@ export function setupBrowserRpc(server: BrowserServer) {
           }
           ctx.state.catchError(error, type)
         },
+        async onQueued(file) {
+          ctx.state.collectFiles(project, [file])
+          const testModule = ctx.state.getReportedEntity(file) as TestModule
+          await ctx.report('onTestModuleQueued', testModule)
+        },
         async onCollected(files) {
           ctx.state.collectFiles(project, files)
           await ctx.report('onCollected', files)

packages/browser/src/node/types.ts

@@ -6,6 +6,7 @@ export interface WebSocketBrowserHandlers {
   resolveSnapshotPath: (testPath: string) => string
   resolveSnapshotRawPath: (testPath: string, rawPath: string) => string
   onUnhandledError: (error: unknown, type: string) => Promise<void>
+  onQueued: (file: RunnerTestFile) => void
   onCollected: (files?: RunnerTestFile[]) => Promise<void>
   onTaskUpdate: (packs: TaskResultPack[]) => void
   onAfterSuiteRun: (meta: AfterSuiteRunMeta) => void

packages/runner/src/collect.ts

@@ -107,6 +107,10 @@ export async function collectTests(
       config.allowOnly,
     )
 
+    if (file.mode === 'queued') {
+      file.mode = 'run'
+    }
+
     files.push(file)
   }
 

packages/runner/src/run.ts

@@ -196,7 +196,7 @@ async function callCleanupHooks(cleanups: HookCleanupCallback[]) {
 export async function runTest(test: Test, runner: VitestRunner): Promise<void> {
   await runner.onBeforeRunTask?.(test)
 
-  if (test.mode !== 'run') {
+  if (test.mode !== 'run' && test.mode !== 'queued') {
     return
   }
 
@@ -458,7 +458,7 @@ export async function runSuite(suite: Suite, runner: VitestRunner): Promise<void
       failTask(suite.result, e, runner.config.diffOptions)
     }
 
-    if (suite.mode === 'run') {
+    if (suite.mode === 'run' || suite.mode === 'queued') {
       if (!runner.config.passWithNoTests && !hasTests(suite)) {
         suite.result.state = 'fail'
         if (!suite.result.errors?.length) {

packages/runner/src/types/tasks.ts

@@ -2,7 +2,7 @@ import type { Awaitable, ErrorWithDiff } from '@vitest/utils'
 import type { FixtureItem } from '../fixture'
 import type { ChainableFunction } from '../utils/chain'
 
-export type RunMode = 'run' | 'skip' | 'only' | 'todo'
+export type RunMode = 'run' | 'skip' | 'only' | 'todo' | 'queued'
 export type TaskState = RunMode | 'pass' | 'fail'
 
 export interface TaskBase {
@@ -23,6 +23,7 @@ export interface TaskBase {
    * - **only**: only this task and other tasks with `only` mode will run
    * - **todo**: task is marked as a todo, alias for `skip`
    * - **run**: task will run or already ran
+   * - **queued**: task will start running next. It can only exist on the File
    */
   mode: RunMode
   /**

packages/runner/src/utils/collect.ts

@@ -72,8 +72,8 @@ export function interpretTaskModes(
     })
 
     // if all subtasks are skipped, mark as skip
-    if (suite.mode === 'run') {
-      if (suite.tasks.length && suite.tasks.every(i => i.mode !== 'run')) {
+    if (suite.mode === 'run' || suite.mode === 'queued') {
+      if (suite.tasks.length && suite.tasks.every(i => i.mode !== 'run' && i.mode !== 'queued')) {
         suite.mode = 'skip'
       }
     }
@@ -115,7 +115,7 @@ export function someTasksAreOnly(suite: Suite): boolean {
 
 function skipAllTasks(suite: Suite) {
   suite.tasks.forEach((t) => {
-    if (t.mode === 'run') {
+    if (t.mode === 'run' || t.mode === 'queued') {
       t.mode = 'skip'
       if (t.type === 'suite') {
         skipAllTasks(t)
@@ -172,7 +172,7 @@ export function createFileTask(
     id: generateFileHash(path, projectName),
     name: path,
     type: 'suite',
-    mode: 'run',
+    mode: 'queued',
     filepath,
     tasks: [],
     meta: Object.create(null),

packages/vitest/src/node/pools/rpc.ts

@@ -1,6 +1,7 @@
 import type { RawSourceMap } from 'vite-node'
 import type { RuntimeRPC } from '../../types/rpc'
 import type { TestProject } from '../project'
+import type { TestModule } from '../reporters/reported-tasks'
 import type { ResolveSnapshotPathHandlerContext } from '../types/config'
 import { mkdir, writeFile } from 'node:fs/promises'
 import { join } from 'pathe'
@@ -78,6 +79,11 @@ export function createMethodsRPC(project: TestProject, options: MethodsOptions =
       ctx.state.collectPaths(paths)
       return ctx.report('onPathsCollected', paths)
     },
+    onQueued(file) {
+      ctx.state.collectFiles(project, [file])
+      const testModule = ctx.state.getReportedEntity(file) as TestModule
+      return ctx.report('onTestModuleQueued', testModule)
+    },
     onCollected(files) {
       ctx.state.collectFiles(project, files)
       return ctx.report('onCollected', files)

packages/vitest/src/node/reporters/base.ts

@@ -75,7 +75,8 @@ export abstract class BaseReporter implements Reporter {
     if (
       !('filepath' in task)
       || !task.result?.state
-      || task.result?.state === 'run') {
+      || task.result?.state === 'run'
+      || task.result?.state === 'queued') {
       return
     }
 

packages/vitest/src/node/reporters/benchmark/table/index.ts

@@ -76,12 +76,13 @@ export class TableReporter extends BaseReporter {
         && task.type === 'suite'
         && task.result?.state
         && task.result?.state !== 'run'
+        && task.result?.state !== 'queued'
       ) {
         // render static table when all benches inside single suite are finished
         const benches = task.tasks.filter(t => t.meta.benchmark)
         if (
           benches.length > 0
-          && benches.every(t => t.result?.state !== 'run')
+          && benches.every(t => t.result?.state !== 'run' && t.result?.state !== 'queued')
         ) {
           let title = ` ${getStateSymbol(task)} ${getFullName(
             task,

packages/vitest/src/node/reporters/default.ts

@@ -1,6 +1,7 @@
 import type { File, TaskResultPack } from '@vitest/runner'
 import type { Vitest } from '../core'
 import type { BaseOptions } from './base'
+import type { TestModule } from './reported-tasks'
 import { BaseReporter } from './base'
 import { SummaryReporter } from './summary'
 
@@ -28,6 +29,10 @@ export class DefaultReporter extends BaseReporter {
     }
   }
 
+  onTestModuleQueued(file: TestModule) {
+    this.summary?.onTestModuleQueued(file)
+  }
+
   onInit(ctx: Vitest) {
     super.onInit(ctx)
     this.summary?.onInit(ctx, { verbose: this.verbose })

packages/vitest/src/node/reporters/json.ts

@@ -26,6 +26,7 @@ const StatusMap: Record<TaskState, Status> = {
   run: 'pending',
   skip: 'skipped',
   todo: 'todo',
+  queued: 'pending',
 }
 
 export interface JsonAssertionResult {
@@ -95,7 +96,7 @@ export class JsonReporter implements Reporter {
 
     const numFailedTestSuites = suites.filter(s => s.result?.state === 'fail').length
     const numPendingTestSuites = suites.filter(
-      s => s.result?.state === 'run' || s.mode === 'todo',
+      s => s.result?.state === 'run' || s.result?.state === 'queued' || s.mode === 'todo',
     ).length
     const numPassedTestSuites = numTotalTestSuites - numFailedTestSuites - numPendingTestSuites
 
@@ -104,7 +105,7 @@ export class JsonReporter implements Reporter {
     ).length
     const numPassedTests = tests.filter(t => t.result?.state === 'pass').length
     const numPendingTests = tests.filter(
-      t => t.result?.state === 'run' || t.mode === 'skip' || t.result?.state === 'skip',
+      t => t.result?.state === 'run' || t.result?.state === 'queued' || t.mode === 'skip' || t.result?.state === 'skip',
     ).length
     const numTodoTests = tests.filter(t => t.mode === 'todo').length
     const testResults: Array<JsonTestResult> = []
@@ -154,7 +155,7 @@ export class JsonReporter implements Reporter {
         } satisfies JsonAssertionResult
       })
 
-      if (tests.some(t => t.result?.state === 'run')) {
+      if (tests.some(t => t.result?.state === 'run' || t.result?.state === 'queued')) {
         this.ctx.logger.warn(
           'WARNING: Some tests are still running when generating the JSON report.'
           + 'This is likely an internal bug in Vitest.'

packages/vitest/src/node/reporters/renderers/utils.ts

@@ -163,7 +163,7 @@ export function getStateSymbol(task: Task) {
     return pending
   }
 
-  if (task.result.state === 'run') {
+  if (task.result.state === 'run' || task.result.state === 'queued') {
     if (task.type === 'suite') {
       return pointer
     }

packages/vitest/src/node/reporters/reported-tasks.ts

@@ -126,7 +126,7 @@ export class TestCase extends ReportedTaskImplementation {
    */
   public result(): TestResult | undefined {
     const result = this.task.result
-    if (!result || result.state === 'run') {
+    if (!result || result.state === 'run' || result.state === 'queued') {
       return undefined
     }
     const state = result.state === 'fail'
@@ -175,7 +175,7 @@ export class TestCase extends ReportedTaskImplementation {
   public diagnostic(): TestDiagnostic | undefined {
     const result = this.task.result
     // startTime should always be available if the test has properly finished
-    if (!result || result.state === 'run' || !result.startTime) {
+    if (!result || result.state === 'run' || result.state === 'queued' || !result.startTime) {
       return undefined
     }
     const duration = result.duration || 0
@@ -450,7 +450,7 @@ export interface TaskOptions {
   shuffle: boolean | undefined
   retry: number | undefined
   repeats: number | undefined
-  mode: 'run' | 'only' | 'skip' | 'todo'
+  mode: 'run' | 'only' | 'skip' | 'todo' | 'queued'
 }
 
 function buildOptions(

packages/vitest/src/node/reporters/summary.ts

@@ -1,6 +1,7 @@
 import type { File, Test } from '@vitest/runner'
 import type { Vitest } from '../core'
 import type { Reporter } from '../types/reporter'
+import type { TestModule } from './reported-tasks'
 import type { HookOptions } from './task-parser'
 import { getTests } from '@vitest/runner/utils'
 import c from 'tinyrainbow'
@@ -87,6 +88,10 @@ export class SummaryReporter extends TaskParser implements Reporter {
     })
   }
 
+  onTestModuleQueued(module: TestModule) {
+    this.onTestFilePrepare(module.task)
+  }
+
   onPathsCollected(paths?: string[]) {
     this.suites.total = (paths || []).length
   }
@@ -111,7 +116,18 @@ export class SummaryReporter extends TaskParser implements Reporter {
   }
 
   onTestFilePrepare(file: File) {
-    if (this.allFinishedTests.has(file.id) || this.runningTests.has(file.id)) {
+    if (this.runningTests.has(file.id)) {
+      const stats = this.runningTests.get(file.id)!
+      // if there are no tests, it means the test was queued but not collected
+      if (!stats.total) {
+        const total = getTests(file).length
+        this.tests.total += total
+        stats.total = total
+      }
+      return
+    }
+
+    if (this.allFinishedTests.has(file.id)) {
       return
     }
 
@@ -266,7 +282,7 @@ export class SummaryReporter extends TaskParser implements Reporter {
     const file = test.file
     let stats = this.runningTests.get(file.id)
 
-    if (!stats) {
+    if (!stats || stats.total === 0) {
       // It's possible that that test finished before it's preparation was even reported
       this.onTestFilePrepare(test.file)
       stats = this.runningTests.get(file.id)!
@@ -303,7 +319,9 @@ export class SummaryReporter extends TaskParser implements Reporter {
         c.bold(c.yellow(` ${F_POINTER} `))
         + formatProjectName(testFile.projectName)
         + testFile.filename
-        + c.dim(` ${testFile.completed}/${testFile.total}`),
+        + c.dim(!testFile.completed && !testFile.total
+          ? ' [queued]'
+          : ` ${testFile.completed}/${testFile.total}`),
       )
 
       const slowTasks = [

packages/vitest/src/node/reporters/task-parser.ts

@@ -39,7 +39,7 @@ export class TaskParser {
       const task = this.ctx.state.idMap.get(pack[0])
 
       if (task?.type === 'suite' && 'filepath' in task && task.result?.state) {
-        if (task?.result?.state === 'run') {
+        if (task?.result?.state === 'run' || task?.result?.state === 'queued') {
           startingTestFiles.push(task)
         }
         else {
@@ -55,7 +55,7 @@ export class TaskParser {
       }
 
       if (task?.type === 'test') {
-        if (task.result?.state === 'run') {
+        if (task.result?.state === 'run' || task.result?.state === 'queued') {
           startingTests.push(task)
         }
         else if (task.result?.hooks?.afterEach !== 'run') {
@@ -65,7 +65,7 @@ export class TaskParser {
 
       if (task?.result?.hooks) {
         for (const [hook, state] of Object.entries(task.result.hooks)) {
-          if (state === 'run') {
+          if (state === 'run' || state === 'queued') {
             startingHooks.push({ name: hook, file: task.file, id: task.id, type: task.type })
           }
           else {
@@ -81,7 +81,6 @@ export class TaskParser {
 
     startingTestFiles.forEach(file => this.onTestFilePrepare(file))
     startingTests.forEach(test => this.onTestStart(test))
-    startingHooks.forEach(hook => this.onHookStart(hook),
-    )
+    startingHooks.forEach(hook => this.onHookStart(hook))
   }
 }

packages/vitest/src/node/reporters/verbose.ts

@@ -20,6 +20,7 @@ export class VerboseReporter extends DefaultReporter {
         && task.type === 'test'
         && task.result?.state
         && task.result?.state !== 'run'
+        && task.result?.state !== 'queued'
       ) {
         let title = ` ${getStateSymbol(task)} `
         if (task.file.projectName) {

packages/vitest/src/node/types/reporter.ts

@@ -2,11 +2,13 @@ import type { File, TaskResultPack } from '@vitest/runner'
 import type { SerializedTestSpecification } from '../../runtime/types/utils'
 import type { Awaitable, UserConsoleLog } from '../../types/general'
 import type { Vitest } from '../core'
+import type { TestModule } from '../reporters/reported-tasks'
 
 export interface Reporter {
   onInit?: (ctx: Vitest) => void
   onPathsCollected?: (paths?: string[]) => Awaitable<void>
   onSpecsCollected?: (specs?: SerializedTestSpecification[]) => Awaitable<void>
+  onTestModuleQueued?: (file: TestModule) => Awaitable<void>
   onCollected?: (files?: File[]) => Awaitable<void>
   onFinished?: (
     files: File[],

packages/vitest/src/runtime/runners/benchmark.ts

@@ -35,7 +35,7 @@ async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) {
   const benchmarkGroup: Benchmark[] = []
   const benchmarkSuiteGroup = []
   for (const task of suite.tasks) {
-    if (task.mode !== 'run') {
+    if (task.mode !== 'run' && task.mode !== 'queued') {
       continue
     }
 

packages/vitest/src/runtime/runners/index.ts

@@ -68,6 +68,12 @@ export async function resolveTestRunner(
     return p
   }
 
+  const originalOnCollectStart = testRunner.onCollectStart
+  testRunner.onCollectStart = async (file) => {
+    await rpc().onQueued(file)
+    await originalOnCollectStart?.call(testRunner, file)
+  }
+
   const originalOnCollected = testRunner.onCollected
   testRunner.onCollected = async (files) => {
     const state = getWorkerState()

packages/vitest/src/runtime/runners/test.ts

@@ -91,7 +91,7 @@ export class VitestTestRunner implements VitestRunner {
       test.mode = 'skip'
     }
 
-    if (test.mode !== 'run') {
+    if (test.mode !== 'run' && test.mode !== 'queued') {
       return
     }