feat!: --merge-reports to support coverage (vitest-dev#5736)

docs/guide/features.md

@@ -232,10 +232,12 @@ test('my types work properly', () => {
 ## Sharding
 
 Run tests on different machines using [`--shard`](/guide/cli#shard) and [`--reporter=blob`](/guide/reporters#blob-reporter) flags.
-All test results can be merged at the end of your CI pipeline using `--merge-reports` command:
+All test and coverage results can be merged at the end of your CI pipeline using `--merge-reports` command:
 
 ```bash
 vitest --shard=1/2 --reporter=blob
 vitest --shard=2/2 --reporter=blob
-vitest --merge-reports --reporter=junit
+vitest --merge-reports --reporter=junit --coverage.reporter=text
 ```
+
+See [`Improving Performance | Sharding`](/guide/improving-performance#sharding) for more information.

docs/guide/improving-performance.md

@@ -72,3 +72,116 @@ export default defineConfig({
 })
 ```
 :::
+
+## Sharding
+
+Test sharding means running a small subset of test cases at a time. It can be useful when you have multiple machines that could be used to run tests simultaneously.
+
+To split Vitest tests on multiple different runs, use [`--shard`](/guide/cli#shard) option with [`--reporter=blob`](/guide/reporters#blob-reporter) option:
+
+```sh
+vitest run --reporter=blob --shard=1/3 # 1st machine
+vitest run --reporter=blob --shard=2/3 # 2nd machine
+vitest run --reporter=blob --shard=3/3 # 3rd machine
+```
+
+Collect the results stored in `.vitest-reports` directory from each machine and merge them with [`--merge-reports`](/guide/cli#merge-reports) option:
+
+```sh
+vitest --merge-reports
+```
+
+<details>
+  <summary>Github action example</summary>
+
+This setup is also used at https://github.com/vitest-tests/test-sharding.
+
+```yaml
+# Inspired from https://playwright.dev/docs/test-sharding
+name: Tests
+on:
+  push:
+    branches:
+      - main
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Install dependencies
+        run: pnpm i
+
+      - name: Run tests
+        run: pnpm run test --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+
+      - name: Upload blob report to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: blob-report-${{ matrix.shardIndex }}
+          path: .vitest-reports/*
+          retention-days: 1
+
+  merge-reports:
+    if: ${{ !cancelled() }}
+    needs: [tests]
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+
+      - name: Install dependencies
+        run: pnpm i
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: .vitest-reports
+          pattern: blob-report-*
+          merge-multiple: true
+
+      - name: Merge reports
+        run: npx vitest --merge-reports
+```
+
+</details>
+
+:::tip
+Test sharding can also become useful on high CPU-count machines.
+
+Vitest will run only a single Vite server in its main thread. Rest of the threads are used to run test files.
+In a high CPU-count machine the main thread can become a bottleneck as it cannot handle all the requests coming from the threads. For example in 32 CPU machine the main thread is responsible to handle load coming from 31 test threads.
+
+To reduce the load from main thread's Vite server you can use test sharding. The load can be balanced on multiple Vite server.
+
+```sh
+# Example for splitting tests on 32 CPU to 4 shards.
+# As each process needs 1 main thread, there's 7 threads for test runners (1+7)*4 = 32
+# Use VITEST_MAX_THREADS or VITEST_MAX_FORKS depending on the pool:
+VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=1/4 & \
+VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=2/4 & \
+VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=3/4 & \
+VITEST_MAX_THREADS=7 vitest run --reporter=blob --shard=4/4 & \
+wait # https://man7.org/linux/man-pages/man2/waitpid.2.html
+
+vitest --merge-reports
+```
+
+:::

packages/coverage-istanbul/src/provider.ts

@@ -162,7 +162,7 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
     this.pendingPromises = []
   }
 
-  async reportCoverage({ allTestsRun }: ReportContext = {}) {
+  async generateCoverage({ allTestsRun }: ReportContext) {
     const coverageMap = libCoverage.createCoverageMap({})
     let index = 0
     const total = this.pendingPromises.length
@@ -202,6 +202,29 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
       coverageMap.merge(await transformCoverage(uncoveredCoverage))
     }
 
+    return coverageMap
+  }
+
+  async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext) {
+    await this.generateReports(
+      coverageMap as CoverageMap || libCoverage.createCoverageMap({}),
+      allTestsRun,
+    )
+
+    // In watch mode we need to preserve the previous results if cleanOnRerun is disabled
+    const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
+
+    if (!keepResults) {
+      this.coverageFiles = new Map()
+      await fs.rm(this.coverageFilesDirectory, { recursive: true })
+
+      // Remove empty reports directory, e.g. when only text-reporter is used
+      if (readdirSync(this.options.reportsDirectory).length === 0)
+        await fs.rm(this.options.reportsDirectory, { recursive: true })
+    }
+  }
+
+  async generateReports(coverageMap: CoverageMap, allTestsRun: boolean | undefined) {
     const context = libReport.createContext({
       dir: this.options.reportsDirectory,
       coverageMap,
@@ -248,21 +271,18 @@ export class IstanbulCoverageProvider extends BaseCoverageProvider implements Co
         })
       }
     }
+  }
 
-    // In watch mode we need to preserve the previous results if cleanOnRerun is disabled
-    const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
+  async mergeReports(coverageMaps: unknown[]) {
+    const coverageMap = libCoverage.createCoverageMap({})
 
-    if (!keepResults) {
-      this.coverageFiles = new Map()
-      await fs.rm(this.coverageFilesDirectory, { recursive: true })
+    for (const coverage of coverageMaps)
+      coverageMap.merge(coverage as CoverageMap)
 
-      // Remove empty reports directory, e.g. when only text-reporter is used
-      if (readdirSync(this.options.reportsDirectory).length === 0)
-        await fs.rm(this.options.reportsDirectory, { recursive: true })
-    }
+    await this.generateReports(coverageMap, true)
   }
 
-  async getCoverageMapForUncoveredFiles(coveredFiles: string[]) {
+  private async getCoverageMapForUncoveredFiles(coveredFiles: string[]) {
     const allFiles = await this.testExclude.glob(this.ctx.config.root)
     let includedFiles = allFiles.map(file => resolve(this.ctx.config.root, file))
 

packages/coverage-v8/src/provider.ts

@@ -148,10 +148,7 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
     this.pendingPromises.push(promise)
   }
 
-  async reportCoverage({ allTestsRun }: ReportContext = {}) {
-    if (provider === 'stackblitz')
-      this.ctx.logger.log(c.blue(' % ') + c.yellow('@vitest/coverage-v8 does not work on Stackblitz. Report will be empty.'))
-
+  async generateCoverage({ allTestsRun }: ReportContext) {
     const coverageMap = libCoverage.createCoverageMap({})
     let index = 0
     const total = this.pendingPromises.length
@@ -193,6 +190,32 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
       coverageMap.merge(await transformCoverage(converted))
     }
 
+    return coverageMap
+  }
+
+  async reportCoverage(coverageMap: unknown, { allTestsRun }: ReportContext) {
+    if (provider === 'stackblitz')
+      this.ctx.logger.log(c.blue(' % ') + c.yellow('@vitest/coverage-v8 does not work on Stackblitz. Report will be empty.'))
+
+    await this.generateReports(
+      coverageMap as CoverageMap || libCoverage.createCoverageMap({}),
+      allTestsRun,
+    )
+
+    // In watch mode we need to preserve the previous results if cleanOnRerun is disabled
+    const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
+
+    if (!keepResults) {
+      this.coverageFiles = new Map()
+      await fs.rm(this.coverageFilesDirectory, { recursive: true })
+
+      // Remove empty reports directory, e.g. when only text-reporter is used
+      if (readdirSync(this.options.reportsDirectory).length === 0)
+        await fs.rm(this.options.reportsDirectory, { recursive: true })
+    }
+  }
+
+  async generateReports(coverageMap: CoverageMap, allTestsRun?: boolean) {
     const context = libReport.createContext({
       dir: this.options.reportsDirectory,
       coverageMap,
@@ -239,18 +262,15 @@ export class V8CoverageProvider extends BaseCoverageProvider implements Coverage
         })
       }
     }
+  }
 
-    // In watch mode we need to preserve the previous results if cleanOnRerun is disabled
-    const keepResults = !this.options.cleanOnRerun && this.ctx.config.watch
+  async mergeReports(coverageMaps: unknown[]) {
+    const coverageMap = libCoverage.createCoverageMap({})
 
-    if (!keepResults) {
-      this.coverageFiles = new Map()
-      await fs.rm(this.coverageFilesDirectory, { recursive: true })
+    for (const coverage of coverageMaps)
+      coverageMap.merge(coverage as CoverageMap)
 
-      // Remove empty reports directory, e.g. when only text-reporter is used
-      if (readdirSync(this.options.reportsDirectory).length === 0)
-        await fs.rm(this.options.reportsDirectory, { recursive: true })
-    }
+    await this.generateReports(coverageMap, true)
   }
 
   private async getUntestedFiles(testedFiles: string[]): Promise<RawCoverage> {

packages/vitest/src/node/core.ts

@@ -393,7 +393,7 @@ export class Vitest {
     if (this.reporters.some(r => r instanceof BlobReporter))
       throw new Error('Cannot merge reports when `--reporter=blob` is used. Remove blob reporter from the config first.')
 
-    const { files, errors } = await readBlobs(this.config.mergeReports, this.projects)
+    const { files, errors, coverages } = await readBlobs(this.config.mergeReports, this.projects)
 
     await this.report('onInit', this)
     await this.report('onPathsCollected', files.flatMap(f => f.filepath))
@@ -439,6 +439,8 @@ export class Vitest {
       process.exitCode = 1
 
     await this.report('onFinished', files, errors)
+    await this.initCoverageProvider()
+    await this.coverageProvider?.mergeReports?.(coverages)
   }
 
   async start(filters?: string[]) {
@@ -459,7 +461,9 @@ export class Vitest {
 
     // if run with --changed, don't exit if no tests are found
     if (!files.length) {
-      await this.reportCoverage(true)
+      // Report coverage for uncovered files
+      const coverage = await this.coverageProvider?.generateCoverage?.({ allTestsRun: true })
+      await this.reportCoverage(coverage, true)
 
       this.logger.printNoTestFound(filters)
 
@@ -645,8 +649,10 @@ export class Vitest {
       .finally(async () => {
         // can be duplicate files if different projects are using the same file
         const files = Array.from(new Set(specs.map(([, p]) => p)))
-        await this.report('onFinished', this.state.getFiles(files), this.state.getUnhandledErrors())
-        await this.reportCoverage(allTestsRun)
+        const coverage = await this.coverageProvider?.generateCoverage({ allTestsRun })
+
+        await this.report('onFinished', this.state.getFiles(files), this.state.getUnhandledErrors(), coverage)
+        await this.reportCoverage(coverage, allTestsRun)
 
         this.runningPromise = undefined
         this.isFirstRun = false
@@ -946,12 +952,12 @@ export class Vitest {
     return Array.from(new Set(files))
   }
 
-  private async reportCoverage(allTestsRun: boolean) {
+  private async reportCoverage(coverage: unknown, allTestsRun: boolean) {
     if (!this.config.coverage.reportOnFailure && this.state.getCountOfFailedTests() > 0)
       return
 
     if (this.coverageProvider) {
-      await this.coverageProvider.reportCoverage({ allTestsRun })
+      await this.coverageProvider.reportCoverage(coverage, { allTestsRun })
       // notify coverage iframe reload
       for (const reporter of this.reporters) {
         if (reporter instanceof WebSocketReporter)

packages/vitest/src/node/reporters/blob.ts

@@ -26,7 +26,7 @@ export class BlobReporter implements Reporter {
     this.ctx = ctx
   }
 
-  async onFinished(files: File[] = [], errors: unknown[] = []) {
+  async onFinished(files: File[] = [], errors: unknown[] = [], coverage: unknown) {
     let outputFile = this.options.outputFile ?? getOutputFile(this.ctx.config, 'blob')
     if (!outputFile) {
       const shard = this.ctx.config.shard
@@ -39,7 +39,7 @@ export class BlobReporter implements Reporter {
       return [project.getName(), [...project.server.moduleGraph.idToModuleMap.keys()]]
     })
 
-    const report = stringify([this.ctx.version, files, errors, moduleKeys] satisfies MergeReport)
+    const report = stringify([this.ctx.version, files, errors, moduleKeys, coverage] satisfies MergeReport)
 
     const reportFile = resolve(this.ctx.config.root, outputFile)
 
@@ -62,8 +62,8 @@ export async function readBlobs(blobsDirectory: string, projectsArray: Workspace
   const blobsFiles = await readdir(resolvedDir)
   const promises = blobsFiles.map(async (file) => {
     const content = await readFile(resolve(resolvedDir, file), 'utf-8')
-    const [version, files, errors, moduleKeys] = parse(content) as MergeReport
-    return { version, files, errors, moduleKeys }
+    const [version, files, errors, moduleKeys, coverage] = parse(content) as MergeReport
+    return { version, files, errors, moduleKeys, coverage }
   })
   const blobs = await Promise.all(promises)
 
@@ -108,10 +108,12 @@ export async function readBlobs(blobsDirectory: string, projectsArray: Workspace
     return time1 - time2
   })
   const errors = blobs.flatMap(blob => blob.errors)
+  const coverages = blobs.map(blob => blob.coverage)
 
   return {
     files,
     errors,
+    coverages,
   }
 }
 
@@ -120,6 +122,7 @@ type MergeReport = [
   files: File[],
   errors: unknown[],
   moduleKeys: MergeReportModuleKeys[],
+  coverage: unknown,
 ]
 
 type MergeReportModuleKeys = [projectName: string, moduleIds: string[]]