diff --git a/docs/config/index.md b/docs/config/index.md index 4a3c717a301d..feb91a78bff2 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -752,6 +752,16 @@ Do not show files with 100% statement, branch, and function coverage. Check thresholds per file. See `lines`, `functions`, `branches` and `statements` for the actual thresholds. +#### thresholdAutoUpdate + +- **Type:** `boolean` +- **Default:** `false` +- **Available for providers:** `'c8' | 'istanbul'` +- **CLI:** `--coverage.thresholdAutoUpdate=` + +Update threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is above the configured thresholds. +This option helps to maintain thresholds when coverage is improved. + #### lines - **Type:** `number` diff --git a/packages/coverage-c8/package.json b/packages/coverage-c8/package.json index 3b1073566a5e..b7e76d130bc7 100644 --- a/packages/coverage-c8/package.json +++ b/packages/coverage-c8/package.json @@ -45,6 +45,7 @@ "vitest": ">=0.29.0 <1" }, "dependencies": { + "@vitest/utils": "workspace:*", "c8": "^7.12.0", "picocolors": "^1.0.0", "std-env": "^3.3.1" diff --git a/packages/coverage-c8/src/provider.ts b/packages/coverage-c8/src/provider.ts index a1a64d6df6ee..9e9add3c35f1 100644 --- a/packages/coverage-c8/src/provider.ts +++ b/packages/coverage-c8/src/provider.ts @@ -6,6 +6,7 @@ import c from 'picocolors' import { provider } from 'std-env' import type { RawSourceMap } from 'vite-node' import { coverageConfigDefaults } from 'vitest/config' +import { updateThresholds } from '@vitest/utils/coverage' // eslint-disable-next-line no-restricted-imports import type { AfterSuiteRunMeta, CoverageC8Options, CoverageProvider, ReportContext, ResolvedCoverageOptions } from 'vitest' import type { Vitest } from 'vitest/node' @@ -147,6 +148,19 @@ export class C8CoverageProvider implements CoverageProvider { await report.run() await checkCoverages(options, report) + + if (this.options.thresholdAutoUpdate && allTestsRun) { + updateThresholds({ + coverageMap: await report.getCoverageMapFromAllCoverageFiles(), + thresholds: { + branches: this.options.branches, + functions: this.options.functions, + lines: this.options.lines, + statements: this.options.statements, + }, + configurationFile: this.ctx.server.config.configFile, + }) + } } } diff --git a/packages/coverage-istanbul/package.json b/packages/coverage-istanbul/package.json index 704d2734c751..f46c3ddfb472 100644 --- a/packages/coverage-istanbul/package.json +++ b/packages/coverage-istanbul/package.json @@ -45,6 +45,7 @@ "vitest": ">=0.28.0 <1" }, "dependencies": { + "@vitest/utils": "workspace:*", "istanbul-lib-coverage": "^3.2.0", "istanbul-lib-instrument": "^5.2.1", "istanbul-lib-report": "^3.0.0", diff --git a/packages/coverage-istanbul/src/provider.ts b/packages/coverage-istanbul/src/provider.ts index 9f019d40dfba..ab441c9db558 100644 --- a/packages/coverage-istanbul/src/provider.ts +++ b/packages/coverage-istanbul/src/provider.ts @@ -4,6 +4,7 @@ import { relative, resolve } from 'pathe' import type { TransformPluginContext } from 'rollup' import type { AfterSuiteRunMeta, CoverageIstanbulOptions, CoverageProvider, ReportContext, ResolvedCoverageOptions, Vitest } from 'vitest' import { coverageConfigDefaults, defaultExclude, defaultInclude } from 'vitest/config' +import { updateThresholds } from '@vitest/utils/coverage' import libReport from 'istanbul-lib-report' import reports from 'istanbul-reports' import type { CoverageMap } from 'istanbul-lib-coverage' @@ -140,6 +141,19 @@ export class IstanbulCoverageProvider implements CoverageProvider { statements: this.options.statements, }) } + + if (this.options.thresholdAutoUpdate && allTestsRun) { + updateThresholds({ + coverageMap, + thresholds: { + branches: this.options.branches, + functions: this.options.functions, + lines: this.options.lines, + statements: this.options.statements, + }, + configurationFile: this.ctx.server.config.configFile, + }) + } } checkThresholds(coverageMap: CoverageMap, thresholds: Record) { diff --git a/packages/utils/coverage.d.ts b/packages/utils/coverage.d.ts new file mode 100644 index 000000000000..cf1145ffa2f6 --- /dev/null +++ b/packages/utils/coverage.d.ts @@ -0,0 +1 @@ +export * from './dist/coverage.js' diff --git a/packages/utils/package.json b/packages/utils/package.json index b08f4045fdce..a119afb6c5a5 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -23,6 +23,10 @@ "types": "./dist/helpers.d.ts", "import": "./dist/helpers.js" }, + "./coverage": { + "types": "./dist/coverage.d.ts", + "import": "./dist/coverage.js" + }, "./*": "./*" }, "main": "./dist/index.js", @@ -44,6 +48,7 @@ "pretty-format": "^27.5.1" }, "devDependencies": { - "@types/diff": "^5.0.2" + "@types/diff": "^5.0.2", + "@types/istanbul-lib-coverage": "^2.0.4" } } diff --git a/packages/utils/rollup.config.js b/packages/utils/rollup.config.js index d9cef53a2618..400c9af8a8e4 100644 --- a/packages/utils/rollup.config.js +++ b/packages/utils/rollup.config.js @@ -11,6 +11,7 @@ const entries = { helpers: 'src/helpers.ts', diff: 'src/diff.ts', types: 'src/types.ts', + coverage: 'src/coverage.ts', } const external = [ diff --git a/packages/utils/src/coverage.ts b/packages/utils/src/coverage.ts new file mode 100644 index 000000000000..d5e0ed1678d8 --- /dev/null +++ b/packages/utils/src/coverage.ts @@ -0,0 +1,55 @@ +import { readFileSync, writeFileSync } from 'node:fs' +import type { CoverageMap } from 'istanbul-lib-coverage' + +type Threshold = 'lines' | 'functions' | 'statements' | 'branches' + +const THRESHOLD_KEYS: Readonly = ['lines', 'functions', 'statements', 'branches'] + +/** + * Check if current coverage is above configured thresholds and bump the thresholds if needed + */ +export function updateThresholds({ configurationFile, coverageMap, thresholds }: { + coverageMap: CoverageMap + thresholds: Record + configurationFile?: string +}) { + // Thresholds cannot be updated if there is no configuration file and + // feature was enabled by CLI, e.g. --coverage.thresholdAutoUpdate + if (!configurationFile) + throw new Error('Missing configurationFile. The "coverage.thresholdAutoUpdate" can only be enabled when configuration file is used.') + + const summary = coverageMap.getCoverageSummary() + const thresholdsToUpdate: Threshold[] = [] + + for (const key of THRESHOLD_KEYS) { + const threshold = thresholds[key] || 100 + const actual = summary[key].pct + + if (actual > threshold) + thresholdsToUpdate.push(key) + } + + if (thresholdsToUpdate.length === 0) + return + + const originalConfig = readFileSync(configurationFile, 'utf8') + let updatedConfig = originalConfig + + for (const threshold of thresholdsToUpdate) { + // Find the exact match from the configuration file and replace the value + const previousThreshold = (thresholds[threshold] || 100).toString() + const pattern = new RegExp(`(${threshold}\\s*:\\s*)${previousThreshold.replace('.', '\\.')}`) + const matches = originalConfig.match(pattern) + + if (matches) + updatedConfig = updatedConfig.replace(matches[0], matches[1] + summary[threshold].pct) + else + console.error(`Unable to update coverage threshold ${threshold}. No threshold found using pattern ${pattern}`) + } + + if (updatedConfig !== originalConfig) { + // eslint-disable-next-line no-console + console.log('Updating thresholds to configuration file. You may want to push with updated coverage thresholds.') + writeFileSync(configurationFile, updatedConfig, 'utf-8') + } +} diff --git a/packages/vitest/src/types/coverage.ts b/packages/vitest/src/types/coverage.ts index 13c306b7f8c1..e536bddd439b 100644 --- a/packages/vitest/src/types/coverage.ts +++ b/packages/vitest/src/types/coverage.ts @@ -192,6 +192,13 @@ export interface BaseCoverageOptions { * @default undefined */ statements?: number + + /** + * Update threshold values automatically when current coverage is higher than earlier thresholds + * + * @default false + */ + thresholdAutoUpdate?: boolean } export interface CoverageIstanbulOptions extends BaseCoverageOptions { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f878a07e6798..6fce3091344e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -650,6 +650,7 @@ importers: packages/coverage-c8: specifiers: + '@vitest/utils': workspace:* c8: ^7.12.0 pathe: ^1.1.0 picocolors: ^1.0.0 @@ -657,6 +658,7 @@ importers: vite-node: workspace:* vitest: workspace:* dependencies: + '@vitest/utils': link:../utils c8: 7.12.0 picocolors: 1.0.0 std-env: 3.3.1 @@ -672,6 +674,7 @@ importers: '@types/istanbul-lib-report': ^3.0.0 '@types/istanbul-lib-source-maps': ^4.0.1 '@types/istanbul-reports': ^3.0.1 + '@vitest/utils': workspace:* istanbul-lib-coverage: ^3.2.0 istanbul-lib-instrument: ^5.2.1 istanbul-lib-report: ^3.0.0 @@ -681,6 +684,7 @@ importers: test-exclude: ^6.0.0 vitest: workspace:* dependencies: + '@vitest/utils': link:../utils istanbul-lib-coverage: 3.2.0 istanbul-lib-instrument: 5.2.1 istanbul-lib-report: 3.0.0 @@ -801,6 +805,7 @@ importers: packages/utils: specifiers: '@types/diff': ^5.0.2 + '@types/istanbul-lib-coverage': ^2.0.4 cli-truncate: ^3.1.0 diff: ^5.1.0 loupe: ^2.3.6 @@ -814,6 +819,7 @@ importers: pretty-format: 27.5.1 devDependencies: '@types/diff': 5.0.2 + '@types/istanbul-lib-coverage': 2.0.4 packages/vite-node: specifiers: diff --git a/test/coverage-test/coverage-report-tests/generic.report.test.ts b/test/coverage-test/coverage-report-tests/generic.report.test.ts index f79271bc7575..4262250041fa 100644 --- a/test/coverage-test/coverage-report-tests/generic.report.test.ts +++ b/test/coverage-test/coverage-report-tests/generic.report.test.ts @@ -63,3 +63,20 @@ test('files should not contain a setup file', () => { expect(srcFiles).not.toContain('another-setup.ts.html') }) + +test('thresholdAutoUpdate updates thresholds', async () => { + const configFilename = resolve('./vitest.config.ts') + const configContents = fs.readFileSync(configFilename, 'utf-8') + + for (const threshold of ['functions', 'branches', 'lines', 'statements']) { + const match = configContents.match(new RegExp(`${threshold}: (?[\\d|\\.]+)`)) + const coverage = match?.groups?.coverage || '0' + + // Configuration has fixed value of 1.01 set for each threshold + expect(parseInt(coverage)).toBeGreaterThan(1.01) + } + + // Update thresholds back to fixed values + const updatedConfig = configContents.replace(/(branches|functions|lines|statements): ([\d|\.])+/g, '$1: 1.01') + fs.writeFileSync(configFilename, updatedConfig) +}) diff --git a/test/coverage-test/vitest.config.ts b/test/coverage-test/vitest.config.ts index 325f715f4112..11c655cc326d 100644 --- a/test/coverage-test/vitest.config.ts +++ b/test/coverage-test/vitest.config.ts @@ -20,6 +20,13 @@ export default defineConfig({ clean: true, all: true, reporter: ['html', 'text', 'lcov', 'json'], + + // These will be updated by tests and reseted back by generic.report.test.ts + thresholdAutoUpdate: true, + functions: 1.01, + branches: 1.01, + lines: 1.01, + statements: 1.01, }, setupFiles: [ resolve(__dirname, './setup.ts'),