Skip to content

Commit

Permalink
feat(coverage): automatic threshold updating (#2886)
Browse files Browse the repository at this point in the history
Closes #1241
  • Loading branch information
AriPerkkio authored Feb 25, 2023
1 parent 615e150 commit e165216
Show file tree
Hide file tree
Showing 19 changed files with 203 additions and 95 deletions.
10 changes: 10 additions & 0 deletions docs/config/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -789,6 +789,16 @@ Do not show files with 100% statement, branch, and function coverage.
Check thresholds per file.
See `lines`, `functions`, `branches` and `statements` for the actual thresholds.

#### thresholdAutoUpdate

- **Type:** `boolean`
- **Default:** `false`
- **Available for providers:** `'c8' | 'istanbul'`
- **CLI:** `--coverage.thresholdAutoUpdate=<boolean>`

Update threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is above the configured thresholds.
This option helps to maintain thresholds when coverage is improved.

#### lines

- **Type:** `number`
Expand Down
1 change: 1 addition & 0 deletions packages/coverage-c8/rollup.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const external = [
'vitest',
'vitest/node',
'vitest/config',
'vitest/coverage',
]

const plugins = [
Expand Down
84 changes: 34 additions & 50 deletions packages/coverage-c8/src/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import c from 'picocolors'
import { provider } from 'std-env'
import type { EncodedSourceMap } from 'vite-node'
import { coverageConfigDefaults } from 'vitest/config'
import { BaseCoverageProvider } from 'vitest/coverage'
// eslint-disable-next-line no-restricted-imports
import type { AfterSuiteRunMeta, CoverageC8Options, CoverageProvider, ReportContext, ResolvedCoverageOptions } from 'vitest'
import type { Vitest } from 'vitest/node'
Expand All @@ -17,16 +18,36 @@ import { checkCoverages } from 'c8/lib/commands/check-coverage.js'

type Options = ResolvedCoverageOptions<'c8'>

export class C8CoverageProvider implements CoverageProvider {
export class C8CoverageProvider extends BaseCoverageProvider implements CoverageProvider {
name = 'c8'

ctx!: Vitest
options!: Options
coverages: Profiler.TakePreciseCoverageReturnType[] = []

initialize(ctx: Vitest) {
const config: CoverageC8Options = ctx.config.coverage

this.ctx = ctx
this.options = resolveC8Options(ctx.config.coverage, ctx.config.root)
this.options = {
...coverageConfigDefaults,

// Provider specific defaults
excludeNodeModules: true,
allowExternal: false,

// User's options
...config,

// Resolved fields
provider: 'c8',
reporter: this.resolveReporters(config.reporter || coverageConfigDefaults.reporter),
reportsDirectory: resolve(ctx.config.root, config.reportsDirectory || coverageConfigDefaults.reportsDirectory),
lines: config['100'] ? 100 : config.lines,
functions: config['100'] ? 100 : config.functions,
branches: config['100'] ? 100 : config.branches,
statements: config['100'] ? 100 : config.statements,
}
}

resolveOptions() {
Expand Down Expand Up @@ -156,55 +177,18 @@ export class C8CoverageProvider implements CoverageProvider {

await report.run()
await checkCoverages(options, report)
}
}

function resolveC8Options(options: CoverageC8Options, root: string): Options {
const reportsDirectory = resolve(root, options.reportsDirectory || coverageConfigDefaults.reportsDirectory)

const resolved: Options = {
...coverageConfigDefaults,

// Provider specific defaults
excludeNodeModules: true,
allowExternal: false,

// User's options
...options,

// Resolved fields
provider: 'c8',
reporter: resolveReporters(options.reporter || coverageConfigDefaults.reporter),
reportsDirectory,
}

if (options['100']) {
resolved.lines = 100
resolved.functions = 100
resolved.branches = 100
resolved.statements = 100
}

return resolved
}

function resolveReporters(configReporters: NonNullable<CoverageC8Options['reporter']>): Options['reporter'] {
// E.g. { reporter: "html" }
if (!Array.isArray(configReporters))
return [[configReporters, {}]]

const resolvedReporters: Options['reporter'] = []

for (const reporter of configReporters) {
if (Array.isArray(reporter)) {
// E.g. { reporter: [ ["html", { skipEmpty: true }], ["lcov"], ["json", { file: "map.json" }] ]}
resolvedReporters.push([reporter[0], reporter[1] || {}])
}
else {
// E.g. { reporter: ["html", "json"]}
resolvedReporters.push([reporter, {}])
if (this.options.thresholdAutoUpdate && allTestsRun) {
this.updateThresholds({
coverageMap: await report.getCoverageMapFromAllCoverageFiles(),
thresholds: {
branches: this.options.branches,
functions: this.options.functions,
lines: this.options.lines,
statements: this.options.statements,
},
configurationFile: this.ctx.server.config.configFile,
})
}
}

return resolvedReporters
}
1 change: 1 addition & 0 deletions packages/coverage-istanbul/rollup.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const external = [
'vitest',
'vitest/node',
'vitest/config',
'vitest/coverage',
]

const plugins = [
Expand Down
69 changes: 28 additions & 41 deletions packages/coverage-istanbul/src/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { relative, resolve } from 'pathe'
import type { TransformPluginContext } from 'rollup'
import type { AfterSuiteRunMeta, CoverageIstanbulOptions, CoverageProvider, ReportContext, ResolvedCoverageOptions, Vitest } from 'vitest'
import { coverageConfigDefaults, defaultExclude, defaultInclude } from 'vitest/config'
import { BaseCoverageProvider } from 'vitest/coverage'
import libReport from 'istanbul-lib-report'
import reports from 'istanbul-reports'
import type { CoverageMap } from 'istanbul-lib-coverage'
Expand Down Expand Up @@ -31,7 +32,7 @@ interface TestExclude {
}
}

export class IstanbulCoverageProvider implements CoverageProvider {
export class IstanbulCoverageProvider extends BaseCoverageProvider implements CoverageProvider {
name = 'istanbul'

ctx!: Vitest
Expand All @@ -48,8 +49,20 @@ export class IstanbulCoverageProvider implements CoverageProvider {
coverages: any[] = []

initialize(ctx: Vitest) {
const config: CoverageIstanbulOptions = ctx.config.coverage

this.ctx = ctx
this.options = resolveIstanbulOptions(ctx.config.coverage, ctx.config.root)
this.options = {
...coverageConfigDefaults,

// User's options
...config,

// Resolved fields
provider: 'istanbul',
reportsDirectory: resolve(ctx.config.root, config.reportsDirectory || coverageConfigDefaults.reportsDirectory),
reporter: this.resolveReporters(config.reporter || coverageConfigDefaults.reporter),
}

this.instrumenter = createInstrumenter({
produceSourceMap: true,
Expand Down Expand Up @@ -141,6 +154,19 @@ export class IstanbulCoverageProvider implements CoverageProvider {
statements: this.options.statements,
})
}

if (this.options.thresholdAutoUpdate && allTestsRun) {
this.updateThresholds({
coverageMap,
thresholds: {
branches: this.options.branches,
functions: this.options.functions,
lines: this.options.lines,
statements: this.options.statements,
},
configurationFile: this.ctx.server.config.configFile,
})
}
}

checkThresholds(coverageMap: CoverageMap, thresholds: Record<Threshold, number | undefined>) {
Expand Down Expand Up @@ -220,24 +246,6 @@ export class IstanbulCoverageProvider implements CoverageProvider {
}
}

function resolveIstanbulOptions(options: CoverageIstanbulOptions, root: string): Options {
const reportsDirectory = resolve(root, options.reportsDirectory || coverageConfigDefaults.reportsDirectory)

const resolved: Options = {
...coverageConfigDefaults,

// User's options
...options,

// Resolved fields
provider: 'istanbul',
reportsDirectory,
reporter: resolveReporters(options.reporter || coverageConfigDefaults.reporter),
}

return resolved
}

/**
* Remove possible query parameters from filenames
* - From `/src/components/Header.component.ts?vue&type=script&src=true&lang.ts`
Expand Down Expand Up @@ -287,24 +295,3 @@ function isEmptyCoverageRange(range: libCoverage.Range) {
|| range.end.column === undefined
)
}

function resolveReporters(configReporters: NonNullable<CoverageIstanbulOptions['reporter']>): Options['reporter'] {
// E.g. { reporter: "html" }
if (!Array.isArray(configReporters))
return [[configReporters, {}]]

const resolvedReporters: Options['reporter'] = []

for (const reporter of configReporters) {
if (Array.isArray(reporter)) {
// E.g. { reporter: [ ["html", { skipEmpty: true }], ["lcov"], ["json", { file: "map.json" }] ]}
resolvedReporters.push([reporter[0], reporter[1] || {}])
}
else {
// E.g. { reporter: ["html", "json"]}
resolvedReporters.push([reporter, {}])
}
}

return resolvedReporters
}
1 change: 1 addition & 0 deletions packages/vitest/coverage.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
export * from './dist/coverage.js'
5 changes: 5 additions & 0 deletions packages/vitest/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@
"types": "./config.d.ts",
"require": "./dist/config.cjs",
"import": "./dist/config.js"
},
"./coverage": {
"types": "./coverage.d.ts",
"import": "./dist/coverage.js"
}
},
"main": "./dist/index.js",
Expand Down Expand Up @@ -144,6 +148,7 @@
"@edge-runtime/vm": "2.0.2",
"@sinonjs/fake-timers": "^10.0.2",
"@types/diff": "^5.0.2",
"@types/istanbul-lib-coverage": "^2.0.4",
"@types/istanbul-reports": "^3.0.1",
"@types/jsdom": "^21.1.0",
"@types/micromatch": "^4.0.2",
Expand Down
2 changes: 2 additions & 0 deletions packages/vitest/rollup.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const entries = [
'src/runtime/loader.ts',
'src/runtime/entry.ts',
'src/integrations/spy.ts',
'src/coverage.ts',
]

const dtsEntries = [
Expand All @@ -36,6 +37,7 @@ const dtsEntries = [
'src/runners.ts',
'src/suite.ts',
'src/config.ts',
'src/coverage.ts',
]

const external = [
Expand Down
1 change: 1 addition & 0 deletions packages/vitest/src/coverage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
export { BaseCoverageProvider } from './utils/coverage'
7 changes: 7 additions & 0 deletions packages/vitest/src/types/coverage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,13 @@ export interface BaseCoverageOptions {
* @default undefined
*/
statements?: number

/**
* Update threshold values automatically when current coverage is higher than earlier thresholds
*
* @default false
*/
thresholdAutoUpdate?: boolean
}

export interface CoverageIstanbulOptions extends BaseCoverageOptions {
Expand Down
82 changes: 82 additions & 0 deletions packages/vitest/src/utils/coverage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
import { readFileSync, writeFileSync } from 'node:fs'
import type { CoverageMap } from 'istanbul-lib-coverage'
import type { BaseCoverageOptions, ResolvedCoverageOptions } from '../types'

type Threshold = 'lines' | 'functions' | 'statements' | 'branches'

const THRESHOLD_KEYS: Readonly<Threshold[]> = ['lines', 'functions', 'statements', 'branches']

export class BaseCoverageProvider {
/**
* Check if current coverage is above configured thresholds and bump the thresholds if needed
*/
updateThresholds({ configurationFile, coverageMap, thresholds }: {
coverageMap: CoverageMap
thresholds: Record<Threshold, number | undefined>
configurationFile?: string
}) {
// Thresholds cannot be updated if there is no configuration file and
// feature was enabled by CLI, e.g. --coverage.thresholdAutoUpdate
if (!configurationFile)
throw new Error('Missing configurationFile. The "coverage.thresholdAutoUpdate" can only be enabled when configuration file is used.')

const summary = coverageMap.getCoverageSummary()
const thresholdsToUpdate: Threshold[] = []

for (const key of THRESHOLD_KEYS) {
const threshold = thresholds[key] || 100
const actual = summary[key].pct

if (actual > threshold)
thresholdsToUpdate.push(key)
}

if (thresholdsToUpdate.length === 0)
return

const originalConfig = readFileSync(configurationFile, 'utf8')
let updatedConfig = originalConfig

for (const threshold of thresholdsToUpdate) {
// Find the exact match from the configuration file and replace the value
const previousThreshold = (thresholds[threshold] || 100).toString()
const pattern = new RegExp(`(${threshold}\\s*:\\s*)${previousThreshold.replace('.', '\\.')}`)
const matches = originalConfig.match(pattern)

if (matches)
updatedConfig = updatedConfig.replace(matches[0], matches[1] + summary[threshold].pct)
else
console.error(`Unable to update coverage threshold ${threshold}. No threshold found using pattern ${pattern}`)
}

if (updatedConfig !== originalConfig) {
// eslint-disable-next-line no-console
console.log('Updating thresholds to configuration file. You may want to push with updated coverage thresholds.')
writeFileSync(configurationFile, updatedConfig, 'utf-8')
}
}

/**
* Resolve reporters from various configuration options
*/
resolveReporters(configReporters: NonNullable<BaseCoverageOptions['reporter']>): ResolvedCoverageOptions['reporter'] {
// E.g. { reporter: "html" }
if (!Array.isArray(configReporters))
return [[configReporters, {}]]

const resolvedReporters: ResolvedCoverageOptions['reporter'] = []

for (const reporter of configReporters) {
if (Array.isArray(reporter)) {
// E.g. { reporter: [ ["html", { skipEmpty: true }], ["lcov"], ["json", { file: "map.json" }] ]}
resolvedReporters.push([reporter[0], reporter[1] || {}])
}
else {
// E.g. { reporter: ["html", "json"]}
resolvedReporters.push([reporter, {}])
}
}

return resolvedReporters
}
}
Loading

0 comments on commit e165216

Please sign in to comment.