Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support non-0 exit code if coverage thresholds not met #59

Merged
merged 5 commits into from
Jan 23, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,30 @@ The above example will output coverage metrics for `foo.js`.

run `c8 report` to regenerate reports after `c8` has already been run.

## Checking coverage

c8 can fail tests if coverage falls below a threshold.
After running your tests with c8, simply run:

```shell
c8 check-coverage --lines 95 --functions 95 --branches 95
```

c8 also accepts a `--check-coverage` shorthand, which can be used to
both run tests and check that coverage falls within the threshold provided:

```shell
c8 --check-coverage --lines 100 npm test
```

The above check fails if coverage falls below 100%.

To check thresholds on a per-file basis run:

```shell
c8 check-coverage --lines 95 --per-file
```

## Supported Node.js Versions

c8 uses
Expand Down
45 changes: 16 additions & 29 deletions bin/c8.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,48 +2,35 @@
'use strict'

const fs = require('fs')
const util = require('util')

const foreground = require('foreground-child')
const report = require('../lib/report')
const { outputReport } = require('../lib/commands/report')
const { checkCoverages } = require('../lib/commands/check-coverage')
const { promisify } = require('util')
const rimraf = require('rimraf')
const {
buildYargs,
hideInstrumenteeArgs,
hideInstrumenterArgs,
yargs
hideInstrumenterArgs
} = require('../lib/parse-args')

const instrumenterArgs = hideInstrumenteeArgs()
let argv = yargs.parse(instrumenterArgs)

const _p = util.promisify

function outputReport () {
report({
include: argv.include,
exclude: argv.exclude,
reporter: Array.isArray(argv.reporter) ? argv.reporter : [argv.reporter],
tempDirectory: argv.tempDirectory,
watermarks: argv.watermarks,
resolve: argv.resolve,
omitRelative: argv.omitRelative,
wrapperLength: argv.wrapperLength
})
}
let argv = buildYargs().parse(instrumenterArgs)

(async function run () {
if (argv._[0] === 'report') {
argv = yargs.parse(process.argv) // support flag arguments after "report".
outputReport()
;(async function run () {
if ([
'check-coverage', 'report'
].indexOf(argv._[0]) !== -1) {
argv = buildYargs(true).parse(process.argv.slice(2))
} else {
if (argv.clean) {
await _p(rimraf)(argv.tempDirectory)
await _p(fs.mkdir)(argv.tempDirectory, { recursive: true })
await promisify(rimraf)(argv.tempDirectory)
await promisify(fs.mkdir)(argv.tempDirectory, { recursive: true })
}
process.env.NODE_V8_COVERAGE = argv.tempDirectory

process.env.NODE_V8_COVERAGE = argv.tempDirectory
foreground(hideInstrumenterArgs(argv), () => {
outputReport()
const report = outputReport(argv)
if (argv.checkCoverage) checkCoverages(argv, report)
})
}
})()
59 changes: 59 additions & 0 deletions lib/commands/check-coverage.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
const { relative } = require('path')
const Report = require('../report')

exports.command = 'check-coverage'

exports.describe = 'check whether coverage is within thresholds provided'

exports.builder = function (yargs) {
yargs
.example('$0 check-coverage --lines 95', "check whether the JSON in c8's output folder meets the thresholds provided")
}

exports.handler = function (argv) {
const report = Report({
include: argv.include,
exclude: argv.exclude,
reporter: Array.isArray(argv.reporter) ? argv.reporter : [argv.reporter],
tempDirectory: argv.tempDirectory,
watermarks: argv.watermarks,
resolve: argv.resolve,
omitRelative: argv.omitRelative,
wrapperLength: argv.wrapperLength
})
exports.checkCoverages(argv, report)
}

exports.checkCoverages = function (argv, report) {
const thresholds = {
lines: argv.lines,
functions: argv.functions,
branches: argv.branches,
statements: argv.statements
}
const map = report.getCoverageMapFromAllCoverageFiles()
if (argv.perFile) {
map.files().forEach(file => {
checkCoverage(map.fileCoverageFor(file).toSummary(), thresholds, file)
})
} else {
checkCoverage(map.getCoverageSummary(), thresholds)
}
}

function checkCoverage (summary, thresholds, file) {
Object.keys(thresholds).forEach(key => {
const coverage = summary[key].pct
if (coverage < thresholds[key]) {
process.exitCode = 1
if (file) {
console.error(
'ERROR: Coverage for ' + key + ' (' + coverage + '%) does not meet threshold (' + thresholds[key] + '%) for ' +
relative('./', file).replace(/\\/g, '/') // standardize path for Windows.
)
} else {
console.error('ERROR: Coverage for ' + key + ' (' + coverage + '%) does not meet global threshold (' + thresholds[key] + '%)')
}
}
})
}
24 changes: 24 additions & 0 deletions lib/commands/report.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
const Report = require('../report')

exports.command = 'report'

exports.describe = 'read V8 coverage data from temp and output report'

exports.handler = function (argv) {
exports.outputReport(argv)
}

exports.outputReport = function (argv) {
const report = Report({
include: argv.include,
exclude: argv.exclude,
reporter: Array.isArray(argv.reporter) ? argv.reporter : [argv.reporter],
tempDirectory: argv.tempDirectory,
watermarks: argv.watermarks,
resolve: argv.resolve,
omitRelative: argv.omitRelative,
wrapperLength: argv.wrapperLength
})
report.run()
return report
}
130 changes: 84 additions & 46 deletions lib/parse-args.js
Original file line number Diff line number Diff line change
@@ -1,56 +1,94 @@
const Exclude = require('test-exclude')
const findUp = require('find-up')
const { readFileSync } = require('fs')
const yargs = require('yargs')
const Yargs = require('yargs/yargs')
const parser = require('yargs-parser')

const configPath = findUp.sync(['.c8rc', '.c8rc.json'])
const config = configPath ? JSON.parse(readFileSync(configPath)) : {}

yargs()
.usage('$0 [opts] [script] [opts]')
.option('reporter', {
alias: 'r',
describe: 'coverage reporter(s) to use',
default: 'text'
})
.option('exclude', {
alias: 'x',
default: Exclude.defaultExclude,
describe: 'a list of specific files and directories that should be excluded from coverage (glob patterns are supported)'
})
.option('include', {
alias: 'n',
default: [],
describe: 'a list of specific files that should be covered (glob patterns are supported)'
})
.option('temp-directory', {
default: './coverage/tmp',
describe: 'directory V8 coverage data is written to and read from'
})
.option('resolve', {
default: '',
describe: 'resolve paths to alternate base directory'
})
.option('wrapper-length', {
describe: 'how many bytes is the wrapper prefix on executed JavaScript',
type: 'number'
})
.option('omit-relative', {
default: true,
type: 'boolean',
describe: 'omit any paths that are not absolute, e.g., internal/net.js'
})
.option('clean', {
default: true,
type: 'boolean',
describe: 'should temp files be deleted before script execution'
})
.command('report', 'read V8 coverage data from temp and output report')
.pkgConf('c8')
.config(config)
.demandCommand(1)
.epilog('visit https://git.io/vHysA for list of available reporters')
function buildYargs (withCommands = false) {
const yargs = Yargs([])
.usage('$0 [opts] [script] [opts]')
.option('reporter', {
alias: 'r',
describe: 'coverage reporter(s) to use',
default: 'text'
})
.option('exclude', {
alias: 'x',
default: Exclude.defaultExclude,
describe: 'a list of specific files and directories that should be excluded from coverage (glob patterns are supported)'
})
.option('include', {
alias: 'n',
default: [],
describe: 'a list of specific files that should be covered (glob patterns are supported)'
})
.option('check-coverage', {
default: false,
type: 'boolean',
description: 'check whether coverage is within thresholds provided'
})
.option('branches', {
default: 0,
description: 'what % of branches must be covered?'
})
.option('functions', {
default: 0,
description: 'what % of functions must be covered?'
})
.option('lines', {
default: 90,
description: 'what % of lines must be covered?'
})
.option('statements', {
default: 0,
description: 'what % of statements must be covered?'
})
.option('per-file', {
default: false,
description: 'check thresholds per file'
})
.option('temp-directory', {
default: './coverage/tmp',
describe: 'directory V8 coverage data is written to and read from'
})
.option('resolve', {
default: '',
describe: 'resolve paths to alternate base directory'
})
.option('wrapper-length', {
describe: 'how many bytes is the wrapper prefix on executed JavaScript',
type: 'number'
})
.option('omit-relative', {
default: true,
type: 'boolean',
describe: 'omit any paths that are not absolute, e.g., internal/net.js'
})
.option('clean', {
default: true,
type: 'boolean',
describe: 'should temp files be deleted before script execution'
})
.pkgConf('c8')
.config(config)
.demandCommand(1)
.epilog('visit https://git.io/vHysA for list of available reporters')

const checkCoverage = require('./commands/check-coverage')
const report = require('./commands/report')
if (withCommands) {
yargs.command(checkCoverage)
yargs.command(report)
} else {
yargs.command(checkCoverage.command, checkCoverage.describe)
yargs.command(report.command, report.describe)
}

return yargs
}

function hideInstrumenterArgs (yargv) {
var argv = process.argv.slice(1)
Expand All @@ -76,7 +114,7 @@ function hideInstrumenteeArgs () {
}

module.exports = {
yargs,
buildYargs,
hideInstrumenterArgs,
hideInstrumenteeArgs
}
16 changes: 11 additions & 5 deletions lib/report.js
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ class Report {
this.wrapperLength = wrapperLength
}
run () {
const map = this._getCoverageMapFromAllCoverageFiles()
const map = this.getCoverageMapFromAllCoverageFiles()
var context = libReport.createContext({
dir: './coverage',
watermarks: this.watermarks
Expand All @@ -45,7 +45,13 @@ class Report {
})
}

_getCoverageMapFromAllCoverageFiles () {
getCoverageMapFromAllCoverageFiles () {
// the merge process can be very expensive, and it's often the case that
// check-coverage is called immediately after a report. We memoize the
// result from getCoverageMapFromAllCoverageFiles() to address this
// use-case.
if (this._allCoverageFiles) return this._allCoverageFiles

const v8ProcessCov = this._getMergedProcessCov()

const map = libCoverage.createCoverageMap({})
Expand All @@ -61,7 +67,8 @@ class Report {
}
}

return map
this._allCoverageFiles = map
return this._allCoverageFiles
}

/**
Expand Down Expand Up @@ -138,6 +145,5 @@ class Report {
}

module.exports = function (opts) {
const report = new Report(opts)
report.run()
return new Report(opts)
}
Loading