From b30f1f3de57578436bba7d5fa36343853cf66a3a Mon Sep 17 00:00:00 2001 From: Chemi Atlow Date: Fri, 20 Sep 2024 16:20:45 +0300 Subject: [PATCH] test_runner: add support for coverage via run() PR-URL: https://github.com/nodejs/node/pull/53937 Fixes: https://github.com/nodejs/node/issues/53867 Refs: https://github.com/nodejs/node/issues/53924 Reviewed-By: Colin Ihrig Reviewed-By: Moshe Atlow Reviewed-By: Matteo Collina --- doc/api/test.md | 27 +++ lib/internal/test_runner/runner.js | 35 +++- test/parallel/test-runner-run-coverage.mjs | 186 +++++++++++++++++++++ 3 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 test/parallel/test-runner-run-coverage.mjs diff --git a/doc/api/test.md b/doc/api/test.md index 93aa0bf32f128c..771563f013f87d 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -1246,6 +1246,9 @@ added: - v18.9.0 - v16.19.0 changes: + - version: REPLACEME + pr-url: https://github.com/nodejs/node/pull/53937 + description: Added coverage options. - version: v22.8.0 pr-url: https://github.com/nodejs/node/pull/53927 description: Added the `isolation` option. @@ -1319,6 +1322,29 @@ changes: that specifies the index of the shard to run. This option is _required_. * `total` {number} is a positive integer that specifies the total number of shards to split the test files to. This option is _required_. + * `coverage` {boolean} enable [code coverage][] collection. + **Default:** `false`. + * `coverageExcludeGlobs` {string|Array} Excludes specific files from code coverage + using a glob pattern, which can match both absolute and relative file paths. + This property is only applicable when `coverage` was set to `true`. + If both `coverageExcludeGlobs` and `coverageIncludeGlobs` are provided, + files must meet **both** criteria to be included in the coverage report. + **Default:** `undefined`. + * `coverageIncludeGlobs` {string|Array} Includes specific files in code coverage + using a glob pattern, which can match both absolute and relative file paths. + This property is only applicable when `coverage` was set to `true`. + If both `coverageExcludeGlobs` and `coverageIncludeGlobs` are provided, + files must meet **both** criteria to be included in the coverage report. + **Default:** `undefined`. + * `lineCoverage` {number} Require a minimum percent of covered lines. If code + coverage does not reach the threshold specified, the process will exit with code `1`. + **Default:** `0`. + * `branchCoverage` {number} Require a minimum percent of covered branches. If code + coverage does not reach the threshold specified, the process will exit with code `1`. + **Default:** `0`. + * `functionCoverage` {number} Require a minimum percent of covered functions. If code + coverage does not reach the threshold specified, the process will exit with code `1`. + **Default:** `0`. * Returns: {TestsStream} **Note:** `shard` is used to horizontally parallelize test running across @@ -3537,6 +3563,7 @@ Can be used to abort test subtasks when the test has been aborted. [`run()`]: #runoptions [`suite()`]: #suitename-options-fn [`test()`]: #testname-options-fn +[code coverage]: #collecting-code-coverage [describe options]: #describename-options-fn [it options]: #testname-options-fn [stream.compose]: stream.md#streamcomposestreams diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index e2237dd73f468d..6df6a9b51818d8 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -55,6 +55,7 @@ const { validateObject, validateOneOf, validateInteger, + validateStringArray, } = require('internal/validators'); const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector'); const { isRegExp } = require('internal/util/types'); @@ -524,7 +525,13 @@ function watchFiles(testFiles, opts) { function run(options = kEmptyObject) { validateObject(options, 'options'); - let { testNamePatterns, testSkipPatterns, shard } = options; + let { + testNamePatterns, + testSkipPatterns, + shard, + coverageExcludeGlobs, + coverageIncludeGlobs, + } = options; const { concurrency, timeout, @@ -537,6 +544,10 @@ function run(options = kEmptyObject) { setup, only, globPatterns, + coverage = false, + lineCoverage = 0, + branchCoverage = 0, + functionCoverage = 0, } = options; if (files != null) { @@ -615,6 +626,22 @@ function run(options = kEmptyObject) { }); } validateOneOf(isolation, 'options.isolation', ['process', 'none']); + validateBoolean(coverage, 'options.coverage'); + if (coverageExcludeGlobs != null) { + if (!ArrayIsArray(coverageExcludeGlobs)) { + coverageExcludeGlobs = [coverageExcludeGlobs]; + } + validateStringArray(coverageExcludeGlobs, 'options.coverageExcludeGlobs'); + } + if (coverageIncludeGlobs != null) { + if (!ArrayIsArray(coverageIncludeGlobs)) { + coverageIncludeGlobs = [coverageIncludeGlobs]; + } + validateStringArray(coverageIncludeGlobs, 'options.coverageIncludeGlobs'); + } + validateInteger(lineCoverage, 'options.lineCoverage', 0, 100); + validateInteger(branchCoverage, 'options.branchCoverage', 0, 100); + validateInteger(functionCoverage, 'options.functionCoverage', 0, 100); const rootTestOptions = { __proto__: null, concurrency, timeout, signal }; const globalOptions = { @@ -623,6 +650,12 @@ function run(options = kEmptyObject) { // behavior has relied on it, so removing it must be done in a semver major. ...parseCommandLine(), setup, // This line can be removed when parseCommandLine() is removed here. + coverage, + coverageExcludeGlobs, + coverageIncludeGlobs, + lineCoverage: lineCoverage, + branchCoverage: branchCoverage, + functionCoverage: functionCoverage, }; const root = createTestTree(rootTestOptions, globalOptions); diff --git a/test/parallel/test-runner-run-coverage.mjs b/test/parallel/test-runner-run-coverage.mjs new file mode 100644 index 00000000000000..15fcfef5238843 --- /dev/null +++ b/test/parallel/test-runner-run-coverage.mjs @@ -0,0 +1,186 @@ +import * as common from '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import { describe, it, run } from 'node:test'; +import assert from 'node:assert'; +import { sep } from 'node:path'; + +const files = [fixtures.path('test-runner', 'coverage.js')]; +const abortedSignal = AbortSignal.abort(); + +describe('require(\'node:test\').run coverage settings', { concurrency: true }, async () => { + await describe('validation', async () => { + await it('should only allow boolean in options.coverage', async () => { + [Symbol(), {}, () => {}, 0, 1, 0n, 1n, '', '1', Promise.resolve(true), []] + .forEach((coverage) => assert.throws(() => run({ coverage }), { + code: 'ERR_INVALID_ARG_TYPE' + })); + }); + + await it('should only allow string|string[] in options.coverageExcludeGlobs', async () => { + [Symbol(), {}, () => {}, 0, 1, 0n, 1n, Promise.resolve([]), true, false] + .forEach((coverageExcludeGlobs) => { + assert.throws(() => run({ coverage: true, coverageExcludeGlobs }), { + code: 'ERR_INVALID_ARG_TYPE' + }); + assert.throws(() => run({ coverage: true, coverageExcludeGlobs: [coverageExcludeGlobs] }), { + code: 'ERR_INVALID_ARG_TYPE' + }); + }); + run({ files: [], signal: abortedSignal, coverage: true, coverageExcludeGlobs: [''] }); + run({ files: [], signal: abortedSignal, coverage: true, coverageExcludeGlobs: '' }); + }); + + await it('should only allow string|string[] in options.coverageIncludeGlobs', async () => { + [Symbol(), {}, () => {}, 0, 1, 0n, 1n, Promise.resolve([]), true, false] + .forEach((coverageIncludeGlobs) => { + assert.throws(() => run({ coverage: true, coverageIncludeGlobs }), { + code: 'ERR_INVALID_ARG_TYPE' + }); + assert.throws(() => run({ coverage: true, coverageIncludeGlobs: [coverageIncludeGlobs] }), { + code: 'ERR_INVALID_ARG_TYPE' + }); + }); + + run({ files: [], signal: abortedSignal, coverage: true, coverageIncludeGlobs: [''] }); + run({ files: [], signal: abortedSignal, coverage: true, coverageIncludeGlobs: '' }); + }); + + await it('should only allow an int within range in options.lineCoverage', async () => { + [Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false] + .forEach((lineCoverage) => { + assert.throws(() => run({ coverage: true, lineCoverage }), { + code: 'ERR_INVALID_ARG_TYPE' + }); + assert.throws(() => run({ coverage: true, lineCoverage: [lineCoverage] }), { + code: 'ERR_INVALID_ARG_TYPE' + }); + }); + assert.throws(() => run({ coverage: true, lineCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => run({ coverage: true, lineCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' }); + + run({ files: [], signal: abortedSignal, coverage: true, lineCoverage: 0 }); + }); + + await it('should only allow an int within range in options.branchCoverage', async () => { + [Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false] + .forEach((branchCoverage) => { + assert.throws(() => run({ coverage: true, branchCoverage }), { + code: 'ERR_INVALID_ARG_TYPE' + }); + assert.throws(() => run({ coverage: true, branchCoverage: [branchCoverage] }), { + code: 'ERR_INVALID_ARG_TYPE' + }); + }); + + assert.throws(() => run({ coverage: true, branchCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => run({ coverage: true, branchCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' }); + + run({ files: [], signal: abortedSignal, coverage: true, branchCoverage: 0 }); + }); + + await it('should only allow an int within range in options.functionCoverage', async () => { + [Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false] + .forEach((functionCoverage) => { + assert.throws(() => run({ coverage: true, functionCoverage }), { + code: 'ERR_INVALID_ARG_TYPE' + }); + assert.throws(() => run({ coverage: true, functionCoverage: [functionCoverage] }), { + code: 'ERR_INVALID_ARG_TYPE' + }); + }); + + assert.throws(() => run({ coverage: true, functionCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' }); + assert.throws(() => run({ coverage: true, functionCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' }); + + run({ files: [], signal: abortedSignal, coverage: true, functionCoverage: 0 }); + }); + }); + + const options = { concurrency: false, skip: !process.features.inspector ? 'inspector disabled' : false }; + await describe('run with coverage', options, async () => { + await it('should run with coverage', async () => { + const stream = run({ files, coverage: true }); + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall()); + stream.on('test:coverage', common.mustCall()); + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); + }); + + await it('should run with coverage and exclude by glob', async () => { + const stream = run({ files, coverage: true, coverageExcludeGlobs: ['test/*/test-runner/invalid-tap.js'] }); + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall(1)); + stream.on('test:coverage', common.mustCall(({ summary: { files } }) => { + const filesPaths = files.map(({ path }) => path); + assert.strictEqual(filesPaths.some((path) => path.includes(`test-runner${sep}invalid-tap.js`)), false); + })); + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); + }); + + await it('should run with coverage and include by glob', async () => { + const stream = run({ + files, + coverage: true, + coverageIncludeGlobs: ['test/fixtures/test-runner/coverage.js', 'test/*/v8-coverage/throw.js'], + }); + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall(1)); + stream.on('test:coverage', common.mustCall(({ summary: { files } }) => { + const filesPaths = files.map(({ path }) => path); + assert.strictEqual(filesPaths.some((path) => path.includes(`v8-coverage${sep}throw.js`)), true); + })); + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); + }); + + await it('should run while including and excluding globs', async () => { + const stream = run({ + files: [...files, fixtures.path('test-runner/invalid-tap.js')], + coverage: true, + coverageIncludeGlobs: ['test/fixtures/test-runner/*.js'], + coverageExcludeGlobs: ['test/fixtures/test-runner/*-tap.js'] + }); + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall(2)); + stream.on('test:coverage', common.mustCall(({ summary: { files } }) => { + const filesPaths = files.map(({ path }) => path); + assert.strictEqual(filesPaths.every((path) => !path.includes(`test-runner${sep}invalid-tap.js`)), true); + assert.strictEqual(filesPaths.some((path) => path.includes(`test-runner${sep}coverage.js`)), true); + })); + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); + }); + + await it('should run with coverage and fail when below line threshold', async () => { + const thresholdErrors = []; + const originalExitCode = process.exitCode; + assert.notStrictEqual(originalExitCode, 1); + const stream = run({ files, coverage: true, lineCoverage: 99, branchCoverage: 99, functionCoverage: 99 }); + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall(1)); + stream.on('test:diagnostic', ({ message }) => { + const match = message.match(/Error: \d{2}\.\d{2}% (line|branch|function) coverage does not meet threshold of 99%/); + if (match) { + thresholdErrors.push(match[1]); + } + }); + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); + assert.deepStrictEqual(thresholdErrors.sort(), ['branch', 'function', 'line']); + assert.strictEqual(process.exitCode, 1); + process.exitCode = originalExitCode; + }); + }); +}); + + +// exitHandler doesn't run until after the tests / after hooks finish. +process.on('exit', () => { + assert.strictEqual(process.listeners('uncaughtException').length, 0); + assert.strictEqual(process.listeners('unhandledRejection').length, 0); + assert.strictEqual(process.listeners('beforeExit').length, 0); + assert.strictEqual(process.listeners('SIGINT').length, 0); + assert.strictEqual(process.listeners('SIGTERM').length, 0); +});