From bb211c19712583056efbe688b53653e301663521 Mon Sep 17 00:00:00 2001 From: pulkit-30 Date: Wed, 20 Dec 2023 16:10:44 +0530 Subject: [PATCH] test_runner: added coverage threshold support for tests --- lib/internal/test_runner/coverage.js | 35 +++++++- lib/internal/test_runner/harness.js | 19 ++++- lib/internal/test_runner/runner.js | 12 ++- lib/internal/test_runner/test.js | 13 ++- lib/internal/test_runner/utils.js | 12 ++- test/parallel/test-runner-coverage.js | 114 ++++++++++++++++---------- test/parallel/test-runner-run.mjs | 38 +++++++++ 7 files changed, 189 insertions(+), 54 deletions(-) diff --git a/lib/internal/test_runner/coverage.js b/lib/internal/test_runner/coverage.js index 7727ab006052ba..75b4ea2bff6b57 100644 --- a/lib/internal/test_runner/coverage.js +++ b/lib/internal/test_runner/coverage.js @@ -31,6 +31,12 @@ const kIgnoreRegex = /\/\* node:coverage ignore next (?\d+ )?\*\//; const kLineEndingRegex = /\r?\n$/u; const kLineSplitRegex = /(?<=\r?\n)/u; const kStatusRegex = /\/\* node:coverage (?enable|disable) \*\//; +const kDefaultMinimumCoverage = { + __proto__: null, + line: 0, + function: 0, + branch: 0, +}; class CoverageLine { #covered; @@ -63,10 +69,11 @@ class CoverageLine { } class TestCoverage { - constructor(coverageDirectory, originalCoverageDirectory, workingDirectory) { + constructor(coverageDirectory, originalCoverageDirectory, workingDirectory, minimumCoverage) { this.coverageDirectory = coverageDirectory; this.originalCoverageDirectory = originalCoverageDirectory; this.workingDirectory = workingDirectory; + this.minimumCoverage = minimumCoverage || kDefaultMinimumCoverage; } summary() { @@ -260,6 +267,24 @@ class TestCoverage { ); coverageSummary.files.sort(sortCoverageFiles); + coverageSummary.minimumCoverage = { + __proto__: null, + line: { + status: doesCoveragePass(this.minimumCoverage.line, coverageSummary.totals.coveredLinePercent), + expected: this.minimumCoverage.line, + actual: coverageSummary.totals.coveredLinePercent, + }, + function: { + status: doesCoveragePass(this.minimumCoverage.function, coverageSummary.totals.coveredFunctionPercent), + expected:this.minimumCoverage.function, + actual: coverageSummary.totals.coveredFunctionPercent + }, + branch: { + status: doesCoveragePass(this.minimumCoverage.branch, coverageSummary.totals.coveredBranchPercent), + expected: this.minimumCoverage.branch, + actual: coverageSummary.totals.coveredBranchPercent + }, + }; return coverageSummary; } @@ -299,7 +324,7 @@ function sortCoverageFiles(a, b) { return StringPrototypeLocaleCompare(a.path, b.path); } -function setupCoverage() { +function setupCoverage(minimumCoverage = null) { let originalCoverageDirectory = process.env.NODE_V8_COVERAGE; const cwd = process.cwd(); @@ -323,7 +348,7 @@ function setupCoverage() { // child processes. process.env.NODE_V8_COVERAGE = coverageDirectory; - return new TestCoverage(coverageDirectory, originalCoverageDirectory, cwd); + return new TestCoverage(coverageDirectory, originalCoverageDirectory, cwd, minimumCoverage); } function mapRangeToLines(range, lines) { @@ -538,4 +563,8 @@ function doesRangeContainOtherRange(range, otherRange) { range.endOffset >= otherRange.endOffset; } +function doesCoveragePass(compareValue, actualValue) { + return compareValue ? actualValue >= compareValue : true; +} + module.exports = { setupCoverage, TestCoverage }; diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index 2f18b0bcf091ac..f34cc90df44eb3 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -1,9 +1,12 @@ 'use strict'; const { + ArrayPrototypeFilter, ArrayPrototypeForEach, FunctionPrototypeBind, PromiseResolve, SafeMap, + StringPrototypeSplit, + StringPrototypeStartsWith, } = primordials; const { getCallerLocation } = internalBinding('util'); const { @@ -78,14 +81,22 @@ function createProcessEventHandler(eventName, rootTest) { } function configureCoverage(rootTest, globalOptions) { - if (!globalOptions.coverage) { - return null; - } + let minimumCoverage = rootTest.coverage; + if (!minimumCoverage) { + if (!globalOptions.coverage) { + return null; + } + const { 1: value } = StringPrototypeSplit( + ArrayPrototypeFilter(process.execArgv, (arg) => + StringPrototypeStartsWith(arg, '--experimental-test-coverage')), '='); + + minimumCoverage = { __proto__: null, line: value, function: value, branch: value }; + } const { setupCoverage } = require('internal/test_runner/coverage'); try { - return setupCoverage(); + return setupCoverage(minimumCoverage); } catch (err) { const msg = `Warning: Code coverage could not be enabled. ${err}`; diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index 34100213ebd935..ecccc3192c0acc 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -439,7 +439,7 @@ function run(options = kEmptyObject) { validateObject(options, 'options'); let { testNamePatterns, shard } = options; - const { concurrency, timeout, signal, files, inspectPort, watch, setup, only } = options; + const { concurrency, timeout, signal, files, inspectPort, watch, setup, only, coverage } = options; if (files != null) { validateArray(files, 'options.files'); @@ -486,7 +486,15 @@ function run(options = kEmptyObject) { }); } - const root = createTestTree({ __proto__: null, concurrency, timeout, signal }); + if (coverage != null) { + if (typeof coverage === 'boolean') { + validateBoolean(coverage, 'options.coverage'); + } else { + validateObject(coverage, 'options.coverage'); + } + } + + const root = createTestTree({ __proto__: null, concurrency, timeout, signal, coverage }); if (process.env.NODE_TEST_CONTEXT !== undefined) { return root.reporter; } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 19ae283ed9ad78..e65e5786cabbef 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -1,5 +1,6 @@ 'use strict'; const { + ArrayPrototypeFilter, ArrayPrototypePush, ArrayPrototypePushApply, ArrayPrototypeReduce, @@ -10,6 +11,8 @@ const { FunctionPrototype, MathMax, Number, + ObjectDefineProperty, + ObjectKeys, ObjectSeal, PromisePrototypeThen, PromiseResolve, @@ -21,7 +24,6 @@ const { SafePromiseAll, SafePromiseRace, SymbolDispose, - ObjectDefineProperty, Symbol, } = primordials; const { getCallerLocation } = internalBinding('util'); @@ -209,7 +211,7 @@ class Test extends AsyncResource { super('Test'); let { fn, name, parent, skip } = options; - const { concurrency, loc, only, timeout, todo, signal } = options; + const { concurrency, loc, only, timeout, todo, signal, coverage } = options; if (typeof fn !== 'function') { fn = noop; @@ -239,6 +241,7 @@ class Test extends AsyncResource { beforeEach: [], afterEach: [], }; + this.coverage = coverage; } else { const nesting = parent.parent === null ? parent.nesting : parent.nesting + 1; @@ -258,6 +261,7 @@ class Test extends AsyncResource { beforeEach: ArrayPrototypeSlice(parent.hooks.beforeEach), afterEach: ArrayPrototypeSlice(parent.hooks.afterEach), }; + this.coverage = parent.coverage; } switch (typeof concurrency) { @@ -754,6 +758,11 @@ class Test extends AsyncResource { if (coverage) { reporter.coverage(nesting, loc, coverage); + const failedCoverage = ArrayPrototypeFilter( + ObjectKeys(coverage.minimumCoverage), (key) => !coverage.minimumCoverage[key].status); + if (failedCoverage.length > 0) { + process.exitCode = 1; + } } reporter.end(); diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 6b4663f14302c3..f13af33b16dca5 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -3,17 +3,19 @@ const { ArrayPrototypeJoin, ArrayPrototypeMap, ArrayPrototypeFlatMap, + ArrayPrototypeForEach, ArrayPrototypePush, ArrayPrototypeReduce, ObjectGetOwnPropertyDescriptor, + ObjectKeys, MathFloor, MathMax, MathMin, NumberPrototypeToFixed, - SafePromiseAllReturnArrayLike, RegExp, RegExpPrototypeExec, SafeMap, + SafePromiseAllReturnArrayLike, StringPrototypePadStart, StringPrototypePadEnd, StringPrototypeRepeat, @@ -316,6 +318,7 @@ const kSeparator = ' | '; function getCoverageReport(pad, summary, symbol, color, table) { const prefix = `${pad}${symbol}`; + color ||= white; let report = `${color}${prefix}start of coverage report\n`; let filePadLength; @@ -405,6 +408,13 @@ function getCoverageReport(pad, summary, symbol, color, table) { `${ArrayPrototypeJoin(ArrayPrototypeMap(kColumnsKeys, (columnKey, j) => getCell(NumberPrototypeToFixed(summary.totals[columnKey], 2), columnPadLengths[j], StringPrototypePadStart, false, summary.totals[columnKey])), kSeparator)} |\n`; if (table) report += addTableLine(prefix, tableWidth); + ArrayPrototypeForEach(ObjectKeys(summary.minimumCoverage), (key) => { + const { actual, expected, status } = summary.minimumCoverage[key] + if (!status) { + report += `${red}${pad}${prefix}minimum coverage failed for ${key}. expected: ${expected} | actual: ${actual}\n${color}`; + } + }); + report += `${prefix}end of coverage report\n`; if (color) { report += white; diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js index d8181417205b46..17ae43019f7ecf 100644 --- a/test/parallel/test-runner-coverage.js +++ b/test/parallel/test-runner-coverage.js @@ -21,46 +21,24 @@ function findCoverageFileForPid(pid) { }); } -function getTapCoverageFixtureReport() { - /* eslint-disable max-len */ - const report = [ - '# start of coverage report', - '# -------------------------------------------------------------------------------------------------------------------', - '# file | line % | branch % | funcs % | uncovered lines', - '# -------------------------------------------------------------------------------------------------------------------', - '# test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72', - '# test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | ', - '# test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5-6', - '# -------------------------------------------------------------------------------------------------------------------', - '# all files | 78.35 | 43.75 | 60.00 |', - '# -------------------------------------------------------------------------------------------------------------------', - '# end of coverage report', - ].join('\n'); - /* eslint-enable max-len */ - - if (common.isWindows) { - return report.replaceAll('/', '\\'); - } +function getCoverageFixtureReport(reporter = 'tap', extraReport = []) { + const symbol = reporter === 'spec' ? '\u2139' : '#'; - return report; -} - -function getSpecCoverageFixtureReport() { - /* eslint-disable max-len */ const report = [ - '\u2139 start of coverage report', - '\u2139 -------------------------------------------------------------------------------------------------------------------', - '\u2139 file | line % | branch % | funcs % | uncovered lines', - '\u2139 -------------------------------------------------------------------------------------------------------------------', - '\u2139 test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72', - '\u2139 test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | ', - '\u2139 test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5-6', - '\u2139 -------------------------------------------------------------------------------------------------------------------', - '\u2139 all files | 78.35 | 43.75 | 60.00 |', - '\u2139 -------------------------------------------------------------------------------------------------------------------', - '\u2139 end of coverage report', + `${symbol} start of coverage report`, + `${symbol} -------------------------------------------------------------------------------------------------------------------`, + `${symbol} file | line % | branch % | funcs % | uncovered lines`, + `${symbol} -------------------------------------------------------------------------------------------------------------------`, + `${symbol} test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72`, + `${symbol} test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | `, + `${symbol} test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5-6`, + `${symbol} -------------------------------------------------------------------------------------------------------------------`, + `${symbol} all files | 78.35 | 43.75 | 60.00 |`, + `${symbol} -------------------------------------------------------------------------------------------------------------------`, + ...extraReport, + `${symbol} end of coverage report`, ].join('\n'); - /* eslint-enable max-len */ + if (common.isWindows) { return report.replaceAll('/', '\\'); @@ -69,6 +47,7 @@ function getSpecCoverageFixtureReport() { return report; } + test('test coverage report', async (t) => { await t.test('handles the inspector not being available', (t) => { if (process.features.inspector) { @@ -92,7 +71,7 @@ test('test tap coverage reporter', skipIfNoInspector, async (t) => { const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture]; const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } }; const result = spawnSync(process.execPath, args, options); - const report = getTapCoverageFixtureReport(); + const report = getCoverageFixtureReport(); assert(result.stdout.toString().includes(report)); assert.strictEqual(result.stderr.toString(), ''); assert.strictEqual(result.status, 0); @@ -103,7 +82,7 @@ test('test tap coverage reporter', skipIfNoInspector, async (t) => { const fixture = fixtures.path('test-runner', 'coverage.js'); const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture]; const result = spawnSync(process.execPath, args); - const report = getTapCoverageFixtureReport(); + const report = getCoverageFixtureReport(); assert(result.stdout.toString().includes(report)); assert.strictEqual(result.stderr.toString(), ''); @@ -118,7 +97,7 @@ test('test spec coverage reporter', skipIfNoInspector, async (t) => { const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture]; const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } }; const result = spawnSync(process.execPath, args, options); - const report = getSpecCoverageFixtureReport(); + const report = getCoverageFixtureReport('spec'); assert(result.stdout.toString().includes(report)); assert.strictEqual(result.stderr.toString(), ''); @@ -130,7 +109,7 @@ test('test spec coverage reporter', skipIfNoInspector, async (t) => { const fixture = fixtures.path('test-runner', 'coverage.js'); const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture]; const result = spawnSync(process.execPath, args); - const report = getSpecCoverageFixtureReport(); + const report = getCoverageFixtureReport('spec'); assert(result.stdout.toString().includes(report)); assert.strictEqual(result.stderr.toString(), ''); @@ -145,7 +124,7 @@ test('single process coverage is the same with --test', skipIfNoInspector, () => '--test', '--experimental-test-coverage', '--test-reporter', 'tap', fixture, ]; const result = spawnSync(process.execPath, args); - const report = getTapCoverageFixtureReport(); + const report = getCoverageFixtureReport(); assert.strictEqual(result.stderr.toString(), ''); assert(result.stdout.toString().includes(report)); @@ -242,3 +221,54 @@ test('coverage reports on lines, functions, and branches', skipIfNoInspector, as }); }); }); + +test('test coverage for tap reporter with min 100% threshold value', async (t) => { + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--experimental-test-coverage=100', '--test-reporter', 'tap', fixture]; + const options = { env: { ...process.env } }; + const result = spawnSync(process.execPath, args, options); + const report = getCoverageFixtureReport('tap', + ['# coverage threshold for lines not met', + '# coverage threshold for branches not met', + '# coverage threshold for functions not met']); + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.stderr.toString(), ''); + assert.strictEqual(result.status, 0); +}); + +test('test coverage for spec reporter with min 100% threshold value', async () => { + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--test', '--experimental-test-coverage=100', '--test-reporter', 'spec', fixture]; + const options = { env: { ...process.env } }; + const result = spawnSync(process.execPath, args, options); + const report = getCoverageFixtureReport('spec', + ['\u2139 coverage threshold for lines not met', + '\u2139 coverage threshold for branches not met', + '\u2139 coverage threshold for functions not met']); + assert.strictEqual(result.stderr.toString(), ''); + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.status, 0); +}); + + +test('test coverage for tap reporter with min 10% threshold value', async (t) => { + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--experimental-test-coverage=10', '--test-reporter', 'tap', fixture]; + const options = { env: { ...process.env } }; + const result = spawnSync(process.execPath, args, options); + const report = getCoverageFixtureReport('tap'); + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.stderr.toString(), ''); + assert.strictEqual(result.status, 0); +}); + +test('test coverage for spec reporter with min 10% threshold value', async () => { + const fixture = fixtures.path('test-runner', 'coverage.js'); + const args = ['--test', '--experimental-test-coverage=10', '--test-reporter', 'spec', fixture]; + const options = { env: { ...process.env } }; + const result = spawnSync(process.execPath, args, options); + const report = getCoverageFixtureReport('spec'); + assert.strictEqual(result.stderr.toString(), ''); + assert(result.stdout.toString().includes(report)); + assert.strictEqual(result.status, 0); +}); diff --git a/test/parallel/test-runner-run.mjs b/test/parallel/test-runner-run.mjs index c712f500153b42..1d41f0d7706668 100644 --- a/test/parallel/test-runner-run.mjs +++ b/test/parallel/test-runner-run.mjs @@ -500,3 +500,41 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { for await (const _ of stream); }); }); + +describe('coverage report for test stream', () => { + it('should run coverage report', async () => { + const stream = run({ files: [join(testFixtures, 'default-behavior/test/random.cjs')], + coverage: { + lines: 100, + functions: 100, + branches: 100 + } }); + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall()); + stream.on('test:coverage', common.mustCall()); + + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); + }); + it('should run coverage report', async () => { + const stream = run({ files: [join(testFixtures, 'default-behavior/test/random.cjs')], + coverage: false }); + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall()); + stream.on('test:coverage', common.mustNotCall()); + + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); + }); + + it('should run coverage report', async () => { + const stream = run({ files: [join(testFixtures, 'default-behavior/test/random.cjs')], + coverage: true }); + stream.on('test:fail', common.mustNotCall()); + stream.on('test:pass', common.mustCall()); + stream.on('test:coverage', common.mustCall()); + + // eslint-disable-next-line no-unused-vars + for await (const _ of stream); + }); +});