From a52c199d9d68ab3dd536d16c4ebdb7981efa1314 Mon Sep 17 00:00:00 2001 From: Aviv Keller Date: Fri, 13 Sep 2024 09:31:37 -0400 Subject: [PATCH] test_runner: report coverage thresholds in `test:coverage` PR-URL: https://github.com/nodejs/node/pull/54813 Reviewed-By: Matteo Collina Reviewed-By: Moshe Atlow Reviewed-By: Chemi Atlow Reviewed-By: James M Snell Reviewed-By: Colin Ihrig --- doc/api/test.md | 5 ++ lib/internal/test_runner/coverage.js | 10 +++- lib/internal/test_runner/test.js | 4 +- .../test-runner-coverage-thresholds.js | 57 +++++++++++++++---- 4 files changed, 61 insertions(+), 15 deletions(-) diff --git a/doc/api/test.md b/doc/api/test.md index 02c34d9437ca1a..48214a4b75f56f 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -2806,6 +2806,11 @@ are defined, while others are emitted in the order that the tests execute. numbers and the number of times they were covered. * `line` {number} The line number. * `count` {number} The number of times the line was covered. + * `thresholds` {Object} An object containing whether or not the coverage for + each coverage type. + * `function` {number} The function coverage threshold. + * `branch` {number} The branch coverage threshold. + * `line` {number} The line coverage threshold. * `totals` {Object} An object containing a summary of coverage for all files. * `totalLineCount` {number} The total number of lines. diff --git a/lib/internal/test_runner/coverage.js b/lib/internal/test_runner/coverage.js index 4f9bd8d20e3409..7ef57020728302 100644 --- a/lib/internal/test_runner/coverage.js +++ b/lib/internal/test_runner/coverage.js @@ -51,12 +51,13 @@ class CoverageLine { } class TestCoverage { - constructor(coverageDirectory, originalCoverageDirectory, workingDirectory, excludeGlobs, includeGlobs) { + constructor(coverageDirectory, originalCoverageDirectory, workingDirectory, excludeGlobs, includeGlobs, thresholds) { this.coverageDirectory = coverageDirectory; this.originalCoverageDirectory = originalCoverageDirectory; this.workingDirectory = workingDirectory; this.excludeGlobs = excludeGlobs; this.includeGlobs = includeGlobs; + this.thresholds = thresholds; } #sourceLines = new SafeMap(); @@ -143,6 +144,7 @@ class TestCoverage { coveredBranchPercent: 0, coveredFunctionPercent: 0, }, + thresholds: this.thresholds, }; if (!coverage) { @@ -505,6 +507,12 @@ function setupCoverage(options) { cwd, options.coverageExcludeGlobs, options.coverageIncludeGlobs, + { + __proto__: null, + line: options.lineCoverage, + branch: options.branchCoverage, + function: options.functionCoverage, + }, ); } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index bfb0712f633b1b..41db230119aab0 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -1043,8 +1043,6 @@ class Test extends AsyncResource { reporter.diagnostic(nesting, loc, `duration_ms ${this.duration()}`); if (coverage) { - reporter.coverage(nesting, loc, coverage); - const coverages = [ { __proto__: null, actual: coverage.totals.coveredLinePercent, threshold: this.config.lineCoverage, name: 'line' }, @@ -1063,6 +1061,8 @@ class Test extends AsyncResource { reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`); } } + + reporter.coverage(nesting, loc, coverage); } if (harness.watching) { diff --git a/test/parallel/test-runner-coverage-thresholds.js b/test/parallel/test-runner-coverage-thresholds.js index 8ee15f4c65b7fe..a57231ea2e661a 100644 --- a/test/parallel/test-runner-coverage-thresholds.js +++ b/test/parallel/test-runner-coverage-thresholds.js @@ -44,10 +44,7 @@ function getTapCoverageFixtureReport() { } const fixture = fixtures.path('test-runner', 'coverage.js'); -const neededArguments = [ - '--experimental-test-coverage', - '--test-reporter', 'tap', -]; +const reporter = fixtures.fileURL('test-runner/custom_reporters/coverage.mjs'); const coverages = [ { flag: '--test-coverage-lines', name: 'line', actual: 78.35 }, @@ -56,10 +53,12 @@ const coverages = [ ]; for (const coverage of coverages) { - test(`test passing ${coverage.flag}`, async (t) => { + test(`test passing ${coverage.flag}`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=25`, + '--test-reporter', 'tap', fixture, ]); @@ -70,10 +69,27 @@ for (const coverage of coverages) { assert(!findCoverageFileForPid(result.pid)); }); - test(`test failing ${coverage.flag}`, async (t) => { + test(`test passing ${coverage.flag} with custom reporter`, () => { + const result = spawnSync(process.execPath, [ + '--test', + '--experimental-test-coverage', + `${coverage.flag}=25`, + '--test-reporter', reporter, + fixture, + ]); + + const stdout = JSON.parse(result.stdout.toString()); + assert.strictEqual(stdout.summary.thresholds[coverage.name], 25); + assert.strictEqual(result.status, 0); + assert(!findCoverageFileForPid(result.pid)); + }); + + test(`test failing ${coverage.flag}`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=99`, + '--test-reporter', 'tap', fixture, ]); @@ -84,9 +100,25 @@ for (const coverage of coverages) { assert(!findCoverageFileForPid(result.pid)); }); - test(`test out-of-range ${coverage.flag} (too high)`, async (t) => { + test(`test failing ${coverage.flag} with custom reporter`, () => { + const result = spawnSync(process.execPath, [ + '--test', + '--experimental-test-coverage', + `${coverage.flag}=99`, + '--test-reporter', reporter, + fixture, + ]); + + const stdout = JSON.parse(result.stdout.toString()); + assert.strictEqual(stdout.summary.thresholds[coverage.name], 99); + assert.strictEqual(result.status, 1); + assert(!findCoverageFileForPid(result.pid)); + }); + + test(`test out-of-range ${coverage.flag} (too high)`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=101`, fixture, ]); @@ -96,9 +128,10 @@ for (const coverage of coverages) { assert(!findCoverageFileForPid(result.pid)); }); - test(`test out-of-range ${coverage.flag} (too low)`, async (t) => { + test(`test out-of-range ${coverage.flag} (too low)`, () => { const result = spawnSync(process.execPath, [ - ...neededArguments, + '--test', + '--experimental-test-coverage', `${coverage.flag}=-1`, fixture, ]);