From 9dba88aa6b8899127d50a8166bcc76686a40e239 Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Fri, 25 Aug 2023 15:46:03 +1000 Subject: [PATCH 1/6] test_runner: report covered lines, functions and branches to reporters This is a breaking change for the format of test:coverage events. But the test coverage is still experimental, so I don't believe it requires a semver-major bump. Fixes https://github.com/nodejs/node/issues/49303 --- doc/api/test.md | 14 +++++++++++-- lib/internal/test_runner/coverage.js | 30 +++++++++++++++++++++++++--- lib/internal/test_runner/utils.js | 16 ++++++++++++--- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/doc/api/test.md b/doc/api/test.md index 36f6851c6acf17..92cf0169d4fbd0 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -2011,8 +2011,18 @@ object, streaming a series of events representing the execution of the tests. * `coveredLinePercent` {number} The percentage of lines covered. * `coveredBranchPercent` {number} The percentage of branches covered. * `coveredFunctionPercent` {number} The percentage of functions covered. - * `uncoveredLineNumbers` {Array} An array of integers representing line - numbers that are uncovered. + * `functions` {Array} An array of functions representing function + coverage. + * `name` {string} The name of the function. + * `line` {number} The line number where the function is defined. + * `count` {number} The number of times the function was called. + * `branches` {Array} An array of branches representing branch coverage. + * `line` {number} The line number where the branch is defined. + * `count` {number} The number of times the branch was taken. + * `lines` {Array} An array of lines representing line + numbers and the number of times they were covered. + * `line` {number} The line number. + * `count` {number} The number of times the line was covered. * `totals` {Object} An object containing a summary of coverage for all files. * `totalLineCount` {number} The total number of lines. diff --git a/lib/internal/test_runner/coverage.js b/lib/internal/test_runner/coverage.js index 70f88984c82150..978537ee8d85c0 100644 --- a/lib/internal/test_runner/coverage.js +++ b/lib/internal/test_runner/coverage.js @@ -13,6 +13,7 @@ const { StringPrototypeIncludes, StringPrototypeLocaleCompare, StringPrototypeStartsWith, + MathMax } = primordials; const { copyFileSync, @@ -43,6 +44,7 @@ class CoverageLine { this.startOffset = startOffset; this.endOffset = startOffset + src.length - newlineLength; this.ignore = false; + this.count = 0; this.#covered = true; } @@ -118,6 +120,8 @@ class TestCoverage { let totalFunctions = 0; let branchesCovered = 0; let functionsCovered = 0; + const functionReports = []; + const branchReports = []; const lines = ArrayPrototypeMap(linesWithBreaks, (line, i) => { const startOffset = offset; @@ -165,6 +169,11 @@ class TestCoverage { mapRangeToLines(range, lines); if (isBlockCoverage) { + ArrayPrototypePush(branchReports, { + line: range.lines[0].line, + count: range.count + }); + if (range.count !== 0 || range.ignoredLines === range.lines.length) { branchesCovered++; @@ -177,6 +186,12 @@ class TestCoverage { if (j > 0 && ranges.length > 0) { const range = ranges[0]; + ArrayPrototypePush(functionReports, { + name: functions[j].functionName, + count: MathMax(...ArrayPrototypeMap(ranges, r => r.count)), + line: range.lines[0].line + }); + if (range.count !== 0 || range.ignoredLines === range.lines.length) { functionsCovered++; } @@ -186,15 +201,18 @@ class TestCoverage { } let coveredCnt = 0; - const uncoveredLineNums = []; + const lineReports = []; for (let j = 0; j < lines.length; ++j) { const line = lines[j]; if (line.covered || line.ignore) { coveredCnt++; + if (!line.ignore) { + ArrayPrototypePush(lineReports, { line: line.line, count: line.count }) + } } else { - ArrayPrototypePush(uncoveredLineNums, line.line); + ArrayPrototypePush(lineReports, { line: line.line, count: 0 }); } } @@ -210,7 +228,9 @@ class TestCoverage { coveredLinePercent: toPercentage(coveredCnt, lines.length), coveredBranchPercent: toPercentage(branchesCovered, totalBranches), coveredFunctionPercent: toPercentage(functionsCovered, totalFunctions), - uncoveredLineNumbers: uncoveredLineNums, + functions: functionReports, + branches: branchReports, + lines: lineReports, }); coverageSummary.totals.totalLineCount += lines.length; @@ -321,6 +341,10 @@ function mapRangeToLines(range, lines) { endOffset >= line.endOffset) { line.covered = false; } + if (count > 0 && startOffset <= line.startOffset && + endOffset >= line.endOffset) { + line.count = count; + } ArrayPrototypePush(mappedLines, line); diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index fba2c3132339aa..7a68c8dba72fde 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -297,6 +297,15 @@ function formatLinesToRanges(values) { }, []), (range) => ArrayPrototypeJoin(range, '-')); } +function getUncoveredLines(lines) { + return ArrayPrototypeReduce(lines, (acc, line) => { + if (line.count === 0) { + ArrayPrototypePush(acc, line.line); + } + return acc; + }, []); +} + function formatUncoveredLines(lines, table) { if (table) return ArrayPrototypeJoin(formatLinesToRanges(lines), ' '); return ArrayPrototypeJoin(lines, ', '); @@ -325,8 +334,9 @@ function getCoverageReport(pad, summary, symbol, color, table) { columnPadLengths = ArrayPrototypeMap(kColumns, (column) => (table ? MathMax(column.length, 6) : 0)); const columnsWidth = ArrayPrototypeReduce(columnPadLengths, (acc, columnPadLength) => acc + columnPadLength + 3, 0); - uncoveredLinesPadLength = table && ArrayPrototypeReduce(summary.files, (acc, file) => - MathMax(acc, formatUncoveredLines(file.uncoveredLineNumbers, table).length), 0); + uncoveredLinesPadLength = table && ArrayPrototypeReduce(summary.files, (acc, file) => { + return MathMax(acc, formatUncoveredLines(getUncoveredLines(file.lines), table).length) + }, 0); uncoveredLinesPadLength = MathMax(uncoveredLinesPadLength, 'uncovered lines'.length); const uncoveredLinesWidth = uncoveredLinesPadLength + 2; @@ -388,7 +398,7 @@ function getCoverageReport(pad, summary, symbol, color, table) { report += `${prefix}${getCell(relativePath, filePadLength, StringPrototypePadEnd, truncateStart, fileCoverage)}${kSeparator}` + `${ArrayPrototypeJoin(ArrayPrototypeMap(coverages, (coverage, j) => getCell(NumberPrototypeToFixed(coverage, 2), columnPadLengths[j], StringPrototypePadStart, false, coverage)), kSeparator)}${kSeparator}` + - `${getCell(formatUncoveredLines(file.uncoveredLineNumbers, table), uncoveredLinesPadLength, false, truncateEnd)}\n`; + `${getCell(formatUncoveredLines(getUncoveredLines(file.lines), table), uncoveredLinesPadLength, false, truncateEnd)}\n`; } // Foot From 85e4ba5ab1d6ec2c8f5d80aa44217fa769ba0a86 Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Fri, 25 Aug 2023 16:53:39 +1000 Subject: [PATCH 2/6] test_runner: fixing style issues --- lib/internal/test_runner/coverage.js | 20 +++++++++++++++----- lib/internal/test_runner/utils.js | 9 +++------ test/parallel/test-runner-coverage.js | 5 +++++ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/lib/internal/test_runner/coverage.js b/lib/internal/test_runner/coverage.js index 978537ee8d85c0..b501b012096923 100644 --- a/lib/internal/test_runner/coverage.js +++ b/lib/internal/test_runner/coverage.js @@ -13,7 +13,7 @@ const { StringPrototypeIncludes, StringPrototypeLocaleCompare, StringPrototypeStartsWith, - MathMax + MathMax, } = primordials; const { copyFileSync, @@ -170,8 +170,9 @@ class TestCoverage { if (isBlockCoverage) { ArrayPrototypePush(branchReports, { + __proto__: null, line: range.lines[0].line, - count: range.count + count: range.count, }); if (range.count !== 0 || @@ -187,9 +188,10 @@ class TestCoverage { const range = ranges[0]; ArrayPrototypePush(functionReports, { + __proto__: null, name: functions[j].functionName, count: MathMax(...ArrayPrototypeMap(ranges, r => r.count)), - line: range.lines[0].line + line: range.lines[0].line, }); if (range.count !== 0 || range.ignoredLines === range.lines.length) { @@ -209,10 +211,18 @@ class TestCoverage { if (line.covered || line.ignore) { coveredCnt++; if (!line.ignore) { - ArrayPrototypePush(lineReports, { line: line.line, count: line.count }) + ArrayPrototypePush(lineReports, { + __proto__: null, + line: line.line, + count: line.count, + }) } } else { - ArrayPrototypePush(lineReports, { line: line.line, count: 0 }); + ArrayPrototypePush(lineReports, { + __proto__: null, + line: line.line, + count: 0, + }); } } diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 7a68c8dba72fde..78e6c259f1598d 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -2,6 +2,7 @@ const { ArrayPrototypeJoin, ArrayPrototypeMap, + ArrayPrototypeFlatMap, ArrayPrototypePush, ArrayPrototypeReduce, ObjectGetOwnPropertyDescriptor, @@ -34,6 +35,7 @@ const { kIsNodeError, } = require('internal/errors'); const { compose } = require('stream'); +const console = require('console'); const coverageColors = { __proto__: null, @@ -298,12 +300,7 @@ function formatLinesToRanges(values) { } function getUncoveredLines(lines) { - return ArrayPrototypeReduce(lines, (acc, line) => { - if (line.count === 0) { - ArrayPrototypePush(acc, line.line); - } - return acc; - }, []); + return ArrayPrototypeFlatMap(lines, (line) => line.count === 0 ? line.line : []); } function formatUncoveredLines(lines, table) { diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js index 9377f1bb509328..00db630f2e3c1f 100644 --- a/test/parallel/test-runner-coverage.js +++ b/test/parallel/test-runner-coverage.js @@ -89,6 +89,11 @@ test('test tap coverage reporter', skipIfNoInspector, async (t) => { const result = spawnSync(process.execPath, args, options); const report = getTapCoverageFixtureReport(); + console.log("=======") + console.log("RESULT\n", result.stdout.toString()) + console.log("EXPECTED\n", report) + console.log("=======") + assert(result.stdout.toString().includes(report)); assert.strictEqual(result.stderr.toString(), ''); assert.strictEqual(result.status, 0); From a3823076c3e522f4db73bd5815f8a56edefd290a Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Fri, 25 Aug 2023 17:00:58 +1000 Subject: [PATCH 3/6] test_runner: remove console.logs and fix lint errors --- lib/internal/test_runner/coverage.js | 4 ++-- lib/internal/test_runner/utils.js | 5 ++--- test/parallel/test-runner-coverage.js | 5 ----- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/internal/test_runner/coverage.js b/lib/internal/test_runner/coverage.js index b501b012096923..2f12db6383c597 100644 --- a/lib/internal/test_runner/coverage.js +++ b/lib/internal/test_runner/coverage.js @@ -190,7 +190,7 @@ class TestCoverage { ArrayPrototypePush(functionReports, { __proto__: null, name: functions[j].functionName, - count: MathMax(...ArrayPrototypeMap(ranges, r => r.count)), + count: MathMax(...ArrayPrototypeMap(ranges, (r) => r.count)), line: range.lines[0].line, }); @@ -215,7 +215,7 @@ class TestCoverage { __proto__: null, line: line.line, count: line.count, - }) + }); } } else { ArrayPrototypePush(lineReports, { diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 78e6c259f1598d..4a8ec3a04a6ebb 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -35,7 +35,6 @@ const { kIsNodeError, } = require('internal/errors'); const { compose } = require('stream'); -const console = require('console'); const coverageColors = { __proto__: null, @@ -300,7 +299,7 @@ function formatLinesToRanges(values) { } function getUncoveredLines(lines) { - return ArrayPrototypeFlatMap(lines, (line) => line.count === 0 ? line.line : []); + return ArrayPrototypeFlatMap(lines, (line) => (line.count === 0 ? line.line : [])); } function formatUncoveredLines(lines, table) { @@ -332,7 +331,7 @@ function getCoverageReport(pad, summary, symbol, color, table) { const columnsWidth = ArrayPrototypeReduce(columnPadLengths, (acc, columnPadLength) => acc + columnPadLength + 3, 0); uncoveredLinesPadLength = table && ArrayPrototypeReduce(summary.files, (acc, file) => { - return MathMax(acc, formatUncoveredLines(getUncoveredLines(file.lines), table).length) + return MathMax(acc, formatUncoveredLines(getUncoveredLines(file.lines), table).length); }, 0); uncoveredLinesPadLength = MathMax(uncoveredLinesPadLength, 'uncovered lines'.length); const uncoveredLinesWidth = uncoveredLinesPadLength + 2; diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js index 00db630f2e3c1f..9377f1bb509328 100644 --- a/test/parallel/test-runner-coverage.js +++ b/test/parallel/test-runner-coverage.js @@ -89,11 +89,6 @@ test('test tap coverage reporter', skipIfNoInspector, async (t) => { const result = spawnSync(process.execPath, args, options); const report = getTapCoverageFixtureReport(); - console.log("=======") - console.log("RESULT\n", result.stdout.toString()) - console.log("EXPECTED\n", report) - console.log("=======") - assert(result.stdout.toString().includes(report)); assert.strictEqual(result.stderr.toString(), ''); assert.strictEqual(result.status, 0); From e0dd2d4b4802438db67873c649c6b85292523016 Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Mon, 28 Aug 2023 10:36:34 +1000 Subject: [PATCH 4/6] test_runner: addressing PR comments Removed additional loop from calculating max counts for functions. Simplified reporting of count for each line. Returned arrow function to implicit return. --- lib/internal/test_runner/coverage.js | 22 +++++++++------------- lib/internal/test_runner/utils.js | 5 ++--- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/lib/internal/test_runner/coverage.js b/lib/internal/test_runner/coverage.js index 2f12db6383c597..1ed45028f61575 100644 --- a/lib/internal/test_runner/coverage.js +++ b/lib/internal/test_runner/coverage.js @@ -163,8 +163,10 @@ class TestCoverage { for (let j = 0; j < functions.length; ++j) { const { isBlockCoverage, ranges } = functions[j]; + let maxCountPerFunction = 0; for (let k = 0; k < ranges.length; ++k) { const range = ranges[k]; + maxCountPerFunction = MathMax(maxCountPerFunction, range.count); mapRangeToLines(range, lines); @@ -190,7 +192,7 @@ class TestCoverage { ArrayPrototypePush(functionReports, { __proto__: null, name: functions[j].functionName, - count: MathMax(...ArrayPrototypeMap(ranges, (r) => r.count)), + count: maxCountPerFunction, line: range.lines[0].line, }); @@ -207,23 +209,16 @@ class TestCoverage { for (let j = 0; j < lines.length; ++j) { const line = lines[j]; - - if (line.covered || line.ignore) { - coveredCnt++; - if (!line.ignore) { - ArrayPrototypePush(lineReports, { - __proto__: null, - line: line.line, - count: line.count, - }); - } - } else { + if (!line.ignore) { ArrayPrototypePush(lineReports, { __proto__: null, line: line.line, - count: 0, + count: line.count, }); } + if (line.covered || line.ignore) { + coveredCnt++; + } } ArrayPrototypePush(coverageSummary.files, { @@ -350,6 +345,7 @@ function mapRangeToLines(range, lines) { if (count === 0 && startOffset <= line.startOffset && endOffset >= line.endOffset) { line.covered = false; + line.count = 0; } if (count > 0 && startOffset <= line.startOffset && endOffset >= line.endOffset) { diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 4a8ec3a04a6ebb..7923732f04dfcb 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -330,9 +330,8 @@ function getCoverageReport(pad, summary, symbol, color, table) { columnPadLengths = ArrayPrototypeMap(kColumns, (column) => (table ? MathMax(column.length, 6) : 0)); const columnsWidth = ArrayPrototypeReduce(columnPadLengths, (acc, columnPadLength) => acc + columnPadLength + 3, 0); - uncoveredLinesPadLength = table && ArrayPrototypeReduce(summary.files, (acc, file) => { - return MathMax(acc, formatUncoveredLines(getUncoveredLines(file.lines), table).length); - }, 0); + uncoveredLinesPadLength = table && ArrayPrototypeReduce(summary.files, (acc, file) => + MathMax(acc, formatUncoveredLines(getUncoveredLines(file.lines), table).length), 0); uncoveredLinesPadLength = MathMax(uncoveredLinesPadLength, 'uncovered lines'.length); const uncoveredLinesWidth = uncoveredLinesPadLength + 2; From 81448af0669ba6956088b1f575f1cc5831fa1d8a Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Mon, 28 Aug 2023 11:37:56 +1000 Subject: [PATCH 5/6] test_runner: tests for coverage event --- .../test-runner/custom_reporters/coverage.mjs | 15 +++++ test/parallel/test-runner-coverage.js | 56 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 test/fixtures/test-runner/custom_reporters/coverage.mjs diff --git a/test/fixtures/test-runner/custom_reporters/coverage.mjs b/test/fixtures/test-runner/custom_reporters/coverage.mjs new file mode 100644 index 00000000000000..c1b8848799030b --- /dev/null +++ b/test/fixtures/test-runner/custom_reporters/coverage.mjs @@ -0,0 +1,15 @@ +import { Transform } from 'node:stream'; + +export default class CoverageReporter extends Transform { + constructor(options) { + super({ ...options, writableObjectMode: true }); + } + + _transform(event, _encoding, callback) { + if (event.type === 'test:coverage') { + callback(null, JSON.stringify(event.data, null, 2)); + } else { + callback(null); + } + } +} diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js index 9377f1bb509328..6527c85f569686 100644 --- a/test/parallel/test-runner-coverage.js +++ b/test/parallel/test-runner-coverage.js @@ -182,3 +182,59 @@ test('coverage is combined for multiple processes', skipIfNoInspector, () => { assert(result.stdout.toString().includes(report)); assert.strictEqual(result.status, 0); }); + +test('coverage reports on lines, functions, and branches', async (t) => { + const fixture = fixtures.path('test-runner', 'coverage.js'); + const child = spawnSync(process.execPath, + ['--test', '--experimental-test-coverage', '--test-reporter', + fixtures.fileURL('test-runner/custom_reporters/coverage.mjs'), + fixture]); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + const coverage = JSON.parse(stdout); + + await t.test('does not include node_modules', () => { + assert.strictEqual(coverage.summary.files.length, 3); + const files = ['coverage.js', 'invalid-tap.js', 'throw.js']; + coverage.summary.files.forEach((file, index) => { + assert.ok(file.path.endsWith(files[index])); + }); + }); + + const file = coverage.summary.files[0]; + + await t.test('reports on function coverage', () => { + const uncalledFunction = file.functions.find((f) => f.name === 'uncalledTopLevelFunction'); + assert.strictEqual(uncalledFunction.count, 0); + assert.strictEqual(uncalledFunction.line, 16); + + const calledTwice = file.functions.find((f) => f.name === 'fnWithControlFlow'); + assert.strictEqual(calledTwice.count, 2); + assert.strictEqual(calledTwice.line, 35); + }); + + await t.test('reports on branch coverage', () => { + const uncalledBranch = file.branches.find((b) => b.line === 6); + assert.strictEqual(uncalledBranch.count, 0); + + const calledTwice = file.branches.find((b) => b.line === 35); + assert.strictEqual(calledTwice.count, 2); + }); + + await t.test('reports on line coverage', () => { + [ + { line: 36, count: 2 }, + { line: 37, count: 1 }, + { line: 38, count: 1 }, + { line: 39, count: 0 }, + { line: 40, count: 1 }, + { line: 41, count: 1 }, + { line: 42, count: 1 }, + { line: 43, count: 0 }, + { line: 44, count: 0 }, + ].forEach((line) => { + const testLine = file.lines.find((l) => l.line === line.line); + assert.strictEqual(testLine.count, line.count); + }); + }); +}); From 741cdf9df92e75843d0e464f732df4551291bba4 Mon Sep 17 00:00:00 2001 From: Phil Nash Date: Mon, 28 Aug 2023 17:03:44 +1000 Subject: [PATCH 6/6] test_runner: skip new coverage tests if inspector is not available --- test/parallel/test-runner-coverage.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js index 6527c85f569686..2532f9971c213d 100644 --- a/test/parallel/test-runner-coverage.js +++ b/test/parallel/test-runner-coverage.js @@ -183,7 +183,7 @@ test('coverage is combined for multiple processes', skipIfNoInspector, () => { assert.strictEqual(result.status, 0); }); -test('coverage reports on lines, functions, and branches', async (t) => { +test('coverage reports on lines, functions, and branches', skipIfNoInspector, async (t) => { const fixture = fixtures.path('test-runner', 'coverage.js'); const child = spawnSync(process.execPath, ['--test', '--experimental-test-coverage', '--test-reporter',