From f1fb090cfaa6156347b075673280e458ce92b961 Mon Sep 17 00:00:00 2001
From: marco-ippolito <marcoippolito54@gmail.com>
Date: Thu, 4 Jan 2024 13:16:45 +0100
Subject: [PATCH] test_runner: create flag --check-coverage to enforce code
 coverage

---
 doc/api/cli.md                                | 63 +++++++++++++++++++
 doc/api/test.md                               | 15 +++++
 doc/node.1                                    | 18 ++++++
 lib/internal/test_runner/test.js              | 21 ++++++-
 lib/internal/test_runner/tests_stream.js      |  3 +-
 lib/internal/test_runner/utils.js             | 41 ++++++++++++
 src/node_options.cc                           | 12 ++++
 src/node_options.h                            |  4 ++
 .../test-runner/output/coverage_check.js      | 25 ++++++++
 .../output/coverage_check.snapshot            | 23 +++++++
 .../output/coverage_insufficient.js           | 25 ++++++++
 .../output/coverage_insufficient.snapshot     | 26 ++++++++
 .../output/coverage_thresholds_no_check.js    | 26 ++++++++
 .../coverage_thresholds_no_check.snapshot     | 23 +++++++
 test/parallel/test-runner-output.mjs          |  3 +
 .../test-runner-test-coverage-check.js        | 52 +++++++++++++++
 16 files changed, 377 insertions(+), 3 deletions(-)
 create mode 100644 test/fixtures/test-runner/output/coverage_check.js
 create mode 100644 test/fixtures/test-runner/output/coverage_check.snapshot
 create mode 100644 test/fixtures/test-runner/output/coverage_insufficient.js
 create mode 100644 test/fixtures/test-runner/output/coverage_insufficient.snapshot
 create mode 100644 test/fixtures/test-runner/output/coverage_thresholds_no_check.js
 create mode 100644 test/fixtures/test-runner/output/coverage_thresholds_no_check.snapshot
 create mode 100644 test/parallel/test-runner-test-coverage-check.js

diff --git a/doc/api/cli.md b/doc/api/cli.md
index 072b3b8ead55b8..bcfbdff0c7837e 100644
--- a/doc/api/cli.md
+++ b/doc/api/cli.md
@@ -892,6 +892,69 @@ generated as part of the test runner output. If no tests are run, a coverage
 report is not generated. See the documentation on
 [collecting code coverage from tests][] for more details.
 
+### `--check-coverage`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+The `--check-coverage` CLI flag, used in conjunction with
+the `--experimental-test-coverage` command, enforces
+that test coverage thresholds for the specified checks
+(`--lines`, `--branches`, `--functions`) are respected.
+
+### `--lines=threshold`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+The `--lines` CLI flag, used in conjunction with the `--check-coverage` flag,
+enforces a coverage threshold check for lines of code covered by the test.
+It is expressed as a numerical value between `0` and `100`,
+representing the percentage (e.g., 80 for 80% coverage).
+If the coverage falls below the threshold,
+the test will exit with non zero code.
+
+### `--branches=threshold`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+The `--branches` CLI flag, used in conjunction with the `--check-coverage` flag,
+enforces a coverage threshold check for branches of code covered by the test.
+It is expressed as a numerical value between `0` and `100`,
+representing the percentage (e.g., 80 for 80% coverage).
+If the coverage falls below the threshold,
+the test will exit with non zero code.
+
+### `--functions=threshold`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+The `--functions` CLI flag, used in conjunction with
+the `--check-coverage`flag, enforces a coverage threshold check
+for functions covered by the test.
+It is expressed as a numerical value between `0` and `100`,
+representing the percentage (e.g., 80 for 80% coverage).
+If the coverage falls below the threshold,
+the test will exit with non zero code.
+
 ### `--experimental-vm-modules`
 
 <!-- YAML
diff --git a/doc/api/test.md b/doc/api/test.md
index 6d4725ba93c9fb..a044ab4b5181dd 100644
--- a/doc/api/test.md
+++ b/doc/api/test.md
@@ -403,6 +403,20 @@ if (anAlwaysFalseCondition) {
 }
 ```
 
+By using the CLI flag [`--check-coverage`][]
+in conjunction with the `--experimental-test-coverage` flag,
+it is possible to enforce specific test coverage threshold checks
+(`--lines`, `--branches`, `--functions`).
+When enabled, it evaluates the test coverage achieved during
+the execution of tests and determines whether it meets
+the specified coverage thresholds.
+If the coverage check falls below the specified threshold,
+the program will exit with a non zero code.
+
+```bash
+node --test --experimental-test-coverage --check-coverage --lines=100 --branches=100 --function=100
+```
+
 ### Coverage reporters
 
 The tap and spec reporters will print a summary of the coverage statistics.
@@ -2966,6 +2980,7 @@ added:
 
 [TAP]: https://testanything.org/
 [TTY]: tty.md
+[`--check-coverage`]: cli.md#--check-coverage
 [`--experimental-test-coverage`]: cli.md#--experimental-test-coverage
 [`--import`]: cli.md#--importmodule
 [`--test-concurrency`]: cli.md#--test-concurrency
diff --git a/doc/node.1 b/doc/node.1
index 2966c14e1c3f5e..2a754fb735020f 100644
--- a/doc/node.1
+++ b/doc/node.1
@@ -91,6 +91,24 @@ Allow spawning process when using the permission model.
 .It Fl -allow-worker
 Allow creating worker threads when using the permission model.
 .
+.It Fl -check-coverage
+Enforce test coverage threshold checks
+when used with the
+.Fl -experimental-test-coverage
+flag.
+.
+.It Fl -lines
+Enforce a minimum threshold of
+lines of code covered by test coverage (0 - 100).
+.
+.It Fl -branches
+Enforce a minimum threshold of
+branches covered by test coverage (0 - 100).
+.
+.It Fl -functions
+Enforce a minimum threshold of
+functions covered by test coverage (0 - 100).
+.
 .It Fl -completion-bash
 Print source-able bash completion script for Node.js.
 .
diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js
index 19ae283ed9ad78..86187cd43b18c3 100644
--- a/lib/internal/test_runner/test.js
+++ b/lib/internal/test_runner/test.js
@@ -10,6 +10,7 @@ const {
   FunctionPrototype,
   MathMax,
   Number,
+  NumberPrototypeToFixed,
   ObjectSeal,
   PromisePrototypeThen,
   PromiseResolve,
@@ -58,6 +59,7 @@ const {
 const { setTimeout } = require('timers');
 const { TIMEOUT_MAX } = require('internal/timers');
 const { availableParallelism } = require('os');
+const { exitCodes: { kGenericUserError } } = internalBinding('errors');
 const { bigint: hrtime } = process.hrtime;
 const kCallbackAndPromisePresent = 'callbackAndPromisePresent';
 const kCancelledByParent = 'cancelledByParent';
@@ -75,7 +77,7 @@ const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']);
 const kUnwrapErrors = new SafeSet()
   .add(kTestCodeFailure).add(kHookFailure)
   .add('uncaughtException').add('unhandledRejection');
-const { testNamePatterns, testOnlyFlag } = parseCommandLine();
+const { testNamePatterns, testOnlyFlag, coverageThreshold } = parseCommandLine();
 let kResistStopPropagation;
 
 function stopTest(timeout, signal) {
@@ -753,13 +755,28 @@ class Test extends AsyncResource {
       reporter.diagnostic(nesting, loc, `duration_ms ${this.duration()}`);
 
       if (coverage) {
-        reporter.coverage(nesting, loc, coverage);
+        if (coverageThreshold) {
+          const cb = (msg) => reporter.stderr(loc, msg);
+          const { lines, branches, functions } = coverageThreshold;
+          this.checkCoverageThreshold(coverage.totals.coveredLinePercent, lines, 'Lines', cb);
+          this.checkCoverageThreshold(coverage.totals.coveredBranchPercent, branches, 'Branches', cb);
+          this.checkCoverageThreshold(coverage.totals.coveredFunctionPercent, functions, 'Functions', cb);
+        }
+        reporter.coverage(nesting, loc, coverage, coverageThreshold);
       }
 
       reporter.end();
     }
   }
 
+  checkCoverageThreshold(actual, expected, type, cb) {
+    if (actual < expected) {
+      const msg = `ERROR: ${type} coverage (${NumberPrototypeToFixed(actual, 2)}%) does not meet expected threshold (${expected}%)\n`;
+      process.exitCode = kGenericUserError;
+      cb(msg);
+    }
+  }
+
   isClearToSend() {
     return this.parent === null ||
       (
diff --git a/lib/internal/test_runner/tests_stream.js b/lib/internal/test_runner/tests_stream.js
index f7730caac00fa7..bf93afd2b880cd 100644
--- a/lib/internal/test_runner/tests_stream.js
+++ b/lib/internal/test_runner/tests_stream.js
@@ -114,11 +114,12 @@ class TestsStream extends Readable {
     this[kEmitMessage]('test:stdout', { __proto__: null, message, ...loc });
   }
 
-  coverage(nesting, loc, summary) {
+  coverage(nesting, loc, summary, coverageThreshold) {
     this[kEmitMessage]('test:coverage', {
       __proto__: null,
       nesting,
       summary,
+      coverageThreshold,
       ...loc,
     });
   }
diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js
index 184d44dce6c162..960f62d70439ad 100644
--- a/lib/internal/test_runner/utils.js
+++ b/lib/internal/test_runner/utils.js
@@ -193,6 +193,46 @@ function parseCommandLine() {
 
   const isTestRunner = getOptionValue('--test');
   const coverage = getOptionValue('--experimental-test-coverage');
+  const checkCoverage = getOptionValue('--check-coverage');
+  let coverageThreshold;
+
+  if (checkCoverage) {
+    const lines = getOptionValue('--lines');
+
+    if (lines < 0 || lines > 100) {
+      throw new ERR_INVALID_ARG_VALUE(
+        '--lines',
+        lines,
+        'must be a value between 0 and 100',
+      );
+    }
+
+    const branches = getOptionValue('--branches');
+    if (branches < 0 || branches > 100) {
+      throw new ERR_INVALID_ARG_VALUE(
+        '--branches',
+        branches,
+        'must be a value between 0 and 100',
+      );
+    }
+
+    const functions = getOptionValue('--functions');
+    if (functions < 0 || functions > 100) {
+      throw new ERR_INVALID_ARG_VALUE(
+        '--functions',
+        functions,
+        'must be a value between 0 and 100',
+      );
+    }
+
+    coverageThreshold = {
+      __proto__: null,
+      lines,
+      branches,
+      functions,
+    };
+  }
+
   const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
   const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
   let destinations;
@@ -244,6 +284,7 @@ function parseCommandLine() {
     __proto__: null,
     isTestRunner,
     coverage,
+    coverageThreshold,
     testOnlyFlag,
     testNamePatterns,
     reporters,
diff --git a/src/node_options.cc b/src/node_options.cc
index 7b5152172c5ce7..59d583af3415a9 100644
--- a/src/node_options.cc
+++ b/src/node_options.cc
@@ -622,6 +622,18 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
   AddOption("--experimental-test-coverage",
             "enable code coverage in the test runner",
             &EnvironmentOptions::test_runner_coverage);
+  AddOption("--check-coverage",
+            "check that coverage falls within the thresholds provided",
+            &EnvironmentOptions::test_runner_check_coverage);
+  AddOption("--lines",
+            "coverage threshold for lines of code",
+            &EnvironmentOptions::test_runner_check_coverage_lines);
+  AddOption("--branches",
+            "coverage threshold for code branches",
+            &EnvironmentOptions::test_runner_check_coverage_branches);
+  AddOption("--functions",
+            "coverage threshold for functions",
+            &EnvironmentOptions::test_runner_check_coverage_functions);
   AddOption("--test-name-pattern",
             "run tests whose name matches this regular expression",
             &EnvironmentOptions::test_name_pattern);
diff --git a/src/node_options.h b/src/node_options.h
index 915151b7dc2904..1a9535366ae3c9 100644
--- a/src/node_options.h
+++ b/src/node_options.h
@@ -166,6 +166,10 @@ class EnvironmentOptions : public Options {
   uint64_t test_runner_concurrency = 0;
   uint64_t test_runner_timeout = 0;
   bool test_runner_coverage = false;
+  bool test_runner_check_coverage = false;
+  uint64_t test_runner_check_coverage_lines = 0;
+  uint64_t test_runner_check_coverage_branches = 0;
+  uint64_t test_runner_check_coverage_functions = 0;
   std::vector<std::string> test_name_pattern;
   std::vector<std::string> test_reporter;
   std::vector<std::string> test_reporter_destination;
diff --git a/test/fixtures/test-runner/output/coverage_check.js b/test/fixtures/test-runner/output/coverage_check.js
new file mode 100644
index 00000000000000..d33780ca30c745
--- /dev/null
+++ b/test/fixtures/test-runner/output/coverage_check.js
@@ -0,0 +1,25 @@
+// Flags: --expose-internals --experimental-test-coverage --check-coverage --lines=100 --branches=100 --functions=100
+
+'use strict';
+require('../../../common');
+const { TestCoverage } = require('internal/test_runner/coverage');
+const { test, mock } = require('node:test');
+
+mock.method(TestCoverage.prototype, 'summary', () => {
+    return {
+        files: [],
+        totals: {
+            totalLineCount: 100,
+            totalBranchCount: 100,
+            totalFunctionCount: 100,
+            coveredLineCount: 100,
+            coveredBranchCount: 100,
+            coveredFunctionCount: 100,
+            coveredLinePercent: 100,
+            coveredBranchPercent: 100,
+            coveredFunctionPercent: 100
+        }
+    }
+});
+
+test('ok');
diff --git a/test/fixtures/test-runner/output/coverage_check.snapshot b/test/fixtures/test-runner/output/coverage_check.snapshot
new file mode 100644
index 00000000000000..30d37bef97e4ea
--- /dev/null
+++ b/test/fixtures/test-runner/output/coverage_check.snapshot
@@ -0,0 +1,23 @@
+TAP version 13
+# Subtest: ok
+ok 1 - ok
+  ---
+  duration_ms: *
+  ...
+1..1
+# tests 1
+# suites 0
+# pass 1
+# fail 0
+# cancelled 0
+# skipped 0
+# todo 0
+# duration_ms *
+# start of coverage report
+# -----------------------------------------------------
+# file | line % | branch % | funcs % | uncovered lines
+# -----------------------------------------------------
+# -----------------------------------------------------
+# all… | 100.00 |   100.00 |  100.00 |
+# -----------------------------------------------------
+# end of coverage report
diff --git a/test/fixtures/test-runner/output/coverage_insufficient.js b/test/fixtures/test-runner/output/coverage_insufficient.js
new file mode 100644
index 00000000000000..6b810b7b1f89cc
--- /dev/null
+++ b/test/fixtures/test-runner/output/coverage_insufficient.js
@@ -0,0 +1,25 @@
+// Flags: --expose-internals --experimental-test-coverage --check-coverage --lines=100 --branches=100 --functions=100
+
+'use strict';
+require('../../../common');
+const { TestCoverage } = require('internal/test_runner/coverage');
+const { test, mock } = require('node:test');
+
+mock.method(TestCoverage.prototype, 'summary', () => {
+    return {
+        files: [],
+        totals: {
+            totalLineCount: 0,
+            totalBranchCount: 0,
+            totalFunctionCount: 0,
+            coveredLineCount: 0,
+            coveredBranchCount: 0,
+            coveredFunctionCount: 0,
+            coveredLinePercent: 0,
+            coveredBranchPercent: 0,
+            coveredFunctionPercent: 0
+        }
+    }
+});
+
+test('ok');
diff --git a/test/fixtures/test-runner/output/coverage_insufficient.snapshot b/test/fixtures/test-runner/output/coverage_insufficient.snapshot
new file mode 100644
index 00000000000000..33b5e6e982ce7c
--- /dev/null
+++ b/test/fixtures/test-runner/output/coverage_insufficient.snapshot
@@ -0,0 +1,26 @@
+TAP version 13
+# Subtest: ok
+ok 1 - ok
+  ---
+  duration_ms: *
+  ...
+1..1
+# tests 1
+# suites 0
+# pass 1
+# fail 0
+# cancelled 0
+# skipped 0
+# todo 0
+# duration_ms *
+# ERROR: Lines coverage (0.00%) does not meet expected threshold (100%)
+# ERROR: Branches coverage (0.00%) does not meet expected threshold (100%)
+# ERROR: Functions coverage (0.00%) does not meet expected threshold (100%)
+# start of coverage report
+# -----------------------------------------------------
+# file | line % | branch % | funcs % | uncovered lines
+# -----------------------------------------------------
+# -----------------------------------------------------
+# all… |   0.00 |     0.00 |    0.00 |
+# -----------------------------------------------------
+# end of coverage report
diff --git a/test/fixtures/test-runner/output/coverage_thresholds_no_check.js b/test/fixtures/test-runner/output/coverage_thresholds_no_check.js
new file mode 100644
index 00000000000000..bf8fcc35124bbf
--- /dev/null
+++ b/test/fixtures/test-runner/output/coverage_thresholds_no_check.js
@@ -0,0 +1,26 @@
+// Flags: --expose-internals --experimental-test-coverage --lines=100 --branches=100 --functions=100
+
+// the flag --check-coverage is missing so thresholds should be ignored
+'use strict';
+require('../../../common');
+const { TestCoverage } = require('internal/test_runner/coverage');
+const { test, mock } = require('node:test');
+
+mock.method(TestCoverage.prototype, 'summary', () => {
+    return {
+        files: [],
+        totals: {
+            totalLineCount: 0,
+            totalBranchCount: 0,
+            totalFunctionCount: 0,
+            coveredLineCount: 0,
+            coveredBranchCount: 0,
+            coveredFunctionCount: 0,
+            coveredLinePercent: 0,
+            coveredBranchPercent: 0,
+            coveredFunctionPercent: 0
+        }
+    }
+});
+
+test('ok');
diff --git a/test/fixtures/test-runner/output/coverage_thresholds_no_check.snapshot b/test/fixtures/test-runner/output/coverage_thresholds_no_check.snapshot
new file mode 100644
index 00000000000000..3db09d8e28ff57
--- /dev/null
+++ b/test/fixtures/test-runner/output/coverage_thresholds_no_check.snapshot
@@ -0,0 +1,23 @@
+TAP version 13
+# Subtest: ok
+ok 1 - ok
+  ---
+  duration_ms: *
+  ...
+1..1
+# tests 1
+# suites 0
+# pass 1
+# fail 0
+# cancelled 0
+# skipped 0
+# todo 0
+# duration_ms *
+# start of coverage report
+# -----------------------------------------------------
+# file | line % | branch % | funcs % | uncovered lines
+# -----------------------------------------------------
+# -----------------------------------------------------
+# all… |   0.00 |     0.00 |    0.00 |
+# -----------------------------------------------------
+# end of coverage report
diff --git a/test/parallel/test-runner-output.mjs b/test/parallel/test-runner-output.mjs
index e40ce896bff465..98fba026f9aa34 100644
--- a/test/parallel/test-runner-output.mjs
+++ b/test/parallel/test-runner-output.mjs
@@ -128,6 +128,9 @@ const tests = [
     ),
   },
   process.features.inspector ? { name: 'test-runner/output/coverage_failure.js' } : false,
+  { name: 'test-runner/output/coverage_insufficient.js' },
+  { name: 'test-runner/output/coverage_check.js' },
+  { name: 'test-runner/output/coverage_thresholds_no_check.js' },
 ]
 .filter(Boolean)
 .map(({ name, tty, transform }) => ({
diff --git a/test/parallel/test-runner-test-coverage-check.js b/test/parallel/test-runner-test-coverage-check.js
new file mode 100644
index 00000000000000..e696a18e27a22b
--- /dev/null
+++ b/test/parallel/test-runner-test-coverage-check.js
@@ -0,0 +1,52 @@
+'use strict';
+require('../common');
+const fixtures = require('../common/fixtures');
+const assert = require('node:assert');
+const { spawnSync } = require('node:child_process');
+const { test } = require('node:test');
+const cwd = fixtures.path('test-runner', 'default-behavior');
+const env = { ...process.env, 'NODE_DEBUG': 'test_runner' };
+
+
+test('lines above 100', () => {
+  const args = ['--test', '--experimental-test-coverage', '--check-coverage', '--lines=101'];
+  const cp = spawnSync(process.execPath, args, { cwd, env });
+  assert.match(cp.stderr.toString(), /ERR_INVALID_ARG_VALUE/);
+  assert.match(cp.stderr.toString(), /The argument '--lines' must be a value between 0 and 100/);
+});
+
+test('lines below 0', () => {
+  const args = ['--test', '--experimental-test-coverage', '--check-coverage', '--lines=-1'];
+  const cp = spawnSync(process.execPath, args, { cwd, env });
+  assert.match(cp.stderr.toString(), /ERR_INVALID_ARG_VALUE/);
+  assert.match(cp.stderr.toString(), /The argument '--lines' must be a value between 0 and 100/);
+});
+
+
+test('branches above 100', () => {
+  const args = ['--test', '--experimental-test-coverage', '--check-coverage', '--branches=101'];
+  const cp = spawnSync(process.execPath, args, { cwd, env });
+  assert.match(cp.stderr.toString(), /ERR_INVALID_ARG_VALUE/);
+  assert.match(cp.stderr.toString(), /The argument '--branches' must be a value between 0 and 100/);
+});
+
+test('branches below 0', () => {
+  const args = ['--test', '--experimental-test-coverage', '--check-coverage', '--branches=-1'];
+  const cp = spawnSync(process.execPath, args, { cwd, env });
+  assert.match(cp.stderr.toString(), /ERR_INVALID_ARG_VALUE/);
+  assert.match(cp.stderr.toString(), /The argument '--branches' must be a value between 0 and 100/);
+});
+
+test('functions above 100', () => {
+  const args = ['--test', '--experimental-test-coverage', '--check-coverage', '--functions=101'];
+  const cp = spawnSync(process.execPath, args, { cwd, env });
+  assert.match(cp.stderr.toString(), /ERR_INVALID_ARG_VALUE/);
+  assert.match(cp.stderr.toString(), /The argument '--functions' must be a value between 0 and 100/);
+});
+
+test('functions below 0', () => {
+  const args = ['--test', '--experimental-test-coverage', '--check-coverage', '--functions=-1'];
+  const cp = spawnSync(process.execPath, args, { cwd, env });
+  assert.match(cp.stderr.toString(), /ERR_INVALID_ARG_VALUE/);
+  assert.match(cp.stderr.toString(), /The argument '--functions' must be a value between 0 and 100/);
+});