test_runner: support handling errors from outside of tests

lib/internal/test_runner/harness.js

@@ -19,7 +19,6 @@ const { exitCodes: { kGenericUserError } } = internalBinding('errors');
 const { kEmptyObject } = require('internal/util');
 const { kCancelledByParent, Test, Suite } = require('internal/test_runner/test');
 const {
-  kAsyncBootstrapFailure,
   parseCommandLine,
   setupTestReporters,
 } = require('internal/test_runner/utils');
@@ -34,27 +33,31 @@ function createTestTree(options = kEmptyObject) {
 
 function createProcessEventHandler(eventName, rootTest) {
   return (err) => {
-    if (err?.failureType === kAsyncBootstrapFailure) {
+    if (!rootTest.harness.bootstrapComplete) {
       // Something went wrong during the asynchronous portion of bootstrapping
       // the test runner. Since the test runner is not setup properly, we can't
       // do anything but throw the error.
-      throw err.cause;
+      throw err;
     }
 
     // Check if this error is coming from a test. If it is, fail the test.
     const test = testResources.get(executionAsyncId());
 
-    if (!test) {
-      throw err;
-    }
-
-    if (test.finished) {
-      // If the test is already finished, report this as a top level
-      // diagnostic since this is a malformed test.
-      const msg = `Warning: Test "${test.name}" generated asynchronous ` +
-        'activity after the test ended. This activity created the error ' +
-        `"${err}" and would have caused the test to fail, but instead ` +
-        `triggered an ${eventName} event.`;
+    if (!test || test.finished) {
+      // If the test is already finished or the resource that created the error
+      // is not mapped to a Test, report this as a top level diagnostic.
+      let msg;
+
+      if (test) {
+        msg = `Warning: Test "${test.name}" generated asynchronous ` +
+          'activity after the test ended. This activity created the error ' +
+          `"${err}" and would have caused the test to fail, but instead ` +
+          `triggered an ${eventName} event.`;
+      } else {
+        msg = 'Warning: A resource generated asynchronous activity after ' +
+          `the test ended. This activity created the error "${err}" which ` +
+          `triggered an ${eventName} event, caught by the test runner.`;
+      }
 
       rootTest.diagnostic(msg);
       process.exitCode = kGenericUserError;
@@ -139,7 +142,7 @@ function setup(root) {
     createProcessEventHandler('unhandledRejection', root);
   const coverage = configureCoverage(root, globalOptions);
   const exitHandler = () => {
-    root.coverage = collectCoverage(root, coverage);
+    root.harness.coverage = collectCoverage(root, coverage);
     root.postRun(new ERR_TEST_FAILURE(
       'Promise resolution is still pending but the event loop has already resolved',
       kCancelledByParent));
@@ -163,6 +166,11 @@ function setup(root) {
     process.on('SIGTERM', terminationHandler);
   }
 
+  root.harness = {
+    __proto__: null,
+    bootstrapComplete: false,
+    coverage: null,
+  };
   root.startTime = hrtime();
 
   wasRootSetup.add(root);
@@ -177,13 +185,14 @@ function getGlobalRoot() {
     globalRoot.reporter.once('test:fail', () => {
       process.exitCode = kGenericUserError;
     });
-    reportersSetup = setupTestReporters(globalRoot.reporter);
+    reportersSetup = setupTestReporters(globalRoot);
   }
   return globalRoot;
 }
 
 async function startSubtest(subtest) {
   await reportersSetup;
+  getGlobalRoot().harness.bootstrapComplete = true;
   await subtest.start();
 }
 

lib/internal/test_runner/runner.js

@@ -387,13 +387,16 @@ function run(options) {
     filesWatcher = watchFiles(testFiles, root, inspectPort);
     postRun = undefined;
   }
-  const runFiles = () => SafePromiseAllSettledReturnVoid(testFiles, (path) => {
-    const subtest = runTestFile(path, root, inspectPort, filesWatcher);
-    runningSubtests.set(path, subtest);
-    return subtest;
-  });
+  const runFiles = () => {
+    root.harness.bootstrapComplete = true;
+    return SafePromiseAllSettledReturnVoid(testFiles, (path) => {
+      const subtest = runTestFile(path, root, inspectPort, filesWatcher);
+      runningSubtests.set(path, subtest);
+      return subtest;
+    });
+  };
 
-  PromisePrototypeThen(PromisePrototypeThen(PromiseResolve(setup?.(root.reporter)), runFiles), postRun);
+  PromisePrototypeThen(PromisePrototypeThen(PromiseResolve(setup?.(root)), runFiles), postRun);
 
   return root.reporter;
 }

lib/internal/test_runner/test.js

@@ -252,7 +252,7 @@ class Test extends AsyncResource {
     this.#outerSignal?.addEventListener('abort', this.#abortHandler);
 
     this.fn = fn;
-    this.coverage = null; // Configured on the root test by the test harness.
+    this.harness = null; // Configured on the root test by the test harness.
     this.mock = null;
     this.name = name;
     this.parent = parent;
@@ -651,8 +651,8 @@ class Test extends AsyncResource {
       this.reporter.diagnostic(this.nesting, kFilename, `todo ${counters.todo}`);
       this.reporter.diagnostic(this.nesting, kFilename, `duration_ms ${this.#duration()}`);
 
-      if (this.coverage) {
-        this.reporter.coverage(this.nesting, kFilename, this.coverage);
+      if (this.harness?.coverage) {
+        this.reporter.coverage(this.nesting, kFilename, this.harness.coverage);
       }
 
       this.reporter.push(null);

lib/internal/test_runner/utils.js

@@ -7,7 +7,6 @@ const {
   RegExp,
   RegExpPrototypeExec,
   SafeMap,
-  Symbol,
 } = primordials;
 const { basename } = require('path');
 const { createWriteStream } = require('fs');
@@ -24,7 +23,6 @@ const {
 } = require('internal/errors');
 const { compose } = require('stream');
 
-const kAsyncBootstrapFailure = Symbol('asyncBootstrapFailure');
 const kMultipleCallbackInvocations = 'multipleCallbackInvocations';
 const kRegExpPattern = /^\/(.*)\/([a-z]*)$/;
 const kSupportedFileExtensions = /\.[cm]?js$/;
@@ -150,16 +148,12 @@ async function getReportersMap(reporters, destinations) {
 }
 
 
-async function setupTestReporters(testsStream) {
-  try {
-    const { reporters, destinations } = parseCommandLine();
-    const reportersMap = await getReportersMap(reporters, destinations);
-    for (let i = 0; i < reportersMap.length; i++) {
-      const { reporter, destination } = reportersMap[i];
-      compose(testsStream, reporter).pipe(destination);
-    }
-  } catch (err) {
-    throw new ERR_TEST_FAILURE(err, kAsyncBootstrapFailure);
+async function setupTestReporters(rootTest) {
+  const { reporters, destinations } = parseCommandLine();
+  const reportersMap = await getReportersMap(reporters, destinations);
+  for (let i = 0; i < reportersMap.length; i++) {
+    const { reporter, destination } = reportersMap[i];
+    compose(rootTest.reporter, reporter).pipe(destination);
   }
 }
 
@@ -225,7 +219,6 @@ module.exports = {
   doesPathMatchFilter,
   isSupportedFileType,
   isTestFailureError,
-  kAsyncBootstrapFailure,
   parseCommandLine,
   setupTestReporters,
 };

test/message/test_runner_output.js

@@ -383,3 +383,9 @@ test('unfinished test with unhandledRejection', async () => {
     setTimeout(() => Promise.reject(new Error('bar')));
   });
 });
+
+// Verify that uncaught exceptions outside of any tests are handled after the
+// test harness has finished bootstrapping itself.
+setImmediate(() => {
+  throw new Error('uncaught from outside of a test');
+});

test/message/test_runner_output.out

@@ -637,6 +637,7 @@ not ok 65 - invalid subtest fail
 1..65
 # Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
+# Warning: A resource generated asynchronous activity after the test ended. This activity created the error "Error: uncaught from outside of a test" which triggered an uncaughtException event, caught by the test runner.
 # Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.

test/message/test_runner_output_cli.out

@@ -636,6 +636,7 @@ not ok 65 - invalid subtest fail
   ...
 # Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
+# Warning: A resource generated asynchronous activity after the test ended. This activity created the error "Error: uncaught from outside of a test" which triggered an uncaughtException event, caught by the test runner.
 # Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.

test/message/test_runner_output_spec_reporter.out

@@ -270,6 +270,7 @@
 
  Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
  Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
+ Warning: A resource generated asynchronous activity after the test ended. This activity created the error "Error: uncaught from outside of a test" which triggered an uncaughtException event, caught by the test runner.
  Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event.
  Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
  Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.