test_runner: detect only tests when --test is not used

This commit updates the way the test runner processes 'only'
tests when node:test files are run without the --test CLI.
This is a breaking change.

PR-URL: #54881
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>

lib/internal/test_runner/harness.js

@@ -38,8 +38,8 @@ function createTestTree(rootTestOptions, globalOptions) {
   const buildPhaseDeferred = createDeferredPromise();
   const isFilteringByName = globalOptions.testNamePatterns ||
                             globalOptions.testSkipPatterns;
-  const isFilteringByOnly = globalOptions.only || (globalOptions.isTestRunner &&
-                                                   globalOptions.isolation === 'none');
+  const isFilteringByOnly = (globalOptions.isolation === 'process' || process.env.NODE_TEST_CONTEXT) ?
+    globalOptions.only : true;
   const harness = {
     __proto__: null,
     buildPromise: buildPhaseDeferred.promise,

test/fixtures/test-runner/output/dot_reporter.snapshot

@@ -1,7 +1,7 @@
 ..XX...X..XXX.X.....
 XXX.....X..X...X....
 .....X...XXX.XX.....
-.XXXXXXX...XXXXX
+XXXXXXX...XXXXX
 
 Failed tests:
 

test/fixtures/test-runner/output/junit_reporter.snapshot

@@ -309,15 +309,11 @@ Error [ERR_TEST_FAILURE]: thrown from callback async throw
 		</failure>
 	</testcase>
 	<testcase name="callback async throw after done" time="*" classname="test"/>
-	<testsuite name="only is set but not in only mode" time="*" disabled="0" errors="0" tests="4" failures="0" skipped="0" hostname="HOSTNAME">
+	<testsuite name="only is set on subtests but not in only mode" time="*" disabled="0" errors="0" tests="3" failures="0" skipped="0" hostname="HOSTNAME">
 		<testcase name="running subtest 1" time="*" classname="test"/>
-		<testcase name="running subtest 2" time="*" classname="test"/>
-		<!-- 'only' and 'runOnly' require the &#45;&#45;test-only command-line option. -->
 		<testcase name="running subtest 3" time="*" classname="test"/>
-		<!-- 'only' and 'runOnly' require the &#45;&#45;test-only command-line option. -->
 		<testcase name="running subtest 4" time="*" classname="test"/>
 	</testsuite>
-	<!-- 'only' and 'runOnly' require the &#45;&#45;test-only command-line option. -->
 	<testcase name="custom inspect symbol fail" time="*" classname="test" failure="customized">
 		<failure type="testCodeFailure" message="customized">
 [Error [ERR_TEST_FAILURE]: customized] { code: 'ERR_TEST_FAILURE', failureType: 'testCodeFailure', cause: customized }
@@ -521,9 +517,9 @@ Error [ERR_TEST_FAILURE]: test could not be started because its parent finished
 	<!-- Error: Test "immediate reject - passes but warns" at test/fixtures/test-runner/output/output.js:86:1 generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. -->
 	<!-- Error: Test "callback called twice in different ticks" at test/fixtures/test-runner/output/output.js:251:1 generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event. -->
 	<!-- Error: Test "callback async throw after done" at test/fixtures/test-runner/output/output.js:269:1 generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. -->
-	<!-- tests 76 -->
+	<!-- tests 75 -->
 	<!-- suites 0 -->
-	<!-- pass 35 -->
+	<!-- pass 34 -->
 	<!-- fail 25 -->
 	<!-- cancelled 3 -->
 	<!-- skipped 9 -->

test/fixtures/test-runner/output/output.js

@@ -274,7 +274,7 @@ test('callback async throw after done', (t, done) => {
   done();
 });
 
-test('only is set but not in only mode', { only: true }, async (t) => {
+test('only is set on subtests but not in only mode', async (t) => {
   // All of these subtests should run.
   await t.test('running subtest 1');
   t.runOnly(true);

test/fixtures/test-runner/output/output.snapshot

@@ -463,35 +463,27 @@ ok 48 - callback async throw after done
   ---
   duration_ms: *
   ...
-# Subtest: only is set but not in only mode
+# Subtest: only is set on subtests but not in only mode
     # Subtest: running subtest 1
     ok 1 - running subtest 1
       ---
       duration_ms: *
       ...
-    # Subtest: running subtest 2
-    ok 2 - running subtest 2
-      ---
-      duration_ms: *
-      ...
-    # 'only' and 'runOnly' require the --test-only command-line option.
     # Subtest: running subtest 3
-    ok 3 - running subtest 3
+    ok 2 - running subtest 3
       ---
       duration_ms: *
       ...
-    # 'only' and 'runOnly' require the --test-only command-line option.
     # Subtest: running subtest 4
-    ok 4 - running subtest 4
+    ok 3 - running subtest 4
       ---
       duration_ms: *
       ...
-    1..4
-ok 49 - only is set but not in only mode
+    1..3
+ok 49 - only is set on subtests but not in only mode
   ---
   duration_ms: *
   ...
-# 'only' and 'runOnly' require the --test-only command-line option.
 # Subtest: custom inspect symbol fail
 not ok 50 - custom inspect symbol fail
   ---
@@ -718,9 +710,9 @@ not ok 62 - invalid subtest fail
 # Error: Test "immediate reject - passes but warns" at test/fixtures/test-runner/output/output.js:(LINE):1 generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Error: Test "callback called twice in different ticks" at test/fixtures/test-runner/output/output.js:(LINE):1 generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Error: Test "callback async throw after done" at test/fixtures/test-runner/output/output.js:(LINE):1 generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event.
-# tests 76
+# tests 75
 # suites 0
-# pass 35
+# pass 34
 # fail 25
 # cancelled 3
 # skipped 9

test/fixtures/test-runner/output/output_cli.snapshot

@@ -463,7 +463,7 @@ ok 48 - callback async throw after done
   ---
   duration_ms: *
   ...
-# Subtest: only is set but not in only mode
+# Subtest: only is set on subtests but not in only mode
     # Subtest: running subtest 1
     ok 1 - running subtest 1
       ---
@@ -487,11 +487,10 @@ ok 48 - callback async throw after done
       duration_ms: *
       ...
     1..4
-ok 49 - only is set but not in only mode
+ok 49 - only is set on subtests but not in only mode
   ---
   duration_ms: *
   ...
-# 'only' and 'runOnly' require the --test-only command-line option.
 # Subtest: custom inspect symbol fail
 not ok 50 - custom inspect symbol fail
   ---

test/fixtures/test-runner/output/spec_reporter.snapshot

@@ -185,15 +185,11 @@
       *
 
  callback async throw after done (*ms)
- only is set but not in only mode
+ only is set on subtests but not in only mode
    running subtest 1 (*ms)
-   running subtest 2 (*ms)
-   'only' and 'runOnly' require the --test-only command-line option.
    running subtest 3 (*ms)
-   'only' and 'runOnly' require the --test-only command-line option.
    running subtest 4 (*ms)
- only is set but not in only mode (*ms)
- 'only' and 'runOnly' require the --test-only command-line option.
+ only is set on subtests but not in only mode (*ms)
  custom inspect symbol fail (*ms)
   customized
 
@@ -304,9 +300,9 @@
  Error: Test "immediate reject - passes but warns" at test/fixtures/test-runner/output/output.js:86:1 generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
  Error: Test "callback called twice in different ticks" at test/fixtures/test-runner/output/output.js:251:1 generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.
  Error: Test "callback async throw after done" at test/fixtures/test-runner/output/output.js:269:1 generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event.
- tests 76
+ tests 75
  suites 0
- pass 35
+ pass 34
  fail 25
  cancelled 3
  skipped 9

test/fixtures/test-runner/output/spec_reporter_cli.snapshot

@@ -185,15 +185,14 @@
       *
 
  callback async throw after done (*ms)
- only is set but not in only mode
+ only is set on subtests but not in only mode
    running subtest 1 (*ms)
    running subtest 2 (*ms)
    'only' and 'runOnly' require the --test-only command-line option.
    running subtest 3 (*ms)
    'only' and 'runOnly' require the --test-only command-line option.
    running subtest 4 (*ms)
- only is set but not in only mode (*ms)
- 'only' and 'runOnly' require the --test-only command-line option.
+ only is set on subtests but not in only mode (*ms)
  custom inspect symbol fail (*ms)
   customized
 

test/parallel/test-runner-no-isolation-filtering.js

@@ -31,6 +31,28 @@ test('works with --test-only', () => {
   assert.match(stdout, /ok 1 - suite two - test/);
 });
 
+test('works without --test-only', () => {
+  const args = [
+    '--test',
+    '--test-reporter=tap',
+    '--experimental-test-isolation=none',
+    fixture1,
+    fixture2,
+  ];
+  const child = spawnSync(process.execPath, args);
+  const stdout = child.stdout.toString();
+
+  assert.strictEqual(child.status, 0);
+  assert.strictEqual(child.signal, null);
+  assert.match(stdout, /# tests 2/);
+  assert.match(stdout, /# suites 2/);
+  assert.match(stdout, /# pass 2/);
+  assert.match(stdout, /ok 1 - suite one/);
+  assert.match(stdout, /ok 1 - suite one - test/);
+  assert.match(stdout, /ok 2 - suite two/);
+  assert.match(stdout, /ok 1 - suite two - test/);
+});
+
 test('works with --test-name-pattern', () => {
   const args = [
     '--test',

test/parallel/test-runner-no-isolation.mjs

@@ -12,7 +12,7 @@ const stream = run({
 });
 
 stream.on('test:fail', mustNotCall());
-stream.on('test:pass', mustCall(5));
+stream.on('test:pass', mustCall(4));
 // eslint-disable-next-line no-unused-vars
 for await (const _ of stream);
 allowGlobals(globalThis.GLOBAL_ORDER);
@@ -28,12 +28,6 @@ deepStrictEqual(globalThis.GLOBAL_ORDER, [
   'afterEach one: suite one - test',
   'afterEach two: suite one - test',
 
-  'beforeEach one: test one',
-  'beforeEach two: test one',
-  'test one',
-  'afterEach one: test one',
-  'afterEach two: test one',
-
   'before suite two: suite two',
 
   'beforeEach one: suite two - test',

test/parallel/test-runner-output.mjs

@@ -225,7 +225,7 @@ const tests = [
   } : false,
   {
     name: 'test-runner/output/test-diagnostic-warning-without-test-only-flag.js',
-    flags: ['--test-reporter=tap'],
+    flags: ['--test', '--test-reporter=tap'],
   },
   process.features.inspector ? {
     name: 'test-runner/output/coverage-width-80.mjs',