From dda64ddfbd93a589e1dbd6f719d510908d0d0cb5 Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Sun, 19 Jun 2022 10:58:48 +0300 Subject: [PATCH] test_runner: add Subtest to tap protocol output PR-URL: https://github.com/nodejs/node/pull/43417 Reviewed-By: Benjamin Gruenbaum Reviewed-By: James M Snell Reviewed-By: Colin Ihrig --- lib/internal/test_runner/tap_stream.js | 4 ++ lib/internal/test_runner/test.js | 6 ++ test/message/test_runner_no_refs.out | 2 + test/message/test_runner_only_tests.out | 23 ++++++ test/message/test_runner_output.out | 71 +++++++++++++++++++ .../test_runner_unresolved_promise.out | 3 + 6 files changed, 109 insertions(+) diff --git a/lib/internal/test_runner/tap_stream.js b/lib/internal/test_runner/tap_stream.js index a6bfbb3367cd79..ca0c101aa9260c 100644 --- a/lib/internal/test_runner/tap_stream.js +++ b/lib/internal/test_runner/tap_stream.js @@ -71,6 +71,10 @@ class TapStream extends Readable { return `TODO${reason ? ` ${tapEscape(reason)}` : ''}`; } + subtest(indent, name) { + this.#tryPush(`${indent}# Subtest: ${tapEscape(name)}\n`); + } + details(indent, duration, error) { let details = `${indent} ---\n`; diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index cac379b32582a4..2876b3d66640cd 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -183,6 +183,10 @@ class Test extends AsyncResource { return; } + if (i === 1 && this.parent !== null) { + this.reporter.subtest(this.indent, this.name); + } + // Report the subtest's results and remove it from the ready map. subtest.finalize(); this.readySubtests.delete(i); @@ -416,6 +420,8 @@ class Test extends AsyncResource { // Output this test's results and update the parent's waiting counter. if (this.subtests.length > 0) { this.reporter.plan(this.subtests[0].indent, this.subtests.length); + } else { + this.reporter.subtest(this.indent, this.name); } this.report(); diff --git a/test/message/test_runner_no_refs.out b/test/message/test_runner_no_refs.out index c5407e3bd330c0..078ab7c390e488 100644 --- a/test/message/test_runner_no_refs.out +++ b/test/message/test_runner_no_refs.out @@ -1,4 +1,6 @@ TAP version 13 +# Subtest: does not keep event loop alive + # Subtest: +does not keep event loop alive not ok 1 - +does not keep event loop alive --- duration_ms: * diff --git a/test/message/test_runner_only_tests.out b/test/message/test_runner_only_tests.out index 76e6337bb8953d..b65958b010f8ad 100644 --- a/test/message/test_runner_only_tests.out +++ b/test/message/test_runner_only_tests.out @@ -1,76 +1,97 @@ TAP version 13 +# Subtest: only = undefined ok 1 - only = undefined # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = undefined, skip = string ok 2 - only = undefined, skip = string # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = undefined, skip = true ok 3 - only = undefined, skip = true # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = undefined, skip = false ok 4 - only = undefined, skip = false # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = false ok 5 - only = false # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = false, skip = string ok 6 - only = false, skip = string # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = false, skip = true ok 7 - only = false, skip = true # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = false, skip = false ok 8 - only = false, skip = false # SKIP 'only' option not set --- duration_ms: * ... +# Subtest: only = true, skip = string ok 9 - only = true, skip = string # SKIP skip message --- duration_ms: * ... +# Subtest: only = true, skip = true ok 10 - only = true, skip = true # SKIP --- duration_ms: * ... +# Subtest: only = true, with subtests + # Subtest: running subtest 1 ok 1 - running subtest 1 --- duration_ms: * ... + # Subtest: running subtest 2 ok 2 - running subtest 2 --- duration_ms: * ... + # Subtest: skipped subtest 1 ok 3 - skipped subtest 1 # SKIP 'only' option not set --- duration_ms: * ... + # Subtest: skipped subtest 2 ok 4 - skipped subtest 2 # SKIP 'only' option not set --- duration_ms: * ... + # Subtest: running subtest 3 ok 5 - running subtest 3 --- duration_ms: * ... + # Subtest: running subtest 4 + # Subtest: running sub-subtest 1 ok 1 - running sub-subtest 1 --- duration_ms: * ... + # Subtest: running sub-subtest 2 ok 2 - running sub-subtest 2 --- duration_ms: * ... + # Subtest: skipped sub-subtest 1 ok 3 - skipped sub-subtest 1 # SKIP 'only' option not set --- duration_ms: * ... + # Subtest: skipped sub-subtest 2 ok 4 - skipped sub-subtest 2 # SKIP 'only' option not set --- duration_ms: * @@ -80,10 +101,12 @@ ok 10 - only = true, skip = true # SKIP --- duration_ms: * ... + # Subtest: skipped subtest 3 ok 7 - skipped subtest 3 # SKIP 'only' option not set --- duration_ms: * ... + # Subtest: skipped subtest 4 ok 8 - skipped subtest 4 # SKIP --- duration_ms: * diff --git a/test/message/test_runner_output.out b/test/message/test_runner_output.out index 6c2fa7eb14cc72..2f6e2502749068 100644 --- a/test/message/test_runner_output.out +++ b/test/message/test_runner_output.out @@ -1,12 +1,15 @@ TAP version 13 +# Subtest: sync pass todo ok 1 - sync pass todo # TODO --- duration_ms: * ... +# Subtest: sync pass todo with message ok 2 - sync pass todo with message # TODO this is a passing todo --- duration_ms: * ... +# Subtest: sync fail todo not ok 3 - sync fail todo # TODO --- duration_ms: * @@ -23,6 +26,7 @@ not ok 3 - sync fail todo # TODO * * ... +# Subtest: sync fail todo with message not ok 4 - sync fail todo with message # TODO this is a failing todo --- duration_ms: * @@ -41,19 +45,23 @@ not ok 4 - sync fail todo with message # TODO this is a failing todo * * ... +# Subtest: sync skip pass ok 5 - sync skip pass # SKIP --- duration_ms: * ... +# Subtest: sync skip pass with message ok 6 - sync skip pass with message # SKIP this is skipped --- duration_ms: * ... +# Subtest: sync pass ok 7 - sync pass --- duration_ms: * ... # this test should pass +# Subtest: sync throw fail not ok 8 - sync throw fail --- duration_ms: * @@ -70,14 +78,17 @@ not ok 8 - sync throw fail * * ... +# Subtest: async skip pass ok 9 - async skip pass # SKIP --- duration_ms: * ... +# Subtest: async pass ok 10 - async pass --- duration_ms: * ... +# Subtest: async throw fail not ok 11 - async throw fail --- duration_ms: * @@ -94,6 +105,7 @@ not ok 11 - async throw fail * * ... +# Subtest: async skip fail not ok 12 - async skip fail # SKIP --- duration_ms: * @@ -110,6 +122,7 @@ not ok 12 - async skip fail # SKIP * * ... +# Subtest: async assertion fail not ok 13 - async assertion fail --- duration_ms: * @@ -130,10 +143,12 @@ not ok 13 - async assertion fail * * ... +# Subtest: resolve pass ok 14 - resolve pass --- duration_ms: * ... +# Subtest: reject fail not ok 15 - reject fail --- duration_ms: * @@ -150,26 +165,33 @@ not ok 15 - reject fail * * ... +# Subtest: unhandled rejection - passes but warns ok 16 - unhandled rejection - passes but warns --- duration_ms: * ... +# Subtest: async unhandled rejection - passes but warns ok 17 - async unhandled rejection - passes but warns --- duration_ms: * ... +# Subtest: immediate throw - passes but warns ok 18 - immediate throw - passes but warns --- duration_ms: * ... +# Subtest: immediate reject - passes but warns ok 19 - immediate reject - passes but warns --- duration_ms: * ... +# Subtest: immediate resolve pass ok 20 - immediate resolve pass --- duration_ms: * ... +# Subtest: subtest sync throw fail + # Subtest: +sync throw fail not ok 1 - +sync throw fail --- duration_ms: * @@ -197,6 +219,7 @@ not ok 21 - subtest sync throw fail error: '1 subtest failed' code: 'ERR_TEST_FAILURE' ... +# Subtest: sync throw non-error fail not ok 22 - sync throw non-error fail --- duration_ms: * @@ -204,18 +227,23 @@ not ok 22 - sync throw non-error fail error: 'Symbol(thrown symbol from sync throw non-error fail)' code: 'ERR_TEST_FAILURE' ... +# Subtest: level 0a + # Subtest: level 1a ok 1 - level 1a --- duration_ms: * ... + # Subtest: level 1b ok 2 - level 1b --- duration_ms: * ... + # Subtest: level 1c ok 3 - level 1c --- duration_ms: * ... + # Subtest: level 1d ok 4 - level 1d --- duration_ms: * @@ -225,6 +253,8 @@ ok 23 - level 0a --- duration_ms: * ... +# Subtest: top level + # Subtest: +long running not ok 1 - +long running --- duration_ms: * @@ -232,6 +262,8 @@ ok 23 - level 0a error: 'test did not finish before its parent and was cancelled' code: 'ERR_TEST_FAILURE' ... + # Subtest: +short running + # Subtest: ++short running ok 1 - ++short running --- duration_ms: * @@ -249,18 +281,22 @@ not ok 24 - top level error: '1 subtest failed' code: 'ERR_TEST_FAILURE' ... +# Subtest: invalid subtest - pass but subtest fails ok 25 - invalid subtest - pass but subtest fails --- duration_ms: * ... +# Subtest: sync skip option ok 26 - sync skip option # SKIP --- duration_ms: * ... +# Subtest: sync skip option with message ok 27 - sync skip option with message # SKIP this is skipped --- duration_ms: * ... +# Subtest: sync skip option is false fail not ok 28 - sync skip option is false fail --- duration_ms: * @@ -276,59 +312,73 @@ not ok 28 - sync skip option is false fail * * ... +# Subtest: ok 29 - --- duration_ms: * ... +# Subtest: functionOnly ok 30 - functionOnly --- duration_ms: * ... +# Subtest: ok 31 - --- duration_ms: * ... +# Subtest: test with only a name provided ok 32 - test with only a name provided --- duration_ms: * ... +# Subtest: ok 33 - --- duration_ms: * ... +# Subtest: ok 34 - # SKIP --- duration_ms: * ... +# Subtest: test with a name and options provided ok 35 - test with a name and options provided # SKIP --- duration_ms: * ... +# Subtest: functionAndOptions ok 36 - functionAndOptions # SKIP --- duration_ms: * ... +# Subtest: escaped description \\ \# \\\#\\ ok 37 - escaped description \\ \# \\\#\\ --- duration_ms: * ... +# Subtest: escaped skip message ok 38 - escaped skip message # SKIP \#skip --- duration_ms: * ... +# Subtest: escaped todo message ok 39 - escaped todo message # TODO \#todo --- duration_ms: * ... +# Subtest: escaped diagnostic ok 40 - escaped diagnostic --- duration_ms: * ... # \#diagnostic +# Subtest: callback pass ok 41 - callback pass --- duration_ms: * ... +# Subtest: callback fail not ok 42 - callback fail --- duration_ms: * @@ -339,18 +389,22 @@ not ok 42 - callback fail * * ... +# Subtest: sync t is this in test ok 43 - sync t is this in test --- duration_ms: * ... +# Subtest: async t is this in test ok 44 - async t is this in test --- duration_ms: * ... +# Subtest: callback t is this in test ok 45 - callback t is this in test --- duration_ms: * ... +# Subtest: callback also returns a Promise not ok 46 - callback also returns a Promise --- duration_ms: * @@ -358,6 +412,7 @@ not ok 46 - callback also returns a Promise error: 'passed a callback but also returned a Promise' code: 'ERR_TEST_FAILURE' ... +# Subtest: callback throw not ok 47 - callback throw --- duration_ms: * @@ -373,6 +428,7 @@ not ok 47 - callback throw * * ... +# Subtest: callback called twice not ok 48 - callback called twice --- duration_ms: * @@ -383,10 +439,12 @@ not ok 48 - callback called twice * * ... +# Subtest: callback called twice in different ticks ok 49 - callback called twice in different ticks --- duration_ms: * ... +# Subtest: callback called twice in future tick not ok 50 - callback called twice in future tick --- duration_ms: * @@ -396,6 +454,7 @@ not ok 50 - callback called twice in future tick stack: |- * ... +# Subtest: callback async throw not ok 51 - callback async throw --- duration_ms: * @@ -405,22 +464,28 @@ not ok 51 - callback async throw stack: |- * ... +# Subtest: callback async throw after done ok 52 - callback async throw after done --- duration_ms: * ... +# Subtest: only is set but not in only mode + # Subtest: running subtest 1 ok 1 - running subtest 1 --- duration_ms: * ... + # Subtest: running subtest 2 ok 2 - running subtest 2 --- duration_ms: * ... + # Subtest: running subtest 3 ok 3 - running subtest 3 --- duration_ms: * ... + # Subtest: running subtest 4 ok 4 - running subtest 4 --- duration_ms: * @@ -430,6 +495,7 @@ ok 53 - only is set but not in only mode --- duration_ms: * ... +# Subtest: custom inspect symbol fail not ok 54 - custom inspect symbol fail --- duration_ms: * @@ -437,6 +503,7 @@ not ok 54 - custom inspect symbol fail error: 'customized' code: 'ERR_TEST_FAILURE' ... +# Subtest: custom inspect symbol that throws fail not ok 55 - custom inspect symbol that throws fail --- duration_ms: * @@ -448,6 +515,8 @@ not ok 55 - custom inspect symbol that throws fail } code: 'ERR_TEST_FAILURE' ... +# Subtest: subtest sync throw fails + # Subtest: sync throw fails at first not ok 1 - sync throw fails at first --- duration_ms: * @@ -466,6 +535,7 @@ not ok 55 - custom inspect symbol that throws fail * * ... + # Subtest: sync throw fails at second not ok 2 - sync throw fails at second --- duration_ms: * @@ -492,6 +562,7 @@ not ok 56 - subtest sync throw fails error: '2 subtests failed' code: 'ERR_TEST_FAILURE' ... +# Subtest: invalid subtest fail not ok 57 - invalid subtest fail --- duration_ms: * diff --git a/test/message/test_runner_unresolved_promise.out b/test/message/test_runner_unresolved_promise.out index 98f52966c33bcb..58b3a4ed89a616 100644 --- a/test/message/test_runner_unresolved_promise.out +++ b/test/message/test_runner_unresolved_promise.out @@ -1,8 +1,10 @@ TAP version 13 +# Subtest: pass ok 1 - pass --- duration_ms: * ... +# Subtest: never resolving promise not ok 2 - never resolving promise --- duration_ms: * @@ -12,6 +14,7 @@ not ok 2 - never resolving promise stack: |- * ... +# Subtest: fail not ok 3 - fail --- duration_ms: *