testament :show duration also for failed tests; improve `tshould_not_…

…work`; mitigate #17946 tchannels timeouts (#17947)

* refs #17946; refactor testament test summary, show test duration for failures; increase timeout tchannels

* revert workarounds from #16698 and add allowPrefixMatch optional param to greedyOrderedSubsetLines

* add test

* workaround for yet another testament bug

testament/lib/stdtest/testutils.nim

@@ -29,19 +29,25 @@ template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) =
 when not defined(js):
   import std/strutils
 
-  proc greedyOrderedSubsetLines*(lhs, rhs: string): bool =
+  proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool =
     ## Returns true if each stripped line in `lhs` appears in rhs, using a greedy matching.
+    # xxx improve error reporting by showing the last matched pair
     iterator splitLinesClosure(): string {.closure.} =
       for line in splitLines(rhs.strip):
         yield line
+    template isMatch(lhsi, rhsi): bool =
+      if allowPrefixMatch:
+        startsWith(rhsi, lhsi):
+      else:
+        lhsi == rhsi
 
     var rhsIter = splitLinesClosure
     var currentLine = strip(rhsIter())
 
     for line in lhs.strip.splitLines:
       let line = line.strip
       if line.len != 0:
-        while line != currentLine:
+        while not isMatch(line, currentLine):
           currentLine = strip(rhsIter())
           if rhsIter.finished:
             return false

testament/testament.nim

@@ -288,16 +288,18 @@ proc addResult(r: var TResults, test: TTest, target: TTarget,
                             expected = expected,
                             given = given)
   r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success)
+  template dispNonSkipped(color, outcome) =
+    maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)"
   template disp(msg) =
     maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name
   if success == reSuccess:
-    maybeStyledEcho fgGreen, "PASS: ", fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)"
+    dispNonSkipped(fgGreen, "PASS: ")
   elif success == reDisabled:
     if test.spec.inCurrentBatch: disp("SKIP:")
     else: disp("NOTINBATCH:")
   elif success == reJoined: disp("JOINED:")
   else:
-    maybeStyledEcho styleBright, fgRed, failString, fgCyan, name
+    dispNonSkipped(fgRed, failString)
     maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\""
     maybeStyledEcho styleBright, fgRed, "Failure: ", $success
     if success in {reBuildFailed, reNimcCrash, reInstallFailed}:

testament/tests/shouldfail/tccodecheck.nim

@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   ccodecheck: "baz"
 """
 

testament/tests/shouldfail/tcolumn.nim

@@ -1,6 +1,5 @@
 discard """
   errormsg: "undeclared identifier: 'undeclared'"
-  targets: "c"
   line: 9
   column: 7
 """

testament/tests/shouldfail/terrormsg.nim

@@ -1,6 +1,5 @@
 discard """
   errormsg: "wrong error message"
-  targets: "c"
   line: 9
   column: 6
 """

testament/tests/shouldfail/texitcode1.nim

@@ -1,4 +1,3 @@
 discard """
-  targets: "c"
   exitcode: 1
 """

testament/tests/shouldfail/tfile.nim

@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   errormsg: "undeclared identifier: 'undefined'"
   file: "notthisfile.nim"
 """

testament/tests/shouldfail/tline.nim

@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   errormsg: "undeclared identifier: 'undeclared'"
   line: 10
   column: 6

testament/tests/shouldfail/tmaxcodesize.nim

@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   maxcodesize: 1
 """
 

testament/tests/shouldfail/tnimout.nim

@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   nimout: "Hello World!"
   action: compile
 """

testament/tests/shouldfail/tnimoutfull.nim

@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   nimout: '''
 msg1
 msg2

testament/tests/shouldfail/toutput.nim

@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   output: '''
   done
   '''

testament/tests/shouldfail/toutputsub.nim

@@ -1,6 +1,5 @@
 discard """
   outputsub: "something else"
-  targets: "c"
 """
 
 echo "Hello World!"

testament/tests/shouldfail/treject.nim

@@ -1,6 +1,5 @@
 discard """
   action: "reject"
-  targets: "c"
 """
 
 # Because we set action="reject", we expect this line not to compile. But the

testament/tests/shouldfail/tsortoutput.nim

@@ -1,6 +1,5 @@
 discard """
   sortoutput: true
-  targets: "c"
   output: '''
 2
 1

testament/tests/shouldfail/ttimeout.nim

@@ -1,6 +1,5 @@
 discard """
   timeout: "0.1"
-  targets: "c"
 """
 
 import os

testament/tests/shouldfail/tvalgrind.nim

@@ -1,6 +1,5 @@
 discard """
   valgrind: true
-  targets: "c"
   cmd: "nim $target --gc:arc -d:useMalloc $options $file"
 """
 

tests/misc/trunner.nim

@@ -6,6 +6,8 @@ discard """
 ## tests that don't quite fit the mold and are easier to handle via `execCmdEx`
 ## A few others could be added to here to simplify code.
 ## Note: this test is a bit slow but tests a lot of things; please don't disable.
+## Note: if needed, we could use `matrix: "-d:case1; -d:case2"` to split this
+## into several independent tests while retaining the common test helpers.
 
 import std/[strformat,os,osproc,unittest,compilesettings]
 from std/sequtils import toSeq,mapIt

tests/stdlib/tchannels.nim

@@ -1,5 +1,5 @@
 discard """
-  timeout:  5.0 # but typically < 1s
+  timeout:  20.0 # but typically < 1s (in isolation but other tests running in parallel can affect this since based on epochTime)
   disabled: "freebsd"
   matrix: "--gc:arc --threads:on; --gc:arc --threads:on -d:danger"
 """

tests/stdlib/ttestutils.nim

@@ -22,3 +22,18 @@ block: # greedyOrderedSubsetLines
     not greedyOrderedSubsetLines("a1\na5", "a0\na1\na2\na3\na4\na5:suffix")
     not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\nprefix:a5")
     not greedyOrderedSubsetLines("a5", "a0\na1\na2\na3\na4\na5:suffix")
+
+block: # greedyOrderedSubsetLines with allowPrefixMatch = true
+  template fn(a, b): bool =
+    greedyOrderedSubsetLines(a, b, allowPrefixMatch = true)
+  assertAll:
+    fn("a1\na3", "a0\na1\na2\na3_suffix\na4")
+    not fn("a1\na3", "a0\na1\na2\nprefix_a3\na4")
+    # these are same as above, could be refactored
+    not fn("a3\na1", "a0\na1\na2\na3\na4") # out of order
+    not fn("a1\na5", "a0\na1\na2\na3\na4") # a5 not in lhs
+
+    not fn("a1\na5", "a0\na1\na2\na3\na4\nprefix:a5")
+    fn("a1\na5", "a0\na1\na2\na3\na4\na5:suffix")
+    not fn("a5", "a0\na1\na2\na3\na4\nprefix:a5")
+    fn("a5", "a0\na1\na2\na3\na4\na5:suffix")

tests/testament/tshould_not_work.nim

@@ -1,39 +1,54 @@
 discard """
-cmd: "testament/testament --directory:testament --colors:off --backendLogging:off --nim:$nim category shouldfail"
-action: compile
-nimout: '''
-FAIL: tests/shouldfail/tccodecheck.nim c
+  joinable: false
+"""
+
+const expected = """
+FAIL: tests/shouldfail/tccodecheck.nim
 Failure: reCodegenFailure
 Expected:
 baz
-FAIL: tests/shouldfail/tcolumn.nim c
+FAIL: tests/shouldfail/tcolumn.nim
 Failure: reLinesDiffer
-FAIL: tests/shouldfail/terrormsg.nim c
+FAIL: tests/shouldfail/terrormsg.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/texitcode1.nim c
+FAIL: tests/shouldfail/texitcode1.nim
 Failure: reExitcodesDiffer
-FAIL: tests/shouldfail/tfile.nim c
+FAIL: tests/shouldfail/tfile.nim
 Failure: reFilesDiffer
-FAIL: tests/shouldfail/tline.nim c
+FAIL: tests/shouldfail/tline.nim
 Failure: reLinesDiffer
-FAIL: tests/shouldfail/tmaxcodesize.nim c
+FAIL: tests/shouldfail/tmaxcodesize.nim
 Failure: reCodegenFailure
 max allowed size: 1
-FAIL: tests/shouldfail/tnimout.nim c
+FAIL: tests/shouldfail/tnimout.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/tnimoutfull.nim c
+FAIL: tests/shouldfail/tnimoutfull.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/toutput.nim c
+FAIL: tests/shouldfail/toutput.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/toutputsub.nim c
+FAIL: tests/shouldfail/toutputsub.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/treject.nim c
+FAIL: tests/shouldfail/treject.nim
 Failure: reFilesDiffer
-FAIL: tests/shouldfail/tsortoutput.nim c
+FAIL: tests/shouldfail/tsortoutput.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/ttimeout.nim c
+FAIL: tests/shouldfail/ttimeout.nim
 Failure: reTimeout
-FAIL: tests/shouldfail/tvalgrind.nim c
+FAIL: tests/shouldfail/tvalgrind.nim
 Failure: reExitcodesDiffer
-'''
 """
+
+import std/[os,strformat,osproc]
+import stdtest/testutils
+
+proc main =
+  const nim = getCurrentCompilerExe()
+  # TODO: bin/testament instead? like other tools (eg bin/nim, bin/nimsuggest etc)
+  let testamentExe = "testament/testament"
+  let cmd = fmt"{testamentExe} --directory:testament --colors:off --backendLogging:off --nim:{nim} category shouldfail"
+  let (outp, status) = execCmdEx(cmd)
+  doAssert status == 1, $status
+
+  let ok = greedyOrderedSubsetLines(expected, outp, allowPrefixMatch = true)
+  doAssert ok, &"\nexpected:\n{expected}\noutp:\n{outp}"
+main()