From cc369751d951f6bf5d64578d48ade657893426d5 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 14 Jun 2023 10:48:15 +0100 Subject: [PATCH 01/15] Test we retry on timeouts --- test/integrationtests.jl | 13 +++++++++++-- test/testfiles/_retry_tests.jl | 10 ++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/test/integrationtests.jl b/test/integrationtests.jl index 59038214..320b64f2 100644 --- a/test/integrationtests.jl +++ b/test/integrationtests.jl @@ -609,7 +609,8 @@ end @testset "test retrying failing testitem" begin file = joinpath(TEST_FILES_DIR, "_retry_tests.jl") - results = encased_testset(()->runtests(file; nworkers=1, retries=2)) + # must run with `testitem_timeout < 60` for test to timeout as expected. + results = encased_testset(()->runtests(file; nworkers=1, retries=2, testitem_timeout=3)) # Test we _ran_ each test-item the expected number of times read_count(x) = parse(Int, read(x, String)) # Passes on second attempt, so only need to retry once. @@ -622,11 +623,19 @@ end # Doesn't pass ever, so need all retries. This testitem set `retries=1` which is less # than the `retries=2` that `runtests` set, so we should retry 2 times. @test read_count(joinpath(tempdir(), "num_runs_4")) == 3 + # Times out always, so should retry as many times as allowed. + # Since it will be a new worker for each retry, we write one file for each. + @test count(contains("num_runs_5"), readdir(tempdir())) == 3 # Test we _report_ the expected number of test-items - @test n_tests(results) == 4 + @test n_tests(results) == 5 # The first two testitems pass on retry. @test n_passed(results) == 2 + + # Clear out any files created by this testset + foreach(readdir(tempdir(); join=true)) do tmp + contains(tmp, "num_runs") && rm(tmp) + end end @testset "testitem timeout" begin diff --git a/test/testfiles/_retry_tests.jl b/test/testfiles/_retry_tests.jl index 162de5f2..0b329907 100644 --- a/test/testfiles/_retry_tests.jl +++ b/test/testfiles/_retry_tests.jl @@ -40,3 +40,13 @@ end write(joinpath(tempdir(), "num_runs_4"), string(NUM_RUNS_4[])) @test false end + + +# For this to timeout, must be run with `testitem_timeout < 60` +# Cannot use `StatefulSetup` for this as it will be a new worker +# every retry, so the `setup` will always have been re-evaluated anew. +@testitem "Timeout always" retries=1 begin + write(tempname() * "_num_runs_5", "1") + sleep(60.0) + @test true +end From 4c6081ab27523c7e30daa24ca5c939f7c99fcd93 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Tue, 20 Jun 2023 15:01:17 +0100 Subject: [PATCH 02/15] Refactor test-error handling to always record the error --- src/ReTestItems.jl | 71 ++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 41 deletions(-) diff --git a/src/ReTestItems.jl b/src/ReTestItems.jl index a39a543d..dcd611bc 100644 --- a/src/ReTestItems.jl +++ b/src/ReTestItems.jl @@ -384,7 +384,7 @@ function start_and_manage_worker( ) ntestitems = length(testitems.testitems) worker = start_worker(proj_name, nworker_threads, worker_init_expr, ntestitems) - nretries = 0 + run_number = 1 while testitem !== nothing ch = Channel{TestItemResult}(1) testitem.workerid[] = worker.pid @@ -415,7 +415,7 @@ function start_and_manage_worker( ts = testitem_result.testset push!(testitem.testsets, ts) push!(testitem.stats, testitem_result.stats) - print_errors_and_captured_logs(testitem, nretries + 1; logs) + print_errors_and_captured_logs(testitem, run_number; logs) report_empty_testsets(testitem, ts) # Run GC to free memory on the worker before next testitem. @debugv 2 "Running GC on $worker" @@ -423,62 +423,51 @@ function start_and_manage_worker( # If the result isn't a pass, we throw to go to the outer try-catch throw_if_failed(ts) testitem = next_testitem(testitems, testitem.id[]) - nretries = 0 + run_number = 1 finally close(timer) end catch e @debugv 2 "Error" exception=e - if !(e isa WorkerTerminatedException || e isa TimeoutException || e isa TestSetFailure) - # we don't expect any other kind of error, so rethrow, which will propagate - # back up to the main coordinator task and throw to the user - rethrow() - end - - if !(e isa TestSetFailure) - println(DEFAULT_STDOUT[]) - # Explicitly show captured logs or say there weren't any in case we're about - # to terminte the worker - _print_captured_logs(DEFAULT_STDOUT[], testitem, nretries + 1) - end - + # Handle the exception if e isa TimeoutException @debugv 1 "Test item $(repr(testitem.name)) timed out. Terminating worker $worker" + # Explicitly show captured logs or say there weren't any before terminating worker + println(DEFAULT_STDOUT[]) + _print_captured_logs(DEFAULT_STDOUT[], testitem, run_number) terminate!(worker) wait(worker) + @warn "$worker timed out evaluating test item $(repr(testitem.name)) afer $timeout seconds. \ + Recording test error." + record_test_error!(testitem, run_number) + elseif e isa WorkerTerminatedException + @warn "$worker died evaluating test item $(repr(testitem.name)). \ + Recording test error." + record_test_error!(testitem, run_number) + elseif e isa TestSetFailure + # We already printed the error and recorded the testset. + else + # We don't expect any other kind of error, so rethrow, which will propagate + # back up to the main coordinator task and throw to the user + rethrow() end - if nretries == retry_limit - if e isa TimeoutException - @warn "$worker timed out evaluating test item $(repr(testitem.name)) afer $timeout seconds. \ - Recording test error, and starting a new worker." - record_test_error!(testitem, nretries + 1) - elseif e isa WorkerTerminatedException - @warn "$worker died evaluating test item $(repr(testitem.name)). \ - Recording test error, and starting a new worker." - record_test_error!(testitem, nretries + 1) - else - @assert e isa TestSetFailure - # We already printed the error and recorded the testset. - end + # Handle retries + if run_number == (1 + retry_limit) testitem = next_testitem(testitems, testitem.id[]) - nretries = 0 + run_number = 1 else - nretries += 1 - if e isa TimeoutException - @warn "$worker timed out evaluating test item $(repr(testitem.name)) afer $timeout seconds. \ - Starting a new worker and retrying. Retry=$nretries." - elseif e isa WorkerTerminatedException - @warn "$worker died evaluating test item $(repr(testitem.name)). \ - Starting a new worker and retrying. Retry=$nretries." + run_number += 1 + if e isa TestSetFailure + @info "Retrying $(repr(testitem.name)) on $worker. Run=$run_number." else - @assert e isa TestSetFailure - @warn "Test item $(repr(testitem.name)) failed. Retrying on $worker. Retry=$nretries." + @info "Retrying $(repr(testitem.name)) on a new worker. Run=$run_number." end end - if (e isa TimeoutException || e isa WorkerTerminatedException) + # Replace worker if necessary + if (e isa TimeoutException || e isa WorkerTerminatedException) && (testitem !== nothing) worker = start_worker(proj_name, nworker_threads, worker_init_expr, ntestitems) end - # now we loop back around to reschedule the testitem + # Now loop back around to reschedule the testitem continue end end From 93e9a9f227c99bb5151b70f60d7e745154d7d62c Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Tue, 20 Jun 2023 16:34:37 +0100 Subject: [PATCH 03/15] Record more informative error: include timeout/abort info --- src/ReTestItems.jl | 29 +++++++++++++++++++++++++---- test/integrationtests.jl | 4 ++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/ReTestItems.jl b/src/ReTestItems.jl index dcd611bc..9072aa7c 100644 --- a/src/ReTestItems.jl +++ b/src/ReTestItems.jl @@ -360,10 +360,31 @@ end struct TestSetFailure <: Exception end throw_if_failed(ts) = ts.anynonpass ? throw(TestSetFailure()) : nothing -function record_test_error!(testitem, ntries) +function record_timeout!(testitem, run_number::Int, timeout_limit::Real) + timeout_s = round(Int, timeout_limit) + time_str = if timeout_s < 60 + string(timeout_s, "s") + else + mins, secs = divrem(timeout_s, 60) + if iszero(secs) + string(mins, "m") + else + string(mins, "m", lpad(secs, 2, "0"), "s") + end + end + msg = "Timed out after $time_str evaluating test item $(repr(testitem.name)) (run=$run_number)" + record_test_error!(testitem, msg) +end + +function record_worker_terminated!(testitem, run_number::Int) + msg = "Worker aborted evaluating test item $(repr(testitem.name)) (run=$run_number)" + record_test_error!(testitem, msg) +end + +function record_test_error!(testitem, msg) Test.TESTSET_PRINT_ENABLE[] = false ts = DefaultTestSet(testitem.name) - err = ErrorException("test item $(repr(testitem.name)) didn't succeed after $ntries tries") + err = ErrorException(msg) Test.record(ts, Test.Error(:nontest_error, Test.Expr(:tuple), err, Base.ExceptionStack([(exception=err, backtrace=Union{Ptr{Nothing}, Base.InterpreterIP}[])]), LineNumberNode(testitem.line, testitem.file))) @@ -439,11 +460,11 @@ function start_and_manage_worker( wait(worker) @warn "$worker timed out evaluating test item $(repr(testitem.name)) afer $timeout seconds. \ Recording test error." - record_test_error!(testitem, run_number) + record_timeout!(testitem, run_number, timeout) elseif e isa WorkerTerminatedException @warn "$worker died evaluating test item $(repr(testitem.name)). \ Recording test error." - record_test_error!(testitem, run_number) + record_worker_terminated!(testitem, run_number) elseif e isa TestSetFailure # We already printed the error and recorded the testset. else diff --git a/test/integrationtests.jl b/test/integrationtests.jl index 320b64f2..8cfbcc5d 100644 --- a/test/integrationtests.jl +++ b/test/integrationtests.jl @@ -604,7 +604,7 @@ end # Test the error is as expected err = only(non_passes(results)) @test err.test_type == :nontest_error - @test err.value == string(ErrorException("test item \"Abort\" didn't succeed after 1 tries")) + @test err.value == string(ErrorException("Worker aborted evaluating test item \"Abort\" (run=1)")) end @testset "test retrying failing testitem" begin @@ -650,7 +650,7 @@ end # Test the error is as expected err = only(non_passes(results)) @test err.test_type == :nontest_error - @test err.value == string(ErrorException("test item \"Test item takes 60 seconds\" didn't succeed after 1 tries")) + @test err.value == string(ErrorException("Timed out after 4s evaluating test item \"Test item takes 60 seconds\" (run=1)")) end @testset "Error outside `@testitem`" begin From 0c07b6eea32b440c45bea8d25dd679af19d7cd4f Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Tue, 20 Jun 2023 17:21:07 +0100 Subject: [PATCH 04/15] Update reference tests --- test/integrationtests.jl | 2 +- test/junit_xml.jl | 11 +- ...ort.xml => retry_tests_report_0worker.xml} | 210 +++++++------ .../references/retry_tests_report_1worker.xml | 294 ++++++++++++++++++ test/testfiles/_retry_tests.jl | 4 +- 5 files changed, 416 insertions(+), 105 deletions(-) rename test/references/{retry_tests_report.xml => retry_tests_report_0worker.xml} (54%) create mode 100644 test/references/retry_tests_report_1worker.xml diff --git a/test/integrationtests.jl b/test/integrationtests.jl index 8cfbcc5d..13c0ae30 100644 --- a/test/integrationtests.jl +++ b/test/integrationtests.jl @@ -609,7 +609,7 @@ end @testset "test retrying failing testitem" begin file = joinpath(TEST_FILES_DIR, "_retry_tests.jl") - # must run with `testitem_timeout < 60` for test to timeout as expected. + # must run with `testitem_timeout < 30` for test to timeout as expected. results = encased_testset(()->runtests(file; nworkers=1, retries=2, testitem_timeout=3)) # Test we _ran_ each test-item the expected number of times read_count(x) = parse(Int, read(x, String)) diff --git a/test/junit_xml.jl b/test/junit_xml.jl index eb8d8ee2..97f3a2b2 100644 --- a/test/junit_xml.jl +++ b/test/junit_xml.jl @@ -100,11 +100,18 @@ end mktempdir() do dir withenv("RETESTITEMS_REPORT_LOCATION" => dir, "RETESTITEMS_NWORKERS" => nworkers) do try # Ignore the fact that the `_retry_tests.jl` testset has failures/errors. - run(`$(Base.julia_cmd()) --project -e 'using ReTestItems; runtests("testfiles/_retry_tests.jl"; report=true, retries=2)'`) + run(`$(Base.julia_cmd()) --project -e 'using ReTestItems; runtests("testfiles/_retry_tests.jl"; testitem_timeout=5, report=true, retries=2)'`) catch end report = only(filter(endswith("xml"), readdir(dir, join=true))) - test_reference(joinpath(REF_DIR, "retry_tests_report.xml"), report) + # timeouts are not supported when running without workers, so + # with 0 workers the "retry tests" that are expected to timeout will pass. + ref = if nworkers == 0 + joinpath(REF_DIR, "retry_tests_report_0worker.xml") + else + joinpath(REF_DIR, "retry_tests_report_1worker.xml") + end + test_reference(ref, report) end end end diff --git a/test/references/retry_tests_report.xml b/test/references/retry_tests_report_0worker.xml similarity index 54% rename from test/references/retry_tests_report.xml rename to test/references/retry_tests_report_0worker.xml index d51eef63..f773b229 100644 --- a/test/references/retry_tests_report.xml +++ b/test/references/retry_tests_report_0worker.xml @@ -1,47 +1,47 @@ - - - + + + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Pass on second run") at test/testfiles/_retry_tests.jl:1 on worker 63497 + Captured logs for test setup "StatefulSetup" (dependency of "Pass on second run") at test/testfiles/_retry_tests.jl:1 on worker 29306 These are the test setup logs. -No Captured Logs for test item "Pass on second run" at test/testfiles/_retry_tests.jl:10 on worker 63497 -Error in testset "Pass on second run" on worker 63497: +No Captured Logs for test item "Pass on second run" at test/testfiles/_retry_tests.jl:10 on worker 29306 +Error in testset "Pass on second run" on worker 29306: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:15 Expression: x - + - - + + - + - + - + - - - - + + + + - + - Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 63497 + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 29306 These are the test setup logs. -Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 63497 +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 29306 These are the logs for run number: 1 -Error in testset "Error, then Fail, then Pass" on worker 63497: +Error in testset "Error, then Fail, then Pass" on worker 29306: Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 Test threw exception Expression: not_defined_err @@ -53,170 +53,180 @@ Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tes @ /repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 63497 + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 29306 These are the test setup logs. -Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 63497 +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 29306 These are the logs for run number: 2 -Error in testset "Error, then Fail, then Pass" on worker 63497: +Error in testset "Error, then Fail, then Pass" on worker 29306: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:26 Expression: 2 + 2 == 5 Evaluated: 4 == 5 - + - - + + - + - + - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63497 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63497 -Error in testset "Has retries=4 and always fails" on worker 63497: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 29306 +Error in testset "Has retries=4 and always fails" on worker 29306: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63497 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63497 -Error in testset "Has retries=4 and always fails" on worker 63497: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 29306 +Error in testset "Has retries=4 and always fails" on worker 29306: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - - + + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63497 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63497 -Error in testset "Has retries=4 and always fails" on worker 63497: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 29306 +Error in testset "Has retries=4 and always fails" on worker 29306: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63497 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63497 -Error in testset "Has retries=4 and always fails" on worker 63497: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 29306 +Error in testset "Has retries=4 and always fails" on worker 29306: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63497 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63497 -Error in testset "Has retries=4 and always fails" on worker 63497: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 29306 +Error in testset "Has retries=4 and always fails" on worker 29306: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63497 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 63497 -Error in testset "Has retries=1 and always fails" on worker 63497: +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 29306 +Error in testset "Has retries=1 and always fails" on worker 29306: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63497 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 63497 -Error in testset "Has retries=1 and always fails" on worker 63497: +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 29306 +Error in testset "Has retries=1 and always fails" on worker 29306: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63497 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 63497 -Error in testset "Has retries=1 and always fails" on worker 63497: +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 29306 +Error in testset "Has retries=1 and always fails" on worker 29306: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 Expression: false + + + + + + + + + + \ No newline at end of file diff --git a/test/references/retry_tests_report_1worker.xml b/test/references/retry_tests_report_1worker.xml new file mode 100644 index 00000000..f8f85326 --- /dev/null +++ b/test/references/retry_tests_report_1worker.xml @@ -0,0 +1,294 @@ + + + + + + + + + + + + + Captured logs for test setup "StatefulSetup" (dependency of "Pass on second run") at test/testfiles/_retry_tests.jl:1 on worker 52041 +These are the test setup logs. +No Captured Logs for test item "Pass on second run" at test/testfiles/_retry_tests.jl:10 on worker 52041 +Error in testset "Pass on second run" on worker 52041: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:15 + Expression: x + + + + + + + + + + + + + + + + + + + + + + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 52041 +These are the test setup logs. +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 52041 +These are the logs for run number: 1 +Error in testset "Error, then Fail, then Pass" on worker 52041: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 + Test threw exception + Expression: not_defined_err + UndefVarError: not_defined_err not defined + Stacktrace: + [1] macro expansion + @ /Applications/Julia-1.8.app/Contents/Resources/julia/share/julia/stdlib/v1.8/Test/src/Test.jl:464 [inlined] + [2] top-level scope + @ /repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 + + + + + + + + + + + + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 52041 +These are the test setup logs. +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 52041 +These are the logs for run number: 2 +Error in testset "Error, then Fail, then Pass" on worker 52041: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:26 + Expression: 2 + 2 == 5 + Evaluated: 4 == 5 + + + + + + + + + + + + + + + + + + + + + + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 +These are the test setup logs. +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 52041 +Error in testset "Has retries=4 and always fails" on worker 52041: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 + Expression: false + + + + + + + + + + + + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 +These are the test setup logs. +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 52041 +Error in testset "Has retries=4 and always fails" on worker 52041: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 + Expression: false + + + + + + + + + + + + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 +These are the test setup logs. +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 52041 +Error in testset "Has retries=4 and always fails" on worker 52041: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 + Expression: false + + + + + + + + + + + + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 +These are the test setup logs. +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 52041 +Error in testset "Has retries=4 and always fails" on worker 52041: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 + Expression: false + + + + + + + + + + + + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 +These are the test setup logs. +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 52041 +Error in testset "Has retries=4 and always fails" on worker 52041: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 + Expression: false + + + + + + + + + + + + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 +These are the test setup logs. +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 52041 +Error in testset "Has retries=1 and always fails" on worker 52041: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 + Expression: false + + + + + + + + + + + + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 +These are the test setup logs. +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 52041 +Error in testset "Has retries=1 and always fails" on worker 52041: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 + Expression: false + + + + + + + + + + + + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 +These are the test setup logs. +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 52041 +Error in testset "Has retries=1 and always fails" on worker 52041: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 + Expression: false + + + + + + + + + + + + Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:48 on worker 52284 + +signal (15): Terminated: 15 +in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:50 +__psynch_cvwait at /usr/lib/system/libsystem_kernel.dylib (unknown line) +unknown function (ip: 0x0) +kevent at /usr/lib/system/libsystem_kernel.dylib (unknown line) +unknown function (ip: 0x0) +Allocations: 5228358 (Pool: 5226025; Big: 2333); GC: 30 +Error in testset "Timeout always" on worker 52284: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:48 + Got exception outside of a @test + Timed out after 5s evaluating test item "Timeout always" (run=1) + + + + + + + + + + + + Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:48 on worker 52284 + +signal (15): Terminated: 15 +in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:50 +__psynch_cvwait at /usr/lib/system/libsystem_kernel.dylib (unknown line) +unknown function (ip: 0x0) +kevent at /usr/lib/system/libsystem_kernel.dylib (unknown line) +unknown function (ip: 0x0) +Allocations: 3068485 (Pool: 3067373; Big: 1112); GC: 3 +Error in testset "Timeout always" on worker 52284: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:48 + Got exception outside of a @test + Timed out after 5s evaluating test item "Timeout always" (run=2) + + + + + + + + + + + + Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:48 on worker 52284 + +signal (15): Terminated: 15 +in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:50 +__psynch_cvwait at /usr/lib/system/libsystem_kernel.dylib (unknown line) +unknown function (ip: 0x0) +kevent at /usr/lib/system/libsystem_kernel.dylib (unknown line) +unknown function (ip: 0x0) +Allocations: 3068503 (Pool: 3067391; Big: 1112); GC: 3 +Error in testset "Timeout always" on worker 52284: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:48 + Got exception outside of a @test + Timed out after 5s evaluating test item "Timeout always" (run=3) + + + + \ No newline at end of file diff --git a/test/testfiles/_retry_tests.jl b/test/testfiles/_retry_tests.jl index 0b329907..3004146e 100644 --- a/test/testfiles/_retry_tests.jl +++ b/test/testfiles/_retry_tests.jl @@ -42,11 +42,11 @@ end end -# For this to timeout, must be run with `testitem_timeout < 60` +# For this to timeout, must be run with `testitem_timeout < 30` # Cannot use `StatefulSetup` for this as it will be a new worker # every retry, so the `setup` will always have been re-evaluated anew. @testitem "Timeout always" retries=1 begin write(tempname() * "_num_runs_5", "1") - sleep(60.0) + sleep(30.0) @test true end From cf723176a62e512347ec36f34e0c46f1733b7f86 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jun 2023 14:45:06 +0100 Subject: [PATCH 05/15] Test case where we timeout then pass on retry --- test/integrationtests.jl | 21 +- test/junit_xml.jl | 8 + .../references/retry_tests_report_0worker.xml | 164 +++++++------ .../references/retry_tests_report_1worker.xml | 222 ++++++++++-------- test/testfiles/_retry_tests.jl | 27 ++- 5 files changed, 262 insertions(+), 180 deletions(-) diff --git a/test/integrationtests.jl b/test/integrationtests.jl index 13c0ae30..87b47fc5 100644 --- a/test/integrationtests.jl +++ b/test/integrationtests.jl @@ -610,6 +610,7 @@ end @testset "test retrying failing testitem" begin file = joinpath(TEST_FILES_DIR, "_retry_tests.jl") # must run with `testitem_timeout < 30` for test to timeout as expected. + # and must run with `nworkers > 0` for retries to be supported. results = encased_testset(()->runtests(file; nworkers=1, retries=2, testitem_timeout=3)) # Test we _ran_ each test-item the expected number of times read_count(x) = parse(Int, read(x, String)) @@ -623,19 +624,27 @@ end # Doesn't pass ever, so need all retries. This testitem set `retries=1` which is less # than the `retries=2` that `runtests` set, so we should retry 2 times. @test read_count(joinpath(tempdir(), "num_runs_4")) == 3 + # This directory must match what's set in `_retry_tests` + tmpdir = joinpath("/tmp", "JL_RETESTITEMS_TEST_TMPDIR") # Times out always, so should retry as many times as allowed. # Since it will be a new worker for each retry, we write one file for each. - @test count(contains("num_runs_5"), readdir(tempdir())) == 3 + @test count(contains("num_runs_5"), readdir(tmpdir)) == 3 + # Times out on first run, then passes on second attempt. + @test count(contains("num_runs_6"), readdir(tmpdir)) == 2 # Test we _report_ the expected number of test-items - @test n_tests(results) == 5 - # The first two testitems pass on retry. - @test n_passed(results) == 2 + @test n_tests(results) == 6 + # Testitems 1, 2, and 6 pass on retry. + @test n_passed(results) == 3 # Clear out any files created by this testset - foreach(readdir(tempdir(); join=true)) do tmp - contains(tmp, "num_runs") && rm(tmp) + for dir in (tempdir(), tmpdir) + foreach(readdir(dir; join=true)) do tmp + # `force` in case it gets cleaned up between `readder` and `rm` + contains(tmp, "num_runs") && rm(tmp; force=true) + end end + rm(tmpdir; force=true) end @testset "testitem timeout" begin diff --git a/test/junit_xml.jl b/test/junit_xml.jl index 97f3a2b2..69d98832 100644 --- a/test/junit_xml.jl +++ b/test/junit_xml.jl @@ -102,6 +102,14 @@ end try # Ignore the fact that the `_retry_tests.jl` testset has failures/errors. run(`$(Base.julia_cmd()) --project -e 'using ReTestItems; runtests("testfiles/_retry_tests.jl"; testitem_timeout=5, report=true, retries=2)'`) catch + finally + # running `_retry_tests` creates files which we need to clean up before + # we run `_retry_tests` again, because these files store state that is + # used to control the behaviour of the tests. + foreach(readdir(joinpath("/tmp", "JL_RETESTITEMS_TEST_TMPDIR"); join=true)) do tmp + rm(tmp; force=true) + end + end end report = only(filter(endswith("xml"), readdir(dir, join=true))) # timeouts are not supported when running without workers, so diff --git a/test/references/retry_tests_report_0worker.xml b/test/references/retry_tests_report_0worker.xml index f773b229..071e568c 100644 --- a/test/references/retry_tests_report_0worker.xml +++ b/test/references/retry_tests_report_0worker.xml @@ -1,47 +1,47 @@ - - - + + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Pass on second run") at test/testfiles/_retry_tests.jl:1 on worker 29306 + Captured logs for test setup "StatefulSetup" (dependency of "Pass on second run") at test/testfiles/_retry_tests.jl:1 on worker 63296 These are the test setup logs. -No Captured Logs for test item "Pass on second run" at test/testfiles/_retry_tests.jl:10 on worker 29306 -Error in testset "Pass on second run" on worker 29306: +No Captured Logs for test item "Pass on second run" at test/testfiles/_retry_tests.jl:10 on worker 63296 +Error in testset "Pass on second run" on worker 63296: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:15 Expression: x - + - + - + - - + + - + - Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 29306 + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 63296 These are the test setup logs. -Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 29306 +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 63296 These are the logs for run number: 1 -Error in testset "Error, then Fail, then Pass" on worker 29306: +Error in testset "Error, then Fail, then Pass" on worker 63296: Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 Test threw exception Expression: not_defined_err @@ -53,179 +53,189 @@ Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tes @ /repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 29306 + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 63296 These are the test setup logs. -Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 29306 +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 63296 These are the logs for run number: 2 -Error in testset "Error, then Fail, then Pass" on worker 29306: +Error in testset "Error, then Fail, then Pass" on worker 63296: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:26 Expression: 2 + 2 == 5 Evaluated: 4 == 5 - + - + - + - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 29306 -Error in testset "Has retries=4 and always fails" on worker 29306: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63296 +Error in testset "Has retries=4 and always fails" on worker 63296: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 29306 -Error in testset "Has retries=4 and always fails" on worker 29306: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63296 +Error in testset "Has retries=4 and always fails" on worker 63296: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 29306 -Error in testset "Has retries=4 and always fails" on worker 29306: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63296 +Error in testset "Has retries=4 and always fails" on worker 63296: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 29306 -Error in testset "Has retries=4 and always fails" on worker 29306: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63296 +Error in testset "Has retries=4 and always fails" on worker 63296: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 29306 -Error in testset "Has retries=4 and always fails" on worker 29306: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63296 +Error in testset "Has retries=4 and always fails" on worker 63296: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 29306 -Error in testset "Has retries=1 and always fails" on worker 29306: +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 63296 +Error in testset "Has retries=1 and always fails" on worker 63296: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 Expression: false - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 29306 -Error in testset "Has retries=1 and always fails" on worker 29306: +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 63296 +Error in testset "Has retries=1 and always fails" on worker 63296: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 Expression: false - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 29306 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 29306 -Error in testset "Has retries=1 and always fails" on worker 29306: +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 63296 +Error in testset "Has retries=1 and always fails" on worker 63296: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 Expression: false - + + + + + + + + + + + - - + + - + - + diff --git a/test/references/retry_tests_report_1worker.xml b/test/references/retry_tests_report_1worker.xml index f8f85326..b267daf0 100644 --- a/test/references/retry_tests_report_1worker.xml +++ b/test/references/retry_tests_report_1worker.xml @@ -1,47 +1,47 @@ - - - + + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Pass on second run") at test/testfiles/_retry_tests.jl:1 on worker 52041 + Captured logs for test setup "StatefulSetup" (dependency of "Pass on second run") at test/testfiles/_retry_tests.jl:1 on worker 51057 These are the test setup logs. -No Captured Logs for test item "Pass on second run" at test/testfiles/_retry_tests.jl:10 on worker 52041 -Error in testset "Pass on second run" on worker 52041: +No Captured Logs for test item "Pass on second run" at test/testfiles/_retry_tests.jl:10 on worker 51057 +Error in testset "Pass on second run" on worker 51057: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:15 Expression: x - + - + - + - + - - - - + + + + - + - Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 52041 + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 51057 These are the test setup logs. -Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 52041 +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 51057 These are the logs for run number: 1 -Error in testset "Error, then Fail, then Pass" on worker 52041: +Error in testset "Error, then Fail, then Pass" on worker 51057: Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 Test threw exception Expression: not_defined_err @@ -53,172 +53,172 @@ Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tes @ /repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 52041 + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 51057 These are the test setup logs. -Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 52041 +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 51057 These are the logs for run number: 2 -Error in testset "Error, then Fail, then Pass" on worker 52041: +Error in testset "Error, then Fail, then Pass" on worker 51057: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:26 Expression: 2 + 2 == 5 Evaluated: 4 == 5 - + - + - + - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 52041 -Error in testset "Has retries=4 and always fails" on worker 52041: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 51057 +Error in testset "Has retries=4 and always fails" on worker 51057: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 52041 -Error in testset "Has retries=4 and always fails" on worker 52041: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 51057 +Error in testset "Has retries=4 and always fails" on worker 51057: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 52041 -Error in testset "Has retries=4 and always fails" on worker 52041: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 51057 +Error in testset "Has retries=4 and always fails" on worker 51057: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 52041 -Error in testset "Has retries=4 and always fails" on worker 52041: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 51057 +Error in testset "Has retries=4 and always fails" on worker 51057: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 52041 -Error in testset "Has retries=4 and always fails" on worker 52041: +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 51057 +Error in testset "Has retries=4 and always fails" on worker 51057: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 Expression: false - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 52041 -Error in testset "Has retries=1 and always fails" on worker 52041: +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 51057 +Error in testset "Has retries=1 and always fails" on worker 51057: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 Expression: false - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 52041 -Error in testset "Has retries=1 and always fails" on worker 52041: +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 51057 +Error in testset "Has retries=1 and always fails" on worker 51057: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 Expression: false - + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 52041 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 52041 -Error in testset "Has retries=1 and always fails" on worker 52041: +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 51057 +Error in testset "Has retries=1 and always fails" on worker 51057: Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 Expression: false - + @@ -227,22 +227,22 @@ Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl: - Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:48 on worker 52284 + Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:52 on worker 51271 signal (15): Terminated: 15 -in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:50 +in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:58 __psynch_cvwait at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) kevent at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) -Allocations: 5228358 (Pool: 5226025; Big: 2333); GC: 30 -Error in testset "Timeout always" on worker 52284: -Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:48 +Allocations: 5228932 (Pool: 5226603; Big: 2329); GC: 30 +Error in testset "Timeout always" on worker 51271: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:52 Got exception outside of a @test Timed out after 5s evaluating test item "Timeout always" (run=1) - + @@ -251,22 +251,22 @@ Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tes - Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:48 on worker 52284 + Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:52 on worker 51271 signal (15): Terminated: 15 -in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:50 +in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:58 __psynch_cvwait at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) kevent at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) -Allocations: 3068485 (Pool: 3067373; Big: 1112); GC: 3 -Error in testset "Timeout always" on worker 52284: -Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:48 +Allocations: 3068788 (Pool: 3067677; Big: 1111); GC: 3 +Error in testset "Timeout always" on worker 51271: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:52 Got exception outside of a @test Timed out after 5s evaluating test item "Timeout always" (run=2) - + @@ -275,20 +275,54 @@ Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tes - Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:48 on worker 52284 + Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:52 on worker 51271 signal (15): Terminated: 15 -in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:50 +in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:58 __psynch_cvwait at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) kevent at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) -Allocations: 3068503 (Pool: 3067391; Big: 1112); GC: 3 -Error in testset "Timeout always" on worker 52284: -Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:48 +Allocations: 3068784 (Pool: 3067673; Big: 1111); GC: 3 +Error in testset "Timeout always" on worker 51271: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:52 Got exception outside of a @test Timed out after 5s evaluating test item "Timeout always" (run=3) + + + + + + + + + + Captured Logs for test item "Timeout first, pass after" at test/testfiles/_retry_tests.jl:62 on worker 51478 + +signal (15): Terminated: 15 +in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:69 +__psynch_cvwait at /usr/lib/system/libsystem_kernel.dylib (unknown line) +unknown function (ip: 0x0) +kevent at /usr/lib/system/libsystem_kernel.dylib (unknown line) +unknown function (ip: 0x0) +Allocations: 3061069 (Pool: 3059958; Big: 1111); GC: 3 +Error in testset "Timeout first, pass after" on worker 51478: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:62 + Got exception outside of a @test + Timed out after 5s evaluating test item "Timeout first, pass after" (run=1) + + + + + + + + + + + + \ No newline at end of file diff --git a/test/testfiles/_retry_tests.jl b/test/testfiles/_retry_tests.jl index 3004146e..41493554 100644 --- a/test/testfiles/_retry_tests.jl +++ b/test/testfiles/_retry_tests.jl @@ -42,11 +42,32 @@ end end -# For this to timeout, must be run with `testitem_timeout < 30` -# Cannot use `StatefulSetup` for this as it will be a new worker +# For these tests to timeout, must be run with `testitem_timeout < 30` +# Cannot use `StatefulSetup` for testing timeouts, as it will be a new worker # every retry, so the `setup` will always have been re-evaluated anew. +# Instead we write a new file for each run. We don't use `tempdir()` in case files written +# there get cleaned up as soon as the worker dies. +# We need to write a new file each time for our counting to be correct, so if the assertion +# fails we need to switch to a more robust ways of creating unique filenames. @testitem "Timeout always" retries=1 begin - write(tempname() * "_num_runs_5", "1") + using Random + tmpdir = mkpath(joinpath("/tmp", "JL_RETESTITEMS_TEST_TMPDIR")) + filename = joinpath(tmpdir, "num_runs_5_" * randstring()) + @assert !isfile(filename) + write(filename, "1") sleep(30.0) @test true end + +@testitem "Timeout first, pass after" retries=1 begin + using Random + tmpdir = mkpath(joinpath("/tmp", "JL_RETESTITEMS_TEST_TMPDIR")) + filename = joinpath(tmpdir, "num_runs_6_" * randstring()) + @assert !isfile(filename) + is_first_run = !any(contains("num_runs_6"), readdir(tmpdir)) + write(filename, "1") + if is_first_run + sleep(30.0) + end + @test true +end From b88fe42402f84351cd4962ca4844f72424dc1ef1 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jun 2023 15:05:14 +0100 Subject: [PATCH 06/15] fixup! Test case where we timeout then pass on retry --- test/junit_xml.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/junit_xml.jl b/test/junit_xml.jl index 69d98832..d64c3d09 100644 --- a/test/junit_xml.jl +++ b/test/junit_xml.jl @@ -110,7 +110,6 @@ end rm(tmp; force=true) end end - end report = only(filter(endswith("xml"), readdir(dir, join=true))) # timeouts are not supported when running without workers, so # with 0 workers the "retry tests" that are expected to timeout will pass. From b4da580bccd5e3b5935d304b7ef1ef130d72750f Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jun 2023 15:08:32 +0100 Subject: [PATCH 07/15] fixup! Test case where we timeout then pass on retry --- Project.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index a1e1c979..5dbef05d 100644 --- a/Project.toml +++ b/Project.toml @@ -23,8 +23,9 @@ DeepDiffs = "ab62b9b5-e342-54a8-a765-a90f495de1a6" IOCapture = "b5f81e59-6552-4d32-b1f0-c071b021bf89" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" XMLDict = "228000da-037f-5747-90a9-8195ccbf91a5" [targets] -test = ["AutoHashEquals", "DeepDiffs", "IOCapture", "Logging", "Pkg", "Test", "XMLDict"] +test = ["AutoHashEquals", "DeepDiffs", "IOCapture", "Logging", "Pkg", "Random", "Test", "XMLDict"] From d939a562a8e7eef2e3ba48872d14e930334c8d72 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jun 2023 15:15:11 +0100 Subject: [PATCH 08/15] Reduce time of test so worker=0 case isn't so slow --- test/integrationtests.jl | 2 +- test/testfiles/_retry_tests.jl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integrationtests.jl b/test/integrationtests.jl index 87b47fc5..e5a2d3fb 100644 --- a/test/integrationtests.jl +++ b/test/integrationtests.jl @@ -609,7 +609,7 @@ end @testset "test retrying failing testitem" begin file = joinpath(TEST_FILES_DIR, "_retry_tests.jl") - # must run with `testitem_timeout < 30` for test to timeout as expected. + # must run with `testitem_timeout < 20` for test to timeout as expected. # and must run with `nworkers > 0` for retries to be supported. results = encased_testset(()->runtests(file; nworkers=1, retries=2, testitem_timeout=3)) # Test we _ran_ each test-item the expected number of times diff --git a/test/testfiles/_retry_tests.jl b/test/testfiles/_retry_tests.jl index 41493554..8a0a4a04 100644 --- a/test/testfiles/_retry_tests.jl +++ b/test/testfiles/_retry_tests.jl @@ -42,7 +42,7 @@ end end -# For these tests to timeout, must be run with `testitem_timeout < 30` +# For these tests to timeout, must be run with `testitem_timeout < 20` # Cannot use `StatefulSetup` for testing timeouts, as it will be a new worker # every retry, so the `setup` will always have been re-evaluated anew. # Instead we write a new file for each run. We don't use `tempdir()` in case files written @@ -55,7 +55,7 @@ end filename = joinpath(tmpdir, "num_runs_5_" * randstring()) @assert !isfile(filename) write(filename, "1") - sleep(30.0) + sleep(20.0) @test true end @@ -67,7 +67,7 @@ end is_first_run = !any(contains("num_runs_6"), readdir(tmpdir)) write(filename, "1") if is_first_run - sleep(30.0) + sleep(20.0) end @test true end From 1052948f189cd646ea7b70c713d609cf466b7e10 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jun 2023 16:36:51 +0100 Subject: [PATCH 09/15] Handle test failure case differently to worker timeout or crash --- src/ReTestItems.jl | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/ReTestItems.jl b/src/ReTestItems.jl index 9072aa7c..ca8ca33b 100644 --- a/src/ReTestItems.jl +++ b/src/ReTestItems.jl @@ -354,12 +354,6 @@ function start_worker(proj_name, nworker_threads, worker_init_expr, ntestitems) return w end - -# This is only used to signal that we need to retry. We don't use Test.TestSetException as -# that requires information that we are not going to use anyway. -struct TestSetFailure <: Exception end -throw_if_failed(ts) = ts.anynonpass ? throw(TestSetFailure()) : nothing - function record_timeout!(testitem, run_number::Int, timeout_limit::Real) timeout_s = round(Int, timeout_limit) time_str = if timeout_s < 60 @@ -441,21 +435,24 @@ function start_and_manage_worker( # Run GC to free memory on the worker before next testitem. @debugv 2 "Running GC on $worker" remote_fetch(worker, :(GC.gc(true); GC.gc(false))) - # If the result isn't a pass, we throw to go to the outer try-catch - throw_if_failed(ts) - testitem = next_testitem(testitems, testitem.id[]) - run_number = 1 + if ts.anynonpass && (run_number != (1 + retry_limit)) + run_number += 1 + @info "Retrying $(repr(testitem.name)) on $worker. Run=$run_number." + else + testitem = next_testitem(testitems, testitem.id[]) + run_number = 1 + end finally close(timer) end catch e @debugv 2 "Error" exception=e + println(DEFAULT_STDOUT[]) + _print_captured_logs(DEFAULT_STDOUT[], testitem, run_number) # Handle the exception if e isa TimeoutException @debugv 1 "Test item $(repr(testitem.name)) timed out. Terminating worker $worker" # Explicitly show captured logs or say there weren't any before terminating worker - println(DEFAULT_STDOUT[]) - _print_captured_logs(DEFAULT_STDOUT[], testitem, run_number) terminate!(worker) wait(worker) @warn "$worker timed out evaluating test item $(repr(testitem.name)) afer $timeout seconds. \ @@ -465,8 +462,6 @@ function start_and_manage_worker( @warn "$worker died evaluating test item $(repr(testitem.name)). \ Recording test error." record_worker_terminated!(testitem, run_number) - elseif e isa TestSetFailure - # We already printed the error and recorded the testset. else # We don't expect any other kind of error, so rethrow, which will propagate # back up to the main coordinator task and throw to the user @@ -478,14 +473,10 @@ function start_and_manage_worker( run_number = 1 else run_number += 1 - if e isa TestSetFailure - @info "Retrying $(repr(testitem.name)) on $worker. Run=$run_number." - else - @info "Retrying $(repr(testitem.name)) on a new worker. Run=$run_number." - end + @info "Retrying $(repr(testitem.name)) on a new worker. Run=$run_number." end - # Replace worker if necessary - if (e isa TimeoutException || e isa WorkerTerminatedException) && (testitem !== nothing) + # Replace worker unless there are no more testitems to run + if testitem !== nothing worker = start_worker(proj_name, nworker_threads, worker_init_expr, ntestitems) end # Now loop back around to reschedule the testitem From 870b3631a458f00676be4184039c90ec985c68b8 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jun 2023 16:45:17 +0100 Subject: [PATCH 10/15] Bump version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 5dbef05d..b3988621 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "ReTestItems" uuid = "817f1d60-ba6b-4fd5-9520-3cf149f6a823" -version = "1.8.1" +version = "1.9.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" From 1babb10ab2d8d310dd69db2e900520029814fce9 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jun 2023 16:47:19 +0100 Subject: [PATCH 11/15] fixup! Handle test failure case differently to worker timeout or crash --- src/ReTestItems.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ReTestItems.jl b/src/ReTestItems.jl index ca8ca33b..f86d3f3c 100644 --- a/src/ReTestItems.jl +++ b/src/ReTestItems.jl @@ -452,7 +452,6 @@ function start_and_manage_worker( # Handle the exception if e isa TimeoutException @debugv 1 "Test item $(repr(testitem.name)) timed out. Terminating worker $worker" - # Explicitly show captured logs or say there weren't any before terminating worker terminate!(worker) wait(worker) @warn "$worker timed out evaluating test item $(repr(testitem.name)) afer $timeout seconds. \ From f66565de07a51fe5fe52d19a29e70f35d1eaf3d4 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jun 2023 16:48:48 +0100 Subject: [PATCH 12/15] fixup! fixup! Handle test failure case differently to worker timeout or crash --- src/ReTestItems.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ReTestItems.jl b/src/ReTestItems.jl index f86d3f3c..efbdbb59 100644 --- a/src/ReTestItems.jl +++ b/src/ReTestItems.jl @@ -474,7 +474,7 @@ function start_and_manage_worker( run_number += 1 @info "Retrying $(repr(testitem.name)) on a new worker. Run=$run_number." end - # Replace worker unless there are no more testitems to run + # The worker was terminated, so replace it unless there are no more testitems to run if testitem !== nothing worker = start_worker(proj_name, nworker_threads, worker_init_expr, ntestitems) end From 71c483fe442928eb7223741a63993aeaa9a0b734 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jun 2023 16:49:20 +0100 Subject: [PATCH 13/15] Update test/integrationtests.jl - typo --- test/integrationtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integrationtests.jl b/test/integrationtests.jl index e5a2d3fb..37c4dbc1 100644 --- a/test/integrationtests.jl +++ b/test/integrationtests.jl @@ -640,7 +640,7 @@ end # Clear out any files created by this testset for dir in (tempdir(), tmpdir) foreach(readdir(dir; join=true)) do tmp - # `force` in case it gets cleaned up between `readder` and `rm` + # `force` in case it gets cleaned up between `readdir` and `rm` contains(tmp, "num_runs") && rm(tmp; force=true) end end From 0e6253f9152d9140ab281dcabfd99ff8a88ee957 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jun 2023 17:18:42 +0100 Subject: [PATCH 14/15] Don't use `tempdir()` --- test/integrationtests.jl | 20 +++++++++----------- test/testfiles/_retry_tests.jl | 10 ++++++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/integrationtests.jl b/test/integrationtests.jl index 37c4dbc1..0da345e3 100644 --- a/test/integrationtests.jl +++ b/test/integrationtests.jl @@ -609,23 +609,23 @@ end @testset "test retrying failing testitem" begin file = joinpath(TEST_FILES_DIR, "_retry_tests.jl") + # This directory must match what's set in `_retry_tests` + tmpdir = joinpath("/tmp", "JL_RETESTITEMS_TEST_TMPDIR") # must run with `testitem_timeout < 20` for test to timeout as expected. # and must run with `nworkers > 0` for retries to be supported. results = encased_testset(()->runtests(file; nworkers=1, retries=2, testitem_timeout=3)) # Test we _ran_ each test-item the expected number of times read_count(x) = parse(Int, read(x, String)) # Passes on second attempt, so only need to retry once. - @test read_count(joinpath(tempdir(), "num_runs_1")) == 2 + @test read_count(joinpath(tmpdir, "num_runs_1")) == 2 # Doesn't pass til third attempt, so needs both retries. - @test read_count(joinpath(tempdir(), "num_runs_2")) == 3 + @test read_count(joinpath(tmpdir, "num_runs_2")) == 3 # Doesn't pass ever, so need all retries. This testitem set `retries=4` which is greater # than the `retries=2` that `runtests` set, so we should retry 4 times. - @test read_count(joinpath(tempdir(), "num_runs_3")) == 5 + @test read_count(joinpath(tmpdir, "num_runs_3")) == 5 # Doesn't pass ever, so need all retries. This testitem set `retries=1` which is less # than the `retries=2` that `runtests` set, so we should retry 2 times. - @test read_count(joinpath(tempdir(), "num_runs_4")) == 3 - # This directory must match what's set in `_retry_tests` - tmpdir = joinpath("/tmp", "JL_RETESTITEMS_TEST_TMPDIR") + @test read_count(joinpath(tmpdir, "num_runs_4")) == 3 # Times out always, so should retry as many times as allowed. # Since it will be a new worker for each retry, we write one file for each. @test count(contains("num_runs_5"), readdir(tmpdir)) == 3 @@ -638,11 +638,9 @@ end @test n_passed(results) == 3 # Clear out any files created by this testset - for dir in (tempdir(), tmpdir) - foreach(readdir(dir; join=true)) do tmp - # `force` in case it gets cleaned up between `readdir` and `rm` - contains(tmp, "num_runs") && rm(tmp; force=true) - end + foreach(readdir(tmpdir; join=true)) do tmp + # `force` in case it gets cleaned up between `readdir` and `rm` + contains(tmp, "num_runs") && rm(tmp; force=true) end rm(tmpdir; force=true) end diff --git a/test/testfiles/_retry_tests.jl b/test/testfiles/_retry_tests.jl index 8a0a4a04..4a3d21a7 100644 --- a/test/testfiles/_retry_tests.jl +++ b/test/testfiles/_retry_tests.jl @@ -1,5 +1,7 @@ @testsetup module StatefulSetup export NUM_RUNS_1, NUM_RUNS_2, NUM_RUNS_3, NUM_RUNS_4 + export tmpdir + const tmpdir = mkpath(joinpath("/tmp", "JL_RETESTITEMS_TEST_TMPDIR")) const NUM_RUNS_1 = Ref{Int}(0) const NUM_RUNS_2 = Ref{Int}(0) const NUM_RUNS_3 = Ref{Int}(0) @@ -9,7 +11,7 @@ end @testitem "Pass on second run" setup=[StatefulSetup] begin NUM_RUNS_1[] += 1 - write(joinpath(tempdir(), "num_runs_1"), string(NUM_RUNS_1[])) + write(joinpath(tmpdir, "num_runs_1"), string(NUM_RUNS_1[])) x = NUM_RUNS_1[] == 2 @test x @@ -17,7 +19,7 @@ end @testitem "Error, then Fail, then Pass" setup=[StatefulSetup] begin NUM_RUNS_2[] += 1 - write(joinpath(tempdir(), "num_runs_2"), string(NUM_RUNS_2[])) + write(joinpath(tmpdir, "num_runs_2"), string(NUM_RUNS_2[])) println("These are the logs for run number: ", NUM_RUNS_2[]) if NUM_RUNS_2[] == 1 @@ -31,13 +33,13 @@ end @testitem "Has retries=4 and always fails" setup=[StatefulSetup] retries=4 begin NUM_RUNS_3[] += 1 - write(joinpath(tempdir(), "num_runs_3"), string(NUM_RUNS_3[])) + write(joinpath(tmpdir, "num_runs_3"), string(NUM_RUNS_3[])) @test false end @testitem "Has retries=1 and always fails" setup=[StatefulSetup] retries=1 begin NUM_RUNS_4[] += 1 - write(joinpath(tempdir(), "num_runs_4"), string(NUM_RUNS_4[])) + write(joinpath(tmpdir, "num_runs_4"), string(NUM_RUNS_4[])) @test false end From e8d40d231313452d97c84102b6b165a2a5097656 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 21 Jun 2023 17:45:41 +0100 Subject: [PATCH 15/15] fixup! Don't use `tempdir()` --- .../references/retry_tests_report_0worker.xml | 246 +++++++-------- .../references/retry_tests_report_1worker.xml | 282 +++++++++--------- 2 files changed, 264 insertions(+), 264 deletions(-) diff --git a/test/references/retry_tests_report_0worker.xml b/test/references/retry_tests_report_0worker.xml index 071e568c..c8d72bea 100644 --- a/test/references/retry_tests_report_0worker.xml +++ b/test/references/retry_tests_report_0worker.xml @@ -1,48 +1,48 @@ - - - + + + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Pass on second run") at test/testfiles/_retry_tests.jl:1 on worker 63296 + Captured logs for test setup "StatefulSetup" (dependency of "Pass on second run") at test/testfiles/_retry_tests.jl:1 on worker 78453 These are the test setup logs. -No Captured Logs for test item "Pass on second run" at test/testfiles/_retry_tests.jl:10 on worker 63296 -Error in testset "Pass on second run" on worker 63296: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:15 +No Captured Logs for test item "Pass on second run" at test/testfiles/_retry_tests.jl:12 on worker 78453 +Error in testset "Pass on second run" on worker 78453: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:17 Expression: x - + - - + + - + - + - + - - - - + + + + - + - Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 63296 + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 78453 These are the test setup logs. -Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 63296 +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:20 on worker 78453 These are the logs for run number: 1 -Error in testset "Error, then Fail, then Pass" on worker 63296: -Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 +Error in testset "Error, then Fail, then Pass" on worker 78453: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:26 Test threw exception Expression: not_defined_err UndefVarError: not_defined_err not defined @@ -50,192 +50,192 @@ Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tes [1] macro expansion @ /Applications/Julia-1.8.app/Contents/Resources/julia/share/julia/stdlib/v1.8/Test/src/Test.jl:464 [inlined] [2] top-level scope - @ /repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 + @ /repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:26 - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 63296 + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 78453 These are the test setup logs. -Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 63296 +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:20 on worker 78453 These are the logs for run number: 2 -Error in testset "Error, then Fail, then Pass" on worker 63296: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:26 +Error in testset "Error, then Fail, then Pass" on worker 78453: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:28 Expression: 2 + 2 == 5 Evaluated: 4 == 5 - + - - + + - + - + - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 78453 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63296 -Error in testset "Has retries=4 and always fails" on worker 63296: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:34 on worker 78453 +Error in testset "Has retries=4 and always fails" on worker 78453: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:37 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 78453 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63296 -Error in testset "Has retries=4 and always fails" on worker 63296: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:34 on worker 78453 +Error in testset "Has retries=4 and always fails" on worker 78453: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:37 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 78453 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63296 -Error in testset "Has retries=4 and always fails" on worker 63296: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:34 on worker 78453 +Error in testset "Has retries=4 and always fails" on worker 78453: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:37 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 78453 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63296 -Error in testset "Has retries=4 and always fails" on worker 63296: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:34 on worker 78453 +Error in testset "Has retries=4 and always fails" on worker 78453: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:37 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 78453 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 63296 -Error in testset "Has retries=4 and always fails" on worker 63296: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:34 on worker 78453 +Error in testset "Has retries=4 and always fails" on worker 78453: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:37 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 78453 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 63296 -Error in testset "Has retries=1 and always fails" on worker 63296: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:40 on worker 78453 +Error in testset "Has retries=1 and always fails" on worker 78453: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:43 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 78453 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 63296 -Error in testset "Has retries=1 and always fails" on worker 63296: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:40 on worker 78453 +Error in testset "Has retries=1 and always fails" on worker 78453: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:43 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 63296 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 78453 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 63296 -Error in testset "Has retries=1 and always fails" on worker 63296: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:40 on worker 78453 +Error in testset "Has retries=1 and always fails" on worker 78453: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:43 Expression: false - + - - + + - + - + - + - - + + - + - + diff --git a/test/references/retry_tests_report_1worker.xml b/test/references/retry_tests_report_1worker.xml index b267daf0..d00e829b 100644 --- a/test/references/retry_tests_report_1worker.xml +++ b/test/references/retry_tests_report_1worker.xml @@ -1,48 +1,48 @@ - - - + + + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Pass on second run") at test/testfiles/_retry_tests.jl:1 on worker 51057 + Captured logs for test setup "StatefulSetup" (dependency of "Pass on second run") at test/testfiles/_retry_tests.jl:1 on worker 81105 These are the test setup logs. -No Captured Logs for test item "Pass on second run" at test/testfiles/_retry_tests.jl:10 on worker 51057 -Error in testset "Pass on second run" on worker 51057: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:15 +No Captured Logs for test item "Pass on second run" at test/testfiles/_retry_tests.jl:12 on worker 81105 +Error in testset "Pass on second run" on worker 81105: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:17 Expression: x - + - - + + - + - + - + - - - - + + + + - + - Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 51057 + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 81105 These are the test setup logs. -Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 51057 +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:20 on worker 81105 These are the logs for run number: 1 -Error in testset "Error, then Fail, then Pass" on worker 51057: -Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 +Error in testset "Error, then Fail, then Pass" on worker 81105: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:26 Test threw exception Expression: not_defined_err UndefVarError: not_defined_err not defined @@ -50,175 +50,175 @@ Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tes [1] macro expansion @ /Applications/Julia-1.8.app/Contents/Resources/julia/share/julia/stdlib/v1.8/Test/src/Test.jl:464 [inlined] [2] top-level scope - @ /repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:24 + @ /repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:26 - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 51057 + Captured logs for test setup "StatefulSetup" (dependency of "Error, then Fail, then Pass") at test/testfiles/_retry_tests.jl:1 on worker 81105 These are the test setup logs. -Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:18 on worker 51057 +Captured Logs for test item "Error, then Fail, then Pass" at test/testfiles/_retry_tests.jl:20 on worker 81105 These are the logs for run number: 2 -Error in testset "Error, then Fail, then Pass" on worker 51057: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:26 +Error in testset "Error, then Fail, then Pass" on worker 81105: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:28 Expression: 2 + 2 == 5 Evaluated: 4 == 5 - + - - + + - + - + - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 81105 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 51057 -Error in testset "Has retries=4 and always fails" on worker 51057: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:34 on worker 81105 +Error in testset "Has retries=4 and always fails" on worker 81105: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:37 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 81105 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 51057 -Error in testset "Has retries=4 and always fails" on worker 51057: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:34 on worker 81105 +Error in testset "Has retries=4 and always fails" on worker 81105: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:37 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 81105 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 51057 -Error in testset "Has retries=4 and always fails" on worker 51057: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:34 on worker 81105 +Error in testset "Has retries=4 and always fails" on worker 81105: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:37 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 81105 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 51057 -Error in testset "Has retries=4 and always fails" on worker 51057: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:34 on worker 81105 +Error in testset "Has retries=4 and always fails" on worker 81105: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:37 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=4 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 81105 These are the test setup logs. -No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:32 on worker 51057 -Error in testset "Has retries=4 and always fails" on worker 51057: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:35 +No Captured Logs for test item "Has retries=4 and always fails" at test/testfiles/_retry_tests.jl:34 on worker 81105 +Error in testset "Has retries=4 and always fails" on worker 81105: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:37 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 81105 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 51057 -Error in testset "Has retries=1 and always fails" on worker 51057: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:40 on worker 81105 +Error in testset "Has retries=1 and always fails" on worker 81105: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:43 Expression: false - + - - + + - + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 81105 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 51057 -Error in testset "Has retries=1 and always fails" on worker 51057: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:40 on worker 81105 +Error in testset "Has retries=1 and always fails" on worker 81105: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:43 Expression: false - + - - + + - + - Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 51057 + Captured logs for test setup "StatefulSetup" (dependency of "Has retries=1 and always fails") at test/testfiles/_retry_tests.jl:1 on worker 81105 These are the test setup logs. -No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:38 on worker 51057 -Error in testset "Has retries=1 and always fails" on worker 51057: -Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:41 +No Captured Logs for test item "Has retries=1 and always fails" at test/testfiles/_retry_tests.jl:40 on worker 81105 +Error in testset "Has retries=1 and always fails" on worker 81105: +Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:43 Expression: false - + @@ -227,22 +227,22 @@ Test Failed at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl: - Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:52 on worker 51271 + Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:54 on worker 81822 signal (15): Terminated: 15 -in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:58 +in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:60 __psynch_cvwait at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) kevent at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) -Allocations: 5228932 (Pool: 5226603; Big: 2329); GC: 30 -Error in testset "Timeout always" on worker 51271: -Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:52 +Allocations: 5231398 (Pool: 5229065; Big: 2333); GC: 30 +Error in testset "Timeout always" on worker 81822: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:54 Got exception outside of a @test Timed out after 5s evaluating test item "Timeout always" (run=1) - + @@ -251,22 +251,22 @@ Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tes - Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:52 on worker 51271 + Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:54 on worker 81822 signal (15): Terminated: 15 -in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:58 +in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:60 __psynch_cvwait at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) kevent at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) -Allocations: 3068788 (Pool: 3067677; Big: 1111); GC: 3 -Error in testset "Timeout always" on worker 51271: -Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:52 +Allocations: 3068722 (Pool: 3067612; Big: 1110); GC: 3 +Error in testset "Timeout always" on worker 81822: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:54 Got exception outside of a @test Timed out after 5s evaluating test item "Timeout always" (run=2) - + @@ -275,22 +275,22 @@ Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tes - Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:52 on worker 51271 + Captured Logs for test item "Timeout always" at test/testfiles/_retry_tests.jl:54 on worker 81822 signal (15): Terminated: 15 -in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:58 +in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:60 __psynch_cvwait at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) kevent at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) -Allocations: 3068784 (Pool: 3067673; Big: 1111); GC: 3 -Error in testset "Timeout always" on worker 51271: -Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:52 +Allocations: 3068695 (Pool: 3067583; Big: 1112); GC: 3 +Error in testset "Timeout always" on worker 81822: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:54 Got exception outside of a @test Timed out after 5s evaluating test item "Timeout always" (run=3) - + @@ -299,29 +299,29 @@ Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tes - Captured Logs for test item "Timeout first, pass after" at test/testfiles/_retry_tests.jl:62 on worker 51478 + Captured Logs for test item "Timeout first, pass after" at test/testfiles/_retry_tests.jl:64 on worker 81835 signal (15): Terminated: 15 -in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:69 +in expression starting at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:71 __psynch_cvwait at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) kevent at /usr/lib/system/libsystem_kernel.dylib (unknown line) unknown function (ip: 0x0) -Allocations: 3061069 (Pool: 3059958; Big: 1111); GC: 3 -Error in testset "Timeout first, pass after" on worker 51478: -Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:62 +Allocations: 3061005 (Pool: 3059893; Big: 1112); GC: 3 +Error in testset "Timeout first, pass after" on worker 81835: +Error During Test at /Users/nickr/repos/ReTestItems.jl/test/testfiles/_retry_tests.jl:64 Got exception outside of a @test Timed out after 5s evaluating test item "Timeout first, pass after" (run=1) - + - - + + - + - +