From 6c2d74ff499194c132aa316779fd64c8bad1e9d9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sun, 28 Jun 2020 16:02:05 -0400 Subject: [PATCH] rerun-fails: Only rerun failed subtests If a root test case has failed subtests, only rerun those that have failed. A new hidden flag is added to restore the old behaviour of re-running the entire root test case if any subtest fails. Because of limitations of the 'go test -run' flag (github.com/golang/go/issues/39904) this required running each test in a separate 'go test' process. --- main.go | 13 ++++ rerunfails.go | 60 +++++++-------- rerunfails_test.go | 27 +++++++ .../reruns_continues_to_fail | 56 ++++++++------ .../reruns_continues_to_fail-go1.14 | 56 ++++++++------ .../TestE2E_RerunFails/reruns_until_success | 75 ++++++++++++------- .../reruns_until_success-go1.14 | 75 ++++++++++++------- testdata/e2e/flaky/flaky_test.go | 14 ++-- testjson/execution.go | 20 ++++- 9 files changed, 253 insertions(+), 143 deletions(-) diff --git a/main.go b/main.go index 43fe7abb..af54f01d 100644 --- a/main.go +++ b/main.go @@ -109,6 +109,9 @@ func setupFlags(name string) (*pflag.FlagSet, *options) { "space separated list of package to test") flags.StringVar(&opts.rerunFailsReportFile, "rerun-fails-report", "", "write a report to the file, of the tests that were rerun") + flags.BoolVar(&opts.rerunFailsOnlyRootCases, "rerun-fails-only-root-testcases", false, + "rerun only root testcaes, instead of only subtests") + flags.Lookup("rerun-fails-only-root-testcases").Hidden = true flags.BoolVar(&opts.debug, "debug", false, "enabled debug logging") flags.BoolVar(&opts.version, "version", false, "show version and exit") @@ -161,6 +164,7 @@ type options struct { rerunFailsMaxAttempts int rerunFailsMaxInitialFailures int rerunFailsReportFile string + rerunFailsOnlyRootCases bool packages []string version bool @@ -178,6 +182,15 @@ func (o options) Validate() error { return nil } +func (o options) rerunFailsFilter() testCaseFilter { + if o.rerunFailsOnlyRootCases { + return func(tcs []testjson.TestCase) []testjson.TestCase { + return tcs + } + } + return testjson.FilterFailedUnique +} + func setupLogging(opts *options) { if opts.debug { log.SetLevel(log.DebugLevel) diff --git a/rerunfails.go b/rerunfails.go index c9d512cd..654d21fa 100644 --- a/rerunfails.go +++ b/rerunfails.go @@ -27,8 +27,18 @@ func (o rerunOpts) Args() []string { return result } +func newRerunOptsFromTestCase(tc testjson.TestCase) rerunOpts { + return rerunOpts{ + runFlag: goTestRunFlagForTestCase(tc.Test), + pkg: tc.Package, + } +} + +type testCaseFilter func([]testjson.TestCase) []testjson.TestCase + func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanConfig) error { - failed := len(scanConfig.Execution.Failed()) + tcFilter := opts.rerunFailsFilter() + failed := len(tcFilter(scanConfig.Execution.Failed())) if failed > opts.rerunFailsMaxInitialFailures { return fmt.Errorf( "number of test failures (%d) exceeds maximum (%d) set by --rerun-fails-max-failures", @@ -42,12 +52,8 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon opts.stdout.Write([]byte("\n")) // nolint: errcheck nextRec := newFailureRecorder(scanConfig.Handler) - for pkg, testCases := range rec.pkgFailures { - rerun := rerunOpts{ - runFlag: goTestRunFlagFromTestCases(testCases), - pkg: pkg, - } - goTestProc, err := startGoTest(ctx, goTestCmdArgs(opts, rerun)) + for _, tc := range tcFilter(rec.failures) { + goTestProc, err := startGoTest(ctx, goTestCmdArgs(opts, newRerunOptsFromTestCase(tc))) if err != nil { return errors.Wrapf(err, "failed to run %s", strings.Join(goTestProc.cmd.Args, " ")) } @@ -66,8 +72,8 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon if err := hasErrors(lastErr, scanConfig.Execution); err != nil { return err } - rec = nextRec } + rec = nextRec } return lastErr } @@ -86,50 +92,36 @@ func hasErrors(err error, exec *testjson.Execution) error { type failureRecorder struct { testjson.EventHandler - pkgFailures map[string][]string + failures []testjson.TestCase } func newFailureRecorder(handler testjson.EventHandler) *failureRecorder { - return &failureRecorder{ - EventHandler: handler, - pkgFailures: make(map[string][]string), - } + return &failureRecorder{EventHandler: handler} } func newFailureRecorderFromExecution(exec *testjson.Execution) *failureRecorder { - r := newFailureRecorder(nil) - for _, tc := range exec.Failed() { - r.pkgFailures[tc.Package] = append(r.pkgFailures[tc.Package], tc.Test) - } - return r + return &failureRecorder{failures: exec.Failed()} } func (r *failureRecorder) Event(event testjson.TestEvent, execution *testjson.Execution) error { if !event.PackageEvent() && event.Action == testjson.ActionFail { - r.pkgFailures[event.Package] = append(r.pkgFailures[event.Package], event.Test) + pkg := execution.Package(event.Package) + tc := pkg.LastFailedByName(event.Test) + r.failures = append(r.failures, tc) } return r.EventHandler.Event(event, execution) } func (r *failureRecorder) count() int { - total := 0 - for _, tcs := range r.pkgFailures { - total += len(tcs) - } - return total + return len(r.failures) } -func goTestRunFlagFromTestCases(tcs []string) string { - buf := new(strings.Builder) - buf.WriteString("-run=^(") - for i, tc := range tcs { - if i != 0 { - buf.WriteString("|") - } - buf.WriteString(tc) +func goTestRunFlagForTestCase(name string) string { + root, sub := testjson.SplitTestName(name) + if sub == "" { + return "-run=^" + name + "$" } - buf.WriteString(")$") - return buf.String() + return "-run=^" + root + "$/^" + sub + "$" } func writeRerunFailsReport(opts *options, exec *testjson.Execution) error { diff --git a/rerunfails_test.go b/rerunfails_test.go index 69a2ef5a..8fc6f55c 100644 --- a/rerunfails_test.go +++ b/rerunfails_test.go @@ -32,3 +32,30 @@ func TestWriteRerunFailsReport(t *testing.T) { assert.NilError(t, err) golden.Assert(t, string(raw), t.Name()+"-expected") } + +func TestGoTestRunFlagFromTestCases(t *testing.T) { + type testCase struct { + input string + expected string + } + fn := func(t *testing.T, tc testCase) { + actual := goTestRunFlagForTestCase(tc.input) + assert.Equal(t, actual, tc.expected) + } + + var testCases = map[string]testCase{ + "root test case": { + input: "TestOne", + expected: "-run=^TestOne$", + }, + "sub test case": { + input: "TestOne/SubtestA", + expected: "-run=^TestOne$/^SubtestA$", + }, + } + for name := range testCases { + t.Run(name, func(t *testing.T) { + fn(t, testCases[name]) + }) + } +} diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail index ed4d4a8a..b968136b 100644 --- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail +++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail @@ -9,37 +9,44 @@ SEED: 0 --- FAIL: TestFailsSometimes flaky_test.go:58: not this time FAIL testdata/e2e/flaky.TestFailsSometimes +PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes +=== RUN TestFailsOften/subtest_may_fail + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail === RUN TestFailsOften SEED: 0 --- FAIL: TestFailsOften - flaky_test.go:65: not this time FAIL testdata/e2e/flaky.TestFailsOften PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch FAIL testdata/e2e/flaky -DONE 6 tests, 3 failures +DONE 8 tests, 4 failures PASS testdata/e2e/flaky.TestFailsRarely (re-run 1) -=== RUN TestFailsSometimes -SEED: 1 ---- FAIL: TestFailsSometimes - flaky_test.go:58: not this time -FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1) +PASS testdata/e2e/flaky +PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1) +PASS testdata/e2e/flaky +=== RUN TestFailsOften/subtest_may_fail + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1) === RUN TestFailsOften -SEED: 1 +SEED: 3 --- FAIL: TestFailsOften - flaky_test.go:65: not this time FAIL testdata/e2e/flaky.TestFailsOften (re-run 1) FAIL testdata/e2e/flaky -DONE 2 runs, 9 tests, 5 failures +DONE 2 runs, 12 tests, 6 failures -PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2) +=== RUN TestFailsOften/subtest_may_fail + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2) === RUN TestFailsOften -SEED: 2 +SEED: 4 --- FAIL: TestFailsOften - flaky_test.go:65: not this time FAIL testdata/e2e/flaky.TestFailsOften (re-run 2) FAIL testdata/e2e/flaky @@ -52,21 +59,26 @@ SEED: 0 SEED: 0 flaky_test.go:58: not this time +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time + === FAIL: testdata/e2e/flaky TestFailsOften SEED: 0 - flaky_test.go:65: not this time -=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1) -SEED: 1 - flaky_test.go:58: not this time +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1) + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time === FAIL: testdata/e2e/flaky TestFailsOften (re-run 1) -SEED: 1 - flaky_test.go:65: not this time +SEED: 3 + +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2) + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time === FAIL: testdata/e2e/flaky TestFailsOften (re-run 2) -SEED: 2 - flaky_test.go:65: not this time +SEED: 4 -DONE 3 runs, 11 tests, 6 failures +DONE 3 runs, 14 tests, 8 failures diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14 b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14 index 4334fc6a..54048eca 100644 --- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14 +++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14 @@ -9,36 +9,43 @@ SEED: 0 TestFailsSometimes: flaky_test.go:58: not this time --- FAIL: TestFailsSometimes FAIL testdata/e2e/flaky.TestFailsSometimes +PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes +=== RUN TestFailsOften/subtest_may_fail + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail === RUN TestFailsOften SEED: 0 - TestFailsOften: flaky_test.go:65: not this time --- FAIL: TestFailsOften FAIL testdata/e2e/flaky.TestFailsOften PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch FAIL testdata/e2e/flaky -DONE 6 tests, 3 failures +DONE 8 tests, 4 failures PASS testdata/e2e/flaky.TestFailsRarely (re-run 1) -=== RUN TestFailsSometimes -SEED: 1 - TestFailsSometimes: flaky_test.go:58: not this time ---- FAIL: TestFailsSometimes -FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1) +PASS testdata/e2e/flaky +PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1) +PASS testdata/e2e/flaky +=== RUN TestFailsOften/subtest_may_fail + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1) === RUN TestFailsOften -SEED: 1 - TestFailsOften: flaky_test.go:65: not this time +SEED: 3 --- FAIL: TestFailsOften FAIL testdata/e2e/flaky.TestFailsOften (re-run 1) FAIL testdata/e2e/flaky -DONE 2 runs, 9 tests, 5 failures +DONE 2 runs, 12 tests, 6 failures -PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2) +=== RUN TestFailsOften/subtest_may_fail + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2) === RUN TestFailsOften -SEED: 2 - TestFailsOften: flaky_test.go:65: not this time +SEED: 4 --- FAIL: TestFailsOften FAIL testdata/e2e/flaky.TestFailsOften (re-run 2) FAIL testdata/e2e/flaky @@ -52,21 +59,26 @@ SEED: 0 SEED: 0 TestFailsSometimes: flaky_test.go:58: not this time +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail + === FAIL: testdata/e2e/flaky TestFailsOften SEED: 0 - TestFailsOften: flaky_test.go:65: not this time -=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1) -SEED: 1 - TestFailsSometimes: flaky_test.go:58: not this time +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1) + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail === FAIL: testdata/e2e/flaky TestFailsOften (re-run 1) -SEED: 1 - TestFailsOften: flaky_test.go:65: not this time +SEED: 3 + +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2) + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail === FAIL: testdata/e2e/flaky TestFailsOften (re-run 2) -SEED: 2 - TestFailsOften: flaky_test.go:65: not this time +SEED: 4 -DONE 3 runs, 11 tests, 6 failures +DONE 3 runs, 14 tests, 8 failures diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success index ebe537e8..dd6e90c2 100644 --- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success +++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success @@ -9,51 +9,62 @@ SEED: 0 --- FAIL: TestFailsSometimes flaky_test.go:58: not this time FAIL testdata/e2e/flaky.TestFailsSometimes +PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes +=== RUN TestFailsOften/subtest_may_fail + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail === RUN TestFailsOften SEED: 0 --- FAIL: TestFailsOften - flaky_test.go:65: not this time FAIL testdata/e2e/flaky.TestFailsOften PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch FAIL testdata/e2e/flaky -DONE 6 tests, 3 failures +DONE 8 tests, 4 failures PASS testdata/e2e/flaky.TestFailsRarely (re-run 1) -=== RUN TestFailsSometimes -SEED: 1 ---- FAIL: TestFailsSometimes - flaky_test.go:58: not this time -FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1) +PASS testdata/e2e/flaky +PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1) +PASS testdata/e2e/flaky +=== RUN TestFailsOften/subtest_may_fail + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1) === RUN TestFailsOften -SEED: 1 +SEED: 3 --- FAIL: TestFailsOften - flaky_test.go:65: not this time FAIL testdata/e2e/flaky.TestFailsOften (re-run 1) FAIL testdata/e2e/flaky -DONE 2 runs, 9 tests, 5 failures +DONE 2 runs, 12 tests, 6 failures -PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2) +=== RUN TestFailsOften/subtest_may_fail + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2) === RUN TestFailsOften -SEED: 2 +SEED: 4 --- FAIL: TestFailsOften - flaky_test.go:65: not this time FAIL testdata/e2e/flaky.TestFailsOften (re-run 2) FAIL testdata/e2e/flaky -DONE 3 runs, 11 tests, 6 failures +DONE 3 runs, 14 tests, 8 failures +=== RUN TestFailsOften/subtest_may_fail + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 3) === RUN TestFailsOften -SEED: 3 +SEED: 5 --- FAIL: TestFailsOften - flaky_test.go:65: not this time FAIL testdata/e2e/flaky.TestFailsOften (re-run 3) FAIL testdata/e2e/flaky -DONE 4 runs, 12 tests, 7 failures +DONE 4 runs, 16 tests, 10 failures +PASS testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 4) PASS testdata/e2e/flaky.TestFailsOften (re-run 4) PASS testdata/e2e/flaky @@ -66,25 +77,33 @@ SEED: 0 SEED: 0 flaky_test.go:58: not this time +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time + === FAIL: testdata/e2e/flaky TestFailsOften SEED: 0 - flaky_test.go:65: not this time -=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1) -SEED: 1 - flaky_test.go:58: not this time +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1) + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time === FAIL: testdata/e2e/flaky TestFailsOften (re-run 1) -SEED: 1 - flaky_test.go:65: not this time +SEED: 3 + +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2) + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time === FAIL: testdata/e2e/flaky TestFailsOften (re-run 2) -SEED: 2 - flaky_test.go:65: not this time +SEED: 4 + +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 3) + --- FAIL: TestFailsOften/subtest_may_fail + flaky_test.go:68: not this time === FAIL: testdata/e2e/flaky TestFailsOften (re-run 3) -SEED: 3 - flaky_test.go:65: not this time +SEED: 5 -DONE 5 runs, 13 tests, 7 failures +DONE 5 runs, 18 tests, 10 failures diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14 b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14 index 63e7612c..c252857f 100644 --- a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14 +++ b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14 @@ -9,51 +9,62 @@ SEED: 0 TestFailsSometimes: flaky_test.go:58: not this time --- FAIL: TestFailsSometimes FAIL testdata/e2e/flaky.TestFailsSometimes +PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes +=== RUN TestFailsOften/subtest_may_fail + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail === RUN TestFailsOften SEED: 0 - TestFailsOften: flaky_test.go:65: not this time --- FAIL: TestFailsOften FAIL testdata/e2e/flaky.TestFailsOften PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch FAIL testdata/e2e/flaky -DONE 6 tests, 3 failures +DONE 8 tests, 4 failures PASS testdata/e2e/flaky.TestFailsRarely (re-run 1) -=== RUN TestFailsSometimes -SEED: 1 - TestFailsSometimes: flaky_test.go:58: not this time ---- FAIL: TestFailsSometimes -FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1) +PASS testdata/e2e/flaky +PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1) +PASS testdata/e2e/flaky +=== RUN TestFailsOften/subtest_may_fail + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1) === RUN TestFailsOften -SEED: 1 - TestFailsOften: flaky_test.go:65: not this time +SEED: 3 --- FAIL: TestFailsOften FAIL testdata/e2e/flaky.TestFailsOften (re-run 1) FAIL testdata/e2e/flaky -DONE 2 runs, 9 tests, 5 failures +DONE 2 runs, 12 tests, 6 failures -PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2) +=== RUN TestFailsOften/subtest_may_fail + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2) === RUN TestFailsOften -SEED: 2 - TestFailsOften: flaky_test.go:65: not this time +SEED: 4 --- FAIL: TestFailsOften FAIL testdata/e2e/flaky.TestFailsOften (re-run 2) FAIL testdata/e2e/flaky -DONE 3 runs, 11 tests, 6 failures +DONE 3 runs, 14 tests, 8 failures +=== RUN TestFailsOften/subtest_may_fail + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail +FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 3) === RUN TestFailsOften -SEED: 3 - TestFailsOften: flaky_test.go:65: not this time +SEED: 5 --- FAIL: TestFailsOften FAIL testdata/e2e/flaky.TestFailsOften (re-run 3) FAIL testdata/e2e/flaky -DONE 4 runs, 12 tests, 7 failures +DONE 4 runs, 16 tests, 10 failures +PASS testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 4) PASS testdata/e2e/flaky.TestFailsOften (re-run 4) PASS testdata/e2e/flaky @@ -66,25 +77,33 @@ SEED: 0 SEED: 0 TestFailsSometimes: flaky_test.go:58: not this time +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail + === FAIL: testdata/e2e/flaky TestFailsOften SEED: 0 - TestFailsOften: flaky_test.go:65: not this time -=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1) -SEED: 1 - TestFailsSometimes: flaky_test.go:58: not this time +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1) + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail === FAIL: testdata/e2e/flaky TestFailsOften (re-run 1) -SEED: 1 - TestFailsOften: flaky_test.go:65: not this time +SEED: 3 + +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2) + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail === FAIL: testdata/e2e/flaky TestFailsOften (re-run 2) -SEED: 2 - TestFailsOften: flaky_test.go:65: not this time +SEED: 4 + +=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 3) + TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time + --- FAIL: TestFailsOften/subtest_may_fail === FAIL: testdata/e2e/flaky TestFailsOften (re-run 3) -SEED: 3 - TestFailsOften: flaky_test.go:65: not this time +SEED: 5 -DONE 5 runs, 13 tests, 7 failures +DONE 5 runs, 18 tests, 10 failures diff --git a/testdata/e2e/flaky/flaky_test.go b/testdata/e2e/flaky/flaky_test.go index c1486249..229fc6cb 100644 --- a/testdata/e2e/flaky/flaky_test.go +++ b/testdata/e2e/flaky/flaky_test.go @@ -47,23 +47,27 @@ func TestAlwaysPasses(t *testing.T) { func TestFailsRarely(t *testing.T) { setup(t) - if seed%10 != 1 { + if seed%20 != 1 { t.Fatal("not this time") } } func TestFailsSometimes(t *testing.T) { setup(t) - if seed%10 != 2 { + if seed%4 != 2 { t.Fatal("not this time") } } func TestFailsOften(t *testing.T) { setup(t) - if seed%10 != 4 { - t.Fatal("not this time") - } + + t.Run("subtest always passes", func(t *testing.T) {}) + t.Run("subtest may fail", func(t *testing.T) { + if seed%20 != 6 { + t.Fatal("not this time") + } + }) } func TestFailsOftenDoesNotPrefixMatch(t *testing.T) {} diff --git a/testjson/execution.go b/testjson/execution.go index 71f0a43f..785e9928 100644 --- a/testjson/execution.go +++ b/testjson/execution.go @@ -143,7 +143,7 @@ func (p *Package) Output(id int) string { // then all output for every subtest under the root test is returned. // See https://github.com/golang/go/issues/29755. func (p *Package) OutputLines(tc TestCase) []string { - _, sub := splitTestName(tc.Test) + _, sub := SplitTestName(tc.Test) lines := p.output[tc.ID] // If this is a subtest, or a root test case with subtest failures the @@ -166,8 +166,8 @@ func (p *Package) addOutput(id int, output string) { p.output[id] = append(p.output[id], output) } -// splitTestName into root test name and any subtest names. -func splitTestName(name string) (root, sub string) { +// SplitTestName into root test name and any subtest names. +func SplitTestName(name string) (root, sub string) { parts := strings.SplitN(name, "/", 2) if len(parts) < 2 { return name, "" @@ -287,7 +287,7 @@ func (e *Execution) addPackageEvent(pkg *Package, event TestEvent) { func (p *Package) addTestEvent(event TestEvent) { tc := p.running[event.Test] - root, subTest := splitTestName(event.Test) + root, subTest := SplitTestName(event.Test) switch event.Action { case ActionRun: @@ -405,6 +405,18 @@ func (e *Execution) Failed() []TestCase { return failed } +// FilterFailedUnique filters a slice of failed TestCases by removing root test +// case that have failed subtests. +func FilterFailedUnique(tcs []TestCase) []TestCase { + var result []TestCase + for _, tc := range tcs { + if !tc.hasSubTestFailed { + result = append(result, tc) + } + } + return result +} + func sortedKeys(pkgs map[string]*Package) []string { keys := make([]string, 0, len(pkgs)) for key := range pkgs {