Skip to content

Commit

Permalink
rerun-fails: Only rerun failed subtests
Browse files Browse the repository at this point in the history 
If a root test case has failed subtests, only rerun those that have
failed. A new hidden flag is added to restore the old behaviour of
re-running the entire root test case if any subtest fails.

Because of limitations of the 'go test -run' flag (github.com/golang/go/issues/39904)
this required running each test in a separate 'go test' process.
  • Loading branch information
@dnephin
dnephin committed Jun 29, 2020
1 parent 8979f92 commit 6c2d74f
Show file tree
Hide file tree
Showing 9 changed files with 253 additions and 143 deletions.
13 changes: 13 additions & 0 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,9 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
"space separated list of package to test")
flags.StringVar(&opts.rerunFailsReportFile, "rerun-fails-report", "",
"write a report to the file, of the tests that were rerun")
flags.BoolVar(&opts.rerunFailsOnlyRootCases, "rerun-fails-only-root-testcases", false,
"rerun only root testcaes, instead of only subtests")
flags.Lookup("rerun-fails-only-root-testcases").Hidden = true

flags.BoolVar(&opts.debug, "debug", false, "enabled debug logging")
flags.BoolVar(&opts.version, "version", false, "show version and exit")
Expand Down Expand Up @@ -161,6 +164,7 @@ type options struct {
rerunFailsMaxAttempts int
rerunFailsMaxInitialFailures int
rerunFailsReportFile string
rerunFailsOnlyRootCases bool
packages []string
version bool

Expand All @@ -178,6 +182,15 @@ func (o options) Validate() error {
return nil
}

func (o options) rerunFailsFilter() testCaseFilter {
if o.rerunFailsOnlyRootCases {
return func(tcs []testjson.TestCase) []testjson.TestCase {
return tcs
}
}
return testjson.FilterFailedUnique
}

func setupLogging(opts *options) {
if opts.debug {
log.SetLevel(log.DebugLevel)
Expand Down
60 changes: 26 additions & 34 deletions rerunfails.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,18 @@ func (o rerunOpts) Args() []string {
return result
}

func newRerunOptsFromTestCase(tc testjson.TestCase) rerunOpts {
return rerunOpts{
runFlag: goTestRunFlagForTestCase(tc.Test),
pkg: tc.Package,
}
}

type testCaseFilter func([]testjson.TestCase) []testjson.TestCase

func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanConfig) error {
failed := len(scanConfig.Execution.Failed())
tcFilter := opts.rerunFailsFilter()
failed := len(tcFilter(scanConfig.Execution.Failed()))
if failed > opts.rerunFailsMaxInitialFailures {
return fmt.Errorf(
"number of test failures (%d) exceeds maximum (%d) set by --rerun-fails-max-failures",
Expand All @@ -42,12 +52,8 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
opts.stdout.Write([]byte("\n")) // nolint: errcheck

nextRec := newFailureRecorder(scanConfig.Handler)
for pkg, testCases := range rec.pkgFailures {
rerun := rerunOpts{
runFlag: goTestRunFlagFromTestCases(testCases),
pkg: pkg,
}
goTestProc, err := startGoTest(ctx, goTestCmdArgs(opts, rerun))
for _, tc := range tcFilter(rec.failures) {
goTestProc, err := startGoTest(ctx, goTestCmdArgs(opts, newRerunOptsFromTestCase(tc)))
if err != nil {
return errors.Wrapf(err, "failed to run %s", strings.Join(goTestProc.cmd.Args, " "))
}
Expand All @@ -66,8 +72,8 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
if err := hasErrors(lastErr, scanConfig.Execution); err != nil {
return err
}
rec = nextRec
}
rec = nextRec
}
return lastErr
}
Expand All @@ -86,50 +92,36 @@ func hasErrors(err error, exec *testjson.Execution) error {

type failureRecorder struct {
testjson.EventHandler
pkgFailures map[string][]string
failures []testjson.TestCase
}

func newFailureRecorder(handler testjson.EventHandler) *failureRecorder {
return &failureRecorder{
EventHandler: handler,
pkgFailures: make(map[string][]string),
}
return &failureRecorder{EventHandler: handler}
}

func newFailureRecorderFromExecution(exec *testjson.Execution) *failureRecorder {
r := newFailureRecorder(nil)
for _, tc := range exec.Failed() {
r.pkgFailures[tc.Package] = append(r.pkgFailures[tc.Package], tc.Test)
}
return r
return &failureRecorder{failures: exec.Failed()}
}

func (r *failureRecorder) Event(event testjson.TestEvent, execution *testjson.Execution) error {
if !event.PackageEvent() && event.Action == testjson.ActionFail {
r.pkgFailures[event.Package] = append(r.pkgFailures[event.Package], event.Test)
pkg := execution.Package(event.Package)
tc := pkg.LastFailedByName(event.Test)
r.failures = append(r.failures, tc)
}
return r.EventHandler.Event(event, execution)
}

func (r *failureRecorder) count() int {
total := 0
for _, tcs := range r.pkgFailures {
total += len(tcs)
}
return total
return len(r.failures)
}

func goTestRunFlagFromTestCases(tcs []string) string {
buf := new(strings.Builder)
buf.WriteString("-run=^(")
for i, tc := range tcs {
if i != 0 {
buf.WriteString("|")
}
buf.WriteString(tc)
func goTestRunFlagForTestCase(name string) string {
root, sub := testjson.SplitTestName(name)
if sub == "" {
return "-run=^" + name + "$"
}
buf.WriteString(")$")
return buf.String()
return "-run=^" + root + "$/^" + sub + "$"
}

func writeRerunFailsReport(opts *options, exec *testjson.Execution) error {
Expand Down
27 changes: 27 additions & 0 deletions rerunfails_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,30 @@ func TestWriteRerunFailsReport(t *testing.T) {
assert.NilError(t, err)
golden.Assert(t, string(raw), t.Name()+"-expected")
}

func TestGoTestRunFlagFromTestCases(t *testing.T) {
type testCase struct {
input string
expected string
}
fn := func(t *testing.T, tc testCase) {
actual := goTestRunFlagForTestCase(tc.input)
assert.Equal(t, actual, tc.expected)
}

var testCases = map[string]testCase{
"root test case": {
input: "TestOne",
expected: "-run=^TestOne$",
},
"sub test case": {
input: "TestOne/SubtestA",
expected: "-run=^TestOne$/^SubtestA$",
},
}
for name := range testCases {
t.Run(name, func(t *testing.T) {
fn(t, testCases[name])
})
}
}
56 changes: 34 additions & 22 deletions testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,37 +9,44 @@ SEED: 0
--- FAIL: TestFailsSometimes
flaky_test.go:58: not this time
FAIL testdata/e2e/flaky.TestFailsSometimes
PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes
=== RUN TestFailsOften/subtest_may_fail
--- FAIL: TestFailsOften/subtest_may_fail
flaky_test.go:68: not this time
FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail
=== RUN TestFailsOften
SEED: 0
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften
PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
FAIL testdata/e2e/flaky

DONE 6 tests, 3 failures
DONE 8 tests, 4 failures

PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
=== RUN TestFailsSometimes
SEED: 1
--- FAIL: TestFailsSometimes
flaky_test.go:58: not this time
FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
PASS testdata/e2e/flaky
PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1)
PASS testdata/e2e/flaky
=== RUN TestFailsOften/subtest_may_fail
--- FAIL: TestFailsOften/subtest_may_fail
flaky_test.go:68: not this time
FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
=== RUN TestFailsOften
SEED: 1
SEED: 3
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
FAIL testdata/e2e/flaky

DONE 2 runs, 9 tests, 5 failures
DONE 2 runs, 12 tests, 6 failures

PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
=== RUN TestFailsOften/subtest_may_fail
--- FAIL: TestFailsOften/subtest_may_fail
flaky_test.go:68: not this time
FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
=== RUN TestFailsOften
SEED: 2
SEED: 4
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
FAIL testdata/e2e/flaky

Expand All @@ -52,21 +59,26 @@ SEED: 0
SEED: 0
flaky_test.go:58: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail
--- FAIL: TestFailsOften/subtest_may_fail
flaky_test.go:68: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
SEED: 0
flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
SEED: 1
flaky_test.go:58: not this time
=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
--- FAIL: TestFailsOften/subtest_may_fail
flaky_test.go:68: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
SEED: 1
flaky_test.go:65: not this time
SEED: 3

=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
--- FAIL: TestFailsOften/subtest_may_fail
flaky_test.go:68: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
SEED: 2
flaky_test.go:65: not this time
SEED: 4


DONE 3 runs, 11 tests, 6 failures
DONE 3 runs, 14 tests, 8 failures
56 changes: 34 additions & 22 deletions testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,36 +9,43 @@ SEED: 0
TestFailsSometimes: flaky_test.go:58: not this time
--- FAIL: TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes
PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes
=== RUN TestFailsOften/subtest_may_fail
TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time
--- FAIL: TestFailsOften/subtest_may_fail
FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail
=== RUN TestFailsOften
SEED: 0
TestFailsOften: flaky_test.go:65: not this time
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften
PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
FAIL testdata/e2e/flaky

DONE 6 tests, 3 failures
DONE 8 tests, 4 failures

PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
=== RUN TestFailsSometimes
SEED: 1
TestFailsSometimes: flaky_test.go:58: not this time
--- FAIL: TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
PASS testdata/e2e/flaky
PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1)
PASS testdata/e2e/flaky
=== RUN TestFailsOften/subtest_may_fail
TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time
--- FAIL: TestFailsOften/subtest_may_fail
FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
=== RUN TestFailsOften
SEED: 1
TestFailsOften: flaky_test.go:65: not this time
SEED: 3
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
FAIL testdata/e2e/flaky

DONE 2 runs, 9 tests, 5 failures
DONE 2 runs, 12 tests, 6 failures

PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
=== RUN TestFailsOften/subtest_may_fail
TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time
--- FAIL: TestFailsOften/subtest_may_fail
FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
=== RUN TestFailsOften
SEED: 2
TestFailsOften: flaky_test.go:65: not this time
SEED: 4
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
FAIL testdata/e2e/flaky
Expand All @@ -52,21 +59,26 @@ SEED: 0
SEED: 0
TestFailsSometimes: flaky_test.go:58: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail
TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time
--- FAIL: TestFailsOften/subtest_may_fail

=== FAIL: testdata/e2e/flaky TestFailsOften
SEED: 0
TestFailsOften: flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
SEED: 1
TestFailsSometimes: flaky_test.go:58: not this time
=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time
--- FAIL: TestFailsOften/subtest_may_fail

=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
SEED: 1
TestFailsOften: flaky_test.go:65: not this time
SEED: 3

=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
TestFailsOften/subtest_may_fail: flaky_test.go:68: not this time
--- FAIL: TestFailsOften/subtest_may_fail

=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
SEED: 2
TestFailsOften: flaky_test.go:65: not this time
SEED: 4


DONE 3 runs, 11 tests, 6 failures
DONE 3 runs, 14 tests, 8 failures

0 comments on commit 6c2d74f

Please sign in to comment.