Merge pull request #135 from dnephin/rerun-fails-only-subtests

rerun-fails: Only rerun failed subtests

do

@@ -7,13 +7,24 @@ binary() {
     go build -o dist/gotestsum .
 }
 
+binary-static() {
+    echo "building static binary: dist/gotestsum"
+    CGO_ENABLED=0 binary
+}
+
 update-golden() {
-    _update-golden
-    GOLANG_VERSION=1.13-alpine ./do shell bash -c 'go build; PATH=$PATH:. ./do _update-golden'
+    #_update-golden
+    if ldd ./dist/gotestsum > /dev/null 2>&1; then
+        binary-static
+    fi
+    GOLANG_VERSION=1.13-alpine ./do shell ./do _update-golden
+    GOLANG_VERSION=1.14.6-alpine ./do shell ./do _update-golden
 }
 
 _update-golden() {
-    gotestsum -- . ./testjson ./internal/junitxml ./cmd/tool/slowest -test.update-golden
+    PATH="$PWD/dist:$PATH" gotestsum -- \
+        . ./testjson ./internal/junitxml ./cmd/tool/slowest \
+        -test.update-golden
 }
 
 lint() {

main.go

@@ -109,6 +109,9 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
 		"space separated list of package to test")
 	flags.StringVar(&opts.rerunFailsReportFile, "rerun-fails-report", "",
 		"write a report to the file, of the tests that were rerun")
+	flags.BoolVar(&opts.rerunFailsOnlyRootCases, "rerun-fails-only-root-testcases", false,
+		"rerun only root testcaes, instead of only subtests")
+	flags.Lookup("rerun-fails-only-root-testcases").Hidden = true
 
 	flags.BoolVar(&opts.debug, "debug", false, "enabled debug logging")
 	flags.BoolVar(&opts.version, "version", false, "show version and exit")
@@ -161,6 +164,7 @@ type options struct {
 	rerunFailsMaxAttempts        int
 	rerunFailsMaxInitialFailures int
 	rerunFailsReportFile         string
+	rerunFailsOnlyRootCases      bool
 	packages                     []string
 	version                      bool
 

main_e2e_test.go

@@ -49,7 +49,7 @@ func TestE2E_RerunFails(t *testing.T) {
 			text.OpRemoveTestElapsedTime,
 			filepath.ToSlash, // for windows
 		)
-		golden.Assert(t, out, expectedFilename(t.Name()))
+		golden.Assert(t, out, "e2e/expected/"+expectedFilename(t.Name()))
 	}
 	var testCases = []testCase{
 		{
@@ -105,10 +105,26 @@ func osEnviron() map[string]string {
 }
 
 func expectedFilename(name string) string {
-	// go1.14 changed how it prints messages from tests. It may be changed back
-	// in go1.15, so special case this version for now.
-	if strings.HasPrefix(runtime.Version(), "go1.14.") {
-		name = name + "-go1.14"
+	ver := runtime.Version()
+	switch {
+	case isPreGo114(ver):
+		return name + "-go1.13"
+	default:
+		return name
 	}
-	return "e2e/expected/" + name
+}
+
+// go1.14.6 changed how it prints messages from tests. go1.14.{0-5} used a format
+// that was different from both go1.14.6 and previous versions of Go. These tests
+// no longer support that format.
+func isPreGo114(ver string) bool {
+	prefix := "go1.1"
+	if !strings.HasPrefix(ver, prefix) || len(ver) < len(prefix)+1 {
+		return false
+	}
+	switch ver[len(prefix)] {
+	case '0', '1', '2', '3':
+		return true
+	}
+	return false
 }

rerunfails.go

@@ -27,8 +27,27 @@ func (o rerunOpts) Args() []string {
 	return result
 }
 
+func newRerunOptsFromTestCase(tc testjson.TestCase) rerunOpts {
+	return rerunOpts{
+		runFlag: goTestRunFlagForTestCase(tc.Test),
+		pkg:     tc.Package,
+	}
+}
+
+type testCaseFilter func([]testjson.TestCase) []testjson.TestCase
+
+func rerunFailsFilter(o *options) testCaseFilter {
+	if o.rerunFailsOnlyRootCases {
+		return func(tcs []testjson.TestCase) []testjson.TestCase {
+			return tcs
+		}
+	}
+	return testjson.FilterFailedUnique
+}
+
 func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanConfig) error {
-	failed := len(scanConfig.Execution.Failed())
+	tcFilter := rerunFailsFilter(opts)
+	failed := len(tcFilter(scanConfig.Execution.Failed()))
 	if failed > opts.rerunFailsMaxInitialFailures {
 		return fmt.Errorf(
 			"number of test failures (%d) exceeds maximum (%d) set by --rerun-fails-max-failures",
@@ -42,12 +61,8 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
 		opts.stdout.Write([]byte("\n")) // nolint: errcheck
 
 		nextRec := newFailureRecorder(scanConfig.Handler)
-		for pkg, testCases := range rec.pkgFailures {
-			rerun := rerunOpts{
-				runFlag: goTestRunFlagFromTestCases(testCases),
-				pkg:     pkg,
-			}
-			goTestProc, err := startGoTest(ctx, goTestCmdArgs(opts, rerun))
+		for _, tc := range tcFilter(rec.failures) {
+			goTestProc, err := startGoTest(ctx, goTestCmdArgs(opts, newRerunOptsFromTestCase(tc)))
 			if err != nil {
 				return errors.Wrapf(err, "failed to run %s", strings.Join(goTestProc.cmd.Args, " "))
 			}
@@ -66,8 +81,8 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
 			if err := hasErrors(lastErr, scanConfig.Execution); err != nil {
 				return err
 			}
-			rec = nextRec
 		}
+		rec = nextRec
 	}
 	return lastErr
 }
@@ -86,50 +101,36 @@ func hasErrors(err error, exec *testjson.Execution) error {
 
 type failureRecorder struct {
 	testjson.EventHandler
-	pkgFailures map[string][]string
+	failures []testjson.TestCase
 }
 
 func newFailureRecorder(handler testjson.EventHandler) *failureRecorder {
-	return &failureRecorder{
-		EventHandler: handler,
-		pkgFailures:  make(map[string][]string),
-	}
+	return &failureRecorder{EventHandler: handler}
 }
 
 func newFailureRecorderFromExecution(exec *testjson.Execution) *failureRecorder {
-	r := newFailureRecorder(nil)
-	for _, tc := range exec.Failed() {
-		r.pkgFailures[tc.Package] = append(r.pkgFailures[tc.Package], tc.Test)
-	}
-	return r
+	return &failureRecorder{failures: exec.Failed()}
 }
 
 func (r *failureRecorder) Event(event testjson.TestEvent, execution *testjson.Execution) error {
 	if !event.PackageEvent() && event.Action == testjson.ActionFail {
-		r.pkgFailures[event.Package] = append(r.pkgFailures[event.Package], event.Test)
+		pkg := execution.Package(event.Package)
+		tc := pkg.LastFailedByName(event.Test)
+		r.failures = append(r.failures, tc)
 	}
 	return r.EventHandler.Event(event, execution)
 }
 
 func (r *failureRecorder) count() int {
-	total := 0
-	for _, tcs := range r.pkgFailures {
-		total += len(tcs)
-	}
-	return total
+	return len(r.failures)
 }
 
-func goTestRunFlagFromTestCases(tcs []string) string {
-	buf := new(strings.Builder)
-	buf.WriteString("-run=^(")
-	for i, tc := range tcs {
-		if i != 0 {
-			buf.WriteString("|")
-		}
-		buf.WriteString(tc)
+func goTestRunFlagForTestCase(name string) string {
+	root, sub := testjson.SplitTestName(name)
+	if sub == "" {
+		return "-run=^" + name + "$"
 	}
-	buf.WriteString(")$")
-	return buf.String()
+	return "-run=^" + root + "$/^" + sub + "$"
 }
 
 func writeRerunFailsReport(opts *options, exec *testjson.Execution) error {

rerunfails_test.go

@@ -32,3 +32,30 @@ func TestWriteRerunFailsReport(t *testing.T) {
 	assert.NilError(t, err)
 	golden.Assert(t, string(raw), t.Name()+"-expected")
 }
+
+func TestGoTestRunFlagFromTestCases(t *testing.T) {
+	type testCase struct {
+		input    string
+		expected string
+	}
+	fn := func(t *testing.T, tc testCase) {
+		actual := goTestRunFlagForTestCase(tc.input)
+		assert.Equal(t, actual, tc.expected)
+	}
+
+	var testCases = map[string]testCase{
+		"root test case": {
+			input:    "TestOne",
+			expected: "-run=^TestOne$",
+		},
+		"sub test case": {
+			input:    "TestOne/SubtestA",
+			expected: "-run=^TestOne$/^SubtestA$",
+		},
+	}
+	for name := range testCases {
+		t.Run(name, func(t *testing.T) {
+			fn(t, testCases[name])
+		})
+	}
+}

testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail

@@ -1,45 +1,52 @@
 PASS testdata/e2e/flaky.TestAlwaysPasses
 === RUN   TestFailsRarely
 SEED:  0
---- FAIL: TestFailsRarely
     flaky_test.go:51: not this time
+--- FAIL: TestFailsRarely
 FAIL testdata/e2e/flaky.TestFailsRarely
 === RUN   TestFailsSometimes
 SEED:  0
---- FAIL: TestFailsSometimes
     flaky_test.go:58: not this time
+--- FAIL: TestFailsSometimes
 FAIL testdata/e2e/flaky.TestFailsSometimes
+PASS testdata/e2e/flaky.TestFailsOften/subtest_always_passes
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail
 === RUN   TestFailsOften
 SEED:  0
 --- FAIL: TestFailsOften
-    flaky_test.go:65: not this time
 FAIL testdata/e2e/flaky.TestFailsOften
 PASS testdata/e2e/flaky.TestFailsOftenDoesNotPrefixMatch
 PASS testdata/e2e/flaky.TestFailsSometimesDoesNotPrefixMatch
 FAIL testdata/e2e/flaky
 
-DONE 6 tests, 3 failures
+DONE 8 tests, 4 failures
 
 PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
-=== RUN   TestFailsSometimes
-SEED:  1
---- FAIL: TestFailsSometimes
-    flaky_test.go:58: not this time
-FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes (re-run 1)
+PASS testdata/e2e/flaky
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 1)
 === RUN   TestFailsOften
-SEED:  1
+SEED:  3
 --- FAIL: TestFailsOften
-    flaky_test.go:65: not this time
 FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
 FAIL testdata/e2e/flaky
 
-DONE 2 runs, 9 tests, 5 failures
+DONE 2 runs, 12 tests, 6 failures
 
-PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
+=== RUN   TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+FAIL testdata/e2e/flaky.TestFailsOften/subtest_may_fail (re-run 2)
 === RUN   TestFailsOften
-SEED:  2
+SEED:  4
 --- FAIL: TestFailsOften
-    flaky_test.go:65: not this time
 FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
 FAIL testdata/e2e/flaky
 
@@ -52,21 +59,26 @@ SEED:  0
 SEED:  0
     flaky_test.go:58: not this time
 
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
+
 === FAIL: testdata/e2e/flaky TestFailsOften
 SEED:  0
-    flaky_test.go:65: not this time
 
-=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
-SEED:  1
-    flaky_test.go:58: not this time
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 1)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
 
 === FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
-SEED:  1
-    flaky_test.go:65: not this time
+SEED:  3
+
+=== FAIL: testdata/e2e/flaky TestFailsOften/subtest_may_fail (re-run 2)
+    flaky_test.go:68: not this time
+    --- FAIL: TestFailsOften/subtest_may_fail
 
 === FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
-SEED:  2
-    flaky_test.go:65: not this time
+SEED:  4
 
 
-DONE 3 runs, 11 tests, 6 failures
+DONE 3 runs, 14 tests, 8 failures