Add test to cover rerun with errors

main_e2e_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"testing"
@@ -18,7 +19,7 @@ func TestE2E_RerunFails(t *testing.T) {
 	type testCase struct {
 		name        string
 		args        []string
-		expectedErr bool
+		expectedErr string
 	}
 	fn := func(t *testing.T, tc testCase) {
 		tmpFile := fs.NewFile(t, t.Name()+"-seedfile", fs.WithContent("0"))
@@ -38,16 +39,17 @@ func TestE2E_RerunFails(t *testing.T) {
 		opts.stderr = bufStderr
 
 		err := run(opts)
-		if tc.expectedErr {
-			assert.Error(t, err, "exit status 1")
+		if tc.expectedErr != "" {
+			assert.Error(t, err, tc.expectedErr)
 		} else {
 			assert.NilError(t, err)
 		}
 		out := text.ProcessLines(t, bufStdout,
 			text.OpRemoveSummaryLineElapsedTime,
-			text.OpRemoveTestElapsedTime)
+			text.OpRemoveTestElapsedTime,
+			filepath.ToSlash, // for windows
+		)
 		golden.Assert(t, out, expectedFilename(t.Name()))
-		assert.Equal(t, bufStderr.String(), "")
 	}
 	var testCases = []testCase{
 		{
@@ -67,7 +69,17 @@ func TestE2E_RerunFails(t *testing.T) {
 				"--packages=./testdata/e2e/flaky/",
 				"--", "-count=1", "-tags=testdata",
 			},
-			expectedErr: true,
+			expectedErr: "exit status 1",
+		},
+		{
+			name: "first run has errors, abort rerun",
+			args: []string{
+				"-f=testname",
+				"--rerun-fails=2",
+				"--packages=./testjson/internal/broken",
+				"--", "-count=1", "-tags=stubpkg",
+			},
+			expectedErr: "rerun aborted because previous run had errors",
 		},
 	}
 	for _, tc := range testCases {

rerunfails.go

@@ -33,14 +33,13 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
 			"number of test failures (%d) exceeds maximum (%d) set by --rerun-fails-max-failures",
 			failed, opts.rerunFailsMaxInitialFailures)
 	}
+	if err := hasErrors(scanConfig.Execution); err != nil {
+		return err
+	}
 
 	rec := newFailureRecorderFromExecution(scanConfig.Execution)
 	var lastErr error
-	for count := 0; rec.count() > 0 && count < opts.rerunFailsMaxAttempts; count++ {
-		if len(scanConfig.Execution.Errors()) > 0 {
-			return fmt.Errorf("re-run cancelled because previous run had errors")
-		}
-
+	for attempts := 0; rec.count() > 0 && attempts < opts.rerunFailsMaxAttempts; attempts++ {
 		testjson.PrintSummary(opts.stdout, scanConfig.Execution, testjson.SummarizeNone)
 		opts.stdout.Write([]byte("\n")) // nolint: errcheck
 
@@ -70,12 +69,22 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
 				log.Warnf("unexpected go test exit code: %v", lastErr)
 				// TODO: will 'go test' exit with 2 if it panics? maybe return err here.
 			}
+			if err := hasErrors(scanConfig.Execution); err != nil {
+				return err
+			}
 			rec = nextRec
 		}
 	}
 	return lastErr
 }
 
+func hasErrors(exec *testjson.Execution) error {
+	if len(exec.Errors()) > 0 {
+		return fmt.Errorf("rerun aborted because previous run had errors")
+	}
+	return nil
+}
+
 type failureRecorder struct {
 	testjson.EventHandler
 	pkgFailures map[string][]string

testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun

@@ -0,0 +1,5 @@
+
+=== Errors
+testjson/internal/broken/broken.go:5:21: undefined: somepackage
+
+DONE 0 tests, 1 error

testdata/e2e/expected/TestE2E_RerunFails/first_run_has_errors,_abort_rerun-go1.14

@@ -0,0 +1,5 @@
+
+=== Errors
+testjson/internal/broken/broken.go:5:21: undefined: somepackage
+
+DONE 0 tests, 1 error