End a run when ScanTestOutput fails

Previously tests would continue to run until they all completed.

cmd/main.go

@@ -211,10 +211,11 @@ func run(opts *options) error {
 		Stdout:  goTestProc.stdout,
 		Stderr:  goTestProc.stderr,
 		Handler: handler,
+		Stop:    cancel,
 	}
 	exec, err := testjson.ScanTestOutput(cfg)
 	if err != nil {
-		return err
+		return finishRun(opts, exec, err)
 	}
 	exitErr := goTestProc.cmd.Wait()
 	if exitErr == nil || opts.rerunFailsMaxAttempts == 0 {

cmd/main_e2e_test.go

@@ -28,6 +28,10 @@ func TestMain(m *testing.M) {
 }
 
 func TestE2E_RerunFails(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for short run")
+	}
+
 	type testCase struct {
 		name        string
 		args        []string
@@ -96,9 +100,6 @@ func TestE2E_RerunFails(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			if testing.Short() {
-				t.Skip("too slow for short run")
-			}
 			fn(t, tc)
 		})
 	}
@@ -215,3 +216,35 @@ func TestE2E_SignalHandler(t *testing.T) {
 
 	result.Assert(t, icmd.Expected{ExitCode: 102})
 }
+
+func TestE2E_MaxFails_EndTestRun(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for short run")
+	}
+
+	tmpFile := fs.NewFile(t, t.Name()+"-seedfile", fs.WithContent("0"))
+	defer tmpFile.Remove()
+
+	envVars := osEnviron()
+	envVars["TEST_SEEDFILE"] = tmpFile.Path()
+	defer env.PatchAll(t, envVars)()
+
+	flags, opts := setupFlags("gotestsum")
+	args := []string{"--max-fails=2", "--packages=./testdata/e2e/flaky/", "--", "-tags=testdata"}
+	assert.NilError(t, flags.Parse(args))
+	opts.args = flags.Args()
+
+	bufStdout := new(bytes.Buffer)
+	opts.stdout = bufStdout
+	bufStderr := new(bytes.Buffer)
+	opts.stderr = bufStderr
+
+	err := run(opts)
+	assert.Error(t, err, "ending test run because max failures was reached")
+	out := text.ProcessLines(t, bufStdout,
+		text.OpRemoveSummaryLineElapsedTime,
+		text.OpRemoveTestElapsedTime,
+		filepath.ToSlash, // for windows
+	)
+	golden.Assert(t, out, "e2e/expected/"+t.Name())
+}

cmd/rerunfails.go

@@ -44,6 +44,8 @@ func rerunFailsFilter(o *options) testCaseFilter {
 }
 
 func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanConfig) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	tcFilter := rerunFailsFilter(opts)
 
 	rec := newFailureRecorderFromExecution(scanConfig.Execution)
@@ -64,6 +66,7 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
 				Stderr:    goTestProc.stderr,
 				Handler:   nextRec,
 				Execution: scanConfig.Execution,
+				Stop:      cancel,
 			}
 			if _, err := testjson.ScanTestOutput(cfg); err != nil {
 				return err

cmd/testdata/e2e/expected/TestE2E_MaxFails_EndTestRun

@@ -0,0 +1,11 @@
+
+=== Failed
+=== FAIL: cmd/testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+DONE 3 tests, 2 failures

testjson/execution.go

@@ -522,6 +522,8 @@ type ScanConfig struct {
 	// Execution to populate while scanning. If nil a new one will be created
 	// and returned from ScanTestOutput.
 	Execution *Execution
+	// Stop is called when ScanTestOutput fails during a scan.
+	Stop func()
 }
 
 // EventHandler is called by ScanTestOutput for each event and write to stderr.
@@ -548,6 +550,9 @@ func ScanTestOutput(config ScanConfig) (*Execution, error) {
 	if config.Stderr == nil {
 		config.Stderr = new(bytes.Reader)
 	}
+	if config.Stop == nil {
+		config.Stop = func() {}
+	}
 	execution := config.Execution
 	if execution == nil {
 		execution = newExecution()
@@ -557,21 +562,27 @@ func ScanTestOutput(config ScanConfig) (*Execution, error) {
 
 	var group errgroup.Group
 	group.Go(func() error {
-		return readStdout(config, execution)
+		if err := readStdout(config, execution); err != nil {
+			config.Stop()
+			return err
+		}
+		return nil
 	})
 	group.Go(func() error {
-		return readStderr(config, execution)
+		if err := readStderr(config, execution); err != nil {
+			config.Stop()
+			return err
+		}
+		return nil
 	})
-	if err := group.Wait(); err != nil {
-		return execution, err
-	}
+
+	err := group.Wait()
 	for _, event := range execution.end() {
 		if err := config.Handler.Event(event, execution); err != nil {
 			return execution, err
 		}
 	}
-
-	return execution, nil
+	return execution, err
 }
 
 func readStdout(config ScanConfig, execution *Execution) error {