Merge pull request #159 from dnephin/max-fails

Add --max-fails flag for ending the test run

cmd/handler.go

@@ -16,6 +16,7 @@ type eventHandler struct {
 	formatter testjson.EventFormatter
 	err       io.Writer
 	jsonFile  io.WriteCloser
+	maxFails  int
 }
 
 func (h *eventHandler) Err(text string) error {
@@ -37,6 +38,10 @@ func (h *eventHandler) Event(event testjson.TestEvent, execution *testjson.Execu
 	if err != nil {
 		return errors.Wrap(err, "failed to format event")
 	}
+
+	if h.maxFails > 0 && len(execution.Failed()) >= h.maxFails {
+		return fmt.Errorf("ending test run because max failures was reached")
+	}
 	return nil
 }
 
@@ -59,6 +64,7 @@ func newEventHandler(opts *options) (*eventHandler, error) {
 	handler := &eventHandler{
 		formatter: formatter,
 		err:       opts.stderr,
+		maxFails:  opts.maxFails,
 	}
 	var err error
 	if opts.jsonFile != "" {

cmd/handler_test.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"bytes"
+	"io/ioutil"
 	"os"
 	"strings"
 	"testing"
@@ -71,3 +72,16 @@ func TestEventHandler_Event_WithMissingActionFail(t *testing.T) {
 	// of the formatter.
 	golden.Assert(t, errBuf.String(), "event-handler-missing-test-fail-expected")
 }
+
+func TestEventHandler_Event_MaxFails(t *testing.T) {
+	format := testjson.NewEventFormatter(ioutil.Discard, "testname")
+
+	source := golden.Get(t, "../../testjson/testdata/go-test-json.out")
+	cfg := testjson.ScanConfig{
+		Stdout:  bytes.NewReader(source),
+		Handler: &eventHandler{formatter: format, maxFails: 2},
+	}
+
+	_, err := testjson.ScanTestOutput(cfg)
+	assert.Error(t, err, "ending test run because max failures was reached")
+}

cmd/main.go

@@ -87,6 +87,8 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
 		"command to run after the tests have completed")
 	flags.BoolVar(&opts.watch, "watch", false,
 		"watch go files, and run tests when a file is modified")
+	flags.IntVar(&opts.maxFails, "max-fails", 0,
+		"end the test run after this number of failures")
 
 	flags.StringVar(&opts.junitFile, "junitfile",
 		lookEnvWithDefault("GOTESTSUM_JUNITFILE", ""),
@@ -163,6 +165,7 @@ type options struct {
 	rerunFailsOnlyRootCases      bool
 	packages                     []string
 	watch                        bool
+	maxFails                     int
 	version                      bool
 
 	// shims for testing
@@ -208,10 +211,11 @@ func run(opts *options) error {
 		Stdout:  goTestProc.stdout,
 		Stderr:  goTestProc.stderr,
 		Handler: handler,
+		Stop:    cancel,
 	}
 	exec, err := testjson.ScanTestOutput(cfg)
 	if err != nil {
-		return err
+		return finishRun(opts, exec, err)
 	}
 	exitErr := goTestProc.cmd.Wait()
 	if exitErr == nil || opts.rerunFailsMaxAttempts == 0 {

cmd/main_e2e_test.go

@@ -28,6 +28,10 @@ func TestMain(m *testing.M) {
 }
 
 func TestE2E_RerunFails(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for short run")
+	}
+
 	type testCase struct {
 		name        string
 		args        []string
@@ -96,9 +100,6 @@ func TestE2E_RerunFails(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			if testing.Short() {
-				t.Skip("too slow for short run")
-			}
 			fn(t, tc)
 		})
 	}
@@ -215,3 +216,35 @@ func TestE2E_SignalHandler(t *testing.T) {
 
 	result.Assert(t, icmd.Expected{ExitCode: 102})
 }
+
+func TestE2E_MaxFails_EndTestRun(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for short run")
+	}
+
+	tmpFile := fs.NewFile(t, t.Name()+"-seedfile", fs.WithContent("0"))
+	defer tmpFile.Remove()
+
+	envVars := osEnviron()
+	envVars["TEST_SEEDFILE"] = tmpFile.Path()
+	defer env.PatchAll(t, envVars)()
+
+	flags, opts := setupFlags("gotestsum")
+	args := []string{"--max-fails=2", "--packages=./testdata/e2e/flaky/", "--", "-tags=testdata"}
+	assert.NilError(t, flags.Parse(args))
+	opts.args = flags.Args()
+
+	bufStdout := new(bytes.Buffer)
+	opts.stdout = bufStdout
+	bufStderr := new(bytes.Buffer)
+	opts.stderr = bufStderr
+
+	err := run(opts)
+	assert.Error(t, err, "ending test run because max failures was reached")
+	out := text.ProcessLines(t, bufStdout,
+		text.OpRemoveSummaryLineElapsedTime,
+		text.OpRemoveTestElapsedTime,
+		filepath.ToSlash, // for windows
+	)
+	golden.Assert(t, out, "e2e/expected/"+t.Name())
+}

cmd/rerunfails.go

@@ -44,6 +44,8 @@ func rerunFailsFilter(o *options) testCaseFilter {
 }
 
 func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanConfig) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	tcFilter := rerunFailsFilter(opts)
 
 	rec := newFailureRecorderFromExecution(scanConfig.Execution)
@@ -64,6 +66,7 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
 				Stderr:    goTestProc.stderr,
 				Handler:   nextRec,
 				Execution: scanConfig.Execution,
+				Stop:      cancel,
 			}
 			if _, err := testjson.ScanTestOutput(cfg); err != nil {
 				return err

cmd/testdata/e2e/expected/TestE2E_MaxFails_EndTestRun

@@ -0,0 +1,11 @@
+
+=== Failed
+=== FAIL: cmd/testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: cmd/testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+DONE 3 tests, 2 failures

cmd/testdata/gotestsum-help-text

@@ -10,6 +10,7 @@ Flags:
       --junitfile string                            write a JUnit XML file
       --junitfile-testcase-classname field-format   format the testcase classname field as: full, relative, short (default full)
       --junitfile-testsuite-name field-format       format the testsuite name field as: full, relative, short (default full)
+      --max-fails int                               end the test run after this number of failures
       --no-color                                    disable color output (default true)
       --packages list                               space separated list of package to test
       --post-run-command command                    command to run after the tests have completed

testjson/execution.go

@@ -522,6 +522,8 @@ type ScanConfig struct {
 	// Execution to populate while scanning. If nil a new one will be created
 	// and returned from ScanTestOutput.
 	Execution *Execution
+	// Stop is called when ScanTestOutput fails during a scan.
+	Stop func()
 }
 
 // EventHandler is called by ScanTestOutput for each event and write to stderr.
@@ -548,6 +550,9 @@ func ScanTestOutput(config ScanConfig) (*Execution, error) {
 	if config.Stderr == nil {
 		config.Stderr = new(bytes.Reader)
 	}
+	if config.Stop == nil {
+		config.Stop = func() {}
+	}
 	execution := config.Execution
 	if execution == nil {
 		execution = newExecution()
@@ -557,21 +562,27 @@ func ScanTestOutput(config ScanConfig) (*Execution, error) {
 
 	var group errgroup.Group
 	group.Go(func() error {
-		return readStdout(config, execution)
+		return stopOnError(config.Stop, readStdout(config, execution))
 	})
 	group.Go(func() error {
-		return readStderr(config, execution)
+		return stopOnError(config.Stop, readStderr(config, execution))
 	})
-	if err := group.Wait(); err != nil {
-		return execution, err
-	}
+
+	err := group.Wait()
 	for _, event := range execution.end() {
 		if err := config.Handler.Event(event, execution); err != nil {
 			return execution, err
 		}
 	}
+	return execution, err
+}
 
-	return execution, nil
+func stopOnError(stop func(), err error) error {
+	if err != nil {
+		stop()
+		return err
+	}
+	return nil
 }
 
 func readStdout(config ScanConfig, execution *Execution) error {

testjson/execution_test.go

@@ -2,6 +2,7 @@ package testjson
 
 import (
 	"bytes"
+	"fmt"
 	"testing"
 	"time"
 
@@ -58,3 +59,34 @@ func TestScanTestOutput_MinimalConfig(t *testing.T) {
 	// a weak check to show that all the stdout was scanned
 	assert.Equal(t, exec.Total(), 46)
 }
+
+func TestScanTestOutput_CallsStopOnError(t *testing.T) {
+	var called bool
+	stop := func() {
+		called = true
+	}
+	cfg := ScanConfig{
+		Stdout:  bytes.NewReader(golden.Get(t, "go-test-json.out")),
+		Handler: &handlerFails{},
+		Stop:    stop,
+	}
+	_, err := ScanTestOutput(cfg)
+	assert.Error(t, err, "something failed")
+	assert.Assert(t, called)
+}
+
+type handlerFails struct {
+	count int
+}
+
+func (s *handlerFails) Event(_ TestEvent, _ *Execution) error {
+	if s.count > 1 {
+		return fmt.Errorf("something failed")
+	}
+	s.count++
+	return nil
+}
+
+func (s *handlerFails) Err(_ string) error {
+	return nil
+}