Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add --max-fails flag for ending the test run #159

Merged
merged 3 commits into from
Nov 8, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions cmd/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ type eventHandler struct {
formatter testjson.EventFormatter
err io.Writer
jsonFile io.WriteCloser
maxFails int
}

func (h *eventHandler) Err(text string) error {
Expand All @@ -37,6 +38,10 @@ func (h *eventHandler) Event(event testjson.TestEvent, execution *testjson.Execu
if err != nil {
return errors.Wrap(err, "failed to format event")
}

if h.maxFails > 0 && len(execution.Failed()) >= h.maxFails {
return fmt.Errorf("ending test run because max failures was reached")
}
return nil
}

Expand All @@ -59,6 +64,7 @@ func newEventHandler(opts *options) (*eventHandler, error) {
handler := &eventHandler{
formatter: formatter,
err: opts.stderr,
maxFails: opts.maxFails,
}
var err error
if opts.jsonFile != "" {
Expand Down
14 changes: 14 additions & 0 deletions cmd/handler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cmd

import (
"bytes"
"io/ioutil"
"os"
"strings"
"testing"
Expand Down Expand Up @@ -71,3 +72,16 @@ func TestEventHandler_Event_WithMissingActionFail(t *testing.T) {
// of the formatter.
golden.Assert(t, errBuf.String(), "event-handler-missing-test-fail-expected")
}

func TestEventHandler_Event_MaxFails(t *testing.T) {
format := testjson.NewEventFormatter(ioutil.Discard, "testname")

source := golden.Get(t, "../../testjson/testdata/go-test-json.out")
cfg := testjson.ScanConfig{
Stdout: bytes.NewReader(source),
Handler: &eventHandler{formatter: format, maxFails: 2},
}

_, err := testjson.ScanTestOutput(cfg)
assert.Error(t, err, "ending test run because max failures was reached")
}
6 changes: 5 additions & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
"command to run after the tests have completed")
flags.BoolVar(&opts.watch, "watch", false,
"watch go files, and run tests when a file is modified")
flags.IntVar(&opts.maxFails, "max-fails", 0,
"end the test run after this number of failures")

flags.StringVar(&opts.junitFile, "junitfile",
lookEnvWithDefault("GOTESTSUM_JUNITFILE", ""),
Expand Down Expand Up @@ -163,6 +165,7 @@ type options struct {
rerunFailsOnlyRootCases bool
packages []string
watch bool
maxFails int
version bool

// shims for testing
Expand Down Expand Up @@ -208,10 +211,11 @@ func run(opts *options) error {
Stdout: goTestProc.stdout,
Stderr: goTestProc.stderr,
Handler: handler,
Stop: cancel,
}
exec, err := testjson.ScanTestOutput(cfg)
if err != nil {
return err
return finishRun(opts, exec, err)
}
exitErr := goTestProc.cmd.Wait()
if exitErr == nil || opts.rerunFailsMaxAttempts == 0 {
Expand Down
39 changes: 36 additions & 3 deletions cmd/main_e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ func TestMain(m *testing.M) {
}

func TestE2E_RerunFails(t *testing.T) {
if testing.Short() {
t.Skip("too slow for short run")
}

type testCase struct {
name string
args []string
Expand Down Expand Up @@ -96,9 +100,6 @@ func TestE2E_RerunFails(t *testing.T) {
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if testing.Short() {
t.Skip("too slow for short run")
}
fn(t, tc)
})
}
Expand Down Expand Up @@ -215,3 +216,35 @@ func TestE2E_SignalHandler(t *testing.T) {

result.Assert(t, icmd.Expected{ExitCode: 102})
}

func TestE2E_MaxFails_EndTestRun(t *testing.T) {
if testing.Short() {
t.Skip("too slow for short run")
}

tmpFile := fs.NewFile(t, t.Name()+"-seedfile", fs.WithContent("0"))
defer tmpFile.Remove()

envVars := osEnviron()
envVars["TEST_SEEDFILE"] = tmpFile.Path()
defer env.PatchAll(t, envVars)()

flags, opts := setupFlags("gotestsum")
args := []string{"--max-fails=2", "--packages=./testdata/e2e/flaky/", "--", "-tags=testdata"}
assert.NilError(t, flags.Parse(args))
opts.args = flags.Args()

bufStdout := new(bytes.Buffer)
opts.stdout = bufStdout
bufStderr := new(bytes.Buffer)
opts.stderr = bufStderr

err := run(opts)
assert.Error(t, err, "ending test run because max failures was reached")
out := text.ProcessLines(t, bufStdout,
text.OpRemoveSummaryLineElapsedTime,
text.OpRemoveTestElapsedTime,
filepath.ToSlash, // for windows
)
golden.Assert(t, out, "e2e/expected/"+t.Name())
}
3 changes: 3 additions & 0 deletions cmd/rerunfails.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ func rerunFailsFilter(o *options) testCaseFilter {
}

func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanConfig) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
tcFilter := rerunFailsFilter(opts)

rec := newFailureRecorderFromExecution(scanConfig.Execution)
Expand All @@ -64,6 +66,7 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
Stderr: goTestProc.stderr,
Handler: nextRec,
Execution: scanConfig.Execution,
Stop: cancel,
}
if _, err := testjson.ScanTestOutput(cfg); err != nil {
return err
Expand Down
11 changes: 11 additions & 0 deletions cmd/testdata/e2e/expected/TestE2E_MaxFails_EndTestRun
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@

=== Failed
=== FAIL: cmd/testdata/e2e/flaky TestFailsRarely
SEED: 0
flaky_test.go:51: not this time

=== FAIL: cmd/testdata/e2e/flaky TestFailsSometimes
SEED: 0
flaky_test.go:58: not this time

DONE 3 tests, 2 failures
1 change: 1 addition & 0 deletions cmd/testdata/gotestsum-help-text
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ Flags:
--junitfile string write a JUnit XML file
--junitfile-testcase-classname field-format format the testcase classname field as: full, relative, short (default full)
--junitfile-testsuite-name field-format format the testsuite name field as: full, relative, short (default full)
--max-fails int end the test run after this number of failures
--no-color disable color output (default true)
--packages list space separated list of package to test
--post-run-command command command to run after the tests have completed
Expand Down
23 changes: 17 additions & 6 deletions testjson/execution.go
Original file line number Diff line number Diff line change
Expand Up @@ -522,6 +522,8 @@ type ScanConfig struct {
// Execution to populate while scanning. If nil a new one will be created
// and returned from ScanTestOutput.
Execution *Execution
// Stop is called when ScanTestOutput fails during a scan.
Stop func()
}

// EventHandler is called by ScanTestOutput for each event and write to stderr.
Expand All @@ -548,6 +550,9 @@ func ScanTestOutput(config ScanConfig) (*Execution, error) {
if config.Stderr == nil {
config.Stderr = new(bytes.Reader)
}
if config.Stop == nil {
config.Stop = func() {}
}
execution := config.Execution
if execution == nil {
execution = newExecution()
Expand All @@ -557,21 +562,27 @@ func ScanTestOutput(config ScanConfig) (*Execution, error) {

var group errgroup.Group
group.Go(func() error {
return readStdout(config, execution)
return stopOnError(config.Stop, readStdout(config, execution))
})
group.Go(func() error {
return readStderr(config, execution)
return stopOnError(config.Stop, readStderr(config, execution))
})
if err := group.Wait(); err != nil {
return execution, err
}

err := group.Wait()
for _, event := range execution.end() {
if err := config.Handler.Event(event, execution); err != nil {
return execution, err
}
}
return execution, err
}

return execution, nil
func stopOnError(stop func(), err error) error {
if err != nil {
stop()
return err
}
return nil
}

func readStdout(config ScanConfig, execution *Execution) error {
Expand Down
32 changes: 32 additions & 0 deletions testjson/execution_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package testjson

import (
"bytes"
"fmt"
"testing"
"time"

Expand Down Expand Up @@ -58,3 +59,34 @@ func TestScanTestOutput_MinimalConfig(t *testing.T) {
// a weak check to show that all the stdout was scanned
assert.Equal(t, exec.Total(), 46)
}

func TestScanTestOutput_CallsStopOnError(t *testing.T) {
var called bool
stop := func() {
called = true
}
cfg := ScanConfig{
Stdout: bytes.NewReader(golden.Get(t, "go-test-json.out")),
Handler: &handlerFails{},
Stop: stop,
}
_, err := ScanTestOutput(cfg)
assert.Error(t, err, "something failed")
assert.Assert(t, called)
}

type handlerFails struct {
count int
}

func (s *handlerFails) Event(_ TestEvent, _ *Execution) error {
if s.count > 1 {
return fmt.Errorf("something failed")
}
s.count++
return nil
}

func (s *handlerFails) Err(_ string) error {
return nil
}