Merge pull request #126 from dnephin/rerun-fails

Rerun failed tests

cmd/tool/slowest/slowest.go

@@ -20,7 +20,7 @@ func Run(name string, args []string) error {
 	case err == pflag.ErrHelp:
 		return nil
 	case err != nil:
-		flags.Usage()
+		usage(os.Stderr, name, flags)
 		return err
 	}
 	return run(opts)
@@ -31,8 +31,22 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
 	flags := pflag.NewFlagSet(name, pflag.ContinueOnError)
 	flags.SetInterspersed(false)
 	flags.Usage = func() {
-		fmt.Fprintf(os.Stderr, `Usage:
-    %s [flags]
+		usage(os.Stdout, name, flags)
+	}
+	flags.StringVar(&opts.jsonfile, "jsonfile", os.Getenv("GOTESTSUM_JSONFILE"),
+		"path to test2json output, defaults to stdin")
+	flags.DurationVar(&opts.threshold, "threshold", 100*time.Millisecond,
+		"test cases with elapsed time greater than threshold are slow tests")
+	flags.StringVar(&opts.skipStatement, "skip-stmt", "",
+		"add this go statement to slow tests, instead of printing the list of slow tests")
+	flags.BoolVar(&opts.debug, "debug", false,
+		"enable debug logging.")
+	return flags, opts
+}
+
+func usage(out io.Writer, name string, flags *pflag.FlagSet) {
+	fmt.Fprintf(out, `Usage:
+    %[1]s [flags]
 
 Read a json file and print or update tests which are slower than threshold.
 The json file may be created with 'gotestsum --jsonfile' or 'go test -json'.
@@ -63,7 +77,7 @@ Alternatively, a custom --skip-stmt may be provided as a string:
             t.Skip("too slow for TEST_FAST")
         }
     '
-    go test -json -short ./... | %s --skip-stmt "$skip_stmt"
+    go test -json -short ./... | %[1]s --skip-stmt "$skip_stmt"
 
 Note that this tool does not add imports, so using a custom statement may require
 you to add imports to the file.
@@ -73,18 +87,9 @@ variable, following the same rules as the go toolchain. See
 https://golang.org/cmd/go/#hdr-Environment_variables.
 
 Flags:
-`, name, name)
-		flags.PrintDefaults()
-	}
-	flags.StringVar(&opts.jsonfile, "jsonfile", os.Getenv("GOTESTSUM_JSONFILE"),
-		"path to test2json output, defaults to stdin")
-	flags.DurationVar(&opts.threshold, "threshold", 100*time.Millisecond,
-		"test cases with elapsed time greater than threshold are slow tests")
-	flags.StringVar(&opts.skipStatement, "skip-stmt", "",
-		"add this go statement to slow tests, instead of printing the list of slow tests")
-	flags.BoolVar(&opts.debug, "debug", false,
-		"enable debug logging.")
-	return flags, opts
+`, name)
+	flags.SetOutput(out)
+	flags.PrintDefaults()
 }
 
 type options struct {

cmd/tool/slowest/slowest_test.go

@@ -1,15 +1,29 @@
 package slowest
 
 import (
+	"bytes"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"gotest.tools/gotestsum/testjson"
 	"gotest.tools/v3/assert"
+	"gotest.tools/v3/env"
+	"gotest.tools/v3/golden"
 )
 
+func TestUsage_WithFlagsFromSetupFlags(t *testing.T) {
+	defer env.PatchAll(t, nil)()
+
+	name := "gotestsum tool slowest"
+	flags, _ := setupFlags(name)
+	buf := new(bytes.Buffer)
+	usage(buf, name, flags)
+
+	golden.Assert(t, buf.String(), "cmd-flags-help-text")
+}
+
 func TestAggregateTestCases(t *testing.T) {
 	cases := []testjson.TestCase{
 		{Test: "TestOne", Package: "pkg", Elapsed: time.Second},

cmd/tool/slowest/testdata/cmd-flags-help-text

@@ -0,0 +1,46 @@
+Usage:
+    gotestsum tool slowest [flags]
+
+Read a json file and print or update tests which are slower than threshold.
+The json file may be created with 'gotestsum --jsonfile' or 'go test -json'.
+If a TestCase appears more than once in the json file, it will only appear once
+in the output, and the median value of all the elapsed times will be used.
+
+By default this command will print the list of tests slower than threshold to stdout.
+The list will be sorted from slowest to fastest.
+
+If --skip-stmt is set, instead of printing the list to stdout, the AST for the
+Go source code in the working directory tree will be modified. The value of
+--skip-stmt will be added to Go test files as the first statement in all the test
+functions which are slower than threshold.
+
+The --skip-stmt flag may be set to the name of a predefined statement, or to
+Go source code which will be parsed as a go/ast.Stmt. Currently there is only one
+predefined statement, --skip-stmt=testing.Short, which uses this Go statement:
+
+    if testing.Short() {
+        t.Skip("too slow for testing.Short")
+    }
+
+
+Alternatively, a custom --skip-stmt may be provided as a string:
+
+    skip_stmt='
+        if os.Getenv("TEST_FAST") {
+            t.Skip("too slow for TEST_FAST")
+        }
+    '
+    go test -json -short ./... | gotestsum tool slowest --skip-stmt "$skip_stmt"
+
+Note that this tool does not add imports, so using a custom statement may require
+you to add imports to the file.
+
+Go build flags, such as build tags, may be set using the GOFLAGS environment
+variable, following the same rules as the go toolchain. See
+https://golang.org/cmd/go/#hdr-Environment_variables.
+
+Flags:
+      --debug                enable debug logging.
+      --jsonfile string      path to test2json output, defaults to stdin
+      --skip-stmt string     add this go statement to slow tests, instead of printing the list of slow tests
+      --threshold duration   test cases with elapsed time greater than threshold are slow tests (default 100ms)

do

@@ -8,7 +8,7 @@ binary() {
 }
 
 update-golden() {
-    gotestsum -- ./testjson ./internal/junitxml -test.update-golden
+    gotestsum -- . ./testjson ./internal/junitxml ./cmd/tool/slowest -test.update-golden
 }
 
 lint() {

flags.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/google/shlex"
 	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
 	"gotest.tools/gotestsum/internal/junitxml"
 	"gotest.tools/gotestsum/testjson"
 )
@@ -112,3 +113,22 @@ func (c *commandValue) Value() []string {
 	}
 	return c.command
 }
+
+var _ pflag.Value = (*stringSlice)(nil)
+
+// stringSlice is a flag.Value which populates the string slice by splitting
+// the raw flag value on spaces.
+type stringSlice []string
+
+func (s *stringSlice) String() string {
+	return strings.Join(*s, " ")
+}
+
+func (s *stringSlice) Set(raw string) error {
+	*s = append(*s, strings.Split(raw, " ")...)
+	return nil
+}
+
+func (s *stringSlice) Type() string {
+	return "list"
+}

handler.go

@@ -25,7 +25,8 @@ func (h *eventHandler) Err(text string) error {
 }
 
 func (h *eventHandler) Event(event testjson.TestEvent, execution *testjson.Execution) error {
-	if h.jsonFile != nil {
+	// ignore artificial events with no raw Bytes()
+	if h.jsonFile != nil && len(event.Bytes()) > 0 {
 		_, err := h.jsonFile.Write(append(event.Bytes(), '\n'))
 		if err != nil {
 			return errors.Wrap(err, "failed to write JSON file")

handler_test.go

@@ -46,3 +46,28 @@ func newExecFromTestData(t *testing.T) *testjson.Execution {
 	assert.NilError(t, err)
 	return exec
 }
+
+type bufferCloser struct {
+	bytes.Buffer
+}
+
+func (bufferCloser) Close() error { return nil }
+
+func TestEventHandler_Event_WithMissingActionFail(t *testing.T) {
+	buf := new(bufferCloser)
+	errBuf := new(bytes.Buffer)
+	format := testjson.NewEventFormatter(errBuf, "testname")
+
+	source := golden.Get(t, "../testjson/testdata/go-test-json-missing-test-fail.out")
+	cfg := testjson.ScanConfig{
+		Stdout:  bytes.NewReader(source),
+		Handler: &eventHandler{jsonFile: buf, formatter: format},
+	}
+	_, err := testjson.ScanTestOutput(cfg)
+	assert.NilError(t, err)
+
+	assert.Equal(t, buf.String(), string(source))
+	// confirm the artificial event was sent to the handler by checking the output
+	// of the formatter.
+	golden.Assert(t, errBuf.String(), "event-handler-missing-test-fail-expected")
+}

internal/text/testing.go

@@ -0,0 +1,42 @@
+package text
+
+import (
+	"bufio"
+	"io"
+	"strings"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+// ProcessLines from the Reader by passing each one to ops. The output of each
+// op is passed to the next. Returns the string created by joining all the
+// processed lines.
+func ProcessLines(t *testing.T, r io.Reader, ops ...func(string) string) string {
+	t.Helper()
+	out := new(strings.Builder)
+	scan := bufio.NewScanner(r)
+	for scan.Scan() {
+		line := scan.Text()
+		for _, op := range ops {
+			line = op(line)
+		}
+		out.WriteString(line + "\n")
+	}
+	assert.NilError(t, scan.Err())
+	return out.String()
+}
+
+func OpRemoveSummaryLineElapsedTime(line string) string {
+	if i := strings.Index(line, " in "); i > 0 {
+		return line[:i]
+	}
+	return line
+}
+
+func OpRemoveTestElapsedTime(line string) string {
+	if i := strings.Index(line, " (0.0"); i > 0 && i+8 == len(line) {
+		return line[:i]
+	}
+	return line
+}