Add an e2e test for rerun-fails

internal/testing/testing.go.go

@@ -0,0 +1,29 @@
+package testing
+
+import (
+	"bufio"
+	"io"
+	"strings"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+// RemoveSummaryTime from the contents of the Reader, and return the contents
+// of the buffer as a string.
+// Summary time is the elapsed time at the end of a DONE line in the test summary.
+func RemoveSummaryTime(t *testing.T, r io.Reader) string {
+	t.Helper()
+	out := new(strings.Builder)
+	scan := bufio.NewScanner(r)
+	for scan.Scan() {
+		line := scan.Text()
+		if i := strings.Index(line, " in "); i > 0 {
+			out.WriteString(line[:i] + "\n")
+			continue
+		}
+		out.WriteString(line + "\n")
+	}
+	assert.NilError(t, scan.Err())
+	return out.String()
+}

main_e2e_test.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+
+	itest "gotest.tools/gotestsum/internal/testing"
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/env"
+	"gotest.tools/v3/fs"
+	"gotest.tools/v3/golden"
+)
+
+func TestE2E_RerunFails(t *testing.T) {
+	type testCase struct {
+		name        string
+		args        []string
+		expectedErr bool
+	}
+	fn := func(t *testing.T, tc testCase) {
+		tmpFile := fs.NewFile(t, t.Name()+"-seedfile", fs.WithContent("0"))
+		defer tmpFile.Remove()
+
+		envVars := osEnviron()
+		envVars["TEST_SEEDFILE"] = tmpFile.Path()
+		defer env.PatchAll(t, envVars)()
+
+		flags, opts := setupFlags("gotestsum")
+		assert.NilError(t, flags.Parse(tc.args))
+		opts.args = flags.Args()
+
+		bufStdout := new(bytes.Buffer)
+		opts.stdout = bufStdout
+		bufStderr := new(bytes.Buffer)
+		opts.stderr = bufStderr
+
+		err := run(opts)
+		if tc.expectedErr {
+			assert.Error(t, err, "exit status 1")
+		} else {
+			assert.NilError(t, err)
+		}
+		golden.Assert(t, itest.RemoveSummaryTime(t, bufStdout), "e2e/expected/"+t.Name())
+		assert.Equal(t, bufStderr.String(), "")
+	}
+	var testCases = []testCase{
+		{
+			name: "reruns until success",
+			args: []string{
+				"-f=testname",
+				"--rerun-fails=4",
+				"--packages=./testdata/e2e/flaky/",
+				"--", "-tags=testdata",
+			},
+		},
+		{
+			name: "reruns continues to fail",
+			args: []string{
+				"-f=testname",
+				"--rerun-fails=2",
+				"--packages=./testdata/e2e/flaky/",
+				"--", "-tags=testdata",
+			},
+			expectedErr: true,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			fn(t, tc)
+		})
+	}
+}
+
+// osEnviron returns os.Environ() as a map, with any GOTESTSUM_ env vars removed
+// so that they do not alert the test results.
+func osEnviron() map[string]string {
+	e := env.ToMap(os.Environ())
+	for k := range e {
+		if strings.HasPrefix(k, "GOTESTSUM_") {
+			delete(e, k)
+		}
+	}
+	return e
+}

testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail

@@ -0,0 +1,64 @@
+PASS testdata/e2e/flaky.TestAlwaysPasses (0.00s)
+=== RUN   TestFailsRarely
+SEED:  0
+    TestFailsRarely: flaky_test.go:52: not this time
+--- FAIL: TestFailsRarely (0.00s)
+FAIL testdata/e2e/flaky.TestFailsRarely (0.00s)
+=== RUN   TestFailsSometimes
+SEED:  0
+    TestFailsSometimes: flaky_test.go:59: not this time
+--- FAIL: TestFailsSometimes (0.00s)
+FAIL testdata/e2e/flaky.TestFailsSometimes (0.00s)
+=== RUN   TestFailsOften
+SEED:  0
+    TestFailsOften: flaky_test.go:66: not this time
+--- FAIL: TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky.TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsRarely (0.00s)
+=== RUN   TestFailsSometimes
+SEED:  1
+    TestFailsSometimes: flaky_test.go:59: not this time
+--- FAIL: TestFailsSometimes (0.00s)
+FAIL testdata/e2e/flaky.TestFailsSometimes (0.00s)
+=== RUN   TestFailsOften
+SEED:  1
+    TestFailsOften: flaky_test.go:66: not this time
+--- FAIL: TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky.TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes (0.00s)
+=== RUN   TestFailsOften
+SEED:  2
+    TestFailsOften: flaky_test.go:66: not this time
+--- FAIL: TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky.TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky
+
+=== Failed
+=== FAIL: testdata/e2e/flaky TestFailsRarely (0.00s)
+SEED:  0
+    TestFailsRarely: flaky_test.go:52: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes (0.00s)
+SEED:  0
+    TestFailsSometimes: flaky_test.go:59: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften (0.00s)
+SEED:  0
+    TestFailsOften: flaky_test.go:66: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes (0.00s)
+SEED:  1
+    TestFailsSometimes: flaky_test.go:59: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften (0.00s)
+SEED:  1
+    TestFailsOften: flaky_test.go:66: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften (0.00s)
+SEED:  2
+    TestFailsOften: flaky_test.go:66: not this time
+
+
+DONE 9 tests, 6 failures

testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success

@@ -0,0 +1,66 @@
+PASS testdata/e2e/flaky.TestAlwaysPasses (0.00s)
+=== RUN   TestFailsRarely
+SEED:  0
+    TestFailsRarely: flaky_test.go:52: not this time
+--- FAIL: TestFailsRarely (0.00s)
+FAIL testdata/e2e/flaky.TestFailsRarely (0.00s)
+=== RUN   TestFailsSometimes
+SEED:  0
+    TestFailsSometimes: flaky_test.go:59: not this time
+--- FAIL: TestFailsSometimes (0.00s)
+FAIL testdata/e2e/flaky.TestFailsSometimes (0.00s)
+=== RUN   TestFailsOften
+SEED:  0
+    TestFailsOften: flaky_test.go:66: not this time
+--- FAIL: TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky.TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsRarely (0.00s)
+=== RUN   TestFailsSometimes
+SEED:  1
+    TestFailsSometimes: flaky_test.go:59: not this time
+--- FAIL: TestFailsSometimes (0.00s)
+FAIL testdata/e2e/flaky.TestFailsSometimes (0.00s)
+=== RUN   TestFailsOften
+SEED:  1
+    TestFailsOften: flaky_test.go:66: not this time
+--- FAIL: TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky.TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes (0.00s)
+=== RUN   TestFailsOften
+SEED:  2
+    TestFailsOften: flaky_test.go:66: not this time
+--- FAIL: TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky.TestFailsOften (0.00s)
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsOften (0.00s)
+PASS testdata/e2e/flaky (cached)
+
+=== Failed
+=== FAIL: testdata/e2e/flaky TestFailsRarely (0.00s)
+SEED:  0
+    TestFailsRarely: flaky_test.go:52: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes (0.00s)
+SEED:  0
+    TestFailsSometimes: flaky_test.go:59: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften (0.00s)
+SEED:  0
+    TestFailsOften: flaky_test.go:66: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes (0.00s)
+SEED:  1
+    TestFailsSometimes: flaky_test.go:59: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften (0.00s)
+SEED:  1
+    TestFailsOften: flaky_test.go:66: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften (0.00s)
+SEED:  2
+    TestFailsOften: flaky_test.go:66: not this time
+
+
+DONE 10 tests, 6 failures

testdata/e2e/flaky/flaky_test.go

@@ -0,0 +1,68 @@
+// +build testdata
+
+package flaky
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"sync"
+	"testing"
+)
+
+var seed int
+var seedfile = seedFile()
+var once = new(sync.Once)
+
+func setup(t *testing.T) {
+	once.Do(func() {
+		raw, _ := ioutil.ReadFile(seedfile)
+		seed = parseSeed(raw)
+
+		err := ioutil.WriteFile(seedfile, []byte(strconv.Itoa(seed+1)), 0644)
+		if err != nil {
+			t.Fatalf("failed to write seed: %v", err)
+		}
+	})
+	fmt.Fprintln(os.Stderr, "SEED: ", seed)
+}
+
+func parseSeed(r []byte) int {
+	n, err := strconv.ParseInt(string(r), 10, 64)
+	if err != nil {
+		return 0
+	}
+	return int(n)
+}
+
+func seedFile() string {
+	if name, ok := os.LookupEnv("TEST_SEEDFILE"); ok {
+		return name
+	}
+	return "/tmp/gotestsum-flaky-seedfile"
+}
+
+func TestAlwaysPasses(t *testing.T) {
+}
+
+func TestFailsRarely(t *testing.T) {
+	setup(t)
+	if seed%10 != 1 {
+		t.Fatal("not this time")
+	}
+}
+
+func TestFailsSometimes(t *testing.T) {
+	setup(t)
+	if seed%10 != 2 {
+		t.Fatal("not this time")
+	}
+}
+
+func TestFailsOften(t *testing.T) {
+	setup(t)
+	if seed%10 != 4 {
+		t.Fatal("not this time")
+	}
+}

testjson/dotformat_test.go

@@ -1,18 +1,16 @@
 package testjson
 
 import (
-	"bufio"
 	"bytes"
-	"io"
 	"math/rand"
 	"runtime"
-	"strings"
 	"testing"
 	"testing/quick"
 	"time"
 	"unicode/utf8"
 
 	"gotest.tools/gotestsum/internal/dotwriter"
+	itest "gotest.tools/gotestsum/internal/testing"
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/assert/cmp"
 	"gotest.tools/v3/golden"
@@ -32,7 +30,7 @@ func TestScanTestOutput_WithDotsFormatter(t *testing.T) {
 	exec, err := ScanTestOutput(shim.Config(t))
 	assert.NilError(t, err)
 
-	actual := removeSummaryTime(t, out)
+	actual := itest.RemoveSummaryTime(t, out)
 	golden.Assert(t, actual, outFile("dots-format"))
 	golden.Assert(t, shim.err.String(), "dots-format.err")
 	assert.DeepEqual(t, exec, expectedExecution, cmpExecutionShallow)
@@ -45,22 +43,6 @@ func outFile(name string) string {
 	return name + ".out"
 }
 
-func removeSummaryTime(t *testing.T, r io.Reader) string {
-	t.Helper()
-	out := new(strings.Builder)
-	scan := bufio.NewScanner(r)
-	for scan.Scan() {
-		line := scan.Text()
-		if i := strings.Index(line, " in "); i > 0 {
-			out.WriteString(line[:i] + "\n")
-			continue
-		}
-		out.WriteString(line + "\n")
-	}
-	assert.NilError(t, scan.Err())
-	return out.String()
-}
-
 func TestFmtDotElapsed(t *testing.T) {
 	var testcases = []struct {
 		cached   bool