Add an e2e test for rerun-fails

Use -count=1 because there appears to be a bug in the 'go test' cache that has it PASS (cached) instead of failing when SEED=3.
gotestyourself · Jun 11, 2020 · 5cc60e2 · 5cc60e2
1 parent 8440739
commit 5cc60e2
Show file tree

Hide file tree

Showing 8 changed files with 493 additions and 20 deletions.
diff --git a/internal/text/testing.go b/internal/text/testing.go
@@ -0,0 +1,42 @@
+package text
+
+import (
+	"bufio"
+	"io"
+	"strings"
+	"testing"
+
+	"gotest.tools/v3/assert"
+)
+
+// ProcessLines from the Reader by passing each one to ops. The output of each
+// op is passed to the next. Returns the string created by joining all the
+// processed lines.
+func ProcessLines(t *testing.T, r io.Reader, ops ...func(string) string) string {
+	t.Helper()
+	out := new(strings.Builder)
+	scan := bufio.NewScanner(r)
+	for scan.Scan() {
+		line := scan.Text()
+		for _, op := range ops {
+			line = op(line)
+		}
+		out.WriteString(line + "\n")
+	}
+	assert.NilError(t, scan.Err())
+	return out.String()
+}
+
+func OpRemoveSummaryLineElapsedTime(line string) string {
+	if i := strings.Index(line, " in "); i > 0 {
+		return line[:i]
+	}
+	return line
+}
+
+func OpRemoveTestElapsedTime(line string) string {
+	if i := strings.Index(line, " (0.0"); i > 0 && i+8 == len(line) {
+		return line[:i]
+	}
+	return line
+}
diff --git a/main_e2e_test.go b/main_e2e_test.go
@@ -0,0 +1,102 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"runtime"
+	"strings"
+	"testing"
+
+	"gotest.tools/gotestsum/internal/text"
+	"gotest.tools/v3/assert"
+	"gotest.tools/v3/env"
+	"gotest.tools/v3/fs"
+	"gotest.tools/v3/golden"
+)
+
+func TestE2E_RerunFails(t *testing.T) {
+	type testCase struct {
+		name        string
+		args        []string
+		expectedErr bool
+	}
+	fn := func(t *testing.T, tc testCase) {
+		tmpFile := fs.NewFile(t, t.Name()+"-seedfile", fs.WithContent("0"))
+		defer tmpFile.Remove()
+
+		envVars := osEnviron()
+		envVars["TEST_SEEDFILE"] = tmpFile.Path()
+		defer env.PatchAll(t, envVars)()
+
+		flags, opts := setupFlags("gotestsum")
+		assert.NilError(t, flags.Parse(tc.args))
+		opts.args = flags.Args()
+
+		bufStdout := new(bytes.Buffer)
+		opts.stdout = bufStdout
+		bufStderr := new(bytes.Buffer)
+		opts.stderr = bufStderr
+
+		err := run(opts)
+		if tc.expectedErr {
+			assert.Error(t, err, "exit status 1")
+		} else {
+			assert.NilError(t, err)
+		}
+		out := text.ProcessLines(t, bufStdout,
+			text.OpRemoveSummaryLineElapsedTime,
+			text.OpRemoveTestElapsedTime)
+		golden.Assert(t, out, expectedFilename(t.Name()))
+		assert.Equal(t, bufStderr.String(), "")
+	}
+	var testCases = []testCase{
+		{
+			name: "reruns until success",
+			args: []string{
+				"-f=testname",
+				"--rerun-fails=4",
+				"--packages=./testdata/e2e/flaky/",
+				"--", "-count=1", "-tags=testdata",
+			},
+		},
+		{
+			name: "reruns continues to fail",
+			args: []string{
+				"-f=testname",
+				"--rerun-fails=2",
+				"--packages=./testdata/e2e/flaky/",
+				"--", "-count=1", "-tags=testdata",
+			},
+			expectedErr: true,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if testing.Short() {
+				t.Skip("too slow for short run")
+			}
+			fn(t, tc)
+		})
+	}
+}
+
+// osEnviron returns os.Environ() as a map, with any GOTESTSUM_ env vars removed
+// so that they do not alter the test results.
+func osEnviron() map[string]string {
+	e := env.ToMap(os.Environ())
+	for k := range e {
+		if strings.HasPrefix(k, "GOTESTSUM_") {
+			delete(e, k)
+		}
+	}
+	return e
+}
+
+func expectedFilename(name string) string {
+	// go1.14 changed how it prints messages from tests. It may be changed back
+	// in go1.15, so special case this version for now.
+	if strings.HasPrefix(runtime.Version(), "go1.14.") {
+		name = name + "-go1.14"
+	}
+	return "e2e/expected/" + name
+}
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
@@ -0,0 +1,64 @@
+PASS testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+--- FAIL: TestFailsRarely
+    flaky_test.go:51: not this time
+FAIL testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  0
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  1
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  1
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  2
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+
+=== Failed
+=== FAIL: testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  0
+    flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  1
+    flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  1
+    flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  2
+    flaky_test.go:65: not this time
+
+
+DONE 9 tests, 6 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14 b/testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14
@@ -0,0 +1,64 @@
+PASS testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+    TestFailsRarely: flaky_test.go:51: not this time
+--- FAIL: TestFailsRarely
+FAIL testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+    TestFailsSometimes: flaky_test.go:58: not this time
+--- FAIL: TestFailsSometimes
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  0
+    TestFailsOften: flaky_test.go:65: not this time
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  1
+    TestFailsSometimes: flaky_test.go:58: not this time
+--- FAIL: TestFailsSometimes
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  1
+    TestFailsOften: flaky_test.go:65: not this time
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  2
+    TestFailsOften: flaky_test.go:65: not this time
+--- FAIL: TestFailsOften
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+
+=== Failed
+=== FAIL: testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    TestFailsRarely: flaky_test.go:51: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    TestFailsSometimes: flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  0
+    TestFailsOften: flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  1
+    TestFailsSometimes: flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  1
+    TestFailsOften: flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  2
+    TestFailsOften: flaky_test.go:65: not this time
+
+
+DONE 9 tests, 6 failures
diff --git a/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success b/testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
@@ -0,0 +1,76 @@
+PASS testdata/e2e/flaky.TestAlwaysPasses
+=== RUN   TestFailsRarely
+SEED:  0
+--- FAIL: TestFailsRarely
+    flaky_test.go:51: not this time
+FAIL testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  0
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  0
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsRarely
+=== RUN   TestFailsSometimes
+SEED:  1
+--- FAIL: TestFailsSometimes
+    flaky_test.go:58: not this time
+FAIL testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  1
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsSometimes
+=== RUN   TestFailsOften
+SEED:  2
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+=== RUN   TestFailsOften
+SEED:  3
+--- FAIL: TestFailsOften
+    flaky_test.go:65: not this time
+FAIL testdata/e2e/flaky.TestFailsOften
+FAIL testdata/e2e/flaky
+PASS testdata/e2e/flaky.TestFailsOften
+PASS testdata/e2e/flaky
+
+=== Failed
+=== FAIL: testdata/e2e/flaky TestFailsRarely
+SEED:  0
+    flaky_test.go:51: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  0
+    flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  0
+    flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsSometimes
+SEED:  1
+    flaky_test.go:58: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  1
+    flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  2
+    flaky_test.go:65: not this time
+
+=== FAIL: testdata/e2e/flaky TestFailsOften
+SEED:  3
+    flaky_test.go:65: not this time
+
+
+DONE 11 tests, 7 failures