-
Notifications
You must be signed in to change notification settings - Fork 118
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use -count=1 because there appears to be a bug in the 'go test' cache that has it PASS (cached) instead of failing when SEED=3.
- Loading branch information
Showing
8 changed files
with
493 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
package text | ||
|
||
import ( | ||
"bufio" | ||
"io" | ||
"strings" | ||
"testing" | ||
|
||
"gotest.tools/v3/assert" | ||
) | ||
|
||
// ProcessLines from the Reader by passing each one to ops. The output of each | ||
// op is passed to the next. Returns the string created by joining all the | ||
// processed lines. | ||
func ProcessLines(t *testing.T, r io.Reader, ops ...func(string) string) string { | ||
t.Helper() | ||
out := new(strings.Builder) | ||
scan := bufio.NewScanner(r) | ||
for scan.Scan() { | ||
line := scan.Text() | ||
for _, op := range ops { | ||
line = op(line) | ||
} | ||
out.WriteString(line + "\n") | ||
} | ||
assert.NilError(t, scan.Err()) | ||
return out.String() | ||
} | ||
|
||
func OpRemoveSummaryLineElapsedTime(line string) string { | ||
if i := strings.Index(line, " in "); i > 0 { | ||
return line[:i] | ||
} | ||
return line | ||
} | ||
|
||
func OpRemoveTestElapsedTime(line string) string { | ||
if i := strings.Index(line, " (0.0"); i > 0 && i+8 == len(line) { | ||
return line[:i] | ||
} | ||
return line | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
package main | ||
|
||
import ( | ||
"bytes" | ||
"os" | ||
"runtime" | ||
"strings" | ||
"testing" | ||
|
||
"gotest.tools/gotestsum/internal/text" | ||
"gotest.tools/v3/assert" | ||
"gotest.tools/v3/env" | ||
"gotest.tools/v3/fs" | ||
"gotest.tools/v3/golden" | ||
) | ||
|
||
func TestE2E_RerunFails(t *testing.T) { | ||
type testCase struct { | ||
name string | ||
args []string | ||
expectedErr bool | ||
} | ||
fn := func(t *testing.T, tc testCase) { | ||
tmpFile := fs.NewFile(t, t.Name()+"-seedfile", fs.WithContent("0")) | ||
defer tmpFile.Remove() | ||
|
||
envVars := osEnviron() | ||
envVars["TEST_SEEDFILE"] = tmpFile.Path() | ||
defer env.PatchAll(t, envVars)() | ||
|
||
flags, opts := setupFlags("gotestsum") | ||
assert.NilError(t, flags.Parse(tc.args)) | ||
opts.args = flags.Args() | ||
|
||
bufStdout := new(bytes.Buffer) | ||
opts.stdout = bufStdout | ||
bufStderr := new(bytes.Buffer) | ||
opts.stderr = bufStderr | ||
|
||
err := run(opts) | ||
if tc.expectedErr { | ||
assert.Error(t, err, "exit status 1") | ||
} else { | ||
assert.NilError(t, err) | ||
} | ||
out := text.ProcessLines(t, bufStdout, | ||
text.OpRemoveSummaryLineElapsedTime, | ||
text.OpRemoveTestElapsedTime) | ||
golden.Assert(t, out, expectedFilename(t.Name())) | ||
assert.Equal(t, bufStderr.String(), "") | ||
} | ||
var testCases = []testCase{ | ||
{ | ||
name: "reruns until success", | ||
args: []string{ | ||
"-f=testname", | ||
"--rerun-fails=4", | ||
"--packages=./testdata/e2e/flaky/", | ||
"--", "-count=1", "-tags=testdata", | ||
}, | ||
}, | ||
{ | ||
name: "reruns continues to fail", | ||
args: []string{ | ||
"-f=testname", | ||
"--rerun-fails=2", | ||
"--packages=./testdata/e2e/flaky/", | ||
"--", "-count=1", "-tags=testdata", | ||
}, | ||
expectedErr: true, | ||
}, | ||
} | ||
for _, tc := range testCases { | ||
t.Run(tc.name, func(t *testing.T) { | ||
if testing.Short() { | ||
t.Skip("too slow for short run") | ||
} | ||
fn(t, tc) | ||
}) | ||
} | ||
} | ||
|
||
// osEnviron returns os.Environ() as a map, with any GOTESTSUM_ env vars removed | ||
// so that they do not alter the test results. | ||
func osEnviron() map[string]string { | ||
e := env.ToMap(os.Environ()) | ||
for k := range e { | ||
if strings.HasPrefix(k, "GOTESTSUM_") { | ||
delete(e, k) | ||
} | ||
} | ||
return e | ||
} | ||
|
||
func expectedFilename(name string) string { | ||
// go1.14 changed how it prints messages from tests. It may be changed back | ||
// in go1.15, so special case this version for now. | ||
if strings.HasPrefix(runtime.Version(), "go1.14.") { | ||
name = name + "-go1.14" | ||
} | ||
return "e2e/expected/" + name | ||
} |
64 changes: 64 additions & 0 deletions
64
testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
PASS testdata/e2e/flaky.TestAlwaysPasses | ||
=== RUN TestFailsRarely | ||
SEED: 0 | ||
--- FAIL: TestFailsRarely | ||
flaky_test.go:51: not this time | ||
FAIL testdata/e2e/flaky.TestFailsRarely | ||
=== RUN TestFailsSometimes | ||
SEED: 0 | ||
--- FAIL: TestFailsSometimes | ||
flaky_test.go:58: not this time | ||
FAIL testdata/e2e/flaky.TestFailsSometimes | ||
=== RUN TestFailsOften | ||
SEED: 0 | ||
--- FAIL: TestFailsOften | ||
flaky_test.go:65: not this time | ||
FAIL testdata/e2e/flaky.TestFailsOften | ||
FAIL testdata/e2e/flaky | ||
PASS testdata/e2e/flaky.TestFailsRarely | ||
=== RUN TestFailsSometimes | ||
SEED: 1 | ||
--- FAIL: TestFailsSometimes | ||
flaky_test.go:58: not this time | ||
FAIL testdata/e2e/flaky.TestFailsSometimes | ||
=== RUN TestFailsOften | ||
SEED: 1 | ||
--- FAIL: TestFailsOften | ||
flaky_test.go:65: not this time | ||
FAIL testdata/e2e/flaky.TestFailsOften | ||
FAIL testdata/e2e/flaky | ||
PASS testdata/e2e/flaky.TestFailsSometimes | ||
=== RUN TestFailsOften | ||
SEED: 2 | ||
--- FAIL: TestFailsOften | ||
flaky_test.go:65: not this time | ||
FAIL testdata/e2e/flaky.TestFailsOften | ||
FAIL testdata/e2e/flaky | ||
|
||
=== Failed | ||
=== FAIL: testdata/e2e/flaky TestFailsRarely | ||
SEED: 0 | ||
flaky_test.go:51: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsSometimes | ||
SEED: 0 | ||
flaky_test.go:58: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsOften | ||
SEED: 0 | ||
flaky_test.go:65: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsSometimes | ||
SEED: 1 | ||
flaky_test.go:58: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsOften | ||
SEED: 1 | ||
flaky_test.go:65: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsOften | ||
SEED: 2 | ||
flaky_test.go:65: not this time | ||
|
||
|
||
DONE 9 tests, 6 failures |
64 changes: 64 additions & 0 deletions
64
testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
PASS testdata/e2e/flaky.TestAlwaysPasses | ||
=== RUN TestFailsRarely | ||
SEED: 0 | ||
TestFailsRarely: flaky_test.go:51: not this time | ||
--- FAIL: TestFailsRarely | ||
FAIL testdata/e2e/flaky.TestFailsRarely | ||
=== RUN TestFailsSometimes | ||
SEED: 0 | ||
TestFailsSometimes: flaky_test.go:58: not this time | ||
--- FAIL: TestFailsSometimes | ||
FAIL testdata/e2e/flaky.TestFailsSometimes | ||
=== RUN TestFailsOften | ||
SEED: 0 | ||
TestFailsOften: flaky_test.go:65: not this time | ||
--- FAIL: TestFailsOften | ||
FAIL testdata/e2e/flaky.TestFailsOften | ||
FAIL testdata/e2e/flaky | ||
PASS testdata/e2e/flaky.TestFailsRarely | ||
=== RUN TestFailsSometimes | ||
SEED: 1 | ||
TestFailsSometimes: flaky_test.go:58: not this time | ||
--- FAIL: TestFailsSometimes | ||
FAIL testdata/e2e/flaky.TestFailsSometimes | ||
=== RUN TestFailsOften | ||
SEED: 1 | ||
TestFailsOften: flaky_test.go:65: not this time | ||
--- FAIL: TestFailsOften | ||
FAIL testdata/e2e/flaky.TestFailsOften | ||
FAIL testdata/e2e/flaky | ||
PASS testdata/e2e/flaky.TestFailsSometimes | ||
=== RUN TestFailsOften | ||
SEED: 2 | ||
TestFailsOften: flaky_test.go:65: not this time | ||
--- FAIL: TestFailsOften | ||
FAIL testdata/e2e/flaky.TestFailsOften | ||
FAIL testdata/e2e/flaky | ||
|
||
=== Failed | ||
=== FAIL: testdata/e2e/flaky TestFailsRarely | ||
SEED: 0 | ||
TestFailsRarely: flaky_test.go:51: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsSometimes | ||
SEED: 0 | ||
TestFailsSometimes: flaky_test.go:58: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsOften | ||
SEED: 0 | ||
TestFailsOften: flaky_test.go:65: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsSometimes | ||
SEED: 1 | ||
TestFailsSometimes: flaky_test.go:58: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsOften | ||
SEED: 1 | ||
TestFailsOften: flaky_test.go:65: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsOften | ||
SEED: 2 | ||
TestFailsOften: flaky_test.go:65: not this time | ||
|
||
|
||
DONE 9 tests, 6 failures |
76 changes: 76 additions & 0 deletions
76
testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
PASS testdata/e2e/flaky.TestAlwaysPasses | ||
=== RUN TestFailsRarely | ||
SEED: 0 | ||
--- FAIL: TestFailsRarely | ||
flaky_test.go:51: not this time | ||
FAIL testdata/e2e/flaky.TestFailsRarely | ||
=== RUN TestFailsSometimes | ||
SEED: 0 | ||
--- FAIL: TestFailsSometimes | ||
flaky_test.go:58: not this time | ||
FAIL testdata/e2e/flaky.TestFailsSometimes | ||
=== RUN TestFailsOften | ||
SEED: 0 | ||
--- FAIL: TestFailsOften | ||
flaky_test.go:65: not this time | ||
FAIL testdata/e2e/flaky.TestFailsOften | ||
FAIL testdata/e2e/flaky | ||
PASS testdata/e2e/flaky.TestFailsRarely | ||
=== RUN TestFailsSometimes | ||
SEED: 1 | ||
--- FAIL: TestFailsSometimes | ||
flaky_test.go:58: not this time | ||
FAIL testdata/e2e/flaky.TestFailsSometimes | ||
=== RUN TestFailsOften | ||
SEED: 1 | ||
--- FAIL: TestFailsOften | ||
flaky_test.go:65: not this time | ||
FAIL testdata/e2e/flaky.TestFailsOften | ||
FAIL testdata/e2e/flaky | ||
PASS testdata/e2e/flaky.TestFailsSometimes | ||
=== RUN TestFailsOften | ||
SEED: 2 | ||
--- FAIL: TestFailsOften | ||
flaky_test.go:65: not this time | ||
FAIL testdata/e2e/flaky.TestFailsOften | ||
FAIL testdata/e2e/flaky | ||
=== RUN TestFailsOften | ||
SEED: 3 | ||
--- FAIL: TestFailsOften | ||
flaky_test.go:65: not this time | ||
FAIL testdata/e2e/flaky.TestFailsOften | ||
FAIL testdata/e2e/flaky | ||
PASS testdata/e2e/flaky.TestFailsOften | ||
PASS testdata/e2e/flaky | ||
|
||
=== Failed | ||
=== FAIL: testdata/e2e/flaky TestFailsRarely | ||
SEED: 0 | ||
flaky_test.go:51: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsSometimes | ||
SEED: 0 | ||
flaky_test.go:58: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsOften | ||
SEED: 0 | ||
flaky_test.go:65: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsSometimes | ||
SEED: 1 | ||
flaky_test.go:58: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsOften | ||
SEED: 1 | ||
flaky_test.go:65: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsOften | ||
SEED: 2 | ||
flaky_test.go:65: not this time | ||
|
||
=== FAIL: testdata/e2e/flaky TestFailsOften | ||
SEED: 3 | ||
flaky_test.go:65: not this time | ||
|
||
|
||
DONE 11 tests, 7 failures |
Oops, something went wrong.