Skip to content

Commit

Permalink
Add RunID to testname format, and summary
Browse files Browse the repository at this point in the history
  • Loading branch information
@dnephin
dnephin committed Jun 20, 2020
1 parent 1263c8c commit 83c209d
Show file tree
Hide file tree
Showing 10 changed files with 110 additions and 40 deletions.
1 change: 1 addition & 0 deletions rerunfails.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
}

cfg := testjson.ScanConfig{
RunID: attempts + 1,
Stdout: goTestProc.stdout,
Stderr: goTestProc.stderr,
Handler: nextRec,
Expand Down
16 changes: 8 additions & 8 deletions testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,27 +20,27 @@ FAIL testdata/e2e/flaky

DONE 6 tests, 3 failures

PASS testdata/e2e/flaky.TestFailsRarely
PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
=== RUN TestFailsSometimes
SEED: 1
--- FAIL: TestFailsSometimes
flaky_test.go:58: not this time
FAIL testdata/e2e/flaky.TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
=== RUN TestFailsOften
SEED: 1
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
FAIL testdata/e2e/flaky

DONE 9 tests, 5 failures

PASS testdata/e2e/flaky.TestFailsSometimes
PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
=== RUN TestFailsOften
SEED: 2
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
FAIL testdata/e2e/flaky

=== Failed
Expand All @@ -56,15 +56,15 @@ SEED: 0
SEED: 0
flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsSometimes
=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
SEED: 1
flaky_test.go:58: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
SEED: 1
flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
SEED: 2
flaky_test.go:65: not this time

Expand Down
16 changes: 8 additions & 8 deletions testdata/e2e/expected/TestE2E_RerunFails/reruns_continues_to_fail-go1.14
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,27 +20,27 @@ FAIL testdata/e2e/flaky

DONE 6 tests, 3 failures

PASS testdata/e2e/flaky.TestFailsRarely
PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
=== RUN TestFailsSometimes
SEED: 1
TestFailsSometimes: flaky_test.go:58: not this time
--- FAIL: TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
=== RUN TestFailsOften
SEED: 1
TestFailsOften: flaky_test.go:65: not this time
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
FAIL testdata/e2e/flaky

DONE 9 tests, 5 failures

PASS testdata/e2e/flaky.TestFailsSometimes
PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
=== RUN TestFailsOften
SEED: 2
TestFailsOften: flaky_test.go:65: not this time
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
FAIL testdata/e2e/flaky

=== Failed
Expand All @@ -56,15 +56,15 @@ SEED: 0
SEED: 0
TestFailsOften: flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsSometimes
=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
SEED: 1
TestFailsSometimes: flaky_test.go:58: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
SEED: 1
TestFailsOften: flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
SEED: 2
TestFailsOften: flaky_test.go:65: not this time

Expand Down
22 changes: 11 additions & 11 deletions testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,27 +20,27 @@ FAIL testdata/e2e/flaky

DONE 6 tests, 3 failures

PASS testdata/e2e/flaky.TestFailsRarely
PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
=== RUN TestFailsSometimes
SEED: 1
--- FAIL: TestFailsSometimes
flaky_test.go:58: not this time
FAIL testdata/e2e/flaky.TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
=== RUN TestFailsOften
SEED: 1
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
FAIL testdata/e2e/flaky

DONE 9 tests, 5 failures

PASS testdata/e2e/flaky.TestFailsSometimes
PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
=== RUN TestFailsOften
SEED: 2
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
FAIL testdata/e2e/flaky

DONE 11 tests, 6 failures
Expand All @@ -49,12 +49,12 @@ DONE 11 tests, 6 failures
SEED: 3
--- FAIL: TestFailsOften
flaky_test.go:65: not this time
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 3)
FAIL testdata/e2e/flaky

DONE 12 tests, 7 failures

PASS testdata/e2e/flaky.TestFailsOften
PASS testdata/e2e/flaky.TestFailsOften (re-run 4)
PASS testdata/e2e/flaky

=== Failed
Expand All @@ -70,19 +70,19 @@ SEED: 0
SEED: 0
flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsSometimes
=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
SEED: 1
flaky_test.go:58: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
SEED: 1
flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
SEED: 2
flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 3)
SEED: 3
flaky_test.go:65: not this time

Expand Down
22 changes: 11 additions & 11 deletions testdata/e2e/expected/TestE2E_RerunFails/reruns_until_success-go1.14
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,27 +20,27 @@ FAIL testdata/e2e/flaky

DONE 6 tests, 3 failures

PASS testdata/e2e/flaky.TestFailsRarely
PASS testdata/e2e/flaky.TestFailsRarely (re-run 1)
=== RUN TestFailsSometimes
SEED: 1
TestFailsSometimes: flaky_test.go:58: not this time
--- FAIL: TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes
FAIL testdata/e2e/flaky.TestFailsSometimes (re-run 1)
=== RUN TestFailsOften
SEED: 1
TestFailsOften: flaky_test.go:65: not this time
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 1)
FAIL testdata/e2e/flaky

DONE 9 tests, 5 failures

PASS testdata/e2e/flaky.TestFailsSometimes
PASS testdata/e2e/flaky.TestFailsSometimes (re-run 2)
=== RUN TestFailsOften
SEED: 2
TestFailsOften: flaky_test.go:65: not this time
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 2)
FAIL testdata/e2e/flaky

DONE 11 tests, 6 failures
Expand All @@ -49,12 +49,12 @@ DONE 11 tests, 6 failures
SEED: 3
TestFailsOften: flaky_test.go:65: not this time
--- FAIL: TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften
FAIL testdata/e2e/flaky.TestFailsOften (re-run 3)
FAIL testdata/e2e/flaky

DONE 12 tests, 7 failures

PASS testdata/e2e/flaky.TestFailsOften
PASS testdata/e2e/flaky.TestFailsOften (re-run 4)
PASS testdata/e2e/flaky

=== Failed
Expand All @@ -70,19 +70,19 @@ SEED: 0
SEED: 0
TestFailsOften: flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsSometimes
=== FAIL: testdata/e2e/flaky TestFailsSometimes (re-run 1)
SEED: 1
TestFailsSometimes: flaky_test.go:58: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 1)
SEED: 1
TestFailsOften: flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 2)
SEED: 2
TestFailsOften: flaky_test.go:65: not this time

=== FAIL: testdata/e2e/flaky TestFailsOften
=== FAIL: testdata/e2e/flaky TestFailsOften (re-run 3)
SEED: 3
TestFailsOften: flaky_test.go:65: not this time

Expand Down
9 changes: 9 additions & 0 deletions testjson/execution.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ type TestEvent struct {
Output string
// raw is the raw JSON bytes of the event
raw []byte
// RunID from the ScanConfig which produced this test event.
RunID int
}

// PackageEvent returns true if the event is a package start or end event
Expand Down Expand Up @@ -231,6 +233,8 @@ type TestCase struct {
Package string
Test string
Elapsed time.Duration
// RunID from the ScanConfig which produced this test case.
RunID int
// hasSubTestFailed is true when a subtest of this TestCase has failed. It is
// used to find root TestCases which have no failing subtests.
hasSubTestFailed bool
Expand Down Expand Up @@ -294,6 +298,7 @@ func (p *Package) addTestEvent(event TestEvent) {
Package: event.Package,
Test: event.Test,
ID: p.Total,
RunID: event.RunID,
}
p.running[event.Test] = tc

Expand Down Expand Up @@ -460,6 +465,9 @@ func newExecution() *Execution {

// ScanConfig used by ScanTestOutput.
type ScanConfig struct {
// RunID is a unique identifier for the run. It may be set to the pid of the
// process, or some other identifier. It will stored as the TestCase.RunID.
RunID int
// Stdout is a reader that yields the test2json output stream.
Stdout io.Reader
// Stderr is a reader that yields stderr from the 'go test' process. Often
Expand Down Expand Up @@ -533,6 +541,7 @@ func readStdout(config ScanConfig, execution *Execution) error {
return errors.Wrapf(err, "failed to parse test output: %s", string(raw))
}

event.RunID = config.RunID
execution.add(event)
if err := config.Handler.Event(event, execution); err != nil {
return err
Expand Down
11 changes: 10 additions & 1 deletion testjson/format.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,10 @@ func testNameFormat(event TestEvent, exec *Execution) (string, error) {
formatTest := func() string {
pkgPath := RelativePackagePath(event.Package)

return fmt.Sprintf("%s %s %s\n",
return fmt.Sprintf("%s %s%s %s\n",
result,
joinPkgToTestName(pkgPath, event.Test),
formatRunID(event.RunID),
event.ElapsedFormatted())
}

Expand Down Expand Up @@ -91,6 +92,14 @@ func joinPkgToTestName(pkg string, test string) string {
return pkg + "." + test
}

// formatRunID returns a formatted string of the runID.
func formatRunID(runID int) string {
if runID <= 0 {
return ""
}
return fmt.Sprintf(" (re-run %d)", runID)
}

// isPkgFailureOutput returns true if the event is package output, and the output
// doesn't match any of the expected framing messages. Events which match this
// pattern should be package-level failures (ex: exit(1) or panic in an init() or
Expand Down
3 changes: 2 additions & 1 deletion testjson/summary.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,10 +168,11 @@ func writeTestCaseSummary(out io.Writer, execution executionSummary, conf testCa
}
fmt.Fprintln(out, "\n=== "+conf.header)
for _, tc := range testCases {
fmt.Fprintf(out, "=== %s: %s %s (%s)\n",
fmt.Fprintf(out, "=== %s: %s %s%s (%s)\n",
conf.prefix,
RelativePackagePath(tc.Package),
tc.Test,
formatRunID(tc.RunID),
FormatDurationAsSeconds(tc.Elapsed, 2))
for _, line := range execution.OutputLines(tc) {
if isRunLine(line) || conf.filter(tc.Test, line) {
Expand Down
15 changes: 15 additions & 0 deletions testjson/summary_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,3 +302,18 @@ func TestPrintSummary_WithRepeatedTestCases(t *testing.T) {
PrintSummary(buf, exec, SummarizeAll)
golden.Assert(t, buf.String(), "bug-repeated-test-case-output.out")
}

func TestPrintSummary_WithRerunID(t *testing.T) {
_, reset := patchClock()
defer reset()

exec, err := ScanTestOutput(ScanConfig{
Stdout: bytes.NewReader(golden.Get(t, "go-test-json.out")),
RunID: 7,
})
assert.NilError(t, err)

buf := new(bytes.Buffer)
PrintSummary(buf, exec, SummarizeAll)
golden.Assert(t, buf.String(), "summary-with-run-id.out")
}
35 changes: 35 additions & 0 deletions testjson/testdata/summary-with-run-id.out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@

=== Skipped
=== SKIP: github.com/gotestyourself/gotestyourself/testjson/internal/good TestSkipped (re-run 7) (0.00s)
good_test.go:23:

=== SKIP: github.com/gotestyourself/gotestyourself/testjson/internal/good TestSkippedWitLog (re-run 7) (0.00s)
good_test.go:27: the skip message

=== SKIP: github.com/gotestyourself/gotestyourself/testjson/internal/stub TestSkipped (re-run 7) (0.00s)
stub_test.go:26:

=== SKIP: github.com/gotestyourself/gotestyourself/testjson/internal/stub TestSkippedWitLog (re-run 7) (0.00s)
stub_test.go:30: the skip message


=== Failed
=== FAIL: github.com/gotestyourself/gotestyourself/testjson/internal/badmain (0.00s)
sometimes main can exit 2
FAIL github.com/gotestyourself/gotestyourself/testjson/internal/badmain 0.010s

=== FAIL: github.com/gotestyourself/gotestyourself/testjson/internal/stub TestFailed (re-run 7) (0.00s)
stub_test.go:34: this failed

=== FAIL: github.com/gotestyourself/gotestyourself/testjson/internal/stub TestFailedWithStderr (re-run 7) (0.00s)
this is stderr
stub_test.go:43: also failed

=== FAIL: github.com/gotestyourself/gotestyourself/testjson/internal/stub TestNestedWithFailure/c (re-run 7) (0.00s)
--- FAIL: TestNestedWithFailure/c (0.00s)
stub_test.go:65: failed

=== FAIL: github.com/gotestyourself/gotestyourself/testjson/internal/stub TestNestedWithFailure (re-run 7) (0.00s)


DONE 46 tests, 4 skipped, 5 failures in 0.000s

0 comments on commit 83c209d

Please sign in to comment.