Skip to content

Commit

Permalink
Merge pull request #134 from dnephin/rerun-summary
Browse files Browse the repository at this point in the history 
Rerun fails report
  • Loading branch information
@dnephin
dnephin committed Jun 24, 2020
2 parents 9a2c7b4 + be1a9c8 commit 14268d4
Show file tree
Hide file tree
Showing 8 changed files with 172 additions and 2 deletions.
2 changes: 1 addition & 1 deletion .golangci.yml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
linters-settings:
gocyclo:
min-complexity: 10
min-complexity: 12
goconst:
min-len: 2
min-occurrences: 4
Expand Down
6 changes: 6 additions & 0 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ func setupFlags(name string) (*pflag.FlagSet, *options) {
"do not rerun any tests if the initial run has more than this number of failures")
flags.Var((*stringSlice)(&opts.packages), "packages",
"space separated list of package to test")
flags.StringVar(&opts.rerunFailsReportFile, "rerun-fails-report", "",
"write a report to the file, of the tests that were rerun")

flags.BoolVar(&opts.debug, "debug", false, "enabled debug logging")
flags.BoolVar(&opts.version, "version", false, "show version and exit")
Expand Down Expand Up @@ -158,6 +160,7 @@ type options struct {
junitTestCaseClassnameFormat *junitFieldFormatValue
rerunFailsMaxAttempts int
rerunFailsMaxInitialFailures int
rerunFailsReportFile string
packages []string
version bool

Expand Down Expand Up @@ -223,6 +226,9 @@ func run(opts *options) error {
if err := writeJUnitFile(opts, exec); err != nil {
return err
}
if err := writeRerunFailsReport(opts, exec); err != nil {
return err
}
if err := postRunHook(opts, exec); err != nil {
return err
}
Expand Down
52 changes: 52 additions & 0 deletions rerunfails.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package main
import (
"context"
"fmt"
"os"
"sort"
"strings"

"github.com/pkg/errors"
Expand Down Expand Up @@ -129,3 +131,53 @@ func goTestRunFlagFromTestCases(tcs []string) string {
buf.WriteString(")$")
return buf.String()
}

func writeRerunFailsReport(opts *options, exec *testjson.Execution) error {
if opts.rerunFailsMaxAttempts == 0 || opts.rerunFailsReportFile == "" {
return nil
}

type testCaseCounts struct {
total int
failed int
}

names := []string{}
results := map[string]testCaseCounts{}
for _, failure := range exec.Failed() {
name := failure.Package + "." + failure.Test
if _, ok := results[name]; ok {
continue
}
names = append(names, name)

pkg := exec.Package(failure.Package)
counts := testCaseCounts{}

for _, tc := range pkg.Failed {
if tc.Test == failure.Test {
counts.total++
counts.failed++
}
}
for _, tc := range pkg.Passed {
if tc.Test == failure.Test {
counts.total++
}
}
// Skipped tests are not counted, but presumably skipped tests can not fail
results[name] = counts
}

fh, err := os.Create(opts.rerunFailsReportFile)
if err != nil {
return err
}

sort.Strings(names)
for _, name := range names {
counts := results[name]
fmt.Fprintf(fh, "%s: %d runs, %d failures\n", name, counts.total, counts.failed)
}
return nil
}
34 changes: 34 additions & 0 deletions rerunfails_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package main

import (
"bytes"
"io/ioutil"
"testing"

"gotest.tools/gotestsum/testjson"
"gotest.tools/v3/assert"
"gotest.tools/v3/fs"
"gotest.tools/v3/golden"
)

func TestWriteRerunFailsReport(t *testing.T) {
reportFile := fs.NewFile(t, t.Name())
defer reportFile.Remove()

opts := &options{
rerunFailsReportFile: reportFile.Path(),
rerunFailsMaxAttempts: 4,
}

exec, err := testjson.ScanTestOutput(testjson.ScanConfig{
Stdout: bytes.NewReader(golden.Get(t, "go-test-json-flaky-rerun.out")),
})
assert.NilError(t, err)

err = writeRerunFailsReport(opts, exec)
assert.NilError(t, err)

raw, err := ioutil.ReadFile(reportFile.Path())
assert.NilError(t, err)
golden.Assert(t, string(raw), t.Name()+"-expected")
}
3 changes: 3 additions & 0 deletions testdata/TestWriteRerunFailsReport-expected
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
gotest.tools/gotestsum/testdata/e2e/flaky.TestFailsOften: 4 runs, 3 failures
gotest.tools/gotestsum/testdata/e2e/flaky.TestFailsRarely: 2 runs, 1 failures
gotest.tools/gotestsum/testdata/e2e/flaky.TestFailsSometimes: 3 runs, 2 failures
75 changes: 75 additions & 0 deletions testdata/go-test-json-flaky-rerun.out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
{"Time":"2020-06-21T21:12:10.815884042-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses"}
{"Time":"2020-06-21T21:12:10.816009964-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses","Output":"=== RUN TestAlwaysPasses\n"}
{"Time":"2020-06-21T21:12:10.816024167-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses","Output":"--- PASS: TestAlwaysPasses (0.00s)\n"}
{"Time":"2020-06-21T21:12:10.81603019-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestAlwaysPasses","Elapsed":0}
{"Time":"2020-06-21T21:12:10.81604135-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely"}
{"Time":"2020-06-21T21:12:10.816045457-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"=== RUN TestFailsRarely\n"}
{"Time":"2020-06-21T21:12:10.816049373-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"SEED: 0\n"}
{"Time":"2020-06-21T21:12:10.816063218-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":" TestFailsRarely: flaky_test.go:51: not this time\n"}
{"Time":"2020-06-21T21:12:10.816068373-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"--- FAIL: TestFailsRarely (0.00s)\n"}
{"Time":"2020-06-21T21:12:10.816072877-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Elapsed":0}
{"Time":"2020-06-21T21:12:10.81607599-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes"}
{"Time":"2020-06-21T21:12:10.81607897-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"=== RUN TestFailsSometimes\n"}
{"Time":"2020-06-21T21:12:10.816082575-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"SEED: 0\n"}
{"Time":"2020-06-21T21:12:10.816086165-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":" TestFailsSometimes: flaky_test.go:58: not this time\n"}
{"Time":"2020-06-21T21:12:10.8160903-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"--- FAIL: TestFailsSometimes (0.00s)\n"}
{"Time":"2020-06-21T21:12:10.816094667-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Elapsed":0}
{"Time":"2020-06-21T21:12:10.816098149-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
{"Time":"2020-06-21T21:12:10.816101281-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN TestFailsOften\n"}
{"Time":"2020-06-21T21:12:10.81610482-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED: 0\n"}
{"Time":"2020-06-21T21:12:10.816108597-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":" TestFailsOften: flaky_test.go:65: not this time\n"}
{"Time":"2020-06-21T21:12:10.816112698-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- FAIL: TestFailsOften (0.00s)\n"}
{"Time":"2020-06-21T21:12:10.816116235-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
{"Time":"2020-06-21T21:12:10.816119536-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch"}
{"Time":"2020-06-21T21:12:10.816122935-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch","Output":"=== RUN TestFailsOftenDoesNotPrefixMatch\n"}
{"Time":"2020-06-21T21:12:10.816127081-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch","Output":"--- PASS: TestFailsOftenDoesNotPrefixMatch (0.00s)\n"}
{"Time":"2020-06-21T21:12:10.816131578-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOftenDoesNotPrefixMatch","Elapsed":0}
{"Time":"2020-06-21T21:12:10.816135262-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch"}
{"Time":"2020-06-21T21:12:10.816139844-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch","Output":"=== RUN TestFailsSometimesDoesNotPrefixMatch\n"}
{"Time":"2020-06-21T21:12:10.816149774-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch","Output":"--- PASS: TestFailsSometimesDoesNotPrefixMatch (0.00s)\n"}
{"Time":"2020-06-21T21:12:10.816155804-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimesDoesNotPrefixMatch","Elapsed":0}
{"Time":"2020-06-21T21:12:10.816159612-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\n"}
{"Time":"2020-06-21T21:12:10.816211999-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\tgotest.tools/gotestsum/testdata/e2e/flaky\t0.001s\n"}
{"Time":"2020-06-21T21:12:10.816224311-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0.001}
{"Time":"2020-06-21T21:12:10.985779906-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely"}
{"Time":"2020-06-21T21:12:10.985890459-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"=== RUN TestFailsRarely\n"}
{"Time":"2020-06-21T21:12:10.985902826-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"SEED: 1\n"}
{"Time":"2020-06-21T21:12:10.985911982-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Output":"--- PASS: TestFailsRarely (0.00s)\n"}
{"Time":"2020-06-21T21:12:10.985916034-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsRarely","Elapsed":0}
{"Time":"2020-06-21T21:12:10.985923087-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes"}
{"Time":"2020-06-21T21:12:10.985926869-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"=== RUN TestFailsSometimes\n"}
{"Time":"2020-06-21T21:12:10.985930857-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"SEED: 1\n"}
{"Time":"2020-06-21T21:12:10.985934726-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":" TestFailsSometimes: flaky_test.go:58: not this time\n"}
{"Time":"2020-06-21T21:12:10.985939499-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"--- FAIL: TestFailsSometimes (0.00s)\n"}
{"Time":"2020-06-21T21:12:10.985943642-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Elapsed":0}
{"Time":"2020-06-21T21:12:10.985945782-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
{"Time":"2020-06-21T21:12:10.985947733-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN TestFailsOften\n"}
{"Time":"2020-06-21T21:12:10.985949865-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED: 1\n"}
{"Time":"2020-06-21T21:12:10.985952001-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":" TestFailsOften: flaky_test.go:65: not this time\n"}
{"Time":"2020-06-21T21:12:10.985954479-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- FAIL: TestFailsOften (0.00s)\n"}
{"Time":"2020-06-21T21:12:10.985956607-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
{"Time":"2020-06-21T21:12:10.985958719-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\n"}
{"Time":"2020-06-21T21:12:10.986048472-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\tgotest.tools/gotestsum/testdata/e2e/flaky\t0.001s\n"}
{"Time":"2020-06-21T21:12:10.986068802-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0.001}
{"Time":"2020-06-21T21:12:11.147554767-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes"}
{"Time":"2020-06-21T21:12:11.147676606-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"=== RUN TestFailsSometimes\n"}
{"Time":"2020-06-21T21:12:11.14768948-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"SEED: 2\n"}
{"Time":"2020-06-21T21:12:11.147700275-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Output":"--- PASS: TestFailsSometimes (0.00s)\n"}
{"Time":"2020-06-21T21:12:11.147705201-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsSometimes","Elapsed":0}
{"Time":"2020-06-21T21:12:11.147710442-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
{"Time":"2020-06-21T21:12:11.147712827-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN TestFailsOften\n"}
{"Time":"2020-06-21T21:12:11.147715318-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED: 2\n"}
{"Time":"2020-06-21T21:12:11.147717797-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":" TestFailsOften: flaky_test.go:65: not this time\n"}
{"Time":"2020-06-21T21:12:11.147721095-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- FAIL: TestFailsOften (0.00s)\n"}
{"Time":"2020-06-21T21:12:11.147723329-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
{"Time":"2020-06-21T21:12:11.147725838-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\n"}
{"Time":"2020-06-21T21:12:11.14783256-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"FAIL\tgotest.tools/gotestsum/testdata/e2e/flaky\t0.001s\n"}
{"Time":"2020-06-21T21:12:11.147849384-04:00","Action":"fail","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0.001}
{"Time":"2020-06-21T21:12:11.226137617-04:00","Action":"run","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften"}
{"Time":"2020-06-21T21:12:11.226247164-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"=== RUN TestFailsOften\n"}
{"Time":"2020-06-21T21:12:11.226256816-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"SEED: 14\n"}
{"Time":"2020-06-21T21:12:11.226264297-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Output":"--- PASS: TestFailsOften (0.00s)\n"}
{"Time":"2020-06-21T21:12:11.226266905-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Test":"TestFailsOften","Elapsed":0}
{"Time":"2020-06-21T21:12:11.22627268-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"PASS\n"}
{"Time":"2020-06-21T21:12:11.226275849-04:00","Action":"output","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Output":"ok \tgotest.tools/gotestsum/testdata/e2e/flaky\t(cached)\n"}
{"Time":"2020-06-21T21:12:11.226279592-04:00","Action":"pass","Package":"gotest.tools/gotestsum/testdata/e2e/flaky","Elapsed":0}
1 change: 1 addition & 0 deletions testdata/gotestsum-help-text
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ Flags:
--raw-command don't prepend 'go test -json' to the 'go test' command
--rerun-fails int[=2] rerun failed tests until they all pass, or attempts exceeds maximum. Defaults to max 2 reruns when enabled.
--rerun-fails-max-failures int do not rerun any tests if the initial run has more than this number of failures (default 10)
--rerun-fails-report string write a report to the file, of the tests that were rerun
--version show version and exit

Formats:
Expand Down
1 change: 0 additions & 1 deletion testjson/execution.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,6 @@ func (e *Execution) addPackageEvent(pkg *Package, event TestEvent) {
}
}

// nolint: gocyclo
func (p *Package) addTestEvent(event TestEvent) {
tc := p.running[event.Test]
root, subTest := splitTestName(event.Test)
Expand Down

0 comments on commit 14268d4

Please sign in to comment.