From d3266fff934418fbf9660faa0174c94a40298709 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 28 Mar 2023 22:59:51 -0400 Subject: [PATCH 01/10] Add run action --- .github/workflows/ci.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..9723df17 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,17 @@ +name: ci + +on: + pull_request: + push: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: '1.20' + cache: true + - run: go build . + - run: ./gotestsum From 128d3bf37202c3a5d64991619eba2cba0999174a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 28 Mar 2023 23:29:33 -0400 Subject: [PATCH 02/10] Add github actions format --- cmd/main.go | 6 ++++- cmd/testdata/gotestsum-help-text | 2 +- testjson/execution.go | 5 ---- testjson/format.go | 42 ++++++++++++++++++++++++++------ 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 7cbbf657..329c63f9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -56,8 +56,12 @@ func setupFlags(name string) (*pflag.FlagSet, *options) { flags.Usage = func() { usage(os.Stdout, name, flags) } + defaultFormat := "pkgname" + if os.Getenv("GITHUB_ACTIONS") == "true" { + defaultFormat = "github-actions" + } flags.StringVarP(&opts.format, "format", "f", - lookEnvWithDefault("GOTESTSUM_FORMAT", "short"), + lookEnvWithDefault("GOTESTSUM_FORMAT", defaultFormat), "print format of test input") flags.BoolVar(&opts.formatOptions.HideEmptyPackages, "format-hide-empty-pkg", false, "do not print empty packages in compact formats") diff --git a/cmd/testdata/gotestsum-help-text b/cmd/testdata/gotestsum-help-text index 048054d7..fff05007 100644 --- a/cmd/testdata/gotestsum-help-text +++ b/cmd/testdata/gotestsum-help-text @@ -6,7 +6,7 @@ See https://pkg.go.dev/gotest.tools/gotestsum#section-readme for detailed docume Flags: --debug enabled debug logging - -f, --format string print format of test input (default "short") + -f, --format string print format of test input (default "pkgname") --format-hide-empty-pkg do not print empty packages in compact formats --format-hivis use high visibility characters in some formats --hide-summary summary hide sections of the summary: skipped,failed,errors,output (default none) diff --git a/testjson/execution.go b/testjson/execution.go index 517b7ec5..99d69295 100644 --- a/testjson/execution.go +++ b/testjson/execution.go @@ -63,11 +63,6 @@ func (e TestEvent) PackageEvent() bool { return e.Test == "" } -// ElapsedFormatted returns Elapsed formatted in the go test format, ex (0.00s). -func (e TestEvent) ElapsedFormatted() string { - return fmt.Sprintf("(%.2fs)", e.Elapsed) -} - // Bytes returns the serialized JSON bytes that were parsed to create the event. func (e TestEvent) Bytes() []byte { return e.raw diff --git a/testjson/format.go b/testjson/format.go index 180086e2..c1962e53 100644 --- a/testjson/format.go +++ b/testjson/format.go @@ -73,18 +73,22 @@ func standardJSONFormat(out io.Writer) EventFormatter { }) } +func testNameFormatTestEvent(out io.Writer, event TestEvent) { + pkgPath := RelativePackagePath(event.Package) + + fmt.Fprintf(out, "%s %s%s %s\n", + colorEvent(event)(strings.ToUpper(string(event.Action))), + joinPkgToTestName(pkgPath, event.Test), + formatRunID(event.RunID), + fmt.Sprintf("(%.2fs)", event.Elapsed)) +} + func testNameFormat(out io.Writer) EventFormatter { buf := bufio.NewWriter(out) // nolint:errcheck return eventFormatterFunc(func(event TestEvent, exec *Execution) error { formatTest := func() error { - pkgPath := RelativePackagePath(event.Package) - - fmt.Fprintf(buf, "%s %s%s %s\n", - colorEvent(event)(strings.ToUpper(string(event.Action))), - joinPkgToTestName(pkgPath, event.Test), - formatRunID(event.RunID), - event.ElapsedFormatted()) + testNameFormatTestEvent(buf, event) return buf.Flush() } @@ -312,7 +316,31 @@ func NewEventFormatter(out io.Writer, format string, formatOpts FormatOptions) E return pkgNameFormat(out, formatOpts) case "pkgname-and-test-fails", "short-with-failures": return pkgNameWithFailuresFormat(out, formatOpts) + case "github-actions", "github-action": + return githubActionsFormat(out) default: return nil } } + +func githubActionsFormat(out io.Writer) eventFormatterFunc { + buf := bufio.NewWriter(out) + return func(event TestEvent, exec *Execution) error { + if event.Test != "" && event.Action.IsTerminal() { + buf.WriteString("::group::") + testNameFormatTestEvent(buf, event) + + pkg := exec.Package(event.Package) + tc := pkg.LastFailedByName(event.Test) + buf.WriteString(strings.TrimRight(pkg.Output(tc.ID), "\n")) + + buf.WriteString("::endgroup::\n") + return buf.Flush() + } + + if err := testNameFormat(buf).Format(event, exec); err != nil { + return err + } + return buf.Flush() + } +} From 6989927bc9cafe43acc3034da052d59c20eaa9b5 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 28 Mar 2023 23:36:43 -0400 Subject: [PATCH 03/10] Add skip output to testname --- testjson/format.go | 2 +- testjson/testdata/format/testname-coverage.out | 4 ++++ testjson/testdata/format/testname-shuffle.out | 5 +++++ testjson/testdata/format/testname.out | 5 +++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/testjson/format.go b/testjson/format.go index c1962e53..2ec76966 100644 --- a/testjson/format.go +++ b/testjson/format.go @@ -120,7 +120,7 @@ func testNameFormat(out io.Writer) EventFormatter { pkg.WriteOutputTo(buf, tc.ID) return formatTest() - case event.Action == ActionPass: + case event.Action == ActionPass || event.Action == ActionSkip: return formatTest() } return nil diff --git a/testjson/testdata/format/testname-coverage.out b/testjson/testdata/format/testname-coverage.out index 4364ae4d..0cac2b4a 100644 --- a/testjson/testdata/format/testname-coverage.out +++ b/testjson/testdata/format/testname-coverage.out @@ -3,6 +3,8 @@ FAIL gotestsum/testjson/internal/badmain PASS gotestsum/testjson/internal/good.TestPassed (0.00s) PASS gotestsum/testjson/internal/good.TestPassedWithLog (0.00s) PASS gotestsum/testjson/internal/good.TestPassedWithStdout (0.00s) +SKIP gotestsum/testjson/internal/good.TestSkipped (0.00s) +SKIP gotestsum/testjson/internal/good.TestSkippedWitLog (0.00s) PASS gotestsum/testjson/internal/good.TestWithStderr (0.00s) PASS gotestsum/testjson/internal/good.TestNestedSuccess/a/sub (0.00s) PASS gotestsum/testjson/internal/good.TestNestedSuccess/a (0.00s) @@ -21,6 +23,8 @@ PASS gotestsum/testjson/internal/good (coverage: 0.0% of statements) PASS gotestsum/testjson/internal/stub.TestPassed (0.00s) PASS gotestsum/testjson/internal/stub.TestPassedWithLog (0.00s) PASS gotestsum/testjson/internal/stub.TestPassedWithStdout (0.00s) +SKIP gotestsum/testjson/internal/stub.TestSkipped (0.00s) +SKIP gotestsum/testjson/internal/stub.TestSkippedWitLog (0.00s) === RUN TestFailed --- FAIL: TestFailed (0.00s) stub_test.go:34: this failed diff --git a/testjson/testdata/format/testname-shuffle.out b/testjson/testdata/format/testname-shuffle.out index 58a02335..8d80b36d 100644 --- a/testjson/testdata/format/testname-shuffle.out +++ b/testjson/testdata/format/testname-shuffle.out @@ -1,6 +1,7 @@ sometimes main can exit 2 FAIL testjson/internal/badmain PASS testjson/internal/good.TestPassedWithLog (0.00s) +SKIP testjson/internal/good.TestSkippedWitLog (0.00s) PASS testjson/internal/good.TestPassedWithStdout (0.00s) PASS testjson/internal/good.TestPassed (0.00s) PASS testjson/internal/good.TestNestedSuccess/a/sub (0.00s) @@ -13,6 +14,7 @@ PASS testjson/internal/good.TestNestedSuccess/d/sub (0.00s) PASS testjson/internal/good.TestNestedSuccess/d (0.00s) PASS testjson/internal/good.TestNestedSuccess (0.00s) PASS testjson/internal/good.TestWithStderr (0.00s) +SKIP testjson/internal/good.TestSkipped (0.00s) PASS testjson/internal/good.TestParallelTheSecond (0.01s) PASS testjson/internal/good.TestParallelTheFirst (0.01s) PASS testjson/internal/good.TestParallelTheThird (0.00s) @@ -68,6 +70,7 @@ FAIL testjson/internal/parallelfails.TestParallelTheFirst (0.01s) FAIL testjson/internal/parallelfails.TestParallelTheThird (0.00s) FAIL testjson/internal/parallelfails (-test.shuffle 123456) PASS testjson/internal/withfails.TestPassedWithStdout (0.00s) +SKIP testjson/internal/withfails.TestSkipped (0.00s) PASS testjson/internal/withfails.TestNestedWithFailure/a/sub (0.00s) PASS testjson/internal/withfails.TestNestedWithFailure/a (0.00s) PASS testjson/internal/withfails.TestNestedWithFailure/b/sub (0.00s) @@ -83,6 +86,7 @@ PASS testjson/internal/withfails.TestNestedWithFailure/d (0.00s) FAIL testjson/internal/withfails.TestNestedWithFailure (0.00s) PASS testjson/internal/withfails.TestWithStderr (0.00s) PASS testjson/internal/withfails.TestPassed (0.00s) +SKIP testjson/internal/withfails.TestSkippedWitLog (0.00s) PASS testjson/internal/withfails.TestNestedSuccess/a/sub (0.00s) PASS testjson/internal/withfails.TestNestedSuccess/a (0.00s) PASS testjson/internal/withfails.TestNestedSuccess/b/sub (0.00s) @@ -93,6 +97,7 @@ PASS testjson/internal/withfails.TestNestedSuccess/d/sub (0.00s) PASS testjson/internal/withfails.TestNestedSuccess/d (0.00s) PASS testjson/internal/withfails.TestNestedSuccess (0.00s) PASS testjson/internal/withfails.TestPassedWithLog (0.00s) +SKIP testjson/internal/withfails.TestTimeout (0.00s) === RUN TestFailedWithStderr this is stderr fails_test.go:43: also failed diff --git a/testjson/testdata/format/testname.out b/testjson/testdata/format/testname.out index 5eaedcad..e39dcd42 100644 --- a/testjson/testdata/format/testname.out +++ b/testjson/testdata/format/testname.out @@ -4,6 +4,8 @@ EMPTY testjson/internal/empty (cached) PASS testjson/internal/good.TestPassed (0.00s) PASS testjson/internal/good.TestPassedWithLog (0.00s) PASS testjson/internal/good.TestPassedWithStdout (0.00s) +SKIP testjson/internal/good.TestSkipped (0.00s) +SKIP testjson/internal/good.TestSkippedWitLog (0.00s) PASS testjson/internal/good.TestWithStderr (0.00s) PASS testjson/internal/good.TestNestedSuccess/a/sub (0.00s) PASS testjson/internal/good.TestNestedSuccess/a (0.00s) @@ -71,6 +73,8 @@ FAIL testjson/internal/parallelfails PASS testjson/internal/withfails.TestPassed (0.00s) PASS testjson/internal/withfails.TestPassedWithLog (0.00s) PASS testjson/internal/withfails.TestPassedWithStdout (0.00s) +SKIP testjson/internal/withfails.TestSkipped (0.00s) +SKIP testjson/internal/withfails.TestSkippedWitLog (0.00s) === RUN TestFailed fails_test.go:34: this failed --- FAIL: TestFailed (0.00s) @@ -103,6 +107,7 @@ PASS testjson/internal/withfails.TestNestedSuccess/c (0.00s) PASS testjson/internal/withfails.TestNestedSuccess/d/sub (0.00s) PASS testjson/internal/withfails.TestNestedSuccess/d (0.00s) PASS testjson/internal/withfails.TestNestedSuccess (0.00s) +SKIP testjson/internal/withfails.TestTimeout (0.00s) PASS testjson/internal/withfails.TestParallelTheFirst (0.01s) PASS testjson/internal/withfails.TestParallelTheThird (0.00s) PASS testjson/internal/withfails.TestParallelTheSecond (0.01s) From 3bd6d96ce0896b745a5d7b9a75cdae9dc5249d0f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 24 Aug 2023 23:07:20 -0400 Subject: [PATCH 04/10] Use local map of output So that output of passed tests is available --- testjson/execution.go | 1 - testjson/format.go | 40 ++++++++++++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/testjson/execution.go b/testjson/execution.go index 99d69295..27e041c8 100644 --- a/testjson/execution.go +++ b/testjson/execution.go @@ -194,7 +194,6 @@ func (p *Package) addOutput(id int, output string) { if strings.HasPrefix(output, "panic: ") { p.panicked = true } - // TODO: limit size of buffered test output p.output[id] = append(p.output[id], output) } diff --git a/testjson/format.go b/testjson/format.go index 2ec76966..2623717e 100644 --- a/testjson/format.go +++ b/testjson/format.go @@ -325,22 +325,50 @@ func NewEventFormatter(out io.Writer, format string, formatOpts FormatOptions) E func githubActionsFormat(out io.Writer) eventFormatterFunc { buf := bufio.NewWriter(out) + + type name struct { + Package string + Test string + } + output := map[name][]string{} + return func(event TestEvent, exec *Execution) error { + key := name{Package: event.Package, Test: event.Test} + // test case output + if event.Test != "" && event.Action == ActionOutput { + output[key] = append(output[key], event.Output) + return nil + } + + // test case end event if event.Test != "" && event.Action.IsTerminal() { buf.WriteString("::group::") testNameFormatTestEvent(buf, event) - pkg := exec.Package(event.Package) - tc := pkg.LastFailedByName(event.Test) - buf.WriteString(strings.TrimRight(pkg.Output(tc.ID), "\n")) - + for _, item := range output[key] { + buf.WriteString(item) + } buf.WriteString("::endgroup::\n") + delete(output, key) return buf.Flush() } - if err := testNameFormat(buf).Format(event, exec); err != nil { - return err + // package event + if !event.Action.IsTerminal() { + return nil } + + result := colorEvent(event)(strings.ToUpper(string(event.Action))) + pkg := exec.Package(event.Package) + if event.Action == ActionSkip || (event.Action == ActionPass && pkg.Total == 0) { + event.Action = ActionSkip + result = colorEvent(event)("EMPTY") + } + + buf.WriteString("\n ") + buf.WriteString(result) + buf.WriteRune(' ') + buf.WriteString(packageLine(event, exec.Package(event.Package))) return buf.Flush() } } From 8f74f08272208e28906ae29d15b78cb8ce376772 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 24 Aug 2023 23:15:01 -0400 Subject: [PATCH 05/10] Fix color of empty packages with test files --- testjson/format.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testjson/format.go b/testjson/format.go index 2623717e..a28a31b6 100644 --- a/testjson/format.go +++ b/testjson/format.go @@ -105,6 +105,7 @@ func testNameFormat(out io.Writer) EventFormatter { result := colorEvent(event)(strings.ToUpper(string(event.Action))) pkg := exec.Package(event.Package) if event.Action == ActionSkip || (event.Action == ActionPass && pkg.Total == 0) { + event.Action = ActionSkip // always color these as skip actions result = colorEvent(event)("EMPTY") } @@ -361,13 +362,13 @@ func githubActionsFormat(out io.Writer) eventFormatterFunc { result := colorEvent(event)(strings.ToUpper(string(event.Action))) pkg := exec.Package(event.Package) if event.Action == ActionSkip || (event.Action == ActionPass && pkg.Total == 0) { - event.Action = ActionSkip + event.Action = ActionSkip // always color these as skip actions result = colorEvent(event)("EMPTY") } buf.WriteString("\n ") buf.WriteString(result) - buf.WriteRune(' ') + buf.WriteString(" ") buf.WriteString(packageLine(event, exec.Package(event.Package))) return buf.Flush() } From 10f00c8411101f2fc3c296a87e85e459c638b752 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 24 Aug 2023 23:24:55 -0400 Subject: [PATCH 06/10] Removing framing lines Also fix package line --- testjson/format.go | 12 ++++++++---- testjson/summary.go | 15 +++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/testjson/format.go b/testjson/format.go index a28a31b6..31ba22b9 100644 --- a/testjson/format.go +++ b/testjson/format.go @@ -335,9 +335,12 @@ func githubActionsFormat(out io.Writer) eventFormatterFunc { return func(event TestEvent, exec *Execution) error { key := name{Package: event.Package, Test: event.Test} + // test case output if event.Test != "" && event.Action == ActionOutput { - output[key] = append(output[key], event.Output) + if !isFramingLine(event.Output, event.Test) { + output[key] = append(output[key], event.Output) + } return nil } @@ -349,7 +352,7 @@ func githubActionsFormat(out io.Writer) eventFormatterFunc { for _, item := range output[key] { buf.WriteString(item) } - buf.WriteString("::endgroup::\n") + buf.WriteString("\n::endgroup::\n") delete(output, key) return buf.Flush() } @@ -366,10 +369,11 @@ func githubActionsFormat(out io.Writer) eventFormatterFunc { result = colorEvent(event)("EMPTY") } - buf.WriteString("\n ") + buf.WriteString(" ") buf.WriteString(result) - buf.WriteString(" ") + buf.WriteString(" Package ") buf.WriteString(packageLine(event, exec.Package(event.Package))) + buf.WriteString("\n") return buf.Flush() } } diff --git a/testjson/summary.go b/testjson/summary.go index 0bbbee00..fa771446 100644 --- a/testjson/summary.go +++ b/testjson/summary.go @@ -178,7 +178,7 @@ func writeTestCaseSummary(out io.Writer, execution executionSummary, conf testCa formatRunID(tc.RunID), FormatDurationAsSeconds(tc.Elapsed, 2)) for _, line := range execution.OutputLines(tc) { - if isFramingLine(line) || conf.filter(tc.Test.Name(), line) { + if isFramingLine(line, tc.Test.Name()) { continue } fmt.Fprint(out, line) @@ -192,7 +192,6 @@ func writeTestCaseSummary(out io.Writer, execution executionSummary, conf testCa type testCaseFormatConfig struct { header string prefix string - filter func(testName string, line string) bool getter func(executionSummary) []TestCase } @@ -201,9 +200,6 @@ func formatFailed() testCaseFormatConfig { return testCaseFormatConfig{ header: withColor("Failed"), prefix: withColor("FAIL"), - filter: func(testName string, line string) bool { - return strings.HasPrefix(line, "--- FAIL: "+testName+" ") - }, getter: func(execution executionSummary) []TestCase { return execution.Failed() }, @@ -215,17 +211,16 @@ func formatSkipped() testCaseFormatConfig { return testCaseFormatConfig{ header: withColor("Skipped"), prefix: withColor("SKIP"), - filter: func(testName string, line string) bool { - return strings.HasPrefix(line, "--- SKIP: "+testName+" ") - }, getter: func(execution executionSummary) []TestCase { return execution.Skipped() }, } } -func isFramingLine(line string) bool { +func isFramingLine(line string, testName string) bool { return strings.HasPrefix(line, "=== RUN Test") || strings.HasPrefix(line, "=== PAUSE Test") || - strings.HasPrefix(line, "=== CONT Test") + strings.HasPrefix(line, "=== CONT Test") || + strings.HasPrefix(line, "--- FAIL: "+testName+" ") || + strings.HasPrefix(line, "--- SKIP: "+testName+" ") } From 70249b46c267ce440d8bb493ebc074c95e32735b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 24 Aug 2023 23:29:12 -0400 Subject: [PATCH 07/10] Only use groups for test cases with output. --- .project/golangci-lint.yml | 1 - testjson/format.go | 10 ++++++++-- testjson/summary.go | 3 ++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.project/golangci-lint.yml b/.project/golangci-lint.yml index 5ef5f797..43d400f5 100644 --- a/.project/golangci-lint.yml +++ b/.project/golangci-lint.yml @@ -31,7 +31,6 @@ linters: - deadcode - depguard - errcheck - - gocognit - goconst - gofmt - goimports diff --git a/testjson/format.go b/testjson/format.go index 31ba22b9..1f82511b 100644 --- a/testjson/format.go +++ b/testjson/format.go @@ -346,13 +346,19 @@ func githubActionsFormat(out io.Writer) eventFormatterFunc { // test case end event if event.Test != "" && event.Action.IsTerminal() { - buf.WriteString("::group::") + if len(output[key]) > 0 { + buf.WriteString("::group::") + } else { + buf.WriteString(" ") + } testNameFormatTestEvent(buf, event) for _, item := range output[key] { buf.WriteString(item) } - buf.WriteString("\n::endgroup::\n") + if len(output[key]) > 0 { + buf.WriteString("\n::endgroup::\n") + } delete(output, key) return buf.Flush() } diff --git a/testjson/summary.go b/testjson/summary.go index fa771446..a06865b4 100644 --- a/testjson/summary.go +++ b/testjson/summary.go @@ -222,5 +222,6 @@ func isFramingLine(line string, testName string) bool { strings.HasPrefix(line, "=== PAUSE Test") || strings.HasPrefix(line, "=== CONT Test") || strings.HasPrefix(line, "--- FAIL: "+testName+" ") || - strings.HasPrefix(line, "--- SKIP: "+testName+" ") + strings.HasPrefix(line, "--- SKIP: "+testName+" ") || + strings.HasPrefix(line, "--- PASS: "+testName+" ") } From 62b821fd05dbe7a1a27d62661f191941fa145622 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 24 Aug 2023 23:41:19 -0400 Subject: [PATCH 08/10] Fix test failure in github actions Don't assume the default format --- cmd/main_e2e_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/main_e2e_test.go b/cmd/main_e2e_test.go index 932cfe09..a6461b8d 100644 --- a/cmd/main_e2e_test.go +++ b/cmd/main_e2e_test.go @@ -220,6 +220,7 @@ func TestE2E_MaxFails_EndTestRun(t *testing.T) { envVars["TEST_SEEDFILE"] = tmpFile.Path() env.PatchAll(t, envVars) + t.Setenv("GOTESTSUM_FORMAT", "pkgname") flags, opts := setupFlags("gotestsum") args := []string{"--max-fails=2", "--packages=./testdata/e2e/flaky/", "--", "-tags=testdata"} assert.NilError(t, flags.Parse(args)) From f814ba5302f065498fac8c02aee0008bbfc1dee3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 24 Aug 2023 23:50:21 -0400 Subject: [PATCH 09/10] Add test case for github-actions format --- testjson/format.go | 6 +- testjson/format_test.go | 5 + testjson/testdata/format/github-actions.out | 213 ++++++++++++++++++++ 3 files changed, 221 insertions(+), 3 deletions(-) create mode 100644 testjson/testdata/format/github-actions.out diff --git a/testjson/format.go b/testjson/format.go index 1f82511b..810484ba 100644 --- a/testjson/format.go +++ b/testjson/format.go @@ -324,7 +324,7 @@ func NewEventFormatter(out io.Writer, format string, formatOpts FormatOptions) E } } -func githubActionsFormat(out io.Writer) eventFormatterFunc { +func githubActionsFormat(out io.Writer) EventFormatter { buf := bufio.NewWriter(out) type name struct { @@ -333,7 +333,7 @@ func githubActionsFormat(out io.Writer) eventFormatterFunc { } output := map[name][]string{} - return func(event TestEvent, exec *Execution) error { + return eventFormatterFunc(func(event TestEvent, exec *Execution) error { key := name{Package: event.Package, Test: event.Test} // test case output @@ -381,5 +381,5 @@ func githubActionsFormat(out io.Writer) eventFormatterFunc { buf.WriteString(packageLine(event, exec.Package(event.Package))) buf.WriteString("\n") return buf.Flush() - } + }) } diff --git a/testjson/format_test.go b/testjson/format_test.go index 552ae221..b2b4605c 100644 --- a/testjson/format_test.go +++ b/testjson/format_test.go @@ -132,6 +132,11 @@ func TestFormats_DefaultGoTestJson(t *testing.T) { format: standardJSONFormat, expectedOut: "input/go-test-json.out", }, + { + name: "github-actions", + format: githubActionsFormat, + expectedOut: "format/github-actions.out", + }, } for _, tc := range testCases { diff --git a/testjson/testdata/format/github-actions.out b/testjson/testdata/format/github-actions.out new file mode 100644 index 00000000..0e6f3250 --- /dev/null +++ b/testjson/testdata/format/github-actions.out @@ -0,0 +1,213 @@ + FAIL Package testjson/internal/badmain (1ms) + + EMPTY Package testjson/internal/empty (cached) + + PASS testjson/internal/good.TestPassed (0.00s) +::group::PASS testjson/internal/good.TestPassedWithLog (0.00s) + good_test.go:15: this is a log + +::endgroup:: +::group::PASS testjson/internal/good.TestPassedWithStdout (0.00s) +this is a Print + +::endgroup:: +::group::SKIP testjson/internal/good.TestSkipped (0.00s) + good_test.go:23: + +::endgroup:: +::group::SKIP testjson/internal/good.TestSkippedWitLog (0.00s) + good_test.go:27: the skip message + +::endgroup:: +::group::PASS testjson/internal/good.TestWithStderr (0.00s) +this is stderr + +::endgroup:: +::group::PASS testjson/internal/good.TestNestedSuccess/a/sub (0.00s) + --- PASS: TestNestedSuccess/a/sub (0.00s) + +::endgroup:: +::group::PASS testjson/internal/good.TestNestedSuccess/a (0.00s) + --- PASS: TestNestedSuccess/a (0.00s) + +::endgroup:: +::group::PASS testjson/internal/good.TestNestedSuccess/b/sub (0.00s) + --- PASS: TestNestedSuccess/b/sub (0.00s) + +::endgroup:: +::group::PASS testjson/internal/good.TestNestedSuccess/b (0.00s) + --- PASS: TestNestedSuccess/b (0.00s) + +::endgroup:: +::group::PASS testjson/internal/good.TestNestedSuccess/c/sub (0.00s) + --- PASS: TestNestedSuccess/c/sub (0.00s) + +::endgroup:: +::group::PASS testjson/internal/good.TestNestedSuccess/c (0.00s) + --- PASS: TestNestedSuccess/c (0.00s) + +::endgroup:: +::group::PASS testjson/internal/good.TestNestedSuccess/d/sub (0.00s) + --- PASS: TestNestedSuccess/d/sub (0.00s) + +::endgroup:: +::group::PASS testjson/internal/good.TestNestedSuccess/d (0.00s) + --- PASS: TestNestedSuccess/d (0.00s) + +::endgroup:: + PASS testjson/internal/good.TestNestedSuccess (0.00s) + PASS testjson/internal/good.TestParallelTheFirst (0.01s) + PASS testjson/internal/good.TestParallelTheThird (0.00s) + PASS testjson/internal/good.TestParallelTheSecond (0.01s) + PASS Package testjson/internal/good (cached) + + PASS testjson/internal/parallelfails.TestPassed (0.00s) +::group::PASS testjson/internal/parallelfails.TestPassedWithLog (0.00s) + fails_test.go:15: this is a log + +::endgroup:: +::group::PASS testjson/internal/parallelfails.TestPassedWithStdout (0.00s) +this is a Print + +::endgroup:: +::group::PASS testjson/internal/parallelfails.TestWithStderr (0.00s) +this is stderr + +::endgroup:: +::group::FAIL testjson/internal/parallelfails.TestNestedParallelFailures/a (0.00s) + fails_test.go:50: failed sub a + --- FAIL: TestNestedParallelFailures/a (0.00s) + +::endgroup:: +::group::FAIL testjson/internal/parallelfails.TestNestedParallelFailures/d (0.00s) + fails_test.go:50: failed sub d + --- FAIL: TestNestedParallelFailures/d (0.00s) + +::endgroup:: +::group::FAIL testjson/internal/parallelfails.TestNestedParallelFailures/c (0.00s) + fails_test.go:50: failed sub c + --- FAIL: TestNestedParallelFailures/c (0.00s) + +::endgroup:: +::group::FAIL testjson/internal/parallelfails.TestNestedParallelFailures/b (0.00s) + fails_test.go:50: failed sub b + --- FAIL: TestNestedParallelFailures/b (0.00s) + +::endgroup:: + FAIL testjson/internal/parallelfails.TestNestedParallelFailures (0.00s) +::group::FAIL testjson/internal/parallelfails.TestParallelTheFirst (0.01s) + fails_test.go:29: failed the first + +::endgroup:: +::group::FAIL testjson/internal/parallelfails.TestParallelTheThird (0.00s) + fails_test.go:41: failed the third + +::endgroup:: +::group::FAIL testjson/internal/parallelfails.TestParallelTheSecond (0.01s) + fails_test.go:35: failed the second + +::endgroup:: + FAIL Package testjson/internal/parallelfails (20ms) + + PASS testjson/internal/withfails.TestPassed (0.00s) +::group::PASS testjson/internal/withfails.TestPassedWithLog (0.00s) + fails_test.go:18: this is a log + +::endgroup:: +::group::PASS testjson/internal/withfails.TestPassedWithStdout (0.00s) +this is a Print + +::endgroup:: +::group::SKIP testjson/internal/withfails.TestSkipped (0.00s) + fails_test.go:26: + +::endgroup:: +::group::SKIP testjson/internal/withfails.TestSkippedWitLog (0.00s) + fails_test.go:30: the skip message + +::endgroup:: +::group::FAIL testjson/internal/withfails.TestFailed (0.00s) + fails_test.go:34: this failed + +::endgroup:: +::group::PASS testjson/internal/withfails.TestWithStderr (0.00s) +this is stderr + +::endgroup:: +::group::FAIL testjson/internal/withfails.TestFailedWithStderr (0.00s) +this is stderr + fails_test.go:43: also failed + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedWithFailure/a/sub (0.00s) + --- PASS: TestNestedWithFailure/a/sub (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedWithFailure/a (0.00s) + --- PASS: TestNestedWithFailure/a (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedWithFailure/b/sub (0.00s) + --- PASS: TestNestedWithFailure/b/sub (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedWithFailure/b (0.00s) + --- PASS: TestNestedWithFailure/b (0.00s) + +::endgroup:: +::group::FAIL testjson/internal/withfails.TestNestedWithFailure/c (0.00s) + fails_test.go:65: failed + --- FAIL: TestNestedWithFailure/c (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedWithFailure/d/sub (0.00s) + --- PASS: TestNestedWithFailure/d/sub (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedWithFailure/d (0.00s) + --- PASS: TestNestedWithFailure/d (0.00s) + +::endgroup:: + FAIL testjson/internal/withfails.TestNestedWithFailure (0.00s) +::group::PASS testjson/internal/withfails.TestNestedSuccess/a/sub (0.00s) + --- PASS: TestNestedSuccess/a/sub (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedSuccess/a (0.00s) + --- PASS: TestNestedSuccess/a (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedSuccess/b/sub (0.00s) + --- PASS: TestNestedSuccess/b/sub (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedSuccess/b (0.00s) + --- PASS: TestNestedSuccess/b (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedSuccess/c/sub (0.00s) + --- PASS: TestNestedSuccess/c/sub (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedSuccess/c (0.00s) + --- PASS: TestNestedSuccess/c (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedSuccess/d/sub (0.00s) + --- PASS: TestNestedSuccess/d/sub (0.00s) + +::endgroup:: +::group::PASS testjson/internal/withfails.TestNestedSuccess/d (0.00s) + --- PASS: TestNestedSuccess/d (0.00s) + +::endgroup:: + PASS testjson/internal/withfails.TestNestedSuccess (0.00s) +::group::SKIP testjson/internal/withfails.TestTimeout (0.00s) + timeout_test.go:13: skipping slow test + +::endgroup:: + PASS testjson/internal/withfails.TestParallelTheFirst (0.01s) + PASS testjson/internal/withfails.TestParallelTheThird (0.00s) + PASS testjson/internal/withfails.TestParallelTheSecond (0.01s) + FAIL Package testjson/internal/withfails (20ms) + From 291320919d657317c47d3956041f6a513ffe08e8 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 25 Aug 2023 23:46:31 -0400 Subject: [PATCH 10/10] Restore default format Only use github-actions format if format is set to testname. The current default pkgname is much more compact. Maybe we could add a github-actions-pkgname format in the future that folds the entire package. For now it seems safe to use github-actions instead of pkgname if the GITHUB_ACTIONS environment variable is set. If someone really wants the output expanded by default they can unset or overwrite that env var for the 'gotestsum' process. --- .github/workflows/ci.yaml | 2 +- cmd/handler_test.go | 2 ++ cmd/main.go | 8 +++----- cmd/main_e2e_test.go | 2 ++ cmd/testdata/gotestsum-help-text | 1 + testjson/format.go | 4 ++++ 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9723df17..010c666c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,4 +14,4 @@ jobs: go-version: '1.20' cache: true - run: go build . - - run: ./gotestsum + - run: ./gotestsum -f testname diff --git a/cmd/handler_test.go b/cmd/handler_test.go index 5d55f9d1..6208c271 100644 --- a/cmd/handler_test.go +++ b/cmd/handler_test.go @@ -71,6 +71,8 @@ func (bufferCloser) Close() error { return nil } func (bufferCloser) Sync() error { return nil } func TestEventHandler_Event_WithMissingActionFail(t *testing.T) { + t.Setenv("GITHUB_ACTIONS", "no") + buf := new(bufferCloser) errBuf := new(bytes.Buffer) format := testjson.NewEventFormatter(errBuf, "testname", testjson.FormatOptions{}) diff --git a/cmd/main.go b/cmd/main.go index 329c63f9..86df7675 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -56,12 +56,9 @@ func setupFlags(name string) (*pflag.FlagSet, *options) { flags.Usage = func() { usage(os.Stdout, name, flags) } - defaultFormat := "pkgname" - if os.Getenv("GITHUB_ACTIONS") == "true" { - defaultFormat = "github-actions" - } + flags.StringVarP(&opts.format, "format", "f", - lookEnvWithDefault("GOTESTSUM_FORMAT", defaultFormat), + lookEnvWithDefault("GOTESTSUM_FORMAT", "pkgname"), "print format of test input") flags.BoolVar(&opts.formatOptions.HideEmptyPackages, "format-hide-empty-pkg", false, "do not print empty packages in compact formats") @@ -143,6 +140,7 @@ Formats: pkgname print a line for each package pkgname-and-test-fails print a line for each package and failed test output testname print a line for each test and package + github-actions testname format with github actions log grouping standard-quiet standard go test format standard-verbose standard go test -v format diff --git a/cmd/main_e2e_test.go b/cmd/main_e2e_test.go index a6461b8d..4478600f 100644 --- a/cmd/main_e2e_test.go +++ b/cmd/main_e2e_test.go @@ -31,6 +31,7 @@ func TestE2E_RerunFails(t *testing.T) { if testing.Short() { t.Skip("too slow for short run") } + t.Setenv("GITHUB_ACTIONS", "no") type testCase struct { name string @@ -245,6 +246,7 @@ func TestE2E_IgnoresWarnings(t *testing.T) { if testing.Short() { t.Skip("too slow for short run") } + t.Setenv("GITHUB_ACTIONS", "no") flags, opts := setupFlags("gotestsum") args := []string{ diff --git a/cmd/testdata/gotestsum-help-text b/cmd/testdata/gotestsum-help-text index fff05007..4ff8982f 100644 --- a/cmd/testdata/gotestsum-help-text +++ b/cmd/testdata/gotestsum-help-text @@ -36,6 +36,7 @@ Formats: pkgname print a line for each package pkgname-and-test-fails print a line for each package and failed test output testname print a line for each test and package + github-actions testname format with github actions log grouping standard-quiet standard go test format standard-verbose standard go test -v format diff --git a/testjson/format.go b/testjson/format.go index 810484ba..f298eb01 100644 --- a/testjson/format.go +++ b/testjson/format.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "io" + "os" "strings" "github.com/fatih/color" @@ -312,6 +313,9 @@ func NewEventFormatter(out io.Writer, format string, formatOpts FormatOptions) E case "dots-v2": return newDotFormatter(out, formatOpts) case "testname", "short-verbose": + if os.Getenv("GITHUB_ACTIONS") == "true" { + return githubActionsFormat(out) + } return testNameFormat(out) case "pkgname", "short": return pkgNameFormat(out, formatOpts)