diff --git a/README.md b/README.md index 436c3f4c..ac70f274 100644 --- a/README.md +++ b/README.md @@ -220,11 +220,17 @@ gotestsum -- -coverprofile=cover.out ./... gotestsum --raw-command -- ./scripts/run_tests.sh ``` -Note: when using `--raw-command` you must ensure that the stdout produced by -the script only contains the `test2json` output. Any stderr produced by the script -will be considered an error (this behaviour is necessary because package build errors -are only reported by writting to stderr, not the `test2json` stdout). Any stderr -produced by tests is not considered an error (it will be in the `test2json` stdout). +Note: when using `--raw-command`, the script must follow a few rules about +stdout and stderr output: + +* The stdout produced by the script must only contain the `test2json` output, or + `gotestsum` will fail. If it isn't possible to change the script to avoid + non-JSON output, you can use `--ignore-non-json-output-lines` to ignore + non-JSON lines and write them to `gotestsum`'s stderr instead. +* Any stderr produced by the script will be considered an error (this behaviour + is necessary because package build errors are only reported by writting to + stderr, not the `test2json` stdout). Any stderr produced by tests is not + considered an error (it will be in the `test2json` stdout). **Example: using `TEST_DIRECTORY`** ``` diff --git a/cmd/main.go b/cmd/main.go index 7c97d714..8121e64e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -61,6 +61,7 @@ func setupFlags(name string) (*pflag.FlagSet, *options) { "don't prepend 'go test -json' to the 'go test' command") flags.BoolVar(&opts.ignoreNonJSONOutputLines, "ignore-non-json-output-lines", false, "write non-JSON 'go test' output lines to stderr instead of failing") + flags.Lookup("ignore-non-json-output-lines").Hidden = true flags.StringVar(&opts.jsonFile, "jsonfile", lookEnvWithDefault("GOTESTSUM_JSONFILE", ""), "write all TestEvents to file") diff --git a/cmd/testdata/gotestsum-help-text b/cmd/testdata/gotestsum-help-text index 02f43eef..d5996d38 100644 --- a/cmd/testdata/gotestsum-help-text +++ b/cmd/testdata/gotestsum-help-text @@ -6,7 +6,6 @@ Flags: --debug enabled debug logging -f, --format string print format of test input (default "short") --hide-summary summary hide sections of the summary: skipped,failed,errors,output (default none) - --ignore-non-json-output-lines write non-JSON 'go test' output lines to stderr instead of failing --jsonfile string write all TestEvents to file --junitfile string write a JUnit XML file --junitfile-testcase-classname field-format format the testcase classname field as: full, relative, short (default full) diff --git a/testjson/execution.go b/testjson/execution.go index eaea5874..8be09395 100644 --- a/testjson/execution.go +++ b/testjson/execution.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "os" "sort" "strings" "sync" diff --git a/testjson/execution_test.go b/testjson/execution_test.go index 9fc316bc..731dd107 100644 --- a/testjson/execution_test.go +++ b/testjson/execution_test.go @@ -205,8 +205,34 @@ func TestScanOutput_WithMissingEvents(t *testing.T) { assert.DeepEqual(t, expected, handler.events[start:], cmpTestEventShallow) } +func TestScanOutput_WithNonJSONLines(t *testing.T) { + source := golden.Get(t, "go-test-json-with-nonjson-stdout.out") + nonJSONLine := "|||This line is not valid test2json output.|||" + + // Test that when we ignore non-JSON lines, scanning completes, and test + // that when we don't ignore non-JSON lines, scanning fails. + for _, ignore := range []bool{true, false} { + handler := &captureHandler{} + cfg := ScanConfig{ + Stdout: bytes.NewReader(source), + Handler: handler, + IgnoreNonJSONOutputLines: ignore, + } + _, err := ScanTestOutput(cfg) + if ignore { + assert.Assert(t, len(handler.errs) == 1) + assert.Assert(t, handler.errs[0] == nonJSONLine) + assert.NilError(t, err) + } else { + assert.Assert(t, len(handler.errs) == 0) + assert.Error(t, err, "failed to parse test output: "+nonJSONLine+": invalid character '|' looking for beginning of value") + } + } +} + type captureHandler struct { events []TestEvent + errs []string } func (s *captureHandler) Event(event TestEvent, _ *Execution) error { @@ -215,5 +241,6 @@ func (s *captureHandler) Event(event TestEvent, _ *Execution) error { } func (s *captureHandler) Err(text string) error { + s.errs = append(s.errs, text) return fmt.Errorf(text) } diff --git a/testjson/testdata/go-test-json-with-nonjson-stdout.out b/testjson/testdata/go-test-json-with-nonjson-stdout.out new file mode 100644 index 00000000..0a92f654 --- /dev/null +++ b/testjson/testdata/go-test-json-with-nonjson-stdout.out @@ -0,0 +1,8 @@ +{"Time":"2021-05-12T13:53:07.462687619-05:00","Action":"run","Package":"gotest.tools/gotestsum/testjson/internal/good","Test":"TestPassed"} +|||This line is not valid test2json output.||| +{"Time":"2021-05-12T13:53:07.46279664-05:00","Action":"output","Package":"gotest.tools/gotestsum/testjson/internal/good","Test":"TestPassed","Output":"=== RUN TestPassed\n"} +{"Time":"2021-05-12T13:53:07.462812837-05:00","Action":"output","Package":"gotest.tools/gotestsum/testjson/internal/good","Test":"TestPassed","Output":"--- PASS: TestPassed (0.00s)\n"} +{"Time":"2021-05-12T13:53:07.462819251-05:00","Action":"pass","Package":"gotest.tools/gotestsum/testjson/internal/good","Test":"TestPassed","Elapsed":0} +{"Time":"2021-05-12T13:53:07.462825108-05:00","Action":"output","Package":"gotest.tools/gotestsum/testjson/internal/good","Output":"PASS\n"} +{"Time":"2021-05-12T13:53:07.462848483-05:00","Action":"output","Package":"gotest.tools/gotestsum/testjson/internal/good","Output":"ok \tgotest.tools/gotestsum/testjson/internal/good\t0.001s\n"} +{"Time":"2021-05-12T13:53:07.46309146-05:00","Action":"pass","Package":"gotest.tools/gotestsum/testjson/internal/good","Elapsed":0.001}