Add test.shuffle to formats

cmd/handler_test.go

@@ -10,7 +10,6 @@ import (
 
 	"gotest.tools/gotestsum/testjson"
 	"gotest.tools/v3/assert"
-	"gotest.tools/v3/env"
 	"gotest.tools/v3/fs"
 	"gotest.tools/v3/golden"
 )
@@ -28,7 +27,7 @@ func TestPostRunHook(t *testing.T) {
 		stdout:         buf,
 	}
 
-	defer env.Patch(t, "GOTESTSUM_FORMAT", "short")()
+	t.Setenv("GOTESTSUM_FORMAT", "short")
 
 	exec := newExecFromTestData(t)
 	err = postRunHook(opts, exec)

internal/junitxml/report_test.go

@@ -11,15 +11,14 @@ import (
 
 	"gotest.tools/gotestsum/testjson"
 	"gotest.tools/v3/assert"
-	"gotest.tools/v3/env"
 	"gotest.tools/v3/golden"
 )
 
 func TestWrite(t *testing.T) {
 	out := new(bytes.Buffer)
 	exec := createExecution(t)
 
-	defer env.Patch(t, "GOVERSION", "go7.7.7")()
+	t.Setenv("GOVERSION", "go7.7.7")
 	err := Write(out, exec, Config{customTimestamp: new(time.Time).Format(time.RFC3339)})
 	assert.NilError(t, err)
 	golden.Assert(t, out.String(), "junitxml-report.golden")
@@ -42,7 +41,7 @@ func readTestData(t *testing.T, stream string) io.Reader {
 
 func TestGoVersion(t *testing.T) {
 	t.Run("unknown", func(t *testing.T) {
-		defer env.Patch(t, "PATH", "/bogus")()
+		t.Setenv("PATH", "/bogus")
 		assert.Equal(t, goVersion(), "unknown")
 	})
 

testjson/execution.go

@@ -107,6 +107,9 @@ type Package struct {
 	// github.com/golang/go/issues/45508. This field may be removed in the future
 	// if the issue is fixed in Go.
 	panicked bool
+	// shuffleSeed is the seed used to shuffle the tests. The value is set when
+	// tests are run with -shuffle
+	shuffleSeed string
 }
 
 // Result returns if the package passed, failed, or was skipped because there
@@ -344,9 +347,12 @@ func (p *Package) addEvent(event TestEvent) {
 		if isCoverageOutput(event.Output) {
 			p.coverage = strings.TrimRight(event.Output, "\n")
 		}
-		if isCachedOutput(event.Output) {
+		if strings.Contains(event.Output, "\t(cached)") {
 			p.cached = true
 		}
+		if isShuffleSeedOutput(event.Output) {
+			p.shuffleSeed = strings.TrimRight(event.Output, "\n")
+		}
 		p.addOutput(0, event.Output)
 	}
 }
@@ -437,8 +443,8 @@ func isCoverageOutput(output string) bool {
 		strings.Contains(output, "% of statements"))
 }
 
-func isCachedOutput(output string) bool {
-	return strings.Contains(output, "\t(cached)")
+func isShuffleSeedOutput(output string) bool {
+	return strings.HasPrefix(output, "-test.shuffle ")
 }
 
 func isWarningNoTestsToRunOutput(output string) bool {

testjson/format.go

@@ -66,14 +66,8 @@ func testNameFormat(event TestEvent, exec *Execution) (string, error) {
 			result = colorEvent(event)("EMPTY")
 		}
 
-		var cached string
-		if pkg.cached {
-			cached = cachedMessage
-		}
-		return fmt.Sprintf("%s %s%s\n",
-			result,
-			RelativePackagePath(event.Package),
-			cached), nil
+		event.Elapsed = 0 // hide elapsed for now, for backwards compat
+		return result + " " + packageLine(event, exec), nil
 
 	case event.Action == ActionFail:
 		pkg := exec.Package(event.Package)
@@ -121,6 +115,7 @@ func isPkgFailureOutput(event TestEvent) bool {
 		!strings.HasPrefix(out, "FAIL\t"+event.Package),
 		!strings.HasPrefix(out, "ok  \t"+event.Package),
 		!strings.HasPrefix(out, "?   \t"+event.Package),
+		!isShuffleSeedOutput(out),
 	)
 }
 
@@ -133,41 +128,18 @@ func all(cond ...bool) bool {
 	return true
 }
 
-const cachedMessage = " (cached)"
-
 func pkgNameFormat(event TestEvent, exec *Execution) (string, error) {
 	if !event.PackageEvent() {
 		return "", nil
 	}
-	return shortFormatPackageEvent(event, exec)
+	return shortFormatPackageEvent(event, exec), nil
 }
 
-func shortFormatPackageEvent(event TestEvent, exec *Execution) (string, error) {
+func shortFormatPackageEvent(event TestEvent, exec *Execution) string {
 	pkg := exec.Package(event.Package)
 
-	fmtElapsed := func() string {
-		if pkg.cached {
-			return cachedMessage
-		}
-		d := elapsedDuration(event.Elapsed)
-		if d == 0 {
-			return ""
-		}
-		return fmt.Sprintf(" (%s)", d)
-	}
-	fmtCoverage := func() string {
-		if pkg.coverage == "" {
-			return ""
-		}
-		return " (" + pkg.coverage + ")"
-	}
-	fmtEvent := func(action string) (string, error) {
-		return fmt.Sprintf("%s  %s%s%s\n",
-			action,
-			RelativePackagePath(event.Package),
-			fmtElapsed(),
-			fmtCoverage(),
-		), nil
+	fmtEvent := func(action string) string {
+		return action + "  " + packageLine(event, exec)
 	}
 	withColor := colorEvent(event)
 	switch event.Action {
@@ -181,7 +153,32 @@ func shortFormatPackageEvent(event TestEvent, exec *Execution) (string, error) {
 	case ActionFail:
 		return fmtEvent(withColor("✖"))
 	}
-	return "", nil
+	return ""
+}
+
+func packageLine(event TestEvent, exec *Execution) string {
+	pkg := exec.Package(event.Package)
+
+	var buf strings.Builder
+	buf.WriteString(RelativePackagePath(event.Package))
+
+	switch {
+	case pkg.cached:
+		buf.WriteString(" (cached)")
+	case event.Elapsed != 0:
+		d := elapsedDuration(event.Elapsed)
+		buf.WriteString(fmt.Sprintf(" (%s)", d))
+	}
+
+	if pkg.coverage != "" {
+		buf.WriteString(" (" + pkg.coverage + ")")
+	}
+
+	if event.Action == ActionFail && pkg.shuffleSeed != "" {
+		buf.WriteString(" (" + pkg.shuffleSeed + ")")
+	}
+	buf.WriteString("\n")
+	return buf.String()
 }
 
 func pkgNameWithFailuresFormat(event TestEvent, exec *Execution) (string, error) {
@@ -193,7 +190,7 @@ func pkgNameWithFailuresFormat(event TestEvent, exec *Execution) (string, error)
 		}
 		return "", nil
 	}
-	return shortFormatPackageEvent(event, exec)
+	return shortFormatPackageEvent(event, exec), nil
 }
 
 func colorEvent(event TestEvent) func(format string, a ...interface{}) string {

testjson/format_test.go

@@ -180,7 +180,7 @@ func TestScanTestOutput_WithPkgNameFormat_WithCoverage(t *testing.T) {
 	assert.DeepEqual(t, exec, expectedCoverageExecution, cmpExecutionShallow)
 }
 
-func TestScanTestOutputWithStandardVerboseFormat(t *testing.T) {
+func TestScanTestOutput_WithStandardVerboseFormat(t *testing.T) {
 	defer patchPkgPathPrefix("github.com/gotestyourself/gotestyourself")()
 
 	shim := newFakeHandlerWithAdapter(standardVerboseFormat, "go-test-json")
@@ -192,7 +192,7 @@ func TestScanTestOutputWithStandardVerboseFormat(t *testing.T) {
 	assert.DeepEqual(t, exec, expectedExecution, cmpExecutionShallow)
 }
 
-func TestScanTestOutputWithStandardQuietFormat(t *testing.T) {
+func TestScanTestOutput_WithStandardQuietFormat(t *testing.T) {
 	defer patchPkgPathPrefix("github.com/gotestyourself/gotestyourself")()
 
 	shim := newFakeHandlerWithAdapter(standardQuietFormat, "go-test-json")
@@ -204,7 +204,7 @@ func TestScanTestOutputWithStandardQuietFormat(t *testing.T) {
 	assert.DeepEqual(t, exec, expectedExecution, cmpExecutionShallow)
 }
 
-func TestScanTestOutputWithStandardQuietFormat_WithCoverage(t *testing.T) {
+func TestScanTestOutput_WithStandardQuietFormat_WithCoverage(t *testing.T) {
 	defer patchPkgPathPrefix("gotest.tools")()
 
 	shim := newFakeHandlerWithAdapter(standardQuietFormat, "go-test-json-with-cover")
@@ -256,3 +256,30 @@ var expectedCoverageExecution = &Execution{
 		},
 	},
 }
+
+func TestScanTestOutput_WithStandardVerboseFormat_WithShuffle(t *testing.T) {
+	shim := newFakeHandlerWithAdapter(standardVerboseFormat, "go-test-json-with-shuffle")
+	_, err := ScanTestOutput(shim.Config(t))
+
+	assert.NilError(t, err)
+	golden.Assert(t, shim.out.String(), "standard-verbose-format-shuffle.out")
+	golden.Assert(t, shim.err.String(), "go-test.err")
+}
+
+func TestScanTestOutput_WithTestNameFormat_WithShuffle(t *testing.T) {
+	shim := newFakeHandlerWithAdapter(testNameFormat, "go-test-json-with-shuffle")
+	_, err := ScanTestOutput(shim.Config(t))
+
+	assert.NilError(t, err)
+	golden.Assert(t, shim.out.String(), "testname-format-shuffle.out")
+	golden.Assert(t, shim.err.String(), "go-test.err")
+}
+
+func TestScanTestOutput_WithPkgNameFormat_WithShuffle(t *testing.T) {
+	shim := newFakeHandlerWithAdapter(pkgNameFormat, "go-test-json-with-shuffle")
+	_, err := ScanTestOutput(shim.Config(t))
+
+	assert.NilError(t, err)
+	golden.Assert(t, shim.out.String(), "pkgname-format-shuffle.out")
+	golden.Assert(t, shim.err.String(), "go-test.err")
+}

testjson/testdata/go-test-json-with-shuffle.err

@@ -0,0 +1,2 @@
+# gotest.tools/gotestsum/testjson/internal/broken
+internal/broken/broken.go:5:21: undefined: somepackage