Merge pull request #256 from gotestyourself/show-shuffle-seed

Show shuffle seed

.golangci.yml

@@ -20,6 +20,9 @@ issues:
       text: 'result .* is always'
     - linters: [unparam]
       text: 'always receives'
+    # Remove once go1.16 is dropped
+    - linters: staticcheck
+      text: 'env.Patch is deprecated'
 
 linters:
   disable-all: true

cmd/handler_test.go

@@ -28,7 +28,7 @@ func TestPostRunHook(t *testing.T) {
 		stdout:         buf,
 	}
 
-	defer env.Patch(t, "GOTESTSUM_FORMAT", "short")()
+	env.Patch(t, "GOTESTSUM_FORMAT", "short")
 
 	exec := newExecFromTestData(t)
 	err = postRunHook(opts, exec)

go.mod

@@ -12,7 +12,7 @@ require (
 	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1
 	golang.org/x/tools v0.1.0
-	gotest.tools/v3 v3.0.3
+	gotest.tools/v3 v3.3.0
 )
 
 go 1.13

go.sum

@@ -4,7 +4,6 @@ github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
@@ -15,7 +14,6 @@ github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -25,7 +23,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -47,7 +44,6 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
@@ -56,5 +52,5 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
-gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
+gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
+gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=

internal/junitxml/report_test.go

@@ -19,7 +19,7 @@ func TestWrite(t *testing.T) {
 	out := new(bytes.Buffer)
 	exec := createExecution(t)
 
-	defer env.Patch(t, "GOVERSION", "go7.7.7")()
+	env.Patch(t, "GOVERSION", "go7.7.7")
 	err := Write(out, exec, Config{customTimestamp: new(time.Time).Format(time.RFC3339)})
 	assert.NilError(t, err)
 	golden.Assert(t, out.String(), "junitxml-report.golden")
@@ -42,7 +42,7 @@ func readTestData(t *testing.T, stream string) io.Reader {
 
 func TestGoVersion(t *testing.T) {
 	t.Run("unknown", func(t *testing.T) {
-		defer env.Patch(t, "PATH", "/bogus")()
+		env.Patch(t, "PATH", "/bogus")
 		assert.Equal(t, goVersion(), "unknown")
 	})
 

testjson/execution.go

@@ -107,6 +107,9 @@ type Package struct {
 	// github.com/golang/go/issues/45508. This field may be removed in the future
 	// if the issue is fixed in Go.
 	panicked bool
+	// shuffleSeed is the seed used to shuffle the tests. The value is set when
+	// tests are run with -shuffle
+	shuffleSeed string
 }
 
 // Result returns if the package passed, failed, or was skipped because there
@@ -344,9 +347,12 @@ func (p *Package) addEvent(event TestEvent) {
 		if isCoverageOutput(event.Output) {
 			p.coverage = strings.TrimRight(event.Output, "\n")
 		}
-		if isCachedOutput(event.Output) {
+		if strings.Contains(event.Output, "\t(cached)") {
 			p.cached = true
 		}
+		if isShuffleSeedOutput(event.Output) {
+			p.shuffleSeed = strings.TrimRight(event.Output, "\n")
+		}
 		p.addOutput(0, event.Output)
 	}
 }
@@ -437,8 +443,8 @@ func isCoverageOutput(output string) bool {
 		strings.Contains(output, "% of statements"))
 }
 
-func isCachedOutput(output string) bool {
-	return strings.Contains(output, "\t(cached)")
+func isShuffleSeedOutput(output string) bool {
+	return strings.HasPrefix(output, "-test.shuffle ")
 }
 
 func isWarningNoTestsToRunOutput(output string) bool {

testjson/format.go

@@ -66,14 +66,8 @@ func testNameFormat(event TestEvent, exec *Execution) (string, error) {
 			result = colorEvent(event)("EMPTY")
 		}
 
-		var cached string
-		if pkg.cached {
-			cached = cachedMessage
-		}
-		return fmt.Sprintf("%s %s%s\n",
-			result,
-			RelativePackagePath(event.Package),
-			cached), nil
+		event.Elapsed = 0 // hide elapsed for now, for backwards compat
+		return result + " " + packageLine(event, exec), nil
 
 	case event.Action == ActionFail:
 		pkg := exec.Package(event.Package)
@@ -121,6 +115,7 @@ func isPkgFailureOutput(event TestEvent) bool {
 		!strings.HasPrefix(out, "FAIL\t"+event.Package),
 		!strings.HasPrefix(out, "ok  \t"+event.Package),
 		!strings.HasPrefix(out, "?   \t"+event.Package),
+		!isShuffleSeedOutput(out),
 	)
 }
 
@@ -133,41 +128,18 @@ func all(cond ...bool) bool {
 	return true
 }
 
-const cachedMessage = " (cached)"
-
 func pkgNameFormat(event TestEvent, exec *Execution) (string, error) {
 	if !event.PackageEvent() {
 		return "", nil
 	}
-	return shortFormatPackageEvent(event, exec)
+	return shortFormatPackageEvent(event, exec), nil
 }
 
-func shortFormatPackageEvent(event TestEvent, exec *Execution) (string, error) {
+func shortFormatPackageEvent(event TestEvent, exec *Execution) string {
 	pkg := exec.Package(event.Package)
 
-	fmtElapsed := func() string {
-		if pkg.cached {
-			return cachedMessage
-		}
-		d := elapsedDuration(event.Elapsed)
-		if d == 0 {
-			return ""
-		}
-		return fmt.Sprintf(" (%s)", d)
-	}
-	fmtCoverage := func() string {
-		if pkg.coverage == "" {
-			return ""
-		}
-		return " (" + pkg.coverage + ")"
-	}
-	fmtEvent := func(action string) (string, error) {
-		return fmt.Sprintf("%s  %s%s%s\n",
-			action,
-			RelativePackagePath(event.Package),
-			fmtElapsed(),
-			fmtCoverage(),
-		), nil
+	fmtEvent := func(action string) string {
+		return action + "  " + packageLine(event, exec)
 	}
 	withColor := colorEvent(event)
 	switch event.Action {
@@ -181,7 +153,32 @@ func shortFormatPackageEvent(event TestEvent, exec *Execution) (string, error) {
 	case ActionFail:
 		return fmtEvent(withColor("✖"))
 	}
-	return "", nil
+	return ""
+}
+
+func packageLine(event TestEvent, exec *Execution) string {
+	pkg := exec.Package(event.Package)
+
+	var buf strings.Builder
+	buf.WriteString(RelativePackagePath(event.Package))
+
+	switch {
+	case pkg.cached:
+		buf.WriteString(" (cached)")
+	case event.Elapsed != 0:
+		d := elapsedDuration(event.Elapsed)
+		buf.WriteString(fmt.Sprintf(" (%s)", d))
+	}
+
+	if pkg.coverage != "" {
+		buf.WriteString(" (" + pkg.coverage + ")")
+	}
+
+	if event.Action == ActionFail && pkg.shuffleSeed != "" {
+		buf.WriteString(" (" + pkg.shuffleSeed + ")")
+	}
+	buf.WriteString("\n")
+	return buf.String()
 }
 
 func pkgNameWithFailuresFormat(event TestEvent, exec *Execution) (string, error) {
@@ -193,7 +190,7 @@ func pkgNameWithFailuresFormat(event TestEvent, exec *Execution) (string, error)
 		}
 		return "", nil
 	}
-	return shortFormatPackageEvent(event, exec)
+	return shortFormatPackageEvent(event, exec), nil
 }
 
 func colorEvent(event TestEvent) func(format string, a ...interface{}) string {

testjson/format_test.go

@@ -180,7 +180,7 @@ func TestScanTestOutput_WithPkgNameFormat_WithCoverage(t *testing.T) {
 	assert.DeepEqual(t, exec, expectedCoverageExecution, cmpExecutionShallow)
 }
 
-func TestScanTestOutputWithStandardVerboseFormat(t *testing.T) {
+func TestScanTestOutput_WithStandardVerboseFormat(t *testing.T) {
 	defer patchPkgPathPrefix("github.com/gotestyourself/gotestyourself")()
 
 	shim := newFakeHandlerWithAdapter(standardVerboseFormat, "go-test-json")
@@ -192,7 +192,7 @@ func TestScanTestOutputWithStandardVerboseFormat(t *testing.T) {
 	assert.DeepEqual(t, exec, expectedExecution, cmpExecutionShallow)
 }
 
-func TestScanTestOutputWithStandardQuietFormat(t *testing.T) {
+func TestScanTestOutput_WithStandardQuietFormat(t *testing.T) {
 	defer patchPkgPathPrefix("github.com/gotestyourself/gotestyourself")()
 
 	shim := newFakeHandlerWithAdapter(standardQuietFormat, "go-test-json")
@@ -204,7 +204,7 @@ func TestScanTestOutputWithStandardQuietFormat(t *testing.T) {
 	assert.DeepEqual(t, exec, expectedExecution, cmpExecutionShallow)
 }
 
-func TestScanTestOutputWithStandardQuietFormat_WithCoverage(t *testing.T) {
+func TestScanTestOutput_WithStandardQuietFormat_WithCoverage(t *testing.T) {
 	defer patchPkgPathPrefix("gotest.tools")()
 
 	shim := newFakeHandlerWithAdapter(standardQuietFormat, "go-test-json-with-cover")
@@ -256,3 +256,30 @@ var expectedCoverageExecution = &Execution{
 		},
 	},
 }
+
+func TestScanTestOutput_WithStandardVerboseFormat_WithShuffle(t *testing.T) {
+	shim := newFakeHandlerWithAdapter(standardVerboseFormat, "go-test-json-with-shuffle")
+	_, err := ScanTestOutput(shim.Config(t))
+
+	assert.NilError(t, err)
+	golden.Assert(t, shim.out.String(), "standard-verbose-format-shuffle.out")
+	golden.Assert(t, shim.err.String(), "go-test.err")
+}
+
+func TestScanTestOutput_WithTestNameFormat_WithShuffle(t *testing.T) {
+	shim := newFakeHandlerWithAdapter(testNameFormat, "go-test-json-with-shuffle")
+	_, err := ScanTestOutput(shim.Config(t))
+
+	assert.NilError(t, err)
+	golden.Assert(t, shim.out.String(), "testname-format-shuffle.out")
+	golden.Assert(t, shim.err.String(), "go-test.err")
+}
+
+func TestScanTestOutput_WithPkgNameFormat_WithShuffle(t *testing.T) {
+	shim := newFakeHandlerWithAdapter(pkgNameFormat, "go-test-json-with-shuffle")
+	_, err := ScanTestOutput(shim.Config(t))
+
+	assert.NilError(t, err)
+	golden.Assert(t, shim.out.String(), "pkgname-format-shuffle.out")
+	golden.Assert(t, shim.err.String(), "go-test.err")
+}

testjson/testdata/go-test-json-with-shuffle.err

@@ -0,0 +1,2 @@
+# gotest.tools/gotestsum/testjson/internal/broken
+internal/broken/broken.go:5:21: undefined: somepackage