release-22.1: roachtest structured errors and ssh/scp retries

README.md

@@ -1,4 +1,5 @@
-<p align="center">
+<
+p align="center">
   <img src='docs/media/cockroach_db.png?raw=true' width='70%'>
 </p>
 

pkg/cmd/internal/issues/formatter_unit.go

@@ -67,20 +67,29 @@ var UnitTestFormatter = IssueFormatter{
 			r.CodeBlock("", data.CondensedMessage.Digest(50))
 		}
 
-		r.Collapsed("Help", func() {
-			if data.HelpCommand != nil {
-				data.HelpCommand(r)
+		if len(data.Parameters) != 0 {
+			params := make([]string, 0, len(data.Parameters))
+			for name := range data.Parameters {
+				params = append(params, name)
 			}
+			sort.Strings(params)
 
-			if len(data.Parameters) != 0 {
-				r.Escaped("Parameters in this failure:\n")
-				for _, p := range data.Parameters {
-					r.Escaped("\n- ")
-					r.Escaped(p)
-					r.Escaped("\n")
+			r.P(func() {
+				r.Escaped("Parameters: ")
+				separator := ""
+				for _, name := range params {
+					r.Escaped(separator)
+					r.Code(fmt.Sprintf("%s=%s", name, data.Parameters[name]))
+					separator = ", "
 				}
-			}
-		})
+			})
+		}
+
+		if data.HelpCommand != nil {
+			r.Collapsed("Help", func() {
+				data.HelpCommand(r)
+			})
+		}
 
 		if len(data.RelatedIssues) > 0 {
 			r.Collapsed("Same failure on other branches", func() {

pkg/cmd/internal/issues/issues.go

@@ -138,12 +138,32 @@ func newPoster(client *github.Client, opts *Options) *poster {
 	}
 }
 
+// parameters returns the parameters to be displayed in the failure
+// report. It adds the default parameters (currently, TAGS and
+// GOFLAGS) to the list of parameters passed by the caller.
+func (p *poster) parameters(extraParams map[string]string) map[string]string {
+	ps := map[string]string{}
+	for name, value := range extraParams {
+		ps[name] = value
+	}
+
+	if p.Tags != "" {
+		ps["TAGS"] = p.Tags
+	}
+	if p.Goflags != "" {
+		ps["GOFLAGS"] = p.Goflags
+	}
+
+	return ps
+}
+
 // Options configures the issue poster.
 type Options struct {
 	Token        string // GitHub API token
 	Org          string
 	Repo         string
 	SHA          string
+	BuildTypeID  string
 	BuildID      string
 	ServerURL    string
 	Branch       string
@@ -163,6 +183,7 @@ func DefaultOptionsFromEnv() *Options {
 		githubRepoEnv          = "GITHUB_REPO"
 		githubAPITokenEnv      = "GITHUB_API_TOKEN"
 		teamcityVCSNumberEnv   = "BUILD_VCS_NUMBER"
+		teamcityBuildTypeIDEnv = "TC_BUILDTYPE_ID"
 		teamcityBuildIDEnv     = "TC_BUILD_ID"
 		teamcityServerURLEnv   = "TC_SERVER_URL"
 		teamcityBuildBranchEnv = "TC_BUILD_BRANCH"
@@ -179,6 +200,7 @@ func DefaultOptionsFromEnv() *Options {
 		// at least it'll be obvious that something went wrong (as an
 		// issue will be posted pointing at that SHA).
 		SHA:          maybeEnv(teamcityVCSNumberEnv, "8548987813ff9e1b8a9878023d3abfc6911c16db"),
+		BuildTypeID:  maybeEnv(teamcityBuildTypeIDEnv, "BUILDTYPE_ID-not-found-in-env"),
 		BuildID:      maybeEnv(teamcityBuildIDEnv, "NOTFOUNDINENV"),
 		ServerURL:    maybeEnv(teamcityServerURLEnv, "https://server-url-not-found-in-env.com"),
 		Branch:       maybeEnv(teamcityBuildBranchEnv, "branch-not-found-in-env"),
@@ -214,8 +236,9 @@ type TemplateData struct {
 	PostRequest
 	// This is foo/bar instead of github.com/cockroachdb/cockroach/pkg/foo/bar.
 	PackageNameShort string
-	// GOFLAGS=-foo TAGS=-race etc.
-	Parameters []string
+	// Parameters includes relevant test or build parameters, such as
+	// build tags or cluster configuration
+	Parameters map[string]string
 	// The message, garnished with helpers that allow extracting the useful
 	// bots.
 	CondensedMessage CondensedMessage
@@ -246,7 +269,7 @@ func (p *poster) templateData(
 	}
 	return TemplateData{
 		PostRequest:      req,
-		Parameters:       p.parameters(),
+		Parameters:       p.parameters(req.ExtraParams),
 		CondensedMessage: CondensedMessage(req.Message),
 		Branch:           p.Branch,
 		Commit:           p.SHA,
@@ -370,39 +393,27 @@ func (p *poster) post(origCtx context.Context, formatter IssueFormatter, req Pos
 
 func (p *poster) teamcityURL(tab, fragment string) *url.URL {
 	options := url.Values{}
-	options.Add("buildId", p.BuildID)
-	options.Add("tab", tab)
+	options.Add("buildTab", tab)
 
 	u, err := url.Parse(p.ServerURL)
 	if err != nil {
 		log.Fatal(err)
 	}
 	u.Scheme = "https"
-	u.Path = "viewLog.html"
+	u.Path = fmt.Sprintf("buildConfiguration/%s/%s", p.BuildTypeID, p.BuildID)
 	u.RawQuery = options.Encode()
 	u.Fragment = fragment
 	return u
 }
 
 func (p *poster) teamcityBuildLogURL() *url.URL {
-	return p.teamcityURL("buildLog", "")
+	return p.teamcityURL("log", "")
 }
 
 func (p *poster) teamcityArtifactsURL(artifacts string) *url.URL {
 	return p.teamcityURL("artifacts", artifacts)
 }
 
-func (p *poster) parameters() []string {
-	var ps []string
-	if p.Tags != "" {
-		ps = append(ps, "TAGS="+p.Tags)
-	}
-	if p.Goflags != "" {
-		ps = append(ps, "GOFLAGS="+p.Goflags)
-	}
-	return ps
-}
-
 // A PostRequest contains the information needed to create an issue about a
 // test failure.
 type PostRequest struct {
@@ -412,6 +423,9 @@ type PostRequest struct {
 	TestName string
 	// The test output.
 	Message string
+	// ExtraParams contains the parameters to be included in a failure
+	// report, other than the defaults (git branch, test flags).
+	ExtraParams map[string]string
 	// A path to the test artifacts relative to the artifacts root. If nonempty,
 	// allows the poster formatter to construct a direct URL to this directory.
 	Artifacts string

pkg/cmd/internal/issues/issues_test.go

@@ -37,15 +37,16 @@ func TestPost(t *testing.T) {
 	)
 
 	opts := Options{
-		Token:     "intentionally-unset",
-		Org:       "cockroachdb",
-		Repo:      "cockroach",
-		SHA:       "abcd123",
-		BuildID:   "8008135",
-		ServerURL: "https://teamcity.example.com",
-		Branch:    "release-0.1",
-		Tags:      "deadlock",
-		Goflags:   "race",
+		Token:       "intentionally-unset",
+		Org:         "cockroachdb",
+		Repo:        "cockroach",
+		SHA:         "abcd123",
+		BuildTypeID: "nightly123",
+		BuildID:     "8008135",
+		ServerURL:   "https://teamcity.example.com",
+		Branch:      "release-0.1",
+		Tags:        "deadlock",
+		Goflags:     "race",
 	}
 
 	type testCase struct {
@@ -308,6 +309,7 @@ test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTe
 				MentionOnCreate: []string{"@cockroachdb/idonotexistbecausethisisatest"},
 				HelpCommand:     repro,
 				ExtraLabels:     []string{"release-blocker"},
+				ExtraParams:     map[string]string{"ROACHTEST_cloud": "gce"},
 			}
 			require.NoError(t, p.post(context.Background(), UnitTestFormatter, req))
 
@@ -349,11 +351,19 @@ func TestPostEndToEnd(t *testing.T) {
 	unset := setEnv(env)
 	defer unset()
 
+	params := map[string]string{
+		"GOFLAGS":         "-race_test",
+		"ROACHTEST_cloud": "test",
+		"ROACHTEST_cpu":   "2",
+	}
+
 	req := PostRequest{
 		PackageName: "github.com/cockroachdb/cockroach/pkg/foo/bar",
 		TestName:    "TestFooBarBaz",
 		Message:     "I'm a message",
 		ExtraLabels: []string{"release-blocker"},
+		ExtraParams: params,
+		HelpCommand: UnitTestHelpCommand(""),
 	}
 
 	require.NoError(t, Post(context.Background(), UnitTestFormatter, req))

pkg/cmd/internal/issues/render.go

@@ -72,6 +72,14 @@ func (r *Renderer) Escaped(txt string) {
 	r.printf("%s", html.EscapeString(txt))
 }
 
+// Code renders a word or phrase as code. Instead of using backticks
+// here (Markdown), we rely on HTML tags since that works even if the
+// this function is called within the context of an HTML tag (such as
+// a paragraph).
+func (r *Renderer) Code(txt string) {
+	r.HTML("code", func() { r.Escaped(txt) })
+}
+
 // CodeBlock renders a code block.
 func (r *Renderer) CodeBlock(typ string, txt string) {
 	r.nl()

pkg/cmd/internal/issues/testdata/post/failure-matching-and-related-issue.txt

@@ -5,21 +5,20 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:
 searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" -label:branch-release-0.1: [github.Issue{Number:31, Title:"boom related", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}]
 createComment owner=cockroachdb repo=cockroach issue=30:
 
-storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123):
+storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123):
 
 
 ```
 	<autogenerated>:12: storage/replicate_queue_test.go:103, condition failed to evaluate within 45s: not balanced: [10 1 10 1 8]
 ```
+<p>Parameters: <code>GOFLAGS=race</code>
+, <code>ROACHTEST_cloud=gce</code>
+, <code>TAGS=deadlock</code>
+</p>
 <details><summary>Help</summary>
 <p>
 
 See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
-Parameters in this failure:
-
-- TAGS=deadlock
-
-- GOFLAGS=race
 </p>
 </details>
 <details><summary>Same failure on other branches</summary>
@@ -34,6 +33,6 @@ Parameters in this failure:
 </sub>
 
 
-Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E
+Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+boom+related+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E
 ----
 ----

pkg/cmd/internal/issues/testdata/post/failure-matching-issue.txt

@@ -5,21 +5,20 @@ searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:
 searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" -label:branch-release-0.1: []
 createComment owner=cockroachdb repo=cockroach issue=30:
 
-storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123):
+storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123):
 
 
 ```
 	<autogenerated>:12: storage/replicate_queue_test.go:103, condition failed to evaluate within 45s: not balanced: [10 1 10 1 8]
 ```
+<p>Parameters: <code>GOFLAGS=race</code>
+, <code>ROACHTEST_cloud=gce</code>
+, <code>TAGS=deadlock</code>
+</p>
 <details><summary>Help</summary>
 <p>
 
 See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
-Parameters in this failure:
-
-- TAGS=deadlock
-
-- GOFLAGS=race
 </p>
 </details>
 <sub>
@@ -28,6 +27,6 @@ Parameters in this failure:
 </sub>
 
 
-Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E
+Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E
 ----
 ----

pkg/cmd/internal/issues/testdata/post/failure-no-issue.txt

@@ -10,21 +10,20 @@ github.IssueRequest{Labels:["O-robot" "C-test-failure" "branch-release-0.1" "rel
 
 storage: TestReplicateQueueRebalance failed
 
-storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/viewLog.html?buildId=8008135&tab=buildLog) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123):
+storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123):
 
 
 ```
 	<autogenerated>:12: storage/replicate_queue_test.go:103, condition failed to evaluate within 45s: not balanced: [10 1 10 1 8]
 ```
+<p>Parameters: <code>GOFLAGS=race</code>
+, <code>ROACHTEST_cloud=gce</code>
+, <code>TAGS=deadlock</code>
+</p>
 <details><summary>Help</summary>
 <p>
 
 See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachlabs.atlassian.net/l/c/HgfXfJgM)
-Parameters in this failure:
-
-- TAGS=deadlock
-
-- GOFLAGS=race
 </p>
 </details>
 /cc @cockroachdb/idonotexistbecausethisisatest
@@ -34,6 +33,6 @@ Parameters in this failure:
 </sub>
 
 
-Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FviewLog.html%3FbuildId%3D8008135%26tab%3DbuildLog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0AParameters+in+this+failure%3A%0A%0A-+TAGS%3Ddeadlock%0A%0A-+GOFLAGS%3Drace%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestReplicateQueueRebalance+failed
+Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgit.luolix.top%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestReplicateQueueRebalance+failed
 ----
 ----