testutils: capture Side-Eye snapshots in SucceedsSoon

Make testutils.SucceedsSoon() capture a snapshot of the test process
with Side-Eye when the deadline expires. The idea is to help with
debugging flaky unit tests, both locally and on CI: the snapshot should
help us figure out where the operation that the test was waiting on has
gotten stuck.

The snapshot is generated if the SIDE_EYE_TOKEN environment variable is
set (it should be set to our organization's API token). The test log will
have a link to app.side-eye.io for visualizing the collected snapshot.

In the future, perhaps we can use Side-Eye to also collect execution
traces and CPU profiles for even more information.

Epic: None
Release note: None

DEPS.bzl

@@ -2405,10 +2405,10 @@ def go_deps():
         name = "com_github_dataexmachina_dev_side_eye_go",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/DataExMachina-dev/side-eye-go",
-        sha256 = "8702e7d34a166207ca2329d9780681edfb18ef6a5a9120d35fe33526d418bc4f",
-        strip_prefix = "github.com/DataExMachina-dev/side-eye-go@v0.0.0-20240528211710-5eb9c7a69e1d",
+        sha256 = "6fb443dab28dff78ee369ffbed2a5a52571d4c551a6a413ee2522a30e51ce8e6",
+        strip_prefix = "github.com/DataExMachina-dev/side-eye-go@v0.0.0-20241205202932-fd550e38f35e",
         urls = [
-            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataExMachina-dev/side-eye-go/com_github_dataexmachina_dev_side_eye_go-v0.0.0-20240528211710-5eb9c7a69e1d.zip",
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataExMachina-dev/side-eye-go/com_github_dataexmachina_dev_side_eye_go-v0.0.0-20241205202932-fd550e38f35e.zip",
         ],
     )
     go_repository(
@@ -6505,6 +6505,16 @@ def go_deps():
             "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/minio/c2goasm/com_github_minio_c2goasm-v0.0.0-20190812172519-36a3d3bbc4f3.zip",
         ],
     )
+    go_repository(
+        name = "com_github_minio_highwayhash",
+        build_file_proto_mode = "disable_global",
+        importpath = "github.com/minio/highwayhash",
+        sha256 = "3ab23da1595a6b8543edf3de80e31afacfba2b1bc9e9f4cf60c6f54ce3f66fa9",
+        strip_prefix = "github.com/minio/highwayhash@v1.0.2",
+        urls = [
+            "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/minio/highwayhash/com_github_minio_highwayhash-v1.0.2.zip",
+        ],
+    )
     go_repository(
         name = "com_github_minio_md5_simd",
         build_file_proto_mode = "disable_global",
@@ -11465,6 +11475,16 @@ def go_deps():
             "https://storage.googleapis.com/cockroach-godeps/gomod/google.golang.org/genproto/org_golang_google_genproto-v0.0.0-20230410155749-daa745c078e1.zip",
         ],
     )
+    go_repository(
+        name = "org_golang_google_genproto_googleapis_rpc",
+        build_file_proto_mode = "disable_global",
+        importpath = "google.golang.org/genproto/googleapis/rpc",
+        sha256 = "5520055455146fd452a014e10bfa069a5e132f30f8fea905431333dabb24c394",
+        strip_prefix = "google.golang.org/genproto/googleapis/rpc@v0.0.0-20240401170217-c3f982113cda",
+        urls = [
+            "https://storage.googleapis.com/cockroach-godeps/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240401170217-c3f982113cda.zip",
+        ],
+    )
     go_repository(
         name = "org_golang_google_grpc",
         build_file_proto_mode = "disable_global",

build/bazelutil/distdir_files.bzl

@@ -178,7 +178,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataDog/datadog-api-client-go/v2/com_github_datadog_datadog_api_client_go_v2-v2.15.0.zip": "1b719dab747449f279830dbb1a5920ec45ad041ea13ffde2ef7dc949c52a59f1",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataDog/datadog-go/com_github_datadog_datadog_go-v3.2.0+incompatible.zip": "ede4a024d3c106b2f57ca04d7bfc7610e0c83f4d8a3bace2cf87b42fd5cf66cd",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataDog/zstd/com_github_datadog_zstd-v1.5.6-0.20230824185856-869dae002e5e.zip": "e4924158bd1abf765a016d2c728fc367b32d20b86a268ef25743ba404c55e097",
-    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataExMachina-dev/side-eye-go/com_github_dataexmachina_dev_side_eye_go-v0.0.0-20240528211710-5eb9c7a69e1d.zip": "8702e7d34a166207ca2329d9780681edfb18ef6a5a9120d35fe33526d418bc4f",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/DataExMachina-dev/side-eye-go/com_github_dataexmachina_dev_side_eye_go-v0.0.0-20241205202932-fd550e38f35e.zip": "6fb443dab28dff78ee369ffbed2a5a52571d4c551a6a413ee2522a30e51ce8e6",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/GeertJohan/go.incremental/com_github_geertjohan_go_incremental-v1.0.0.zip": "ce46b3b717f8d2927046bcfb99c6f490b1b547a681e6b23240ac2c2292a891e8",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/GeertJohan/go.rice/com_github_geertjohan_go_rice-v1.0.0.zip": "2fc48b9422bf356c18ed3fe32ec52f6a8b87ac168f83d2eed249afaebcc3eeb8",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/GoogleCloudPlatform/cloudsql-proxy/com_github_googlecloudplatform_cloudsql_proxy-v0.0.0-20190129172621-c8b1d7a94ddf.zip": "d18ff41309efc943c71d5c8faa5b1dd792700a79fa4f61508c5e50f17fc9ca6f",
@@ -795,6 +795,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/mileusna/useragent/com_github_mileusna_useragent-v0.0.0-20190129205925-3e331f0949a5.zip": "169eabdbd206177d55bcf544ec99437d5e10cea4104f8d542aa16515202e584f",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/minio/asm2plan9s/com_github_minio_asm2plan9s-v0.0.0-20200509001527-cdd76441f9d8.zip": "39a2e28284764fd5423247d7469875046d0c8c4c2773333abf1c544197e9d946",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/minio/c2goasm/com_github_minio_c2goasm-v0.0.0-20190812172519-36a3d3bbc4f3.zip": "04367ddf0fc5cd0f293e2c4f1acefb131b572539d88b5804d92efc905eb718b5",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/minio/highwayhash/com_github_minio_highwayhash-v1.0.2.zip": "3ab23da1595a6b8543edf3de80e31afacfba2b1bc9e9f4cf60c6f54ce3f66fa9",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/minio/md5-simd/com_github_minio_md5_simd-v1.1.2.zip": "f829d35a6e6897db415af8888c4b074d1a253aee0e8fb7054b4d95477a81c3d6",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/minio/minio-go/com_github_minio_minio_go-v0.0.0-20190131015406-c8a261de75c1.zip": "329d7e50f7e20014fa563aa8ff7a789106660e4b6fed87b2ca17fe3387cecb86",
     "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/minio/minio-go/v7/com_github_minio_minio_go_v7-v7.0.21.zip": "826952231f5c7622b7c2d4b5180a4e12cf3379bd3842e1ef4934dfe115786218",
@@ -1112,6 +1113,7 @@ DISTDIR_FILES = {
     "https://storage.googleapis.com/cockroach-godeps/gomod/google.golang.org/api/org_golang_google_api-v0.114.0.zip": "42c62aaba1d76efede08c70d8aef7889c5c8ee9c9c4f1e7c455b07838cabb785",
     "https://storage.googleapis.com/cockroach-godeps/gomod/google.golang.org/appengine/org_golang_google_appengine-v1.6.7.zip": "79f80dfac18681788f1414e21a4a7734eff4cdf992070be9163103eb8d9f92cd",
     "https://storage.googleapis.com/cockroach-godeps/gomod/google.golang.org/cloud/org_golang_google_cloud-v0.0.0-20151119220103-975617b05ea8.zip": "b1d5595a11b88273665d35d4316edbd4545731c979d046c82844fafef2039c2a",
+    "https://storage.googleapis.com/cockroach-godeps/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240401170217-c3f982113cda.zip": "5520055455146fd452a014e10bfa069a5e132f30f8fea905431333dabb24c394",
     "https://storage.googleapis.com/cockroach-godeps/gomod/google.golang.org/genproto/org_golang_google_genproto-v0.0.0-20230410155749-daa745c078e1.zip": "28f0317e6948788a33c07698109005675062f0203ed06bc866350a575bc974bf",
     "https://storage.googleapis.com/cockroach-godeps/gomod/google.golang.org/grpc/cmd/protoc-gen-go-grpc/org_golang_google_grpc_cmd_protoc_gen_go_grpc-v1.1.0.zip": "13877d86cbfa30bde4d62fef2bc58dd56377dcb502c16cf78197f6934193009a",
     "https://storage.googleapis.com/cockroach-godeps/gomod/google.golang.org/grpc/examples/org_golang_google_grpc_examples-v0.0.0-20210324172016-702608ffae4d.zip": "f5cad7b05a93557c91864a02890a35c6bc5c394897222978cff2b880a78f7a11",

go.mod

@@ -96,7 +96,7 @@ require (
 	github.com/BurntSushi/toml v1.2.1
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/DataDog/datadog-api-client-go/v2 v2.15.0
-	github.com/DataExMachina-dev/side-eye-go v0.0.0-20240528211710-5eb9c7a69e1d
+	github.com/DataExMachina-dev/side-eye-go v0.0.0-20241205202932-fd550e38f35e
 	github.com/IBM/sarama v1.43.1
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/MichaelTJones/walk v0.0.0-20161122175330-4748e29d5718
@@ -387,6 +387,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
+	github.com/minio/highwayhash v1.0.2 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/minio/minio-go/v7 v7.0.21 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect

go.sum

@@ -193,8 +193,8 @@ github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/uf
 github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE=
 github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
-github.com/DataExMachina-dev/side-eye-go v0.0.0-20240528211710-5eb9c7a69e1d h1:0NRhNOBaRnkXED8mftbOSCNGAf8MZhv4zu840hIUpIc=
-github.com/DataExMachina-dev/side-eye-go v0.0.0-20240528211710-5eb9c7a69e1d/go.mod h1:FukCpc3od3BzYgxUtTWm3iB4ALtc4UknLNMQ0rq+V3A=
+github.com/DataExMachina-dev/side-eye-go v0.0.0-20241205202932-fd550e38f35e h1:NSIxncrsbk194r6JOUDNWS1Mp5kpn/TNnqvDcB5OcpI=
+github.com/DataExMachina-dev/side-eye-go v0.0.0-20241205202932-fd550e38f35e/go.mod h1:LnR2h6AbRNTnV2APqvG+CekOFC8ANLR/arskhOOuKnM=
 github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
 github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
 github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190129172621-c8b1d7a94ddf/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo=
@@ -1742,6 +1742,8 @@ github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpsp
 github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
 github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
 github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
+github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
+github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
@@ -2714,6 +2716,7 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

pkg/cmd/roachtest/test_runner.go

@@ -1864,7 +1864,7 @@ func (r *testRunner) serveHTTP(wr http.ResponseWriter, req *http.Request) {
 			}
 			sideEyeEnv := w.Cluster().sideEyeEnvName()
 			if sideEyeEnv != "" {
-				clusterBuilder.WriteString(fmt.Sprintf(" (<a href='%s'>Side-Eye</a>)", sideeyeclient.SnapshotsURL(sideEyeEnv)))
+				clusterBuilder.WriteString(fmt.Sprintf(" (<a href='%s'>Side-Eye</a>)", sideeyeclient.RecordingsURL(sideEyeEnv)))
 			}
 		}
 		t := w.Test()

pkg/testutils/BUILD.bazel

@@ -31,6 +31,7 @@ go_library(
         "//pkg/util/syncutil",
         "//pkg/util/timeutil",
         "@com_github_cockroachdb_errors//:errors",
+        "@com_github_dataexmachina_dev_side_eye_go//sideeye",
     ],
 )
 

pkg/testutils/soon.go

@@ -7,8 +7,10 @@ package testutils
 
 import (
 	"context"
+	"os"
 	"time"
 
+	"github.com/DataExMachina-dev/side-eye-go/sideeye"
 	"github.com/cockroachdb/cockroach/pkg/util"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
@@ -57,10 +59,49 @@ func SucceedsWithin(t TestFataler, fn func() error, duration time.Duration) {
 		if f, l, _, ok := errors.GetOneLineSource(err); ok {
 			err = errors.Wrapf(err, "from %s:%d", f, l)
 		}
+		maybeCaptureSideEyeSnapshot(t)
 		t.Fatalf("condition failed to evaluate within %s: %s", duration, err)
 	}
 }
 
+// maybeCaptureSideEyeSnapshot captures a Side-Eye snapshot if the
+// SIDE_EYE_TOKEN env var is set. If the snapshot is captured, the snapshot's
+// URL is logged. Snapshots are captured with a 30s timeout.
+func maybeCaptureSideEyeSnapshot(t TestFataler) {
+	// If the Side-Eye token is not set, don't do anything.
+	if os.Getenv("SIDE_EYE_TOKEN") == "" {
+		log.Infof(context.Background(), "not capturing Side-Eye snapshot for timed-out test; SIDE_EYE_TOKEN env var not set")
+		return
+	}
+
+	var name string
+	if t, ok := t.(TestNamedFatalerLogger); ok {
+		name = t.Name()
+	} else {
+		name = "unknown SucceedsWithin test"
+	}
+	ctx := context.Background()
+	snapshotCtx, cancel := context.WithTimeoutCause(
+		ctx, 30*time.Second, errors.New("timed out waiting for Side-Eye snapshot"),
+	)
+	defer cancel()
+	snapshotURL, err := sideeye.CaptureSelfSnapshot(snapshotCtx, name, sideeye.WithEnvironment("unit tests"))
+	if err != nil {
+		if errors.As(err, &sideeye.BinaryStrippedError{}) {
+			log.Infof(ctx, "failed to capture Side-Eye snapshot because the binary is stripped of debug info; "+
+				"if running with `go test` instead of bazel, use `go test -o test.out` "+
+				"for creating a non-stripped binary")
+		}
+
+		if ctx.Err() != nil {
+			err = context.Cause(ctx)
+		}
+		log.Warningf(ctx, "failed to capture Side-Eye snapshot: %s", err)
+	} else {
+		log.Infof(ctx, "captured Side-Eye snapshot: %s", snapshotURL)
+	}
+}
+
 // SucceedsWithinError returns an error unless the supplied function
 // runs without error within the given duration. The function is
 // invoked immediately at first and then successively with an