Merge cockroachdb#121279

121279: rpc: expect eventual connection in remote offset unhealthy test r=arulajmani a=kvoli

`TestRemoteOffsetUnhealthy` manually connects a grid of peers to later assert on the remote offset between rpc connections. The initial connection setup can timeout under resource constraints.

Retry the initial connection when setting up the test.

Fixes: cockroachdb#121238
Fixes: cockroachdb#116898
Release note: None

Co-authored-by: Austen McClernon <austen@cockroachlabs.com>

pkg/rpc/context_test.go

@@ -1163,9 +1163,16 @@ func TestRemoteOffsetUnhealthy(t *testing.T) {
 			if i == j {
 				continue
 			}
-			if _, err := clientNodeContext.ctx.GRPCDialNode(serverNodeContext.ctx.AdvertiseAddr, serverNodeContext.ctx.NodeID.Get(), DefaultClass).Connect(ctx); err != nil {
-				t.Fatal(err)
-			}
+			testutils.SucceedsSoon(t, func() error {
+				if _, err := clientNodeContext.ctx.GRPCDialNode(
+					serverNodeContext.ctx.AdvertiseAddr,
+					serverNodeContext.ctx.NodeID.Get(),
+					DefaultClass,
+				).Connect(ctx); err != nil {
+					return err
+				}
+				return nil
+			})
 		}
 	}