sql: trace meta2 scans made by crdb_internal.probe_ranges

With this commit, CRDB traces the meta2 scan made by
crdb_internal.probe_ranges. If the scan fails, the trace is attached to the
error returned to the SQL client. This way, outages involving a down meta2
range are observable.

According to my testing, meta2 is the only system range that can make
crdb_internal.probe_ranges unavailable. As a result, with this commit, we
should get a trace of a query to the unavailable range, regardless of which
range is unavailable, either in the SQL table output of
crdb_internal.probe_ranges or in an error string.

Release note. None.

pkg/sql/sem/builtins/generator_probe_ranges.go

@@ -134,16 +134,31 @@ func makeProbeRangeGenerator(ctx *eval.Context, args tree.Datums) (eval.ValueGen
 			"only users with the admin role are allowed to use crdb_internal.probe_range",
 		)
 	}
-	// Handle args passed in.
+	// Trace the query to meta2. Return it as part of the error string if the query fails.
+	// This improves observability into a meta2 outage. We expect crdb_internal.probe_range
+	// to be available, unless meta2 is down.
+	var ranges []kv.KeyValue
+	{
+		txn := ctx.Txn
+		ctx, sp := tracing.EnsureChildSpan(
+			ctx.Context, ctx.Tracer, "meta2scan",
+			tracing.WithForceRealSpan(),
+		)
+		sp.SetRecordingType(tracing.RecordingVerbose)
+		defer func() {
+			sp.Finish()
+		}()
+		// Handle args passed in.
+		ranges, err = kvclient.ScanMetaKVs(ctx, txn, roachpb.Span{
+			Key:    keys.MinKey,
+			EndKey: keys.MaxKey,
+		})
+		if err != nil {
+			return nil, errors.Wrapf(err, "%s", sp.FinishAndGetConfiguredRecording().String())
+		}
+	}
 	timeout := time.Duration(tree.MustBeDInterval(args[0]).Duration.Nanos())
 	isWrite := args[1].(*tree.DEnum).LogicalRep
-	ranges, err := kvclient.ScanMetaKVs(ctx.Context, ctx.Txn, roachpb.Span{
-		Key:    keys.MinKey,
-		EndKey: keys.MaxKey,
-	})
-	if err != nil {
-		return nil, err
-	}
 	return &probeRangeGenerator{
 		rangeProber: ctx.RangeProber,
 		timeout:     timeout,