Merge cockroachdb#25851

25851: opt: support SHOW TRACE r=RaduBerinde a=RaduBerinde

#### sql: move explain ResultColumns to sqlbase

We are duplicating the information about the result columns returned
by EXPLAIN statements (in sql and in opt/optbuilder). This change
moves the definitions to sqlbase and uses them in both cases.

Release note: None

#### opt: support SHOW TRACE

Adding optimizer support for SHOW TRACE statements.

We use two operators, one for `SHOW TRACE FOR <query>` and one for
`SHOW TRACE FOR SESSION`. They both use the same `ShowTraceOpDef` for
the private.

We don't add non-explain execbuilder tests because statement tracing
doesn't work with the TestEngine implementation. This will be
addressed in a subsequent PR.

Release note: None


Co-authored-by: Radu Berinde <radu@cockroachlabs.com>

pkg/sql/explain_plan.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
-	"github.com/cockroachdb/cockroach/pkg/sql/sem/types"
 	"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
 	"github.com/cockroachdb/cockroach/pkg/util/treeprinter"
 )
@@ -48,34 +47,6 @@ type explainPlanNode struct {
 	run explainPlanRun
 }
 
-var explainPlanColumns = sqlbase.ResultColumns{
-	// Tree shows the node type with the tree structure.
-	{Name: "Tree", Typ: types.String},
-	// Field is the part of the node that a row of output pertains to.
-	{Name: "Field", Typ: types.String},
-	// Description contains details about the field.
-	{Name: "Description", Typ: types.String},
-}
-
-var explainPlanVerboseColumns = sqlbase.ResultColumns{
-	// Tree shows the node type with the tree structure.
-	{Name: "Tree", Typ: types.String},
-	// Level is the depth of the node in the tree. Hidden by default; can be
-	// retrieved using:
-	//   SELECT "Level" FROM [ EXPLAIN (VERBOSE) ... ].
-	{Name: "Level", Typ: types.Int, Hidden: true},
-	// Type is the node type. Hidden by default.
-	{Name: "Type", Typ: types.String, Hidden: true},
-	// Field is the part of the node that a row of output pertains to.
-	{Name: "Field", Typ: types.String},
-	// Description contains details about the field.
-	{Name: "Description", Typ: types.String},
-	// Columns is the type signature of the data source.
-	{Name: "Columns", Typ: types.String},
-	// Ordering indicates the known ordering of the data from this source.
-	{Name: "Ordering", Typ: types.String},
-}
-
 // newExplainPlanNode instantiates a planNode that runs an EXPLAIN query.
 func (p *planner) makeExplainPlanNode(
 	ctx context.Context, opts *tree.ExplainOptions, origStmt tree.Statement,
@@ -116,9 +87,9 @@ func (p *planner) makeExplainPlanNodeWithPlan(
 		flags.showTypes = true
 	}
 
-	columns := explainPlanColumns
+	columns := sqlbase.ExplainPlanColumns
 	if flags.showMetadata {
-		columns = explainPlanVerboseColumns
+		columns = sqlbase.ExplainPlanVerboseColumns
 	}
 
 	e := explainer{explainFlags: flags}

pkg/sql/logictest/testdata/logic_test/alter_table

@@ -1,4 +1,4 @@
-# LogicTest: default opt parallel-stmts distsql distsql-metadata distsql-opt
+# LogicTest: default parallel-stmts distsql distsql-metadata
 
 statement ok
 CREATE TABLE other (b INT PRIMARY KEY)

pkg/sql/opt/exec/execbuilder/relational_builder.go

@@ -131,6 +131,9 @@ func (b *Builder) buildRelational(ev memo.ExprView) (execPlan, error) {
 	case opt.ExplainOp:
 		ep, err = b.buildExplain(ev)
 
+	case opt.ShowTraceOp, opt.ShowTraceForSessionOp:
+		ep, err = b.buildShowTrace(ev)
+
 	case opt.RowNumberOp:
 		ep, err = b.buildRowNumber(ev)
 
@@ -623,3 +626,27 @@ func (b *Builder) buildExplain(ev memo.ExprView) (execPlan, error) {
 	b.subqueries = b.subqueries[:0]
 	return ep, nil
 }
+
+func (b *Builder) buildShowTrace(ev memo.ExprView) (execPlan, error) {
+	var inputPlan exec.Node
+	if ev.Operator() == opt.ShowTraceOp {
+		input, err := b.buildRelational(ev.Child(0))
+		if err != nil {
+			return execPlan{}, err
+		}
+		inputPlan = input.root
+	}
+
+	def := ev.Private().(*memo.ShowTraceOpDef)
+	node, err := b.factory.ConstructShowTrace(def.Type, def.Compact, inputPlan)
+	if err != nil {
+		return execPlan{}, err
+	}
+	ep := execPlan{root: node}
+	for i := range def.ColList {
+		ep.outputCols.Set(int(def.ColList[i]), i)
+	}
+	// The subqueries are now owned by the explain node; remove them so they don't
+	// also show up in the final plan.
+	return ep, nil
+}

pkg/sql/opt/exec/execbuilder/testdata/show_trace

@@ -0,0 +1,43 @@
+exec-raw
+CREATE TABLE xy (x INT PRIMARY KEY, y INT)
+----
+
+exec nodist
+EXPLAIN (VERBOSE) SHOW TRACE FOR SELECT x FROM xy ORDER BY y
+----
+Tree:string          Field:string  Description:string  Columns:string                                          Ordering:string
+show trace for       ·             ·                   ("timestamp", age, message, tag, loc, operation, span)  ·
+ └── render          ·             ·                   (x)                                                     ·
+      │              render 0      x                   ·                                                       ·
+      └── sort       ·             ·                   (x, y)                                                  +y
+           │         order         +y                  ·                                                       ·
+           └── scan  ·             ·                   (x, y)                                                  ·
+·                    table         xy@primary          ·                                                       ·
+·                    spans         ALL                 ·                                                       ·
+
+exec nodist
+EXPLAIN (VERBOSE) SHOW COMPACT KV TRACE FOR SELECT x FROM xy ORDER BY y
+----
+Tree:string          Field:string  Description:string  Columns:string                  Ordering:string
+show trace for       ·             ·                   (age, message, tag, operation)  ·
+ └── render          ·             ·                   (x)                             ·
+      │              render 0      x                   ·                               ·
+      └── sort       ·             ·                   (x, y)                          +y
+           │         order         +y                  ·                               ·
+           └── scan  ·             ·                   (x, y)                          ·
+·                    table         xy@primary          ·                               ·
+·                    spans         ALL                 ·                               ·
+
+exec nodist
+EXPLAIN (VERBOSE) SHOW EXPERIMENTAL_REPLICA TRACE FOR SELECT x FROM xy ORDER BY y
+----
+Tree:string               Field:string  Description:string  Columns:string                                          Ordering:string
+replica trace             ·             ·                   ("timestamp", node_id, store_id, replica_id)            ·
+ └── show trace for       ·             ·                   ("timestamp", age, message, tag, loc, operation, span)  ·
+      └── render          ·             ·                   (x)                                                     ·
+           │              render 0      x                   ·                                                       ·
+           └── sort       ·             ·                   (x, y)                                                  +y
+                │         order         +y                  ·                                                       ·
+                └── scan  ·             ·                   (x, y)                                                  ·
+·                         table         xy@primary          ·                                                       ·
+·                         spans         ALL                 ·                                                       ·

pkg/sql/opt/exec/factory.go

@@ -121,6 +121,10 @@ type Factory interface {
 	// ConstructExplain returns a node that implements EXPLAIN, showing
 	// information about the given plan.
 	ConstructExplain(options *tree.ExplainOptions, plan Plan) (Node, error)
+
+	// ConstructShowTrace returns a node that implements a SHOW TRACE statement.
+	// If the input is nil, it creates a node for SHOW TRACE FOR SESSION.
+	ConstructShowTrace(typ tree.ShowTraceType, compact bool, input Node) (Node, error)
 }
 
 // Subquery encapsulates information about a subquery that is part of a plan.

pkg/sql/opt/memo/expr_view.go

@@ -252,7 +252,7 @@ func (ev ExprView) formatRelational(f *opt.ExprFmtCtx, tp treeprinter.Node) {
 	fmt.Fprintf(&buf, "%v", ev.op)
 
 	switch ev.Operator() {
-	case opt.ScanOp:
+	case opt.ScanOp, opt.ShowTraceOp, opt.ShowTraceForSessionOp:
 		formatter := ev.mem.makeExprFormatter(&buf)
 		formatter.formatPrivate(ev.Private(), formatNormal)
 	}

pkg/sql/opt/memo/logical_props_builder.go

@@ -87,6 +87,9 @@ func (b *logicalPropsBuilder) buildRelationalProps(ev ExprView) props.Logical {
 	case opt.ExplainOp:
 		return b.buildExplainProps(ev)
 
+	case opt.ShowTraceOp, opt.ShowTraceForSessionOp:
+		return b.buildShowTraceProps(ev)
+
 	case opt.RowNumberOp:
 		return b.buildRowNumberProps(ev)
 	}
@@ -439,6 +442,18 @@ func (b *logicalPropsBuilder) buildExplainProps(ev ExprView) props.Logical {
 	return logical
 }
 
+func (b *logicalPropsBuilder) buildShowTraceProps(ev ExprView) props.Logical {
+	logical := props.Logical{Relational: &props.Relational{}}
+
+	def := ev.Private().(*ShowTraceOpDef)
+	logical.Relational.OutputCols = opt.ColListToSet(def.ColList)
+	logical.Relational.Cardinality = props.AnyCardinality
+
+	// Zero value for Stats is ok for ShowTrace.
+
+	return logical
+}
+
 func (b *logicalPropsBuilder) buildLimitProps(ev ExprView) props.Logical {
 	logical := props.Logical{Relational: &props.Relational{}}
 

pkg/sql/opt/memo/memo_format.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/cockroachdb/cockroach/pkg/sql/opt"
 	"github.com/cockroachdb/cockroach/pkg/sql/opt/props"
+	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
 	"github.com/cockroachdb/cockroach/pkg/util/treeprinter"
 )
 
@@ -309,6 +310,24 @@ func (f exprFormatter) formatPrivate(private interface{}, mode formatMode) {
 			}
 		}
 
+	case *ShowTraceOpDef:
+		if t.Compact {
+			f.buf.WriteString(" compact")
+		}
+		switch t.Type {
+		case tree.ShowTraceKV:
+			f.buf.WriteString(" kv")
+		case tree.ShowTraceReplica:
+			f.buf.WriteString(" replica")
+		}
+
+		if mode == formatMemo {
+			propsStr := t.Props.String()
+			if propsStr != "" {
+				fmt.Fprintf(f.buf, " %s", propsStr)
+			}
+		}
+
 	case opt.ColSet, opt.ColList:
 		// Don't show anything, because it's mostly redundant.
 

pkg/sql/opt/memo/private_defs.go

@@ -147,6 +147,21 @@ type ExplainOpDef struct {
 	Props props.Physical
 }
 
+// ShowTraceOpDef defines the value of the Def private field of the Explain operator.
+type ShowTraceOpDef struct {
+	Type tree.ShowTraceType
+
+	// Compact indicates that we output a smaller set of columns; set
+	// when SHOW COMPACT [KV] TRACE is used.
+	Compact bool
+
+	// ColList stores the column IDs for the SHOW TRACE columns.
+	ColList opt.ColList
+
+	// Props stores the required physical properties for the enclosed expression.
+	Props props.Physical
+}
+
 // RowNumberDef defines the value of the Def private field of the RowNumber
 // operator.
 type RowNumberDef struct {

pkg/sql/opt/memo/private_storage.go

@@ -245,7 +245,7 @@ func (ps *privateStorage) internLookupJoinDef(def *LookupJoinDef) PrivateID {
 
 // internExplainOpDef adds the given value to storage and returns an id that can
 // later be used to retrieve the value by calling the lookup method. If the
-// value has been previously added to storage, then internLookupJoinDef always
+// value has been previously added to storage, then internExplainOpDef always
 // returns the same private id that was returned from the previous call.
 func (ps *privateStorage) internExplainOpDef(def *ExplainOpDef) PrivateID {
 	ps.keyBuf.Reset()
@@ -261,6 +261,27 @@ func (ps *privateStorage) internExplainOpDef(def *ExplainOpDef) PrivateID {
 	return ps.addValue(privateKey{iface: typ, str: ps.keyBuf.String()}, def)
 }
 
+// internShowTraceOpDef adds the given value to storage and returns an id that can
+// later be used to retrieve the value by calling the lookup method. If the
+// value has been previously added to storage, then internShowTraceOpDef always
+// returns the same private id that was returned from the previous call.
+func (ps *privateStorage) internShowTraceOpDef(def *ShowTraceOpDef) PrivateID {
+	ps.keyBuf.Reset()
+	ps.keyBuf.WriteString(string(def.Type))
+	if def.Compact {
+		ps.keyBuf.WriteByte(1)
+	} else {
+		ps.keyBuf.WriteByte(0)
+	}
+	ps.keyBuf.writeColList(def.ColList)
+	ps.keyBuf.WriteString(def.Props.Fingerprint())
+	typ := (*ShowTraceOpDef)(nil)
+	if id, ok := ps.privatesMap[privateKey{iface: typ, str: ps.keyBuf.String()}]; ok {
+		return id
+	}
+	return ps.addValue(privateKey{iface: typ, str: ps.keyBuf.String()}, def)
+}
+
 // internRowNumberDef adds the given value to storage and returns an id that can
 // later be used to retrieve the value by calling the lookup method. If the
 // value has been previously added to storage, then internRowNumberDef always

pkg/sql/opt/norm/rule_props_builder.go

@@ -84,12 +84,14 @@ func (b *rulePropsBuilder) buildProps(ev memo.ExprView) {
 	case opt.Max1RowOp:
 		b.buildMax1RowProps(ev)
 
-	case opt.ExplainOp:
-		b.buildExplainProps(ev)
-
 	case opt.RowNumberOp:
 		b.buildRowNumberProps(ev)
 
+	case opt.ExplainOp, opt.ShowTraceOp, opt.ShowTraceForSessionOp:
+		// Don't allow any columns to be pruned, since that would trigger the
+		// creation of a wrapper Project around the Explain (it's not capable
+		// of pruning columns or of passing through Project operators).
+
 	default:
 		panic(fmt.Sprintf("unrecognized relational expression type: %v", ev.Operator()))
 	}
@@ -177,12 +179,6 @@ func (b *rulePropsBuilder) buildValuesProps(ev memo.ExprView) {
 	relational.Rule.PruneCols = ev.Logical().Relational.OutputCols
 }
 
-func (b *rulePropsBuilder) buildExplainProps(ev memo.ExprView) {
-	// Don't allow any columns to be pruned, since that would trigger the
-	// creation of a wrapper Project around the Explain, since it's not capable
-	// of pruning columns or of passing through Project operators.
-}
-
 func (b *rulePropsBuilder) buildLimitProps(ev memo.ExprView) {
 	relational := ev.Logical().Relational
 

pkg/sql/opt/ops/relational.opt

@@ -311,20 +311,36 @@ define Offset {
     Ordering Ordering
 }
 
-# Max1Row is an operator which enforces that its input must return at most one
-# row. It is used as input to the Subquery operator. See the comment above
-# Subquery for more details.
+# Max1Row enforces that its input must return at most one row. It is used as
+# input to the Subquery operator. See the comment above Subquery for more
+# details.
 [Relational]
 define Max1Row {
     Input Expr
 }
 
+# Explain returns information about the execution plan of the "input"
+# expression.
 [Relational]
 define Explain {
     Input Expr
     Def   ExplainOpDef
 }
 
+# ShowTrace runs the "input" expressions in a special tracing mode and returns
+# trace results.
+[Relational]
+define ShowTrace {
+    Input Expr
+    Def   ShowTraceOpDef
+}
+
+# ShowTraceForSession returns the current session traces.
+[Relational]
+define ShowTraceForSession {
+    Def   ShowTraceOpDef
+}
+
 # RowNumber adds a column to each row in its input containing a unique,
 # increasing number.
 [Relational]

pkg/sql/opt/optbuilder/builder.go

@@ -182,6 +182,9 @@ func (b *Builder) buildStmt(stmt tree.Statement, inScope *scope) (outScope *scop
 	case *tree.Explain:
 		return b.buildExplain(stmt, inScope)
 
+	case *tree.ShowTrace:
+		return b.buildShowTrace(stmt, inScope)
+
 	default:
 		panic(unimplementedf("unsupported statement: %T", stmt))
 	}