opt: fix hidden column handling for WITH

pkg/sql/opt/optbuilder/scope.go

@@ -399,11 +399,14 @@ func (s *scope) hasSameColumns(other *scope) bool {
 	return s.colSetWithExtraCols().Equals(other.colSetWithExtraCols())
 }
 
-// removeHiddenCols removes hidden columns from the scope.
+// removeHiddenCols removes hidden columns from the scope (and moves them to
+// extraCols, in case they are referenced by ORDER BY or DISTINCT ON).
 func (s *scope) removeHiddenCols() {
 	n := 0
 	for i := range s.cols {
-		if !s.cols[i].hidden {
+		if s.cols[i].hidden {
+			s.extraCols = append(s.extraCols, s.cols[i])
+		} else {
 			if n != i {
 				s.cols[n] = s.cols[i]
 			}

pkg/sql/opt/optbuilder/select.go

@@ -579,6 +579,7 @@ func (b *Builder) buildCTE(
 	outScope.ctes = make(map[string]*cteSource)
 	for i := range ctes {
 		cteScope := b.buildStmt(ctes[i].Stmt, nil /* desiredTypes */, outScope)
+		cteScope.removeHiddenCols()
 		cols := cteScope.cols
 		name := ctes[i].Name.Alias
 

pkg/sql/opt/optbuilder/testdata/with

@@ -762,3 +762,29 @@ with &1 (t)
                      │    └── mapping:
                      │         └──  xy.x:1(int) => x:16(int)
                      └── variable: xy.x [type=int]
+
+# Check hidden column handling: level, node_type should not be output.
+build
+WITH cte AS (EXPLAIN (VERBOSE) SELECT 1) SELECT * FROM cte
+----
+with &1 (cte)
+ ├── columns: tree:9(string) field:10(string) description:11(string) columns:12(string) ordering:13(string)
+ ├── project
+ │    ├── columns: tree:2(string) field:5(string) description:6(string) columns:7(string) ordering:8(string)
+ │    └── explain
+ │         ├── columns: tree:2(string) level:3(int) node_type:4(string) field:5(string) description:6(string) columns:7(string) ordering:8(string)
+ │         ├── mode: verbose
+ │         └── project
+ │              ├── columns: "?column?":1(int!null)
+ │              ├── values
+ │              │    └── tuple [type=tuple]
+ │              └── projections
+ │                   └── const: 1 [type=int]
+ └── with-scan &1 (cte)
+      ├── columns: tree:9(string) field:10(string) description:11(string) columns:12(string) ordering:13(string)
+      └── mapping:
+           ├──  tree:2(string) => tree:9(string)
+           ├──  field:5(string) => field:10(string)
+           ├──  description:6(string) => description:11(string)
+           ├──  columns:7(string) => columns:12(string)
+           └──  ordering:8(string) => ordering:13(string)