Skip to content

Commit

Permalink
opt: add TES, SES, and rules to reorderjoins
Browse files Browse the repository at this point in the history 
This commit updates the output of the `reorderjoins` opt test command to
display the initial state of the `JoinOrderBuilder`. It adds additional
information to the output including the TES, SES, and conflict rules for
each edge.

Release note: None
  • Loading branch information
@mgartner
mgartner committed Feb 14, 2022
1 parent f1fcb83 commit aac0834
Show file tree
Hide file tree
Showing 3 changed files with 1,446 additions and 1,345 deletions.
220 changes: 120 additions & 100 deletions pkg/sql/opt/testutils/opttester/reorder_joins.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
package opttester

import (
"bytes"
"fmt"
"strconv"
"strings"
Expand All @@ -38,10 +37,7 @@ import (
func (ot *OptTester) ReorderJoins() (string, error) {
ot.builder.Reset()
o := ot.makeOptimizer()

// Map from the first ColumnID of each base relation to its assigned name.
// this ensures that aliases are consistent across different calls to Reorder.
names := map[opt.ColumnID]string{}
jof := newJoinOrderFormatter(o)

// joinsConsidered counts the number of joins which joinOrderBuilder attempts
// to add to the memo during each call to Reorder.
Expand All @@ -53,47 +49,43 @@ func (ot *OptTester) ReorderJoins() (string, error) {
func(
join memo.RelExpr,
vertexes []memo.RelExpr,
edges []memo.FiltersExpr,
edgeOps []opt.Operator,
edges []xform.OnReorderEdgeParam,
) {
if treeNum > 1 {
// This isn't the first Reorder call. Output the number of joins added to
// the memo by the last call to Reorder.
ot.builder.WriteString(fmt.Sprintf("\nJoins Considered: %v\n", joinsConsidered))
ot.output(fmt.Sprintf("Joins Considered: %v\n", joinsConsidered))
joinsConsidered = 0
}
ot.builder.WriteString(separator("-"))
ot.builder.WriteString(fmt.Sprintf("----Join Tree #%v----\n", treeNum))
ot.builder.WriteString(o.FormatExpr(join, memo.ExprFmtHideAll))
ot.builder.WriteString("\n----Vertexes----\n")
ot.builder.WriteString(outputVertexes(vertexes, names, o))
ot.builder.WriteString("----Edges----\n")
ot.separator("-")
ot.output(fmt.Sprintf("Join Tree #%d\n", treeNum))
ot.separator("-")
ot.indent(o.FormatExpr(join, memo.ExprFmtHideAll))
ot.output("Vertexes\n")
for i := range vertexes {
ot.indent(jof.formatVertex(vertexes[i]))
}
ot.output("Edges\n")
for i := range edges {
ot.builder.WriteString(outputEdge(edges[i], edgeOps[i], o))
ot.indent(jof.formatEdge(edges[i]))
}
ot.builder.WriteString("\n")
treeNum++
relsJoinedLast = ""
})

o.JoinOrderBuilder().NotifyOnAddJoin(func(left, right, all, refs []memo.RelExpr, op opt.Operator) {
relsToJoin := outputRels(all, names)
relsToJoin := jof.formatVertexSet(all)
if relsToJoin != relsJoinedLast {
ot.builder.WriteString(
fmt.Sprintf(
"----Joining %s----\n",
relsToJoin,
),
)
ot.output(fmt.Sprintf("Joining %s\n", relsToJoin))
relsJoinedLast = relsToJoin
}
ot.builder.WriteString(
ot.indent(
fmt.Sprintf(
"%s %s refs [%s] [%s]\n",
outputRels(left, names),
outputRels(right, names),
outputRels(refs, names),
outputOp(op),
"%s %s [%s, refs=%s]",
jof.formatVertexSet(left),
jof.formatVertexSet(right),
joinOpLabel(op),
jof.formatVertexSet(refs),
),
)
joinsConsidered++
Expand All @@ -103,61 +95,120 @@ func (ot *OptTester) ReorderJoins() (string, error) {
if err != nil {
return "", err
}
ot.builder.WriteString(fmt.Sprintf("\nJoins Considered: %v\n", joinsConsidered))
ot.builder.WriteString(separator("-"))
ot.builder.WriteString("----Final Plan----\n")
ot.builder.WriteString(ot.FormatExpr(expr))
ot.builder.WriteString(separator("-"))
ot.output(fmt.Sprintf("Joins Considered: %d\n", joinsConsidered))
ot.separator("=")
ot.output("Final Plan\n")
ot.separator("=")
ot.output(ot.FormatExpr(expr))
return ot.builder.String(), err
}

// outputVertexes outputs each base relation in the vertexes slice along with
type joinOrderFormatter struct {
o *xform.Optimizer

// relLabels is a map from the first ColumnID of each base relation to its
// assigned label.
relLabels map[opt.ColumnID]string
}

// newJoinOrderFormatter returns an initialized joinOrderFormatter.
func newJoinOrderFormatter(o *xform.Optimizer) *joinOrderFormatter {
return &joinOrderFormatter{
o: o,
relLabels: make(map[opt.ColumnID]string),
}
}

// formatVertex outputs each base relation in the vertexes slice along with
// its alias.
func outputVertexes(
vertexes []memo.RelExpr, names map[opt.ColumnID]string, o *xform.Optimizer,
) string {
buf := bytes.Buffer{}
for i := range vertexes {
firstCol, ok := vertexes[i].Relational().OutputCols.Next(0)
if !ok {
panic(errors.AssertionFailedf("failed to retrieve column from %v", vertexes[i].Op()))
}
name, ok := names[firstCol]
if !ok {
name = getRelationName(len(names))
names[firstCol] = name
}
buf.WriteString(
fmt.Sprintf(
"%s:\n%s",
name,
o.FormatExpr(vertexes[i], memo.ExprFmtHideAll),
),
)
buf.WriteString("\n")
func (jof *joinOrderFormatter) formatVertex(vertex memo.RelExpr) string {
var b strings.Builder
b.WriteString(jof.relLabel(vertex))
b.WriteString(":\n")
expr := jof.o.FormatExpr(vertex, memo.ExprFmtHideAll)
expr = strings.TrimRight(expr, " \n\t\r")
lines := strings.Split(expr, "\n")
for _, line := range lines {
b.WriteString(fmt.Sprintf(" %s\n", line))
}
return b.String()
}

// formatVertexSet outputs each base relation in the vertexes slice along with
// its alias.
func (jof *joinOrderFormatter) formatVertexSet(vertexSet []memo.RelExpr) string {
var b strings.Builder
for i := range vertexSet {
b.WriteString(jof.relLabel(vertexSet[i]))
}
return buf.String()
return b.String()
}

// outputEdge returns a formatted string for the given FiltersItem along with
// formatEdge returns a formatted string for the given FiltersItem along with
// the type of join the edge came from, like so: "x = a left".
func outputEdge(edge memo.FiltersExpr, op opt.Operator, o *xform.Optimizer) string {
buf := bytes.Buffer{}
if len(edge) == 0 {
buf.WriteString("cross")
func (jof *joinOrderFormatter) formatEdge(edge xform.OnReorderEdgeParam) string {
var b strings.Builder
if len(edge.Filters) == 0 {
b.WriteString("cross")
} else {
for i := range edge {
for i := range edge.Filters {
if i != 0 {
buf.WriteString(", ")
b.WriteString(", ")
}
buf.WriteString(strings.TrimSuffix(o.FormatExpr(&edge[i], memo.ExprFmtHideAll), "\n"))
b.WriteString(strings.TrimSuffix(jof.o.FormatExpr(&edge.Filters[i], memo.ExprFmtHideAll), "\n"))
}
}
b.WriteString(fmt.Sprintf(
" [%s, ses=%s, tes=%s, rules=%s]",
joinOpLabel(edge.Op),
jof.formatVertexSet(edge.SES),
jof.formatVertexSet(edge.TES),
jof.formatRules(edge.Rules),
))
return b.String()
}

func (jof *joinOrderFormatter) formatRules(rules []xform.OnReorderRuleParam) string {
var b strings.Builder
b.WriteRune('(')
for i, rule := range rules {
if i > 0 {
b.WriteRune(',')
}
b.WriteString(fmt.Sprintf(
"%s->%s",
jof.formatVertexSet(rule.From),
jof.formatVertexSet(rule.To),
))
}
buf.WriteString(fmt.Sprintf(" [%s]\n", outputOp(op)))
return buf.String()
b.WriteRune(')')
return b.String()
}

func outputOp(op opt.Operator) string {
// relLabel returns the label for the given relation. Labels will follow the
// pattern A, B, ..., Z, A1, B1, etc.
func (jof *joinOrderFormatter) relLabel(e memo.RelExpr) string {
firstCol, ok := e.Relational().OutputCols.Next(0)
if !ok {
panic(errors.AssertionFailedf("failed to retrieve column from %v", e.Op()))
}
if label, ok := jof.relLabels[firstCol]; ok {
return label
}
const lenAlphabet = 26
labelCount := len(jof.relLabels)
label := string(rune(int('A') + (labelCount % lenAlphabet)))
number := labelCount / lenAlphabet
if number > 0 {
// Names will follow the pattern: A, B, ..., Z, A1, B1, etc.
label += strconv.Itoa(number)
}
jof.relLabels[firstCol] = label
return label
}

// joinOpLabel returns an abbreviated string representation of a join operator.
func joinOpLabel(op opt.Operator) string {
switch op {
case opt.InnerJoinOp:
return "inner"
Expand All @@ -178,34 +229,3 @@ func outputOp(op opt.Operator) string {
panic(errors.AssertionFailedf("unexpected operator: %v", op))
}
}

// outputRels returns a string with the aliases of the given base relations
// concatenated together. Panics if there is no alias for a base relation.
func outputRels(baseRels []memo.RelExpr, names map[opt.ColumnID]string) string {
buf := bytes.Buffer{}
for i := range baseRels {
firstCol, ok := baseRels[i].Relational().OutputCols.Next(0)
if !ok {
panic(errors.AssertionFailedf("failed to retrieve column from %v", baseRels[i].Op()))
}
buf.WriteString(names[firstCol])
}
return buf.String()
}

// getRelationName returns a simple alias for a base relation given the number
// of names generated so far.
func getRelationName(nameCount int) string {
const lenAlphabet = 26
name := string(rune(int('A') + (nameCount % lenAlphabet)))
number := nameCount / lenAlphabet
if number > 0 {
// Names will follow the pattern: A, B, ..., Z, A1, B1, etc.
name += strconv.Itoa(number)
}
return name
}

func separator(sep string) string {
return fmt.Sprintf("%s\n", strings.Repeat(sep, 80))
}
49 changes: 40 additions & 9 deletions pkg/sql/opt/xform/join_order_builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,32 @@ import (
type OnReorderFunc func(
join memo.RelExpr,
vertexes []memo.RelExpr,
edges []memo.FiltersExpr,
edgeOps []opt.Operator,
edges []OnReorderEdgeParam,
)

// OnReorderEdgeParam is a struct representing an edge in the join graph. This
// type is only used for the OnReorderFunc during testing and debugging. See the
// more efficient edge type that is used in the join reordering algorithm.
type OnReorderEdgeParam struct {
// Op is the original join operator from which the edge was constructed.
Op opt.Operator
// Filters is the edge's set of join filters
Filters memo.FiltersExpr
// SES is the edge's syntactic eligibility set.
SES []memo.RelExpr
// TES is the edge's total eligibility set.
TES []memo.RelExpr
// Rules is the set of conflict rules of the edge.
Rules []OnReorderRuleParam
}

// OnReorderRuleParam is a struct representing a conflict rule. This type is
// only used for the OnReorderFunc during testing and debugging. See the more
// efficient conflictRule type that is used in the join reordering algorithm.
type OnReorderRuleParam struct {
From, To []memo.RelExpr
}

// OnAddJoinFunc defines the callback function for the NotifyOnAddJoin event
// supported by JoinOrderBuilder. OnAddJoinFunc is called when JoinOrderBuilder
// attempts to add a join to the memo via addJoin. The callback parameters give
Expand Down Expand Up @@ -906,14 +928,23 @@ func (jb *JoinOrderBuilder) NotifyOnAddJoin(onAddJoin OnAddJoinFunc) {
// function is nil.
func (jb *JoinOrderBuilder) callOnReorderFunc(join memo.RelExpr) {
// Get a slice with all edges of the join graph.
edgeSlice := make([]memo.FiltersExpr, 0, len(jb.edges))
edgeOps := make([]opt.Operator, 0, len(jb.edges))
for i := range jb.edges {
edgeSlice = append(edgeSlice, jb.edges[i].filters)
edgeOps = append(edgeOps, jb.edges[i].op.joinType)
edges := make([]OnReorderEdgeParam, 0, len(jb.edges))
for _, edge := range jb.edges {
ep := OnReorderEdgeParam{
Op: edge.op.joinType,
Filters: edge.filters,
SES: jb.getRelationSlice(edge.ses),
TES: jb.getRelationSlice(edge.tes),
}
for _, rule := range edge.rules {
ep.Rules = append(ep.Rules, OnReorderRuleParam{
From: jb.getRelationSlice(rule.from),
To: jb.getRelationSlice(rule.to),
})
}
edges = append(edges, ep)
}

jb.onReorderFunc(join, jb.getRelationSlice(jb.allVertexes()), edgeSlice, edgeOps)
jb.onReorderFunc(join, jb.getRelationSlice(jb.allVertexes()), edges)
}

// callOnAddJoinFunc calls the onAddJoinFunc callback function. Panics if the
Expand Down
Loading

0 comments on commit aac0834

Please sign in to comment.