Skip to content

Commit

Permalink
Merge #47316
Browse files Browse the repository at this point in the history 
47316: sql: add virtual table pushdown infrastructure, and add to several spots r=jordanlewis a=jordanlewis

See individual commits for details. This basically adds the infrastructure to the optimizer and execution engine for pushdown, and adds virtual indexes to a few choice tables.

The second commit rearranges how SHOW CREATE TABLE is done so that it can use a virtual index. Now it's no longer O(n) in the number of tables, hooray!

This is still a WIP - there are some things that need cleaning / more comments, and I think the optimizer part is still probably not complete. Also, I need to figure out how to write optimizer tests for the virtual indexes. I tried to add support to the opt catalog for them but ran into some weird issues. I'm pushing this up so we can start making progress on it as a team - I've been sitting on it for too long.

Co-authored-by: Jordan Lewis <jordanthelewis@gmail.com>
  • Loading branch information
@jordanlewis
craig[bot] and jordanlewis committed Apr 25, 2020
2 parents d620f62 + c44619c commit 29cead1
Show file tree
Hide file tree
Showing 35 changed files with 1,334 additions and 823 deletions.
216 changes: 78 additions & 138 deletions pkg/sql/crdb_internal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror"
"github.com/cockroachdb/cockroach/pkg/sql/sem/builtins"
"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
"github.com/cockroachdb/cockroach/pkg/sql/types"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
Expand All @@ -46,10 +47,9 @@ import (
"github.com/cockroachdb/cockroach/pkg/util/protoutil"
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
"github.com/cockroachdb/errors"
"gopkg.in/yaml.v2"
)

const crdbInternalName = "crdb_internal"
const crdbInternalName = sessiondata.CRDBInternalSchemaName

// Naming convention:
// - if the response is served from memory, prefix with node_
Expand Down Expand Up @@ -1371,13 +1371,18 @@ CREATE TABLE crdb_internal.builtin_functions (
},
}

// Prepare the row populate function.
var typeView = tree.NewDString("view")
var typeTable = tree.NewDString("table")
var typeSequence = tree.NewDString("sequence")

// crdbInternalCreateStmtsTable exposes the CREATE TABLE/CREATE VIEW
// statements.
//
// TODO(tbg): prefix with kv_.
var crdbInternalCreateStmtsTable = virtualSchemaTable{
comment: `CREATE and ALTER statements for all tables accessible by current user in current database (KV scan)`,
schema: `
var crdbInternalCreateStmtsTable = makeAllRelationsVirtualTableWithDescriptorIDIndex(
`CREATE and ALTER statements for all tables accessible by current user in current database (KV scan)`,
`
CREATE TABLE crdb_internal.create_statements (
database_id INT,
database_name STRING,
Expand All @@ -1390,148 +1395,83 @@ CREATE TABLE crdb_internal.create_statements (
create_nofks STRING NOT NULL,
alter_statements STRING[] NOT NULL,
validate_statements STRING[] NOT NULL,
zone_configuration_statements STRING[] NOT NULL
has_partitions BOOL NOT NULL,
INDEX(descriptor_id)
)
`,
populate: func(ctx context.Context, p *planner, dbContext *DatabaseDescriptor, addRow func(...tree.Datum) error) error {
`, virtualOnce, false, /* includesIndexEntries */
func(ctx context.Context, p *planner, h oidHasher, db *sqlbase.DatabaseDescriptor, scName string,
table *sqlbase.TableDescriptor, lookup simpleSchemaResolver, addRow func(...tree.Datum) error) error {
contextName := ""
if dbContext != nil {
contextName = dbContext.Name
}

// Prepare the row populate function.
typeView := tree.NewDString("view")
typeTable := tree.NewDString("table")
typeSequence := tree.NewDString("sequence")

// Hold the configuration statements for each table
zoneConfigStmts := make(map[string][]string)
// Prepare a query used to see zones configuations on this table.
configStmtsQuery := `
SELECT
table_name, raw_config_yaml, raw_config_sql
FROM
crdb_internal.zones
WHERE
database_name = '%[1]s'
AND table_name IS NOT NULL
AND raw_config_yaml IS NOT NULL
AND raw_config_sql IS NOT NULL
ORDER BY
database_name, table_name, index_name, partition_name
`
// The create_statements table is used at times where other internal
// tables have not been created, or are unaccessible (perhaps during
// certain tests (TestDumpAsOf in pkg/cli/dump_test.go)). So if something
// goes wrong querying this table, proceed without any constraint data.
zoneConstraintRows, err := p.ExtendedEvalContext().ExecCfg.InternalExecutor.Query(
ctx, "zone-constraints-for-show-create-table", p.txn,
fmt.Sprintf(configStmtsQuery, contextName))
if err != nil {
log.VEventf(ctx, 1, "%q", err)
parentNameStr := tree.DNull
if db != nil {
contextName = db.Name
parentNameStr = tree.NewDString(db.Name)
}
scNameStr := tree.NewDString(scName)

var descType tree.Datum
var stmt, createNofk string
alterStmts := tree.NewDArray(types.String)
validateStmts := tree.NewDArray(types.String)
var err error
if table.IsView() {
descType = typeView
stmt, err = ShowCreateView(ctx, (*tree.Name)(&table.Name), table)
} else if table.IsSequence() {
descType = typeSequence
stmt, err = ShowCreateSequence(ctx, (*tree.Name)(&table.Name), table)
} else {
for _, row := range zoneConstraintRows {
tableName := string(tree.MustBeDString(row[0]))
var zoneConfig zonepb.ZoneConfig
yamlString := string(tree.MustBeDString(row[1]))
err := yaml.UnmarshalStrict([]byte(yamlString), &zoneConfig)
if err != nil {
return err
}
// If all constraints are default, then don't show anything.
if !zoneConfig.Equal(zonepb.ZoneConfig{}) {
sqlString := string(tree.MustBeDString(row[2]))
zoneConfigStmts[tableName] = append(zoneConfigStmts[tableName], sqlString)
}
descType = typeTable
tn := (*tree.Name)(&table.Name)
createNofk, err = ShowCreateTable(ctx, p, tn, contextName, table, lookup, OmitFKClausesFromCreate)
if err != nil {
return err
}
allIdx := append(table.Indexes, table.PrimaryIndex)
if err := showAlterStatementWithInterleave(ctx, tn, contextName, lookup, allIdx, table, alterStmts,
validateStmts); err != nil {
return err
}
stmt, err = ShowCreateTable(ctx, p, tn, contextName, table, lookup, IncludeFkClausesInCreate)
}
if err != nil {
return err
}

return forEachTableDescWithTableLookupInternal(ctx, p, dbContext, virtualOnce, true, /*allowAdding*/
func(db *DatabaseDescriptor, scName string, table *TableDescriptor, lCtx tableLookupFn) error {
parentNameStr := tree.DNull
if db != nil {
parentNameStr = tree.NewDString(db.Name)
}
scNameStr := tree.NewDString(scName)

var descType tree.Datum
var stmt, createNofk string
alterStmts := tree.NewDArray(types.String)
validateStmts := tree.NewDArray(types.String)
var err error
if table.IsView() {
descType = typeView
stmt, err = ShowCreateView(ctx, (*tree.Name)(&table.Name), table)
} else if table.IsSequence() {
descType = typeSequence
stmt, err = ShowCreateSequence(ctx, (*tree.Name)(&table.Name), table)
} else {
descType = typeTable
tn := (*tree.Name)(&table.Name)
createNofk, err = ShowCreateTable(ctx, p, tn, contextName, table, lCtx, OmitFKClausesFromCreate)
if err != nil {
return err
}
allIdx := append(table.Indexes, table.PrimaryIndex)
if err := showAlterStatementWithInterleave(ctx, tn, contextName, lCtx, allIdx, table, alterStmts, validateStmts); err != nil {
return err
}
stmt, err = ShowCreateTable(ctx, p, tn, contextName, table, lCtx, IncludeFkClausesInCreate)
}
if err != nil {
return err
}

zoneRows := tree.NewDArray(types.String)
if val, ok := zoneConfigStmts[table.Name]; ok {
for _, s := range val {
if err := zoneRows.Append(tree.NewDString(s)); err != nil {
return err
}
}
} else {
// If there are partitions applied to this table and no zone configurations, display a warning.
hasPartitions := false
for i := range table.Indexes {
if table.Indexes[i].Partitioning.NumColumns != 0 {
hasPartitions = true
break
}
}
hasPartitions = hasPartitions || table.PrimaryIndex.Partitioning.NumColumns != 0
if hasPartitions {
stmt += "\n-- Warning: Partitioned table with no zone configurations."
}
}

descID := tree.NewDInt(tree.DInt(table.ID))
dbDescID := tree.NewDInt(tree.DInt(table.GetParentID()))
if createNofk == "" {
createNofk = stmt
}
return addRow(
dbDescID,
parentNameStr,
scNameStr,
descID,
descType,
tree.NewDString(table.Name),
tree.NewDString(stmt),
tree.NewDString(table.State.String()),
tree.NewDString(createNofk),
alterStmts,
validateStmts,
zoneRows,
)
})
},
}
descID := tree.NewDInt(tree.DInt(table.ID))
dbDescID := tree.NewDInt(tree.DInt(table.GetParentID()))
if createNofk == "" {
createNofk = stmt
}
hasPartitions := false
for i := range table.Indexes {
if table.Indexes[i].Partitioning.NumColumns != 0 {
hasPartitions = true
break
}
}
hasPartitions = hasPartitions || table.PrimaryIndex.Partitioning.NumColumns != 0
return addRow(
dbDescID,
parentNameStr,
scNameStr,
descID,
descType,
tree.NewDString(table.Name),
tree.NewDString(stmt),
tree.NewDString(table.State.String()),
tree.NewDString(createNofk),
alterStmts,
validateStmts,
tree.MakeDBool(tree.DBool(hasPartitions)),
)
})

func showAlterStatementWithInterleave(
ctx context.Context,
tn *tree.Name,
contextName string,
lCtx tableLookupFn,
lCtx simpleSchemaResolver,
allIdx []sqlbase.IndexDescriptor,
table *sqlbase.TableDescriptor,
alterStmts *tree.DArray,
Expand Down Expand Up @@ -1574,7 +1514,7 @@ func showAlterStatementWithInterleave(
var err error
var parentName tree.TableName
if lCtx != nil {
parentName, err = lCtx.getParentAsTableName(parentTableID, contextName)
parentName, err = getParentAsTableName(lCtx, parentTableID, contextName)
if err != nil {
return err
}
Expand All @@ -1586,7 +1526,7 @@ func showAlterStatementWithInterleave(

var tableName tree.TableName
if lCtx != nil {
tableName, err = lCtx.getTableAsTableName(table, contextName)
tableName, err = getTableAsTableName(lCtx, table, contextName)
if err != nil {
return err
}
Expand Down
10 changes: 6 additions & 4 deletions pkg/sql/delayed.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ package sql
import (
"context"

"github.com/cockroachdb/cockroach/pkg/sql/opt/constraint"
"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
)
Expand All @@ -21,10 +22,11 @@ import (
// constructor must be delayed during query execution (as opposed to
// SQL prepare) for resource tracking purposes.
type delayedNode struct {
name string
columns sqlbase.ResultColumns
constructor nodeConstructor
plan planNode
name string
columns sqlbase.ResultColumns
indexConstraint *constraint.Constraint
constructor nodeConstructor
plan planNode
}

type nodeConstructor func(context.Context, *planner) (planNode, error)
Expand Down
32 changes: 20 additions & 12 deletions pkg/sql/delegate/show_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,23 +21,30 @@ import (

func (d *delegator) delegateShowCreate(n *tree.ShowCreate) (tree.Statement, error) {
sqltelemetry.IncrementShowCounter(sqltelemetry.Create)

const showCreateQuery = `
WITH zone_configs AS (
SELECT string_agg(raw_config_sql, e';\n') FROM crdb_internal.zones
WHERE database_name = %[1]s
AND table_name = %[2]s
AND raw_config_yaml IS NOT NULL
AND raw_config_sql IS NOT NULL
)
SELECT
%[3]s AS table_name,
array_to_string(
array_cat(
ARRAY[create_statement],
zone_configuration_statements
),
e';\n'
)
AS create_statement
concat(create_statement,
CASE
WHEN NOT has_partitions
THEN NULL
WHEN (SELECT * FROM zone_configs) IS NULL
THEN e'\n-- Warning: Partitioned table with no zone configurations.'
ELSE concat(e';\n', (SELECT * FROM zone_configs))
END
) AS create_statement
FROM
%[4]s.crdb_internal.create_statements
WHERE
(database_name IS NULL OR database_name = %[1]s)
AND schema_name = %[5]s
AND descriptor_name = %[2]s
descriptor_id = %[6]d
`

return d.showTableDetails(n.Name, showCreateQuery)
Expand Down Expand Up @@ -164,6 +171,7 @@ func (d *delegator) delegateShowConstraints(n *tree.ShowConstraints) (tree.State
// %[3]s the given table name as SQL string literal.
// %[4]s the database name as SQL identifier.
// %[5]s the schema name as SQL string literal.
// %[6]s the table ID.
func (d *delegator) showTableDetails(
name *tree.UnresolvedObjectName, query string,
) (tree.Statement, error) {
Expand All @@ -185,7 +193,7 @@ func (d *delegator) showTableDetails(
lex.EscapeSQLString(resName.String()),
resName.CatalogName.String(), // note: CatalogName.String() != Catalog()
lex.EscapeSQLString(resName.Schema()),
dataSource.ID(),
dataSource.PostgresDescriptorID(),
)

return parse(fullQuery)
Expand Down
Loading

0 comments on commit 29cead1

Please sign in to comment.