From 4607a7430d7d132a3dbbfa18d65b29a31cc1c2db Mon Sep 17 00:00:00 2001 From: Faizan Qazi Date: Fri, 10 Feb 2023 10:26:37 -0500 Subject: [PATCH 1/3] workload/schemachange: skip tsquery/tsvector types in mixed version envs Fixes: #96489 Previously, the randomized schema change workload in a mixed version environment attempted to create tables with the tsvector/tsquery types. This could lead to syntax errors on older nodes that do not know of this. This patch, avoids using these types until the version of the cluster hits 23.1. Epic: none Release note: None --- .../schemachange/operation_generator.go | 38 ++++--------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/pkg/workload/schemachange/operation_generator.go b/pkg/workload/schemachange/operation_generator.go index 20f6a5035eae..ea4a49de2737 100644 --- a/pkg/workload/schemachange/operation_generator.go +++ b/pkg/workload/schemachange/operation_generator.go @@ -1211,44 +1211,21 @@ func (og *operationGenerator) createTable(ctx context.Context, tx pgx.Tx) (*opSt stmt := randgen.RandCreateTableWithColumnIndexNumberGenerator(og.params.rng, "table", tableIdx, databaseHasMultiRegion, og.newUniqueSeqNum) stmt.Table = *tableName stmt.IfNotExists = og.randIntn(2) == 0 - trigramIsNotSupported, err := isClusterVersionLessThan( + tsQueryNotSupported, err := isClusterVersionLessThan( ctx, tx, - clusterversion.ByKey(clusterversion.TODODelete_V22_2TrigramInvertedIndexes)) + clusterversion.ByKey(clusterversion.V23_1)) if err != nil { return nil, err } - hasTrigramIdxUnsupported := func() bool { - if !trigramIsNotSupported { + hasUnsupportedTSQuery := func() bool { + if !tsQueryNotSupported { return false } // Check if any of the indexes have trigrams involved. for _, def := range stmt.Defs { - if idx, ok := def.(*tree.IndexTableDef); ok && idx.Inverted { - lastColumn := idx.Columns[len(idx.Columns)-1] - switch lastColumn.OpClass { - case "gin_trgm_ops", "gist_trgm_ops": - return true - } - } - } - return false - }() - - invisibleIndexesIsNotSupported, err := isClusterVersionLessThan( - ctx, - tx, - clusterversion.ByKey(clusterversion.TODODelete_V22_2Start)) - if err != nil { - return nil, err - } - hasInvisibleIndexesUnsupported := func() bool { - if !invisibleIndexesIsNotSupported { - return false - } - // Check if any of the indexes have trigrams involved. - for _, def := range stmt.Defs { - if idx, ok := def.(*tree.IndexTableDef); ok && idx.NotVisible { + if col, ok := def.(*tree.ColumnTableDef); ok && + (col.Type.SQLString() == "TSQUERY" || col.Type.SQLString() == "TSVECTOR") { return true } } @@ -1271,8 +1248,7 @@ func (og *operationGenerator) createTable(ctx context.Context, tx pgx.Tx) (*opSt // Compatibility errors aren't guaranteed since the cluster version update is not // fully transaction aware. codesWithConditions{ - {code: pgcode.FeatureNotSupported, condition: hasTrigramIdxUnsupported}, - {code: pgcode.Syntax, condition: hasInvisibleIndexesUnsupported}, + {code: pgcode.Syntax, condition: hasUnsupportedTSQuery}, }.add(opStmt.potentialExecErrors) opStmt.sql = tree.Serialize(stmt) return opStmt, nil From e135c6754bd47d1de83a3c0976659b0a74b48243 Mon Sep 17 00:00:00 2001 From: j82w Date: Thu, 9 Feb 2023 10:10:59 -0500 Subject: [PATCH 2/3] ui: add statement fingerprint to insights 1. Removes contention events from insights. This avoids tracking and storing duplicate information. 2. Add database_name, schema_name, table_name, index_name, contending_pretty_key to crdb_internal.transaction_contention_events. This avoids needing to join with multiple tables and makes it easier for users to understand. 3. Add waiting statement info to the insights transaction details page. This way if users have a large transaction with multiple statements with multiple contention events they can see which specific statement was waited on. 4. Add blocking transaction fingerprint to the insight statement details page. The user can now see which transaction is blocking the statement ,and it has a link to the blocking transaction activity page. closes: #91665 Release note (ui change): Add waiting statement id and fingerprint to insights transaction details page. Add blocking transaction id and fingerprint to the insights statement page. --- pkg/sql/crdb_internal.go | 129 +++++++------ pkg/sql/crdb_internal_test.go | 30 ++- .../testdata/logic_test/crdb_internal_catalog | 6 +- pkg/sql/sqlstats/insights/insights.proto | 3 +- .../insights/integration/insights_test.go | 28 +-- .../sqlstatsutil/json_encoding.go | 21 --- .../sqlstatsutil/json_impl.go | 44 ----- pkg/sql/sqlstats/ssmemstorage/BUILD.bazel | 1 - .../sqlstats/ssmemstorage/ss_mem_writer.go | 4 - .../cluster-ui/src/api/contentionApi.ts | 173 ++++++++++++++++++ .../cluster-ui/src/api/stmtInsightsApi.ts | 68 ++++--- .../cluster-ui/src/api/txnInsightsApi.ts | 139 +++----------- .../cluster-ui/src/insights/types.ts | 26 ++- .../cluster-ui/src/insights/utils.spec.ts | 26 +-- .../insightDetailsTables.tsx | 77 +++++--- .../statementInsightDetailsOverviewTab.tsx | 3 +- .../transactionInsightDetailsOverviewTab.tsx | 44 +++-- .../workloadInsights/util/insightsColumns.tsx | 16 ++ 18 files changed, 467 insertions(+), 371 deletions(-) create mode 100644 pkg/ui/workspaces/cluster-ui/src/api/contentionApi.ts diff --git a/pkg/sql/crdb_internal.go b/pkg/sql/crdb_internal.go index a97d76ec9dd7..b5de846b294c 100644 --- a/pkg/sql/crdb_internal.go +++ b/pkg/sql/crdb_internal.go @@ -57,6 +57,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/catalog/systemschema" "github.com/cockroachdb/cockroach/pkg/sql/catalog/typedesc" "github.com/cockroachdb/cockroach/pkg/sql/clusterunique" + "github.com/cockroachdb/cockroach/pkg/sql/contentionpb" "github.com/cockroachdb/cockroach/pkg/sql/idxusage" "github.com/cockroachdb/cockroach/pkg/sql/isql" "github.com/cockroachdb/cockroach/pkg/sql/parser" @@ -6473,9 +6474,15 @@ CREATE TABLE crdb_internal.transaction_contention_events ( contention_duration INTERVAL NOT NULL, contending_key BYTES NOT NULL, - - waiting_stmt_id string NOT NULL, - waiting_stmt_fingerprint_id BYTES NOT NULL + contending_pretty_key STRING NOT NULL, + + waiting_stmt_id string NOT NULL, + waiting_stmt_fingerprint_id BYTES NOT NULL, + + database_name STRING NOT NULL, + schema_name STRING NOT NULL, + table_name STRING NOT NULL, + index_name STRING );`, generator: func(ctx context.Context, p *planner, db catalog.DatabaseDescriptor, stopper *stop.Stopper) (virtualTableGenerator, cleanupFunc, error) { // Check permission first before making RPC fanout. @@ -6541,10 +6548,13 @@ CREATE TABLE crdb_internal.transaction_contention_events ( types.DefaultIntervalTypeMetadata, ) + contendingPrettyKey := tree.NewDString("") contendingKey := tree.NewDBytes("") if !shouldRedactContendingKey { + decodedKey, _, _ := keys.DecodeTenantPrefix(resp.Events[i].BlockingEvent.Key) + contendingPrettyKey = tree.NewDString(keys.PrettyPrint(nil /* valDirs */, decodedKey)) contendingKey = tree.NewDBytes( - tree.DBytes(resp.Events[i].BlockingEvent.Key)) + tree.DBytes(decodedKey)) } waitingStmtFingerprintID := tree.NewDBytes( @@ -6552,17 +6562,27 @@ CREATE TABLE crdb_internal.transaction_contention_events ( waitingStmtId := tree.NewDString(hex.EncodeToString(resp.Events[i].WaitingStmtID.GetBytes())) + schemaName, dbName, tableName, indexName, err := getContentionEventInfo(ctx, p, resp.Events[i]) + if err != nil { + return err + } + row = row[:0] row = append(row, collectionTs, // collection_ts tree.NewDUuid(tree.DUuid{UUID: resp.Events[i].BlockingEvent.TxnMeta.ID}), // blocking_txn_id blockingFingerprintID, // blocking_fingerprint_id tree.NewDUuid(tree.DUuid{UUID: resp.Events[i].WaitingTxnID}), // waiting_txn_id - waitingFingerprintID, // waiting_fingerprint_id - contentionDuration, // contention_duration - contendingKey, // contending_key, - waitingStmtId, // waiting_stmt_id - waitingStmtFingerprintID, // waiting_stmt_fingerprint_id + waitingFingerprintID, // waiting_fingerprint_id + contentionDuration, // contention_duration + contendingKey, // contending_key, + contendingPrettyKey, // contending_pretty_key + waitingStmtId, // waiting_stmt_id + waitingStmtFingerprintID, // waiting_stmt_fingerprint_id + tree.NewDString(dbName), // database_name + tree.NewDString(schemaName), // schema_name + tree.NewDString(tableName), // table_name + tree.NewDString(indexName), // index_name ) if err = pusher.pushRow(row...); err != nil { @@ -7207,7 +7227,6 @@ CREATE TABLE crdb_internal.%s ( last_retry_reason STRING, exec_node_ids INT[] NOT NULL, contention INTERVAL, - contention_events JSONB, index_recommendations STRING[] NOT NULL, implicit_txn BOOL NOT NULL, cpu_sql_nanos INT8 @@ -7295,17 +7314,6 @@ func populateStmtInsights( ) } - contentionEvents := tree.DNull - if len(s.ContentionEvents) > 0 { - var contentionEventsJSON json.JSON - contentionEventsJSON, err = convertContentionEventsToJSON(ctx, p, s.ContentionEvents) - if err != nil { - return err - } - - contentionEvents = tree.NewDJSON(contentionEventsJSON) - } - indexRecommendations := tree.NewDArray(types.String) for _, recommendation := range s.IndexRecommendations { if err = indexRecommendations.Append(tree.NewDString(recommendation)); err != nil { @@ -7337,7 +7345,6 @@ func populateStmtInsights( autoRetryReason, execNodeIDs, contentionTime, - contentionEvents, indexRecommendations, tree.MakeDBool(tree.DBool(insight.Transaction.ImplicitTxn)), tree.NewDInt(tree.DInt(s.CPUSQLNanos)), @@ -7347,57 +7354,45 @@ func populateStmtInsights( return } -func convertContentionEventsToJSON( - ctx context.Context, p *planner, contentionEvents []roachpb.ContentionEvent, -) (json json.JSON, err error) { +func getContentionEventInfo( + ctx context.Context, p *planner, contentionEvent contentionpb.ExtendedContentionEvent, +) (schemaName, dbName, tableName, indexName string, err error) { - eventWithNames := make([]sqlstatsutil.ContentionEventWithNames, len(contentionEvents)) - for i, contentionEvent := range contentionEvents { - _, tableID, err := p.ExecCfg().Codec.DecodeTablePrefix(contentionEvent.Key) - if err != nil { - return nil, err - } - _, _, indexID, err := p.ExecCfg().Codec.DecodeIndexPrefix(contentionEvent.Key) - if err != nil { - return nil, err - } - - desc := p.Descriptors() - var tableDesc catalog.TableDescriptor - tableDesc, err = desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Table(ctx, descpb.ID(tableID)) - if err != nil { - return nil, err - } + _, tableID, err := p.ExecCfg().Codec.DecodeTablePrefix(contentionEvent.BlockingEvent.Key) + if err != nil { + return "", "", "", "", err + } + _, _, indexID, err := p.ExecCfg().Codec.DecodeIndexPrefix(contentionEvent.BlockingEvent.Key) + if err != nil { + return "", "", "", "", err + } - idxDesc, err := catalog.MustFindIndexByID(tableDesc, descpb.IndexID(indexID)) - if err != nil { - return nil, err - } + desc := p.Descriptors() + var tableDesc catalog.TableDescriptor + tableDesc, err = desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Table(ctx, descpb.ID(tableID)) + if err != nil { + return "", "", "", "", err + } - dbDesc, err := desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Database(ctx, tableDesc.GetParentID()) - if err != nil { - return nil, err - } + idxDesc, err := catalog.MustFindIndexByID(tableDesc, descpb.IndexID(indexID)) + if err != nil { + return "", "", "", "", err + } - schemaDesc, err := desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Schema(ctx, tableDesc.GetParentSchemaID()) - if err != nil { - return nil, err - } + dbDesc, err := desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Database(ctx, tableDesc.GetParentID()) + if err != nil { + return "", "", "", "", err + } - var idxName string - if idxDesc != nil { - idxName = idxDesc.GetName() - } + schemaDesc, err := desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Schema(ctx, tableDesc.GetParentSchemaID()) + if err != nil { + return "", "", "", "", err + } - eventWithNames[i] = sqlstatsutil.ContentionEventWithNames{ - BlockingTransactionID: contentionEvent.TxnMeta.ID.String(), - SchemaName: schemaDesc.GetName(), - DatabaseName: dbDesc.GetName(), - TableName: tableDesc.GetName(), - IndexName: idxName, - DurationInMs: float64(contentionEvent.Duration) / float64(time.Millisecond), - } + var idxName string + if idxDesc != nil { + idxName = idxDesc.GetName() } - return sqlstatsutil.BuildContentionEventsJSON(eventWithNames) + return schemaDesc.GetName(), dbDesc.GetName(), tableDesc.GetName(), idxName, nil } diff --git a/pkg/sql/crdb_internal_test.go b/pkg/sql/crdb_internal_test.go index 603c84fc5c55..f7c8c3b0c13a 100644 --- a/pkg/sql/crdb_internal_test.go +++ b/pkg/sql/crdb_internal_test.go @@ -982,7 +982,12 @@ func TestTxnContentionEventsTable(t *testing.T) { waiting_stmt_id, encode( waiting_txn_fingerprint_id, 'hex' - ) AS waiting_txn_fingerprint_id + ) AS waiting_txn_fingerprint_id, + contending_pretty_key, + database_name, + schema_name, + table_name, + index_name FROM crdb_internal.transaction_contention_events tce inner join ( select @@ -1001,7 +1006,8 @@ func TestTxnContentionEventsTable(t *testing.T) { rowCount++ var blockingTxnId, waitingTxnId, waitingStmtId, waitingStmtFingerprint string - errVerify = rows.Scan(&blockingTxnId, &waitingTxnId, &waitingStmtId, &waitingStmtFingerprint) + var prettyKey, dbName, schemaName, tableName, indexName string + errVerify = rows.Scan(&blockingTxnId, &waitingTxnId, &waitingStmtId, &waitingStmtFingerprint, &prettyKey, &dbName, &schemaName, &tableName, &indexName) if errVerify != nil { return errVerify } @@ -1015,8 +1021,24 @@ func TestTxnContentionEventsTable(t *testing.T) { return fmt.Errorf("transaction_contention_events had default waiting txn id %s, blocking txn id %s", waitingTxnId, blockingTxnId) } - if waitingStmtId == defaultIdString { - return fmt.Errorf("transaction_contention_events had default waiting stmt id %s, blocking txn id %s, waiting txn id %s", waitingStmtId, blockingTxnId, waitingTxnId) + if !strings.HasPrefix(prettyKey, "/Table/") { + return fmt.Errorf("prettyKey should be defaultdb: %s, %s, %s, %s, %s", prettyKey, dbName, schemaName, tableName, indexName) + } + + if dbName != "defaultdb" { + return fmt.Errorf("dbName should be defaultdb: %s, %s, %s, %s, %s", prettyKey, dbName, schemaName, tableName, indexName) + } + + if schemaName != "public" { + return fmt.Errorf("schemaName should be public: %s, %s, %s, %s, %s", prettyKey, dbName, schemaName, tableName, indexName) + } + + if tableName != "t" { + return fmt.Errorf("tableName should be t: %s, %s, %s, %s, %s", prettyKey, dbName, schemaName, tableName, indexName) + } + + if indexName != "t_pkey" { + return fmt.Errorf("indexName should be t_pkey: %s, %s, %s, %s, %s", prettyKey, dbName, schemaName, tableName, indexName) } } diff --git a/pkg/sql/logictest/testdata/logic_test/crdb_internal_catalog b/pkg/sql/logictest/testdata/logic_test/crdb_internal_catalog index e2f38e77e3b7..14f0a502a9a8 100644 --- a/pkg/sql/logictest/testdata/logic_test/crdb_internal_catalog +++ b/pkg/sql/logictest/testdata/logic_test/crdb_internal_catalog @@ -420,7 +420,7 @@ SELECT id, strip_volatile(descriptor) FROM crdb_internal.kv_catalog_descriptor 4294967245 {"table": {"columns": [{"id": 1, "name": "node_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "session_id", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 3, "name": "user_name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "client_address", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 5, "name": "application_name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 6, "name": "active_queries", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 7, "name": "last_active_query", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 8, "name": "num_txns_executed", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 9, "name": "session_start", "nullable": true, "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 10, "name": "active_query_start", "nullable": true, "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 11, "name": "kv_txn", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 12, "name": "alloc_bytes", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 13, "name": "max_alloc_bytes", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 14, "name": "status", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 15, "name": "session_end", "nullable": true, "type": {"family": "TimestampFamily", "oid": 1114}}], "formatVersion": 3, "id": 4294967245, "name": "node_sessions", "nextColumnId": 16, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967246 {"table": {"columns": [{"id": 1, "name": "id", "nullable": true, "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 2, "name": "node_id", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "session_id", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "start", "nullable": true, "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 5, "name": "txn_string", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 6, "name": "application_name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 7, "name": "num_stmts", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 8, "name": "num_retries", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 9, "name": "num_auto_retries", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 10, "name": "last_auto_retry_reason", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}], "formatVersion": 3, "id": 4294967246, "name": "node_transactions", "nextColumnId": 11, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967247 {"table": {"columns": [{"id": 1, "name": "query_id", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 2, "name": "txn_id", "nullable": true, "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 3, "name": "node_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 4, "name": "session_id", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 5, "name": "user_name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 6, "name": "start", "nullable": true, "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 7, "name": "query", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 8, "name": "client_address", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "application_name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 10, "name": "distributed", "nullable": true, "type": {"oid": 16}}, {"id": 11, "name": "phase", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 12, "name": "full_scan", "nullable": true, "type": {"oid": 16}}, {"id": 13, "name": "plan_gist", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 14, "name": "database", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}], "formatVersion": 3, "id": 4294967247, "name": "node_queries", "nextColumnId": 15, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} -4294967248 {"table": {"columns": [{"id": 1, "name": "session_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 2, "name": "txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 3, "name": "txn_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 4, "name": "stmt_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 5, "name": "stmt_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 6, "name": "problem", "type": {"family": "StringFamily", "oid": 25}}, {"id": 7, "name": "causes", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 8, "name": "query", "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "status", "type": {"family": "StringFamily", "oid": 25}}, {"id": 10, "name": "start_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 11, "name": "end_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 12, "name": "full_scan", "type": {"oid": 16}}, {"id": 13, "name": "user_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 14, "name": "app_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 15, "name": "database_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 16, "name": "plan_gist", "type": {"family": "StringFamily", "oid": 25}}, {"id": 17, "name": "rows_read", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 18, "name": "rows_written", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 19, "name": "priority", "type": {"family": "StringFamily", "oid": 25}}, {"id": 20, "name": "retries", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 21, "name": "last_retry_reason", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 22, "name": "exec_node_ids", "type": {"arrayContents": {"family": "IntFamily", "oid": 20, "width": 64}, "arrayElemType": "IntFamily", "family": "ArrayFamily", "oid": 1016, "width": 64}}, {"id": 23, "name": "contention", "nullable": true, "type": {"family": "IntervalFamily", "intervalDurationField": {}, "oid": 1186}}, {"id": 24, "name": "contention_events", "nullable": true, "type": {"family": "JsonFamily", "oid": 3802}}, {"id": 25, "name": "index_recommendations", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 26, "name": "implicit_txn", "type": {"oid": 16}}, {"id": 27, "name": "cpu_sql_nanos", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 4294967248, "name": "node_execution_insights", "nextColumnId": 28, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} +4294967248 {"table": {"columns": [{"id": 1, "name": "session_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 2, "name": "txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 3, "name": "txn_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 4, "name": "stmt_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 5, "name": "stmt_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 6, "name": "problem", "type": {"family": "StringFamily", "oid": 25}}, {"id": 7, "name": "causes", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 8, "name": "query", "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "status", "type": {"family": "StringFamily", "oid": 25}}, {"id": 10, "name": "start_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 11, "name": "end_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 12, "name": "full_scan", "type": {"oid": 16}}, {"id": 13, "name": "user_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 14, "name": "app_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 15, "name": "database_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 16, "name": "plan_gist", "type": {"family": "StringFamily", "oid": 25}}, {"id": 17, "name": "rows_read", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 18, "name": "rows_written", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 19, "name": "priority", "type": {"family": "StringFamily", "oid": 25}}, {"id": 20, "name": "retries", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 21, "name": "last_retry_reason", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 22, "name": "exec_node_ids", "type": {"arrayContents": {"family": "IntFamily", "oid": 20, "width": 64}, "arrayElemType": "IntFamily", "family": "ArrayFamily", "oid": 1016, "width": 64}}, {"id": 23, "name": "contention", "nullable": true, "type": {"family": "IntervalFamily", "intervalDurationField": {}, "oid": 1186}}, {"id": 24, "name": "index_recommendations", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 25, "name": "implicit_txn", "type": {"oid": 16}}, {"id": 26, "name": "cpu_sql_nanos", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 4294967248, "name": "node_execution_insights", "nextColumnId": 27, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967249 {"table": {"columns": [{"id": 1, "name": "flow_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 2, "name": "node_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "stmt", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "since", "type": {"family": "TimestampTZFamily", "oid": 1184}}, {"id": 5, "name": "status", "type": {"family": "StringFamily", "oid": 25}}], "formatVersion": 3, "id": 4294967249, "name": "node_distsql_flows", "nextColumnId": 6, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967250 {"table": {"columns": [{"id": 1, "name": "table_id", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "index_id", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "num_contention_events", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 4, "name": "cumulative_contention_time", "type": {"family": "IntervalFamily", "intervalDurationField": {}, "oid": 1186}}, {"id": 5, "name": "key", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 6, "name": "txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 7, "name": "count", "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 4294967250, "name": "node_contention_events", "nextColumnId": 8, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967251 {"table": {"columns": [{"id": 1, "name": "node_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "table_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "parent_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 5, "name": "expiration", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 6, "name": "deleted", "type": {"oid": 16}}], "formatVersion": 3, "id": 4294967251, "name": "leases", "nextColumnId": 7, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} @@ -432,7 +432,7 @@ SELECT id, strip_volatile(descriptor) FROM crdb_internal.kv_catalog_descriptor 4294967257 {"table": {"columns": [{"id": 1, "name": "table_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "index_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "total_reads", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 4, "name": "last_read", "nullable": true, "type": {"family": "TimestampTZFamily", "oid": 1184}}], "formatVersion": 3, "id": 4294967257, "name": "index_usage_statistics", "nextColumnId": 5, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967258 {"table": {"columns": [{"id": 1, "name": "descriptor_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "index_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "start_key", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 4, "name": "end_key", "type": {"family": "BytesFamily", "oid": 17}}], "formatVersion": 3, "id": 4294967258, "indexes": [{"foreignKey": {}, "geoConfig": {}, "id": 2, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [1], "keyColumnNames": ["descriptor_id"], "name": "index_spans_descriptor_id_idx", "partitioning": {}, "sharded": {}, "storeColumnIds": [2, 3, 4], "storeColumnNames": ["index_id", "start_key", "end_key"], "version": 3}], "name": "index_spans", "nextColumnId": 5, "nextConstraintId": 2, "nextIndexId": 3, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967259 {"table": {"columns": [{"id": 1, "name": "descriptor_id", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "descriptor_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 3, "name": "index_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 4, "name": "index_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 5, "name": "column_type", "type": {"family": "StringFamily", "oid": 25}}, {"id": 6, "name": "column_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 7, "name": "column_name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 8, "name": "column_direction", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "implicit", "nullable": true, "type": {"oid": 16}}], "formatVersion": 3, "id": 4294967259, "name": "index_columns", "nextColumnId": 10, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} -4294967260 {"table": {"columns": [{"id": 1, "name": "collection_ts", "type": {"family": "TimestampTZFamily", "oid": 1184}}, {"id": 2, "name": "blocking_txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 3, "name": "blocking_txn_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 4, "name": "waiting_txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 5, "name": "waiting_txn_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 6, "name": "contention_duration", "type": {"family": "IntervalFamily", "intervalDurationField": {}, "oid": 1186}}, {"id": 7, "name": "contending_key", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 8, "name": "waiting_stmt_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "waiting_stmt_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}], "formatVersion": 3, "id": 4294967260, "name": "transaction_contention_events", "nextColumnId": 10, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} +4294967260 {"table": {"columns": [{"id": 1, "name": "collection_ts", "type": {"family": "TimestampTZFamily", "oid": 1184}}, {"id": 2, "name": "blocking_txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 3, "name": "blocking_txn_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 4, "name": "waiting_txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 5, "name": "waiting_txn_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 6, "name": "contention_duration", "type": {"family": "IntervalFamily", "intervalDurationField": {}, "oid": 1186}}, {"id": 7, "name": "contending_key", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 8, "name": "contending_pretty_key", "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "waiting_stmt_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 10, "name": "waiting_stmt_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 11, "name": "database_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 12, "name": "schema_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 13, "name": "table_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 14, "name": "index_name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}], "formatVersion": 3, "id": 4294967260, "name": "transaction_contention_events", "nextColumnId": 15, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967261 {"table": {"columns": [{"id": 1, "name": "source_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "target_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 4294967261, "name": "gossip_network", "nextColumnId": 3, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967262 {"table": {"columns": [{"id": 1, "name": "node_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "epoch", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "expiration", "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "draining", "type": {"oid": 16}}, {"id": 5, "name": "decommissioning", "type": {"oid": 16}}, {"id": 6, "name": "membership", "type": {"family": "StringFamily", "oid": 25}}, {"id": 7, "name": "updated_at", "nullable": true, "type": {"family": "TimestampFamily", "oid": 1114}}], "formatVersion": 3, "id": 4294967262, "name": "gossip_liveness", "nextColumnId": 8, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967263 {"table": {"columns": [{"id": 1, "name": "node_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "store_id", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "category", "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "description", "type": {"family": "StringFamily", "oid": 25}}, {"id": 5, "name": "value", "type": {"family": "FloatFamily", "oid": 701, "width": 64}}], "formatVersion": 3, "id": 4294967263, "name": "gossip_alerts", "nextColumnId": 6, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} @@ -454,7 +454,7 @@ SELECT id, strip_volatile(descriptor) FROM crdb_internal.kv_catalog_descriptor 4294967279 {"table": {"columns": [{"id": 1, "name": "range_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "table_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "database_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "schema_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 5, "name": "table_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 6, "name": "index_name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 7, "name": "lock_key", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 8, "name": "lock_key_pretty", "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "txn_id", "nullable": true, "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 10, "name": "ts", "nullable": true, "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 11, "name": "lock_strength", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 12, "name": "durability", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 13, "name": "granted", "nullable": true, "type": {"oid": 16}}, {"id": 14, "name": "contended", "type": {"oid": 16}}, {"id": 15, "name": "duration", "nullable": true, "type": {"family": "IntervalFamily", "intervalDurationField": {}, "oid": 1186}}], "formatVersion": 3, "id": 4294967279, "indexes": [{"foreignKey": {}, "geoConfig": {}, "id": 2, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [2], "keyColumnNames": ["table_id"], "name": "cluster_locks_table_id_idx", "partitioning": {}, "sharded": {}, "storeColumnIds": [1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], "storeColumnNames": ["range_id", "database_name", "schema_name", "table_name", "index_name", "lock_key", "lock_key_pretty", "txn_id", "ts", "lock_strength", "durability", "granted", "contended", "duration"], "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 3, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [3], "keyColumnNames": ["database_name"], "name": "cluster_locks_database_name_idx", "partitioning": {}, "sharded": {}, "storeColumnIds": [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], "storeColumnNames": ["range_id", "table_id", "schema_name", "table_name", "index_name", "lock_key", "lock_key_pretty", "txn_id", "ts", "lock_strength", "durability", "granted", "contended", "duration"], "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 4, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [5], "keyColumnNames": ["table_name"], "name": "cluster_locks_table_name_idx", "partitioning": {}, "sharded": {}, "storeColumnIds": [1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], "storeColumnNames": ["range_id", "table_id", "database_name", "schema_name", "index_name", "lock_key", "lock_key_pretty", "txn_id", "ts", "lock_strength", "durability", "granted", "contended", "duration"], "version": 3}, {"foreignKey": {}, "geoConfig": {}, "id": 5, "interleave": {}, "keyColumnDirections": ["ASC"], "keyColumnIds": [14], "keyColumnNames": ["contended"], "name": "cluster_locks_contended_idx", "partitioning": {}, "sharded": {}, "storeColumnIds": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 15], "storeColumnNames": ["range_id", "table_id", "database_name", "schema_name", "table_name", "index_name", "lock_key", "lock_key_pretty", "txn_id", "ts", "lock_strength", "durability", "granted", "duration"], "version": 3}], "name": "cluster_locks", "nextColumnId": 16, "nextConstraintId": 2, "nextIndexId": 6, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967280 {"table": {"columns": [{"id": 1, "name": "txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 2, "name": "txn_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 3, "name": "query", "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "implicit_txn", "type": {"oid": 16}}, {"id": 5, "name": "session_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 6, "name": "start_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 7, "name": "end_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 8, "name": "user_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "app_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 10, "name": "rows_read", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 11, "name": "rows_written", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 12, "name": "priority", "type": {"family": "StringFamily", "oid": 25}}, {"id": 13, "name": "retries", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 14, "name": "last_retry_reason", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 15, "name": "contention", "nullable": true, "type": {"family": "IntervalFamily", "intervalDurationField": {}, "oid": 1186}}, {"id": 16, "name": "problems", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 17, "name": "causes", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 18, "name": "stmt_execution_ids", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 19, "name": "cpu_sql_nanos", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 4294967280, "name": "node_txn_execution_insights", "nextColumnId": 20, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967281 {"table": {"columns": [{"id": 1, "name": "txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 2, "name": "txn_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 3, "name": "query", "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "implicit_txn", "type": {"oid": 16}}, {"id": 5, "name": "session_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 6, "name": "start_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 7, "name": "end_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 8, "name": "user_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "app_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 10, "name": "rows_read", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 11, "name": "rows_written", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 12, "name": "priority", "type": {"family": "StringFamily", "oid": 25}}, {"id": 13, "name": "retries", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 14, "name": "last_retry_reason", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 15, "name": "contention", "nullable": true, "type": {"family": "IntervalFamily", "intervalDurationField": {}, "oid": 1186}}, {"id": 16, "name": "problems", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 17, "name": "causes", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 18, "name": "stmt_execution_ids", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 19, "name": "cpu_sql_nanos", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 4294967281, "name": "cluster_txn_execution_insights", "nextColumnId": 20, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} -4294967282 {"table": {"columns": [{"id": 1, "name": "session_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 2, "name": "txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 3, "name": "txn_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 4, "name": "stmt_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 5, "name": "stmt_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 6, "name": "problem", "type": {"family": "StringFamily", "oid": 25}}, {"id": 7, "name": "causes", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 8, "name": "query", "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "status", "type": {"family": "StringFamily", "oid": 25}}, {"id": 10, "name": "start_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 11, "name": "end_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 12, "name": "full_scan", "type": {"oid": 16}}, {"id": 13, "name": "user_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 14, "name": "app_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 15, "name": "database_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 16, "name": "plan_gist", "type": {"family": "StringFamily", "oid": 25}}, {"id": 17, "name": "rows_read", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 18, "name": "rows_written", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 19, "name": "priority", "type": {"family": "StringFamily", "oid": 25}}, {"id": 20, "name": "retries", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 21, "name": "last_retry_reason", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 22, "name": "exec_node_ids", "type": {"arrayContents": {"family": "IntFamily", "oid": 20, "width": 64}, "arrayElemType": "IntFamily", "family": "ArrayFamily", "oid": 1016, "width": 64}}, {"id": 23, "name": "contention", "nullable": true, "type": {"family": "IntervalFamily", "intervalDurationField": {}, "oid": 1186}}, {"id": 24, "name": "contention_events", "nullable": true, "type": {"family": "JsonFamily", "oid": 3802}}, {"id": 25, "name": "index_recommendations", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 26, "name": "implicit_txn", "type": {"oid": 16}}, {"id": 27, "name": "cpu_sql_nanos", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 4294967282, "name": "cluster_execution_insights", "nextColumnId": 28, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} +4294967282 {"table": {"columns": [{"id": 1, "name": "session_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 2, "name": "txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 3, "name": "txn_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 4, "name": "stmt_id", "type": {"family": "StringFamily", "oid": 25}}, {"id": 5, "name": "stmt_fingerprint_id", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 6, "name": "problem", "type": {"family": "StringFamily", "oid": 25}}, {"id": 7, "name": "causes", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 8, "name": "query", "type": {"family": "StringFamily", "oid": 25}}, {"id": 9, "name": "status", "type": {"family": "StringFamily", "oid": 25}}, {"id": 10, "name": "start_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 11, "name": "end_time", "type": {"family": "TimestampFamily", "oid": 1114}}, {"id": 12, "name": "full_scan", "type": {"oid": 16}}, {"id": 13, "name": "user_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 14, "name": "app_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 15, "name": "database_name", "type": {"family": "StringFamily", "oid": 25}}, {"id": 16, "name": "plan_gist", "type": {"family": "StringFamily", "oid": 25}}, {"id": 17, "name": "rows_read", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 18, "name": "rows_written", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 19, "name": "priority", "type": {"family": "StringFamily", "oid": 25}}, {"id": 20, "name": "retries", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 21, "name": "last_retry_reason", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 22, "name": "exec_node_ids", "type": {"arrayContents": {"family": "IntFamily", "oid": 20, "width": 64}, "arrayElemType": "IntFamily", "family": "ArrayFamily", "oid": 1016, "width": 64}}, {"id": 23, "name": "contention", "nullable": true, "type": {"family": "IntervalFamily", "intervalDurationField": {}, "oid": 1186}}, {"id": 24, "name": "index_recommendations", "type": {"arrayContents": {"family": "StringFamily", "oid": 25}, "arrayElemType": "StringFamily", "family": "ArrayFamily", "oid": 1009}}, {"id": 25, "name": "implicit_txn", "type": {"oid": 16}}, {"id": 26, "name": "cpu_sql_nanos", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 4294967282, "name": "cluster_execution_insights", "nextColumnId": 27, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967283 {"table": {"columns": [{"id": 1, "name": "flow_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 2, "name": "node_id", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "stmt", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "since", "type": {"family": "TimestampTZFamily", "oid": 1184}}, {"id": 5, "name": "status", "type": {"family": "StringFamily", "oid": 25}}], "formatVersion": 3, "id": 4294967283, "name": "cluster_distsql_flows", "nextColumnId": 6, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967284 {"table": {"columns": [{"id": 1, "name": "table_id", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 2, "name": "index_id", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 3, "name": "num_contention_events", "type": {"family": "IntFamily", "oid": 20, "width": 64}}, {"id": 4, "name": "cumulative_contention_time", "type": {"family": "IntervalFamily", "intervalDurationField": {}, "oid": 1186}}, {"id": 5, "name": "key", "type": {"family": "BytesFamily", "oid": 17}}, {"id": 6, "name": "txn_id", "type": {"family": "UuidFamily", "oid": 2950}}, {"id": 7, "name": "count", "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 4294967284, "name": "cluster_contention_events", "nextColumnId": 8, "nextConstraintId": 2, "nextIndexId": 2, "nextMutationId": 1, "primaryIndex": {"constraintId": 1, "foreignKey": {}, "geoConfig": {}, "id": 1, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1"}} 4294967285 {"table": {"columns": [{"id": 1, "name": "database_name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 2, "name": "schema_name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 3, "name": "table_name", "nullable": true, "type": {"family": "StringFamily", "oid": 25}}, {"id": 4, "name": "num_contention_events", "nullable": true, "type": {"family": "IntFamily", "oid": 20, "width": 64}}], "formatVersion": 3, "id": 4294967285, "name": "cluster_contended_tables", "nextColumnId": 5, "nextConstraintId": 1, "nextMutationId": 1, "primaryIndex": {"foreignKey": {}, "geoConfig": {}, "interleave": {}, "partitioning": {}, "sharded": {}}, "privileges": {"ownerProto": "node", "users": [{"privileges": "32", "userProto": "public"}], "version": 2}, "replacementOf": {"time": {}}, "unexposedParentSchemaId": 4294967295, "version": "1", "viewQuery": "SELECT database_name, schema_name, name, sum(num_contention_events) FROM (SELECT DISTINCT database_name, schema_name, name, index_id, num_contention_events FROM crdb_internal.cluster_contention_events JOIN crdb_internal.tables ON crdb_internal.cluster_contention_events.table_id = crdb_internal.tables.table_id) GROUP BY database_name, schema_name, name"}} diff --git a/pkg/sql/sqlstats/insights/insights.proto b/pkg/sql/sqlstats/insights/insights.proto index effb8556a8e3..36e5e76df179 100644 --- a/pkg/sql/sqlstats/insights/insights.proto +++ b/pkg/sql/sqlstats/insights/insights.proto @@ -115,8 +115,7 @@ message Statement { repeated int64 nodes = 17; google.protobuf.Duration contention = 18 [(gogoproto.stdduration) = true]; repeated string index_recommendations = 19; - // contention_events hit at the statement level. - repeated cockroach.roachpb.ContentionEvent contention_events = 20 [(gogoproto.nullable) = false]; + reserved 20; // previously contention_events Problem problem = 21; repeated Cause causes = 22; int64 cpu_sql_nanos = 23 [(gogoproto.customname) = "CPUSQLNanos"]; diff --git a/pkg/sql/sqlstats/insights/integration/insights_test.go b/pkg/sql/sqlstats/insights/integration/insights_test.go index 900a405e0a3c..2fe5aca68aac 100644 --- a/pkg/sql/sqlstats/insights/integration/insights_test.go +++ b/pkg/sql/sqlstats/insights/integration/insights_test.go @@ -372,16 +372,17 @@ func TestInsightsIntegrationForContention(t *testing.T) { testutils.SucceedsWithin(t, func() error { rows, err := conn.QueryContext(ctx, `SELECT query, - contention::FLOAT, - contention_events->0->>'durationInMs' AS durationMs, - contention_events->0->>'schemaName', - contention_events->0->>'databaseName', - contention_events->0->>'tableName', - contention_events->0->>'indexName', - txn_contention.blocking_txn_fingerprint_id + insight.contention::FLOAT, + sum(txn_contention.contention_duration)::FLOAT AS durationMs, + txn_contention.schema_name, + txn_contention.database_name, + txn_contention.table_name, + txn_contention.index_name, + txn_contention.waiting_txn_fingerprint_id FROM crdb_internal.cluster_execution_insights insight - left join crdb_internal.transaction_contention_events txn_contention on (contention_events->0->>'blockingTxnID')::uuid = txn_contention.blocking_txn_id - where query like 'UPDATE t SET s =%'`) + left join crdb_internal.transaction_contention_events txn_contention on insight.stmt_id = txn_contention.waiting_stmt_id + where query like 'UPDATE t SET s =%' + group by query, insight.contention, txn_contention.schema_name, txn_contention.database_name, txn_contention.table_name, txn_contention.index_name, waiting_txn_fingerprint_id;`) if err != nil { return err } @@ -393,19 +394,18 @@ func TestInsightsIntegrationForContention(t *testing.T) { return err } - var totalContentionFromQuerySeconds, contentionFromEventMs float64 + var totalContentionFromQueryMs, contentionFromEventMs float64 var queryText, schemaName, dbName, tableName, indexName string var blockingTxnFingerprintID gosql.NullString - err = rows.Scan(&queryText, &totalContentionFromQuerySeconds, &contentionFromEventMs, &schemaName, &dbName, &tableName, &indexName, &blockingTxnFingerprintID) + err = rows.Scan(&queryText, &totalContentionFromQueryMs, &contentionFromEventMs, &schemaName, &dbName, &tableName, &indexName, &blockingTxnFingerprintID) if err != nil { return err } - if totalContentionFromQuerySeconds < .2 { - return fmt.Errorf("contention time is %f should be greater than .2 since block is delayed by .5 seconds", totalContentionFromQuerySeconds) + if totalContentionFromQueryMs < .2 { + return fmt.Errorf("contention time is %f should be greater than .2 since block is delayed by .5 seconds", totalContentionFromQueryMs) } - totalContentionFromQueryMs := totalContentionFromQuerySeconds * 1000 diff := totalContentionFromQueryMs - contentionFromEventMs if math.Abs(diff) > .1 { return fmt.Errorf("contention time from column: %f should be the same as event value %f", totalContentionFromQueryMs, contentionFromEventMs) diff --git a/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_encoding.go b/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_encoding.go index 5b6d76b5adde..2fc6be743f56 100644 --- a/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_encoding.go +++ b/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_encoding.go @@ -311,27 +311,6 @@ func BuildStmtDetailsMetadataJSON( return (*aggregatedMetadata)(metadata).jsonFields().encodeJSON() } -// BuildContentionEventsJSON returns a json.JSON object for contention events -// roachpb.ContentionEvent. -// JSON Schema for contention events -// -// { -// "$schema": "https://json-schema.org/draft/2020-12/schema", -// "title": "system.statement_statistics.contention_events", -// "type": "object", -// [{ -// "blockingTxnID": { "type": "string" }, -// "durationInMs": { "type": "number" }, -// "schemaName": { "type": "string" }, -// "databaseName": { "type": "string" }, -// "tableName": { "type": "string" }, -// "indexName": { "type": "string" } -// }] -// } -func BuildContentionEventsJSON(events []ContentionEventWithNames) (json.JSON, error) { - return (*contentionEvents)(&events).encodeJSON() -} - // EncodeUint64ToBytes returns the []byte representation of an uint64 value. func EncodeUint64ToBytes(id uint64) []byte { result := make([]byte, 0, 8) diff --git a/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_impl.go b/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_impl.go index 1587c43a4ebd..e1c7fb16dcfb 100644 --- a/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_impl.go +++ b/pkg/sql/sqlstats/persistedsqlstats/sqlstatsutil/json_impl.go @@ -124,50 +124,6 @@ func (s *aggregatedMetadata) jsonFields() jsonFields { } } -// ContentionEventWithNames is used to serialize into -// cluster_execution_insights as a JSON array. This type -// has the names instead of ids to avoid doing joins to get -// the user-friendly names. -type ContentionEventWithNames struct { - BlockingTransactionID string - SchemaName string - DatabaseName string - TableName string - IndexName string - DurationInMs float64 -} - -type contentionEvents []ContentionEventWithNames - -func (s *contentionEvents) encodeJSON() (json.JSON, error) { - builder := json.NewArrayBuilder(len(*s)) - - for _, value := range *s { - jsVal := (*contentionEvent)(&value).jsonFields() - jsObj, err := jsVal.encodeJSON() - if err != nil { - return nil, err - } - - builder.Add(jsObj) - } - - return builder.Build(), nil -} - -type contentionEvent ContentionEventWithNames - -func (s *contentionEvent) jsonFields() jsonFields { - return jsonFields{ - {"blockingTxnID", (*jsonString)(&s.BlockingTransactionID)}, - {"durationInMs", (*jsonFloat)(&s.DurationInMs)}, - {"schemaName", (*jsonString)(&s.SchemaName)}, - {"databaseName", (*jsonString)(&s.DatabaseName)}, - {"tableName", (*jsonString)(&s.TableName)}, - {"indexName", (*jsonString)(&s.IndexName)}, - } -} - type int64Array []int64 func (a *int64Array) decodeJSON(js json.JSON) error { diff --git a/pkg/sql/sqlstats/ssmemstorage/BUILD.bazel b/pkg/sql/sqlstats/ssmemstorage/BUILD.bazel index ea7716c6aecc..7df2696d8dd0 100644 --- a/pkg/sql/sqlstats/ssmemstorage/BUILD.bazel +++ b/pkg/sql/sqlstats/ssmemstorage/BUILD.bazel @@ -12,7 +12,6 @@ go_library( importpath = "github.com/cockroachdb/cockroach/pkg/sql/sqlstats/ssmemstorage", visibility = ["//visibility:public"], deps = [ - "//pkg/roachpb", "//pkg/server/serverpb", "//pkg/settings", "//pkg/settings/cluster", diff --git a/pkg/sql/sqlstats/ssmemstorage/ss_mem_writer.go b/pkg/sql/sqlstats/ssmemstorage/ss_mem_writer.go index 2117c5c81797..4888d01475f0 100644 --- a/pkg/sql/sqlstats/ssmemstorage/ss_mem_writer.go +++ b/pkg/sql/sqlstats/ssmemstorage/ss_mem_writer.go @@ -15,7 +15,6 @@ import ( "time" "unsafe" - "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/sql/appstatspb" "github.com/cockroachdb/cockroach/pkg/sql/execstats" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" @@ -194,11 +193,9 @@ func (s *Container) RecordStatement( } var contention *time.Duration - var contentionEvents []roachpb.ContentionEvent var cpuSQLNanos int64 if value.ExecStats != nil { contention = &value.ExecStats.ContentionTime - contentionEvents = value.ExecStats.ContentionEvents cpuSQLNanos = value.ExecStats.CPUTime.Nanoseconds() } @@ -218,7 +215,6 @@ func (s *Container) RecordStatement( RowsWritten: value.RowsWritten, Nodes: value.Nodes, Contention: contention, - ContentionEvents: contentionEvents, IndexRecommendations: value.IndexRecommendations, Database: value.Database, CPUSQLNanos: cpuSQLNanos, diff --git a/pkg/ui/workspaces/cluster-ui/src/api/contentionApi.ts b/pkg/ui/workspaces/cluster-ui/src/api/contentionApi.ts new file mode 100644 index 000000000000..13665eca94e7 --- /dev/null +++ b/pkg/ui/workspaces/cluster-ui/src/api/contentionApi.ts @@ -0,0 +1,173 @@ +// Copyright 2022 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +import { + executeInternalSql, + LARGE_RESULT_SIZE, + LONG_TIMEOUT, + sqlApiErrorMessage, + SqlExecutionRequest, + sqlResultsAreEmpty, +} from "./sqlApi"; +import { ContentionDetails } from "src/insights"; +import moment from "moment"; + +export type ContentionFilters = { + waitingTxnID?: string; + waitingStmtID?: string; + start?: moment.Moment; + end?: moment.Moment; +}; + +export type ContentionResponseColumns = { + waiting_txn_id: string; + waiting_txn_fingerprint_id: string; + collection_ts: string; + contention_duration: string; + blocking_txn_id: string; + blocking_txn_fingerprint_id: string; + waiting_stmt_id: string; + waiting_stmt_fingerprint_id: string; + schema_name: string; + database_name: string; + table_name: string; + index_name: string; + key: string; +}; + +export async function getContentionDetailsApi( + filters?: ContentionFilters, +): Promise { + const request: SqlExecutionRequest = { + statements: [ + { + sql: contentionDetailsQuery(filters), + }, + ], + execute: true, + max_result_size: LARGE_RESULT_SIZE, + timeout: LONG_TIMEOUT, + }; + + const result = await executeInternalSql(request); + if (result.error) { + throw new Error( + `Error while retrieving insights information: ${sqlApiErrorMessage( + result.error.message, + )}`, + ); + } + + if (sqlResultsAreEmpty(result)) { + return []; + } + + const contentionDetails: ContentionDetails[] = []; + result.execution.txn_results.forEach(x => { + x.rows.forEach(row => { + contentionDetails.push({ + blockingExecutionID: row.blocking_txn_id, + blockingTxnFingerprintID: row.blocking_txn_fingerprint_id, + blockingTxnQuery: null, + waitingTxnID: row.waiting_txn_id, + waitingTxnFingerprintID: row.waiting_txn_fingerprint_id, + waitingStmtID: row.waiting_stmt_id, + waitingStmtFingerprintID: row.waiting_stmt_fingerprint_id, + collectionTimeStamp: moment(row.collection_ts).utc(), + contendedKey: row.key, + contentionTimeMs: moment + .duration(row.contention_duration) + .asMilliseconds(), + databaseName: row.database_name, + schemaName: row.schema_name, + tableName: row.table_name, + indexName: + row.index_name && row.index_name !== "" + ? row.index_name + : "index not found", + }); + }); + }); + + return contentionDetails; +} + +function isFiltered(filters: ContentionFilters): boolean { + if (filters == null) { + return false; + } + + return ( + filters.waitingStmtID != null || + filters.waitingTxnID != null || + filters.end != null || + filters.start != null + ); +} + +function getContentionWhereClause(filters?: ContentionFilters): string { + if (!isFiltered(filters)) { + return ""; + } + const defaultWhereClause = " where "; + let whereClause = defaultWhereClause; + if (filters?.waitingStmtID) { + whereClause = + whereClause + ` waiting_stmt_id = '${filters.waitingStmtID}' `; + } + + if (filters?.waitingTxnID) { + if (whereClause != defaultWhereClause) { + whereClause += " and "; + } + whereClause = whereClause + ` waiting_txn_id >= '${filters.waitingTxnID}' `; + } + + if (filters?.start) { + if (whereClause != defaultWhereClause) { + whereClause += " and "; + } + whereClause = + whereClause + ` collection_ts >= '${filters.start.toISOString()}' `; + } + if (filters?.end) { + if (whereClause != defaultWhereClause) { + whereClause += " and "; + } + + whereClause = + whereClause + + ` (collection_ts + contention_duration) <= '${filters.end.toISOString()}' `; + } + return whereClause; +} + +// txnContentionDetailsQuery selects information about a specific transaction contention event. +function contentionDetailsQuery(filters?: ContentionFilters) { + const whereClause = getContentionWhereClause(filters); + return ` + SELECT DISTINCT collection_ts, + blocking_txn_id, + encode(blocking_txn_fingerprint_id, 'hex') AS blocking_txn_fingerprint_id, + waiting_txn_id, + encode(waiting_txn_fingerprint_id, 'hex') AS waiting_txn_fingerprint_id, + waiting_stmt_id, + encode(waiting_stmt_fingerprint_id, 'hex') AS waiting_stmt_fingerprint_id, + contention_duration, + contending_pretty_key AS key, + database_name, + schema_name, + table_name, + index_name + FROM + crdb_internal.transaction_contention_events AS tce + ${whereClause} + `; +} diff --git a/pkg/ui/workspaces/cluster-ui/src/api/stmtInsightsApi.ts b/pkg/ui/workspaces/cluster-ui/src/api/stmtInsightsApi.ts index 93c8f7479d73..f35619149aaf 100644 --- a/pkg/ui/workspaces/cluster-ui/src/api/stmtInsightsApi.ts +++ b/pkg/ui/workspaces/cluster-ui/src/api/stmtInsightsApi.ts @@ -18,6 +18,7 @@ import { SqlTxnResult, } from "./sqlApi"; import { + ContentionDetails, getInsightsFromProblemsAndCauses, InsightExecEnum, StmtInsightEvent, @@ -25,6 +26,7 @@ import { import moment from "moment"; import { INTERNAL_APP_NAME_PREFIX } from "src/recentExecutions/recentStatementUtils"; import { FixFingerprintHexValue } from "../util"; +import { getContentionDetailsApi } from "./contentionApi"; export type StmtInsightsReq = { start?: moment.Moment; @@ -33,15 +35,6 @@ export type StmtInsightsReq = { stmtFingerprintId?: string; }; -type InsightsContentionResponseEvent = { - blockingTxnID: string; - durationInMs: number; - schemaName: string; - databaseName: string; - tableName: string; - indexName: string; -}; - export type StmtInsightsResponseRow = { session_id: string; txn_id: string; @@ -62,7 +55,7 @@ export type StmtInsightsResponseRow = { retries: number; exec_node_ids: number[]; contention: string; // interval - contention_events: InsightsContentionResponseEvent[]; + contention_events: ContentionDetails[]; last_retry_reason?: string; causes: string[]; problem: string; @@ -91,7 +84,6 @@ priority, retries, exec_node_ids, contention, -contention_events, last_retry_reason, causes, problem, @@ -143,7 +135,7 @@ export const stmtInsightsByTxnExecutionQuery = (id: string): string => ` WHERE txn_id = '${id}' `; -export function getStmtInsightsApi( +export async function getStmtInsightsApi( req?: StmtInsightsReq, ): Promise { const request: SqlExecutionRequest = { @@ -156,21 +148,48 @@ export function getStmtInsightsApi( max_result_size: LARGE_RESULT_SIZE, timeout: LONG_TIMEOUT, }; - return executeInternalSql(request).then(result => { - if (result.error) { - throw new Error( - `Error while retrieving insights information: ${sqlApiErrorMessage( - result.error.message, - )}`, - ); - } - if (sqlResultsAreEmpty(result)) { - return []; + const result = await executeInternalSql(request); + if (result.error) { + throw new Error( + `Error while retrieving insights information: ${sqlApiErrorMessage( + result.error.message, + )}`, + ); + } + + if (sqlResultsAreEmpty(result)) { + return []; + } + + const stmtInsightEvent = formatStmtInsights(result.execution?.txn_results[0]); + await addStmtContentionInfoApi(stmtInsightEvent); + return stmtInsightEvent; +} + +async function addStmtContentionInfoApi( + input: StmtInsightEvent[], +): Promise { + if (!input || input.length === 0) { + return; + } + + for (let i = 0; i < input.length; i++) { + const event = input[i]; + if ( + event.contentionTime == null || + event.contentionTime.asMilliseconds() <= 0 + ) { + continue; } - return formatStmtInsights(result.execution?.txn_results[0]); - }); + event.contentionEvents = await getContentionDetailsApi({ + waitingTxnID: null, + waitingStmtID: event.statementExecutionID, + start: null, + end: null, + }); + } } export function formatStmtInsights( @@ -204,7 +223,6 @@ export function formatStmtInsights( isFullScan: row.full_scan, rowsRead: row.rows_read, rowsWritten: row.rows_written, - contentionEvents: row.contention_events, // This is the total stmt contention. contentionTime: row.contention ? moment.duration(row.contention) : null, indexRecommendations: row.index_recommendations, diff --git a/pkg/ui/workspaces/cluster-ui/src/api/txnInsightsApi.ts b/pkg/ui/workspaces/cluster-ui/src/api/txnInsightsApi.ts index 7f51fdee1bdd..d74384302fbf 100644 --- a/pkg/ui/workspaces/cluster-ui/src/api/txnInsightsApi.ts +++ b/pkg/ui/workspaces/cluster-ui/src/api/txnInsightsApi.ts @@ -19,7 +19,7 @@ import { sqlResultsAreEmpty, } from "./sqlApi"; import { - BlockedContentionDetails, + ContentionDetails, getInsightsFromProblemsAndCauses, InsightExecEnum, InsightNameEnum, @@ -35,6 +35,7 @@ import { StmtInsightsResponseRow, } from "./stmtInsightsApi"; import { INTERNAL_APP_NAME_PREFIX } from "src/recentExecutions/recentStatementUtils"; +import { getContentionDetailsApi } from "./contentionApi"; export const TXN_QUERY_PREVIEW_MAX = 800; export const QUERY_MAX = 1500; @@ -128,69 +129,10 @@ function createStmtFingerprintToQueryMap( return idToQuery; } -// txnContentionDetailsQuery selects information about a specific transaction contention event. -function txnContentionDetailsQuery(filters: TxnContentionDetailsRequest) { - let whereClause = ` WHERE waiting_txn_id = '${filters.txnExecutionID}'`; - if (filters?.start) { - whereClause = - whereClause + ` AND collection_ts >= '${filters.start.toISOString()}'`; - } - if (filters?.end) { - whereClause = - whereClause + - ` AND (collection_ts + contention_duration) <= '${filters.end.toISOString()}'`; - } - return ` -SELECT DISTINCT - collection_ts, - blocking_txn_id, - encode( blocking_txn_fingerprint_id, 'hex' ) AS blocking_txn_fingerprint_id, - waiting_txn_id, - encode( waiting_txn_fingerprint_id, 'hex' ) AS waiting_txn_fingerprint_id, - contention_duration, - crdb_internal.pretty_key(contending_key, 0) AS key, - database_name, - schema_name, - table_name, - index_name -FROM - crdb_internal.transaction_contention_events AS tce - JOIN [SELECT database_name, - schema_name, - name AS table_name, - table_id - FROM - "".crdb_internal.tables] AS tables ON tce.contending_key BETWEEN crdb_internal.table_span(tables.table_id)[1] - AND crdb_internal.table_span(tables.table_id)[2] - LEFT OUTER JOIN [SELECT index_name, - descriptor_id, - index_id - FROM - "".crdb_internal.table_indexes] AS indexes ON tce.contending_key BETWEEN crdb_internal.index_span( - indexes.descriptor_id, - indexes.index_id - )[1] - AND crdb_internal.index_span( - indexes.descriptor_id, - indexes.index_id - )[2] - ${whereClause} -`; -} - -type TxnContentionDetailsResponseColumns = { - waiting_txn_id: string; - waiting_txn_fingerprint_id: string; - collection_ts: string; - contention_duration: string; - blocking_txn_id: string; - blocking_txn_fingerprint_id: string; - schema_name: string; - database_name: string; - table_name: string; - index_name: string; - key: string; -}; +export type TransactionContentionEventDetails = Omit< + TxnContentionInsightDetails, + "application" | "queries" | "blockingQueries" +>; type PartialTxnContentionDetails = Omit< TxnContentionInsightDetails, @@ -198,48 +140,20 @@ type PartialTxnContentionDetails = Omit< >; function formatTxnContentionDetailsResponse( - response: SqlExecutionResponse, + response: ContentionDetails[], ): PartialTxnContentionDetails { - const resultsRows = response.execution.txn_results[0].rows; - if (!resultsRows) { + if (!response || response.length === 9) { // No data. return; } - const blockingContentionDetails = new Array( - resultsRows.length, - ); - - resultsRows.forEach((value, idx) => { - const contentionTimeInMs = moment - .duration(value.contention_duration) - .asMilliseconds(); - blockingContentionDetails[idx] = { - blockingExecutionID: value.blocking_txn_id, - blockingTxnFingerprintID: FixFingerprintHexValue( - value.blocking_txn_fingerprint_id, - ), - blockingQueries: null, - collectionTimeStamp: moment(value.collection_ts).utc(), - contentionTimeMs: contentionTimeInMs, - contendedKey: value.key, - schemaName: value.schema_name, - databaseName: value.database_name, - tableName: value.table_name, - indexName: - value.index_name && value.index_name !== "" - ? value.index_name - : "index not found", - }; - }); - - const row = resultsRows[0]; + const row = response[0]; return { - transactionExecutionID: row.waiting_txn_id, + transactionExecutionID: row.waitingTxnID, transactionFingerprintID: FixFingerprintHexValue( - row.waiting_txn_fingerprint_id, + row.waitingTxnFingerprintID, ), - blockingContentionDetails: blockingContentionDetails, + blockingContentionDetails: response, insightName: InsightNameEnum.highContention, execType: InsightExecEnum.TRANSACTION, }; @@ -263,24 +177,15 @@ export async function getTxnInsightsContentionDetailsApi( // 3. Get the query strings for ALL statements involved in the transaction. // Get contention results for requested transaction. - const contentionResults = - await executeInternalSql( - makeInsightsSqlRequest([ - txnContentionDetailsQuery({ - txnExecutionID: req.txnExecutionID, - start: req.start, - end: req.end, - }), - ]), - ); - if (contentionResults.error) { - throw new Error( - `Error while retrieving contention information: ${sqlApiErrorMessage( - contentionResults.error.message, - )}`, - ); - } - if (sqlResultsAreEmpty(contentionResults)) { + + const contentionResults = await getContentionDetailsApi({ + waitingTxnID: req.txnExecutionID, + waitingStmtID: null, + start: null, + end: null, + }); + + if (contentionResults.length === 0) { return; } const contentionDetails = @@ -358,7 +263,7 @@ function buildTxnContentionInsightDetails( return; } - blockedRow.blockingQueries = currBlockedFingerprintStmts.queryIDs.map( + blockedRow.blockingTxnQuery = currBlockedFingerprintStmts.queryIDs.map( id => stmtFingerprintToQuery.get(id) ?? "", ); }); diff --git a/pkg/ui/workspaces/cluster-ui/src/insights/types.ts b/pkg/ui/workspaces/cluster-ui/src/insights/types.ts index e7a069dc2022..41e5bd5f99ef 100644 --- a/pkg/ui/workspaces/cluster-ui/src/insights/types.ts +++ b/pkg/ui/workspaces/cluster-ui/src/insights/types.ts @@ -53,11 +53,15 @@ export type TxnInsightEvent = InsightEventBase & { }; // Information about the blocking transaction and schema. -export type BlockedContentionDetails = { +export type ContentionDetails = { collectionTimeStamp: Moment; blockingExecutionID: string; blockingTxnFingerprintID: string; - blockingQueries: string[]; + blockingTxnQuery: string[]; + waitingTxnID: string; + waitingTxnFingerprintID: string; + waitingStmtID: string; + waitingStmtFingerprintID: string; contendedKey: string; schemaName: string; databaseName: string; @@ -70,7 +74,7 @@ export type TxnContentionInsightDetails = { transactionExecutionID: string; application: string; transactionFingerprintID: string; - blockingContentionDetails: BlockedContentionDetails[]; + blockingContentionDetails: ContentionDetails[]; execType: InsightExecEnum; insightName: string; }; @@ -80,26 +84,17 @@ export type TxnInsightDetails = { // This data is segmented into 3 parts so that we can // selective fetch missing info on the details page. txnDetails?: TxnInsightEvent; - blockingContentionDetails?: BlockedContentionDetails[]; + blockingContentionDetails?: ContentionDetails[]; statements?: StmtInsightEvent[]; execType?: InsightExecEnum; }; -export type BlockedStatementContentionDetails = { - blockingTxnID: string; - durationInMs: number; - schemaName: string; - databaseName: string; - tableName: string; - indexName: string; -}; - // Shown on the stmt insights overview page. export type StmtInsightEvent = InsightEventBase & { statementExecutionID: string; statementFingerprintID: string; isFullScan: boolean; - contentionEvents?: BlockedStatementContentionDetails[]; + contentionEvents?: ContentionDetails[]; indexRecommendations: string[]; planGist: string; databaseName: string; @@ -116,6 +111,8 @@ export type Insight = { export type ContentionEvent = { executionID: string; fingerprintID: string; + waitingStmtID: string; + waitingStmtFingerprintID: string; queries: string[]; startTime: Moment; contentionTimeMs: number; @@ -124,6 +121,7 @@ export type ContentionEvent = { tableName: string; indexName: string; execType: InsightExecEnum; + stmtInsightEvent: StmtInsightEvent; }; export const highContentionInsight = ( diff --git a/pkg/ui/workspaces/cluster-ui/src/insights/utils.spec.ts b/pkg/ui/workspaces/cluster-ui/src/insights/utils.spec.ts index 0a796f7e2213..bb8c55a19eaf 100644 --- a/pkg/ui/workspaces/cluster-ui/src/insights/utils.spec.ts +++ b/pkg/ui/workspaces/cluster-ui/src/insights/utils.spec.ts @@ -10,35 +10,39 @@ import moment from "moment"; import { - filterTransactionInsights, - getAppsFromTransactionInsights, filterStatementInsights, + filterTransactionInsights, getAppsFromStatementInsights, + getAppsFromTransactionInsights, getInsightsFromProblemsAndCauses, mergeTxnInsightDetails, } from "./utils"; import { - TxnInsightEvent, - InsightNameEnum, + ContentionDetails, failedExecutionInsight, - StmtInsightEvent, - InsightExecEnum, highContentionInsight, - slowExecutionInsight, + highRetryCountInsight, + InsightExecEnum, + InsightNameEnum, planRegressionInsight, + slowExecutionInsight, + StmtInsightEvent, suboptimalPlanInsight, - highRetryCountInsight, - BlockedContentionDetails, TxnInsightDetails, + TxnInsightEvent, } from "./types"; const INTERNAL_APP_PREFIX = "$ internal"; -const blockedContentionMock: BlockedContentionDetails = { +const blockedContentionMock: ContentionDetails = { collectionTimeStamp: moment(), blockingExecutionID: "execution", blockingTxnFingerprintID: "block", - blockingQueries: ["select 1"], + blockingTxnQuery: ["select 1"], + waitingStmtFingerprintID: "waitingStmtFingerprintID", + waitingStmtID: "waitingStmtID", + waitingTxnFingerprintID: "waitingTxnFingerprintID", + waitingTxnID: "waitingTxnID", contendedKey: "key", schemaName: "schema", databaseName: "db", diff --git a/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsightDetails/insightDetailsTables.tsx b/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsightDetails/insightDetailsTables.tsx index f00c59c18623..f06e4269728c 100644 --- a/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsightDetails/insightDetailsTables.tsx +++ b/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsightDetails/insightDetailsTables.tsx @@ -11,14 +11,11 @@ import React, { useState } from "react"; import { ColumnDescriptor, SortedTable, SortSetting } from "src/sortedtable"; import { DATE_WITH_SECONDS_AND_MILLISECONDS_FORMAT, Duration } from "src/util"; -import { - BlockedStatementContentionDetails, - ContentionEvent, - InsightExecEnum, -} from "../types"; +import { ContentionDetails, ContentionEvent, InsightExecEnum } from "../types"; import { insightsTableTitles, QueriesCell, + StatementDetailsLink, TransactionDetailsLink, } from "../workloadInsights/util"; import { TimeScale } from "../../timeScaleDropdown"; @@ -51,6 +48,23 @@ export function makeInsightDetailsColumns( ), sort: (item: ContentionEvent) => item.fingerprintID, }, + { + name: "waitingStmtId", + title: insightsTableTitles.waitingID(InsightExecEnum.STATEMENT), + cell: (item: ContentionEvent) => String(item.waitingStmtID), + sort: (item: ContentionEvent) => item.waitingStmtID, + }, + { + name: "waitingStmtFingerprintID", + title: insightsTableTitles.waitingFingerprintID( + InsightExecEnum.STATEMENT, + ), + cell: (item: ContentionEvent) => + item.stmtInsightEvent + ? StatementDetailsLink(item.stmtInsightEvent, setTimeScale) + : item.waitingStmtFingerprintID, + sort: (item: ContentionEvent) => item.waitingStmtFingerprintID, + }, { name: "query", title: insightsTableTitles.query(execType), @@ -116,59 +130,72 @@ export const WaitTimeDetailsTable: React.FC< ); }; -export function makeInsightStatementContentionColumns(): ColumnDescriptor[] { +export function makeInsightStatementContentionColumns( + setTimeScale: (ts: TimeScale) => void, +): ColumnDescriptor[] { const execType = InsightExecEnum.STATEMENT; return [ { name: "executionID", title: insightsTableTitles.executionID(InsightExecEnum.TRANSACTION), - cell: (item: BlockedStatementContentionDetails) => item.blockingTxnID, - sort: (item: BlockedStatementContentionDetails) => item.blockingTxnID, + cell: (item: ContentionDetails) => item.blockingExecutionID, + sort: (item: ContentionDetails) => item.blockingExecutionID, }, { - name: "duration", - title: insightsTableTitles.contention(execType), - cell: (item: BlockedStatementContentionDetails) => - Duration(item.durationInMs * 1e6), - sort: (item: BlockedStatementContentionDetails) => item.durationInMs, + name: "fingerprintId", + title: insightsTableTitles.fingerprintID(InsightExecEnum.TRANSACTION), + cell: (item: ContentionDetails) => + TransactionDetailsLink( + item.blockingTxnFingerprintID, + item.collectionTimeStamp, + setTimeScale, + ), + sort: (item: ContentionDetails) => item.blockingTxnFingerprintID, }, { - name: "schemaName", - title: insightsTableTitles.schemaName(execType), - cell: (item: BlockedStatementContentionDetails) => item.schemaName, - sort: (item: BlockedStatementContentionDetails) => item.schemaName, + name: "duration", + title: insightsTableTitles.contention(execType), + cell: (item: ContentionDetails) => Duration(item.contentionTimeMs * 1e6), + sort: (item: ContentionDetails) => item.contentionTimeMs, }, { name: "databaseName", title: insightsTableTitles.databaseName(execType), - cell: (item: BlockedStatementContentionDetails) => item.databaseName, - sort: (item: BlockedStatementContentionDetails) => item.databaseName, + cell: (item: ContentionDetails) => item.databaseName, + sort: (item: ContentionDetails) => item.databaseName, + }, + { + name: "schemaName", + title: insightsTableTitles.schemaName(execType), + cell: (item: ContentionDetails) => item.schemaName, + sort: (item: ContentionDetails) => item.schemaName, }, { name: "tableName", title: insightsTableTitles.tableName(execType), - cell: (item: BlockedStatementContentionDetails) => item.tableName, - sort: (item: BlockedStatementContentionDetails) => item.tableName, + cell: (item: ContentionDetails) => item.tableName, + sort: (item: ContentionDetails) => item.tableName, }, { name: "indexName", title: insightsTableTitles.indexName(execType), - cell: (item: BlockedStatementContentionDetails) => item.indexName, - sort: (item: BlockedStatementContentionDetails) => item.indexName, + cell: (item: ContentionDetails) => item.indexName, + sort: (item: ContentionDetails) => item.indexName, }, ]; } interface InsightContentionTableProps { - data: BlockedStatementContentionDetails[]; + data: ContentionDetails[]; sortSetting?: SortSetting; onChangeSortSetting?: (ss: SortSetting) => void; + setTimeScale: (ts: TimeScale) => void; } export const ContentionStatementDetailsTable: React.FC< InsightContentionTableProps > = props => { - const columns = makeInsightStatementContentionColumns(); + const columns = makeInsightStatementContentionColumns(props.setTimeScale); return ( @@ -85,6 +85,7 @@ export const StatementInsightDetailsOverviewTab: React.FC< data={insightDetails.contentionEvents} sortSetting={insightsDetailsContentionSortSetting} onChangeSortSetting={setDetailsContentionSortSetting} + setTimeScale={setTimeScale} /> diff --git a/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsightDetails/transactionInsightDetailsOverviewTab.tsx b/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsightDetails/transactionInsightDetailsOverviewTab.tsx index 02ad4db39d92..00db10d01a77 100644 --- a/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsightDetails/transactionInsightDetailsOverviewTab.tsx +++ b/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsightDetails/transactionInsightDetailsOverviewTab.tsx @@ -28,12 +28,12 @@ import { } from "src/insightsTable/insightsTable"; import { WaitTimeDetailsTable } from "./insightDetailsTables"; import { - BlockedContentionDetails, + ContentionDetails, ContentionEvent, - TxnInsightEvent, InsightExecEnum, - StmtInsightEvent, InsightNameEnum, + StmtInsightEvent, + TxnInsightEvent, } from "../types"; import classNames from "classnames/bind"; @@ -55,7 +55,7 @@ const tableCx = classNames.bind(insightTableStyles); type Props = { txnDetails: TxnInsightEvent | null; statements: StmtInsightEvent[] | null; - contentionDetails?: BlockedContentionDetails[]; + contentionDetails?: ContentionDetails[]; setTimeScale: (ts: TimeScale) => void; hasAdminRole: boolean; errors: TxnInsightDetailsReqErrs | null; @@ -87,20 +87,28 @@ export const TransactionInsightDetailsOverviewTab: React.FC = ({ true, ); - const blockingExecutions: ContentionEvent[] = contentionDetails?.map(x => { - return { - executionID: x.blockingExecutionID, - fingerprintID: x.blockingTxnFingerprintID, - queries: x.blockingQueries, - startTime: x.collectionTimeStamp, - contentionTimeMs: x.contentionTimeMs, - execType: InsightExecEnum.TRANSACTION, - schemaName: x.schemaName, - databaseName: x.databaseName, - tableName: x.tableName, - indexName: x.indexName, - }; - }); + const blockingExecutions: ContentionEvent[] = contentionDetails?.map( + event => { + const stmtInsight = statements.find( + stmt => stmt.statementExecutionID == event.waitingStmtID, + ); + return { + executionID: event.blockingExecutionID, + fingerprintID: event.blockingTxnFingerprintID, + waitingStmtID: event.waitingStmtID, + waitingStmtFingerprintID: event.waitingStmtFingerprintID, + queries: event.blockingTxnQuery, + startTime: event.collectionTimeStamp, + contentionTimeMs: event.contentionTimeMs, + execType: InsightExecEnum.TRANSACTION, + schemaName: event.schemaName, + databaseName: event.databaseName, + tableName: event.tableName, + indexName: event.indexName, + stmtInsightEvent: stmtInsight, + }; + }, + ); const insightRecs = getTxnInsightRecommendations(txnDetails); const hasContentionInsights = diff --git a/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/util/insightsColumns.tsx b/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/util/insightsColumns.tsx index 1a57c3ed9b97..d6bf4f7f3625 100644 --- a/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/util/insightsColumns.tsx +++ b/pkg/ui/workspaces/cluster-ui/src/insights/workloadInsights/util/insightsColumns.tsx @@ -18,6 +18,8 @@ import { contentionTime, readFromDisk, writtenToDisk } from "../../../util"; export const insightsColumnLabels = { executionID: "Execution ID", latestExecutionID: "Latest Execution ID", + waitingID: "Waiting Execution ID", + waitingFingerprintID: "Waiting Fingerprint ID", query: "Execution", insights: "Insights", startTime: "Start Time (UTC)", @@ -90,6 +92,20 @@ export const insightsTableTitles: InsightsTableTitleType = { execType, ); }, + waitingFingerprintID: (execType: InsightExecEnum) => { + return makeToolTip( +

The {execType} fingerprint ID.

, + "waitingFingerprintID", + execType, + ); + }, + waitingID: (execType: InsightExecEnum) => { + return makeToolTip( +

The ID of the waiting {execType}.

, + "waitingID", + execType, + ); + }, latestExecutionID: (execType: InsightExecEnum) => { return makeToolTip(

From 12aec72005fd69bd8f18813c19764d6107e5b813 Mon Sep 17 00:00:00 2001 From: Radu Berinde Date: Mon, 13 Feb 2023 07:28:21 -0800 Subject: [PATCH 3/3] storage: remove pre-22.2 code This change removes any code that handles pre-22.2 versions. Features that are in 22.2 (like range keys) are now always enabled. Any new store always uses at least the 22.2 pebble format version. Release note: None Epic: none Fixes: #96764 --- pkg/ccl/cliccl/ear_test.go | 27 ++- .../storageccl/engineccl/encrypted_fs_test.go | 5 +- .../kvstorage/cluster_version_test.go | 42 +++-- pkg/kv/kvserver/spanset/batch.go | 5 - pkg/sql/gcjob_test/BUILD.bazel | 7 - pkg/sql/gcjob_test/gc_job_test.go | 154 ---------------- pkg/storage/engine.go | 5 - pkg/storage/engine_test.go | 164 ------------------ pkg/storage/intent_interleaving_iter.go | 3 +- pkg/storage/min_version_test.go | 31 ++-- pkg/storage/mvcc.go | 7 +- pkg/storage/pebble.go | 86 ++------- pkg/storage/pebble_batch.go | 24 +-- pkg/storage/pebble_iterator.go | 49 +----- pkg/storage/pebble_iterator_test.go | 4 +- pkg/storage/pebble_test.go | 16 +- pkg/storage/sst_writer.go | 11 +- pkg/storage/sst_writer_test.go | 62 ++----- pkg/storage/temp_engine.go | 1 + .../upgrademanager/manager_external_test.go | 21 +-- .../upgrades/schema_changes_external_test.go | 6 +- 21 files changed, 127 insertions(+), 603 deletions(-) diff --git a/pkg/ccl/cliccl/ear_test.go b/pkg/ccl/cliccl/ear_test.go index 43f9e534d396..b29a06cc1ac5 100644 --- a/pkg/ccl/cliccl/ear_test.go +++ b/pkg/ccl/cliccl/ear_test.go @@ -156,13 +156,13 @@ func TestList(t *testing.T) { 000004.log: env type: Data, AES128_CTR keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef - nonce: 80 18 c0 79 61 c7 cf ef b4 25 4e 78 - counter: 1483615076 + nonce: 31 d3 cd 5a 69 e2 13 64 21 53 57 64 + counter: 3952287331 000005.sst: env type: Data, AES128_CTR keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef - nonce: 71 12 f7 22 9a fb 90 24 4e 58 27 01 - counter: 3082989236 + nonce: 23 d9 b2 e1 39 b0 87 ed f9 6d 49 20 + counter: 3481614039 COCKROACHDB_DATA_KEYS_000001_monolith: env type: Store, AES128_CTR keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867 @@ -171,8 +171,8 @@ COCKROACHDB_DATA_KEYS_000001_monolith: CURRENT: env type: Data, AES128_CTR keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef - nonce: 18 c2 a6 23 cc 6e 2e 7c 8e bf 84 77 - counter: 3159373900 + nonce: 71 12 f7 22 9a fb 90 24 4e 58 27 01 + counter: 3082989236 MANIFEST-000001: env type: Data, AES128_CTR keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef @@ -181,14 +181,25 @@ MANIFEST-000001: OPTIONS-000003: env type: Data, AES128_CTR keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef - nonce: d3 97 11 b3 1a ed 22 2b 74 fb 02 0c - counter: 1229228536 + nonce: c3 6d b2 7b 3f 3e 67 b9 28 b9 81 b1 + counter: 3050109376 marker.datakeys.000001.COCKROACHDB_DATA_KEYS_000001_monolith: env type: Store, AES128_CTR keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867 nonce: 55 d7 d4 27 6c 97 9b dd f1 5d 40 c8 counter: 467030050 +marker.format-version.000009.010: + env type: Data, AES128_CTR + keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef + nonce: 6e 34 f4 3c 11 43 1a f5 69 ce 33 f1 + counter: 2398097086 +marker.manifest.000001.MANIFEST-000001: + env type: Data, AES128_CTR + keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef + nonce: d3 97 11 b3 1a ed 22 2b 74 fb 02 0c + counter: 1229228536 ` + require.Equal(t, want, b.String()) } diff --git a/pkg/ccl/storageccl/engineccl/encrypted_fs_test.go b/pkg/ccl/storageccl/engineccl/encrypted_fs_test.go index 74a00395eca4..a207b1e1d921 100644 --- a/pkg/ccl/storageccl/engineccl/encrypted_fs_test.go +++ b/pkg/ccl/storageccl/engineccl/encrypted_fs_test.go @@ -268,9 +268,10 @@ func TestPebbleEncryption(t *testing.T) { stats, err := db.GetEnvStats() require.NoError(t, err) // Opening the DB should've created OPTIONS, CURRENT, MANIFEST and the - // WAL, all under the active key. + // WAL. require.Equal(t, uint64(4), stats.TotalFiles) - require.Equal(t, uint64(4), stats.ActiveKeyFiles) + // We also created markers for the format version and the manifest. + require.Equal(t, uint64(6), stats.ActiveKeyFiles) var s enginepbccl.EncryptionStatus require.NoError(t, protoutil.Unmarshal(stats.EncryptionStatus, &s)) require.Equal(t, "16.key", s.ActiveStoreKey.Source) diff --git a/pkg/kv/kvserver/kvstorage/cluster_version_test.go b/pkg/kv/kvserver/kvstorage/cluster_version_test.go index c800907915d5..8340c98e5d34 100644 --- a/pkg/kv/kvserver/kvstorage/cluster_version_test.go +++ b/pkg/kv/kvserver/kvstorage/cluster_version_test.go @@ -31,8 +31,14 @@ func TestStoresClusterVersionIncompatible(t *testing.T) { ctx := context.Background() - vOneDashOne := roachpb.Version{Major: 1, Internal: 1} - vOne := roachpb.Version{Major: 1} + current := clusterversion.ByKey(clusterversion.BinaryVersionKey) + minSupported := clusterversion.ByKey(clusterversion.BinaryMinSupportedVersionKey) + + future := current + future.Major++ + + tooOld := minSupported + tooOld.Major-- type testCase struct { binV, minV roachpb.Version // binary version and min supported version @@ -42,28 +48,21 @@ func TestStoresClusterVersionIncompatible(t *testing.T) { for name, tc := range map[string]testCase{ "StoreTooNew": { // This is what the node is running. - binV: vOneDashOne, + binV: current, // This is what the running node requires from its stores. - minV: vOne, + minV: minSupported, // Version is way too high for this node. - engV: roachpb.Version{Major: 9}, - expErr: `cockroach version v1\.0-1 is incompatible with data in store =; use version v9\.0 or later`, + engV: future, + expErr: `cockroach version .* is incompatible with data in store =; use version .* or later`, }, "StoreTooOldVersion": { // This is what the node is running. - binV: roachpb.Version{Major: 9}, + binV: current, // This is what the running node requires from its stores. - minV: roachpb.Version{Major: 5}, + minV: minSupported, // Version is way too low. - engV: roachpb.Version{Major: 4}, - expErr: `store =, last used with cockroach version v4\.0, is too old for running version v9\.0 \(which requires data from v5\.0 or later\)`, - }, - "StoreTooOldMinVersion": { - // Like the previous test case, but this time cv.MinimumVersion is the culprit. - binV: roachpb.Version{Major: 9}, - minV: roachpb.Version{Major: 5}, - engV: roachpb.Version{Major: 4}, - expErr: `store =, last used with cockroach version v4\.0, is too old for running version v9\.0 \(which requires data from v5\.0 or later\)`, + engV: tooOld, + expErr: `store =, last used with cockroach version .*, is too old for running version .* \(which requires data from .* or later\)`, }, } { t.Run(name, func(t *testing.T) { @@ -71,12 +70,11 @@ func TestStoresClusterVersionIncompatible(t *testing.T) { defer engs[0].Close() // Configure versions and write. cv := clusterversion.ClusterVersion{Version: tc.engV} - if err := WriteClusterVersionToEngines(ctx, engs, cv); err != nil { - t.Fatal(err) + err := WriteClusterVersionToEngines(ctx, engs, cv) + if err == nil { + cv, err = SynthesizeClusterVersionFromEngines(ctx, engs, tc.binV, tc.minV) } - if cv, err := SynthesizeClusterVersionFromEngines( - ctx, engs, tc.binV, tc.minV, - ); !testutils.IsError(err, tc.expErr) { + if !testutils.IsError(err, tc.expErr) { t.Fatalf("unexpected error: %+v, got version %v", err, cv) } }) diff --git a/pkg/kv/kvserver/spanset/batch.go b/pkg/kv/kvserver/spanset/batch.go index 2198e2cf8371..81baf7ecd9f2 100644 --- a/pkg/kv/kvserver/spanset/batch.go +++ b/pkg/kv/kvserver/spanset/batch.go @@ -495,11 +495,6 @@ func (s spanSetReader) ConsistentIterators() bool { return s.r.ConsistentIterators() } -// SupportsRangeKeys implements the storage.Reader interface. -func (s spanSetReader) SupportsRangeKeys() bool { - return s.r.SupportsRangeKeys() -} - // PinEngineStateForIterators implements the storage.Reader interface. func (s spanSetReader) PinEngineStateForIterators() error { return s.r.PinEngineStateForIterators() diff --git a/pkg/sql/gcjob_test/BUILD.bazel b/pkg/sql/gcjob_test/BUILD.bazel index 40e297c9d2d3..af9e229d561a 100644 --- a/pkg/sql/gcjob_test/BUILD.bazel +++ b/pkg/sql/gcjob_test/BUILD.bazel @@ -11,9 +11,6 @@ go_test( args = ["-test.timeout=295s"], deps = [ "//pkg/base", - "//pkg/clusterversion", - "//pkg/config", - "//pkg/config/zonepb", "//pkg/jobs", "//pkg/jobs/jobspb", "//pkg/keys", @@ -33,7 +30,6 @@ go_test( "//pkg/sql/catalog/descs", "//pkg/sql/catalog/tabledesc", "//pkg/sql/gcjob", - "//pkg/sql/gcjob/gcjobnotifier", "//pkg/sql/isql", "//pkg/sql/sem/catid", "//pkg/storage", @@ -46,10 +42,7 @@ go_test( "//pkg/util/leaktest", "//pkg/util/log", "//pkg/util/randutil", - "//pkg/util/stop", - "//pkg/util/syncutil", "//pkg/util/timeutil", - "@com_github_cockroachdb_errors//:errors", "@com_github_stretchr_testify//require", ], ) diff --git a/pkg/sql/gcjob_test/gc_job_test.go b/pkg/sql/gcjob_test/gc_job_test.go index 8ce019a52492..71abf4780b36 100644 --- a/pkg/sql/gcjob_test/gc_job_test.go +++ b/pkg/sql/gcjob_test/gc_job_test.go @@ -19,9 +19,6 @@ import ( "time" "github.com/cockroachdb/cockroach/pkg/base" - "github.com/cockroachdb/cockroach/pkg/clusterversion" - "github.com/cockroachdb/cockroach/pkg/config" - "github.com/cockroachdb/cockroach/pkg/config/zonepb" "github.com/cockroachdb/cockroach/pkg/jobs" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" "github.com/cockroachdb/cockroach/pkg/keys" @@ -30,7 +27,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/multitenant/mtinfopb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/security/username" - "github.com/cockroachdb/cockroach/pkg/server" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/sql" "github.com/cockroachdb/cockroach/pkg/sql/catalog/bootstrap" @@ -39,7 +35,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/catalog/descs" "github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc" "github.com/cockroachdb/cockroach/pkg/sql/gcjob" - "github.com/cockroachdb/cockroach/pkg/sql/gcjob/gcjobnotifier" "github.com/cockroachdb/cockroach/pkg/sql/isql" "github.com/cockroachdb/cockroach/pkg/sql/sem/catid" "github.com/cockroachdb/cockroach/pkg/storage" @@ -51,10 +46,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" - "github.com/cockroachdb/cockroach/pkg/util/stop" - "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil" - "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" ) @@ -663,149 +655,3 @@ SELECT descriptor_id, index_id require.NoError(t, jr.WaitForJobs(ctx, []jobspb.JobID{jobID})) }) } - -// TestGCJobNoSystemConfig tests that the GC job is robust to running with -// no system config provided by the SystemConfigProvider. It is a regression -// test for a panic which could occur due to a slow systemconfigwatcher -// initialization. -// -// TODO(ajwerner): Remove this test in 23.1. -func TestGCJobNoSystemConfig(t *testing.T) { - defer leaktest.AfterTest(t)() - - provider := fakeSystemConfigProvider{} - var ( - v0 = clusterversion.ByKey(clusterversion.TODODelete_V22_2UseDelRangeInGCJob - 1) - v1 = clusterversion.ByKey(clusterversion.TODODelete_V22_2UseDelRangeInGCJob) - ) - settings := cluster.MakeTestingClusterSettingsWithVersions(v1, v0, false /* initializeVersion */) - ctx := context.Background() - require.NoError(t, clusterversion.Initialize(ctx, v0, &settings.SV)) - stopper := stop.NewStopper() - gcKnobs := &sql.GCJobTestingKnobs{} - s, sqlDB, kvDB := serverutils.StartServer(t, base.TestServerArgs{ - Settings: settings, - Stopper: stopper, - Knobs: base.TestingKnobs{ - GCJob: gcKnobs, - Server: &server.TestingKnobs{ - DisableAutomaticVersionUpgrade: make(chan struct{}), - BinaryVersionOverride: v0, - }, - }, - }) - defer stopper.Stop(ctx) - codec := s.ExecutorConfig().(sql.ExecutorConfig).Codec - n := gcjobnotifier.New(settings, &provider, codec, stopper) - n.Start(ctx) - gcKnobs.Notifier = n - - tdb := sqlutils.MakeSQLRunner(sqlDB) - tdb.Exec(t, "CREATE TABLE foo (i INT PRIMARY KEY)") - tdb.Exec(t, "SET CLUSTER SETTING kv.protectedts.poll_interval = '10ms'") - var id uint32 - tdb.QueryRow(t, "SELECT 'foo'::regclass::int").Scan(&id) - tdb.Exec(t, "DROP TABLE foo") - // We want to make sure there's a notifyee and that the job attempted - // to read the status twice. We expect it once for the notifier and - // once for the job itself. - testutils.SucceedsSoon(t, func() error { - if n := provider.numNotifyees(); n == 0 { - return errors.Errorf("expected 1 notifyee, got %d", n) - } - if n := provider.numCalls(); n < 2 { - return errors.Errorf("expected at least 2 calls, got %d", n) - } - return nil - }) - cfgProto := &zonepb.ZoneConfig{ - GC: &zonepb.GCPolicy{TTLSeconds: 0}, - } - cfg := config.NewSystemConfig(cfgProto) - descKV, err := kvDB.Get(ctx, codec.DescMetadataKey(id)) - require.NoError(t, err) - var zoneKV roachpb.KeyValue - zoneKV.Key = config.MakeZoneKey(codec, descpb.ID(id)) - require.NoError(t, zoneKV.Value.SetProto(cfgProto)) - defaultKV := zoneKV - defaultKV.Key = config.MakeZoneKey(codec, 0) - // We need to put in an entry for the descriptor both so that the notifier - // fires and so that we don't think the descriptor is missing. We also - // need a zone config KV to make the delta filter happy. - cfg.Values = []roachpb.KeyValue{ - {Key: descKV.Key, Value: *descKV.Value}, - defaultKV, - zoneKV, - } - - provider.setConfig(cfg) - tdb.CheckQueryResultsRetry(t, ` -SELECT status - FROM crdb_internal.jobs - WHERE description = 'GC for DROP TABLE defaultdb.public.foo'`, - [][]string{{"succeeded"}}) -} - -type fakeSystemConfigProvider struct { - mu struct { - syncutil.Mutex - - calls int - n int - config *config.SystemConfig - notifyees map[int]chan struct{} - } -} - -func (f *fakeSystemConfigProvider) GetSystemConfig() *config.SystemConfig { - f.mu.Lock() - defer f.mu.Unlock() - f.mu.calls++ - return f.mu.config -} - -func (f *fakeSystemConfigProvider) RegisterSystemConfigChannel() ( - _ <-chan struct{}, - unregister func(), -) { - f.mu.Lock() - defer f.mu.Unlock() - ch := make(chan struct{}, 1) - n := f.mu.n - f.mu.n++ - if f.mu.notifyees == nil { - f.mu.notifyees = map[int]chan struct{}{} - } - f.mu.notifyees[n] = ch - return ch, func() { - f.mu.Lock() - defer f.mu.Unlock() - delete(f.mu.notifyees, n) - } -} - -func (f *fakeSystemConfigProvider) setConfig(c *config.SystemConfig) { - f.mu.Lock() - defer f.mu.Unlock() - f.mu.config = c - for _, ch := range f.mu.notifyees { - select { - case ch <- struct{}{}: - default: - } - } -} - -func (f *fakeSystemConfigProvider) numNotifyees() int { - f.mu.Lock() - defer f.mu.Unlock() - return len(f.mu.notifyees) -} - -func (f *fakeSystemConfigProvider) numCalls() int { - f.mu.Lock() - defer f.mu.Unlock() - return f.mu.calls -} - -var _ config.SystemConfigProvider = (*fakeSystemConfigProvider)(nil) diff --git a/pkg/storage/engine.go b/pkg/storage/engine.go index 315a832932d5..23f678e19f0b 100644 --- a/pkg/storage/engine.go +++ b/pkg/storage/engine.go @@ -591,11 +591,6 @@ type Reader interface { // underlying Engine state. This is not true about Batch writes: new iterators // will see new writes made to the batch, existing iterators won't. ConsistentIterators() bool - // SupportsRangeKeys returns true if the Reader implementation supports - // range keys. - // - // TODO(erikgrinaker): Remove this after 22.2. - SupportsRangeKeys() bool // PinEngineStateForIterators ensures that the state seen by iterators // without timestamp hints (see IterOptions) is pinned and will not see diff --git a/pkg/storage/engine_test.go b/pkg/storage/engine_test.go index f397ef5f7d31..1b418a58067e 100644 --- a/pkg/storage/engine_test.go +++ b/pkg/storage/engine_test.go @@ -26,7 +26,6 @@ import ( "testing" "time" - "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" @@ -1911,8 +1910,6 @@ func TestEngineRangeKeyMutations(t *testing.T) { defer rw.Close() } - require.True(t, rw.SupportsRangeKeys()) - // Check errors for invalid, empty, and zero-length range keys. Not // exhaustive, since we assume validation dispatches to // MVCCRangeKey.Validate() which is tested separately. @@ -2034,167 +2031,6 @@ func TestEngineRangeKeyMutations(t *testing.T) { }) } -// TestEngineRangeKeysUnsupported tests that engines without range key -// support behave as expected, i.e. writes fail but reads degrade gracefully. -func TestEngineRangeKeysUnsupported(t *testing.T) { - defer leaktest.AfterTest(t)() - defer log.Scope(t).Close(t) - - // Set up an engine with a version that doesn't support range keys. - version := clusterversion.ByKey(clusterversion.TODODelete_V22_2EnsurePebbleFormatVersionRangeKeys - 1) - st := cluster.MakeTestingClusterSettingsWithVersions(version, version, true) - - eng, err := Open(context.Background(), InMemory(), st, MaxSize(1<<20)) - if err != nil { - panic(err) - } - defer eng.Close() - - require.NoError(t, eng.PutMVCC(pointKey("a", 1), stringValue("a1"))) - - batch := eng.NewBatch() - defer batch.Close() - snapshot := eng.NewSnapshot() - defer snapshot.Close() - readOnly := eng.NewReadOnly(StandardDurability) - defer readOnly.Close() - - writers := map[string]Writer{ - "engine": eng, - "batch": batch, - } - readers := map[string]Reader{ - "engine": eng, - "batch": batch, - "snapshot": snapshot, - "readonly": readOnly, - } - - // Range key puts should error, but clears are noops (since old databases - // cannot contain range keys by definition). - for name, w := range writers { - t.Run(fmt.Sprintf("write/%s", name), func(t *testing.T) { - rangeKey := rangeKey("a", "b", 2) - - err := w.PutMVCCRangeKey(rangeKey, MVCCValue{}) - require.Error(t, err) - require.Contains(t, err.Error(), "range keys not supported") - - err = w.PutRawMVCCRangeKey(rangeKey, []byte{}) - require.Error(t, err) - require.Contains(t, err.Error(), "range keys not supported") - - err = w.PutEngineRangeKey(rangeKey.StartKey, rangeKey.EndKey, nil, nil) - require.Error(t, err) - require.Contains(t, err.Error(), "range keys not supported") - - require.NoError(t, w.ClearMVCCRangeKey(rangeKey)) - require.NoError(t, w.ClearEngineRangeKey( - rangeKey.StartKey, rangeKey.EndKey, EncodeMVCCTimestampSuffix(rangeKey.Timestamp))) - require.NoError(t, w.ClearRawRange( - rangeKey.StartKey, rangeKey.EndKey, false /* pointKeys */, true /* rangeKeys */)) - }) - } - - // All range key iterators should degrade gracefully to point key iterators, - // and be empty for IterKeyTypeRangesOnly. - keyTypes := map[string]IterKeyType{ - "PointsOnly": IterKeyTypePointsOnly, - "PointsAndRanges": IterKeyTypePointsAndRanges, - "RangesOnly": IterKeyTypeRangesOnly, - } - for name, r := range readers { - for keyTypeName, keyType := range keyTypes { - t.Run(fmt.Sprintf("read/%s/%s", name, keyTypeName), func(t *testing.T) { - require.False(t, r.SupportsRangeKeys()) - - t.Run("MVCCIterator", func(t *testing.T) { - iter := r.NewMVCCIterator(MVCCKeyAndIntentsIterKind, IterOptions{ - KeyTypes: keyType, - UpperBound: keys.MaxKey, - RangeKeyMaskingBelow: hlc.Timestamp{WallTime: 1}, // should get disabled when unsupported - }) - defer iter.Close() - - iter.SeekGE(pointKey("a", 0)) - - ok, err := iter.Valid() - require.NoError(t, err) - - if keyType == IterKeyTypeRangesOnly { - // With RangesOnly, the iterator must be empty. - require.False(t, ok) - hasPoint, hasRange := iter.HasPointAndRange() - require.False(t, hasPoint) - require.False(t, hasRange) - return - } - - require.True(t, ok) - require.Equal(t, pointKey("a", 1), iter.UnsafeKey()) - v, err := iter.UnsafeValue() - require.NoError(t, err) - require.Equal(t, stringValueRaw("a1"), v) - - hasPoint, hasRange := iter.HasPointAndRange() - require.True(t, hasPoint) - require.False(t, hasRange) - require.Empty(t, iter.RangeBounds()) - require.Empty(t, iter.RangeKeys()) - - // Exhaust the iterator. - iter.Next() - ok, err = iter.Valid() - require.NoError(t, err) - require.False(t, ok) - }) - - t.Run("EngineIterator", func(t *testing.T) { - iter := r.NewEngineIterator(IterOptions{ - KeyTypes: keyType, - UpperBound: keys.MaxKey, - RangeKeyMaskingBelow: hlc.Timestamp{WallTime: 1}, // should get disabled when unsupported - }) - defer iter.Close() - - ok, err := iter.SeekEngineKeyGE(engineKey("a", 0)) - require.NoError(t, err) - - if keyType == IterKeyTypeRangesOnly { - // With RangesOnly, the iterator must be empty. - require.False(t, ok) - hasPoint, hasRange := iter.HasPointAndRange() - require.False(t, hasPoint) - require.False(t, hasRange) - return - } - - require.True(t, ok) - key, err := iter.UnsafeEngineKey() - require.NoError(t, err) - require.Equal(t, engineKey("a", 1), key) - v, err := iter.UnsafeValue() - require.NoError(t, err) - require.Equal(t, stringValueRaw("a1"), v) - - hasPoint, hasRange := iter.HasPointAndRange() - require.True(t, hasPoint) - require.False(t, hasRange) - rangeBounds, err := iter.EngineRangeBounds() - require.NoError(t, err) - require.Empty(t, rangeBounds) - require.Empty(t, iter.EngineRangeKeys()) - - // Exhaust the iterator. - ok, err = iter.NextEngineKey() - require.NoError(t, err) - require.False(t, ok) - }) - }) - } - } -} - // TODO(erikgrinaker): The below test helpers should be moved to // testutils/storageutils instead, but that requires storage tests to be in the // storage_test package to avoid import cycles. diff --git a/pkg/storage/intent_interleaving_iter.go b/pkg/storage/intent_interleaving_iter.go index 2cf67c686e1c..80be79c9c1ab 100644 --- a/pkg/storage/intent_interleaving_iter.go +++ b/pkg/storage/intent_interleaving_iter.go @@ -267,8 +267,7 @@ func newIntentInterleavingIterator(reader Reader, opts IterOptions) MVCCIterator if reader.ConsistentIterators() { iter = maybeUnwrapUnsafeIter(reader.NewMVCCIterator(MVCCKeyIterKind, opts)).(*pebbleIterator) } else { - iter = newPebbleIteratorByCloning( - intentIter.GetRawIter(), opts, StandardDurability, reader.SupportsRangeKeys()) + iter = newPebbleIteratorByCloning(intentIter.GetRawIter(), opts, StandardDurability) } *iiIter = intentInterleavingIter{ diff --git a/pkg/storage/min_version_test.go b/pkg/storage/min_version_test.go index 8ae595cb317e..2bd12c8bb53a 100644 --- a/pkg/storage/min_version_test.go +++ b/pkg/storage/min_version_test.go @@ -96,16 +96,18 @@ func TestSetMinVersion(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + st := cluster.MakeClusterSettings() p, err := Open(context.Background(), InMemory(), cluster.MakeClusterSettings(), CacheSize(0)) require.NoError(t, err) defer p.Close() - require.Equal(t, pebble.FormatMostCompatible, p.db.FormatMajorVersion()) + require.Equal(t, pebble.FormatPrePebblev1Marked, p.db.FormatMajorVersion()) - // Advancing the store cluster version to V22_2 should also advance the - // store's format major version. - err = p.SetMinVersion(clusterversion.ByKey(clusterversion.V22_2)) + ValueBlocksEnabled.Override(context.Background(), &st.SV, true) + // Advancing the store cluster version to one that supports value blocks + // should also advance the store's format major version. + err = p.SetMinVersion(clusterversion.ByKey(clusterversion.V23_1EnablePebbleFormatSSTableValueBlocks)) require.NoError(t, err) - require.Equal(t, pebble.FormatPrePebblev1Marked, p.db.FormatMajorVersion()) + require.Equal(t, pebble.FormatSSTableValueBlocks, p.db.FormatMajorVersion()) } func TestMinVersion_IsNotEncrypted(t *testing.T) { @@ -119,34 +121,23 @@ func TestMinVersion_IsNotEncrypted(t *testing.T) { defer func() { NewEncryptedEnvFunc = oldNewEncryptedEnvFunc }() NewEncryptedEnvFunc = fauxNewEncryptedEnvFunc + st := cluster.MakeClusterSettings() fs := vfs.NewMem() p, err := Open( context.Background(), Location{dir: "", fs: fs}, - cluster.MakeClusterSettings(), + st, EncryptionAtRest(nil)) require.NoError(t, err) defer p.Close() - - v1 := roachpb.Version{Major: 21, Minor: 1, Patch: 0, Internal: 122} - v2 := roachpb.Version{Major: 21, Minor: 1, Patch: 0, Internal: 126} - - ok, err := MinVersionIsAtLeastTargetVersion(p.unencryptedFS, p.path, v1) - require.NoError(t, err) - require.False(t, ok) - - require.NoError(t, p.SetMinVersion(v2)) - - ok, err = MinVersionIsAtLeastTargetVersion(p.unencryptedFS, p.path, v1) - require.NoError(t, err) - require.True(t, ok) + require.NoError(t, p.SetMinVersion(st.Version.BinaryVersion())) // Reading the file directly through the unencrypted MemFS should // succeed and yield the correct version. v, ok, err := getMinVersion(fs, "") require.NoError(t, err) require.True(t, ok) - require.Equal(t, v2, v) + require.Equal(t, st.Version.BinaryVersion(), v) } func fauxNewEncryptedEnvFunc( diff --git a/pkg/storage/mvcc.go b/pkg/storage/mvcc.go index 91e5c56c783d..df1df9ddbd38 100644 --- a/pkg/storage/mvcc.go +++ b/pkg/storage/mvcc.go @@ -22,7 +22,6 @@ import ( "sync" "time" - "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/kv/kvnemesis/kvnemesisutil" "github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock" @@ -94,8 +93,7 @@ var MVCCRangeTombstonesEnabled = settings.RegisterBoolSetting( // It requires the MVCCRangeTombstones version gate to be active, and the // setting storage.mvcc.range_tombstones.enabled to be enabled. func CanUseMVCCRangeTombstones(ctx context.Context, st *cluster.Settings) bool { - return st.Version.IsActive(ctx, clusterversion.TODODelete_V22_2MVCCRangeTombstones) && - MVCCRangeTombstonesEnabled.Get(&st.SV) + return MVCCRangeTombstonesEnabled.Get(&st.SV) } // MaxIntentsPerWriteIntentError sets maximum number of intents returned in @@ -4954,8 +4952,7 @@ func MVCCResolveWriteIntentRange( mvccIter = rw.NewMVCCIterator(MVCCKeyIterKind, iterOpts) } else { // For correctness, we need mvccIter to be consistent with engineIter. - mvccIter = newPebbleIteratorByCloning( - engineIter.GetRawIter(), iterOpts, StandardDurability, rw.SupportsRangeKeys()) + mvccIter = newPebbleIteratorByCloning(engineIter.GetRawIter(), iterOpts, StandardDurability) } iterAndBuf := GetBufUsingIter(mvccIter) defer func() { diff --git a/pkg/storage/pebble.go b/pkg/storage/pebble.go index 199c1aec7691..ca6ede5f0866 100644 --- a/pkg/storage/pebble.go +++ b/pkg/storage/pebble.go @@ -553,6 +553,8 @@ func DefaultPebbleOptions() *pebble.Options { MemTableStopWritesThreshold: 4, Merger: MVCCMerger, BlockPropertyCollectors: PebbleBlockPropertyCollectors, + // Minimum supported format. + FormatMajorVersion: pebble.FormatPrePebblev1Marked, } // Automatically flush 10s after the first range tombstone is added to a // memtable. This ensures that we can reclaim space even when there's no @@ -741,12 +743,6 @@ type Pebble struct { } asyncDone sync.WaitGroup - // supportsRangeKeys is 1 if the database supports range keys. It must - // be accessed atomically. - // - // TODO(erikgrinaker): Remove this after 22.2 when all databases support it. - supportsRangeKeys int32 - // closer is populated when the database is opened. The closer is associated // with the filesyetem closer io.Closer @@ -1093,9 +1089,6 @@ func NewPebble(ctx context.Context, cfg PebbleConfig) (p *Pebble, err error) { return nil, err } } - if p.db.FormatMajorVersion() >= pebble.FormatRangeKeys { - atomic.StoreInt32(&p.supportsRangeKeys, 1) - } return p, nil } @@ -1262,13 +1255,13 @@ func (p *Pebble) NewMVCCIterator(iterKind MVCCIterKind, opts IterOptions) MVCCIt return maybeWrapInUnsafeIter(iter) } - iter := newPebbleIterator(p.db, opts, StandardDurability, p.SupportsRangeKeys()) + iter := newPebbleIterator(p.db, opts, StandardDurability) return maybeWrapInUnsafeIter(iter) } // NewEngineIterator implements the Engine interface. func (p *Pebble) NewEngineIterator(opts IterOptions) EngineIterator { - return newPebbleIterator(p.db, opts, StandardDurability, p.SupportsRangeKeys()) + return newPebbleIterator(p.db, opts, StandardDurability) } // ConsistentIterators implements the Engine interface. @@ -1276,11 +1269,6 @@ func (p *Pebble) ConsistentIterators() bool { return false } -// SupportsRangeKeys implements the Engine interface. -func (p *Pebble) SupportsRangeKeys() bool { - return atomic.LoadInt32(&p.supportsRangeKeys) == 1 -} - // PinEngineStateForIterators implements the Engine interface. func (p *Pebble) PinEngineStateForIterators() error { return errors.AssertionFailedf( @@ -1355,7 +1343,7 @@ func (p *Pebble) ClearRawRange(start, end roachpb.Key, pointKeys, rangeKeys bool return err } } - if rangeKeys && p.SupportsRangeKeys() { + if rangeKeys { if err := p.db.RangeKeyDelete(startRaw, endRaw, pebble.Sync); err != nil { return err } @@ -1475,20 +1463,12 @@ func (p *Pebble) put(key MVCCKey, value []byte) error { // PutEngineRangeKey implements the Engine interface. func (p *Pebble) PutEngineRangeKey(start, end roachpb.Key, suffix, value []byte) error { - if !p.SupportsRangeKeys() { - return errors.Errorf("range keys not supported by Pebble database version %s", - p.db.FormatMajorVersion()) - } return p.db.RangeKeySet( EngineKey{Key: start}.Encode(), EngineKey{Key: end}.Encode(), suffix, value, pebble.Sync) } // ClearEngineRangeKey implements the Engine interface. func (p *Pebble) ClearEngineRangeKey(start, end roachpb.Key, suffix []byte) error { - if !p.SupportsRangeKeys() { - // These databases cannot contain range keys, so clearing is a noop. - return nil - } return p.db.RangeKeyUnset( EngineKey{Key: start}.Encode(), EngineKey{Key: end}.Encode(), suffix, pebble.Sync) } @@ -1522,17 +1502,7 @@ var LocalTimestampsEnabled = settings.RegisterBoolSetting( ) func shouldWriteLocalTimestamps(ctx context.Context, settings *cluster.Settings) bool { - if !LocalTimestampsEnabled.Get(&settings.SV) { - // Not enabled. - return false - } - ver := settings.Version.ActiveVersionOrEmpty(ctx) - if ver == (clusterversion.ClusterVersion{}) { - // Some tests fail to configure settings. In these cases, assume that it - // is safe to write local timestamps. - return true - } - return ver.IsActive(clusterversion.TODODelete_V22_2LocalTimestamps) + return LocalTimestampsEnabled.Get(&settings.SV) } // ShouldWriteLocalTimestamps implements the Writer interface. @@ -1984,27 +1954,22 @@ func (p *Pebble) SetMinVersion(version roachpb.Version) error { case !version.Less(clusterversion.ByKey(clusterversion.V23_1EnsurePebbleFormatSSTableValueBlocks)): formatVers = pebble.FormatSSTableValueBlocks - case !version.Less(clusterversion.ByKey(clusterversion.TODODelete_V22_2PebbleFormatPrePebblev1Marked)): + case !version.Less(clusterversion.ByKey(clusterversion.V22_2)): + // This is the earliest supported format. The code assumes that the features + // provided by this format are always available. formatVers = pebble.FormatPrePebblev1Marked - case !version.Less(clusterversion.ByKey(clusterversion.TODODelete_V22_2EnsurePebbleFormatVersionRangeKeys)): - formatVers = pebble.FormatRangeKeys - - case !version.Less(clusterversion.ByKey(clusterversion.TODODelete_V22_2PebbleFormatSplitUserKeysMarkedCompacted)): - formatVers = pebble.FormatSplitUserKeysMarkedCompacted - default: - // Corresponds to TODODelete_V22_1. - formatVers = pebble.FormatSplitUserKeysMarked + // This should never happen in production. But we tolerate tests creating + // imaginary older versions; we must still use the earliest supported + // format. + formatVers = pebble.FormatPrePebblev1Marked } if p.db.FormatMajorVersion() < formatVers { if err := p.db.RatchetFormatMajorVersion(formatVers); err != nil { return errors.Wrap(err, "ratcheting format major version") } - if formatVers >= pebble.FormatRangeKeys { - atomic.StoreInt32(&p.supportsRangeKeys, 1) - } } return nil } @@ -2141,14 +2106,13 @@ func (p *pebbleReadOnly) NewMVCCIterator(iterKind MVCCIterKind, opts IterOptions iter = &p.prefixIter } if iter.inuse { - return newPebbleIteratorByCloning(p.iter, opts, p.durability, p.SupportsRangeKeys()) + return newPebbleIteratorByCloning(p.iter, opts, p.durability) } if iter.iter != nil { iter.setOptions(opts, p.durability) } else { - iter.initReuseOrCreate( - p.parent.db, p.iter, p.iterUsed, opts, p.durability, p.SupportsRangeKeys()) + iter.initReuseOrCreate(p.parent.db, p.iter, p.iterUsed, opts, p.durability) if p.iter == nil { // For future cloning. p.iter = iter.iter @@ -2172,14 +2136,13 @@ func (p *pebbleReadOnly) NewEngineIterator(opts IterOptions) EngineIterator { iter = &p.prefixEngineIter } if iter.inuse { - return newPebbleIteratorByCloning(p.iter, opts, p.durability, p.SupportsRangeKeys()) + return newPebbleIteratorByCloning(p.iter, opts, p.durability) } if iter.iter != nil { iter.setOptions(opts, p.durability) } else { - iter.initReuseOrCreate( - p.parent.db, p.iter, p.iterUsed, opts, p.durability, p.SupportsRangeKeys()) + iter.initReuseOrCreate(p.parent.db, p.iter, p.iterUsed, opts, p.durability) if p.iter == nil { // For future cloning. p.iter = iter.iter @@ -2197,11 +2160,6 @@ func (p *pebbleReadOnly) ConsistentIterators() bool { return true } -// SupportsRangeKeys implements the Engine interface. -func (p *pebbleReadOnly) SupportsRangeKeys() bool { - return p.parent.SupportsRangeKeys() -} - // PinEngineStateForIterators implements the Engine interface. func (p *pebbleReadOnly) PinEngineStateForIterators() error { if p.iter == nil { @@ -2373,14 +2331,13 @@ func (p *pebbleSnapshot) NewMVCCIterator(iterKind MVCCIterKind, opts IterOptions return maybeWrapInUnsafeIter(iter) } - iter := MVCCIterator(newPebbleIterator( - p.snapshot, opts, StandardDurability, p.SupportsRangeKeys())) + iter := MVCCIterator(newPebbleIterator(p.snapshot, opts, StandardDurability)) return maybeWrapInUnsafeIter(iter) } // NewEngineIterator implements the Reader interface. func (p pebbleSnapshot) NewEngineIterator(opts IterOptions) EngineIterator { - return newPebbleIterator(p.snapshot, opts, StandardDurability, p.SupportsRangeKeys()) + return newPebbleIterator(p.snapshot, opts, StandardDurability) } // ConsistentIterators implements the Reader interface. @@ -2388,11 +2345,6 @@ func (p pebbleSnapshot) ConsistentIterators() bool { return true } -// SupportsRangeKeys implements the Reader interface. -func (p *pebbleSnapshot) SupportsRangeKeys() bool { - return p.parent.SupportsRangeKeys() -} - // PinEngineStateForIterators implements the Reader interface. func (p *pebbleSnapshot) PinEngineStateForIterators() error { // Snapshot already pins state, so nothing to do. diff --git a/pkg/storage/pebble_batch.go b/pkg/storage/pebble_batch.go index 8711e7a535ec..0bc9dc625263 100644 --- a/pkg/storage/pebble_batch.go +++ b/pkg/storage/pebble_batch.go @@ -177,14 +177,13 @@ func (p *pebbleBatch) NewMVCCIterator(iterKind MVCCIterKind, opts IterOptions) M handle = p.db } if iter.inuse { - return newPebbleIteratorByCloning(p.iter, opts, StandardDurability, p.SupportsRangeKeys()) + return newPebbleIteratorByCloning(p.iter, opts, StandardDurability) } if iter.iter != nil { iter.setOptions(opts, StandardDurability) } else { - iter.initReuseOrCreate( - handle, p.iter, p.iterUsed, opts, StandardDurability, p.SupportsRangeKeys()) + iter.initReuseOrCreate(handle, p.iter, p.iterUsed, opts, StandardDurability) if p.iter == nil { // For future cloning. p.iter = iter.iter @@ -211,14 +210,13 @@ func (p *pebbleBatch) NewEngineIterator(opts IterOptions) EngineIterator { handle = p.db } if iter.inuse { - return newPebbleIteratorByCloning(p.iter, opts, StandardDurability, p.SupportsRangeKeys()) + return newPebbleIteratorByCloning(p.iter, opts, StandardDurability) } if iter.iter != nil { iter.setOptions(opts, StandardDurability) } else { - iter.initReuseOrCreate( - handle, p.iter, p.iterUsed, opts, StandardDurability, p.SupportsRangeKeys()) + iter.initReuseOrCreate(handle, p.iter, p.iterUsed, opts, StandardDurability) if p.iter == nil { // For future cloning. p.iter = iter.iter @@ -235,11 +233,6 @@ func (p *pebbleBatch) ConsistentIterators() bool { return true } -// SupportsRangeKeys implements the Batch interface. -func (p *pebbleBatch) SupportsRangeKeys() bool { - return p.db.FormatMajorVersion() >= pebble.FormatRangeKeys -} - // PinEngineStateForIterators implements the Batch interface. func (p *pebbleBatch) PinEngineStateForIterators() error { if p.iter == nil { @@ -323,7 +316,7 @@ func (p *pebbleBatch) ClearRawRange(start, end roachpb.Key, pointKeys, rangeKeys return err } } - if rangeKeys && p.SupportsRangeKeys() { + if rangeKeys { if err := p.batch.RangeKeyDelete(p.buf, endRaw, pebble.Sync); err != nil { return err } @@ -450,19 +443,12 @@ func (p *pebbleBatch) PutRawMVCCRangeKey(rangeKey MVCCRangeKey, value []byte) er // PutEngineRangeKey implements the Engine interface. func (p *pebbleBatch) PutEngineRangeKey(start, end roachpb.Key, suffix, value []byte) error { - if !p.SupportsRangeKeys() { - return errors.Errorf("range keys not supported by Pebble database version %s", - p.db.FormatMajorVersion()) - } return p.batch.RangeKeySet( EngineKey{Key: start}.Encode(), EngineKey{Key: end}.Encode(), suffix, value, nil) } // ClearEngineRangeKey implements the Engine interface. func (p *pebbleBatch) ClearEngineRangeKey(start, end roachpb.Key, suffix []byte) error { - if !p.SupportsRangeKeys() { - return nil // noop - } return p.batch.RangeKeyUnset( EngineKey{Key: start}.Encode(), EngineKey{Key: end}.Encode(), suffix, nil) } diff --git a/pkg/storage/pebble_iterator.go b/pkg/storage/pebble_iterator.go index b045e8ca7d3e..39327291b104 100644 --- a/pkg/storage/pebble_iterator.go +++ b/pkg/storage/pebble_iterator.go @@ -19,7 +19,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/storage/pebbleiter" "github.com/cockroachdb/cockroach/pkg/util" - "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" @@ -48,10 +47,6 @@ type pebbleIterator struct { // initialized the first time an iterator's RangeKeys() method is called. mvccRangeKeyVersions []MVCCRangeKeyVersion - // True if the iterator's underlying reader supports range keys. - // - // TODO(erikgrinaker): Remove after 22.2. - supportsRangeKeys bool // Set to true to govern whether to call SeekPrefixGE or SeekGE. Skips // SSTables based on MVCC/Engine key when true. prefix bool @@ -87,11 +82,11 @@ var pebbleIterPool = sync.Pool{ // newPebbleIterator creates a new Pebble iterator for the given Pebble reader. func newPebbleIterator( - handle pebble.Reader, opts IterOptions, durability DurabilityRequirement, supportsRangeKeys bool, + handle pebble.Reader, opts IterOptions, durability DurabilityRequirement, ) *pebbleIterator { p := pebbleIterPool.Get().(*pebbleIterator) p.reusable = false // defensive - p.init(nil, opts, durability, supportsRangeKeys) + p.init(nil, opts, durability) p.iter = pebbleiter.MaybeWrap(handle.NewIter(&p.options)) return p } @@ -99,15 +94,12 @@ func newPebbleIterator( // newPebbleIteratorByCloning creates a new Pebble iterator by cloning the given // iterator and reconfiguring it. func newPebbleIteratorByCloning( - iter pebbleiter.Iterator, - opts IterOptions, - durability DurabilityRequirement, - supportsRangeKeys bool, + iter pebbleiter.Iterator, opts IterOptions, durability DurabilityRequirement, ) *pebbleIterator { var err error p := pebbleIterPool.Get().(*pebbleIterator) p.reusable = false // defensive - p.init(nil, opts, durability, supportsRangeKeys) + p.init(nil, opts, durability) p.iter, err = iter.Clone(pebble.CloneOptions{ IterOptions: &p.options, RefreshBatchView: true, @@ -125,7 +117,7 @@ func newPebbleSSTIterator( ) (*pebbleIterator, error) { p := pebbleIterPool.Get().(*pebbleIterator) p.reusable = false // defensive - p.init(nil, opts, StandardDurability, true /* supportsRangeKeys */) + p.init(nil, opts, StandardDurability) var externalIterOpts []pebble.ExternalIterOption if forwardOnly { @@ -146,10 +138,7 @@ func newPebbleSSTIterator( // reconfiguring the given iter. It is valid to pass a nil iter and then create // p.iter using p.options, to avoid redundant reconfiguration via SetOptions(). func (p *pebbleIterator) init( - iter pebbleiter.Iterator, - opts IterOptions, - durability DurabilityRequirement, - supportsRangeKeys bool, + iter pebbleiter.Iterator, opts IterOptions, durability DurabilityRequirement, ) { *p = pebbleIterator{ iter: iter, @@ -158,7 +147,6 @@ func (p *pebbleIterator) init( upperBoundBuf: p.upperBoundBuf, rangeKeyMaskingBuf: p.rangeKeyMaskingBuf, reusable: p.reusable, - supportsRangeKeys: supportsRangeKeys, } p.setOptions(opts, durability) p.inuse = true // after setOptions(), so panic won't cause reader to panic too @@ -176,14 +164,13 @@ func (p *pebbleIterator) initReuseOrCreate( clone bool, opts IterOptions, durability DurabilityRequirement, - supportsRangeKeys bool, // TODO(erikgrinaker): remove after 22.2 ) { if iter != nil && !clone { - p.init(iter, opts, durability, supportsRangeKeys) + p.init(iter, opts, durability) return } - p.init(nil, opts, durability, supportsRangeKeys) + p.init(nil, opts, durability) if iter == nil { p.iter = pebbleiter.MaybeWrap(handle.NewIter(&p.options)) } else if clone { @@ -212,19 +199,6 @@ func (p *pebbleIterator) setOptions(opts IterOptions, durability DurabilityRequi panic("can't use range key masking with prefix iterators") // very high overhead } - // If this Pebble database does not support range keys yet, fall back to - // only iterating over point keys to avoid panics. This is effectively the - // same, since a database without range key support contains no range keys, - // except in the case of RangesOnly where the iterator must always be empty. - if !p.supportsRangeKeys { - if opts.KeyTypes == IterKeyTypeRangesOnly { - opts.LowerBound = nil - opts.UpperBound = []byte{0} - } - opts.KeyTypes = IterKeyTypePointsOnly - opts.RangeKeyMaskingBelow = hlc.Timestamp{} - } - // Generate new Pebble iterator options. p.options = pebble.IterOptions{ OnlyReadGuaranteedDurable: durability == GuaranteedDurability, @@ -1008,12 +982,5 @@ func (p *pebbleIterator) assertMVCCInvariants() error { return errors.AssertionFailedf("IsPrefix() does not match prefix=%v", p.prefix) } - // Ensure !supportsRangeKeys never exposes range keys. - if !p.supportsRangeKeys { - if _, hasRange := p.HasPointAndRange(); hasRange { - return errors.AssertionFailedf("hasRange=true but supportsRangeKeys=false") - } - } - return nil } diff --git a/pkg/storage/pebble_iterator_test.go b/pkg/storage/pebble_iterator_test.go index 21ecad9cf98c..d1c9eb3b1c37 100644 --- a/pkg/storage/pebble_iterator_test.go +++ b/pkg/storage/pebble_iterator_test.go @@ -72,7 +72,7 @@ func TestPebbleIterator_Corruption(t *testing.T) { LowerBound: []byte("a"), UpperBound: []byte("z"), } - iter := newPebbleIterator(p.db, iterOpts, StandardDurability, false /* range keys */) + iter := newPebbleIterator(p.db, iterOpts, StandardDurability) // Seeking into the table catches the corruption. ok, err := iter.SeekEngineKeyGE(ek) @@ -94,7 +94,7 @@ func randStr(fill []byte, rng *rand.Rand) { func TestPebbleIterator_ExternalCorruption(t *testing.T) { defer leaktest.AfterTest(t)() - version := clusterversion.ByKey(clusterversion.TODODelete_V22_2EnsurePebbleFormatVersionRangeKeys) + version := clusterversion.ByKey(clusterversion.V22_2) st := cluster.MakeTestingClusterSettingsWithVersions(version, version, true) ctx := context.Background() rng := rand.New(rand.NewSource(timeutil.Now().UnixNano())) diff --git a/pkg/storage/pebble_test.go b/pkg/storage/pebble_test.go index 5cba54859c40..00885abe281e 100644 --- a/pkg/storage/pebble_test.go +++ b/pkg/storage/pebble_test.go @@ -24,10 +24,12 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/storage/enginepb" + "github.com/cockroachdb/cockroach/pkg/storage/fs" "github.com/cockroachdb/cockroach/pkg/testutils" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/cockroachdb/errors" @@ -1287,13 +1289,17 @@ func TestIncompatibleVersion(t *testing.T) { dir: "", fs: vfs.NewMem(), } - oldVer := roachpb.Version{Major: 21, Minor: 1} - stOld := cluster.MakeTestingClusterSettingsWithVersions(oldVer, oldVer, true /* initializeVersion */) - p, err := Open(ctx, loc, stOld) + + p, err := Open(ctx, loc, cluster.MakeTestingClusterSettings()) require.NoError(t, err) p.Close() - stNew := cluster.MakeTestingClusterSettings() - _, err = Open(ctx, loc, stNew) + // Overwrite the min version file with an unsupported version. + version := roachpb.Version{Major: 21, Minor: 1} + b, err := protoutil.Marshal(&version) + require.NoError(t, err) + require.NoError(t, fs.SafeWriteToFile(loc.fs, loc.dir, MinVersionFilename, b)) + + _, err = Open(ctx, loc, cluster.MakeTestingClusterSettings()) require.ErrorContains(t, err, "is too old for running version") } diff --git a/pkg/storage/sst_writer.go b/pkg/storage/sst_writer.go index ccccbdd669d7..f9bbdb80143f 100644 --- a/pkg/storage/sst_writer.go +++ b/pkg/storage/sst_writer.go @@ -62,10 +62,7 @@ func MakeIngestionWriterOptions(ctx context.Context, cs *cluster.Settings) sstab // By default, take a conservative approach and assume we don't have newer // table features available. Upgrade to an appropriate version only if the // cluster supports it. - format := sstable.TableFormatPebblev1 // Block properties. - if cs.Version.IsActive(ctx, clusterversion.TODODelete_V22_2EnablePebbleFormatVersionRangeKeys) { - format = sstable.TableFormatPebblev2 // Range keys. - } + format := sstable.TableFormatPebblev2 if cs.Version.IsActive(ctx, clusterversion.V23_1EnablePebbleFormatSSTableValueBlocks) && ValueBlocksEnabled.Get(&cs.SV) { format = sstable.TableFormatPebblev3 @@ -81,10 +78,8 @@ func MakeBackupSSTWriter(ctx context.Context, cs *cluster.Settings, f io.Writer) // By default, take a conservative approach and assume we don't have newer // table features available. Upgrade to an appropriate version only if the // cluster supports it. - format := sstable.TableFormatPebblev1 // Block properties. - if cs.Version.IsActive(ctx, clusterversion.TODODelete_V22_2EnablePebbleFormatVersionRangeKeys) { - format = sstable.TableFormatPebblev2 // Range keys. - } + format := sstable.TableFormatPebblev2 + // TODO(sumeer): add code to use TableFormatPebblev3 after confirming that // we won't run afoul of any stale tooling that reads backup ssts. opts := DefaultPebbleOptions().MakeWriterOptions(0, format) diff --git a/pkg/storage/sst_writer_test.go b/pkg/storage/sst_writer_test.go index af2a94db22c4..b46b3641c898 100644 --- a/pkg/storage/sst_writer_test.go +++ b/pkg/storage/sst_writer_test.go @@ -19,7 +19,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings/cluster" - "github.com/cockroachdb/cockroach/pkg/testutils/skip" "github.com/cockroachdb/cockroach/pkg/util/encoding" "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" @@ -79,30 +78,28 @@ func makePebbleSST(t testing.TB, kvs []MVCCKeyValue, ingestion bool) []byte { func TestMakeIngestionWriterOptions(t *testing.T) { defer leaktest.AfterTest(t)() - skip.WithIssue(t, 95530, "bump minBinary to 22.2. Skip 22.2 mixed-version tests for future cleanup") - testCases := []struct { name string st *cluster.Settings want sstable.TableFormat }{ { - name: "before feature gate", + name: "22.2", st: cluster.MakeTestingClusterSettingsWithVersions( - clusterversion.ByKey(clusterversion.TODODelete_V22_2EnablePebbleFormatVersionRangeKeys-1), + clusterversion.ByKey(clusterversion.V22_2), clusterversion.TestingBinaryMinSupportedVersion, true, ), - want: sstable.TableFormatPebblev1, + want: sstable.TableFormatPebblev2, }, { - name: "at feature gate", - st: cluster.MakeTestingClusterSettingsWithVersions( - clusterversion.ByKey(clusterversion.TODODelete_V22_2EnablePebbleFormatVersionRangeKeys), - clusterversion.TestingBinaryMinSupportedVersion, - true, - ), - want: sstable.TableFormatPebblev2, + name: "with value blocks", + st: func() *cluster.Settings { + st := cluster.MakeTestingClusterSettings() + ValueBlocksEnabled.Override(context.Background(), &st.SV, true) + return st + }(), + want: sstable.TableFormatPebblev3, }, } @@ -115,45 +112,6 @@ func TestMakeIngestionWriterOptions(t *testing.T) { } } -func TestSSTWriterRangeKeysUnsupported(t *testing.T) { - defer leaktest.AfterTest(t)() - defer log.Scope(t).Close(t) - - ctx := context.Background() - - // Set up a version that doesn't support range keys. - version := clusterversion.ByKey(clusterversion.TODODelete_V22_2EnsurePebbleFormatVersionRangeKeys - 1) - st := cluster.MakeTestingClusterSettingsWithVersions(version, version, true) - - writers := map[string]SSTWriter{ - "ingestion": MakeIngestionSSTWriter(ctx, st, &MemFile{}), - "backup": MakeBackupSSTWriter(ctx, st, &MemFile{}), - } - - for name, w := range writers { - t.Run(name, func(t *testing.T) { - defer w.Close() - - rangeKey := rangeKey("a", "b", 2) - - // Put should error, but clears are noops. - err := w.PutMVCCRangeKey(rangeKey, MVCCValue{}) - require.Error(t, err) - require.Contains(t, err.Error(), "range keys not supported") - - err = w.PutEngineRangeKey(rangeKey.StartKey, rangeKey.EndKey, - EncodeMVCCTimestampSuffix(rangeKey.Timestamp), nil) - require.Error(t, err) - require.Contains(t, err.Error(), "range keys not supported") - - require.NoError(t, w.ClearMVCCRangeKey(rangeKey)) - require.NoError(t, w.ClearEngineRangeKey(rangeKey.StartKey, rangeKey.EndKey, - EncodeMVCCTimestampSuffix(rangeKey.Timestamp))) - require.NoError(t, w.ClearRawRange(rangeKey.StartKey, rangeKey.EndKey, false, true)) - }) - } -} - func TestSSTWriterRangeKeys(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) diff --git a/pkg/storage/temp_engine.go b/pkg/storage/temp_engine.go index 103037cce18c..0a5aef019f45 100644 --- a/pkg/storage/temp_engine.go +++ b/pkg/storage/temp_engine.go @@ -88,6 +88,7 @@ func newPebbleTempEngine( cfg.Opts.Comparer = pebble.DefaultComparer cfg.Opts.DisableWAL = true cfg.Opts.Experimental.KeyValidationFunc = nil + cfg.Opts.BlockPropertyCollectors = nil return nil }, ) diff --git a/pkg/upgrade/upgrademanager/manager_external_test.go b/pkg/upgrade/upgrademanager/manager_external_test.go index 64b6bd6a7cd3..fb32d3b895e7 100644 --- a/pkg/upgrade/upgrademanager/manager_external_test.go +++ b/pkg/upgrade/upgrademanager/manager_external_test.go @@ -301,23 +301,20 @@ func TestConcurrentMigrationAttempts(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - // We're going to be migrating from startKey to endKey. We end up needing - // to use real versions because the ListBetween uses the keys compiled into - // the clusterversion package. - const ( - startMajor = 42 - endMajor = 48 - ) - migrationRunCounts := make(map[clusterversion.ClusterVersion]int) + // We're going to be migrating from the current version to imaginary future versions. + current := clusterversion.ByKey(clusterversion.BinaryVersionKey) + versions := []roachpb.Version{current} + for i := int32(1); i <= 6; i++ { + v := current + v.Major += i + versions = append(versions, v) + } // RegisterKVMigration the upgrades to update the map with run counts. // There should definitely not be any concurrency of execution, so the race // detector should not fire. - var versions []roachpb.Version + migrationRunCounts := make(map[clusterversion.ClusterVersion]int) - for major := int32(startMajor); major <= endMajor; major++ { - versions = append(versions, roachpb.Version{Major: major}) - } ctx := context.Background() var active int32 // used to detect races tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ diff --git a/pkg/upgrade/upgrades/schema_changes_external_test.go b/pkg/upgrade/upgrades/schema_changes_external_test.go index 898a600da07a..b98303aa6600 100644 --- a/pkg/upgrade/upgrades/schema_changes_external_test.go +++ b/pkg/upgrade/upgrades/schema_changes_external_test.go @@ -242,9 +242,9 @@ func testMigrationWithFailures( skip.UnderRace(t, "very slow") - // We're going to be migrating from startCV to endCV. - startCV := roachpb.Version{Major: 2041} - endCV := roachpb.Version{Major: 2042} + // We're going to be migrating from the minimum supported version to the current version. + startCV := clusterversion.ByKey(clusterversion.BinaryMinSupportedVersionKey) + endCV := clusterversion.ByKey(clusterversion.BinaryVersionKey) // The tests follows the following procedure. //