From 7a57e7ea1e4b188c76407edf4d318099a9f2bcbf Mon Sep 17 00:00:00 2001 From: irfan sharif Date: Thu, 16 Mar 2023 17:21:49 -0400 Subject: [PATCH] spanconfig: enable range coalescing by default Fixes #81008. We built the basic infrastructure to coalesce ranges across table boundaries back in 22.2 as part of #66063. We've enabled this optimization for secondary tenants since then, but hadn't for the system tenant because of two primary blockers: - #93617: SHOW RANGES was broken by coalesced ranges. - #84105: APIs to compute sizes for schema objects (used in our UI) was broken by coalesced ranges. In both these cases we baked in assumptions about there being a minimum of one-{table,index,partition}-per-range. These blockers didn't apply to secondary tenants at the time since they didn't have access to SHOW RANGES, nor the UI pages where these schema statistics were displayed. We've addressed both these blockers in the 23.1 cycle as part of bridging the compatibility between secondary tenants and yesteryear's system tenant. - #93644 revised SHOW RANGES and crdb_internal.ranges{,_no_leases}, both internally and its external UX, to accommodate ranges straddling table/database boundaries. - #96223 re-worked our SpanStats API to work in the face of coalesced ranges, addressing #84105. Release note (general change): CockroachDB would previously use separate ranges for each table, index, or partition. This is no longer true -- it's possible now to have multiple tables, indexes, and partitions to get packed into the same range. For users with many such "schema objects", this will reduce the total range count in their clusters. This is especially true if individual tables, indexes, or partitions are smaller than the default configured maximum range size (controlled using zone configs, specifically the range_max_bytes parameter). We made this change to improve scalability with respect to the number of schema objects, since the underlying range count now no longer a bottleneck. Users upgrading from 22.2, once finalizing their upgrade, may observe a round of range merges and snapshot transfers (to power said range merges) as a result of this change. If users want to opt-out of this optimization, they can use the following cluster setting: SET CLUSTER SETTING spanconfig.storage_coalesce_adjacent.enabled = false; --- .../settings/settings-for-tenants.txt | 2 +- docs/generated/settings/settings.html | 4 +++- pkg/clusterversion/cockroach_versions.go | 8 +++++++ pkg/server/admin_test.go | 4 ++++ pkg/spanconfig/spanconfigstore/BUILD.bazel | 1 + pkg/spanconfig/spanconfigstore/span_store.go | 24 +++++++++++-------- .../spanconfigstore/span_store_test.go | 1 + pkg/spanconfig/spanconfigstore/store.go | 6 +++-- pkg/spanconfig/spanconfigstore/store_test.go | 14 +++++++---- pkg/spanconfig/testing_knobs.go | 2 +- .../containers/dataDistribution/index.tsx | 9 +++++++ 11 files changed, 55 insertions(+), 20 deletions(-) diff --git a/docs/generated/settings/settings-for-tenants.txt b/docs/generated/settings/settings-for-tenants.txt index a5e5992aa6ed..a97b32fc4fe7 100644 --- a/docs/generated/settings/settings-for-tenants.txt +++ b/docs/generated/settings/settings-for-tenants.txt @@ -294,4 +294,4 @@ trace.opentelemetry.collector string address of an OpenTelemetry trace collecto trace.snapshot.rate duration 0s if non-zero, interval at which background trace snapshots are captured tenant-rw trace.span_registry.enabled boolean true if set, ongoing traces can be seen at https:///#/debug/tracez tenant-rw trace.zipkin.collector string the address of a Zipkin instance to receive traces, as :. If no port is specified, 9411 will be used. tenant-rw -version version 1000023.1-6 set the active cluster version in the format '.' tenant-rw +version version 1000023.1-8 set the active cluster version in the format '.' tenant-rw diff --git a/docs/generated/settings/settings.html b/docs/generated/settings/settings.html index 80321402a9e2..857fe971c848 100644 --- a/docs/generated/settings/settings.html +++ b/docs/generated/settings/settings.html @@ -119,6 +119,8 @@
server.web_session.purge.ttl
duration1h0m0sif nonzero, entries in system.web_sessions older than this duration are periodically purgedServerless/Dedicated/Self-Hosted
server.web_session_timeout
duration168h0m0sthe duration that a newly created web session will be validServerless/Dedicated/Self-Hosted
spanconfig.bounds.enabled
booleantruedictates whether span config bounds are consulted when serving span configs for secondary tenantsDedicated/Self-Hosted +
spanconfig.storage_coalesce_adjacent.enabled
booleantruecollapse adjacent ranges with the same span configs, for the ranges specific to the system tenantDedicated/Self-Hosted +
spanconfig.tenant_coalesce_adjacent.enabled
booleantruecollapse adjacent ranges with the same span configs across all secondary tenant keyspacesDedicated/Self-Hosted
sql.auth.change_own_password.enabled
booleanfalsecontrols whether a user is allowed to change their own password, even if they have no other privilegesServerless/Dedicated/Self-Hosted
sql.auth.resolve_membership_single_scan.enabled
booleantruedetermines whether to populate the role membership cache with a single scanServerless/Dedicated/Self-Hosted
sql.closed_session_cache.capacity
integer1000the maximum number of sessions in the cacheServerless/Dedicated/Self-Hosted @@ -246,6 +248,6 @@
trace.snapshot.rate
duration0sif non-zero, interval at which background trace snapshots are capturedServerless/Dedicated/Self-Hosted
trace.span_registry.enabled
booleantrueif set, ongoing traces can be seen at https://<ui>/#/debug/tracezServerless/Dedicated/Self-Hosted
trace.zipkin.collector
stringthe address of a Zipkin instance to receive traces, as <host>:<port>. If no port is specified, 9411 will be used.Serverless/Dedicated/Self-Hosted -
version
version1000023.1-6set the active cluster version in the format '<major>.<minor>'Serverless/Dedicated/Self-Hosted +
version
version1000023.1-8set the active cluster version in the format '<major>.<minor>'Serverless/Dedicated/Self-Hosted diff --git a/pkg/clusterversion/cockroach_versions.go b/pkg/clusterversion/cockroach_versions.go index 9a3438c3e732..5e30740f1e9b 100644 --- a/pkg/clusterversion/cockroach_versions.go +++ b/pkg/clusterversion/cockroach_versions.go @@ -534,6 +534,10 @@ const ( // indexes are enabled. V23_2_PartiallyVisibleIndexes + // V23_2_EnableRangeCoalescingForSystemTenant enables range coalescing for + // the system tenant. + V23_2_EnableRangeCoalescingForSystemTenant + // ************************************************* // Step (1) Add new versions here. // Do not add new versions to a patch release. @@ -927,6 +931,10 @@ var rawVersionsSingleton = keyedVersions{ Key: V23_2_PartiallyVisibleIndexes, Version: roachpb.Version{Major: 23, Minor: 1, Internal: 6}, }, + { + Key: V23_2_EnableRangeCoalescingForSystemTenant, + Version: roachpb.Version{Major: 23, Minor: 1, Internal: 8}, + }, // ************************************************* // Step (2): Add new versions here. diff --git a/pkg/server/admin_test.go b/pkg/server/admin_test.go index 5d28e8277732..ee8decc5bcbc 100644 --- a/pkg/server/admin_test.go +++ b/pkg/server/admin_test.go @@ -2206,6 +2206,10 @@ func TestAdminAPIDataDistribution(t *testing.T) { firstServer := testCluster.Server(0) sqlDB := sqlutils.MakeSQLRunner(testCluster.ServerConn(0)) + // TODO(irfansharif): The data-distribution page and underyling APIs don't + // know how to deal with coalesced ranges. See #97942. + sqlDB.Exec(t, `SET CLUSTER SETTING spanconfig.storage_coalesce_adjacent.enabled = false`) + // Create some tables. sqlDB.Exec(t, `CREATE DATABASE roachblog`) sqlDB.Exec(t, `CREATE TABLE roachblog.posts (id INT PRIMARY KEY, title text, body text)`) diff --git a/pkg/spanconfig/spanconfigstore/BUILD.bazel b/pkg/spanconfig/spanconfigstore/BUILD.bazel index 4b09ec6c55c4..2402b2e80f64 100644 --- a/pkg/spanconfig/spanconfigstore/BUILD.bazel +++ b/pkg/spanconfig/spanconfigstore/BUILD.bazel @@ -15,6 +15,7 @@ go_library( importpath = "github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigstore", visibility = ["//visibility:public"], deps = [ + "//pkg/clusterversion", "//pkg/keys", "//pkg/multitenant/tenantcapabilities", "//pkg/roachpb", diff --git a/pkg/spanconfig/spanconfigstore/span_store.go b/pkg/spanconfig/spanconfigstore/span_store.go index e1a9f2833446..3b2c9adc5096 100644 --- a/pkg/spanconfig/spanconfigstore/span_store.go +++ b/pkg/spanconfig/spanconfigstore/span_store.go @@ -14,6 +14,7 @@ import ( "context" "sort" + "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/settings" @@ -24,7 +25,7 @@ import ( "github.com/cockroachdb/errors" ) -// tenantCoalesceAdjacentSetting is a hidden cluster setting that +// tenantCoalesceAdjacentSetting is a public cluster setting that // controls whether we coalesce adjacent ranges across all secondary // tenant keyspaces if they have the same span config. var tenantCoalesceAdjacentSetting = settings.RegisterBoolSetting( @@ -32,17 +33,17 @@ var tenantCoalesceAdjacentSetting = settings.RegisterBoolSetting( "spanconfig.tenant_coalesce_adjacent.enabled", `collapse adjacent ranges with the same span configs across all secondary tenant keyspaces`, true, -) +).WithPublic() -// StorageCoalesceAdjacentSetting is a hidden cluster setting that -// controls whether we coalesce adjacent ranges outside of the -// secondary tenant keyspaces if they have the same span config. +// StorageCoalesceAdjacentSetting is a public cluster setting that controls +// whether we coalesce adjacent ranges outside of the secondary tenant keyspaces +// if they have the same span config. var StorageCoalesceAdjacentSetting = settings.RegisterBoolSetting( settings.SystemOnly, "spanconfig.storage_coalesce_adjacent.enabled", - `collapse adjacent ranges with the same span configs for the ranges specific to the system tenant`, - false, -) + `collapse adjacent ranges with the same span configs, for the ranges specific to the system tenant`, + true, +).WithPublic() // spanConfigStore is an in-memory data structure to store and retrieve // SpanConfigs associated with a single span. Internally it makes use of a @@ -104,7 +105,9 @@ func (s *spanConfigStore) forEachOverlapping( // computeSplitKey returns the first key we should split on because of the // presence a span config given a start and end key pair. -func (s *spanConfigStore) computeSplitKey(start, end roachpb.RKey) (roachpb.RKey, error) { +func (s *spanConfigStore) computeSplitKey( + ctx context.Context, start, end roachpb.RKey, +) (roachpb.RKey, error) { sp := roachpb.Span{Key: start.AsRawKey(), EndKey: end.AsRawKey()} // Generally split keys are going to be the start keys of span config entries. @@ -152,7 +155,8 @@ func (s *spanConfigStore) computeSplitKey(start, end roachpb.RKey) (roachpb.RKey // ranges. systemTableUpperBound := keys.SystemSQLCodec.TablePrefix(keys.MaxReservedDescID + 1) if roachpb.Key(rem).Compare(systemTableUpperBound) < 0 || - !StorageCoalesceAdjacentSetting.Get(&s.settings.SV) { + !(StorageCoalesceAdjacentSetting.Get(&s.settings.SV) && + s.settings.Version.IsActive(ctx, clusterversion.V23_2_EnableRangeCoalescingForSystemTenant)) { return roachpb.RKey(match.span.Key), nil } } else { diff --git a/pkg/spanconfig/spanconfigstore/span_store_test.go b/pkg/spanconfig/spanconfigstore/span_store_test.go index fa8e53bf3494..b66aa6351e5e 100644 --- a/pkg/spanconfig/spanconfigstore/span_store_test.go +++ b/pkg/spanconfig/spanconfigstore/span_store_test.go @@ -234,6 +234,7 @@ func TestRandomized(t *testing.T) { // span and asserts on all the properties above. querySpan := getRandomSpan() splitKeys := store.TestingSplitKeys(t, + ctx, roachpb.RKey(querySpan.Key), roachpb.RKey(querySpan.EndKey), ) diff --git a/pkg/spanconfig/spanconfigstore/store.go b/pkg/spanconfig/spanconfigstore/store.go index 76cce4c13d93..c8c6a242e274 100644 --- a/pkg/spanconfig/spanconfigstore/store.go +++ b/pkg/spanconfig/spanconfigstore/store.go @@ -123,11 +123,13 @@ func (s *Store) NeedsSplit(ctx context.Context, start, end roachpb.RKey) (bool, } // ComputeSplitKey is part of the spanconfig.StoreReader interface. -func (s *Store) ComputeSplitKey(_ context.Context, start, end roachpb.RKey) (roachpb.RKey, error) { +func (s *Store) ComputeSplitKey( + ctx context.Context, start, end roachpb.RKey, +) (roachpb.RKey, error) { s.mu.RLock() defer s.mu.RUnlock() - return s.mu.spanConfigStore.computeSplitKey(start, end) + return s.mu.spanConfigStore.computeSplitKey(ctx, start, end) } // GetSpanConfigForKey is part of the spanconfig.StoreReader interface. diff --git a/pkg/spanconfig/spanconfigstore/store_test.go b/pkg/spanconfig/spanconfigstore/store_test.go index 4c37f638d110..9ef7c9f08934 100644 --- a/pkg/spanconfig/spanconfigstore/store_test.go +++ b/pkg/spanconfig/spanconfigstore/store_test.go @@ -37,20 +37,24 @@ func (s *Store) TestingApplyInternal( // TestingSplitKeys returns the computed list of range split points between // [start, end). -func (s *Store) TestingSplitKeys(tb testing.TB, start, end roachpb.RKey) []roachpb.RKey { +func (s *Store) TestingSplitKeys( + tb testing.TB, ctx context.Context, start, end roachpb.RKey, +) []roachpb.RKey { s.mu.RLock() defer s.mu.RUnlock() - return s.mu.spanConfigStore.TestingSplitKeys(tb, start, end) + return s.mu.spanConfigStore.TestingSplitKeys(tb, ctx, start, end) } // TestingSplitKeys returns the computed list of range split points between // [start, end). -func (s *spanConfigStore) TestingSplitKeys(tb testing.TB, start, end roachpb.RKey) []roachpb.RKey { +func (s *spanConfigStore) TestingSplitKeys( + tb testing.TB, ctx context.Context, start, end roachpb.RKey, +) []roachpb.RKey { var splitKeys []roachpb.RKey computeStart := start for { - splitKey, err := s.computeSplitKey(computeStart, end) + splitKey, err := s.computeSplitKey(ctx, computeStart, end) require.NoError(tb, err) if splitKey == nil { break @@ -184,7 +188,7 @@ func TestDataDriven(t *testing.T) { span := spanconfigtestutils.ParseSpan(t, spanStr) start, end := roachpb.RKey(span.Key), roachpb.RKey(span.EndKey) - splitKeys := store.TestingSplitKeys(t, start, end) + splitKeys := store.TestingSplitKeys(t, ctx, start, end) var b strings.Builder for _, splitKey := range splitKeys { b.WriteString(fmt.Sprintf("key=%s\n", string(splitKey))) diff --git a/pkg/spanconfig/testing_knobs.go b/pkg/spanconfig/testing_knobs.go index e6c19893a80e..df5899360c01 100644 --- a/pkg/spanconfig/testing_knobs.go +++ b/pkg/spanconfig/testing_knobs.go @@ -123,7 +123,7 @@ type TestingKnobs struct { StoreDisableCoalesceAdjacent bool // StoreIgnoreCoalesceAdjacentExceptions, if set, ignores the cluster settings - // spanconfig.{host,tenant}_coalesce_adjacent.enabled. It also allows + // spanconfig.{storage,tenant}_coalesce_adjacent.enabled. It also allows // coalescing system database ranges for the host tenant. StoreIgnoreCoalesceAdjacentExceptions bool diff --git a/pkg/ui/workspaces/db-console/src/views/cluster/containers/dataDistribution/index.tsx b/pkg/ui/workspaces/db-console/src/views/cluster/containers/dataDistribution/index.tsx index 9a8c493dda51..937e833e2e70 100644 --- a/pkg/ui/workspaces/db-console/src/views/cluster/containers/dataDistribution/index.tsx +++ b/pkg/ui/workspaces/db-console/src/views/cluster/containers/dataDistribution/index.tsx @@ -182,6 +182,15 @@ export class DataDistributionPage extends React.Component<

Data Distribution

+

+ Note: the data distribution render does not work with coalesced + ranges. To disable coalesced ranges (would increase range count), + run the following SQL statement: +

+ 
+            SET CLUSTER SETTING spanconfig.storage_coalesce_adjacent.enabled =
+            false;
+