Skip to content

Commit

Permalink
Merge #127034 #127374 #127404
Browse files Browse the repository at this point in the history 
127034: bootstrap: create an explicit zoneconfig for timeseries data r=rafiss a=rafiss

This will make it more clear that the timeseries zone config can be changed independently from all the other zone configs.

fixes #123762
Release note: None

127374: ui: change 'Physcial Replication Producer' category to 'Replication Producer' r=msbutler a=msbutler

Epic: none

Release note: none

127404: roachtest: rename DSC job compat test for 24.3; avoid pooling r=rafiss a=rafiss

See individual commits.

fixes #124442
fixes #124791
Release note: None

Co-authored-by: Rafi Shamim <rafi@cockroachlabs.com>
Co-authored-by: Michael Butler <butler@cockroachlabs.com>
  • Loading branch information
@rafiss @msbutler
3 people committed Jul 18, 2024
4 parents 51926bf + 59f5cb9 + 8e7952d + 2614bab commit 6154ebf
Show file tree
Hide file tree
Showing 16 changed files with 84 additions and 80 deletions.
8 changes: 8 additions & 0 deletions pkg/ccl/logictestccl/testdata/logic_test/crdb_internal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ SELECT zone_id, target FROM crdb_internal.zones ORDER BY 1
0 RANGE default
16 RANGE meta
17 RANGE system
18 RANGE timeseries
22 RANGE liveness
111 TABLE test.public.t4
111 INDEX test.public.t4@myt4index
Expand Down Expand Up @@ -109,6 +110,13 @@ RANGE system ALTER RANGE system CONFIGURE ZONE USING
num_replicas = 5,
constraints = '[]',
lease_preferences = '[]'
RANGE timeseries ALTER RANGE timeseries CONFIGURE ZONE USING
range_min_bytes = 134217728,
range_max_bytes = 536870912,
gc.ttlseconds = 14400,
num_replicas = 3,
constraints = '[]',
lease_preferences = '[]'
TABLE test.public.t4 ALTER TABLE test.public.t4 CONFIGURE ZONE USING
num_replicas = 7

Expand Down
55 changes: 28 additions & 27 deletions pkg/ccl/logictestccl/testdata/logic_test/zone
View file
Original file line number Diff line number Diff line change
Expand Up @@ -616,33 +616,34 @@ SELECT target, range_name, database_name, table_name, index_name, partition_name
FROM crdb_internal.zones
ORDER BY target
----
DATABASE "my database" NULL my database NULL NULL NULL
DATABASE system NULL system NULL NULL NULL
INDEX "my database".public."my table"@"my index" NULL my database my table my index NULL
INDEX test.public.t36642@secondary NULL test t36642 secondary NULL
INDEX test.public.t36642@tertiary NULL test t36642 tertiary NULL
INDEX test.public.t36644@secondary NULL test t36644 secondary NULL
INDEX test.public.t36644@tertiary NULL test t36644 tertiary NULL
INDEX test.public.t@secondary NULL test t secondary NULL
INDEX test.public.t@tertiary NULL test t tertiary NULL
PARTITION "my partition" OF INDEX "my database".public."my table"@"my index" NULL my database my table my index my partition
PARTITION "my partition" OF INDEX "my database".public."my table"@"my table_pkey" NULL my database my table my table_pkey my partition
PARTITION x1_idx OF INDEX test.public.t38391@foo NULL test t38391 foo x1_idx
RANGE default default NULL NULL NULL NULL
RANGE liveness liveness NULL NULL NULL NULL
RANGE meta meta NULL NULL NULL NULL
RANGE system system NULL NULL NULL NULL
TABLE "my database".public."my table" NULL my database my table NULL NULL
TABLE system.public.lease NULL system lease NULL NULL
TABLE system.public.replication_constraint_stats NULL system replication_constraint_stats NULL NULL
TABLE system.public.replication_stats NULL system replication_stats NULL NULL
TABLE system.public.span_stats_tenant_boundaries NULL system span_stats_tenant_boundaries NULL NULL
TABLE system.public.statement_activity NULL system statement_activity NULL NULL
TABLE system.public.statement_statistics NULL system statement_statistics NULL NULL
TABLE system.public.tenant_usage NULL system tenant_usage NULL NULL
TABLE system.public.transaction_activity NULL system transaction_activity NULL NULL
TABLE system.public.transaction_statistics NULL system transaction_statistics NULL NULL
TABLE test.public.t NULL test t NULL NULL
DATABASE "my database" NULL my database NULL NULL NULL
DATABASE system NULL system NULL NULL NULL
INDEX "my database".public."my table"@"my index" NULL my database my table my index NULL
INDEX test.public.t36642@secondary NULL test t36642 secondary NULL
INDEX test.public.t36642@tertiary NULL test t36642 tertiary NULL
INDEX test.public.t36644@secondary NULL test t36644 secondary NULL
INDEX test.public.t36644@tertiary NULL test t36644 tertiary NULL
INDEX test.public.t@secondary NULL test t secondary NULL
INDEX test.public.t@tertiary NULL test t tertiary NULL
PARTITION "my partition" OF INDEX "my database".public."my table"@"my index" NULL my database my table my index my partition
PARTITION "my partition" OF INDEX "my database".public."my table"@"my table_pkey" NULL my database my table my table_pkey my partition
PARTITION x1_idx OF INDEX test.public.t38391@foo NULL test t38391 foo x1_idx
RANGE default default NULL NULL NULL NULL
RANGE liveness liveness NULL NULL NULL NULL
RANGE meta meta NULL NULL NULL NULL
RANGE system system NULL NULL NULL NULL
RANGE timeseries timeseries NULL NULL NULL NULL
TABLE "my database".public."my table" NULL my database my table NULL NULL
TABLE system.public.lease NULL system lease NULL NULL
TABLE system.public.replication_constraint_stats NULL system replication_constraint_stats NULL NULL
TABLE system.public.replication_stats NULL system replication_stats NULL NULL
TABLE system.public.span_stats_tenant_boundaries NULL system span_stats_tenant_boundaries NULL NULL
TABLE system.public.statement_activity NULL system statement_activity NULL NULL
TABLE system.public.statement_statistics NULL system statement_statistics NULL NULL
TABLE system.public.tenant_usage NULL system tenant_usage NULL NULL
TABLE system.public.transaction_activity NULL system transaction_activity NULL NULL
TABLE system.public.transaction_statistics NULL system transaction_statistics NULL NULL
TABLE test.public.t NULL test t NULL NULL

# Test the zone information being displayed in SHOW CREATE
statement ok
Expand Down
1 change: 1 addition & 0 deletions pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/testdata/full_translate_named_zones_deleted
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ SELECT id FROM system.zones
11
16
17
18
22
25
27
Expand Down
7 changes: 3 additions & 4 deletions pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/testdata/named_zones
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,12 @@
# RANGE DEFAULT is akin to a full reconciliation.


# ID 18 is for the time series range which is the only named zone that doesn't
# have an entry in `system.zones` at bootstrap. It should inherit from RANGE
# DEFAULT.
# ID 18 is for the time series range. At bootstrap, it has an explicit zone
# config which should inherit from RANGE DEFAULT.
query-sql
SELECT count(*) FROM system.zones WHERE id=18
----
0
1

translate named-zone=timeseries
----
Expand Down
1 change: 1 addition & 0 deletions pkg/cmd/roachtest/tests/gossip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,7 @@ func runGossipRestartNodeOne(ctx context.Context, t test.Test, c cluster.Cluster
run(`ALTER DATABASE system %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`)
run(`ALTER RANGE meta %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`)
run(`ALTER RANGE liveness %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`)
run(`ALTER RANGE timeseries %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`)
if t.IsBuildVersion("v19.2.0") {
run(`ALTER TABLE system.replication_stats %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`)
run(`ALTER TABLE system.replication_constraint_stats %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`)
Expand Down
56 changes: 14 additions & 42 deletions pkg/cmd/roachtest/tests/mixed_version_job_compatibility_in_declarative_schema_changer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func registerDeclarativeSchemaChangerJobCompatibilityInMixedVersion(r registry.R
// This test requires us to come back and change the stmts in executeSupportedDDLs to be those
// supported in the "previous" major release.
r.Add(registry.TestSpec{
Name: "declarative_schema_changer/job-compatibility-mixed-version-V241-V242",
Name: "declarative_schema_changer/job-compatibility-mixed-version-V242-V243",
Owner: registry.OwnerSQLFoundations,
Cluster: r.MakeClusterSpec(4),
CompatibleClouds: registry.AllExceptAWS,
Expand Down Expand Up @@ -97,40 +97,13 @@ func executeSupportedDDLs(
return err
}

// DDLs supported in V23_2.
v232DDLs := []string{
`COMMENT ON DATABASE testdb IS 'this is a database comment'`,
`COMMENT ON SCHEMA testdb.testsc IS 'this is a schema comment'`,
`COMMENT ON TABLE testdb.testsc.t IS 'this is a table comment'`,
`COMMENT ON COLUMN testdb.testsc.t.i IS 'this is a column comment'`,
`COMMENT ON INDEX testdb.testsc.t@idx IS 'this is a index comment'`,
`COMMENT ON CONSTRAINT check_j ON testdb.testsc.t IS 'this is a constraint comment'`,
`ALTER TABLE testdb.testsc.t ADD COLUMN k INT DEFAULT 35`,
`ALTER TABLE testdb.testsc.t DROP COLUMN k`,
`ALTER TABLE testdb.testsc.t2 ADD PRIMARY KEY (i)`,
`ALTER TABLE testdb.testsc.t2 ALTER PRIMARY KEY USING COLUMNS (j)`,
`CREATE FUNCTION fn(a INT) RETURNS INT AS 'SELECT a*a' LANGUAGE SQL`,
`CREATE INDEX ON testdb.testsc.t3 (i)`,
`ALTER TABLE testdb.testsc.t2 ALTER COLUMN k SET NOT NULL`,
`ALTER TABLE testdb.testsc.t2 ALTER PRIMARY KEY USING COLUMNS (k) USING HASH`,
`ALTER TABLE testdb.testsc.t3 ADD CONSTRAINT j_unique UNIQUE WITHOUT INDEX (k)`,
`ALTER TABLE testdb.testsc.t3 ADD CONSTRAINT j_unique_not_valid UNIQUE WITHOUT INDEX (k) NOT VALID`,
`ALTER TABLE testdb.testsc.t3 ADD CONSTRAINT fk FOREIGN KEY (j) REFERENCES testdb.testsc.t2 (j)`,
`ALTER TABLE testdb.testsc.t3 ADD CONSTRAINT fk_not_valid FOREIGN KEY (j) REFERENCES testdb.testsc.t2 (j) NOT VALID`,
`ALTER TABLE testdb.testsc.t3 ADD CONSTRAINT check_positive CHECK (j > 0)`,
`ALTER TABLE testdb.testsc.t3 ADD CONSTRAINT check_positive_not_valid CHECK (j > 0) NOT VALID`,
`ALTER TABLE testdb.testsc.t3 DROP CONSTRAINT check_positive`,
`ALTER TABLE testdb.testsc.t3 VALIDATE CONSTRAINT check_positive_not_valid`,
}

// DDLs supported in V24_1.
v241DDLs := []string{
`ALTER TABLE testdb.testsc.t ADD COLUMN k int, ADD COLUMN l int, DROP COLUMN l`,
`ALTER TABLE testdb.testsc.t ALTER COLUMN k SET DEFAULT 42`,
`ALTER TABLE testdb.testsc.t ALTER COLUMN k DROP DEFAULT`,
`CREATE DATABASE testdb2`,
`CREATE SCHEMA testdb2.testsc`,
`CREATE SEQUENCE testdb2.testsc.s`,
// DDLs supported in V24_2.
// TODO(sql-foundations): uncomment these when the final 24.2 cluster version
// is created.
v242DDLs := []string{
// `ALTER DATABASE testdb CONFIGURE ZONE USING gc.ttlseconds=1000`,
// `ALTER TABLE testdb.testsc.t CONFIGURE ZONE USING gc.ttlseconds=2000`,
// `COMMENT ON TYPE testdb.testsc.typ IS 'comment'`,
}

// Used to clean up our CREATE-d elements after we are done with them.
Expand All @@ -151,7 +124,7 @@ func executeSupportedDDLs(
`DROP DATABASE testdb2 CASCADE`,
}

ddls := append(v232DDLs, append(v241DDLs, cleanup...)...)
ddls := append(v242DDLs, cleanup...)

for _, ddl := range ddls {
if err := helper.ExecWithGateway(r, nodes, ddl); err != nil {
Expand All @@ -178,13 +151,8 @@ func runDeclarativeSchemaChangerJobCompatibilityInMixedVersion(

// Ensure that the declarative schema changer is off so that we do not get failures related to unimplemented
// statements in the dsc.
for _, node := range c.All() {
if err := helper.ExecWithGateway(r, option.NodeListOption{node}, "SET use_declarative_schema_changer = off"); err != nil {
return err
}
}

setUpQuery := `
SET use_declarative_schema_changer = off;
CREATE DATABASE IF NOT EXISTS testdb;
CREATE SCHEMA IF NOT EXISTS testdb.testsc;
CREATE TABLE IF NOT EXISTS testdb.testsc.t (i INT PRIMARY KEY, j INT NOT NULL, INDEX idx (j), CONSTRAINT check_j CHECK (j > 0));
Expand All @@ -205,7 +173,11 @@ CREATE VIEW IF NOT EXISTS testdb.testsc.v AS (SELECT i*2 FROM testdb.testsc.t);
// so that we don't fall back to legacy schema changer implicitly.
// Being explicit can help catch bugs that will otherwise be
// buried by the fallback.
// To make sure the session variables are applied correctly, we limit each
// connection pool to have at most 1 connection.
for _, node := range c.All() {
db := helper.Connect(node)
db.SetMaxOpenConns(1)
if err := helper.ExecWithGateway(r, option.NodeListOption{node}, "SET use_declarative_schema_changer = unsafe_always"); err != nil {
return err
}
Expand Down
1 change: 1 addition & 0 deletions pkg/kv/kvserver/replicate_queue_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2167,6 +2167,7 @@ func TestPromoteNonVoterInAddVoter(t *testing.T) {
setConstraintFn("RANGE system", 7, 7, "")
setConstraintFn("RANGE liveness", 7, 7, "")
setConstraintFn("RANGE meta", 7, 7, "")
setConstraintFn("RANGE timeseries", 7, 7, "")
setConstraintFn("RANGE default", 7, 7, "")
testutils.SucceedsSoon(t, func() error {
if err := forceScanOnAllReplicationQueues(tc); err != nil {
Expand Down
6 changes: 5 additions & 1 deletion pkg/kv/kvserver/reports/reporter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,9 +196,13 @@ func TestCriticalLocalitiesReportIntegration(t *testing.T) {
}
require.Greater(t, len(systemZoneIDs), 0, "expected some system zones, got none")
// Remove the entries in systemZoneIDs that don't get critical locality reports.
// The default range (zoneID=0) does get reports, but since there is no system
// data in the default range that is not part of a more granular zone (e.g.,
// the meta, liveness, timeseries, or system range), there won't be a report
// for it.
i := 0
for j, zid := range systemZoneIDs {
if zoneChangesReplication(&systemZones[j]) {
if zoneChangesReplication(&systemZones[j]) && zid != keys.RootNamespaceID {
systemZoneIDs[i] = zid
i++
}
Expand Down
5 changes: 5 additions & 0 deletions pkg/sql/catalog/bootstrap/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -511,6 +511,10 @@ func InitialZoneConfigKVs(
metaRangeZoneConf := protoutil.Clone(defaultSystemZoneConfig).(*zonepb.ZoneConfig)
livenessZoneConf := protoutil.Clone(defaultSystemZoneConfig).(*zonepb.ZoneConfig)

// The timeseries zone is the same as the default zone, but we create it
// explicitly here so it's more clearly reported with SHOW ALL ZONE CONFIGURATIONS.
timeseriesZoneConf := protoutil.Clone(defaultZoneConfig).(*zonepb.ZoneConfig)

// .meta zone config entry with a shorter GC time.
metaRangeZoneConf.GC.TTLSeconds = 60 * 60 // 1h

Expand All @@ -534,6 +538,7 @@ func InitialZoneConfigKVs(
add(keys.MetaRangesID, metaRangeZoneConf)
add(keys.LivenessRangesID, livenessZoneConf)
add(keys.SystemRangesID, systemZoneConf)
add(keys.TimeseriesRangesID, timeseriesZoneConf)
add(keys.SystemDatabaseID, systemZoneConf)
add(keys.ReplicationConstraintStatsTableID, replicationConstraintStatsZoneConf)
add(keys.ReplicationStatsTableID, replicationStatsZoneConf)
Expand Down
3 changes: 2 additions & 1 deletion pkg/sql/catalog/bootstrap/testdata/testdata
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
system hash=ccfcf37e550c9bceebe02e444e7b9269f7565df315eccde68b1ff384e742c8b8
system hash=4f192396c59085346e3e43f000323964a38c038f3e4b28bab0c09d92bb0ae6eb
----
[{"key":"8b"}
,{"key":"8b89898a89","value":"0312450a0673797374656d10011a250a0d0a0561646d696e1080101880100a0c0a04726f6f7410801018801012046e6f646518032200280140004a006a0a08d8843d10011800200a"}
Expand Down Expand Up @@ -65,6 +65,7 @@ system hash=ccfcf37e550c9bceebe02e444e7b9269f7565df315eccde68b1ff384e742c8b8
,{"key":"8d89938a89","value":"03220308d804500158017800"}
,{"key":"8d89988a89","value":"031080808040188080808002220308901c2805500058007801"}
,{"key":"8d89998a89","value":"031080808040188080808002220308c0702805500058007801"}
,{"key":"8d899a8a89","value":"031080808040188080808002220308c0702803500058007801"}
,{"key":"8d899e8a89","value":"031080808040188080808002220308d8042805500058007801"}
,{"key":"8d89a18a89","value":"03220308d804500158017800"}
,{"key":"8d89a38a89","value":"03220308d804500158017800"}
Expand Down
2 changes: 2 additions & 0 deletions pkg/sql/catalog/internal/catkv/testdata/testdata_system
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,8 @@ catalog:
zone: gc.ttlseconds=3600
"017":
zone: gc.ttlseconds=14400
"018":
zone: gc.ttlseconds=14400
"019":
descriptor: relation
namespace: (1, 29, "web_sessions")
Expand Down
1 change: 0 additions & 1 deletion pkg/sql/logictest/testdata/logic_test/crdb_internal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -567,7 +567,6 @@ SELECT 'schema.bar'::regclass::int

statement ok
INSERT INTO system.zones (id, config) VALUES
(18, (SELECT raw_config_protobuf FROM crdb_internal.zones WHERE zone_id = 0)),
($testdb_id, (SELECT raw_config_protobuf FROM crdb_internal.zones WHERE zone_id = 0)),
($testdb_foo_id, (SELECT raw_config_protobuf FROM crdb_internal.zones WHERE zone_id = 0)),
($schema_bar_id, (SELECT raw_config_protobuf FROM crdb_internal.zones WHERE zone_id = 0))
Expand Down
1 change: 1 addition & 0 deletions pkg/sql/logictest/testdata/logic_test/zone_config_system_tenant
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ SELECT zone_id, target FROM crdb_internal.zones ORDER BY 1
11 TABLE system.public.lease
16 RANGE meta
17 RANGE system
18 RANGE timeseries
22 RANGE liveness
25 TABLE system.public.replication_constraint_stats
27 TABLE system.public.replication_stats
Expand Down
2 changes: 1 addition & 1 deletion pkg/sql/tests/system_table_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func TestInitialKeys(t *testing.T) {
var nonDescKeys int
if systemTenant {
codec = keys.SystemSQLCodec
nonDescKeys = 16
nonDescKeys = 17
} else {
codec = keys.MakeSQLCodec(roachpb.MustMakeTenantID(5))
nonDescKeys = 8
Expand Down
Loading

0 comments on commit 6154ebf

Please sign in to comment.