From 68994d7309ca10d9a2d29e7d61abb9739abe5b19 Mon Sep 17 00:00:00 2001 From: Mark Sirek Date: Thu, 14 Jul 2022 16:10:30 -0700 Subject: [PATCH] opt: session setting to enforce queries access only home region rows Fixes #86228 This commit adds a new session setting, `enforce_home_region`, which causes queries to error out if they need to talk to regions other than the gateway region to answer the query. A home region specifies the region(s) from which consistent reads from a set of rows in a table can be served locally. The home region for a set of rows in a multiregion table is determined differently depending on the type of multiregion table involved: | Locality | Home Region | | -------- | ----------- | | REGIONAL BY ROW | Home region determined by crdb_region column value | | REGIONAL BY TABLE | All rows share a single home region | | GLOBAL | Any region can act as the home region | When `enforce_home_region` is true, and a query has no home region (for example, reading from different home regions in a REGIONAL BY ROW table), error code 42899 (`QueryHasNoHomeRegion`) is returned. When `enforce_home_region` is true, and a query's home region differs from the gateway region, error code 42898 (`QueryNotRunningInHomeRegion`) is returned. The error message, in some instances, provides a useful tip on possible steps to take to allow the query to run entirely in the gateway region, e.g., ``` Query is not running in its home region. Try running the query from region 'ap-southeast-2'. Query has no home region. Try removing the join expression. Query has no home region. Try accessing only tables in multi-region databases with ZONE survivability. ``` Support for this new session mode is being added in 3 phases. This commit consists of phase 1, which include only simple static checks during query compilation for the following allowed cases: - A scan of a table with `LOCALITY REGIONAL BY TABLE` with primary region matching the gateway region - A scan of a table with `LOCALITY GLOBAL` - A scan of a table with `LOCALITY REGIONAL BY ROW` using only local constraints (e.g. crdb_region = 'ca-central-1') - A scan of a table with `LOCALITY REGIONAL BY ROW` using locality-optimized search. - A lookup join into a table with `LOCALITY REGIONAL BY ROW` using locality-optimized lookup. Only tables in multiregion databases with ZONE survivability may be scanned without error because with REGION survivability, ranges in a down region may be served non-local to the gateway region, so are not guaranteed to have low latency. Note that locality-optimized search and lookup join are not guaranteed to scan no remote rows, but are still allowed. Release note (sql change): A new session setting, enforce_home_region, is added, which when true causes queries which may scan rows via a database connection outside of the query's home region to error out. Also, only tables in multiregion databases with ZONE survivability may be scanned without error when this setting is true because with REGION survivability, ranges in a down region may be served non-local to the gateway region, so are not guaranteed to have low latency. --- .../multi_region_remote_access_error | 493 ++++++++++++++++++ .../generated_test.go | 7 + pkg/sql/exec_util.go | 4 + pkg/sql/faketreeeval/BUILD.bazel | 1 + pkg/sql/faketreeeval/evalctx.go | 22 +- pkg/sql/importer/import_table_creation.go | 8 +- pkg/sql/internal.go | 3 + .../testdata/logic_test/information_schema | 1 + .../logictest/testdata/logic_test/pg_catalog | 3 + .../logictest/testdata/logic_test/show_source | 1 + pkg/sql/opt/BUILD.bazel | 3 + pkg/sql/opt/cat/table.go | 16 + pkg/sql/opt/constraint/constraint.go | 24 + pkg/sql/opt/distribution/BUILD.bazel | 5 + pkg/sql/opt/distribution/distribution.go | 79 ++- pkg/sql/opt/exec/execbuilder/BUILD.bazel | 1 + pkg/sql/opt/exec/execbuilder/builder.go | 3 +- pkg/sql/opt/exec/execbuilder/relational.go | 172 +++++- pkg/sql/opt/exec/explain/plan_gist_factory.go | 20 + pkg/sql/opt/memo/expr.go | 42 ++ pkg/sql/opt/memo/memo.go | 5 +- pkg/sql/opt/memo/memo_test.go | 6 + pkg/sql/opt/metadata.go | 25 +- pkg/sql/opt/optgen/exprgen/BUILD.bazel | 1 + pkg/sql/opt/optgen/exprgen/expr_gen.go | 2 + pkg/sql/opt/props/physical/BUILD.bazel | 2 + pkg/sql/opt/props/physical/distribution.go | 89 +++- pkg/sql/opt/table_meta.go | 77 ++- pkg/sql/opt/testutils/testcat/test_catalog.go | 20 + pkg/sql/opt/xform/coster.go | 39 +- pkg/sql/opt/xform/optimizer.go | 2 - pkg/sql/opt_catalog.go | 68 +++ pkg/sql/pgwire/pgcode/codes.go | 2 + pkg/sql/region_util.go | 19 +- pkg/sql/sem/eval/BUILD.bazel | 1 + pkg/sql/sem/eval/context.go | 6 + pkg/sql/sem/eval/deps.go | 7 + .../local_only_session_data.proto | 4 + pkg/sql/vars.go | 17 + 39 files changed, 1245 insertions(+), 55 deletions(-) create mode 100644 pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error diff --git a/pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error b/pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error new file mode 100644 index 000000000000..9cb956bba244 --- /dev/null +++ b/pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error @@ -0,0 +1,493 @@ +# tenant-cluster-setting-override-opt: allow-multi-region-abstractions-for-secondary-tenants +# LogicTest: multiregion-9node-3region-3azs !metamorphic + +# Set the closed timestamp interval to be short to shorten the amount of time +# we need to wait for the system config to propagate. +statement ok +SET CLUSTER SETTING kv.closed_timestamp.side_transport_interval = '10ms'; + +statement ok +SET CLUSTER SETTING kv.closed_timestamp.target_duration = '10ms'; + +# Start with SURVIVE ZONE FAILURE for positive tests. +# SURVIVE REGION FAILURE cases will always error out. +statement ok +CREATE DATABASE multi_region_test_db PRIMARY REGION "ap-southeast-2" REGIONS "ca-central-1", "us-east-1" SURVIVE ZONE FAILURE; + +statement ok +USE multi_region_test_db + +query T +SELECT gateway_region(); +---- +ap-southeast-2 + +statement ok +CREATE TABLE messages_global ( + account_id INT NOT NULL, + message_id UUID DEFAULT gen_random_uuid(), + message STRING NOT NULL, + PRIMARY KEY (account_id), + INDEX msg_idx(message) +) LOCALITY GLOBAL + +statement ok +CREATE TABLE messages_rbt ( + account_id INT NOT NULL, + message_id UUID DEFAULT gen_random_uuid(), + message STRING NOT NULL, + PRIMARY KEY (account_id), + INDEX msg_idx(message) +) LOCALITY REGIONAL BY TABLE + +statement ok +CREATE TABLE messages_rbr ( + account_id INT NOT NULL, + message_id UUID DEFAULT gen_random_uuid(), + message STRING NOT NULL, + crdb_region crdb_internal_region NOT NULL, + PRIMARY KEY (account_id), + INDEX msg_idx(message) +) +LOCALITY REGIONAL BY ROW + +statement ok +CREATE TABLE customers ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name STRING NOT NULL +) LOCALITY REGIONAL BY ROW; + +statement ok +ALTER TABLE customers INJECT STATISTICS '[ + { + "columns": ["id"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 100 + }, + { + "columns": ["crdb_region"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 3 + }, + { + "columns": ["crdb_region","id"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 100 + } +]' + +statement ok +CREATE TABLE orders ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + cust_id UUID NOT NULL, + items STRING NOT NULL, + INDEX (cust_id), + FOREIGN KEY (cust_id, crdb_region) REFERENCES customers (id, crdb_region) ON UPDATE CASCADE +) LOCALITY REGIONAL BY ROW; + +statement ok +ALTER TABLE orders INJECT STATISTICS '[ + { + "columns": ["id"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 100 + }, + { + "columns": ["cust_id"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 10 + }, + { + "columns": ["crdb_region"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 3 + }, + { + "columns": ["crdb_region","id"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 100 + } +]' + +statement ok +SET enforce_home_region = true + +statement error pq: Query has no home region\. Try adding a filter on messages_rbr\.crdb_region and/or on key column \(messages_rbr\.account_id\)\. +SELECT * FROM messages_rbr rbr INNER LOOKUP JOIN messages_global g2 ON rbr.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id + +statement ok +CREATE TABLE parent ( + p_id INT PRIMARY KEY, + FAMILY (p_id) +) LOCALITY REGIONAL BY ROW; + +statement ok +CREATE TABLE child ( + c_id INT PRIMARY KEY, + c_p_id INT REFERENCES parent (p_id), + INDEX (c_p_id), + FAMILY (c_id, c_p_id) +) LOCALITY REGIONAL BY ROW; + +# Non-locality-optimized lookup join with no home region should error out. +statement error pq: Query has no home region\. Try adding a filter on parent\.crdb_region and/or on key column \(parent\.p_id\)\. Try adding a filter on child\.crdb_region and/or on key column \(child\.c_id\)\. +SELECT * FROM parent p, child c WHERE p_id = c_p_id AND +p.crdb_region = c.crdb_region LIMIT 1 + +query T +EXPLAIN(OPT) SELECT * FROM parent p, child c WHERE c_id = 10 AND p_id = c_p_id +---- +project + └── inner-join (lookup parent [as=p]) + ├── lookup columns are key + ├── locality-optimized-search + │ ├── scan child [as=c] + │ │ └── constraint: /13/11: [/'ap-southeast-2'/10 - /'ap-southeast-2'/10] + │ └── scan child [as=c] + │ └── constraint: /18/16 + │ ├── [/'ca-central-1'/10 - /'ca-central-1'/10] + │ └── [/'us-east-1'/10 - /'us-east-1'/10] + └── filters (true) + + +# Locality optimized lookup join is allowed in phase 1, though it is +# not guaranteed to run with low latency. +query TT +SELECT * FROM child WHERE NOT EXISTS (SELECT * FROM parent WHERE p_id = c_p_id) AND c_id = 10 +---- + +query T +EXPLAIN(OPT) SELECT * FROM child WHERE NOT EXISTS (SELECT * FROM parent WHERE p_id = c_p_id) AND c_id = 10 +---- +anti-join (lookup parent) + ├── lookup columns are key + ├── anti-join (lookup parent) + │ ├── lookup columns are key + │ ├── locality-optimized-search + │ │ ├── scan child + │ │ │ └── constraint: /13/11: [/'ap-southeast-2'/10 - /'ap-southeast-2'/10] + │ │ └── scan child + │ │ └── constraint: /18/16 + │ │ ├── [/'ca-central-1'/10 - /'ca-central-1'/10] + │ │ └── [/'us-east-1'/10 - /'us-east-1'/10] + │ └── filters (true) + └── filters (true) + +statement ok +SET locality_optimized_partitioned_index_scan = false + +# This query should error out because it is not locality optimized. +statement error pq: Query has no home region\. Try adding a filter on parent\.crdb_region and/or on key column \(parent\.p_id\)\. Try adding a filter on child\.crdb_region and/or on key column \(child\.c_id\)\. +SELECT * FROM child WHERE NOT EXISTS (SELECT * FROM parent WHERE p_id = c_p_id) AND c_id = 10 + +statement ok +RESET locality_optimized_partitioned_index_scan + +# Locality optimized search is allowed. +query T +SELECT * FROM parent LIMIT 1 +---- + +query T +EXPLAIN(OPT) SELECT * FROM parent LIMIT 1 +---- +locality-optimized-search + ├── scan parent + │ ├── constraint: /7/6: [/'ap-southeast-2' - /'ap-southeast-2'] + │ └── limit: 1 + └── scan parent + ├── constraint: /11/10: [/'ca-central-1' - /'us-east-1'] + └── limit: 1 + +# Locality optimized search with lookup join will be supported in phase 2 or 3 +# when we can dynamically determine if the lookup will access a remote region. +statement error pq: Query has no home region\. Try adding a filter on orders\.crdb_region and/or on key column \(orders\.id\)\. +SELECT * FROM customers c JOIN orders o ON c.id = o.cust_id AND + (c.crdb_region = o.crdb_region) WHERE c.id = '69a1c2c2-5b18-459e-94d2-079dc53a4dd0' + +# Locality optimized lookup join is allowed. +query TTTTTTT +SELECT * FROM messages_rbr rbr, messages_rbt rbt WHERE rbr.account_id = rbt.account_id LIMIT 1 +---- + +query T +EXPLAIN SELECT * FROM messages_rbr rbr, messages_rbt rbt WHERE rbr.account_id = rbt.account_id LIMIT 1 +---- +distribution: local +vectorized: true +· +• limit +│ count: 1 +│ +└── • lookup join + │ table: messages_rbr@messages_rbr_pkey + │ equality cols are key + │ lookup condition: (crdb_region = 'ap-southeast-2') AND (account_id = account_id) + │ remote lookup condition: (crdb_region IN ('ca-central-1', 'us-east-1')) AND (account_id = account_id) + │ + └── • scan + missing stats + table: messages_rbt@messages_rbt_pkey + spans: FULL SCAN (SOFT LIMIT) + +# Select from a global table is OK with ZONE survivability. +query TTT +SELECT * FROM messages_global@messages_global_pkey +---- + +# Select from REGIONAL BY TABLE is OK with ZONE survivability. +query T +SELECT message from messages_rbt@messages_rbt_pkey +---- + +# A local join between an RBR and RBT table should be allowed. +query TTTTTTT +SELECT * FROM messages_rbt rbt INNER LOOKUP JOIN messages_rbr rbr ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' +---- + +query T +EXPLAIN(OPT) SELECT * FROM messages_rbt rbt INNER LOOKUP JOIN messages_rbr rbr ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' +---- +inner-join (lookup messages_rbr [as=rbr]) + ├── flags: force lookup join (into right side) + ├── lookup columns are key + ├── project + │ ├── scan messages_rbt [as=rbt] + │ └── projections + │ └── 'ap-southeast-2' + └── filters (true) + +# A local join between an RBR and RBT table should be allowed. +query TTTTTTT +SELECT * FROM messages_rbr rbr INNER LOOKUP JOIN messages_rbt rbt ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' +---- + +query T +EXPLAIN(OPT) SELECT * FROM messages_rbr rbr INNER LOOKUP JOIN messages_rbt rbt ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' +---- +inner-join (lookup messages_rbt [as=rbt]) + ├── flags: force lookup join (into right side) + ├── lookup columns are key + ├── scan messages_rbr [as=rbr] + │ └── constraint: /4/1: [/'ap-southeast-2' - /'ap-southeast-2'] + └── filters (true) + +# A lookup join between with a global table as either input should be allowed. +query TTTTTT +SELECT * FROM messages_global g1 INNER LOOKUP JOIN messages_global g2 ON g1.account_id = g2.account_id +---- + +query T +EXPLAIN(OPT) SELECT * FROM messages_global g1 INNER LOOKUP JOIN messages_global g2 ON g1.account_id = g2.account_id +---- +inner-join (lookup messages_global [as=g2]) + ├── flags: force lookup join (into right side) + ├── lookup columns are key + ├── scan messages_global [as=g1] + └── filters (true) + +# A join relation with local home region as the left input of lookup join should be allowed. +query TTTTTTTTT +SELECT * FROM messages_global g1 INNER LOOKUP JOIN messages_global g2 ON g1.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id +---- + +query T +EXPLAIN(OPT) SELECT * FROM messages_global g1 INNER LOOKUP JOIN messages_global g2 ON g1.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id +---- +inner-join (lookup messages_global [as=g3]) + ├── flags: force lookup join (into right side) + ├── lookup columns are key + ├── inner-join (lookup messages_global [as=g2]) + │ ├── flags: force lookup join (into right side) + │ ├── lookup columns are key + │ ├── scan messages_global [as=g1] + │ └── filters (true) + └── filters (true) + +# A join relation with no home region as the left input of lookup join should +# not be allowed. +statement error pq: Query has no home region\. Try adding a filter on messages_rbr\.crdb_region and/or on key column \(messages_rbr\.account_id\)\. +SELECT * FROM messages_rbr rbr INNER LOOKUP JOIN messages_global g2 ON rbr.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id + +# A lookup join relation with a left input join relation which uses locality +# optimized scan in one of the tables of the lookup join should be allowed. +query TTTTTTTTTT +SELECT * FROM (SELECT * FROM messages_rbr LIMIT 1) rbr INNER LOOKUP JOIN + messages_global g2 ON rbr.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id +---- + +query T +EXPLAIN(OPT) SELECT * FROM (SELECT * FROM messages_rbr LIMIT 1) rbr INNER LOOKUP JOIN + messages_global g2 ON rbr.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id +---- +inner-join (lookup messages_global [as=g3]) + ├── flags: force lookup join (into right side) + ├── lookup columns are key + ├── inner-join (lookup messages_global [as=g2]) + │ ├── flags: force lookup join (into right side) + │ ├── lookup columns are key + │ ├── locality-optimized-search + │ │ ├── scan messages_rbr + │ │ │ ├── constraint: /33/30: [/'ap-southeast-2' - /'ap-southeast-2'] + │ │ │ └── limit: 1 + │ │ └── scan messages_rbr + │ │ ├── constraint: /39/36: [/'ca-central-1' - /'us-east-1'] + │ │ └── limit: 1 + │ └── filters (true) + └── filters (true) + +statement ok +ALTER TABLE messages_rbt SET LOCALITY REGIONAL BY TABLE IN "us-east-1"; + +# Select from REGIONAL BY TABLE should indicate the gateway region to use. +statement error pq: Query is not running in its home region. Try running the query from region 'us-east-1'. +SELECT message from messages_rbt@messages_rbt_pkey + +# Logging in through the appropriate gateway region allows reading from an RBR +# table with a span on crdb_region. +query T nodeidx=4 +SET enforce_home_region = true; +USE multi_region_test_db; +SELECT message from messages_rbr@msg_idx WHERE crdb_region = 'ca-central-1' +---- + +query T nodeidx=4 +SET enforce_home_region = true; +USE multi_region_test_db; +EXPLAIN(OPT) SELECT message from messages_rbr@msg_idx WHERE crdb_region = 'ca-central-1' +---- +project + └── scan messages_rbr@msg_idx + ├── constraint: /4/3/1: [/'ca-central-1' - /'ca-central-1'] + └── flags: force-index=msg_idx + +# Lookup join should detect REGIONAL BY TABLE in the wrong region. +statement error pq: Query has no home region\. The home region \('us-east-1'\) of table 'messages_rbt' does not match the home region \('ap-southeast-2'\) of lookup table 'messages_rbr'\. +SELECT * FROM messages_rbt rbt inner lookup join messages_rbr rbr ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' + +# Lookup join should detect REGIONAL BY TABLE in the wrong region. +statement error pq: Query has no home region\. The home region \('ap-southeast-2'\) of table 'messages_rbr' does not match the home region \('us-east-1'\) of lookup table 'messages_rbt'\. +SELECT * FROM messages_rbr rbr inner lookup join messages_rbt rbt ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' + +# Equality predicate on crdb_region of an RBR table is allowed. +query T +SELECT message from messages_rbr@msg_idx WHERE crdb_region = 'ap-southeast-2' +---- + +query T +EXPLAIN(OPT) SELECT message from messages_rbr@msg_idx WHERE crdb_region = 'ap-southeast-2' +---- +project + └── scan messages_rbr@msg_idx + ├── constraint: /4/3/1: [/'ap-southeast-2' - /'ap-southeast-2'] + └── flags: force-index=msg_idx + +statement ok +PREPARE s AS SELECT message from messages_rbr@msg_idx WHERE crdb_region = $1 + +# Prepared statement accessing the local span is allowed. +query T +EXECUTE s('ap-southeast-2') +---- + +# Prepared statement accessing a remote span is disallowed. +statement error pq: Query is not running in its home region. Try running the query from region 'us-east-1'. +EXECUTE s('us-east-1') + +statement ok +RESET enforce_home_region + +statement ok +CREATE DATABASE non_multiregion_test_db; + +statement ok +USE non_multiregion_test_db + +statement ok +CREATE TABLE messages ( + account_id INT NOT NULL, + message_id UUID DEFAULT gen_random_uuid(), + message STRING NOT NULL, + PRIMARY KEY (account_id), + INDEX msg_idx(message) +) + +statement ok +SET enforce_home_region = true + +# Tables in non-multiregion databases have no home region. +statement error pq: Query has no home region. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM messages + +# If any table in a query has no home region, error out. +statement error pq: Query has no home region. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM non_multiregion_test_db.messages, multi_region_test_db.messages_global + +# Scans with contradictions in predicates are allowed. +query TTT +SELECT * FROM messages WHERE account_id = 1 AND account_id = 2 +---- + +query T +EXPLAIN SELECT * FROM messages WHERE account_id = 1 AND account_id = 2 +---- +distribution: local +vectorized: true +· +• norows + +# A lookup join from a multiregion table to non-multiregion table is not +# allowed. +statement error pq: Query has no home region. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM multi_region_test_db.messages_global mr INNER LOOKUP JOIN non_multiregion_test_db.messages nmr + ON mr.account_id = nmr.account_id + +statement ok +ALTER DATABASE multi_region_test_db SURVIVE REGION FAILURE + +statement ok +USE multi_region_test_db + +# Statements which previously succeeded should now fail under REGION survivability. +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM parent p, child c WHERE c_id = 10 AND p_id = c_p_id + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM child WHERE NOT EXISTS (SELECT * FROM parent WHERE p_id = c_p_id) AND c_id = 10 + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM parent LIMIT 1 + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM messages_global@messages_global_pkey + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT message from messages_rbt@messages_rbt_pkey + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT message from messages_rbr@msg_idx WHERE crdb_region = 'ap-southeast-2' + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +EXECUTE s('ap-southeast-2') + +statement ok +RESET enforce_home_region diff --git a/pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs/generated_test.go b/pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs/generated_test.go index 7972fa8b48bb..098843c5eb1d 100644 --- a/pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs/generated_test.go +++ b/pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs/generated_test.go @@ -136,6 +136,13 @@ func TestCCLLogic_multi_region_query_behavior( runCCLLogicTest(t, "multi_region_query_behavior") } +func TestCCLLogic_multi_region_remote_access_error( + t *testing.T, +) { + defer leaktest.AfterTest(t)() + runCCLLogicTest(t, "multi_region_remote_access_error") +} + func TestCCLLogic_multi_region_show( t *testing.T, ) { diff --git a/pkg/sql/exec_util.go b/pkg/sql/exec_util.go index 9b36733e5925..5ab534dd6207 100644 --- a/pkg/sql/exec_util.go +++ b/pkg/sql/exec_util.go @@ -3281,6 +3281,10 @@ func (m *sessionDataMutator) SetTroubleshootingModeEnabled(val bool) { m.data.TroubleshootingMode = val } +func (m *sessionDataMutator) SetEnforceHomeRegion(val bool) { + m.data.EnforceHomeRegion = val +} + // Utility functions related to scrubbing sensitive information on SQL Stats. // quantizeCounts ensures that the Count field in the diff --git a/pkg/sql/faketreeeval/BUILD.bazel b/pkg/sql/faketreeeval/BUILD.bazel index f4217bd519bc..77df33d5096d 100644 --- a/pkg/sql/faketreeeval/BUILD.bazel +++ b/pkg/sql/faketreeeval/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//pkg/clusterversion", "//pkg/security/username", "//pkg/sql/catalog/catpb", + "//pkg/sql/catalog/descpb", "//pkg/sql/parser", "//pkg/sql/pgwire/pgcode", "//pkg/sql/pgwire/pgerror", diff --git a/pkg/sql/faketreeeval/evalctx.go b/pkg/sql/faketreeeval/evalctx.go index 3909724e8ad6..9d93deb4e88e 100644 --- a/pkg/sql/faketreeeval/evalctx.go +++ b/pkg/sql/faketreeeval/evalctx.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/security/username" "github.com/cockroachdb/cockroach/pkg/sql/catalog/catpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/parser" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" @@ -124,21 +125,21 @@ var _ eval.RegionOperator = &DummyRegionOperator{} var errRegionOperator = unimplemented.NewWithIssue(42508, "cannot evaluate scalar expressions containing region operations in this context") -// CurrentDatabaseRegionConfig is part of the eval.DatabaseCatalog interface. +// CurrentDatabaseRegionConfig is part of the eval.RegionOperator interface. func (so *DummyRegionOperator) CurrentDatabaseRegionConfig( _ context.Context, ) (eval.DatabaseRegionConfig, error) { return nil, errors.WithStack(errRegionOperator) } -// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.DatabaseCatalog interface. +// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.RegionOperator interface. func (so *DummyRegionOperator) ValidateAllMultiRegionZoneConfigsInCurrentDatabase( _ context.Context, ) error { return errors.WithStack(errRegionOperator) } -// ResetMultiRegionZoneConfigsForTable is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForTable is part of the eval.RegionOperator // interface. func (so *DummyRegionOperator) ResetMultiRegionZoneConfigsForTable( _ context.Context, id int64, @@ -146,7 +147,7 @@ func (so *DummyRegionOperator) ResetMultiRegionZoneConfigsForTable( return errors.WithStack(errRegionOperator) } -// ResetMultiRegionZoneConfigsForDatabase is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForDatabase is part of the eval.RegionOperator // interface. func (so *DummyRegionOperator) ResetMultiRegionZoneConfigsForDatabase( _ context.Context, id int64, @@ -310,20 +311,20 @@ var _ eval.Planner = &DummyEvalPlanner{} var errEvalPlanner = pgerror.New(pgcode.ScalarOperationCannotRunWithoutFullSessionContext, "cannot evaluate scalar expressions using table lookups in this context") -// CurrentDatabaseRegionConfig is part of the eval.DatabaseCatalog interface. +// CurrentDatabaseRegionConfig is part of the eval.RegionOperator interface. func (ep *DummyEvalPlanner) CurrentDatabaseRegionConfig( _ context.Context, ) (eval.DatabaseRegionConfig, error) { return nil, errors.WithStack(errEvalPlanner) } -// ResetMultiRegionZoneConfigsForTable is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForTable is part of the eval.RegionOperator // interface. func (ep *DummyEvalPlanner) ResetMultiRegionZoneConfigsForTable(_ context.Context, _ int64) error { return errors.WithStack(errEvalPlanner) } -// ResetMultiRegionZoneConfigsForDatabase is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForDatabase is part of the eval.RegionOperator // interface. func (ep *DummyEvalPlanner) ResetMultiRegionZoneConfigsForDatabase( _ context.Context, _ int64, @@ -331,7 +332,7 @@ func (ep *DummyEvalPlanner) ResetMultiRegionZoneConfigsForDatabase( return errors.WithStack(errEvalPlanner) } -// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.DatabaseCatalog interface. +// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.RegionOperator interface. func (ep *DummyEvalPlanner) ValidateAllMultiRegionZoneConfigsInCurrentDatabase( _ context.Context, ) error { @@ -428,6 +429,11 @@ func (ep *DummyEvalPlanner) IsActive(_ context.Context, _ clusterversion.Key) bo return true } +// GetMultiregionConfig is part of the eval.Planner interface. +func (ep *DummyEvalPlanner) GetMultiregionConfig(databaseID descpb.ID) (interface{}, bool) { + return nil /* regionConfig */, false +} + // DummyPrivilegedAccessor implements the tree.PrivilegedAccessor interface by returning errors. type DummyPrivilegedAccessor struct{} diff --git a/pkg/sql/importer/import_table_creation.go b/pkg/sql/importer/import_table_creation.go index 1d9b7c1141f7..73e21a22e4f5 100644 --- a/pkg/sql/importer/import_table_creation.go +++ b/pkg/sql/importer/import_table_creation.go @@ -256,21 +256,21 @@ func (i importDatabaseRegionConfig) PrimaryRegionString() string { var _ eval.DatabaseRegionConfig = &importDatabaseRegionConfig{} -// CurrentDatabaseRegionConfig is part of the eval.DatabaseCatalog interface. +// CurrentDatabaseRegionConfig is part of the eval.RegionOperator interface. func (so *importRegionOperator) CurrentDatabaseRegionConfig( _ context.Context, ) (eval.DatabaseRegionConfig, error) { return importDatabaseRegionConfig{primaryRegion: so.primaryRegion}, nil } -// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.DatabaseCatalog interface. +// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.RegionOperator interface. func (so *importRegionOperator) ValidateAllMultiRegionZoneConfigsInCurrentDatabase( _ context.Context, ) error { return errors.WithStack(errRegionOperator) } -// ResetMultiRegionZoneConfigsForTable is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForTable is part of the eval.RegionOperator // interface. func (so *importRegionOperator) ResetMultiRegionZoneConfigsForTable( _ context.Context, _ int64, @@ -278,7 +278,7 @@ func (so *importRegionOperator) ResetMultiRegionZoneConfigsForTable( return errors.WithStack(errRegionOperator) } -// ResetMultiRegionZoneConfigsForDatabase is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForDatabase is part of the eval.RegionOperator // interface. func (so *importRegionOperator) ResetMultiRegionZoneConfigsForDatabase( _ context.Context, _ int64, diff --git a/pkg/sql/internal.go b/pkg/sql/internal.go index 520cec4ea509..55428782d32e 100644 --- a/pkg/sql/internal.go +++ b/pkg/sql/internal.go @@ -649,6 +649,9 @@ func (ie *InternalExecutor) execInternal( if ie.sessionDataStack != nil { // TODO(andrei): Properly clone (deep copy) ie.sessionData. sd = ie.sessionDataStack.Top().Clone() + // Even if session queries are told to error on non-home region accesses, + // internal queries spawned from the same context should never do so. + sd.LocalOnlySessionData.EnforceHomeRegion = false } else { sd = ie.s.newSessionData(SessionArgs{}) } diff --git a/pkg/sql/logictest/testdata/logic_test/information_schema b/pkg/sql/logictest/testdata/logic_test/information_schema index 4688f856689a..ae1c4fc3ad5c 100644 --- a/pkg/sql/logictest/testdata/logic_test/information_schema +++ b/pkg/sql/logictest/testdata/logic_test/information_schema @@ -4690,6 +4690,7 @@ enable_multiregion_placement_policy off enable_seqscan on enable_super_regions off enable_zigzag_join on +enforce_home_region off escape_string_warning on expect_and_ignore_not_visible_columns_in_copy off experimental_computed_column_rewrites · diff --git a/pkg/sql/logictest/testdata/logic_test/pg_catalog b/pkg/sql/logictest/testdata/logic_test/pg_catalog index 2007bfaa0c91..c3a3bf9d5470 100644 --- a/pkg/sql/logictest/testdata/logic_test/pg_catalog +++ b/pkg/sql/logictest/testdata/logic_test/pg_catalog @@ -4174,6 +4174,7 @@ enable_multiregion_placement_policy off NULL enable_seqscan on NULL NULL NULL string enable_super_regions off NULL NULL NULL string enable_zigzag_join on NULL NULL NULL string +enforce_home_region off NULL NULL NULL string escape_string_warning on NULL NULL NULL string expect_and_ignore_not_visible_columns_in_copy off NULL NULL NULL string experimental_distsql_planning off NULL NULL NULL string @@ -4299,6 +4300,7 @@ enable_multiregion_placement_policy off NULL enable_seqscan on NULL user NULL on on enable_super_regions off NULL user NULL off off enable_zigzag_join on NULL user NULL on on +enforce_home_region off NULL user NULL off off escape_string_warning on NULL user NULL on on expect_and_ignore_not_visible_columns_in_copy off NULL user NULL off off experimental_distsql_planning off NULL user NULL off off @@ -4420,6 +4422,7 @@ enable_multiregion_placement_policy NULL NULL NULL enable_seqscan NULL NULL NULL NULL NULL enable_super_regions NULL NULL NULL NULL NULL enable_zigzag_join NULL NULL NULL NULL NULL +enforce_home_region NULL NULL NULL NULL NULL escape_string_warning NULL NULL NULL NULL NULL expect_and_ignore_not_visible_columns_in_copy NULL NULL NULL NULL NULL experimental_distsql_planning NULL NULL NULL NULL NULL diff --git a/pkg/sql/logictest/testdata/logic_test/show_source b/pkg/sql/logictest/testdata/logic_test/show_source index 0fdb1eb5be67..81ff65ee3a6b 100644 --- a/pkg/sql/logictest/testdata/logic_test/show_source +++ b/pkg/sql/logictest/testdata/logic_test/show_source @@ -60,6 +60,7 @@ enable_multiregion_placement_policy off enable_seqscan on enable_super_regions off enable_zigzag_join on +enforce_home_region off escape_string_warning on expect_and_ignore_not_visible_columns_in_copy off experimental_distsql_planning off diff --git a/pkg/sql/opt/BUILD.bazel b/pkg/sql/opt/BUILD.bazel index 0fe0a7a77b99..ce61ed50b4e7 100644 --- a/pkg/sql/opt/BUILD.bazel +++ b/pkg/sql/opt/BUILD.bazel @@ -24,7 +24,10 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/server/telemetry", + "//pkg/sql/catalog/catpb", "//pkg/sql/catalog/colinfo", + "//pkg/sql/catalog/descpb", + "//pkg/sql/catalog/multiregion", "//pkg/sql/opt/cat", "//pkg/sql/opt/partition", "//pkg/sql/pgwire/pgcode", diff --git a/pkg/sql/opt/cat/table.go b/pkg/sql/opt/cat/table.go index 2f7e0b674290..49e0b71dbc84 100644 --- a/pkg/sql/opt/cat/table.go +++ b/pkg/sql/opt/cat/table.go @@ -13,6 +13,7 @@ package cat import ( "time" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/types" ) @@ -149,6 +150,21 @@ type Table interface { // created with the NO DATA option and is yet to be refreshed. Accessing // such a view prior to running refresh returns an error. IsRefreshViewRequired() bool + + // HomeRegion returns the home region of the table, if any, for example if + // a table is defined with LOCALITY REGIONAL BY TABLE. + HomeRegion() (region string, ok bool) + + // IsGlobalTable returns true if the table is defined with LOCALITY GLOBAL. + IsGlobalTable() bool + + // IsRegionalByRow returns true if the table is defined with LOCALITY REGIONAL + // BY ROW. + IsRegionalByRow() bool + + // GetDatabaseID returns the owning database id of the table, or zero, if the + // owning database could not be determined. + GetDatabaseID() descpb.ID } // CheckConstraint contains the SQL text and the validity status for a check diff --git a/pkg/sql/opt/constraint/constraint.go b/pkg/sql/opt/constraint/constraint.go index e52b42990cce..ad743298df7c 100644 --- a/pkg/sql/opt/constraint/constraint.go +++ b/pkg/sql/opt/constraint/constraint.go @@ -830,3 +830,27 @@ func (c *Constraint) CollectFirstColumnValues( } return values, hasNullValue, true } + +// HasRemoteSpans returns true if any of the constraint spans in `c` belong to +// partitions remote to the gateway region. +func (c *Constraint) HasRemoteSpans(ps *partition.PrefixSorter) bool { + if c.IsUnconstrained() { + return true + } + if c.IsContradiction() { + return false + } + // Iterate through the spans and determine whether each one matches + // with a prefix from a remote partition. + for i, n := 0, c.Spans.Count(); i < n; i++ { + span := c.Spans.Get(i) + if match, ok := FindMatch(span, ps); ok { + if !match.IsLocal { + return true + } + } else { + return true + } + } + return false +} diff --git a/pkg/sql/opt/distribution/BUILD.bazel b/pkg/sql/opt/distribution/BUILD.bazel index 627d56b959d5..4de5775e383e 100644 --- a/pkg/sql/opt/distribution/BUILD.bazel +++ b/pkg/sql/opt/distribution/BUILD.bazel @@ -7,9 +7,14 @@ go_library( importpath = "github.com/cockroachdb/cockroach/pkg/sql/opt/distribution", visibility = ["//visibility:public"], deps = [ + "//pkg/sql/catalog/descpb", + "//pkg/sql/opt", "//pkg/sql/opt/memo", "//pkg/sql/opt/props/physical", + "//pkg/sql/pgwire/pgcode", + "//pkg/sql/pgwire/pgerror", "//pkg/sql/sem/eval", + "//pkg/sql/sem/tree", "//pkg/util/buildutil", "@com_github_cockroachdb_errors//:errors", ], diff --git a/pkg/sql/opt/distribution/distribution.go b/pkg/sql/opt/distribution/distribution.go index 4edb00e308b1..73207a236bc5 100644 --- a/pkg/sql/opt/distribution/distribution.go +++ b/pkg/sql/opt/distribution/distribution.go @@ -11,9 +11,14 @@ package distribution import ( + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/opt/props/physical" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" + "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/util/buildutil" "github.com/cockroachdb/errors" ) @@ -34,20 +39,47 @@ func CanProvide(evalCtx *eval.Context, expr memo.RelExpr, required *physical.Dis return true case *memo.LocalityOptimizedSearchExpr: + // Locality optimized search is legal with the EnforceHomeRegion flag, + // but only with the SURVIVE ZONE FAILURE option. + if evalCtx.SessionData().EnforceHomeRegion && t.Local.Op() == opt.ScanOp { + scanExpr := t.Local.(*memo.ScanExpr) + tabMeta := t.Memo().Metadata().TableMeta(scanExpr.Table) + errorOnInvalidMultiregionDB(evalCtx, tabMeta) + } provided.FromLocality(evalCtx.Locality) case *memo.ScanExpr: - md := expr.Memo().Metadata() - index := md.Table(t.Table).Index(t.Index) - provided.FromIndexScan(evalCtx, index, t.Constraint) + tabMeta := t.Memo().Metadata().TableMeta(t.Table) + // Tables in database that don't use the SURVIVE ZONE FAILURE option are + // disallowed when EnforceHomeRegion is true. + if evalCtx.SessionData().EnforceHomeRegion { + errorOnInvalidMultiregionDB(evalCtx, tabMeta) + } + provided.FromIndexScan(evalCtx, tabMeta, t.Index, t.Constraint) default: // Other operators can pass through the distribution to their children. } - return provided.Any() || provided.Equals(*required) } +// errorOnInvalidMultiregionDB panics if the table described by tabMeta is owned +// by a non-multiregion database or a multiregion database with SURVIVE REGION +// FAILURE goal. +func errorOnInvalidMultiregionDB(evalCtx *eval.Context, tabMeta *opt.TableMeta) { + survivalGoal, ok := tabMeta.GetDatabaseSurvivalGoal(evalCtx.Planner) + // non-multiregional database or SURVIVE REGION FAILURE option + if !ok { + err := pgerror.New(pgcode.QueryHasNoHomeRegion, + "Query has no home region. Try accessing only tables in multi-region databases with ZONE survivability.") + panic(err) + } else if survivalGoal == descpb.SurvivalGoal_REGION_FAILURE { + err := pgerror.New(pgcode.QueryHasNoHomeRegion, + "The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability.") + panic(err) + } +} + // BuildChildRequired returns the distribution that must be required of its // given child in order to satisfy a required distribution. Can only be called if // CanProvide is true for the required distribution. @@ -89,17 +121,29 @@ func BuildProvided( return *required case *memo.LocalityOptimizedSearchExpr: + // Locality optimized search is legal with the EnforceHomeRegion flag, + // but only with the SURVIVE ZONE FAILURE option. + if evalCtx.SessionData().EnforceHomeRegion && t.Local.Op() == opt.ScanOp { + scanExpr := t.Local.(*memo.ScanExpr) + tabMeta := t.Memo().Metadata().TableMeta(scanExpr.Table) + errorOnInvalidMultiregionDB(evalCtx, tabMeta) + } provided.FromLocality(evalCtx.Locality) case *memo.ScanExpr: - md := expr.Memo().Metadata() - index := md.Table(t.Table).Index(t.Index) - provided.FromIndexScan(evalCtx, index, t.Constraint) + // Tables in database that don't use the SURVIVE ZONE FAILURE option are + // disallowed when EnforceHomeRegion is true. + tabMeta := t.Memo().Metadata().TableMeta(t.Table) + if evalCtx.SessionData().EnforceHomeRegion { + errorOnInvalidMultiregionDB(evalCtx, tabMeta) + } + provided.FromIndexScan(evalCtx, tabMeta, t.Index, t.Constraint) default: for i, n := 0, expr.ChildCount(); i < n; i++ { if relExpr, ok := expr.Child(i).(memo.RelExpr); ok { - provided = provided.Union(relExpr.ProvidedPhysical().Distribution) + childDistribution := relExpr.ProvidedPhysical().Distribution + provided = provided.Union(childDistribution) } } } @@ -111,6 +155,25 @@ func BuildProvided( return provided } +// GetProjectedEnumConstant looks for the projection with target colID in +// projectExpr, and if it contains a constant enum, returns its string +// representation, or the empty string if not found. +func GetProjectedEnumConstant(projectExpr *memo.ProjectExpr, colID opt.ColumnID) string { + for _, projection := range projectExpr.Projections { + if projection.Col == colID { + if projection.Element.Op() == opt.ConstOp { + constExpr := projection.Element.(*memo.ConstExpr) + if regionName, ok := constExpr.Value.(*tree.DEnum); ok { + return regionName.LogicalRep + } + } else { + return "" + } + } + } + return "" +} + func checkRequired(required *physical.Distribution) { // There should be exactly one region in the required distribution (for now, // assuming this is coming from the gateway). diff --git a/pkg/sql/opt/exec/execbuilder/BUILD.bazel b/pkg/sql/opt/exec/execbuilder/BUILD.bazel index b524d65e36a7..68acb72b2610 100644 --- a/pkg/sql/opt/exec/execbuilder/BUILD.bazel +++ b/pkg/sql/opt/exec/execbuilder/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "//pkg/sql/opt", "//pkg/sql/opt/cat", "//pkg/sql/opt/constraint", + "//pkg/sql/opt/distribution", "//pkg/sql/opt/exec", "//pkg/sql/opt/exec/explain", "//pkg/sql/opt/memo", diff --git a/pkg/sql/opt/exec/execbuilder/builder.go b/pkg/sql/opt/exec/execbuilder/builder.go index c76e4a360652..fce9be490da1 100644 --- a/pkg/sql/opt/exec/execbuilder/builder.go +++ b/pkg/sql/opt/exec/execbuilder/builder.go @@ -320,8 +320,7 @@ func (b *Builder) findBuiltWithExpr(id opt.WithID) *builtWithExpr { // boundedStaleness returns true if this query uses bounded staleness. func (b *Builder) boundedStaleness() bool { - return b.evalCtx != nil && b.evalCtx.AsOfSystemTime != nil && - b.evalCtx.AsOfSystemTime.BoundedStaleness + return b.evalCtx != nil && b.evalCtx.BoundedStaleness() } // mdVarContainer is an IndexedVarContainer implementation used by BuildScalar - diff --git a/pkg/sql/opt/exec/execbuilder/relational.go b/pkg/sql/opt/exec/execbuilder/relational.go index 203af8dff338..5b0ab19c200e 100644 --- a/pkg/sql/opt/exec/execbuilder/relational.go +++ b/pkg/sql/opt/exec/execbuilder/relational.go @@ -14,6 +14,7 @@ import ( "bytes" "context" "fmt" + "strings" "github.com/cockroachdb/cockroach/pkg/server/telemetry" "github.com/cockroachdb/cockroach/pkg/sql/catalog/colinfo" @@ -21,6 +22,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/opt/constraint" + "github.com/cockroachdb/cockroach/pkg/sql/opt/distribution" "github.com/cockroachdb/cockroach/pkg/sql/opt/exec" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/opt/norm" @@ -1666,21 +1668,39 @@ func (b *Builder) buildSort(sort *memo.SortExpr) (execPlan, error) { return execPlan{root: node, outputCols: input.outputCols}, nil } -func (b *Builder) buildDistribute(distribute *memo.DistributeExpr) (execPlan, error) { - input, err := b.buildRelational(distribute.Input) +func (b *Builder) buildDistribute(distribute *memo.DistributeExpr) (input execPlan, err error) { + input, err = b.buildRelational(distribute.Input) if err != nil { return execPlan{}, err } - distribution := distribute.ProvidedPhysical().Distribution - inputDistribution := distribute.Input.ProvidedPhysical().Distribution - if distribution.Equals(inputDistribution) { + if distribute.NoOpDistribution() { // Don't bother creating a no-op distribution. This likely exists because // the input is a Sort expression, and this is an artifact of how physical // properties are enforced. return input, err } + if b.evalCtx.SessionData().EnforceHomeRegion { + homeRegion, ok := distribute.GetInputHomeRegion() + var errorStringBuilder strings.Builder + var errCode pgcode.Code + if ok { + errCode = pgcode.QueryNotRunningInHomeRegion + errorStringBuilder.WriteString("Query is not running in its home region.") + errorStringBuilder.WriteString(fmt.Sprintf(` Try running the query from region '%s'.`, homeRegion)) + } else if distribute.Input.Op() != opt.LookupJoinOp { + // More detailed error message handling for lookup join occurs in the + // execbuilder. + errCode = pgcode.QueryHasNoHomeRegion + errorStringBuilder.WriteString("Query has no home region.") + errorStringBuilder.WriteString(` Try adding a LIMIT clause.`) + } + msgString := errorStringBuilder.String() + err = pgerror.Newf(errCode, "%s", msgString) + return execPlan{}, err + } + // TODO(rytaft): This is currently a no-op. We should pass this distribution // info to the DistSQL planner. return input, err @@ -1743,9 +1763,137 @@ func (b *Builder) buildIndexJoin(join *memo.IndexJoinExpr) (execPlan, error) { return res, nil } +func indexColumnNames(tableName string, index cat.Index, startColumn int) string { + if startColumn < 0 { + return "" + } + sb := strings.Builder{} + for i, n := startColumn, index.KeyColumnCount(); i < n; i++ { + if i > startColumn { + sb.WriteString(", ") + } + col := index.Column(i) + sb.WriteString(fmt.Sprintf("%s.%s", tableName, string(col.ColName()))) + } + return sb.String() +} + +func filterSuggestionError( + table cat.Table, indexOrdinal cat.IndexOrdinal, table2 cat.Table, indexOrdinal2 cat.IndexOrdinal, +) (err error) { + var index cat.Index + if table != nil { + index = table.Index(indexOrdinal) + + if table.IsRegionalByRow() { + plural := "" + if index.KeyColumnCount() > 2 { + plural = "s" + } + args := make([]interface{}, 0, 6) + args = append(args, string(table.Name())) + args = append(args, plural) + args = append(args, indexColumnNames(string(table.Name()), index, 1)) + if table2 != nil && table2.IsRegionalByRow() { + index = table2.Index(indexOrdinal2) + plural = "" + if index.KeyColumnCount() > 2 { + plural = "s" + } + args = append(args, string(table2.Name())) + args = append(args, plural) + args = append(args, indexColumnNames(string(table2.Name()), index, 1)) + err = pgerror.Newf(pgcode.QueryHasNoHomeRegion, + "Query has no home region. Try adding a filter on %s.crdb_region and/or on key column%s (%s). Try adding a filter on %s.crdb_region and/or on key column%s (%s).", args...) + } else { + err = pgerror.Newf(pgcode.QueryHasNoHomeRegion, + "Query has no home region. Try adding a filter on %s.crdb_region and/or on key column%s (%s).", args...) + } + } + } + return err +} + +func (b *Builder) handleRemoteLookupJoinError(join *memo.LookupJoinExpr) (err error) { + lookupTable := join.Memo().Metadata().Table(join.Table) + var input opt.Expr + input = join.Input + for input.ChildCount() == 1 || input.Op() == opt.ProjectOp { + input = input.Child(0) + } + gatewayRegion, foundLocalRegion := b.evalCtx.Locality.Find("region") + inputTableName := "" + // ScanExprs from global tables will have filled in a provided distribution + // of the gateway region by now. + queryHomeRegion := input.(memo.RelExpr).ProvidedPhysical().Distribution.GetRegionOfDistribution() + var inputTable cat.Table + var inputIndexOrdinal cat.IndexOrdinal + switch t := input.(type) { + case *memo.ScanExpr: + inputTable = join.Memo().Metadata().Table(t.Table) + inputTableName = string(inputTable.Name()) + inputIndexOrdinal = t.Index + } + + homeRegion := "" + if lookupTable.IsGlobalTable() { + // HomeRegion() does not automatically fill in the home region of a global + // table as the gateway region, so let's manually set it here. + homeRegion = gatewayRegion + } else { + homeRegion, _ = lookupTable.HomeRegion() + } + if homeRegion == "" { + if lookupTable.IsRegionalByRow() { + if len(join.LookupJoinPrivate.KeyCols) > 0 { + if projectExpr, ok := join.Input.(*memo.ProjectExpr); ok { + colID := join.LookupJoinPrivate.KeyCols[0] + homeRegion = distribution.GetProjectedEnumConstant(projectExpr, colID) + } + } + } + } + + if homeRegion != "" { + if foundLocalRegion { + if queryHomeRegion != "" { + if homeRegion != queryHomeRegion { + if inputTableName == "" { + return pgerror.Newf(pgcode.QueryHasNoHomeRegion, + `Query has no home region. The home region ('%s') of lookup table '%s' does not match the home region ('%s') of the other relation in the join.'`, + homeRegion, + lookupTable.Name(), + queryHomeRegion, + ) + } + return pgerror.Newf(pgcode.QueryHasNoHomeRegion, + `Query has no home region. The home region ('%s') of table '%s' does not match the home region ('%s') of lookup table '%s'.`, + queryHomeRegion, + inputTableName, + homeRegion, + string(lookupTable.Name()), + ) + } else if gatewayRegion != homeRegion { + return pgerror.Newf(pgcode.QueryNotRunningInHomeRegion, + `Query is not running in its home region. Try running the query from region '%s'.`, + homeRegion, + ) + } + } else { + return filterSuggestionError(inputTable, inputIndexOrdinal, nil, 0) + } + } + } else { + if queryHomeRegion == "" { + return filterSuggestionError(lookupTable, join.Index, inputTable, inputIndexOrdinal) + } + return filterSuggestionError(lookupTable, join.Index, nil, 0) + } + return err +} + func (b *Builder) buildLookupJoin(join *memo.LookupJoinExpr) (execPlan, error) { md := b.mem.Metadata() - if !b.disableTelemetry { telemetry.Inc(sqltelemetry.JoinAlgoLookupUseCounter) telemetry.Inc(opt.JoinTypeToUseCounter(join.JoinType)) @@ -1754,12 +1902,20 @@ func (b *Builder) buildLookupJoin(join *memo.LookupJoinExpr) (execPlan, error) { telemetry.Inc(sqltelemetry.PartialIndexLookupJoinUseCounter) } } - input, err := b.buildRelational(join.Input) if err != nil { return execPlan{}, err } - + if b.evalCtx.SessionData().EnforceHomeRegion { + // TODO(msirek): Remove this in phase 2 or 3 when we can dynamically + // determine if the lookup will be local. + if !join.LocalityOptimized { + err = b.handleRemoteLookupJoinError(join) + if err != nil { + return execPlan{}, err + } + } + } keyCols := make([]exec.NodeColumnOrdinal, len(join.KeyCols)) for i, c := range join.KeyCols { keyCols[i] = input.getNodeColumnOrdinal(c) diff --git a/pkg/sql/opt/exec/explain/plan_gist_factory.go b/pkg/sql/opt/exec/explain/plan_gist_factory.go index 19807090fcf0..2646b45294fa 100644 --- a/pkg/sql/opt/exec/explain/plan_gist_factory.go +++ b/pkg/sql/opt/exec/explain/plan_gist_factory.go @@ -562,6 +562,26 @@ func (u *unknownTable) IsRefreshViewRequired() bool { return false } +// HomeRegion is part of the cat.Table interface. +func (u *unknownTable) HomeRegion() (region string, ok bool) { + return "", false +} + +// IsGlobalTable is part of the cat.Table interface. +func (u *unknownTable) IsGlobalTable() bool { + return false +} + +// IsRegionalByRow is part of the cat.Table interface. +func (u *unknownTable) IsRegionalByRow() bool { + return false +} + +// GetDatabaseID is part of the cat.Table interface. +func (u *unknownTable) GetDatabaseID() descpb.ID { + return 0 +} + var _ cat.Table = &unknownTable{} // unknownTable implements the cat.Index interface and is used to represent diff --git a/pkg/sql/opt/memo/expr.go b/pkg/sql/opt/memo/expr.go index d5919bddba8f..eb087e8fa449 100644 --- a/pkg/sql/opt/memo/expr.go +++ b/pkg/sql/opt/memo/expr.go @@ -274,6 +274,48 @@ func (n FiltersExpr) Difference(other FiltersExpr) FiltersExpr { return newFilters } +// NoOpDistribution returns true if a DistributeExpr has the same distribution +// as its input. +func (e *DistributeExpr) NoOpDistribution() bool { + distributionProvidedPhysical := e.ProvidedPhysical() + inputDistributionProvidedPhysical := e.Input.ProvidedPhysical() + + if distributionProvidedPhysical != nil && inputDistributionProvidedPhysical != nil { + distribution := distributionProvidedPhysical.Distribution + inputDistribution := inputDistributionProvidedPhysical.Distribution + return distribution.Equals(inputDistribution) + } + return false +} + +// GetRegionOfDistribution returns the single region name of the provided +// distribution, if there is exactly one. +func (e *DistributeExpr) GetRegionOfDistribution() (region string, ok bool) { + distributionProvidedPhysical := e.ProvidedPhysical() + + if distributionProvidedPhysical != nil { + distribution := distributionProvidedPhysical.Distribution + if len(distribution.Regions) == 1 { + return distribution.Regions[0], true + } + } + return "", false +} + +// GetInputHomeRegion returns the single region name of the home region of the +// input expression tree to the Distribute operation, if there is exactly one. +func (e *DistributeExpr) GetInputHomeRegion() (inputHomeRegion string, ok bool) { + inputDistributionProvidedPhysical := e.Input.ProvidedPhysical() + + if inputDistributionProvidedPhysical != nil { + distribution := inputDistributionProvidedPhysical.Distribution + if len(distribution.Regions) == 1 { + return distribution.Regions[0], true + } + } + return "", false +} + // OutputCols returns the set of columns constructed by the Aggregations // expression. func (n AggregationsExpr) OutputCols() opt.ColSet { diff --git a/pkg/sql/opt/memo/memo.go b/pkg/sql/opt/memo/memo.go index 965229328296..0215b59a6a89 100644 --- a/pkg/sql/opt/memo/memo.go +++ b/pkg/sql/opt/memo/memo.go @@ -154,6 +154,7 @@ type Memo struct { testingOptimizerRandomSeed int64 testingOptimizerCostPerturbation float64 testingOptimizerDisableRuleProbability float64 + enforceHomeRegion bool // curRank is the highest currently in-use scalar expression rank. curRank opt.ScalarRank @@ -203,6 +204,7 @@ func (m *Memo) Init(evalCtx *eval.Context) { testingOptimizerRandomSeed: evalCtx.SessionData().TestingOptimizerRandomSeed, testingOptimizerCostPerturbation: evalCtx.SessionData().TestingOptimizerCostPerturbation, testingOptimizerDisableRuleProbability: evalCtx.SessionData().TestingOptimizerDisableRuleProbability, + enforceHomeRegion: evalCtx.SessionData().EnforceHomeRegion, } m.metadata.Init() m.logPropsBuilder.init(evalCtx, m) @@ -335,7 +337,8 @@ func (m *Memo) IsStale( m.allowUnconstrainedNonCoveringIndexScan != evalCtx.SessionData().UnconstrainedNonCoveringIndexScanEnabled || m.testingOptimizerRandomSeed != evalCtx.SessionData().TestingOptimizerRandomSeed || m.testingOptimizerCostPerturbation != evalCtx.SessionData().TestingOptimizerCostPerturbation || - m.testingOptimizerDisableRuleProbability != evalCtx.SessionData().TestingOptimizerDisableRuleProbability { + m.testingOptimizerDisableRuleProbability != evalCtx.SessionData().TestingOptimizerDisableRuleProbability || + m.enforceHomeRegion != evalCtx.SessionData().EnforceHomeRegion { return true, nil } diff --git a/pkg/sql/opt/memo/memo_test.go b/pkg/sql/opt/memo/memo_test.go index 276c0596cd9c..29cd538639dd 100644 --- a/pkg/sql/opt/memo/memo_test.go +++ b/pkg/sql/opt/memo/memo_test.go @@ -274,6 +274,12 @@ func TestMemoIsStale(t *testing.T) { evalCtx.SessionData().UnconstrainedNonCoveringIndexScanEnabled = false notStale() + // Stale enforce home region. + evalCtx.SessionData().EnforceHomeRegion = true + stale() + evalCtx.SessionData().EnforceHomeRegion = false + notStale() + // Stale testing_optimizer_random_seed. evalCtx.SessionData().TestingOptimizerRandomSeed = 100 stale() diff --git a/pkg/sql/opt/metadata.go b/pkg/sql/opt/metadata.go index 060aa495d9e8..057dab43ac52 100644 --- a/pkg/sql/opt/metadata.go +++ b/pkg/sql/opt/metadata.go @@ -16,6 +16,7 @@ import ( "math/bits" "strings" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/multiregion" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" @@ -196,7 +197,7 @@ func (md *Metadata) Init() { // This metadata can then be modified independent of the copied metadata. // // Table annotations are not transferred over; all annotations are unset on -// the copy. +// the copy, except for regionConfig, which is read-only, and can be shared. // // copyScalarFn must be a function that returns a copy of the given scalar // expression. @@ -226,8 +227,17 @@ func (md *Metadata) CopyFrom(from *Metadata, copyScalarFn func(Expr) Expr) { md.tables = make([]TableMeta, len(from.tables)) } for i := range from.tables { - // Note: annotations inside TableMeta are not retained. + // Note: annotations inside TableMeta are not retained... md.tables[i].copyFrom(&from.tables[i], copyScalarFn) + + // ...except for the regionConfig annotation. + tabID := from.tables[i].MetaID + regionConfig, ok := md.TableAnnotation(tabID, regionConfigAnnID).(*multiregion.RegionConfig) + if ok { + // Don't waste time looking up a database descriptor and constructing a + // RegionConfig more than once for a given table. + md.SetTableAnnotation(tabID, regionConfigAnnID, regionConfig) + } } md.sequences = append(md.sequences, from.sequences...) @@ -485,7 +495,7 @@ func (md *Metadata) DuplicateTable( } } - md.tables = append(md.tables, TableMeta{ + newTabMeta := TableMeta{ MetaID: newTabID, Table: tabMeta.Table, Alias: tabMeta.Alias, @@ -495,7 +505,14 @@ func (md *Metadata) DuplicateTable( partialIndexPredicates: partialIndexPredicates, indexPartitionLocalities: tabMeta.indexPartitionLocalities, checkConstraintsStats: checkConstraintsStats, - }) + } + md.tables = append(md.tables, newTabMeta) + regionConfig, ok := md.TableAnnotation(tabID, regionConfigAnnID).(*multiregion.RegionConfig) + if ok { + // Don't waste time looking up a database descriptor and constructing a + // RegionConfig more than once for a given table. + md.SetTableAnnotation(newTabID, regionConfigAnnID, regionConfig) + } return newTabID } diff --git a/pkg/sql/opt/optgen/exprgen/BUILD.bazel b/pkg/sql/opt/optgen/exprgen/BUILD.bazel index 896e2d061e8c..85381c861017 100644 --- a/pkg/sql/opt/optgen/exprgen/BUILD.bazel +++ b/pkg/sql/opt/optgen/exprgen/BUILD.bazel @@ -14,6 +14,7 @@ go_library( deps = [ "//pkg/sql/opt", "//pkg/sql/opt/cat", + "//pkg/sql/opt/distribution", "//pkg/sql/opt/memo", "//pkg/sql/opt/norm", "//pkg/sql/opt/optgen/lang", diff --git a/pkg/sql/opt/optgen/exprgen/expr_gen.go b/pkg/sql/opt/optgen/exprgen/expr_gen.go index 28de4cf92304..b84d47bfbbb7 100644 --- a/pkg/sql/opt/optgen/exprgen/expr_gen.go +++ b/pkg/sql/opt/optgen/exprgen/expr_gen.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" + "github.com/cockroachdb/cockroach/pkg/sql/opt/distribution" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/opt/norm" "github.com/cockroachdb/cockroach/pkg/sql/opt/optgen/lang" @@ -399,6 +400,7 @@ func (eg *exprGen) populateBestProps(expr opt.Expr, required *physical.Required) // BuildProvided relies on ProvidedPhysical() being set in the children, so // it must run after the recursive calls on the children. provided.Ordering = ordering.BuildProvided(rel, &required.Ordering) + provided.Distribution = distribution.BuildProvided(eg.f.EvalContext(), rel, &required.Distribution) cost += eg.coster.ComputeCost(rel, required) eg.mem.SetBestProps(rel, required, provided, cost) diff --git a/pkg/sql/opt/props/physical/BUILD.bazel b/pkg/sql/opt/props/physical/BUILD.bazel index db73cf61b611..e75562839350 100644 --- a/pkg/sql/opt/props/physical/BUILD.bazel +++ b/pkg/sql/opt/props/physical/BUILD.bazel @@ -16,6 +16,8 @@ go_library( "//pkg/sql/opt/cat", "//pkg/sql/opt/constraint", "//pkg/sql/opt/props", + "//pkg/sql/pgwire/pgcode", + "//pkg/sql/pgwire/pgerror", "//pkg/sql/sem/eval", ], ) diff --git a/pkg/sql/opt/props/physical/distribution.go b/pkg/sql/opt/props/physical/distribution.go index 10b6ba4cc38a..2deb6cb59b3d 100644 --- a/pkg/sql/opt/props/physical/distribution.go +++ b/pkg/sql/opt/props/physical/distribution.go @@ -15,8 +15,11 @@ import ( "sort" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/opt/constraint" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" ) @@ -107,8 +110,10 @@ func (d *Distribution) FromLocality(locality roachpb.Locality) { // FromIndexScan sets the Distribution that results from scanning the given // index with the given constraint c (c can be nil). func (d *Distribution) FromIndexScan( - evalCtx *eval.Context, index cat.Index, c *constraint.Constraint, + evalCtx *eval.Context, tabMeta *opt.TableMeta, ord cat.IndexOrdinal, c *constraint.Constraint, ) { + tab := tabMeta.Table + index := tab.Index(ord) if index.Table().IsVirtualTable() { // Virtual tables do not have zone configurations. return @@ -153,8 +158,43 @@ func (d *Distribution) FromIndexScan( } } + // Populate the distribution for GLOBAL tables and REGIONAL BY TABLE. if len(regions) == 0 { - regions = getRegionsFromZone(index.Zone()) + regionsPopulated := false + zone := index.Zone() + + if tab.IsGlobalTable() { + // Global tables can always be treated as local to the gateway region. + gatewayRegion, found := evalCtx.Locality.Find("region") + if found { + regions = make(map[string]struct{}) + regions[gatewayRegion] = struct{}{} + regionsPopulated = true + } + } + if !regionsPopulated && zone.ReplicaConstraintsCount() == 0 { + // If there are no replica constraints, then the distribution is all + // regions in the database. + regionsNames, ok := tabMeta.GetRegionsInDatabase(evalCtx.Planner) + if !ok && evalCtx.SessionData().EnforceHomeRegion { + err := pgerror.New(pgcode.QueryHasNoHomeRegion, + "Query has no home region. Try accessing only tables defined in multi-region databases.") + panic(err) + } + regions = make(map[string]struct{}) + for _, regionName := range regionsNames { + regions[string(regionName)] = struct{}{} + } + } + if !regionsPopulated { + if homeRegion, ok := tab.HomeRegion(); ok { + regions = make(map[string]struct{}) + regions[homeRegion] = struct{}{} + } else { + // Use the leaseholder region(s). + regions = getRegionsFromZone(index.Zone()) + } + } } // Convert to a slice and sort regions. @@ -165,24 +205,24 @@ func (d *Distribution) FromIndexScan( sort.Strings(d.Regions) } +// GetRegionOfDistribution returns the single region name of the distribution, +// if there is exactly one. +func (d *Distribution) GetRegionOfDistribution() string { + if d == nil { + return "" + } + if len(d.Regions) == 1 { + return d.Regions[0] + } + return "" +} + // getRegionsFromZone returns the regions of the given zone config, if any. It // attempts to find the smallest set of regions likely to hold the leaseholder. func getRegionsFromZone(zone cat.Zone) map[string]struct{} { // First find any regional replica constraints. If there is exactly one, we // can return early. - var regions map[string]struct{} - for i, n := 0, zone.ReplicaConstraintsCount(); i < n; i++ { - replicaConstraint := zone.ReplicaConstraints(i) - for j, m := 0, replicaConstraint.ConstraintCount(); j < m; j++ { - constraint := replicaConstraint.Constraint(j) - if region, ok := getRegionFromConstraint(constraint); ok { - if regions == nil { - regions = make(map[string]struct{}) - } - regions[region] = struct{}{} - } - } - } + regions := getReplicaRegionsFromZone(zone) if len(regions) == 1 { return regions } @@ -238,3 +278,22 @@ func getRegionFromConstraint(constraint cat.Constraint) (region string, ok bool) // The region is prohibited. return "", false /* ok */ } + +// getReplicaRegionsFromZone returns the replica regions of the given zone +// config, if any. +func getReplicaRegionsFromZone(zone cat.Zone) map[string]struct{} { + var regions map[string]struct{} + for i, n := 0, zone.ReplicaConstraintsCount(); i < n; i++ { + replicaConstraint := zone.ReplicaConstraints(i) + for j, m := 0, replicaConstraint.ConstraintCount(); j < m; j++ { + constraint := replicaConstraint.Constraint(j) + if region, ok := getRegionFromConstraint(constraint); ok { + if regions == nil { + regions = make(map[string]struct{}) + } + regions[region] = struct{}{} + } + } + } + return regions +} diff --git a/pkg/sql/opt/table_meta.go b/pkg/sql/opt/table_meta.go index 9bc94f1d78ef..ea015155dc7f 100644 --- a/pkg/sql/opt/table_meta.go +++ b/pkg/sql/opt/table_meta.go @@ -11,6 +11,9 @@ package opt import ( + "github.com/cockroachdb/cockroach/pkg/sql/catalog/catpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/multiregion" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/opt/partition" "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" @@ -111,7 +114,9 @@ var tableAnnIDCount TableAnnID // called. Calling more than this number of times results in a panic. Having // a maximum enables a static annotation array to be inlined into the metadata // table struct. -const maxTableAnnIDCount = 2 +const maxTableAnnIDCount = 3 + +var regionConfigAnnID = NewTableAnnID() // TableMeta stores information about one of the tables stored in the metadata. // @@ -188,6 +193,23 @@ type TableMeta struct { anns [maxTableAnnIDCount]interface{} } +// TableAnnotation returns the given annotation that is associated with the +// given table. If the table has no such annotation, TableAnnotation returns +// nil. +func (tm *TableMeta) TableAnnotation(annID TableAnnID) interface{} { + return tm.anns[annID] +} + +// SetTableAnnotation associates the given annotation with the given table. The +// annotation is associated by the given ID, which was allocated by calling +// NewTableAnnID. If an annotation with the ID already exists on the table, then +// it is overwritten. +// +// See the TableAnnID comment for more details and a usage example. +func (tm *TableMeta) SetTableAnnotation(tabAnnID TableAnnID, ann interface{}) { + tm.anns[tabAnnID] = ann +} + // copyFrom initializes the receiver with a copy of the given TableMeta, which // is considered immutable. // @@ -419,6 +441,59 @@ func (tm *TableMeta) VirtualComputedColumns() ColSet { return virtualCols } +// GetRegionsInDatabase finds the full set of regions in the multiregion +// database owning the table described by `tm`, or returns ok=false if not +// multiregion. The result is cached in TableMeta. +func (tm *TableMeta) GetRegionsInDatabase( + planner eval.Planner, +) (regionNames catpb.RegionNames, ok bool) { + multiregionConfig, ok := tm.TableAnnotation(regionConfigAnnID).(*multiregion.RegionConfig) + if ok { + if multiregionConfig == nil { + return nil /* regionNames */, false + } + return multiregionConfig.Regions(), true + } + dbID := tm.Table.GetDatabaseID() + if dbID == 0 { + tm.SetTableAnnotation(regionConfigAnnID, nil) + return nil /* regionNames */, false + } + + regionConfig, ok := planner.GetMultiregionConfig(dbID) + if !ok { + tm.SetTableAnnotation(regionConfigAnnID, nil) + return nil /* regionNames */, false + } + multiregionConfig, _ = regionConfig.(*multiregion.RegionConfig) + tm.SetTableAnnotation(regionConfigAnnID, multiregionConfig) + return multiregionConfig.Regions(), true +} + +// GetDatabaseSurvivalGoal finds the survival goal of the multiregion database +// owning the table described by `tm`, or returns ok=false if not multiregion. +// The result is cached in TableMeta. +func (tm *TableMeta) GetDatabaseSurvivalGoal( + planner eval.Planner, +) (survivalGoal descpb.SurvivalGoal, ok bool) { + multiregionConfig, ok := tm.TableAnnotation(regionConfigAnnID).(*multiregion.RegionConfig) + if ok { + if multiregionConfig == nil { + return descpb.SurvivalGoal_ZONE_FAILURE /* survivalGoal */, false + } + return multiregionConfig.SurvivalGoal(), true + } + dbID := tm.Table.GetDatabaseID() + regionConfig, ok := planner.GetMultiregionConfig(dbID) + if !ok { + tm.SetTableAnnotation(regionConfigAnnID, nil) + return descpb.SurvivalGoal_ZONE_FAILURE /* survivalGoal */, false + } + multiregionConfig, _ = regionConfig.(*multiregion.RegionConfig) + tm.SetTableAnnotation(regionConfigAnnID, multiregionConfig) + return multiregionConfig.SurvivalGoal(), true +} + // TableAnnotation returns the given annotation that is associated with the // given table. If the table has no such annotation, TableAnnotation returns // nil. diff --git a/pkg/sql/opt/testutils/testcat/test_catalog.go b/pkg/sql/opt/testutils/testcat/test_catalog.go index 285a4151a7d1..1dcde1ac1f4a 100644 --- a/pkg/sql/opt/testutils/testcat/test_catalog.go +++ b/pkg/sql/opt/testutils/testcat/test_catalog.go @@ -846,6 +846,26 @@ func (tt *Table) IsPartitionAllBy() bool { return false } +// HomeRegion is part of the cat.Table interface. +func (tt *Table) HomeRegion() (region string, ok bool) { + return "", false +} + +// IsGlobalTable is part of the cat.Table interface. +func (tt *Table) IsGlobalTable() bool { + return false +} + +// IsRegionalByRow is part of the cat.Table interface. +func (tt *Table) IsRegionalByRow() bool { + return false +} + +// GetDatabaseID is part of the cat.Table interface. +func (tt *Table) GetDatabaseID() descpb.ID { + return 0 +} + // FindOrdinal returns the ordinal of the column with the given name. func (tt *Table) FindOrdinal(name string) int { for i, col := range tt.Columns { diff --git a/pkg/sql/opt/xform/coster.go b/pkg/sql/opt/xform/coster.go index b20c29924c73..12508223a72a 100644 --- a/pkg/sql/opt/xform/coster.go +++ b/pkg/sql/opt/xform/coster.go @@ -156,6 +156,11 @@ const ( // stale. largeMaxCardinalityScanCostPenalty = unboundedMaxCardinalityScanCostPenalty / 2 + // remoteCostPenalty is the penalty that should be added to the cost of joins + // which are not locality-optimized or Distribute operations when a session + // mode is set to error out on remote node accesses. + remoteCostPenalty = 1e8 + // preferLookupJoinFactor is a scale factor for the cost of a lookup join when // we have a hint for preferring a lookup join. preferLookupJoinFactor = 1e-6 @@ -657,6 +662,20 @@ func (c *coster) computeSortCost(sort *memo.SortExpr, required *physical.Require func (c *coster) computeDistributeCost( distribute *memo.DistributeExpr, required *physical.Required, ) memo.Cost { + if distribute.NoOpDistribution() { + // If the distribution will be elided, the cost is zero. + return memo.Cost(0) + } + if c.evalCtx.SessionData().EnforceHomeRegion { + // When forcing an index, we always want the access path with the index to + // be cheaper than a primary index scan, so that we do not error out with: + // `index "%s" cannot be used for this query`. The forced index case sets + // the cost of using other indexes to `hugeCost`, so let's make the + // current plan's costs expensive, but not on par with the illegal index + // plan since we never want the plan with the illegal index to win out. + return remoteCostPenalty + } + // TODO(rytaft): Compute a real cost here. Currently we just add a tiny cost // as a placeholder. return cpuCostFactor @@ -848,6 +867,7 @@ func (c *coster) computeHashJoinCost(join memo.RelExpr) memo.Cost { rowsProcessed = join.Relational().Stats.RowCount } cost += memo.Cost(rowsProcessed) * filterPerRow + cost += c.nonLocalityOptimizedJoinPenalty() return cost } @@ -886,6 +906,7 @@ func (c *coster) computeMergeJoinCost(join *memo.MergeJoinExpr) memo.Cost { panic(errors.AssertionFailedf("could not get rows processed for merge join")) } cost += memo.Cost(rowsProcessed) * filterPerRow + cost += c.nonLocalityOptimizedJoinPenalty() return cost } @@ -911,7 +932,7 @@ func (c *coster) computeLookupJoinCost( if join.LookupJoinPrivate.Flags.Has(memo.DisallowLookupJoinIntoRight) { return hugeCost } - return c.computeIndexLookupJoinCost( + cost := c.computeIndexLookupJoinCost( join, required, join.LookupColsAreTableKey, @@ -922,6 +943,10 @@ func (c *coster) computeLookupJoinCost( join.Flags, join.LocalityOptimized, ) + if !join.LocalityOptimized { + cost += c.nonLocalityOptimizedJoinPenalty() + } + return cost } func (c *coster) computeIndexLookupJoinCost( @@ -1078,6 +1103,7 @@ func (c *coster) computeInvertedJoinCost( c.rowScanCost(join, join.Table, join.Index, lookupCols, join.Relational().Stats) cost += memo.Cost(rowsProcessed) * perRowCost + cost += c.nonLocalityOptimizedJoinPenalty() return cost } @@ -1462,6 +1488,17 @@ func (c *coster) largeCardinalityCostPenalty( return 0 } +// remoteCostPenalty returns a penalty that should be added to +// the cost of joins which are not locality-optimized when a session mode is set +// to error out on remote table accesses. We want to favor joins which could be +// evaluated locally to avoid erroring out if possible. +func (c *coster) nonLocalityOptimizedJoinPenalty() memo.Cost { + if c.evalCtx != nil && c.evalCtx.SessionData().EnforceHomeRegion { + return remoteCostPenalty + } + return 0 +} + // localityMatchScore returns a number from 0.0 to 1.0 that describes how well // the current node's locality matches the given zone constraints and // leaseholder preferences, with 0.0 indicating 0% and 1.0 indicating 100%. This diff --git a/pkg/sql/opt/xform/optimizer.go b/pkg/sql/opt/xform/optimizer.go index b837e3f80748..abd2ec2eb431 100644 --- a/pkg/sql/opt/xform/optimizer.go +++ b/pkg/sql/opt/xform/optimizer.go @@ -761,13 +761,11 @@ func (o *Optimizer) setLowestCostTree(parent opt.Expr, parentProps *physical.Req var childProps *physical.Required for i, n := 0, parent.ChildCount(); i < n; i++ { before := parent.Child(i) - if relParent != nil { childProps = BuildChildPhysicalProps(o.mem, relParent, i, parentProps) } else { childProps = BuildChildPhysicalPropsScalar(o.mem, parent, i) } - after := o.setLowestCostTree(before, childProps) if after != before { if mutable == nil { diff --git a/pkg/sql/opt_catalog.go b/pkg/sql/opt_catalog.go index a7c187476d47..764874c245cc 100644 --- a/pkg/sql/opt_catalog.go +++ b/pkg/sql/opt_catalog.go @@ -1255,6 +1255,54 @@ func (ot *optTable) IsPartitionAllBy() bool { return ot.desc.IsPartitionAllBy() } +// HomeRegion is part of the cat.Table interface. +func (ot *optTable) HomeRegion() (region string, ok bool) { + localityConfig := ot.desc.GetLocalityConfig() + if localityConfig == nil { + return "", false + } + regionalByTable := localityConfig.GetRegionalByTable() + if regionalByTable == nil { + return "", false + } + if regionalByTable.Region != nil { + return regionalByTable.Region.String(), true + } + if ot.zone.LeasePreferenceCount() != 1 { + return "", false + } + if ot.zone.LeasePreference(0).ConstraintCount() != 1 { + return "", false + } + if ot.zone.LeasePreference(0).Constraint(0).GetKey() != "region" { + return "", false + } + return ot.zone.LeasePreference(0).Constraint(0).GetValue(), true +} + +// IsGlobalTable is part of the cat.Table interface. +func (ot *optTable) IsGlobalTable() bool { + localityConfig := ot.desc.GetLocalityConfig() + if localityConfig == nil { + return false + } + return localityConfig.GetGlobal() != nil +} + +// IsRegionalByRow is part of the cat.Table interface. +func (ot *optTable) IsRegionalByRow() bool { + localityConfig := ot.desc.GetLocalityConfig() + if localityConfig == nil { + return false + } + return localityConfig.GetRegionalByRow() != nil +} + +// GetDatabaseID is part of the cat.Table interface. +func (ot *optTable) GetDatabaseID() descpb.ID { + return ot.desc.GetParentID() +} + // lookupColumnOrdinal returns the ordinal of the column with the given ID. A // cache makes the lookup O(1). func (ot *optTable) lookupColumnOrdinal(colID descpb.ColumnID) (int, error) { @@ -2190,6 +2238,26 @@ func (ot *optVirtualTable) IsPartitionAllBy() bool { return false } +// HomeRegion is part of the cat.Table interface. +func (ot *optVirtualTable) HomeRegion() (region string, ok bool) { + return "", false +} + +// IsGlobalTable is part of the cat.Table interface. +func (ot *optVirtualTable) IsGlobalTable() bool { + return false +} + +// IsRegionalByRow is part of the cat.Table interface. +func (ot *optVirtualTable) IsRegionalByRow() bool { + return false +} + +// GetDatabaseID is part of the cat.Table interface. +func (ot *optVirtualTable) GetDatabaseID() descpb.ID { + return 0 +} + // CollectTypes is part of the cat.DataSource interface. func (ot *optVirtualTable) CollectTypes(ord int) (descpb.IDs, error) { col := ot.desc.AllColumns()[ord] diff --git a/pkg/sql/pgwire/pgcode/codes.go b/pkg/sql/pgwire/pgcode/codes.go index 06e6b63558a3..d9831d7de05f 100644 --- a/pkg/sql/pgwire/pgcode/codes.go +++ b/pkg/sql/pgwire/pgcode/codes.go @@ -202,6 +202,8 @@ var ( Windowing = MakeCode("42P20") InvalidRecursion = MakeCode("42P19") InvalidForeignKey = MakeCode("42830") + QueryNotRunningInHomeRegion = MakeCode("42898") + QueryHasNoHomeRegion = MakeCode("42899") InvalidName = MakeCode("42602") NameTooLong = MakeCode("42622") ReservedName = MakeCode("42939") diff --git a/pkg/sql/region_util.go b/pkg/sql/region_util.go index 9cfc3874cb94..c1ebfab6d555 100644 --- a/pkg/sql/region_util.go +++ b/pkg/sql/region_util.go @@ -1298,7 +1298,7 @@ func (p *planner) validateAllMultiRegionZoneConfigsInDatabase( ) } -// CurrentDatabaseRegionConfig is part of the eval.DatabaseCatalog interface. +// CurrentDatabaseRegionConfig is part of the eval.RegionOperator interface. // CurrentDatabaseRegionConfig uses the cache to synthesize the RegionConfig // and as such is intended for DML use. It returns nil // if the current database is not multi-region enabled. @@ -2249,3 +2249,20 @@ func (p *planner) checkNoRegionChangeUnderway( } return nil } + +// GetMultiregionConfig is part of the eval.Planner interface. +func (p *planner) GetMultiregionConfig(databaseID descpb.ID) (interface{}, bool) { + + regionConfig, err := SynthesizeRegionConfig( + p.EvalContext().Ctx(), + p.txn, + databaseID, + p.Descriptors(), + SynthesizeRegionConfigOptionUseCache, + ) + + if err != nil { + return nil /* regionConfig */, false + } + return ®ionConfig, true +} diff --git a/pkg/sql/sem/eval/BUILD.bazel b/pkg/sql/sem/eval/BUILD.bazel index d24f1a17da74..5025f39938b7 100644 --- a/pkg/sql/sem/eval/BUILD.bazel +++ b/pkg/sql/sem/eval/BUILD.bazel @@ -49,6 +49,7 @@ go_library( "//pkg/settings", "//pkg/settings/cluster", "//pkg/sql/catalog/catpb", + "//pkg/sql/catalog/descpb", "//pkg/sql/lex", "//pkg/sql/pgwire/pgcode", "//pkg/sql/pgwire/pgerror", diff --git a/pkg/sql/sem/eval/context.go b/pkg/sql/sem/eval/context.go index afb7694ed9a4..15a98682d82c 100644 --- a/pkg/sql/sem/eval/context.go +++ b/pkg/sql/sem/eval/context.go @@ -572,6 +572,12 @@ func (ec *Context) Ctx() context.Context { return ec.Context } +// BoundedStaleness returns true if this query uses bounded staleness. +func (ec *Context) BoundedStaleness() bool { + return ec.AsOfSystemTime != nil && + ec.AsOfSystemTime.BoundedStaleness +} + // ensureExpectedType will return an error if a datum does not match the // provided type. If the expected type is Any or if the datum is a Null // type, then no error will be returned. diff --git a/pkg/sql/sem/eval/deps.go b/pkg/sql/sem/eval/deps.go index 9df07272cfc1..6e7ed54fab9b 100644 --- a/pkg/sql/sem/eval/deps.go +++ b/pkg/sql/sem/eval/deps.go @@ -17,6 +17,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/security/username" "github.com/cockroachdb/cockroach/pkg/sql/catalog/catpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgnotice" "github.com/cockroachdb/cockroach/pkg/sql/privilege" "github.com/cockroachdb/cockroach/pkg/sql/roleoption" @@ -339,6 +340,12 @@ type Planner interface { privilegeObjectPath string, privilegeObjectType privilege.ObjectType, ) (*catpb.PrivilegeDescriptor, error) + + // GetMultiregionConfig synthesizes a new multiregion.RegionConfig describing + // the multiregion properties of the database identified via databaseID. The + // second return value is false if the database doesn't exist or is not + // multiregion. + GetMultiregionConfig(databaseID descpb.ID) (interface{}, bool) } // InternalRows is an iterator interface that's exposed by the internal diff --git a/pkg/sql/sessiondatapb/local_only_session_data.proto b/pkg/sql/sessiondatapb/local_only_session_data.proto index a94e11c0b90c..6c045cfc605f 100644 --- a/pkg/sql/sessiondatapb/local_only_session_data.proto +++ b/pkg/sql/sessiondatapb/local_only_session_data.proto @@ -271,6 +271,10 @@ message LocalOnlySessionData { // given probability. This should only be used in test scenarios and is very // much a non-production setting. double testing_optimizer_disable_rule_probability = 73; + // EnforceHomeRegion, when true, causes queries which scan rows from multiple + // regions, or which scan rows from a single home region, but initiated from + // a gateway region which differs from that home region, to error out. + bool enforce_home_region = 74; /////////////////////////////////////////////////////////////////////////// // WARNING: consider whether a session parameter you're adding needs to // diff --git a/pkg/sql/vars.go b/pkg/sql/vars.go index 672f6631f771..36af8ec11942 100644 --- a/pkg/sql/vars.go +++ b/pkg/sql/vars.go @@ -2120,6 +2120,23 @@ var varGen = map[string]sessionVar{ return formatFloatAsPostgresSetting(0) }, }, + + // CockroachDB extension. + `enforce_home_region`: { + GetStringVal: makePostgresBoolGetStringValFn(`enforce_home_region`), + Set: func(_ context.Context, m sessionDataMutator, s string) error { + b, err := paramparse.ParseBoolVar("enforce_home_region", s) + if err != nil { + return err + } + m.SetEnforceHomeRegion(b) + return nil + }, + Get: func(evalCtx *extendedEvalContext, _ *kv.Txn) (string, error) { + return formatBoolAsPostgresSetting(evalCtx.SessionData().EnforceHomeRegion), nil + }, + GlobalDefault: globalFalse, + }, } const compatErrMsg = "this parameter is currently recognized only for compatibility and has no effect in CockroachDB."