diff --git a/pkg/ccl/logictestccl/testdata/logic_test/as_of b/pkg/ccl/logictestccl/testdata/logic_test/as_of index c1c8b45c08f1..722406252bef 100644 --- a/pkg/ccl/logictestccl/testdata/logic_test/as_of +++ b/pkg/ccl/logictestccl/testdata/logic_test/as_of @@ -179,16 +179,10 @@ EXPLAIN (OPT, MEMO) SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('1ms') ---- memo (optimized, ~8KB, required=[presentation: info:6] [distribution: test]) ├── G1: (explain G2 [presentation: i:1,j:2,k:3] [distribution: test]) - │ ├── [presentation: info:6] [distribution: test] - │ │ ├── best: (explain G2="[presentation: i:1,j:2,k:3] [distribution: test]" [presentation: i:1,j:2,k:3] [distribution: test]) - │ │ └── cost: 5.18 - │ └── [] - │ ├── best: (explain G2="[presentation: i:1,j:2,k:3]" [presentation: i:1,j:2,k:3] [distribution: test]) + │ └── [presentation: info:6] [distribution: test] + │ ├── best: (explain G2="[presentation: i:1,j:2,k:3] [distribution: test]" [presentation: i:1,j:2,k:3] [distribution: test]) │ └── cost: 5.18 ├── G2: (select G3 G4) (select G5 G6) - │ ├── [presentation: i:1,j:2,k:3] - │ │ ├── best: (select G5 G6) - │ │ └── cost: 5.16 │ ├── [presentation: i:1,j:2,k:3] [distribution: test] │ │ ├── best: (select G5="[distribution: test]" G6) │ │ └── cost: 5.16 diff --git a/pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error b/pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error new file mode 100644 index 000000000000..c0de5ac06250 --- /dev/null +++ b/pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error @@ -0,0 +1,695 @@ +# tenant-cluster-setting-override-opt: allow-multi-region-abstractions-for-secondary-tenants +# LogicTest: multiregion-9node-3region-3azs !metamorphic + +# Set the closed timestamp interval to be short to shorten the amount of time +# we need to wait for the system config to propagate. +statement ok +SET CLUSTER SETTING kv.closed_timestamp.side_transport_interval = '10ms'; + +statement ok +SET CLUSTER SETTING kv.closed_timestamp.target_duration = '10ms'; + +# Start with SURVIVE ZONE FAILURE for positive tests. +# SURVIVE REGION FAILURE cases will always error out. +statement ok +CREATE DATABASE multi_region_test_db PRIMARY REGION "ap-southeast-2" REGIONS "ca-central-1", "us-east-1" SURVIVE ZONE FAILURE; + +statement ok +USE multi_region_test_db + +query T +SELECT gateway_region(); +---- +ap-southeast-2 + +statement ok +CREATE TABLE parent ( + p_id INT PRIMARY KEY, + FAMILY (p_id) +) LOCALITY REGIONAL BY ROW; + +statement ok +CREATE TABLE child ( + c_id INT PRIMARY KEY, + c_p_id INT REFERENCES parent (p_id), + INDEX (c_p_id), + FAMILY (c_id, c_p_id) +) LOCALITY REGIONAL BY ROW; + +statement ok +CREATE TABLE messages_global ( + account_id INT NOT NULL, + message_id UUID DEFAULT gen_random_uuid(), + message STRING NOT NULL, + PRIMARY KEY (account_id), + INDEX msg_idx(message) +) LOCALITY GLOBAL + +statement ok +CREATE TABLE messages_rbt ( + account_id INT NOT NULL, + message_id UUID DEFAULT gen_random_uuid(), + message STRING NOT NULL, + PRIMARY KEY (account_id), + INDEX msg_idx(message) +) LOCALITY REGIONAL BY TABLE + +statement ok +CREATE TABLE messages_rbr ( + account_id INT NOT NULL, + message_id UUID DEFAULT gen_random_uuid(), + message STRING NOT NULL, + crdb_region crdb_internal_region NOT NULL, + PRIMARY KEY (account_id), + INDEX msg_idx(message) +) +LOCALITY REGIONAL BY ROW + +statement ok +CREATE TABLE messages_rbr_alt ( + account_id INT NOT NULL, + message_id UUID DEFAULT gen_random_uuid(), + message STRING NOT NULL, + crdb_region_alt crdb_internal_region NOT NULL, + PRIMARY KEY (account_id), + INDEX msg_idx(message) +) +LOCALITY REGIONAL BY ROW AS crdb_region_alt + +statement ok +CREATE TABLE customers ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name STRING NOT NULL +) LOCALITY REGIONAL BY ROW; + +statement ok +ALTER TABLE customers INJECT STATISTICS '[ + { + "columns": ["id"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 100 + }, + { + "columns": ["crdb_region"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 3 + }, + { + "columns": ["crdb_region","id"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 100 + } +]' + +statement ok +CREATE TABLE orders ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + cust_id UUID NOT NULL, + items STRING NOT NULL, + INDEX (cust_id), + FOREIGN KEY (cust_id, crdb_region) REFERENCES customers (id, crdb_region) ON UPDATE CASCADE +) LOCALITY REGIONAL BY ROW; + +statement ok +ALTER TABLE orders INJECT STATISTICS '[ + { + "columns": ["id"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 100 + }, + { + "columns": ["cust_id"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 10 + }, + { + "columns": ["crdb_region"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 3 + }, + { + "columns": ["crdb_region","id"], + "created_at": "2018-05-01 1:00:00.00000+00:00", + "row_count": 100, + "distinct_count": 100 + } +]' + +statement ok +CREATE TABLE json_arr1_rbt ( + k INT PRIMARY KEY, + i INT, + j JSONB, + a STRING[], + INVERTED INDEX j_idx (j), + INVERTED INDEX a_idx (a) +) LOCALITY REGIONAL BY TABLE + +statement ok +CREATE TABLE json_arr1_rbr ( + k INT PRIMARY KEY, + i INT, + j JSONB, + a STRING[], + INVERTED INDEX j_idx (j), + INVERTED INDEX a_idx (a) +) LOCALITY REGIONAL BY ROW + +statement ok +CREATE TABLE json_arr2_rbt ( + k INT PRIMARY KEY, + l INT, + j JSONB, + a STRING[] +) LOCALITY REGIONAL BY TABLE + +# Zone configs sometimes are not available right away. Issue a couple queries +# that take a while to compile to make sure they're available. +query I +SELECT 1 FROM child c1, child c2, child c3, child c4, child c5, child c6, child c7, + child c8, child c9, child c10, child c11, child c12 +---- + +query I +SELECT 1 FROM parent p1, parent p2, parent p3, parent p4, parent p5, parent p6, parent p7, + parent p8, parent p9, parent p10, parent p11, parent p12 +---- + +statement ok +SET enforce_home_region = true + +# A join relation with no home region as the left input of lookup join should +# not be allowed. +statement error pq: Query has no home region\. Try adding a filter on rbr\.crdb_region and/or on key column \(rbr\.account_id\)\. +SELECT * FROM messages_rbr rbr INNER LOOKUP JOIN messages_global g2 ON rbr.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id + +# A join involving a VALUES clause should succeed. +query I retry +SELECT c_id FROM child, (SELECT * FROM [VALUES (1)]) v WHERE crdb_region = 'ap-southeast-2' +---- + +# Joins which may access all regions should error out in phase 1. +statement error pq: Query has no home region\. Try adding a filter on p\.crdb_region and/or on key column \(p\.p_id\)\. Try adding a filter on c\.crdb_region and/or on key column \(c\.c_p_id\)\. +SELECT * FROM parent p, child c WHERE p_id = c_p_id AND +p.crdb_region = c.crdb_region LIMIT 1 + +# Locality optimized lookup join should not error out in phase 1. +query TTT retry +SELECT * FROM parent p, child c WHERE p_id = c_p_id LIMIT 1 +---- + +# Locality optimized lookup join should not error out in phase 1. +query TT retry +SELECT * FROM child WHERE NOT EXISTS (SELECT * FROM parent WHERE p_id = c_p_id) AND c_id = 10 +---- + +query T retry +EXPLAIN(OPT,VERBOSE) SELECT * FROM child WHERE NOT EXISTS (SELECT * FROM parent WHERE p_id = c_p_id) AND c_id = 10 +---- +anti-join (lookup parent) + ├── columns: c_id:1 c_p_id:2 + ├── lookup expression + │ └── filters + │ ├── parent.crdb_region:7 IN ('ca-central-1', 'us-east-1') [outer=(7), constraints=(/7: [/'ca-central-1' - /'ca-central-1'] [/'us-east-1' - /'us-east-1']; tight)] + │ └── c_p_id:2 = p_id:6 [outer=(2,6), constraints=(/2: (/NULL - ]; /6: (/NULL - ]), fd=(2)==(6), (6)==(2)] + ├── lookup columns are key + ├── cardinality: [0 - 1] + ├── stats: [rows=1e-10] + ├── cost: 8.29105556 + ├── key: () + ├── fd: ()-->(1,2) + ├── distribution: ap-southeast-2 + ├── anti-join (lookup parent) + │ ├── columns: c_id:1 c_p_id:2 + │ ├── lookup expression + │ │ └── filters + │ │ ├── parent.crdb_region:7 = 'ap-southeast-2' [outer=(7), immutable, constraints=(/7: [/'ap-southeast-2' - /'ap-southeast-2']; tight), fd=()-->(7)] + │ │ └── c_p_id:2 = p_id:6 [outer=(2,6), constraints=(/2: (/NULL - ]; /6: (/NULL - ]), fd=(2)==(6), (6)==(2)] + │ ├── lookup columns are key + │ ├── cardinality: [0 - 1] + │ ├── stats: [rows=0.6666667, distinct(1)=0.666667, null(1)=0, distinct(2)=0.665314, null(2)=0.00666667] + │ ├── cost: 6.65852222 + │ ├── key: () + │ ├── fd: ()-->(1,2) + │ ├── distribution: ap-southeast-2 + │ ├── locality-optimized-search + │ │ ├── columns: c_id:1 c_p_id:2 + │ │ ├── left columns: c_id:11 c_p_id:12 + │ │ ├── right columns: c_id:16 c_p_id:17 + │ │ ├── cardinality: [0 - 1] + │ │ ├── stats: [rows=1, distinct(1)=1, null(1)=0, distinct(2)=0.995512, null(2)=0.01] + │ │ ├── cost: 4.76472222 + │ │ ├── key: () + │ │ ├── fd: ()-->(1,2) + │ │ ├── distribution: ap-southeast-2 + │ │ ├── prune: (2) + │ │ ├── scan child + │ │ │ ├── columns: c_id:11 c_p_id:12 + │ │ │ ├── constraint: /13/11: [/'ap-southeast-2'/10 - /'ap-southeast-2'/10] + │ │ │ ├── cardinality: [0 - 1] + │ │ │ ├── stats: [rows=0.9333333, distinct(11)=0.933333, null(11)=0, distinct(13)=0.933333, null(13)=0, distinct(11,13)=0.933333, null(11,13)=0] + │ │ │ ├── cost: 1.70518519 + │ │ │ ├── key: () + │ │ │ └── fd: ()-->(11,12) + │ │ └── scan child + │ │ ├── columns: c_id:16 c_p_id:17 + │ │ ├── constraint: /18/16 + │ │ │ ├── [/'ca-central-1'/10 - /'ca-central-1'/10] + │ │ │ └── [/'us-east-1'/10 - /'us-east-1'/10] + │ │ ├── cardinality: [0 - 1] + │ │ ├── stats: [rows=0.9666667, distinct(16)=0.966667, null(16)=0, distinct(18)=0.966667, null(18)=0, distinct(16,18)=0.966667, null(16,18)=0] + │ │ ├── cost: 3.03953704 + │ │ ├── key: () + │ │ └── fd: ()-->(16,17) + │ └── filters (true) + └── filters (true) + +statement ok +SET locality_optimized_partitioned_index_scan = false + +# This query should error out because it is not locality optimized. +statement error pq: Query has no home region\. Try adding a filter on parent\.crdb_region and/or on key column \(parent\.p_id\)\. Try adding a filter on child\.crdb_region and/or on key column \(child\.c_id\)\. +SELECT * FROM child WHERE NOT EXISTS (SELECT * FROM parent WHERE p_id = c_p_id) AND c_id = 10 + +statement ok +RESET locality_optimized_partitioned_index_scan + +# Locality optimized search is allowed. +query T retry +SELECT * FROM parent LIMIT 1 +---- + +query T retry +EXPLAIN(OPT) SELECT * FROM parent LIMIT 1 +---- +locality-optimized-search + ├── scan parent + │ ├── constraint: /7/6: [/'ap-southeast-2' - /'ap-southeast-2'] + │ └── limit: 1 + └── scan parent + ├── constraint: /11/10: [/'ca-central-1' - /'us-east-1'] + └── limit: 1 + +# Locality optimized search with lookup join will be supported in phase 2 or 3 +# when we can dynamically determine if the lookup will access a remote region. +statement error pq: Query has no home region\. Try adding a filter on o\.crdb_region and/or on key column \(o\.cust_id\)\. +SELECT * FROM customers c JOIN orders o ON c.id = o.cust_id AND + (c.crdb_region = o.crdb_region) WHERE c.id = '69a1c2c2-5b18-459e-94d2-079dc53a4dd0' + +# A join relation with no home region as the left input of lookup join should +# not be allowed. +statement error pq: Query has no home region\. Try adding a filter on rbr\.crdb_region and/or on key column \(rbr\.account_id\)\. +SELECT * FROM messages_rbr rbr INNER LOOKUP JOIN messages_global g2 ON rbr.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id + +# Locality optimized lookup join is allowed. +query TTTTTTT retry +SELECT * FROM messages_rbr rbr, messages_rbt rbt WHERE rbr.account_id = rbt.account_id LIMIT 1 +---- + +query T retry +EXPLAIN SELECT * FROM messages_rbr rbr, messages_rbt rbt WHERE rbr.account_id = rbt.account_id LIMIT 1 +---- +distribution: local +vectorized: true +· +• limit +│ count: 1 +│ +└── • lookup join + │ table: messages_rbr@messages_rbr_pkey + │ equality cols are key + │ lookup condition: (crdb_region = 'ap-southeast-2') AND (account_id = account_id) + │ remote lookup condition: (crdb_region IN ('ca-central-1', 'us-east-1')) AND (account_id = account_id) + │ + └── • scan + missing stats + table: messages_rbt@messages_rbt_pkey + spans: FULL SCAN (SOFT LIMIT) + +# Select from a global table is OK with ZONE survivability. +query TTT retry +SELECT * FROM messages_global@messages_global_pkey +---- + +# Select from REGIONAL BY TABLE is OK with ZONE survivability. +query T retry +SELECT message from messages_rbt@messages_rbt_pkey +---- + +# A local join between an RBR and RBT table should be allowed. +query TTTTTTT retry +SELECT * FROM messages_rbt rbt INNER LOOKUP JOIN messages_rbr rbr ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' +---- + +query T retry +EXPLAIN(OPT) SELECT * FROM messages_rbt rbt INNER LOOKUP JOIN messages_rbr rbr ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' +---- +inner-join (lookup messages_rbr [as=rbr]) + ├── flags: force lookup join (into right side) + ├── lookup columns are key + ├── project + │ ├── scan messages_rbt [as=rbt] + │ └── projections + │ └── 'ap-southeast-2' + └── filters (true) + +# A local join between an RBR and RBT table should be allowed. +query TTTTTTT retry +SELECT * FROM messages_rbr rbr INNER LOOKUP JOIN messages_rbt rbt ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' +---- + +query T retry +EXPLAIN(OPT) SELECT * FROM messages_rbr rbr INNER LOOKUP JOIN messages_rbt rbt ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' +---- +inner-join (lookup messages_rbt [as=rbt]) + ├── flags: force lookup join (into right side) + ├── lookup columns are key + ├── scan messages_rbr [as=rbr] + │ └── constraint: /4/1: [/'ap-southeast-2' - /'ap-southeast-2'] + └── filters (true) + +# A lookup join between with a global table as either input should be allowed. +query TTTTTT retry +SELECT * FROM messages_global g1 INNER LOOKUP JOIN messages_global g2 ON g1.account_id = g2.account_id +---- + +query T retry +EXPLAIN(OPT) SELECT * FROM messages_global g1 INNER LOOKUP JOIN messages_global g2 ON g1.account_id = g2.account_id +---- +inner-join (lookup messages_global [as=g2]) + ├── flags: force lookup join (into right side) + ├── lookup columns are key + ├── scan messages_global [as=g1] + └── filters (true) + +# A join relation with local home region as the left input of lookup join should be allowed. +query TTTTTTTTT retry +SELECT * FROM messages_global g1 INNER LOOKUP JOIN messages_global g2 ON g1.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id +---- + +query T retry +EXPLAIN(OPT) SELECT * FROM messages_global g1 INNER LOOKUP JOIN messages_global g2 ON g1.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id +---- +inner-join (lookup messages_global [as=g3]) + ├── flags: force lookup join (into right side) + ├── lookup columns are key + ├── inner-join (lookup messages_global [as=g2]) + │ ├── flags: force lookup join (into right side) + │ ├── lookup columns are key + │ ├── scan messages_global [as=g1] + │ └── filters (true) + └── filters (true) + +# A join relation with no home region as the left input of lookup join should +# not be allowed. +statement error pq: Query has no home region\. Try adding a filter on rbr\.crdb_region and/or on key column \(rbr\.account_id\)\. +SELECT * FROM messages_rbr rbr INNER LOOKUP JOIN messages_global g2 ON rbr.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id + +# The explicit REGIONAL BY ROW AS column name should be used in the error +# message if it differs from the default crdb_region. +statement error pq: Query has no home region\. Try adding a filter on rbr\.crdb_region_alt and/or on key column \(rbr\.account_id\)\. +SELECT * FROM messages_rbr_alt rbr INNER LOOKUP JOIN messages_global g2 ON rbr.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id + +# A lookup join relation with a left input join relation which uses locality +# optimized scan in one of the tables of the lookup join should be allowed. +query TTTTTTTTTT retry +SELECT * FROM (SELECT * FROM messages_rbr LIMIT 1) rbr INNER LOOKUP JOIN + messages_global g2 ON rbr.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id +---- + +query T retry +EXPLAIN(OPT) SELECT * FROM (SELECT * FROM messages_rbr LIMIT 1) rbr INNER LOOKUP JOIN + messages_global g2 ON rbr.account_id = g2.account_id + INNER LOOKUP JOIN messages_global g3 ON g2.account_id = g3.account_id +---- +inner-join (lookup messages_global [as=g3]) + ├── flags: force lookup join (into right side) + ├── lookup columns are key + ├── inner-join (lookup messages_global [as=g2]) + │ ├── flags: force lookup join (into right side) + │ ├── lookup columns are key + │ ├── locality-optimized-search + │ │ ├── scan messages_rbr + │ │ │ ├── constraint: /33/30: [/'ap-southeast-2' - /'ap-southeast-2'] + │ │ │ └── limit: 1 + │ │ └── scan messages_rbr + │ │ ├── constraint: /39/36: [/'ca-central-1' - /'us-east-1'] + │ │ └── limit: 1 + │ └── filters (true) + └── filters (true) + +statement ok +ALTER TABLE messages_rbt SET LOCALITY REGIONAL BY TABLE IN "us-east-1"; + +# Select from REGIONAL BY TABLE should indicate the gateway region to use. +statement error pq: Query is not running in its home region. Try running the query from region 'us-east-1'. +SELECT message from messages_rbt@messages_rbt_pkey + +# Logging in through the appropriate gateway region allows reading from an RBR +# table with a span on crdb_region. +query T nodeidx=4 +SET enforce_home_region = true; +USE multi_region_test_db; +SELECT message from messages_rbr@msg_idx WHERE crdb_region = 'ca-central-1' +---- + +query T nodeidx=4 +SET enforce_home_region = true; +USE multi_region_test_db; +EXPLAIN(OPT) SELECT message from messages_rbr@msg_idx WHERE crdb_region = 'ca-central-1' +---- +project + └── scan messages_rbr@msg_idx + ├── constraint: /4/3/1: [/'ca-central-1' - /'ca-central-1'] + └── flags: force-index=msg_idx + +# Lookup join should detect REGIONAL BY TABLE in the wrong region. +statement error pq: Query has no home region\. The home region \('us-east-1'\) of table 'messages_rbt' does not match the home region \('ap-southeast-2'\) of lookup table 'messages_rbr'\. +SELECT * FROM messages_rbt rbt inner lookup join messages_rbr rbr ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' + +# Lookup join should detect REGIONAL BY TABLE in the wrong region. +statement error pq: Query has no home region\. The home region \('ap-southeast-2'\) of table 'messages_rbr' does not match the home region \('us-east-1'\) of lookup table 'messages_rbt'\. +SELECT * FROM messages_rbr rbr inner lookup join messages_rbt rbt ON rbr.account_id = rbt.account_id +AND rbr.crdb_region = 'ap-southeast-2' + +# Equality predicate on crdb_region of an RBR table is allowed. +query T retry +SELECT message from messages_rbr@msg_idx WHERE crdb_region = 'ap-southeast-2' +---- + +query T retry +EXPLAIN(OPT) SELECT message from messages_rbr@msg_idx WHERE crdb_region = 'ap-southeast-2' +---- +project + └── scan messages_rbr@msg_idx + ├── constraint: /4/3/1: [/'ap-southeast-2' - /'ap-southeast-2'] + └── flags: force-index=msg_idx + +statement ok +PREPARE s AS SELECT message from messages_rbr@msg_idx WHERE crdb_region = $1 + +# Prepared statement accessing the local span is allowed. +query T retry +EXECUTE s('ap-southeast-2') +---- + +# Prepared statement accessing a remote span is disallowed. +statement error pq: Query is not running in its home region. Try running the query from region 'us-east-1'. +EXECUTE s('us-east-1') + +statement ok +RESET enforce_home_region + +statement ok +CREATE DATABASE non_multiregion_test_db; + +statement ok +USE non_multiregion_test_db + +statement ok +CREATE TABLE messages ( + account_id INT NOT NULL, + message_id UUID DEFAULT gen_random_uuid(), + message STRING NOT NULL, + PRIMARY KEY (account_id), + INDEX msg_idx(message) +) + +statement ok +SET enforce_home_region = true + +# Tables in non-multiregion databases have no home region. +statement error pq: Query has no home region. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM messages + +# If any table in a query has no home region, error out. +statement error pq: Query has no home region. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM non_multiregion_test_db.messages, multi_region_test_db.messages_global + +# Scans from tables in non-multiregion databases with contradictions in +# predicates are not allowed. +statement error pq: Query has no home region. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM messages WHERE account_id = 1 AND account_id = 2 + +# A lookup join from a multiregion table to non-multiregion table is not +# allowed. +statement error pq: Query has no home region. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM multi_region_test_db.messages_global mr INNER LOOKUP JOIN non_multiregion_test_db.messages nmr + ON mr.account_id = nmr.account_id + +statement ok +ALTER DATABASE multi_region_test_db SURVIVE REGION FAILURE + +statement ok +USE multi_region_test_db + +# Statements which previously succeeded should now fail under REGION survivability. +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM parent p, child c WHERE c_id = 10 AND p_id = c_p_id + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM child WHERE NOT EXISTS (SELECT * FROM parent WHERE p_id = c_p_id) AND c_id = 10 + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM parent LIMIT 1 + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT * FROM messages_global@messages_global_pkey + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT message from messages_rbt@messages_rbt_pkey + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +SELECT message from messages_rbr@msg_idx WHERE crdb_region = 'ap-southeast-2' + +statement error pq: The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability. +EXECUTE s('ap-southeast-2') + +####################### +# Inverted join tests # +####################### + +statement ok +ALTER DATABASE multi_region_test_db SURVIVE ZONE FAILURE + +statement ok +USE multi_region_test_db + +# Inverted join on REGIONAL BY TABLE tables is allowed. +query T retry +SELECT t1.k FROM json_arr2_rbt AS t2 INNER INVERTED JOIN json_arr1_rbt AS t1 ON t1.j @> t2.j +---- + +query T retry +EXPLAIN(OPT) SELECT t1.k FROM json_arr2_rbt AS t2 INNER INVERTED JOIN json_arr1_rbt AS t1 ON t1.j @> t2.j +---- +project + └── inner-join (lookup json_arr1_rbt [as=t1]) + ├── lookup columns are key + ├── inner-join (inverted json_arr1_rbt@j_idx [as=t1]) + │ ├── flags: force inverted join (into right side) + │ ├── inverted-expr + │ │ └── t1.j @> t2.j + │ ├── scan json_arr2_rbt [as=t2] + │ └── filters (true) + └── filters + └── t1.j @> t2.j + +# Inverted join doing lookup into a REGIONAL BY ROW table is not allowed. +statement error pq: Query has no home region\. Try adding a filter on t1\.crdb_region and/or on key column \(t1\.j_inverted_key\)\. +SELECT t1.k FROM json_arr2_rbt AS t2 INNER INVERTED JOIN json_arr1_rbr AS t1 ON t1.j @> t2.j + +# Inverted join with lookup into a REGIONAL BY ROW table in local region is allowed. +query T retry +SELECT t1.k FROM json_arr2_rbt AS t2 INNER INVERTED JOIN json_arr1_rbr AS t1 ON t1.j @> t2.j +AND t1.crdb_region = 'ap-southeast-2' +---- + +# A local inverted join should not have high cost estimates (due to +# `largeDistributeCost`). +query T retry +EXPLAIN(OPT,VERBOSE) SELECT t1.k FROM json_arr2_rbt AS t2 INNER INVERTED JOIN json_arr1_rbr AS t1 ON t1.j @> t2.j +AND t1.crdb_region = 'ap-southeast-2' LIMIT 1 +---- +project + ├── columns: k:7 + ├── cardinality: [0 - 1] + ├── immutable + ├── stats: [rows=1] + ├── cost: 4309.15778 + ├── key: () + ├── fd: ()-->(7) + ├── distribution: ap-southeast-2 + ├── prune: (7) + └── limit + ├── columns: t2.j:3 t1.k:7 t1.j:9 crdb_region:11 + ├── cardinality: [0 - 1] + ├── immutable + ├── stats: [rows=1] + ├── cost: 4309.13778 + ├── key: () + ├── fd: ()-->(3,7,9,11) + ├── distribution: ap-southeast-2 + ├── inner-join (lookup json_arr1_rbr [as=t1]) + │ ├── columns: t2.j:3 t1.k:7 t1.j:9 crdb_region:11 + │ ├── key columns: [22 18] = [11 7] + │ ├── lookup columns are key + │ ├── immutable + │ ├── stats: [rows=3333.333] + │ ├── cost: 4309.11778 + │ ├── fd: ()-->(11), (7)-->(9) + │ ├── limit hint: 1.00 + │ ├── distribution: ap-southeast-2 + │ ├── prune: (7) + │ ├── inner-join (inverted json_arr1_rbr@j_idx [as=t1]) + │ │ ├── columns: t2.j:3 t1.k:18 crdb_region:22 + │ │ ├── flags: force inverted join (into right side) + │ │ ├── prefix key columns: [17] = [22] + │ │ ├── inverted-expr + │ │ │ └── t1.j:20 @> t2.j:3 + │ │ ├── stats: [rows=3333.333, distinct(17)=1, null(17)=0, distinct(18)=964.524, null(18)=0, distinct(22)=1, null(22)=0] + │ │ ├── cost: 3837.19889 + │ │ ├── fd: ()-->(22) + │ │ ├── limit hint: 100.00 + │ │ ├── distribution: ap-southeast-2 + │ │ ├── project + │ │ │ ├── columns: "inverted_join_const_col_@11":17 t2.j:3 + │ │ │ ├── stats: [rows=1000, distinct(17)=1, null(17)=0] + │ │ │ ├── cost: 1136.62333 + │ │ │ ├── fd: ()-->(17) + │ │ │ ├── distribution: ap-southeast-2 + │ │ │ ├── scan json_arr2_rbt [as=t2] + │ │ │ │ ├── columns: t2.j:3 + │ │ │ │ ├── stats: [rows=1000] + │ │ │ │ ├── cost: 1116.60333 + │ │ │ │ ├── distribution: ap-southeast-2 + │ │ │ │ ├── prune: (3) + │ │ │ │ └── unfiltered-cols: (1-6) + │ │ │ └── projections + │ │ │ └── 'ap-southeast-2' [as="inverted_join_const_col_@11":17] + │ │ └── filters (true) + │ └── filters + │ └── t1.j:9 @> t2.j:3 [outer=(3,9), immutable] + └── 1 + +statement ok +RESET enforce_home_region + diff --git a/pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs/generated_test.go b/pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs/generated_test.go index 565c902c782d..f9b1b86f8464 100644 --- a/pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs/generated_test.go +++ b/pkg/ccl/logictestccl/tests/multiregion-9node-3region-3azs/generated_test.go @@ -136,6 +136,13 @@ func TestCCLLogic_multi_region_query_behavior( runCCLLogicTest(t, "multi_region_query_behavior") } +func TestCCLLogic_multi_region_remote_access_error( + t *testing.T, +) { + defer leaktest.AfterTest(t)() + runCCLLogicTest(t, "multi_region_remote_access_error") +} + func TestCCLLogic_multi_region_show( t *testing.T, ) { diff --git a/pkg/sql/copy.go b/pkg/sql/copy.go index 6c68436d2b11..b4e80d967652 100644 --- a/pkg/sql/copy.go +++ b/pkg/sql/copy.go @@ -429,9 +429,6 @@ func (c *copyMachine) processCopyData(ctx context.Context, data string, final bo } }() - // When this many rows are in the copy buffer, they are inserted. - const copyBatchRowSize = 100 - if len(data) > (c.buf.Cap() - c.buf.Len()) { // If it looks like the buffer will need to allocate to accommodate data, // account for the memory here. This is not particularly accurate - we don't diff --git a/pkg/sql/exec_util.go b/pkg/sql/exec_util.go index 565cde36463e..16dc9f35734d 100644 --- a/pkg/sql/exec_util.go +++ b/pkg/sql/exec_util.go @@ -3337,6 +3337,10 @@ func (m *sessionDataMutator) SetCopyFromAtomicEnabled(val bool) { m.data.CopyFromAtomicEnabled = val } +func (m *sessionDataMutator) SetEnforceHomeRegion(val bool) { + m.data.EnforceHomeRegion = val +} + // Utility functions related to scrubbing sensitive information on SQL Stats. // quantizeCounts ensures that the Count field in the diff --git a/pkg/sql/faketreeeval/BUILD.bazel b/pkg/sql/faketreeeval/BUILD.bazel index f4217bd519bc..77df33d5096d 100644 --- a/pkg/sql/faketreeeval/BUILD.bazel +++ b/pkg/sql/faketreeeval/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "//pkg/clusterversion", "//pkg/security/username", "//pkg/sql/catalog/catpb", + "//pkg/sql/catalog/descpb", "//pkg/sql/parser", "//pkg/sql/pgwire/pgcode", "//pkg/sql/pgwire/pgerror", diff --git a/pkg/sql/faketreeeval/evalctx.go b/pkg/sql/faketreeeval/evalctx.go index 59c60619af40..24bbdb5546a2 100644 --- a/pkg/sql/faketreeeval/evalctx.go +++ b/pkg/sql/faketreeeval/evalctx.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/clusterversion" "github.com/cockroachdb/cockroach/pkg/security/username" "github.com/cockroachdb/cockroach/pkg/sql/catalog/catpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/parser" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" @@ -124,21 +125,21 @@ var _ eval.RegionOperator = &DummyRegionOperator{} var errRegionOperator = unimplemented.NewWithIssue(42508, "cannot evaluate scalar expressions containing region operations in this context") -// CurrentDatabaseRegionConfig is part of the eval.DatabaseCatalog interface. +// CurrentDatabaseRegionConfig is part of the eval.RegionOperator interface. func (so *DummyRegionOperator) CurrentDatabaseRegionConfig( _ context.Context, ) (eval.DatabaseRegionConfig, error) { return nil, errors.WithStack(errRegionOperator) } -// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.DatabaseCatalog interface. +// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.RegionOperator interface. func (so *DummyRegionOperator) ValidateAllMultiRegionZoneConfigsInCurrentDatabase( _ context.Context, ) error { return errors.WithStack(errRegionOperator) } -// ResetMultiRegionZoneConfigsForTable is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForTable is part of the eval.RegionOperator // interface. func (so *DummyRegionOperator) ResetMultiRegionZoneConfigsForTable( _ context.Context, id int64, @@ -146,7 +147,7 @@ func (so *DummyRegionOperator) ResetMultiRegionZoneConfigsForTable( return errors.WithStack(errRegionOperator) } -// ResetMultiRegionZoneConfigsForDatabase is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForDatabase is part of the eval.RegionOperator // interface. func (so *DummyRegionOperator) ResetMultiRegionZoneConfigsForDatabase( _ context.Context, id int64, @@ -310,20 +311,20 @@ var _ eval.Planner = &DummyEvalPlanner{} var errEvalPlanner = pgerror.New(pgcode.ScalarOperationCannotRunWithoutFullSessionContext, "cannot evaluate scalar expressions using table lookups in this context") -// CurrentDatabaseRegionConfig is part of the eval.DatabaseCatalog interface. +// CurrentDatabaseRegionConfig is part of the eval.RegionOperator interface. func (ep *DummyEvalPlanner) CurrentDatabaseRegionConfig( _ context.Context, ) (eval.DatabaseRegionConfig, error) { return nil, errors.WithStack(errEvalPlanner) } -// ResetMultiRegionZoneConfigsForTable is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForTable is part of the eval.RegionOperator // interface. func (ep *DummyEvalPlanner) ResetMultiRegionZoneConfigsForTable(_ context.Context, _ int64) error { return errors.WithStack(errEvalPlanner) } -// ResetMultiRegionZoneConfigsForDatabase is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForDatabase is part of the eval.RegionOperator // interface. func (ep *DummyEvalPlanner) ResetMultiRegionZoneConfigsForDatabase( _ context.Context, _ int64, @@ -331,7 +332,7 @@ func (ep *DummyEvalPlanner) ResetMultiRegionZoneConfigsForDatabase( return errors.WithStack(errEvalPlanner) } -// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.DatabaseCatalog interface. +// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.RegionOperator interface. func (ep *DummyEvalPlanner) ValidateAllMultiRegionZoneConfigsInCurrentDatabase( _ context.Context, ) error { @@ -449,6 +450,11 @@ func (ep *DummyEvalPlanner) ResolveFunctionByOID( return "", nil, errors.AssertionFailedf("ResolveFunctionByOID unimplemented") } +// GetMultiregionConfig is part of the eval.Planner interface. +func (ep *DummyEvalPlanner) GetMultiregionConfig(databaseID descpb.ID) (interface{}, bool) { + return nil /* regionConfig */, false +} + // DummyPrivilegedAccessor implements the tree.PrivilegedAccessor interface by returning errors. type DummyPrivilegedAccessor struct{} diff --git a/pkg/sql/importer/import_table_creation.go b/pkg/sql/importer/import_table_creation.go index 66a1ce8bd825..7b78c53f4256 100644 --- a/pkg/sql/importer/import_table_creation.go +++ b/pkg/sql/importer/import_table_creation.go @@ -245,21 +245,21 @@ func (i importDatabaseRegionConfig) PrimaryRegionString() string { var _ eval.DatabaseRegionConfig = &importDatabaseRegionConfig{} -// CurrentDatabaseRegionConfig is part of the eval.DatabaseCatalog interface. +// CurrentDatabaseRegionConfig is part of the eval.RegionOperator interface. func (so *importRegionOperator) CurrentDatabaseRegionConfig( _ context.Context, ) (eval.DatabaseRegionConfig, error) { return importDatabaseRegionConfig{primaryRegion: so.primaryRegion}, nil } -// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.DatabaseCatalog interface. +// ValidateAllMultiRegionZoneConfigsInCurrentDatabase is part of the eval.RegionOperator interface. func (so *importRegionOperator) ValidateAllMultiRegionZoneConfigsInCurrentDatabase( _ context.Context, ) error { return errors.WithStack(errRegionOperator) } -// ResetMultiRegionZoneConfigsForTable is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForTable is part of the eval.RegionOperator // interface. func (so *importRegionOperator) ResetMultiRegionZoneConfigsForTable( _ context.Context, _ int64, @@ -267,7 +267,7 @@ func (so *importRegionOperator) ResetMultiRegionZoneConfigsForTable( return errors.WithStack(errRegionOperator) } -// ResetMultiRegionZoneConfigsForDatabase is part of the eval.DatabaseCatalog +// ResetMultiRegionZoneConfigsForDatabase is part of the eval.RegionOperator // interface. func (so *importRegionOperator) ResetMultiRegionZoneConfigsForDatabase( _ context.Context, _ int64, diff --git a/pkg/sql/internal.go b/pkg/sql/internal.go index 4091ea6f0980..902d54f31ffb 100644 --- a/pkg/sql/internal.go +++ b/pkg/sql/internal.go @@ -823,6 +823,9 @@ func (ie *InternalExecutor) execInternal( if ie.sessionDataStack != nil { // TODO(andrei): Properly clone (deep copy) ie.sessionData. sd = ie.sessionDataStack.Top().Clone() + // Even if session queries are told to error on non-home region accesses, + // internal queries spawned from the same context should never do so. + sd.LocalOnlySessionData.EnforceHomeRegion = false } else { sd = ie.s.newSessionData(SessionArgs{}) } diff --git a/pkg/sql/logictest/testdata/logic_test/information_schema b/pkg/sql/logictest/testdata/logic_test/information_schema index fcd2bfc567d6..9c80c9a9d422 100644 --- a/pkg/sql/logictest/testdata/logic_test/information_schema +++ b/pkg/sql/logictest/testdata/logic_test/information_schema @@ -4706,6 +4706,7 @@ enable_multiregion_placement_policy off enable_seqscan on enable_super_regions off enable_zigzag_join on +enforce_home_region off escape_string_warning on expect_and_ignore_not_visible_columns_in_copy off experimental_computed_column_rewrites · diff --git a/pkg/sql/logictest/testdata/logic_test/pg_catalog b/pkg/sql/logictest/testdata/logic_test/pg_catalog index fe49a2501e31..d9b37d378767 100644 --- a/pkg/sql/logictest/testdata/logic_test/pg_catalog +++ b/pkg/sql/logictest/testdata/logic_test/pg_catalog @@ -4189,6 +4189,7 @@ enable_multiregion_placement_policy off NULL enable_seqscan on NULL NULL NULL string enable_super_regions off NULL NULL NULL string enable_zigzag_join on NULL NULL NULL string +enforce_home_region off NULL NULL NULL string escape_string_warning on NULL NULL NULL string expect_and_ignore_not_visible_columns_in_copy off NULL NULL NULL string experimental_distsql_planning off NULL NULL NULL string @@ -4317,6 +4318,7 @@ enable_multiregion_placement_policy off NULL enable_seqscan on NULL user NULL on on enable_super_regions off NULL user NULL off off enable_zigzag_join on NULL user NULL on on +enforce_home_region off NULL user NULL off off escape_string_warning on NULL user NULL on on expect_and_ignore_not_visible_columns_in_copy off NULL user NULL off off experimental_distsql_planning off NULL user NULL off off @@ -4442,6 +4444,7 @@ enable_multiregion_placement_policy NULL NULL NULL enable_seqscan NULL NULL NULL NULL NULL enable_super_regions NULL NULL NULL NULL NULL enable_zigzag_join NULL NULL NULL NULL NULL +enforce_home_region NULL NULL NULL NULL NULL escape_string_warning NULL NULL NULL NULL NULL expect_and_ignore_not_visible_columns_in_copy NULL NULL NULL NULL NULL experimental_distsql_planning NULL NULL NULL NULL NULL diff --git a/pkg/sql/logictest/testdata/logic_test/prepare b/pkg/sql/logictest/testdata/logic_test/prepare index 51443a0b2261..663cc7f4de07 100644 --- a/pkg/sql/logictest/testdata/logic_test/prepare +++ b/pkg/sql/logictest/testdata/logic_test/prepare @@ -1183,14 +1183,14 @@ select ├── columns: k:1 str:2 ├── immutable ├── stats: [rows=333.3333] - ├── cost: 1074.45 + ├── cost: 1114.85 ├── key: (1) ├── fd: (1)-->(2) ├── prune: (2) ├── scan t2 │ ├── columns: k:1 str:2 │ ├── stats: [rows=1000] - │ ├── cost: 1064.42 + │ ├── cost: 1104.82 │ ├── key: (1) │ ├── fd: (1)-->(2) │ └── prune: (1,2) diff --git a/pkg/sql/logictest/testdata/logic_test/show_source b/pkg/sql/logictest/testdata/logic_test/show_source index 05a0e6e39efe..3d887e08c257 100644 --- a/pkg/sql/logictest/testdata/logic_test/show_source +++ b/pkg/sql/logictest/testdata/logic_test/show_source @@ -62,6 +62,7 @@ enable_multiregion_placement_policy off enable_seqscan on enable_super_regions off enable_zigzag_join on +enforce_home_region off escape_string_warning on expect_and_ignore_not_visible_columns_in_copy off experimental_distsql_planning off diff --git a/pkg/sql/opt/BUILD.bazel b/pkg/sql/opt/BUILD.bazel index af609658d610..9dbce7eac2a7 100644 --- a/pkg/sql/opt/BUILD.bazel +++ b/pkg/sql/opt/BUILD.bazel @@ -25,12 +25,16 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/server/telemetry", + "//pkg/sql/catalog/catpb", "//pkg/sql/catalog/colinfo", + "//pkg/sql/catalog/descpb", + "//pkg/sql/catalog/multiregion", "//pkg/sql/opt/cat", "//pkg/sql/opt/partition", "//pkg/sql/pgwire/pgcode", "//pkg/sql/pgwire/pgerror", "//pkg/sql/privilege", + "//pkg/sql/sem/eval", "//pkg/sql/sem/tree", "//pkg/sql/sem/tree/treebin", "//pkg/sql/sem/tree/treecmp", diff --git a/pkg/sql/opt/cat/table.go b/pkg/sql/opt/cat/table.go index ef24aa2ef8d7..5b8841c9d642 100644 --- a/pkg/sql/opt/cat/table.go +++ b/pkg/sql/opt/cat/table.go @@ -13,6 +13,7 @@ package cat import ( "time" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/types" ) @@ -149,6 +150,28 @@ type Table interface { // created with the NO DATA option and is yet to be refreshed. Accessing // such a view prior to running refresh returns an error. IsRefreshViewRequired() bool + + // HomeRegion returns the home region of the table, if any, for example if + // a table is defined with LOCALITY REGIONAL BY TABLE. + HomeRegion() (region string, ok bool) + + // IsGlobalTable returns true if the table is defined with LOCALITY GLOBAL. + IsGlobalTable() bool + + // IsRegionalByRow returns true if the table is defined with LOCALITY REGIONAL + // BY ROW. + IsRegionalByRow() bool + + // HomeRegionColName returns the name of the crdb_internal_region column name + // specifying the home region of each row in the table, if this table is a + // REGIONAL BY ROW TABLE, otherwise "", false is returned. + // This column is name `crdb_region` by default, but may be overridden with a + // different name in a `REGIONAL BY ROW AS` DDL clause. + HomeRegionColName() (colName string, ok bool) + + // GetDatabaseID returns the owning database id of the table, or zero, if the + // owning database could not be determined. + GetDatabaseID() descpb.ID } // CheckConstraint contains the SQL text and the validity status for a check diff --git a/pkg/sql/opt/constraint/constraint.go b/pkg/sql/opt/constraint/constraint.go index e42d52a980a4..9440018ad444 100644 --- a/pkg/sql/opt/constraint/constraint.go +++ b/pkg/sql/opt/constraint/constraint.go @@ -830,3 +830,27 @@ func (c *Constraint) CollectFirstColumnValues( } return values, hasNullValue, true } + +// HasRemoteSpans returns true if any of the constraint spans in `c` belong to +// partitions remote to the gateway region. +func (c *Constraint) HasRemoteSpans(ps partition.PrefixSorter) bool { + if c.IsUnconstrained() { + return true + } + if c.IsContradiction() { + return false + } + // Iterate through the spans and determine whether each one matches + // with a prefix from a remote partition. + for i, n := 0, c.Spans.Count(); i < n; i++ { + span := c.Spans.Get(i) + if match, ok := FindMatch(span, ps); ok { + if !match.IsLocal { + return true + } + } else { + return true + } + } + return false +} diff --git a/pkg/sql/opt/distribution/BUILD.bazel b/pkg/sql/opt/distribution/BUILD.bazel index 627d56b959d5..753c98ec26e5 100644 --- a/pkg/sql/opt/distribution/BUILD.bazel +++ b/pkg/sql/opt/distribution/BUILD.bazel @@ -7,9 +7,11 @@ go_library( importpath = "github.com/cockroachdb/cockroach/pkg/sql/opt/distribution", visibility = ["//visibility:public"], deps = [ + "//pkg/sql/opt", "//pkg/sql/opt/memo", "//pkg/sql/opt/props/physical", "//pkg/sql/sem/eval", + "//pkg/sql/sem/tree", "//pkg/util/buildutil", "@com_github_cockroachdb_errors//:errors", ], diff --git a/pkg/sql/opt/distribution/distribution.go b/pkg/sql/opt/distribution/distribution.go index 4edb00e308b1..13ff52608763 100644 --- a/pkg/sql/opt/distribution/distribution.go +++ b/pkg/sql/opt/distribution/distribution.go @@ -11,9 +11,11 @@ package distribution import ( + "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/opt/props/physical" "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" + "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/util/buildutil" "github.com/cockroachdb/errors" ) @@ -37,14 +39,17 @@ func CanProvide(evalCtx *eval.Context, expr memo.RelExpr, required *physical.Dis provided.FromLocality(evalCtx.Locality) case *memo.ScanExpr: - md := expr.Memo().Metadata() - index := md.Table(t.Table).Index(t.Index) - provided.FromIndexScan(evalCtx, index, t.Constraint) + tabMeta := t.Memo().Metadata().TableMeta(t.Table) + provided.FromIndexScan(evalCtx, tabMeta, t.Index, t.Constraint) + + case *memo.LookupJoinExpr: + if t.LocalityOptimized { + provided.FromLocality(evalCtx.Locality) + } default: // Other operators can pass through the distribution to their children. } - return provided.Any() || provided.Equals(*required) } @@ -92,14 +97,24 @@ func BuildProvided( provided.FromLocality(evalCtx.Locality) case *memo.ScanExpr: - md := expr.Memo().Metadata() - index := md.Table(t.Table).Index(t.Index) - provided.FromIndexScan(evalCtx, index, t.Constraint) + tabMeta := t.Memo().Metadata().TableMeta(t.Table) + provided.FromIndexScan(evalCtx, tabMeta, t.Index, t.Constraint) default: + // TODO(msirek): Clarify the distinction between a distribution which can + // provide any required distribution and one which can provide + // none. See issue #86641. + if lookupJoinExpr, ok := expr.(*memo.LookupJoinExpr); ok { + if lookupJoinExpr.LocalityOptimized { + // Locality-optimized join is considered to be local. + provided.FromLocality(evalCtx.Locality) + break + } + } for i, n := 0, expr.ChildCount(); i < n; i++ { if relExpr, ok := expr.Child(i).(memo.RelExpr); ok { - provided = provided.Union(relExpr.ProvidedPhysical().Distribution) + childDistribution := relExpr.ProvidedPhysical().Distribution + provided = provided.Union(childDistribution) } } } @@ -111,6 +126,99 @@ func BuildProvided( return provided } +// GetDEnumAsStringFromConstantExpr returns the string representation of a DEnum +// ConstantExpr, if expr is such a ConstantExpr. +func GetDEnumAsStringFromConstantExpr(expr opt.Expr) (enumAsString string, ok bool) { + if constExpr, ok := expr.(*memo.ConstExpr); ok { + if enumValue, ok := constExpr.Value.(*tree.DEnum); ok { + return enumValue.LogicalRep, true + } + } + return "", false +} + +// BuildLookupJoinLookupTableDistribution builds the Distribution that results +// from performing lookups of a LookupJoin, if that distribution can be +// statically determined. +func BuildLookupJoinLookupTableDistribution( + evalCtx *eval.Context, lookupJoin *memo.LookupJoinExpr, +) (provided physical.Distribution) { + lookupTableMeta := lookupJoin.Memo().Metadata().TableMeta(lookupJoin.Table) + lookupTable := lookupTableMeta.Table + + if lookupJoin.LocalityOptimized { + provided.FromLocality(evalCtx.Locality) + return provided + } else if lookupTable.IsGlobalTable() { + provided.FromLocality(evalCtx.Locality) + return provided + } else if homeRegion, ok := lookupTable.HomeRegion(); ok { + provided.Regions = []string{homeRegion} + return provided + } else if lookupTable.IsRegionalByRow() { + if len(lookupJoin.KeyCols) > 0 { + inputExpr := lookupJoin.Input + firstKeyColID := lookupJoin.LookupJoinPrivate.KeyCols[0] + if invertedJoinExpr, ok := inputExpr.(*memo.InvertedJoinExpr); ok { + if filterExpr, ok := invertedJoinExpr.GetConstExprFromFilter(firstKeyColID); ok { + if homeRegion, ok = GetDEnumAsStringFromConstantExpr(filterExpr); ok { + provided.Regions = []string{homeRegion} + return provided + } + } + } else if projectExpr, ok := inputExpr.(*memo.ProjectExpr); ok { + regionName := projectExpr.GetProjectedEnumConstant(firstKeyColID) + if regionName != "" { + provided.Regions = []string{regionName} + return provided + } + } + } else if lookupJoin.LookupJoinPrivate.LookupColsAreTableKey && + len(lookupJoin.LookupJoinPrivate.LookupExpr) > 0 { + if filterIdx, ok := lookupJoin.GetConstPrefixFilter(lookupJoin.Memo().Metadata()); ok { + firstIndexColEqExpr := lookupJoin.LookupJoinPrivate.LookupExpr[filterIdx].Condition + if firstIndexColEqExpr.Op() == opt.EqOp { + if regionName, ok := GetDEnumAsStringFromConstantExpr(firstIndexColEqExpr.Child(1)); ok { + provided.Regions = []string{regionName} + return provided + } + } + } + } + } + provided.FromIndexScan(evalCtx, lookupTableMeta, lookupJoin.Index, nil) + return provided +} + +// BuildInvertedJoinLookupTableDistribution builds the Distribution that results +// from performing lookups of an inverted join, if that distribution can be +// statically determined. +func BuildInvertedJoinLookupTableDistribution( + evalCtx *eval.Context, invertedJoin *memo.InvertedJoinExpr, +) (provided physical.Distribution) { + lookupTableMeta := invertedJoin.Memo().Metadata().TableMeta(invertedJoin.Table) + lookupTable := lookupTableMeta.Table + + if lookupTable.IsGlobalTable() { + provided.FromLocality(evalCtx.Locality) + return provided + } else if homeRegion, ok := lookupTable.HomeRegion(); ok { + provided.Regions = []string{homeRegion} + return provided + } else if lookupTable.IsRegionalByRow() { + if len(invertedJoin.PrefixKeyCols) > 0 { + if projectExpr, ok := invertedJoin.Input.(*memo.ProjectExpr); ok { + colID := invertedJoin.PrefixKeyCols[0] + homeRegion = projectExpr.GetProjectedEnumConstant(colID) + provided.Regions = []string{homeRegion} + return provided + } + } + } + provided.FromIndexScan(evalCtx, lookupTableMeta, invertedJoin.Index, nil) + return provided +} + func checkRequired(required *physical.Distribution) { // There should be exactly one region in the required distribution (for now, // assuming this is coming from the gateway). diff --git a/pkg/sql/opt/exec/execbuilder/BUILD.bazel b/pkg/sql/opt/exec/execbuilder/BUILD.bazel index b524d65e36a7..68acb72b2610 100644 --- a/pkg/sql/opt/exec/execbuilder/BUILD.bazel +++ b/pkg/sql/opt/exec/execbuilder/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "//pkg/sql/opt", "//pkg/sql/opt/cat", "//pkg/sql/opt/constraint", + "//pkg/sql/opt/distribution", "//pkg/sql/opt/exec", "//pkg/sql/opt/exec/explain", "//pkg/sql/opt/memo", diff --git a/pkg/sql/opt/exec/execbuilder/builder.go b/pkg/sql/opt/exec/execbuilder/builder.go index 4de0e49edf1c..8f52c284a7a9 100644 --- a/pkg/sql/opt/exec/execbuilder/builder.go +++ b/pkg/sql/opt/exec/execbuilder/builder.go @@ -157,6 +157,14 @@ type Builder struct { // The default can be overridden by calling SetBuiltinFuncWrapper method to provide // custom search path implementation. wrapFunctionOverride func(fnName string) tree.ResolvableFunctionReference + + // builtScans collects all scans in the operation tree so post-build checking + // for non-local execution can be done. + builtScans []*memo.ScanExpr + + // doScanExprCollection, when true, causes buildScan to add any ScanExprs it + // processes to the builtScans slice. + doScanExprCollection bool } // New constructs an instance of the execution node builder using the @@ -329,8 +337,7 @@ func (b *Builder) findBuiltWithExpr(id opt.WithID) *builtWithExpr { // boundedStaleness returns true if this query uses bounded staleness. func (b *Builder) boundedStaleness() bool { - return b.evalCtx != nil && b.evalCtx.AsOfSystemTime != nil && - b.evalCtx.AsOfSystemTime.BoundedStaleness + return b.evalCtx != nil && b.evalCtx.BoundedStaleness() } // mdVarContainer is an IndexedVarContainer implementation used by BuildScalar - diff --git a/pkg/sql/opt/exec/execbuilder/relational.go b/pkg/sql/opt/exec/execbuilder/relational.go index a76b26be3235..0e60d3cfff71 100644 --- a/pkg/sql/opt/exec/execbuilder/relational.go +++ b/pkg/sql/opt/exec/execbuilder/relational.go @@ -14,6 +14,7 @@ import ( "bytes" "context" "fmt" + "strings" "github.com/cockroachdb/cockroach/pkg/server/telemetry" "github.com/cockroachdb/cockroach/pkg/sql/catalog/colinfo" @@ -21,6 +22,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/opt/constraint" + "github.com/cockroachdb/cockroach/pkg/sql/opt/distribution" "github.com/cockroachdb/cockroach/pkg/sql/opt/exec" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/opt/norm" @@ -757,6 +759,14 @@ func (b *Builder) buildScan(scan *memo.ScanExpr) (execPlan, error) { } res.root = root + if b.evalCtx.SessionData().EnforceHomeRegion && b.doScanExprCollection { + if b.builtScans == nil { + // Make this large enough to handle simple 2-table join queries without + // wasting memory. + b.builtScans = make([]*memo.ScanExpr, 0, 2) + } + b.builtScans = append(b.builtScans, scan) + } return res, nil } @@ -1733,21 +1743,101 @@ func (b *Builder) buildSort(sort *memo.SortExpr) (execPlan, error) { return execPlan{root: node, outputCols: input.outputCols}, nil } -func (b *Builder) buildDistribute(distribute *memo.DistributeExpr) (execPlan, error) { - input, err := b.buildRelational(distribute.Input) +func (b *Builder) enforceScanWithHomeRegion() error { + homeRegion := "" + firstTable := "" + gatewayRegion, foundLocalRegion := b.evalCtx.Locality.Find("region") + if !foundLocalRegion { + return errors.AssertionFailedf("The gateway region could not be determined while enforcing query home region.") + } + for i, scan := range b.builtScans { + inputTableMeta := scan.Memo().Metadata().TableMeta(scan.Table) + inputTable := inputTableMeta.Table + inputTableName := string(inputTable.Name()) + inputIndexOrdinal := scan.Index + + queryHomeRegion, queryHasHomeRegion := scan.ProvidedPhysical().Distribution.GetSingleRegion() + if homeRegion == "" { + homeRegion = queryHomeRegion + firstTable = inputTableName + } + + if queryHasHomeRegion { + if homeRegion != queryHomeRegion { + return pgerror.Newf(pgcode.QueryHasNoHomeRegion, + `Query has no home region. The home region ('%s') of table '%s' does not match the home region ('%s') of lookup table '%s'.`, + queryHomeRegion, + inputTableName, + homeRegion, + firstTable) + } else if gatewayRegion != homeRegion { + return pgerror.Newf(pgcode.QueryNotRunningInHomeRegion, + `Query is not running in its home region. Try running the query from region '%s'.`, + homeRegion, + ) + } + } else { + var inputTableMeta2 *opt.TableMeta + var inputIndexOrdinal2 cat.IndexOrdinal + if len(b.builtScans) > 1 && i == 0 { + scan2 := b.builtScans[1] + inputTableMeta2 = scan2.Memo().Metadata().TableMeta(scan2.Table) + inputIndexOrdinal2 = scan2.Index + } + return b.filterSuggestionError(inputTableMeta, inputIndexOrdinal, inputTableMeta2, inputIndexOrdinal2) + } + } + b.builtScans = nil + return nil +} + +func (b *Builder) buildDistribute(distribute *memo.DistributeExpr) (input execPlan, err error) { + input, err = b.buildRelational(distribute.Input) if err != nil { return execPlan{}, err } - distribution := distribute.ProvidedPhysical().Distribution - inputDistribution := distribute.Input.ProvidedPhysical().Distribution - if distribution.Equals(inputDistribution) { + if distribute.NoOpDistribution() { // Don't bother creating a no-op distribution. This likely exists because // the input is a Sort expression, and this is an artifact of how physical // properties are enforced. return input, err } + if b.evalCtx.SessionData().EnforceHomeRegion { + saveDoScanExprCollection := b.doScanExprCollection + b.doScanExprCollection = true + // Traverse the tree again, this time collecting ScanExprs that should + // be processed for error handling. + _, err = b.buildRelational(distribute.Input) + b.doScanExprCollection = saveDoScanExprCollection + if err != nil { + return execPlan{}, err + } + err = b.enforceScanWithHomeRegion() + if err != nil { + return execPlan{}, err + } + + homeRegion, ok := distribute.GetInputHomeRegion() + var errorStringBuilder strings.Builder + var errCode pgcode.Code + if ok { + errCode = pgcode.QueryNotRunningInHomeRegion + errorStringBuilder.WriteString("Query is not running in its home region.") + errorStringBuilder.WriteString(fmt.Sprintf(` Try running the query from region '%s'.`, homeRegion)) + } else if distribute.Input.Op() != opt.LookupJoinOp { + // More detailed error message handling for lookup join occurs in the + // execbuilder. + errCode = pgcode.QueryHasNoHomeRegion + errorStringBuilder.WriteString("Query has no home region.") + errorStringBuilder.WriteString(` Try adding a LIMIT clause.`) + } + msgString := errorStringBuilder.String() + err = pgerror.Newf(errCode, "%s", msgString) + return execPlan{}, err + } + // TODO(rytaft): This is currently a no-op. We should pass this distribution // info to the DistSQL planner. return input, err @@ -1811,9 +1901,181 @@ func (b *Builder) buildIndexJoin(join *memo.IndexJoinExpr) (execPlan, error) { return res, nil } +func (b *Builder) indexColumnNames( + tableMeta *opt.TableMeta, index cat.Index, startColumn int, +) string { + if startColumn < 0 { + return "" + } + sb := strings.Builder{} + for i, n := startColumn, index.ExplicitColumnCount(); i < n; i++ { + if i > startColumn { + sb.WriteString(", ") + } + col := index.Column(i) + ord := col.Ordinal() + colID := tableMeta.MetaID.ColumnID(ord) + colName := b.mem.Metadata().QualifiedAlias(colID, false /* fullyQualify */, true /* alwaysQualify */, b.catalog) + sb.WriteString(colName) + } + return sb.String() +} + +func (b *Builder) filterSuggestionError( + tableMeta *opt.TableMeta, + indexOrdinal cat.IndexOrdinal, + table2Meta *opt.TableMeta, + indexOrdinal2 cat.IndexOrdinal, +) (err error) { + var index cat.Index + var table2 cat.Table + if table2Meta != nil { + table2 = table2Meta.Table + } + if tableMeta != nil { + table := tableMeta.Table + index = table.Index(indexOrdinal) + + if crdbRegionColName, ok := table.HomeRegionColName(); ok { + plural := "" + if index.ExplicitColumnCount() > 2 { + plural = "s" + } + tableName := tableMeta.Alias.Table() + args := make([]interface{}, 0, 8) + args = append(args, tableName) + args = append(args, crdbRegionColName) + args = append(args, plural) + args = append(args, b.indexColumnNames(tableMeta, index, 1)) + if table2 == nil { + err = pgerror.Newf(pgcode.QueryHasNoHomeRegion, + "Query has no home region. Try adding a filter on %s.%s and/or on key column%s (%s).", args...) + } else if crdbRegionColName2, ok := table2.HomeRegionColName(); ok { + index = table2.Index(indexOrdinal2) + plural = "" + if index.ExplicitColumnCount() > 2 { + plural = "s" + } + table2Name := table2Meta.Alias.Table() + args = append(args, table2Name) + args = append(args, crdbRegionColName2) + args = append(args, plural) + args = append(args, b.indexColumnNames(table2Meta, index, 1)) + err = pgerror.Newf(pgcode.QueryHasNoHomeRegion, + "Query has no home region. Try adding a filter on %s.%s and/or on key column%s (%s). Try adding a filter on %s.%s and/or on key column%s (%s).", args...) + } + } + } + return err +} + +func (b *Builder) handleRemoteLookupJoinError(join *memo.LookupJoinExpr) (err error) { + if join.LocalityOptimized { + // Locality optimized joins are considered local in phase 1. + return nil + } + lookupTableMeta := join.Memo().Metadata().TableMeta(join.Table) + lookupTable := lookupTableMeta.Table + + var input opt.Expr + input = join.Input + for input.ChildCount() == 1 || input.Op() == opt.ProjectOp { + input = input.Child(0) + } + gatewayRegion, foundLocalRegion := b.evalCtx.Locality.Find("region") + inputTableName := "" + // ScanExprs from global tables will have filled in a provided distribution + // of the gateway region by now. + queryHomeRegion, queryHasHomeRegion := input.(memo.RelExpr).ProvidedPhysical().Distribution.GetSingleRegion() + var inputTableMeta *opt.TableMeta + var inputTable cat.Table + var inputIndexOrdinal cat.IndexOrdinal + switch t := input.(type) { + case *memo.ScanExpr: + inputTableMeta = join.Memo().Metadata().TableMeta(t.Table) + inputTable = inputTableMeta.Table + inputTableName = string(inputTable.Name()) + inputIndexOrdinal = t.Index + } + + homeRegion := "" + if lookupTable.IsGlobalTable() { + // HomeRegion() does not automatically fill in the home region of a global + // table as the gateway region, so let's manually set it here. + homeRegion = gatewayRegion + } else { + homeRegion, _ = lookupTable.HomeRegion() + } + if homeRegion == "" { + if lookupTable.IsRegionalByRow() { + if len(join.KeyCols) > 0 { + inputExpr := join.Input + firstKeyColID := join.KeyCols[0] + if invertedJoinExpr, ok := inputExpr.(*memo.InvertedJoinExpr); ok { + if constExpr, ok := invertedJoinExpr.GetConstExprFromFilter(firstKeyColID); ok { + if regionName, ok := distribution.GetDEnumAsStringFromConstantExpr(constExpr); ok { + homeRegion = regionName + } + } + } else if projectExpr, ok := inputExpr.(*memo.ProjectExpr); ok { + homeRegion = projectExpr.GetProjectedEnumConstant(firstKeyColID) + } + } else if join.LookupColsAreTableKey && + len(join.LookupExpr) > 0 { + if filterIdx, ok := join.GetConstPrefixFilter(join.Memo().Metadata()); ok { + firstIndexColEqExpr := join.LookupJoinPrivate.LookupExpr[filterIdx].Condition + if firstIndexColEqExpr.Op() == opt.EqOp { + if regionName, ok := distribution.GetDEnumAsStringFromConstantExpr(firstIndexColEqExpr.Child(1)); ok { + homeRegion = regionName + } + } + } + } + } + } + + if homeRegion != "" { + if foundLocalRegion { + if queryHasHomeRegion { + if homeRegion != queryHomeRegion { + if inputTableName == "" { + return pgerror.Newf(pgcode.QueryHasNoHomeRegion, + `Query has no home region. The home region ('%s') of lookup table '%s' does not match the home region ('%s') of the other relation in the join.'`, + homeRegion, + lookupTable.Name(), + queryHomeRegion, + ) + } + return pgerror.Newf(pgcode.QueryHasNoHomeRegion, + `Query has no home region. The home region ('%s') of table '%s' does not match the home region ('%s') of lookup table '%s'.`, + queryHomeRegion, + inputTableName, + homeRegion, + string(lookupTable.Name()), + ) + } else if gatewayRegion != homeRegion { + return pgerror.Newf(pgcode.QueryNotRunningInHomeRegion, + `Query is not running in its home region. Try running the query from region '%s'.`, + homeRegion, + ) + } + } else { + return b.filterSuggestionError(inputTableMeta, inputIndexOrdinal, nil /* table2Meta */, 0 /* indexOrdinal2 */) + } + } else { + return errors.AssertionFailedf("The gateway region could not be determined while enforcing query home region.") + } + } else { + if !queryHasHomeRegion { + return b.filterSuggestionError(lookupTableMeta, join.Index, inputTableMeta, inputIndexOrdinal) + } + return b.filterSuggestionError(lookupTableMeta, join.Index, nil /* table2Meta */, 0 /* indexOrdinal2 */) + } + return nil +} + func (b *Builder) buildLookupJoin(join *memo.LookupJoinExpr) (execPlan, error) { md := b.mem.Metadata() - if !b.disableTelemetry { telemetry.Inc(sqltelemetry.JoinAlgoLookupUseCounter) telemetry.Inc(opt.JoinTypeToUseCounter(join.JoinType)) @@ -1822,12 +2084,33 @@ func (b *Builder) buildLookupJoin(join *memo.LookupJoinExpr) (execPlan, error) { telemetry.Inc(sqltelemetry.PartialIndexLookupJoinUseCounter) } } - + saveDoScanExprCollection := false + enforceHomeRegion := b.evalCtx.SessionData().EnforceHomeRegion + if enforceHomeRegion { + saveDoScanExprCollection = b.doScanExprCollection + var rel opt.Expr + rel = join.Input + for rel.ChildCount() == 1 || rel.Op() == opt.ProjectOp { + rel = rel.Child(0) + } + if rel.Op() == opt.ScanOp { + b.doScanExprCollection = false + } + } input, err := b.buildRelational(join.Input) if err != nil { return execPlan{}, err } - + if enforceHomeRegion { + b.doScanExprCollection = saveDoScanExprCollection + // TODO(msirek): Update this in phase 2 or 3 when we can dynamically + // determine if the lookup will be local, or after #69617 is + // merged. + err = b.handleRemoteLookupJoinError(join) + if err != nil { + return execPlan{}, err + } + } keyCols := make([]exec.NodeColumnOrdinal, len(join.KeyCols)) for i, c := range join.KeyCols { keyCols[i] = input.getNodeColumnOrdinal(c) @@ -1935,12 +2218,119 @@ func (b *Builder) buildLookupJoin(join *memo.LookupJoinExpr) (execPlan, error) { return res, nil } +func (b *Builder) handleRemoteInvertedJoinError(join *memo.InvertedJoinExpr) (err error) { + lookupTableMeta := join.Memo().Metadata().TableMeta(join.Table) + lookupTable := lookupTableMeta.Table + + var input opt.Expr + input = join.Input + for input.ChildCount() == 1 || input.Op() == opt.ProjectOp { + input = input.Child(0) + } + gatewayRegion, foundLocalRegion := b.evalCtx.Locality.Find("region") + inputTableName := "" + // ScanExprs from global tables will have filled in a provided distribution + // of the gateway region by now. + queryHomeRegion, queryHasHomeRegion := input.(memo.RelExpr).ProvidedPhysical().Distribution.GetSingleRegion() + var inputTableMeta *opt.TableMeta + var inputTable cat.Table + var inputIndexOrdinal cat.IndexOrdinal + switch t := input.(type) { + case *memo.ScanExpr: + inputTableMeta = join.Memo().Metadata().TableMeta(t.Table) + inputTable = inputTableMeta.Table + inputTableName = string(inputTable.Name()) + inputIndexOrdinal = t.Index + } + + homeRegion := "" + if lookupTable.IsGlobalTable() { + // HomeRegion() does not automatically fill in the home region of a global + // table as the gateway region, so let's manually set it here. + homeRegion = gatewayRegion + } else { + homeRegion, _ = lookupTable.HomeRegion() + } + if homeRegion == "" { + if lookupTable.IsRegionalByRow() { + if len(join.PrefixKeyCols) > 0 { + if projectExpr, ok := join.Input.(*memo.ProjectExpr); ok { + colID := join.PrefixKeyCols[0] + homeRegion = projectExpr.GetProjectedEnumConstant(colID) + } + } + } + } + + if homeRegion != "" { + if foundLocalRegion { + if queryHasHomeRegion { + if homeRegion != queryHomeRegion { + if inputTableName == "" { + return pgerror.Newf(pgcode.QueryHasNoHomeRegion, + `Query has no home region. The home region ('%s') of lookup table '%s' does not match the home region ('%s') of the other relation in the join.'`, + homeRegion, + lookupTable.Name(), + queryHomeRegion, + ) + } + return pgerror.Newf(pgcode.QueryHasNoHomeRegion, + `Query has no home region. The home region ('%s') of table '%s' does not match the home region ('%s') of lookup table '%s'.`, + queryHomeRegion, + inputTableName, + homeRegion, + string(lookupTable.Name()), + ) + } else if gatewayRegion != homeRegion { + return pgerror.Newf(pgcode.QueryNotRunningInHomeRegion, + `Query is not running in its home region. Try running the query from region '%s'.`, + homeRegion, + ) + } + } else { + return b.filterSuggestionError(inputTableMeta, inputIndexOrdinal, nil /* table2Meta */, 0 /* indexOrdinal2 */) + } + } else { + return errors.AssertionFailedf("The gateway region could not be determined while enforcing query home region.") + } + } else { + if !queryHasHomeRegion { + return b.filterSuggestionError(lookupTableMeta, join.Index, inputTableMeta, inputIndexOrdinal) + } + return b.filterSuggestionError(lookupTableMeta, join.Index, nil /* table2Meta */, 0 /* indexOrdinal2 */) + } + return nil +} + func (b *Builder) buildInvertedJoin(join *memo.InvertedJoinExpr) (execPlan, error) { + enforceHomeRegion := b.evalCtx.SessionData().EnforceHomeRegion + saveDoScanExprCollection := false + if enforceHomeRegion { + saveDoScanExprCollection = b.doScanExprCollection + var rel opt.Expr + rel = join.Input + for rel.ChildCount() == 1 || rel.Op() == opt.ProjectOp { + rel = rel.Child(0) + } + if rel.Op() == opt.ScanOp { + b.doScanExprCollection = false + } + } input, err := b.buildRelational(join.Input) if err != nil { return execPlan{}, err } + if enforceHomeRegion { + b.doScanExprCollection = saveDoScanExprCollection + // TODO(msirek): Update this in phase 2 or 3 when we can dynamically + // determine if the lookup will be local, or after #69617 is + // merged. + err = b.handleRemoteInvertedJoinError(join) + if err != nil { + return execPlan{}, err + } + } md := b.mem.Metadata() tab := md.Table(join.Table) idx := tab.Index(join.Index) diff --git a/pkg/sql/opt/exec/execbuilder/testdata/explain b/pkg/sql/opt/exec/execbuilder/testdata/explain index ea78684d8b77..49456148f722 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/explain +++ b/pkg/sql/opt/exec/execbuilder/testdata/explain @@ -1290,58 +1290,77 @@ values query T EXPLAIN (OPT) SELECT * FROM tc WHERE a = 10 ORDER BY b ---- -sort - └── index-join tc - └── scan tc@c - └── constraint: /1/3: [/10 - /10] +distribute + └── sort + └── index-join tc + └── scan tc@c + └── constraint: /1/3: [/10 - /10] query T EXPLAIN (OPT,VERBOSE) SELECT * FROM tc WHERE a = 10 ORDER BY b ---- -sort +distribute ├── columns: a:1 b:2 ├── stats: [rows=10, distinct(1)=1, null(1)=0] - ├── cost: 86.8243864 + ├── cost: 86.8543864 ├── fd: ()-->(1) ├── ordering: +2 opt(1) [actual: +2] + ├── distribution: test + ├── input distribution: ├── prune: (2) - └── index-join tc + └── sort ├── columns: a:1 b:2 ├── stats: [rows=10, distinct(1)=1, null(1)=0] - ├── cost: 85.7400007 + ├── cost: 86.8243864 ├── fd: ()-->(1) + ├── ordering: +2 opt(1) [actual: +2] ├── prune: (2) - └── scan tc@c - ├── columns: a:1 rowid:3 - ├── constraint: /1/3: [/10 - /10] + └── index-join tc + ├── columns: a:1 b:2 ├── stats: [rows=10, distinct(1)=1, null(1)=0] - ├── cost: 24.8200001 - ├── key: (3) - └── fd: ()-->(1) + ├── cost: 85.7400007 + ├── fd: ()-->(1) + ├── prune: (2) + └── scan tc@c + ├── columns: a:1 rowid:3 + ├── constraint: /1/3: [/10 - /10] + ├── stats: [rows=10, distinct(1)=1, null(1)=0] + ├── cost: 24.8200001 + ├── key: (3) + └── fd: ()-->(1) query T EXPLAIN (OPT,TYPES) SELECT * FROM tc WHERE a = 10 ORDER BY b ---- -sort +distribute ├── columns: a:1(int!null) b:2(int) ├── stats: [rows=10, distinct(1)=1, null(1)=0] - ├── cost: 86.8243864 + ├── cost: 86.8543864 ├── fd: ()-->(1) ├── ordering: +2 opt(1) [actual: +2] + ├── distribution: test + ├── input distribution: ├── prune: (2) - └── index-join tc + └── sort ├── columns: a:1(int!null) b:2(int) ├── stats: [rows=10, distinct(1)=1, null(1)=0] - ├── cost: 85.7400007 + ├── cost: 86.8243864 ├── fd: ()-->(1) + ├── ordering: +2 opt(1) [actual: +2] ├── prune: (2) - └── scan tc@c - ├── columns: a:1(int!null) rowid:3(int!null) - ├── constraint: /1/3: [/10 - /10] + └── index-join tc + ├── columns: a:1(int!null) b:2(int) ├── stats: [rows=10, distinct(1)=1, null(1)=0] - ├── cost: 24.8200001 - ├── key: (3) - └── fd: ()-->(1) + ├── cost: 85.7400007 + ├── fd: ()-->(1) + ├── prune: (2) + └── scan tc@c + ├── columns: a:1(int!null) rowid:3(int!null) + ├── constraint: /1/3: [/10 - /10] + ├── stats: [rows=10, distinct(1)=1, null(1)=0] + ├── cost: 24.8200001 + ├── key: (3) + └── fd: ()-->(1) query T EXPLAIN (OPT,CATALOG) SELECT * FROM tc WHERE a = 10 ORDER BY b @@ -1357,10 +1376,11 @@ TABLE tc └── INDEX c ├── a int └── rowid int not null default (unique_rowid()) [hidden] -sort - └── index-join tc - └── scan tc@c - └── constraint: /1/3: [/10 - /10] +distribute + └── sort + └── index-join tc + └── scan tc@c + └── constraint: /1/3: [/10 - /10] query T EXPLAIN (OPT,VERBOSE,CATALOG) SELECT * FROM tc JOIN t ON k=a @@ -1415,96 +1435,119 @@ inner-join (hash) query T EXPLAIN (OPT) SELECT * FROM tc WHERE a + 2 * b > 1 ORDER BY a*b ---- -sort - └── project - ├── select - │ ├── scan tc - │ └── filters - │ └── (a + (b * 2)) > 1 - └── projections - └── a * b +distribute + └── sort + └── project + ├── select + │ ├── scan tc + │ └── filters + │ └── (a + (b * 2)) > 1 + └── projections + └── a * b query T EXPLAIN (OPT, VERBOSE) SELECT * FROM tc WHERE a + 2 * b > 1 ORDER BY a*b ---- -sort +distribute ├── columns: a:1 b:2 [hidden: column6:6] ├── immutable ├── stats: [rows=333.3333] - ├── cost: 1214.30951 + ├── cost: 1214.33951 ├── fd: (1,2)-->(6) ├── ordering: +6 + ├── distribution: test + ├── input distribution: ├── prune: (1,2,6) ├── interesting orderings: (+1) - └── project - ├── columns: column6:6 a:1 b:2 + └── sort + ├── columns: a:1 b:2 column6:6 ├── immutable ├── stats: [rows=333.3333] - ├── cost: 1141.73667 + ├── cost: 1214.30951 ├── fd: (1,2)-->(6) + ├── ordering: +6 ├── prune: (1,2,6) ├── interesting orderings: (+1) - ├── select - │ ├── columns: a:1 b:2 - │ ├── immutable - │ ├── stats: [rows=333.3333] - │ ├── cost: 1135.05 - │ ├── interesting orderings: (+1) - │ ├── scan tc - │ │ ├── columns: a:1 b:2 - │ │ ├── stats: [rows=1000] - │ │ ├── cost: 1125.02 - │ │ ├── prune: (1,2) - │ │ └── interesting orderings: (+1) - │ └── filters - │ └── (a:1 + (b:2 * 2)) > 1 [outer=(1,2), immutable] - └── projections - └── a:1 * b:2 [as=column6:6, outer=(1,2), immutable] + └── project + ├── columns: column6:6 a:1 b:2 + ├── immutable + ├── stats: [rows=333.3333] + ├── cost: 1141.73667 + ├── fd: (1,2)-->(6) + ├── prune: (1,2,6) + ├── interesting orderings: (+1) + ├── select + │ ├── columns: a:1 b:2 + │ ├── immutable + │ ├── stats: [rows=333.3333] + │ ├── cost: 1135.05 + │ ├── interesting orderings: (+1) + │ ├── scan tc + │ │ ├── columns: a:1 b:2 + │ │ ├── stats: [rows=1000] + │ │ ├── cost: 1125.02 + │ │ ├── prune: (1,2) + │ │ └── interesting orderings: (+1) + │ └── filters + │ └── (a:1 + (b:2 * 2)) > 1 [outer=(1,2), immutable] + └── projections + └── a:1 * b:2 [as=column6:6, outer=(1,2), immutable] query T EXPLAIN (OPT, TYPES) SELECT * FROM tc WHERE a + 2 * b > 1 ORDER BY a*b ---- -sort +distribute ├── columns: a:1(int) b:2(int) [hidden: column6:6(int)] ├── immutable ├── stats: [rows=333.3333] - ├── cost: 1214.30951 + ├── cost: 1214.33951 ├── fd: (1,2)-->(6) ├── ordering: +6 + ├── distribution: test + ├── input distribution: ├── prune: (1,2,6) ├── interesting orderings: (+1) - └── project - ├── columns: column6:6(int) a:1(int) b:2(int) + └── sort + ├── columns: a:1(int) b:2(int) column6:6(int) ├── immutable ├── stats: [rows=333.3333] - ├── cost: 1141.73667 + ├── cost: 1214.30951 ├── fd: (1,2)-->(6) + ├── ordering: +6 ├── prune: (1,2,6) ├── interesting orderings: (+1) - ├── select - │ ├── columns: a:1(int) b:2(int) - │ ├── immutable - │ ├── stats: [rows=333.3333] - │ ├── cost: 1135.05 - │ ├── interesting orderings: (+1) - │ ├── scan tc - │ │ ├── columns: a:1(int) b:2(int) - │ │ ├── stats: [rows=1000] - │ │ ├── cost: 1125.02 - │ │ ├── prune: (1,2) - │ │ └── interesting orderings: (+1) - │ └── filters - │ └── gt [type=bool, outer=(1,2), immutable] - │ ├── plus [type=int] - │ │ ├── variable: a:1 [type=int] - │ │ └── mult [type=int] - │ │ ├── variable: b:2 [type=int] - │ │ └── const: 2 [type=int] - │ └── const: 1 [type=int] - └── projections - └── mult [as=column6:6, type=int, outer=(1,2), immutable] - ├── variable: a:1 [type=int] - └── variable: b:2 [type=int] + └── project + ├── columns: column6:6(int) a:1(int) b:2(int) + ├── immutable + ├── stats: [rows=333.3333] + ├── cost: 1141.73667 + ├── fd: (1,2)-->(6) + ├── prune: (1,2,6) + ├── interesting orderings: (+1) + ├── select + │ ├── columns: a:1(int) b:2(int) + │ ├── immutable + │ ├── stats: [rows=333.3333] + │ ├── cost: 1135.05 + │ ├── interesting orderings: (+1) + │ ├── scan tc + │ │ ├── columns: a:1(int) b:2(int) + │ │ ├── stats: [rows=1000] + │ │ ├── cost: 1125.02 + │ │ ├── prune: (1,2) + │ │ └── interesting orderings: (+1) + │ └── filters + │ └── gt [type=bool, outer=(1,2), immutable] + │ ├── plus [type=int] + │ │ ├── variable: a:1 [type=int] + │ │ └── mult [type=int] + │ │ ├── variable: b:2 [type=int] + │ │ └── const: 2 [type=int] + │ └── const: 1 [type=int] + └── projections + └── mult [as=column6:6, type=int, outer=(1,2), immutable] + ├── variable: a:1 [type=int] + └── variable: b:2 [type=int] query T EXPLAIN SELECT string_agg(x, y) FROM (VALUES ('foo', 'foo'), ('bar', 'bar')) t(x, y) @@ -1802,16 +1845,10 @@ EXPLAIN (OPT, MEMO) SELECT * FROM tc JOIN t ON k=a ---- memo (optimized, ~11KB, required=[presentation: info:10] [distribution: test]) ├── G1: (explain G2 [presentation: a:1,b:2,k:6,v:7] [distribution: test]) - │ ├── [presentation: info:10] [distribution: test] - │ │ ├── best: (explain G2="[presentation: a:1,b:2,k:6,v:7] [distribution: test]" [presentation: a:1,b:2,k:6,v:7] [distribution: test]) - │ │ └── cost: 2269.93 - │ └── [] - │ ├── best: (explain G2="[presentation: a:1,b:2,k:6,v:7]" [presentation: a:1,b:2,k:6,v:7] [distribution: test]) + │ └── [presentation: info:10] [distribution: test] + │ ├── best: (explain G2="[presentation: a:1,b:2,k:6,v:7] [distribution: test]" [presentation: a:1,b:2,k:6,v:7] [distribution: test]) │ └── cost: 2269.93 ├── G2: (inner-join G3 G4 G5) (inner-join G4 G3 G5) (merge-join G3 G4 G6 inner-join,+1,+6) (lookup-join G3 G6 t,keyCols=[1],outCols=(1,2,6,7)) (merge-join G4 G3 G6 inner-join,+6,+1) (lookup-join G7 G6 tc,keyCols=[3],outCols=(1,2,6,7)) - │ ├── [presentation: a:1,b:2,k:6,v:7] - │ │ ├── best: (inner-join G3 G4 G5) - │ │ └── cost: 2269.91 │ ├── [presentation: a:1,b:2,k:6,v:7] [distribution: test] │ │ ├── best: (inner-join G3="[distribution: test]" G4="[distribution: test]" G5) │ │ └── cost: 2269.91 @@ -1885,16 +1922,10 @@ TABLE t └── k int not null memo (optimized, ~11KB, required=[presentation: info:10] [distribution: test]) ├── G1: (explain G2 [presentation: a:1,b:2,k:6,v:7] [distribution: test]) - │ ├── [presentation: info:10] [distribution: test] - │ │ ├── best: (explain G2="[presentation: a:1,b:2,k:6,v:7] [distribution: test]" [presentation: a:1,b:2,k:6,v:7] [distribution: test]) - │ │ └── cost: 2269.93 - │ └── [] - │ ├── best: (explain G2="[presentation: a:1,b:2,k:6,v:7]" [presentation: a:1,b:2,k:6,v:7] [distribution: test]) + │ └── [presentation: info:10] [distribution: test] + │ ├── best: (explain G2="[presentation: a:1,b:2,k:6,v:7] [distribution: test]" [presentation: a:1,b:2,k:6,v:7] [distribution: test]) │ └── cost: 2269.93 ├── G2: (inner-join G3 G4 G5) (inner-join G4 G3 G5) (merge-join G3 G4 G6 inner-join,+1,+6) (lookup-join G3 G6 t,keyCols=[1],outCols=(1,2,6,7)) (merge-join G4 G3 G6 inner-join,+6,+1) (lookup-join G7 G6 tc,keyCols=[3],outCols=(1,2,6,7)) - │ ├── [presentation: a:1,b:2,k:6,v:7] - │ │ ├── best: (inner-join G3 G4 G5) - │ │ └── cost: 2269.91 │ ├── [presentation: a:1,b:2,k:6,v:7] [distribution: test] │ │ ├── best: (inner-join G3="[distribution: test]" G4="[distribution: test]" G5) │ │ └── cost: 2269.91 diff --git a/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local b/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local index 6a5643f0a88d..c24101396ba0 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local +++ b/pkg/sql/opt/exec/execbuilder/testdata/vectorize_local @@ -107,43 +107,50 @@ Diagram: https://cockroachdb.github.io/distsqlplan/decode.html#eJy0U9Fq1EAUffcrL query T EXPLAIN (OPT, VERBOSE) SELECT c.a FROM c INNER MERGE JOIN d ON c.a = d.b ---- -project +distribute ├── columns: a:1 ├── stats: [rows=10] - ├── cost: 1121.819 + ├── cost: 1121.849 + ├── distribution: test + ├── input distribution: ├── prune: (1) - └── inner-join (merge) - ├── columns: c.a:1 d.b:8 - ├── flags: force merge join - ├── left ordering: +1 - ├── right ordering: +8 - ├── stats: [rows=10, distinct(1)=1, null(1)=0, distinct(8)=1, null(8)=0] - ├── cost: 1121.699 - ├── fd: (1)==(8), (8)==(1) - ├── sort - │ ├── columns: c.a:1 - │ ├── stats: [rows=1, distinct(1)=1, null(1)=0] - │ ├── cost: 25.95 - │ ├── ordering: +1 - │ ├── prune: (1) - │ ├── interesting orderings: (+1) - │ ├── unfiltered-cols: (1-6) - │ └── scan c@sec - │ ├── columns: c.a:1 - │ ├── stats: [rows=1, distinct(1)=1, null(1)=0] - │ ├── cost: 25.9 - │ ├── prune: (1) - │ ├── interesting orderings: (+1) - │ └── unfiltered-cols: (1-6) - ├── scan d - │ ├── columns: d.b:8 - │ ├── stats: [rows=1000, distinct(8)=100, null(8)=0] - │ ├── cost: 1084.62 - │ ├── ordering: +8 - │ ├── prune: (8) - │ ├── interesting orderings: (+8) - │ └── unfiltered-cols: (7-10) - └── filters (true) + └── project + ├── columns: c.a:1 + ├── stats: [rows=10] + ├── cost: 1121.819 + ├── prune: (1) + └── inner-join (merge) + ├── columns: c.a:1 d.b:8 + ├── flags: force merge join + ├── left ordering: +1 + ├── right ordering: +8 + ├── stats: [rows=10, distinct(1)=1, null(1)=0, distinct(8)=1, null(8)=0] + ├── cost: 1121.699 + ├── fd: (1)==(8), (8)==(1) + ├── sort + │ ├── columns: c.a:1 + │ ├── stats: [rows=1, distinct(1)=1, null(1)=0] + │ ├── cost: 25.95 + │ ├── ordering: +1 + │ ├── prune: (1) + │ ├── interesting orderings: (+1) + │ ├── unfiltered-cols: (1-6) + │ └── scan c@sec + │ ├── columns: c.a:1 + │ ├── stats: [rows=1, distinct(1)=1, null(1)=0] + │ ├── cost: 25.9 + │ ├── prune: (1) + │ ├── interesting orderings: (+1) + │ └── unfiltered-cols: (1-6) + ├── scan d + │ ├── columns: d.b:8 + │ ├── stats: [rows=1000, distinct(8)=100, null(8)=0] + │ ├── cost: 1084.62 + │ ├── ordering: +8 + │ ├── prune: (8) + │ ├── interesting orderings: (+8) + │ └── unfiltered-cols: (7-10) + └── filters (true) query T EXPLAIN ANALYZE (DISTSQL) SELECT c.a FROM c INNER MERGE JOIN d ON c.a = d.b diff --git a/pkg/sql/opt/exec/explain/plan_gist_factory.go b/pkg/sql/opt/exec/explain/plan_gist_factory.go index 8199f1266875..d2e72c24d7d1 100644 --- a/pkg/sql/opt/exec/explain/plan_gist_factory.go +++ b/pkg/sql/opt/exec/explain/plan_gist_factory.go @@ -572,6 +572,31 @@ func (u *unknownTable) IsRefreshViewRequired() bool { return false } +// HomeRegion is part of the cat.Table interface. +func (u *unknownTable) HomeRegion() (region string, ok bool) { + return "", false +} + +// IsGlobalTable is part of the cat.Table interface. +func (u *unknownTable) IsGlobalTable() bool { + return false +} + +// IsRegionalByRow is part of the cat.Table interface. +func (u *unknownTable) IsRegionalByRow() bool { + return false +} + +// HomeRegionColName is part of the cat.Table interface. +func (u *unknownTable) HomeRegionColName() (colName string, ok bool) { + return "", false +} + +// GetDatabaseID is part of the cat.Table interface. +func (u *unknownTable) GetDatabaseID() descpb.ID { + return 0 +} + var _ cat.Table = &unknownTable{} // unknownTable implements the cat.Index interface and is used to represent diff --git a/pkg/sql/opt/memo/expr.go b/pkg/sql/opt/memo/expr.go index 51c7969b484d..7d76fd3e9e19 100644 --- a/pkg/sql/opt/memo/expr.go +++ b/pkg/sql/opt/memo/expr.go @@ -281,6 +281,42 @@ func (n FiltersExpr) Difference(other FiltersExpr) FiltersExpr { return newFilters } +// NoOpDistribution returns true if a DistributeExpr has the same distribution +// as its input. +func (e *DistributeExpr) NoOpDistribution() bool { + distributionProvidedPhysical := e.ProvidedPhysical() + inputDistributionProvidedPhysical := e.Input.ProvidedPhysical() + + if distributionProvidedPhysical != nil && inputDistributionProvidedPhysical != nil { + distribution := distributionProvidedPhysical.Distribution + inputDistribution := inputDistributionProvidedPhysical.Distribution + return distribution.Equals(inputDistribution) + } + return false +} + +// GetRegionOfDistribution returns the single region name of the provided +// distribution, if there is exactly one. +func (e *DistributeExpr) GetRegionOfDistribution() (region string, ok bool) { + distributionProvidedPhysical := e.ProvidedPhysical() + + if distributionProvidedPhysical != nil { + return distributionProvidedPhysical.Distribution.GetSingleRegion() + } + return "", false +} + +// GetInputHomeRegion returns the single region name of the home region of the +// input expression tree to the Distribute operation, if there is exactly one. +func (e *DistributeExpr) GetInputHomeRegion() (inputHomeRegion string, ok bool) { + inputDistributionProvidedPhysical := e.Input.ProvidedPhysical() + + if inputDistributionProvidedPhysical != nil { + return inputDistributionProvidedPhysical.Distribution.GetSingleRegion() + } + return "", false +} + // OutputCols returns the set of columns constructed by the Aggregations // expression. func (n AggregationsExpr) OutputCols() opt.ColSet { @@ -765,6 +801,24 @@ func (s *ScanPrivate) SetConstraint(evalCtx *eval.Context, c *constraint.Constra } } +// GetConstExprFromFilter finds the constant expression which is equated with +// the column with the given `colID` in the inverted join constant filters, if +// one exists. Otherwise, returns nil, ok=false. +func (ij *InvertedJoinPrivate) GetConstExprFromFilter(colID opt.ColumnID) (expr opt.Expr, ok bool) { + + for _, filter := range ij.ConstFilters { + if filter.Condition.Op() == opt.EqOp { + leftChild := filter.Condition.Child(0) + if variableExpr, ok := leftChild.(*VariableExpr); ok { + if variableExpr.Col == colID { + return filter.Condition.Child(1), true + } + } + } + } + return nil, false +} + // UsesPartialIndex returns true if the LookupJoinPrivate looks-up via a // partial index. func (lj *LookupJoinPrivate) UsesPartialIndex(md *opt.Metadata) bool { @@ -772,6 +826,31 @@ func (lj *LookupJoinPrivate) UsesPartialIndex(md *opt.Metadata) bool { return isPartialIndex } +// GetConstPrefixFilter finds the position of the filter in the lookup join +// expression filters that constrains the first index column to one or more +// constant values. If such a filter is found, GetConstPrefixFilter returns the +// position of the filter and ok=true. Otherwise, returns ok=false. +func (lj *LookupJoinPrivate) GetConstPrefixFilter(md *opt.Metadata) (pos int, ok bool) { + lookupTable := md.Table(lj.Table) + lookupIndex := lookupTable.Index(lj.Index) + + idxCol := lj.Table.IndexColumnID(lookupIndex, 0) + for i := range lj.LookupExpr { + props := lj.LookupExpr[i].ScalarProps() + if !props.TightConstraints { + continue + } + if props.OuterCols.Len() != 1 { + continue + } + col := props.OuterCols.SingleColumn() + if col == idxCol { + return i, true + } + } + return 0, false +} + // NeedResults returns true if the mutation operator can return the rows that // were mutated. func (m *MutationPrivate) NeedResults() bool { @@ -896,6 +975,25 @@ func (prj *ProjectExpr) InternalFDs() *props.FuncDepSet { return &prj.internalFuncDeps } +// GetProjectedEnumConstant looks for the projection with target colID in prj, +// and if it contains a constant enum, returns its string representation, or the +// empty string if not found. +func (prj *ProjectExpr) GetProjectedEnumConstant(colID opt.ColumnID) string { + for _, projection := range prj.Projections { + if projection.Col == colID { + if projection.Element.Op() == opt.ConstOp { + constExpr := projection.Element.(*ConstExpr) + if enumValue, ok := constExpr.Value.(*tree.DEnum); ok { + return enumValue.LogicalRep + } + } else { + return "" + } + } + } + return "" +} + // FindInlinableConstants returns the set of input columns that are synthesized // constant value expressions: ConstOp, TrueOp, FalseOp, or NullOp. Constant // value expressions can often be inlined into referencing expressions. Only diff --git a/pkg/sql/opt/memo/expr_format.go b/pkg/sql/opt/memo/expr_format.go index 9ecb901a82ab..e63853a89f57 100644 --- a/pkg/sql/opt/memo/expr_format.go +++ b/pkg/sql/opt/memo/expr_format.go @@ -1425,7 +1425,7 @@ func (f *ExprFmtCtx) formatColSimpleToBuffer(buf *bytes.Buffer, label string, id if f.Memo != nil { md := f.Memo.metadata fullyQualify := !f.HasFlags(ExprFmtHideQualifications) - label = md.QualifiedAlias(id, fullyQualify, f.Catalog) + label = md.QualifiedAlias(id, fullyQualify, false /* alwaysQualify */, f.Catalog) } else { label = fmt.Sprintf("unknown%d", id) } diff --git a/pkg/sql/opt/memo/memo.go b/pkg/sql/opt/memo/memo.go index 3a8d2954d53d..3c6e23a048ce 100644 --- a/pkg/sql/opt/memo/memo.go +++ b/pkg/sql/opt/memo/memo.go @@ -155,6 +155,7 @@ type Memo struct { testingOptimizerRandomSeed int64 testingOptimizerCostPerturbation float64 testingOptimizerDisableRuleProbability float64 + enforceHomeRegion bool // curRank is the highest currently in-use scalar expression rank. curRank opt.ScalarRank @@ -205,6 +206,7 @@ func (m *Memo) Init(evalCtx *eval.Context) { testingOptimizerRandomSeed: evalCtx.SessionData().TestingOptimizerRandomSeed, testingOptimizerCostPerturbation: evalCtx.SessionData().TestingOptimizerCostPerturbation, testingOptimizerDisableRuleProbability: evalCtx.SessionData().TestingOptimizerDisableRuleProbability, + enforceHomeRegion: evalCtx.SessionData().EnforceHomeRegion, } m.metadata.Init() m.logPropsBuilder.init(evalCtx, m) @@ -338,7 +340,8 @@ func (m *Memo) IsStale( m.allowUnconstrainedNonCoveringIndexScan != evalCtx.SessionData().UnconstrainedNonCoveringIndexScanEnabled || m.testingOptimizerRandomSeed != evalCtx.SessionData().TestingOptimizerRandomSeed || m.testingOptimizerCostPerturbation != evalCtx.SessionData().TestingOptimizerCostPerturbation || - m.testingOptimizerDisableRuleProbability != evalCtx.SessionData().TestingOptimizerDisableRuleProbability { + m.testingOptimizerDisableRuleProbability != evalCtx.SessionData().TestingOptimizerDisableRuleProbability || + m.enforceHomeRegion != evalCtx.SessionData().EnforceHomeRegion { return true, nil } @@ -494,6 +497,11 @@ func (m *Memo) DisableCheckExpr() { m.disableCheckExpr = true } +// EvalContext returns the eval.Context of the current SQL request. +func (m *Memo) EvalContext() *eval.Context { + return m.logPropsBuilder.evalCtx +} + // ValuesContainer lets ValuesExpr and LiteralValuesExpr share code. type ValuesContainer interface { RelExpr diff --git a/pkg/sql/opt/memo/memo_test.go b/pkg/sql/opt/memo/memo_test.go index 2c4505c0b69d..04cf0ac1f043 100644 --- a/pkg/sql/opt/memo/memo_test.go +++ b/pkg/sql/opt/memo/memo_test.go @@ -280,6 +280,12 @@ func TestMemoIsStale(t *testing.T) { evalCtx.SessionData().UnconstrainedNonCoveringIndexScanEnabled = false notStale() + // Stale enforce home region. + evalCtx.SessionData().EnforceHomeRegion = true + stale() + evalCtx.SessionData().EnforceHomeRegion = false + notStale() + // Stale testing_optimizer_random_seed. evalCtx.SessionData().TestingOptimizerRandomSeed = 100 stale() diff --git a/pkg/sql/opt/metadata.go b/pkg/sql/opt/metadata.go index ebeae345be2a..8329cada0382 100644 --- a/pkg/sql/opt/metadata.go +++ b/pkg/sql/opt/metadata.go @@ -16,6 +16,7 @@ import ( "math/bits" "strings" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/multiregion" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" @@ -196,7 +197,7 @@ func (md *Metadata) Init() { // This metadata can then be modified independent of the copied metadata. // // Table annotations are not transferred over; all annotations are unset on -// the copy. +// the copy, except for regionConfig, which is read-only, and can be shared. // // copyScalarFn must be a function that returns a copy of the given scalar // expression. @@ -226,8 +227,17 @@ func (md *Metadata) CopyFrom(from *Metadata, copyScalarFn func(Expr) Expr) { md.tables = make([]TableMeta, len(from.tables)) } for i := range from.tables { - // Note: annotations inside TableMeta are not retained. + // Note: annotations inside TableMeta are not retained... md.tables[i].copyFrom(&from.tables[i], copyScalarFn) + + // ...except for the regionConfig annotation. + tabID := from.tables[i].MetaID + regionConfig, ok := md.TableAnnotation(tabID, regionConfigAnnID).(*multiregion.RegionConfig) + if ok { + // Don't waste time looking up a database descriptor and constructing a + // RegionConfig more than once for a given table. + md.SetTableAnnotation(tabID, regionConfigAnnID, regionConfig) + } } md.sequences = append(md.sequences, from.sequences...) @@ -485,7 +495,7 @@ func (md *Metadata) DuplicateTable( } } - md.tables = append(md.tables, TableMeta{ + newTabMeta := TableMeta{ MetaID: newTabID, Table: tabMeta.Table, Alias: tabMeta.Alias, @@ -495,7 +505,14 @@ func (md *Metadata) DuplicateTable( partialIndexPredicates: partialIndexPredicates, indexPartitionLocalities: tabMeta.indexPartitionLocalities, checkConstraintsStats: checkConstraintsStats, - }) + } + md.tables = append(md.tables, newTabMeta) + regionConfig, ok := md.TableAnnotation(tabID, regionConfigAnnID).(*multiregion.RegionConfig) + if ok { + // Don't waste time looking up a database descriptor and constructing a + // RegionConfig more than once for a given table. + md.SetTableAnnotation(newTabID, regionConfigAnnID, regionConfig) + } return newTabID } @@ -550,9 +567,12 @@ func (md *Metadata) ColumnMeta(colID ColumnID) *ColumnMeta { // // 2. If there's another column in the metadata with the same column alias but // a different table name, then prefix the column alias with the table -// name: "tabName.columnAlias". +// name: "tabName.columnAlias". If alwaysQualify is true, then the column +// alias is always prefixed with the table alias. // -func (md *Metadata) QualifiedAlias(colID ColumnID, fullyQualify bool, catalog cat.Catalog) string { +func (md *Metadata) QualifiedAlias( + colID ColumnID, fullyQualify, alwaysQualify bool, catalog cat.Catalog, +) string { cm := md.ColumnMeta(colID) if cm.Table == 0 { // Column doesn't belong to a table, so no need to qualify it further. @@ -562,8 +582,9 @@ func (md *Metadata) QualifiedAlias(colID ColumnID, fullyQualify bool, catalog ca // If a fully qualified alias has not been requested, then only qualify it if // it would otherwise be ambiguous. var tabAlias tree.TableName - qualify := fullyQualify + qualify := fullyQualify || alwaysQualify if !fullyQualify { + tabAlias = md.TableMeta(cm.Table).Alias for i := range md.cols { if i == int(cm.MetaID-1) { continue @@ -572,7 +593,6 @@ func (md *Metadata) QualifiedAlias(colID ColumnID, fullyQualify bool, catalog ca // If there are two columns with same alias, then column is ambiguous. cm2 := &md.cols[i] if cm2.Alias == cm.Alias { - tabAlias = md.TableMeta(cm.Table).Alias if cm2.Table == 0 { qualify = true } else { diff --git a/pkg/sql/opt/optbuilder/select.go b/pkg/sql/opt/optbuilder/select.go index 9d6060b68ad7..d59ae3b7b976 100644 --- a/pkg/sql/opt/optbuilder/select.go +++ b/pkg/sql/opt/optbuilder/select.go @@ -13,6 +13,7 @@ package optbuilder import ( "github.com/cockroachdb/cockroach/pkg/server/telemetry" "github.com/cockroachdb/cockroach/pkg/sql/catalog/colinfo" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc" "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" @@ -24,6 +25,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/privilege" "github.com/cockroachdb/cockroach/pkg/sql/sem/asof" + "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/sql/sqltelemetry" "github.com/cockroachdb/cockroach/pkg/sql/types" @@ -422,6 +424,23 @@ func (b *Builder) addTable(tab cat.Table, alias *tree.TableName) *opt.TableMeta return md.TableMeta(tabID) } +// errorOnInvalidMultiregionDB panics if the table described by tabMeta is owned +// by a non-multiregion database or a multiregion database with SURVIVE REGION +// FAILURE goal. +func errorOnInvalidMultiregionDB(evalCtx *eval.Context, tabMeta *opt.TableMeta) { + survivalGoal, ok := tabMeta.GetDatabaseSurvivalGoal(evalCtx.Planner) + // non-multiregional database or SURVIVE REGION FAILURE option + if !ok { + err := pgerror.New(pgcode.QueryHasNoHomeRegion, + "Query has no home region. Try accessing only tables in multi-region databases with ZONE survivability.") + panic(err) + } else if survivalGoal == descpb.SurvivalGoal_REGION_FAILURE { + err := pgerror.New(pgcode.QueryHasNoHomeRegion, + "The enforce_home_region setting cannot be combined with REGION survivability. Try accessing only tables in multi-region databases with ZONE survivability.") + panic(err) + } +} + // buildScan builds a memo group for a ScanOp expression on the given table. If // the ordinals list contains any VirtualComputed columns, a ProjectOp is built // on top. @@ -517,6 +536,12 @@ func (b *Builder) buildScan( return outScope } + // Scanning tables in databases that don't use the SURVIVE ZONE FAILURE option + // is disallowed when EnforceHomeRegion is true. + if b.evalCtx.SessionData().EnforceHomeRegion { + errorOnInvalidMultiregionDB(b.evalCtx, tabMeta) + } + private := memo.ScanPrivate{Table: tabID, Cols: scanColIDs} if indexFlags != nil { private.Flags.NoIndexJoin = indexFlags.NoIndexJoin diff --git a/pkg/sql/opt/optgen/exprgen/BUILD.bazel b/pkg/sql/opt/optgen/exprgen/BUILD.bazel index 896e2d061e8c..85381c861017 100644 --- a/pkg/sql/opt/optgen/exprgen/BUILD.bazel +++ b/pkg/sql/opt/optgen/exprgen/BUILD.bazel @@ -14,6 +14,7 @@ go_library( deps = [ "//pkg/sql/opt", "//pkg/sql/opt/cat", + "//pkg/sql/opt/distribution", "//pkg/sql/opt/memo", "//pkg/sql/opt/norm", "//pkg/sql/opt/optgen/lang", diff --git a/pkg/sql/opt/optgen/exprgen/expr_gen.go b/pkg/sql/opt/optgen/exprgen/expr_gen.go index 28de4cf92304..8d51a1a88520 100644 --- a/pkg/sql/opt/optgen/exprgen/expr_gen.go +++ b/pkg/sql/opt/optgen/exprgen/expr_gen.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" + "github.com/cockroachdb/cockroach/pkg/sql/opt/distribution" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/opt/norm" "github.com/cockroachdb/cockroach/pkg/sql/opt/optgen/lang" @@ -101,7 +102,7 @@ func buildAndOptimize( mem: factory.Memo(), cat: catalog, }, - coster: xform.MakeDefaultCoster(factory.Memo()), + coster: xform.MakeDefaultCoster(factory.Memo(), factory.EvalContext()), } // To create a valid optgen "file", we create a rule with a bogus match. @@ -399,6 +400,7 @@ func (eg *exprGen) populateBestProps(expr opt.Expr, required *physical.Required) // BuildProvided relies on ProvidedPhysical() being set in the children, so // it must run after the recursive calls on the children. provided.Ordering = ordering.BuildProvided(rel, &required.Ordering) + provided.Distribution = distribution.BuildProvided(eg.f.EvalContext(), rel, &required.Distribution) cost += eg.coster.ComputeCost(rel, required) eg.mem.SetBestProps(rel, required, provided, cost) diff --git a/pkg/sql/opt/props/physical/BUILD.bazel b/pkg/sql/opt/props/physical/BUILD.bazel index db73cf61b611..e75562839350 100644 --- a/pkg/sql/opt/props/physical/BUILD.bazel +++ b/pkg/sql/opt/props/physical/BUILD.bazel @@ -16,6 +16,8 @@ go_library( "//pkg/sql/opt/cat", "//pkg/sql/opt/constraint", "//pkg/sql/opt/props", + "//pkg/sql/pgwire/pgcode", + "//pkg/sql/pgwire/pgerror", "//pkg/sql/sem/eval", ], ) diff --git a/pkg/sql/opt/props/physical/distribution.go b/pkg/sql/opt/props/physical/distribution.go index 10b6ba4cc38a..3cae19e40765 100644 --- a/pkg/sql/opt/props/physical/distribution.go +++ b/pkg/sql/opt/props/physical/distribution.go @@ -15,8 +15,11 @@ import ( "sort" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/opt/constraint" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode" + "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgerror" "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" ) @@ -107,8 +110,10 @@ func (d *Distribution) FromLocality(locality roachpb.Locality) { // FromIndexScan sets the Distribution that results from scanning the given // index with the given constraint c (c can be nil). func (d *Distribution) FromIndexScan( - evalCtx *eval.Context, index cat.Index, c *constraint.Constraint, + evalCtx *eval.Context, tabMeta *opt.TableMeta, ord cat.IndexOrdinal, c *constraint.Constraint, ) { + tab := tabMeta.Table + index := tab.Index(ord) if index.Table().IsVirtualTable() { // Virtual tables do not have zone configurations. return @@ -153,8 +158,44 @@ func (d *Distribution) FromIndexScan( } } + // Populate the distribution for GLOBAL tables and REGIONAL BY TABLE. if len(regions) == 0 { - regions = getRegionsFromZone(index.Zone()) + regionsPopulated := false + + if tab.IsGlobalTable() { + // Global tables can always be treated as local to the gateway region. + gatewayRegion, found := evalCtx.Locality.Find("region") + if found { + regions = make(map[string]struct{}) + regions[gatewayRegion] = struct{}{} + regionsPopulated = true + } + } else if homeRegion, ok := tab.HomeRegion(); ok { + regions = make(map[string]struct{}) + regions[homeRegion] = struct{}{} + regionsPopulated = true + } else { + // Use the leaseholder region(s), which should be the same as the home + // region of REGIONAL BY TABLE tables. + regions = getRegionsFromZone(index.Zone()) + regionsPopulated = regions != nil + } + if !regionsPopulated { + // If the above methods failed to find a distribution, then the + // distribution is all regions in the database. + regionsNames, ok := tabMeta.GetRegionsInDatabase(evalCtx.Planner) + if !ok && evalCtx.SessionData().EnforceHomeRegion { + err := pgerror.New(pgcode.QueryHasNoHomeRegion, + "Query has no home region. Try accessing only tables defined in multi-region databases.") + panic(err) + } + if ok { + regions = make(map[string]struct{}) + for _, regionName := range regionsNames { + regions[string(regionName)] = struct{}{} + } + } + } } // Convert to a slice and sort regions. @@ -165,24 +206,39 @@ func (d *Distribution) FromIndexScan( sort.Strings(d.Regions) } +// GetSingleRegion returns the single region name of the distribution, +// if there is exactly one. +func (d *Distribution) GetSingleRegion() (region string, ok bool) { + if d == nil { + return "", false + } + if len(d.Regions) == 1 { + return d.Regions[0], true + } + return "", false +} + +// IsLocal returns true if this distribution matches +// the gateway region of the connection. +func (d *Distribution) IsLocal(evalCtx *eval.Context) bool { + if d == nil { + return false + } + gatewayRegion, foundLocalRegion := evalCtx.Locality.Find("region") + if foundLocalRegion { + if distributionRegion, ok := d.GetSingleRegion(); ok { + return distributionRegion == gatewayRegion + } + } + return false +} + // getRegionsFromZone returns the regions of the given zone config, if any. It // attempts to find the smallest set of regions likely to hold the leaseholder. func getRegionsFromZone(zone cat.Zone) map[string]struct{} { // First find any regional replica constraints. If there is exactly one, we // can return early. - var regions map[string]struct{} - for i, n := 0, zone.ReplicaConstraintsCount(); i < n; i++ { - replicaConstraint := zone.ReplicaConstraints(i) - for j, m := 0, replicaConstraint.ConstraintCount(); j < m; j++ { - constraint := replicaConstraint.Constraint(j) - if region, ok := getRegionFromConstraint(constraint); ok { - if regions == nil { - regions = make(map[string]struct{}) - } - regions[region] = struct{}{} - } - } - } + regions := getReplicaRegionsFromZone(zone) if len(regions) == 1 { return regions } @@ -238,3 +294,22 @@ func getRegionFromConstraint(constraint cat.Constraint) (region string, ok bool) // The region is prohibited. return "", false /* ok */ } + +// getReplicaRegionsFromZone returns the replica regions of the given zone +// config, if any. +func getReplicaRegionsFromZone(zone cat.Zone) map[string]struct{} { + var regions map[string]struct{} + for i, n := 0, zone.ReplicaConstraintsCount(); i < n; i++ { + replicaConstraint := zone.ReplicaConstraints(i) + for j, m := 0, replicaConstraint.ConstraintCount(); j < m; j++ { + constraint := replicaConstraint.Constraint(j) + if region, ok := getRegionFromConstraint(constraint); ok { + if regions == nil { + regions = make(map[string]struct{}) + } + regions[region] = struct{}{} + } + } + } + return regions +} diff --git a/pkg/sql/opt/table_meta.go b/pkg/sql/opt/table_meta.go index 9d69506a5c69..a7e2c8b625d9 100644 --- a/pkg/sql/opt/table_meta.go +++ b/pkg/sql/opt/table_meta.go @@ -11,8 +11,12 @@ package opt import ( + "github.com/cockroachdb/cockroach/pkg/sql/catalog/catpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/multiregion" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" "github.com/cockroachdb/cockroach/pkg/sql/opt/partition" + "github.com/cockroachdb/cockroach/pkg/sql/sem/eval" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" "github.com/cockroachdb/cockroach/pkg/util/buildutil" "github.com/cockroachdb/errors" @@ -111,11 +115,14 @@ var tableAnnIDCount TableAnnID // called. Calling more than this number of times results in a panic. Having // a maximum enables a static annotation array to be inlined into the metadata // table struct. -const maxTableAnnIDCount = 3 +const maxTableAnnIDCount = 4 // NotNullAnnID is the annotation ID for table not null columns. var NotNullAnnID = NewTableAnnID() +// regionConfigAnnID is the annotation ID for multiregion table config. +var regionConfigAnnID = NewTableAnnID() + // TableMeta stores information about one of the tables stored in the metadata. // // NOTE: Metadata.DuplicateTable and TableMeta.copyFrom must be kept in sync @@ -195,6 +202,23 @@ type TableMeta struct { anns [maxTableAnnIDCount]interface{} } +// TableAnnotation returns the given annotation that is associated with the +// given table. If the table has no such annotation, TableAnnotation returns +// nil. +func (tm *TableMeta) TableAnnotation(annID TableAnnID) interface{} { + return tm.anns[annID] +} + +// SetTableAnnotation associates the given annotation with the given table. The +// annotation is associated by the given ID, which was allocated by calling +// NewTableAnnID. If an annotation with the ID already exists on the table, then +// it is overwritten. +// +// See the TableAnnID comment for more details and a usage example. +func (tm *TableMeta) SetTableAnnotation(tabAnnID TableAnnID, ann interface{}) { + tm.anns[tabAnnID] = ann +} + // copyFrom initializes the receiver with a copy of the given TableMeta, which // is considered immutable. // @@ -418,6 +442,64 @@ func (tm *TableMeta) VirtualComputedColumns() ColSet { return virtualCols } +// GetRegionsInDatabase finds the full set of regions in the multiregion +// database owning the table described by `tm`, or returns ok=false if not +// multiregion. The result is cached in TableMeta. +func (tm *TableMeta) GetRegionsInDatabase( + planner eval.Planner, +) (regionNames catpb.RegionNames, ok bool) { + multiregionConfig, ok := tm.TableAnnotation(regionConfigAnnID).(*multiregion.RegionConfig) + if ok { + if multiregionConfig == nil { + return nil /* regionNames */, false + } + return multiregionConfig.Regions(), true + } + dbID := tm.Table.GetDatabaseID() + if dbID == 0 { + tm.SetTableAnnotation(regionConfigAnnID, nil) + return nil /* regionNames */, false + } + + regionConfig, ok := planner.GetMultiregionConfig(dbID) + if !ok { + tm.SetTableAnnotation(regionConfigAnnID, nil) + return nil /* regionNames */, false + } + multiregionConfig, _ = regionConfig.(*multiregion.RegionConfig) + tm.SetTableAnnotation(regionConfigAnnID, multiregionConfig) + return multiregionConfig.Regions(), true +} + +// GetDatabaseSurvivalGoal finds the survival goal of the multiregion database +// owning the table described by `tm`, or returns ok=false if not multiregion. +// The result is cached in TableMeta. +func (tm *TableMeta) GetDatabaseSurvivalGoal( + planner eval.Planner, +) (survivalGoal descpb.SurvivalGoal, ok bool) { + // If planner is nil, we could be running an internal query or something else + // which is not a user query, so make sure we don't error out this case. + if planner == nil { + return descpb.SurvivalGoal_ZONE_FAILURE /* survivalGoal */, true + } + multiregionConfig, ok := tm.TableAnnotation(regionConfigAnnID).(*multiregion.RegionConfig) + if ok { + if multiregionConfig == nil { + return descpb.SurvivalGoal_ZONE_FAILURE /* survivalGoal */, false + } + return multiregionConfig.SurvivalGoal(), true + } + dbID := tm.Table.GetDatabaseID() + regionConfig, ok := planner.GetMultiregionConfig(dbID) + if !ok { + tm.SetTableAnnotation(regionConfigAnnID, nil) + return descpb.SurvivalGoal_ZONE_FAILURE /* survivalGoal */, false + } + multiregionConfig, _ = regionConfig.(*multiregion.RegionConfig) + tm.SetTableAnnotation(regionConfigAnnID, multiregionConfig) + return multiregionConfig.SurvivalGoal(), true +} + // TableAnnotation returns the given annotation that is associated with the // given table. If the table has no such annotation, TableAnnotation returns // nil. diff --git a/pkg/sql/opt/testutils/testcat/test_catalog.go b/pkg/sql/opt/testutils/testcat/test_catalog.go index 747a90bd203f..6c673fa54ab6 100644 --- a/pkg/sql/opt/testutils/testcat/test_catalog.go +++ b/pkg/sql/opt/testutils/testcat/test_catalog.go @@ -847,6 +847,31 @@ func (tt *Table) IsPartitionAllBy() bool { return false } +// HomeRegion is part of the cat.Table interface. +func (tt *Table) HomeRegion() (region string, ok bool) { + return "", false +} + +// IsGlobalTable is part of the cat.Table interface. +func (tt *Table) IsGlobalTable() bool { + return false +} + +// IsRegionalByRow is part of the cat.Table interface. +func (tt *Table) IsRegionalByRow() bool { + return false +} + +// HomeRegionColName is part of the cat.Table interface. +func (tt *Table) HomeRegionColName() (colName string, ok bool) { + return "", false +} + +// GetDatabaseID is part of the cat.Table interface. +func (tt *Table) GetDatabaseID() descpb.ID { + return 0 +} + // FindOrdinal returns the ordinal of the column with the given name. func (tt *Table) FindOrdinal(name string) int { for i, col := range tt.Columns { diff --git a/pkg/sql/opt/xform/coster.go b/pkg/sql/opt/xform/coster.go index 1c24dde92f34..2de8a48d2297 100644 --- a/pkg/sql/opt/xform/coster.go +++ b/pkg/sql/opt/xform/coster.go @@ -17,6 +17,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" + "github.com/cockroachdb/cockroach/pkg/sql/opt/distribution" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/opt/ordering" "github.com/cockroachdb/cockroach/pkg/sql/opt/props" @@ -91,8 +92,11 @@ type coster struct { var _ Coster = &coster{} // MakeDefaultCoster creates an instance of the default coster. -func MakeDefaultCoster(mem *memo.Memo) Coster { - return &coster{mem: mem} +func MakeDefaultCoster(mem *memo.Memo, evalCtx *eval.Context) Coster { + return &coster{evalCtx: evalCtx, + mem: mem, + locality: evalCtx.Locality, + } } const ( @@ -156,6 +160,10 @@ const ( // stale. largeMaxCardinalityScanCostPenalty = unboundedMaxCardinalityScanCostPenalty / 2 + // LargeDistributeCost is the cost to use for Distribute operations when a + // session mode is set to error out on access of rows from remote regions. + LargeDistributeCost = hugeCost / 100 + // preferLookupJoinFactor is a scale factor for the cost of a lookup join when // we have a hint for preferring a lookup join. preferLookupJoinFactor = 1e-6 @@ -657,6 +665,14 @@ func (c *coster) computeSortCost(sort *memo.SortExpr, required *physical.Require func (c *coster) computeDistributeCost( distribute *memo.DistributeExpr, required *physical.Required, ) memo.Cost { + if distribute.NoOpDistribution() { + // If the distribution will be elided, the cost is zero. + return memo.Cost(0) + } + if c.evalCtx != nil && c.evalCtx.SessionData().EnforceHomeRegion { + return LargeDistributeCost + } + // TODO(rytaft): Compute a real cost here. Currently we just add a tiny cost // as a placeholder. return cpuCostFactor @@ -911,7 +927,7 @@ func (c *coster) computeLookupJoinCost( if join.LookupJoinPrivate.Flags.Has(memo.DisallowLookupJoinIntoRight) { return hugeCost } - return c.computeIndexLookupJoinCost( + cost := c.computeIndexLookupJoinCost( join, required, join.LookupColsAreTableKey, @@ -922,6 +938,18 @@ func (c *coster) computeLookupJoinCost( join.Flags, join.LocalityOptimized, ) + if c.evalCtx != nil && c.evalCtx.SessionData().EnforceHomeRegion { + provided := distribution.BuildLookupJoinLookupTableDistribution(c.evalCtx, join) + if provided.Any() || len(provided.Regions) != 1 { + cost += LargeDistributeCost + } + var localDist physical.Distribution + localDist.FromLocality(c.evalCtx.Locality) + if !localDist.Equals(provided) { + cost += LargeDistributeCost + } + } + return cost } func (c *coster) computeIndexLookupJoinCost( @@ -1078,6 +1106,18 @@ func (c *coster) computeInvertedJoinCost( c.rowScanCost(join, join.Table, join.Index, lookupCols, join.Relational().Stats) cost += memo.Cost(rowsProcessed) * perRowCost + + if c.evalCtx != nil && c.evalCtx.SessionData().EnforceHomeRegion { + provided := distribution.BuildInvertedJoinLookupTableDistribution(c.evalCtx, join) + if provided.Any() || len(provided.Regions) != 1 { + cost += LargeDistributeCost + } + var localDist physical.Distribution + localDist.FromLocality(c.evalCtx.Locality) + if !localDist.Equals(provided) { + cost += LargeDistributeCost + } + } return cost } diff --git a/pkg/sql/opt/xform/join_funcs.go b/pkg/sql/opt/xform/join_funcs.go index 70b057988347..b2357677b34a 100644 --- a/pkg/sql/opt/xform/join_funcs.go +++ b/pkg/sql/opt/xform/join_funcs.go @@ -1285,7 +1285,6 @@ func (c *CustomFuncs) GetLocalityOptimizedLookupJoinExprs( } } tabMeta := c.e.mem.Metadata().TableMeta(private.Table) - index := tabMeta.Table.Index(private.Index) // The PrefixSorter has collected all the prefixes from all the different // partitions (remembering which ones came from local partitions), and has @@ -1301,7 +1300,7 @@ func (c *CustomFuncs) GetLocalityOptimizedLookupJoinExprs( } // Find a filter that constrains the first column of the index. - filterIdx, ok := c.getConstPrefixFilter(index, private.Table, private.LookupExpr) + filterIdx, ok := private.GetConstPrefixFilter(c.e.mem.Metadata()) if !ok { return nil, nil, false } @@ -1339,30 +1338,6 @@ func (c *CustomFuncs) GetLocalityOptimizedLookupJoinExprs( return localExpr, remoteExpr, true } -// getConstPrefixFilter finds the position of the filter in the given slice of -// filters that constrains the first index column to one or more constant -// values. If such a filter is found, getConstPrefixFilter returns the position -// of the filter and ok=true. Otherwise, returns ok=false. -func (c CustomFuncs) getConstPrefixFilter( - index cat.Index, table opt.TableID, filters memo.FiltersExpr, -) (pos int, ok bool) { - idxCol := table.IndexColumnID(index, 0) - for i := range filters { - props := filters[i].ScalarProps() - if !props.TightConstraints { - continue - } - if props.OuterCols.Len() != 1 { - continue - } - col := props.OuterCols.SingleColumn() - if col == idxCol { - return i, true - } - } - return 0, false -} - // getLocalValues returns the indexes of the values in the given Datums slice // that target local partitions. func (c *CustomFuncs) getLocalValues( diff --git a/pkg/sql/opt/xform/optimizer.go b/pkg/sql/opt/xform/optimizer.go index aae3a726d0c0..2f2c47d84326 100644 --- a/pkg/sql/opt/xform/optimizer.go +++ b/pkg/sql/opt/xform/optimizer.go @@ -644,13 +644,13 @@ func (o *Optimizer) enforceProps( // stripped by recursively optimizing the group with successively fewer // properties. The properties are stripped off in a heuristic order, from // least likely to be expensive to enforce to most likely. - if !required.Distribution.Any() { + if !required.Distribution.Any() && member.Op() != opt.ExplainOp { enforcer := &memo.DistributeExpr{Input: member} memberProps := BuildChildPhysicalProps(o.mem, enforcer, 0, required) return o.optimizeEnforcer(state, enforcer, required, member, memberProps) } - if !required.Ordering.Any() { + if !required.Ordering.Any() && member.Op() != opt.ExplainOp { // Try Sort enforcer that requires no ordering from its input. enforcer := &memo.SortExpr{Input: member} memberProps := BuildChildPhysicalProps(o.mem, enforcer, 0, required) @@ -762,13 +762,11 @@ func (o *Optimizer) setLowestCostTree(parent opt.Expr, parentProps *physical.Req var childProps *physical.Required for i, n := 0, parent.ChildCount(); i < n; i++ { before := parent.Child(i) - if relParent != nil { childProps = BuildChildPhysicalProps(o.mem, relParent, i, parentProps) } else { childProps = BuildChildPhysicalPropsScalar(o.mem, parent, i) } - after := o.setLowestCostTree(before, childProps) if after != before { if mutable == nil { diff --git a/pkg/sql/opt/xform/testdata/rules/join b/pkg/sql/opt/xform/testdata/rules/join index 4298e30ba478..7a3afa35394e 100644 --- a/pkg/sql/opt/xform/testdata/rules/join +++ b/pkg/sql/opt/xform/testdata/rules/join @@ -10965,7 +10965,7 @@ distribute ├── key: (2) ├── fd: ()-->(4), (2)-->(1,3), (3)~~>(1,2) ├── distribution: east - ├── input distribution: central,east,west + ├── input distribution: east └── anti-join (lookup abc_part) ├── columns: def_part.r:1!null d:2!null e:3 f:4!null ├── lookup expression @@ -11511,7 +11511,7 @@ distribute ├── key: (2) ├── fd: ()-->(4), (2)-->(1,3), (3)-->(1,2), (8)-->(7,9-11), (9)~~>(7,8,10), (3)==(8), (8)==(3) ├── distribution: east - ├── input distribution: central,east,west + ├── input distribution: east └── project ├── columns: v:11!null def_part.r:1!null d:2!null e:3!null f:4!null abc_part.r:7!null a:8!null b:9 c:10 ├── immutable diff --git a/pkg/sql/opt_catalog.go b/pkg/sql/opt_catalog.go index 0eb86fa7e540..2ef646b1892e 100644 --- a/pkg/sql/opt_catalog.go +++ b/pkg/sql/opt_catalog.go @@ -1249,6 +1249,70 @@ func (ot *optTable) IsPartitionAllBy() bool { return ot.desc.IsPartitionAllBy() } +// HomeRegion is part of the cat.Table interface. +func (ot *optTable) HomeRegion() (region string, ok bool) { + localityConfig := ot.desc.GetLocalityConfig() + if localityConfig == nil { + return "", false + } + regionalByTable := localityConfig.GetRegionalByTable() + if regionalByTable == nil { + return "", false + } + if regionalByTable.Region != nil { + return regionalByTable.Region.String(), true + } + if ot.zone.LeasePreferenceCount() != 1 { + return "", false + } + if ot.zone.LeasePreference(0).ConstraintCount() != 1 { + return "", false + } + if ot.zone.LeasePreference(0).Constraint(0).GetKey() != "region" { + return "", false + } + return ot.zone.LeasePreference(0).Constraint(0).GetValue(), true +} + +// IsGlobalTable is part of the cat.Table interface. +func (ot *optTable) IsGlobalTable() bool { + localityConfig := ot.desc.GetLocalityConfig() + if localityConfig == nil { + return false + } + return localityConfig.GetGlobal() != nil +} + +// IsRegionalByRow is part of the cat.Table interface. +func (ot *optTable) IsRegionalByRow() bool { + localityConfig := ot.desc.GetLocalityConfig() + if localityConfig == nil { + return false + } + return localityConfig.GetRegionalByRow() != nil +} + +// HomeRegionColName is part of the cat.Table interface. +func (ot *optTable) HomeRegionColName() (colName string, ok bool) { + localityConfig := ot.desc.GetLocalityConfig() + if localityConfig == nil { + return "", false + } + regionalByRowConfig := localityConfig.GetRegionalByRow() + if regionalByRowConfig == nil { + return "", false + } + if regionalByRowConfig.As == nil { + return "crdb_region", true + } + return *regionalByRowConfig.As, true +} + +// GetDatabaseID is part of the cat.Table interface. +func (ot *optTable) GetDatabaseID() descpb.ID { + return ot.desc.GetParentID() +} + // lookupColumnOrdinal returns the ordinal of the column with the given ID. A // cache makes the lookup O(1). func (ot *optTable) lookupColumnOrdinal(colID descpb.ColumnID) (int, error) { @@ -2189,6 +2253,31 @@ func (ot *optVirtualTable) IsPartitionAllBy() bool { return false } +// HomeRegion is part of the cat.Table interface. +func (ot *optVirtualTable) HomeRegion() (region string, ok bool) { + return "", false +} + +// IsGlobalTable is part of the cat.Table interface. +func (ot *optVirtualTable) IsGlobalTable() bool { + return false +} + +// IsRegionalByRow is part of the cat.Table interface. +func (ot *optVirtualTable) IsRegionalByRow() bool { + return false +} + +// HomeRegionColName is part of the cat.Table interface. +func (ot *optVirtualTable) HomeRegionColName() (colName string, ok bool) { + return "", false +} + +// GetDatabaseID is part of the cat.Table interface. +func (ot *optVirtualTable) GetDatabaseID() descpb.ID { + return 0 +} + // CollectTypes is part of the cat.DataSource interface. func (ot *optVirtualTable) CollectTypes(ord int) (descpb.IDs, error) { col := ot.desc.AllColumns()[ord] diff --git a/pkg/sql/pgwire/pgcode/codes.go b/pkg/sql/pgwire/pgcode/codes.go index 06e6b63558a3..d9831d7de05f 100644 --- a/pkg/sql/pgwire/pgcode/codes.go +++ b/pkg/sql/pgwire/pgcode/codes.go @@ -202,6 +202,8 @@ var ( Windowing = MakeCode("42P20") InvalidRecursion = MakeCode("42P19") InvalidForeignKey = MakeCode("42830") + QueryNotRunningInHomeRegion = MakeCode("42898") + QueryHasNoHomeRegion = MakeCode("42899") InvalidName = MakeCode("42602") NameTooLong = MakeCode("42622") ReservedName = MakeCode("42939") diff --git a/pkg/sql/region_util.go b/pkg/sql/region_util.go index e2e896f4f591..cedd81d3249f 100644 --- a/pkg/sql/region_util.go +++ b/pkg/sql/region_util.go @@ -1307,7 +1307,7 @@ func (p *planner) validateAllMultiRegionZoneConfigsInDatabase( ) } -// CurrentDatabaseRegionConfig is part of the eval.DatabaseCatalog interface. +// CurrentDatabaseRegionConfig is part of the eval.RegionOperator interface. // CurrentDatabaseRegionConfig uses the cache to synthesize the RegionConfig // and as such is intended for DML use. It returns nil // if the current database is not multi-region enabled. @@ -2258,3 +2258,20 @@ func (p *planner) checkNoRegionChangeUnderway( } return nil } + +// GetMultiregionConfig is part of the eval.Planner interface. +func (p *planner) GetMultiregionConfig(databaseID descpb.ID) (interface{}, bool) { + + regionConfig, err := SynthesizeRegionConfig( + p.EvalContext().Ctx(), + p.txn, + databaseID, + p.Descriptors(), + SynthesizeRegionConfigOptionUseCache, + ) + + if err != nil { + return nil /* regionConfig */, false + } + return ®ionConfig, true +} diff --git a/pkg/sql/sem/eval/BUILD.bazel b/pkg/sql/sem/eval/BUILD.bazel index d24f1a17da74..5025f39938b7 100644 --- a/pkg/sql/sem/eval/BUILD.bazel +++ b/pkg/sql/sem/eval/BUILD.bazel @@ -49,6 +49,7 @@ go_library( "//pkg/settings", "//pkg/settings/cluster", "//pkg/sql/catalog/catpb", + "//pkg/sql/catalog/descpb", "//pkg/sql/lex", "//pkg/sql/pgwire/pgcode", "//pkg/sql/pgwire/pgerror", diff --git a/pkg/sql/sem/eval/context.go b/pkg/sql/sem/eval/context.go index 3c4c019f128e..bf336d86fcdf 100644 --- a/pkg/sql/sem/eval/context.go +++ b/pkg/sql/sem/eval/context.go @@ -596,6 +596,12 @@ func (ec *Context) Ctx() context.Context { return ec.Context } +// BoundedStaleness returns true if this query uses bounded staleness. +func (ec *Context) BoundedStaleness() bool { + return ec.AsOfSystemTime != nil && + ec.AsOfSystemTime.BoundedStaleness +} + // ensureExpectedType will return an error if a datum does not match the // provided type. If the expected type is Any or if the datum is a Null // type, then no error will be returned. diff --git a/pkg/sql/sem/eval/deps.go b/pkg/sql/sem/eval/deps.go index f42fd291303a..430827875d2a 100644 --- a/pkg/sql/sem/eval/deps.go +++ b/pkg/sql/sem/eval/deps.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/security/username" "github.com/cockroachdb/cockroach/pkg/sql/catalog/catpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgnotice" "github.com/cockroachdb/cockroach/pkg/sql/privilege" "github.com/cockroachdb/cockroach/pkg/sql/roleoption" @@ -347,6 +348,12 @@ type Planner interface { privilegeObjectPath string, privilegeObjectType privilege.ObjectType, ) (*catpb.PrivilegeDescriptor, error) + + // GetMultiregionConfig synthesizes a new multiregion.RegionConfig describing + // the multiregion properties of the database identified via databaseID. The + // second return value is false if the database doesn't exist or is not + // multiregion. + GetMultiregionConfig(databaseID descpb.ID) (interface{}, bool) } // InternalRows is an iterator interface that's exposed by the internal diff --git a/pkg/sql/sessiondatapb/local_only_session_data.proto b/pkg/sql/sessiondatapb/local_only_session_data.proto index 272cdd5b1d22..f4614f8f186c 100644 --- a/pkg/sql/sessiondatapb/local_only_session_data.proto +++ b/pkg/sql/sessiondatapb/local_only_session_data.proto @@ -284,6 +284,10 @@ message LocalOnlySessionData { // CopyFromAtomicEnabled controls whether implicit txn copy from operations // are atomic or segmented. bool copy_from_atomic_enabled = 77; + // EnforceHomeRegion, when true, causes queries which scan rows from multiple + // regions, or which scan rows from a single home region, but initiated from + // a gateway region which differs from that home region, to error out. + bool enforce_home_region = 78; /////////////////////////////////////////////////////////////////////////// // WARNING: consider whether a session parameter you're adding needs to // diff --git a/pkg/sql/vars.go b/pkg/sql/vars.go index aee9958238b8..5428fe3d47eb 100644 --- a/pkg/sql/vars.go +++ b/pkg/sql/vars.go @@ -2190,6 +2190,23 @@ var varGen = map[string]sessionVar{ }, GlobalDefault: globalTrue, }, + + // CockroachDB extension. + `enforce_home_region`: { + GetStringVal: makePostgresBoolGetStringValFn(`enforce_home_region`), + Set: func(_ context.Context, m sessionDataMutator, s string) error { + b, err := paramparse.ParseBoolVar("enforce_home_region", s) + if err != nil { + return err + } + m.SetEnforceHomeRegion(b) + return nil + }, + Get: func(evalCtx *extendedEvalContext, _ *kv.Txn) (string, error) { + return formatBoolAsPostgresSetting(evalCtx.SessionData().EnforceHomeRegion), nil + }, + GlobalDefault: globalFalse, + }, } // We want test coverage for this on and off so make it metamorphic.