Skip to content

Commit

Permalink
sql: dynamically detect a query's home region with enforce_home_region
Browse files Browse the repository at this point in the history 
enforce_home_region phase 3
----

This extends the `enforce_home_region` session setting to not error out
right away when executing a locality-optimized search or join operation,
and the operation must read rows from a remote region. Instead, it will
retry the transaction, but with the local region, as marked in `evalCtx`,
updated to indicate one of the remote regions in the cluster. This causes
the locality-optimized Scan or Join operation to read from that remote
region first. A follower_read_timestamp() is used when re-running the
query with AOST, so no remote reads actually occur. If the query succeeds,
the current fake local region is reported back to the user in an error,
for example:
```
Query is not running in its home region.
Try running the query from region 'us-east-1'.
```
All of the remote regions in the cluster are tried until the home region
is found, or all remote regions are exhausted, in which case the original
"Query has no home region" error is returned.

This preview feature is only supported for SELECT queries.

Epic: CRDB-18645
Fixes: #83819

Release note (sql change): The enforce_home_region session setting was
extended with a new optional preview feature and session setting, which is
disabled by default, to dynamically detect and report the home region for
SELECT queries based on the locality of the queried rows, if different
from the gateway region.
  • Loading branch information
Mark Sirek committed Mar 7, 2023
1 parent 2afe786 commit d2594fc
Show file tree
Hide file tree
Showing 26 changed files with 547 additions and 29 deletions.
193 changes: 186 additions & 7 deletions pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,9 @@ statement ok
CREATE TABLE t1 (a INT, b INT, c INT, primary key(a)) LOCALITY REGIONAL BY ROW;

statement ok
INSERT INTO t1 (crdb_region, a, b, c) VALUES ('ap-southeast-2', 1, 1, 1);
INSERT INTO t1 (crdb_region, a, b, c) VALUES ('us-east-1', 1, 1, 1);
INSERT INTO t1 (crdb_region, a, b, c) VALUES ('ca-central-1', 2, 1, 1);
INSERT INTO t1 (crdb_region, a, b, c) VALUES ('ap-southeast-2', 3, 1, 1);

statement ok
CREATE TABLE parent (
Expand Down Expand Up @@ -75,6 +76,10 @@ CREATE TABLE messages_rbt (
INDEX msg_idx(message)
) LOCALITY REGIONAL BY TABLE

# Sleep so that the follow_read_timestamp() used by phase 3 dynamic checking
# of a query's home region isn't executing before the tables existed.
sleep 5s

statement ok
CREATE TABLE messages_rbr (
account_id INT NOT NULL,
Expand Down Expand Up @@ -844,22 +849,69 @@ INSERT INTO parent (crdb_region, p_id) VALUES('ca-central-1', 2);
statement ok
INSERT INTO child (crdb_region, c_id, c_p_id) VALUES('ap-southeast-2', 11, 2);

subtest enforce_home_region_phase_2
subtest enforce_home_region_phase_2_3

retry
statement ok
INSERT INTO messages_rbr (crdb_region, account_id, message) VALUES ('us-east-1', 2, 'Hola, Region!');
INSERT INTO messages_rbr (crdb_region, account_id, message) VALUES ('ca-central-1', 3, 'Guten Tag, Region!');

retry
statement ok
INSERT INTO messages_global SELECT * FROM messages_rbr WHERE crdb_region != 'ap-southeast-2'

statement ok
SET enforce_home_region = true;

# Querying the row in the local region succeeds.
query III retry
SELECT * FROM t1 WHERE a=1;
SELECT * FROM t1 WHERE a=3;
----
1 1 1
3 1 1

# Querying the row in the remote region errors out.
## Phase 3, dynamically detect a query's home region.
# Querying a row in a remote region errors out, and indicates the home region.
retry
statement error pq: Query has no home region\. Try using a lower LIMIT value or running the query from a different region\.
statement error pq: Query is not running in its home region\. Try running the query from region 'us-east-1'\.
SELECT * FROM t1 WHERE a=1;

# Phase 3, dynamically detect a query's home region.
# Querying a row in a remote region errors out, and indicates the home region.
retry
statement error pq: Query is not running in its home region\. Try running the query from region 'ca-central-1'\.
SELECT * FROM t1 WHERE a=2;

retry
statement ok
PREPARE s1 AS SELECT * FROM t1 WHERE a=$1

# Phase 3, dynamically detect a query's home region.
# Prepared statement accessing a row in a remote region errors out,
# and indicates the home region.
retry
statement error pq: Query is not running in its home region\. Try running the query from region 'ca-central-1'\.
EXECUTE s1('2')

statement ok
BEGIN

statement ok
SAVEPOINT foo

# Phase 3, dynamically detect a query's home region.
# An explicit transaction accessing a remote region errors out,
# and indicates the home region.
retry
statement error pq: Query is not running in its home region\. Try running the query from region 'ca-central-1'\.
SELECT * FROM t1 WHERE a=2;

# Verify we can still roll back to the savepoint.
statement ok
ROLLBACK TO SAVEPOINT foo

statement ok
COMMIT

# Locality-optimized join in the local region succeeds.
query III retry
SELECT * FROM parent p, child c WHERE p_id = c_p_id AND c_id = 10 LIMIT 1
Expand All @@ -886,6 +938,14 @@ SELECT rbr.message FROM messages_rbr rbr, (SELECT * FROM messages_rbt ORDER BY a
----
Hello, Region!

# Phase 3, dynamically detect a query's home region.
# Locality-optimized lookup join should report a query's home region.
retry
statement error pq: Query is not running in its home region\. Try running the query from region 'us-east-1'\.
SELECT rbr.message FROM messages_rbr rbr, (SELECT * FROM messages_global ORDER BY account_id LIMIT 1) rbg
WHERE rbr.account_id = rbg.account_id LIMIT 1


# Locality-optimized semijoin in the local region succeeds.
query T retry
SELECT message FROM (SELECT * FROM messages_rbt ORDER BY account_id LIMIT 1) rbt WHERE account_id IN
Expand All @@ -903,7 +963,7 @@ SELECT message FROM (SELECT * FROM messages_rbt ORDER BY account_id LIMIT 2) rbt

# Locality-optimized semijoin in the local region with an ordered join reader
# succeeds.
query I
query I retry
SELECT account_id FROM (SELECT * FROM messages_rbt ORDER BY account_id LIMIT 1) rbt WHERE account_id IN
(SELECT account_id FROM messages_rbr rbr) ORDER BY 1 LIMIT 1;
----
Expand Down Expand Up @@ -935,6 +995,13 @@ SELECT message FROM (SELECT * FROM messages_rbt ORDER BY account_id LIMIT 1) rbt
(SELECT account_id FROM messages_rbr rbr) LIMIT 1
----

# Phase 3, dynamically detect a query's home region.
# Locality-optimized lookup antijoin should report a query's home region.
retry
statement error pq: Query is not running in its home region\. Try running the query from region 'us-east-1'\.
SELECT message FROM (SELECT * FROM messages_global ORDER BY account_id LIMIT 1) rbg WHERE account_id NOT IN
(SELECT account_id FROM messages_rbr rbr) LIMIT 1

# Locality-optimized antijoin reading into a remote region fails.
retry
statement error pq: Query has no home region\. Try using a lower LIMIT value or running the query from a different region\.
Expand All @@ -960,5 +1027,117 @@ statement error pq: Query has no home region\. Try using a lower LIMIT value or
SELECT rbr.message FROM messages_rbr rbr INNER LOOKUP JOIN messages_rbt rbt
ON rbr.account_id = rbt.account_id LIMIT 2

# Run a locality-optimized scan that dynamically detects the home region with
# tracing.
retry
statement error pq: Query is not running in its home region\. Try running the query from region 'ca-central-1'\.
SET TRACING = "on", kv, results;
SELECT * FROM t1 WHERE a=2;
SET TRACING = off

# All of the batch requests should be sent to (n1,s1) only.
query T
SELECT
CASE WHEN message LIKE '%sending%'
THEN SUBSTRING(message FOR 14 FROM 6) || SUBSTRING(message FOR 20 FROM POSITION('to' IN message))
ELSE message END
FROM
[SHOW KV TRACE FOR SESSION] WITH ORDINALITY
WHERE
message LIKE 'output row%' OR message LIKE '%sending%' OR message LIKE 'execution%'
ORDER BY
"ordinality" ASC
----
sending batch to (n1,s1):1
execution failed after 0 rows: Query has no home region. Try using a lower LIMIT value or running the query from a different region.
sending batch to (n1,s1):1
output row: [2 1 1]

# Run a locality-optimized join that dynamically detects the home region with
# tracing.
retry
statement error pq: Query is not running in its home region\. Try running the query from region 'us-east-1'\.
SET TRACING = "on", kv, results;
SELECT rbr.message FROM messages_rbr rbr, (SELECT * FROM messages_global ORDER BY account_id LIMIT 1) rbg
WHERE rbr.account_id = rbg.account_id LIMIT 1;
SET TRACING = off

# All of the batch requests should be sent to (n1,s1) only.
query T
SELECT
CASE WHEN message LIKE '%sending%'
THEN SUBSTRING(message FOR 14 FROM POSITION('sending' IN message)) || SUBSTRING(message FOR 12 FROM POSITION('to' IN message))
ELSE message END
FROM
[SHOW KV TRACE FOR SESSION] WITH ORDINALITY
WHERE
message LIKE 'output row%' OR message LIKE '%sending%' OR message LIKE 'execution%'
ORDER BY
"ordinality" ASC
----
sending batch to (n1,s1):1
execution failed after 0 rows: Query has no home region. Try using a lower LIMIT value or running the query from a different region.
sending batch to (n1,s1):1
output row: [2 1 1]
sending batch to (n1,s1):1
output row: ['execution failed after 0 rows: Query has no home region. Try using a lower LIMIT value or running the query from a different region.']
sending batch to (n1,s1):1
output row: ['output row: [2 1 1]']
sending batch to (n1,s1):1
sending batch to (n1,s1):1
execution failed after 0 rows: Query has no home region. Try using a lower LIMIT value or running the query from a different region.
sending batch to (n1,s1):1
sending batch to (n1,s1):1
execution failed after 0 rows: Query has no home region. Try using a lower LIMIT value or running the query from a different region.
sending batch to (n1,s1):1
sending batch to (n1,s1):1
output row: ['Hola, Region!']

statement ok
BEGIN

statement ok
SAVEPOINT foo

# Phase 3, dynamically detect a query's home region.
# An explicit transaction accessing a remote region errors out,
# and indicates the home region.
statement error pq: Query is not running in its home region. Try running the query from region 'ca-central-1'\.
SELECT * FROM t1 WHERE a=2;

# Verify we can still roll back to the savepoint.
statement ok
ROLLBACK TO SAVEPOINT foo

statement ok
COMMIT

statement ok
BEGIN

statement ok
SAVEPOINT foo

# Read some rows in an explicit transaction, which invalidates use of
# switching to AOST follower_read_timestamp() internally in the next statement.
query III
SELECT * FROM t1 WHERE a=3;
----
3 1 1

# Phase 3, dynamically detect a query's home region.
# An explicit transaction accessing a remote region errors out,
# and indicates "Query has no home region" if any rows were read
# in the transaction already.
statement error pq: Query has no home region\. Try using a lower LIMIT value or running the query from a different region\.
SELECT * FROM t1 WHERE a=2;

# Verify we can still roll back to the savepoint.
statement ok
ROLLBACK TO SAVEPOINT foo

statement ok
COMMIT

statement ok
RESET enforce_home_region
13 changes: 13 additions & 0 deletions pkg/roachpb/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -706,6 +706,19 @@ func (l *Locality) Set(value string) error {
return nil
}

// CopyReplaceKeyValue makes a copy of this locality, replacing any tier in the
// copy having the specified `key` with the new specified `value`.
func (l *Locality) CopyReplaceKeyValue(key, value string) Locality {
tiers := make([]Tier, len(l.Tiers))
for i := range l.Tiers {
tiers[i] = l.Tiers[i]
if tiers[i].Key == key {
tiers[i].Value = value
}
}
return Locality{Tiers: tiers}
}

// Find searches the locality's tiers for the input key, returning its value if
// present.
func (l *Locality) Find(key string) (value string, ok bool) {
Expand Down
26 changes: 25 additions & 1 deletion pkg/sql/conn_executor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/sql/catalog/schematelemetry/schematelemetrycontroller"
"github.com/cockroachdb/cockroach/pkg/sql/clusterunique"
"github.com/cockroachdb/cockroach/pkg/sql/contention/txnidcache"
"github.com/cockroachdb/cockroach/pkg/sql/execinfra"
"github.com/cockroachdb/cockroach/pkg/sql/execstats"
"github.com/cockroachdb/cockroach/pkg/sql/idxrecommendations"
"github.com/cockroachdb/cockroach/pkg/sql/idxusage"
Expand Down Expand Up @@ -2982,6 +2983,7 @@ func errIsRetriable(err error) bool {
scerrors.ConcurrentSchemaChangeDescID(err) != descpb.InvalidID ||
errors.Is(err, retriableMinTimestampBoundUnsatisfiableError) ||
errors.Is(err, descidgen.ErrDescIDSequenceMigrationInProgress) ||
execinfra.IsDynamicQueryHasNoHomeRegionError(err) ||
descs.IsTwoVersionInvariantViolationError(err)
}

Expand Down Expand Up @@ -3016,6 +3018,14 @@ func (ex *connExecutor) makeErrEvent(err error, stmt tree.Statement) (fsm.Event,
}

retriable := errIsRetriable(err)
if retriable && execinfra.IsDynamicQueryHasNoHomeRegionError(err) {
// Retry only # of remote regions times if the retry is due to the
// enforce_home_region setting.
retriable = int(ex.state.mu.autoRetryCounter) < len(ex.planner.EvalContext().RemoteRegions)
if !retriable {
err = execinfra.MaybeGetNonRetryableDynamicQueryHasNoHomeRegionError(err)
}
}
if retriable {
var rc rewindCapability
var canAutoRetry bool
Expand Down Expand Up @@ -3269,16 +3279,30 @@ func (ex *connExecutor) resetPlanner(
ctx context.Context, p *planner, txn *kv.Txn, stmtTS time.Time,
) {
p.resetPlanner(ctx, txn, stmtTS, ex.sessionData(), ex.state.mon)
autoRetryReason := ex.state.mu.autoRetryReason
// If we are retrying due to an unsatisfiable timestamp bound which is
// retriable, it means we were unable to serve the previous minimum timestamp
// as there was a schema update in between. When retrying, we want to keep the
// same minimum timestamp for the AOST read, but set the maximum timestamp
// to the point just before our failed read to ensure we don't try to read
// data which may be after the schema change when we retry.
var minTSErr *kvpb.MinTimestampBoundUnsatisfiableError
if err := ex.state.mu.autoRetryReason; err != nil && errors.As(err, &minTSErr) {
// Make sure the default locality specifies the actual gateway region at the
// start of query compilation. It could have been overridden to a remote
// region when the enforce_home_region session setting is true.
p.EvalContext().Locality = p.EvalContext().OriginalLocality
if err := autoRetryReason; err != nil && errors.As(err, &minTSErr) {
nextMax := minTSErr.MinTimestampBound
ex.extraTxnState.descCollection.SetMaxTimestampBound(nextMax)
} else if execinfra.IsDynamicQueryHasNoHomeRegionError(autoRetryReason) {
if int(ex.state.mu.autoRetryCounter) <= len(p.EvalContext().RemoteRegions) {
// Set a fake gateway region for use by the optimizer to inform its
// decision on which region to access first in locality-optimized scan
// and join operations. This setting does not affect the distsql planner,
// and local plans will continue to be run from the actual gateway region.
p.EvalContext().Locality =
p.EvalContext().Locality.CopyReplaceKeyValue("region", string(p.EvalContext().RemoteRegions[ex.state.mu.autoRetryCounter-1]))
}
} else if newTxn := txn == nil || p.extendedEvalCtx.Txn != txn; newTxn {
// Otherwise, only change the historical timestamps if this is a new txn.
// This is because resetPlanner can be called multiple times for the same
Expand Down
Loading

0 comments on commit d2594fc

Please sign in to comment.