diff --git a/pkg/kv/integration_test.go b/pkg/kv/integration_test.go index c1750fe1eecb..f8600b54f2bd 100644 --- a/pkg/kv/integration_test.go +++ b/pkg/kv/integration_test.go @@ -51,6 +51,7 @@ import ( // 3) Another txn runs into the intents on r2, tries to push, and succeeds // because the txn record doesn't exist yet. It writes the txn record as // Aborted. +// TODO(nvanbenschoten): This test will change when #25437 is fully addressed. // 4) If the Begin were to execute now, it'd discover the Aborted txn and return // a retryable TxnAbortedError. But it doesn't execute now; it's still delayed // somehow. @@ -67,6 +68,10 @@ import ( // result of this convoluted scenario. func TestDelayedBeginRetryable(t *testing.T) { defer leaktest.AfterTest(t)() + t.Skip(`WIP: @andrei what should we do here? Now that the timestamp + cache check is below the abort span, we're hitting ABORT_REASON_ABORT_SPAN + instead. Can we rely on the abort span or do we need to clear it and test + that we still hit the timestamp cache without its protection?`) // Here's how this test is gonna go: // - We're going to send a BeginTxn+Put(a)+Put(c). The first two will be split @@ -183,7 +188,7 @@ func TestDelayedBeginRetryable(t *testing.T) { if _, ok := pErr.GetDetail().(*roachpb.HandledRetryableTxnError); !ok { t.Fatalf("expected HandledRetryableTxnError, got: %v", pErr) } - exp := "TransactionAbortedError(ABORT_REASON_ALREADY_COMMITTED_OR_ROLLED_BACK_POSSIBLE_REPLAY)" + exp := "TransactionAbortedError(ABORT_REASON_ABORT_SPAN)" if !testutils.IsPError(pErr, regexp.QuoteMeta(exp)) { t.Fatalf("expected %s, got: %s", exp, pErr) } diff --git a/pkg/kv/txn_interceptor_heartbeat.go b/pkg/kv/txn_interceptor_heartbeat.go index 60e71b0bcb1c..f0eeaf582aad 100644 --- a/pkg/kv/txn_interceptor_heartbeat.go +++ b/pkg/kv/txn_interceptor_heartbeat.go @@ -97,6 +97,16 @@ type txnHeartbeat struct { // EndTransaction can be elided - we want to allow multiple rollback attempts // to be sent and the first one stops the heartbeat loop. everSentBeginTxn bool + + // sendingEndTxn is set when an EndTransactionRequest is in flight to the + // server. It is used to signify to the heartbeat loop that it should ignore + // the results of any heartbeat request. + // + // The field is not needed for client-perceived correctness, but prevents the + // coordinator's local transaction status from flipping to ABORTED after a + // heartbeat that observes the effect of a concurrent EndTransactionRequest + // and then over to COMMITTED after the EndTransactionRequest returns. + sendingEndTxn bool } } @@ -203,6 +213,7 @@ func (h *txnHeartbeat) SendLocked( var elideEndTxn bool var commitTurnedToRollback bool if haveEndTxn { + h.mu.sendingEndTxn = true // Are we writing now or have we written in the past? elideEndTxn = !h.mu.everSentBeginTxn if elideEndTxn { @@ -252,6 +263,10 @@ func (h *txnHeartbeat) SendLocked( } } + if haveEndTxn { + h.mu.sendingEndTxn = false + } + if pErr != nil { return nil, pErr } @@ -451,6 +466,13 @@ func (h *txnHeartbeat) heartbeat(ctx context.Context) bool { log.VEvent(ctx, 2, "heartbeat") br, pErr := h.gatekeeper.SendLocked(ctx, ba) + // If there is an in-flight EndTransaction request, avoid updating our local + // state based on the result of heartbeat requests. Instead, wait for the + // EndTransaction to return. + if h.mu.sendingEndTxn { + return true + } + var respTxn *roachpb.Transaction if pErr != nil { log.VEventf(ctx, 2, "heartbeat failed: %s", pErr) @@ -459,20 +481,14 @@ func (h *txnHeartbeat) heartbeat(ctx context.Context) bool { // then we ignore the error. This is possible if the heartbeat loop was // started before a BeginTxn request succeeds because of ambiguity in the // first write request's response. + // + // TODO(nvanbenschoten): Remove this in 2.3. if tse, ok := pErr.GetDetail().(*roachpb.TransactionStatusError); ok && tse.Reason == roachpb.TransactionStatusError_REASON_TXN_NOT_FOUND { return true } - if pErr.GetTxn() != nil { - // It is not expected for a 2.1 node to return an error with a transaction - // in it. For one, heartbeats are not supposed to return - // TransactionAbortedErrors. - // TODO(andrei): Remove this in 2.2. - respTxn = pErr.GetTxn() - } else { - return true - } + respTxn = pErr.GetTxn() } else { respTxn = br.Responses[0].GetInner().(*roachpb.HeartbeatTxnResponse).Txn } @@ -494,12 +510,6 @@ func (h *txnHeartbeat) heartbeat(ctx context.Context) bool { // abortTxnAsyncLocked send an EndTransaction(commmit=false) asynchronously. // The asyncAbortCallbackLocked callback is also called. func (h *txnHeartbeat) abortTxnAsyncLocked(ctx context.Context) { - // Stop the heartbeat loop if it is still running. - if h.mu.txnEnd != nil { - close(h.mu.txnEnd) - h.mu.txnEnd = nil - } - if h.mu.txn.Status != roachpb.ABORTED { log.Fatalf(ctx, "abortTxnAsyncLocked called for non-aborted txn: %s", h.mu.txn) } diff --git a/pkg/roachpb/errors.go b/pkg/roachpb/errors.go index 111868f23ac4..f89bb21d47eb 100644 --- a/pkg/roachpb/errors.go +++ b/pkg/roachpb/errors.go @@ -407,15 +407,6 @@ func NewTransactionStatusError(msg string) *TransactionStatusError { } } -// NewTransactionNotFoundStatusError initializes a new TransactionStatusError with -// a REASON_TXN_NOT_FOUND reason. -func NewTransactionNotFoundStatusError() *TransactionStatusError { - return &TransactionStatusError{ - Msg: "txn record not found", - Reason: TransactionStatusError_REASON_TXN_NOT_FOUND, - } -} - // NewTransactionCommittedStatusError initializes a new TransactionStatusError // with a REASON_TXN_COMMITTED. func NewTransactionCommittedStatusError() *TransactionStatusError { diff --git a/pkg/roachpb/errors.pb.go b/pkg/roachpb/errors.pb.go index 5dab9cefa22b..fd0dda5350fa 100644 --- a/pkg/roachpb/errors.pb.go +++ b/pkg/roachpb/errors.pb.go @@ -36,10 +36,11 @@ const ( // transaction simply pushed us, or we failed to heartbeat for a while and // another txn (of any priority) considered us abandoned and pushed us. ABORT_REASON_ABORTED_RECORD_FOUND TransactionAbortedReason = 1 - // The BeginTransaction has a timestamp below the TxnSpanGCThreshold, so there - // might have been an ABORTED txn record that got GCed (so, we might be in the - // ABORT_REASON_ABORTED_RECORD_FOUND case and not know). - ABORT_REASON_BEGIN_TOO_OLD TransactionAbortedReason = 2 + // The request attempting to create a transaction record has a timestamp below + // the TxnSpanGCThreshold, so there might have been an ABORTED txn record that + // got GCed (so, we might be in the ABORT_REASON_ABORTED_RECORD_FOUND case and + // not know). + ABORT_REASON_NEW_TXN_RECORD_TOO_OLD TransactionAbortedReason = 2 // The client is trying to use a transaction that's already been aborted. The // TxnCoordSender detects this. Either the client is misusing a txn, or the // TxnCoordSender found out about the transaction being aborted async through @@ -70,7 +71,7 @@ const ( var TransactionAbortedReason_name = map[int32]string{ 0: "ABORT_REASON_UNKNOWN", 1: "ABORT_REASON_ABORTED_RECORD_FOUND", - 2: "ABORT_REASON_BEGIN_TOO_OLD", + 2: "ABORT_REASON_NEW_TXN_RECORD_TOO_OLD", 3: "ABORT_REASON_CLIENT_REJECT", 4: "ABORT_REASON_PUSHER_ABORTED", 5: "ABORT_REASON_ABORT_SPAN", @@ -80,7 +81,7 @@ var TransactionAbortedReason_name = map[int32]string{ var TransactionAbortedReason_value = map[string]int32{ "ABORT_REASON_UNKNOWN": 0, "ABORT_REASON_ABORTED_RECORD_FOUND": 1, - "ABORT_REASON_BEGIN_TOO_OLD": 2, + "ABORT_REASON_NEW_TXN_RECORD_TOO_OLD": 2, "ABORT_REASON_CLIENT_REJECT": 3, "ABORT_REASON_PUSHER_ABORTED": 4, "ABORT_REASON_ABORT_SPAN": 5, @@ -120,6 +121,7 @@ const ( RETRY_SERIALIZABLE TransactionRetryReason = 3 // A possible replay caused by duplicate begin txn or out-of-order // txn sequence number. + // TODO(nvanbenschoten): This is no longer in use. Remove. RETRY_POSSIBLE_REPLAY TransactionRetryReason = 4 // An asynchronous write was observed to have failed. RETRY_ASYNC_WRITE_FAILURE TransactionRetryReason = 5 @@ -9504,178 +9506,179 @@ var ( func init() { proto.RegisterFile("roachpb/errors.proto", fileDescriptorErrors) } var fileDescriptorErrors = []byte{ - // 2767 bytes of a gzipped FileDescriptorProto + // 2773 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0xcb, 0x73, 0xdb, 0xd6, 0xd5, 0x27, 0x24, 0x4a, 0x94, 0x8e, 0x5e, 0xd0, 0xb5, 0x22, 0xc3, 0x72, 0x4c, 0xd9, 0x72, 0x1e, 0x8e, 0xbf, 0x89, 0xf4, 0x8d, 0xf3, 0x79, 0xbe, 0x89, 0x9b, 0x2c, 0xf8, 0x80, 0x44, 0x48, 0x7c, 0x28, 0x20, 0x15, 0xc7, 0x49, 0x3b, 0x18, 0x88, 0xb8, 0xa2, 0x10, 0x93, 0x00, 0x7b, 0x01, 0x58, - 0xd4, 0xae, 0xcb, 0xec, 0xda, 0xee, 0xba, 0x6b, 0x66, 0xda, 0x4d, 0xa6, 0xdb, 0x4e, 0xfe, 0x86, - 0x2c, 0xbb, 0xcc, 0x74, 0x26, 0x9e, 0xd6, 0xdd, 0x74, 0xa6, 0xff, 0x81, 0x57, 0x9d, 0xfb, 0x00, - 0x08, 0x92, 0x00, 0xc3, 0x64, 0x45, 0xe2, 0x3c, 0x7e, 0xf7, 0xdc, 0x7b, 0xcf, 0x3d, 0xf7, 0x77, - 0x2e, 0x6c, 0x11, 0xd7, 0x6c, 0x5f, 0xf6, 0xcf, 0x0f, 0x30, 0x21, 0x2e, 0xf1, 0xf6, 0xfb, 0xc4, - 0xf5, 0x5d, 0xb4, 0xd9, 0x76, 0xdb, 0xcf, 0x99, 0x66, 0x5f, 0xe8, 0x77, 0x50, 0x68, 0x68, 0x99, - 0xbe, 0xc9, 0xcd, 0x76, 0xb6, 0x43, 0x59, 0x0f, 0xfb, 0x66, 0x4c, 0x7e, 0xdf, 0xf3, 0x5d, 0x62, - 0x76, 0xf0, 0x01, 0x76, 0x3a, 0xb6, 0x13, 0xfe, 0x50, 0xbb, 0x17, 0xed, 0xf6, 0x07, 0xc2, 0x48, - 0x09, 0x7c, 0xbb, 0x7b, 0x70, 0xd9, 0x6d, 0x1f, 0xf8, 0x76, 0x0f, 0x7b, 0xbe, 0xd9, 0xeb, 0x0b, - 0xcd, 0x56, 0xc7, 0xed, 0xb8, 0xec, 0xef, 0x01, 0xfd, 0xc7, 0xa5, 0x7b, 0xdf, 0xce, 0xc1, 0x8d, - 0xba, 0xeb, 0x57, 0xb1, 0xe9, 0xe1, 0x8a, 0xdb, 0xb5, 0x30, 0x51, 0x69, 0xc8, 0xa8, 0x0c, 0x39, - 0x82, 0xfb, 0x5d, 0xbb, 0x6d, 0x2a, 0xd2, 0x5d, 0xe9, 0xc1, 0xca, 0xa3, 0xb7, 0xf6, 0x27, 0xa2, - 0xdf, 0xd7, 0xb9, 0x45, 0x19, 0x7b, 0x6d, 0x62, 0xf7, 0x7d, 0x97, 0x14, 0xb3, 0xdf, 0xbd, 0xdc, - 0xcd, 0xe8, 0xa1, 0x2b, 0x3a, 0x82, 0xd5, 0x2e, 0x45, 0x36, 0x2e, 0x19, 0xb4, 0x32, 0x37, 0x3b, - 0x94, 0xbe, 0xd2, 0x1d, 0xc6, 0x84, 0x1e, 0xc3, 0x12, 0x31, 0x9d, 0x0e, 0x36, 0x6c, 0x4b, 0x99, - 0xbf, 0x2b, 0x3d, 0x98, 0x2f, 0xee, 0xd0, 0x91, 0x5e, 0xbd, 0xdc, 0xcd, 0xe9, 0x54, 0xae, 0x95, - 0x5f, 0x0f, 0xff, 0xea, 0x39, 0x66, 0xab, 0x59, 0x68, 0x1f, 0x16, 0x18, 0x8a, 0x92, 0x65, 0x03, - 0x2b, 0x09, 0x03, 0xb3, 0x99, 0xeb, 0xdc, 0x0c, 0xdd, 0x07, 0x68, 0x07, 0x9e, 0xef, 0xf6, 0x8c, - 0x9e, 0xd7, 0x51, 0x16, 0xee, 0x4a, 0x0f, 0x96, 0xc5, 0x94, 0x96, 0xb9, 0xbc, 0xe6, 0x75, 0x9e, - 0x64, 0xff, 0xfd, 0xf5, 0xae, 0xb4, 0xf7, 0x26, 0x6c, 0xd5, 0x5d, 0x0b, 0x9f, 0x39, 0xe6, 0x0b, - 0xd3, 0xee, 0x9a, 0xe7, 0x5d, 0xcc, 0x16, 0x4e, 0x68, 0x77, 0xe1, 0xe6, 0x99, 0xe3, 0x05, 0xfd, - 0xbe, 0x4b, 0x7c, 0x6c, 0xe9, 0xf8, 0xd7, 0x01, 0xf6, 0xfc, 0xb8, 0xc1, 0x57, 0x12, 0x20, 0x16, - 0x6e, 0xdd, 0xf5, 0x0f, 0xdd, 0xc0, 0xb1, 0xf8, 0xb2, 0xc7, 0xe7, 0x29, 0xcd, 0x3e, 0xcf, 0xc7, - 0xb0, 0x44, 0x93, 0x83, 0xb9, 0xcd, 0x8d, 0xba, 0x35, 0xa9, 0x9c, 0xbb, 0x89, 0xbf, 0x7a, 0x8e, - 0xd9, 0x6a, 0x96, 0x08, 0xe5, 0x8f, 0x73, 0xf0, 0x06, 0x43, 0x3c, 0xc1, 0xd7, 0x35, 0xdb, 0xeb, - 0x99, 0x7e, 0xfb, 0x92, 0x47, 0xf3, 0x01, 0x6c, 0x12, 0x1e, 0xba, 0xe1, 0xf9, 0x26, 0xf1, 0x8d, - 0xe7, 0xf8, 0x9a, 0x85, 0xb5, 0x5a, 0xcc, 0xbd, 0x7e, 0xb9, 0x3b, 0x7f, 0x82, 0xaf, 0xf5, 0x0d, - 0x61, 0xd1, 0xa4, 0x06, 0x27, 0xf8, 0x1a, 0x1d, 0x40, 0x28, 0x32, 0xb0, 0x63, 0x31, 0x97, 0xb9, - 0x51, 0x97, 0x35, 0xa1, 0x57, 0x1d, 0x8b, 0x3a, 0xd4, 0x40, 0xee, 0x89, 0x61, 0xb1, 0x65, 0xb0, - 0x29, 0xb1, 0x3d, 0x5e, 0x79, 0xb4, 0x97, 0x94, 0x28, 0x54, 0x1f, 0x4b, 0x93, 0x8d, 0xa1, 0x2f, - 0x53, 0xa1, 0x13, 0xd8, 0xf0, 0x82, 0x4e, 0x07, 0x7b, 0x7e, 0x84, 0x96, 0x9d, 0x19, 0x6d, 0x3d, - 0x72, 0x65, 0x1a, 0xb1, 0x42, 0xff, 0x99, 0x83, 0x3d, 0x1d, 0x9b, 0xd6, 0x53, 0xdb, 0xbf, 0xb4, - 0x9d, 0x33, 0xa7, 0x8d, 0x89, 0x6f, 0xda, 0x8e, 0x7f, 0xad, 0x39, 0x3e, 0x26, 0x2f, 0xcc, 0x2e, - 0x5f, 0xae, 0x63, 0x58, 0x27, 0xd8, 0xb4, 0x8c, 0xe8, 0xe4, 0x89, 0xa3, 0x73, 0x27, 0x36, 0x30, - 0x3d, 0x9e, 0xfb, 0x97, 0xdd, 0xf6, 0x7e, 0x2b, 0x34, 0x12, 0x09, 0xb6, 0x46, 0x5d, 0x23, 0x21, - 0xd2, 0x01, 0xe1, 0x81, 0xed, 0xf9, 0xb6, 0xd3, 0x89, 0xe1, 0xcd, 0xcd, 0x8e, 0xb7, 0x19, 0xba, - 0x0f, 0x31, 0x8b, 0xb0, 0xd6, 0x33, 0x07, 0x31, 0xb8, 0xf9, 0x19, 0xe0, 0xf4, 0xd5, 0x9e, 0x39, - 0x18, 0x62, 0x7c, 0x01, 0x37, 0xdc, 0x73, 0x0f, 0x93, 0x17, 0x38, 0x36, 0x4f, 0x4f, 0xc9, 0xde, - 0x9d, 0x4f, 0x39, 0xd8, 0x0d, 0x61, 0x3d, 0x1e, 0x1f, 0x72, 0xc7, 0x15, 0x9e, 0x58, 0xed, 0x2f, - 0xe1, 0x66, 0x8b, 0x98, 0x8e, 0x67, 0xb6, 0x7d, 0xdb, 0x75, 0x0a, 0xe7, 0xec, 0x08, 0xf1, 0x15, - 0xd6, 0x60, 0x91, 0x60, 0xd3, 0x73, 0x1d, 0xb6, 0xb2, 0xeb, 0x8f, 0xfe, 0x27, 0x61, 0xc0, 0x49, - 0x5f, 0x9d, 0xb9, 0x88, 0x71, 0x05, 0x80, 0x18, 0xcb, 0x84, 0xad, 0x98, 0xfd, 0x69, 0xe0, 0x89, - 0xcc, 0x2f, 0x01, 0xf4, 0x03, 0xef, 0x12, 0x63, 0xc3, 0x1f, 0x38, 0x62, 0x1b, 0xf3, 0xd3, 0x07, - 0x0b, 0x0b, 0x05, 0xf7, 0x6b, 0x0d, 0xc2, 0x21, 0x2e, 0xe0, 0x8d, 0x98, 0x95, 0x8e, 0x7d, 0x72, - 0xcd, 0xc7, 0x38, 0x1a, 0x9b, 0xcc, 0x7b, 0xd3, 0xf1, 0x99, 0xe7, 0x94, 0xa9, 0x7c, 0x2f, 0xc1, - 0x76, 0xcc, 0xbc, 0xe9, 0x9b, 0x7e, 0xe0, 0xf1, 0x91, 0xb6, 0x61, 0x9e, 0xd6, 0x33, 0x29, 0x56, - 0xcf, 0xa8, 0x00, 0xd5, 0xa3, 0x08, 0xe6, 0x58, 0x04, 0xff, 0x3b, 0x3d, 0x82, 0x18, 0xe4, 0x7e, - 0x52, 0x20, 0x7b, 0xa7, 0xb0, 0xc8, 0xe5, 0x08, 0xc1, 0xba, 0xae, 0x16, 0x9a, 0x8d, 0xba, 0x71, - 0x56, 0x3f, 0xa9, 0x37, 0x9e, 0xd6, 0xe5, 0x0c, 0x52, 0x60, 0x4b, 0xc8, 0x5a, 0x9f, 0xd5, 0x8d, - 0x7a, 0xa3, 0x65, 0x1c, 0x36, 0xce, 0xea, 0x65, 0x59, 0x1a, 0xd3, 0x94, 0x1a, 0xb5, 0x9a, 0xd6, - 0x6a, 0xa9, 0x65, 0x79, 0x4e, 0x4c, 0xed, 0x19, 0xc8, 0x4f, 0x89, 0xed, 0x63, 0x7a, 0xdc, 0x1c, - 0x5e, 0x46, 0xd1, 0x87, 0x90, 0xb3, 0xd9, 0xa7, 0xa7, 0x48, 0x2c, 0xf9, 0x6e, 0x25, 0x04, 0xcf, - 0x1d, 0xc2, 0x5b, 0x49, 0xd8, 0x73, 0xd0, 0xe3, 0xec, 0xd2, 0x9c, 0x3c, 0xbf, 0xf7, 0x17, 0x49, - 0x60, 0xb7, 0x5c, 0xb7, 0xd1, 0x15, 0x69, 0x56, 0x80, 0xe5, 0x9f, 0x75, 0x86, 0x87, 0x5e, 0xa8, - 0x0e, 0xb2, 0xd9, 0xf6, 0x03, 0xb3, 0xfb, 0xf3, 0x4e, 0xef, 0x06, 0x77, 0x8e, 0xc4, 0x62, 0x21, - 0x76, 0x00, 0x35, 0xfa, 0xf4, 0x36, 0xb1, 0x09, 0xf6, 0x5a, 0x03, 0x27, 0x7e, 0xa3, 0x3c, 0x83, - 0xad, 0x92, 0xeb, 0x58, 0x36, 0xdd, 0xa9, 0x43, 0xd3, 0xee, 0x86, 0x67, 0xe6, 0x17, 0xb0, 0x2a, - 0x22, 0x79, 0x61, 0x76, 0x03, 0x2c, 0xe6, 0x93, 0x74, 0x15, 0x7e, 0x4a, 0xf5, 0xfa, 0x0a, 0xb7, - 0x66, 0x1f, 0x02, 0xfa, 0xaf, 0x12, 0x20, 0x7e, 0x4f, 0xe2, 0x2f, 0x71, 0x3b, 0x3a, 0x8d, 0x79, - 0xc8, 0xf5, 0xb0, 0xe7, 0x99, 0x1d, 0x3c, 0x92, 0x5a, 0xa1, 0x10, 0x7d, 0x04, 0xcb, 0xa2, 0xd2, - 0x63, 0x4b, 0x4c, 0x3e, 0xf5, 0x06, 0x0e, 0x57, 0x30, 0x72, 0x40, 0x4f, 0x60, 0x29, 0x2c, 0x61, - 0xa2, 0x50, 0xfd, 0x98, 0x73, 0x64, 0x2f, 0xc2, 0xfe, 0x7f, 0x58, 0x6e, 0x62, 0x67, 0xb6, 0x60, - 0x47, 0x92, 0xe2, 0x0a, 0xb6, 0x0a, 0xbd, 0x73, 0xbb, 0x13, 0xb8, 0x81, 0xa7, 0x63, 0x2f, 0xe8, - 0xfa, 0xb3, 0x4d, 0xf8, 0x43, 0x58, 0xb9, 0x22, 0x66, 0xbf, 0x8f, 0x2d, 0x03, 0x13, 0x32, 0x65, - 0xca, 0x0c, 0x4e, 0x07, 0x61, 0xac, 0x92, 0x70, 0x0f, 0xef, 0xd0, 0x9b, 0xf8, 0xc2, 0x3f, 0x22, - 0x6e, 0xd0, 0x2f, 0xe3, 0x2e, 0x0e, 0x97, 0x5a, 0xa8, 0x31, 0x6c, 0x0b, 0x9e, 0x54, 0x72, 0x09, - 0x09, 0xfa, 0x74, 0xab, 0x79, 0x64, 0xf7, 0x60, 0x99, 0x51, 0x4d, 0x63, 0xfc, 0x9c, 0x2f, 0x31, - 0x71, 0xcd, 0xeb, 0xa0, 0x3d, 0x58, 0xee, 0x13, 0xb7, 0x8d, 0x3d, 0x4f, 0xec, 0xc6, 0x52, 0x54, - 0xb1, 0x42, 0x71, 0x94, 0x49, 0x48, 0x0c, 0x13, 0x3f, 0x14, 0x1f, 0x03, 0x08, 0x5a, 0x17, 0x92, - 0x93, 0x85, 0x62, 0x5e, 0xb0, 0x8c, 0x65, 0x61, 0xcf, 0x78, 0xc6, 0xf0, 0x83, 0x6e, 0x27, 0xff, - 0x1b, 0x42, 0x7f, 0x02, 0x88, 0xb1, 0x90, 0x09, 0xd6, 0x13, 0xd1, 0x17, 0xe9, 0xa7, 0xd2, 0x97, - 0x1a, 0xa5, 0x5a, 0x97, 0xa6, 0x63, 0x75, 0x69, 0xa5, 0xf7, 0xc9, 0x75, 0xc4, 0xc5, 0xd0, 0x23, - 0xc8, 0xf6, 0x55, 0x42, 0xa6, 0xa4, 0x3c, 0xb3, 0x13, 0xeb, 0xc0, 0x6c, 0xf7, 0x7e, 0x90, 0x40, - 0xa9, 0x8c, 0xa1, 0x85, 0x27, 0x2d, 0xb5, 0x90, 0x7e, 0x01, 0x8b, 0xfe, 0xc0, 0x09, 0xd9, 0xd7, - 0x6a, 0xb1, 0x4c, 0x55, 0x7f, 0x7f, 0xb9, 0xfb, 0x41, 0xc7, 0xf6, 0x2f, 0x83, 0xf3, 0xfd, 0xb6, - 0xdb, 0x3b, 0x88, 0x06, 0xb7, 0xce, 0x87, 0xff, 0x0f, 0xfa, 0xcf, 0x3b, 0x07, 0x8c, 0xb2, 0x07, - 0x81, 0x6d, 0xed, 0x9f, 0x9d, 0x69, 0xe5, 0x57, 0x2f, 0x77, 0x17, 0x5a, 0x03, 0x47, 0x2b, 0xeb, - 0x0b, 0xfe, 0xc0, 0xd1, 0x2c, 0x74, 0x08, 0x2b, 0xfe, 0xb0, 0x08, 0x8b, 0xb3, 0x30, 0xdb, 0x65, - 0x14, 0x77, 0x14, 0xcb, 0xf5, 0x2e, 0xec, 0xb6, 0x06, 0x4e, 0xa1, 0x4b, 0xe9, 0xc6, 0xb5, 0xea, - 0xb4, 0xdd, 0x80, 0x72, 0x18, 0x91, 0x67, 0xf1, 0x64, 0xfb, 0xbd, 0x04, 0x5b, 0xb4, 0x7e, 0x76, - 0x30, 0x69, 0xbc, 0xc0, 0xe4, 0xa2, 0xeb, 0x5e, 0xf1, 0x45, 0xb8, 0x05, 0xf3, 0x09, 0x3c, 0x90, - 0xca, 0xd0, 0x7b, 0xb0, 0xd6, 0x0e, 0x08, 0xc1, 0x8e, 0x2f, 0x8a, 0x0d, 0x27, 0xa3, 0x3c, 0x98, - 0x55, 0xa1, 0x62, 0x95, 0x05, 0xbd, 0x0f, 0x1b, 0xb6, 0xd3, 0x26, 0xb8, 0x37, 0x34, 0x9e, 0x8f, - 0x19, 0xaf, 0x47, 0xca, 0x78, 0x21, 0xaa, 0xc1, 0x66, 0xcd, 0x1e, 0x60, 0xab, 0x19, 0xb4, 0x69, - 0xc6, 0x86, 0xbb, 0x9c, 0x13, 0x07, 0xe9, 0xc7, 0x36, 0x5a, 0x0f, 0x0d, 0x05, 0xdc, 0x37, 0x12, - 0xdc, 0x2e, 0x52, 0xee, 0x38, 0x2c, 0xbf, 0xf8, 0xc2, 0x25, 0xf8, 0xa8, 0x14, 0xdd, 0x03, 0xad, - 0x9f, 0x75, 0x0f, 0x0c, 0xf9, 0x12, 0x85, 0xb8, 0x24, 0xd8, 0xa3, 0x0d, 0xd0, 0x4f, 0xb9, 0x00, - 0x86, 0x5e, 0x22, 0xd6, 0xcf, 0x00, 0xf1, 0xdb, 0xac, 0x66, 0x7b, 0x9e, 0xed, 0x74, 0x78, 0x84, - 0x1f, 0xc1, 0xea, 0x15, 0x71, 0x9d, 0x8e, 0xc1, 0xef, 0x36, 0x11, 0x64, 0xfa, 0x55, 0xa8, 0xaf, - 0x30, 0x73, 0xfe, 0x31, 0xec, 0x64, 0x6a, 0x98, 0x74, 0xb0, 0xe6, 0x9c, 0x12, 0xb7, 0x43, 0xc2, - 0x75, 0x15, 0xda, 0xd7, 0x12, 0xdc, 0x60, 0x5c, 0xf8, 0x10, 0x8b, 0x13, 0xc1, 0x47, 0x3e, 0x19, - 0x63, 0x2f, 0xef, 0xa7, 0xb1, 0xeb, 0x51, 0xbf, 0x64, 0xe2, 0xf0, 0x5b, 0x29, 0x62, 0x0e, 0x3b, - 0xb0, 0x2d, 0xb8, 0x80, 0xae, 0x9e, 0x56, 0xb5, 0x52, 0xc1, 0xd0, 0xd5, 0x5a, 0xe3, 0x53, 0xb5, - 0x2c, 0x67, 0xd0, 0x36, 0xa0, 0x50, 0x57, 0xa8, 0x1f, 0xa9, 0x46, 0xf3, 0xb4, 0xaa, 0xb5, 0x64, - 0x09, 0xdd, 0x84, 0x1b, 0x23, 0xf2, 0x9a, 0xaa, 0x1f, 0x51, 0xfa, 0x10, 0x23, 0x16, 0x7a, 0xe1, - 0xb0, 0x65, 0x34, 0xeb, 0x85, 0xd3, 0x66, 0xa5, 0xd1, 0x92, 0xe7, 0x51, 0x1e, 0x76, 0x84, 0xa6, - 0xda, 0x38, 0xd2, 0x4a, 0x85, 0xaa, 0xd1, 0x38, 0x6d, 0x1a, 0x35, 0xad, 0xd9, 0xd4, 0xea, 0x47, - 0x72, 0x56, 0x4c, 0xfe, 0xcf, 0x5b, 0xb0, 0xc2, 0xc2, 0x2e, 0x63, 0xdf, 0xb4, 0xbb, 0x48, 0x07, - 0xd9, 0x71, 0x7d, 0x63, 0xa4, 0xa7, 0xe5, 0x4b, 0xfe, 0x4e, 0xc2, 0xf4, 0x13, 0xfa, 0xea, 0x4a, - 0x46, 0x5f, 0x77, 0x46, 0xc4, 0xa8, 0x01, 0x1b, 0xbc, 0xe5, 0xa3, 0xc8, 0x17, 0xb4, 0x28, 0x8a, - 0x3c, 0x79, 0x3b, 0x6d, 0x45, 0x47, 0x8a, 0x67, 0x85, 0xb6, 0x0e, 0x71, 0x29, 0xfa, 0x0c, 0x10, - 0x07, 0x7c, 0x8e, 0xaf, 0x8d, 0xb0, 0x3b, 0x12, 0x65, 0xe3, 0x41, 0x1a, 0xe6, 0x78, 0xef, 0x57, - 0xc9, 0xe8, 0x32, 0x19, 0x53, 0xa0, 0xdf, 0x48, 0x70, 0x97, 0x75, 0x38, 0x57, 0xac, 0x11, 0x32, - 0x82, 0x61, 0x27, 0xc4, 0x12, 0x90, 0xb6, 0x42, 0xa2, 0xd9, 0x7a, 0x9c, 0xd8, 0xe3, 0xff, 0x58, - 0x0b, 0x55, 0xc9, 0xe8, 0x77, 0xc8, 0x34, 0x2b, 0xf4, 0x2b, 0xb8, 0x11, 0xab, 0x69, 0x86, 0xc9, - 0x19, 0x3e, 0x6b, 0xd5, 0x57, 0x1e, 0x3d, 0x9c, 0xa9, 0x1d, 0x08, 0x47, 0x42, 0xfe, 0x84, 0x0a, - 0xb5, 0x40, 0x8e, 0xc3, 0x53, 0x2e, 0xaf, 0x2c, 0x32, 0xec, 0x77, 0xa7, 0x63, 0x47, 0xad, 0x43, - 0x25, 0xa3, 0x6f, 0xf8, 0xa3, 0x72, 0xf4, 0x14, 0x36, 0xe3, 0xa8, 0x84, 0x9e, 0x06, 0x25, 0x97, - 0xba, 0x21, 0x89, 0xed, 0x02, 0xdd, 0x10, 0x7f, 0x4c, 0x81, 0x3e, 0x87, 0xf8, 0x24, 0x68, 0x93, - 0xee, 0x07, 0x9e, 0xb2, 0xc4, 0x90, 0xdf, 0x9b, 0x99, 0xcc, 0x57, 0x32, 0x7a, 0x3c, 0x3e, 0xae, - 0x41, 0x15, 0x5a, 0x5a, 0x6c, 0x1f, 0x87, 0xa5, 0x65, 0x99, 0xa1, 0xde, 0x4f, 0x40, 0x1d, 0xe7, - 0xe6, 0x95, 0x0c, 0x2d, 0x33, 0x91, 0x0c, 0x69, 0xb0, 0xc6, 0x91, 0x7c, 0xd7, 0x35, 0x68, 0x1d, - 0x84, 0xe9, 0x50, 0x31, 0xd6, 0x11, 0x41, 0x71, 0x19, 0x3d, 0x2c, 0x6e, 0xdf, 0x20, 0x82, 0x01, - 0xb3, 0xe6, 0x6c, 0x25, 0xf5, 0xb0, 0x4c, 0x52, 0x65, 0x7a, 0x58, 0xdc, 0xb8, 0x94, 0x6e, 0x78, - 0x3b, 0x64, 0xcd, 0xc6, 0x05, 0xa3, 0xcd, 0xca, 0x6a, 0xea, 0x86, 0x27, 0x11, 0x6c, 0xba, 0xe1, - 0xed, 0x51, 0x39, 0xaa, 0xc3, 0x3a, 0xaf, 0x11, 0x44, 0x10, 0x66, 0x65, 0x2d, 0x35, 0xca, 0x49, - 0x62, 0x4d, 0xa3, 0xec, 0xc6, 0xa5, 0x34, 0x4a, 0xc7, 0xb5, 0xb0, 0x11, 0x0c, 0x5f, 0x9b, 0x94, - 0xf5, 0xd4, 0x28, 0x93, 0xde, 0xa5, 0x68, 0x94, 0xce, 0xa8, 0x9c, 0xd2, 0x23, 0x0f, 0x3b, 0x96, - 0xb2, 0xc1, 0x90, 0xde, 0x4c, 0x40, 0x8a, 0xe8, 0x73, 0x25, 0xa3, 0x33, 0x5b, 0x5e, 0x5c, 0x2e, - 0x7c, 0xa3, 0x43, 0x29, 0xaa, 0x61, 0x71, 0x8e, 0xaa, 0xc8, 0x53, 0x8a, 0x4b, 0x02, 0x9d, 0xe5, - 0xc5, 0x65, 0x54, 0x41, 0x73, 0x39, 0xe4, 0x97, 0xed, 0x88, 0xdd, 0x2a, 0x9b, 0xa9, 0xb9, 0x9c, - 0xcc, 0x84, 0x69, 0x2e, 0x93, 0x71, 0x0d, 0xab, 0xb1, 0x02, 0x3b, 0xcc, 0x41, 0x94, 0x5e, 0x63, - 0x27, 0xb8, 0x2f, 0xab, 0xb1, 0x71, 0x29, 0xdd, 0x10, 0x33, 0xec, 0x10, 0x0c, 0xc2, 0x5a, 0x04, - 0x65, 0x27, 0x75, 0x43, 0x92, 0x9a, 0x09, 0xba, 0x21, 0xe6, 0xa8, 0x9c, 0x86, 0xc9, 0x79, 0xf0, - 0xf0, 0x2a, 0xb8, 0x9d, 0x1a, 0xe6, 0x24, 0x8f, 0xa6, 0x61, 0x7a, 0x71, 0x29, 0xea, 0xc2, 0x6d, - 0xc1, 0x8c, 0x79, 0xd1, 0xa1, 0xdb, 0x4e, 0x0f, 0x8d, 0xc1, 0xba, 0x02, 0xe5, 0x4d, 0x06, 0x9e, - 0xf4, 0x88, 0x92, 0xc6, 0x80, 0x2b, 0x19, 0x5d, 0xb9, 0x4c, 0x63, 0xc7, 0x2d, 0x90, 0x6d, 0x4e, - 0x18, 0x0d, 0x57, 0x30, 0x46, 0x65, 0x37, 0x75, 0x51, 0x92, 0xb8, 0x25, 0x5d, 0x14, 0x7b, 0x54, - 0x4e, 0x2b, 0x7e, 0x30, 0x7c, 0x4a, 0x35, 0x44, 0x83, 0xa8, 0xdc, 0x4d, 0xad, 0xf8, 0x29, 0x0f, - 0xaf, 0xb4, 0xe2, 0x07, 0x13, 0x2a, 0x74, 0x02, 0x6b, 0x3d, 0x4a, 0x29, 0x0d, 0x8f, 0x73, 0x4a, - 0xe5, 0x5e, 0xea, 0x1b, 0xf5, 0x04, 0xf5, 0xac, 0x64, 0xf4, 0xd5, 0x5e, 0x4c, 0x88, 0xbe, 0x00, - 0x39, 0x6a, 0xf7, 0x8d, 0x73, 0xc6, 0x25, 0x95, 0x3d, 0x86, 0xb7, 0x9f, 0x80, 0x37, 0x85, 0x7a, - 0xb2, 0x5b, 0x64, 0x54, 0x83, 0xae, 0xe0, 0x0e, 0xdd, 0x3a, 0x93, 0x53, 0x77, 0x03, 0x0f, 0xb9, - 0xbb, 0xd8, 0xce, 0xfb, 0x6c, 0xa4, 0x47, 0x49, 0x75, 0x7f, 0x3a, 0xe3, 0xaf, 0x64, 0xf4, 0x1d, - 0x3f, 0xd5, 0x84, 0x56, 0x33, 0x7e, 0x07, 0x50, 0x36, 0x41, 0xb9, 0xa7, 0xf2, 0x56, 0x6a, 0x56, - 0x4e, 0x72, 0x54, 0x9a, 0x95, 0x76, 0x5c, 0x8a, 0xce, 0x60, 0xb3, 0x47, 0x09, 0xa7, 0x61, 0x3b, - 0x46, 0x5f, 0x50, 0x4e, 0xe5, 0xed, 0xd4, 0x44, 0x49, 0x22, 0xa7, 0x74, 0x7d, 0x7a, 0xa3, 0x72, - 0xf4, 0x89, 0x20, 0x52, 0x17, 0x38, 0x4c, 0x77, 0xe5, 0x9d, 0x54, 0x6e, 0x96, 0x40, 0x4d, 0x29, - 0x37, 0x8b, 0x00, 0x98, 0x98, 0xb3, 0xc0, 0x62, 0x0e, 0x16, 0x58, 0x83, 0x72, 0x9c, 0x5d, 0xda, - 0x96, 0x6f, 0x1e, 0x67, 0x97, 0x6e, 0xc9, 0x3b, 0xc7, 0xd9, 0xa5, 0x3b, 0x72, 0xfe, 0x38, 0xbb, - 0x94, 0x97, 0x77, 0xf7, 0x0e, 0x18, 0x4b, 0x3c, 0x75, 0x3d, 0x76, 0x07, 0xa0, 0x1d, 0x58, 0xb0, - 0x1d, 0x0b, 0x0f, 0x44, 0x93, 0xcc, 0xa9, 0x2e, 0x17, 0x09, 0x5e, 0xf9, 0xed, 0x3c, 0x2c, 0xcc, - 0xf6, 0xa4, 0xf0, 0xcb, 0x51, 0xbe, 0x43, 0x30, 0x7b, 0x88, 0x67, 0x6c, 0x6e, 0x3d, 0x71, 0x03, - 0x46, 0xc8, 0x03, 0x33, 0x0e, 0x1f, 0x5c, 0xfd, 0x09, 0x0d, 0x2a, 0xc1, 0x5a, 0xe0, 0xe0, 0x41, - 0xdf, 0xf5, 0xb0, 0xc5, 0x2e, 0xd3, 0xec, 0x2c, 0xcd, 0xa5, 0xbe, 0x1a, 0x39, 0xd1, 0x2b, 0xf4, - 0x00, 0x56, 0x5c, 0x62, 0x77, 0x6c, 0xc7, 0xa0, 0x17, 0x0c, 0xa3, 0x62, 0x0b, 0xc5, 0x75, 0x3a, - 0xe6, 0xeb, 0x97, 0xbb, 0x8b, 0xf4, 0x32, 0xd2, 0xca, 0x3a, 0x70, 0x13, 0xfa, 0x85, 0x3e, 0x82, - 0x45, 0x8b, 0xf1, 0x69, 0x41, 0xad, 0xf2, 0x69, 0xfd, 0x1a, 0x67, 0xdd, 0x61, 0xaf, 0xc0, 0x7d, - 0xd0, 0xff, 0x85, 0xab, 0x9b, 0x9b, 0xe6, 0x1c, 0x6e, 0x86, 0x58, 0x77, 0xf4, 0x18, 0xe6, 0x1d, - 0xf7, 0x4a, 0x50, 0xa3, 0x99, 0x3a, 0x30, 0x6a, 0xff, 0x64, 0xe9, 0x0f, 0x5f, 0xef, 0x66, 0x86, - 0x2f, 0x43, 0x0f, 0x7f, 0x98, 0x03, 0x25, 0xed, 0x81, 0x99, 0x76, 0x1b, 0x85, 0x62, 0x43, 0x6f, - 0x19, 0x13, 0x4f, 0x9f, 0x6f, 0xc3, 0xbd, 0x11, 0x0d, 0xfb, 0x50, 0xcb, 0x86, 0xae, 0x96, 0x1a, - 0x7a, 0x39, 0x7a, 0x07, 0xcd, 0xc3, 0xce, 0x88, 0x59, 0x51, 0x3d, 0xd2, 0xea, 0x46, 0xab, 0xd1, - 0x30, 0x1a, 0x55, 0xda, 0xce, 0x8c, 0xeb, 0x4b, 0x55, 0x4d, 0xad, 0xd3, 0xaf, 0x63, 0xb5, 0x44, - 0x9b, 0x9a, 0x5d, 0xb8, 0x3d, 0xa2, 0x3f, 0x3d, 0x6b, 0x56, 0x54, 0x3d, 0x1c, 0x4d, 0xce, 0xa2, - 0xdb, 0x70, 0x73, 0x32, 0x0e, 0xa3, 0x79, 0x5a, 0xa8, 0xcb, 0x0b, 0xa8, 0x00, 0x1f, 0x8f, 0x2a, - 0xab, 0xba, 0x5a, 0x28, 0x3f, 0x1b, 0xbe, 0xc7, 0x1a, 0x0d, 0xdd, 0xd0, 0x1b, 0xd5, 0xaa, 0x5a, - 0x36, 0x8a, 0x85, 0xd2, 0x89, 0x71, 0xda, 0x68, 0x36, 0xb5, 0x62, 0x55, 0x65, 0x9d, 0x5a, 0xe1, - 0x99, 0xbc, 0x88, 0x3e, 0x84, 0xc7, 0x23, 0x10, 0x2d, 0xad, 0xa6, 0x36, 0x5b, 0x85, 0xda, 0xa9, - 0x51, 0x2a, 0x94, 0x2a, 0xaa, 0x88, 0x54, 0x2d, 0x4f, 0xb8, 0xe6, 0x76, 0xb2, 0x5f, 0xfd, 0x29, - 0x9f, 0x79, 0xf8, 0xcd, 0xe8, 0x23, 0x76, 0xec, 0xcd, 0x9b, 0xf7, 0x72, 0x2d, 0xfd, 0xd9, 0xe4, - 0xea, 0xb2, 0xf6, 0x8f, 0x6a, 0x9e, 0xea, 0x5a, 0x4b, 0x8d, 0xd6, 0x4b, 0xe2, 0xfd, 0x22, 0x55, - 0x34, 0x55, 0x5d, 0x2b, 0x54, 0xb5, 0xcf, 0x0b, 0xc5, 0xaa, 0x2a, 0xcf, 0xa3, 0x5b, 0xf0, 0x06, - 0x97, 0x8f, 0x87, 0x91, 0x45, 0x77, 0xe0, 0x16, 0x57, 0x15, 0x9a, 0xcf, 0xea, 0x25, 0x81, 0x78, - 0x58, 0xd0, 0xaa, 0x67, 0xba, 0x2a, 0x2f, 0xf0, 0x28, 0xf7, 0x68, 0x2e, 0xcc, 0x3d, 0x7c, 0x02, - 0x68, 0xf2, 0xb0, 0xa1, 0x25, 0xc8, 0xd6, 0x1b, 0x75, 0x55, 0xce, 0xa0, 0x15, 0xc8, 0xd1, 0x65, - 0x6a, 0x1c, 0x1e, 0xca, 0x12, 0x5a, 0x83, 0x65, 0xad, 0x56, 0x53, 0xcb, 0x5a, 0xa1, 0xa5, 0xca, - 0x73, 0xc5, 0x7b, 0xdf, 0xfd, 0x33, 0x9f, 0xf9, 0xee, 0x55, 0x5e, 0xfa, 0xdb, 0xab, 0xbc, 0xf4, - 0xfd, 0xab, 0xbc, 0xf4, 0x8f, 0x57, 0x79, 0xe9, 0x77, 0xff, 0xca, 0x67, 0x3e, 0xcf, 0x89, 0x24, - 0xfe, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x44, 0x6d, 0x68, 0x9c, 0x44, 0x1e, 0x00, 0x00, + 0xd2, 0xae, 0xcb, 0xec, 0xda, 0xee, 0xba, 0x6b, 0x66, 0xda, 0x4d, 0xa6, 0xdb, 0x4e, 0xfe, 0x86, + 0x2c, 0xbb, 0xcc, 0x74, 0xa6, 0x9e, 0xd6, 0x9d, 0xe9, 0x74, 0xa6, 0xff, 0x81, 0x57, 0x9d, 0xfb, + 0x00, 0x08, 0x88, 0x00, 0xc3, 0x64, 0x45, 0xe2, 0x3c, 0x7e, 0xf7, 0xdc, 0x7b, 0xcf, 0x3d, 0xf7, + 0x77, 0x2e, 0x6c, 0x10, 0xd7, 0xec, 0x9e, 0x0f, 0x4f, 0xf7, 0x30, 0x21, 0x2e, 0xf1, 0x76, 0x87, + 0xc4, 0xf5, 0x5d, 0xb4, 0xde, 0x75, 0xbb, 0xcf, 0x99, 0x66, 0x57, 0xe8, 0xb7, 0x50, 0x68, 0x68, + 0x99, 0xbe, 0xc9, 0xcd, 0xb6, 0x36, 0x43, 0xd9, 0x00, 0xfb, 0x66, 0x4c, 0x7e, 0xdf, 0xf3, 0x5d, + 0x62, 0xf6, 0xf0, 0x1e, 0x76, 0x7a, 0xb6, 0x13, 0xfe, 0x50, 0xbb, 0x17, 0xdd, 0xee, 0x07, 0xc2, + 0x48, 0x09, 0x7c, 0xbb, 0xbf, 0x77, 0xde, 0xef, 0xee, 0xf9, 0xf6, 0x00, 0x7b, 0xbe, 0x39, 0x18, + 0x0a, 0xcd, 0x46, 0xcf, 0xed, 0xb9, 0xec, 0xef, 0x1e, 0xfd, 0xc7, 0xa5, 0x3b, 0xdf, 0xce, 0xc0, + 0x8d, 0xa6, 0xeb, 0xd7, 0xb1, 0xe9, 0xe1, 0x9a, 0xdb, 0xb7, 0x30, 0x51, 0x69, 0xc8, 0xa8, 0x0a, + 0x05, 0x82, 0x87, 0x7d, 0xbb, 0x6b, 0x2a, 0xd2, 0x5d, 0xe9, 0xc1, 0xd2, 0xa3, 0xb7, 0x76, 0xc7, + 0xa2, 0xdf, 0xd5, 0xb9, 0x45, 0x15, 0x7b, 0x5d, 0x62, 0x0f, 0x7d, 0x97, 0x94, 0xf3, 0xdf, 0xbd, + 0xdc, 0xce, 0xe9, 0xa1, 0x2b, 0x3a, 0x80, 0xe5, 0x3e, 0x45, 0x36, 0xce, 0x19, 0xb4, 0x32, 0x33, + 0x3d, 0x94, 0xbe, 0xd4, 0x1f, 0xc5, 0x84, 0x1e, 0xc3, 0x02, 0x31, 0x9d, 0x1e, 0x36, 0x6c, 0x4b, + 0x99, 0xbd, 0x2b, 0x3d, 0x98, 0x2d, 0x6f, 0xd1, 0x91, 0x5e, 0xbd, 0xdc, 0x2e, 0xe8, 0x54, 0xae, + 0x55, 0x5f, 0x8f, 0xfe, 0xea, 0x05, 0x66, 0xab, 0x59, 0x68, 0x17, 0xe6, 0x18, 0x8a, 0x92, 0x67, + 0x03, 0x2b, 0x29, 0x03, 0xb3, 0x99, 0xeb, 0xdc, 0x0c, 0xdd, 0x07, 0xe8, 0x06, 0x9e, 0xef, 0x0e, + 0x8c, 0x81, 0xd7, 0x53, 0xe6, 0xee, 0x4a, 0x0f, 0x16, 0xc5, 0x94, 0x16, 0xb9, 0xbc, 0xe1, 0xf5, + 0x9e, 0xe4, 0xff, 0xfd, 0xf5, 0xb6, 0xb4, 0xf3, 0x26, 0x6c, 0x34, 0x5d, 0x0b, 0x9f, 0x38, 0xe6, + 0x0b, 0xd3, 0xee, 0x9b, 0xa7, 0x7d, 0xcc, 0x16, 0x4e, 0x68, 0xb7, 0xe1, 0xe6, 0x89, 0xe3, 0x05, + 0xc3, 0xa1, 0x4b, 0x7c, 0x6c, 0xe9, 0xf8, 0x97, 0x01, 0xf6, 0xfc, 0xb8, 0xc1, 0x57, 0x12, 0x20, + 0x16, 0x6e, 0xd3, 0xf5, 0xf7, 0xdd, 0xc0, 0xb1, 0xf8, 0xb2, 0xc7, 0xe7, 0x29, 0x4d, 0x3f, 0xcf, + 0xc7, 0xb0, 0x40, 0x93, 0x83, 0xb9, 0xcd, 0x24, 0xdd, 0xda, 0x54, 0xce, 0xdd, 0xc4, 0x5f, 0xbd, + 0xc0, 0x6c, 0x35, 0x4b, 0x84, 0xf2, 0xfb, 0x19, 0x78, 0x83, 0x21, 0x1e, 0xe1, 0xab, 0x86, 0xed, + 0x0d, 0x4c, 0xbf, 0x7b, 0xce, 0xa3, 0xf9, 0x00, 0xd6, 0x09, 0x0f, 0xdd, 0xf0, 0x7c, 0x93, 0xf8, + 0xc6, 0x73, 0x7c, 0xc5, 0xc2, 0x5a, 0x2e, 0x17, 0x5e, 0xbf, 0xdc, 0x9e, 0x3d, 0xc2, 0x57, 0xfa, + 0x9a, 0xb0, 0x68, 0x53, 0x83, 0x23, 0x7c, 0x85, 0xf6, 0x20, 0x14, 0x19, 0xd8, 0xb1, 0x98, 0xcb, + 0x4c, 0xd2, 0x65, 0x45, 0xe8, 0x55, 0xc7, 0xa2, 0x0e, 0x0d, 0x90, 0x07, 0x62, 0x58, 0x6c, 0x19, + 0x6c, 0x4a, 0x6c, 0x8f, 0x97, 0x1e, 0xed, 0xa4, 0x25, 0x0a, 0xd5, 0xc7, 0xd2, 0x64, 0x6d, 0xe4, + 0xcb, 0x54, 0xe8, 0x08, 0xd6, 0xbc, 0xa0, 0xd7, 0xc3, 0x9e, 0x1f, 0xa1, 0xe5, 0xa7, 0x46, 0x5b, + 0x8d, 0x5c, 0x99, 0x46, 0xac, 0xd0, 0x7f, 0x66, 0x60, 0x47, 0xc7, 0xa6, 0xf5, 0xd4, 0xf6, 0xcf, + 0x6d, 0xe7, 0xc4, 0xe9, 0x62, 0xe2, 0x9b, 0xb6, 0xe3, 0x5f, 0x69, 0x8e, 0x8f, 0xc9, 0x0b, 0xb3, + 0xcf, 0x97, 0xeb, 0x10, 0x56, 0x09, 0x36, 0x2d, 0x23, 0x3a, 0x79, 0xe2, 0xe8, 0xdc, 0x89, 0x0d, + 0x4c, 0x8f, 0xe7, 0xee, 0x79, 0xbf, 0xbb, 0xdb, 0x09, 0x8d, 0x44, 0x82, 0xad, 0x50, 0xd7, 0x48, + 0x88, 0x74, 0x40, 0xf8, 0xd2, 0xf6, 0x7c, 0xdb, 0xe9, 0xc5, 0xf0, 0x66, 0xa6, 0xc7, 0x5b, 0x0f, + 0xdd, 0x47, 0x98, 0x65, 0x58, 0x19, 0x98, 0x97, 0x31, 0xb8, 0xd9, 0x29, 0xe0, 0xf4, 0xe5, 0x81, + 0x79, 0x39, 0xc2, 0xf8, 0x02, 0x6e, 0xb8, 0xa7, 0x1e, 0x26, 0x2f, 0x70, 0x6c, 0x9e, 0x9e, 0x92, + 0xbf, 0x3b, 0x9b, 0x71, 0xb0, 0x5b, 0xc2, 0xfa, 0x7a, 0x7c, 0xc8, 0xbd, 0xae, 0xf0, 0xc4, 0x6a, + 0x7f, 0x09, 0x37, 0x3b, 0xc4, 0x74, 0x3c, 0xb3, 0xeb, 0xdb, 0xae, 0x53, 0x3a, 0x65, 0x47, 0x88, + 0xaf, 0xb0, 0x06, 0xf3, 0x04, 0x9b, 0x9e, 0xeb, 0xb0, 0x95, 0x5d, 0x7d, 0xf4, 0x3f, 0x29, 0x03, + 0x8e, 0xfb, 0xea, 0xcc, 0x45, 0x8c, 0x2b, 0x00, 0xc4, 0x58, 0x26, 0x6c, 0xc4, 0xec, 0x8f, 0x03, + 0x4f, 0x64, 0x7e, 0x05, 0x60, 0x18, 0x78, 0xe7, 0x18, 0x1b, 0xfe, 0xa5, 0x23, 0xb6, 0xb1, 0x38, + 0x79, 0xb0, 0xb0, 0x50, 0x70, 0xbf, 0xce, 0x65, 0x38, 0xc4, 0x19, 0xbc, 0x11, 0xb3, 0xd2, 0xb1, + 0x4f, 0xae, 0xf8, 0x18, 0x07, 0xd7, 0x26, 0xf3, 0xde, 0x64, 0x7c, 0xe6, 0x39, 0x61, 0x2a, 0xdf, + 0x4b, 0xb0, 0x19, 0x33, 0x6f, 0xfb, 0xa6, 0x1f, 0x78, 0x7c, 0xa4, 0x4d, 0x98, 0xa5, 0xf5, 0x4c, + 0x8a, 0xd5, 0x33, 0x2a, 0x40, 0xcd, 0x28, 0x82, 0x19, 0x16, 0xc1, 0xff, 0x4e, 0x8e, 0x20, 0x06, + 0xb9, 0x9b, 0x16, 0xc8, 0xce, 0x31, 0xcc, 0x73, 0x39, 0x42, 0xb0, 0xaa, 0xab, 0xa5, 0x76, 0xab, + 0x69, 0x9c, 0x34, 0x8f, 0x9a, 0xad, 0xa7, 0x4d, 0x39, 0x87, 0x14, 0xd8, 0x10, 0xb2, 0xce, 0x67, + 0x4d, 0xa3, 0xd9, 0xea, 0x18, 0xfb, 0xad, 0x93, 0x66, 0x55, 0x96, 0xae, 0x69, 0x2a, 0xad, 0x46, + 0x43, 0xeb, 0x74, 0xd4, 0xaa, 0x3c, 0x23, 0xa6, 0xf6, 0x0c, 0xe4, 0xa7, 0xc4, 0xf6, 0x31, 0x3d, + 0x6e, 0x0e, 0x2f, 0xa3, 0xe8, 0x43, 0x28, 0xd8, 0xec, 0xd3, 0x53, 0x24, 0x96, 0x7c, 0xb7, 0x52, + 0x82, 0xe7, 0x0e, 0xe1, 0xad, 0x24, 0xec, 0x39, 0xe8, 0x61, 0x7e, 0x61, 0x46, 0x9e, 0xdd, 0xf9, + 0x93, 0x24, 0xb0, 0x3b, 0xae, 0xdb, 0xea, 0x8b, 0x34, 0x2b, 0xc1, 0xe2, 0x4f, 0x3a, 0xc3, 0x23, + 0x2f, 0xd4, 0x04, 0xd9, 0xec, 0xfa, 0x81, 0xd9, 0xff, 0x69, 0xa7, 0x77, 0x8d, 0x3b, 0x47, 0x62, + 0xb1, 0x10, 0x5b, 0x80, 0x5a, 0x43, 0x7a, 0x9b, 0xd8, 0x04, 0x7b, 0x9d, 0x4b, 0x27, 0x7e, 0xa3, + 0x3c, 0x83, 0x8d, 0x8a, 0xeb, 0x58, 0x36, 0xdd, 0xa9, 0x7d, 0xd3, 0xee, 0x87, 0x67, 0xe6, 0x67, + 0xb0, 0x2c, 0x22, 0x79, 0x61, 0xf6, 0x03, 0x2c, 0xe6, 0x93, 0x76, 0x15, 0x7e, 0x4a, 0xf5, 0xfa, + 0x12, 0xb7, 0x66, 0x1f, 0x02, 0xfa, 0xcf, 0x12, 0x20, 0x7e, 0x4f, 0xe2, 0x2f, 0x71, 0x37, 0x3a, + 0x8d, 0x45, 0x28, 0x0c, 0xb0, 0xe7, 0x99, 0x3d, 0x9c, 0x48, 0xad, 0x50, 0x88, 0x3e, 0x82, 0x45, + 0x51, 0xe9, 0xb1, 0x25, 0x26, 0x9f, 0x79, 0x03, 0x87, 0x2b, 0x18, 0x39, 0xa0, 0x27, 0xb0, 0x10, + 0x96, 0x30, 0x51, 0xa8, 0x7e, 0xc8, 0x39, 0xb2, 0x17, 0x61, 0xff, 0x3f, 0x2c, 0xb6, 0xb1, 0x33, + 0x5d, 0xb0, 0x89, 0xa4, 0xb8, 0x80, 0x8d, 0xd2, 0xe0, 0xd4, 0xee, 0x05, 0x6e, 0xe0, 0xe9, 0xd8, + 0x0b, 0xfa, 0xfe, 0x74, 0x13, 0xfe, 0x10, 0x96, 0x2e, 0x88, 0x39, 0x1c, 0x62, 0xcb, 0xc0, 0x84, + 0x4c, 0x98, 0x32, 0x83, 0xd3, 0x41, 0x18, 0xab, 0x24, 0xdc, 0xc3, 0x3b, 0xf4, 0x26, 0x3e, 0xf3, + 0x0f, 0x88, 0x1b, 0x0c, 0xab, 0xb8, 0x8f, 0xc3, 0xa5, 0x16, 0x6a, 0x0c, 0x9b, 0x82, 0x27, 0x55, + 0x5c, 0x42, 0x82, 0x21, 0xdd, 0x6a, 0x1e, 0xd9, 0x3d, 0x58, 0x64, 0x54, 0xd3, 0xb8, 0x7e, 0xce, + 0x17, 0x98, 0xb8, 0xe1, 0xf5, 0xd0, 0x0e, 0x2c, 0x0e, 0x89, 0xdb, 0xc5, 0x9e, 0x27, 0x76, 0x63, + 0x21, 0xaa, 0x58, 0xa1, 0x38, 0xca, 0x24, 0x24, 0x86, 0x89, 0x1f, 0x8a, 0x8f, 0x01, 0x04, 0xad, + 0x0b, 0xc9, 0xc9, 0x5c, 0xb9, 0x28, 0x58, 0xc6, 0xa2, 0xb0, 0x67, 0x3c, 0x63, 0xf4, 0x41, 0xb7, + 0x93, 0xff, 0x0d, 0xa1, 0x3f, 0x01, 0xc4, 0x58, 0xc8, 0x18, 0xeb, 0x89, 0xe8, 0x8b, 0xf4, 0x63, + 0xe9, 0x4b, 0x83, 0x52, 0xad, 0x73, 0xd3, 0xb1, 0xfa, 0xb4, 0xd2, 0xfb, 0xe4, 0x2a, 0xe2, 0x62, + 0xe8, 0x11, 0xe4, 0x87, 0x2a, 0x21, 0x13, 0x52, 0x9e, 0xd9, 0x89, 0x75, 0x60, 0xb6, 0x3b, 0x7f, + 0x93, 0x40, 0xa9, 0x5d, 0x43, 0x0b, 0x4f, 0x5a, 0x66, 0x21, 0xfd, 0x02, 0xe6, 0xfd, 0x4b, 0x27, + 0x64, 0x5f, 0xcb, 0xe5, 0x2a, 0x55, 0xfd, 0xf5, 0xe5, 0xf6, 0x07, 0x3d, 0xdb, 0x3f, 0x0f, 0x4e, + 0x77, 0xbb, 0xee, 0x60, 0x2f, 0x1a, 0xdc, 0x3a, 0x1d, 0xfd, 0xdf, 0x1b, 0x3e, 0xef, 0xed, 0x31, + 0xca, 0x1e, 0x04, 0xb6, 0xb5, 0x7b, 0x72, 0xa2, 0x55, 0x5f, 0xbd, 0xdc, 0x9e, 0xeb, 0x5c, 0x3a, + 0x5a, 0x55, 0x9f, 0xf3, 0x2f, 0x1d, 0xcd, 0x42, 0xfb, 0xb0, 0xe4, 0x8f, 0x8a, 0xb0, 0x38, 0x0b, + 0xd3, 0x5d, 0x46, 0x71, 0x47, 0xb1, 0x5c, 0xef, 0xc2, 0x76, 0xe7, 0xd2, 0x29, 0xf5, 0x29, 0xdd, + 0xb8, 0x52, 0x9d, 0xae, 0x1b, 0x50, 0x0e, 0x23, 0xf2, 0x2c, 0x9e, 0x6c, 0xbf, 0x95, 0x60, 0x83, + 0xd6, 0xcf, 0x1e, 0x26, 0xad, 0x17, 0x98, 0x9c, 0xf5, 0xdd, 0x0b, 0xbe, 0x08, 0xb7, 0x60, 0x36, + 0x85, 0x07, 0x52, 0x19, 0x7a, 0x0f, 0x56, 0xba, 0x01, 0x21, 0xd8, 0xf1, 0x45, 0xb1, 0xe1, 0x64, + 0x94, 0x07, 0xb3, 0x2c, 0x54, 0xac, 0xb2, 0xa0, 0xf7, 0x61, 0xcd, 0x76, 0xba, 0x04, 0x0f, 0x46, + 0xc6, 0xb3, 0x31, 0xe3, 0xd5, 0x48, 0x19, 0x2f, 0x44, 0x0d, 0x58, 0x6f, 0xd8, 0x97, 0xd8, 0x6a, + 0x07, 0x5d, 0x9a, 0xb1, 0xe1, 0x2e, 0x17, 0xc4, 0x41, 0xfa, 0xa1, 0x8d, 0xd6, 0x43, 0x43, 0x01, + 0xf7, 0x8d, 0x04, 0xb7, 0xcb, 0x94, 0x3b, 0x8e, 0xca, 0x2f, 0x3e, 0x73, 0x09, 0x3e, 0xa8, 0x44, + 0xf7, 0x40, 0xe7, 0x27, 0xdd, 0x03, 0x23, 0xbe, 0x44, 0x21, 0xce, 0x09, 0xf6, 0x68, 0x03, 0xf4, + 0x63, 0x2e, 0x80, 0x91, 0x97, 0x88, 0xf5, 0x33, 0x40, 0xfc, 0x36, 0x6b, 0xd8, 0x9e, 0x67, 0x3b, + 0x3d, 0x1e, 0xe1, 0x47, 0xb0, 0x7c, 0x41, 0x5c, 0xa7, 0x67, 0xf0, 0xbb, 0x4d, 0x04, 0x99, 0x7d, + 0x15, 0xea, 0x4b, 0xcc, 0x9c, 0x7f, 0x8c, 0x3a, 0x99, 0x06, 0x26, 0x3d, 0xac, 0x39, 0xc7, 0xc4, + 0xed, 0x91, 0x70, 0x5d, 0x85, 0xf6, 0xb5, 0x04, 0x37, 0x18, 0x17, 0xde, 0xc7, 0xe2, 0x44, 0xf0, + 0x91, 0x8f, 0xae, 0xb1, 0x97, 0xf7, 0xb3, 0xd8, 0x75, 0xd2, 0x2f, 0x9d, 0x38, 0xfc, 0x5a, 0x8a, + 0x98, 0xc3, 0x16, 0x6c, 0x0a, 0x2e, 0xa0, 0xab, 0xc7, 0x75, 0xad, 0x52, 0x32, 0x74, 0xb5, 0xd1, + 0xfa, 0x54, 0xad, 0xca, 0x39, 0xb4, 0x09, 0x28, 0xd4, 0x95, 0x9a, 0x07, 0xaa, 0xd1, 0x3e, 0xae, + 0x6b, 0x1d, 0x59, 0x42, 0x37, 0xe1, 0x46, 0x42, 0xde, 0x50, 0xf5, 0x03, 0x4a, 0x1f, 0x62, 0xc4, + 0x42, 0x2f, 0xed, 0x77, 0x8c, 0x76, 0xb3, 0x74, 0xdc, 0xae, 0xb5, 0x3a, 0xf2, 0x2c, 0x2a, 0xc2, + 0x96, 0xd0, 0xd4, 0x5b, 0x07, 0x5a, 0xa5, 0x54, 0x37, 0x5a, 0xc7, 0x6d, 0xa3, 0xa1, 0xb5, 0xdb, + 0x5a, 0xf3, 0x40, 0xce, 0x8b, 0xc9, 0xff, 0x71, 0x03, 0x96, 0x58, 0xd8, 0x55, 0xec, 0x9b, 0x76, + 0x1f, 0xe9, 0x20, 0x3b, 0xae, 0x6f, 0x24, 0x7a, 0x5a, 0xbe, 0xe4, 0xef, 0xa4, 0x4c, 0x3f, 0xa5, + 0xaf, 0xae, 0xe5, 0xf4, 0x55, 0x27, 0x21, 0x46, 0x2d, 0x58, 0xe3, 0x2d, 0x1f, 0x45, 0x3e, 0xa3, + 0x45, 0x51, 0xe4, 0xc9, 0xdb, 0x59, 0x2b, 0x9a, 0x28, 0x9e, 0x35, 0xda, 0x3a, 0xc4, 0xa5, 0xe8, + 0x33, 0x40, 0x1c, 0xf0, 0x39, 0xbe, 0x32, 0xc2, 0xee, 0x48, 0x94, 0x8d, 0x07, 0x59, 0x98, 0xd7, + 0x7b, 0xbf, 0x5a, 0x4e, 0x97, 0xc9, 0x35, 0x05, 0xfa, 0x95, 0x04, 0x77, 0x59, 0x87, 0x73, 0xc1, + 0x1a, 0x21, 0x23, 0x18, 0x75, 0x42, 0x2c, 0x01, 0x69, 0x2b, 0x24, 0x9a, 0xad, 0xc7, 0xa9, 0x3d, + 0xfe, 0x0f, 0xb5, 0x50, 0xb5, 0x9c, 0x7e, 0x87, 0x4c, 0xb2, 0x42, 0xbf, 0x80, 0x1b, 0xb1, 0x9a, + 0x66, 0x98, 0x9c, 0xe1, 0xb3, 0x56, 0x7d, 0xe9, 0xd1, 0xc3, 0xa9, 0xda, 0x81, 0x70, 0x24, 0xe4, + 0x8f, 0xa9, 0x50, 0x07, 0xe4, 0x38, 0x3c, 0xe5, 0xf2, 0xca, 0x3c, 0xc3, 0x7e, 0x77, 0x32, 0x76, + 0xd4, 0x3a, 0xd4, 0x72, 0xfa, 0x9a, 0x9f, 0x94, 0xa3, 0xa7, 0xb0, 0x1e, 0x47, 0x25, 0xf4, 0x34, + 0x28, 0x85, 0xcc, 0x0d, 0x49, 0x6d, 0x17, 0xe8, 0x86, 0xf8, 0xd7, 0x14, 0xe8, 0x73, 0x88, 0x4f, + 0x82, 0x36, 0xe9, 0x7e, 0xe0, 0x29, 0x0b, 0x0c, 0xf9, 0xbd, 0xa9, 0xc9, 0x7c, 0x2d, 0xa7, 0xc7, + 0xe3, 0xe3, 0x1a, 0x54, 0xa3, 0xa5, 0xc5, 0xf6, 0x71, 0x58, 0x5a, 0x16, 0x19, 0xea, 0xfd, 0x14, + 0xd4, 0xeb, 0xdc, 0xbc, 0x96, 0xa3, 0x65, 0x26, 0x92, 0x21, 0x0d, 0x56, 0x38, 0x92, 0xef, 0xba, + 0x06, 0xad, 0x83, 0x30, 0x19, 0x2a, 0xc6, 0x3a, 0x22, 0x28, 0x2e, 0xa3, 0x87, 0xc5, 0x1d, 0x1a, + 0x44, 0x30, 0x60, 0xd6, 0x9c, 0x2d, 0x65, 0x1e, 0x96, 0x71, 0xaa, 0x4c, 0x0f, 0x8b, 0x1b, 0x97, + 0xd2, 0x0d, 0xef, 0x86, 0xac, 0xd9, 0x38, 0x63, 0xb4, 0x59, 0x59, 0xce, 0xdc, 0xf0, 0x34, 0x82, + 0x4d, 0x37, 0xbc, 0x9b, 0x94, 0xa3, 0x26, 0xac, 0xf2, 0x1a, 0x41, 0x04, 0x61, 0x56, 0x56, 0x32, + 0xa3, 0x1c, 0x27, 0xd6, 0x34, 0xca, 0x7e, 0x5c, 0x4a, 0xa3, 0x74, 0x5c, 0x0b, 0x1b, 0xc1, 0xe8, + 0xb5, 0x49, 0x59, 0xcd, 0x8c, 0x32, 0xed, 0x5d, 0x8a, 0x46, 0xe9, 0x24, 0xe5, 0x94, 0x1e, 0x79, + 0xd8, 0xb1, 0x94, 0x35, 0x86, 0xf4, 0x66, 0x0a, 0x52, 0x44, 0x9f, 0x6b, 0x39, 0x9d, 0xd9, 0xf2, + 0xe2, 0x72, 0xe6, 0x1b, 0x3d, 0x4a, 0x51, 0x0d, 0x8b, 0x73, 0x54, 0x45, 0x9e, 0x50, 0x5c, 0x52, + 0xe8, 0x2c, 0x2f, 0x2e, 0x49, 0x05, 0xcd, 0xe5, 0x90, 0x5f, 0x76, 0x23, 0x76, 0xab, 0xac, 0x67, + 0xe6, 0x72, 0x3a, 0x13, 0xa6, 0xb9, 0x4c, 0xae, 0x6b, 0x58, 0x8d, 0x15, 0xd8, 0x61, 0x0e, 0xa2, + 0xec, 0x1a, 0x3b, 0xc6, 0x7d, 0x59, 0x8d, 0x8d, 0x4b, 0xe9, 0x86, 0x98, 0x61, 0x87, 0x60, 0x10, + 0xd6, 0x22, 0x28, 0x5b, 0x99, 0x1b, 0x92, 0xd6, 0x4c, 0xd0, 0x0d, 0x31, 0x93, 0x72, 0x1a, 0x26, + 0xe7, 0xc1, 0xa3, 0xab, 0xe0, 0x76, 0x66, 0x98, 0xe3, 0x3c, 0x9a, 0x86, 0xe9, 0xc5, 0xa5, 0xa8, + 0x0f, 0xb7, 0x05, 0x33, 0xe6, 0x45, 0x87, 0x6e, 0x3b, 0x3d, 0x34, 0x06, 0xeb, 0x0a, 0x94, 0x37, + 0x19, 0x78, 0xda, 0x23, 0x4a, 0x16, 0x03, 0xae, 0xe5, 0x74, 0xe5, 0x3c, 0x8b, 0x1d, 0x77, 0x40, + 0xb6, 0x39, 0x61, 0x34, 0x5c, 0xc1, 0x18, 0x95, 0xed, 0xcc, 0x45, 0x49, 0xe3, 0x96, 0x74, 0x51, + 0xec, 0xa4, 0x9c, 0x56, 0xfc, 0x60, 0xf4, 0x94, 0x6a, 0x88, 0x06, 0x51, 0xb9, 0x9b, 0x59, 0xf1, + 0x33, 0x1e, 0x5e, 0x69, 0xc5, 0x0f, 0xc6, 0x54, 0xe8, 0x08, 0x56, 0x06, 0x94, 0x52, 0x1a, 0x1e, + 0xe7, 0x94, 0xca, 0xbd, 0xcc, 0x37, 0xea, 0x31, 0xea, 0x59, 0xcb, 0xe9, 0xcb, 0x83, 0x98, 0x10, + 0x7d, 0x01, 0x72, 0xd4, 0xee, 0x1b, 0xa7, 0x8c, 0x4b, 0x2a, 0x3b, 0x0c, 0x6f, 0x37, 0x05, 0x6f, + 0x02, 0xf5, 0x64, 0xb7, 0x48, 0x52, 0x83, 0x2e, 0xe0, 0x0e, 0xdd, 0x3a, 0x93, 0x53, 0x77, 0x03, + 0x8f, 0xb8, 0xbb, 0xd8, 0xce, 0xfb, 0x6c, 0xa4, 0x47, 0x69, 0x75, 0x7f, 0x32, 0xe3, 0xaf, 0xe5, + 0xf4, 0x2d, 0x3f, 0xd3, 0x84, 0x56, 0x33, 0x7e, 0x07, 0x50, 0x36, 0x41, 0xb9, 0xa7, 0xf2, 0x56, + 0x66, 0x56, 0x8e, 0x73, 0x54, 0x9a, 0x95, 0x76, 0x5c, 0x8a, 0x4e, 0x60, 0x7d, 0x40, 0x09, 0xa7, + 0x61, 0x3b, 0xc6, 0x50, 0x50, 0x4e, 0xe5, 0xed, 0xcc, 0x44, 0x49, 0x23, 0xa7, 0x74, 0x7d, 0x06, + 0x49, 0x39, 0xfa, 0x44, 0x10, 0xa9, 0x33, 0x1c, 0xa6, 0xbb, 0xf2, 0x4e, 0x26, 0x37, 0x4b, 0xa1, + 0xa6, 0x94, 0x9b, 0x45, 0x00, 0x4c, 0xcc, 0x59, 0x60, 0xb9, 0x00, 0x73, 0xac, 0x41, 0x39, 0xcc, + 0x2f, 0x6c, 0xca, 0x37, 0x0f, 0xf3, 0x0b, 0xb7, 0xe4, 0xad, 0xc3, 0xfc, 0xc2, 0x1d, 0xb9, 0x78, + 0x98, 0x5f, 0x28, 0xca, 0xdb, 0x3b, 0x7b, 0x8c, 0x25, 0x1e, 0xbb, 0x1e, 0xbb, 0x03, 0xd0, 0x16, + 0xcc, 0xd9, 0x8e, 0x85, 0x2f, 0x45, 0x93, 0xcc, 0xa9, 0x2e, 0x17, 0x09, 0x5e, 0xf9, 0xed, 0x2c, + 0xcc, 0x4d, 0xf7, 0xa4, 0xf0, 0xf3, 0x24, 0xdf, 0x21, 0x98, 0x3d, 0xc4, 0x33, 0x36, 0xb7, 0x9a, + 0xba, 0x01, 0x09, 0xf2, 0xc0, 0x8c, 0xc3, 0x07, 0x57, 0x7f, 0x4c, 0x83, 0x2a, 0xb0, 0x12, 0x38, + 0xf8, 0x72, 0xe8, 0x7a, 0xd8, 0x62, 0x97, 0x69, 0x7e, 0x9a, 0xe6, 0x52, 0x5f, 0x8e, 0x9c, 0xe8, + 0x15, 0xba, 0x07, 0x4b, 0x2e, 0xb1, 0x7b, 0xb6, 0x63, 0xd0, 0x0b, 0x86, 0x51, 0xb1, 0xb9, 0xf2, + 0x2a, 0x1d, 0xf3, 0xf5, 0xcb, 0xed, 0x79, 0x7a, 0x19, 0x69, 0x55, 0x1d, 0xb8, 0x09, 0xfd, 0x42, + 0x1f, 0xc1, 0xbc, 0xc5, 0xf8, 0xb4, 0xa0, 0x56, 0xc5, 0xac, 0x7e, 0x8d, 0xb3, 0xee, 0xb0, 0x57, + 0xe0, 0x3e, 0xe8, 0xff, 0xc2, 0xd5, 0x2d, 0x4c, 0x72, 0x0e, 0x37, 0x43, 0xac, 0x3b, 0x7a, 0x0c, + 0xb3, 0x8e, 0x7b, 0x21, 0xa8, 0xd1, 0x54, 0x1d, 0x18, 0xb5, 0x7f, 0xb2, 0xf0, 0xbb, 0xaf, 0xb7, + 0x73, 0xa3, 0x97, 0xa1, 0x87, 0xff, 0x9a, 0x01, 0x25, 0xeb, 0x81, 0x99, 0x76, 0x1b, 0xa5, 0x72, + 0x4b, 0xef, 0x18, 0x63, 0x4f, 0x9f, 0x6f, 0xc3, 0xbd, 0x84, 0x86, 0x7d, 0xa8, 0x55, 0x43, 0x57, + 0x2b, 0x2d, 0xbd, 0x1a, 0xbd, 0x83, 0xbe, 0x0b, 0xf7, 0x13, 0x66, 0x4d, 0xf5, 0x29, 0x7b, 0x11, + 0x15, 0x66, 0x9d, 0x56, 0xcb, 0x68, 0xd5, 0x69, 0x5f, 0x53, 0x84, 0xad, 0x84, 0x61, 0xa5, 0xae, + 0xa9, 0x4d, 0xfa, 0x75, 0xa8, 0x56, 0x68, 0x77, 0xb3, 0x0d, 0xb7, 0x13, 0xfa, 0xe3, 0x93, 0x76, + 0x4d, 0xd5, 0xc3, 0x61, 0xe5, 0x3c, 0xba, 0x0d, 0x37, 0xc7, 0x03, 0x32, 0xda, 0xc7, 0xa5, 0xa6, + 0x3c, 0x87, 0x4a, 0xf0, 0x71, 0x52, 0x59, 0xd7, 0xd5, 0x52, 0xf5, 0xd9, 0xe8, 0x61, 0xd6, 0x68, + 0xe9, 0x86, 0xde, 0xaa, 0xd7, 0xd5, 0xaa, 0x51, 0x2e, 0x55, 0x8e, 0x8c, 0xe3, 0x56, 0xbb, 0xad, + 0x95, 0xeb, 0x2a, 0x6b, 0xd9, 0x4a, 0xcf, 0xe4, 0x79, 0xf4, 0x21, 0x3c, 0x4e, 0x40, 0x74, 0xb4, + 0x86, 0xda, 0xee, 0x94, 0x1a, 0xc7, 0x46, 0xa5, 0x54, 0xa9, 0xa9, 0x22, 0x52, 0xb5, 0x3a, 0xe6, + 0x5a, 0xd8, 0xca, 0x7f, 0xf5, 0x87, 0x62, 0xee, 0xe1, 0x37, 0xc9, 0xd7, 0xec, 0xd8, 0xe3, 0x37, + 0x6f, 0xea, 0x3a, 0xfa, 0xb3, 0xf1, 0x65, 0x66, 0x7d, 0x20, 0xd5, 0x3c, 0xd5, 0xb5, 0x8e, 0x1a, + 0xad, 0x97, 0xc4, 0x1b, 0x47, 0xaa, 0x68, 0xab, 0xba, 0x56, 0xaa, 0x6b, 0x9f, 0x97, 0xca, 0x75, + 0x55, 0x9e, 0x45, 0xb7, 0xe0, 0x0d, 0x2e, 0xbf, 0x1e, 0x46, 0x1e, 0xdd, 0x81, 0x5b, 0x5c, 0x55, + 0x6a, 0x3f, 0x6b, 0x56, 0x04, 0xe2, 0x7e, 0x49, 0xab, 0x9f, 0xe8, 0xaa, 0x3c, 0xc7, 0xa3, 0xdc, + 0xa1, 0x49, 0x31, 0xf3, 0xf0, 0x09, 0xa0, 0xf1, 0x53, 0x87, 0x16, 0x20, 0xdf, 0x6c, 0x35, 0x55, + 0x39, 0x87, 0x96, 0xa0, 0x40, 0x97, 0xa9, 0xb5, 0xbf, 0x2f, 0x4b, 0x68, 0x05, 0x16, 0xb5, 0x46, + 0x43, 0xad, 0x6a, 0xa5, 0x8e, 0x2a, 0xcf, 0x94, 0xef, 0x7d, 0xf7, 0x8f, 0x62, 0xee, 0xbb, 0x57, + 0x45, 0xe9, 0x2f, 0xaf, 0x8a, 0xd2, 0xf7, 0xaf, 0x8a, 0xd2, 0xdf, 0x5f, 0x15, 0xa5, 0xdf, 0xfc, + 0xb3, 0x98, 0xfb, 0xbc, 0x20, 0xb2, 0xf9, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xf8, 0x1d, 0x1c, + 0x5b, 0x4d, 0x1e, 0x00, 0x00, } diff --git a/pkg/roachpb/errors.proto b/pkg/roachpb/errors.proto index 1c492541ba69..6e8d88f1f41f 100644 --- a/pkg/roachpb/errors.proto +++ b/pkg/roachpb/errors.proto @@ -131,10 +131,11 @@ enum TransactionAbortedReason { // another txn (of any priority) considered us abandoned and pushed us. ABORT_REASON_ABORTED_RECORD_FOUND = 1; - // The BeginTransaction has a timestamp below the TxnSpanGCThreshold, so there - // might have been an ABORTED txn record that got GCed (so, we might be in the - // ABORT_REASON_ABORTED_RECORD_FOUND case and not know). - ABORT_REASON_BEGIN_TOO_OLD = 2; + // The request attempting to create a transaction record has a timestamp below + // the TxnSpanGCThreshold, so there might have been an ABORTED txn record that + // got GCed (so, we might be in the ABORT_REASON_ABORTED_RECORD_FOUND case and + // not know). + ABORT_REASON_NEW_TXN_RECORD_TOO_OLD = 2; // The client is trying to use a transaction that's already been aborted. The // TxnCoordSender detects this. Either the client is misusing a txn, or the @@ -205,6 +206,7 @@ enum TransactionRetryReason { RETRY_SERIALIZABLE = 3; // A possible replay caused by duplicate begin txn or out-of-order // txn sequence number. + // TODO(nvanbenschoten): This is no longer in use. Remove. RETRY_POSSIBLE_REPLAY = 4; // An asynchronous write was observed to have failed. RETRY_ASYNC_WRITE_FAILURE = 5; diff --git a/pkg/storage/batcheval/cmd_begin_transaction.go b/pkg/storage/batcheval/cmd_begin_transaction.go index e86946ea9aa3..f4fc56ffa66e 100644 --- a/pkg/storage/batcheval/cmd_begin_transaction.go +++ b/pkg/storage/batcheval/cmd_begin_transaction.go @@ -40,6 +40,9 @@ func declareKeysWriteTransaction( spans.Add(spanset.SpanReadWrite, roachpb.Span{ Key: keys.TransactionKey(req.Header().Key, header.Txn.ID), }) + spans.Add(spanset.SpanReadOnly, roachpb.Span{ + Key: keys.RangeTxnSpanGCThresholdKey(header.RangeID), + }) } } @@ -47,7 +50,6 @@ func declareKeysBeginTransaction( desc roachpb.RangeDescriptor, header roachpb.Header, req roachpb.Request, spans *spanset.SpanSet, ) { declareKeysWriteTransaction(desc, header, req, spans) - spans.Add(spanset.SpanReadOnly, roachpb.Span{Key: keys.RangeTxnSpanGCThresholdKey(header.RangeID)}) spans.Add(spanset.SpanReadOnly, roachpb.Span{ Key: keys.AbortSpanKey(header.RangeID, header.Txn.ID), }) @@ -95,10 +97,12 @@ func BeginTransaction( // this command's txn and rewrite the record. reply.Txn.Update(&tmpTxn) } else { - // Our txn record already exists. This is either a client error, sending - // a duplicate BeginTransaction, or it's an artifact of DistSender - // re-sending a batch. Assume the latter and ask the client to restart. - return result.Result{}, roachpb.NewTransactionRetryError(roachpb.RETRY_POSSIBLE_REPLAY) + // Our txn record already exists. This is possible if the first + // transaction heartbeat evaluated before this BeginTransaction + // request or if the DistSender re-sent the batch. Either way, + // this request will contain no new information about the + // transaction, so treat the BeginTransaction as a no-op. + return result.Result{}, nil } case roachpb.COMMITTED: @@ -113,15 +117,10 @@ func BeginTransaction( } } - // Disallow creation or modification of a transaction record if it's at a - // timestamp before the TxnSpanGCThreshold, as in that case our transaction - // may already have been aborted by a concurrent actor which encountered one - // of our intents (which may have been written before this entry). - // - // See #9265. - threshold := cArgs.EvalCtx.GetTxnSpanGCThreshold() - if reply.Txn.LastActive().Less(threshold) { - return result.Result{}, roachpb.NewTransactionAbortedError(roachpb.ABORT_REASON_BEGIN_TOO_OLD) + // Verify that it is safe to create the transaction record. + if ok, errTxn, err := CanCreateTxnRecord(ctx, cArgs.EvalCtx, reply.Txn); !ok { + reply.Txn = errTxn + return result.Result{}, err } // Initialize the LastHeartbeat field to the present time. This allows the diff --git a/pkg/storage/batcheval/cmd_end_transaction.go b/pkg/storage/batcheval/cmd_end_transaction.go index ce50472fc7eb..89f7f72baa0e 100644 --- a/pkg/storage/batcheval/cmd_end_transaction.go +++ b/pkg/storage/batcheval/cmd_end_transaction.go @@ -177,91 +177,93 @@ func evalEndTransaction( ); err != nil { return result.Result{}, err } else if !ok { - if args.Commit { - return result.Result{}, roachpb.NewTransactionNotFoundStatusError() - } - // For rollbacks, we don't consider not finding the txn record an error; - // we accept without fuss rollbacks for transactions where the txn record - // was never written (because the BeginTransaction's batch failed, or - // even where the BeginTransaction was never sent to server). + // No existing transaction record was found - create one. txn := h.Txn.Clone() reply.Txn = &txn - return result.Result{}, nil - } - // We're using existingTxn on the reply, although it can be stale - // compared to the Transaction in the request (e.g. the Sequence, - // and various timestamps). We must be careful to update it with the - // supplied ba.Txn if we return it with an error which might be - // retried, as for example to avoid client-side serializable restart. - reply.Txn = &existingTxn - - // Verify that we can either commit it or abort it (according - // to args.Commit), and also that the Timestamp and Epoch have - // not suffered regression. - switch reply.Txn.Status { - case roachpb.COMMITTED: - return result.Result{}, roachpb.NewTransactionCommittedStatusError() - - case roachpb.ABORTED: - if !args.Commit { - // The transaction has already been aborted by other. - // Do not return TransactionAbortedError since the client anyway - // wanted to abort the transaction. - desc := cArgs.EvalCtx.Desc() - externalIntents, err := resolveLocalIntents(ctx, desc, batch, ms, *args, reply.Txn, cArgs.EvalCtx) - if err != nil { - return result.Result{}, err - } - if err := updateTxnWithExternalIntents( - ctx, batch, ms, *args, reply.Txn, externalIntents, - ); err != nil { + + // Verify that it is safe to create the transaction record. + if args.Commit { + if ok, errTxn, err := CanCreateTxnRecord(ctx, cArgs.EvalCtx, reply.Txn); !ok { + reply.Txn = errTxn return result.Result{}, err } - // Use alwaysReturn==true because the transaction is definitely - // aborted, no matter what happens to this command. - return result.FromEndTxn(reply.Txn, true /* alwaysReturn */, args.Poison), nil } - // If the transaction was previously aborted by a concurrent writer's - // push, any intents written are still open. It's only now that we know - // them, so we return them all for asynchronous resolution (we're - // currently not able to write on error, but see #1989). - // - // Similarly to above, use alwaysReturn==true. The caller isn't trying - // to abort, but the transaction is definitely aborted and its intents - // can go. - reply.Txn.Intents = args.IntentSpans - return result.FromEndTxn(reply.Txn, true /* alwaysReturn */, args.Poison), - roachpb.NewTransactionAbortedError(roachpb.ABORT_REASON_ABORTED_RECORD_FOUND) - - case roachpb.PENDING: - if h.Txn.Epoch < reply.Txn.Epoch { - // TODO(tschottdorf): this leaves the Txn record (and more - // importantly, intents) dangling; we can't currently write on - // error. Would panic, but that makes TestEndTransactionWithErrors - // awkward. - return result.Result{}, roachpb.NewTransactionStatusError( - fmt.Sprintf("epoch regression: %d", h.Txn.Epoch), - ) - } else if h.Txn.Epoch == reply.Txn.Epoch && reply.Txn.Timestamp.Less(h.Txn.OrigTimestamp) { - // The transaction record can only ever be pushed forward, so it's an - // error if somehow the transaction record has an earlier timestamp - // than the original transaction timestamp. + } else { + // We're using existingTxn on the reply, although it can be stale + // compared to the Transaction in the request (e.g. the Sequence, + // and various timestamps). We must be careful to update it with the + // supplied ba.Txn if we return it with an error which might be + // retried, as for example to avoid client-side serializable restart. + reply.Txn = &existingTxn + + // Verify that we can either commit it or abort it (according + // to args.Commit), and also that the Timestamp and Epoch have + // not suffered regression. + switch reply.Txn.Status { + case roachpb.COMMITTED: + return result.Result{}, roachpb.NewTransactionCommittedStatusError() + + case roachpb.ABORTED: + if !args.Commit { + // The transaction has already been aborted by other. + // Do not return TransactionAbortedError since the client anyway + // wanted to abort the transaction. + desc := cArgs.EvalCtx.Desc() + externalIntents, err := resolveLocalIntents(ctx, desc, batch, ms, *args, reply.Txn, cArgs.EvalCtx) + if err != nil { + return result.Result{}, err + } + if err := updateTxnWithExternalIntents( + ctx, batch, ms, *args, reply.Txn, externalIntents, + ); err != nil { + return result.Result{}, err + } + // Use alwaysReturn==true because the transaction is definitely + // aborted, no matter what happens to this command. + return result.FromEndTxn(reply.Txn, true /* alwaysReturn */, args.Poison), nil + } + // If the transaction was previously aborted by a concurrent writer's + // push, any intents written are still open. It's only now that we know + // them, so we return them all for asynchronous resolution (we're + // currently not able to write on error, but see #1989). + // + // Similarly to above, use alwaysReturn==true. The caller isn't trying + // to abort, but the transaction is definitely aborted and its intents + // can go. + reply.Txn.Intents = args.IntentSpans + return result.FromEndTxn(reply.Txn, true /* alwaysReturn */, args.Poison), + roachpb.NewTransactionAbortedError(roachpb.ABORT_REASON_ABORTED_RECORD_FOUND) + + case roachpb.PENDING: + if h.Txn.Epoch < reply.Txn.Epoch { + // TODO(tschottdorf): this leaves the Txn record (and more + // importantly, intents) dangling; we can't currently write on + // error. Would panic, but that makes TestEndTransactionWithErrors + // awkward. + return result.Result{}, roachpb.NewTransactionStatusError( + fmt.Sprintf("epoch regression: %d", h.Txn.Epoch), + ) + } else if h.Txn.Epoch == reply.Txn.Epoch && reply.Txn.Timestamp.Less(h.Txn.OrigTimestamp) { + // The transaction record can only ever be pushed forward, so it's an + // error if somehow the transaction record has an earlier timestamp + // than the original transaction timestamp. + + // TODO(tschottdorf): see above comment on epoch regression. + return result.Result{}, roachpb.NewTransactionStatusError( + fmt.Sprintf("timestamp regression: %s", h.Txn.OrigTimestamp), + ) + } - // TODO(tschottdorf): see above comment on epoch regression. + default: return result.Result{}, roachpb.NewTransactionStatusError( - fmt.Sprintf("timestamp regression: %s", h.Txn.OrigTimestamp), + fmt.Sprintf("bad txn status: %s", reply.Txn), ) } - default: - return result.Result{}, roachpb.NewTransactionStatusError( - fmt.Sprintf("bad txn status: %s", reply.Txn), - ) + // Update the existing txn with the supplied txn. + reply.Txn.Update(h.Txn) } - // Update the existing txn with the supplied txn. - reply.Txn.Update(h.Txn) - var pd result.Result // Set transaction status to COMMITTED or ABORTED as per the diff --git a/pkg/storage/batcheval/cmd_heartbeat_txn.go b/pkg/storage/batcheval/cmd_heartbeat_txn.go index 1bbf239fc745..66d3c014d55a 100644 --- a/pkg/storage/batcheval/cmd_heartbeat_txn.go +++ b/pkg/storage/batcheval/cmd_heartbeat_txn.go @@ -62,11 +62,19 @@ func HeartbeatTxn( ); err != nil { return result.Result{}, err } else if !ok { - // If no existing transaction record was found, skip heartbeat. - // This could mean the heartbeat is a delayed relic or it could - // mean that the BeginTransaction call was delayed. In either - // case, there's no reason to persist a new transaction record. - return result.Result{}, roachpb.NewTransactionNotFoundStatusError() + // No existing transaction record was found - create one. + txn = h.Txn.Clone() + if txn.Status != roachpb.PENDING { + return result.Result{}, roachpb.NewTransactionStatusError( + fmt.Sprintf("cannot heartbeat txn with status %v: %s", txn.Status, txn), + ) + } + + // Verify that it is safe to create the transaction record. + if ok, errTxn, err := CanCreateTxnRecord(ctx, cArgs.EvalCtx, &txn); !ok { + reply.Txn = errTxn + return result.Result{}, err + } } if txn.Status == roachpb.PENDING { diff --git a/pkg/storage/batcheval/cmd_resolve_intent_test.go b/pkg/storage/batcheval/cmd_resolve_intent_test.go index d308df972e4c..4c05e63b51c9 100644 --- a/pkg/storage/batcheval/cmd_resolve_intent_test.go +++ b/pkg/storage/batcheval/cmd_resolve_intent_test.go @@ -109,6 +109,11 @@ func (m *mockEvalCtx) GetMVCCStats() enginepb.MVCCStats { func (m *mockEvalCtx) GetSplitQPS() float64 { return m.qps } +func (m *mockEvalCtx) GetTxnTombstoneFromTimestampCache( + *roachpb.Transaction, +) (hlc.Timestamp, uuid.UUID) { + panic("unimplemented") +} func (m *mockEvalCtx) GetGCThreshold() hlc.Timestamp { return m.gcThreshold } diff --git a/pkg/storage/batcheval/eval_context.go b/pkg/storage/batcheval/eval_context.go index c5a95120a2aa..6e6c05dccaa8 100644 --- a/pkg/storage/batcheval/eval_context.go +++ b/pkg/storage/batcheval/eval_context.go @@ -28,6 +28,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/storage/txnwait" "github.com/cockroachdb/cockroach/pkg/util/hlc" "github.com/cockroachdb/cockroach/pkg/util/limit" + "github.com/cockroachdb/cockroach/pkg/util/uuid" opentracing "github.com/opentracing/opentracing-go" "golang.org/x/time/rate" ) @@ -80,6 +81,14 @@ type EvalContext interface { // setting is disabled. GetSplitQPS() float64 + // GetTxnTombstoneFromTimestampCache returns the tombstone timestamp for the + // provided transaction, along with an associated txnID marker. If the txnID + // is not empty then the tombstone was explicitly set by an EndTxn req. If + // it is empty then the tombstone is an implicit artifact of the timestamp + // cache's low-water mark. Either way, transactions should not write records + // at or below this timestamp. + GetTxnTombstoneFromTimestampCache(*roachpb.Transaction) (hlc.Timestamp, uuid.UUID) + GetGCThreshold() hlc.Timestamp GetTxnSpanGCThreshold() hlc.Timestamp GetLastReplicaGCTimestamp(context.Context) (hlc.Timestamp, error) diff --git a/pkg/storage/batcheval/transaction.go b/pkg/storage/batcheval/transaction.go index 2b37498f61cb..f984464a8dfe 100644 --- a/pkg/storage/batcheval/transaction.go +++ b/pkg/storage/batcheval/transaction.go @@ -21,6 +21,8 @@ import ( "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/storage/engine" "github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/uuid" "github.com/pkg/errors" ) @@ -91,3 +93,85 @@ func CanPushWithPriority(pusher, pushee *roachpb.Transaction) bool { return (pusher.Priority > roachpb.MinTxnPriority && pushee.Priority == roachpb.MinTxnPriority) || (pusher.Priority == roachpb.MaxTxnPriority && pushee.Priority < pusher.Priority) } + +// CanCreateTxnRecord determines whether a transaction record can be created +// for the provided transaction. If not, it returns an updated transaction +// that would be allowed to write a transaction record, along with an error. +// +// TODO(nvanbenschoten): This function can be used in cmd_push_txn.go and +// cmd_query_txn.go to determine whether a transaction record is allowed to be +// synthesized to get around the complications of eagerly GCed transaction +// records. I believe this is exactly what @andreimatei was suggesting. To do +// this, we'll need to address the TODO comment in below. If we do make this the +// case, update this comment to reflect that once this method returns false, it +// will never return true for the same txn. +func CanCreateTxnRecord( + ctx context.Context, rec EvalContext, txn *roachpb.Transaction, +) (ok bool, errTxn *roachpb.Transaction, err error) { + // We look in the timestamp cache to see if there is a tombstone entry for + // this transaction, which would indicate this transaction has already been + // finalized. If there is one, then we return a retriable into an ambiguous + // one higher up. Otherwise, if the client is still waiting for a result, + // then this cannot be a "replay" of any sort. + // + // The retriable error we return is a TransactionAbortedError, instructing + // the client to create a new transaction. Since a transaction record + // doesn't exist, there's no point in the client to continue with the + // existing transaction at a new epoch. + tombTS, tombTxnID := rec.GetTxnTombstoneFromTimestampCache(txn) + // GetTxnTombstoneFromTimestampCache will only find a timestamp interval + // with an associated txnID on the TransactionKey if an EndTxnReq has been + // processed. All other timestamp intervals will have no associated txnID + // and will be due to the low-water mark. + switch tombTxnID { + case txn.ID: + newTxn := txn.Clone() + newTxn.Status = roachpb.ABORTED + newTxn.Timestamp.Forward(tombTS.Next()) + return false, &newTxn, roachpb.NewTransactionAbortedError( + roachpb.ABORT_REASON_ALREADY_COMMITTED_OR_ROLLED_BACK_POSSIBLE_REPLAY, + ) + case uuid.UUID{} /* noTxnID */ : + // TODO(nvanbenschoten): I think we should be comparing against + // txn.OrigTimestamp, not txn.Timestamp, if we want this to make + // definitive decisions about whether a transaction record will *never* + // be able to be written. We can't allow a concurrent txn to use this to + // decided that a transaction can never write a txn record, then have + // that txn bump its timestamp and squeeze around this protection. + if !tombTS.Less(txn.Timestamp) { + // On lease transfers the timestamp cache is reset with the transfer + // time as the low-water mark, so if this replica recently obtained + // the lease, this case will be true for new txns, even if they're + // not a replay. We move the timestamp forward and return retry. + newTxn := txn.Clone() + newTxn.Status = roachpb.ABORTED + newTxn.Timestamp.Forward(tombTS.Next()) + return false, &newTxn, roachpb.NewTransactionAbortedError( + roachpb.ABORT_REASON_TIMESTAMP_CACHE_REJECTED_POSSIBLE_REPLAY, + ) + } + default: + log.Fatalf(ctx, "unexpected tscache interval (%s,%s) for txn %s", + tombTS, tombTxnID, txn) + } + + // Disallow creation or modification of a transaction record if its original + // timestamp is before the TxnSpanGCThreshold, as in that case our transaction + // may already have been aborted by a concurrent actor which encountered one + // of our intents (which may have been written before this entry). We compare + // against the original timestamp because the transaction's provisional commit + // timestamp may be moved forward over the course of a single epoch. + // + // See #9265. + threshold := rec.GetTxnSpanGCThreshold() + if txn.OrigTimestamp.Less(threshold) { + newTxn := txn.Clone() + newTxn.Status = roachpb.ABORTED + newTxn.Timestamp.Forward(threshold) + return false, &newTxn, roachpb.NewTransactionAbortedError( + roachpb.ABORT_REASON_NEW_TXN_RECORD_TOO_OLD, + ) + } + + return true, nil, nil +} diff --git a/pkg/storage/replica_eval_context_span.go b/pkg/storage/replica_eval_context_span.go index 05969a024647..d702905773bc 100644 --- a/pkg/storage/replica_eval_context_span.go +++ b/pkg/storage/replica_eval_context_span.go @@ -29,6 +29,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/storage/storagebase" "github.com/cockroachdb/cockroach/pkg/storage/txnwait" "github.com/cockroachdb/cockroach/pkg/util/hlc" + "github.com/cockroachdb/cockroach/pkg/util/uuid" opentracing "github.com/opentracing/opentracing-go" ) @@ -148,6 +149,18 @@ func (rec SpanSetReplicaEvalContext) GetSplitQPS() float64 { return rec.i.GetSplitQPS() } +// GetTxnTombstoneFromTimestampCache returns the tombstone timestamp for the +// provided transaction, along with an associated txnID marker. If the txnID +// is not empty then the tombstone was explicitly set by an EndTxn req. If +// it is empty then the tombstone is an implicit artifact of the timestamp +// cache's low-water mark. Either way, transactions should not write records +// at or below this timestamp. +func (rec SpanSetReplicaEvalContext) GetTxnTombstoneFromTimestampCache( + txn *roachpb.Transaction, +) (hlc.Timestamp, uuid.UUID) { + return rec.i.GetTxnTombstoneFromTimestampCache(txn) +} + // GetGCThreshold returns the GC threshold of the Range, typically updated when // keys are garbage collected. Reads and writes at timestamps <= this time will // not be served. diff --git a/pkg/storage/replica_test.go b/pkg/storage/replica_test.go index 3e54a0f6e97b..e27b67c33d71 100644 --- a/pkg/storage/replica_test.go +++ b/pkg/storage/replica_test.go @@ -3050,11 +3050,9 @@ func TestReplicaAbortSpanOnlyWithIntent(t *testing.T) { args, h := heartbeatArgs(txn, tc.Clock().Now()) // If the AbortSpan were active for this request, we'd catch a txn retry. - // Instead, we expect the error from heartbeating a nonexistent txn. - _, pErr := tc.SendWrappedWith(h, &args) - if tse, ok := pErr.GetDetail().(*roachpb.TransactionStatusError); !ok || - tse.Reason != roachpb.TransactionStatusError_REASON_TXN_NOT_FOUND { - t.Fatalf("expected TransactionStatusError with REASON_TXN_NOT_FOUND, found %v", pErr) + // Instead, we expect no error and a successfully created transaction record. + if _, pErr := tc.SendWrappedWith(h, &args); pErr != nil { + t.Fatalf("unexpected error: %v", pErr) } } @@ -3243,15 +3241,37 @@ func TestEndTransactionTxnSpanGCThreshold(t *testing.T) { // we're in trouble because other written intents may have been aborted, // i.e. the transaction might commit but lose some of its writes. // - // It should not succeed because BeginTransaction checks the transaction's - // last activity against the persisted TxnSpanGCThreshold. + // It should not succeed because all request types that can create txn + // records check the transaction's original timestamp against the persisted + // TxnSpanGCThreshold. { - beginArgs, header := beginTxnArgs(key, pushee) - resp, pErr := tc.SendWrappedWith(header, &beginArgs) + expErr := "TransactionAbortedError(ABORT_REASON_NEW_TXN_RECORD_TOO_OLD)" + + // BeginTransaction. + bt, btH := beginTxnArgs(key, pushee) + resp, pErr := tc.SendWrappedWith(btH, &bt) if pErr == nil { t.Fatalf("unexpected success: %+v", resp) - } else if _, ok := pErr.GetDetail().(*roachpb.TransactionAbortedError); !ok { - t.Fatalf("expected txn aborted error, got %v and response %+v", pErr, resp) + } else if !testutils.IsPError(pErr, regexp.QuoteMeta(expErr)) { + t.Fatalf("expected %s, got %v and response %+v", expErr, pErr, resp) + } + + // HeartbeatTxn. + hb, hbH := heartbeatArgs(pushee, tc.Clock().Now()) + resp, pErr = tc.SendWrappedWith(hbH, &hb) + if pErr == nil { + t.Fatalf("unexpected success: %+v", resp) + } else if !testutils.IsPError(pErr, regexp.QuoteMeta(expErr)) { + t.Fatalf("expected %s, got %v and response %+v", expErr, pErr, resp) + } + + // EndTransaction. + et, etH := endTxnArgs(pushee, true) + resp, pErr = tc.SendWrappedWith(etH, &et) + if pErr == nil { + t.Fatalf("unexpected success: %+v", resp) + } else if !testutils.IsPError(pErr, regexp.QuoteMeta(expErr)) { + t.Fatalf("expected %s, got %v and response %+v", expErr, pErr, resp) } } @@ -3259,9 +3279,24 @@ func TestEndTransactionTxnSpanGCThreshold(t *testing.T) { // be prevented from writing its record. // See #9522. { + // BeginTransaction. txn := newTransaction("foo", key, 1, tc.Clock()) - beginArgs, header := beginTxnArgs(key, txn) - if _, pErr := tc.SendWrappedWith(header, &beginArgs); pErr != nil { + bt, btH := beginTxnArgs(key, txn) + if _, pErr := tc.SendWrappedWith(btH, &bt); pErr != nil { + t.Fatal(pErr) + } + + // HeartbeatTxn. + txn2 := newTransaction("foo", key, 1, tc.Clock()) + hb, hbH := heartbeatArgs(txn2, tc.Clock().Now()) + if _, pErr := tc.SendWrappedWith(hbH, &hb); pErr != nil { + t.Fatal(pErr) + } + + // EndTransaction. + txn3 := newTransaction("foo", key, 1, tc.Clock()) + et, etH := endTxnArgs(txn3, true) + if _, pErr := tc.SendWrappedWith(etH, &et); pErr != nil { t.Fatal(pErr) } } @@ -3390,49 +3425,55 @@ func TestEndTransactionBeforeHeartbeat(t *testing.T) { tc.Start(t, stopper) key := []byte("a") - for _, commit := range []bool{true, false} { - txn := newTransaction("test", key, 1, tc.Clock()) - _, btH := beginTxnArgs(key, txn) - put := putArgs(key, key) - assignSeqNumsForReqs(txn, &put) - beginReply, pErr := maybeWrapWithBeginTransaction(context.Background(), tc.Sender(), btH, &put) - if pErr != nil { - t.Fatal(pErr) - } - txn.Writing = true - args, h := endTxnArgs(txn, commit) - assignSeqNumsForReqs(txn, &args) - resp, pErr := tc.SendWrappedWith(h, &args) - if pErr != nil { - t.Error(pErr) - } - reply := resp.(*roachpb.EndTransactionResponse) - expStatus := roachpb.COMMITTED - if !commit { - expStatus = roachpb.ABORTED - } - if reply.Txn.Status != expStatus { - t.Errorf("expected transaction status to be %s; got %s", expStatus, reply.Txn.Status) - } + testutils.RunTrueAndFalse(t, "begin", func(t *testing.T, begin bool) { + testutils.RunTrueAndFalse(t, "commit", func(t *testing.T, commit bool) { + key = roachpb.Key(key).Next() + txn := newTransaction("test", key, 1, tc.Clock()) + txn.Writing = true - // Try a heartbeat to the already-committed transaction; should get - // committed txn back, but without last heartbeat timestamp set. - txn.Epoch++ // need to fake a higher epoch to sneak past abort span - hBA, h := heartbeatArgs(txn, tc.Clock().Now()) + var ba roachpb.BatchRequest + bt, btH := beginTxnArgs(key, txn) + put := putArgs(key, key) + assignSeqNumsForReqs(txn, &put) + ba.Header = btH + if begin { + ba.Add(&bt) + } + ba.Add(&put) + if _, pErr := tc.Sender().Send(context.Background(), ba); pErr != nil { + t.Fatal(pErr) + } - resp, pErr = tc.SendWrappedWith(h, &hBA) - if pErr != nil { - t.Error(pErr) - } - hBR := resp.(*roachpb.HeartbeatTxnResponse) - if hBR.Txn.Status != expStatus { - t.Errorf("expected transaction status to be %s, but got %s", hBR.Txn.Status, expStatus) - } - if initHeartbeat := beginReply.Header().Txn.LastHeartbeat; hBR.Txn.LastHeartbeat != initHeartbeat { - t.Errorf("expected transaction last heartbeat to be %s, but got %s", reply.Txn.LastHeartbeat, initHeartbeat) - } - key = roachpb.Key(key).Next() - } + args, h := endTxnArgs(txn, commit) + assignSeqNumsForReqs(txn, &args) + resp, pErr := tc.SendWrappedWith(h, &args) + if pErr != nil { + t.Fatal(pErr) + } + reply := resp.(*roachpb.EndTransactionResponse) + expStatus := roachpb.COMMITTED + if !commit { + expStatus = roachpb.ABORTED + } + if reply.Txn.Status != expStatus { + t.Errorf("expected transaction status to be %s; got %s", expStatus, reply.Txn.Status) + } + + // Try a heartbeat to the already-committed transaction; should get + // committed txn back, but without last heartbeat timestamp set. + txn.Epoch++ // need to fake a higher epoch to sneak past abort span + hBA, h := heartbeatArgs(txn, tc.Clock().Now()) + + resp, pErr = tc.SendWrappedWith(h, &hBA) + if pErr != nil { + t.Error(pErr) + } + hBR := resp.(*roachpb.HeartbeatTxnResponse) + if hBR.Txn.Status != expStatus { + t.Errorf("expected transaction status to be %s, but got %s", hBR.Txn.Status, expStatus) + } + }) + }) } // TestEndTransactionAfterHeartbeat verifies that a transaction @@ -3445,52 +3486,57 @@ func TestEndTransactionAfterHeartbeat(t *testing.T) { tc.Start(t, stopper) key := roachpb.Key("a") - for _, commit := range []bool{true, false} { - txn := newTransaction("test", key, 1, tc.Clock()) - _, btH := beginTxnArgs(key, txn) - put := putArgs(key, key) - assignSeqNumsForReqs(txn, &put) - beginReply, pErr := maybeWrapWithBeginTransaction(context.Background(), tc.Sender(), btH, &put) - if pErr != nil { - t.Fatal(pErr) - } + testutils.RunTrueAndFalse(t, "begin", func(t *testing.T, begin bool) { + testutils.RunTrueAndFalse(t, "commit", func(t *testing.T, commit bool) { + txn := newTransaction("test", key, 1, tc.Clock()) - // Start out with a heartbeat to the transaction. - hBA, h := heartbeatArgs(txn, tc.Clock().Now()) + var ba roachpb.BatchRequest + bt, btH := beginTxnArgs(key, txn) + put := putArgs(key, key) + assignSeqNumsForReqs(txn, &put) + ba.Header = btH + if begin { + ba.Add(&bt) + } + ba.Add(&put) + if _, pErr := tc.Sender().Send(context.Background(), ba); pErr != nil { + t.Fatal(pErr) + } - resp, pErr := tc.SendWrappedWith(h, &hBA) - if pErr != nil { - t.Fatal(pErr) - } - hBR := resp.(*roachpb.HeartbeatTxnResponse) - if hBR.Txn.Status != roachpb.PENDING { - t.Errorf("expected transaction status to be %s, but got %s", hBR.Txn.Status, roachpb.PENDING) - } - if initHeartbeat := beginReply.Header().Txn.LastHeartbeat; hBR.Txn.LastHeartbeat == initHeartbeat { - t.Errorf("expected transaction last heartbeat to advance, but it remained at %s", initHeartbeat) - } + // Start out with a heartbeat to the transaction. + hBA, h := heartbeatArgs(txn, tc.Clock().Now()) - args, h := endTxnArgs(txn, commit) - assignSeqNumsForReqs(txn, &args) + resp, pErr := tc.SendWrappedWith(h, &hBA) + if pErr != nil { + t.Fatal(pErr) + } + hBR := resp.(*roachpb.HeartbeatTxnResponse) + if hBR.Txn.Status != roachpb.PENDING { + t.Errorf("expected transaction status to be %s, but got %s", hBR.Txn.Status, roachpb.PENDING) + } - resp, pErr = tc.SendWrappedWith(h, &args) - if pErr != nil { - t.Error(pErr) - } - reply := resp.(*roachpb.EndTransactionResponse) - expStatus := roachpb.COMMITTED - if !commit { - expStatus = roachpb.ABORTED - } - if reply.Txn.Status != expStatus { - t.Errorf("expected transaction status to be %s; got %s", expStatus, reply.Txn.Status) - } - if reply.Txn.LastHeartbeat != hBR.Txn.LastHeartbeat { - t.Errorf("expected heartbeats to remain equal: %+v != %+v", - reply.Txn.LastHeartbeat, hBR.Txn.LastHeartbeat) - } - key = key.Next() - } + args, h := endTxnArgs(txn, commit) + assignSeqNumsForReqs(txn, &args) + + resp, pErr = tc.SendWrappedWith(h, &args) + if pErr != nil { + t.Error(pErr) + } + reply := resp.(*roachpb.EndTransactionResponse) + expStatus := roachpb.COMMITTED + if !commit { + expStatus = roachpb.ABORTED + } + if reply.Txn.Status != expStatus { + t.Errorf("expected transaction status to be %s; got %s", expStatus, reply.Txn.Status) + } + if reply.Txn.LastHeartbeat != hBR.Txn.LastHeartbeat { + t.Errorf("expected heartbeats to remain equal: %+v != %+v", + reply.Txn.LastHeartbeat, hBR.Txn.LastHeartbeat) + } + key = key.Next() + }) + }) } // TestEndTransactionWithPushedTimestamp verifies that txn can be @@ -3619,8 +3665,6 @@ func TestEndTransactionWithErrors(t *testing.T) { regressTS := tc.Clock().Now() txn := newTransaction("test", roachpb.Key(""), 1, tc.Clock()) - doesNotExist := roachpb.TransactionStatus(-1) - testCases := []struct { key roachpb.Key existStatus roachpb.TransactionStatus @@ -3628,7 +3672,6 @@ func TestEndTransactionWithErrors(t *testing.T) { existTS hlc.Timestamp expErrRegexp string }{ - {roachpb.Key("a"), doesNotExist, txn.Epoch, txn.Timestamp, "txn record not found"}, {roachpb.Key("a"), roachpb.COMMITTED, txn.Epoch, txn.Timestamp, "already committed"}, {roachpb.Key("b"), roachpb.ABORTED, txn.Epoch, txn.Timestamp, regexp.QuoteMeta("TransactionAbortedError(ABORT_REASON_ABORTED_RECORD_FOUND)")}, @@ -3643,12 +3686,9 @@ func TestEndTransactionWithErrors(t *testing.T) { existTxn.Epoch = test.existEpoch existTxn.Timestamp = test.existTS txnKey := keys.TransactionKey(test.key, txn.ID) - - if test.existStatus != doesNotExist { - if err := engine.MVCCPutProto(context.Background(), tc.repl.store.Engine(), nil, txnKey, hlc.Timestamp{}, - nil, &existTxn); err != nil { - t.Fatal(err) - } + if err := engine.MVCCPutProto(context.Background(), tc.repl.store.Engine(), nil, txnKey, hlc.Timestamp{}, + nil, &existTxn); err != nil { + t.Fatal(err) } // End the transaction, verify expected error. @@ -3788,7 +3828,7 @@ func TestRaftRetryProtectionInTxn(t *testing.T) { // - the first one will fail because of a WriteTooOldError that pushes the // transaction, which fails the 1PC path and forces the txn to execute // normally at which point the WriteTooOld gets indirectly turned into a - // TransactionRetryError. + // TransactionAbortedError. // - the second one fails because the BeginTxn is detected to be a duplicate. for i := 0; i < 2; i++ { // Reach in and manually send to raft (to simulate Raft retry) and @@ -3800,8 +3840,8 @@ func TestRaftRetryProtectionInTxn(t *testing.T) { t.Fatalf("%d: unexpected error: %s", i, err) } respWithErr := <-ch - if _, ok := respWithErr.Err.GetDetail().(*roachpb.TransactionRetryError); !ok { - t.Fatalf("%d: expected TransactionRetryError; got %s", i, respWithErr.Err) + if _, ok := respWithErr.Err.GetDetail().(*roachpb.TransactionAbortedError); !ok { + t.Fatalf("%d: expected TransactionAbortedError; got %s", i, respWithErr.Err) } } } @@ -3903,6 +3943,13 @@ func TestRaftRetryCantCommitIntents(t *testing.T) { t.Fatalf("unexpected error: %s", pErr) } + // HeartbeatTxn. + hbTxn := br.Txn.Clone() + hb, hbH := heartbeatArgs(&hbTxn, tc.Clock().Now()) + if _, pErr := tc.SendWrappedWith(hbH, &hb); pErr != nil { + t.Fatalf("unexpected error: %s", pErr) + } + // EndTransaction. etTxn := br.Txn.Clone() et, etH := endTxnArgs(&etTxn, true) @@ -3941,11 +3988,16 @@ func TestRaftRetryCantCommitIntents(t *testing.T) { t.Error(pErr) } - // EndTransaction should fail with a txn not found error. + // Heartbeat should fail with a TransactionAbortedError. + _, pErr = tc.SendWrappedWith(hbH, &hb) + if !testutils.IsPError(pErr, regexp.QuoteMeta(expErr)) { + t.Errorf("expected %s; got %v", expErr, pErr) + } + + // EndTransaction should fail with a TransactionAbortedError. _, pErr = tc.SendWrappedWith(etH, &et) - if tse, ok := pErr.GetDetail().(*roachpb.TransactionStatusError); !ok || - tse.Reason != roachpb.TransactionStatusError_REASON_TXN_NOT_FOUND { - t.Fatalf("expected TransactionStatusError with REASON_TXN_NOT_FOUND, found %v", pErr) + if !testutils.IsPError(pErr, regexp.QuoteMeta(expErr)) { + t.Errorf("expected %s; got %v", expErr, pErr) } // Expect that keyB intent did not get written! @@ -3956,9 +4008,9 @@ func TestRaftRetryCantCommitIntents(t *testing.T) { } } -// Test that a duplicate BeginTransaction results in a TransactionRetryError, as -// such recognizing that it's likely the result of the batch being retried by -// DistSender. +// Test that a duplicate BeginTransaction becomes a no-op, as such recognizing +// that it's likely the result of the batch being retried by DistSender or the +// request being evaluated after the first HeartbeatTxn. func TestDuplicateBeginTransaction(t *testing.T) { defer leaktest.AfterTest(t)() tc := testContext{} @@ -3978,8 +4030,8 @@ func TestDuplicateBeginTransaction(t *testing.T) { } // Send the batch again. _, pErr = tc.Sender().Send(context.Background(), ba) - if _, ok := pErr.GetDetail().(*roachpb.TransactionRetryError); !ok { - t.Fatalf("expected retry error; got %v", pErr) + if pErr != nil { + t.Fatal(pErr) } } @@ -8382,10 +8434,6 @@ func TestNoopRequestsNotProposed(t *testing.T) { deleteReq := &roachpb.DeleteRequest{ RequestHeader: rh, } - commitTxnReq := &roachpb.EndTransactionRequest{ - RequestHeader: rh, - Commit: true, - } pushTxnReq := &roachpb.PushTxnRequest{ RequestHeader: roachpb.RequestHeader{ Key: txn.TxnMeta.Key, @@ -8464,14 +8512,6 @@ func TestNoopRequestsNotProposed(t *testing.T) { // NB: a tombstone intent is written even if no value exists at the key. expProposal: true, }, - { - name: "failed commit txn req", - useTxn: true, - req: commitTxnReq, - expFailure: "txn record not found", - // No-op - the request fails. - expProposal: false, - }, { name: "push txn req", req: pushTxnReq, diff --git a/pkg/storage/replica_tscache.go b/pkg/storage/replica_tscache.go index 3d9aa6ed32c8..43cd8a43db3c 100644 --- a/pkg/storage/replica_tscache.go +++ b/pkg/storage/replica_tscache.go @@ -20,7 +20,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/util/hlc" - "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/util/uuid" ) @@ -65,8 +64,17 @@ func (r *Replica) updateTimestampCache( switch t := args.(type) { case *roachpb.EndTransactionRequest: // EndTransaction adds the transaction key to the write - // timestamp cache to ensure replays create a transaction - // record with WriteTooOld set. + // timestamp cache as a tombstone to ensure replays and + // concurrent requests aren't able to create a new + // transaction record. + // + // It inserts the timestamp of the final batch in the + // transaction. This timestamp must necessarily be equal + // to or greater than all other batch timestamps used + // throughout the transaction. + // TODO(nvanbenschoten): This assumption wouldn't be needed + // if we compared against for the OrigTimestamp. See + // CanCreateTxnRecord. key := keys.TransactionKey(start, txnID) tc.Add(key, nil, ts, txnID, false /* readCache */) case *roachpb.ConditionalPutRequest: @@ -157,51 +165,6 @@ func (r *Replica) applyTimestampCache( args := union.GetInner() if roachpb.ConsultsTimestampCache(args) { header := args.Header() - // BeginTransaction is a special case. We use the transaction - // key to look for an entry which would indicate this transaction - // has already been finalized, in which case this BeginTxn might be a - // replay (it might also be delayed, coming in behind an async EndTxn). - // If the request hits the timestamp cache, then we return a retriable - // error: if this is a re-evaluation, then the error will be transformed - // into an ambiguous one higher up. Otherwise, if the client is still - // waiting for a result, then this cannot be a "replay" of any sort. - // - // The retriable error we return is a TransactionAbortedError, instructing - // the client to create a new transaction. Since a transaction record - // doesn't exist, there's no point in the client to continue with the - // existing transaction at a new epoch. - if _, ok := args.(*roachpb.BeginTransactionRequest); ok { - key := keys.TransactionKey(header.Key, ba.Txn.ID) - wTS, wTxnID := r.store.tsCache.GetMaxWrite(key, nil /* end */) - // GetMaxWrite will only find a timestamp interval with an - // associated txnID on the TransactionKey if an EndTxnReq has - // been processed. All other timestamp intervals will have no - // associated txnID and will be due to the low-water mark. - switch wTxnID { - case ba.Txn.ID: - newTxn := ba.Txn.Clone() - newTxn.Status = roachpb.ABORTED - newTxn.Timestamp.Forward(wTS.Next()) - return false, roachpb.NewErrorWithTxn(roachpb.NewTransactionAbortedError( - roachpb.ABORT_REASON_ALREADY_COMMITTED_OR_ROLLED_BACK_POSSIBLE_REPLAY), &newTxn) - case uuid.UUID{} /* noTxnID */ : - if !wTS.Less(ba.Txn.Timestamp) { - // On lease transfers the timestamp cache is reset with the transfer - // time as the low-water mark, so if this replica recently obtained - // the lease, this case will be true for new txns, even if they're - // not a replay. We move the timestamp forward and return retry. - newTxn := ba.Txn.Clone() - newTxn.Status = roachpb.ABORTED - newTxn.Timestamp.Forward(wTS.Next()) - return false, roachpb.NewErrorWithTxn(roachpb.NewTransactionAbortedError( - roachpb.ABORT_REASON_TIMESTAMP_CACHE_REJECTED_POSSIBLE_REPLAY), &newTxn) - } - default: - log.Fatalf(ctx, "unexpected tscache interval (%s,%s) on TxnKey %s", - wTS, wTxnID, key) - } - continue - } // Forward the timestamp if there's been a more recent read (by someone else). rTS, rTxnID := r.store.tsCache.GetMaxRead(header.Key, header.EndKey) @@ -242,3 +205,17 @@ func (r *Replica) applyTimestampCache( } return bumped, nil } + +// GetTxnTombstoneFromTimestampCache returns the tombstone timestamp for the +// provided transaction, along with an associated txnID marker. If the txnID +// is not empty then the tombstone was explicitly set by an EndTxn req. If +// it is empty then the tombstone is an implicit artifact of the timestamp +// cache's low-water mark. Either way, transactions should not write records +// at or below this timestamp. +func (r *Replica) GetTxnTombstoneFromTimestampCache( + txn *roachpb.Transaction, +) (hlc.Timestamp, uuid.UUID) { + // Use the transaction key to look for an entry in the write timestamp cache. + key := keys.TransactionKey(txn.Key, txn.ID) + return r.store.tsCache.GetMaxWrite(key, nil /* end */) +}