[#5789, #5783, #5755] Don't cleanup transaction that potentially was …

…committed

Summary:
Since commit could be retried we could get into situation when transaction was actually committed, but during commit transaction reported as aborted.
So we should not cleanup intents for such transactions, because it could delete committed transaction intents from participant.

Also this diff relaxes requirements in CqlIndexTest.TxnCleanup, so tablet could contain small amount of running transactions.

Test Plan:
ybd --gtest_filter CqlIndexTest.TxnCleanup
ybd tsan --gtest_filter SnapshotTxnTest.MultiWriteWithRestartAndLongApply -n 20

Reviewers: mikhail, timur

Reviewed By: timur

Subscribers: ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D9440

src/yb/client/snapshot-txn-test.cc

@@ -589,6 +589,7 @@ bool IntermittentTxnFailure(const Status& status) {
     "Transaction expired"s,
     "Transaction metadata missing"s,
     "Unknown transaction, could be recently aborted"s,
+    "Transaction was recently aborted"s,
   };
   auto msg = status.ToString();
   for (const auto& allowed : kAllowedMessages) {
@@ -685,10 +686,14 @@ void SnapshotTxnTest::TestMultiWriteWithRestart() {
         op = ReadRow(session, key->value);
         auto flush_result = session->Flush();
         if (flush_result.ok()) {
-          break;
+          if (op->succeeded()) {
+            break;
+          }
+          if (op->response().error_message().find("timed out after") == std::string::npos) {
+            ASSERT_TRUE(op->succeeded()) << "Read failed: " << op->response().ShortDebugString();
+          }
         }
       }
-      ASSERT_TRUE(op->succeeded()) << "Read failed: " << op->response().ShortDebugString();
       auto rowblock = yb::ql::RowsResult(op.get()).GetRowBlock();
       ASSERT_EQ(rowblock->row_count(), 1);
       const auto& first_column = rowblock->row(0).column(0);

src/yb/client/transaction.cc

@@ -712,10 +712,6 @@ class YBTransaction::Impl final {
     }
     VLOG_WITH_PREFIX(4) << "Commit done: " << actual_status;
     commit_callback_(actual_status);
-
-    if (actual_status.IsExpired()) {
-      DoAbortCleanup(transaction);
-    }
   }
 
   void AbortDone(const Status& status,

src/yb/integration-tests/cql-index-test.cc

@@ -190,7 +190,7 @@ TEST_F(CqlIndexTest, TxnCleanup) {
 
   CleanFutures(&futures, CheckReady::kFalse);
 
-  AssertNoRunningTransactions(cluster_.get());
+  AssertRunningTransactionsCountLessOrEqualTo(cluster_.get(), 5);
 }
 
 } // namespace yb

src/yb/integration-tests/mini_cluster_utils.cc

@@ -37,7 +37,7 @@ size_t CountRunningTransactions(MiniCluster* cluster) {
   return result;
 }
 
-void AssertNoRunningTransactions(MiniCluster* cluster) {
+void AssertRunningTransactionsCountLessOrEqualTo(MiniCluster* cluster, size_t limit_per_tablet) {
   MonoTime deadline = MonoTime::Now() + 7s * kTimeMultiplier;
   bool has_bad = false;
   for (int i = 0; i != cluster->num_tablet_servers(); ++i) {
@@ -67,8 +67,8 @@ void AssertNoRunningTransactions(MiniCluster* cluster) {
     for (const auto& peer : tablets) {
       auto participant = peer->tablet()->transaction_participant();
       if (participant) {
-        auto status = Wait([participant] {
-              return participant->TEST_GetNumRunningTransactions() == 0;
+        auto status = Wait([participant, limit_per_tablet] {
+              return participant->TEST_GetNumRunningTransactions() <= limit_per_tablet;
             },
             deadline,
             "Wait until no transactions are running");
@@ -85,4 +85,8 @@ void AssertNoRunningTransactions(MiniCluster* cluster) {
   ASSERT_FALSE(has_bad);
 }
 
+void AssertNoRunningTransactions(MiniCluster* cluster) {
+  AssertRunningTransactionsCountLessOrEqualTo(cluster, 0);
+}
+
 } // namespace yb

src/yb/integration-tests/mini_cluster_utils.h

@@ -22,6 +22,7 @@ class MiniCluster;
 
 size_t CountRunningTransactions(MiniCluster* cluster);
 void AssertNoRunningTransactions(MiniCluster* cluster);
+void AssertRunningTransactionsCountLessOrEqualTo(MiniCluster* cluster, size_t limit_per_tablet);
 
 } // namespace yb