Index crash long run for put (eBay#537)

* add crash long run

* add adjustable max node keys in node

* add put crash to long running

.jenkins/Dockerfile

@@ -13,6 +13,7 @@ RUN set -eux; \
     rm -rf /var/lib/apt/lists/*;
 
 COPY test_index_btree /usr/local/bin/test_index_btree
+COPY test_index_crash_recovery /usr/local/bin/test_index_crash_recovery
 COPY test_meta_blk_mgr /usr/local/bin/test_meta_blk_mgr
 COPY test_log_store /usr/local/bin/test_log_store
 COPY test_home_raft_logstore /usr/local/bin/test_home_raft_logstore

.jenkins/jenkinsfile_nightly

@@ -42,6 +42,7 @@ pipeline {
                     steps {
                         sh "conan create --build missing -o homestore:sanitize=True -pr debug . ${PROJECT}/${VER}@"
                         sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_index_btree' -exec cp {} .jenkins/test_index_btree \\;"
+                        sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_index_crash_recovery' -exec cp {} .jenkins/test_index_crash_recovery \\;"
                         sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_meta_blk_mgr' -exec cp {} .jenkins/test_meta_blk_mgr \\;"
                         sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_log_store' -exec cp {} .jenkins/test_log_store \\;"
                         sh "find ${CONAN_USER_HOME} -type f -wholename '*tests/test_home_raft_logstore' -exec cp {} .jenkins/test_home_raft_logstore \\;"

conanfile.py

@@ -9,7 +9,7 @@
 
 class HomestoreConan(ConanFile):
     name = "homestore"
-    version = "6.4.55"
+    version = "6.4.56"
 
     homepage = "https://github.com/eBay/Homestore"
     description = "HomeStore Storage Engine"

src/include/homestore/btree/detail/btree_internal.hpp

@@ -244,6 +244,9 @@ struct BtreeConfig {
     uint8_t m_suggested_min_pct{30};
     uint8_t m_split_pct{50};
     uint32_t m_max_merge_nodes{3};
+#ifdef _PRERELEASE
+        uint64_t m_max_keys_in_node{0};
+#endif
     bool m_rebalance_turned_on{false};
     bool m_merge_turned_on{true};
 

src/include/homestore/btree/detail/btree_node.hpp

@@ -36,6 +36,7 @@ struct transient_hdr_t {
 
     /* these variables are accessed without taking lock and are not expected to change after init */
     uint8_t leaf_node{0};
+    uint64_t max_keys_in_node{0};
 
     bool is_leaf() const { return (leaf_node != 0); }
 };
@@ -113,6 +114,10 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
             DEBUG_ASSERT_EQ(version(), BTREE_NODE_VERSION);
         }
         m_trans_hdr.leaf_node = is_leaf;
+#ifdef _PRERELEASE
+        m_trans_hdr.max_keys_in_node = cfg.m_max_keys_in_node;
+#endif
+
     }
     virtual ~BtreeNode() = default;
 
@@ -327,6 +332,7 @@ class BtreeNode : public sisl::ObjLifeCounter< BtreeNode > {
     uint16_t level() const { return get_persistent_header_const()->level; }
 
     // uint32_t total_entries() const { return (has_valid_edge() ? total_entries() + 1 : total_entries()); }
+    uint64_t max_keys_in_node() const { return m_trans_hdr.max_keys_in_node; }
 
     void lock(locktype_t l) const {
         if (l == locktype_t::READ) {

src/include/homestore/btree/detail/simple_node.hpp

@@ -42,6 +42,7 @@ class SimpleNode : public VariantNode< K, V > {
     using BtreeNode::get_nth_value_size;
     using BtreeNode::to_string;
     using VariantNode< K, V >::get_nth_value;
+    using VariantNode< K, V >::max_keys_in_node;
 
     // Insert the key and value in provided index
     // Assumption: Node lock is already taken
@@ -202,7 +203,8 @@ class SimpleNode : public VariantNode< K, V > {
 
     bool has_room_for_put(btree_put_type put_type, uint32_t key_size, uint32_t value_size) const override {
 #ifdef _PRERELEASE
-        // return (this->total_entries() <= 3);
+        auto max_keys = max_keys_in_node();
+            if(max_keys) {return (this->total_entries() < max_keys);}
 #endif
         return ((put_type == btree_put_type::UPSERT) || (put_type == btree_put_type::INSERT))
             ? (get_available_entries() > 0)

src/lib/index/wb_cache.cpp

@@ -551,9 +551,10 @@ folly::Future< bool > IndexWBCache::async_cp_flush(IndexCPContext* cp_ctx) {
 void IndexWBCache::do_flush_one_buf(IndexCPContext* cp_ctx, IndexBufferPtr const& buf, bool part_of_batch) {
 #ifdef _PRERELEASE
     if (buf->m_crash_flag_on) {
-        std::string filename = "crash_buf_" + std::to_string(cp_ctx->id()) + ".dot";
-        LOGINFOMOD(wbcache, "Simulating crash while writing buffer {},  stored in file {}", buf->to_string(), filename);
-        cp_ctx->to_string_dot(filename);
+//        std::string filename = "crash_buf_" + std::to_string(cp_ctx->id()) + ".dot";
+//        LOGINFOMOD(wbcache, "Simulating crash while writing buffer {},  stored in file {}", buf->to_string(), filename);
+//        cp_ctx->to_string_dot(filename);
+        LOGINFOMOD(wbcache, "Simulating crash while writing buffer {}", buf->to_string());
         hs()->crash_simulator().crash();
         cp_ctx->complete(true);
         return;
@@ -581,6 +582,7 @@ void IndexWBCache::do_flush_one_buf(IndexCPContext* cp_ctx, IndexBufferPtr const
                     BtreeNode::to_string_buf(buf->raw_buffer()));
         m_vdev->async_write(r_cast< const char* >(buf->raw_buffer()), m_node_size, buf->m_blkid, part_of_batch)
             .thenValue([buf, cp_ctx](auto) {
+                // TODO: crash may cause wb_cache() to be destroyed and return null pointer
                 auto& pthis = s_cast< IndexWBCache& >(wb_cache()); // Avoiding more than 16 bytes capture
                 pthis.process_write_completion(cp_ctx, buf);
             });

src/tests/btree_helpers/btree_test_helper.hpp

@@ -217,18 +217,24 @@ struct BtreeTestHelper {
     }
 
     ////////////////////// All remove operation variants ///////////////////////////////
-    void remove_one(uint32_t k) {
+    void remove_one(uint32_t k, bool care_success = true) {
         auto existing_v = std::make_unique< V >();
         auto pk = std::make_unique< K >(k);
 
         auto rreq = BtreeSingleRemoveRequest{pk.get(), existing_v.get()};
         rreq.enable_route_tracing();
         bool removed = (m_bt->remove(rreq) == btree_status_t::success);
 
-        ASSERT_EQ(removed, m_shadow_map.exists(*pk))
-            << "Removal of key " << pk->key() << " status doesn't match with shadow";
+        if(care_success) {
+            ASSERT_EQ(removed, m_shadow_map.exists(*pk))
+                << "Removal of key " << pk->key() << " status doesn't match with shadow";
+            if (removed) { m_shadow_map.remove_and_check(*pk, *existing_v); }
+        }else {
+            // Do not care if the key is not present in the btree, just cleanup the shadow map
+            m_shadow_map.erase(*pk);
+        }
+
 
-        if (removed) { m_shadow_map.remove_and_check(*pk, *existing_v); }
     }
 
     void remove_random() {