Merge pull request #2573 from bitshares/pr-2298-budget-data-in-es

Store budget records in ES

libraries/plugins/elasticsearch/elasticsearch_plugin.cpp

@@ -661,7 +661,7 @@ operation_history_object elasticsearch_plugin::get_operation_by_id(operation_his
       "query": {
          "match":
          {
-            "account_history.operation_id": )" + operation_id_string + R"("
+            "account_history.operation_id": ")" + operation_id_string + R"("
          }
       }
    }

libraries/plugins/es_objects/es_objects.cpp

@@ -30,6 +30,7 @@
 #include <graphene/chain/market_object.hpp>
 #include <graphene/chain/asset_object.hpp>
 #include <graphene/chain/account_object.hpp>
+#include <graphene/chain/budget_record_object.hpp>
 
 #include <graphene/utilities/elasticsearch.hpp>
 #include <graphene/utilities/boost_program_options.hpp>
@@ -83,6 +84,7 @@ class es_objects_plugin_impl
          object_options balances       { true, false, true,  "balance"    };
          object_options limit_orders   { true, false, false, "limitorder" };
          object_options asset_bitasset { true, false, true,  "bitasset"   };
+         object_options budget         { true, false, true,  "budget"     };
 
          std::string index_prefix = "objects-";
          uint32_t start_es_after_block = 0;
@@ -173,6 +175,7 @@ void es_objects_plugin_impl::sync_db()
    loader.load<account_balance_object     >( _options.balances );
    loader.load<proposal_object            >( _options.proposals );
    loader.load<limit_order_object         >( _options.limit_orders );
+   loader.load<budget_record_object       >( _options.budget );
 }
 
 void es_objects_plugin_impl::index_database(const vector<object_id_type>& ids, action_type action)
@@ -198,7 +201,8 @@ void es_objects_plugin_impl::index_database(const vector<object_id_type>& ids, a
       { asset_id_type::space_type,               _options.assets         },
       { asset_bitasset_data_id_type::space_type, _options.asset_bitasset },
       { limit_order_id_type::space_type,         _options.limit_orders   },
-      { proposal_id_type::space_type,            _options.proposals      }
+      { proposal_id_type::space_type,            _options.proposals      },
+      { budget_record_id_type::space_type,       _options.budget         }
    };
 
    for( const auto& value: ids )
@@ -231,6 +235,9 @@ void es_objects_plugin_impl::index_database(const vector<object_id_type>& ids, a
          case proposal_id_type::space_type:
             prepareTemplate( db.get<proposal_object>(value), opt );
             break;
+         case budget_record_id_type::space_type:
+            prepareTemplate( db.get<budget_record_object>(value), opt );
+            break;
          default:
             break;
          }
@@ -381,6 +388,8 @@ void es_objects_plugin::plugin_set_program_options(
          ("es-objects-asset-bitasset-store-updates", boost::program_options::value<bool>(),
                "Store all updates to the bitasset data (false)")
 
+         ("es-objects-budget-records", boost::program_options::value<bool>(), "Store budget records (true)")
+
          ("es-objects-index-prefix", boost::program_options::value<std::string>(),
                "Add a prefix to the index(objects-)")
          ("es-objects-keep-only-current", boost::program_options::value<bool>(),
@@ -419,6 +428,7 @@ void detail::es_objects_plugin_impl::plugin_options::init(const boost::program_o
    utilities::get_program_option( options, "es-objects-limit-orders-no-delete",       limit_orders.no_delete );
    utilities::get_program_option( options, "es-objects-asset-bitasset",               asset_bitasset.enabled );
    utilities::get_program_option( options, "es-objects-asset-bitasset-store-updates", asset_bitasset.store_updates );
+   utilities::get_program_option( options, "es-objects-budget-records",               budget.enabled );
    utilities::get_program_option( options, "es-objects-index-prefix",         index_prefix );
    utilities::get_program_option( options, "es-objects-start-es-after-block", start_es_after_block );
    utilities::get_program_option( options, "es-objects-sync-db-on-startup",   sync_db_on_startup );

tests/elasticsearch/main.cpp

@@ -224,7 +224,10 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
       es.elasticsearch_url = GRAPHENE_TESTING_ES_URL;
       es.index_prefix = es_obj_index_prefix;
 
-      // delete all first
+      // The head block number is 1
+      BOOST_CHECK_EQUAL( db.head_block_num(), 1u );
+
+      // delete all first, this will delete genesis data and data inserted at block 1
       auto delete_objects = graphene::utilities::deleteAll(es);
       BOOST_REQUIRE(delete_objects); // require successful deletion
 
@@ -259,11 +262,30 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
 
          auto bitasset_data_id = j["hits"]["hits"][size_t(0)]["_source"]["bitasset_data_id"].as_string();
          es.endpoint = es.index_prefix + "bitasset/_doc/_search";
-         es.query = "{ \"query\" : { \"bool\": { \"must\" : [{ \"term\": { \"object_id\": \""+bitasset_data_id+"\"}}] } } }";
+         es.query = "{ \"query\" : { \"bool\": { \"must\" : [{ \"term\": { \"object_id\": \""
+                  + bitasset_data_id + "\"}}] } } }";
          res = graphene::utilities::simpleQuery(es);
          j = fc::json::from_string(res);
          auto bitasset_object_id = j["hits"]["hits"][size_t(0)]["_source"]["object_id"].as_string();
          BOOST_CHECK_EQUAL(bitasset_object_id, bitasset_data_id);
+
+         // maintenance, for budget records
+         generate_blocks( db.get_dynamic_global_properties().next_maintenance_time );
+
+         es.endpoint = es.index_prefix + "budget/_doc/_count";
+         es.query = "";
+         fc::wait_for( ES_WAIT_TIME,  [&]() {
+            res = graphene::utilities::getEndPoint(es);
+            j = fc::json::from_string(res);
+            if( !j.is_object() )
+               return false;
+            const auto& obj = j.get_object();
+            if( obj.find("count") == obj.end() )
+               return false;
+            total = obj["count"].as_string();
+            return (total == "1"); // new record inserted at the first maintenance block
+         });
+
       }
    }
    catch (fc::exception &e) {