From e1244eb7ab23f648b08a6034136e6ea27731efcb Mon Sep 17 00:00:00 2001 From: Peter Conrad Date: Wed, 24 May 2017 19:16:09 +0200 Subject: [PATCH 01/17] Implemented "plugins" config variable --- libraries/app/application.cpp | 39 +++++++++++++++---- .../app/include/graphene/app/application.hpp | 5 ++- .../delayed_node/delayed_node_plugin.cpp | 3 +- programs/witness_node/main.cpp | 6 ++- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index bcbe66595..5e5c7b8cb 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -891,7 +891,8 @@ namespace detail { std::shared_ptr _websocket_server; std::shared_ptr _websocket_tls_server; - std::map> _plugins; + std::map> _active_plugins; + std::map> _available_plugins; bool _is_finished_syncing = false; }; @@ -933,6 +934,7 @@ void application::set_program_options(boost::program_options::options_descriptio ("enable-standby-votes-tracking", bpo::value()->implicit_value(true), "Whether to enable tracking of votes of standby witnesses and committee members. " "Set it to true to provide accurate data to API clients, set to false for slightly better performance.") + ("plugins", bpo::value(), "Space-separated list of plugins to activate") ; command_line_options.add(configuration_file_options); command_line_options.add_options() @@ -978,6 +980,22 @@ void application::initialize(const fc::path& data_dir, const boost::program_opti std::exit(EXIT_SUCCESS); } + + std::vector wanted; + if( options.count("plugins") ) + { + boost::split(wanted, options.at("plugins").as(), [](char c){return c == ' ';}); + } + else + { + wanted.push_back("witness"); + wanted.push_back("account_history"); + wanted.push_back("market_history"); + } + for (auto it = wanted.cbegin(); it != wanted.cend(); it++) + { + if (!it->empty()) enable_plugin(*it); + } } void application::startup() @@ -995,7 +1013,7 @@ void application::startup() std::shared_ptr application::get_plugin(const string& name) const { - return my->_plugins[name]; + return my->_active_plugins[name]; } net::node_ptr application::p2p_node() @@ -1028,14 +1046,21 @@ bool application::is_finished_syncing() const return my->_is_finished_syncing; } -void graphene::app::application::add_plugin(const string& name, std::shared_ptr p) +void graphene::app::application::enable_plugin(const string& name) +{ + FC_ASSERT(my->_available_plugins[name], "Unknown plugin '" + name + "'"); + my->_active_plugins[name] = my->_available_plugins[name]; + my->_active_plugins[name]->plugin_set_app(this); +} + +void graphene::app::application::add_available_plugin(std::shared_ptr p) { - my->_plugins[name] = p; + my->_available_plugins[p->plugin_name()] = p; } void application::shutdown_plugins() { - for( auto& entry : my->_plugins ) + for( auto& entry : my->_active_plugins ) entry.second->plugin_shutdown(); return; } @@ -1049,14 +1074,14 @@ void application::shutdown() void application::initialize_plugins( const boost::program_options::variables_map& options ) { - for( auto& entry : my->_plugins ) + for( auto& entry : my->_active_plugins ) entry.second->plugin_initialize( options ); return; } void application::startup_plugins() { - for( auto& entry : my->_plugins ) + for( auto& entry : my->_active_plugins ) entry.second->plugin_startup(); return; } diff --git a/libraries/app/include/graphene/app/application.hpp b/libraries/app/include/graphene/app/application.hpp index 26ae78efd..758069a0a 100644 --- a/libraries/app/include/graphene/app/application.hpp +++ b/libraries/app/include/graphene/app/application.hpp @@ -63,7 +63,7 @@ namespace graphene { namespace app { if( !plugin_cfg_options.options().empty() ) _cfg_options.add(plugin_cfg_options); - add_plugin( plug->plugin_name(), plug ); + add_available_plugin( plug ); return plug; } std::shared_ptr get_plugin( const string& name )const; @@ -89,7 +89,8 @@ namespace graphene { namespace app { boost::signals2::signal syncing_finished; private: - void add_plugin( const string& name, std::shared_ptr p ); + void enable_plugin( const string& name ); + void add_available_plugin( std::shared_ptr p ); std::shared_ptr my; boost::program_options::options_description _cli_options; diff --git a/libraries/plugins/delayed_node/delayed_node_plugin.cpp b/libraries/plugins/delayed_node/delayed_node_plugin.cpp index f9db2ccda..d49129b08 100644 --- a/libraries/plugins/delayed_node/delayed_node_plugin.cpp +++ b/libraries/plugins/delayed_node/delayed_node_plugin.cpp @@ -58,7 +58,7 @@ delayed_node_plugin::~delayed_node_plugin() void delayed_node_plugin::plugin_set_program_options(bpo::options_description& cli, bpo::options_description& cfg) { cli.add_options() - ("trusted-node", boost::program_options::value()->required(), "RPC endpoint of a trusted validating node (required)") + ("trusted-node", boost::program_options::value(), "RPC endpoint of a trusted validating node (required)") ; cfg.add(cli); } @@ -74,6 +74,7 @@ void delayed_node_plugin::connect() void delayed_node_plugin::plugin_initialize(const boost::program_options::variables_map& options) { + FC_ASSERT(options.count("trusted-node") > 0); my->remote_endpoint = "ws://" + options.at("trusted-node").as(); } diff --git a/programs/witness_node/main.cpp b/programs/witness_node/main.cpp index 8c613067a..ec08feb66 100644 --- a/programs/witness_node/main.cpp +++ b/programs/witness_node/main.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -71,6 +72,7 @@ int main(int argc, char** argv) { bpo::variables_map options; auto witness_plug = node->register_plugin(); + auto debug_witness_plug = node->register_plugin(); auto history_plug = node->register_plugin(); auto market_history_plug = node->register_plugin(); //auto generate_genesis_plug = node->register_plugin(); @@ -142,7 +144,7 @@ int main(int argc, char** argv) { exit_promise->set_value(signal); }, SIGTERM); - ilog("Started witness node on a chain with ${h} blocks.", ("h", node->chain_database()->head_block_num())); + ilog("Started BitShares node on a chain with ${h} blocks.", ("h", node->chain_database()->head_block_num())); ilog("Chain ID is ${id}", ("id", node->chain_database()->get_chain_id()) ); int signal = exit_promise->wait(); @@ -163,4 +165,4 @@ int main(int argc, char** argv) { delete node; return 1; } -} \ No newline at end of file +} From f3d961bb700391ea8b8cfe3ca43255da3f0c819f Mon Sep 17 00:00:00 2001 From: Alfredo Date: Fri, 10 Nov 2017 16:18:31 -0300 Subject: [PATCH 02/17] allow plugin to have descriptions --- libraries/app/include/graphene/app/application.hpp | 3 ++- libraries/app/include/graphene/app/plugin.hpp | 2 ++ libraries/app/plugin.cpp | 5 +++++ .../plugins/snapshot/include/graphene/snapshot/snapshot.hpp | 1 + libraries/plugins/snapshot/snapshot.cpp | 5 +++++ 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/libraries/app/include/graphene/app/application.hpp b/libraries/app/include/graphene/app/application.hpp index 758069a0a..6f55c390b 100644 --- a/libraries/app/include/graphene/app/application.hpp +++ b/libraries/app/include/graphene/app/application.hpp @@ -56,7 +56,8 @@ namespace graphene { namespace app { auto plug = std::make_shared(); plug->plugin_set_app(this); - boost::program_options::options_description plugin_cli_options("Options for plugin " + plug->plugin_name()), plugin_cfg_options; + boost::program_options::options_description plugin_cli_options(plug->plugin_name() + " plugin. " + plug->plugin_description() + "\nOptions"), plugin_cfg_options; + //boost::program_options::options_description plugin_cli_options("Options for plugin " + plug->plugin_name()), plugin_cfg_options; plug->plugin_set_program_options(plugin_cli_options, plugin_cfg_options); if( !plugin_cli_options.options().empty() ) _cli_options.add(plugin_cli_options); diff --git a/libraries/app/include/graphene/app/plugin.hpp b/libraries/app/include/graphene/app/plugin.hpp index c242130b9..45336f677 100644 --- a/libraries/app/include/graphene/app/plugin.hpp +++ b/libraries/app/include/graphene/app/plugin.hpp @@ -35,6 +35,7 @@ class abstract_plugin public: virtual ~abstract_plugin(){} virtual std::string plugin_name()const = 0; + virtual std::string plugin_description()const = 0; /** * @brief Perform early startup routines and register plugin indexes, callbacks, etc. @@ -100,6 +101,7 @@ class plugin : public abstract_plugin virtual ~plugin() override; virtual std::string plugin_name()const override; + virtual std::string plugin_description()const override; virtual void plugin_initialize( const boost::program_options::variables_map& options ) override; virtual void plugin_startup() override; virtual void plugin_shutdown() override; diff --git a/libraries/app/plugin.cpp b/libraries/app/plugin.cpp index 8568d3711..cae488a66 100644 --- a/libraries/app/plugin.cpp +++ b/libraries/app/plugin.cpp @@ -43,6 +43,11 @@ std::string plugin::plugin_name()const return ""; } +std::string plugin::plugin_description()const +{ + return ""; +} + void plugin::plugin_initialize( const boost::program_options::variables_map& options ) { return; diff --git a/libraries/plugins/snapshot/include/graphene/snapshot/snapshot.hpp b/libraries/plugins/snapshot/include/graphene/snapshot/snapshot.hpp index b3ee30c6f..eb8d3a16c 100644 --- a/libraries/plugins/snapshot/include/graphene/snapshot/snapshot.hpp +++ b/libraries/plugins/snapshot/include/graphene/snapshot/snapshot.hpp @@ -35,6 +35,7 @@ class snapshot_plugin : public graphene::app::plugin { ~snapshot_plugin() {} std::string plugin_name()const override; + std::string plugin_description()const override; virtual void plugin_set_program_options( boost::program_options::options_description &command_line_options, diff --git a/libraries/plugins/snapshot/snapshot.cpp b/libraries/plugins/snapshot/snapshot.cpp index fe856ecb1..f74ad5894 100644 --- a/libraries/plugins/snapshot/snapshot.cpp +++ b/libraries/plugins/snapshot/snapshot.cpp @@ -54,6 +54,11 @@ std::string snapshot_plugin::plugin_name()const return "snapshot"; } +std::string snapshot_plugin::plugin_description()const +{ + return "Create snapshots at a specified time or block number."; +} + void snapshot_plugin::plugin_initialize(const boost::program_options::variables_map& options) { try { ilog("snapshot plugin: plugin_initialize() begin"); From 00f14c472934e752ca920fff91226acd9c6f5b2a Mon Sep 17 00:00:00 2001 From: oxarbitrage Date: Sat, 11 Nov 2017 10:32:53 -0300 Subject: [PATCH 03/17] Merge pull request #444 from oxarbitrage/elasticsearch Elasticsearch plugin --- libraries/app/application.cpp | 14 +- .../chain/operation_history_object.hpp | 10 + libraries/plugins/CMakeLists.txt | 1 + .../plugins/elasticsearch/CMakeLists.txt | 23 ++ .../elasticsearch/elasticsearch_plugin.cpp | 388 ++++++++++++++++++ .../elasticsearch/elasticsearch_plugin.hpp | 159 +++++++ programs/witness_node/CMakeLists.txt | 2 +- programs/witness_node/main.cpp | 2 + 8 files changed, 596 insertions(+), 3 deletions(-) create mode 100644 libraries/plugins/elasticsearch/CMakeLists.txt create mode 100644 libraries/plugins/elasticsearch/elasticsearch_plugin.cpp create mode 100644 libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index 5e5c7b8cb..b366be490 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -992,9 +992,19 @@ void application::initialize(const fc::path& data_dir, const boost::program_opti wanted.push_back("account_history"); wanted.push_back("market_history"); } - for (auto it = wanted.cbegin(); it != wanted.cend(); it++) + int es_ah_conflict_counter = 0; + for (auto& it : wanted) { - if (!it->empty()) enable_plugin(*it); + if(it == "account_history") + ++es_ah_conflict_counter; + if(it == "elasticsearch") + ++es_ah_conflict_counter; + + if(es_ah_conflict_counter > 1) { + elog("Can't start program with elasticsearch and account_history plugin at the same time"); + std::exit(EXIT_FAILURE); + } + if (!it.empty()) enable_plugin(it); } } diff --git a/libraries/chain/include/graphene/chain/operation_history_object.hpp b/libraries/chain/include/graphene/chain/operation_history_object.hpp index b35b171f9..891994727 100644 --- a/libraries/chain/include/graphene/chain/operation_history_object.hpp +++ b/libraries/chain/include/graphene/chain/operation_history_object.hpp @@ -102,6 +102,16 @@ namespace graphene { namespace chain { struct by_seq; struct by_op; struct by_opid; + +typedef multi_index_container< + operation_history_object, + indexed_by< + ordered_unique< tag, member< object, object_id_type, &object::id > > + > +> operation_history_multi_index_type; + +typedef generic_index operation_history_index; + typedef multi_index_container< account_transaction_history_object, indexed_by< diff --git a/libraries/plugins/CMakeLists.txt b/libraries/plugins/CMakeLists.txt index 01079fe29..b3fe52d29 100644 --- a/libraries/plugins/CMakeLists.txt +++ b/libraries/plugins/CMakeLists.txt @@ -2,6 +2,7 @@ add_subdirectory( witness ) add_subdirectory( account_history ) add_subdirectory( accounts_list ) add_subdirectory( affiliate_stats ) +add_subdirectory( elasticsearch ) add_subdirectory( market_history ) add_subdirectory( delayed_node ) add_subdirectory( bookie ) diff --git a/libraries/plugins/elasticsearch/CMakeLists.txt b/libraries/plugins/elasticsearch/CMakeLists.txt new file mode 100644 index 000000000..f4815576d --- /dev/null +++ b/libraries/plugins/elasticsearch/CMakeLists.txt @@ -0,0 +1,23 @@ +file(GLOB HEADERS "include/graphene/elasticsearch/*.hpp") + +add_library( graphene_elasticsearch + elasticsearch_plugin.cpp + ) + +target_link_libraries( graphene_elasticsearch graphene_chain graphene_app curl ) +target_include_directories( graphene_elasticsearch + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) + +if(MSVC) + set_source_files_properties(elasticsearch_plugin.cpp PROPERTIES COMPILE_FLAGS "/bigobj" ) +endif(MSVC) + +install( TARGETS + graphene_elasticsearch + + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) +INSTALL( FILES ${HEADERS} DESTINATION "include/graphene/elasticsearch" ) + diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp new file mode 100644 index 000000000..b63802db2 --- /dev/null +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -0,0 +1,388 @@ +/* + * Copyright (c) 2017 Cryptonomex, Inc., and contributors. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace graphene { namespace elasticsearch { + +namespace detail +{ + +class elasticsearch_plugin_impl +{ + public: + elasticsearch_plugin_impl(elasticsearch_plugin& _plugin) + : _self( _plugin ) + { curl = curl_easy_init(); } + virtual ~elasticsearch_plugin_impl(); + + void update_account_histories( const signed_block& b ); + + graphene::chain::database& database() + { + return _self.database(); + } + + elasticsearch_plugin& _self; + primary_index< operation_history_index >* _oho_index; + + std::string _elasticsearch_node_url = "http://localhost:9200/"; + uint32_t _elasticsearch_bulk_replay = 10000; + uint32_t _elasticsearch_bulk_sync = 100; + bool _elasticsearch_logs = true; + bool _elasticsearch_visitor = false; + CURL *curl; // curl handler + vector bulk; // vector of op lines + private: + void add_elasticsearch( const account_id_type account_id, const optional& oho, const signed_block& b ); + void createBulkLine(account_transaction_history_object ath, operation_history_struct os, int op_type, block_struct bs, visitor_struct vs); + void sendBulk(std::string _elasticsearch_node_url, bool _elasticsearch_logs); + +}; + +elasticsearch_plugin_impl::~elasticsearch_plugin_impl() +{ + return; +} + +void elasticsearch_plugin_impl::update_account_histories( const signed_block& b ) +{ + graphene::chain::database& db = database(); + const vector >& hist = db.get_applied_operations(); + for( const optional< operation_history_object >& o_op : hist ) { + optional oho; + + auto create_oho = [&]() { + return optional( + db.create([&](operation_history_object &h) { + if (o_op.valid()) + { + h.op = o_op->op; + h.result = o_op->result; + h.block_num = o_op->block_num; + h.trx_in_block = o_op->trx_in_block; + h.op_in_trx = o_op->op_in_trx; + h.virtual_op = o_op->virtual_op; + } + })); + }; + + if( !o_op.valid() ) { + _oho_index->use_next_id(); + continue; + } + oho = create_oho(); + + const operation_history_object& op = *o_op; + + // get the set of accounts this operation applies to + flat_set impacted; + vector other; + operation_get_required_authorities( op.op, impacted, impacted, other ); // fee_payer is added here + + if( op.op.which() == operation::tag< account_create_operation >::value ) + impacted.insert( op.result.get() ); + else + graphene::app::operation_get_impacted_accounts( op.op, impacted ); + + for( auto& a : other ) + for( auto& item : a.account_auths ) + impacted.insert( item.first ); + + for( auto& account_id : impacted ) + { + add_elasticsearch( account_id, oho, b ); + } + } +} + +void elasticsearch_plugin_impl::add_elasticsearch( const account_id_type account_id, const optional & oho, const signed_block& b) +{ + graphene::chain::database& db = database(); + const auto &stats_obj = account_id(db).statistics(db); + + // add new entry + const auto &ath = db.create([&](account_transaction_history_object &obj) { + obj.operation_id = oho->id; + obj.account = account_id; + obj.sequence = stats_obj.total_ops + 1; + obj.next = stats_obj.most_recent_op; + }); + + // keep stats growing as no op will be removed + db.modify(stats_obj, [&](account_statistics_object &obj) { + obj.most_recent_op = ath.id; + obj.total_ops = ath.sequence; + }); + + // operation_type + int op_type = -1; + if (!oho->id.is_null()) + op_type = oho->op.which(); + + // operation history data + operation_history_struct os; + os.trx_in_block = oho->trx_in_block; + os.op_in_trx = oho->op_in_trx; + os.operation_result = fc::json::to_string(oho->result); + os.virtual_op = oho->virtual_op; + os.op = fc::json::to_string(oho->op); + + // visitor data + visitor_struct vs; + if(_elasticsearch_visitor) { + operation_visitor o_v; + oho->op.visit(o_v); + + vs.fee_data.asset = o_v.fee_asset; + vs.fee_data.amount = o_v.fee_amount; + vs.transfer_data.asset = o_v.transfer_asset_id; + vs.transfer_data.amount = o_v.transfer_amount; + vs.transfer_data.from = o_v.transfer_from; + vs.transfer_data.to = o_v.transfer_to; + } + + // block data + std::string trx_id = ""; + if(!b.transactions.empty() && oho->trx_in_block < b.transactions.size()) { + trx_id = b.transactions[oho->trx_in_block].id().str(); + } + block_struct bs; + bs.block_num = b.block_num(); + bs.block_time = b.timestamp; + bs.trx_id = trx_id; + + // check if we are in replay or in sync and change number of bulk documents accordingly + uint32_t limit_documents = 0; + if((fc::time_point::now() - b.timestamp) < fc::seconds(30)) + limit_documents = _elasticsearch_bulk_sync; + else + limit_documents = _elasticsearch_bulk_replay; + + createBulkLine(ath, os, op_type, bs, vs); // we have everything, creating bulk line + + if (curl && bulk.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech + sendBulk(_elasticsearch_node_url, _elasticsearch_logs); + } + + // remove everything except current object from ath + const auto &his_idx = db.get_index_type(); + const auto &by_seq_idx = his_idx.indices().get(); + auto itr = by_seq_idx.lower_bound(boost::make_tuple(account_id, 0)); + if (itr != by_seq_idx.end() && itr->account == account_id && itr->id != ath.id) { + // if found, remove the entry + const auto remove_op_id = itr->operation_id; + const auto itr_remove = itr; + ++itr; + db.remove( *itr_remove ); + // modify previous node's next pointer + // this should be always true, but just have a check here + if( itr != by_seq_idx.end() && itr->account == account_id ) + { + db.modify( *itr, [&]( account_transaction_history_object& obj ){ + obj.next = account_transaction_history_id_type(); + }); + } + // do the same on oho + const auto &by_opid_idx = his_idx.indices().get(); + if (by_opid_idx.find(remove_op_id) == by_opid_idx.end()) { + db.remove(remove_op_id(db)); + } + } +} + +void elasticsearch_plugin_impl::createBulkLine(account_transaction_history_object ath, operation_history_struct os, int op_type, block_struct bs, visitor_struct vs) +{ + bulk_struct bulks; + bulks.account_history = ath; + bulks.operation_history = os; + bulks.operation_type = op_type; + bulks.block_data = bs; + bulks.additional_data = vs; + + std::string alltogether = fc::json::to_string(bulks); + + auto block_date = bulks.block_data.block_time.to_iso_string(); + std::vector parts; + boost::split(parts, block_date, boost::is_any_of("-")); + std::string index_name = "graphene-" + parts[0] + "-" + parts[1]; + + // bulk header before each line, op_type = create to avoid dups, index id will be ath id(2.9.X). + std::string _id = fc::json::to_string(ath.id); + bulk.push_back("{ \"index\" : { \"_index\" : \""+index_name+"\", \"_type\" : \"data\", \"op_type\" : \"create\", \"_id\" : "+_id+" } }"); // header + bulk.push_back(alltogether); +} + +void elasticsearch_plugin_impl::sendBulk(std::string _elasticsearch_node_url, bool _elasticsearch_logs) +{ + + // curl buffers to read + std::string readBuffer; + std::string readBuffer_logs; + + std::string bulking = ""; + + bulking = boost::algorithm::join(bulk, "\n"); + bulking = bulking + "\n"; + bulk.clear(); + + //wlog((bulking)); + + struct curl_slist *headers = NULL; + curl_slist_append(headers, "Content-Type: application/json"); + std::string url = _elasticsearch_node_url + "_bulk"; + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_POST, true); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, bulking.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&readBuffer); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcrp/0.1"); + //curl_easy_setopt(curl, CURLOPT_VERBOSE, true); + curl_easy_perform(curl); + + long http_code = 0; + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + if(http_code == 200) { + // all good, do nothing + } + else if(http_code == 429) { + // repeat request? + } + else { + // exit everything ? + } + + if(_elasticsearch_logs) { + auto logs = readBuffer; + // do logs + std::string url_logs = _elasticsearch_node_url + "logs/data/"; + curl_easy_setopt(curl, CURLOPT_URL, url_logs.c_str()); + curl_easy_setopt(curl, CURLOPT_POST, true); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, logs.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &readBuffer_logs); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcrp/0.1"); + //curl_easy_setopt(curl, CURLOPT_VERBOSE, true); + //ilog("log here curl: ${output}", ("output", readBuffer_logs)); + curl_easy_perform(curl); + + http_code = 0; + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + if(http_code == 200) { + // all good, do nothing + } + else if(http_code == 429) { + // repeat request? + } + else { + // exit everything ? + } + } +} + +} // end namespace detail + +elasticsearch_plugin::elasticsearch_plugin() : + my( new detail::elasticsearch_plugin_impl(*this) ) +{ +} + +elasticsearch_plugin::~elasticsearch_plugin() +{ +} + +std::string elasticsearch_plugin::plugin_name()const +{ + return "elasticsearch"; +} +std::string elasticsearch_plugin::plugin_description()const +{ + return "Stores account history data in elasticsearch database(EXPERIMENTAL)."; +} + +void elasticsearch_plugin::plugin_set_program_options( + boost::program_options::options_description& cli, + boost::program_options::options_description& cfg + ) +{ + cli.add_options() + ("elasticsearch-node-url", boost::program_options::value(), "Elastic Search database node url") + ("elasticsearch-bulk-replay", boost::program_options::value(), "Number of bulk documents to index on replay(5000)") + ("elasticsearch-bulk-sync", boost::program_options::value(), "Number of bulk documents to index on a syncronied chain(10)") + ("elasticsearch-logs", boost::program_options::value(), "Log bulk events to database") + ("elasticsearch-visitor", boost::program_options::value(), "Use visitor to index additional data(slows down the replay)") + ; + cfg.add(cli); +} + +void elasticsearch_plugin::plugin_initialize(const boost::program_options::variables_map& options) +{ + database().applied_block.connect( [&]( const signed_block& b){ my->update_account_histories(b); } ); + my->_oho_index = database().add_index< primary_index< operation_history_index > >(); + database().add_index< primary_index< account_transaction_history_index > >(); + + if (options.count("elasticsearch-node-url")) { + my->_elasticsearch_node_url = options["elasticsearch-node-url"].as(); + } + if (options.count("elasticsearch-bulk-replay")) { + my->_elasticsearch_bulk_replay = options["elasticsearch-bulk-replay"].as(); + } + if (options.count("elasticsearch-bulk-sync")) { + my->_elasticsearch_bulk_sync = options["elasticsearch-bulk-sync"].as(); + } + if (options.count("elasticsearch-logs")) { + my->_elasticsearch_logs = options["elasticsearch-logs"].as(); + } + if (options.count("elasticsearch-visitor")) { + my->_elasticsearch_visitor = options["elasticsearch-visitor"].as(); + } +} + +void elasticsearch_plugin::plugin_startup() +{ +} + +} } diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp new file mode 100644 index 000000000..cc51c2478 --- /dev/null +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2017 Cryptonomex, Inc., and contributors. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#pragma once + +#include +#include +#include + +namespace graphene { namespace elasticsearch { + using namespace chain; + //using namespace graphene::db; + //using boost::multi_index_container; + //using namespace boost::multi_index; + +// +// Plugins should #define their SPACE_ID's so plugins with +// conflicting SPACE_ID assignments can be compiled into the +// same binary (by simply re-assigning some of the conflicting #defined +// SPACE_ID's in a build script). +// +// Assignment of SPACE_ID's cannot be done at run-time because +// various template automagic depends on them being known at compile +// time. +// +#ifndef ELASTICSEARCH_SPACE_ID +#define ELASTICSEARCH_SPACE_ID 6 +#endif + +static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp) +{ + ((std::string*)userp)->append((char*)contents, size * nmemb); + return size * nmemb; +} + +namespace detail +{ + class elasticsearch_plugin_impl; +} + +class elasticsearch_plugin : public graphene::app::plugin +{ + public: + elasticsearch_plugin(); + virtual ~elasticsearch_plugin(); + + std::string plugin_name()const override; + std::string plugin_description()const override; + virtual void plugin_set_program_options( + boost::program_options::options_description& cli, + boost::program_options::options_description& cfg) override; + virtual void plugin_initialize(const boost::program_options::variables_map& options) override; + virtual void plugin_startup() override; + + friend class detail::elasticsearch_plugin_impl; + std::unique_ptr my; +}; + + +struct operation_visitor +{ + typedef void result_type; + + share_type fee_amount; + asset_id_type fee_asset; + + asset_id_type transfer_asset_id; + share_type transfer_amount; + account_id_type transfer_from; + account_id_type transfer_to; + + void operator()( const graphene::chain::transfer_operation& o ) + { + fee_asset = o.fee.asset_id; + fee_amount = o.fee.amount; + + transfer_asset_id = o.amount.asset_id; + transfer_amount = o.amount.amount; + transfer_from = o.from; + transfer_to = o.to; + } + template + void operator()( const T& o ) + { + fee_asset = o.fee.asset_id; + fee_amount = o.fee.amount; + } +}; + +struct operation_history_struct { + int trx_in_block; + int op_in_trx; + std::string operation_result; + int virtual_op; + std::string op; +}; + +struct block_struct { + int block_num; + fc::time_point_sec block_time; + std::string trx_id; +}; + +struct fee_struct { + asset_id_type asset; + share_type amount; +}; + +struct transfer_struct { + asset_id_type asset; + share_type amount; + account_id_type from; + account_id_type to; +}; + +struct visitor_struct { + fee_struct fee_data; + transfer_struct transfer_data; +}; + +struct bulk_struct { + account_transaction_history_object account_history; + operation_history_struct operation_history; + int operation_type; + block_struct block_data; + visitor_struct additional_data; +}; + + +} } //graphene::elasticsearch + +FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op) ) +FC_REFLECT( graphene::elasticsearch::block_struct, (block_num)(block_time)(trx_id) ) +FC_REFLECT( graphene::elasticsearch::fee_struct, (asset)(amount) ) +FC_REFLECT( graphene::elasticsearch::transfer_struct, (asset)(amount)(from)(to) ) +FC_REFLECT( graphene::elasticsearch::visitor_struct, (fee_data)(transfer_data) ) +FC_REFLECT( graphene::elasticsearch::bulk_struct, (account_history)(operation_history)(operation_type)(block_data)(additional_data) ) + + diff --git a/programs/witness_node/CMakeLists.txt b/programs/witness_node/CMakeLists.txt index 3d03253b3..0aa73cf17 100644 --- a/programs/witness_node/CMakeLists.txt +++ b/programs/witness_node/CMakeLists.txt @@ -11,7 +11,7 @@ endif() # We have to link against graphene_debug_witness because deficiency in our API infrastructure doesn't allow plugins to be fully abstracted #246 target_link_libraries( witness_node - PRIVATE graphene_app graphene_account_history graphene_affiliate_stats graphene_market_history graphene_witness graphene_chain graphene_debug_witness graphene_bookie graphene_egenesis_full fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) + PRIVATE graphene_app graphene_account_history graphene_affiliate_stats graphene_elasticsearch graphene_market_history graphene_witness graphene_chain graphene_debug_witness graphene_bookie graphene_egenesis_full fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) # also add dependencies to graphene_generate_genesis graphene_generate_uia_sharedrop_genesis if you want those plugins install( TARGETS diff --git a/programs/witness_node/main.cpp b/programs/witness_node/main.cpp index ec08feb66..65b36c049 100644 --- a/programs/witness_node/main.cpp +++ b/programs/witness_node/main.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include //#include //#include @@ -74,6 +75,7 @@ int main(int argc, char** argv) { auto witness_plug = node->register_plugin(); auto debug_witness_plug = node->register_plugin(); auto history_plug = node->register_plugin(); + auto elasticsearch_plug = node->register_plugin(); auto market_history_plug = node->register_plugin(); //auto generate_genesis_plug = node->register_plugin(); //auto generate_uia_sharedrop_genesis_plug = node->register_plugin(); From 8d900a5276af9ad4724818b89d9bda02fe37c975 Mon Sep 17 00:00:00 2001 From: oxarbitrage Date: Tue, 20 Mar 2018 17:06:05 -0300 Subject: [PATCH 04/17] Merge pull request #500 from oxarbitrage/elasticsearch-extras es_objects plugin --- libraries/plugins/CMakeLists.txt | 1 + libraries/plugins/es_objects/CMakeLists.txt | 23 ++ libraries/plugins/es_objects/es_objects.cpp | 355 ++++++++++++++++++ .../graphene/es_objects/es_objects.hpp | 137 +++++++ libraries/utilities/CMakeLists.txt | 1 + libraries/utilities/elasticsearch.cpp | 123 ++++++ .../graphene/utilities/elasticsearch.hpp | 42 +++ programs/witness_node/CMakeLists.txt | 2 +- programs/witness_node/main.cpp | 2 + 9 files changed, 685 insertions(+), 1 deletion(-) create mode 100644 libraries/plugins/es_objects/CMakeLists.txt create mode 100644 libraries/plugins/es_objects/es_objects.cpp create mode 100644 libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp create mode 100644 libraries/utilities/elasticsearch.cpp create mode 100644 libraries/utilities/include/graphene/utilities/elasticsearch.hpp diff --git a/libraries/plugins/CMakeLists.txt b/libraries/plugins/CMakeLists.txt index b3fe52d29..58728a9e5 100644 --- a/libraries/plugins/CMakeLists.txt +++ b/libraries/plugins/CMakeLists.txt @@ -10,3 +10,4 @@ add_subdirectory( generate_genesis ) add_subdirectory( generate_uia_sharedrop_genesis ) add_subdirectory( debug_witness ) add_subdirectory( snapshot ) +add_subdirectory( es_objects ) diff --git a/libraries/plugins/es_objects/CMakeLists.txt b/libraries/plugins/es_objects/CMakeLists.txt new file mode 100644 index 000000000..92e3d1502 --- /dev/null +++ b/libraries/plugins/es_objects/CMakeLists.txt @@ -0,0 +1,23 @@ +file(GLOB HEADERS "include/graphene/es_objects/*.hpp") + +add_library( graphene_es_objects + es_objects.cpp + ) + +target_link_libraries( graphene_es_objects graphene_chain graphene_app curl ) +target_include_directories( graphene_es_objects + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) + +if(MSVC) + set_source_files_properties(es_objects.cpp PROPERTIES COMPILE_FLAGS "/bigobj" ) +endif(MSVC) + +install( TARGETS + graphene_es_objects + + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) +INSTALL( FILES ${HEADERS} DESTINATION "include/graphene/es_objects" ) + diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp new file mode 100644 index 000000000..7c9c2b613 --- /dev/null +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -0,0 +1,355 @@ +/* + * Copyright (c) 2018 oxarbitrage, and contributors. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include + +#include +#include +#include +#include + +#include + + +namespace graphene { namespace es_objects { + +namespace detail +{ + +class es_objects_plugin_impl +{ + public: + es_objects_plugin_impl(es_objects_plugin& _plugin) + : _self( _plugin ) + { curl = curl_easy_init(); } + virtual ~es_objects_plugin_impl(); + + void updateDatabase( const vector& ids , bool isNew); + + es_objects_plugin& _self; + std::string _es_objects_elasticsearch_url = "http://localhost:9200/"; + uint32_t _es_objects_bulk_replay = 5000; + uint32_t _es_objects_bulk_sync = 10; + bool _es_objects_proposals = true; + bool _es_objects_accounts = true; + bool _es_objects_assets = true; + bool _es_objects_balances = true; + bool _es_objects_limit_orders = true; + bool _es_objects_asset_bitasset = true; + bool _es_objects_logs = true; + CURL *curl; // curl handler + vector bulk; + vector prepare; + map bitassets; + //uint32_t bitasset_seq; + private: + void PrepareProposal(const proposal_object* proposal_object, const fc::time_point_sec block_time, uint32_t block_number); + void PrepareAccount(const account_object* account_object, const fc::time_point_sec block_time, uint32_t block_number); + void PrepareAsset(const asset_object* asset_object, const fc::time_point_sec block_time, uint32_t block_number); + void PrepareBalance(const balance_object* balance_object, const fc::time_point_sec block_time, uint32_t block_number); + void PrepareLimit(const limit_order_object* limit_object, const fc::time_point_sec block_time, uint32_t block_number); + void PrepareBitAsset(const asset_bitasset_data_object* bitasset_object, const fc::time_point_sec block_time, uint32_t block_number); +}; + +void es_objects_plugin_impl::updateDatabase( const vector& ids , bool isNew) +{ + + graphene::chain::database &db = _self.database(); + + const fc::time_point_sec block_time = db.head_block_time(); + const uint32_t block_number = db.head_block_num(); + + // check if we are in replay or in sync and change number of bulk documents accordingly + uint32_t limit_documents = 0; + if((fc::time_point::now() - block_time) < fc::seconds(30)) + limit_documents = _es_objects_bulk_sync; + else + limit_documents = _es_objects_bulk_replay; + + if (curl && bulk.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech + if(!graphene::utilities::SendBulk(curl, bulk, _es_objects_elasticsearch_url, _es_objects_logs, "objects_logs")) + elog("Error sending data to database"); + bulk.clear(); + } + + for(auto const& value: ids) { + if(value.is() && _es_objects_proposals) { + auto obj = db.find_object(value); + auto p = static_cast(obj); + if(p != nullptr) + PrepareProposal(p, block_time, block_number); + } + else if(value.is() && _es_objects_accounts) { + auto obj = db.find_object(value); + auto a = static_cast(obj); + if(a != nullptr) + PrepareAccount(a, block_time, block_number); + } + else if(value.is() && _es_objects_assets) { + auto obj = db.find_object(value); + auto a = static_cast(obj); + if(a != nullptr) + PrepareAsset(a, block_time, block_number); + } + else if(value.is() && _es_objects_balances) { + auto obj = db.find_object(value); + auto b = static_cast(obj); + if(b != nullptr) + PrepareBalance(b, block_time, block_number); + } + else if(value.is() && _es_objects_limit_orders) { + auto obj = db.find_object(value); + auto l = static_cast(obj); + if(l != nullptr) + PrepareLimit(l, block_time, block_number); + } + else if(value.is() && _es_objects_asset_bitasset) { + auto obj = db.find_object(value); + auto ba = static_cast(obj); + if(ba != nullptr) + PrepareBitAsset(ba, block_time, block_number); + } + } +} + +void es_objects_plugin_impl::PrepareProposal(const proposal_object* proposal_object, const fc::time_point_sec block_time, uint32_t block_number) +{ + proposal_struct prop; + prop.object_id = proposal_object->id; + prop.block_time = block_time; + prop.block_number = block_number; + prop.expiration_time = proposal_object->expiration_time; + prop.review_period_time = proposal_object->review_period_time; + prop.proposed_transaction = fc::json::to_string(proposal_object->proposed_transaction); + prop.required_owner_approvals = fc::json::to_string(proposal_object->required_owner_approvals); + prop.available_owner_approvals = fc::json::to_string(proposal_object->available_owner_approvals); + prop.required_active_approvals = fc::json::to_string(proposal_object->required_active_approvals); + prop.available_key_approvals = fc::json::to_string(proposal_object->available_key_approvals); + prop.proposer = proposal_object->proposer; + + std::string data = fc::json::to_string(prop); + prepare = graphene::utilities::createBulk("bitshares-proposal", data, "", 1); + bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare.clear(); +} + +void es_objects_plugin_impl::PrepareAccount(const account_object* account_object, const fc::time_point_sec block_time, uint32_t block_number) +{ + account_struct acct; + acct.object_id = account_object->id; + acct.block_time = block_time; + acct.block_number = block_number; + acct.membership_expiration_date = account_object->membership_expiration_date; + acct.registrar = account_object->registrar; + acct.referrer = account_object->referrer; + acct.lifetime_referrer = account_object->lifetime_referrer; + acct.network_fee_percentage = account_object->network_fee_percentage; + acct.lifetime_referrer_fee_percentage = account_object->lifetime_referrer_fee_percentage; + acct.referrer_rewards_percentage = account_object->referrer_rewards_percentage; + acct.name = account_object->name; + acct.owner_account_auths = fc::json::to_string(account_object->owner.account_auths); + acct.owner_key_auths = fc::json::to_string(account_object->owner.key_auths); + acct.owner_address_auths = fc::json::to_string(account_object->owner.address_auths); + acct.active_account_auths = fc::json::to_string(account_object->active.account_auths); + acct.active_key_auths = fc::json::to_string(account_object->active.key_auths); + acct.active_address_auths = fc::json::to_string(account_object->active.address_auths); + acct.voting_account = account_object->options.voting_account; + + std::string data = fc::json::to_string(acct); + prepare = graphene::utilities::createBulk("bitshares-account", data, "", 1); + bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare.clear(); +} + +void es_objects_plugin_impl::PrepareAsset(const asset_object* asset_object, const fc::time_point_sec block_time, uint32_t block_number) +{ + asset_struct _asset; + _asset.object_id = asset_object->id; + _asset.block_time = block_time; + _asset.block_number = block_number; + _asset.symbol = asset_object->symbol; + _asset.issuer = asset_object->issuer; + _asset.is_market_issued = asset_object->is_market_issued(); + _asset.dynamic_asset_data_id = asset_object->dynamic_asset_data_id; + _asset.bitasset_data_id = asset_object->bitasset_data_id; + + std::string data = fc::json::to_string(_asset); + prepare = graphene::utilities::createBulk("bitshares-asset", data, fc::json::to_string(asset_object->id), 0); + bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare.clear(); +} + +void es_objects_plugin_impl::PrepareBalance(const balance_object* balance_object, const fc::time_point_sec block_time, uint32_t block_number) +{ + balance_struct balance; + balance.object_id = balance_object->id; + balance.block_time = block_time; + balance.block_number = block_number;balance.owner = balance_object->owner; + balance.asset_id = balance_object->balance.asset_id; + balance.amount = balance_object->balance.amount; + + std::string data = fc::json::to_string(balance); + prepare = graphene::utilities::createBulk("bitshares-balance", data, "", 1); + bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare.clear(); +} + +void es_objects_plugin_impl::PrepareLimit(const limit_order_object* limit_object, const fc::time_point_sec block_time, uint32_t block_number) +{ + limit_order_struct limit; + limit.object_id = limit_object->id; + limit.block_time = block_time; + limit.block_number = block_number; + limit.expiration = limit_object->expiration; + limit.seller = limit_object->seller; + limit.for_sale = limit_object->for_sale; + limit.sell_price = limit_object->sell_price; + limit.deferred_fee = limit_object->deferred_fee; + + std::string data = fc::json::to_string(limit); + prepare = graphene::utilities::createBulk("bitshares-limitorder", data, "", 1); + bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare.clear(); +} + +void es_objects_plugin_impl::PrepareBitAsset(const asset_bitasset_data_object* bitasset_object, const fc::time_point_sec block_time, uint32_t block_number) +{ + if(!bitasset_object->is_prediction_market) { + + auto object_id = bitasset_object->id; + auto it = bitassets.find(object_id); + if(it == bitassets.end()) + bitassets[object_id] = fc::json::to_string(bitasset_object->current_feed); + else { + if(it->second == fc::json::to_string(bitasset_object->current_feed)) return; + else bitassets[object_id] = fc::json::to_string(bitasset_object->current_feed); + } + + bitasset_struct bitasset; + + bitasset.object_id = bitasset_object->id; + bitasset.block_time = block_time; + bitasset.block_number = block_number; + bitasset.current_feed = fc::json::to_string(bitasset_object->current_feed); + bitasset.current_feed_publication_time = bitasset_object->current_feed_publication_time; + + std::string data = fc::json::to_string(bitasset); + prepare = graphene::utilities::createBulk("bitshares-bitasset", data, "", 1); + bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare.clear(); + } +} + +es_objects_plugin_impl::~es_objects_plugin_impl() +{ + return; +} + + +} // end namespace detail + +es_objects_plugin::es_objects_plugin() : + my( new detail::es_objects_plugin_impl(*this) ) +{ +} + +es_objects_plugin::~es_objects_plugin() +{ +} + +std::string es_objects_plugin::plugin_name()const +{ + return "es_objects"; +} +std::string es_objects_plugin::plugin_description()const +{ + return "Stores blockchain objects in ES database. Experimental."; +} + +void es_objects_plugin::plugin_set_program_options( + boost::program_options::options_description& cli, + boost::program_options::options_description& cfg + ) +{ + cli.add_options() + ("es-objects-elasticsearch-url", boost::program_options::value(), "Elasticsearch node url") + ("es-objects-logs", boost::program_options::value(), "Log bulk events to database") + ("es-objects-bulk-replay", boost::program_options::value(), "Number of bulk documents to index on replay(5000)") + ("es-objects-bulk-sync", boost::program_options::value(), "Number of bulk documents to index on a syncronied chain(10)") + ("es-objects-proposals", boost::program_options::value(), "Store proposal objects") + ("es-objects-accounts", boost::program_options::value(), "Store account objects") + ("es-objects-assets", boost::program_options::value(), "Store asset objects") + ("es-objects-balances", boost::program_options::value(), "Store balances objects") + ("es-objects-limit-orders", boost::program_options::value(), "Store limit order objects") + ("es-objects-asset-bitasset", boost::program_options::value(), "Store feed data") + + ; + cfg.add(cli); +} + +void es_objects_plugin::plugin_initialize(const boost::program_options::variables_map& options) +{ + database().new_objects.connect([&]( const vector& ids, const flat_set& impacted_accounts ){ my->updateDatabase(ids, 1); }); + database().changed_objects.connect([&]( const vector& ids, const flat_set& impacted_accounts ){ my->updateDatabase(ids, 0); }); + + if (options.count("es-objects-elasticsearch-url")) { + my->_es_objects_elasticsearch_url = options["es-objects-elasticsearch-url"].as(); + } + if (options.count("es-objects-logs")) { + my->_es_objects_logs = options["es-objects-logs"].as(); + } + if (options.count("es-objects-bulk-replay")) { + my->_es_objects_bulk_replay = options["es-objects-bulk-replay"].as(); + } + if (options.count("es-objects-bulk-sync")) { + my->_es_objects_bulk_sync = options["es-objects-bulk-sync"].as(); + } + if (options.count("es-objects-proposals")) { + my->_es_objects_proposals = options["es-objects-proposals"].as(); + } + if (options.count("es-objects-accounts")) { + my->_es_objects_accounts = options["es-objects-accounts"].as(); + } + if (options.count("es-objects-assets")) { + my->_es_objects_assets = options["es-objects-assets"].as(); + } + if (options.count("es-objects-balances")) { + my->_es_objects_balances = options["es-objects-balances"].as(); + } + if (options.count("es-objects-limit-orders")) { + my->_es_objects_limit_orders = options["es-objects-limit-orders"].as(); + } + if (options.count("es-objects-asset-bitasset")) { + my->_es_objects_asset_bitasset = options["es-objects-asset-bitasset"].as(); + } +} + +void es_objects_plugin::plugin_startup() +{ + ilog("elasticsearch objects: plugin_startup() begin"); +} + +} } \ No newline at end of file diff --git a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp new file mode 100644 index 000000000..31809e040 --- /dev/null +++ b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2018 oxarbitrage, and contributors. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#pragma once + +#include +#include + +namespace graphene { namespace es_objects { + +using namespace chain; + + +namespace detail +{ + class es_objects_plugin_impl; +} + +class es_objects_plugin : public graphene::app::plugin +{ + public: + es_objects_plugin(); + virtual ~es_objects_plugin(); + + std::string plugin_name()const override; + std::string plugin_description()const override; + virtual void plugin_set_program_options( + boost::program_options::options_description& cli, + boost::program_options::options_description& cfg) override; + virtual void plugin_initialize(const boost::program_options::variables_map& options) override; + virtual void plugin_startup() override; + + friend class detail::es_objects_plugin_impl; + std::unique_ptr my; +}; + +struct proposal_struct { + object_id_type object_id; + fc::time_point_sec block_time; + uint32_t block_number; + time_point_sec expiration_time; + optional review_period_time; + string proposed_transaction; + string required_active_approvals; + string available_active_approvals; + string required_owner_approvals; + string available_owner_approvals; + string available_key_approvals; + account_id_type proposer; + +}; +struct account_struct { + object_id_type object_id; + fc::time_point_sec block_time; + uint32_t block_number; + time_point_sec membership_expiration_date; + account_id_type registrar; + account_id_type referrer; + account_id_type lifetime_referrer; + uint16_t network_fee_percentage; + uint16_t lifetime_referrer_fee_percentage; + uint16_t referrer_rewards_percentage; + string name; + string owner_account_auths; + string owner_key_auths; + string owner_address_auths; + string active_account_auths; + string active_key_auths; + string active_address_auths; + account_id_type voting_account; +}; +struct asset_struct { + object_id_type object_id; + fc::time_point_sec block_time; + uint32_t block_number; + string symbol; + account_id_type issuer; + bool is_market_issued; + asset_dynamic_data_id_type dynamic_asset_data_id; + optional bitasset_data_id; + +}; +struct balance_struct { + object_id_type object_id; + fc::time_point_sec block_time; + uint32_t block_number; + address owner; + asset_id_type asset_id; + share_type amount; +}; +struct limit_order_struct { + object_id_type object_id; + fc::time_point_sec block_time; + uint32_t block_number; + time_point_sec expiration; + account_id_type seller; + share_type for_sale; + price sell_price; + share_type deferred_fee; +}; +struct bitasset_struct { + object_id_type object_id; + fc::time_point_sec block_time; + uint32_t block_number; + string current_feed; + time_point_sec current_feed_publication_time; + time_point_sec feed_expiration_time; +}; + +} } //graphene::es_objects + +FC_REFLECT( graphene::es_objects::proposal_struct, (object_id)(block_time)(block_number)(expiration_time)(review_period_time)(proposed_transaction)(required_active_approvals)(available_active_approvals)(required_owner_approvals)(available_owner_approvals)(available_key_approvals)(proposer) ) +FC_REFLECT( graphene::es_objects::account_struct, (object_id)(block_time)(block_number)(membership_expiration_date)(registrar)(referrer)(lifetime_referrer)(network_fee_percentage)(lifetime_referrer_fee_percentage)(referrer_rewards_percentage)(name)(owner_account_auths)(owner_key_auths)(owner_address_auths)(active_account_auths)(active_key_auths)(active_address_auths)(voting_account) ) +FC_REFLECT( graphene::es_objects::asset_struct, (object_id)(block_time)(block_number)(symbol)(issuer)(is_market_issued)(dynamic_asset_data_id)(bitasset_data_id) ) +FC_REFLECT( graphene::es_objects::balance_struct, (object_id)(block_time)(block_number)(block_time)(owner)(asset_id)(amount) ) +FC_REFLECT( graphene::es_objects::limit_order_struct, (object_id)(block_time)(block_number)(expiration)(seller)(for_sale)(sell_price)(deferred_fee) ) +FC_REFLECT( graphene::es_objects::bitasset_struct, (object_id)(block_time)(block_number)(current_feed)(current_feed_publication_time) ) \ No newline at end of file diff --git a/libraries/utilities/CMakeLists.txt b/libraries/utilities/CMakeLists.txt index f2d646d56..98086b105 100644 --- a/libraries/utilities/CMakeLists.txt +++ b/libraries/utilities/CMakeLists.txt @@ -14,6 +14,7 @@ set(sources string_escape.cpp tempdir.cpp words.cpp + elasticsearch.cpp ${HEADERS}) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/git_revision.cpp.in" "${CMAKE_CURRENT_BINARY_DIR}/git_revision.cpp" @ONLY) diff --git a/libraries/utilities/elasticsearch.cpp b/libraries/utilities/elasticsearch.cpp new file mode 100644 index 000000000..1674a12ae --- /dev/null +++ b/libraries/utilities/elasticsearch.cpp @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2018 oxarbitrage, and contributors. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include + +#include +#include + +namespace graphene { namespace utilities { + +bool SendBulk(CURL *curl, std::vector& bulk, std::string elasticsearch_url, bool do_logs, std::string logs_index) +{ + // curl buffers to read + std::string readBuffer; + std::string readBuffer_logs; + + std::string bulking = ""; + + bulking = boost::algorithm::join(bulk, "\n"); + bulking = bulking + "\n"; + bulk.clear(); + + struct curl_slist *headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/json"); + std::string url = elasticsearch_url + "_bulk"; + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_POST, true); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, bulking.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&readBuffer); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcrp/0.1"); + //curl_easy_setopt(curl, CURLOPT_VERBOSE, true); + curl_easy_perform(curl); + + long http_code = 0; + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + if(http_code == 200) { + // all good, do nothing + } + else if(http_code == 413) { + elog("413 error: Can be low space disk"); + return 0; + } + else { + elog(http_code + "error: Unknown error"); + return 0; + } + + if(do_logs) { + auto logs = readBuffer; + // do logs + std::string url_logs = elasticsearch_url + logs_index + "/data/"; + curl_easy_setopt(curl, CURLOPT_URL, url_logs.c_str()); + curl_easy_setopt(curl, CURLOPT_POST, true); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, logs.c_str()); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &readBuffer_logs); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcrp/0.1"); + curl_easy_perform(curl); + + http_code = 0; + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + if(http_code == 200) { + // all good, do nothing + return 1; + } + else if(http_code == 201) { + // 201 is ok + return 1; + } + else if(http_code == 409) { + // 409 for record already exist is ok + return 1; + } + else if(http_code == 413) { + elog("413 error: Can be low space disk"); + return 0; + } + else { + elog(http_code + "error: Unknown error"); + return 0; + } + } + return 0; +} + +std::vector createBulk(std::string index_name, std::string data, std::string id, bool onlycreate) +{ + std::vector bulk; + std::string create_string = ""; + if(!onlycreate) + create_string = ",\"_id\" : "+id; + + bulk.push_back("{ \"index\" : { \"_index\" : \""+index_name+"\", \"_type\" : \"data\" "+create_string+" } }"); + bulk.push_back(data); + + return bulk; +} + + +} } // end namespace graphene::utilities diff --git a/libraries/utilities/include/graphene/utilities/elasticsearch.hpp b/libraries/utilities/include/graphene/utilities/elasticsearch.hpp new file mode 100644 index 000000000..517f23456 --- /dev/null +++ b/libraries/utilities/include/graphene/utilities/elasticsearch.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2018 oxarbitrage, and contributors. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#pragma once +#include +#include +#include + +#include + +static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp) +{ + ((std::string*)userp)->append((char*)contents, size * nmemb); + return size * nmemb; +} + +namespace graphene { namespace utilities { + + bool SendBulk(CURL *curl, std::vector & bulk, std::string elasticsearch_url, bool do_logs, std::string logs_index); + std::vector createBulk(std::string type, std::string data, std::string id, bool onlycreate); + +} } // end namespace graphene::utilities diff --git a/programs/witness_node/CMakeLists.txt b/programs/witness_node/CMakeLists.txt index 0aa73cf17..0c4c1db45 100644 --- a/programs/witness_node/CMakeLists.txt +++ b/programs/witness_node/CMakeLists.txt @@ -11,7 +11,7 @@ endif() # We have to link against graphene_debug_witness because deficiency in our API infrastructure doesn't allow plugins to be fully abstracted #246 target_link_libraries( witness_node - PRIVATE graphene_app graphene_account_history graphene_affiliate_stats graphene_elasticsearch graphene_market_history graphene_witness graphene_chain graphene_debug_witness graphene_bookie graphene_egenesis_full fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) + PRIVATE graphene_app graphene_account_history graphene_affiliate_stats graphene_elasticsearch graphene_market_history graphene_witness graphene_chain graphene_debug_witness graphene_bookie graphene_egenesis_full graphene_es_objects fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) # also add dependencies to graphene_generate_genesis graphene_generate_uia_sharedrop_genesis if you want those plugins install( TARGETS diff --git a/programs/witness_node/main.cpp b/programs/witness_node/main.cpp index 65b36c049..0d6e65c67 100644 --- a/programs/witness_node/main.cpp +++ b/programs/witness_node/main.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include //#include //#include @@ -76,6 +77,7 @@ int main(int argc, char** argv) { auto debug_witness_plug = node->register_plugin(); auto history_plug = node->register_plugin(); auto elasticsearch_plug = node->register_plugin(); + auto es_objects_plug = node->register_plugin(); auto market_history_plug = node->register_plugin(); //auto generate_genesis_plug = node->register_plugin(); //auto generate_uia_sharedrop_genesis_plug = node->register_plugin(); From e91e61e6cba0dc75d6a8c69eef93faacc1d10d10 Mon Sep 17 00:00:00 2001 From: Abit Date: Wed, 25 Apr 2018 16:34:37 +0200 Subject: [PATCH 05/17] Merge pull request #873 from pmconrad/585_fix_history_ids Fix history ids --- .../plugins/elasticsearch/elasticsearch_plugin.cpp | 13 ++++++++++++- tests/tests/history_api_tests.cpp | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index b63802db2..a7d3fefae 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -90,10 +90,21 @@ void elasticsearch_plugin_impl::update_account_histories( const signed_block& b { graphene::chain::database& db = database(); const vector >& hist = db.get_applied_operations(); + bool is_first = true; + auto skip_oho_id = [&is_first,&db,this]() { + if( is_first && db._undo_db.enabled() ) // this ensures that the current id is rolled back on undo + { + db.remove( db.create( []( operation_history_object& obj) {} ) ); + is_first = false; + } + else + _oho_index->use_next_id(); + }; for( const optional< operation_history_object >& o_op : hist ) { optional oho; auto create_oho = [&]() { + is_first = false; return optional( db.create([&](operation_history_object &h) { if (o_op.valid()) @@ -109,7 +120,7 @@ void elasticsearch_plugin_impl::update_account_histories( const signed_block& b }; if( !o_op.valid() ) { - _oho_index->use_next_id(); + skip_oho_id(); continue; } oho = create_oho(); diff --git a/tests/tests/history_api_tests.cpp b/tests/tests/history_api_tests.cpp index 4edccce52..943b8265d 100644 --- a/tests/tests/history_api_tests.cpp +++ b/tests/tests/history_api_tests.cpp @@ -595,4 +595,4 @@ BOOST_AUTO_TEST_CASE(get_account_history_operations) { } } -BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END() From c9583f4486605ecb5152881d69ec6de1c833f834 Mon Sep 17 00:00:00 2001 From: Abit Date: Thu, 2 Aug 2018 15:31:54 +0000 Subject: [PATCH 06/17] Merge pull request #1201 from oxarbitrage/elasticsearch_tests2 Elasticsearch refactor --- .../elasticsearch/elasticsearch_plugin.cpp | 377 ++++++++++-------- .../elasticsearch/elasticsearch_plugin.hpp | 58 ++- libraries/plugins/es_objects/es_objects.cpp | 317 ++++++++++----- .../graphene/es_objects/es_objects.hpp | 74 +++- libraries/utilities/elasticsearch.cpp | 221 ++++++---- .../graphene/utilities/elasticsearch.hpp | 40 +- tests/CMakeLists.txt | 20 +- tests/common/database_fixture.cpp | 51 ++- tests/elasticsearch/main.cpp | 213 ++++++++++ 9 files changed, 981 insertions(+), 390 deletions(-) create mode 100644 tests/elasticsearch/main.cpp diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index a7d3fefae..b69ff64aa 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -23,26 +23,11 @@ */ #include - #include - #include -#include -#include -#include -#include -#include -#include - #include -#include - #include -#include -#include -#include -#include -#include +#include namespace graphene { namespace elasticsearch { @@ -57,7 +42,7 @@ class elasticsearch_plugin_impl { curl = curl_easy_init(); } virtual ~elasticsearch_plugin_impl(); - void update_account_histories( const signed_block& b ); + bool update_account_histories( const signed_block& b ); graphene::chain::database& database() { @@ -70,15 +55,39 @@ class elasticsearch_plugin_impl std::string _elasticsearch_node_url = "http://localhost:9200/"; uint32_t _elasticsearch_bulk_replay = 10000; uint32_t _elasticsearch_bulk_sync = 100; - bool _elasticsearch_logs = true; bool _elasticsearch_visitor = false; + std::string _elasticsearch_basic_auth = ""; + std::string _elasticsearch_index_prefix = "bitshares-"; CURL *curl; // curl handler - vector bulk; // vector of op lines + vector bulk_lines; // vector of op lines + vector prepare; + + graphene::utilities::ES es; + uint32_t limit_documents; + int16_t op_type; + operation_history_struct os; + block_struct bs; + visitor_struct vs; + bulk_struct bulk_line_struct; + std::string bulk_line; + std::string index_name; + bool is_sync = false; private: - void add_elasticsearch( const account_id_type account_id, const optional& oho, const signed_block& b ); - void createBulkLine(account_transaction_history_object ath, operation_history_struct os, int op_type, block_struct bs, visitor_struct vs); - void sendBulk(std::string _elasticsearch_node_url, bool _elasticsearch_logs); - + bool add_elasticsearch( const account_id_type account_id, const optional& oho, const signed_block& b ); + const account_transaction_history_object& addNewEntry(const account_statistics_object& stats_obj, + const account_id_type account_id, + const optional & oho); + const account_statistics_object& getStatsObject(const account_id_type account_id); + void growStats(const account_statistics_object& stats_obj, const account_transaction_history_object& ath); + void getOperationType(const optional & oho); + void doOperationHistory(const optional & oho); + void doBlock(const optional & oho, const signed_block& b); + void doVisitor(const optional & oho); + void checkState(const fc::time_point_sec& block_time); + void cleanObjects(const account_transaction_history_object& ath, account_id_type account_id); + void createBulkLine(const account_transaction_history_object& ath); + void prepareBulk(const account_transaction_history_id_type& ath_id); + void populateESstruct(); }; elasticsearch_plugin_impl::~elasticsearch_plugin_impl() @@ -86,8 +95,11 @@ elasticsearch_plugin_impl::~elasticsearch_plugin_impl() return; } -void elasticsearch_plugin_impl::update_account_histories( const signed_block& b ) +bool elasticsearch_plugin_impl::update_account_histories( const signed_block& b ) { + checkState(b.timestamp); + index_name = graphene::utilities::generateIndexName(b.timestamp, _elasticsearch_index_prefix); + graphene::chain::database& db = database(); const vector >& hist = db.get_applied_operations(); bool is_first = true; @@ -125,6 +137,13 @@ void elasticsearch_plugin_impl::update_account_histories( const signed_block& b } oho = create_oho(); + // populate what we can before impacted loop + getOperationType(oho); + doOperationHistory(oho); + doBlock(oho, b); + if(_elasticsearch_visitor) + doVisitor(oho); + const operation_history_object& op = *o_op; // get the set of accounts this operation applies to @@ -143,80 +162,170 @@ void elasticsearch_plugin_impl::update_account_histories( const signed_block& b for( auto& account_id : impacted ) { - add_elasticsearch( account_id, oho, b ); + if(!add_elasticsearch( account_id, oho, b )) + return false; } } + // we send bulk at end of block when we are in sync for better real time client experience + if(is_sync) + { + populateESstruct(); + if(es.bulk_lines.size() > 0) + { + prepare.clear(); + if(!graphene::utilities::SendBulk(es)) + return false; + else + bulk_lines.clear(); + } + } + + return true; } -void elasticsearch_plugin_impl::add_elasticsearch( const account_id_type account_id, const optional & oho, const signed_block& b) +void elasticsearch_plugin_impl::checkState(const fc::time_point_sec& block_time) { - graphene::chain::database& db = database(); - const auto &stats_obj = account_id(db).statistics(db); - - // add new entry - const auto &ath = db.create([&](account_transaction_history_object &obj) { - obj.operation_id = oho->id; - obj.account = account_id; - obj.sequence = stats_obj.total_ops + 1; - obj.next = stats_obj.most_recent_op; - }); - - // keep stats growing as no op will be removed - db.modify(stats_obj, [&](account_statistics_object &obj) { - obj.most_recent_op = ath.id; - obj.total_ops = ath.sequence; - }); + if((fc::time_point::now() - block_time) < fc::seconds(30)) + { + limit_documents = _elasticsearch_bulk_sync; + is_sync = true; + } + else + { + limit_documents = _elasticsearch_bulk_replay; + is_sync = false; + } +} - // operation_type - int op_type = -1; +void elasticsearch_plugin_impl::getOperationType(const optional & oho) +{ if (!oho->id.is_null()) op_type = oho->op.which(); +} - // operation history data - operation_history_struct os; +void elasticsearch_plugin_impl::doOperationHistory(const optional & oho) +{ os.trx_in_block = oho->trx_in_block; os.op_in_trx = oho->op_in_trx; os.operation_result = fc::json::to_string(oho->result); os.virtual_op = oho->virtual_op; os.op = fc::json::to_string(oho->op); +} - // visitor data - visitor_struct vs; - if(_elasticsearch_visitor) { - operation_visitor o_v; - oho->op.visit(o_v); - - vs.fee_data.asset = o_v.fee_asset; - vs.fee_data.amount = o_v.fee_amount; - vs.transfer_data.asset = o_v.transfer_asset_id; - vs.transfer_data.amount = o_v.transfer_amount; - vs.transfer_data.from = o_v.transfer_from; - vs.transfer_data.to = o_v.transfer_to; - } - - // block data +void elasticsearch_plugin_impl::doBlock(const optional & oho, const signed_block& b) +{ std::string trx_id = ""; - if(!b.transactions.empty() && oho->trx_in_block < b.transactions.size()) { + if(oho->trx_in_block < b.transactions.size()) trx_id = b.transactions[oho->trx_in_block].id().str(); - } - block_struct bs; bs.block_num = b.block_num(); bs.block_time = b.timestamp; bs.trx_id = trx_id; +} - // check if we are in replay or in sync and change number of bulk documents accordingly - uint32_t limit_documents = 0; - if((fc::time_point::now() - b.timestamp) < fc::seconds(30)) - limit_documents = _elasticsearch_bulk_sync; - else - limit_documents = _elasticsearch_bulk_replay; - - createBulkLine(ath, os, op_type, bs, vs); // we have everything, creating bulk line +void elasticsearch_plugin_impl::doVisitor(const optional & oho) +{ + operation_visitor o_v; + oho->op.visit(o_v); + + vs.fee_data.asset = o_v.fee_asset; + vs.fee_data.amount = o_v.fee_amount; + + vs.transfer_data.asset = o_v.transfer_asset_id; + vs.transfer_data.amount = o_v.transfer_amount; + vs.transfer_data.from = o_v.transfer_from; + vs.transfer_data.to = o_v.transfer_to; + + vs.fill_data.order_id = o_v.fill_order_id; + vs.fill_data.account_id = o_v.fill_account_id; + vs.fill_data.pays_asset_id = o_v.fill_pays_asset_id; + vs.fill_data.pays_amount = o_v.fill_pays_amount; + vs.fill_data.receives_asset_id = o_v.fill_receives_asset_id; + vs.fill_data.receives_amount = o_v.fill_receives_amount; + //vs.fill_data.fill_price = o_v.fill_fill_price; + //vs.fill_data.is_maker = o_v.fill_is_maker; +} - if (curl && bulk.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech - sendBulk(_elasticsearch_node_url, _elasticsearch_logs); +bool elasticsearch_plugin_impl::add_elasticsearch( const account_id_type account_id, + const optional & oho, + const signed_block& b) +{ + const auto &stats_obj = getStatsObject(account_id); + const auto &ath = addNewEntry(stats_obj, account_id, oho); + growStats(stats_obj, ath); + createBulkLine(ath); + prepareBulk(ath.id); + cleanObjects(ath, account_id); + + if (curl && bulk_lines.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech + prepare.clear(); + populateESstruct(); + if(!graphene::utilities::SendBulk(es)) + return false; + else + bulk_lines.clear(); } + return true; +} + +const account_statistics_object& elasticsearch_plugin_impl::getStatsObject(const account_id_type account_id) +{ + graphene::chain::database& db = database(); + const auto &acct = db.get(account_id); + return acct.statistics(db); +} + +const account_transaction_history_object& elasticsearch_plugin_impl::addNewEntry(const account_statistics_object& stats_obj, + const account_id_type account_id, + const optional & oho) +{ + graphene::chain::database& db = database(); + const auto &ath = db.create([&](account_transaction_history_object &obj) { + obj.operation_id = oho->id; + obj.account = account_id; + obj.sequence = stats_obj.total_ops + 1; + obj.next = stats_obj.most_recent_op; + }); + + return ath; +} + +void elasticsearch_plugin_impl::growStats(const account_statistics_object& stats_obj, + const account_transaction_history_object& ath) +{ + graphene::chain::database& db = database(); + db.modify(stats_obj, [&](account_statistics_object &obj) { + obj.most_recent_op = ath.id; + obj.total_ops = ath.sequence; + }); +} + +void elasticsearch_plugin_impl::createBulkLine(const account_transaction_history_object& ath) +{ + bulk_line_struct.account_history = ath; + bulk_line_struct.operation_history = os; + bulk_line_struct.operation_type = op_type; + bulk_line_struct.operation_id_num = ath.operation_id.instance.value; + bulk_line_struct.block_data = bs; + if(_elasticsearch_visitor) + bulk_line_struct.additional_data = vs; + bulk_line = fc::json::to_string(bulk_line_struct); +} + +void elasticsearch_plugin_impl::prepareBulk(const account_transaction_history_id_type& ath_id) +{ + const std::string _id = fc::json::to_string(ath_id); + fc::mutable_variant_object bulk_header; + bulk_header["_index"] = index_name; + bulk_header["_type"] = "data"; + bulk_header["_id"] = fc::to_string(ath_id.space_id) + "." + fc::to_string(ath_id.type_id) + "." + ath_id.instance; + prepare = graphene::utilities::createBulk(bulk_header, bulk_line); + bulk_lines.insert(bulk_lines.end(), prepare.begin(), prepare.end()); +} + +void elasticsearch_plugin_impl::cleanObjects(const account_transaction_history_object& ath, account_id_type account_id) +{ + graphene::chain::database& db = database(); // remove everything except current object from ath const auto &his_idx = db.get_index_type(); const auto &by_seq_idx = his_idx.indices().get(); @@ -243,95 +352,12 @@ void elasticsearch_plugin_impl::add_elasticsearch( const account_id_type account } } -void elasticsearch_plugin_impl::createBulkLine(account_transaction_history_object ath, operation_history_struct os, int op_type, block_struct bs, visitor_struct vs) -{ - bulk_struct bulks; - bulks.account_history = ath; - bulks.operation_history = os; - bulks.operation_type = op_type; - bulks.block_data = bs; - bulks.additional_data = vs; - - std::string alltogether = fc::json::to_string(bulks); - - auto block_date = bulks.block_data.block_time.to_iso_string(); - std::vector parts; - boost::split(parts, block_date, boost::is_any_of("-")); - std::string index_name = "graphene-" + parts[0] + "-" + parts[1]; - - // bulk header before each line, op_type = create to avoid dups, index id will be ath id(2.9.X). - std::string _id = fc::json::to_string(ath.id); - bulk.push_back("{ \"index\" : { \"_index\" : \""+index_name+"\", \"_type\" : \"data\", \"op_type\" : \"create\", \"_id\" : "+_id+" } }"); // header - bulk.push_back(alltogether); -} - -void elasticsearch_plugin_impl::sendBulk(std::string _elasticsearch_node_url, bool _elasticsearch_logs) +void elasticsearch_plugin_impl::populateESstruct() { - - // curl buffers to read - std::string readBuffer; - std::string readBuffer_logs; - - std::string bulking = ""; - - bulking = boost::algorithm::join(bulk, "\n"); - bulking = bulking + "\n"; - bulk.clear(); - - //wlog((bulking)); - - struct curl_slist *headers = NULL; - curl_slist_append(headers, "Content-Type: application/json"); - std::string url = _elasticsearch_node_url + "_bulk"; - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_POST, true); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, bulking.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&readBuffer); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcrp/0.1"); - //curl_easy_setopt(curl, CURLOPT_VERBOSE, true); - curl_easy_perform(curl); - - long http_code = 0; - curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); - if(http_code == 200) { - // all good, do nothing - } - else if(http_code == 429) { - // repeat request? - } - else { - // exit everything ? - } - - if(_elasticsearch_logs) { - auto logs = readBuffer; - // do logs - std::string url_logs = _elasticsearch_node_url + "logs/data/"; - curl_easy_setopt(curl, CURLOPT_URL, url_logs.c_str()); - curl_easy_setopt(curl, CURLOPT_POST, true); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, logs.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &readBuffer_logs); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcrp/0.1"); - //curl_easy_setopt(curl, CURLOPT_VERBOSE, true); - //ilog("log here curl: ${output}", ("output", readBuffer_logs)); - curl_easy_perform(curl); - - http_code = 0; - curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); - if(http_code == 200) { - // all good, do nothing - } - else if(http_code == 429) { - // repeat request? - } - else { - // exit everything ? - } - } + es.curl = curl; + es.bulk_lines = bulk_lines; + es.elasticsearch_url = _elasticsearch_node_url; + es.auth = _elasticsearch_basic_auth; } } // end namespace detail @@ -363,15 +389,21 @@ void elasticsearch_plugin::plugin_set_program_options( ("elasticsearch-node-url", boost::program_options::value(), "Elastic Search database node url") ("elasticsearch-bulk-replay", boost::program_options::value(), "Number of bulk documents to index on replay(5000)") ("elasticsearch-bulk-sync", boost::program_options::value(), "Number of bulk documents to index on a syncronied chain(10)") - ("elasticsearch-logs", boost::program_options::value(), "Log bulk events to database") ("elasticsearch-visitor", boost::program_options::value(), "Use visitor to index additional data(slows down the replay)") + ("elasticsearch-basic-auth", boost::program_options::value(), "Pass basic auth to elasticsearch database ") + ("elasticsearch-index-prefix", boost::program_options::value(), "Add a prefix to the index(bitshares-)") ; cfg.add(cli); } void elasticsearch_plugin::plugin_initialize(const boost::program_options::variables_map& options) { - database().applied_block.connect( [&]( const signed_block& b){ my->update_account_histories(b); } ); + database().applied_block.connect( [&]( const signed_block& b) { + if(!my->update_account_histories(b)) + { + FC_THROW_EXCEPTION(fc::exception, "Error populating ES database, we are going to keep trying."); + } + } ); my->_oho_index = database().add_index< primary_index< operation_history_index > >(); database().add_index< primary_index< account_transaction_history_index > >(); @@ -384,16 +416,27 @@ void elasticsearch_plugin::plugin_initialize(const boost::program_options::varia if (options.count("elasticsearch-bulk-sync")) { my->_elasticsearch_bulk_sync = options["elasticsearch-bulk-sync"].as(); } - if (options.count("elasticsearch-logs")) { - my->_elasticsearch_logs = options["elasticsearch-logs"].as(); - } if (options.count("elasticsearch-visitor")) { my->_elasticsearch_visitor = options["elasticsearch-visitor"].as(); } + if (options.count("elasticsearch-basic-auth")) { + my->_elasticsearch_basic_auth = options["elasticsearch-basic-auth"].as(); + } + if (options.count("elasticsearch-index-prefix")) { + my->_elasticsearch_index_prefix = options["elasticsearch-index-prefix"].as(); + } } void elasticsearch_plugin::plugin_startup() { + graphene::utilities::ES es; + es.curl = my->curl; + es.elasticsearch_url = my->_elasticsearch_node_url; + es.auth = my->_elasticsearch_basic_auth; + + if(!graphene::utilities::checkES(es)) + FC_THROW_EXCEPTION(fc::exception, "ES database is not up in url ${url}", ("url", my->_elasticsearch_node_url)); + ilog("elasticsearch ACCOUNT HISTORY: plugin_startup() begin"); } } } diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index cc51c2478..19a488436 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -29,9 +29,6 @@ namespace graphene { namespace elasticsearch { using namespace chain; - //using namespace graphene::db; - //using boost::multi_index_container; - //using namespace boost::multi_index; // // Plugins should #define their SPACE_ID's so plugins with @@ -47,12 +44,6 @@ namespace graphene { namespace elasticsearch { #define ELASTICSEARCH_SPACE_ID 6 #endif -static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp) -{ - ((std::string*)userp)->append((char*)contents, size * nmemb); - return size * nmemb; -} - namespace detail { class elasticsearch_plugin_impl; @@ -76,7 +67,6 @@ class elasticsearch_plugin : public graphene::app::plugin std::unique_ptr my; }; - struct operation_visitor { typedef void result_type; @@ -99,6 +89,31 @@ struct operation_visitor transfer_from = o.from; transfer_to = o.to; } + + object_id_type fill_order_id; + account_id_type fill_account_id; + asset_id_type fill_pays_asset_id; + share_type fill_pays_amount; + asset_id_type fill_receives_asset_id; + share_type fill_receives_amount; + //double fill_fill_price; + //bool fill_is_maker; + + void operator()( const graphene::chain::fill_order_operation& o ) + { + fee_asset = o.fee.asset_id; + fee_amount = o.fee.amount; + + fill_order_id = o.order_id; + fill_account_id = o.account_id; + fill_pays_asset_id = o.pays.asset_id; + fill_pays_amount = o.pays.amount; + fill_receives_asset_id = o.receives.asset_id; + fill_receives_amount = o.receives.amount; + //fill_fill_price = o.fill_price.to_real(); + //fill_is_maker = o.is_maker; + } + template void operator()( const T& o ) { @@ -133,27 +148,38 @@ struct transfer_struct { account_id_type to; }; +struct fill_struct { + object_id_type order_id; + account_id_type account_id; + asset_id_type pays_asset_id; + share_type pays_amount; + asset_id_type receives_asset_id; + share_type receives_amount; + double fill_price; + bool is_maker; +}; + struct visitor_struct { fee_struct fee_data; transfer_struct transfer_data; + fill_struct fill_data; }; struct bulk_struct { account_transaction_history_object account_history; operation_history_struct operation_history; int operation_type; + int operation_id_num; block_struct block_data; - visitor_struct additional_data; + optional additional_data; }; - } } //graphene::elasticsearch FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op) ) FC_REFLECT( graphene::elasticsearch::block_struct, (block_num)(block_time)(trx_id) ) FC_REFLECT( graphene::elasticsearch::fee_struct, (asset)(amount) ) FC_REFLECT( graphene::elasticsearch::transfer_struct, (asset)(amount)(from)(to) ) -FC_REFLECT( graphene::elasticsearch::visitor_struct, (fee_data)(transfer_data) ) -FC_REFLECT( graphene::elasticsearch::bulk_struct, (account_history)(operation_history)(operation_type)(block_data)(additional_data) ) - - +FC_REFLECT( graphene::elasticsearch::fill_struct, (order_id)(account_id)(pays_asset_id)(pays_amount)(receives_asset_id)(receives_amount)(fill_price)(is_maker)) +FC_REFLECT( graphene::elasticsearch::visitor_struct, (fee_data)(transfer_data)(fill_data) ) +FC_REFLECT( graphene::elasticsearch::bulk_struct, (account_history)(operation_history)(operation_type)(operation_id_num)(block_data)(additional_data) ) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 7c9c2b613..5f04e40c9 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -47,10 +47,11 @@ class es_objects_plugin_impl { curl = curl_easy_init(); } virtual ~es_objects_plugin_impl(); - void updateDatabase( const vector& ids , bool isNew); + bool updateDatabase( const vector& ids , bool isNew); es_objects_plugin& _self; std::string _es_objects_elasticsearch_url = "http://localhost:9200/"; + std::string _es_objects_auth = ""; uint32_t _es_objects_bulk_replay = 5000; uint32_t _es_objects_bulk_sync = 10; bool _es_objects_proposals = true; @@ -59,24 +60,27 @@ class es_objects_plugin_impl bool _es_objects_balances = true; bool _es_objects_limit_orders = true; bool _es_objects_asset_bitasset = true; - bool _es_objects_logs = true; + std::string _es_objects_index_prefix = "objects-"; CURL *curl; // curl handler vector bulk; vector prepare; - map bitassets; - //uint32_t bitasset_seq; + map bitassets; + map accounts; + map proposals; + map assets; + map balances; + map limit_orders; private: - void PrepareProposal(const proposal_object* proposal_object, const fc::time_point_sec block_time, uint32_t block_number); - void PrepareAccount(const account_object* account_object, const fc::time_point_sec block_time, uint32_t block_number); - void PrepareAsset(const asset_object* asset_object, const fc::time_point_sec block_time, uint32_t block_number); - void PrepareBalance(const balance_object* balance_object, const fc::time_point_sec block_time, uint32_t block_number); - void PrepareLimit(const limit_order_object* limit_object, const fc::time_point_sec block_time, uint32_t block_number); - void PrepareBitAsset(const asset_bitasset_data_object* bitasset_object, const fc::time_point_sec block_time, uint32_t block_number); + void PrepareProposal(const proposal_object& proposal_object, const fc::time_point_sec& block_time, const uint32_t& block_number); + void PrepareAccount(const account_object& account_object, const fc::time_point_sec& block_time, const uint32_t& block_number); + void PrepareAsset(const asset_object& asset_object, const fc::time_point_sec& block_time, const uint32_t& block_number); + void PrepareBalance(const balance_object& balance_object, const fc::time_point_sec& block_time, const uint32_t& block_number); + void PrepareLimit(const limit_order_object& limit_object, const fc::time_point_sec& block_time, const uint32_t& block_number); + void PrepareBitAsset(const asset_bitasset_data_object& bitasset_object, const fc::time_point_sec& block_time, const uint32_t& block_number); }; -void es_objects_plugin_impl::updateDatabase( const vector& ids , bool isNew) +bool es_objects_plugin_impl::updateDatabase( const vector& ids , bool isNew) { - graphene::chain::database &db = _self.database(); const fc::time_point_sec block_time = db.head_block_time(); @@ -89,175 +93,259 @@ void es_objects_plugin_impl::updateDatabase( const vector& ids , else limit_documents = _es_objects_bulk_replay; - if (curl && bulk.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech - if(!graphene::utilities::SendBulk(curl, bulk, _es_objects_elasticsearch_url, _es_objects_logs, "objects_logs")) - elog("Error sending data to database"); - bulk.clear(); - } - for(auto const& value: ids) { if(value.is() && _es_objects_proposals) { auto obj = db.find_object(value); auto p = static_cast(obj); if(p != nullptr) - PrepareProposal(p, block_time, block_number); + PrepareProposal(*p, block_time, block_number); } else if(value.is() && _es_objects_accounts) { auto obj = db.find_object(value); auto a = static_cast(obj); if(a != nullptr) - PrepareAccount(a, block_time, block_number); + PrepareAccount(*a, block_time, block_number); } else if(value.is() && _es_objects_assets) { auto obj = db.find_object(value); auto a = static_cast(obj); if(a != nullptr) - PrepareAsset(a, block_time, block_number); + PrepareAsset(*a, block_time, block_number); } else if(value.is() && _es_objects_balances) { auto obj = db.find_object(value); auto b = static_cast(obj); if(b != nullptr) - PrepareBalance(b, block_time, block_number); + PrepareBalance(*b, block_time, block_number); } else if(value.is() && _es_objects_limit_orders) { auto obj = db.find_object(value); auto l = static_cast(obj); if(l != nullptr) - PrepareLimit(l, block_time, block_number); + PrepareLimit(*l, block_time, block_number); } else if(value.is() && _es_objects_asset_bitasset) { auto obj = db.find_object(value); auto ba = static_cast(obj); if(ba != nullptr) - PrepareBitAsset(ba, block_time, block_number); + PrepareBitAsset(*ba, block_time, block_number); } } + + if (curl && bulk.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech + + graphene::utilities::ES es; + es.curl = curl; + es.bulk_lines = bulk; + es.elasticsearch_url = _es_objects_elasticsearch_url; + es.auth = _es_objects_auth; + + if(!graphene::utilities::SendBulk(es)) + return false; + else + bulk.clear(); + } + + return true; } -void es_objects_plugin_impl::PrepareProposal(const proposal_object* proposal_object, const fc::time_point_sec block_time, uint32_t block_number) +void es_objects_plugin_impl::PrepareProposal(const proposal_object& proposal_object, + const fc::time_point_sec& block_time, const uint32_t& block_number) { proposal_struct prop; - prop.object_id = proposal_object->id; + prop.object_id = proposal_object.id; prop.block_time = block_time; prop.block_number = block_number; - prop.expiration_time = proposal_object->expiration_time; - prop.review_period_time = proposal_object->review_period_time; - prop.proposed_transaction = fc::json::to_string(proposal_object->proposed_transaction); - prop.required_owner_approvals = fc::json::to_string(proposal_object->required_owner_approvals); - prop.available_owner_approvals = fc::json::to_string(proposal_object->available_owner_approvals); - prop.required_active_approvals = fc::json::to_string(proposal_object->required_active_approvals); - prop.available_key_approvals = fc::json::to_string(proposal_object->available_key_approvals); - prop.proposer = proposal_object->proposer; + prop.expiration_time = proposal_object.expiration_time; + prop.review_period_time = proposal_object.review_period_time; + prop.proposed_transaction = fc::json::to_string(proposal_object.proposed_transaction); + prop.required_owner_approvals = fc::json::to_string(proposal_object.required_owner_approvals); + prop.available_owner_approvals = fc::json::to_string(proposal_object.available_owner_approvals); + prop.required_active_approvals = fc::json::to_string(proposal_object.required_active_approvals); + prop.available_key_approvals = fc::json::to_string(proposal_object.available_key_approvals); + prop.proposer = proposal_object.proposer; + + auto it = proposals.find(proposal_object.id); + if(it == proposals.end()) + proposals[proposal_object.id] = prop; + else { + if(it->second == prop) return; + else proposals[proposal_object.id] = prop; + } std::string data = fc::json::to_string(prop); - prepare = graphene::utilities::createBulk("bitshares-proposal", data, "", 1); + + fc::mutable_variant_object bulk_header; + bulk_header["_index"] = _es_objects_index_prefix + "proposal"; + bulk_header["_type"] = "data"; + + prepare = graphene::utilities::createBulk(bulk_header, data); bulk.insert(bulk.end(), prepare.begin(), prepare.end()); prepare.clear(); } -void es_objects_plugin_impl::PrepareAccount(const account_object* account_object, const fc::time_point_sec block_time, uint32_t block_number) +void es_objects_plugin_impl::PrepareAccount(const account_object& account_object, + const fc::time_point_sec& block_time, const uint32_t& block_number) { account_struct acct; - acct.object_id = account_object->id; + acct.object_id = account_object.id; acct.block_time = block_time; acct.block_number = block_number; - acct.membership_expiration_date = account_object->membership_expiration_date; - acct.registrar = account_object->registrar; - acct.referrer = account_object->referrer; - acct.lifetime_referrer = account_object->lifetime_referrer; - acct.network_fee_percentage = account_object->network_fee_percentage; - acct.lifetime_referrer_fee_percentage = account_object->lifetime_referrer_fee_percentage; - acct.referrer_rewards_percentage = account_object->referrer_rewards_percentage; - acct.name = account_object->name; - acct.owner_account_auths = fc::json::to_string(account_object->owner.account_auths); - acct.owner_key_auths = fc::json::to_string(account_object->owner.key_auths); - acct.owner_address_auths = fc::json::to_string(account_object->owner.address_auths); - acct.active_account_auths = fc::json::to_string(account_object->active.account_auths); - acct.active_key_auths = fc::json::to_string(account_object->active.key_auths); - acct.active_address_auths = fc::json::to_string(account_object->active.address_auths); - acct.voting_account = account_object->options.voting_account; + acct.membership_expiration_date = account_object.membership_expiration_date; + acct.registrar = account_object.registrar; + acct.referrer = account_object.referrer; + acct.lifetime_referrer = account_object.lifetime_referrer; + acct.network_fee_percentage = account_object.network_fee_percentage; + acct.lifetime_referrer_fee_percentage = account_object.lifetime_referrer_fee_percentage; + acct.referrer_rewards_percentage = account_object.referrer_rewards_percentage; + acct.name = account_object.name; + acct.owner_account_auths = fc::json::to_string(account_object.owner.account_auths); + acct.owner_key_auths = fc::json::to_string(account_object.owner.key_auths); + acct.owner_address_auths = fc::json::to_string(account_object.owner.address_auths); + acct.active_account_auths = fc::json::to_string(account_object.active.account_auths); + acct.active_key_auths = fc::json::to_string(account_object.active.key_auths); + acct.active_address_auths = fc::json::to_string(account_object.active.address_auths); + acct.voting_account = account_object.options.voting_account; + + auto it = accounts.find(account_object.id); + if(it == accounts.end()) + accounts[account_object.id] = acct; + else { + if(it->second == acct) return; + else accounts[account_object.id] = acct; + } std::string data = fc::json::to_string(acct); - prepare = graphene::utilities::createBulk("bitshares-account", data, "", 1); + + fc::mutable_variant_object bulk_header; + bulk_header["_index"] = _es_objects_index_prefix + "acount"; + bulk_header["_type"] = "data"; + + prepare = graphene::utilities::createBulk(bulk_header, data); bulk.insert(bulk.end(), prepare.begin(), prepare.end()); prepare.clear(); } -void es_objects_plugin_impl::PrepareAsset(const asset_object* asset_object, const fc::time_point_sec block_time, uint32_t block_number) +void es_objects_plugin_impl::PrepareAsset(const asset_object& asset_object, + const fc::time_point_sec& block_time, const uint32_t& block_number) { - asset_struct _asset; - _asset.object_id = asset_object->id; - _asset.block_time = block_time; - _asset.block_number = block_number; - _asset.symbol = asset_object->symbol; - _asset.issuer = asset_object->issuer; - _asset.is_market_issued = asset_object->is_market_issued(); - _asset.dynamic_asset_data_id = asset_object->dynamic_asset_data_id; - _asset.bitasset_data_id = asset_object->bitasset_data_id; - - std::string data = fc::json::to_string(_asset); - prepare = graphene::utilities::createBulk("bitshares-asset", data, fc::json::to_string(asset_object->id), 0); + asset_struct asset; + asset.object_id = asset_object.id; + asset.block_time = block_time; + asset.block_number = block_number; + asset.symbol = asset_object.symbol; + asset.issuer = asset_object.issuer; + asset.is_market_issued = asset_object.is_market_issued(); + asset.dynamic_asset_data_id = asset_object.dynamic_asset_data_id; + asset.bitasset_data_id = asset_object.bitasset_data_id; + + auto it = assets.find(asset_object.id); + if(it == assets.end()) + assets[asset_object.id] = asset; + else { + if(it->second == asset) return; + else assets[asset_object.id] = asset; + } + + std::string data = fc::json::to_string(asset); + + fc::mutable_variant_object bulk_header; + bulk_header["_index"] = _es_objects_index_prefix + "asset"; + bulk_header["_type"] = "data"; + + prepare = graphene::utilities::createBulk(bulk_header, data); bulk.insert(bulk.end(), prepare.begin(), prepare.end()); prepare.clear(); } -void es_objects_plugin_impl::PrepareBalance(const balance_object* balance_object, const fc::time_point_sec block_time, uint32_t block_number) +void es_objects_plugin_impl::PrepareBalance(const balance_object& balance_object, + const fc::time_point_sec& block_time, const uint32_t& block_number) { balance_struct balance; - balance.object_id = balance_object->id; + balance.object_id = balance_object.id; balance.block_time = block_time; - balance.block_number = block_number;balance.owner = balance_object->owner; - balance.asset_id = balance_object->balance.asset_id; - balance.amount = balance_object->balance.amount; + balance.block_number = block_number;balance.owner = balance_object.owner; + balance.asset_id = balance_object.balance.asset_id; + balance.amount = balance_object.balance.amount; + + auto it = balances.find(balance_object.id); + if(it == balances.end()) + balances[balance_object.id] = balance; + else { + if(it->second == balance) return; + else balances[balance_object.id] = balance; + } std::string data = fc::json::to_string(balance); - prepare = graphene::utilities::createBulk("bitshares-balance", data, "", 1); + + fc::mutable_variant_object bulk_header; + bulk_header["_index"] = _es_objects_index_prefix + "balance"; + bulk_header["_type"] = "data"; + + prepare = graphene::utilities::createBulk(bulk_header, data); bulk.insert(bulk.end(), prepare.begin(), prepare.end()); prepare.clear(); } -void es_objects_plugin_impl::PrepareLimit(const limit_order_object* limit_object, const fc::time_point_sec block_time, uint32_t block_number) +void es_objects_plugin_impl::PrepareLimit(const limit_order_object& limit_object, + const fc::time_point_sec& block_time, const uint32_t& block_number) { limit_order_struct limit; - limit.object_id = limit_object->id; + limit.object_id = limit_object.id; limit.block_time = block_time; limit.block_number = block_number; - limit.expiration = limit_object->expiration; - limit.seller = limit_object->seller; - limit.for_sale = limit_object->for_sale; - limit.sell_price = limit_object->sell_price; - limit.deferred_fee = limit_object->deferred_fee; + limit.expiration = limit_object.expiration; + limit.seller = limit_object.seller; + limit.for_sale = limit_object.for_sale; + limit.sell_price = limit_object.sell_price; + limit.deferred_fee = limit_object.deferred_fee; + + auto it = limit_orders.find(limit_object.id); + if(it == limit_orders.end()) + limit_orders[limit_object.id] = limit; + else { + if(it->second == limit) return; + else limit_orders[limit_object.id] = limit; + } std::string data = fc::json::to_string(limit); - prepare = graphene::utilities::createBulk("bitshares-limitorder", data, "", 1); + + fc::mutable_variant_object bulk_header; + bulk_header["_index"] = _es_objects_index_prefix + "limitorder"; + bulk_header["_type"] = "data"; + + prepare = graphene::utilities::createBulk(bulk_header, data); bulk.insert(bulk.end(), prepare.begin(), prepare.end()); prepare.clear(); } -void es_objects_plugin_impl::PrepareBitAsset(const asset_bitasset_data_object* bitasset_object, const fc::time_point_sec block_time, uint32_t block_number) +void es_objects_plugin_impl::PrepareBitAsset(const asset_bitasset_data_object& bitasset_object, + const fc::time_point_sec& block_time, const uint32_t& block_number) { - if(!bitasset_object->is_prediction_market) { + if(!bitasset_object.is_prediction_market) { - auto object_id = bitasset_object->id; - auto it = bitassets.find(object_id); + bitasset_struct bitasset; + bitasset.object_id = bitasset_object.id; + bitasset.block_time = block_time; + bitasset.block_number = block_number; + bitasset.current_feed = fc::json::to_string(bitasset_object.current_feed); + bitasset.current_feed_publication_time = bitasset_object.current_feed_publication_time; + + auto it = bitassets.find(bitasset_object.id); if(it == bitassets.end()) - bitassets[object_id] = fc::json::to_string(bitasset_object->current_feed); + bitassets[bitasset_object.id] = bitasset; else { - if(it->second == fc::json::to_string(bitasset_object->current_feed)) return; - else bitassets[object_id] = fc::json::to_string(bitasset_object->current_feed); + if(it->second == bitasset) return; + else bitassets[bitasset_object.id] = bitasset; } - bitasset_struct bitasset; + std::string data = fc::json::to_string(bitasset); - bitasset.object_id = bitasset_object->id; - bitasset.block_time = block_time; - bitasset.block_number = block_number; - bitasset.current_feed = fc::json::to_string(bitasset_object->current_feed); - bitasset.current_feed_publication_time = bitasset_object->current_feed_publication_time; + fc::mutable_variant_object bulk_header; + bulk_header["_index"] = _es_objects_index_prefix + "bitasset"; + bulk_header["_type"] = "data"; - std::string data = fc::json::to_string(bitasset); - prepare = graphene::utilities::createBulk("bitshares-bitasset", data, "", 1); + prepare = graphene::utilities::createBulk(bulk_header, data); bulk.insert(bulk.end(), prepare.begin(), prepare.end()); prepare.clear(); } @@ -268,7 +356,6 @@ es_objects_plugin_impl::~es_objects_plugin_impl() return; } - } // end namespace detail es_objects_plugin::es_objects_plugin() : @@ -296,7 +383,7 @@ void es_objects_plugin::plugin_set_program_options( { cli.add_options() ("es-objects-elasticsearch-url", boost::program_options::value(), "Elasticsearch node url") - ("es-objects-logs", boost::program_options::value(), "Log bulk events to database") + ("es-objects-auth", boost::program_options::value(), "Basic auth username:password") ("es-objects-bulk-replay", boost::program_options::value(), "Number of bulk documents to index on replay(5000)") ("es-objects-bulk-sync", boost::program_options::value(), "Number of bulk documents to index on a syncronied chain(10)") ("es-objects-proposals", boost::program_options::value(), "Store proposal objects") @@ -305,21 +392,30 @@ void es_objects_plugin::plugin_set_program_options( ("es-objects-balances", boost::program_options::value(), "Store balances objects") ("es-objects-limit-orders", boost::program_options::value(), "Store limit order objects") ("es-objects-asset-bitasset", boost::program_options::value(), "Store feed data") - + ("es-objects-index-prefix", boost::program_options::value(), "Add a prefix to the index(objects-)") ; cfg.add(cli); } void es_objects_plugin::plugin_initialize(const boost::program_options::variables_map& options) { - database().new_objects.connect([&]( const vector& ids, const flat_set& impacted_accounts ){ my->updateDatabase(ids, 1); }); - database().changed_objects.connect([&]( const vector& ids, const flat_set& impacted_accounts ){ my->updateDatabase(ids, 0); }); - + database().new_objects.connect([&]( const vector& ids, const flat_set& impacted_accounts ) { + if(!my->updateDatabase(ids, 1)) + { + FC_THROW_EXCEPTION(fc::exception, "Error populating ES database, we are going to keep trying."); + } + }); + database().changed_objects.connect([&]( const vector& ids, const flat_set& impacted_accounts ) { + if(!my->updateDatabase(ids, 0)) + { + FC_THROW_EXCEPTION(fc::exception, "Error populating ES database, we are going to keep trying."); + } + }); if (options.count("es-objects-elasticsearch-url")) { my->_es_objects_elasticsearch_url = options["es-objects-elasticsearch-url"].as(); } - if (options.count("es-objects-logs")) { - my->_es_objects_logs = options["es-objects-logs"].as(); + if (options.count("es-objects-auth")) { + my->_es_objects_auth = options["es-objects-auth"].as(); } if (options.count("es-objects-bulk-replay")) { my->_es_objects_bulk_replay = options["es-objects-bulk-replay"].as(); @@ -345,11 +441,22 @@ void es_objects_plugin::plugin_initialize(const boost::program_options::variable if (options.count("es-objects-asset-bitasset")) { my->_es_objects_asset_bitasset = options["es-objects-asset-bitasset"].as(); } + if (options.count("es-objects-index-prefix")) { + my->_es_objects_index_prefix = options["es-objects-index-prefix"].as(); + } } void es_objects_plugin::plugin_startup() { - ilog("elasticsearch objects: plugin_startup() begin"); + graphene::utilities::ES es; + es.curl = my->curl; + es.elasticsearch_url = my->_es_objects_elasticsearch_url; + es.auth = my->_es_objects_auth; + es.auth = my->_es_objects_index_prefix; + + if(!graphene::utilities::checkES(es)) + FC_THROW_EXCEPTION(fc::exception, "ES database is not up in url ${url}", ("url", my->_es_objects_elasticsearch_url)); + ilog("elasticsearch OBJECTS: plugin_startup() begin"); } -} } \ No newline at end of file +} } diff --git a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp index 31809e040..d9c387112 100644 --- a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp +++ b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp @@ -68,6 +68,20 @@ struct proposal_struct { string available_key_approvals; account_id_type proposer; + friend bool operator==(const proposal_struct& l, const proposal_struct& r) + { + return std::tie(l.object_id, l.block_time, l.block_number, l.expiration_time, l.review_period_time, + l.proposed_transaction, l.required_active_approvals, l.available_active_approvals, + l.required_owner_approvals, l.available_owner_approvals, l.available_key_approvals, + l.proposer) == std::tie(r.object_id, r.block_time, r.block_number, r.expiration_time, r.review_period_time, + r.proposed_transaction, r.required_active_approvals, r.available_active_approvals, + r.required_owner_approvals, r.available_owner_approvals, r.available_key_approvals, + r.proposer); + } + friend bool operator!=(const proposal_struct& l, const proposal_struct& r) + { + return !operator==(l, r); + } }; struct account_struct { object_id_type object_id; @@ -88,6 +102,23 @@ struct account_struct { string active_key_auths; string active_address_auths; account_id_type voting_account; + + friend bool operator==(const account_struct& l, const account_struct& r) + { + return std::tie(l.object_id, l.block_time, l.block_number, l.membership_expiration_date, l.registrar, l.referrer, + l.lifetime_referrer, l.network_fee_percentage, l.lifetime_referrer_fee_percentage, + l.referrer_rewards_percentage, l.name, l.owner_account_auths, l.owner_key_auths, + l.owner_address_auths, l.active_account_auths, l.active_key_auths, l.active_address_auths, + l.voting_account) == std::tie(r.object_id, r.block_time, r.block_number, r.membership_expiration_date, r.registrar, r.referrer, + r.lifetime_referrer, r.network_fee_percentage, r.lifetime_referrer_fee_percentage, + r.referrer_rewards_percentage, r.name, r.owner_account_auths, r.owner_key_auths, + r.owner_address_auths, r.active_account_auths, r.active_key_auths, r.active_address_auths, + r.voting_account); + } + friend bool operator!=(const account_struct& l, const account_struct& r) + { + return !operator==(l, r); + } }; struct asset_struct { object_id_type object_id; @@ -99,6 +130,17 @@ struct asset_struct { asset_dynamic_data_id_type dynamic_asset_data_id; optional bitasset_data_id; + friend bool operator==(const asset_struct& l, const asset_struct& r) + { + return std::tie(l.object_id, l.block_time, l.block_number, l.symbol, l.issuer, l.is_market_issued, + l.dynamic_asset_data_id, l.bitasset_data_id) == std::tie(r.object_id, r.block_time, + r.block_number, r.symbol, r.issuer, r.is_market_issued, r.dynamic_asset_data_id, + r.bitasset_data_id); + } + friend bool operator!=(const asset_struct& l, const asset_struct& r) + { + return !operator==(l, r); + } }; struct balance_struct { object_id_type object_id; @@ -107,6 +149,16 @@ struct balance_struct { address owner; asset_id_type asset_id; share_type amount; + + friend bool operator==(const balance_struct& l, const balance_struct& r) + { + return std::tie(l.object_id, l.block_time, l.block_number, l.block_time, l.owner, l.asset_id, l.amount) + == std::tie(r.object_id, r.block_time, r.block_number, r.block_time, r.owner, r.asset_id, r.amount); + } + friend bool operator!=(const balance_struct& l, const balance_struct& r) + { + return !operator==(l, r); + } }; struct limit_order_struct { object_id_type object_id; @@ -117,6 +169,16 @@ struct limit_order_struct { share_type for_sale; price sell_price; share_type deferred_fee; + + friend bool operator==(const limit_order_struct& l, const limit_order_struct& r) + { + return std::tie(l.object_id, l.block_time, l.block_number, l.expiration, l.seller, l.for_sale, l.sell_price, l.deferred_fee) + == std::tie(r.object_id, r.block_time, r.block_number, r.expiration, r.seller, r.for_sale, r.sell_price, r.deferred_fee); + } + friend bool operator!=(const limit_order_struct& l, const limit_order_struct& r) + { + return !operator==(l, r); + } }; struct bitasset_struct { object_id_type object_id; @@ -125,6 +187,16 @@ struct bitasset_struct { string current_feed; time_point_sec current_feed_publication_time; time_point_sec feed_expiration_time; + + friend bool operator==(const bitasset_struct& l, const bitasset_struct& r) + { + return std::tie(l.object_id, l.block_time, l.block_number, l.current_feed, l.current_feed_publication_time) + == std::tie(r.object_id, r.block_time, r.block_number, r.current_feed, r.current_feed_publication_time); + } + friend bool operator!=(const bitasset_struct& l, const bitasset_struct& r) + { + return !operator==(l, r); + } }; } } //graphene::es_objects @@ -134,4 +206,4 @@ FC_REFLECT( graphene::es_objects::account_struct, (object_id)(block_time)(block_ FC_REFLECT( graphene::es_objects::asset_struct, (object_id)(block_time)(block_number)(symbol)(issuer)(is_market_issued)(dynamic_asset_data_id)(bitasset_data_id) ) FC_REFLECT( graphene::es_objects::balance_struct, (object_id)(block_time)(block_number)(block_time)(owner)(asset_id)(amount) ) FC_REFLECT( graphene::es_objects::limit_order_struct, (object_id)(block_time)(block_number)(expiration)(seller)(for_sale)(sell_price)(deferred_fee) ) -FC_REFLECT( graphene::es_objects::bitasset_struct, (object_id)(block_time)(block_number)(current_feed)(current_feed_publication_time) ) \ No newline at end of file +FC_REFLECT( graphene::es_objects::bitasset_struct, (object_id)(block_time)(block_number)(current_feed)(current_feed_publication_time) ) diff --git a/libraries/utilities/elasticsearch.cpp b/libraries/utilities/elasticsearch.cpp index 1674a12ae..11a9561bf 100644 --- a/libraries/utilities/elasticsearch.cpp +++ b/libraries/utilities/elasticsearch.cpp @@ -24,100 +24,167 @@ #include #include +#include #include +#include + +size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp) +{ + ((std::string*)userp)->append((char*)contents, size * nmemb); + return size * nmemb; +} namespace graphene { namespace utilities { -bool SendBulk(CURL *curl, std::vector& bulk, std::string elasticsearch_url, bool do_logs, std::string logs_index) +bool checkES(ES& es) +{ + graphene::utilities::CurlRequest curl_request; + curl_request.handler = es.curl; + curl_request.url = es.elasticsearch_url + "_nodes"; + curl_request.auth = es.auth; + curl_request.type = "GET"; + + if(doCurl(curl_request).empty()) + return false; + return true; + +} +const std::string simpleQuery(ES& es) +{ + graphene::utilities::CurlRequest curl_request; + curl_request.handler = es.curl; + curl_request.url = es.elasticsearch_url + es.endpoint; + curl_request.auth = es.auth; + curl_request.type = "POST"; + curl_request.query = es.query; + + return doCurl(curl_request); +} + +bool SendBulk(ES& es) +{ + std::string bulking = joinBulkLines(es.bulk_lines); + + graphene::utilities::CurlRequest curl_request; + curl_request.handler = es.curl; + curl_request.url = es.elasticsearch_url + "_bulk"; + curl_request.auth = es.auth; + curl_request.type = "POST"; + curl_request.query = bulking; + + auto curlResponse = doCurl(curl_request); + + if(handleBulkResponse(getResponseCode(curl_request.handler), curlResponse)) + return true; + return false; +} + +const std::string joinBulkLines(const std::vector& bulk) +{ + auto bulking = boost::algorithm::join(bulk, "\n"); + bulking = bulking + "\n"; + + return bulking; +} +long getResponseCode(CURL *handler) +{ + long http_code = 0; + curl_easy_getinfo (handler, CURLINFO_RESPONSE_CODE, &http_code); + return http_code; +} + +bool handleBulkResponse(long http_code, const std::string& CurlReadBuffer) { - // curl buffers to read - std::string readBuffer; - std::string readBuffer_logs; - - std::string bulking = ""; - - bulking = boost::algorithm::join(bulk, "\n"); - bulking = bulking + "\n"; - bulk.clear(); - - struct curl_slist *headers = NULL; - headers = curl_slist_append(headers, "Content-Type: application/json"); - std::string url = elasticsearch_url + "_bulk"; - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_POST, true); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, bulking.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&readBuffer); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcrp/0.1"); - //curl_easy_setopt(curl, CURLOPT_VERBOSE, true); - curl_easy_perform(curl); - - long http_code = 0; - curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); if(http_code == 200) { - // all good, do nothing - } - else if(http_code == 413) { - elog("413 error: Can be low space disk"); - return 0; + // all good, but check errors in response + fc::variant j = fc::json::from_string(CurlReadBuffer); + bool errors = j["errors"].as_bool(); + if(errors == true) { + return false; + } } else { - elog(http_code + "error: Unknown error"); - return 0; + if(http_code == 413) { + elog( "413 error: Can be low disk space" ); + } + else if(http_code == 401) { + elog( "401 error: Unauthorized" ); + } + else { + elog( std::to_string(http_code) + " error: Unknown error" ); + } + return false; } - - if(do_logs) { - auto logs = readBuffer; - // do logs - std::string url_logs = elasticsearch_url + logs_index + "/data/"; - curl_easy_setopt(curl, CURLOPT_URL, url_logs.c_str()); - curl_easy_setopt(curl, CURLOPT_POST, true); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, logs.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *) &readBuffer_logs); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcrp/0.1"); - curl_easy_perform(curl); - - http_code = 0; - curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); - if(http_code == 200) { - // all good, do nothing - return 1; - } - else if(http_code == 201) { - // 201 is ok - return 1; - } - else if(http_code == 409) { - // 409 for record already exist is ok - return 1; - } - else if(http_code == 413) { - elog("413 error: Can be low space disk"); - return 0; - } - else { - elog(http_code + "error: Unknown error"); - return 0; - } - } - return 0; + return true; } -std::vector createBulk(std::string index_name, std::string data, std::string id, bool onlycreate) +const std::vector createBulk(const fc::mutable_variant_object& bulk_header, const std::string& data) { std::vector bulk; - std::string create_string = ""; - if(!onlycreate) - create_string = ",\"_id\" : "+id; - - bulk.push_back("{ \"index\" : { \"_index\" : \""+index_name+"\", \"_type\" : \"data\" "+create_string+" } }"); + fc::mutable_variant_object final_bulk_header; + final_bulk_header["index"] = bulk_header; + bulk.push_back(fc::json::to_string(final_bulk_header)); bulk.push_back(data); return bulk; } +bool deleteAll(ES& es) +{ + graphene::utilities::CurlRequest curl_request; + curl_request.handler = es.curl; + curl_request.url = es.elasticsearch_url + es.index_prefix + "*"; + curl_request.auth = es.auth; + curl_request.type = "DELETE"; + + auto curl_response = doCurl(curl_request); + if(curl_response.empty()) + return false; + else + return true; +} +const std::string getEndPoint(ES& es) +{ + graphene::utilities::CurlRequest curl_request; + curl_request.handler = es.curl; + curl_request.url = es.elasticsearch_url + es.endpoint; + curl_request.auth = es.auth; + curl_request.type = "GET"; + + return doCurl(curl_request); +} + +const std::string generateIndexName(const fc::time_point_sec& block_date, const std::string& _elasticsearch_index_prefix) +{ + auto block_date_string = block_date.to_iso_string(); + std::vector parts; + boost::split(parts, block_date_string, boost::is_any_of("-")); + std::string index_name = _elasticsearch_index_prefix + parts[0] + "-" + parts[1]; + return index_name; +} + +const std::string doCurl(CurlRequest& curl) +{ + std::string CurlReadBuffer; + struct curl_slist *headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/json"); + + curl_easy_setopt(curl.handler, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl.handler, CURLOPT_URL, curl.url.c_str()); + curl_easy_setopt(curl.handler, CURLOPT_CUSTOMREQUEST, curl.type.c_str()); + if(curl.type == "POST") + { + curl_easy_setopt(curl.handler, CURLOPT_POST, true); + curl_easy_setopt(curl.handler, CURLOPT_POSTFIELDS, curl.query.c_str()); + } + curl_easy_setopt(curl.handler, CURLOPT_WRITEFUNCTION, WriteCallback); + curl_easy_setopt(curl.handler, CURLOPT_WRITEDATA, (void *)&CurlReadBuffer); + curl_easy_setopt(curl.handler, CURLOPT_USERAGENT, "libcrp/0.1"); + if(!curl.auth.empty()) + curl_easy_setopt(curl.handler, CURLOPT_USERPWD, curl.auth.c_str()); + curl_easy_perform(curl.handler); + + return CurlReadBuffer; +} } } // end namespace graphene::utilities diff --git a/libraries/utilities/include/graphene/utilities/elasticsearch.hpp b/libraries/utilities/include/graphene/utilities/elasticsearch.hpp index 517f23456..898464b16 100644 --- a/libraries/utilities/include/graphene/utilities/elasticsearch.hpp +++ b/libraries/utilities/include/graphene/utilities/elasticsearch.hpp @@ -27,16 +27,42 @@ #include #include +#include +#include -static size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp) -{ - ((std::string*)userp)->append((char*)contents, size * nmemb); - return size * nmemb; -} +size_t WriteCallback(void *contents, size_t size, size_t nmemb, void *userp); namespace graphene { namespace utilities { - bool SendBulk(CURL *curl, std::vector & bulk, std::string elasticsearch_url, bool do_logs, std::string logs_index); - std::vector createBulk(std::string type, std::string data, std::string id, bool onlycreate); + class ES { + public: + CURL *curl; + std::vector bulk_lines; + std::string elasticsearch_url; + std::string index_prefix; + std::string auth; + std::string endpoint; + std::string query; + }; + class CurlRequest { + public: + CURL *handler; + std::string url; + std::string type; + std::string auth; + std::string query; + }; + + bool SendBulk(ES& es); + const std::vector createBulk(const fc::mutable_variant_object& bulk_header, const std::string& data); + bool checkES(ES& es); + const std::string simpleQuery(ES& es); + bool deleteAll(ES& es); + bool handleBulkResponse(long http_code, const std::string& CurlReadBuffer); + const std::string getEndPoint(ES& es); + const std::string generateIndexName(const fc::time_point_sec& block_date, const std::string& _elasticsearch_index_prefix); + const std::string doCurl(CurlRequest& curl); + const std::string joinBulkLines(const std::vector& bulk); + long getResponseCode(CURL *handler); } } // end namespace graphene::utilities diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 44af778be..e57e3374a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,38 +8,38 @@ endif() file(GLOB UNIT_TESTS "tests/*.cpp") add_executable( chain_test ${UNIT_TESTS} ${COMMON_SOURCES} ) -target_link_libraries( chain_test graphene_chain graphene_app graphene_account_history graphene_bookie graphene_egenesis_none fc graphene_wallet ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( chain_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_bookie graphene_egenesis_none fc graphene_wallet ${PLATFORM_SPECIFIC_LIBS} ) if(MSVC) set_source_files_properties( tests/serialization_tests.cpp PROPERTIES COMPILE_FLAGS "/bigobj" ) endif(MSVC) file(GLOB PERFORMANCE_TESTS "performance/*.cpp") add_executable( performance_test ${PERFORMANCE_TESTS} ${COMMON_SOURCES} ) -target_link_libraries( performance_test graphene_chain graphene_app graphene_account_history graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( performance_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) file(GLOB BENCH_MARKS "benchmarks/*.cpp") add_executable( chain_bench ${BENCH_MARKS} ${COMMON_SOURCES} ) -target_link_libraries( chain_bench graphene_chain graphene_app graphene_account_history graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( chain_bench graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) file(GLOB APP_SOURCES "app/*.cpp") add_executable( app_test ${APP_SOURCES} ) -target_link_libraries( app_test graphene_app graphene_account_history graphene_bookie graphene_net graphene_chain graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( app_test graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_witness graphene_bookie graphene_net graphene_chain graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) file(GLOB INTENSE_SOURCES "intense/*.cpp") add_executable( intense_test ${INTENSE_SOURCES} ${COMMON_SOURCES} ) -target_link_libraries( intense_test graphene_chain graphene_app graphene_account_history graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( intense_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_bookie graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) file(GLOB BETTING_TESTS "betting/*.cpp") add_executable( betting_test ${BETTING_TESTS} ${COMMON_SOURCES} ) -target_link_libraries( betting_test graphene_chain graphene_app graphene_account_history graphene_bookie graphene_egenesis_none fc graphene_wallet ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( betting_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_bookie graphene_egenesis_none fc graphene_wallet ${PLATFORM_SPECIFIC_LIBS} ) file(GLOB TOURNAMENT_TESTS "tournament/*.cpp") add_executable( tournament_test ${TOURNAMENT_TESTS} ${COMMON_SOURCES} ) -target_link_libraries( tournament_test graphene_chain graphene_app graphene_account_history graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( tournament_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) file(GLOB RANDOM_SOURCES "random/*.cpp") add_executable( random_test ${RANDOM_SOURCES} ${COMMON_SOURCES} ) -target_link_libraries( random_test graphene_chain graphene_app graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) +target_link_libraries( random_test graphene_chain graphene_app graphene_elasticsearch graphene_es_objects graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) file(GLOB CLI_SOURCES "cli/*.cpp") add_executable( cli_test ${CLI_SOURCES} ) @@ -51,4 +51,8 @@ if(MSVC) set_source_files_properties( cli/main.cpp PROPERTIES COMPILE_FLAGS "/bigobj" ) endif(MSVC) +file(GLOB ES_SOURCES "elasticsearch/*.cpp") +add_executable( es_test ${ES_SOURCES} ${COMMON_SOURCES} ) +target_link_libraries( es_test graphene_chain graphene_app graphene_account_history graphene_elasticsearch graphene_es_objects graphene_egenesis_none fc ${PLATFORM_SPECIFIC_LIBS} ) + add_subdirectory( generate_empty_blocks ) diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index 0728ce2df..8cd3dc400 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -29,11 +29,9 @@ #include #include #include +#include +#include -#include - -#include -#include #include #include #include @@ -51,9 +49,7 @@ #include #include -#include #include -#include #include "database_fixture.hpp" @@ -134,8 +130,46 @@ database_fixture::database_fixture() } // app.initialize(); - ahplugin->plugin_set_app(&app); - ahplugin->plugin_initialize(options); + + auto test_name = boost::unit_test::framework::current_test_case().p_name.value; + if(test_name == "elasticsearch_account_history" || test_name == "elasticsearch_suite") { + auto esplugin = app.register_plugin(); + esplugin->plugin_set_app(&app); + + options.insert(std::make_pair("elasticsearch-node-url", boost::program_options::variable_value(string("http://localhost:9200/"), false))); + options.insert(std::make_pair("elasticsearch-bulk-replay", boost::program_options::variable_value(uint32_t(2), false))); + options.insert(std::make_pair("elasticsearch-bulk-sync", boost::program_options::variable_value(uint32_t(2), false))); + options.insert(std::make_pair("elasticsearch-visitor", boost::program_options::variable_value(true, false))); + //options.insert(std::make_pair("elasticsearch-basic-auth", boost::program_options::variable_value(string("elastic:changeme"), false))); + + esplugin->plugin_initialize(options); + esplugin->plugin_startup(); + } + else { + auto ahplugin = app.register_plugin(); + ahplugin->plugin_set_app(&app); + ahplugin->plugin_initialize(options); + ahplugin->plugin_startup(); + } + + if(test_name == "elasticsearch_objects" || test_name == "elasticsearch_suite") { + auto esobjects_plugin = app.register_plugin(); + esobjects_plugin->plugin_set_app(&app); + + options.insert(std::make_pair("es-objects-elasticsearch-url", boost::program_options::variable_value(string("http://localhost:9200/"), false))); + options.insert(std::make_pair("es-objects-bulk-replay", boost::program_options::variable_value(uint32_t(2), false))); + options.insert(std::make_pair("es-objects-bulk-sync", boost::program_options::variable_value(uint32_t(2), false))); + options.insert(std::make_pair("es-objects-proposals", boost::program_options::variable_value(true, false))); + options.insert(std::make_pair("es-objects-accounts", boost::program_options::variable_value(true, false))); + options.insert(std::make_pair("es-objects-assets", boost::program_options::variable_value(true, false))); + options.insert(std::make_pair("es-objects-balances", boost::program_options::variable_value(true, false))); + options.insert(std::make_pair("es-objects-limit-orders", boost::program_options::variable_value(true, false))); + options.insert(std::make_pair("es-objects-asset-bitasset", boost::program_options::variable_value(true, false))); + + esobjects_plugin->plugin_initialize(options); + esobjects_plugin->plugin_startup(); + } + mhplugin->plugin_set_app(&app); mhplugin->plugin_initialize(options); bookieplugin->plugin_set_app(&app); @@ -143,7 +177,6 @@ database_fixture::database_fixture() affiliateplugin->plugin_set_app(&app); affiliateplugin->plugin_initialize(options); - ahplugin->plugin_startup(); mhplugin->plugin_startup(); bookieplugin->plugin_startup(); affiliateplugin->plugin_startup(); diff --git a/tests/elasticsearch/main.cpp b/tests/elasticsearch/main.cpp new file mode 100644 index 000000000..18674a3b8 --- /dev/null +++ b/tests/elasticsearch/main.cpp @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2018 oxarbitrage and contributors. + * + * The MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include +#include + +#include + +#include "../common/database_fixture.hpp" + +#define BOOST_TEST_MODULE Elastic Search Database Tests +#include + +using namespace graphene::chain; +using namespace graphene::chain::test; +using namespace graphene::app; + +BOOST_FIXTURE_TEST_SUITE( elasticsearch_tests, database_fixture ) + +BOOST_AUTO_TEST_CASE(elasticsearch_account_history) { + try { + + CURL *curl; // curl handler + curl = curl_easy_init(); + + graphene::utilities::ES es; + es.curl = curl; + es.elasticsearch_url = "http://localhost:9200/"; + es.index_prefix = "bitshares-"; + //es.auth = "elastic:changeme"; + + // delete all first + auto delete_account_history = graphene::utilities::deleteAll(es); + fc::usleep(fc::milliseconds(1000)); // this is because index.refresh_interval, nothing to worry + + if(delete_account_history) { // all records deleted + + //account_id_type() do 3 ops + create_bitasset("USD", account_id_type()); + auto dan = create_account("dan"); + auto bob = create_account("bob"); + + generate_block(); + fc::usleep(fc::milliseconds(1000)); + + // for later use + //int asset_create_op_id = operation::tag::value; + //int account_create_op_id = operation::tag::value; + + string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }"; + es.endpoint = es.index_prefix + "*/data/_count"; + es.query = query; + + auto res = graphene::utilities::simpleQuery(es); + variant j = fc::json::from_string(res); + auto total = j["count"].as_string(); + BOOST_CHECK_EQUAL(total, "5"); + + es.endpoint = es.index_prefix + "*/data/_search"; + res = graphene::utilities::simpleQuery(es); + j = fc::json::from_string(res); + auto first_id = j["hits"]["hits"][size_t(0)]["_id"].as_string(); + BOOST_CHECK_EQUAL(first_id, "2.9.0"); + + generate_block(); + auto willie = create_account("willie"); + generate_block(); + + fc::usleep(fc::milliseconds(1000)); // index.refresh_interval + + es.endpoint = es.index_prefix + "*/data/_count"; + res = graphene::utilities::simpleQuery(es); + j = fc::json::from_string(res); + + total = j["count"].as_string(); + BOOST_CHECK_EQUAL(total, "7"); + + // do some transfers in 1 block + transfer(account_id_type()(db), bob, asset(100)); + transfer(account_id_type()(db), bob, asset(200)); + transfer(account_id_type()(db), bob, asset(300)); + + generate_block(); + fc::usleep(fc::milliseconds(1000)); // index.refresh_interval + + res = graphene::utilities::simpleQuery(es); + j = fc::json::from_string(res); + + total = j["count"].as_string(); + BOOST_CHECK_EQUAL(total, "13"); + + // check the visitor data + auto block_date = db.head_block_time(); + std::string index_name = graphene::utilities::generateIndexName(block_date, "bitshares-"); + + es.endpoint = index_name + "/data/2.9.12"; // we know last op is a transfer of amount 300 + res = graphene::utilities::getEndPoint(es); + j = fc::json::from_string(res); + auto last_transfer_amount = j["_source"]["additional_data"]["transfer_data"]["amount"].as_string(); + BOOST_CHECK_EQUAL(last_transfer_amount, "300"); + } + } + catch (fc::exception &e) { + edump((e.to_detail_string())); + throw; + } +} + +BOOST_AUTO_TEST_CASE(elasticsearch_objects) { + try { + + CURL *curl; // curl handler + curl = curl_easy_init(); + + graphene::utilities::ES es; + es.curl = curl; + es.elasticsearch_url = "http://localhost:9200/"; + es.index_prefix = "objects-"; + //es.auth = "elastic:changeme"; + + // delete all first + auto delete_objects = graphene::utilities::deleteAll(es); + + generate_block(); + fc::usleep(fc::milliseconds(1000)); + + if(delete_objects) { // all records deleted + + // asset and bitasset + create_bitasset("USD", account_id_type()); + generate_block(); + fc::usleep(fc::milliseconds(1000)); + + string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }"; + es.endpoint = es.index_prefix + "*/data/_count"; + es.query = query; + + auto res = graphene::utilities::simpleQuery(es); + variant j = fc::json::from_string(res); + auto total = j["count"].as_string(); + BOOST_CHECK_EQUAL(total, "2"); + + es.endpoint = es.index_prefix + "asset/data/_search"; + res = graphene::utilities::simpleQuery(es); + j = fc::json::from_string(res); + auto first_id = j["hits"]["hits"][size_t(0)]["_source"]["symbol"].as_string(); + BOOST_CHECK_EQUAL(first_id, "USD"); + + auto bitasset_data_id = j["hits"]["hits"][size_t(0)]["_source"]["bitasset_data_id"].as_string(); + es.endpoint = es.index_prefix + "bitasset/data/_search"; + es.query = "{ \"query\" : { \"bool\": { \"must\" : [{ \"term\": { \"object_id\": \""+bitasset_data_id+"\"}}] } } }"; + res = graphene::utilities::simpleQuery(es); + j = fc::json::from_string(res); + auto bitasset_object_id = j["hits"]["hits"][size_t(0)]["_source"]["object_id"].as_string(); + BOOST_CHECK_EQUAL(bitasset_object_id, bitasset_data_id); + } + } + catch (fc::exception &e) { + edump((e.to_detail_string())); + throw; + } +} + +BOOST_AUTO_TEST_CASE(elasticsearch_suite) { + try { + + CURL *curl; // curl handler + curl = curl_easy_init(); + + graphene::utilities::ES es; + es.curl = curl; + es.elasticsearch_url = "http://localhost:9200/"; + es.index_prefix = "bitshares-"; + auto delete_account_history = graphene::utilities::deleteAll(es); + fc::usleep(fc::milliseconds(1000)); + es.index_prefix = "objects-"; + auto delete_objects = graphene::utilities::deleteAll(es); + fc::usleep(fc::milliseconds(1000)); + + if(delete_account_history && delete_objects) { // all records deleted + + + } + } + catch (fc::exception &e) { + edump((e.to_detail_string())); + throw; + } +} + +BOOST_AUTO_TEST_SUITE_END() From 2d19aa3de10ae145d9d4644379ff2502a91c6cc0 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 15 Oct 2018 14:42:45 -0300 Subject: [PATCH 07/17] Merge pull request #1271 from oxarbitrage/es_objects refine es_objects plugin --- libraries/plugins/es_objects/es_objects.cpp | 271 ++++++++++-------- .../graphene/es_objects/es_objects.hpp | 48 +++- 2 files changed, 183 insertions(+), 136 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 5f04e40c9..82e6ff231 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -47,13 +47,14 @@ class es_objects_plugin_impl { curl = curl_easy_init(); } virtual ~es_objects_plugin_impl(); - bool updateDatabase( const vector& ids , bool isNew); + bool index_database( const vector& ids, std::string action); + void remove_from_database( object_id_type id, std::string index); es_objects_plugin& _self; std::string _es_objects_elasticsearch_url = "http://localhost:9200/"; std::string _es_objects_auth = ""; - uint32_t _es_objects_bulk_replay = 5000; - uint32_t _es_objects_bulk_sync = 10; + uint32_t _es_objects_bulk_replay = 10000; + uint32_t _es_objects_bulk_sync = 100; bool _es_objects_proposals = true; bool _es_objects_accounts = true; bool _es_objects_assets = true; @@ -64,27 +65,27 @@ class es_objects_plugin_impl CURL *curl; // curl handler vector bulk; vector prepare; - map bitassets; - map accounts; - map proposals; - map assets; - map balances; - map limit_orders; + + bool _es_objects_keep_only_current = true; + + uint32_t block_number; + fc::time_point_sec block_time; + private: - void PrepareProposal(const proposal_object& proposal_object, const fc::time_point_sec& block_time, const uint32_t& block_number); - void PrepareAccount(const account_object& account_object, const fc::time_point_sec& block_time, const uint32_t& block_number); - void PrepareAsset(const asset_object& asset_object, const fc::time_point_sec& block_time, const uint32_t& block_number); - void PrepareBalance(const balance_object& balance_object, const fc::time_point_sec& block_time, const uint32_t& block_number); - void PrepareLimit(const limit_order_object& limit_object, const fc::time_point_sec& block_time, const uint32_t& block_number); - void PrepareBitAsset(const asset_bitasset_data_object& bitasset_object, const fc::time_point_sec& block_time, const uint32_t& block_number); + void prepare_proposal(const proposal_object& proposal_object); + void prepare_account(const account_object& account_object); + void prepare_asset(const asset_object& asset_object); + void prepare_balance(const account_balance_object& account_balance_object); + void prepare_limit(const limit_order_object& limit_object); + void prepare_bitasset(const asset_bitasset_data_object& bitasset_object); }; -bool es_objects_plugin_impl::updateDatabase( const vector& ids , bool isNew) +bool es_objects_plugin_impl::index_database( const vector& ids, std::string action) { graphene::chain::database &db = _self.database(); - const fc::time_point_sec block_time = db.head_block_time(); - const uint32_t block_number = db.head_block_num(); + block_time = db.head_block_time(); + block_number = db.head_block_num(); // check if we are in replay or in sync and change number of bulk documents accordingly uint32_t limit_documents = 0; @@ -97,38 +98,62 @@ bool es_objects_plugin_impl::updateDatabase( const vector& ids , if(value.is() && _es_objects_proposals) { auto obj = db.find_object(value); auto p = static_cast(obj); - if(p != nullptr) - PrepareProposal(*p, block_time, block_number); + if(p != nullptr) { + if(action == "delete") + remove_from_database(p->id, "proposal"); + else + prepare_proposal(*p); + } } else if(value.is() && _es_objects_accounts) { auto obj = db.find_object(value); auto a = static_cast(obj); - if(a != nullptr) - PrepareAccount(*a, block_time, block_number); + if(a != nullptr) { + if(action == "delete") + remove_from_database(a->id, "account"); + else + prepare_account(*a); + } } else if(value.is() && _es_objects_assets) { auto obj = db.find_object(value); auto a = static_cast(obj); - if(a != nullptr) - PrepareAsset(*a, block_time, block_number); + if(a != nullptr) { + if(action == "delete") + remove_from_database(a->id, "asset"); + else + prepare_asset(*a); + } } - else if(value.is() && _es_objects_balances) { + else if(value.is() && _es_objects_balances) { auto obj = db.find_object(value); - auto b = static_cast(obj); - if(b != nullptr) - PrepareBalance(*b, block_time, block_number); + auto b = static_cast(obj); + if(b != nullptr) { + if(action == "delete") + remove_from_database(b->id, "balance"); + else + prepare_balance(*b); + } } else if(value.is() && _es_objects_limit_orders) { auto obj = db.find_object(value); auto l = static_cast(obj); - if(l != nullptr) - PrepareLimit(*l, block_time, block_number); + if(l != nullptr) { + if(action == "delete") + remove_from_database(l->id, "limitorder"); + else + prepare_limit(*l); + } } else if(value.is() && _es_objects_asset_bitasset) { auto obj = db.find_object(value); auto ba = static_cast(obj); - if(ba != nullptr) - PrepareBitAsset(*ba, block_time, block_number); + if(ba != nullptr) { + if(action == "delete") + remove_from_database(ba->id, "bitasset"); + else + prepare_bitasset(*ba); + } } } @@ -149,8 +174,23 @@ bool es_objects_plugin_impl::updateDatabase( const vector& ids , return true; } -void es_objects_plugin_impl::PrepareProposal(const proposal_object& proposal_object, - const fc::time_point_sec& block_time, const uint32_t& block_number) +void es_objects_plugin_impl::remove_from_database( object_id_type id, std::string index) +{ + if(_es_objects_keep_only_current) + { + fc::mutable_variant_object delete_line; + delete_line["_id"] = string(id); + delete_line["_index"] = _es_objects_index_prefix + index; + delete_line["_type"] = "data"; + fc::mutable_variant_object final_delete_line; + final_delete_line["delete"] = delete_line; + prepare.push_back(fc::json::to_string(final_delete_line)); + std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); + prepare.clear(); + } +} + +void es_objects_plugin_impl::prepare_proposal(const proposal_object& proposal_object) { proposal_struct prop; prop.object_id = proposal_object.id; @@ -165,27 +205,22 @@ void es_objects_plugin_impl::PrepareProposal(const proposal_object& proposal_obj prop.available_key_approvals = fc::json::to_string(proposal_object.available_key_approvals); prop.proposer = proposal_object.proposer; - auto it = proposals.find(proposal_object.id); - if(it == proposals.end()) - proposals[proposal_object.id] = prop; - else { - if(it->second == prop) return; - else proposals[proposal_object.id] = prop; - } - std::string data = fc::json::to_string(prop); fc::mutable_variant_object bulk_header; bulk_header["_index"] = _es_objects_index_prefix + "proposal"; bulk_header["_type"] = "data"; + if(_es_objects_keep_only_current) + { + bulk_header["_id"] = string(prop.object_id); + } - prepare = graphene::utilities::createBulk(bulk_header, data); - bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); + std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); prepare.clear(); } -void es_objects_plugin_impl::PrepareAccount(const account_object& account_object, - const fc::time_point_sec& block_time, const uint32_t& block_number) +void es_objects_plugin_impl::prepare_account(const account_object& account_object) { account_struct acct; acct.object_id = account_object.id; @@ -206,28 +241,24 @@ void es_objects_plugin_impl::PrepareAccount(const account_object& account_object acct.active_key_auths = fc::json::to_string(account_object.active.key_auths); acct.active_address_auths = fc::json::to_string(account_object.active.address_auths); acct.voting_account = account_object.options.voting_account; - - auto it = accounts.find(account_object.id); - if(it == accounts.end()) - accounts[account_object.id] = acct; - else { - if(it->second == acct) return; - else accounts[account_object.id] = acct; - } + acct.votes = fc::json::to_string(account_object.options.votes); std::string data = fc::json::to_string(acct); fc::mutable_variant_object bulk_header; - bulk_header["_index"] = _es_objects_index_prefix + "acount"; + bulk_header["_index"] = _es_objects_index_prefix + "account"; bulk_header["_type"] = "data"; + if(_es_objects_keep_only_current) + { + bulk_header["_id"] = string(acct.object_id); + } - prepare = graphene::utilities::createBulk(bulk_header, data); - bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); + std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); prepare.clear(); } -void es_objects_plugin_impl::PrepareAsset(const asset_object& asset_object, - const fc::time_point_sec& block_time, const uint32_t& block_number) +void es_objects_plugin_impl::prepare_asset(const asset_object& asset_object) { asset_struct asset; asset.object_id = asset_object.id; @@ -239,56 +270,47 @@ void es_objects_plugin_impl::PrepareAsset(const asset_object& asset_object, asset.dynamic_asset_data_id = asset_object.dynamic_asset_data_id; asset.bitasset_data_id = asset_object.bitasset_data_id; - auto it = assets.find(asset_object.id); - if(it == assets.end()) - assets[asset_object.id] = asset; - else { - if(it->second == asset) return; - else assets[asset_object.id] = asset; - } - std::string data = fc::json::to_string(asset); fc::mutable_variant_object bulk_header; bulk_header["_index"] = _es_objects_index_prefix + "asset"; bulk_header["_type"] = "data"; + if(_es_objects_keep_only_current) + { + bulk_header["_id"] = string(asset.object_id); + } - prepare = graphene::utilities::createBulk(bulk_header, data); - bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); + std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); prepare.clear(); } -void es_objects_plugin_impl::PrepareBalance(const balance_object& balance_object, - const fc::time_point_sec& block_time, const uint32_t& block_number) +void es_objects_plugin_impl::prepare_balance(const account_balance_object& account_balance_object) { balance_struct balance; - balance.object_id = balance_object.id; + balance.object_id = account_balance_object.id; balance.block_time = block_time; - balance.block_number = block_number;balance.owner = balance_object.owner; - balance.asset_id = balance_object.balance.asset_id; - balance.amount = balance_object.balance.amount; - - auto it = balances.find(balance_object.id); - if(it == balances.end()) - balances[balance_object.id] = balance; - else { - if(it->second == balance) return; - else balances[balance_object.id] = balance; - } + balance.block_number = block_number; + balance.owner = account_balance_object.owner; + balance.asset_type = account_balance_object.asset_type; + balance.balance = account_balance_object.balance; std::string data = fc::json::to_string(balance); fc::mutable_variant_object bulk_header; bulk_header["_index"] = _es_objects_index_prefix + "balance"; bulk_header["_type"] = "data"; + if(_es_objects_keep_only_current) + { + bulk_header["_id"] = string(balance.object_id); + } - prepare = graphene::utilities::createBulk(bulk_header, data); - bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); + std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); prepare.clear(); } -void es_objects_plugin_impl::PrepareLimit(const limit_order_object& limit_object, - const fc::time_point_sec& block_time, const uint32_t& block_number) +void es_objects_plugin_impl::prepare_limit(const limit_order_object& limit_object) { limit_order_struct limit; limit.object_id = limit_object.id; @@ -300,27 +322,22 @@ void es_objects_plugin_impl::PrepareLimit(const limit_order_object& limit_object limit.sell_price = limit_object.sell_price; limit.deferred_fee = limit_object.deferred_fee; - auto it = limit_orders.find(limit_object.id); - if(it == limit_orders.end()) - limit_orders[limit_object.id] = limit; - else { - if(it->second == limit) return; - else limit_orders[limit_object.id] = limit; - } - std::string data = fc::json::to_string(limit); fc::mutable_variant_object bulk_header; bulk_header["_index"] = _es_objects_index_prefix + "limitorder"; bulk_header["_type"] = "data"; + if(_es_objects_keep_only_current) + { + bulk_header["_id"] = string(limit.object_id); + } - prepare = graphene::utilities::createBulk(bulk_header, data); - bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); + std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); prepare.clear(); } -void es_objects_plugin_impl::PrepareBitAsset(const asset_bitasset_data_object& bitasset_object, - const fc::time_point_sec& block_time, const uint32_t& block_number) +void es_objects_plugin_impl::prepare_bitasset(const asset_bitasset_data_object& bitasset_object) { if(!bitasset_object.is_prediction_market) { @@ -331,22 +348,18 @@ void es_objects_plugin_impl::PrepareBitAsset(const asset_bitasset_data_object& b bitasset.current_feed = fc::json::to_string(bitasset_object.current_feed); bitasset.current_feed_publication_time = bitasset_object.current_feed_publication_time; - auto it = bitassets.find(bitasset_object.id); - if(it == bitassets.end()) - bitassets[bitasset_object.id] = bitasset; - else { - if(it->second == bitasset) return; - else bitassets[bitasset_object.id] = bitasset; - } - std::string data = fc::json::to_string(bitasset); fc::mutable_variant_object bulk_header; bulk_header["_index"] = _es_objects_index_prefix + "bitasset"; bulk_header["_type"] = "data"; + if(_es_objects_keep_only_current) + { + bulk_header["_id"] = string(bitasset.object_id); + } - prepare = graphene::utilities::createBulk(bulk_header, data); - bulk.insert(bulk.end(), prepare.begin(), prepare.end()); + prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); + std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); prepare.clear(); } } @@ -382,17 +395,18 @@ void es_objects_plugin::plugin_set_program_options( ) { cli.add_options() - ("es-objects-elasticsearch-url", boost::program_options::value(), "Elasticsearch node url") - ("es-objects-auth", boost::program_options::value(), "Basic auth username:password") - ("es-objects-bulk-replay", boost::program_options::value(), "Number of bulk documents to index on replay(5000)") - ("es-objects-bulk-sync", boost::program_options::value(), "Number of bulk documents to index on a syncronied chain(10)") - ("es-objects-proposals", boost::program_options::value(), "Store proposal objects") - ("es-objects-accounts", boost::program_options::value(), "Store account objects") - ("es-objects-assets", boost::program_options::value(), "Store asset objects") - ("es-objects-balances", boost::program_options::value(), "Store balances objects") - ("es-objects-limit-orders", boost::program_options::value(), "Store limit order objects") - ("es-objects-asset-bitasset", boost::program_options::value(), "Store feed data") + ("es-objects-elasticsearch-url", boost::program_options::value(), "Elasticsearch node url(http://localhost:9200/)") + ("es-objects-auth", boost::program_options::value(), "Basic auth username:password('')") + ("es-objects-bulk-replay", boost::program_options::value(), "Number of bulk documents to index on replay(10000)") + ("es-objects-bulk-sync", boost::program_options::value(), "Number of bulk documents to index on a synchronized chain(100)") + ("es-objects-proposals", boost::program_options::value(), "Store proposal objects(true)") + ("es-objects-accounts", boost::program_options::value(), "Store account objects(true)") + ("es-objects-assets", boost::program_options::value(), "Store asset objects(true)") + ("es-objects-balances", boost::program_options::value(), "Store balances objects(true)") + ("es-objects-limit-orders", boost::program_options::value(), "Store limit order objects(true)") + ("es-objects-asset-bitasset", boost::program_options::value(), "Store feed data(true)") ("es-objects-index-prefix", boost::program_options::value(), "Add a prefix to the index(objects-)") + ("es-objects-keep-only-current", boost::program_options::value(), "Keep only current state of the objects(true)") ; cfg.add(cli); } @@ -400,17 +414,25 @@ void es_objects_plugin::plugin_set_program_options( void es_objects_plugin::plugin_initialize(const boost::program_options::variables_map& options) { database().new_objects.connect([&]( const vector& ids, const flat_set& impacted_accounts ) { - if(!my->updateDatabase(ids, 1)) + if(!my->index_database(ids, "create")) { - FC_THROW_EXCEPTION(fc::exception, "Error populating ES database, we are going to keep trying."); + FC_THROW_EXCEPTION(fc::exception, "Error creating object from ES database, we are going to keep trying."); } }); database().changed_objects.connect([&]( const vector& ids, const flat_set& impacted_accounts ) { - if(!my->updateDatabase(ids, 0)) + if(!my->index_database(ids, "update")) { - FC_THROW_EXCEPTION(fc::exception, "Error populating ES database, we are going to keep trying."); + FC_THROW_EXCEPTION(fc::exception, "Error updating object from ES database, we are going to keep trying."); } }); + database().removed_objects.connect([this](const vector& ids, const vector& objs, const flat_set& impacted_accounts) { + if(!my->index_database(ids, "delete")) + { + FC_THROW_EXCEPTION(fc::exception, "Error deleting object from ES database, we are going to keep trying."); + } + }); + + if (options.count("es-objects-elasticsearch-url")) { my->_es_objects_elasticsearch_url = options["es-objects-elasticsearch-url"].as(); } @@ -444,6 +466,9 @@ void es_objects_plugin::plugin_initialize(const boost::program_options::variable if (options.count("es-objects-index-prefix")) { my->_es_objects_index_prefix = options["es-objects-index-prefix"].as(); } + if (options.count("es-objects-keep-only-current")) { + my->_es_objects_keep_only_current = options["es-objects-keep-only-current"].as(); + } } void es_objects_plugin::plugin_startup() diff --git a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp index d9c387112..886129c88 100644 --- a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp +++ b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp @@ -102,6 +102,7 @@ struct account_struct { string active_key_auths; string active_address_auths; account_id_type voting_account; + string votes; friend bool operator==(const account_struct& l, const account_struct& r) { @@ -109,11 +110,11 @@ struct account_struct { l.lifetime_referrer, l.network_fee_percentage, l.lifetime_referrer_fee_percentage, l.referrer_rewards_percentage, l.name, l.owner_account_auths, l.owner_key_auths, l.owner_address_auths, l.active_account_auths, l.active_key_auths, l.active_address_auths, - l.voting_account) == std::tie(r.object_id, r.block_time, r.block_number, r.membership_expiration_date, r.registrar, r.referrer, + l.voting_account, l.votes) == std::tie(r.object_id, r.block_time, r.block_number, r.membership_expiration_date, r.registrar, r.referrer, r.lifetime_referrer, r.network_fee_percentage, r.lifetime_referrer_fee_percentage, r.referrer_rewards_percentage, r.name, r.owner_account_auths, r.owner_key_auths, r.owner_address_auths, r.active_account_auths, r.active_key_auths, r.active_address_auths, - r.voting_account); + r.voting_account, r.votes); } friend bool operator!=(const account_struct& l, const account_struct& r) { @@ -146,14 +147,14 @@ struct balance_struct { object_id_type object_id; fc::time_point_sec block_time; uint32_t block_number; - address owner; - asset_id_type asset_id; - share_type amount; + account_id_type owner; + asset_id_type asset_type; + share_type balance; friend bool operator==(const balance_struct& l, const balance_struct& r) { - return std::tie(l.object_id, l.block_time, l.block_number, l.block_time, l.owner, l.asset_id, l.amount) - == std::tie(r.object_id, r.block_time, r.block_number, r.block_time, r.owner, r.asset_id, r.amount); + return std::tie(l.object_id, l.block_time, l.block_number, l.block_time, l.owner, l.asset_type, l.balance) + == std::tie(r.object_id, r.block_time, r.block_number, r.block_time, r.owner, r.asset_type, r.balance); } friend bool operator!=(const balance_struct& l, const balance_struct& r) { @@ -201,9 +202,30 @@ struct bitasset_struct { } } //graphene::es_objects -FC_REFLECT( graphene::es_objects::proposal_struct, (object_id)(block_time)(block_number)(expiration_time)(review_period_time)(proposed_transaction)(required_active_approvals)(available_active_approvals)(required_owner_approvals)(available_owner_approvals)(available_key_approvals)(proposer) ) -FC_REFLECT( graphene::es_objects::account_struct, (object_id)(block_time)(block_number)(membership_expiration_date)(registrar)(referrer)(lifetime_referrer)(network_fee_percentage)(lifetime_referrer_fee_percentage)(referrer_rewards_percentage)(name)(owner_account_auths)(owner_key_auths)(owner_address_auths)(active_account_auths)(active_key_auths)(active_address_auths)(voting_account) ) -FC_REFLECT( graphene::es_objects::asset_struct, (object_id)(block_time)(block_number)(symbol)(issuer)(is_market_issued)(dynamic_asset_data_id)(bitasset_data_id) ) -FC_REFLECT( graphene::es_objects::balance_struct, (object_id)(block_time)(block_number)(block_time)(owner)(asset_id)(amount) ) -FC_REFLECT( graphene::es_objects::limit_order_struct, (object_id)(block_time)(block_number)(expiration)(seller)(for_sale)(sell_price)(deferred_fee) ) -FC_REFLECT( graphene::es_objects::bitasset_struct, (object_id)(block_time)(block_number)(current_feed)(current_feed_publication_time) ) +FC_REFLECT( + graphene::es_objects::proposal_struct, + (object_id)(block_time)(block_number)(expiration_time)(review_period_time)(proposed_transaction)(required_active_approvals) + (available_active_approvals)(required_owner_approvals)(available_owner_approvals)(available_key_approvals)(proposer) +) +FC_REFLECT( + graphene::es_objects::account_struct, + (object_id)(block_time)(block_number)(membership_expiration_date)(registrar)(referrer)(lifetime_referrer) + (network_fee_percentage)(lifetime_referrer_fee_percentage)(referrer_rewards_percentage)(name)(owner_account_auths) + (owner_key_auths)(owner_address_auths)(active_account_auths)(active_key_auths)(active_address_auths)(voting_account)(votes) +) +FC_REFLECT( + graphene::es_objects::asset_struct, + (object_id)(block_time)(block_number)(symbol)(issuer)(is_market_issued)(dynamic_asset_data_id)(bitasset_data_id) +) +FC_REFLECT( + graphene::es_objects::balance_struct, + (object_id)(block_time)(block_number)(owner)(asset_type)(balance) +) +FC_REFLECT( + graphene::es_objects::limit_order_struct, + (object_id)(block_time)(block_number)(expiration)(seller)(for_sale)(sell_price)(deferred_fee) +) +FC_REFLECT( + graphene::es_objects::bitasset_struct, + (object_id)(block_time)(block_number)(current_feed)(current_feed_publication_time) +) From 4f5f8f44799c5279661158335334c3b7b52e36f1 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 19 Dec 2018 09:48:43 -0300 Subject: [PATCH 08/17] Merge pull request #1429 from oxarbitrage/es_objects_templates Add an adaptor to es_objects and template function to reduce code --- libraries/plugins/es_objects/es_objects.cpp | 194 ++------------- .../graphene/es_objects/es_objects.hpp | 222 ++++-------------- 2 files changed, 74 insertions(+), 342 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 82e6ff231..9896772d1 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -30,10 +30,11 @@ #include #include #include +#include +#include #include - namespace graphene { namespace es_objects { namespace detail @@ -72,12 +73,8 @@ class es_objects_plugin_impl fc::time_point_sec block_time; private: - void prepare_proposal(const proposal_object& proposal_object); - void prepare_account(const account_object& account_object); - void prepare_asset(const asset_object& asset_object); - void prepare_balance(const account_balance_object& account_balance_object); - void prepare_limit(const limit_order_object& limit_object); - void prepare_bitasset(const asset_bitasset_data_object& bitasset_object); + template + void prepareTemplate(T blockchain_object, string index_name); }; bool es_objects_plugin_impl::index_database( const vector& ids, std::string action) @@ -102,7 +99,7 @@ bool es_objects_plugin_impl::index_database( const vector& ids, if(action == "delete") remove_from_database(p->id, "proposal"); else - prepare_proposal(*p); + prepareTemplate(*p, "proposal"); } } else if(value.is() && _es_objects_accounts) { @@ -112,7 +109,7 @@ bool es_objects_plugin_impl::index_database( const vector& ids, if(action == "delete") remove_from_database(a->id, "account"); else - prepare_account(*a); + prepareTemplate(*a, "account"); } } else if(value.is() && _es_objects_assets) { @@ -122,7 +119,7 @@ bool es_objects_plugin_impl::index_database( const vector& ids, if(action == "delete") remove_from_database(a->id, "asset"); else - prepare_asset(*a); + prepareTemplate(*a, "asset"); } } else if(value.is() && _es_objects_balances) { @@ -132,7 +129,7 @@ bool es_objects_plugin_impl::index_database( const vector& ids, if(action == "delete") remove_from_database(b->id, "balance"); else - prepare_balance(*b); + prepareTemplate(*b, "balance"); } } else if(value.is() && _es_objects_limit_orders) { @@ -142,7 +139,7 @@ bool es_objects_plugin_impl::index_database( const vector& ids, if(action == "delete") remove_from_database(l->id, "limitorder"); else - prepare_limit(*l); + prepareTemplate(*l, "limitorder"); } } else if(value.is() && _es_objects_asset_bitasset) { @@ -152,7 +149,7 @@ bool es_objects_plugin_impl::index_database( const vector& ids, if(action == "delete") remove_from_database(ba->id, "bitasset"); else - prepare_bitasset(*ba); + prepareTemplate(*ba, "bitasset"); } } } @@ -190,180 +187,33 @@ void es_objects_plugin_impl::remove_from_database( object_id_type id, std::strin } } -void es_objects_plugin_impl::prepare_proposal(const proposal_object& proposal_object) -{ - proposal_struct prop; - prop.object_id = proposal_object.id; - prop.block_time = block_time; - prop.block_number = block_number; - prop.expiration_time = proposal_object.expiration_time; - prop.review_period_time = proposal_object.review_period_time; - prop.proposed_transaction = fc::json::to_string(proposal_object.proposed_transaction); - prop.required_owner_approvals = fc::json::to_string(proposal_object.required_owner_approvals); - prop.available_owner_approvals = fc::json::to_string(proposal_object.available_owner_approvals); - prop.required_active_approvals = fc::json::to_string(proposal_object.required_active_approvals); - prop.available_key_approvals = fc::json::to_string(proposal_object.available_key_approvals); - prop.proposer = proposal_object.proposer; - - std::string data = fc::json::to_string(prop); - - fc::mutable_variant_object bulk_header; - bulk_header["_index"] = _es_objects_index_prefix + "proposal"; - bulk_header["_type"] = "data"; - if(_es_objects_keep_only_current) - { - bulk_header["_id"] = string(prop.object_id); - } - - prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); - std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); - prepare.clear(); -} - -void es_objects_plugin_impl::prepare_account(const account_object& account_object) -{ - account_struct acct; - acct.object_id = account_object.id; - acct.block_time = block_time; - acct.block_number = block_number; - acct.membership_expiration_date = account_object.membership_expiration_date; - acct.registrar = account_object.registrar; - acct.referrer = account_object.referrer; - acct.lifetime_referrer = account_object.lifetime_referrer; - acct.network_fee_percentage = account_object.network_fee_percentage; - acct.lifetime_referrer_fee_percentage = account_object.lifetime_referrer_fee_percentage; - acct.referrer_rewards_percentage = account_object.referrer_rewards_percentage; - acct.name = account_object.name; - acct.owner_account_auths = fc::json::to_string(account_object.owner.account_auths); - acct.owner_key_auths = fc::json::to_string(account_object.owner.key_auths); - acct.owner_address_auths = fc::json::to_string(account_object.owner.address_auths); - acct.active_account_auths = fc::json::to_string(account_object.active.account_auths); - acct.active_key_auths = fc::json::to_string(account_object.active.key_auths); - acct.active_address_auths = fc::json::to_string(account_object.active.address_auths); - acct.voting_account = account_object.options.voting_account; - acct.votes = fc::json::to_string(account_object.options.votes); - - std::string data = fc::json::to_string(acct); - - fc::mutable_variant_object bulk_header; - bulk_header["_index"] = _es_objects_index_prefix + "account"; - bulk_header["_type"] = "data"; - if(_es_objects_keep_only_current) - { - bulk_header["_id"] = string(acct.object_id); - } - - prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); - std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); - prepare.clear(); -} - -void es_objects_plugin_impl::prepare_asset(const asset_object& asset_object) -{ - asset_struct asset; - asset.object_id = asset_object.id; - asset.block_time = block_time; - asset.block_number = block_number; - asset.symbol = asset_object.symbol; - asset.issuer = asset_object.issuer; - asset.is_market_issued = asset_object.is_market_issued(); - asset.dynamic_asset_data_id = asset_object.dynamic_asset_data_id; - asset.bitasset_data_id = asset_object.bitasset_data_id; - - std::string data = fc::json::to_string(asset); - - fc::mutable_variant_object bulk_header; - bulk_header["_index"] = _es_objects_index_prefix + "asset"; - bulk_header["_type"] = "data"; - if(_es_objects_keep_only_current) - { - bulk_header["_id"] = string(asset.object_id); - } - - prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); - std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); - prepare.clear(); -} - -void es_objects_plugin_impl::prepare_balance(const account_balance_object& account_balance_object) +template +void es_objects_plugin_impl::prepareTemplate(T blockchain_object, string index_name) { - balance_struct balance; - balance.object_id = account_balance_object.id; - balance.block_time = block_time; - balance.block_number = block_number; - balance.owner = account_balance_object.owner; - balance.asset_type = account_balance_object.asset_type; - balance.balance = account_balance_object.balance; - - std::string data = fc::json::to_string(balance); - fc::mutable_variant_object bulk_header; - bulk_header["_index"] = _es_objects_index_prefix + "balance"; + bulk_header["_index"] = _es_objects_index_prefix + index_name; bulk_header["_type"] = "data"; if(_es_objects_keep_only_current) { - bulk_header["_id"] = string(balance.object_id); + bulk_header["_id"] = string(blockchain_object.id); } - prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); - std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); - prepare.clear(); -} + adaptor_struct adaptor; + fc::variant blockchain_object_variant; + fc::to_variant( blockchain_object, blockchain_object_variant, GRAPHENE_NET_MAX_NESTED_OBJECTS ); + fc::mutable_variant_object o = adaptor.adapt(blockchain_object_variant.get_object()); -void es_objects_plugin_impl::prepare_limit(const limit_order_object& limit_object) -{ - limit_order_struct limit; - limit.object_id = limit_object.id; - limit.block_time = block_time; - limit.block_number = block_number; - limit.expiration = limit_object.expiration; - limit.seller = limit_object.seller; - limit.for_sale = limit_object.for_sale; - limit.sell_price = limit_object.sell_price; - limit.deferred_fee = limit_object.deferred_fee; - - std::string data = fc::json::to_string(limit); + o["object_id"] = string(blockchain_object.id); + o["block_time"] = block_time; + o["block_number"] = block_number; - fc::mutable_variant_object bulk_header; - bulk_header["_index"] = _es_objects_index_prefix + "limitorder"; - bulk_header["_type"] = "data"; - if(_es_objects_keep_only_current) - { - bulk_header["_id"] = string(limit.object_id); - } + string data = fc::json::to_string(o); prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); prepare.clear(); } -void es_objects_plugin_impl::prepare_bitasset(const asset_bitasset_data_object& bitasset_object) -{ - if(!bitasset_object.is_prediction_market) { - - bitasset_struct bitasset; - bitasset.object_id = bitasset_object.id; - bitasset.block_time = block_time; - bitasset.block_number = block_number; - bitasset.current_feed = fc::json::to_string(bitasset_object.current_feed); - bitasset.current_feed_publication_time = bitasset_object.current_feed_publication_time; - - std::string data = fc::json::to_string(bitasset); - - fc::mutable_variant_object bulk_header; - bulk_header["_index"] = _es_objects_index_prefix + "bitasset"; - bulk_header["_type"] = "data"; - if(_es_objects_keep_only_current) - { - bulk_header["_id"] = string(bitasset.object_id); - } - - prepare = graphene::utilities::createBulk(bulk_header, std::move(data)); - std::move(prepare.begin(), prepare.end(), std::back_inserter(bulk)); - prepare.clear(); - } -} - es_objects_plugin_impl::~es_objects_plugin_impl() { return; diff --git a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp index 886129c88..fa91e3bde 100644 --- a/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp +++ b/libraries/plugins/es_objects/include/graphene/es_objects/es_objects.hpp @@ -30,7 +30,6 @@ namespace graphene { namespace es_objects { using namespace chain; - namespace detail { class es_objects_plugin_impl; @@ -54,178 +53,61 @@ class es_objects_plugin : public graphene::app::plugin std::unique_ptr my; }; -struct proposal_struct { - object_id_type object_id; - fc::time_point_sec block_time; - uint32_t block_number; - time_point_sec expiration_time; - optional review_period_time; - string proposed_transaction; - string required_active_approvals; - string available_active_approvals; - string required_owner_approvals; - string available_owner_approvals; - string available_key_approvals; - account_id_type proposer; - - friend bool operator==(const proposal_struct& l, const proposal_struct& r) - { - return std::tie(l.object_id, l.block_time, l.block_number, l.expiration_time, l.review_period_time, - l.proposed_transaction, l.required_active_approvals, l.available_active_approvals, - l.required_owner_approvals, l.available_owner_approvals, l.available_key_approvals, - l.proposer) == std::tie(r.object_id, r.block_time, r.block_number, r.expiration_time, r.review_period_time, - r.proposed_transaction, r.required_active_approvals, r.available_active_approvals, - r.required_owner_approvals, r.available_owner_approvals, r.available_key_approvals, - r.proposer); - } - friend bool operator!=(const proposal_struct& l, const proposal_struct& r) - { - return !operator==(l, r); - } -}; -struct account_struct { - object_id_type object_id; - fc::time_point_sec block_time; - uint32_t block_number; - time_point_sec membership_expiration_date; - account_id_type registrar; - account_id_type referrer; - account_id_type lifetime_referrer; - uint16_t network_fee_percentage; - uint16_t lifetime_referrer_fee_percentage; - uint16_t referrer_rewards_percentage; - string name; - string owner_account_auths; - string owner_key_auths; - string owner_address_auths; - string active_account_auths; - string active_key_auths; - string active_address_auths; - account_id_type voting_account; - string votes; - - friend bool operator==(const account_struct& l, const account_struct& r) - { - return std::tie(l.object_id, l.block_time, l.block_number, l.membership_expiration_date, l.registrar, l.referrer, - l.lifetime_referrer, l.network_fee_percentage, l.lifetime_referrer_fee_percentage, - l.referrer_rewards_percentage, l.name, l.owner_account_auths, l.owner_key_auths, - l.owner_address_auths, l.active_account_auths, l.active_key_auths, l.active_address_auths, - l.voting_account, l.votes) == std::tie(r.object_id, r.block_time, r.block_number, r.membership_expiration_date, r.registrar, r.referrer, - r.lifetime_referrer, r.network_fee_percentage, r.lifetime_referrer_fee_percentage, - r.referrer_rewards_percentage, r.name, r.owner_account_auths, r.owner_key_auths, - r.owner_address_auths, r.active_account_auths, r.active_key_auths, r.active_address_auths, - r.voting_account, r.votes); - } - friend bool operator!=(const account_struct& l, const account_struct& r) - { - return !operator==(l, r); - } -}; -struct asset_struct { - object_id_type object_id; - fc::time_point_sec block_time; - uint32_t block_number; - string symbol; - account_id_type issuer; - bool is_market_issued; - asset_dynamic_data_id_type dynamic_asset_data_id; - optional bitasset_data_id; - - friend bool operator==(const asset_struct& l, const asset_struct& r) - { - return std::tie(l.object_id, l.block_time, l.block_number, l.symbol, l.issuer, l.is_market_issued, - l.dynamic_asset_data_id, l.bitasset_data_id) == std::tie(r.object_id, r.block_time, - r.block_number, r.symbol, r.issuer, r.is_market_issued, r.dynamic_asset_data_id, - r.bitasset_data_id); - } - friend bool operator!=(const asset_struct& l, const asset_struct& r) - { - return !operator==(l, r); - } -}; -struct balance_struct { - object_id_type object_id; - fc::time_point_sec block_time; - uint32_t block_number; - account_id_type owner; - asset_id_type asset_type; - share_type balance; - - friend bool operator==(const balance_struct& l, const balance_struct& r) - { - return std::tie(l.object_id, l.block_time, l.block_number, l.block_time, l.owner, l.asset_type, l.balance) - == std::tie(r.object_id, r.block_time, r.block_number, r.block_time, r.owner, r.asset_type, r.balance); +struct adaptor_struct { + fc::mutable_variant_object adapt(const variant_object &obj) { + fc::mutable_variant_object o(obj); + vector keys_to_rename; + for (auto i = o.begin(); i != o.end(); ++i) { + auto &element = (*i).value(); + if (element.is_object()) { + const string &name = (*i).key(); + auto &vo = element.get_object(); + if (vo.contains(name.c_str())) + keys_to_rename.emplace_back(name); + element = adapt(vo); + } else if (element.is_array()) + adapt(element.get_array()); + } + for (const auto &i : keys_to_rename) { + string new_name = i + "_"; + o[new_name] = variant(o[i]); + o.erase(i); + } + if (o.find("owner") != o.end() && o["owner"].is_string()) + { + o["owner_"] = o["owner"].as_string(); + o.erase("owner"); + } + if (o.find("active_special_authority") != o.end()) + { + o["active_special_authority"] = fc::json::to_string(o["active_special_authority"]); + } + if (o.find("owner_special_authority") != o.end()) + { + o["owner_special_authority"] = fc::json::to_string(o["owner_special_authority"]); + } + if (o.find("feeds") != o.end()) + { + o["feeds"] = fc::json::to_string(o["feeds"]); + } + if (o.find("operations") != o.end()) + { + o["operations"] = fc::json::to_string(o["operations"]); + } + + return o; } - friend bool operator!=(const balance_struct& l, const balance_struct& r) - { - return !operator==(l, r); - } -}; -struct limit_order_struct { - object_id_type object_id; - fc::time_point_sec block_time; - uint32_t block_number; - time_point_sec expiration; - account_id_type seller; - share_type for_sale; - price sell_price; - share_type deferred_fee; - friend bool operator==(const limit_order_struct& l, const limit_order_struct& r) - { - return std::tie(l.object_id, l.block_time, l.block_number, l.expiration, l.seller, l.for_sale, l.sell_price, l.deferred_fee) - == std::tie(r.object_id, r.block_time, r.block_number, r.expiration, r.seller, r.for_sale, r.sell_price, r.deferred_fee); - } - friend bool operator!=(const limit_order_struct& l, const limit_order_struct& r) - { - return !operator==(l, r); - } -}; -struct bitasset_struct { - object_id_type object_id; - fc::time_point_sec block_time; - uint32_t block_number; - string current_feed; - time_point_sec current_feed_publication_time; - time_point_sec feed_expiration_time; - - friend bool operator==(const bitasset_struct& l, const bitasset_struct& r) - { - return std::tie(l.object_id, l.block_time, l.block_number, l.current_feed, l.current_feed_publication_time) - == std::tie(r.object_id, r.block_time, r.block_number, r.current_feed, r.current_feed_publication_time); - } - friend bool operator!=(const bitasset_struct& l, const bitasset_struct& r) - { - return !operator==(l, r); + void adapt(fc::variants &v) { + for (auto &array_element : v) { + if (array_element.is_object()) + array_element = adapt(array_element.get_object()); + else if (array_element.is_array()) + adapt(array_element.get_array()); + else + array_element = array_element.as_string(); + } } }; } } //graphene::es_objects - -FC_REFLECT( - graphene::es_objects::proposal_struct, - (object_id)(block_time)(block_number)(expiration_time)(review_period_time)(proposed_transaction)(required_active_approvals) - (available_active_approvals)(required_owner_approvals)(available_owner_approvals)(available_key_approvals)(proposer) -) -FC_REFLECT( - graphene::es_objects::account_struct, - (object_id)(block_time)(block_number)(membership_expiration_date)(registrar)(referrer)(lifetime_referrer) - (network_fee_percentage)(lifetime_referrer_fee_percentage)(referrer_rewards_percentage)(name)(owner_account_auths) - (owner_key_auths)(owner_address_auths)(active_account_auths)(active_key_auths)(active_address_auths)(voting_account)(votes) -) -FC_REFLECT( - graphene::es_objects::asset_struct, - (object_id)(block_time)(block_number)(symbol)(issuer)(is_market_issued)(dynamic_asset_data_id)(bitasset_data_id) -) -FC_REFLECT( - graphene::es_objects::balance_struct, - (object_id)(block_time)(block_number)(owner)(asset_type)(balance) -) -FC_REFLECT( - graphene::es_objects::limit_order_struct, - (object_id)(block_time)(block_number)(expiration)(seller)(for_sale)(sell_price)(deferred_fee) -) -FC_REFLECT( - graphene::es_objects::bitasset_struct, - (object_id)(block_time)(block_number)(current_feed)(current_feed_publication_time) -) From c4612a522b9c1819a1d0712a0e9bab4f6ec40a2f Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 21 Dec 2018 08:24:45 -0300 Subject: [PATCH 09/17] Merge pull request #1458 from oxarbitrage/issue1455 add option elasticsearch-start-es-after-block to es plugin --- .../elasticsearch/elasticsearch_plugin.cpp | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index b69ff64aa..23cb31aa8 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -58,6 +58,8 @@ class elasticsearch_plugin_impl bool _elasticsearch_visitor = false; std::string _elasticsearch_basic_auth = ""; std::string _elasticsearch_index_prefix = "bitshares-"; + bool _elasticsearch_operation_object = false; + uint32_t _elasticsearch_start_es_after_block = 0; // disabled CURL *curl; // curl handler vector bulk_lines; // vector of op lines vector prepare; @@ -73,7 +75,7 @@ class elasticsearch_plugin_impl std::string index_name; bool is_sync = false; private: - bool add_elasticsearch( const account_id_type account_id, const optional& oho, const signed_block& b ); + bool add_elasticsearch( const account_id_type account_id, const optional& oho, const uint32_t block_number ); const account_transaction_history_object& addNewEntry(const account_statistics_object& stats_obj, const account_id_type account_id, const optional & oho); @@ -162,7 +164,7 @@ bool elasticsearch_plugin_impl::update_account_histories( const signed_block& b for( auto& account_id : impacted ) { - if(!add_elasticsearch( account_id, oho, b )) + if(!add_elasticsearch( account_id, oho, b.block_num() )) return false; } } @@ -247,13 +249,15 @@ void elasticsearch_plugin_impl::doVisitor(const optional & oho, - const signed_block& b) + const uint32_t block_number) { const auto &stats_obj = getStatsObject(account_id); const auto &ath = addNewEntry(stats_obj, account_id, oho); growStats(stats_obj, ath); - createBulkLine(ath); - prepareBulk(ath.id); + if(_elasticsearch_start_es_after_block == 0 || block_number > _elasticsearch_start_es_after_block) { + createBulkLine(ath); + prepareBulk(ath.id); + } cleanObjects(ath, account_id); if (curl && bulk_lines.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech @@ -392,6 +396,8 @@ void elasticsearch_plugin::plugin_set_program_options( ("elasticsearch-visitor", boost::program_options::value(), "Use visitor to index additional data(slows down the replay)") ("elasticsearch-basic-auth", boost::program_options::value(), "Pass basic auth to elasticsearch database ") ("elasticsearch-index-prefix", boost::program_options::value(), "Add a prefix to the index(bitshares-)") + ("elasticsearch-operation-object", boost::program_options::value(), "Save operation as object(false)") + ("elasticsearch-start-es-after-block", boost::program_options::value(), "Start doing ES job after block(0)") ; cfg.add(cli); } @@ -399,11 +405,10 @@ void elasticsearch_plugin::plugin_set_program_options( void elasticsearch_plugin::plugin_initialize(const boost::program_options::variables_map& options) { database().applied_block.connect( [&]( const signed_block& b) { - if(!my->update_account_histories(b)) - { + if (!my->update_account_histories(b)) FC_THROW_EXCEPTION(fc::exception, "Error populating ES database, we are going to keep trying."); - } } ); + my->_oho_index = database().add_index< primary_index< operation_history_index > >(); database().add_index< primary_index< account_transaction_history_index > >(); @@ -425,6 +430,12 @@ void elasticsearch_plugin::plugin_initialize(const boost::program_options::varia if (options.count("elasticsearch-index-prefix")) { my->_elasticsearch_index_prefix = options["elasticsearch-index-prefix"].as(); } + if (options.count("elasticsearch-operation-object")) { + my->_elasticsearch_operation_object = options["elasticsearch-operation-object"].as(); + } + if (options.count("elasticsearch-start-es-after-block")) { + my->_elasticsearch_start_es_after_block = options["elasticsearch-start-es-after-block"].as(); + } } void elasticsearch_plugin::plugin_startup() From 6d9ad94e20a07903f5dd6e95b66bda80aac58844 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 30 Jan 2019 11:46:17 -0300 Subject: [PATCH 10/17] Merge pull request #1541 from oxarbitrage/es_objects_start_after_block add es-objects-start-es-after-block option --- .../elasticsearch/elasticsearch_plugin.cpp | 4 +- libraries/plugins/es_objects/es_objects.cpp | 158 +++++++++--------- 2 files changed, 83 insertions(+), 79 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 23cb31aa8..1ca911f98 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -59,7 +59,7 @@ class elasticsearch_plugin_impl std::string _elasticsearch_basic_auth = ""; std::string _elasticsearch_index_prefix = "bitshares-"; bool _elasticsearch_operation_object = false; - uint32_t _elasticsearch_start_es_after_block = 0; // disabled + uint32_t _elasticsearch_start_es_after_block = 0; CURL *curl; // curl handler vector bulk_lines; // vector of op lines vector prepare; @@ -254,7 +254,7 @@ bool elasticsearch_plugin_impl::add_elasticsearch( const account_id_type account const auto &stats_obj = getStatsObject(account_id); const auto &ath = addNewEntry(stats_obj, account_id, oho); growStats(stats_obj, ath); - if(_elasticsearch_start_es_after_block == 0 || block_number > _elasticsearch_start_es_after_block) { + if(block_number > _elasticsearch_start_es_after_block) { createBulkLine(ath); prepareBulk(ath.id); } diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 9896772d1..58517349e 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -63,6 +63,7 @@ class es_objects_plugin_impl bool _es_objects_limit_orders = true; bool _es_objects_asset_bitasset = true; std::string _es_objects_index_prefix = "objects-"; + uint32_t _es_objects_start_es_after_block = 0; CURL *curl; // curl handler vector bulk; vector prepare; @@ -84,88 +85,87 @@ bool es_objects_plugin_impl::index_database( const vector& ids, block_time = db.head_block_time(); block_number = db.head_block_num(); - // check if we are in replay or in sync and change number of bulk documents accordingly - uint32_t limit_documents = 0; - if((fc::time_point::now() - block_time) < fc::seconds(30)) - limit_documents = _es_objects_bulk_sync; - else - limit_documents = _es_objects_bulk_replay; - - for(auto const& value: ids) { - if(value.is() && _es_objects_proposals) { - auto obj = db.find_object(value); - auto p = static_cast(obj); - if(p != nullptr) { - if(action == "delete") - remove_from_database(p->id, "proposal"); - else - prepareTemplate(*p, "proposal"); - } - } - else if(value.is() && _es_objects_accounts) { - auto obj = db.find_object(value); - auto a = static_cast(obj); - if(a != nullptr) { - if(action == "delete") - remove_from_database(a->id, "account"); - else - prepareTemplate(*a, "account"); - } - } - else if(value.is() && _es_objects_assets) { - auto obj = db.find_object(value); - auto a = static_cast(obj); - if(a != nullptr) { - if(action == "delete") - remove_from_database(a->id, "asset"); - else - prepareTemplate(*a, "asset"); - } - } - else if(value.is() && _es_objects_balances) { - auto obj = db.find_object(value); - auto b = static_cast(obj); - if(b != nullptr) { - if(action == "delete") - remove_from_database(b->id, "balance"); - else - prepareTemplate(*b, "balance"); - } - } - else if(value.is() && _es_objects_limit_orders) { - auto obj = db.find_object(value); - auto l = static_cast(obj); - if(l != nullptr) { - if(action == "delete") - remove_from_database(l->id, "limitorder"); - else - prepareTemplate(*l, "limitorder"); - } - } - else if(value.is() && _es_objects_asset_bitasset) { - auto obj = db.find_object(value); - auto ba = static_cast(obj); - if(ba != nullptr) { - if(action == "delete") - remove_from_database(ba->id, "bitasset"); - else - prepareTemplate(*ba, "bitasset"); + if(block_number > _es_objects_start_es_after_block) { + + // check if we are in replay or in sync and change number of bulk documents accordingly + uint32_t limit_documents = 0; + if ((fc::time_point::now() - block_time) < fc::seconds(30)) + limit_documents = _es_objects_bulk_sync; + else + limit_documents = _es_objects_bulk_replay; + + + for (auto const &value: ids) { + if (value.is() && _es_objects_proposals) { + auto obj = db.find_object(value); + auto p = static_cast(obj); + if (p != nullptr) { + if (action == "delete") + remove_from_database(p->id, "proposal"); + else + prepareTemplate(*p, "proposal"); + } + } else if (value.is() && _es_objects_accounts) { + auto obj = db.find_object(value); + auto a = static_cast(obj); + if (a != nullptr) { + if (action == "delete") + remove_from_database(a->id, "account"); + else + prepareTemplate(*a, "account"); + } + } else if (value.is() && _es_objects_assets) { + auto obj = db.find_object(value); + auto a = static_cast(obj); + if (a != nullptr) { + if (action == "delete") + remove_from_database(a->id, "asset"); + else + prepareTemplate(*a, "asset"); + } + } else if (value.is() && _es_objects_balances) { + auto obj = db.find_object(value); + auto b = static_cast(obj); + if (b != nullptr) { + if (action == "delete") + remove_from_database(b->id, "balance"); + else + prepareTemplate(*b, "balance"); + } + } else if (value.is() && _es_objects_limit_orders) { + auto obj = db.find_object(value); + auto l = static_cast(obj); + if (l != nullptr) { + if (action == "delete") + remove_from_database(l->id, "limitorder"); + else + prepareTemplate(*l, "limitorder"); + } + } else if (value.is() && _es_objects_asset_bitasset) { + auto obj = db.find_object(value); + auto ba = static_cast(obj); + if (ba != nullptr) { + if (action == "delete") + remove_from_database(ba->id, "bitasset"); + else + prepareTemplate(*ba, "bitasset"); + } } } - } - if (curl && bulk.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech + if (curl && bulk.size() >= limit_documents) { // we are in bulk time, ready to add data to elasticsearech - graphene::utilities::ES es; - es.curl = curl; - es.bulk_lines = bulk; - es.elasticsearch_url = _es_objects_elasticsearch_url; - es.auth = _es_objects_auth; + graphene::utilities::ES es; + es.curl = curl; + es.bulk_lines = bulk; + es.elasticsearch_url = _es_objects_elasticsearch_url; + es.auth = _es_objects_auth; - if(!graphene::utilities::SendBulk(es)) - return false; - else - bulk.clear(); + if (!graphene::utilities::SendBulk(es)) + return false; + else + bulk.clear(); + } } return true; @@ -257,6 +257,7 @@ void es_objects_plugin::plugin_set_program_options( ("es-objects-asset-bitasset", boost::program_options::value(), "Store feed data(true)") ("es-objects-index-prefix", boost::program_options::value(), "Add a prefix to the index(objects-)") ("es-objects-keep-only-current", boost::program_options::value(), "Keep only current state of the objects(true)") + ("es-objects-start-es-after-block", boost::program_options::value(), "Start doing ES job after block(0)") ; cfg.add(cli); } @@ -319,6 +320,9 @@ void es_objects_plugin::plugin_initialize(const boost::program_options::variable if (options.count("es-objects-keep-only-current")) { my->_es_objects_keep_only_current = options["es-objects-keep-only-current"].as(); } + if (options.count("es-objects-start-es-after-block")) { + my->_es_objects_start_es_after_block = options["es-objects-start-es-after-block"].as(); + } } void es_objects_plugin::plugin_startup() From 82ef3a51d1037295d8de3c06339d809e04a4c960 Mon Sep 17 00:00:00 2001 From: crypto-ape <43807588+crypto-ape@users.noreply.github.com> Date: Tue, 2 Apr 2019 12:55:25 +0200 Subject: [PATCH 11/17] explicitly cleanup external library facilities --- libraries/plugins/elasticsearch/elasticsearch_plugin.cpp | 4 ++++ libraries/plugins/es_objects/es_objects.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 1ca911f98..d67539c44 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -94,6 +94,10 @@ class elasticsearch_plugin_impl elasticsearch_plugin_impl::~elasticsearch_plugin_impl() { + if (curl) { + curl_easy_cleanup(curl); + curl = nullptr; + } return; } diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 58517349e..4b95343b8 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -216,6 +216,10 @@ void es_objects_plugin_impl::prepareTemplate(T blockchain_object, string index_n es_objects_plugin_impl::~es_objects_plugin_impl() { + if (curl) { + curl_easy_cleanup(curl); + curl = nullptr; + } return; } From 2f054ac619c7ac3a5f886053933442a504a8a8a8 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 23 Apr 2019 07:56:14 -0300 Subject: [PATCH 12/17] Merge pull request #1717 from oxarbitrage/issue1652 add genesis data to es_objects --- libraries/plugins/es_objects/es_objects.cpp | 66 +++++++++++++++++++-- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 4b95343b8..5b3f29e7e 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -48,8 +48,9 @@ class es_objects_plugin_impl { curl = curl_easy_init(); } virtual ~es_objects_plugin_impl(); - bool index_database( const vector& ids, std::string action); - void remove_from_database( object_id_type id, std::string index); + bool index_database(const vector& ids, std::string action); + bool genesis(); + void remove_from_database(object_id_type id, std::string index); es_objects_plugin& _self; std::string _es_objects_elasticsearch_url = "http://localhost:9200/"; @@ -78,7 +79,55 @@ class es_objects_plugin_impl void prepareTemplate(T blockchain_object, string index_name); }; -bool es_objects_plugin_impl::index_database( const vector& ids, std::string action) +bool es_objects_plugin_impl::genesis() +{ + + ilog("elasticsearch OBJECTS: inserting data from genesis"); + + graphene::chain::database &db = _self.database(); + + block_number = db.head_block_num(); + block_time = db.head_block_time(); + + if (_es_objects_accounts) { + auto &index_accounts = db.get_index(1, 2); + index_accounts.inspect_all_objects([this, &db](const graphene::db::object &o) { + auto obj = db.find_object(o.id); + auto a = static_cast(obj); + prepareTemplate(*a, "account"); + }); + } + if (_es_objects_assets) { + auto &index_assets = db.get_index(1, 3); + index_assets.inspect_all_objects([this, &db](const graphene::db::object &o) { + auto obj = db.find_object(o.id); + auto a = static_cast(obj); + prepareTemplate(*a, "asset"); + }); + } + if (_es_objects_balances) { + auto &index_balances = db.get_index(2, 5); + index_balances.inspect_all_objects([this, &db](const graphene::db::object &o) { + auto obj = db.find_object(o.id); + auto b = static_cast(obj); + prepareTemplate(*b, "balance"); + }); + } + + graphene::utilities::ES es; + es.curl = curl; + es.bulk_lines = bulk; + es.elasticsearch_url = _es_objects_elasticsearch_url; + es.auth = _es_objects_auth; + if (!graphene::utilities::SendBulk(es)) + FC_THROW_EXCEPTION(fc::exception, "Error inserting genesis data."); + else + bulk.clear(); + + return true; +} + +bool es_objects_plugin_impl::index_database(const vector& ids, std::string action) { graphene::chain::database &db = _self.database(); @@ -268,13 +317,20 @@ void es_objects_plugin::plugin_set_program_options( void es_objects_plugin::plugin_initialize(const boost::program_options::variables_map& options) { - database().new_objects.connect([&]( const vector& ids, const flat_set& impacted_accounts ) { + database().applied_block.connect([this](const signed_block &b) { + if(b.block_num() == 1) { + if (!my->genesis()) + FC_THROW_EXCEPTION(fc::exception, "Error populating genesis data."); + } + }); + + database().new_objects.connect([this]( const vector& ids, const flat_set& impacted_accounts ) { if(!my->index_database(ids, "create")) { FC_THROW_EXCEPTION(fc::exception, "Error creating object from ES database, we are going to keep trying."); } }); - database().changed_objects.connect([&]( const vector& ids, const flat_set& impacted_accounts ) { + database().changed_objects.connect([this]( const vector& ids, const flat_set& impacted_accounts ) { if(!my->index_database(ids, "update")) { FC_THROW_EXCEPTION(fc::exception, "Error updating object from ES database, we are going to keep trying."); From de4faee7f0a6aa12874303ab444df5ccd77bedee Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 21 Jun 2018 09:01:51 -0300 Subject: [PATCH 13/17] Merge pull request #1073 from xiangxn/merge-impacted merge impacted into db_notify --- libraries/app/CMakeLists.txt | 1 - libraries/app/api.cpp | 1 - libraries/app/impacted.cpp | 315 ------------------ libraries/chain/db_notify.cpp | 10 +- .../include/graphene/chain}/impacted.hpp | 4 +- .../account_history_plugin.cpp | 7 +- .../accounts_list/accounts_list_plugin.cpp | 2 +- .../affiliate_stats_plugin.cpp | 2 +- libraries/plugins/bookie/bookie_plugin.cpp | 2 +- .../elasticsearch/elasticsearch_plugin.cpp | 4 +- 10 files changed, 19 insertions(+), 329 deletions(-) delete mode 100644 libraries/app/impacted.cpp rename libraries/{app/include/graphene/app => chain/include/graphene/chain}/impacted.hpp (96%) diff --git a/libraries/app/CMakeLists.txt b/libraries/app/CMakeLists.txt index 93e540f98..ac69dfe5d 100644 --- a/libraries/app/CMakeLists.txt +++ b/libraries/app/CMakeLists.txt @@ -5,7 +5,6 @@ add_library( graphene_app api.cpp application.cpp database_api.cpp - impacted.cpp plugin.cpp config_util.cpp ${HEADERS} diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 138e0560e..94f01330d 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include diff --git a/libraries/app/impacted.cpp b/libraries/app/impacted.cpp deleted file mode 100644 index 08253417b..000000000 --- a/libraries/app/impacted.cpp +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Copyright (c) 2015 Cryptonomex, Inc., and contributors. - * - * The MIT License - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include - -namespace graphene { namespace app { - -using namespace fc; -using namespace graphene::chain; - -// TODO: Review all of these, especially no-ops -struct get_impacted_account_visitor -{ - flat_set& _impacted; - get_impacted_account_visitor( flat_set& impact ):_impacted(impact) {} - typedef void result_type; - - void operator()( const transfer_operation& op ) - { - _impacted.insert( op.to ); - } - - void operator()( const asset_claim_fees_operation& op ){} - void operator()( const limit_order_create_operation& op ) {} - void operator()( const limit_order_cancel_operation& op ) - { - _impacted.insert( op.fee_paying_account ); - } - void operator()( const call_order_update_operation& op ) {} - void operator()( const fill_order_operation& op ) - { - _impacted.insert( op.account_id ); - } - - void operator()( const account_create_operation& op ) - { - _impacted.insert( op.registrar ); - _impacted.insert( op.referrer ); - add_authority_accounts( _impacted, op.owner ); - add_authority_accounts( _impacted, op.active ); - } - - void operator()( const account_update_operation& op ) - { - _impacted.insert( op.account ); - if( op.owner ) - add_authority_accounts( _impacted, *(op.owner) ); - if( op.active ) - add_authority_accounts( _impacted, *(op.active) ); - } - - void operator()( const account_whitelist_operation& op ) - { - _impacted.insert( op.account_to_list ); - } - - void operator()( const account_upgrade_operation& op ) {} - void operator()( const account_transfer_operation& op ) - { - _impacted.insert( op.new_owner ); - } - - void operator()( const asset_create_operation& op ) {} - void operator()( const asset_update_operation& op ) - { - if( op.new_issuer ) - _impacted.insert( *(op.new_issuer) ); - } - - void operator()( const asset_update_bitasset_operation& op ) {} - void operator()( const asset_update_dividend_operation& op ) {} - void operator()( const asset_dividend_distribution_operation& op ) - { - _impacted.insert( op.account_id ); - } - - void operator()( const asset_update_feed_producers_operation& op ) {} - - void operator()( const asset_issue_operation& op ) - { - _impacted.insert( op.issue_to_account ); - } - - void operator()( const asset_reserve_operation& op ) {} - void operator()( const asset_fund_fee_pool_operation& op ) {} - void operator()( const asset_settle_operation& op ) {} - void operator()( const asset_global_settle_operation& op ) {} - void operator()( const asset_publish_feed_operation& op ) {} - void operator()( const witness_create_operation& op ) - { - _impacted.insert( op.witness_account ); - } - void operator()( const witness_update_operation& op ) - { - _impacted.insert( op.witness_account ); - } - - void operator()( const proposal_create_operation& op ) - { - vector other; - for( const auto& proposed_op : op.proposed_ops ) - operation_get_required_authorities( proposed_op.op, _impacted, _impacted, other ); - for( auto& o : other ) - add_authority_accounts( _impacted, o ); - } - - void operator()( const proposal_update_operation& op ) {} - void operator()( const proposal_delete_operation& op ) {} - - void operator()( const withdraw_permission_create_operation& op ) - { - _impacted.insert( op.authorized_account ); - } - - void operator()( const withdraw_permission_update_operation& op ) - { - _impacted.insert( op.authorized_account ); - } - - void operator()( const withdraw_permission_claim_operation& op ) - { - _impacted.insert( op.withdraw_from_account ); - } - - void operator()( const withdraw_permission_delete_operation& op ) - { - _impacted.insert( op.authorized_account ); - } - - void operator()( const committee_member_create_operation& op ) - { - _impacted.insert( op.committee_member_account ); - } - void operator()( const committee_member_update_operation& op ) - { - _impacted.insert( op.committee_member_account ); - } - void operator()( const committee_member_update_global_parameters_operation& op ) {} - - void operator()( const vesting_balance_create_operation& op ) - { - _impacted.insert( op.owner ); - } - - void operator()( const vesting_balance_withdraw_operation& op ) {} - void operator()( const worker_create_operation& op ) {} - void operator()( const custom_operation& op ) {} - void operator()( const assert_operation& op ) {} - void operator()( const balance_claim_operation& op ) {} - - void operator()( const override_transfer_operation& op ) - { - _impacted.insert( op.to ); - _impacted.insert( op.from ); - _impacted.insert( op.issuer ); - } - - void operator()( const transfer_to_blind_operation& op ) - { - _impacted.insert( op.from ); - for( const auto& out : op.outputs ) - add_authority_accounts( _impacted, out.owner ); - } - - void operator()( const blind_transfer_operation& op ) - { - for( const auto& in : op.inputs ) - add_authority_accounts( _impacted, in.owner ); - for( const auto& out : op.outputs ) - add_authority_accounts( _impacted, out.owner ); - } - - void operator()( const transfer_from_blind_operation& op ) - { - _impacted.insert( op.to ); - for( const auto& in : op.inputs ) - add_authority_accounts( _impacted, in.owner ); - } - - void operator()( const asset_settle_cancel_operation& op ) - { - _impacted.insert( op.account ); - } - - void operator()( const fba_distribute_operation& op ) - { - _impacted.insert( op.account_id ); - } - - void operator()( const sport_create_operation& op ) {} - void operator()( const sport_update_operation& op ) {} - void operator()( const sport_delete_operation& op ) {} - void operator()( const event_group_create_operation& op ) {} - void operator()( const event_group_update_operation& op ) {} - void operator()( const event_group_delete_operation& op ) {} - void operator()( const event_create_operation& op ) {} - void operator()( const event_update_operation& op ) {} - void operator()( const event_update_status_operation& op ) {} - void operator()( const betting_market_rules_create_operation& op ) {} - void operator()( const betting_market_rules_update_operation& op ) {} - void operator()( const betting_market_group_create_operation& op ) {} - void operator()( const betting_market_group_update_operation& op ) {} - void operator()( const betting_market_create_operation& op ) {} - void operator()( const betting_market_update_operation& op ) {} - void operator()( const betting_market_group_resolve_operation& op ) {} - void operator()( const betting_market_group_cancel_unmatched_bets_operation& op ) {} - - void operator()( const bet_place_operation& op ) - { - _impacted.insert( op.bettor_id ); - } - void operator()( const bet_cancel_operation& op ) - { - _impacted.insert( op.bettor_id ); - } - void operator()( const bet_canceled_operation& op ) - { - _impacted.insert( op.bettor_id ); - } - void operator()( const bet_adjusted_operation& op ) - { - _impacted.insert( op.bettor_id ); - } - void operator()( const bet_matched_operation& op ) - { - _impacted.insert( op.bettor_id ); - } - void operator()( const betting_market_group_resolved_operation& op ) - { - _impacted.insert( op.bettor_id ); - } - - void operator()( const tournament_create_operation& op ) - { - _impacted.insert( op.creator ); - _impacted.insert( op.options.whitelist.begin(), op.options.whitelist.end() ); - } - void operator()( const tournament_join_operation& op ) - { - _impacted.insert( op.payer_account_id ); - _impacted.insert( op.player_account_id ); - } - void operator()( const tournament_leave_operation& op ) - { - //if account canceling registration is not the player, it must be the payer - if (op.canceling_account_id != op.player_account_id) - _impacted.erase( op.canceling_account_id ); - _impacted.erase( op.player_account_id ); - } - void operator()( const game_move_operation& op ) - { - _impacted.insert( op.player_account_id ); - } - void operator()( const tournament_payout_operation& op ) - { - _impacted.insert( op.payout_account_id ); - } - void operator()( const affiliate_payout_operation& op ) - { - _impacted.insert( op.affiliate ); - } - void operator()( const affiliate_referral_payout_operation& op ) { } - void operator()( const lottery_asset_create_operation& op) { } - void operator()( const ticket_purchase_operation& op ) - { - _impacted.insert( op.buyer ); - } - void operator()( const lottery_reward_operation& op ) { - _impacted.insert( op.winner ); - } - void operator()( const lottery_end_operation& op ) { - for( auto participant : op.participants ) { - _impacted.insert(participant.first); - } - } - void operator()( const sweeps_vesting_claim_operation& op ) { - _impacted.insert( op.account ); - } -}; - -void operation_get_impacted_accounts( const operation& op, flat_set& result ) -{ - get_impacted_account_visitor vtor = get_impacted_account_visitor( result ); - op.visit( vtor ); -} - -void transaction_get_impacted_accounts( const transaction& tx, flat_set& result ) -{ - for( const auto& op : tx.operations ) - operation_get_impacted_accounts( op, result ); -} - -} } diff --git a/libraries/chain/db_notify.cpp b/libraries/chain/db_notify.cpp index 3404989a8..d41c2a26c 100644 --- a/libraries/chain/db_notify.cpp +++ b/libraries/chain/db_notify.cpp @@ -33,6 +33,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include using namespace fc; using namespace graphene::chain; @@ -287,13 +293,13 @@ struct get_impacted_account_visitor } }; -void operation_get_impacted_accounts( const operation& op, flat_set& result ) +void graphene::chain::operation_get_impacted_accounts( const operation& op, flat_set& result ) { get_impacted_account_visitor vtor = get_impacted_account_visitor( result ); op.visit( vtor ); } -void transaction_get_impacted_accounts( const transaction& tx, flat_set& result ) +void graphene::chain::transaction_get_impacted_accounts( const transaction& tx, flat_set& result ) { for( const auto& op : tx.operations ) operation_get_impacted_accounts( op, result ); diff --git a/libraries/app/include/graphene/app/impacted.hpp b/libraries/chain/include/graphene/chain/impacted.hpp similarity index 96% rename from libraries/app/include/graphene/app/impacted.hpp rename to libraries/chain/include/graphene/chain/impacted.hpp index 2e59b9104..2a22cbd12 100644 --- a/libraries/app/include/graphene/app/impacted.hpp +++ b/libraries/chain/include/graphene/chain/impacted.hpp @@ -28,7 +28,7 @@ #include #include -namespace graphene { namespace app { +namespace graphene { namespace chain { void operation_get_impacted_accounts( const graphene::chain::operation& op, @@ -39,4 +39,4 @@ void transaction_get_impacted_accounts( fc::flat_set& result ); -} } // graphene::app +} } // graphene::app \ No newline at end of file diff --git a/libraries/plugins/account_history/account_history_plugin.cpp b/libraries/plugins/account_history/account_history_plugin.cpp index 81acb01ed..67322f800 100644 --- a/libraries/plugins/account_history/account_history_plugin.cpp +++ b/libraries/plugins/account_history/account_history_plugin.cpp @@ -24,7 +24,7 @@ #include -#include +#include #include #include @@ -128,8 +128,8 @@ void account_history_plugin_impl::update_account_histories( const signed_block& if( op.op.which() == operation::tag< account_create_operation >::value ) impacted.insert( op.result.get() ); else - graphene::app::operation_get_impacted_accounts( op.op, impacted ); - if( op.op.which() == operation::tag< lottery_end_operation >::value ) + graphene::chain::operation_get_impacted_accounts( op.op, impacted ); + if( op.op.which() == operation::tag< lottery_end_operation >::value ) { auto lop = op.op.get< lottery_end_operation >(); auto asset_object = lop.lottery( db ); @@ -137,6 +137,7 @@ void account_history_plugin_impl::update_account_histories( const signed_block& for( auto benefactor : asset_object.lottery_options->benefactors ) impacted.insert( benefactor.id ); } + for( auto& a : other ) for( auto& item : a.account_auths ) impacted.insert( item.first ); diff --git a/libraries/plugins/accounts_list/accounts_list_plugin.cpp b/libraries/plugins/accounts_list/accounts_list_plugin.cpp index aabf711d1..757891ea9 100644 --- a/libraries/plugins/accounts_list/accounts_list_plugin.cpp +++ b/libraries/plugins/accounts_list/accounts_list_plugin.cpp @@ -24,7 +24,7 @@ #include -#include +#include #include #include diff --git a/libraries/plugins/affiliate_stats/affiliate_stats_plugin.cpp b/libraries/plugins/affiliate_stats/affiliate_stats_plugin.cpp index 438b1acab..da9c8a04b 100644 --- a/libraries/plugins/affiliate_stats/affiliate_stats_plugin.cpp +++ b/libraries/plugins/affiliate_stats/affiliate_stats_plugin.cpp @@ -25,7 +25,7 @@ #include #include -#include +#include #include #include diff --git a/libraries/plugins/bookie/bookie_plugin.cpp b/libraries/plugins/bookie/bookie_plugin.cpp index f15ac2f7c..5b59d14a4 100644 --- a/libraries/plugins/bookie/bookie_plugin.cpp +++ b/libraries/plugins/bookie/bookie_plugin.cpp @@ -24,7 +24,7 @@ #include #include -#include +#include #include #include diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index d67539c44..98f91c590 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -23,7 +23,7 @@ */ #include -#include +#include #include #include #include @@ -160,7 +160,7 @@ bool elasticsearch_plugin_impl::update_account_histories( const signed_block& b if( op.op.which() == operation::tag< account_create_operation >::value ) impacted.insert( op.result.get() ); else - graphene::app::operation_get_impacted_accounts( op.op, impacted ); + graphene::chain::operation_get_impacted_accounts( op.op, impacted ); for( auto& a : other ) for( auto& item : a.account_auths ) From 62247c543d2422345334a239179463dc431d7e95 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 14 Aug 2019 07:57:10 -0300 Subject: [PATCH 14/17] Merge pull request #1725 from oxarbitrage/issue1682 elasticsearch history api #1682 --- libraries/app/CMakeLists.txt | 2 +- libraries/app/api.cpp | 12 + libraries/app/application.cpp | 5 + libraries/app/include/graphene/app/api.hpp | 2 + .../app/include/graphene/app/application.hpp | 7 +- .../elasticsearch/elasticsearch_plugin.cpp | 188 +++++++++- .../elasticsearch/elasticsearch_plugin.hpp | 106 +++++- tests/common/database_fixture.cpp | 11 +- tests/elasticsearch/main.cpp | 324 +++++++++++++++++- 9 files changed, 634 insertions(+), 23 deletions(-) diff --git a/libraries/app/CMakeLists.txt b/libraries/app/CMakeLists.txt index ac69dfe5d..d66b4052b 100644 --- a/libraries/app/CMakeLists.txt +++ b/libraries/app/CMakeLists.txt @@ -13,7 +13,7 @@ add_library( graphene_app # need to link graphene_debug_witness because plugins aren't sufficiently isolated #246 #target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_chain fc graphene_db graphene_net graphene_utilities graphene_debug_witness ) -target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_accounts_list graphene_affiliate_stats graphene_chain fc graphene_db graphene_net graphene_time graphene_utilities graphene_debug_witness graphene_bookie ) +target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_accounts_list graphene_affiliate_stats graphene_chain fc graphene_db graphene_net graphene_time graphene_utilities graphene_debug_witness graphene_bookie graphene_elasticsearch ) target_include_directories( graphene_app PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../egenesis/include" ) diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp index 94f01330d..c808a27c7 100644 --- a/libraries/app/api.cpp +++ b/libraries/app/api.cpp @@ -580,6 +580,18 @@ namespace graphene { namespace app { start = node.operation_id; } catch(...) { return result; } + if(_app.is_plugin_enabled("elasticsearch")) { + auto es = _app.get_plugin("elasticsearch"); + if(es.get()->get_running_mode() != elasticsearch::mode::only_save) { + if(!_app.elasticsearch_thread) + _app.elasticsearch_thread= std::make_shared("elasticsearch"); + + return _app.elasticsearch_thread->async([&es, &account, &stop, &limit, &start]() { + return es->get_account_history(account, stop, limit, start); + }, "thread invoke for method " BOOST_PP_STRINGIZE(method_name)).wait(); + } + } + const auto& hist_idx = db.get_index_type(); const auto& by_op_idx = hist_idx.indices().get(); auto index_start = by_op_idx.begin(); diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index b366be490..b20ffc4cc 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -1026,6 +1026,11 @@ std::shared_ptr application::get_plugin(const string& name) con return my->_active_plugins[name]; } +bool application::is_plugin_enabled(const string& name) const +{ + return !(my->_active_plugins.find(name) == my->_active_plugins.end()); +} + net::node_ptr application::p2p_node() { return my->_p2p_network; diff --git a/libraries/app/include/graphene/app/api.hpp b/libraries/app/include/graphene/app/api.hpp index 9e468dca9..4adf73a3a 100644 --- a/libraries/app/include/graphene/app/api.hpp +++ b/libraries/app/include/graphene/app/api.hpp @@ -31,6 +31,8 @@ #include #include +#include + #include #include #include diff --git a/libraries/app/include/graphene/app/application.hpp b/libraries/app/include/graphene/app/application.hpp index 6f55c390b..a313e2f8c 100644 --- a/libraries/app/include/graphene/app/application.hpp +++ b/libraries/app/include/graphene/app/application.hpp @@ -89,8 +89,13 @@ namespace graphene { namespace app { /// Emitted when syncing finishes (is_finished_syncing will return true) boost::signals2::signal syncing_finished; - private: void enable_plugin( const string& name ); + + bool is_plugin_enabled(const string& name) const; + + std::shared_ptr elasticsearch_thread; + + private: void add_available_plugin( std::shared_ptr p ); std::shared_ptr my; diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 98f91c590..7db8ccee4 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -27,7 +27,6 @@ #include #include #include -#include namespace graphene { namespace elasticsearch { @@ -60,6 +59,8 @@ class elasticsearch_plugin_impl std::string _elasticsearch_index_prefix = "bitshares-"; bool _elasticsearch_operation_object = false; uint32_t _elasticsearch_start_es_after_block = 0; + bool _elasticsearch_operation_string = true; + mode _elasticsearch_mode = mode::only_save; CURL *curl; // curl handler vector bulk_lines; // vector of op lines vector prepare; @@ -215,7 +216,14 @@ void elasticsearch_plugin_impl::doOperationHistory(const optional op_in_trx; os.operation_result = fc::json::to_string(oho->result); os.virtual_op = oho->virtual_op; - os.op = fc::json::to_string(oho->op); + + if(_elasticsearch_operation_object) { + oho->op.visit(fc::from_static_variant(os.op_object, FC_PACK_MAX_DEPTH)); + adaptor_struct adaptor; + os.op_object = adaptor.adapt(os.op_object.get_object()); + } + if(_elasticsearch_operation_string) + os.op = fc::json::to_string(oho->op); } void elasticsearch_plugin_impl::doBlock(const optional & oho, const signed_block& b) @@ -394,25 +402,32 @@ void elasticsearch_plugin::plugin_set_program_options( ) { cli.add_options() - ("elasticsearch-node-url", boost::program_options::value(), "Elastic Search database node url") - ("elasticsearch-bulk-replay", boost::program_options::value(), "Number of bulk documents to index on replay(5000)") - ("elasticsearch-bulk-sync", boost::program_options::value(), "Number of bulk documents to index on a syncronied chain(10)") - ("elasticsearch-visitor", boost::program_options::value(), "Use visitor to index additional data(slows down the replay)") - ("elasticsearch-basic-auth", boost::program_options::value(), "Pass basic auth to elasticsearch database ") - ("elasticsearch-index-prefix", boost::program_options::value(), "Add a prefix to the index(bitshares-)") - ("elasticsearch-operation-object", boost::program_options::value(), "Save operation as object(false)") - ("elasticsearch-start-es-after-block", boost::program_options::value(), "Start doing ES job after block(0)") + ("elasticsearch-node-url", boost::program_options::value(), + "Elastic Search database node url(http://localhost:9200/)") + ("elasticsearch-bulk-replay", boost::program_options::value(), + "Number of bulk documents to index on replay(10000)") + ("elasticsearch-bulk-sync", boost::program_options::value(), + "Number of bulk documents to index on a syncronied chain(100)") + ("elasticsearch-visitor", boost::program_options::value(), + "Use visitor to index additional data(slows down the replay(false))") + ("elasticsearch-basic-auth", boost::program_options::value(), + "Pass basic auth to elasticsearch database('')") + ("elasticsearch-index-prefix", boost::program_options::value(), + "Add a prefix to the index(bitshares-)") + ("elasticsearch-operation-object", boost::program_options::value(), + "Save operation as object(false)") + ("elasticsearch-start-es-after-block", boost::program_options::value(), + "Start doing ES job after block(0)") + ("elasticsearch-operation-string", boost::program_options::value(), + "Save operation as string. Needed to serve history api calls(true)") + ("elasticsearch-mode", boost::program_options::value(), + "Mode of operation: only_save(0), only_query(1), all(2) - Default: 0") ; cfg.add(cli); } void elasticsearch_plugin::plugin_initialize(const boost::program_options::variables_map& options) { - database().applied_block.connect( [&]( const signed_block& b) { - if (!my->update_account_histories(b)) - FC_THROW_EXCEPTION(fc::exception, "Error populating ES database, we are going to keep trying."); - } ); - my->_oho_index = database().add_index< primary_index< operation_history_index > >(); database().add_index< primary_index< account_transaction_history_index > >(); @@ -439,7 +454,28 @@ void elasticsearch_plugin::plugin_initialize(const boost::program_options::varia } if (options.count("elasticsearch-start-es-after-block")) { my->_elasticsearch_start_es_after_block = options["elasticsearch-start-es-after-block"].as(); - } + } + if (options.count("elasticsearch-operation-string")) { + my->_elasticsearch_operation_string = options["elasticsearch-operation-string"].as(); + } + if (options.count("elasticsearch-mode")) { + const auto option_number = options["elasticsearch-mode"].as(); + if(option_number > mode::all) + FC_THROW_EXCEPTION(fc::exception, "Elasticsearch mode not valid"); + my->_elasticsearch_mode = static_cast(options["elasticsearch-mode"].as()); + } + + if(my->_elasticsearch_mode != mode::only_query) { + if (my->_elasticsearch_mode == mode::all && !my->_elasticsearch_operation_string) + FC_THROW_EXCEPTION(fc::exception, + "If elasticsearch-mode is set to all then elasticsearch-operation-string need to be true"); + + database().applied_block.connect([this](const signed_block &b) { + if (!my->update_account_histories(b)) + FC_THROW_EXCEPTION(fc::exception, + "Error populating ES database, we are going to keep trying."); + }); + } } void elasticsearch_plugin::plugin_startup() @@ -454,4 +490,124 @@ void elasticsearch_plugin::plugin_startup() ilog("elasticsearch ACCOUNT HISTORY: plugin_startup() begin"); } +operation_history_object elasticsearch_plugin::get_operation_by_id(operation_history_id_type id) +{ + const string operation_id_string = std::string(object_id_type(id)); + + const string query = R"( + { + "query": { + "match": + { + "account_history.operation_id": )" + operation_id_string + R"(" + } + } + } + )"; + + auto es = prepareHistoryQuery(query); + const auto response = graphene::utilities::simpleQuery(es); + variant variant_response = fc::json::from_string(response); + const auto source = variant_response["hits"]["hits"][size_t(0)]["_source"]; + return fromEStoOperation(source); +} + +vector elasticsearch_plugin::get_account_history( + const account_id_type account_id, + operation_history_id_type stop = operation_history_id_type(), + unsigned limit = 100, + operation_history_id_type start = operation_history_id_type()) +{ + const string account_id_string = std::string(object_id_type(account_id)); + + const auto stop_number = stop.instance.value; + const auto start_number = start.instance.value; + + string range = ""; + if(stop_number == 0) + range = " AND operation_id_num: ["+fc::to_string(stop_number)+" TO "+fc::to_string(start_number)+"]"; + else if(stop_number > 0) + range = " AND operation_id_num: {"+fc::to_string(stop_number)+" TO "+fc::to_string(start_number)+"]"; + + const string query = R"( + { + "size": )" + fc::to_string(limit) + R"(, + "sort" : [{ "operation_id_num" : {"order" : "desc"}}], + "query": { + "bool": { + "must": [ + { + "query_string": { + "query": "account_history.account: )" + account_id_string + range + R"(" + } + } + ] + } + } + } + )"; + + auto es = prepareHistoryQuery(query); + + vector result; + + if(!graphene::utilities::checkES(es)) + return result; + + const auto response = graphene::utilities::simpleQuery(es); + variant variant_response = fc::json::from_string(response); + + const auto hits = variant_response["hits"]["total"]["value"]; + const auto size = std::min(static_cast(hits.as_uint64()), limit); + + for(unsigned i=0; i_elasticsearch_node_url; + es.index_prefix = my->_elasticsearch_index_prefix; + es.endpoint = es.index_prefix + "*/data/_search"; + es.query = query; + + return es; +} + +mode elasticsearch_plugin::get_running_mode() +{ + return my->_elasticsearch_mode; +} + + } } diff --git a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp index 19a488436..01a832448 100644 --- a/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp +++ b/libraries/plugins/elasticsearch/include/graphene/elasticsearch/elasticsearch_plugin.hpp @@ -26,6 +26,7 @@ #include #include #include +#include namespace graphene { namespace elasticsearch { using namespace chain; @@ -49,6 +50,8 @@ namespace detail class elasticsearch_plugin_impl; } +enum mode { only_save = 0 , only_query = 1, all = 2 }; + class elasticsearch_plugin : public graphene::app::plugin { public: @@ -63,10 +66,20 @@ class elasticsearch_plugin : public graphene::app::plugin virtual void plugin_initialize(const boost::program_options::variables_map& options) override; virtual void plugin_startup() override; + operation_history_object get_operation_by_id(operation_history_id_type id); + vector get_account_history(const account_id_type account_id, + operation_history_id_type stop, unsigned limit, operation_history_id_type start); + mode get_running_mode(); + friend class detail::elasticsearch_plugin_impl; std::unique_ptr my; + + private: + operation_history_object fromEStoOperation(variant source); + graphene::utilities::ES prepareHistoryQuery(string query); }; + struct operation_visitor { typedef void result_type; @@ -128,6 +141,7 @@ struct operation_history_struct { std::string operation_result; int virtual_op; std::string op; + variant op_object; }; struct block_struct { @@ -174,9 +188,99 @@ struct bulk_struct { optional additional_data; }; +struct adaptor_struct { + variant adapt(const variant_object& op) + { + fc::mutable_variant_object o(op); + vector keys_to_rename; + for (auto i = o.begin(); i != o.end(); ++i) + { + auto& element = (*i).value(); + if (element.is_object()) + { + const string& name = (*i).key(); + auto& vo = element.get_object(); + if (vo.contains(name.c_str())) + keys_to_rename.emplace_back(name); + element = adapt(vo); + } + else if (element.is_array()) + adapt(element.get_array()); + } + for (const auto& i : keys_to_rename) + { + string new_name = i + "_"; + o[new_name] = variant(o[i]); + o.erase(i); + } + + if (o.find("memo") != o.end()) + { + auto& memo = o["memo"]; + if (memo.is_string()) + { + o["memo_"] = o["memo"]; + o.erase("memo"); + } + else if (memo.is_object()) + { + fc::mutable_variant_object tmp(memo.get_object()); + if (tmp.find("nonce") != tmp.end()) + { + tmp["nonce"] = tmp["nonce"].as_string(); + o["memo"] = tmp; + } + } + } + if (o.find("new_parameters") != o.end()) + { + auto& tmp = o["new_parameters"]; + if (tmp.is_object()) + { + fc::mutable_variant_object tmp2(tmp.get_object()); + if (tmp2.find("current_fees") != tmp2.end()) + { + tmp2.erase("current_fees"); + o["new_parameters"] = tmp2; + } + } + } + if (o.find("owner") != o.end() && o["owner"].is_string()) + { + o["owner_"] = o["owner"].as_string(); + o.erase("owner"); + } + if (o.find("proposed_ops") != o.end()) + { + o["proposed_ops"] = fc::json::to_string(o["proposed_ops"]); + } + if (o.find("initializer") != o.end()) + { + o["initializer"] = fc::json::to_string(o["initializer"]); + } + + variant v; + fc::to_variant(o, v, FC_PACK_MAX_DEPTH); + return v; + } + + void adapt(fc::variants& v) + { + for (auto& array_element : v) + { + if (array_element.is_object()) + array_element = adapt(array_element.get_object()); + else if (array_element.is_array()) + adapt(array_element.get_array()); + else + array_element = array_element.as_string(); + } + } +}; } } //graphene::elasticsearch -FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op) ) +FC_REFLECT_ENUM( graphene::elasticsearch::mode, (only_save)(only_query)(all) ) +FC_REFLECT( graphene::elasticsearch::operation_history_struct, (trx_in_block)(op_in_trx)(operation_result)(virtual_op)(op)(op_object) ) FC_REFLECT( graphene::elasticsearch::block_struct, (block_num)(block_time)(trx_id) ) FC_REFLECT( graphene::elasticsearch::fee_struct, (asset)(amount) ) FC_REFLECT( graphene::elasticsearch::transfer_struct, (asset)(amount)(from)(to) ) diff --git a/tests/common/database_fixture.cpp b/tests/common/database_fixture.cpp index 8cd3dc400..14b10fa32 100644 --- a/tests/common/database_fixture.cpp +++ b/tests/common/database_fixture.cpp @@ -132,21 +132,26 @@ database_fixture::database_fixture() // app.initialize(); auto test_name = boost::unit_test::framework::current_test_case().p_name.value; - if(test_name == "elasticsearch_account_history" || test_name == "elasticsearch_suite") { + if(test_name == "elasticsearch_account_history" || test_name == "elasticsearch_suite" || + test_name == "elasticsearch_history_api") { auto esplugin = app.register_plugin(); esplugin->plugin_set_app(&app); options.insert(std::make_pair("elasticsearch-node-url", boost::program_options::variable_value(string("http://localhost:9200/"), false))); options.insert(std::make_pair("elasticsearch-bulk-replay", boost::program_options::variable_value(uint32_t(2), false))); options.insert(std::make_pair("elasticsearch-bulk-sync", boost::program_options::variable_value(uint32_t(2), false))); - options.insert(std::make_pair("elasticsearch-visitor", boost::program_options::variable_value(true, false))); - //options.insert(std::make_pair("elasticsearch-basic-auth", boost::program_options::variable_value(string("elastic:changeme"), false))); + options.insert(std::make_pair("elasticsearch-start-es-after-block", boost::program_options::variable_value(uint32_t(0), false))); + options.insert(std::make_pair("elasticsearch-visitor", boost::program_options::variable_value(false, false))); + options.insert(std::make_pair("elasticsearch-operation-object", boost::program_options::variable_value(true, false))); + options.insert(std::make_pair("elasticsearch-operation-string", boost::program_options::variable_value(true, false))); + options.insert(std::make_pair("elasticsearch-mode", boost::program_options::variable_value(uint16_t(2), false))); esplugin->plugin_initialize(options); esplugin->plugin_startup(); } else { auto ahplugin = app.register_plugin(); + app.enable_plugin("affiliate_stats"); ahplugin->plugin_set_app(&app); ahplugin->plugin_initialize(options); ahplugin->plugin_startup(); diff --git a/tests/elasticsearch/main.cpp b/tests/elasticsearch/main.cpp index 18674a3b8..fc4599354 100644 --- a/tests/elasticsearch/main.cpp +++ b/tests/elasticsearch/main.cpp @@ -27,6 +27,7 @@ #include #include +#include #include "../common/database_fixture.hpp" @@ -118,7 +119,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) { es.endpoint = index_name + "/data/2.9.12"; // we know last op is a transfer of amount 300 res = graphene::utilities::getEndPoint(es); j = fc::json::from_string(res); - auto last_transfer_amount = j["_source"]["additional_data"]["transfer_data"]["amount"].as_string(); + auto last_transfer_amount = j["_source"]["operation_history"]["op_object"]["amount_"]["amount"].as_string(); BOOST_CHECK_EQUAL(last_transfer_amount, "300"); } } @@ -210,4 +211,325 @@ BOOST_AUTO_TEST_CASE(elasticsearch_suite) { } } +BOOST_AUTO_TEST_CASE(elasticsearch_history_api) { + try { + CURL *curl; // curl handler + curl = curl_easy_init(); + + graphene::utilities::ES es; + es.curl = curl; + es.elasticsearch_url = "http://localhost:9200/"; + es.index_prefix = "bitshares-"; + + auto delete_account_history = graphene::utilities::deleteAll(es); + + generate_block(); + fc::usleep(fc::milliseconds(1000)); + + if(delete_account_history) { + + create_bitasset("USD", account_id_type()); // create op 0 + const account_object& dan = create_account("dan"); // create op 1 + create_bitasset("CNY", dan.id); // create op 2 + create_bitasset("BTC", account_id_type()); // create op 3 + create_bitasset("XMR", dan.id); // create op 4 + create_bitasset("EUR", account_id_type()); // create op 5 + create_bitasset("OIL", dan.id); // create op 6 + + generate_block(); + fc::usleep(fc::milliseconds(1000)); + + graphene::app::history_api hist_api(app); + app.enable_plugin("elasticsearch"); + + // f(A, 0, 4, 9) = { 5, 3, 1, 0 } + auto histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(9)); + + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u); + + // f(A, 0, 4, 6) = { 5, 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(6)); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u); + + // f(A, 0, 4, 5) = { 5, 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(5)); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u); + + // f(A, 0, 4, 4) = { 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(4)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u); + + // f(A, 0, 4, 3) = { 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(3)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u); + + // f(A, 0, 4, 2) = { 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(2)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u); + + // f(A, 0, 4, 1) = { 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type(1)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u); + + // f(A, 0, 4, 0) = { 5, 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 4, operation_history_id_type()); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 0u); + + // f(A, 1, 5, 9) = { 5, 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(9)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + + // f(A, 1, 5, 6) = { 5, 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(6)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + + // f(A, 1, 5, 5) = { 5, 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(5)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + + // f(A, 1, 5, 4) = { 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(4)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + + // f(A, 1, 5, 3) = { 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(3)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + + // f(A, 1, 5, 2) = { } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(2)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(A, 1, 5, 1) = { } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(1)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(A, 1, 5, 0) = { 5, 3 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(1), 5, operation_history_id_type(0)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + + // f(A, 0, 3, 9) = { 5, 3, 1 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(9)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(A, 0, 3, 6) = { 5, 3, 1 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(6)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(A, 0, 3, 5) = { 5, 3, 1 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(5)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(A, 0, 3, 4) = { 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(4)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u); + + // f(A, 0, 3, 3) = { 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(3)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 0u); + + // f(A, 0, 3, 2) = { 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(2)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u); + + // f(A, 0, 3, 1) = { 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type(1)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 0u); + + // f(A, 0, 3, 0) = { 5, 3, 1 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(), 3, operation_history_id_type()); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(B, 0, 4, 9) = { 6, 4, 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(9)); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u); + + // f(B, 0, 4, 6) = { 6, 4, 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(6)); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u); + + // f(B, 0, 4, 5) = { 4, 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(5)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(B, 0, 4, 4) = { 4, 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(4)); + BOOST_CHECK_EQUAL(histories.size(), 3u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 1u); + + // f(B, 0, 4, 3) = { 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(3)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + + // f(B, 0, 4, 2) = { 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(2)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 1u); + + // f(B, 0, 4, 1) = { 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type(1)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 1u); + + // f(B, 0, 4, 0) = { 6, 4, 2, 1 } + histories = hist_api.get_account_history("dan", operation_history_id_type(), 4, operation_history_id_type()); + BOOST_CHECK_EQUAL(histories.size(), 4u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 2u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u); + + // f(B, 2, 4, 9) = { 6, 4 } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(9)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + + // f(B, 2, 4, 6) = { 6, 4 } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(6)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + + // f(B, 2, 4, 5) = { 4 } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(5)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u); + + // f(B, 2, 4, 4) = { 4 } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(4)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 4u); + + // f(B, 2, 4, 3) = { } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(3)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(B, 2, 4, 2) = { } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(2)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(B, 2, 4, 1) = { } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(1)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(B, 2, 4, 0) = { 6, 4 } + histories = hist_api.get_account_history("dan", operation_history_id_type(2), 4, operation_history_id_type(0)); + BOOST_CHECK_EQUAL(histories.size(), 2u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 6u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 4u); + + // 0 limits + histories = hist_api.get_account_history("dan", operation_history_id_type(0), 0, operation_history_id_type(0)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(3), 0, operation_history_id_type(9)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // non existent account + histories = hist_api.get_account_history("1.2.18", operation_history_id_type(0), 4, operation_history_id_type(0)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // create a new account C = alice { 7 } + auto alice = create_account("alice"); + + generate_block(); + fc::usleep(fc::milliseconds(1000)); + + // f(C, 0, 4, 10) = { 7 } + histories = hist_api.get_account_history("alice", operation_history_id_type(0), 4, operation_history_id_type(10)); + BOOST_CHECK_EQUAL(histories.size(), 1u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 7u); + + // f(C, 8, 4, 10) = { } + histories = hist_api.get_account_history("alice", operation_history_id_type(8), 4, operation_history_id_type(10)); + BOOST_CHECK_EQUAL(histories.size(), 0u); + + // f(A, 0, 10, 0) = { 7, 5, 3, 1, 0 } + histories = hist_api.get_account_history("1.2.0", operation_history_id_type(0), 10, operation_history_id_type(0)); + BOOST_CHECK_EQUAL(histories.size(), 5u); + BOOST_CHECK_EQUAL(histories[0].id.instance(), 7u); + BOOST_CHECK_EQUAL(histories[1].id.instance(), 5u); + BOOST_CHECK_EQUAL(histories[2].id.instance(), 3u); + BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u); + BOOST_CHECK_EQUAL(histories[4].id.instance(), 0u); + } + } + catch (fc::exception &e) { + edump((e.to_detail_string())); + throw; + } +} BOOST_AUTO_TEST_SUITE_END() From bfa878f9b285571419cdfdb477d4ddea3b0867b2 Mon Sep 17 00:00:00 2001 From: gladcow Date: Fri, 13 Dec 2019 18:55:49 +0300 Subject: [PATCH 15/17] change ES index prefixes to Peerplays-specific --- .../plugins/elasticsearch/elasticsearch_plugin.cpp | 4 ++-- libraries/plugins/es_objects/es_objects.cpp | 4 ++-- tests/elasticsearch/main.cpp | 14 +++++++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 7db8ccee4..11eee3e15 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -56,7 +56,7 @@ class elasticsearch_plugin_impl uint32_t _elasticsearch_bulk_sync = 100; bool _elasticsearch_visitor = false; std::string _elasticsearch_basic_auth = ""; - std::string _elasticsearch_index_prefix = "bitshares-"; + std::string _elasticsearch_index_prefix = "peerplays-"; bool _elasticsearch_operation_object = false; uint32_t _elasticsearch_start_es_after_block = 0; bool _elasticsearch_operation_string = true; @@ -413,7 +413,7 @@ void elasticsearch_plugin::plugin_set_program_options( ("elasticsearch-basic-auth", boost::program_options::value(), "Pass basic auth to elasticsearch database('')") ("elasticsearch-index-prefix", boost::program_options::value(), - "Add a prefix to the index(bitshares-)") + "Add a prefix to the index(peerplays-)") ("elasticsearch-operation-object", boost::program_options::value(), "Save operation as object(false)") ("elasticsearch-start-es-after-block", boost::program_options::value(), diff --git a/libraries/plugins/es_objects/es_objects.cpp b/libraries/plugins/es_objects/es_objects.cpp index 5b3f29e7e..b9083cbc5 100644 --- a/libraries/plugins/es_objects/es_objects.cpp +++ b/libraries/plugins/es_objects/es_objects.cpp @@ -63,7 +63,7 @@ class es_objects_plugin_impl bool _es_objects_balances = true; bool _es_objects_limit_orders = true; bool _es_objects_asset_bitasset = true; - std::string _es_objects_index_prefix = "objects-"; + std::string _es_objects_index_prefix = "ppobjects-"; uint32_t _es_objects_start_es_after_block = 0; CURL *curl; // curl handler vector bulk; @@ -308,7 +308,7 @@ void es_objects_plugin::plugin_set_program_options( ("es-objects-balances", boost::program_options::value(), "Store balances objects(true)") ("es-objects-limit-orders", boost::program_options::value(), "Store limit order objects(true)") ("es-objects-asset-bitasset", boost::program_options::value(), "Store feed data(true)") - ("es-objects-index-prefix", boost::program_options::value(), "Add a prefix to the index(objects-)") + ("es-objects-index-prefix", boost::program_options::value(), "Add a prefix to the index(ppobjects-)") ("es-objects-keep-only-current", boost::program_options::value(), "Keep only current state of the objects(true)") ("es-objects-start-es-after-block", boost::program_options::value(), "Start doing ES job after block(0)") ; diff --git a/tests/elasticsearch/main.cpp b/tests/elasticsearch/main.cpp index fc4599354..28d3522ca 100644 --- a/tests/elasticsearch/main.cpp +++ b/tests/elasticsearch/main.cpp @@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) { graphene::utilities::ES es; es.curl = curl; es.elasticsearch_url = "http://localhost:9200/"; - es.index_prefix = "bitshares-"; + es.index_prefix = "peerplays-"; //es.auth = "elastic:changeme"; // delete all first @@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) { fc::usleep(fc::milliseconds(1000)); // for later use - //int asset_create_op_id = operation::tag::value; + //int asset_crobjeate_op_id = operation::tag::value; //int account_create_op_id = operation::tag::value; string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }"; @@ -114,7 +114,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) { // check the visitor data auto block_date = db.head_block_time(); - std::string index_name = graphene::utilities::generateIndexName(block_date, "bitshares-"); + std::string index_name = graphene::utilities::generateIndexName(block_date, "peerplays-"); es.endpoint = index_name + "/data/2.9.12"; // we know last op is a transfer of amount 300 res = graphene::utilities::getEndPoint(es); @@ -138,7 +138,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) { graphene::utilities::ES es; es.curl = curl; es.elasticsearch_url = "http://localhost:9200/"; - es.index_prefix = "objects-"; + es.index_prefix = "ppobjects-"; //es.auth = "elastic:changeme"; // delete all first @@ -193,10 +193,10 @@ BOOST_AUTO_TEST_CASE(elasticsearch_suite) { graphene::utilities::ES es; es.curl = curl; es.elasticsearch_url = "http://localhost:9200/"; - es.index_prefix = "bitshares-"; + es.index_prefix = "peerplays-"; auto delete_account_history = graphene::utilities::deleteAll(es); fc::usleep(fc::milliseconds(1000)); - es.index_prefix = "objects-"; + es.index_prefix = "ppobjects-"; auto delete_objects = graphene::utilities::deleteAll(es); fc::usleep(fc::milliseconds(1000)); @@ -219,7 +219,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_history_api) { graphene::utilities::ES es; es.curl = curl; es.elasticsearch_url = "http://localhost:9200/"; - es.index_prefix = "bitshares-"; + es.index_prefix = "peerplays-"; auto delete_account_history = graphene::utilities::deleteAll(es); From 0a904429660461ac07aa2a38a0e731a31728210f Mon Sep 17 00:00:00 2001 From: gladcow Date: Mon, 13 Jan 2020 15:35:18 +0300 Subject: [PATCH 16/17] fix the data writing to ES during sync issues --- .../elasticsearch/elasticsearch_plugin.cpp | 5 ++- programs/witness_node/genesis.json | 37 ++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp index 11eee3e15..96dbab7c6 100644 --- a/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp +++ b/libraries/plugins/elasticsearch/elasticsearch_plugin.cpp @@ -75,6 +75,7 @@ class elasticsearch_plugin_impl std::string bulk_line; std::string index_name; bool is_sync = false; + fc::time_point last_sync; private: bool add_elasticsearch( const account_id_type account_id, const optional& oho, const uint32_t block_number ); const account_transaction_history_object& addNewEntry(const account_statistics_object& stats_obj, @@ -192,10 +193,12 @@ bool elasticsearch_plugin_impl::update_account_histories( const signed_block& b void elasticsearch_plugin_impl::checkState(const fc::time_point_sec& block_time) { - if((fc::time_point::now() - block_time) < fc::seconds(30)) + fc::time_point current_time(fc::time_point::now()); + if(((current_time - block_time) < fc::seconds(30)) || (current_time - last_sync > fc::seconds(60))) { limit_documents = _elasticsearch_bulk_sync; is_sync = true; + last_sync = current_time; } else { diff --git a/programs/witness_node/genesis.json b/programs/witness_node/genesis.json index ab153e7bd..e07cec92b 100644 --- a/programs/witness_node/genesis.json +++ b/programs/witness_node/genesis.json @@ -1,5 +1,5 @@ { - "initial_timestamp": "2019-05-14T18:47:51", + "initial_timestamp": "2020-01-13T06:03:15", "max_core_supply": "1000000000000000", "initial_parameters": { "current_fees": { @@ -307,6 +307,27 @@ 75,{} ],[ 76,{} + ],[ + 77,{ + "lottery_asset": 2000000, + "price_per_kbyte": 10 + } + ],[ + 78,{ + "fee": 0 + } + ],[ + 79,{ + "fee": 0 + } + ],[ + 80,{ + "fee": 0 + } + ],[ + 81,{ + "fee": 2000000 + } ] ], "scale": 10000 @@ -352,7 +373,19 @@ "maximum_tournament_start_time_in_future": 2419200, "maximum_tournament_start_delay": 604800, "maximum_tournament_number_of_wins": 100, - "extensions": {} + "extensions": { + "min_bet_multiplier": 10100, + "max_bet_multiplier": 10000000, + "betting_rake_fee_percentage": 300, + "live_betting_delay_time": 5, + "sweeps_distribution_percentage": 200, + "sweeps_distribution_asset": "1.3.0", + "sweeps_vesting_accumulator_account": "1.2.0", + "gpos_period": 15552000, + "gpos_subperiod": 2592000, + "gpos_period_start": 1601528400, + "gpos_vesting_lockin_period": 2592000 + } }, "initial_bts_accounts": [], "initial_accounts": [{ From e55075ec6d5aeacfb9e69a43bead9d7d75dc159b Mon Sep 17 00:00:00 2001 From: gladcow Date: Fri, 17 Jan 2020 18:51:44 +0300 Subject: [PATCH 17/17] fix CLI tests --- libraries/app/application.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/app/application.cpp b/libraries/app/application.cpp index b20ffc4cc..adfd84022 100644 --- a/libraries/app/application.cpp +++ b/libraries/app/application.cpp @@ -991,6 +991,7 @@ void application::initialize(const fc::path& data_dir, const boost::program_opti wanted.push_back("witness"); wanted.push_back("account_history"); wanted.push_back("market_history"); + wanted.push_back("bookie"); } int es_ah_conflict_counter = 0; for (auto& it : wanted)