Fix Github Actions es_test failing issue

.github/workflows/build-and-test.ubuntu-debug.yml

@@ -94,6 +94,9 @@ jobs:
       run: |
         _build/tests/app_test -l test_suite
         df -h
+        curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
+          -d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
+        echo
         _build/tests/es_test -l test_suite
         df -h
         libraries/fc/tests/run-parallel-tests.sh _build/tests/chain_test -l test_suite

.github/workflows/build-and-test.ubuntu-release.yml

@@ -73,6 +73,9 @@ jobs:
     - name: Unit-Tests
       run: |
         _build/tests/app_test -l test_suite
+        curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
+          -d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
+        echo
         _build/tests/es_test -l test_suite
         libraries/fc/tests/run-parallel-tests.sh _build/tests/chain_test -l test_suite
         _build/tests/cli_test -l test_suite

.github/workflows/sonar-scan.yml

@@ -118,6 +118,9 @@ jobs:
       run: |
         _build/tests/app_test -l test_suite
         df -h
+        curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
+          -d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
+        echo
         _build/tests/es_test -l test_suite
         df -h
         libraries/fc/tests/run-parallel-tests.sh _build/tests/chain_test -l test_suite

tests/app/main.cpp

@@ -42,28 +42,16 @@
 
 #include "../../libraries/app/application_impl.hxx"
 
-#define BOOST_TEST_MODULE Test Application
-#include <boost/test/included/unit_test.hpp>
-
+#include "../common/init_unit_test_suite.hpp"
 #include "../common/genesis_file_util.hpp"
+#include "../common/utils.hpp"
 
 using namespace graphene;
 namespace bpo = boost::program_options;
 
 namespace fc {
    extern std::unordered_map<std::string, logger> &get_logger_map();
    extern std::unordered_map<std::string, appender::ptr> &get_appender_map();
-
-   /** Waits for F() to return true before max_duration has passed.
-    */
-   template<typename Functor>
-   static void wait_for( const fc::microseconds max_duration, const Functor&& f )
-   {
-      const auto start = fc::time_point::now();
-      while( !f() && fc::time_point::now() < start + max_duration )
-         fc::usleep(fc::milliseconds(100));
-      BOOST_REQUIRE( f() );
-   }
 }
 
 BOOST_AUTO_TEST_CASE(load_configuration_options_test_config_logging_files_created)

tests/cli/main.cpp

@@ -60,8 +60,7 @@
 
 #include <boost/filesystem/path.hpp>
 
-#define BOOST_TEST_MODULE Test Application
-#include <boost/test/included/unit_test.hpp>
+#include "../common/init_unit_test_suite.hpp"
 
 /*****
  * Global Initialization for Windows

tests/common/database_fixture.cpp

@@ -54,7 +54,8 @@
 
 using namespace graphene::chain::test;
 
-uint32_t GRAPHENE_TESTING_GENESIS_TIMESTAMP = 1431700000;
+extern uint32_t    GRAPHENE_TESTING_GENESIS_TIMESTAMP;
+extern std::string GRAPHENE_TESTING_ES_URL;
 
 namespace graphene { namespace chain {
 
@@ -292,14 +293,22 @@ database_fixture::database_fixture(const fc::time_point_sec &initial_timestamp)
          current_test_name == "elasticsearch_history_api") {
       auto esplugin = app.register_plugin<graphene::elasticsearch::elasticsearch_plugin>(true);
 
-      options.insert(std::make_pair("elasticsearch-node-url", boost::program_options::variable_value(string("http://localhost:9200/"), false)));
-      options.insert(std::make_pair("elasticsearch-bulk-replay", boost::program_options::variable_value(uint32_t(2), false)));
-      options.insert(std::make_pair("elasticsearch-bulk-sync", boost::program_options::variable_value(uint32_t(2), false)));
-      options.insert(std::make_pair("elasticsearch-start-es-after-block", boost::program_options::variable_value(uint32_t(0), false)));
-      options.insert(std::make_pair("elasticsearch-visitor", boost::program_options::variable_value(false, false)));
-      options.insert(std::make_pair("elasticsearch-operation-object", boost::program_options::variable_value(true, false)));
-      options.insert(std::make_pair("elasticsearch-operation-string", boost::program_options::variable_value(true, false)));
-      options.insert(std::make_pair("elasticsearch-mode", boost::program_options::variable_value(uint16_t(2), false)));
+      options.insert(std::make_pair("elasticsearch-node-url",
+            boost::program_options::variable_value(GRAPHENE_TESTING_ES_URL, false)));
+      options.insert(std::make_pair("elasticsearch-bulk-replay",
+            boost::program_options::variable_value(uint32_t(2), false)));
+      options.insert(std::make_pair("elasticsearch-bulk-sync",
+            boost::program_options::variable_value(uint32_t(2), false)));
+      options.insert(std::make_pair("elasticsearch-start-es-after-block",
+            boost::program_options::variable_value(uint32_t(0), false)));
+      options.insert(std::make_pair("elasticsearch-visitor",
+            boost::program_options::variable_value(false, false)));
+      options.insert(std::make_pair("elasticsearch-operation-object",
+            boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("elasticsearch-operation-string",
+            boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("elasticsearch-mode",
+            boost::program_options::variable_value(uint16_t(2), false)));
 
       es_index_prefix = string("bitshares-") + fc::to_string(uint64_t(rand())) + "-";
       BOOST_TEST_MESSAGE( string("ES index prefix is ") + es_index_prefix );
@@ -319,15 +328,24 @@ database_fixture::database_fixture(const fc::time_point_sec &initial_timestamp)
    if(current_test_name == "elasticsearch_objects" || current_test_name == "elasticsearch_suite") {
       auto esobjects_plugin = app.register_plugin<graphene::es_objects::es_objects_plugin>(true);
 
-      options.insert(std::make_pair("es-objects-elasticsearch-url", boost::program_options::variable_value(string("http://localhost:9200/"), false)));
-      options.insert(std::make_pair("es-objects-bulk-replay", boost::program_options::variable_value(uint32_t(2), false)));
-      options.insert(std::make_pair("es-objects-bulk-sync", boost::program_options::variable_value(uint32_t(2), false)));
-      options.insert(std::make_pair("es-objects-proposals", boost::program_options::variable_value(true, false)));
-      options.insert(std::make_pair("es-objects-accounts", boost::program_options::variable_value(true, false)));
-      options.insert(std::make_pair("es-objects-assets", boost::program_options::variable_value(true, false)));
-      options.insert(std::make_pair("es-objects-balances", boost::program_options::variable_value(true, false)));
-      options.insert(std::make_pair("es-objects-limit-orders", boost::program_options::variable_value(true, false)));
-      options.insert(std::make_pair("es-objects-asset-bitasset", boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("es-objects-elasticsearch-url",
+            boost::program_options::variable_value(GRAPHENE_TESTING_ES_URL, false)));
+      options.insert(std::make_pair("es-objects-bulk-replay",
+            boost::program_options::variable_value(uint32_t(2), false)));
+      options.insert(std::make_pair("es-objects-bulk-sync",
+            boost::program_options::variable_value(uint32_t(2), false)));
+      options.insert(std::make_pair("es-objects-proposals",
+            boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("es-objects-accounts",
+            boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("es-objects-assets",
+            boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("es-objects-balances",
+            boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("es-objects-limit-orders",
+            boost::program_options::variable_value(true, false)));
+      options.insert(std::make_pair("es-objects-asset-bitasset",
+            boost::program_options::variable_value(true, false)));
 
       es_obj_index_prefix = string("objects-") + fc::to_string(uint64_t(rand())) + "-";
       BOOST_TEST_MESSAGE( string("ES_OBJ index prefix is ") + es_obj_index_prefix );
@@ -401,6 +419,41 @@ database_fixture::~database_fixture()
    } catch (...) {
       BOOST_FAIL( "Uncaught exception in ~database_fixture" );
    }
+
+   // cleanup data in ES
+   if( !es_index_prefix.empty() || !es_obj_index_prefix.empty() )
+   {
+      CURL *curl; // curl handler
+      curl = curl_easy_init();
+      curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
+
+      graphene::utilities::ES es;
+      es.curl = curl;
+      es.elasticsearch_url = GRAPHENE_TESTING_ES_URL;
+
+      if( !es_index_prefix.empty() )
+      {
+         es.index_prefix = es_index_prefix;
+         // delete all
+         try {
+            graphene::utilities::deleteAll(es);
+         } catch (...) {
+            // nothing to do
+         }
+      }
+
+      if( !es_obj_index_prefix.empty() )
+      {
+         es.index_prefix = es_obj_index_prefix;
+         // delete all
+         try {
+            graphene::utilities::deleteAll(es);
+         } catch (...) {
+            // nothing to do
+         }
+      }
+   }
+
 } 
 
 void database_fixture::vote_for_committee_and_witnesses(uint16_t num_committee, uint16_t num_witness)

tests/common/init_unit_test_suite.hpp

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2015 Cryptonomex, Inc., and contributors.
+ *
+ * The MIT License
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include <cstdlib>
+#include <iostream>
+#include <boost/test/included/unit_test.hpp>
+#include <chrono>
+#include <string>
+
+uint32_t    GRAPHENE_TESTING_GENESIS_TIMESTAMP = 1431700000;
+std::string GRAPHENE_TESTING_ES_URL            = "http://127.0.0.1:9200/";
+
+boost::unit_test::test_suite* init_unit_test_suite(int argc, char* argv[]) {
+   const auto seed = std::chrono::high_resolution_clock::now().time_since_epoch().count();
+   std::srand( seed );
+   std::cout << "Random number generator seeded to " << seed << std::endl;
+   const char* genesis_timestamp_str = getenv("GRAPHENE_TESTING_GENESIS_TIMESTAMP");
+   if( genesis_timestamp_str != nullptr )
+   {
+      GRAPHENE_TESTING_GENESIS_TIMESTAMP = std::stoul( genesis_timestamp_str );
+   }
+   std::cout << "GRAPHENE_TESTING_GENESIS_TIMESTAMP is " << GRAPHENE_TESTING_GENESIS_TIMESTAMP << std::endl;
+   const char* env_es_url = getenv("GRAPHENE_TESTING_ES_URL");
+   if( env_es_url != nullptr )
+   {
+      std::string tmp_es_url( env_es_url );
+      if( tmp_es_url.substr(0, 7) == "http://" || tmp_es_url.substr(0, 8) == "https://" )
+         GRAPHENE_TESTING_ES_URL = tmp_es_url;
+   }
+   std::cout << "GRAPHENE_TESTING_ES_URL is " << GRAPHENE_TESTING_ES_URL << std::endl;
+   return nullptr;
+}

tests/common/utils.hpp

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2015 Cryptonomex, Inc., and contributors.
+ *
+ * The MIT License
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <fc/thread/thread.hpp>
+#include <boost/test/included/unit_test.hpp>
+
+namespace fc {
+
+   /** Waits for F() to return true before max_duration has passed.
+    */
+   template<typename Functor>
+   static void wait_for( const fc::microseconds max_duration, const Functor&& f )
+   {
+      const auto start = fc::time_point::now();
+      while( !f() && fc::time_point::now() < start + max_duration )
+         fc::usleep(fc::milliseconds(100));
+      BOOST_REQUIRE( f() );
+   }
+}