Skip to content

Commit

Permalink
Merge pull request #2373 from bitshares/fix-es-test-concurrency
Browse files Browse the repository at this point in the history
Fix Github Actions es_test failing issue
  • Loading branch information
abitmore authored Mar 6, 2021
2 parents db6a715 + d8c9131 commit 9fb62c4
Show file tree
Hide file tree
Showing 11 changed files with 279 additions and 148 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/build-and-test.ubuntu-debug.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,9 @@ jobs:
run: |
_build/tests/app_test -l test_suite
df -h
curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
-d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
echo
_build/tests/es_test -l test_suite
df -h
libraries/fc/tests/run-parallel-tests.sh _build/tests/chain_test -l test_suite
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/build-and-test.ubuntu-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ jobs:
- name: Unit-Tests
run: |
_build/tests/app_test -l test_suite
curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
-d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
echo
_build/tests/es_test -l test_suite
libraries/fc/tests/run-parallel-tests.sh _build/tests/chain_test -l test_suite
_build/tests/cli_test -l test_suite
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/sonar-scan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ jobs:
run: |
_build/tests/app_test -l test_suite
df -h
curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
-d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
echo
_build/tests/es_test -l test_suite
df -h
libraries/fc/tests/run-parallel-tests.sh _build/tests/chain_test -l test_suite
Expand Down
16 changes: 2 additions & 14 deletions tests/app/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -42,28 +42,16 @@

#include "../../libraries/app/application_impl.hxx"

#define BOOST_TEST_MODULE Test Application
#include <boost/test/included/unit_test.hpp>

#include "../common/init_unit_test_suite.hpp"
#include "../common/genesis_file_util.hpp"
#include "../common/utils.hpp"

using namespace graphene;
namespace bpo = boost::program_options;

namespace fc {
extern std::unordered_map<std::string, logger> &get_logger_map();
extern std::unordered_map<std::string, appender::ptr> &get_appender_map();

/** Waits for F() to return true before max_duration has passed.
*/
template<typename Functor>
static void wait_for( const fc::microseconds max_duration, const Functor&& f )
{
const auto start = fc::time_point::now();
while( !f() && fc::time_point::now() < start + max_duration )
fc::usleep(fc::milliseconds(100));
BOOST_REQUIRE( f() );
}
}

BOOST_AUTO_TEST_CASE(load_configuration_options_test_config_logging_files_created)
Expand Down
3 changes: 1 addition & 2 deletions tests/cli/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,7 @@

#include <boost/filesystem/path.hpp>

#define BOOST_TEST_MODULE Test Application
#include <boost/test/included/unit_test.hpp>
#include "../common/init_unit_test_suite.hpp"

/*****
* Global Initialization for Windows
Expand Down
89 changes: 71 additions & 18 deletions tests/common/database_fixture.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@

using namespace graphene::chain::test;

uint32_t GRAPHENE_TESTING_GENESIS_TIMESTAMP = 1431700000;
extern uint32_t GRAPHENE_TESTING_GENESIS_TIMESTAMP;
extern std::string GRAPHENE_TESTING_ES_URL;

namespace graphene { namespace chain {

Expand Down Expand Up @@ -292,14 +293,22 @@ database_fixture::database_fixture(const fc::time_point_sec &initial_timestamp)
current_test_name == "elasticsearch_history_api") {
auto esplugin = app.register_plugin<graphene::elasticsearch::elasticsearch_plugin>(true);

options.insert(std::make_pair("elasticsearch-node-url", boost::program_options::variable_value(string("http://localhost:9200/"), false)));
options.insert(std::make_pair("elasticsearch-bulk-replay", boost::program_options::variable_value(uint32_t(2), false)));
options.insert(std::make_pair("elasticsearch-bulk-sync", boost::program_options::variable_value(uint32_t(2), false)));
options.insert(std::make_pair("elasticsearch-start-es-after-block", boost::program_options::variable_value(uint32_t(0), false)));
options.insert(std::make_pair("elasticsearch-visitor", boost::program_options::variable_value(false, false)));
options.insert(std::make_pair("elasticsearch-operation-object", boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("elasticsearch-operation-string", boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("elasticsearch-mode", boost::program_options::variable_value(uint16_t(2), false)));
options.insert(std::make_pair("elasticsearch-node-url",
boost::program_options::variable_value(GRAPHENE_TESTING_ES_URL, false)));
options.insert(std::make_pair("elasticsearch-bulk-replay",
boost::program_options::variable_value(uint32_t(2), false)));
options.insert(std::make_pair("elasticsearch-bulk-sync",
boost::program_options::variable_value(uint32_t(2), false)));
options.insert(std::make_pair("elasticsearch-start-es-after-block",
boost::program_options::variable_value(uint32_t(0), false)));
options.insert(std::make_pair("elasticsearch-visitor",
boost::program_options::variable_value(false, false)));
options.insert(std::make_pair("elasticsearch-operation-object",
boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("elasticsearch-operation-string",
boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("elasticsearch-mode",
boost::program_options::variable_value(uint16_t(2), false)));

es_index_prefix = string("bitshares-") + fc::to_string(uint64_t(rand())) + "-";
BOOST_TEST_MESSAGE( string("ES index prefix is ") + es_index_prefix );
Expand All @@ -319,15 +328,24 @@ database_fixture::database_fixture(const fc::time_point_sec &initial_timestamp)
if(current_test_name == "elasticsearch_objects" || current_test_name == "elasticsearch_suite") {
auto esobjects_plugin = app.register_plugin<graphene::es_objects::es_objects_plugin>(true);

options.insert(std::make_pair("es-objects-elasticsearch-url", boost::program_options::variable_value(string("http://localhost:9200/"), false)));
options.insert(std::make_pair("es-objects-bulk-replay", boost::program_options::variable_value(uint32_t(2), false)));
options.insert(std::make_pair("es-objects-bulk-sync", boost::program_options::variable_value(uint32_t(2), false)));
options.insert(std::make_pair("es-objects-proposals", boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("es-objects-accounts", boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("es-objects-assets", boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("es-objects-balances", boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("es-objects-limit-orders", boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("es-objects-asset-bitasset", boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("es-objects-elasticsearch-url",
boost::program_options::variable_value(GRAPHENE_TESTING_ES_URL, false)));
options.insert(std::make_pair("es-objects-bulk-replay",
boost::program_options::variable_value(uint32_t(2), false)));
options.insert(std::make_pair("es-objects-bulk-sync",
boost::program_options::variable_value(uint32_t(2), false)));
options.insert(std::make_pair("es-objects-proposals",
boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("es-objects-accounts",
boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("es-objects-assets",
boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("es-objects-balances",
boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("es-objects-limit-orders",
boost::program_options::variable_value(true, false)));
options.insert(std::make_pair("es-objects-asset-bitasset",
boost::program_options::variable_value(true, false)));

es_obj_index_prefix = string("objects-") + fc::to_string(uint64_t(rand())) + "-";
BOOST_TEST_MESSAGE( string("ES_OBJ index prefix is ") + es_obj_index_prefix );
Expand Down Expand Up @@ -401,6 +419,41 @@ database_fixture::~database_fixture()
} catch (...) {
BOOST_FAIL( "Uncaught exception in ~database_fixture" );
}

// cleanup data in ES
if( !es_index_prefix.empty() || !es_obj_index_prefix.empty() )
{
CURL *curl; // curl handler
curl = curl_easy_init();
curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);

graphene::utilities::ES es;
es.curl = curl;
es.elasticsearch_url = GRAPHENE_TESTING_ES_URL;

if( !es_index_prefix.empty() )
{
es.index_prefix = es_index_prefix;
// delete all
try {
graphene::utilities::deleteAll(es);
} catch (...) {
// nothing to do
}
}

if( !es_obj_index_prefix.empty() )
{
es.index_prefix = es_obj_index_prefix;
// delete all
try {
graphene::utilities::deleteAll(es);
} catch (...) {
// nothing to do
}
}
}

}

void database_fixture::vote_for_committee_and_witnesses(uint16_t num_committee, uint16_t num_witness)
Expand Down
52 changes: 52 additions & 0 deletions tests/common/init_unit_test_suite.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
* Copyright (c) 2015 Cryptonomex, Inc., and contributors.
*
* The MIT License
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#include <cstdlib>
#include <iostream>
#include <boost/test/included/unit_test.hpp>
#include <chrono>
#include <string>

uint32_t GRAPHENE_TESTING_GENESIS_TIMESTAMP = 1431700000;
std::string GRAPHENE_TESTING_ES_URL = "http://127.0.0.1:9200/";

boost::unit_test::test_suite* init_unit_test_suite(int argc, char* argv[]) {
const auto seed = std::chrono::high_resolution_clock::now().time_since_epoch().count();
std::srand( seed );
std::cout << "Random number generator seeded to " << seed << std::endl;
const char* genesis_timestamp_str = getenv("GRAPHENE_TESTING_GENESIS_TIMESTAMP");
if( genesis_timestamp_str != nullptr )
{
GRAPHENE_TESTING_GENESIS_TIMESTAMP = std::stoul( genesis_timestamp_str );
}
std::cout << "GRAPHENE_TESTING_GENESIS_TIMESTAMP is " << GRAPHENE_TESTING_GENESIS_TIMESTAMP << std::endl;
const char* env_es_url = getenv("GRAPHENE_TESTING_ES_URL");
if( env_es_url != nullptr )
{
std::string tmp_es_url( env_es_url );
if( tmp_es_url.substr(0, 7) == "http://" || tmp_es_url.substr(0, 8) == "https://" )
GRAPHENE_TESTING_ES_URL = tmp_es_url;
}
std::cout << "GRAPHENE_TESTING_ES_URL is " << GRAPHENE_TESTING_ES_URL << std::endl;
return nullptr;
}
40 changes: 40 additions & 0 deletions tests/common/utils.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright (c) 2015 Cryptonomex, Inc., and contributors.
*
* The MIT License
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/

#include <fc/thread/thread.hpp>
#include <boost/test/included/unit_test.hpp>

namespace fc {

/** Waits for F() to return true before max_duration has passed.
*/
template<typename Functor>
static void wait_for( const fc::microseconds max_duration, const Functor&& f )
{
const auto start = fc::time_point::now();
while( !f() && fc::time_point::now() < start + max_duration )
fc::usleep(fc::milliseconds(100));
BOOST_REQUIRE( f() );
}
}
Loading

0 comments on commit 9fb62c4

Please sign in to comment.