From b15c06f95d7a9658b1a85d2de62d5fdfd633380d Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Fri, 15 Jan 2016 04:34:02 +0000 Subject: [PATCH 1/6] Bugfix: The var is LIBUNIVALUE,not LIBBITCOIN_UNIVALUE --- src/Makefile.bench.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index b8b7fcbf4ef74..05477dac1c11d 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -41,7 +41,7 @@ bench_bench_pivx_LDADD = \ $(LIBBITCOIN_SERVER) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_UNIVALUE) \ + $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) \ $(LIBLEVELDB) \ From 7f8f03082fa4cd0584655666a5123df1e8cec239 Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 30 Aug 2021 13:01:41 -0300 Subject: [PATCH 2/6] Build: Add cmake support for benchmarking framework --- CMakeLists.txt | 1 + src/bench/CMakeLists.txt | 104 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/bench/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index e320cf9d1e556..d3466cac6cb78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -593,3 +593,4 @@ target_link_libraries(pivxd ${sodium_LIBRARY_RELEASE} -ldl -lpthread) add_subdirectory(src/qt) add_subdirectory(src/test) +add_subdirectory(src/bench) diff --git a/src/bench/CMakeLists.txt b/src/bench/CMakeLists.txt new file mode 100644 index 0000000000000..2208c80220d04 --- /dev/null +++ b/src/bench/CMakeLists.txt @@ -0,0 +1,104 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.14) +set(CMAKE_INCLUDE_CURRENT_DIR ON) + +set(Boost_USE_STATIC_LIBS ON) +find_package(Boost COMPONENTS system filesystem thread unit_test_framework REQUIRED) + +set(RAW_TEST_FILES ${CMAKE_CURRENT_SOURCE_DIR}/data/block2680960.raw) + +# Generate raw files +function(GenerateRaws) + set(fileList "") + foreach(file IN LISTS ARGN) + get_filename_component(filename ${file} NAME_WE) + set(outFile ${file}.h) + set(runCmd ${CMAKE_SOURCE_DIR}/contrib/devtools/hexdump_util.sh) + add_custom_command( + OUTPUT ${outFile} + COMMAND ${CMAKE_COMMAND} -E echo "static unsigned const char ${filename}_raw[] = {" > ${outFile} + COMMAND ${runCmd} ${file} ${outFile} + COMMAND ${CMAKE_COMMAND} -E echo "};" >> ${outFile} + DEPENDS ${file} + COMMENT "Generating raw ${file}.h" + VERBATIM + ) + list(APPEND fileList ${outFile}) + endforeach() + install(FILES ${fileList} DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/data) + add_custom_target( + genRaws2 ALL + DEPENDS ${fileList} + COMMENT "Processing raw files..." + ) +endfunction() + +GenerateRaws(${RAW_TEST_FILES}) + +set(BITCOIN_BENCH_SUITE + ${CMAKE_CURRENT_SOURCE_DIR}/bench_pivx.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/bench.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/bench.h + ${CMAKE_CURRENT_SOURCE_DIR}/Examples.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/base58.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/bls.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/bls_dkg.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/checkblock.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/checkqueue.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/data.h + ${CMAKE_CURRENT_SOURCE_DIR}/data.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/chacha20.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/crypto_hash.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ecdsa.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/lockedpool.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/perf.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/perf.h + ${CMAKE_CURRENT_SOURCE_DIR}/prevector.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/util_time.cpp + ) + +set(bench_bench_pivx_SOURCES ${BITCOIN_BENCH_SUITE} ${BITCOIN_TESTS}) +add_executable(bench_pivx ${bench_bench_pivx_SOURCES} ${BitcoinHeaders}) +add_dependencies(bench_pivx genHeaders genRaws2 libunivalue libsecp256k1 libzcashrust leveldb crc32c bls) +target_include_directories(bench_pivx PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/src/leveldb + ${CMAKE_SOURCE_DIR}/src/leveldb/include + ${CMAKE_SOURCE_DIR}/src/leveldb/helpers/memenv + ${LIBEVENT_INCLUDE_DIR} + ${GMP_INCLUDE_DIR}) +target_link_libraries(bench_pivx PRIVATE + SERVER_A + CLI_A + WALLET_A + COMMON_A + univalue + ZEROCOIN_A + UTIL_A + SAPLING_A + BITCOIN_CRYPTO_A + leveldb + crc32c + secp256k1 + rustzcash + bls + ${BerkeleyDB_LIBRARIES} ${Boost_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} ${LIBEVENT_LIB} ${GMP_LIBRARY} pthread + ) +if(ZMQ_FOUND) + target_link_libraries(bench_pivx PRIVATE ZMQ_A ${ZMQ_LIB}) + target_include_directories(bench_pivx PRIVATE ${ZMQ_INCLUDE_DIR}) +endif() +if(MINIUPNP_FOUND) + target_compile_definitions(bench_pivx PRIVATE "-DSTATICLIB -DMINIUPNP_STATICLIB") + target_link_libraries(bench_pivx PRIVATE ${MINIUPNP_LIBRARY}) + target_include_directories(bench_pivx PRIVATE ${MINIUPNP_INCLUDE_DIR}) +endif() +if(NAT-PMP_FOUND) + target_link_libraries(bench_pivx PRIVATE ${NAT-PMP_LIBRARY}) + target_include_directories(bench_pivx PRIVATE ${NAT-PMP_INCLUDE_DIR}) +endif() + +if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + target_link_libraries(bench_pivx PRIVATE "-framework Cocoa") +endif() + +target_link_libraries(bench_pivx PRIVATE ${sodium_LIBRARY_RELEASE} -ldl -lpthread) + From 17c4bcfcadb746623466120f8d2f037bb658d0c0 Mon Sep 17 00:00:00 2001 From: Martin Ankerl Date: Tue, 17 Oct 2017 16:48:02 +0200 Subject: [PATCH 3/6] Improved microbenchmarking with multiple features. * inline performance critical code * Average runtime is specified and used to calculate iterations. * Console: show median of multiple runs * plot: show box plot * filter benchmarks * specify scaling factor * ignore src/test and src/bench in command line check script * number of iterations instead of time * Replaced runtime in BENCHMARK makro number of iterations. * Added -? to bench_bitcoin * Benchmark plotly.js URL, width, height can be customized * Fixed incorrect precision warning --- src/bench/Examples.cpp | 4 +- src/bench/base58.cpp | 6 +- src/bench/bench.cpp | 184 +++++++++++++++++++++++--------------- src/bench/bench.h | 144 +++++++++++++++++++---------- src/bench/bench_pivx.cpp | 64 +++++++++++-- src/bench/bls.cpp | 24 ++--- src/bench/bls_dkg.cpp | 44 ++++----- src/bench/chacha20.cpp | 6 +- src/bench/checkblock.cpp | 4 +- src/bench/checkqueue.cpp | 4 +- src/bench/crypto_hash.cpp | 31 ++++--- src/bench/ecdsa.cpp | 6 +- src/bench/lockedpool.cpp | 3 +- src/bench/prevector.cpp | 22 ++--- src/bench/util_time.cpp | 8 +- 15 files changed, 348 insertions(+), 206 deletions(-) diff --git a/src/bench/Examples.cpp b/src/bench/Examples.cpp index 69942029e661f..a8e0e80d836c1 100644 --- a/src/bench/Examples.cpp +++ b/src/bench/Examples.cpp @@ -14,7 +14,7 @@ static void Sleep100ms(benchmark::State& state) } } -BENCHMARK(Sleep100ms); +BENCHMARK(Sleep100ms, 10); // Extremely fast-running benchmark: #include @@ -30,4 +30,4 @@ static void Trig(benchmark::State& state) } } -BENCHMARK(Trig); +BENCHMARK(Trig, 12 * 1000 * 1000); diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index c107ece5aaf1b..e7cf998b21061 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -52,6 +52,6 @@ static void Base58Decode(benchmark::State& state) } -BENCHMARK(Base58Encode); -BENCHMARK(Base58CheckEncode); -BENCHMARK(Base58Decode); +BENCHMARK(Base58Encode, 470 * 1000); +BENCHMARK(Base58CheckEncode, 320 * 1000); +BENCHMARK(Base58Decode, 800 * 1000); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 622b8d14ad7f8..1de61fd4d2ad8 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -8,98 +8,138 @@ #include #include +#include +#include +#include + #include -benchmark::BenchRunner::BenchmarkMap &benchmark::BenchRunner::benchmarks() { - static std::map benchmarks_map; - return benchmarks_map; +void benchmark::ConsolePrinter::header() +{ + std::cout << "# Benchmark, evals, iterations, total, min, max, median" << std::endl; } -benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func) +void benchmark::ConsolePrinter::result(const State& state) { - benchmarks().emplace(name, func); + auto results = state.m_elapsed_results; + std::sort(results.begin(), results.end()); + + double total = state.m_num_iters * std::accumulate(results.begin(), results.end(), 0.0); + + double front = 0; + double back = 0; + double median = 0; + + if (!results.empty()) { + front = results.front(); + back = results.back(); + + size_t mid = results.size() / 2; + median = results[mid]; + if (0 == results.size() % 2) { + median = (results[mid] + results[mid + 1]) / 2; + } + } + + std::cout << std::setprecision(6); + std::cout << state.m_name << ", " << state.m_num_evals << ", " << state.m_num_iters << ", " << total << ", " << front << ", " << back << ", " << median << std::endl; } -void -benchmark::BenchRunner::RunAll(benchmark::duration elapsedTimeForOne) +void benchmark::ConsolePrinter::footer() {} +benchmark::PlotlyPrinter::PlotlyPrinter(std::string plotly_url, int64_t width, int64_t height) + : m_plotly_url(plotly_url), m_width(width), m_height(height) { - perf_init(); - if (std::ratio_less_equal::value) { - std::cerr << "WARNING: Clock precision is worse than microsecond - benchmarks may be less accurate!\n"; - } - std::cout << "#Benchmark" << "," << "count" << "," << "min(ns)" << "," << "max(ns)" << "," << "average(ns)" << "," - << "min_cycles" << "," << "max_cycles" << "," << "average_cycles" << "\n"; +} - for (const auto &p: benchmarks()) { - State state(p.first, elapsedTimeForOne); - p.second(state); - } - perf_fini(); +void benchmark::PlotlyPrinter::header() +{ + std::cout << "" + << "" + << "
" + << ""; +} + + +benchmark::BenchRunner::BenchmarkMap& benchmark::BenchRunner::benchmarks() +{ + static std::map benchmarks_map; + return benchmarks_map; +} + +benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func, uint64_t num_iters_for_one_second) +{ + benchmarks().insert(std::make_pair(name, Bench{func, num_iters_for_one_second})); +} + +void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double scaling, const std::string& filter, bool is_list_only) +{ + perf_init(); + if (!std::ratio_less_equal::value) { + std::cerr << "WARNING: Clock precision is worse than microsecond - benchmarks may be less accurate!\n"; } - else { - now = clock::now(); - auto elapsed = now - lastTime; - auto elapsedOne = elapsed / (countMask + 1); - if (elapsedOne < minTime) minTime = elapsedOne; - if (elapsedOne > maxTime) maxTime = elapsedOne; - - // We only use relative values, so don't have to handle 64-bit wrap-around specially - nowCycles = perf_cpucycles(); - uint64_t elapsedOneCycles = (nowCycles - lastCycles) / (countMask + 1); - if (elapsedOneCycles < minCycles) minCycles = elapsedOneCycles; - if (elapsedOneCycles > maxCycles) maxCycles = elapsedOneCycles; - - if (elapsed*128 < maxElapsed) { - // If the execution was much too fast (1/128th of maxElapsed), increase the count mask by 8x and restart timing. - // The restart avoids including the overhead of this code in the measurement. - countMask = ((countMask<<3)|7) & ((1LL<<60)-1); - count = 0; - minTime = duration::max(); - maxTime = duration::zero(); - minCycles = std::numeric_limits::max(); - maxCycles = std::numeric_limits::min(); - return true; + + std::regex reFilter(filter); + std::smatch baseMatch; + + printer.header(); + + for (const auto& p : benchmarks()) { + if (!std::regex_match(p.first, baseMatch, reFilter)) { + continue; + } + + uint64_t num_iters = static_cast(p.second.num_iters_for_one_second * scaling); + if (0 == num_iters) { + num_iters = 1; } - if (elapsed*16 < maxElapsed) { - uint64_t newCountMask = ((countMask<<1)|1) & ((1LL<<60)-1); - if ((count & newCountMask)==0) { - countMask = newCountMask; - } + State state(p.first, num_evals, num_iters, printer); + if (!is_list_only) { + p.second.func(state); } + printer.result(state); } - lastTime = now; - lastCycles = nowCycles; - ++count; - if (now - beginTime < maxElapsed) return true; // Keep going + printer.footer(); - --count; + perf_fini(); +} - assert(count != 0 && "count == 0 => (now == 0 && beginTime == 0) => return above"); +bool benchmark::State::UpdateTimer(const benchmark::time_point current_time) +{ + if (m_start_time != time_point()) { + std::chrono::duration diff = current_time - m_start_time; + m_elapsed_results.push_back(diff.count() / m_num_iters); - // Output results - // Duration casts are only necessary here because hardware with sub-nanosecond clocks - // will lose precision. - int64_t min_elapsed = std::chrono::duration_cast(minTime).count(); - int64_t max_elapsed = std::chrono::duration_cast(maxTime).count(); - int64_t avg_elapsed = std::chrono::duration_cast((now-beginTime)/count).count(); - int64_t averageCycles = (nowCycles-beginCycles)/count; - std::cout << std::fixed << std::setprecision(15) << name << "," << count << "," << min_elapsed << "," << max_elapsed << "," << avg_elapsed << "," - << minCycles << "," << maxCycles << "," << averageCycles << "\n"; - std::cout.copyfmt(std::ios(nullptr)); + if (m_elapsed_results.size() == m_num_evals) { + return false; + } + } - return false; + m_num_iters_left = m_num_iters - 1; + return true; } diff --git a/src/bench/bench.h b/src/bench/bench.h index ea91ecd657040..c64b5ea99bf95 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include @@ -30,65 +32,111 @@ static void CODE_TO_TIME(benchmark::State& state) } ... do any cleanup needed... } -BENCHMARK(CODE_TO_TIME); - */ +// default to running benchmark for 5000 iterations +BENCHMARK(CODE_TO_TIME, 5000); + + */ namespace benchmark { +// In case high_resolution_clock is steady, prefer that, otherwise use steady_clock. +struct best_clock { + using hi_res_clock = std::chrono::high_resolution_clock; + using steady_clock = std::chrono::steady_clock; + using type = std::conditional::type; +}; +using clock = best_clock::type; +using time_point = clock::time_point; +using duration = clock::duration; - // In case high_resolution_clock is steady, prefer that, otherwise use steady_clock. - struct best_clock { - using hi_res_clock = std::chrono::high_resolution_clock; - using steady_clock = std::chrono::steady_clock; - using type = std::conditional::type; - }; - using clock = best_clock::type; - using time_point = clock::time_point; - using duration = clock::duration; - - class State { - std::string name; - duration maxElapsed; - time_point beginTime, lastTime; - duration minTime, maxTime; - uint64_t count; - uint64_t countMask; - uint64_t beginCycles; - uint64_t lastCycles; - uint64_t minCycles; - uint64_t maxCycles; - public: - State(std::string _name, duration _maxElapsed) : - name(_name), - maxElapsed(_maxElapsed), - minTime(duration::max()), - maxTime(duration::zero()), - count(0), - countMask(1), - beginCycles(0), - lastCycles(0), - minCycles(std::numeric_limits::max()), - maxCycles(std::numeric_limits::min()) { - } - bool KeepRunning(); - }; +class Printer; - typedef std::function BenchFunction; +class State +{ +public: + std::string m_name; + uint64_t m_num_iters_left; + const uint64_t m_num_iters; + const uint64_t m_num_evals; + std::vector m_elapsed_results; + time_point m_start_time; + + bool UpdateTimer(time_point finish_time); + + State(std::string name, uint64_t num_evals, double num_iters, Printer& printer) : m_name(name), m_num_iters_left(0), m_num_iters(num_iters), m_num_evals(num_evals) + { + } - class BenchRunner + inline bool KeepRunning() { - typedef std::map BenchmarkMap; - static BenchmarkMap &benchmarks(); + if (m_num_iters_left--) { + return true; + } - public: - BenchRunner(std::string name, BenchFunction func); + bool result = UpdateTimer(clock::now()); + // measure again so runtime of UpdateTimer is not included + m_start_time = clock::now(); + return result; + } +}; + +typedef std::function BenchFunction; - static void RunAll(duration elapsedTimeForOne = std::chrono::seconds(1)); +class BenchRunner +{ + struct Bench { + BenchFunction func; + uint64_t num_iters_for_one_second; }; + typedef std::map BenchmarkMap; + static BenchmarkMap& benchmarks(); + +public: + BenchRunner(std::string name, BenchFunction func, uint64_t num_iters_for_one_second); + + static void RunAll(Printer& printer, uint64_t num_evals, double scaling, const std::string& filter, bool is_list_only); +}; + +// interface to output benchmark results. +class Printer +{ +public: + virtual ~Printer() {} + virtual void header() = 0; + virtual void result(const State& state) = 0; + virtual void footer() = 0; +}; + +// default printer to console, shows min, max, median. +class ConsolePrinter : public Printer +{ +public: + void header(); + void result(const State& state); + void footer(); +}; + +// creates box plot with plotly.js +class PlotlyPrinter : public Printer +{ +public: + PlotlyPrinter(std::string plotly_url, int64_t width, int64_t height); + void header(); + void result(const State& state); + void footer(); + +private: + std::string m_plotly_url; + int64_t m_width; + int64_t m_height; +}; } -// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); -#define BENCHMARK(n) \ - benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n); + +// BENCHMARK(foo, num_iters_for_one_second) expands to: benchmark::BenchRunner bench_11foo("foo", num_iterations); +// Choose a num_iters_for_one_second that takes roughly 1 second. The goal is that all benchmarks should take approximately +// the same time, and scaling factor can be used that the total time is appropriate for your system. +#define BENCHMARK(n, num_iters_for_one_second) \ + benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n, (num_iters_for_one_second)); #endif // BITCOIN_BENCH_BENCH_H diff --git a/src/bench/bench_pivx.cpp b/src/bench/bench_pivx.cpp index 459fba9d38ce1..020bb19a93395 100644 --- a/src/bench/bench_pivx.cpp +++ b/src/bench/bench_pivx.cpp @@ -1,14 +1,27 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers -// Copyright (c) 2020 The PIVX developers +// Copyright (c) 2015-2016 The Bitcoin Core developers +// Copyright (c) 2021 The PIVX developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" +#include -#include "bls/bls_wrapper.h" -#include "key.h" -#include "random.h" -#include "util/system.h" +#include +#include +#include +#include +#include + +#include + +#include + +static const int64_t DEFAULT_BENCH_EVALUATIONS = 5; +static const char* DEFAULT_BENCH_FILTER = ".*"; +static const char* DEFAULT_BENCH_SCALING = "1.0"; +static const char* DEFAULT_BENCH_PRINTER = "console"; +static const char* DEFAULT_PLOT_PLOTLYURL = "https://cdn.plot.ly/plotly-latest.min.js"; +static const int64_t DEFAULT_PLOT_WIDTH = 1024; +static const int64_t DEFAULT_PLOT_HEIGHT = 768; void InitBLSTests(); void CleanupBLSTests(); @@ -16,6 +29,24 @@ void CleanupBLSDkgTests(); int main(int argc, char** argv) { + gArgs.ParseParameters(argc, argv); + + if (gArgs.IsArgSet("-?") || gArgs.IsArgSet("-h") || gArgs.IsArgSet("-help")) { + std::cout << HelpMessageGroup(_("Options:")) + << HelpMessageOpt("-?", _("Print this help message and exit")) + << HelpMessageOpt("-list", _("List benchmarks without executing them. Can be combined with -scaling and -filter")) + << HelpMessageOpt("-evals=", strprintf(_("Number of measurement evaluations to perform. (default: %u)"), DEFAULT_BENCH_EVALUATIONS)) + << HelpMessageOpt("-filter=", strprintf(_("Regular expression filter to select benchmark by name (default: %s)"), DEFAULT_BENCH_FILTER)) + << HelpMessageOpt("-scaling=", strprintf(_("Scaling factor for benchmark's runtime (default: %u)"), DEFAULT_BENCH_SCALING)) + << HelpMessageOpt("-printer=(console|plot)", strprintf(_("Choose printer format. console: print data to console. plot: Print results as HTML graph (default: %s)"), DEFAULT_BENCH_PRINTER)) + << HelpMessageOpt("-plot-plotlyurl=", strprintf(_("URL to use for plotly.js (default: %s)"), DEFAULT_PLOT_PLOTLYURL)) + << HelpMessageOpt("-plot-width=", strprintf(_("Plot width in pixel (default: %u)"), DEFAULT_PLOT_WIDTH)) + << HelpMessageOpt("-plot-height=", strprintf(_("Plot height in pixel (default: %u)"), DEFAULT_PLOT_HEIGHT)); + + return 0; + } + + RandomInit(); ECC_Start(); ECCVerifyHandle globalVerifyHandle; RandomInit(); @@ -24,7 +55,24 @@ int main(int argc, char** argv) SetupEnvironment(); g_logger->m_print_to_file = false; // don't want to write to debug.log file - benchmark::BenchRunner::RunAll(); + int64_t evaluations = gArgs.GetArg("-evals", DEFAULT_BENCH_EVALUATIONS); + std::string regex_filter = gArgs.GetArg("-filter", DEFAULT_BENCH_FILTER); + std::string scaling_str = gArgs.GetArg("-scaling", DEFAULT_BENCH_SCALING); + bool is_list_only = gArgs.GetBoolArg("-list", false); + + double scaling_factor = boost::lexical_cast(scaling_str); + + + std::unique_ptr printer(new benchmark::ConsolePrinter()); + std::string printer_arg = gArgs.GetArg("-printer", DEFAULT_BENCH_PRINTER); + if ("plot" == printer_arg) { + printer.reset(new benchmark::PlotlyPrinter( + gArgs.GetArg("-plot-plotlyurl", DEFAULT_PLOT_PLOTLYURL), + gArgs.GetArg("-plot-width", DEFAULT_PLOT_WIDTH), + gArgs.GetArg("-plot-height", DEFAULT_PLOT_HEIGHT))); + } + + benchmark::BenchRunner::RunAll(*printer, evaluations, scaling_factor, regex_filter, is_list_only); // need to be called before global destructors kick in (PoolAllocator is needed due to many BLSSecretKeys) CleanupBLSDkgTests(); diff --git a/src/bench/bls.cpp b/src/bench/bls.cpp index ed767cddc2178..321afce93679d 100644 --- a/src/bench/bls.cpp +++ b/src/bench/bls.cpp @@ -346,15 +346,15 @@ static void BLSVerify_BatchedParallel(benchmark::State& state) } } -BENCHMARK(BLSPubKeyAggregate_Normal) -BENCHMARK(BLSSecKeyAggregate_Normal) -BENCHMARK(BLSSign_Normal) -BENCHMARK(BLSVerify_Normal) -BENCHMARK(BLSVerify_LargeBlock1000) -BENCHMARK(BLSVerify_LargeBlockSelfAggregated1000) -BENCHMARK(BLSVerify_LargeBlockSelfAggregated10000) -BENCHMARK(BLSVerify_LargeAggregatedBlock1000) -BENCHMARK(BLSVerify_LargeAggregatedBlock10000) -BENCHMARK(BLSVerify_LargeAggregatedBlock1000PreVerified) -BENCHMARK(BLSVerify_Batched) -BENCHMARK(BLSVerify_BatchedParallel) +BENCHMARK(BLSPubKeyAggregate_Normal, 700 * 1000) +BENCHMARK(BLSSecKeyAggregate_Normal, 1300 * 1000) +BENCHMARK(BLSSign_Normal, 600) +BENCHMARK(BLSVerify_Normal, 350) +BENCHMARK(BLSVerify_LargeBlock1000, 1) +BENCHMARK(BLSVerify_LargeBlockSelfAggregated1000, 1) +BENCHMARK(BLSVerify_LargeBlockSelfAggregated10000, 1) +BENCHMARK(BLSVerify_LargeAggregatedBlock1000, 1) +BENCHMARK(BLSVerify_LargeAggregatedBlock10000, 1) +BENCHMARK(BLSVerify_LargeAggregatedBlock1000PreVerified, 7) +BENCHMARK(BLSVerify_Batched, 500) +BENCHMARK(BLSVerify_BatchedParallel, 1000) diff --git a/src/bench/bls_dkg.cpp b/src/bench/bls_dkg.cpp index 156886b339cf9..0e07b8fac4d59 100644 --- a/src/bench/bls_dkg.cpp +++ b/src/bench/bls_dkg.cpp @@ -140,45 +140,45 @@ void CleanupBLSDkgTests() -#define BENCH_BuildQuorumVerificationVectors(name, quorumSize, parallel) \ +#define BENCH_BuildQuorumVerificationVectors(name, quorumSize, parallel, num_iters_for_one_second) \ static void BLSDKG_BuildQuorumVerificationVectors_##name##_##quorumSize(benchmark::State& state) \ { \ InitIfNeeded(); \ dkg##quorumSize->Bench_BuildQuorumVerificationVectors(state, parallel); \ } \ - BENCHMARK(BLSDKG_BuildQuorumVerificationVectors_##name##_##quorumSize) + BENCHMARK(BLSDKG_BuildQuorumVerificationVectors_##name##_##quorumSize, num_iters_for_one_second) -BENCH_BuildQuorumVerificationVectors(simple, 10, false) -BENCH_BuildQuorumVerificationVectors(simple, 100, false) -BENCH_BuildQuorumVerificationVectors(simple, 400, false) -BENCH_BuildQuorumVerificationVectors(parallel, 10, true) -BENCH_BuildQuorumVerificationVectors(parallel, 100, true) -BENCH_BuildQuorumVerificationVectors(parallel, 400, true) +BENCH_BuildQuorumVerificationVectors(simple, 10, false, 11000) +BENCH_BuildQuorumVerificationVectors(simple, 100, false, 110) +BENCH_BuildQuorumVerificationVectors(simple, 400, false, 6) +BENCH_BuildQuorumVerificationVectors(parallel, 10, true, 12000) +BENCH_BuildQuorumVerificationVectors(parallel, 100, true, 170) +BENCH_BuildQuorumVerificationVectors(parallel, 400, true, 8) /////////////////////////////// -#define BENCH_VerifyContributionShares(name, quorumSize, invalidCount, parallel, aggregated) \ +#define BENCH_VerifyContributionShares(name, quorumSize, invalidCount, parallel, aggregated, num_iters_for_one_second) \ static void BLSDKG_VerifyContributionShares_##name##_##quorumSize(benchmark::State& state) \ { \ InitIfNeeded(); \ dkg##quorumSize->Bench_VerifyContributionShares(state, invalidCount, parallel, aggregated); \ } \ - BENCHMARK(BLSDKG_VerifyContributionShares_##name##_##quorumSize) + BENCHMARK(BLSDKG_VerifyContributionShares_##name##_##quorumSize, num_iters_for_one_second) -BENCH_VerifyContributionShares(simple, 10, 5, false, false) -BENCH_VerifyContributionShares(simple, 100, 5, false, false) -BENCH_VerifyContributionShares(simple, 400, 5, false, false) +BENCH_VerifyContributionShares(simple, 10, 5, false, false, 70) +BENCH_VerifyContributionShares(simple, 100, 5, false, false, 1) +BENCH_VerifyContributionShares(simple, 400, 5, false, false, 1) -BENCH_VerifyContributionShares(aggregated, 10, 5, false, true) -BENCH_VerifyContributionShares(aggregated, 100, 5, false, true) -BENCH_VerifyContributionShares(aggregated, 400, 5, false, true) +BENCH_VerifyContributionShares(aggregated, 10, 5, false, true, 70) +BENCH_VerifyContributionShares(aggregated, 100, 5, false, true, 2) +BENCH_VerifyContributionShares(aggregated, 400, 5, false, true, 1) -BENCH_VerifyContributionShares(parallel, 10, 5, true, false) -BENCH_VerifyContributionShares(parallel, 100, 5, true, false) -BENCH_VerifyContributionShares(parallel, 400, 5, true, false) +BENCH_VerifyContributionShares(parallel, 10, 5, true, false, 200) +BENCH_VerifyContributionShares(parallel, 100, 5, true, false, 2) +BENCH_VerifyContributionShares(parallel, 400, 5, true, false, 1) -BENCH_VerifyContributionShares(parallel_aggregated, 10, 5, true, true) -BENCH_VerifyContributionShares(parallel_aggregated, 100, 5, true, true) -BENCH_VerifyContributionShares(parallel_aggregated, 400, 5, true, true) +BENCH_VerifyContributionShares(parallel_aggregated, 10, 5, true, true, 150) +BENCH_VerifyContributionShares(parallel_aggregated, 100, 5, true, true, 4) +BENCH_VerifyContributionShares(parallel_aggregated, 400, 5, true, true, 1) diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp index 7c25adf9cbaf2..69d8c96ec0cbd 100644 --- a/src/bench/chacha20.cpp +++ b/src/bench/chacha20.cpp @@ -41,6 +41,6 @@ static void CHACHA20_1MB(benchmark::State& state) CHACHA20(state, BUFFER_SIZE_LARGE); } -BENCHMARK(CHACHA20_64BYTES); -BENCHMARK(CHACHA20_256BYTES); -BENCHMARK(CHACHA20_1MB); +BENCHMARK(CHACHA20_64BYTES, 500000); +BENCHMARK(CHACHA20_256BYTES, 250000); +BENCHMARK(CHACHA20_1MB, 340); diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 2516eabdcd898..5408749a23086 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -44,5 +44,5 @@ static void DeserializeAndCheckBlockTest(benchmark::State& state) } } -BENCHMARK(DeserializeBlockTest); -BENCHMARK(DeserializeAndCheckBlockTest); +BENCHMARK(DeserializeBlockTest, 130); +BENCHMARK(DeserializeAndCheckBlockTest, 160); diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 6a1bfacfdd016..e84710bcdc429 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -99,5 +99,5 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::State& state) tg.interrupt_all(); tg.join_all(); } -BENCHMARK(CCheckQueueSpeed); -BENCHMARK(CCheckQueueSpeedPrevectorJob); +BENCHMARK(CCheckQueueSpeed, 29000); +BENCHMARK(CCheckQueueSpeedPrevectorJob, 1400); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index b9a36b579e0a9..27c79a3777075 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -39,6 +39,16 @@ static void SHA256(benchmark::State& state) CSHA256().Write(in.data(), in.size()).Finalize(hash); } +static void SHA256_32b(benchmark::State& state) +{ + std::vector in(32,0); + while (state.KeepRunning()) { + CSHA256() + .Write(in.data(), in.size()) + .Finalize(in.data()); + } +} + static void SHA512(benchmark::State& state) { uint8_t hash[CSHA512::OUTPUT_SIZE]; @@ -52,9 +62,7 @@ static void FastRandom_32bit(benchmark::State& state) FastRandomContext rng(true); uint32_t x = 0; while (state.KeepRunning()) { - for (int i = 0; i < 1000000; i++) { - x += rng.rand32(); - } + x += rng.rand32(); } } @@ -63,16 +71,15 @@ static void FastRandom_1bit(benchmark::State& state) FastRandomContext rng(true); uint32_t x = 0; while (state.KeepRunning()) { - for (int i = 0; i < 1000000; i++) { - x += rng.randbool(); - } + x += rng.randbool(); } } -BENCHMARK(RIPEMD160); -BENCHMARK(SHA1); -BENCHMARK(SHA256); -BENCHMARK(SHA512); +BENCHMARK(RIPEMD160, 440); +BENCHMARK(SHA1, 570); +BENCHMARK(SHA256, 340); +BENCHMARK(SHA512, 330); -BENCHMARK(FastRandom_32bit); -BENCHMARK(FastRandom_1bit); +BENCHMARK(SHA256_32b, 4700 * 1000); +BENCHMARK(FastRandom_32bit, 110 * 1000 * 1000); +BENCHMARK(FastRandom_1bit, 440 * 1000 * 1000); diff --git a/src/bench/ecdsa.cpp b/src/bench/ecdsa.cpp index 706608eac208e..310f2cf2be633 100644 --- a/src/bench/ecdsa.cpp +++ b/src/bench/ecdsa.cpp @@ -72,6 +72,6 @@ static void ECDSAVerify_LargeBlock(benchmark::State& state) } } -BENCHMARK(ECDSASign) -BENCHMARK(ECDSAVerify) -BENCHMARK(ECDSAVerify_LargeBlock) +BENCHMARK(ECDSASign, 22 * 1000) +BENCHMARK(ECDSAVerify, 15 * 1000) +BENCHMARK(ECDSAVerify_LargeBlock, 15) diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp index 5df5b1ac6e640..725422fe32b87 100644 --- a/src/bench/lockedpool.cpp +++ b/src/bench/lockedpool.cpp @@ -43,5 +43,4 @@ static void LockedPool(benchmark::State& state) addr.clear(); } -BENCHMARK(LockedPool); - +BENCHMARK(LockedPool, 530); diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index 75bfd77730569..96f269e04cdd5 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -75,16 +75,16 @@ void PrevectorResize(benchmark::State& state) } } -#define PREVECTOR_TEST(name) \ +#define PREVECTOR_TEST(name, nontrivops, trivops) \ static void Prevector ## name ## Nontrivial(benchmark::State& state) { \ - PrevectorResize(state); \ - } \ - BENCHMARK(Prevector ## name ## Nontrivial); \ - static void Prevector ## name ## Trivial(benchmark::State& state) { \ - PrevectorResize(state); \ - } \ - BENCHMARK(Prevector ## name ## Trivial); + PrevectorResize(state); \ + } \ + BENCHMARK(Prevector ## name ## Nontrivial, nontrivops); \ + static void Prevector ## name ## Trivial(benchmark::State& state) { \ + PrevectorResize(state); \ + } \ + BENCHMARK(Prevector ## name ## Trivial, trivops); -PREVECTOR_TEST(Clear) -PREVECTOR_TEST(Destructor) -PREVECTOR_TEST(Resize) +PREVECTOR_TEST(Clear, 28300, 88600) +PREVECTOR_TEST(Destructor, 28800, 88900) +PREVECTOR_TEST(Resize, 28900, 90300) diff --git a/src/bench/util_time.cpp b/src/bench/util_time.cpp index fd3569c5f0cfd..467ccbffbeda8 100644 --- a/src/bench/util_time.cpp +++ b/src/bench/util_time.cpp @@ -36,7 +36,7 @@ static void BenchTimeMillisSys(benchmark::State& state) } } -BENCHMARK(BenchTimeDeprecated/*, 100000000*/); -BENCHMARK(BenchTimeMillis/*, 6000000*/); -BENCHMARK(BenchTimeMillisSys/*, 6000000*/); -BENCHMARK(BenchTimeMock/*, 300000000*/); +BENCHMARK(BenchTimeDeprecated, 100000000); +BENCHMARK(BenchTimeMillis, 6000000); +BENCHMARK(BenchTimeMillisSys, 6000000); +BENCHMARK(BenchTimeMock, 300000000); From 0519e228f50e28ccb3764c290ee20d2498575855 Mon Sep 17 00:00:00 2001 From: Martin Ankerl Date: Wed, 20 Dec 2017 19:53:28 +0100 Subject: [PATCH 4/6] Removed CCheckQueueSpeed benchmark This benchmark's runtime was rather unpredictive on different machines, not really a useful benchmark. --- src/bench/checkqueue.cpp | 43 +--------------------------------------- 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index e84710bcdc429..d57b6d207d564 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -12,51 +12,11 @@ #include -// This Benchmark tests the CheckQueue with the lightest -// weight Checks, so it should make any lock contention -// particularly visible static const int MIN_CORES = 2; static const size_t BATCHES = 101; static const size_t BATCH_SIZE = 30; static const int PREVECTOR_SIZE = 28; -static const int QUEUE_BATCH_SIZE = 128; -static void CCheckQueueSpeed(benchmark::State& state) -{ - struct FakeJobNoWork { - bool operator()() - { - return true; - } - void swap(FakeJobNoWork& x){}; - }; - CCheckQueue queue {QUEUE_BATCH_SIZE}; - boost::thread_group tg; - for (auto x = 0; x < std::max(MIN_CORES, GetNumCores()); ++x) { - tg.create_thread([&]{queue.Thread();}); - } - while (state.KeepRunning()) { - CCheckQueueControl control(&queue); - - // We call Add a number of times to simulate the behavior of adding - // a block of transactions at once. - - std::vector> vBatches(BATCHES); - for (auto& vChecks : vBatches) { - vChecks.resize(BATCH_SIZE); - } - for (auto& vChecks : vBatches) { - // We can't make vChecks in the inner loop because we want to measure - // the cost of getting the memory to each thread and we might get the same - // memory - control.Add(vChecks); - } - // control waits for completion by RAII, but - // it is done explicitly here for clarity - control.Wait(); - } - tg.interrupt_all(); - tg.join_all(); -} +static const unsigned int QUEUE_BATCH_SIZE = 128; // This Benchmark tests the CheckQueue with a slightly realistic workload, // where checks all contain a prevector that is indirect 50% of the time @@ -99,5 +59,4 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::State& state) tg.interrupt_all(); tg.join_all(); } -BENCHMARK(CCheckQueueSpeed, 29000); BENCHMARK(CCheckQueueSpeedPrevectorJob, 1400); From 0c97fbe23a56a9d1179fac94a17b8ce2f53a5295 Mon Sep 17 00:00:00 2001 From: Nikolay Mitev Date: Fri, 20 Jul 2018 09:48:49 +0300 Subject: [PATCH 5/6] trivial: Replace CPubKey::operator[] with CPubKey::vch where possible --- src/pubkey.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 3ee8cc0926a5f..317bc7e54c233 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -172,7 +172,7 @@ bool CPubKey::Verify(const uint256& hash, const std::vector& vchS return false; secp256k1_pubkey pubkey; secp256k1_ecdsa_signature sig; - if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, &(*this)[0], size())) { + if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { return false; } if (!ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig, vchSig.data(), vchSig.size())) { @@ -210,7 +210,7 @@ bool CPubKey::IsFullyValid() const if (!IsValid()) return false; secp256k1_pubkey pubkey; - return secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, &(*this)[0], size()); + return secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size()); } bool CPubKey::Decompress() @@ -218,7 +218,7 @@ bool CPubKey::Decompress() if (!IsValid()) return false; secp256k1_pubkey pubkey; - if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, &(*this)[0], size())) { + if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { return false; } unsigned char pub[PUBLIC_KEY_SIZE]; @@ -237,7 +237,7 @@ bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChi BIP32Hash(cc, nChild, *begin(), begin()+1, out); memcpy(ccChild.begin(), out+32, 32); secp256k1_pubkey pubkey; - if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, &(*this)[0], size())) { + if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { return false; } if (!secp256k1_ec_pubkey_tweak_add(secp256k1_context_verify, &pubkey, out)) { From a5fcdacf6aea7164e3854585c5c5b5a739a1770b Mon Sep 17 00:00:00 2001 From: practicalswift Date: Sun, 27 Oct 2019 21:46:50 +0000 Subject: [PATCH 6/6] pubkey: Assert CPubKey's ECCVerifyHandle precondition --- src/pubkey.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 317bc7e54c233..a21e85a27abc0 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -172,6 +172,7 @@ bool CPubKey::Verify(const uint256& hash, const std::vector& vchS return false; secp256k1_pubkey pubkey; secp256k1_ecdsa_signature sig; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!secp256k1_ec_pubkey_parse(secp256k1_context_verify, &pubkey, vch, size())) { return false; } @@ -192,6 +193,7 @@ bool CPubKey::RecoverCompact(const uint256& hash, const std::vector& vchSig) { secp256k1_ecdsa_signature sig; + assert(secp256k1_context_verify && "secp256k1_context_verify must be initialized to use CPubKey."); if (!ecdsa_signature_parse_der_lax(secp256k1_context_verify, &sig, vchSig.data(), vchSig.size())) { return false; }