[Tests] Improve benchmark precision

CMakeLists.txt

@@ -593,3 +593,4 @@ target_link_libraries(pivxd ${sodium_LIBRARY_RELEASE} -ldl -lpthread)
 
 add_subdirectory(src/qt)
 add_subdirectory(src/test)
+add_subdirectory(src/bench)

src/Makefile.bench.include

@@ -41,7 +41,7 @@ bench_bench_pivx_LDADD = \
   $(LIBBITCOIN_SERVER) \
   $(LIBBITCOIN_WALLET) \
   $(LIBBITCOIN_COMMON) \
-  $(LIBBITCOIN_UNIVALUE) \
+  $(LIBUNIVALUE) \
   $(LIBBITCOIN_UTIL) \
   $(LIBBITCOIN_CRYPTO) \
   $(LIBLEVELDB) \

src/bench/CMakeLists.txt

@@ -0,0 +1,104 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 3.14)
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+
+set(Boost_USE_STATIC_LIBS ON)
+find_package(Boost COMPONENTS system filesystem thread unit_test_framework REQUIRED)
+
+set(RAW_TEST_FILES  ${CMAKE_CURRENT_SOURCE_DIR}/data/block2680960.raw)
+
+# Generate raw files
+function(GenerateRaws)
+    set(fileList "")
+    foreach(file IN LISTS ARGN)
+        get_filename_component(filename ${file} NAME_WE)
+        set(outFile ${file}.h)
+        set(runCmd ${CMAKE_SOURCE_DIR}/contrib/devtools/hexdump_util.sh)
+        add_custom_command(
+                OUTPUT ${outFile}
+                COMMAND ${CMAKE_COMMAND} -E echo "static unsigned const char ${filename}_raw[] = {" > ${outFile}
+                COMMAND ${runCmd} ${file} ${outFile}
+                COMMAND ${CMAKE_COMMAND} -E echo "};" >> ${outFile}
+                DEPENDS ${file}
+                COMMENT "Generating raw ${file}.h"
+                VERBATIM
+        )
+        list(APPEND fileList ${outFile})
+    endforeach()
+    install(FILES ${fileList} DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/data)
+    add_custom_target(
+            genRaws2 ALL
+            DEPENDS ${fileList}
+            COMMENT "Processing raw files..."
+    )
+endfunction()
+
+GenerateRaws(${RAW_TEST_FILES})
+
+set(BITCOIN_BENCH_SUITE
+        ${CMAKE_CURRENT_SOURCE_DIR}/bench_pivx.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/bench.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/bench.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/Examples.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/base58.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/bls.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/bls_dkg.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/checkblock.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/checkqueue.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/data.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/data.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/chacha20.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/crypto_hash.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/ecdsa.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/lockedpool.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/perf.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/perf.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/prevector.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/util_time.cpp
+        )
+
+set(bench_bench_pivx_SOURCES ${BITCOIN_BENCH_SUITE} ${BITCOIN_TESTS})
+add_executable(bench_pivx ${bench_bench_pivx_SOURCES} ${BitcoinHeaders})
+add_dependencies(bench_pivx genHeaders genRaws2 libunivalue libsecp256k1 libzcashrust leveldb crc32c bls)
+target_include_directories(bench_pivx PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
+        ${CMAKE_SOURCE_DIR}/src/leveldb
+        ${CMAKE_SOURCE_DIR}/src/leveldb/include
+        ${CMAKE_SOURCE_DIR}/src/leveldb/helpers/memenv
+        ${LIBEVENT_INCLUDE_DIR}
+        ${GMP_INCLUDE_DIR})
+target_link_libraries(bench_pivx PRIVATE
+        SERVER_A
+        CLI_A
+        WALLET_A
+        COMMON_A
+        univalue
+        ZEROCOIN_A
+        UTIL_A
+        SAPLING_A
+        BITCOIN_CRYPTO_A
+        leveldb
+        crc32c
+        secp256k1
+        rustzcash
+        bls
+        ${BerkeleyDB_LIBRARIES} ${Boost_LIBRARIES} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} ${LIBEVENT_LIB} ${GMP_LIBRARY} pthread
+        )
+if(ZMQ_FOUND)
+    target_link_libraries(bench_pivx PRIVATE ZMQ_A ${ZMQ_LIB})
+    target_include_directories(bench_pivx PRIVATE ${ZMQ_INCLUDE_DIR})
+endif()
+if(MINIUPNP_FOUND)
+    target_compile_definitions(bench_pivx PRIVATE "-DSTATICLIB -DMINIUPNP_STATICLIB")
+    target_link_libraries(bench_pivx PRIVATE ${MINIUPNP_LIBRARY})
+    target_include_directories(bench_pivx PRIVATE ${MINIUPNP_INCLUDE_DIR})
+endif()
+if(NAT-PMP_FOUND)
+    target_link_libraries(bench_pivx PRIVATE ${NAT-PMP_LIBRARY})
+    target_include_directories(bench_pivx PRIVATE ${NAT-PMP_INCLUDE_DIR})
+endif()
+
+if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    target_link_libraries(bench_pivx PRIVATE "-framework Cocoa")
+endif()
+
+target_link_libraries(bench_pivx PRIVATE ${sodium_LIBRARY_RELEASE} -ldl -lpthread)
+

src/bench/Examples.cpp

@@ -14,7 +14,7 @@ static void Sleep100ms(benchmark::State& state)
     }
 }
 
-BENCHMARK(Sleep100ms);
+BENCHMARK(Sleep100ms, 10);
 
 // Extremely fast-running benchmark:
 #include <math.h>
@@ -30,4 +30,4 @@ static void Trig(benchmark::State& state)
     }
 }
 
-BENCHMARK(Trig);
+BENCHMARK(Trig, 12 * 1000 * 1000);

src/bench/base58.cpp

@@ -52,6 +52,6 @@ static void Base58Decode(benchmark::State& state)
 }
 
 
-BENCHMARK(Base58Encode);
-BENCHMARK(Base58CheckEncode);
-BENCHMARK(Base58Decode);
+BENCHMARK(Base58Encode, 470 * 1000);
+BENCHMARK(Base58CheckEncode, 320 * 1000);
+BENCHMARK(Base58Decode, 800 * 1000);

src/bench/bench.cpp

@@ -8,98 +8,138 @@
 
 #include <assert.h>
 #include <iomanip>
+#include <algorithm>
+#include <regex>
+#include <numeric>
+
 #include <iostream>
 
-benchmark::BenchRunner::BenchmarkMap &benchmark::BenchRunner::benchmarks() {
-    static std::map<std::string, benchmark::BenchFunction> benchmarks_map;
-    return benchmarks_map;
+void benchmark::ConsolePrinter::header()
+{
+    std::cout << "# Benchmark, evals, iterations, total, min, max, median" << std::endl;
 }
 
-benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func)
+void benchmark::ConsolePrinter::result(const State& state)
 {
-    benchmarks().emplace(name, func);
+    auto results = state.m_elapsed_results;
+    std::sort(results.begin(), results.end());
+
+    double total = state.m_num_iters * std::accumulate(results.begin(), results.end(), 0.0);
+
+    double front = 0;
+    double back = 0;
+    double median = 0;
+
+    if (!results.empty()) {
+        front = results.front();
+        back = results.back();
+
+        size_t mid = results.size() / 2;
+        median = results[mid];
+        if (0 == results.size() % 2) {
+            median = (results[mid] + results[mid + 1]) / 2;
+        }
+    }
+
+    std::cout << std::setprecision(6);
+    std::cout << state.m_name << ", " << state.m_num_evals << ", " << state.m_num_iters << ", " << total << ", " << front << ", " << back << ", " << median << std::endl;
 }
 
-void
-benchmark::BenchRunner::RunAll(benchmark::duration elapsedTimeForOne)
+void benchmark::ConsolePrinter::footer() {}
+benchmark::PlotlyPrinter::PlotlyPrinter(std::string plotly_url, int64_t width, int64_t height)
+    : m_plotly_url(plotly_url), m_width(width), m_height(height)
 {
-    perf_init();
-    if (std::ratio_less_equal<benchmark::clock::period, std::micro>::value) {
-        std::cerr << "WARNING: Clock precision is worse than microsecond - benchmarks may be less accurate!\n";
-    }
-    std::cout << "#Benchmark" << "," << "count" << "," << "min(ns)" << "," << "max(ns)" << "," << "average(ns)" << ","
-              << "min_cycles" << "," << "max_cycles" << "," << "average_cycles" << "\n";
+}
 
-    for (const auto &p: benchmarks()) {
-        State state(p.first, elapsedTimeForOne);
-        p.second(state);
-    }
-    perf_fini();
+void benchmark::PlotlyPrinter::header()
+{
+    std::cout << "<html><head>"
+              << "<script src=\"" << m_plotly_url << "\"></script>"
+              << "</head><body><div id=\"myDiv\" style=\"width:" << m_width << "px; height:" << m_height << "px\"></div>"
+              << "<script> var data = ["
+              << std::endl;
 }
 
-bool benchmark::State::KeepRunning()
+void benchmark::PlotlyPrinter::result(const State& state)
 {
-    if (count & countMask) {
-      ++count;
-      return true;
+    std::cout << "{ " << std::endl
+              << "  name: '" << state.m_name << "', " << std::endl
+              << "  y: [";
+
+    const char* prefix = "";
+    for (const auto& e : state.m_elapsed_results) {
+        std::cout << prefix << std::setprecision(6) << e;
+        prefix = ", ";
     }
-    time_point now;
-    uint64_t nowCycles;
-    if (count == 0) {
-        beginTime = now = clock::now();
-        lastCycles = beginCycles = nowCycles = perf_cpucycles();
+    std::cout << "]," << std::endl
+              << "  boxpoints: 'all', jitter: 0.3, pointpos: 0, type: 'box',"
+              << std::endl
+              << "}," << std::endl;
+}
+
+void benchmark::PlotlyPrinter::footer()
+{
+    std::cout << "]; var layout = { showlegend: false, yaxis: { rangemode: 'tozero', autorange: true } };"
+              << "Plotly.newPlot('myDiv', data, layout);"
+              << "</script></body></html>";
+}
+
+
+benchmark::BenchRunner::BenchmarkMap& benchmark::BenchRunner::benchmarks()
+{
+    static std::map<std::string, Bench> benchmarks_map;
+    return benchmarks_map;
+}
+
+benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func, uint64_t num_iters_for_one_second)
+{
+    benchmarks().insert(std::make_pair(name, Bench{func, num_iters_for_one_second}));
+}
+
+void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double scaling, const std::string& filter, bool is_list_only)
+{
+    perf_init();
+    if (!std::ratio_less_equal<benchmark::clock::period, std::micro>::value) {
+        std::cerr << "WARNING: Clock precision is worse than microsecond - benchmarks may be less accurate!\n";
     }
-    else {
-        now = clock::now();
-        auto elapsed = now - lastTime;
-        auto elapsedOne = elapsed / (countMask + 1);
-        if (elapsedOne < minTime) minTime = elapsedOne;
-        if (elapsedOne > maxTime) maxTime = elapsedOne;
-
-        // We only use relative values, so don't have to handle 64-bit wrap-around specially
-        nowCycles = perf_cpucycles();
-        uint64_t elapsedOneCycles = (nowCycles - lastCycles) / (countMask + 1);
-        if (elapsedOneCycles < minCycles) minCycles = elapsedOneCycles;
-        if (elapsedOneCycles > maxCycles) maxCycles = elapsedOneCycles;
-
-        if (elapsed*128 < maxElapsed) {
-          // If the execution was much too fast (1/128th of maxElapsed), increase the count mask by 8x and restart timing.
-          // The restart avoids including the overhead of this code in the measurement.
-          countMask = ((countMask<<3)|7) & ((1LL<<60)-1);
-          count = 0;
-          minTime = duration::max();
-          maxTime = duration::zero();
-          minCycles = std::numeric_limits<uint64_t>::max();
-          maxCycles = std::numeric_limits<uint64_t>::min();
-          return true;
+
+    std::regex reFilter(filter);
+    std::smatch baseMatch;
+
+    printer.header();
+
+    for (const auto& p : benchmarks()) {
+        if (!std::regex_match(p.first, baseMatch, reFilter)) {
+            continue;
+        }
+
+        uint64_t num_iters = static_cast<uint64_t>(p.second.num_iters_for_one_second * scaling);
+        if (0 == num_iters) {
+            num_iters = 1;
         }
-        if (elapsed*16 < maxElapsed) {
-            uint64_t newCountMask = ((countMask<<1)|1) & ((1LL<<60)-1);
-            if ((count & newCountMask)==0) {
-                countMask = newCountMask;
-            }
+        State state(p.first, num_evals, num_iters, printer);
+        if (!is_list_only) {
+            p.second.func(state);
         }
+        printer.result(state);
     }
-    lastTime = now;
-    lastCycles = nowCycles;
-    ++count;
 
-    if (now - beginTime < maxElapsed) return true; // Keep going
+    printer.footer();
 
-    --count;
+    perf_fini();
+}
 
-    assert(count != 0 && "count == 0 => (now == 0 && beginTime == 0) => return above");
+bool benchmark::State::UpdateTimer(const benchmark::time_point current_time)
+{
+    if (m_start_time != time_point()) {
+        std::chrono::duration<double> diff = current_time - m_start_time;
+        m_elapsed_results.push_back(diff.count() / m_num_iters);
 
-    // Output results
-    // Duration casts are only necessary here because hardware with sub-nanosecond clocks
-    // will lose precision.
-    int64_t min_elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>(minTime).count();
-    int64_t max_elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>(maxTime).count();
-    int64_t avg_elapsed = std::chrono::duration_cast<std::chrono::nanoseconds>((now-beginTime)/count).count();
-    int64_t averageCycles = (nowCycles-beginCycles)/count;
-    std::cout << std::fixed << std::setprecision(15) << name << "," << count << "," << min_elapsed << "," << max_elapsed << "," << avg_elapsed << ","
-              << minCycles << "," << maxCycles << "," << averageCycles << "\n";
-    std::cout.copyfmt(std::ios(nullptr));
+        if (m_elapsed_results.size() == m_num_evals) {
+            return false;
+        }
+    }
 
-    return false;
+    m_num_iters_left = m_num_iters - 1;
+    return true;
 }