[Refactor] Update RNG code from upstream

CMakeLists.txt

@@ -431,6 +431,7 @@ set(UTIL_SOURCES
         ./src/fs.cpp
         ./src/logging.cpp
         ./src/random.cpp
+        ./src/randomenv.cpp
         ./src/rpc/protocol.cpp
         ./src/sync.cpp
         ./src/threadinterrupt.cpp

build-aux/m4/l_socket.m4

@@ -0,0 +1,36 @@
+# Illumos/SmartOS requires linking with -lsocket if
+# using getifaddrs & freeifaddrs
+
+m4_define([_CHECK_SOCKET_testbody], [[
+  #include <sys/types.h>
+  #include <ifaddrs.h>
+
+  int main() {
+    struct ifaddrs *ifaddr;
+    getifaddrs(&ifaddr);
+    freeifaddrs(ifaddr);
+  }
+]])
+
+AC_DEFUN([CHECK_SOCKET], [
+
+  AC_LANG_PUSH(C++)
+
+  AC_MSG_CHECKING([whether ifaddrs funcs can be used without link library])
+
+  AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_SOCKET_testbody])],[
+      AC_MSG_RESULT([yes])
+    ],[
+      AC_MSG_RESULT([no])
+      LIBS="$LIBS -lsocket"
+      AC_MSG_CHECKING([whether getifaddrs needs -lsocket])
+      AC_LINK_IFELSE([AC_LANG_SOURCE([_CHECK_SOCKET_testbody])],[
+          AC_MSG_RESULT([yes])
+        ],[
+          AC_MSG_RESULT([no])
+          AC_MSG_FAILURE([cannot figure out how to use getifaddrs])
+        ])
+    ])
+
+  AC_LANG_POP
+])

configure.ac

@@ -749,8 +749,12 @@ if test x$TARGET_OS = xdarwin; then
   AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"])
 fi
 
-AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h])
+AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h])
 
+AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],,
+    [#include <sys/types.h>
+    #include <ifaddrs.h>]
+)
 AC_CHECK_DECLS([strnlen])
 
 # Check for daemon(3), unrelated to --with-daemon (although used by it)
@@ -888,6 +892,18 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h>
  [ AC_MSG_RESULT(no)]
 )
 
+AC_MSG_CHECKING(for sysctl)
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h>
+  #include <sys/sysctl.h>]],
+ [[ static const int name[2] = {CTL_KERN, KERN_VERSION};
+    #ifdef __linux__
+    #error "Don't use sysctl on Linux, it's deprecated even when it works"
+    #endif
+    sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])],
+ [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL, 1,[Define this symbol if the BSD sysctl() is available]) ],
+ [ AC_MSG_RESULT(no)]
+)
+
 AC_MSG_CHECKING(for sysctl KERN_ARND)
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h>
   #include <sys/sysctl.h>]],

contrib/devtools/test_deterministic_coverage.sh

@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# Test for deterministic coverage across unit test runs.
+
+export LC_ALL=C
+
+# Use GCOV_EXECUTABLE="gcov" if compiling with gcc.
+# Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang.
+GCOV_EXECUTABLE="gcov"
+
+# Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism.
+NON_DETERMINISTIC_TESTS=(
+    "dummy_tests"                                # We currently don't have any unit tests that have determinism issues
+)
+
+TEST_BITCOIN_BINARY="src/test/test_pivx"
+
+print_usage() {
+    echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]"
+}
+
+N_TEST_RUNS=2
+BOOST_TEST_RUN_FILTERS=""
+if [[ $# != 0 ]]; then
+    if [[ $1 == "--help" ]]; then
+        print_usage
+        exit
+    fi
+    PARSED_ARGUMENTS=0
+    if [[ $1 =~ [a-z] ]]; then
+        BOOST_TEST_RUN_FILTERS=$1
+        PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1))
+        shift
+    fi
+    if [[ $1 =~ ^[0-9]+$ ]]; then
+        N_TEST_RUNS=$1
+        PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1))
+        shift
+    fi
+    if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then
+        print_usage
+        exit
+    fi
+fi
+if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then
+    BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')"
+else
+    echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}"
+    echo
+fi
+
+if ! command -v gcov > /dev/null; then
+    echo "Error: gcov not installed. Exiting."
+    exit 1
+fi
+
+if ! command -v gcovr > /dev/null; then
+    echo "Error: gcovr not installed. Exiting."
+    exit 1
+fi
+
+if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then
+    echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"./configure --enable-lcov\" and compile."
+    exit 1
+fi
+
+get_file_suffix_count() {
+    find src/ -type f -name "*.$1" | wc -l
+}
+
+if [[ $(get_file_suffix_count gcno) == 0 ]]; then
+    echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"./configure --enable-lcov\" and re-compile."
+    exit 1
+fi
+
+get_covr_filename() {
+    echo "gcovr.run-$1.txt"
+}
+
+TEST_RUN_ID=0
+while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do
+    TEST_RUN_ID=$((TEST_RUN_ID + 1))
+    echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}"
+    find src/ -type f -name "*.gcda" -exec rm {} \;
+    if [[ $(get_file_suffix_count gcda) != 0 ]]; then
+        echo "Error: Stale *.gcda files found. Exiting."
+        exit 1
+    fi
+    TEST_OUTPUT_TEMPFILE=$(mktemp)
+    if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_BITCOIN_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then
+        cat "${TEST_OUTPUT_TEMPFILE}"
+        rm "${TEST_OUTPUT_TEMPFILE}"
+        exit 1
+    fi
+    rm "${TEST_OUTPUT_TEMPFILE}"
+    if [[ $(get_file_suffix_count gcda) == 0 ]]; then
+        echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"./configure --enable-lcov\" and re-compile."
+        exit 1
+    fi
+    GCOVR_TEMPFILE=$(mktemp)
+    if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then
+        echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting."
+        exit 1
+    fi
+    GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID})
+    mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}"
+    if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then
+        echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)."
+        exit 1
+    fi
+    if [[ ${TEST_RUN_ID} != 1 ]]; then
+        COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}")
+        if [[ ${COVERAGE_DIFF} != "" ]]; then
+            echo
+            echo "The line coverage is non-deterministic between runs. Exiting."
+            echo
+            echo "The test suite must be deterministic in the sense that the set of lines executed at least"
+            echo "once must be identical between runs. This is a necessary condition for meaningful"
+            echo "coverage measuring."
+            echo
+            echo "${COVERAGE_DIFF}"
+            exit 1
+        fi
+        rm "${GCOVR_FILENAME}"
+    fi
+done
+
+echo
+echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs."
+exit

src/Makefile.am

@@ -165,6 +165,7 @@ BITCOIN_CORE_H = \
   coins.h \
   compat.h \
   compat/byteswap.h \
+  compat/cpuid.h \
   compat/endian.h \
   compat/sanity.h \
   compressor.h \
@@ -239,6 +240,7 @@ BITCOIN_CORE_H = \
   protocol.h \
   pubkey.h \
   random.h \
+  randomenv.h \
   reverselock.h \
   reverse_iterate.h \
   rpc/client.h \
@@ -538,6 +540,7 @@ libbitcoin_util_a_SOURCES = \
   interfaces/handler.cpp \
   logging.cpp \
   random.cpp \
+  randomenv.cpp \
   rpc/protocol.cpp \
   support/cleanse.cpp \
   support/lockedpool.cpp \

src/Makefile.bench.include

@@ -15,6 +15,7 @@ bench_bench_pivx_SOURCES = \
   bench/base58.cpp \
   bench/checkblock.cpp \
   bench/checkqueue.cpp \
+  bench/chacha20.cpp \
   bench/crypto_hash.cpp \
   bench/lockedpool.cpp \
   bench/perf.cpp \

src/addrman.cpp

@@ -219,7 +219,7 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime
         return;
 
     // find a bucket it is in now
-    int nRnd = RandomInt(ADDRMAN_NEW_BUCKET_COUNT);
+    int nRnd = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT);
     int nUBucket = -1;
     for (unsigned int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) {
         int nB = (n + nRnd) % ADDRMAN_NEW_BUCKET_COUNT;
@@ -295,7 +295,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP
         int nFactor = 1;
         for (int n = 0; n < pinfo->nRefCount; n++)
             nFactor *= 2;
-        if (nFactor > 1 && (RandomInt(nFactor) != 0))
+        if (nFactor > 1 && (insecure_rand.randrange(nFactor) != 0))
             return false;
     } else {
         pinfo = Create(addr, source, &nId);
@@ -359,37 +359,37 @@ CAddrInfo CAddrMan::Select_(bool newOnly)
         return CAddrInfo();
 
     // Use a 50% chance for choosing between tried and new table entries.
-    if (!newOnly && (nTried > 0 && (nNew == 0 || RandomInt(2) == 0))) {
+    if (!newOnly && (nTried > 0 && (nNew == 0 || insecure_rand.randbool() == 0))) {
         // use a tried node
         double fChanceFactor = 1.0;
         while (1) {
-            int nKBucket = RandomInt(ADDRMAN_TRIED_BUCKET_COUNT);
-            int nKBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE);
+            int nKBucket = insecure_rand.randrange(ADDRMAN_TRIED_BUCKET_COUNT);
+            int nKBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE);
             while (vvTried[nKBucket][nKBucketPos] == -1) {
                 nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT;
                 nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE;
             }
             int nId = vvTried[nKBucket][nKBucketPos];
             assert(mapInfo.count(nId) == 1);
             CAddrInfo& info = mapInfo[nId];
-            if (RandomInt(1 << 30) < fChanceFactor * info.GetChance() * (1 << 30))
+            if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30))
                 return info;
             fChanceFactor *= 1.2;
         }
     } else {
         // use a new node
         double fChanceFactor = 1.0;
         while (1) {
-            int nUBucket = RandomInt(ADDRMAN_NEW_BUCKET_COUNT);
-            int nUBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE);
+            int nUBucket = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT);
+            int nUBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE);
             while (vvNew[nUBucket][nUBucketPos] == -1) {
                 nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT;
                 nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE;
             }
             int nId = vvNew[nUBucket][nUBucketPos];
             assert(mapInfo.count(nId) == 1);
             CAddrInfo& info = mapInfo[nId];
-            if (RandomInt(1 << 30) < fChanceFactor * info.GetChance() * (1 << 30))
+            if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30))
                 return info;
             fChanceFactor *= 1.2;
         }
@@ -485,7 +485,7 @@ void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr)
         if (vAddr.size() >= nNodes)
             break;
 
-        int nRndPos = RandomInt(vRandom.size() - n) + n;
+        int nRndPos = insecure_rand.randrange(vRandom.size() - n) + n;
         SwapRandom(n, nRndPos);
         assert(mapInfo.count(vRandom[n]) == 1);
 
@@ -533,10 +533,6 @@ void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices)
     info.nServices = nServices;
 }
 
-int CAddrMan::RandomInt(int nMax){
-    return GetRandInt(nMax);
-}
-
 void CAddrMan::ResolveCollisions_()
 {
     for (std::set<int>::iterator it = m_tried_collisions.begin(); it != m_tried_collisions.end();) {
@@ -603,7 +599,7 @@ CAddrInfo CAddrMan::SelectTriedCollision_()
     std::set<int>::iterator it = m_tried_collisions.begin();
 
     // Selects a random element from m_tried_collisions
-    std::advance(it, GetRandInt(m_tried_collisions.size()));
+    std::advance(it, insecure_rand.randrange(m_tried_collisions.size()));
     int id_new = *it;
 
     // If id_new not found in mapInfo remove it from m_tried_collisions

src/addrman.h

@@ -269,9 +269,6 @@ class CAddrMan
     //! Return a random to-be-evicted tried table address.
     CAddrInfo SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs);
 
-    //! Wraps GetRandInt to allow tests to override RandomInt and make it determinismistic.
-    virtual int RandomInt(int nMax);
-
 #ifdef DEBUG_ADDRMAN
     //! Perform consistency check. Returns an error code or zero.
     int Check_() EXCLUSIVE_LOCKS_REQUIRED(cs);
@@ -469,7 +466,7 @@ class CAddrMan
     {
         LOCK(cs);
         std::vector<int>().swap(vRandom);
-        nKey = GetRandHash();
+        nKey = insecure_rand.rand256();
         for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) {
             for (size_t entry = 0; entry < ADDRMAN_BUCKET_SIZE; entry++) {
                 vvNew[bucket][entry] = -1;