Merge #2273: [TierTwo] Deterministic masternode lists

6dbdf37 [Consensus] Serialize also shield inputs for ProReg inputshash (random-zebra)
e0ff60f [Consensus] Serialize the whole scriptPayout for ProReg sign string (random-zebra)
e659d7e [Trivial] Fix styling/logging nits (random-zebra)
170ab92 [Tests] Check for protx inputs hash in evo_specialtx_tests (random-zebra)
a1cb227 [BUG] Add proper virtual dtor for CActiveDeterministicMasternodeManager (random-zebra)
0520ea9 [Init] Add -mnoperatorprivatekey to flag the active mn as deterministic (random-zebra)
dcb5df9 [Refactoring] CADMM: GetLocalAddress static (random-zebra)
6cd5b0c [Refactoring] encapsulate activeMNINfo inside activeMNManager (random-zebra)
ca9d10c Implement CActiveDeterministicMasternodeManager (Alexander Block)
66302c3 [Tests] Check deterministic masternodes unique properties (random-zebra)
ddad010 [Validation] Check duplicate unique-properties for masternodes (random-zebra)
41869ec [Refactoring] Add IsDIP3Enforced/LegacyMNObsolete funcs to DMNManager (random-zebra)
c013bf1 [Refactoring] Add evo Notification Interface (random-zebra)
180a311 [Validation] Connect deterministic manager to block processing (random-zebra)
7f2d2b5 [MOVE-ONLY] move special_tx_validation_test inside evo_* test file (random-zebra)
9ac0f37 [Tests] Introduce evo_deterministicmns_tests.cpp (random-zebra)
48c66dc Conflict handling for ProRegTx in mempool (Alexander Block)
ae33972 [Core] Deterministic Masternode List implementation (random-zebra)
50c277b Implement std::unordered_map/set compatible hasher classes for salted hashes (Alexander Block)
0c064f8 [Tests] Add tests for SetTxPayload/GetTxPayload and CheckStringSig (random-zebra)
693772a [Core] Introduce ProRegTx payload (random-zebra)
6fb2496 Define hash function for uint256 to be used in STL-like containers (random-zebra)
3f74c88 [Build] CMake: add immer headers (random-zebra)
e067443 [Build] Add "immer" functional/immutable containers library (random-zebra)
b92de4d [Consensus] Reject special txes before V6 enforcement (random-zebra)

Pull request description:

  Extracted from #2267.
  This is the first fundamental PR.
  Here we introduce the `PROREG` transaction type, the DMN manager (and active-masternode manager), and all the required classes (`CDeterministicMN`, `CDeterministicMNState`, `CDeterministicMNStateDiff`, `CDeterministicMNList` and `CDeterministicMNListDiff`).

  This contains also the headers-only immer library for persistent immutable data structures. Reasons for the inclusion are explained in #2267 (under "code architecture").
  The headers are copied over, at the same commit used by Dash (`0a718d2d76bab6ebdcf43de943bd6c7d2dbfe2f9`). Later on, we might want to properly include this as a git subtree.

  Built on top of:
  - [x] #2271

ACKs for top commit:
  furszy:
    Code review ACK 6dbdf37.
  Fuzzbawls:
    Code ACK 6dbdf37

Tree-SHA512: 3b175eef5cd03c73ee4d125063a4a5f5a5eff6064fb6ff3f54c01b6f2d48562c6181cbfa2687025a2fc9a65a80b17d799581cba1f2937333dfee4c3212b5cfac

CMakeLists.txt

@@ -168,6 +168,7 @@ file(GLOB CONSENSUS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/consensus/*.h)
 file(GLOB CTAES_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/crypto/ctaes/*.h)
 file(GLOB ZRUST_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/rust/include/*.h)
 file(GLOB SAPLING_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/sapling/*.h)
+file(GLOB IMMER_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/immer/*.h)
 file(GLOB EVO_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/evo/*.h)
 
 source_group("BitcoinHeaders" FILES
@@ -182,6 +183,7 @@ source_group("BitcoinHeaders" FILES
         ${CTAES_HEADERS}
         ${ZRUST_HEADERS}
         ${SAPLING_HEADERS}
+        ${IMMER_HEADERS}
         ${EVO_HEADERS}
         ./src/support/cleanse.h
         )
@@ -372,7 +374,10 @@ set(COMMON_SOURCES
         ./src/coins.cpp
         ./src/key_io.cpp
         ./src/compressor.cpp
+        ./src/evo/deterministicmns.cpp
+        ./src/evo/evonotificationinterface.cpp
         ./src/evo/evodb.cpp
+        ./src/evo/providertx.cpp
         ./src/evo/specialtx.cpp
         ./src/consensus/merkle.cpp
         ./src/consensus/zerocoin_verify.cpp
@@ -390,6 +395,7 @@ set(COMMON_SOURCES
         ./src/policy/feerate.cpp
         ./src/protocol.cpp
         ./src/pubkey.cpp
+        ./src/saltedhasher.cpp
         ./src/scheduler.cpp
         ./src/script/interpreter.cpp
         ./src/script/script.cpp

src/Makefile.am

@@ -182,7 +182,10 @@ BITCOIN_CORE_H = \
   cuckoocache.h \
   crypter.h \
   cyclingvector.h \
+  evo/deterministicmns.h \
   evo/evodb.h \
+  evo/evonotificationinterface.h \
+  evo/providertx.h \
   evo/specialtx.h \
   pairresult.h \
   addressbook.h \
@@ -247,6 +250,7 @@ BITCOIN_CORE_H = \
   rpc/protocol.h \
   rpc/register.h \
   rpc/server.h \
+  saltedhasher.h \
   scheduler.h \
   script/interpreter.h \
   script/keyorigin.h \
@@ -325,7 +329,10 @@ libbitcoin_server_a_SOURCES = \
   consensus/params.cpp \
   consensus/tx_verify.cpp \
   consensus/zerocoin_verify.cpp \
+  evo/deterministicmns.cpp \
   evo/evodb.cpp \
+  evo/evonotificationinterface.cpp \
+  evo/providertx.cpp \
   evo/specialtx.cpp \
   httprpc.cpp \
   httpserver.cpp \
@@ -506,6 +513,7 @@ libbitcoin_common_a_SOURCES = \
   policy/feerate.cpp \
   protocol.cpp \
   pubkey.cpp \
+  saltedhasher.cpp \
   scheduler.cpp \
   script/interpreter.cpp \
   script/script.cpp \
@@ -729,7 +737,10 @@ CTAES_DIST += crypto/ctaes/ctaes.h
 CTAES_DIST += crypto/ctaes/README.md
 CTAES_DIST += crypto/ctaes/test.c
 
+IMMER_DIST = immer
+
 EXTRA_DIST = $(CTAES_DIST) rust
+EXTRA_DIST += $(IMMER_DIST)
 
 
 config/pivx-config.h: config/stamp-h1

src/Makefile.test.include

@@ -62,6 +62,8 @@ BITCOIN_TESTS =\
   test/crypto_tests.cpp \
   test/cuckoocache_tests.cpp \
   test/DoS_tests.cpp \
+  test/evo_deterministicmns_tests.cpp \
+  test/evo_specialtx_tests.cpp \
   test/getarg_tests.cpp \
   test/hash_tests.cpp \
   test/key_tests.cpp \

src/activemasternode.cpp

@@ -6,6 +6,8 @@
 #include "activemasternode.h"
 
 #include "addrman.h"
+#include "evo/providertx.h"
+#include "evo/deterministicmns.h"
 #include "masternode-sync.h"
 #include "masternode.h"
 #include "masternodeconfig.h"
@@ -14,6 +16,180 @@
 #include "netbase.h"
 #include "protocol.h"
 
+// Keep track of the active Masternode
+CActiveDeterministicMasternodeManager* activeMasternodeManager{nullptr};
+
+static bool GetLocalAddress(CService& addrRet)
+{
+    // First try to find whatever local address is specified by externalip option
+    bool fFound = GetLocal(addrRet) && CActiveDeterministicMasternodeManager::IsValidNetAddr(addrRet);
+    if (!fFound && Params().IsRegTestNet()) {
+        if (Lookup("127.0.0.1", addrRet, GetListenPort(), false)) {
+            fFound = true;
+        }
+    }
+    if(!fFound) {
+        // If we have some peers, let's try to find our local address from one of them
+        g_connman->ForEachNodeContinueIf([&fFound, &addrRet](CNode* pnode) {
+            if (pnode->addr.IsIPv4())
+                fFound = GetLocal(addrRet, &pnode->addr) && CActiveDeterministicMasternodeManager::IsValidNetAddr(addrRet);
+            return !fFound;
+        });
+    }
+    return fFound;
+}
+
+std::string CActiveDeterministicMasternodeManager::GetStatus() const
+{
+    switch (state) {
+        case MASTERNODE_WAITING_FOR_PROTX:    return "Waiting for ProTx to appear on-chain";
+        case MASTERNODE_POSE_BANNED:          return "Masternode was PoSe banned";
+        case MASTERNODE_REMOVED:              return "Masternode removed from list";
+        case MASTERNODE_OPERATOR_KEY_CHANGED: return "Operator key changed or revoked";
+        case MASTERNODE_PROTX_IP_CHANGED:     return "IP address specified in ProTx changed";
+        case MASTERNODE_READY:                return "Ready";
+        case MASTERNODE_ERROR:                return "Error. " + strError;
+        default:                              return "Unknown";
+    }
+}
+
+OperationResult CActiveDeterministicMasternodeManager::SetOperatorKey(const std::string& strMNOperatorPrivKey)
+{
+
+    LOCK(cs_main); // Lock cs_main so the node doesn't perform any action while we setup the Masternode
+    LogPrintf("Initializing deterministic masternode...\n");
+    if (strMNOperatorPrivKey.empty()) {
+        return errorOut("ERROR: Masternode operator priv key cannot be empty.");
+    }
+    if (!CMessageSigner::GetKeysFromSecret(strMNOperatorPrivKey, info.keyOperator, info.keyIDOperator)) {
+        return errorOut(_("Invalid mnoperatorprivatekey. Please see the documentation."));
+    }
+    return OperationResult(true);
+}
+
+void CActiveDeterministicMasternodeManager::Init()
+{
+    if (!fMasterNode || !deterministicMNManager->IsDIP3Enforced())
+        return;
+
+    LOCK(cs_main);
+
+    // Check that our local network configuration is correct
+    if (!fListen) {
+        // listen option is probably overwritten by smth else, no good
+        state = MASTERNODE_ERROR;
+        strError = "Masternode must accept connections from outside. Make sure listen configuration option is not overwritten by some another parameter.";
+        LogPrintf("%s ERROR: %s\n", __func__, strError);
+        return;
+    }
+
+    if (!GetLocalAddress(info.service)) {
+        state = MASTERNODE_ERROR;
+        strError = "Can't detect valid external address. Please consider using the externalip configuration option if problem persists. Make sure to use IPv4 address only.";
+        LogPrintf("%s ERROR: %s\n", __func__, strError);
+        return;
+    }
+
+    CDeterministicMNList mnList = deterministicMNManager->GetListAtChainTip();
+
+    CDeterministicMNCPtr dmn = mnList.GetMNByOperatorKey(info.keyIDOperator);
+    if (!dmn) {
+        // MN not appeared on the chain yet
+        return;
+    }
+
+    if (!mnList.IsMNValid(dmn->proTxHash)) {
+        if (mnList.IsMNPoSeBanned(dmn->proTxHash)) {
+            state = MASTERNODE_POSE_BANNED;
+        } else {
+            state = MASTERNODE_REMOVED;
+        }
+        return;
+    }
+
+    LogPrintf("%s: proTxHash=%s, proTx=%s\n", __func__, dmn->proTxHash.ToString(), dmn->ToString());
+
+    if (info.service != dmn->pdmnState->addr) {
+        state = MASTERNODE_ERROR;
+        strError = strprintf("Local address %s does not match the address from ProTx (%s)",
+                             info.service.ToStringIPPort(), dmn->pdmnState->addr.ToStringIPPort());
+        LogPrintf("%s ERROR: %s\n", __func__, strError);
+        return;
+    }
+
+    if (!Params().IsRegTestNet()) {
+        // Check socket connectivity
+        const std::string& strService = info.service.ToString();
+        LogPrintf("%s: Checking inbound connection to '%s'\n", __func__, strService);
+        SOCKET hSocket;
+        bool fConnected = ConnectSocket(info.service, hSocket, nConnectTimeout) && IsSelectableSocket(hSocket);
+        CloseSocket(hSocket);
+
+        if (!fConnected) {
+            state = MASTERNODE_ERROR;
+            LogPrintf("%s ERROR: Could not connect to %s\n", __func__, strService);
+            return;
+        }
+    }
+
+    info.proTxHash = dmn->proTxHash;
+    state = MASTERNODE_READY;
+}
+
+void CActiveDeterministicMasternodeManager::Reset(masternode_state_t _state)
+{
+    state = _state;
+    SetNullProTx();
+    // MN might have reappeared in same block with a new ProTx
+    Init();
+}
+
+void CActiveDeterministicMasternodeManager::UpdatedBlockTip(const CBlockIndex* pindexNew, const CBlockIndex* pindexFork, bool fInitialDownload)
+{
+    if (fInitialDownload)
+        return;
+
+    if (!fMasterNode || !deterministicMNManager->IsDIP3Enforced())
+        return;
+
+    if (state == MASTERNODE_READY) {
+        auto oldMNList = deterministicMNManager->GetListForBlock(pindexNew->pprev);
+        auto newMNList = deterministicMNManager->GetListForBlock(pindexNew);
+        if (!newMNList.IsMNValid(info.proTxHash)) {
+            // MN disappeared from MN list
+            Reset(MASTERNODE_REMOVED);
+            return;
+        }
+
+        auto oldDmn = oldMNList.GetMN(info.proTxHash);
+        auto newDmn = newMNList.GetMN(info.proTxHash);
+        if (newDmn->pdmnState->keyIDOperator != oldDmn->pdmnState->keyIDOperator) {
+            // MN operator key changed or revoked
+            Reset(MASTERNODE_OPERATOR_KEY_CHANGED);
+            return;
+        }
+
+        if (newDmn->pdmnState->addr != oldDmn->pdmnState->addr) {
+            // MN IP changed
+            Reset(MASTERNODE_PROTX_IP_CHANGED);
+            return;
+        }
+    } else {
+        // MN might have (re)appeared with a new ProTx or we've found some peers
+        // and figured out our local address
+        Init();
+    }
+}
+
+bool CActiveDeterministicMasternodeManager::IsValidNetAddr(const CService& addrIn)
+{
+    // TODO: check IPv6 and TOR addresses
+    return Params().IsRegTestNet() || (addrIn.IsIPv4() && IsReachable(addrIn) && addrIn.IsRoutable());
+}
+
+
+/********* LEGACY *********/
+
 OperationResult initMasternode(const std::string& _strMasterNodePrivKey, const std::string& _strMasterNodeAddr, bool isFromInit)
 {
     if (!isFromInit && fMasterNode) {
@@ -79,6 +255,10 @@ OperationResult initMasternode(const std::string& _strMasterNodePrivKey, const s
 void CActiveMasternode::ManageStatus()
 {
     if (!fMasterNode) return;
+    if (activeMasternodeManager != nullptr) {
+        // Deterministic masternode
+        return;
+    }
 
     LogPrint(BCLog::MASTERNODE, "CActiveMasternode::ManageStatus() - Begin\n");
 

src/activemasternode.h

@@ -12,17 +12,69 @@
 #include "net.h"
 #include "operationresult.h"
 #include "sync.h"
+#include "validationinterface.h"
 #include "wallet/wallet.h"
 
+class CActiveDeterministicMasternodeManager;
+
 #define ACTIVE_MASTERNODE_INITIAL 0 // initial state
 #define ACTIVE_MASTERNODE_SYNC_IN_PROCESS 1
 #define ACTIVE_MASTERNODE_NOT_CAPABLE 3
 #define ACTIVE_MASTERNODE_STARTED 4
 
+extern CActiveDeterministicMasternodeManager* activeMasternodeManager;
+
+struct CActiveMasternodeInfo
+{
+    // Keys for the active Masternode
+    CKeyID keyIDOperator;
+    CKey keyOperator;
+    // Initialized while registering Masternode
+    uint256 proTxHash{UINT256_ZERO};
+    CService service;
+};
+
+class CActiveDeterministicMasternodeManager : public CValidationInterface
+{
+public:
+    enum masternode_state_t {
+        MASTERNODE_WAITING_FOR_PROTX,
+        MASTERNODE_POSE_BANNED,
+        MASTERNODE_REMOVED,
+        MASTERNODE_OPERATOR_KEY_CHANGED,
+        MASTERNODE_PROTX_IP_CHANGED,
+        MASTERNODE_READY,
+        MASTERNODE_ERROR,
+    };
+
+private:
+    masternode_state_t state{MASTERNODE_WAITING_FOR_PROTX};
+    std::string strError;
+    CActiveMasternodeInfo info;
+
+public:
+    virtual ~CActiveDeterministicMasternodeManager() = default;
+    virtual void UpdatedBlockTip(const CBlockIndex* pindexNew, const CBlockIndex* pindexFork, bool fInitialDownload);
+
+    void Init();
+    void Reset(masternode_state_t _state);
+    // Sets the Deterministic Masternode Operator's private/public key
+    OperationResult SetOperatorKey(const std::string& strMNOperatorPrivKey);
+    void SetNullProTx() { info.proTxHash = UINT256_ZERO; }
+
+    const CActiveMasternodeInfo* GetInfo() const { return &info; }
+    masternode_state_t GetState() const { return state; }
+    std::string GetStatus() const;
+    bool IsReady() const { return state == MASTERNODE_READY; }
+
+    static bool IsValidNetAddr(const CService& addrIn);
+};
+
 // Responsible for initializing the masternode
 OperationResult initMasternode(const std::string& strMasterNodePrivKey, const std::string& strMasterNodeAddr, bool isFromInit);
 
-// Responsible for activating the Masternode and pinging the network
+
+// Responsible for activating the Masternode and pinging the network (legacy MN list)
 class CActiveMasternode
 {
 private: