Merge #2412: [Backport] Serialization framework updates

bd4b846 Remove old serialization primitives (Pieter Wuille)
060d62b Convert the last, non-trivial, serialization functions to the new form (furszy)
8c74c09 Convert LimitedString to formatter (Pieter Wuille)
f021897 Fix CDiskBlockIndex serialization of dummy fields for old DB versions (random-zebra)
1ee0cb2 Convert CDiskBlockIndex to new serialization. (furszy)
221bf49 Convert wallet to new serialization (furszy)
cf06950 Convert to new serialization (step 3) (furszy)
dc0fc95 Remove old MESS_VER_STRMESS message version try-catch. (furszy)
35fca11 Convert Qt to new serialization (Pieter Wuille)
3f7826e Add comments to CustomUintFormatter (Pieter Wuille)
eccd473 Convert to new serialization (step 2). Focused on object's serializations that doesn't require an special treatment. (furszy)
0f15784 Convert everything except wallet/qt to new serialization (step 1) (Pieter Wuille)
3d3ee64 Convert merkleblock to new serialization (Pieter Wuille)
13577fb Add SER_READ and SER_WRITE for read/write-dependent statements (Russell Yanofsky)
7344c1a Extend CustomUintFormatter to support enums (Russell Yanofsky)
c4d6228 Merge BigEndian functionality into CustomUintFormatter (Pieter Wuille)
3765d6c Add static_asserts to ser_X_to_Y() methods (Samer Afach)
806213a Fix a violation of C++ standard rules that unions cannot be switched. (Samer Afach)
d6380c4 Add CustomUintFormatter (Pieter Wuille)
fd29a50 Make VectorFormatter support stateful formatters (Russell Yanofsky)
4e2afad Convert CCompactSize to proper formatter (Pieter Wuille)
bb99030 Get rid of VARINT default argument (Pieter Wuille)
e107a0c Convert undo.h to new serialization framework (Pieter Wuille)
a926ba3 Make std::vector and prevector reuse the VectorFormatter logic (Pieter Wuille)
1dfddce Add custom vector-element formatter (Pieter Wuille)
df4e1ba Add a constant for the maximum vector allocation (5 Mbyte) (Pieter Wuille)
c2fdeaf Convert compression.h to new serialization framework (Pieter Wuille)
aa35991 Add FORMATTER_METHODS, similar to SERIALIZE_METHODS, but for formatters (Pieter Wuille)
3e38199 Move compressor utility functions out of class (Pieter Wuille)
7376a95 Convert chain to new serialization (Pieter Wuille)
bbfc55c Convert VARINT to the formatter/Using approach (Pieter Wuille)
39c58a1 Add a generic approach for (de)serialization of objects using code in other classes (Pieter Wuille)
ace3895 Convert addrdb/addrman to new serialization (Pieter Wuille)
6bb135e Introduce new serialization macros without casts (Pieter Wuille)
ace7d14 Drop minor GetSerializeSize template (Ben Woosley)
f05e692 Drop unused GetType() from CSizeComputer (furszy)
5c36b3d Introduce BigEndian wrapper and use it for netaddress ports (Pieter Wuille)
fb3c646 Migrate last FLATDATA calls to use Span. (furszy)
1ef2d90 Support serializing Span<unsigned char> and use that instead of FLATDATA (Pieter Wuille)
8fef544 Add Slice: a (pointer, size) array view that acts like a container (Pieter Wuille)

Pull request description:

  Decoupled from #2411, built on top of #2359.

  Focused on creating the Span class and updating the serialization framework and every object using it up to latest upstream structure (3-4 years ahead of what we currently are in master). We will be up-to-date with them in the area after finishing with #2411 entirely (there are few more updates to the serialization code that comes down #2411 commits line that cannot cherry-pick here).

  Adapted the following PRs:

  *  bitcoin#12886.
  *  bitcoin#12916.
  *  bitcoin#13558.
  *  bitcoin#17850.
  *  bitcoin#17896.
  *  bitcoin#12752.
  *  bitcoin#17957.
  *  bitcoin#18021.
  *  bitcoin#18087.
  *  bitcoin#18112 (only from 353f376 that we don't support).
  *  bitcoin#18167.
  *  bitcoin#18317.
  *  bitcoin#19032.

ACKs for top commit:
  random-zebra:
    ACK bd4b846
  Fuzzbawls:
    ACK bd4b846

Tree-SHA512: fe1b31d0976dff97bfeed0f9efeeb4c6c161277529880ede990c9b3fb0ea680f25b4be01b739f6bf7eeca79fa7687c9c2146c403c96e86bc6b052c6dd88e6930

src/Makefile.am

@@ -262,6 +262,7 @@ BITCOIN_CORE_H = \
   script/standard.h \
   script/script_error.h \
   serialize.h \
+  span.h \
   spork.h \
   sporkdb.h \
   sporkid.h \

src/addrdb.cpp

@@ -83,7 +83,7 @@ bool DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true)
         unsigned char pchMsgTmp[4];
         verifier >> pchMsgTmp;
         // ... verify the network matches ours
-        if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp)))
+        if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp)) != 0)
             return error("%s: Invalid network magic number", __func__);
 
         // de-serialize data

src/addrdb.h

@@ -45,15 +45,7 @@ class CBanEntry
         nCreateTime = nCreateTimeIn;
     }
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITE(nVersion);
-        READWRITE(nCreateTime);
-        READWRITE(nBanUntil);
-        READWRITE(banReason);
-    }
+    SERIALIZE_METHODS(CBanEntry, obj) { READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, obj.banReason); }
 
     void SetNull()
     {

src/addrman.h

@@ -55,15 +55,11 @@ class CAddrInfo : public CAddress
     friend class CAddrMan;
 
 public:
-    ADD_SERIALIZE_METHODS;
 
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
+    SERIALIZE_METHODS(CAddrInfo, obj)
     {
-        READWRITEAS(CAddress, *this);
-        READWRITE(source);
-        READWRITE(nLastSuccess);
-        READWRITE(nAttempts);
+        READWRITEAS(CAddress, obj);
+        READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts);
     }
 
     void Init()
@@ -310,7 +306,7 @@ class CAddrMan
      * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports
      * changes to the ADDRMAN_ parameters without breaking the on-disk structure.
      *
-     * We don't use ADD_SERIALIZE_METHODS since the serialization and deserialization code has
+     * We don't use SERIALIZE_METHODS since the serialization and deserialization code has
      * very little in common.
      */
     template <typename Stream>

src/bench/prevector.cpp

@@ -22,6 +22,7 @@
 struct nontrivial_t {
     int x;
     nontrivial_t() :x(-1) {}
+    SERIALIZE_METHODS(nontrivial_t, obj) { READWRITE(obj.x); }
 };
 static_assert(!IS_TRIVIALLY_CONSTRUCTIBLE<nontrivial_t>::value,
               "expected nontrivial_t to not be trivially constructible");

src/bloom.h

@@ -33,11 +33,11 @@ enum bloomflags {
 /**
  * BloomFilter is a probabilistic filter which SPV clients provide
  * so that we can filter the transactions we sends them.
- * 
+ *
  * This allows for significantly more efficient transaction and block downloads.
- * 
+ *
  * Because bloom filters are probabilistic, an SPV node can increase the false-
- * positive rate, making us send them transactions which aren't actually theirs, 
+ * positive rate, making us send them transactions which aren't actually theirs,
  * allowing clients to trade more bandwidth for more privacy by obfuscating which
  * keys are owned by them.
  */
@@ -66,16 +66,7 @@ class CBloomFilter
     CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak, unsigned char nFlagsIn);
     CBloomFilter() : isFull(true), isEmpty(false), nHashFuncs(0), nTweak(0), nFlags(0) {}
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(vData);
-        READWRITE(nHashFuncs);
-        READWRITE(nTweak);
-        READWRITE(nFlags);
-    }
+    SERIALIZE_METHODS(CBloomFilter, obj) { READWRITE(obj.vData, obj.nHashFuncs, obj.nTweak, obj.nFlags); }
 
     void insert(const std::vector<unsigned char>& vKey);
     void insert(const COutPoint& outpoint);

src/budget/budgetdb.cpp

@@ -93,7 +93,6 @@ CBudgetDB::ReadResult CBudgetDB::Read(CBudgetManager& objToLoad, bool fDryRun)
     }
 
     int version;
-    unsigned char pchMsgTmp[4];
     std::string strMagicMessageTmp;
     try {
         // de-serialize file header
@@ -106,12 +105,12 @@ CBudgetDB::ReadResult CBudgetDB::Read(CBudgetManager& objToLoad, bool fDryRun)
             return IncorrectMagicMessage;
         }
 
-
         // de-serialize file header (network specific magic number) and ..
-        ssObj >> FLATDATA(pchMsgTmp);
+        std::vector<unsigned char> pchMsgTmp(4);
+        ssObj >> MakeSpan(pchMsgTmp);
 
         // ... verify the network matches ours
-        if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) {
+        if (memcmp(pchMsgTmp.data(), Params().MessageStart(), pchMsgTmp.size()) != 0) {
             error("%s : Invalid network magic number", __func__);
             return IncorrectMagicNumber;
         }

src/budget/budgetmanager.h

@@ -166,30 +166,23 @@ class CBudgetManager
     // Remove proposal/budget by FeeTx (called when a block is disconnected)
     void RemoveByFeeTxId(const uint256& feeTxId);
 
-    ADD_SERIALIZE_METHODS;
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
+    SERIALIZE_METHODS(CBudgetManager, obj)
     {
         {
-            LOCK(cs_proposals);
-            READWRITE(mapProposals);
-            READWRITE(mapFeeTxToProposal);
+            LOCK(obj.cs_proposals);
+            READWRITE(obj.mapProposals, obj.mapFeeTxToProposal);
         }
         {
-            LOCK(cs_votes);
-            READWRITE(mapSeenProposalVotes);
-            READWRITE(mapOrphanProposalVotes);
+            LOCK(obj.cs_votes);
+            READWRITE(obj.mapSeenProposalVotes, obj.mapOrphanProposalVotes);
         }
         {
-            LOCK(cs_budgets);
-            READWRITE(mapFinalizedBudgets);
-            READWRITE(mapFeeTxToBudget);
-            READWRITE(mapUnconfirmedFeeTx);
+            LOCK(obj.cs_budgets);
+            READWRITE(obj.mapFinalizedBudgets, obj.mapFeeTxToBudget, obj.mapUnconfirmedFeeTx);
         }
         {
-            LOCK(cs_finalizedvotes);
-            READWRITE(mapSeenFinalizedBudgetVotes);
-            READWRITE(mapOrphanFinalizedBudgetVotes);
+            LOCK(obj.cs_finalizedvotes);
+            READWRITE(obj.mapSeenFinalizedBudgetVotes, obj.mapOrphanFinalizedBudgetVotes);
         }
     }
 };

src/budget/budgetproposal.h

@@ -106,19 +106,17 @@ class CBudgetProposal
     }
 
     // Serialization for local DB
-    ADD_SERIALIZE_METHODS;
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
+    SERIALIZE_METHODS(CBudgetProposal, obj)
     {
-        READWRITE(LIMITED_STRING(strProposalName, 20));
-        READWRITE(LIMITED_STRING(strURL, 64));
-        READWRITE(nBlockStart);
-        READWRITE(nBlockEnd);
-        READWRITE(nAmount);
-        READWRITE(address);
-        READWRITE(nFeeTXHash);
-        READWRITE(nTime);
-        READWRITE(mapVotes);
+        READWRITE(LIMITED_STRING(obj.strProposalName, 20));
+        READWRITE(LIMITED_STRING(obj.strURL, 64));
+        READWRITE(obj.nBlockStart);
+        READWRITE(obj.nBlockEnd);
+        READWRITE(obj.nAmount);
+        READWRITE(obj.address);
+        READWRITE(obj.nFeeTXHash);
+        READWRITE(obj.nTime);
+        READWRITE(obj.mapVotes);
     }
 
     // Serialization for network messages.

src/budget/budgetvote.h

@@ -17,7 +17,7 @@
 class CBudgetVote : public CSignedMessage
 {
 public:
-    enum VoteDirection {
+    enum VoteDirection : uint32_t {
         VOTE_ABSTAIN = 0,
         VOTE_YES = 1,
         VOTE_NO = 2
@@ -64,25 +64,7 @@ class CBudgetVote : public CSignedMessage
     void SetTime(const int64_t& _nTime) { nTime = _nTime; }
     void SetValid(bool _fValid) { fValid = _fValid; }
 
-    ADD_SERIALIZE_METHODS;
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(vin);
-        READWRITE(nProposalHash);
-        int nVoteInt = (int) nVote;
-        READWRITE(nVoteInt);
-        if (ser_action.ForRead())
-            nVote = (VoteDirection) nVoteInt;
-        READWRITE(nTime);
-        READWRITE(vchSig);
-        try
-        {
-            READWRITE(nMessVersion);
-        } catch (...) {
-            nMessVersion = MessageVersion::MESS_VER_STRMESS;
-        }
-    }
+    SERIALIZE_METHODS(CBudgetVote, obj) { READWRITE(obj.vin, obj.nProposalHash, Using<CustomUintFormatter<4>>(obj.nVote), obj.nTime, obj.vchSig, obj.nMessVersion); }
 };
 
 #endif // BUDGET_VOTE_H

src/budget/finalizedbudget.h

@@ -106,18 +106,16 @@ class CFinalizedBudget
     }
 
     // Serialization for local DB
-    ADD_SERIALIZE_METHODS;
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
+    SERIALIZE_METHODS(CFinalizedBudget, obj)
     {
-        READWRITE(LIMITED_STRING(strBudgetName, 20));
-        READWRITE(nFeeTXHash);
-        READWRITE(nTime);
-        READWRITE(nBlockStart);
-        READWRITE(vecBudgetPayments);
-        READWRITE(fAutoChecked);
-        READWRITE(mapVotes);
-        READWRITE(strProposals);
+        READWRITE(LIMITED_STRING(obj.strBudgetName, 20));
+        READWRITE(obj.nFeeTXHash);
+        READWRITE(obj.nTime);
+        READWRITE(obj.nBlockStart);
+        READWRITE(obj.vecBudgetPayments);
+        READWRITE(obj.fAutoChecked);
+        READWRITE(obj.mapVotes);
+        READWRITE(obj.strProposals);
     }
 
     // Serialization for network messages.
@@ -155,16 +153,8 @@ class CTxBudgetPayment
         nAmount(_nAmount)
     {}
 
-    ADD_SERIALIZE_METHODS;
-
     //for saving to the serialized db
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(payee);
-        READWRITE(nAmount);
-        READWRITE(nProposalHash);
-    }
+    SERIALIZE_METHODS(CTxBudgetPayment, obj) { READWRITE(obj.payee, obj.nAmount, obj.nProposalHash); }
 
     // compare payments by proposal hash
     inline bool operator>(const CTxBudgetPayment& other) const

src/budget/finalizedbudgetvote.h

@@ -47,21 +47,7 @@ class CFinalizedBudgetVote : public CSignedMessage
     void SetTime(const int64_t& _nTime) { nTime = _nTime; }
     void SetValid(bool _fValid) { fValid = _fValid; }
 
-    ADD_SERIALIZE_METHODS;
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action)
-    {
-        READWRITE(vin);
-        READWRITE(nBudgetHash);
-        READWRITE(nTime);
-        READWRITE(vchSig);
-        try
-        {
-            READWRITE(nMessVersion);
-        } catch (...) {
-            nMessVersion = MessageVersion::MESS_VER_STRMESS;
-        }
-    }
+    SERIALIZE_METHODS(CFinalizedBudgetVote, obj) { READWRITE(obj.vin, obj.nBudgetHash, obj.nTime, obj.vchSig, obj.nMessVersion); }
 };
 
 #endif // FINALIZED_BUDGET_VOTE_H