Add RPA Support

Fulcrum.pro

@@ -323,9 +323,11 @@ SOURCES += \
     Mixins.cpp \
     Mgr.cpp \
     Options.cpp \
+    PackedNumView.cpp \
     PeerMgr.cpp \
     RecordFile.cpp \
     RollingBloomFilter.cpp \
+    Rpa.cpp \
     RPC.cpp \
     RPCMsgId.cpp \
     ServerMisc.cpp \
@@ -369,9 +371,11 @@ HEADERS += \
     Mgr.h \
     Mixins.h \
     Options.h \
+    PackedNumView.h \
     PeerMgr.h \
     RecordFile.h \
     RollingBloomFilter.h \
+    Rpa.h \
     RPC.h \
     RPCMsgId.h \
     ServerMisc.h \
@@ -398,6 +402,10 @@ HEADERS += robin_hood/robin_hood.h
 RESOURCES += \
     resources.qrc
 
+contains(DEFINES, ENABLE_TESTS) {
+    RESOURCES += resources/testdata/testdata.qrc
+}
+
 # Bitcoin related sources & headers
 SOURCES += \
     bitcoin/amount.cpp \

contrib/rpm/fulcrum.spec

@@ -1,5 +1,5 @@
 Name:    {{{ git_repo_name name="fulcrum" }}}
-Version: 1.9.8
+Version: 1.10.0
 Release: {{{ git_repo_version }}}%{?dist}
 Summary: A fast & nimble SPV server for Bitcoin Cash & Bitcoin BTC
 

doc/fulcrum-example-config.conf

@@ -1013,3 +1013,76 @@ rpcpassword = hunter1
 # useful for admins wishing to integrate Fulcrum with monitoring software.
 #
 #pidfile = /path/to/fulcrum.pid
+
+
+
+#-------------------------------------------------------------------------------
+# Reusable Payment Address (RPA) Options
+#-------------------------------------------------------------------------------
+
+# Enable RPA indexing - `rpa` - DEFAULT: 1 for BCH, 0 for all other coins
+#
+# Whether or not to enable the BCH-specific "RPA" index.
+#
+# See: https://github.com/imaginaryusername/Reusable_specs/blob/master/reusable_addresses.md
+#
+# This index takes ~42M of space currently on mainnet (but may grow to >3GB as
+# the blockchain advances over time).  If this index is enabled, the
+# `blockchain.rpa.*` and/or the `blockchain.reusable.*` RPC methods will be
+# available to clients, and the `server.features` map will contain an "rpa"
+# key to indicate that the server supports RPA.
+#
+# If unspecified, then the RPA index and associated RPCs will only be enabled
+# for BCH, and will be disabled for all coins.
+#
+#rpa = 1
+
+
+# RPA starting block height - `rpa_start_height` - DEFAULT: 825000 for mainnet
+#                                                           0 all other nets
+#
+# Limit the RPA index to start at this block height. Blocks before this height
+# will not have their data indexed by the RPA index. The default for mainnet is
+# to save space and cycles since before a certain block height, no RPA wallets
+# were in existence anyway since RPA had not yet been invented.
+#
+#rpa_start_height = 825000
+
+
+# RPA history scan block limit - `rpa_history_blocks` - DEFAULT: 60
+#
+# For the `blockchain.rpa.get_history` and/or `blockchain.reusable.get_history`
+# RPC methods, limit the number of blocks that client can request to scan for RPA
+# transactions in a single RPC call to this number of blocks. In other words,
+# results will be truncated if the client requests a wider height range in their
+# request than this number. The reason for this limit is that clients should be
+# making many frequent fast calls to the server so as to maximize the server's
+# ability to multiplex requests (many small requests is better than a few larger
+# ones when it comes to perceived server responsiveness). Specifying this to be
+# a large value (say, >1000) is a potential DoS vector.
+#
+#rpa_history_blocks = 60
+
+
+# RPA maximum history results limit - `rpa_max_history` - DEFAULT: `max_history`
+#
+# This is similar to the configuration option `max_history` (search for it above),
+# but it can be independently specified for the RPA subsystem to be larger or
+# smaller than the app-level `max_history`.  If unspecified, this option will
+# inherit whatever the app-level `max_history` setting is.
+#
+#rpa_max_history = 125000
+
+
+# RPA prefix bits minimum - `rpa_prefix_bits_min` - DEFAULT: 8
+#
+# Affects the minimum "prefix" value that is accepted by the
+# `blockchain.rpa.get_history` RPC method, in terms of number of bits. Specify
+# a value in the range: [4, 16]. This is a low-level configuration variable and
+# the default of 8 should be good for all extant RPA clients. 4 offers a larger
+# anonymity set to clients when they perform queries (as they will get more
+# haystack to their 1 needled they are looking for), but it comes with a
+# performance penalty on the server-side, which is why we set the default
+# minimum to 8 in Fulcrum.
+#
+#rpa_prefix_bits_min = 8

doc/unix-man-page.md

@@ -1,6 +1,6 @@
-% FULCRUM(1) Version 1.9.8 | Fulcrum Manual
+% FULCRUM(1) Version 1.10.0 | Fulcrum Manual
 % Fulcrum is written by Calin Culianu (cculianu)
-% January 13, 2024
+% March 01, 2024
 
 # NAME
 

resources/testdata/testdata.qrc

@@ -0,0 +1,5 @@
+<RCC>
+    <qresource prefix="/testdata">
+        <file>bch_block_833705.bin</file>
+    </qresource>
+</RCC>

src/App.cpp

@@ -22,6 +22,7 @@
 #include "Controller.h"
 #include "Json/Json.h"
 #include "Logger.h"
+#include "Rpa.h"
 #include "ServerMisc.h"
 #include "Servers.h"
 #include "Storage.h"
@@ -521,6 +522,15 @@ void App::parseArgs()
                " option only takes effect on initial sync, otherwise this option has no effect.\n"),
        QString("MB"),
     },
+    {
+        "rpa",
+        QString("Explicitly enable the Reusable Payment Address index and offer the associated \"blockchain.rpa.*\" RPC"
+                " methods to clients. To explicitly disable this facility, use the CLI arg --no-rpa. Default is: %1.\n")
+             .arg(options->rpa.enabledSpecToString())
+    },
+    {
+        "no-rpa", QString("<hidden>")
+    },
     {
        "dump-sh",
        QString("*** This is an advanced debugging option ***   Dump script hashes. If specified, after the database"
@@ -554,6 +564,13 @@ void App::parseArgs()
         });
     }
 
+    // Hide options that we marked above as hidden by setting the description to: "<hidden>"
+    for (auto & opt : allOptions) {
+        if (opt.description() == "<hidden>") {
+            opt.setFlags(opt.flags() | QCommandLineOption::HiddenFromHelp);
+        }
+    }
+
     parser.addOptions(allOptions);
     QString configArgDesc = "Configuration file (optional). To read configuration variables from the environment instead, ";
 #ifdef Q_OS_LINUX
@@ -1379,6 +1396,90 @@ void App::parseArgs()
             DebugM("config: pidfile = ", options->pidFileAbsPath, " (size: ", QFileInfo(options->pidFileAbsPath).size(), " bytes)");
         });
     }
+
+    // CLI: --rpa
+    // conf: rpa
+    if (const bool psetYes = parser.isSet("rpa"), psetNo = parser.isSet("no-rpa"); psetYes || psetNo || conf.hasValue("rpa")) {
+        bool val{};
+        if (!psetYes && !psetNo) {
+            bool ok{};
+            val = conf.boolValue("rpa", false, &ok);
+            if (!ok) throw BadArgs("rpa: bad value. Specify a boolean value such as 0, 1, true, false, yes, no");
+        }
+        else if (psetYes && psetNo) throw BadArgs("Cannot specify --rpa and --no-rpa at the same time!");
+        else val = psetYes; // will be false if psetNo here
+        options->rpa.enabledSpec = val ? Options::Rpa::Enabled : Options::Rpa::Disabled;
+        Util::AsyncOnObject(this, [val] { DebugM("config: rpa = ", val); });
+    }
+
+    // conf: rpa_max_history
+    if (conf.hasValue("rpa_max_history")) {
+        bool ok;
+        int mh = conf.intValue("rpa_max_history", -1, &ok);
+        if (!ok || mh < options->maxHistoryMin || mh > options->maxHistoryMax)
+            throw BadArgs(QString("rpa_max_history: bad value. Specify a value in the range [%1, %2]")
+                              .arg(options->maxHistoryMin).arg(options->maxHistoryMax));
+        options->rpa.maxHistory = mh;
+        // log this later in case we are in syslog mode
+        Util::AsyncOnObject(this, [mh]{ Debug() << "config: rpa_max_history = " << mh; });
+    } else {
+        // Otherwise, if nothing specified, we have special logic here:
+        // We inherit whatever the user specified for max_history, if anything (may be default)
+        options->rpa.maxHistory = options->maxHistory;
+        if (conf.hasValue("max_history")) {
+            Util::AsyncOnObject(this, [mh = options->maxHistory]{
+                Debug() << "config: rpa_max_history = " << mh << " (inherited from max_history)";
+            });
+        }
+    }
+
+    // conf: rpa_history_block_limit / rpa_history_blocks
+    if (const bool b1 = conf.hasValue("rpa_history_blocks"), b2 = conf.hasValue("rpa_history_block_limit"); b1 || b2) {
+        // support either: "rpa_history_block_limit" or "rpa_history_blocks", but not both
+        if (b1 && b2) throw BadArgs("Both `rpa_history_blocks` and `rpa_history_block_limit` were found in the config file; this looks like a typo.");
+        const QString confKey(b1 ? "rpa_history_blocks" : "rpa_history_block_limit");
+        bool ok;
+        const int limit = conf.intValue(confKey, -1, &ok);
+        if (!ok || limit < 0 || unsigned(limit) < options->rpa.historyBlockLimitMin || unsigned(limit) > options->rpa.historyBlockLimitMax)
+            throw BadArgs(QString("%1: bad value. Specify a value in the range [%2, %3]")
+                              .arg(confKey).arg(options->rpa.historyBlockLimitMin).arg(options->rpa.historyBlockLimitMax));
+        options->rpa.historyBlockLimit = unsigned(limit);
+        // log this later in case we are in syslog mode
+        Util::AsyncOnObject(this, [limit, confKey]{ Debug() << "config: " << confKey << " = " << limit; });
+    }
+
+    // conf: rpa_prefix_bits_min
+    static_assert(Options::Rpa::defaultPrefixBitsMin >= Rpa::PrefixBitsMin && Options::Rpa::defaultPrefixBitsMin <= Rpa::PrefixBits
+                  && !(Options::Rpa::defaultPrefixBitsMin & 0b11));
+    if (conf.hasValue("rpa_prefix_bits_min")) {
+        bool ok;
+        int pbm = conf.intValue("rpa_prefix_bits_min", -1, &ok);
+        if (!ok || pbm < int(Rpa::PrefixBitsMin) || pbm > int(Rpa::PrefixBits) || pbm & 0b11 /* fancy way to check if multiple of 4 */) {
+            throw BadArgs(QString("rpa_prefix_bits_min: bad value. Specify a number that is a multiple of 4 and that is in the range [%1, %2].")
+                              .arg(Rpa::PrefixBitsMin).arg(Rpa::PrefixBits));
+        }
+        options->rpa.prefixBitsMin = pbm;
+        // log this later in case we are in syslog mode
+        Util::AsyncOnObject(this, [pbm]{ Debug() << "config: rpa_prefix_bits_min = " << pbm; });
+    }
+
+    // conf: rpa_start_height
+    if (const auto b1 = conf.hasValue("rpa_start_height"), b2 = conf.hasValue("rpa_starting_height"); b1 || b2) {
+        // support either: "rpa_start_height" or "rpa_starting_height", but not both
+        if (b1 && b2) throw BadArgs("Both `rpa_start_height` and `rpa_starting_height` were found in the config file; this looks like a typo.");
+        const QString confKey(b1 ? "rpa_start_height" : "rpa_starting_height");
+        bool ok;
+        int ht = conf.intValue(confKey, -1, &ok);
+        if (!ok || ht < -1 /* -1 ok, -2 not, etc*/ || (ht >= 0 && ht > int(Storage::MAX_HEADERS)))
+            throw BadArgs(QString("%1: bad value. Specify a block height between [0, %2], or use -1 to"
+                                  " auto-configure this setting with a chain-specific default (%3 for mainnet, %4 for"
+                                  " all other nets).")
+                              .arg(confKey).arg(Storage::MAX_HEADERS).arg(Options::Rpa::defaultStartHeightForMainnet)
+                              .arg(Options::Rpa::defaultStartHeightOtherNets));
+        options->rpa.requestedStartHeight = ht;
+        // log this later in case we are in syslog mode
+        Util::AsyncOnObject(this, [ht, confKey]{ Debug() << "config: " << confKey << " = " << ht; });
+    }
 }
 
 namespace {

src/BTC.cpp

@@ -23,15 +23,10 @@
 #include "bitcoin/crypto/endian.h"
 #include "bitcoin/crypto/sha256.h"
 #include "bitcoin/hash.h"
-#include "bitcoin/pubkey.h"
-#include "bitcoin/streams.h"
-#include "bitcoin/utilstrencodings.h"
-#include "bitcoin/version.h"
 
 #include <QMap>
 
 #include <algorithm>
-#include <atomic>
 #include <utility>
 
 namespace bitcoin
@@ -236,3 +231,44 @@ namespace BTC
 
 
 } // end namespace BTC
+
+
+#ifdef ENABLE_TESTS
+#include "App.h"
+
+#include "bitcoin/transaction.h"
+#include "bitcoin/uint256.h"
+
+namespace {
+    void test()
+    {
+        // Misc. unit tests for BTC namespace utility functions
+        Log() << "Testing Hash2ByteArrayRev ...";
+        bitcoin::uint256 hash = bitcoin::uint256S("080bb1010c4d32f3cb16c6a7f1ac2a949d0b5b0f0396f183870be7032cfc4da9");
+        if (hash.ToString() != "080bb1010c4d32f3cb16c6a7f1ac2a949d0b5b0f0396f183870be7032cfc4da9") throw Exception("Hash parse fail");
+        const QByteArray qba(reinterpret_cast<const char *>(std::as_const(hash).data()), hash.size());
+        if (ByteView{hash} != ByteView{qba}) throw Exception("2");
+        if (qba.toHex() != "a94dfc2c03e70b8783f196030f5b0b9d942aacf1a7c616cbf3324d0c01b10b08") throw Exception("Hash parse did not yield expected result");
+        auto rhash = BTC::Hash2ByteArrayRev(hash);
+        Debug() << "Expected hash: " << rhash.toHex();
+        if (rhash.toHex() != "080bb1010c4d32f3cb16c6a7f1ac2a949d0b5b0f0396f183870be7032cfc4da9") throw Exception("BTC::Hash2ByteArrayRev is broken");
+
+        Log() << "Testing Deserialize ...";
+        const auto txnhex = "0100000001e7b81293c58fa088412949e485f7a7310c386a267a1825284e79c083d26b55670000000084410b00"
+                            "086668d9c26c3bf44b4f136512d7edae0f01ddd66844e312fa00f54250e93457b5e2c823ca31ab452d22f27181"
+                            "b13ce3560b974130b5e8a9e1b3ab820d0d414104e8806002111e3dfb6944e63a42461832437f2bbd616facc269"
+                            "10becfa388642972aaf555ffcdc2cdc07a248e7881efa7f456634e1bdb11485dbbc9db20cb669dfeffffff01fb"
+                            "0cfe00000000001976a914590888ac04b1f1cf01f08110cca83dd3e3da7f7388accbb90c00";
+        auto tx = BTC::Deserialize<bitcoin::CTransaction>(Util::ParseHexFast(txnhex));
+        if (hash != tx.GetHash()) throw Exception("Txn did not deserialize ok");
+
+        Log() << "Testing HashInPlace ...";
+        if (BTC::HashInPlace(tx) != qba) throw Exception("Txn hash in place failed");
+        if (BTC::HashInPlace(tx, false, /* reversed = */true) != rhash) throw Exception("Txn hash in place reversed failed");
+
+        Log(Log::BrightWhite) << "All btcmisc unit tests passed!";
+    }
+
+    auto t1 = App::registerTest("btcmisc", test);
+} // namespace
+#endif

src/BTC.h

@@ -1,6 +1,6 @@
 //
 // Fulcrum - A fast & nimble SPV Server for Bitcoin Cash
-// Copyright (C) 2019-2023 Calin A. Culianu <calin.culianu@gmail.com>
+// Copyright (C) 2019-2024 Calin A. Culianu <calin.culianu@gmail.com>
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@
 #include "Util.h"
 
 #include "bitcoin/block.h"
+#include "bitcoin/hash.h"
 #include "bitcoin/script.h"
 #include "bitcoin/streams.h"
 #include "bitcoin/transaction.h"
@@ -31,9 +32,11 @@
 #include <QMetaType>
 #include <QString>
 
+#include <algorithm>
 #include <cstddef> // for std::byte, etc
 #include <cstring> // for memcpy
 #include <ios>
+#include <iterator>
 #include <type_traits>
 #include <utility> // for pair, etc
 
@@ -170,17 +173,27 @@ namespace BTC
     inline QByteArray HashOnce(const QByteArray &b) { return Hash(b, true); }
     /// Like the Hash() function above, except does hash160 once. (not reversed).
     extern QByteArray Hash160(const QByteArray &);
+    /// Hash any Bitcoin object in-place and return the hash. If `once` == true, we do single-sha256 hashing. If
+    /// `reversed` == true, we reverse the result (making it big-endian ready for JSON).
+    template <typename BitcoinObject>
+    QByteArray HashInPlace(const BitcoinObject &bo, bool once = false, bool reversed = false) {
+        QByteArray ret(bitcoin::CHash256::OUTPUT_SIZE, Qt::Uninitialized); // allocate without initializing
+        bitcoin::SerializeHashInPlace(ret.data(), bo, bitcoin::SER_GETHASH, bitcoin::PROTOCOL_VERSION, once);
+        if (reversed) std::reverse(ret.begin(), ret.end());
+        return ret;
+    }
 
     /// Takes a hash in bitcoin memory order and returns a deep copy QByteArray of the data, reversed
     /// (this is intended to keep our representation of bitcoin data closer to how we will send it to clients down
     /// the wire -- we send all hex encoded hashes in reverse order as is customary when representing bitcoin
     /// hashes in hex). See BlockProc.cpp for an example of where this is used.
     template <class BitcoinHashT>
     QByteArray Hash2ByteArrayRev(const BitcoinHashT &hash) {
-        QByteArray ret(reinterpret_cast<const char *>(hash.begin()), hash.width()); // deep copy
-        std::reverse(ret.begin(), ret.end()); // reverse it
+        QByteArray ret(hash.width(), Qt::Uninitialized);
+        std::copy(std::reverse_iterator(hash.end()), std::reverse_iterator(hash.begin()),
+                  reinterpret_cast<uint8_t *>(ret.data())); // reversed copy
         return ret;
-    };
+    }
 
     /// returns true iff cscript is OP_RETURN, false otherwise
     inline bool IsOpReturn(const bitcoin::CScript &cs) {

src/BitcoinD.cpp

@@ -510,6 +510,8 @@ BitcoinDInfo BitcoinDMgr::getBitcoinDInfo() const
     return bitcoinDInfo;
 }
 
+void BitcoinDMgr::requestBitcoinDInfoRefresh() { refreshBitcoinDNetworkInfo(); }
+
 bool BitcoinDMgr::isZeroArgEstimateFee() const
 {
     std::shared_lock g(bitcoinDInfoLock);