rpc: allow dumptxoutset to dump human-readable data

luke-jr · May 6, 2021 · 65d0697 · 65d0697
1 parent 06d573f
commit 65d0697
Show file tree

Hide file tree

Showing 6 changed files with 126 additions and 30 deletions.
diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp
@@ -33,6 +33,7 @@
 #include <txmempool.h>
 #include <undo.h>
 #include <util/strencodings.h>
+#include <util/string.h>
 #include <util/system.h>
 #include <util/translation.h>
 #include <validation.h>
@@ -2516,15 +2517,35 @@ static RPCHelpMan getblockfilter()
  */
 static RPCHelpMan dumptxoutset()
 {
+    const std::vector<std::pair<std::string, coinascii_cb_t>> ascii_types{
+        {"txid",         [](const COutPoint& k, const Coin& c) { return k.hash.GetHex(); }},
+        {"vout",         [](const COutPoint& k, const Coin& c) { return ToString(static_cast<int32_t>(k.n)); }},
+        {"value",        [](const COutPoint& k, const Coin& c) { return ToString(c.out.nValue); }},
+        {"coinbase",     [](const COutPoint& k, const Coin& c) { return ToString(c.fCoinBase); }},
+        {"height",       [](const COutPoint& k, const Coin& c) { return ToString(static_cast<uint32_t>(c.nHeight)); }},
+        {"scriptPubKey", [](const COutPoint& k, const Coin& c) { return HexStr(c.out.scriptPubKey); }},
+        // add any other desired items here
+    };
+
+    std::vector<RPCArg> ascii_args;
+    std::transform(std::begin(ascii_types), std::end(ascii_types), std::back_inserter(ascii_args),
+            [](const std::pair<std::string, coinascii_cb_t>& t) { return RPCArg{t.first, RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Info to write for a given UTXO"}; });
+
     return RPCHelpMan{
         "dumptxoutset",
-        "\nWrite the serialized UTXO set to disk.\n",
+        "\nWrite the UTXO set to disk.\n",
         {
             {"path",
                 RPCArg::Type::STR,
                 RPCArg::Optional::NO,
                 /* default_val */ "",
                 "path to the output file. If relative, will be prefixed by datadir."},
+            {"format", RPCArg::Type::ARR, RPCArg::DefaultHint{"compact serialized format"},
+                                        "If no argument is provided, a compact binary serialized format is used; otherwise only requested items "
+                                        "available below are written in ASCII format (if an empty array is provided, all items are written in ASCII).",
+                                        ascii_args, "format"},
+            {"show_header", RPCArg::Type::BOOL, RPCArg::Default{true}, "Whether to include the header line in non-serialized (ASCII) mode"},
+            {"separator", RPCArg::Type::STR, RPCArg::Default{","}, "Field separator to use in non-serialized (ASCII) mode"},
         },
         RPCResult{
             RPCResult::Type::OBJ, "", "",
@@ -2536,10 +2557,33 @@ static RPCHelpMan dumptxoutset()
                 }
         },
         RPCExamples{
-            HelpExampleCli("dumptxoutset", "utxo.dat")
+            HelpExampleCli("dumptxoutset", "utxo.dat") +
+            HelpExampleCli("dumptxoutset", "utxo.dat '[]'") +
+            HelpExampleCli("dumptxoutset", "utxo.dat '[\"txid\", \"vout\"]' false ':'")
         },
         [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
 {
+    // handle optional ASCII parameters
+    const bool is_compact = request.params[1].isNull();
+    const bool show_header = request.params[2].isNull() || request.params[2].get_bool();
+    const std::string separator = request.params[3].isNull() ? "," : request.params[3].get_str();
+    std::vector<std::pair<std::string, coinascii_cb_t>> requested;
+    if (!is_compact) {
+        const auto& arr = request.params[1].get_array();
+        const std::unordered_map<std::string, coinascii_cb_t> ascii_map(std::begin(ascii_types), std::end(ascii_types));
+        for(size_t i = 0; i < arr.size(); ++i) {
+            const auto it = ascii_map.find(arr[i].get_str());
+            if (it == std::end(ascii_map))
+                throw JSONRPCError(RPC_INVALID_PARAMETER, "unable to find item '"+arr[i].get_str()+"'");
+
+            requested.push_back(*it);
+        }
+
+        // if nothing was found, shows everything by default
+        if (requested.size() == 0)
+            requested = ascii_types;
+    }
+
     const fs::path path = fsbridge::AbsPathJoin(GetDataDir(), request.params[0].get_str());
     // Write to a temporary path and then move into `path` on completion
     // to avoid confusion due to an interruption.
@@ -2552,10 +2596,10 @@ static RPCHelpMan dumptxoutset()
             "move it out of the way first");
     }
 
-    FILE* file{fsbridge::fopen(temppath, "wb")};
+    FILE* file{fsbridge::fopen(temppath, is_compact ? "wb" : "w")};
     CAutoFile afile{file, SER_DISK, CLIENT_VERSION};
     NodeContext& node = EnsureAnyNodeContext(request.context);
-    UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), afile);
+    UniValue result = CreateUTXOSnapshot(is_compact, show_header, separator, node, node.chainman->ActiveChainstate(), afile, requested);
     fs::rename(temppath, path);
 
     result.pushKV("path", path.string());
@@ -2564,7 +2608,7 @@ static RPCHelpMan dumptxoutset()
     };
 }
 
-UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile)
+UniValue CreateUTXOSnapshot(const bool is_compact, const bool show_header, const std::string& separator, NodeContext& node, CChainState& chainstate, CAutoFile& afile, const std::vector<std::pair<std::string, coinascii_cb_t>>& requested)
 {
     std::unique_ptr<CCoinsViewCursor> pcursor;
     CCoinsStats stats{CoinStatsHashType::NONE};
@@ -2596,9 +2640,18 @@ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFil
         CHECK_NONFATAL(tip);
     }
 
-    SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx};
-
-    afile << metadata;
+    if (is_compact) {
+        SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx};
+        afile << metadata;
+    } else if (show_header) {
+        afile.write("#(blockhash " + tip->GetBlockHash().ToString() + " ) ");
+        for (auto it = std::begin(requested); it != std::end(requested); ++it) {
+            if (it != std::begin(requested))
+                afile.write(separator);
+            afile.write(it->first);
+        }
+        afile.write("\n");
+    }
 
     COutPoint key;
     Coin coin;
@@ -2608,8 +2661,17 @@ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFil
         if (iter % 5000 == 0) node.rpc_interruption_point();
         ++iter;
         if (pcursor->GetKey(key) && pcursor->GetValue(coin)) {
-            afile << key;
-            afile << coin;
+            if (is_compact) {
+                afile << key;
+                afile << coin;
+            } else {
+                for (auto it = std::begin(requested); it != std::end(requested); ++it) {
+                    if (it != std::begin(requested))
+                        afile.write(separator);
+                    afile.write(it->second(key, coin));
+                }
+                afile.write("\n");
+            }
         }
 
         pcursor->Next();

diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h
@@ -9,6 +9,7 @@
 #include <core_io.h>
 #include <streams.h>
 #include <sync.h>
+#include <coins.h>
 
 #include <any>
 #include <stdint.h>
@@ -26,6 +27,7 @@ class UniValue;
 struct NodeContext;
 
 static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5;
+using coinascii_cb_t = std::function<std::string(const COutPoint&, const Coin&)>;
 
 /**
  * Get the difficulty of the net wrt to the given block index.
@@ -68,6 +70,6 @@ CBlockPolicyEstimator& EnsureAnyFeeEstimator(const std::any& context);
  * Helper to create UTXO snapshots given a chainstate and a file handle.
  * @return a UniValue map containing metadata about the snapshot.
  */
-UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile);
+UniValue CreateUTXOSnapshot(const bool is_compact, const bool show_header, const std::string& separator, NodeContext& node, CChainState& chainstate, CAutoFile& afile, const std::vector<std::pair<std::string, coinascii_cb_t>>& requested);
 
 #endif
diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp
@@ -77,6 +77,8 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "sendmany", 8, "fee_rate"},
     { "sendmany", 9, "verbose" },
     { "deriveaddresses", 1, "range" },
+    { "dumptxoutset", 1, "format" },
+    { "dumptxoutset", 2, "show_header" },
     { "scantxoutset", 1, "scanobjects" },
     { "addmultisigaddress", 0, "nrequired" },
     { "addmultisigaddress", 1, "keys" },

diff --git a/src/streams.h b/src/streams.h
@@ -643,6 +643,11 @@ class CAutoFile
             throw std::ios_base::failure("CAutoFile::write: write failed");
     }
 
+    void write(const std::string& s)
+    {
+        write(s.c_str(), s.size());
+    }
+
     template<typename T>
     CAutoFile& operator<<(const T& obj)
     {

diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp
@@ -183,7 +183,7 @@ CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleati
     FILE* outfile{fsbridge::fopen(snapshot_path, "wb")};
     CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION};
 
-    UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile);
+    UniValue result = CreateUTXOSnapshot(false, false, "", node, node.chainman->ActiveChainstate(), auto_outfile, {});
     BOOST_TEST_MESSAGE(
         "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write());
 

diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py
@@ -18,34 +18,59 @@ def set_test_params(self):
 
     def run_test(self):
         """Test a trivial usage of the dumptxoutset RPC command."""
+
+        # format: test title, kwargs, file hash
+        TESTS = [["no_option",      {},
+                  'be032e5f248264ba08e11099ac09dbd001f6f87ffc68bf0f87043d8146d50664'],
+                 ["all_data",       {"format": []},
+                  '5554c7d08c2f9aaacbbc66617eb59f13aab4b8c0574f4d8b12f728c60dc7d287'],
+                 ["partial_data_1", {"format": ["txid"]},
+                  'eaec3b56b285dcae610be0975d494befa5a6a130211dda0e1ec1ef2c4afa4389'],
+                 ["partial_data_order", {"format": ["height", "vout"]},
+                  '3e5d6d1cb44595eb7c9d13b3370d14b8826c0d81798c29339794623d4ab6091c'],
+                 ["partial_data_double", {"format": ["scriptPubKey", "scriptPubKey"]},
+                  '0eb83a3bf6a7580333fdaf7fd6cebebe93096e032d49049229124ca699222919'],
+                 ["no_header",      {"format": [], "show_header": False},
+                  'ba85c1db5df6de80c783f2c9a617de4bd7e0e92125a0d318532218eaaed28bfa'],
+                 ["separator",      {"format": [], "separator": ":"},
+                  '3352b4db7a9f63629cf255c1a805241f1bee2b557e5f113993669cd3085e9b0f'],
+                 ["all_options",    {"format": [], "show_header": False, "separator": ":"},
+                  '7df9588375f8bd01d0b6f902a55e086c2d0549c3f08f389baa28b398e987f8a2']]
+
         node = self.nodes[0]
         mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1
         node.setmocktime(mocktime)
         node.generate(100)
 
-        FILENAME = 'txoutset.dat'
-        out = node.dumptxoutset(FILENAME)
-        expected_path = Path(node.datadir) / self.chain / FILENAME
-
-        assert expected_path.is_file()
+        for test in TESTS:
+            self.log.info(test[0])
+            test[1]["path"] = test[0]+'_txoutset.dat'
+            out = node.dumptxoutset(**test[1])
+            expected_path = Path(node.datadir) / self.chain / test[1]["path"]
 
-        assert_equal(out['coins_written'], 100)
-        assert_equal(out['base_height'], 100)
-        assert_equal(out['path'], str(expected_path))
-        # Blockhash should be deterministic based on mocked time.
-        assert_equal(
-            out['base_hash'],
-            '6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6')
+            assert expected_path.is_file()
 
-        with open(str(expected_path), 'rb') as f:
-            digest = hashlib.sha256(f.read()).hexdigest()
-            # UTXO snapshot hash should be deterministic based on mocked time.
+            assert_equal(out['coins_written'], 100)
+            assert_equal(out['base_height'], 100)
+            assert_equal(out['path'], str(expected_path))
+            # Blockhash should be deterministic based on mocked time.
             assert_equal(
-                digest, '7ae82c986fa5445678d2a21453bb1c86d39e47af13da137640c2b1cf8093691c')
+                out['base_hash'],
+                '6fd417acba2a8738b06fee43330c50d58e6a725046c3d843c8dd7e51d46d1ed6')
 
-        # Specifying a path to an existing file will fail.
+            with open(str(expected_path), 'rb') as f:
+                digest = hashlib.sha256(f.read()).hexdigest()
+                # UTXO snapshot hash should be deterministic based on mocked time.
+                assert_equal(digest, test[2])
+
+            # Specifying a path to an existing file will fail.
+            assert_raises_rpc_error(
+                -8, '{} already exists'.format(test[1]["path"]),  node.dumptxoutset, test[1]["path"])
+
+        # Other failing tests
         assert_raises_rpc_error(
-            -8, '{} already exists'.format(FILENAME),  node.dumptxoutset, FILENAME)
+            -8, 'unable to find item \'sample\'',  node.dumptxoutset, path='xxx', format=['sample'])
+
 
 if __name__ == '__main__':
     DumptxoutsetTest().main()