Legacy Indexing Overhaul (#42)

This PR removes "universal"-style indexing for legacy CLSAG rings, and replaces it with a reference set scheme that uses
(amount, index in amount) indexing pairs to reference on-chain enotes. This is the same method that Cryptonote txs use, and is how
the current Monero Core LMDB database is referenced. Doing things this way means that the database will not have to be re-indexed,
saving at a very minimum 1.6 GB (100M on-chain enotes * (16 bytes for extra table keys)) of storage space, and an expensive
database migration involving moving all existing enote data to a new table. We change the MockLedgerContext to support this indexing
scheme.

In practice, serialized txs under this method shouldn't take up much more space than pre-PR if compressed clever-ly, and assuming
most ring members will RingCT enotes.

We also add LegacyEnoteOriginContext for contextualized enote records so we can better keep tracked of scanned legacy enotes under
the legacy indexing scheme.

Co-authored-by: SNeedlewoods <sneedlewoods_1@protonmail.com>

src/seraphis_core/legacy_decoy_selector.h

@@ -31,12 +31,13 @@
 #pragma once
 
 //local headers
+#include "legacy_output_index.h"
 
 //third party headers
 
 //standard headers
 #include <cstdint>
-#include <vector>
+#include <set>
 
 //forward declarations
 
@@ -60,9 +61,9 @@ class LegacyDecoySelector
 
 //member functions
     /// request a set of ring members as on-chain enote indices
-    virtual void get_ring_members(const std::uint64_t real_ring_member_index,
+    virtual void get_ring_members(const legacy_output_index_t real_ring_member_index,
         const std::uint64_t num_ring_members,
-        std::vector<std::uint64_t> &ring_members_out,
+        std::set<legacy_output_index_t> &ring_members_out,
         std::uint64_t &real_ring_member_index_in_ref_set_out) const = 0;
 };
 

src/seraphis_core/legacy_decoy_selector_flat.cpp

@@ -45,50 +45,52 @@
 namespace sp
 {
 //-------------------------------------------------------------------------------------------------------------------
-LegacyDecoySelectorFlat::LegacyDecoySelectorFlat(const std::uint64_t min_index, const std::uint64_t max_index) :
-    m_min_index{min_index},
-    m_max_index{max_index}
+LegacyDecoySelectorFlat::LegacyDecoySelectorFlat(index_bounds_by_amount_t index_bounds_by_amount) :
+    m_index_bounds_by_amount(std::move(index_bounds_by_amount))
 {
     // checks
-    CHECK_AND_ASSERT_THROW_MES(m_max_index >= m_min_index, "legacy decoy selector (flat): invalid element range.");
+    for (const auto &p : m_index_bounds_by_amount)
+    {
+        const std::uint64_t min_ind{p.second.first};
+        const std::uint64_t max_ind{p.second.second};
+        CHECK_AND_ASSERT_THROW_MES(max_ind >= min_ind, "legacy decoy selector (flat): min > max index.");
+    }
 }
 //-------------------------------------------------------------------------------------------------------------------
-void LegacyDecoySelectorFlat::get_ring_members(const std::uint64_t real_ring_member_index,
+void LegacyDecoySelectorFlat::get_ring_members(const legacy_output_index_t real_ring_member_index,
         const std::uint64_t num_ring_members,
-        std::vector<std::uint64_t> &ring_members_out,
+        std::set<legacy_output_index_t> &ring_members_out,
         std::uint64_t &real_ring_member_index_in_ref_set_out) const
 {
-    CHECK_AND_ASSERT_THROW_MES(real_ring_member_index >= m_min_index,
+    const rct::xmr_amount amount{real_ring_member_index.ledger_indexing_amount};
+    const std::uint64_t min_index{this->get_min_index(amount)};
+    const std::uint64_t max_index{this->get_max_index(amount)};
+
+    CHECK_AND_ASSERT_THROW_MES(real_ring_member_index.index >= min_index,
         "legacy decoy selector (flat): real ring member index below available index range.");
-    CHECK_AND_ASSERT_THROW_MES(real_ring_member_index <= m_max_index,
+    CHECK_AND_ASSERT_THROW_MES(real_ring_member_index.index <= max_index,
         "legacy decoy selector (flat): real ring member index above available index range.");
-    CHECK_AND_ASSERT_THROW_MES(num_ring_members <= m_max_index - m_min_index + 1,
+    CHECK_AND_ASSERT_THROW_MES(num_ring_members <= max_index - min_index + 1,
         "legacy decoy selector (flat): insufficient available legacy enotes to have unique ring members.");
 
     // fill in ring members
     ring_members_out.clear();
-    ring_members_out.reserve(num_ring_members);
-    ring_members_out.emplace_back(real_ring_member_index);
+    ring_members_out.insert(real_ring_member_index);
 
     while (ring_members_out.size() < num_ring_members)
     {
-        // select a new ring member from indices in the specified range that aren't used yet (only unique ring members
-        //   are allowed)
-        std::uint64_t new_ring_member;
-        do { new_ring_member = crypto::rand_range<std::uint64_t>(m_min_index, m_max_index); }
-        while (std::find(ring_members_out.begin(), ring_members_out.end(), new_ring_member) != ring_members_out.end());
-
-        ring_members_out.emplace_back(new_ring_member);
+        // select a new ring member from indices in the specified range with uniform distribution
+        const std::uint64_t new_ring_member_index{crypto::rand_range<std::uint64_t>(min_index, max_index)};
+        // add to set (only unique values will remain)
+        ring_members_out.insert({amount, new_ring_member_index});
     }
 
-    // sort reference set
-    std::sort(ring_members_out.begin(), ring_members_out.end());
-
     // find location in reference set where the real reference sits
     // note: the reference set does not contain duplicates, so we don't have to handle the case of multiple real references
+    // note2: `ring_members_out` is a `std::set`, which contains ordered keys, so the index selected will be correct.
     real_ring_member_index_in_ref_set_out = 0;
 
-    for (const std::uint64_t reference : ring_members_out)
+    for (const legacy_output_index_t reference : ring_members_out)
     {
         if (reference == real_ring_member_index)
             return;
@@ -97,4 +99,14 @@ void LegacyDecoySelectorFlat::get_ring_members(const std::uint64_t real_ring_mem
     }
 }
 //-------------------------------------------------------------------------------------------------------------------
+std::uint64_t LegacyDecoySelectorFlat::get_min_index(const rct::xmr_amount amount) const
+{
+    return m_index_bounds_by_amount.at(amount).first;
+}
+//-------------------------------------------------------------------------------------------------------------------
+std::uint64_t LegacyDecoySelectorFlat::get_max_index(const rct::xmr_amount amount) const
+{
+    return m_index_bounds_by_amount.at(amount).second;
+}
+//-------------------------------------------------------------------------------------------------------------------
 } //namespace sp

src/seraphis_core/legacy_decoy_selector_flat.h

@@ -36,8 +36,8 @@
 //third party headers
 
 //standard headers
-#include <cstdint>
-#include <vector>
+#include <map>
+#include <utility>
 
 //forward declarations
 
@@ -47,29 +47,37 @@ namespace sp
 
 ////
 // LegacyDecoySelectorFlat
-// - get a set of unique legacy ring members, selected from a flat distribution across the range of available enotes
+// - get a set of unique legacy ring members, selected from a flat distribution across the range of available
+//   enotes with the same ledger indexing amount
 ///
 class LegacyDecoySelectorFlat final : public LegacyDecoySelector
 {
 public:
+//member types
+    ///                                       [ ledger amount : {min index, max index} ]
+    using index_bounds_by_amount_t = std::map<rct::xmr_amount, std::pair<std::uint64_t, std::uint64_t>>;
+
 //constructors
     /// default constructor: disabled
     /// normal constructor
-    LegacyDecoySelectorFlat(const std::uint64_t min_index, const std::uint64_t max_index);
+    LegacyDecoySelectorFlat(index_bounds_by_amount_t index_bounds_by_amount);
 
 //destructor: default
 
 //member functions
     /// request a set of ring members from range [min_index, max_index]
-    void get_ring_members(const std::uint64_t real_ring_member_index,
+    void get_ring_members(const legacy_output_index_t real_ring_member_index,
         const std::uint64_t num_ring_members,
-        std::vector<std::uint64_t> &ring_members_out,
+        std::set<legacy_output_index_t> &ring_members_out,
         std::uint64_t &real_ring_member_index_in_ref_set_out) const override;
 
 //member variables
 private:
-    std::uint64_t m_min_index;
-    std::uint64_t m_max_index;
+    index_bounds_by_amount_t m_index_bounds_by_amount;
+
+//member functions (private)
+    std::uint64_t get_min_index(const rct::xmr_amount amount) const;
+    std::uint64_t get_max_index(const rct::xmr_amount amount) const;
 };
 
 } //namespace sp

src/seraphis_core/legacy_enote_types.cpp

@@ -76,6 +76,21 @@ rct::key amount_commitment_ref(const LegacyEnoteVariant &variant)
     return variant.visit(visitor());
 }
 //-------------------------------------------------------------------------------------------------------------------
+rct::xmr_amount cleartext_amount_ref(const LegacyEnoteVariant &variant)
+{
+    struct visitor final : public tools::variant_static_visitor<rct::xmr_amount>
+    {
+        using variant_static_visitor::operator();  //for blank overload
+        rct::xmr_amount operator()(const LegacyEnoteV1 &enote) const { return enote.amount; }
+        rct::xmr_amount operator()(const LegacyEnoteV2 &enote) const { return 0; }
+        rct::xmr_amount operator()(const LegacyEnoteV3 &enote) const { return 0; }
+        rct::xmr_amount operator()(const LegacyEnoteV4 &enote) const { return enote.amount; }
+        rct::xmr_amount operator()(const LegacyEnoteV5 &enote) const { return 0; }
+    };
+
+    return variant.visit(visitor());
+}
+//-------------------------------------------------------------------------------------------------------------------
 LegacyEnoteV1 gen_legacy_enote_v1()
 {
     LegacyEnoteV1 temp;

src/seraphis_core/legacy_enote_types.h

@@ -47,7 +47,7 @@ namespace sp
 {
 
 ////
-// LegacyEnoteV1
+// LegacyEnoteV1 (all pre-RingCT enotes, then post-RingCT pre-viewtag coinbase)
 // - onetime address
 // - cleartext amount
 ///
@@ -104,7 +104,7 @@ struct LegacyEnoteV3 final
 inline std::size_t legacy_enote_v3_size_bytes() { return 2*32 + 8; }
 
 ////
-// LegacyEnoteV4
+// LegacyEnoteV4 (post-viewtag coinbase, also post-viewtag v1 unmixable dust txs)
 // - onetime address
 // - cleartext amount
 // - view tag
@@ -151,10 +151,12 @@ inline std::size_t legacy_enote_v5_size_bytes() { return 2*32 + 8 + sizeof(crypt
 // onetime_address_ref(): get the enote's onetime address
 // amount_commitment_ref(): get the enote's amount commitment (this is a copy because V1 enotes need to
 //                          compute the commitment)
+// cleartext_amount_ref(): get the enote's cleartext amount if applicable, or 0 otherwise
 ///
 using LegacyEnoteVariant = tools::variant<LegacyEnoteV1, LegacyEnoteV2, LegacyEnoteV3, LegacyEnoteV4, LegacyEnoteV5>;
 const rct::key& onetime_address_ref(const LegacyEnoteVariant &variant);
 rct::key amount_commitment_ref(const LegacyEnoteVariant &variant);
+rct::xmr_amount cleartext_amount_ref(const LegacyEnoteVariant &variant);
 
 /**
 * brief: gen_legacy_enote_v1() - generate a legacy v1 enote (all random)

src/seraphis_core/legacy_enote_utils.cpp

@@ -58,6 +58,11 @@ void get_legacy_enote_identifier(const rct::key &onetime_address, const rct::xmr
     sp_hash_to_32(transcript.data(), transcript.size(), identifier_out.bytes);
 }
 //-------------------------------------------------------------------------------------------------------------------
+rct::xmr_amount get_legacy_ledger_indexing_amount(const LegacyEnoteVariant &enote, const bool is_rct)
+{
+    return is_rct ? 0 : cleartext_amount_ref(enote);
+}
+//-------------------------------------------------------------------------------------------------------------------
 void make_legacy_enote_v1(const rct::key &destination_spendkey,
     const rct::key &destination_viewkey,
     const rct::xmr_amount amount,

src/seraphis_core/legacy_enote_utils.h

@@ -59,6 +59,13 @@ namespace sp
 * outparam: identifier_out - H_32(Ko, a)
 */
 void get_legacy_enote_identifier(const rct::key &onetime_address, const rct::xmr_amount amount, rct::key &identifier_out);
+/**
+ * brief: get_legacy_ledger_indexing_amount - get the "amount" key used for indexing this enote in the ledger
+ * param: enote -
+ * param: is_rct - true if this is a RingCT enote
+ * return: the enote's cleartext amount if pre-RingCT or 0 if RingCT
+*/
+rct::xmr_amount get_legacy_ledger_indexing_amount(const LegacyEnoteVariant &enote, const bool is_rct);
 /**
 * brief: make_legacy_enote_v1 - make a v1 legacy enote sending to an address or subaddress
 * param: destination_spendkey - [address: K^s = k^s G] [subaddress: K^{s,i} = (Hn(k^v, i) + k^s) G]

src/seraphis_core/legacy_output_index.h

@@ -0,0 +1,99 @@
+// Copyright (c) 2022, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Convenience type for indexing legacy enotes in (ledger amount, index in amount) form.
+
+#pragma once
+
+//local headers
+
+//third party headers
+
+//standard headers
+#include <cstdint>    // std::uint64_t
+#include <functional> // std::hash
+
+//forward declarations
+namespace rct { typedef uint64_t xmr_amount; }
+
+namespace sp
+{
+////
+// legacy_output_index_t
+// - used to index legacy enotes the same way cryptonote inputs do: (ledger amount, index in amount)
+//
+struct legacy_output_index_t
+{
+    /// the public ledger amount used to index the enote, 0 for everything post RingCT (even coinbase)
+    rct::xmr_amount ledger_indexing_amount;
+    /// the nth position of this enote in the chain for the given amount
+    std::uint64_t index;
+};
+
+inline bool operator==(const legacy_output_index_t &a, const legacy_output_index_t &b)
+{
+    return a.ledger_indexing_amount == b.ledger_indexing_amount && a.index == b.index;
+}
+
+inline bool operator!=(const legacy_output_index_t &a, const legacy_output_index_t &b)
+{
+    return !(a == b);
+}
+
+/// the only purpose of this total ordering is consistency, it can't tell which came first on-chain
+inline bool operator<(const legacy_output_index_t &a, const legacy_output_index_t &b)
+{
+    if (a.ledger_indexing_amount < b.ledger_indexing_amount) return true;
+    else if (a.ledger_indexing_amount > b.ledger_indexing_amount) return false;
+    else if (a.index < b.index) return true;
+    else return false;
+}
+
+inline bool operator>(const legacy_output_index_t &a, const legacy_output_index_t &b)
+{
+    if (a.ledger_indexing_amount > b.ledger_indexing_amount) return true;
+    else if (a.ledger_indexing_amount < b.ledger_indexing_amount) return false;
+    else if (a.index > b.index) return true;
+    else return false;
+}
+} // namespace sp
+
+namespace std
+{
+template <>
+struct hash<sp::legacy_output_index_t>
+{
+    size_t operator()(const sp::legacy_output_index_t x) const
+    {
+        const std::size_t h1{hash<rct::xmr_amount>{}(x.ledger_indexing_amount)};
+        const std::size_t h2{hash<std::uint64_t>{}(x.index)};
+        return h2 ^ (h1 + 0x9e3779b9 + (h2 << 6) + (h2 >> 2)); // see boost::hash_combine
+    }
+};
+} // namespace std
+

src/seraphis_impl/enote_finding_context_legacy.cpp

@@ -63,7 +63,7 @@ void EnoteFindingContextLegacySimple::view_scan_chunk(const LegacyUnscannedChunk
                     blk.block_index,
                     blk.block_timestamp,
                     tx.transaction_id,
-                    tx.total_enotes_before_tx,
+                    tx.legacy_output_index_per_enote,
                     tx.unlock_time,
                     tx.tx_memo,
                     tx.enotes,
@@ -138,7 +138,7 @@ void EnoteFindingContextLegacyMultithreaded::view_scan_chunk(
                             blk.block_index,
                             blk.block_timestamp,
                             tx.transaction_id,
-                            tx.total_enotes_before_tx,
+                            tx.legacy_output_index_per_enote,
                             tx.unlock_time,
                             tx.tx_memo,
                             tx.enotes,