Skip to content

Commit

Permalink
Upgrade evoDB and llmqDB to V2 serializer
Browse files Browse the repository at this point in the history
  • Loading branch information
@panleone
panleone committed Sep 12, 2023
1 parent 0ab665d commit d6424b8
Show file tree
Hide file tree
Showing 8 changed files with 75 additions and 59 deletions.
3 changes: 2 additions & 1 deletion src/dbwrapper.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ static leveldb::Options GetOptions(size_t nCacheSize)
return options;
}

CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe)
CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, int nVersion)
{
penv = nullptr;
readoptions.verify_checksums = true;
Expand All @@ -61,6 +61,7 @@ CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bo
syncoptions.sync = true;
options = GetOptions(nCacheSize);
options.create_if_missing = true;
this->nVersion = nVersion;
if (fMemory) {
penv = leveldb::NewMemEnv(leveldb::Env::Default());
options.env = penv;
Expand Down
78 changes: 42 additions & 36 deletions src/dbwrapper.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,13 @@ class CDBBatch

CDataStream ssKey;
CDataStream ssValue;

size_t size_estimate;

public:
/**
* @param[in] _parent CDBWrapper that this batch is to be submitted to
* @param[in] _nVersion The version used to serialize data.
*/
CDBBatch() : ssKey(SER_DISK, CLIENT_VERSION), ssValue(SER_DISK, CLIENT_VERSION), size_estimate(0) { };
explicit CDBBatch(int nVersion) : ssKey(SER_DISK, nVersion), ssValue(SER_DISK, nVersion), size_estimate(0){};

void Clear()
{
Expand All @@ -80,7 +79,6 @@ class CDBBatch
{
leveldb::Slice slKey(_ssKey.data(), _ssKey.size());

CDataStream ssValue(SER_DISK, CLIENT_VERSION);
ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE);
ssValue << value;
leveldb::Slice slValue(ssValue.data(), ssValue.size());
Expand Down Expand Up @@ -128,12 +126,14 @@ class CDBIterator
{
private:
leveldb::Iterator *piter;
int nVersion;

public:
/**
* @param[in] _piter The original leveldb iterator.
* @param[in] _nVersion The version used to serialize data.
*/
CDBIterator(leveldb::Iterator *_piter) : piter(_piter) { };
CDBIterator(leveldb::Iterator* _piter, int _nVersion) : piter(_piter), nVersion(_nVersion){};
~CDBIterator();

bool Valid();
Expand All @@ -142,7 +142,7 @@ class CDBIterator

template<typename K> void Seek(const K& key)
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
CDataStream ssKey(SER_DISK, nVersion);
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
Seek(ssKey);
Expand Down Expand Up @@ -170,14 +170,14 @@ class CDBIterator
CDataStream GetKey()
{
leveldb::Slice slKey = piter->key();
return CDataStream(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION);
return CDataStream(slKey.data(), slKey.data() + slKey.size(), SER_DISK, nVersion);
}

template<typename V> bool GetValue(V& value)
{
leveldb::Slice slValue = piter->value();
try {
CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION);
CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, nVersion);
ssValue >> value;
} catch(const std::exception& e) {
return false;
Expand Down Expand Up @@ -216,20 +216,24 @@ class CDBWrapper
//! the database itself
leveldb::DB* pdb;

//! the version used to serialize data
int nVersion;

public:
/**
* @param[in] path Location in the filesystem where leveldb data will be stored.
* @param[in] nCacheSize Configures various leveldb cache settings.
* @param[in] fMemory If true, use leveldb's memory environment.
* @param[in] fWipe If true, remove all existing data.
* @param[in] nVersion The version used to serialize data.
*/
CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false);
CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, int nVersion = CLIENT_VERSION);
~CDBWrapper();

template <typename K>
bool ReadDataStream(const K& key, CDataStream& ssValue) const
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
CDataStream ssKey(SER_DISK, nVersion);
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
return ReadDataStream(ssKey, ssValue);
Expand All @@ -247,15 +251,15 @@ class CDBWrapper
LogPrintf("LevelDB read failure: %s\n", status.ToString());
dbwrapper_private::HandleError(status);
}
CDataStream ssValueTmp(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION);
CDataStream ssValueTmp(strValue.data(), strValue.data() + strValue.size(), SER_DISK, nVersion);
ssValue = std::move(ssValueTmp);
return true;
}

template <typename K, typename V>
bool Read(const K& key, V& value) const
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
CDataStream ssKey(SER_DISK, nVersion);
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
return Read(ssKey, value);
Expand All @@ -264,7 +268,7 @@ class CDBWrapper
template <typename V>
bool Read(const CDataStream& ssKey, V& value) const
{
CDataStream ssValue(SER_DISK, CLIENT_VERSION);
CDataStream ssValue(SER_DISK, nVersion);
if (!ReadDataStream(ssKey, ssValue)) {
return false;
}
Expand All @@ -279,15 +283,15 @@ class CDBWrapper
template <typename K, typename V>
bool Write(const K& key, const V& value, bool fSync = false)
{
CDBBatch batch;
CDBBatch batch(nVersion);
batch.Write(key, value);
return WriteBatch(batch, fSync);
}

template <typename K>
bool Exists(const K& key) const
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
CDataStream ssKey(SER_DISK, nVersion);
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
return Exists(ssKey);
Expand All @@ -311,7 +315,7 @@ class CDBWrapper
template <typename K>
bool Erase(const K& key, bool fSync = false)
{
CDBBatch batch;
CDBBatch batch(nVersion);
batch.Erase(key);
return WriteBatch(batch, fSync);
}
Expand All @@ -326,14 +330,14 @@ class CDBWrapper

bool Sync()
{
CDBBatch batch;
CDBBatch batch(nVersion);
return WriteBatch(batch, true);
}

// not exactly clean encapsulation, but it's easiest for now
CDBIterator* NewIterator()
{
return new CDBIterator(pdb->NewIterator(iteroptions));
return new CDBIterator(pdb->NewIterator(iteroptions), nVersion);
}

/**
Expand All @@ -344,7 +348,7 @@ class CDBWrapper
template<typename K>
size_t EstimateSize(const K& key_begin, const K& key_end) const
{
CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
CDataStream ssKey1(SER_DISK, nVersion), ssKey2(SER_DISK, nVersion);
ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey1 << key_begin;
Expand All @@ -363,7 +367,7 @@ class CDBWrapper
template<typename K>
void CompactRange(const K& key_begin, const K& key_end) const
{
CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION);
CDataStream ssKey1(SER_DISK, nVersion), ssKey2(SER_DISK, nVersion);
ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey1 << key_begin;
Expand Down Expand Up @@ -395,12 +399,13 @@ class CDBTransactionIterator
typename CDBTransaction::WritesMap::iterator transactionIt;
std::unique_ptr<ParentIterator> parentIt;
CDataStream parentKey;
int nVersion;
bool curIsParent{false};

public:
CDBTransactionIterator(CDBTransaction& _transaction) :
transaction(_transaction),
parentKey(SER_DISK, CLIENT_VERSION)
CDBTransactionIterator(CDBTransaction& _transaction, int _nVersion) : transaction(_transaction),
parentKey(SER_DISK, _nVersion),
nVersion(_nVersion)
{
transactionIt = transaction.writes.end();
parentIt = std::unique_ptr<ParentIterator>(transaction.parent.NewIterator());
Expand All @@ -417,7 +422,7 @@ class CDBTransactionIterator
template<typename K>
void Seek(const K& key)
{
Seek(CDBTransaction::KeyToDataStream(key));
Seek(CDBTransaction::KeyToDataStream(key, nVersion));
}

void Seek(const CDataStream& ssKey)
Expand Down Expand Up @@ -473,7 +478,7 @@ class CDBTransactionIterator
CDataStream GetKey()
{
if (!Valid()) {
return CDataStream(SER_DISK, CLIENT_VERSION);
return CDataStream(SER_DISK, nVersion);
}
if (curIsParent) {
return parentIt->GetKey();
Expand Down Expand Up @@ -563,10 +568,10 @@ class CDBTransaction {
V value;
};

template<typename K>
static CDataStream KeyToDataStream(const K& key)
template <typename K>
static CDataStream KeyToDataStream(const K& key, int nVersion)
{
CDataStream ssKey(SER_DISK, CLIENT_VERSION);
CDataStream ssKey(SER_DISK, nVersion);
ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE);
ssKey << key;
return ssKey;
Expand All @@ -577,20 +582,21 @@ class CDBTransaction {

WritesMap writes;
DeletesSet deletes;
int nVersion;

public:
CDBTransaction(Parent &_parent, CommitTarget &_commitTarget) : parent(_parent), commitTarget(_commitTarget) {}
CDBTransaction(Parent& _parent, CommitTarget& _commitTarget, int _nVersion) : parent(_parent), commitTarget(_commitTarget), nVersion(_nVersion) {}

template <typename K, typename V>
void Write(const K& key, const V& v)
{
Write(KeyToDataStream(key), v);
Write(KeyToDataStream(key, nVersion), v);
}

template <typename V>
void Write(const CDataStream& ssKey, const V& v)
{
auto valueMemoryUsage = ::GetSerializeSize(v, CLIENT_VERSION);
auto valueMemoryUsage = ::GetSerializeSize(v, nVersion);
if (deletes.erase(ssKey)) {
memoryUsage -= ssKey.size();
}
Expand All @@ -606,7 +612,7 @@ class CDBTransaction {
template <typename K, typename V>
bool Read(const K& key, V& value)
{
return Read(KeyToDataStream(key), value);
return Read(KeyToDataStream(key, nVersion), value);
}

template <typename V>
Expand All @@ -632,7 +638,7 @@ class CDBTransaction {
template <typename K>
bool Exists(const K& key)
{
return Exists(KeyToDataStream(key));
return Exists(KeyToDataStream(key, nVersion));
}

bool Exists(const CDataStream& ssKey)
Expand All @@ -651,7 +657,7 @@ class CDBTransaction {
template <typename K>
void Erase(const K& key)
{
return Erase(KeyToDataStream(key));
return Erase(KeyToDataStream(key, nVersion));
}

void Erase(const CDataStream& ssKey)
Expand Down Expand Up @@ -705,11 +711,11 @@ class CDBTransaction {

CDBTransactionIterator<CDBTransaction>* NewIterator()
{
return new CDBTransactionIterator<CDBTransaction>(*this);
return new CDBTransactionIterator<CDBTransaction>(*this, nVersion);
}
std::unique_ptr<CDBTransactionIterator<CDBTransaction>> NewIteratorUniquePtr()
{
return std::make_unique<CDBTransactionIterator<CDBTransaction>>(*this);
return std::make_unique<CDBTransactionIterator<CDBTransaction>>(*this, nVersion);
}
};

Expand Down
11 changes: 7 additions & 4 deletions src/evo/deterministicmns.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,11 @@
#include "evo/evodb.h"
#include "evo/providertx.h"
#include "llmq/quorums_commitment.h"
#include "netaddress.h"
#include "saltedhasher.h"
#include "serialize.h"
#include "sync.h"
#include "version.h"

#include <immer/map.hpp>
#include <immer/map_transient.hpp>
Expand Down Expand Up @@ -431,12 +434,12 @@ class CDeterministicMNList
template <typename T>
bool HasUniqueProperty(const T& v) const
{
return mnUniquePropertyMap.count(::SerializeHash(v)) != 0;
return mnUniquePropertyMap.count(::SerializeHash(v, SER_GETHASH, PROTOCOL_VERSION | ADDRV2_FORMAT)) != 0;
}
template <typename T>
CDeterministicMNCPtr GetUniquePropertyMN(const T& v) const
{
auto p = mnUniquePropertyMap.find(::SerializeHash(v));
auto p = mnUniquePropertyMap.find(::SerializeHash(v, SER_GETHASH, PROTOCOL_VERSION | ADDRV2_FORMAT));
if (!p) {
return nullptr;
}
Expand All @@ -450,7 +453,7 @@ class CDeterministicMNList
static const T nullValue;
assert(v != nullValue);

auto hash = ::SerializeHash(v);
auto hash = ::SerializeHash(v, SER_GETHASH, PROTOCOL_VERSION | ADDRV2_FORMAT);
auto oldEntry = mnUniquePropertyMap.find(hash);
assert(!oldEntry || oldEntry->first == dmn->proTxHash);
std::pair<uint256, uint32_t> newEntry(dmn->proTxHash, 1);
Expand All @@ -465,7 +468,7 @@ class CDeterministicMNList
static const T nullValue;
assert(oldValue != nullValue);

auto oldHash = ::SerializeHash(oldValue);
auto oldHash = ::SerializeHash(oldValue, SER_GETHASH, PROTOCOL_VERSION | ADDRV2_FORMAT);
auto p = mnUniquePropertyMap.find(oldHash);
assert(p && p->first == dmn->proTxHash);
if (p->second == 1) {
Expand Down
11 changes: 6 additions & 5 deletions src/evo/evodb.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include "evodb.h"
#include "clientversion.h"
#include "netaddress.h"

std::unique_ptr<CEvoDB> evoDb;

Expand Down Expand Up @@ -32,11 +34,10 @@ void CEvoDBScopedCommitter::Rollback()
evoDB.RollbackCurTransaction();
}

CEvoDB::CEvoDB(size_t nCacheSize, bool fMemory, bool fWipe) :
db(fMemory ? "" : (GetDataDir() / "evodb"), nCacheSize, fMemory, fWipe),
rootBatch(),
rootDBTransaction(db, rootBatch),
curDBTransaction(rootDBTransaction, rootDBTransaction)
CEvoDB::CEvoDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(fMemory ? "" : (GetDataDir() / "evodb"), nCacheSize, fMemory, fWipe, CLIENT_VERSION | ADDRV2_FORMAT),
rootBatch(CLIENT_VERSION | ADDRV2_FORMAT),
rootDBTransaction(db, rootBatch, CLIENT_VERSION | ADDRV2_FORMAT),
curDBTransaction(rootDBTransaction, rootDBTransaction, CLIENT_VERSION | ADDRV2_FORMAT)
{
}

Expand Down
Loading

0 comments on commit d6424b8

Please sign in to comment.