From a30481b154f49cbb9ab8387575f7141ca30c7c1a Mon Sep 17 00:00:00 2001 From: Peter Shugalev Date: Sat, 13 May 2023 12:37:42 +0200 Subject: [PATCH 01/12] Use single instance of CWalletDB in CSparkWallet --- src/spark/sparkwallet.cpp | 48 +++++++++++++++------------------------ src/spark/sparkwallet.h | 20 ++++++++-------- src/wallet/rpcwallet.cpp | 4 ++-- src/wallet/wallet.cpp | 2 +- src/wallet/wallet.h | 1 - 5 files changed, 31 insertions(+), 44 deletions(-) diff --git a/src/spark/sparkwallet.cpp b/src/spark/sparkwallet.cpp index 7c4d6531f6..082e0b2b4f 100644 --- a/src/spark/sparkwallet.cpp +++ b/src/spark/sparkwallet.cpp @@ -13,11 +13,7 @@ const uint32_t DEFAULT_SPARK_NCOUNT = 1; -CSparkWallet::CSparkWallet(const std::string& strWalletFile) { - - CWalletDB walletdb(strWalletFile); - this->strWalletFile = strWalletFile; - +CSparkWallet::CSparkWallet(const std::string &strWalletFile) : walletdb(strWalletFile) { const spark::Params* params = spark::Params::get_default(); fullViewKey = spark::FullViewKey(params); @@ -66,11 +62,11 @@ CSparkWallet::~CSparkWallet() { delete (ParallelOpThreadPool*)threadPool; } -void CSparkWallet::resetDiversifierFromDB(CWalletDB& walletdb) { +void CSparkWallet::resetDiversifierFromDB() { walletdb.readDiversifier(lastDiversifier); } -void CSparkWallet::updatetDiversifierInDB(CWalletDB& walletdb) { +void CSparkWallet::updatetDiversifierInDB() { walletdb.writeDiversifier(lastDiversifier); } @@ -173,8 +169,7 @@ spark::Address CSparkWallet::generateNewAddress() { spark::Address address(viewKey, lastDiversifier); addresses[lastDiversifier] = address; - CWalletDB walletdb(strWalletFile); - updatetDiversifierInDB(walletdb); + updatetDiversifierInDB(); return address; } @@ -281,9 +276,8 @@ std::vector CSparkWallet::ListSparkMints(bool fUnusedOnly, bool return setMints; } -std::list CSparkWallet::ListSparkSpends() const { +std::list CSparkWallet::ListSparkSpends() { std::list result; - CWalletDB walletdb(strWalletFile); walletdb.ListSparkSpends(result); return result; } @@ -316,7 +310,7 @@ spark::Coin CSparkWallet::getCoinFromLTag(const GroupElement& lTag) const { } -void CSparkWallet::clearAllMints(CWalletDB& walletdb) { +void CSparkWallet::clearAllMints() { LOCK(cs_spark_wallet); for (auto& itr : coinMeta) { walletdb.EraseSparkMint(itr.first); @@ -327,13 +321,13 @@ void CSparkWallet::clearAllMints(CWalletDB& walletdb) { walletdb.writeDiversifier(lastDiversifier); } -void CSparkWallet::eraseMint(const uint256& hash, CWalletDB& walletdb) { +void CSparkWallet::eraseMint(const uint256& hash) { LOCK(cs_spark_wallet); walletdb.EraseSparkMint(hash); coinMeta.erase(hash); } -void CSparkWallet::addOrUpdateMint(const CSparkMintMeta& mint, const uint256& lTagHash, CWalletDB& walletdb) { +void CSparkWallet::addOrUpdateMint(const CSparkMintMeta& mint, const uint256& lTagHash) { LOCK(cs_spark_wallet); if (mint.i > lastDiversifier) { @@ -344,24 +338,23 @@ void CSparkWallet::addOrUpdateMint(const CSparkMintMeta& mint, const uint256& lT walletdb.WriteSparkMint(lTagHash, mint); } -void CSparkWallet::updateMint(const CSparkMintMeta& mint, CWalletDB& walletdb) { +void CSparkWallet::updateMint(const CSparkMintMeta& mint) { LOCK(cs_spark_wallet); for (const auto& coin : coinMeta) { if (mint == coin.second) { - addOrUpdateMint(mint, coin.first, walletdb); + addOrUpdateMint(mint, coin.first); } } } void CSparkWallet::setCoinUnused(const GroupElement& lTag) { LOCK(cs_spark_wallet); - CWalletDB walletdb(strWalletFile); uint256 lTagHash = primitives::GetLTagHash(lTag); CSparkMintMeta coinMeta = getMintMeta(lTagHash); if (coinMeta != CSparkMintMeta()) { coinMeta.isUsed = false; - updateMint(coinMeta, walletdb); + updateMint(coinMeta); } } @@ -402,12 +395,11 @@ void CSparkWallet::UpdateSpendState(const GroupElement& lTag, const uint256& lTa spendEntry.hashTx = txHash; spendEntry.amount = mintMeta.v; - CWalletDB walletdb(strWalletFile); walletdb.WriteSparkSpendEntry(spendEntry); if (fUpdateMint) { mintMeta.isUsed = true; - addOrUpdateMint(mintMeta, lTagHash, walletdb); + addOrUpdateMint(mintMeta, lTagHash); } // pwalletMain->NotifyZerocoinChanged( @@ -502,7 +494,7 @@ CAmount CSparkWallet::getMySpendAmount(const std::vector& lTags) c return result; } -void CSparkWallet::UpdateMintState(const std::vector& coins, const uint256& txHash, CWalletDB& walletdb) { +void CSparkWallet::UpdateMintState(const std::vector& coins, const uint256& txHash) { spark::CSparkState *sparkState = spark::CSparkState::GetState(); for (auto coin : coins) { try { @@ -528,7 +520,7 @@ void CSparkWallet::UpdateMintState(const std::vector& coins, const } uint256 lTagHash = primitives::GetLTagHash(recoveredCoinData.T); - addOrUpdateMint(mintMeta, lTagHash, walletdb); + addOrUpdateMint(mintMeta, lTagHash); if (mintMeta.isUsed) { uint256 spendTxHash; @@ -553,8 +545,7 @@ void CSparkWallet::UpdateMintState(const std::vector& coins, const void CSparkWallet::UpdateMintStateFromMempool(const std::vector& coins, const uint256& txHash) { ((ParallelOpThreadPool*)threadPool)->PostTask([=]() mutable { LOCK(cs_spark_wallet); - CWalletDB walletdb(strWalletFile); - UpdateMintState(coins, txHash, walletdb); + UpdateMintState(coins, txHash); }); } @@ -563,12 +554,11 @@ void CSparkWallet::UpdateMintStateFromBlock(const CBlock& block) { ((ParallelOpThreadPool*)threadPool)->PostTask([=] () mutable { LOCK(cs_spark_wallet); - CWalletDB walletdb(strWalletFile); for (const auto& tx : transactions) { if (tx->IsSparkTransaction()) { auto coins = spark::GetSparkMintCoins(*tx); uint256 txHash = tx->GetHash(); - UpdateMintState(coins, txHash, walletdb); + UpdateMintState(coins, txHash); } } }); @@ -580,10 +570,9 @@ void CSparkWallet::RemoveSparkMints(const std::vector& mints) { spark::IdentifiedCoinData identifiedCoinData = coin.identify(this->viewKey); spark::RecoveredCoinData recoveredCoinData = coin.recover(this->fullViewKey, identifiedCoinData); - CWalletDB walletdb(strWalletFile); uint256 lTagHash = primitives::GetLTagHash(recoveredCoinData.T); - eraseMint(lTagHash, walletdb); + eraseMint(lTagHash); } catch (const std::runtime_error &e) { continue; } @@ -598,8 +587,7 @@ void CSparkWallet::RemoveSparkSpends(const std::unordered_map if (coinMeta.count(lTagHash)) { auto mintMeta = coinMeta[lTagHash]; mintMeta.isUsed = false; - CWalletDB walletdb(strWalletFile); - addOrUpdateMint(mintMeta, lTagHash, walletdb); + addOrUpdateMint(mintMeta, lTagHash); walletdb.EraseSparkSpendEntry(spend.first); } } diff --git a/src/spark/sparkwallet.h b/src/spark/sparkwallet.h index 7153f0b360..32b56bf78a 100644 --- a/src/spark/sparkwallet.h +++ b/src/spark/sparkwallet.h @@ -21,16 +21,16 @@ const uint32_t BIP44_SPARK_INDEX = 0x6; class CSparkWallet { public: - CSparkWallet(const std::string& strWalletFile); + CSparkWallet(const std::string &strWalletFile); ~CSparkWallet(); // increment diversifier and generate address for that spark::Address generateNextAddress(); spark::Address generateNewAddress(); spark::Address getDefaultAddress(); // assign difersifier to the value from db - void resetDiversifierFromDB(CWalletDB& walletdb); + void resetDiversifierFromDB(); // assign diversifier in to to current value - void updatetDiversifierInDB(CWalletDB& walletdb); + void updatetDiversifierInDB(); // functions for key set generation spark::SpendKey generateSpendKey(const spark::Params* params); @@ -45,7 +45,7 @@ class CSparkWallet { // list spark mint, mint metadata in memory and in db should be the same at this moment, so get from memory std::vector ListSparkMints(bool fUnusedOnly = false, bool fMatureOnly = false) const; - std::list ListSparkSpends() const; + std::list ListSparkSpends(); std::unordered_map getMintMap() const; // generate spark Coin from meta data spark::Coin getCoinFromMeta(const CSparkMintMeta& meta) const; @@ -62,12 +62,12 @@ class CSparkWallet { CAmount getAddressUnconfirmedBalance(const spark::Address& address); // function to be used for zap wallet - void clearAllMints(CWalletDB& walletdb); + void clearAllMints(); // erase mint meta data from memory and from db - void eraseMint(const uint256& hash, CWalletDB& walletdb); + void eraseMint(const uint256& hash); // add mint meta data to memory and to db - void addOrUpdateMint(const CSparkMintMeta& mint, const uint256& lTagHash, CWalletDB& walletdb); - void updateMint(const CSparkMintMeta& mint, CWalletDB& walletdb); + void addOrUpdateMint(const CSparkMintMeta& mint, const uint256& lTagHash); + void updateMint(const CSparkMintMeta& mint); void setCoinUnused(const GroupElement& lTag); @@ -87,7 +87,7 @@ class CSparkWallet { void UpdateSpendState(const GroupElement& lTag, const uint256& txHash, bool fUpdateMint = true); void UpdateSpendStateFromMempool(const std::vector& lTags, const uint256& txHash, bool fUpdateMint = true); void UpdateSpendStateFromBlock(const CBlock& block); - void UpdateMintState(const std::vector& coins, const uint256& txHash, CWalletDB& walletdb); + void UpdateMintState(const std::vector& coins, const uint256& txHash); void UpdateMintStateFromMempool(const std::vector& coins, const uint256& txHash); void UpdateMintStateFromBlock(const CBlock& block); void RemoveSparkMints(const std::vector& mints); @@ -136,7 +136,7 @@ class CSparkWallet { mutable CCriticalSection cs_spark_wallet; private: - std::string strWalletFile; + CWalletDB walletdb; // this is latest used diversifier int32_t lastDiversifier; diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index aab51b2bca..1db1d2d455 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3467,7 +3467,7 @@ UniValue resetsparkmints(const JSONRPCRequest& request) { BOOST_FOREACH(CSparkMintMeta& mint, listMints) { mint.isUsed = false; mint.nHeight = -1; - pwallet->sparkWallet->updateMint(mint, walletdb); + pwallet->sparkWallet->updateMint(mint); } return NullUniValue; @@ -3498,7 +3498,7 @@ UniValue setsparkmintstatus(const JSONRPCRequest& request) { if (coinMeta != CSparkMintMeta()) { coinMeta.isUsed = fStatus; - pwallet->sparkWallet->updateMint(coinMeta, walletdb); + pwallet->sparkWallet->updateMint(coinMeta); } return NullUniValue; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index c29a7d11e3..e9a9f98a87 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -7658,7 +7658,7 @@ void CWallet::HandleSparkTransaction(CWalletTx const & wtx) { // get spark coins and add into wallet std::vector coins = spark::GetSparkMintCoins(*wtx.tx); - sparkWallet->UpdateMintState(coins, txHash, walletdb); + sparkWallet->UpdateMintState(coins, txHash); } void CWallet::LabelSendingPcode(bip47::CPaymentCode const & pcode_, std::string const & label, bool remove) diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 33148312bc..4d983de711 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -755,7 +755,6 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface unsigned int nMasterKeyMaxID; std::unique_ptr zwallet; - std::unique_ptr sparkWallet; CWallet() From 4e1b20419b905a3ea385c5fa39bc2ccbd911cda7 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Thu, 9 Mar 2017 20:11:36 +0000 Subject: [PATCH 02/12] wallet: Move nAccountingEntryNumber from static/global to CWallet --- src/wallet/wallet.cpp | 2 +- src/wallet/wallet.h | 2 ++ src/wallet/walletdb.cpp | 12 +++--------- src/wallet/walletdb.h | 1 - 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index e9a9f98a87..f0a4741eae 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -6009,7 +6009,7 @@ bool CWallet::AddAccountingEntry(const CAccountingEntry& acentry) bool CWallet::AddAccountingEntry(const CAccountingEntry& acentry, CWalletDB *pwalletdb) { - if (!pwalletdb->WriteAccountingEntry_Backend(acentry)) + if (!pwalletdb->WriteAccountingEntry(++nAccountingEntryNumber, acentry)) return false; laccentries.push_back(acentry); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 4d983de711..faaa5f0a3c 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -782,6 +782,7 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface nMasterKeyMaxID = 0; pwalletdbEncryption = NULL; nOrderPosNext = 0; + nAccountingEntryNumber = 0; nNextResend = 0; nLastResend = 0; nTimeFirstKey = 0; @@ -802,6 +803,7 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface TxItems wtxOrdered; int64_t nOrderPosNext; + uint64_t nAccountingEntryNumber; std::map mapRequestCount; std::map mapAddressBook; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index c9efbdc4a9..36cf92c4cf 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -24,8 +24,6 @@ #include #include -static uint64_t nAccountingEntryNumber = 0; - static std::atomic nWalletDBUpdateCounter; // @@ -211,11 +209,6 @@ bool CWalletDB::WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccount return Write(std::make_pair(std::string("acentry"), std::make_pair(acentry.strAccount, nAccEntryNum)), acentry); } -bool CWalletDB::WriteAccountingEntry_Backend(const CAccountingEntry& acentry) -{ - return WriteAccountingEntry(++nAccountingEntryNumber, acentry); -} - CAmount CWalletDB::GetAccountCreditDebit(const std::string& strAccount) { std::list entries; @@ -630,8 +623,9 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, ssKey >> strAccount; uint64_t nNumber; ssKey >> nNumber; - if (nNumber > nAccountingEntryNumber) - nAccountingEntryNumber = nNumber; + if (nNumber > pwallet->nAccountingEntryNumber) { + pwallet->nAccountingEntryNumber = nNumber; + } if (!wss.fAnyUnordered) { diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index ac5093ac88..4bfc0afb9f 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -228,7 +228,6 @@ class CWalletDB : public CDB /// This writes directly to the database, and will not update the CWallet's cached accounting entries! /// Use wallet.AddAccountingEntry instead, to write *and* update its caches. bool WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry); - bool WriteAccountingEntry_Backend(const CAccountingEntry& acentry); bool ReadAccount(const std::string& strAccount, CAccount& account); bool WriteAccount(const std::string& strAccount, const CAccount& account); From c747a7aa2324334076ab66bedcd155727f7a4890 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Tue, 3 Jan 2017 20:33:55 +0000 Subject: [PATCH 03/12] Bugfix: ForceSetArg should replace entr(ies) in mapMultiArgs, not append --- src/util.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/util.cpp b/src/util.cpp index 2eb872995c..a44d25321a 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -465,6 +465,8 @@ void ForceSetArg(const std::string& strArg, const std::string& strValue) { LOCK(cs_args); mapArgs[strArg] = strValue; + mapMultiArgs[strArg].clear(); + mapMultiArgs[strArg].push_back(strValue); } From e1a67d86b410ef0004de22dbe6791cb94cb4116d Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Thu, 9 Mar 2017 20:56:58 +0000 Subject: [PATCH 04/12] CWalletDB: Store the update counter per wallet --- src/wallet/db.cpp | 10 +++++++++ src/wallet/db.h | 50 +++++++++++++++++++++++++++++++++++++++++ src/wallet/wallet.cpp | 2 +- src/wallet/walletdb.cpp | 25 +++++++-------------- src/wallet/walletdb.h | 41 +++++++++++++++++++++++++++++---- 5 files changed, 106 insertions(+), 22 deletions(-) diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index c0fd858469..b02d27668c 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -307,6 +307,16 @@ void CDB::Flush() bitdb.dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); } +void CWalletDBWrapper::IncrementUpdateCounter() +{ + ++nUpdateCounter; +} + +unsigned int CWalletDBWrapper::GetUpdateCounter() +{ + return nUpdateCounter.load(); +} + void CDB::Close() { if (!pdb) diff --git a/src/wallet/db.h b/src/wallet/db.h index ea56200323..eb450b6c62 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -87,6 +87,56 @@ class CDBEnv extern CDBEnv bitdb; +/** An instance of this class represents one database. + * For BerkeleyDB this is just a (env, strFile) tuple. + **/ +class CWalletDBWrapper +{ + friend class CDB; +public: + /** Create dummy DB handle */ + CWalletDBWrapper(): env(nullptr) + { + } + + /** Create DB handle to real database */ + CWalletDBWrapper(CDBEnv *env_in, const std::string &strFile_in): + env(env_in), strFile(strFile_in) + { + } + + /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero + */ + bool Rewrite(const char* pszSkip=nullptr); + + /** Back up the entire database to a file. + */ + bool Backup(const std::string& strDest); + + /** Get a name for this database, for debugging etc. + */ + std::string GetName() const { return strFile; } + + /** Make sure all changes are flushed to disk. + */ + void Flush(bool shutdown); + + void IncrementUpdateCounter(); + unsigned int GetUpdateCounter(); + +private: + /** BerkeleyDB specific */ + CDBEnv *env; + std::string strFile; + std::atomic nUpdateCounter; + + /** Return whether this database handle is a dummy for testing. + * Only to be used at a low level, application should ideally not care + * about this. + */ + bool IsDummy() { return env == nullptr; } +}; + /** RAII class that provides access to a Berkeley database */ class CDB diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index f0a4741eae..5ad9cfd170 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -7038,7 +7038,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) walletInstance->ScanForWalletTransactions(pindexRescan, true, fRecoverMnemonic); LogPrintf(" rescan %15dms\n", GetTimeMillis() - nStart); walletInstance->SetBestChain(chainActive.GetLocator()); - CWalletDB::IncrementUpdateCounter(); + walletInstance->dbw->IncrementUpdateCounter(); // Restore wallet transaction metadata after -zapwallettxes=1 if (GetBoolArg("-zapwallettxes", false)) diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 36cf92c4cf..31023a4f41 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -24,8 +24,6 @@ #include #include -static std::atomic nWalletDBUpdateCounter; - // // CWalletDB // @@ -1128,20 +1126,23 @@ void ThreadFlushWalletDB() if (!GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) return; - unsigned int nLastSeen = CWalletDB::GetUpdateCounter(); - unsigned int nLastFlushed = CWalletDB::GetUpdateCounter(); + CWalletDBWrapper& dbh = pwalletMain->GetDBHandle(); + + static unsigned int nLastSeen = dbh.GetUpdateCounter(); + static unsigned int nLastFlushed = dbh.GetUpdateCounter(); + static int64_t nLastWalletUpdate = GetTime(); int64_t nLastWalletUpdate = GetTime(); while (true) { MilliSleep(500); - if (nLastSeen != CWalletDB::GetUpdateCounter()) + if (nLastSeen != dbh.GetUpdateCounter()) { - nLastSeen = CWalletDB::GetUpdateCounter(); + nLastSeen = dbh.GetUpdateCounter(); nLastWalletUpdate = GetTime(); } - if (nLastFlushed != CWalletDB::GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2) + if (nLastFlushed != dbh.GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2) { TRY_LOCK(bitdb.cs_db,lockDb); if (lockDb) @@ -1688,16 +1689,6 @@ bool CWalletDB::UnarchiveSigmaMint(const uint256& hashPubcoin, CSigmaEntry& sigm return true; } -void CWalletDB::IncrementUpdateCounter() -{ - nWalletDBUpdateCounter++; -} - -unsigned int CWalletDB::GetUpdateCounter() -{ - return nWalletDBUpdateCounter; -} - std::unordered_map CWalletDB::ListSparkMints() { std::unordered_map listMints; diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 4bfc0afb9f..5b6d31e5cb 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -186,8 +186,31 @@ class CKeyMetadata /** Access to the wallet database */ class CWalletDB : public CDB { +private: + template + bool WriteIC(const K& key, const T& value, bool fOverwrite = true) + { + if (!batch.Write(key, value, fOverwrite)) { + return false; + } + m_dbw.IncrementUpdateCounter(); + return true; + } + + template + bool EraseIC(const K& key) + { + if (!batch.Erase(key)) { + return false; + } + m_dbw.IncrementUpdateCounter(); + return true; + } + public: - CWalletDB(const std::string& strFilename, const char* pszMode = "r+", bool fFlushOnClose = true) : CDB(strFilename, pszMode, fFlushOnClose) + CWalletDB(CWalletDBWrapper& dbw, const char* pszMode = "r+", bool _fFlushOnClose = true) : + batch(dbw, pszMode, _fFlushOnClose), + m_dbw(dbw) { } @@ -316,9 +339,6 @@ class CWalletDB : public CDB bool WriteHDChain(const CHDChain& chain); bool WriteMnemonic(const MnemonicContainer& mnContainer); - static void IncrementUpdateCounter(); - static unsigned int GetUpdateCounter(); - #ifdef ENABLE_ELYSIUM public: @@ -434,6 +454,19 @@ class CWalletDB : public CDB { return Erase(std::make_pair(std::string("exodus_mint_id_v1"), k)); } + //! Begin a new transaction + bool TxnBegin(); + //! Commit current transaction + bool TxnCommit(); + //! Abort current transaction + bool TxnAbort(); + //! Read wallet version + bool ReadVersion(int& nVersion); + //! Write wallet version + bool WriteVersion(int nVersion); +private: + CDB batch; + CWalletDBWrapper& m_dbw; template bool ReadElysiumMintV1(const K& k, V& v) From f481533b8eb698fd2fd2c36ec9d9a72eb59a62e9 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Fri, 9 Sep 2016 08:42:30 +0000 Subject: [PATCH 05/12] Wallet: Replace pwalletMain with a vector of wallet pointers --- src/init.cpp | 21 +++++--- src/qt/bitcoin.cpp | 5 +- src/wallet/db.cpp | 5 -- src/wallet/db.h | 13 +++-- src/wallet/rpcwallet.cpp | 3 +- src/wallet/test/wallet_test_fixture.cpp | 2 + src/wallet/test/wallet_tests.cpp | 24 ++++++--- src/wallet/wallet.cpp | 6 +-- src/wallet/wallet.h | 3 +- src/wallet/walletdb.cpp | 68 ++++++------------------- 10 files changed, 64 insertions(+), 86 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index b8f7234ba7..cc290b33db 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -259,8 +259,9 @@ void Shutdown() BatchProofContainer::get_instance()->verify(); #ifdef ENABLE_WALLET - if (pwalletMain) - pwalletMain->Flush(false); + for (CWalletRef pwallet : vpwallets) { + pwallet->Flush(false); + } #endif GenerateBitcoins(false, 0, Params()); MapPort(false); @@ -311,8 +312,9 @@ void Shutdown() #endif #ifdef ENABLE_WALLET - if (pwalletMain) - pwalletMain->Flush(true); + for (CWalletRef pwallet : vpwallets) { + pwallet->Flush(true); + } #endif #if ENABLE_ZMQ @@ -345,8 +347,10 @@ void Shutdown() #endif UnregisterAllValidationInterfaces(); #ifdef ENABLE_WALLET - delete pwalletMain; - pwalletMain = NULL; + for (CWalletRef pwallet : vpwallets) { + delete pwallet; + } + vpwallets.clear(); #endif globalVerifyHandle.reset(); ECC_Stop(); @@ -2170,8 +2174,9 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) uiInterface.InitMessage(_("Done loading")); #ifdef ENABLE_WALLET - if (pwalletMain) - pwalletMain->postInitProcess(threadGroup); + for (CWalletRef pwallet : vpwallets) { + pwallet->postInitProcess(scheduler); + } #endif return !fRequestShutdown; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index cbea7f2b5b..21d4dd98b1 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -480,9 +480,10 @@ void BitcoinApplication::initializeResult(int retval) window->setClientModel(clientModel); #ifdef ENABLE_WALLET - if(pwalletMain) + // TODO: Expose secondary wallets + if (!vpwallets.empty()) { - walletModel = new WalletModel(platformStyle, pwalletMain, optionsModel); + walletModel = new WalletModel(platformStyle, vpwallets[0], optionsModel); window->addWallet(BitcoinGUI::DEFAULT_WALLET, walletModel); window->setCurrentWallet(BitcoinGUI::DEFAULT_WALLET); diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index b02d27668c..996c01271e 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -312,11 +312,6 @@ void CWalletDBWrapper::IncrementUpdateCounter() ++nUpdateCounter; } -unsigned int CWalletDBWrapper::GetUpdateCounter() -{ - return nUpdateCounter.load(); -} - void CDB::Close() { if (!pdb) diff --git a/src/wallet/db.h b/src/wallet/db.h index eb450b6c62..5c92218579 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -95,13 +95,13 @@ class CWalletDBWrapper friend class CDB; public: /** Create dummy DB handle */ - CWalletDBWrapper(): env(nullptr) + CWalletDBWrapper() : nLastSeen(0), nLastFlushed(0), nLastWalletUpdate(0), env(nullptr) { } /** Create DB handle to real database */ - CWalletDBWrapper(CDBEnv *env_in, const std::string &strFile_in): - env(env_in), strFile(strFile_in) + CWalletDBWrapper(CDBEnv *env_in, const std::string &strFile_in) : + nLastSeen(0), nLastFlushed(0), nLastWalletUpdate(0), env(env_in), strFile(strFile_in) { } @@ -122,13 +122,16 @@ class CWalletDBWrapper void Flush(bool shutdown); void IncrementUpdateCounter(); - unsigned int GetUpdateCounter(); + + std::atomic nUpdateCounter; + unsigned int nLastSeen; + unsigned int nLastFlushed; + int64_t nLastWalletUpdate; private: /** BerkeleyDB specific */ CDBEnv *env; std::string strFile; - std::atomic nUpdateCounter; /** Return whether this database handle is a dummy for testing. * Only to be used at a low level, application should ideally not care diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 1db1d2d455..07dc945b27 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -43,7 +43,8 @@ static CCriticalSection cs_nWalletUnlockTime; CWallet *GetWalletForJSONRPCRequest(const JSONRPCRequest& request) { - return pwalletMain; + // TODO: Some way to access secondary wallets + return vpwallets.empty() ? nullptr : vpwallets[0]; } std::string HelpRequiringPassphrase(CWallet * const pwallet) diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index d0891a97b1..7c5ba24c7b 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -8,6 +8,8 @@ #include "wallet/db.h" #include "wallet/wallet.h" +CWallet *pwalletMain; + WalletTestingSetup::WalletTestingSetup(const std::string& chainName): TestingSetup(chainName) { diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index e17a05f90d..ca0382c9d2 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -18,6 +18,8 @@ #include #include +extern CWallet* pwalletMain; + extern UniValue importmulti(const JSONRPCRequest& request); extern UniValue dumpwallet(const JSONRPCRequest& request); extern UniValue importwallet(const JSONRPCRequest& request); @@ -402,8 +404,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) // after. { CWallet wallet; - CWallet *backup = ::pwalletMain; - ::pwalletMain = &wallet; + vpwallets.insert(vpwallets.begin(), &wallet); UniValue keys; keys.setArray(); UniValue key; @@ -425,8 +426,16 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) request.params.push_back(keys); UniValue response = importmulti(request); - BOOST_CHECK_EQUAL(response.write(), strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Failed to rescan before time %d, transactions may be missing.\"}},{\"success\":true}]", newTip->GetBlockTimeMax())); - ::pwalletMain = backup; + BOOST_CHECK_EQUAL(response.write(), + strprintf("[{\"success\":false,\"error\":{\"code\":-1,\"message\":\"Rescan failed for key with creation " + "timestamp %d. There was an error reading a block from time %d, which is after or within %d " + "seconds of key creation, and could contain transactions pertaining to the key. As a result, " + "transactions and coins using this key may not appear in the wallet. This error could be caused " + "by pruning or data corruption (see bitcoind log for details) and could be dealt with by " + "downloading and rescanning the relevant blocks (see -reindex and -rescan " + "options).\"}},{\"success\":true}]", + 0, oldTip->GetBlockTimeMax(), TIMESTAMP_WINDOW)); + vpwallets.erase(vpwallets.begin()); } } @@ -436,7 +445,6 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) // than or equal to key birthday. BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) { - CWallet *pwalletMainBackup = ::pwalletMain; LOCK(cs_main); // Create two blocks with same timestamp to verify that importwallet rescan @@ -466,7 +474,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) JSONRPCRequest request; request.params.setArray(); request.params.push_back("wallet.backup"); - ::pwalletMain = &wallet; + vpwallets.insert(vpwallets.begin(), &wallet); ::dumpwallet(request); } @@ -481,7 +489,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) JSONRPCRequest request; request.params.setArray(); request.params.push_back("wallet.backup"); - ::pwalletMain = &wallet; + vpwallets[0] = &wallet; ::importwallet(request); BOOST_CHECK_EQUAL(wallet.mapWallet.size(), 3); @@ -494,7 +502,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) } SetMockTime(0); - ::pwalletMain = pwalletMainBackup; + vpwallets.erase(vpwallets.begin()); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 5ad9cfd170..750fdb5f86 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -60,8 +60,7 @@ #include "bip47/paymentcode.h" #include "bip47/bip47utils.h" -CWallet* pwalletMain = NULL; - +std::vector vpwallets; /** Transaction fee set by the user */ CFeeRate payTxFee(DEFAULT_TRANSACTION_FEE); unsigned int nTxConfirmTarget = DEFAULT_TX_CONFIRM_TARGET; @@ -7102,7 +7101,6 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) bool CWallet::InitLoadWallet() { if (GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) { - pwalletMain = NULL; LogPrintf("Wallet disabled!\n"); return true; } @@ -7119,7 +7117,7 @@ bool CWallet::InitLoadWallet() if (!pwallet) { return false; } - pwalletMain = pwallet; + vpwallets.push_back(pwallet); return true; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index faaa5f0a3c..8759377d16 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -46,7 +46,8 @@ #include #include -extern CWallet* pwalletMain; +typedef CWallet* CWalletRef; +extern std::vector vpwallets; /** * Settings diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 31023a4f41..4e0e67cfdd 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -1116,68 +1116,32 @@ DBErrors CWalletDB::ZapWalletTx(CWallet* pwallet, std::vector& vWtx) void ThreadFlushWalletDB() { - // Make this thread recognisable as the wallet flushing thread - RenameThread("firo-wallet"); - - static bool fOneThread; - if (fOneThread) + static std::atomic fOneThread; + if (fOneThread.exchange(true)) { return; - fOneThread = true; - if (!GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) + } + if (!GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { return; + } - CWalletDBWrapper& dbh = pwalletMain->GetDBHandle(); + for (CWalletRef pwallet : vpwallets) { + CWalletDBWrapper& dbh = pwallet->GetDBHandle(); - static unsigned int nLastSeen = dbh.GetUpdateCounter(); - static unsigned int nLastFlushed = dbh.GetUpdateCounter(); - static int64_t nLastWalletUpdate = GetTime(); - int64_t nLastWalletUpdate = GetTime(); - while (true) - { - MilliSleep(500); + unsigned int nUpdateCounter = dbh.nUpdateCounter; - if (nLastSeen != dbh.GetUpdateCounter()) - { - nLastSeen = dbh.GetUpdateCounter(); - nLastWalletUpdate = GetTime(); + if (dbh.nLastSeen != nUpdateCounter) { + dbh.nLastSeen = nUpdateCounter; + dbh.nLastWalletUpdate = GetTime(); } - if (nLastFlushed != dbh.GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2) - { - TRY_LOCK(bitdb.cs_db,lockDb); - if (lockDb) - { - // Don't do this if any databases are in use - int nRefCount = 0; - std::map::iterator mi = bitdb.mapFileUseCount.begin(); - while (mi != bitdb.mapFileUseCount.end()) - { - nRefCount += (*mi).second; - mi++; - } - - if (nRefCount == 0) - { - boost::this_thread::interruption_point(); - const std::string& strFile = pwalletMain->strWalletFile; - std::map::iterator _mi = bitdb.mapFileUseCount.find(strFile); - if (_mi != bitdb.mapFileUseCount.end()) - { - LogPrint("db", "Flushing %s\n", strFile); - nLastFlushed = CWalletDB::GetUpdateCounter(); - int64_t nStart = GetTimeMillis(); - - // Flush wallet file so it's self contained - bitdb.CloseDb(strFile); - bitdb.CheckpointLSN(strFile); - - bitdb.mapFileUseCount.erase(_mi++); - LogPrint("db", "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); - } - } + if (dbh.nLastFlushed != nUpdateCounter && GetTime() - dbh.nLastWalletUpdate >= 2) { + if (CDB::PeriodicFlush(dbh)) { + dbh.nLastFlushed = nUpdateCounter; } } } + + fOneThread = false; } // This should be called carefully: From c153a3e6aea08b5a26902f542e512ab795971063 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Fri, 9 Sep 2016 08:50:09 +0000 Subject: [PATCH 06/12] Wallet: Support loading multiple wallets if -wallet used more than once --- src/wallet/wallet.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 750fdb5f86..d43e7ecff6 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -7105,19 +7105,21 @@ bool CWallet::InitLoadWallet() return true; } - std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); + SoftSetArg("-wallet", DEFAULT_WALLET_DAT); - if (walletFile.find_first_of("/\\") != std::string::npos) { - return InitError(_("-wallet parameter must only specify a filename (not a path)")); - } else if (SanitizeString(walletFile, SAFE_CHARS_FILENAME) != walletFile) { - return InitError(_("Invalid characters in -wallet filename")); - } + for (const std::string& walletFile : gArgs.GetArgs("-wallet")) { + if (boost::filesystem::path(walletFile).filename() != walletFile) { + return InitError(_("-wallet parameter must only specify a filename (not a path)")); + } else if (SanitizeString(walletFile, SAFE_CHARS_FILENAME) != walletFile) { + return InitError(_("Invalid characters in -wallet filename")); + } - CWallet * const pwallet = CreateWalletFromFile(walletFile); - if (!pwallet) { - return false; + CWallet * const pwallet = CreateWalletFromFile(walletFile); + if (!pwallet) { + return false; + } + vpwallets.push_back(pwallet); } - vpwallets.push_back(pwallet); return true; } From 4498cbe1e92e5b9d64548f20883fdcb2af53bab9 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Thu, 9 Mar 2017 23:45:58 +0000 Subject: [PATCH 07/12] Wallet: Move multiwallet sanity checks to CWallet::Verify, and do other checks on all wallets --- src/wallet/wallet.cpp | 73 ++++++++++++++----------------------------- 1 file changed, 24 insertions(+), 49 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index d43e7ecff6..0556759f87 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -633,55 +633,38 @@ bool CWallet::Verify() if (GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) return true; - LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); - std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); - - LogPrintf("Using wallet %s\n", walletFile); - uiInterface.InitMessage(_("Verifying wallet...")); + SoftSetArg("-wallet", DEFAULT_WALLET_DAT); - // Wallet file must be a plain filename without a directory - if (walletFile != boost::filesystem::basename(walletFile) + boost::filesystem::extension(walletFile)) - return InitError(strprintf(_("Wallet %s resides outside data directory %s"), walletFile, GetDataDir().string())); + uiInterface.InitMessage(_("Verifying wallet(s)...")); - if (!bitdb.Open(GetDataDir())) - { - // try moving the database env out of the way - boost::filesystem::path pathDatabase = GetDataDir() / "database"; - boost::filesystem::path pathDatabaseBak = GetDataDir() / strprintf("database.%d.bak", GetTime()); - try { - boost::filesystem::rename(pathDatabase, pathDatabaseBak); - LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string()); - } catch (const boost::filesystem::filesystem_error&) { - // failure is ok (well, not really, but it's not worse than what we started with) + for (const std::string& walletFile : gArgs.GetArgs("-wallet")) { + if (boost::filesystem::path(walletFile).filename() != walletFile) { + return InitError(_("-wallet parameter must only specify a filename (not a path)")); + } else if (SanitizeString(walletFile, SAFE_CHARS_FILENAME) != walletFile) { + return InitError(_("Invalid characters in -wallet filename")); } - // try again - if (!bitdb.Open(GetDataDir())) { - // if it still fails, it probably means we can't even create the database env - return InitError(strprintf(_("Error initializing wallet database environment %s!"), GetDataDir())); - } - } + std::string strError; + if (!CWalletDB::VerifyEnvironment(walletFile, GetDataDir().string(), strError)) + return InitError(strError); - if (GetBoolArg("-salvagewallet", false)) - { - // Recover readable keypairs: - if (!CWalletDB::Recover(bitdb, walletFile, true)) - return false; - } + if (GetBoolArg("-salvagewallet", false)) + { + // Recover readable keypairs: + CWallet dummyWallet; + if (!CWalletDB::Recover(walletFile, (void *)&dummyWallet, CWalletDB::RecoverKeysOnlyFilter)) + return false; + } - if (boost::filesystem::exists(GetDataDir() / walletFile)) - { - CDBEnv::VerifyResult r = bitdb.Verify(walletFile, CWalletDB::Recover); - if (r == CDBEnv::RECOVER_OK) + std::string strWarning; + bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetDataDir().string(), strWarning, strError); + if (!strWarning.empty()) + InitWarning(strWarning); + if (!dbV) { - InitWarning(strprintf(_("Warning: Wallet file corrupt, data salvaged!" - " Original %s saved as %s in %s; if" - " your balance or transactions are incorrect you should" - " restore from a backup."), - walletFile, "wallet.{timestamp}.bak", GetDataDir())); + InitError(strError); + return false; } - if (r == CDBEnv::RECOVER_FAIL) - return InitError(strprintf(_("%s corrupt, salvage failed"), walletFile)); } return true; @@ -7105,15 +7088,7 @@ bool CWallet::InitLoadWallet() return true; } - SoftSetArg("-wallet", DEFAULT_WALLET_DAT); - for (const std::string& walletFile : gArgs.GetArgs("-wallet")) { - if (boost::filesystem::path(walletFile).filename() != walletFile) { - return InitError(_("-wallet parameter must only specify a filename (not a path)")); - } else if (SanitizeString(walletFile, SAFE_CHARS_FILENAME) != walletFile) { - return InitError(_("Invalid characters in -wallet filename")); - } - CWallet * const pwallet = CreateWalletFromFile(walletFile); if (!pwallet) { return false; From 3f9899486126c2a70708da88cb357065fcc13d4c Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Mon, 5 Jun 2017 21:36:02 +0000 Subject: [PATCH 08/12] Bugfix: wallet: Fix warningStr, errorStr argument order --- src/wallet/walletdb.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 4e0e67cfdd..16733671da 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -1352,6 +1352,11 @@ bool CWalletDB::Recover(CDBEnv& dbenv, const std::string& filename) return CWalletDB::Recover(dbenv, filename, false); } +bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr) +{ + return CDB::VerifyDatabaseFile(walletFile, dataDir, warningStr, errorStr, CWalletDB::Recover); +} + bool CWalletDB::WriteDestData(const std::string &address, const std::string &key, const std::string &value) { nWalletDBUpdateCounter++; From 6412711847e5abd6fdd3b2032c81f7ae46ea30ed Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Mon, 5 Jun 2017 22:01:48 +0000 Subject: [PATCH 09/12] wallet: Include actual backup filename in recovery warning message --- src/wallet/db.cpp | 129 +++++++++++++++++++++++++++++++++++++++- src/wallet/db.h | 12 +++- src/wallet/wallet.cpp | 3 +- src/wallet/walletdb.cpp | 13 ++++ src/wallet/walletdb.h | 23 +++++-- 5 files changed, 171 insertions(+), 9 deletions(-) diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 996c01271e..f85d094768 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -143,7 +143,7 @@ void CDBEnv::MakeMock() fMockDb = true; } -CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFunc)(CDBEnv& dbenv, const std::string& strFile)) +CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); @@ -156,10 +156,135 @@ CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFu return RECOVER_FAIL; // Try to recover: - bool fRecovered = (*recoverFunc)(*this, strFile); + bool fRecovered = (*recoverFunc)(strFile, out_backup_filename); return (fRecovered ? RECOVER_OK : RECOVER_FAIL); } +bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename) +{ + // Recovery procedure: + // move wallet file to wallet.timestamp.bak + // Call Salvage with fAggressive=true to + // get as much data as possible. + // Rewrite salvaged data to fresh wallet file + // Set -rescan so any missing transactions will be + // found. + int64_t now = GetTime(); + newFilename = strprintf("wallet.%d.bak", now); + + int result = bitdb.dbenv->dbrename(NULL, filename.c_str(), NULL, + newFilename.c_str(), DB_AUTO_COMMIT); + if (result == 0) + LogPrintf("Renamed %s to %s\n", filename, newFilename); + else + { + LogPrintf("Failed to rename %s to %s\n", filename, newFilename); + return false; + } + + std::vector salvagedData; + bool fSuccess = bitdb.Salvage(newFilename, true, salvagedData); + if (salvagedData.empty()) + { + LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); + return false; + } + LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); + + std::unique_ptr pdbCopy(new Db(bitdb.dbenv, 0)); + int ret = pdbCopy->open(NULL, // Txn pointer + filename.c_str(), // Filename + "main", // Logical db name + DB_BTREE, // Database type + DB_CREATE, // Flags + 0); + if (ret > 0) + { + LogPrintf("Cannot create database file %s\n", filename); + return false; + } + + DbTxn* ptxn = bitdb.TxnBegin(); + BOOST_FOREACH(CDBEnv::KeyValPair& row, salvagedData) + { + if (recoverKVcallback) + { + CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); + CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); + std::string strType, strErr; + if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue)) + continue; + } + Dbt datKey(&row.first[0], row.first.size()); + Dbt datValue(&row.second[0], row.second.size()); + int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); + if (ret2 > 0) + fSuccess = false; + } + ptxn->commit(0); + pdbCopy->close(0); + + return fSuccess; +} + +bool CDB::VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr) +{ + LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); + LogPrintf("Using wallet %s\n", walletFile); + + // Wallet file must be a plain filename without a directory + if (walletFile != fs::basename(walletFile) + fs::extension(walletFile)) + { + errorStr = strprintf(_("Wallet %s resides outside data directory %s"), walletFile, dataDir.string()); + return false; + } + + if (!bitdb.Open(dataDir)) + { + // try moving the database env out of the way + fs::path pathDatabase = dataDir / "database"; + fs::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime()); + try { + fs::rename(pathDatabase, pathDatabaseBak); + LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string()); + } catch (const fs::filesystem_error&) { + // failure is ok (well, not really, but it's not worse than what we started with) + } + + // try again + if (!bitdb.Open(dataDir)) { + // if it still fails, it probably means we can't even create the database env + errorStr = strprintf(_("Error initializing wallet database environment %s!"), GetDataDir()); + return false; + } + } + return true; +} + +bool CDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc) +{ + if (fs::exists(dataDir / walletFile)) + { + std::string backup_filename; + CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc, backup_filename); + if (r == CDBEnv::RECOVER_OK) + { + warningStr = strprintf(_("Warning: Wallet file corrupt, data salvaged!" + " Original %s saved as %s in %s; if" + " your balance or transactions are incorrect you should" + " restore from a backup."), + walletFile, backup_filename, dataDir); + } + if (r == CDBEnv::RECOVER_FAIL) + { + errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile); + return false; + } + } + // also return true if files does not exists + return true; +} + /* End of headers, beginning of key/value data */ static const char *HEADER_END = "HEADER=END"; /* End of key/value data */ diff --git a/src/wallet/db.h b/src/wallet/db.h index 5c92218579..ea08faeb69 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -56,7 +56,8 @@ class CDBEnv enum VerifyResult { VERIFY_OK, RECOVER_OK, RECOVER_FAIL }; - VerifyResult Verify(const std::string& strFile, bool (*recoverFunc)(CDBEnv& dbenv, const std::string& strFile)); + typedef bool (*recoverFunc_type)(const std::string& strFile, std::string& out_backup_filename); + VerifyResult Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename); /** * Salvage data from a file that Verify says is bad. * fAggressive sets the DB_AGGRESSIVE flag (see berkeley DB->verify() method documentation). @@ -157,6 +158,15 @@ class CDB public: void Flush(); void Close(); + static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& out_backup_filename); + + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + static bool PeriodicFlush(CWalletDBWrapper& dbw); + /* verifies the database environment */ + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); + /* verifies the database file */ + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr, CDBEnv::recoverFunc_type recoverFunc); private: CDB(const CDB&); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 0556759f87..d39bda349b 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -652,7 +652,8 @@ bool CWallet::Verify() { // Recover readable keypairs: CWallet dummyWallet; - if (!CWalletDB::Recover(walletFile, (void *)&dummyWallet, CWalletDB::RecoverKeysOnlyFilter)) + std::string backup_filename; + if (!CWalletDB::Recover(walletFile, (void *)&dummyWallet, CWalletDB::RecoverKeysOnlyFilter, backup_filename)) return false; } diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 16733671da..c897dc7388 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -1261,6 +1261,19 @@ bool AutoBackupWallet (CWallet* wallet, std::string strWalletFile, std::string& LogPrintf("Automatic wallet backups are disabled!\n"); return false; +// +// Try to (very carefully!) recover wallet file if there is a problem. +// +bool CWalletDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& out_backup_filename) +{ + return CDB::Recover(filename, callbackDataIn, recoverKVcallback, out_backup_filename); +} + +bool CWalletDB::Recover(const std::string& filename, std::string& out_backup_filename) +{ + // recover without a key filter callback + // results in recovering all record types + return CWalletDB::Recover(filename, NULL, NULL, out_backup_filename); } // diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 5b6d31e5cb..733c7b5d53 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -282,11 +282,24 @@ class CWalletDB : public CDB DBErrors ReorderTransactions(CWallet* pwallet); DBErrors LoadWallet(CWallet* pwallet); - DBErrors FindWalletTx(CWallet* pwallet, std::vector& vTxHash, std::vector& vWtx); - DBErrors ZapWalletTx(CWallet* pwallet, std::vector& vWtx); - DBErrors ZapSelectTx(CWallet* pwallet, std::vector& vHashIn, std::vector& vHashOut); - DBErrors ZapSigmaMints(CWallet* pwallet); - DBErrors ZapLelantusMints(CWallet *pwallet); + DBErrors FindWalletTx(std::vector& vTxHash, std::vector& vWtx); + DBErrors ZapWalletTx(std::vector& vWtx); + DBErrors ZapSelectTx(std::vector& vHashIn, std::vector& vHashOut); + /* Try to (very carefully!) recover wallet database (with a possible key type filter) */ + static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& out_backup_filename); + /* Recover convenience-function to bypass the key filter callback, called when verify fails, recovers everything */ + static bool Recover(const std::string& filename, std::string& out_backup_filename); + /* Recover filter (used as callback), will only let keys (cryptographical keys) as KV/key-type pass through */ + static bool RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue); + /* Function to determine if a certain KV/key-type is a key (cryptographical key) type */ + static bool IsKeyType(const std::string& strType); + /* verifies the database environment */ + static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); + /* verifies the database file */ + static bool VerifyDatabaseFile(const std::string& walletFile, const fs::path& dataDir, std::string& warningStr, std::string& errorStr); + + DBErrors ZapSigmaMints(); + DBErrors ZapLelantusMints(); static bool Recover(CDBEnv& dbenv, const std::string& filename, bool fOnlyKeys); static bool Recover(CDBEnv& dbenv, const std::string& filename); From 387f5d71667f39f5f53f4e39712c3830881e9546 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Mon, 5 Jun 2017 22:02:32 +0000 Subject: [PATCH 10/12] wallet: Base backup filenames on original wallet filename --- src/wallet/db.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index f85d094768..2f4caddc89 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -163,14 +163,14 @@ CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, recoverFunc_type bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename) { // Recovery procedure: - // move wallet file to wallet.timestamp.bak + // move wallet file to walletfilename.timestamp.bak // Call Salvage with fAggressive=true to // get as much data as possible. // Rewrite salvaged data to fresh wallet file // Set -rescan so any missing transactions will be // found. int64_t now = GetTime(); - newFilename = strprintf("wallet.%d.bak", now); + newFilename = strprintf("%s.%d.bak", filename, now); int result = bitdb.dbenv->dbrename(NULL, filename.c_str(), NULL, newFilename.c_str(), DB_AUTO_COMMIT); From 19fa3899ab2709519c257f879311af18c679e800 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Mon, 5 Jun 2017 22:21:20 +0000 Subject: [PATCH 11/12] wallet: Forbid -salvagewallet, -zapwallettxes, and -upgradewallet with multiple wallets --- src/wallet/wallet.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index d39bda349b..63c366548d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -633,8 +633,6 @@ bool CWallet::Verify() if (GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) return true; - SoftSetArg("-wallet", DEFAULT_WALLET_DAT); - uiInterface.InitMessage(_("Verifying wallet(s)...")); for (const std::string& walletFile : gArgs.GetArgs("-wallet")) { @@ -7116,6 +7114,9 @@ void CWallet::postInitProcess(boost::thread_group& threadGroup) bool CWallet::ParameterInteraction() { + SoftSetArg("-wallet", DEFAULT_WALLET_DAT); + const bool is_multiwallet = gArgs.GetArgs("-wallet").size() > 1; + if (GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) return true; @@ -7124,15 +7125,27 @@ bool CWallet::ParameterInteraction() } if (GetBoolArg("-salvagewallet", false) && SoftSetBoolArg("-rescan", true)) { + if (is_multiwallet) { + return InitError(strprintf("%s is only allowed with a single wallet file", "-salvagewallet")); + } // Rewrite just private keys: rescan to find transactions LogPrintf("%s: parameter interaction: -salvagewallet=1 -> setting -rescan=1\n", __func__); } // -zapwallettx implies a rescan if (GetBoolArg("-zapwallettxes", false) && SoftSetBoolArg("-rescan", true)) { + if (is_multiwallet) { + return InitError(strprintf("%s is only allowed with a single wallet file", "-zapwallettxes")); + } LogPrintf("%s: parameter interaction: -zapwallettxes= -> setting -rescan=1\n", __func__); } + if (is_multiwallet) { + if (GetBoolArg("-upgradewallet", false)) { + return InitError(strprintf("%s is only allowed with a single wallet file", "-upgradewallet")); + } + } + if (GetBoolArg("-sysperms", false)) return InitError("-sysperms is not allowed in combination with enabled wallet functionality"); if (GetArg("-prune", 0) && GetBoolArg("-rescan", false)) From e4c0237e12d321105b4a9b5bbe9c042932abd10f Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Mon, 5 Jun 2017 22:23:20 +0000 Subject: [PATCH 12/12] wallet: Update formatting --- src/wallet/wallet.cpp | 18 ++++++++++-------- src/wallet/walletdb.cpp | 30 +++++++++++------------------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 63c366548d..da39fef8a9 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -643,24 +643,25 @@ bool CWallet::Verify() } std::string strError; - if (!CWalletDB::VerifyEnvironment(walletFile, GetDataDir().string(), strError)) + if (!CWalletDB::VerifyEnvironment(walletFile, GetDataDir().string(), strError)) { return InitError(strError); + } - if (GetBoolArg("-salvagewallet", false)) - { + if (GetBoolArg("-salvagewallet", false)) { // Recover readable keypairs: CWallet dummyWallet; std::string backup_filename; - if (!CWalletDB::Recover(walletFile, (void *)&dummyWallet, CWalletDB::RecoverKeysOnlyFilter, backup_filename)) + if (!CWalletDB::Recover(walletFile, (void *)&dummyWallet, CWalletDB::RecoverKeysOnlyFilter, backup_filename)) { return false; + } } std::string strWarning; bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetDataDir().string(), strWarning, strError); - if (!strWarning.empty()) + if (!strWarning.empty()) { InitWarning(strWarning); - if (!dbV) - { + } + if (!dbV) { InitError(strError); return false; } @@ -5990,8 +5991,9 @@ bool CWallet::AddAccountingEntry(const CAccountingEntry& acentry) bool CWallet::AddAccountingEntry(const CAccountingEntry& acentry, CWalletDB *pwalletdb) { - if (!pwalletdb->WriteAccountingEntry(++nAccountingEntryNumber, acentry)) + if (!pwalletdb->WriteAccountingEntry(++nAccountingEntryNumber, acentry)) { return false; + } laccentries.push_back(acentry); CAccountingEntry & entry = laccentries.back(); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index c897dc7388..d138b0d716 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -80,11 +80,9 @@ bool CWalletDB::EraseTx(uint256 hash) bool CWalletDB::WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, const CKeyMetadata& keyMeta) { - nWalletDBUpdateCounter++; - - if (!Write(std::make_pair(std::string("keymeta"), vchPubKey), - keyMeta, false)) + if (!WriteIC(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta, false)) { return false; + } // hash pubkey/privkey to accelerate wallet load std::vector vchKey; @@ -102,12 +100,13 @@ bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, const bool fEraseUnencryptedKey = true; nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("keymeta"), vchPubKey), - keyMeta)) + if (!WriteIC(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta)) { return false; + } - if (!Write(std::make_pair(std::string("ckey"), vchPubKey), vchCryptedSecret, false)) + if (!WriteIC(std::make_pair(std::string("ckey"), vchPubKey), vchCryptedSecret, false)) { return false; + } if (fEraseUnencryptedKey) { Erase(std::make_pair(std::string("key"), vchPubKey)); @@ -130,29 +129,22 @@ bool CWalletDB::WriteCScript(const uint160& hash, const CScript& redeemScript) bool CWalletDB::WriteWatchOnly(const CScript &dest, const CKeyMetadata& keyMeta) { +<<<<<<< HEAD nWalletDBUpdateCounter++; if (!Write(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)), keyMeta)) return false; - return Write(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest)), '1'); +>>>>>>> c237bd750e... wallet: Update formatting } bool CWalletDB::EraseWatchOnly(const CScript &dest) -{ - nWalletDBUpdateCounter++; - if (!Erase(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) + if (!EraseIC(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) { return false; - return Erase(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); -} - -bool CWalletDB::WriteBestBlock(const CBlockLocator& locator) -{ - nWalletDBUpdateCounter++; - Write(std::string("bestblock"), CBlockLocator()); // Write empty block locator so versions that require a merkle branch automatically rescan + } + return EraseIC(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); return Write(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::ReadBestBlock(CBlockLocator& locator) -{ if (Read(std::string("bestblock"), locator) && !locator.vHave.empty()) return true; return Read(std::string("bestblock_nomerkle"), locator); }