diff --git a/CHANGELOG.md b/CHANGELOG.md index 10475cf02..3300a92f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,21 @@ you run it for the first time.** ### Wallet Changes: +#### Configuration + Wallet now has option `wallet-migrate-no-rescan`/`migrate-no-rescan` if you +want to disable rescan when migration recommends it. It may result in the +incorrect txdb state, but can be useful if you know the issue does not affect +your wallet or is not critical. + +#### Wallet API - Add migration that recalculates txdb balances to fix any inconsistencies. +- WalletDB Now emits events for: `open`, `close`, `connect`, `disconnect`. +- WalletDB + - `open()` no longer calls `connect` and needs separate call `connect`. + - `open()` no longer calls scan, instead only rollbacks and waits for + sync to do the rescan. + - emits events for: `open`, `close`, `connect`, `disconnect`, `sync done`. - HTTP Changes: - All transaction creating endpoints now accept `hardFee` for specifying the exact fee. diff --git a/lib/wallet/node.js b/lib/wallet/node.js index d1d4b2f87..b91b92a76 100644 --- a/lib/wallet/node.js +++ b/lib/wallet/node.js @@ -52,6 +52,7 @@ class WalletNode extends Node { wipeNoReally: this.config.bool('wipe-no-really'), spv: this.config.bool('spv'), walletMigrate: this.config.uint('migrate'), + migrateNoRescan: this.config.bool('migrate-no-rescan', false), icannlockup: this.config.bool('icannlockup', false) }); @@ -107,6 +108,7 @@ class WalletNode extends Node { await this.openPlugins(); await this.http.open(); + await this.wdb.connect(); await this.handleOpen(); this.logger.info('Wallet node is loaded.'); @@ -128,6 +130,7 @@ class WalletNode extends Node { this.rpc.wallet = null; + await this.wdb.disconnect(); await this.wdb.close(); await this.handleClose(); } diff --git a/lib/wallet/plugin.js b/lib/wallet/plugin.js index 5c514475d..539789839 100644 --- a/lib/wallet/plugin.js +++ b/lib/wallet/plugin.js @@ -57,6 +57,7 @@ class Plugin extends EventEmitter { wipeNoReally: this.config.bool('wipe-no-really'), spv: node.spv, walletMigrate: this.config.uint('migrate'), + migrateNoRescan: this.config.bool('migrate-no-rescan', false), icannlockup: this.config.bool('icannlockup', false) }); @@ -90,11 +91,13 @@ class Plugin extends EventEmitter { await this.wdb.open(); this.rpc.wallet = this.wdb.primary; await this.http.open(); + await this.wdb.connect(); } async close() { await this.http.close(); this.rpc.wallet = null; + await this.wdb.disconnect(); await this.wdb.close(); } } diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 7a8c25277..e0736bd59 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -64,12 +64,18 @@ class WalletDB extends EventEmitter { this.name = 'wallet'; this.version = 2; - this.primary = null; + // chain state. + this.hasStateCache = false; this.state = new ChainState(); - this.confirming = false; this.height = 0; + + // wallets + this.primary = null; this.wallets = new Map(); this.depth = 0; + + // guards + this.confirming = false; this.rescanning = false; this.filterSent = false; @@ -122,6 +128,7 @@ class WalletDB extends EventEmitter { }); this.client.on('connect', async () => { + this.emit('connect'); try { await this.syncNode(); this.emit('sync done', this.state); @@ -131,6 +138,7 @@ class WalletDB extends EventEmitter { }); this.client.on('disconnect', async () => { + this.emit('disconnect'); this.filterSent = false; }); @@ -201,7 +209,7 @@ class WalletDB extends EventEmitter { await this.wipe(); await this.watch(); - await this.connect(); + await this.loadState(); this.logger.info( 'WalletDB loaded (depth=%d, height=%d, start=%d).', @@ -222,9 +230,17 @@ class WalletDB extends EventEmitter { this.primary = wallet; if (migrationResult.rescan) { - this.logger.info('Rescanning...'); - await this.scan(0); + if (!this.options.migrateNoRescan) { + this.logger.info('Migration rollback...'); + await this.rollback(0); + } else { + this.logger.warning( + 'Migration rescan skipped, state may be incorrect.'); + } } + + this.logger.info('WalletDB opened.'); + this.emit('open'); } /** @@ -270,14 +286,17 @@ class WalletDB extends EventEmitter { */ async close() { - await this.disconnect(); + if (this.client.opened) + await this.disconnect(); for (const wallet of this.wallets.values()) { await wallet.destroy(); this.unregister(wallet); } - return this.db.close(); + await this.db.close(); + this.logger.info('WalletDB Closed.'); + this.emit('close'); } /** @@ -368,7 +387,7 @@ class WalletDB extends EventEmitter { const unlock = await this.txLock.lock(); try { this.logger.info('Resyncing from server...'); - await this.syncState(); + await this.syncInitState(); await this.syncFilter(); await this.syncChain(); await this.resend(); @@ -378,18 +397,31 @@ class WalletDB extends EventEmitter { } /** - * Initialize and write initial sync state. + * Recover state from the cache. * @returns {Promise} */ - async syncState() { + async loadState() { const cache = await this.getState(); - if (cache) { - this.state = cache; - this.height = cache.height; - return undefined; - } + if (!cache) + return; + + this.logger.info('Initialized chain state from the database.'); + this.hasStateCache = true; + this.state = cache; + this.height = cache.height; + } + + /** + * Initialize and write initial sync state. + * @returns {Promise} + */ + + async syncInitState() { + // We have recovered from the cache. + if (this.hasStateCache) + return; this.logger.info('Initializing database state from server.'); @@ -420,7 +452,7 @@ class WalletDB extends EventEmitter { this.state = state; this.height = state.height; - return undefined; + return; } /** @@ -2185,6 +2217,7 @@ class WalletDB extends EventEmitter { await iter.each(async (key, value) => { const [height] = layout.b.decode(key); const block = MapRecord.decode(value); + this.logger.info('Reverting block: %d', height); for (const wid of block.wids) { const wallet = await this.get(wid); @@ -2494,6 +2527,7 @@ class WalletOptions { this.spv = false; this.wipeNoReally = false; this.walletMigrate = -1; + this.migrateNoRescan = false; this.icannlockup = false; if (options) @@ -2582,6 +2616,11 @@ class WalletOptions { this.icannlockup = options.icannlockup; } + if (options.migrateNoRescan != null) { + assert(typeof options.migrateNoRescan === 'boolean'); + this.migrateNoRescan = options.migrateNoRescan; + } + return this; } diff --git a/test/mempool-reorg-test.js b/test/mempool-reorg-test.js index 330558ca9..16a870a83 100644 --- a/test/mempool-reorg-test.js +++ b/test/mempool-reorg-test.js @@ -8,6 +8,7 @@ const plugin = require('../lib/wallet/plugin'); const Coin = require('../lib/primitives/coin'); const Address = require('../lib/primitives/address'); const MTX = require('../lib/primitives/mtx'); +const {forEvent} = require('./util/common'); const network = Network.get('regtest'); const { @@ -25,8 +26,13 @@ describe('Mempool Covenant Reorg', function () { let wallet, name; before(async () => { + const wdb = node.require('walletdb').wdb; + const syncDone = forEvent(wdb, 'sync done'); await node.open(); + wallet = node.get('walletdb').wdb.primary; + + await syncDone; }); after(async () => { diff --git a/test/wallet-auction-test.js b/test/wallet-auction-test.js index a9abf7d8f..69c09a69b 100644 --- a/test/wallet-auction-test.js +++ b/test/wallet-auction-test.js @@ -64,6 +64,7 @@ describe('Wallet Auction', function() { await chain.open(); await miner.open(); await wdb.open(); + await wdb.connect(); // Set up wallet winner = await wdb.create(); @@ -81,6 +82,7 @@ describe('Wallet Auction', function() { }); after(async () => { + await wdb.disconnect(); await wdb.close(); await miner.close(); await chain.close(); diff --git a/test/wallet-events-test.js b/test/wallet-events-test.js index 536d8485d..3e404c78f 100644 --- a/test/wallet-events-test.js +++ b/test/wallet-events-test.js @@ -7,11 +7,13 @@ const Wallet = require('../lib/wallet/wallet'); const WalletKey = require('../lib/wallet/walletkey'); const ChainEntry = require('../lib/blockchain/chainentry'); const MemWallet = require('./util/memwallet'); +const {forEvent} = require('./util/common'); describe('WalletDB Events', function () { const node = new FullNode({ memory: true, network: 'regtest', + noDNS: true, plugins: [require('../lib/wallet/plugin')] }); @@ -35,15 +37,31 @@ describe('WalletDB Events', function () { await node.close(); }); - it('should emit `tx` events', async () => { - const waiter = new Promise((resolve) => { - wdb.once('tx', (w, tx) => resolve([w, tx])); - }); + it('should emit `open`/`close` and `connect`/`disconnect` events', async () => { + const closeEvent = forEvent(wdb, 'close'); + const disconnectEvent = forEvent(wdb, 'disconnect'); + await node.close(); + await disconnectEvent; + await closeEvent; + const openEvent = forEvent(wdb, 'open'); + const connectEvent = forEvent(wdb, 'connect'); + const syncDone = forEvent(wdb, 'sync done'); + await node.open(); + await openEvent; + await connectEvent; + await syncDone; + + wallet = await wdb.get('primary'); + }); + + it('should emit `tx` events', async () => { + const waiter = forEvent(wdb, 'tx'); const walletReceive = await wallet.receiveAddress(); await mineBlocks(1, walletReceive); - const [w, tx] = await waiter; + const events = await waiter; + const [w, tx] = events[0].values; assert(w); assert(w instanceof Wallet); @@ -55,14 +73,13 @@ describe('WalletDB Events', function () { }); it('should emit `address` events', async () => { - const waiter = new Promise((resolve) => { - wdb.once('address', (w, walletKey) => resolve([w, walletKey])); - }); + const waiter = forEvent(wdb, 'address'); const walletReceive = await wallet.receiveAddress(); await mineBlocks(1, walletReceive); - const [w, walletKey] = await waiter; + const events = await waiter; + const [w, walletKey] = events[0].values; assert(w); assert(w instanceof Wallet); @@ -75,10 +92,7 @@ describe('WalletDB Events', function () { describe('should emit `block connect` events', () => { it('with a block that includes a wallet tx', async () => { - const waiter = new Promise((resolve) => { - wdb.once('block connect', (entry, txs) => resolve([entry, txs])); - }); - + const waiter = forEvent(wdb, 'block connect'); const walletReceive = await wallet.receiveAddress(); await wallet.send({ @@ -89,7 +103,8 @@ describe('WalletDB Events', function () { await mineBlocks(1); - const [entry, txs] = await waiter; + const events = await waiter; + const [entry, txs] = events[0].values; assert(entry); assert(entry instanceof ChainEntry); @@ -127,15 +142,14 @@ describe('WalletDB Events', function () { otherTx = otherMtx.toTX(); } - const waiter = new Promise((resolve) => { - wdb.once('block connect', (entry, txs) => resolve([entry, txs])); - }); + const waiter = forEvent(wdb, 'block connect'); await node.sendTX(otherTx); await mineBlocks(1); - const [entry, txs] = await waiter; + const events = await waiter; + const [entry, txs] = events[0].values; assert(entry); assert(entry instanceof ChainEntry); @@ -158,14 +172,13 @@ describe('WalletDB Events', function () { await mineBlocks(1); // Disconnect it - const waiter = new Promise((resolve) => { - wdb.once('block disconnect', entry => resolve(entry)); - }); + const waiter = forEvent(wdb, 'block disconnect'); const entryToDisconnect = node.chain.tip; await node.chain.disconnect(entryToDisconnect); - const entry = await waiter; + const events = await waiter; + const entry = events[0].values[0]; assert(entry); assert(entry instanceof ChainEntry); @@ -173,18 +186,16 @@ describe('WalletDB Events', function () { }); it('with a block that does not include a wallet tx', async () => { - const waiter = new Promise((resolve) => { - wdb.once('block disconnect', entry => resolve(entry)); - }); - - const entryToDisconnect = node.chain.tip; - await node.chain.disconnect(entryToDisconnect); + const waiter = forEvent(wdb, 'block disconnect'); + const entryToDisconnect = node.chain.tip; + await node.chain.disconnect(entryToDisconnect); - const entry = await waiter; + const events = await waiter; + const entry = events[0].values[0]; - assert(entry); - assert(entry instanceof ChainEntry); - assert(entry.hash = entryToDisconnect.hash); + assert(entry); + assert(entry instanceof ChainEntry); + assert(entry.hash = entryToDisconnect.hash); }); }); }); diff --git a/test/wallet-migration-test.js b/test/wallet-migration-test.js index f25e1e995..8dfc4371e 100644 --- a/test/wallet-migration-test.js +++ b/test/wallet-migration-test.js @@ -19,7 +19,7 @@ const { oldLayout } = require('../lib/migrations/migrator'); const {migrationError} = require('./util/migrations'); -const {forEvent, rimraf, testdir} = require('./util/common'); +const {rimraf, testdir} = require('./util/common'); const NETWORK = 'regtest'; const network = Network.get(NETWORK); @@ -82,7 +82,7 @@ describe('Wallet Migrations', function() { return ids; }; - let walletDB, ldb, wdbOpenSync; + let walletDB, ldb; beforeEach(async () => { await fs.mkdirp(location); @@ -91,9 +91,7 @@ describe('Wallet Migrations', function() { WalletMigrator.migrations = mockMigrations; - wdbOpenSync = wdbOpenSyncFn(walletDB); - - await wdbOpenSync(); + await walletDB.open(); }); afterEach(async () => { @@ -181,7 +179,7 @@ describe('Wallet Migrations', function() { walletDB.options.walletMigrate = lastMigrationID; walletDB.version = 1; - await wdbOpenSync(); + await walletDB.open(); const versionData = await ldb.get(layout.V.encode()); const version = getVersion(versionData, 'wallet'); @@ -244,7 +242,7 @@ describe('Wallet Migrations', function() { network }; - let walletDB, ldb, wdbOpenSync; + let walletDB, ldb; beforeEach(async () => { await fs.mkdirp(location); @@ -253,8 +251,6 @@ describe('Wallet Migrations', function() { ldb = walletDB.db; WalletMigrator.migrations = testMigrations; - - wdbOpenSync = wdbOpenSyncFn(walletDB); }); afterEach(async () => { @@ -268,8 +264,7 @@ describe('Wallet Migrations', function() { }); it('should initialize fresh walletdb migration state', async () => { - await wdbOpenSync(); - + await walletDB.open(); const rawState = await ldb.get(layout.M.encode()); const state = MigrationState.decode(rawState); @@ -281,8 +276,7 @@ describe('Wallet Migrations', function() { }); it('should not migrate pre-old migration state w/o flag', async () => { - await wdbOpenSync(); - + await walletDB.open(); const b = ldb.batch(); b.del(layout.M.encode()); await b.write(); @@ -308,8 +302,7 @@ describe('Wallet Migrations', function() { }); it('should migrate pre-old migration state with flag', async () => { - await wdbOpenSync(); - + await walletDB.open(); const b = ldb.batch(); b.del(layout.M.encode()); writeVersion(b, 'wallet', 0); @@ -317,7 +310,7 @@ describe('Wallet Migrations', function() { await walletDB.close(); walletDB.options.walletMigrate = 1; - await wdbOpenSync(); + await walletDB.open(); const versionData = await ldb.get(layout.V.encode()); const version = getVersion(versionData, 'wallet'); @@ -334,7 +327,7 @@ describe('Wallet Migrations', function() { }); it('should not migrate from last old migration state w/o flag', async () => { - await wdbOpenSync(); + await walletDB.open(); const b = ldb.batch(); b.del(layout.M.encode()); @@ -363,7 +356,7 @@ describe('Wallet Migrations', function() { }); it('should not migrate from last old migration state with flag', async () => { - await wdbOpenSync(); + await walletDB.open(); const b = ldb.batch(); b.del(layout.M.encode()); @@ -372,7 +365,7 @@ describe('Wallet Migrations', function() { await walletDB.close(); walletDB.options.walletMigrate = 1; - await wdbOpenSync(); + await walletDB.open(); const rawState = await ldb.get(layout.M.encode()); const state = MigrationState.decode(rawState); @@ -398,7 +391,7 @@ describe('Wallet Migrations', function() { } }; - await wdbOpenSync(); + await walletDB.open(); const b = ldb.batch(); b.del(layout.M.encode()); @@ -450,7 +443,7 @@ describe('Wallet Migrations', function() { } }; - await wdbOpenSync(); + await walletDB.open(); const b = ldb.batch(); b.del(layout.M.encode()); @@ -460,7 +453,7 @@ describe('Wallet Migrations', function() { await walletDB.close(); walletDB.options.walletMigrate = 2; - await wdbOpenSync(); + await walletDB.open(); assert.strictEqual(migrated1, false); assert.strictEqual(migrated2, true); @@ -492,7 +485,7 @@ describe('Wallet Migrations', function() { const ADD_CHANGE_DEPTH = 10; - let walletDB, ldb, wdbOpenSync; + let walletDB, ldb; const missingAddrs = []; before(async () => { WalletMigrator.migrations = {}; @@ -507,7 +500,6 @@ describe('Wallet Migrations', function() { beforeEach(async () => { walletDB = new WalletDB(walletOptions); ldb = walletDB.db; - wdbOpenSync = wdbOpenSyncFn(walletDB); }); afterEach(async () => { @@ -516,7 +508,7 @@ describe('Wallet Migrations', function() { }); it('should set incorrect walletdb state', async () => { - await wdbOpenSync(); + await walletDB.open(); const wallet = walletDB.primary; const account = await wallet.getAccount(0); @@ -535,7 +527,7 @@ describe('Wallet Migrations', function() { }); it('should have missing addresses', async () => { - await wdbOpenSync(); + await walletDB.open(); const wallet = walletDB.primary; for (const addr of missingAddrs) { @@ -568,12 +560,12 @@ describe('Wallet Migrations', function() { it('should migrate with migrate flag', async () => { walletDB.options.walletMigrate = 0; - let rescan = false; - walletDB.scan = () => { - rescan = true; + let rollback = false; + walletDB.rollback = () => { + rollback = true; }; - await wdbOpenSync(); + await walletDB.open(); const wallet = walletDB.primary; for (const addr of missingAddrs) { @@ -581,7 +573,7 @@ describe('Wallet Migrations', function() { assert.strictEqual(hasAddr, true); } - assert.strictEqual(rescan, true); + assert.strictEqual(rollback, true); await walletDB.close(); }); @@ -598,7 +590,7 @@ describe('Wallet Migrations', function() { network: network }; - let walletDB, ldb, wdbOpenSync; + let walletDB, ldb; before(async () => { WalletMigrator.migrations = {}; await fs.mkdirp(location); @@ -612,7 +604,6 @@ describe('Wallet Migrations', function() { beforeEach(async () => { walletDB = new WalletDB(walletOptions); ldb = walletDB.db; - wdbOpenSync = wdbOpenSyncFn(walletDB); }); afterEach(async () => { @@ -658,7 +649,7 @@ describe('Wallet Migrations', function() { await b.write(); }; - await wdbOpenSync(); + await walletDB.open(); const wallet = walletDB.primary; await wallet.createAccount({}); @@ -703,7 +694,7 @@ describe('Wallet Migrations', function() { walletDB.options.walletMigrate = 0; - await wdbOpenSync(); + await walletDB.open(); const wallet = walletDB.primary; const wallet2 = await walletDB.get(1); await checkLookahead(wallet, TEST_LOOKAHEAD + 0); @@ -731,7 +722,7 @@ describe('Wallet Migrations', function() { assert.strictEqual(balance.clocked, expected.clocked); }; - let walletDB, ldb, wdbOpenSync; + let walletDB, ldb; before(async () => { WalletMigrator.migrations = {}; await fs.mkdirp(location); @@ -745,7 +736,6 @@ describe('Wallet Migrations', function() { beforeEach(async () => { walletDB = new WalletDB(walletOptions); ldb = walletDB.db; - wdbOpenSync = wdbOpenSyncFn(walletDB); }); afterEach(async () => { @@ -755,7 +745,7 @@ describe('Wallet Migrations', function() { it('should write some coins w/o updating balance', async () => { // generate credits for the first 10 addresses stored on initialization. - await wdbOpenSync(); + await walletDB.open(); const wallet = walletDB.primary; @@ -880,7 +870,7 @@ describe('Wallet Migrations', function() { }); it('should have incorrect balance before migration', async () => { - await wdbOpenSync(); + await walletDB.open(); const wallet = walletDB.primary; const balance = await wallet.getBalance(-1); @@ -912,7 +902,7 @@ describe('Wallet Migrations', function() { it('should migrate', async () => { walletDB.options.walletMigrate = 0; - await wdbOpenSync(); + await walletDB.open(); const wallet = walletDB.primary; const balance = await wallet.getBalance(-1); @@ -981,11 +971,3 @@ function getVersion(data, name) { return data.readUInt32LE(name.length); } - -function wdbOpenSyncFn(wdb) { - return async () => { - const forSync = forEvent(wdb, 'sync done'); - await wdb.open(); - await forSync; - }; -}; diff --git a/test/wallet-test.js b/test/wallet-test.js index 2d8d4e5f1..76b5d3c13 100644 --- a/test/wallet-test.js +++ b/test/wallet-test.js @@ -2158,6 +2158,7 @@ describe('Wallet', function() { before(async () => { await wdb.open(); + await wdb.connect(); wallet = await wdb.create(); recip = await wdb.create(); // rollout all names @@ -2165,6 +2166,7 @@ describe('Wallet', function() { }); after(async () => { + await wdb.disconnect(); await wdb.close(); }); @@ -2962,6 +2964,7 @@ describe('Wallet', function() { before(async () => { await wdb.open(); + await wdb.connect(); wallet = await wdb.create(); for (let i = 0; i < 3; i++) { @@ -2971,6 +2974,7 @@ describe('Wallet', function() { }); after(async () => { + await wdb.disconnect(); await wdb.close(); });