From c4a2e185c2442d43f65027d61a0ca52d84d6cfdb Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Tue, 31 Mar 2020 06:24:43 -0700 Subject: [PATCH 1/5] wallet: add migration to regenerate missing change addresses. see #414, #413, #411. --- lib/wallet/layout.js | 7 ++--- lib/wallet/wallet.js | 42 ++++++++++++++++++++++++++++++ lib/wallet/walletdb.js | 59 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 3 deletions(-) diff --git a/lib/wallet/layout.js b/lib/wallet/layout.js index 57a9295ef..5d21d00a3 100644 --- a/lib/wallet/layout.js +++ b/lib/wallet/layout.js @@ -28,6 +28,8 @@ const bdb = require('bdb'); * o[hash][index] -> outpoint->wid map * T[hash] -> tx->wid map * t[wid]* -> txdb + * N[hash256] -> name map + * M[migration-id] -> dummy */ exports.wdb = { @@ -49,9 +51,8 @@ exports.wdb = { o: bdb.key('o', ['hash256', 'uint32']), T: bdb.key('T', ['hash256']), t: bdb.key('t', ['uint32']), - - // Name Map - N: bdb.key('N', ['hash256']) + N: bdb.key('N', ['hash256']), + M: bdb.key('M', ['uint32']) }; /* diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index aae276230..5d1f628b1 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -224,6 +224,48 @@ class Wallet extends EventEmitter { } } + /** + * Run change address migration. + * @param {Batch} b + */ + + async migrateChange(b) { + const unlock1 = await this.writeLock.lock(); + const unlock2 = await this.fundLock.lock(); + + try { + return await this._migrateChange(b); + } finally { + unlock2(); + unlock1(); + } + } + + /** + * Run change address migration (without a lock). + * @param {Batch} b + */ + + async _migrateChange(b) { + let total = 0; + + for (let i = 0; i < this.accountDepth; i++) { + const account = await this.getAccount(i); + + for (let i = 0; i < account.changeDepth + account.lookahead; i++) { + const key = account.deriveChange(i); + const path = key.toPath(); + + if (!await this.wdb.hasPath(account.wid, path.hash)) { + await this.wdb.savePath(b, account.wid, path); + total += 1; + } + } + } + + return total; + } + /** * Add a public account key to the wallet (multisig). * Saves the key in the wallet database. diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index e00ab1531..dd416da9b 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -206,6 +206,65 @@ class WalletDB extends EventEmitter { wallet.id, wallet.wid, addr.toString(this.network)); this.primary = wallet; + + await this.migrateChange(); + } + + /** + * Run change address migration. + * @param {Wallet} wallet + */ + + async migrateChange() { + if (this.options.memory) + return; + + if (await this.db.has(layout.M.encode(0))) + return; + + const {prefix} = this.options; + const bak = path.join(prefix, `wallet.${Date.now()}.bak`); + const b = this.db.batch(); + + b.put(layout.M.encode(0), null); + + if (!this.state.marked) { + await b.write(); + return; + } + + this.logger.warning('Running change address migration.'); + this.logger.warning('Backing up database to %s.', bak); + + await this.db.backup(bak); + + const wids = await this.db.keys({ + gte: layout.W.min(), + lte: layout.W.max(), + parse: key => layout.W.decode(key)[0] + }); + + let total = 0; + + for (const wid of wids) { + const wallet = await this.get(wid); + + this.logger.info('Regenerating change addresses (id=%s, wid=%d).', + wallet.id, wid); + + total += await wallet.migrateChange(b); + } + + await b.write(); + + if (total > 0) { + this.logger.warning('Regenerated %d change addresses.', total); + this.logger.warning('Rescanning...'); + + await this.scan(); + } else { + this.logger.info('All change addresses present.'); + } } /** From 07e23587dc02251023fabbc22b64e3cbd9c23f6b Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Tue, 31 Mar 2020 11:41:19 -0400 Subject: [PATCH 2/5] test: cover create change fix and wallet migration --- test/wallet-change-test.js | 155 +++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 test/wallet-change-test.js diff --git a/test/wallet-change-test.js b/test/wallet-change-test.js new file mode 100644 index 000000000..39376570e --- /dev/null +++ b/test/wallet-change-test.js @@ -0,0 +1,155 @@ +/* eslint-env mocha */ +/* eslint prefer-arrow-callback: "off" */ + +'use strict'; + +const assert = require('bsert'); +const FullNode = require('../lib/node/fullnode'); +const Address = require('../lib/primitives/address'); +const {tmpdir} = require('os'); +const {randomBytes} = require('bcrypto/lib/random'); +const Path = require('path'); +const layouts = require('../lib/wallet/layout'); +const layout = layouts.wdb; + +const uniq = randomBytes(4).toString('hex'); +const path = Path.join(tmpdir(), `hsd-test-${uniq}`); + +const node = new FullNode({ + prefix: path, + memory: false, + network: 'regtest', + plugins: [require('../lib/wallet/plugin')] +}); + +const {wdb} = node.require('walletdb'); + +let wallet, recAddr; +const changeAddrs = []; +const manualChangeAddrs = []; +const missingChangeAddrs = []; + +async function mineBlocks(n, addr) { + addr = addr ? addr : new Address().toString('regtest'); + for (let i = 0; i < n; i++) { + const block = await node.miner.mineBlock(null, addr); + await node.chain.add(block); + } +} + +describe('Derive and save change addresses', function() { + before(async () => { + await node.ensure(); + await node.open(); + + wallet = await wdb.create(); + recAddr = await wallet.receiveAddress(); + }); + + after(async () => { + await node.close(); + }); + + it('should fund account', async () => { + await mineBlocks(2, recAddr); + + // Wallet rescan is an effective way to ensure that + // wallet and chain are synced before proceeding. + await wdb.rescan(0); + + const aliceBal = await wallet.getBalance(0); + assert(aliceBal.confirmed === 2000 * 2 * 1e6); + }); + + it('should send 20 transactions', async () => { + for (let i = 0; i < 20; i++) { + const tx = await wallet.send({outputs: [{ + address: Address.fromHash(Buffer.alloc(32, 1)), + value: 10000 + }]}); + + for (const output of tx.outputs) { + if (output.value !== 10000) + changeAddrs.push(output.address); + } + } + }); + + it('should have incremented changeDepth by 20', async () => { + const info = await wallet.getAccount(0); + assert.strictEqual(info.changeDepth, 21); + assert.strictEqual(changeAddrs.length, 20); + }); + + it('should have all change addresses saved', async () => { + for (const addr of changeAddrs) { + assert(await wallet.hasAddress(addr)); + } + }); + + it('should manually generate 20 change addresses', async () => { + for (let i = 0; i < 20; i++) { + const addr = await wallet.createChange(); + manualChangeAddrs.push(addr.getAddress()); + } + }); + + it('should have incremented changeDepth by 20', async () => { + const info = await wallet.getAccount(0); + assert.strictEqual(info.changeDepth, 41); + assert.strictEqual(manualChangeAddrs.length, 20); + }); + + it('should have all change addresses saved', async () => { + for (const addr of manualChangeAddrs) { + assert(await wallet.hasAddress(addr)); + } + }); + + it('should recreate the missing change address bug', async () => { + for (let i = 0; i < 20; i++) { + const acct = await wallet.getAccount(0); + const key = acct.deriveChange(acct.changeDepth); + acct.changeDepth += 1; + const b = wdb.db.batch(); + await wdb.saveAccount(b, acct); + await b.write(); + missingChangeAddrs.push(key.getAddress()); + } + }); + + it('should have no missing change addresses beyond lookahead', async () => { + const acct = await wallet.getAccount(0); + const lookahead = acct.lookahead; + + for (let i = 0; i < missingChangeAddrs.length; i++) { + const addr = await wallet.hasAddress(missingChangeAddrs[i]); + + if (i < lookahead) + assert(addr); + else + assert(!addr); + } + }); + + it('should migrate wallet and recover change addresses', async () => { + // Fake an old db state + await wdb.db.del(layout.M.encode(0)); + + // Run migration script + await wdb.migrateChange(); + + // Fixed + for (const addr of missingChangeAddrs) { + assert(await wallet.hasAddress(addr)); + } + + // Sanity checks + for (const addr of changeAddrs) { + assert(await wallet.hasAddress(addr)); + } + for (const addr of manualChangeAddrs) { + assert(await wallet.hasAddress(addr)); + } + }); +}); From 4d7a14ba4d883cf4cc7e3e9d6d332645203aed7a Mon Sep 17 00:00:00 2001 From: Mark Tyneway Date: Wed, 22 Jul 2020 18:17:20 -0700 Subject: [PATCH 3/5] migration: wallet 0 migration script --- lib/wallet/walletdb.js | 2 -- migrate/wallet0.js | 50 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100755 migrate/wallet0.js diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index dd416da9b..543922de5 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -206,8 +206,6 @@ class WalletDB extends EventEmitter { wallet.id, wallet.wid, addr.toString(this.network)); this.primary = wallet; - - await this.migrateChange(); } /** diff --git a/migrate/wallet0.js b/migrate/wallet0.js new file mode 100755 index 000000000..8373ca0cc --- /dev/null +++ b/migrate/wallet0.js @@ -0,0 +1,50 @@ +#!/usr/bin/env node + +'use strict'; + +const WalletDB = require('../lib/wallet/walletdb'); +const path = require('path'); +const os = require('os'); +const Logger = require('blgr'); + +(async () => { + const args = process.argv.slice(2); + + let network = 'main'; + let prefix = path.join(os.homedir(), '.hsd'); + let level = 'info'; + + while (args.length > 0) { + const arg = args.shift() + switch (arg) { + case '-n': + case '--network': + network = args.shift(); + continue; + case '-p': + case '--prefix': + prefix = args.shift(); + case '-l': + case '--log-level': + level = args.shift(); + } + } + + const logger = new Logger(level); + await logger.open + + const wdb = new WalletDB({ + network: network, + logger: logger, + prefix: prefix, + memory: false + }); + + await wdb.open(); + await wdb.migrateChange(); + await wdb.close() + +})().catch(err => { + console.log(err) + process.exit(1); +}); From 741b13454b2ec668c540bbf806feded26d921b27 Mon Sep 17 00:00:00 2001 From: Mark Tyneway Date: Mon, 27 Jul 2020 11:44:36 -0700 Subject: [PATCH 4/5] eslintfiles: add migrate directory --- .eslintfiles | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintfiles b/.eslintfiles index a30ce329a..6afc4eee9 100644 --- a/.eslintfiles +++ b/.eslintfiles @@ -10,4 +10,5 @@ bin/hsd-cli bin/hsw-cli etc/genesis lib/ +migrate/ test/ From 63147c3faeba1c2023194b35480353d245a5adbc Mon Sep 17 00:00:00 2001 From: Mark Tyneway Date: Mon, 27 Jul 2020 11:45:05 -0700 Subject: [PATCH 5/5] migrate: lint --- migrate/wallet0.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/migrate/wallet0.js b/migrate/wallet0.js index 8373ca0cc..b2d65762e 100755 --- a/migrate/wallet0.js +++ b/migrate/wallet0.js @@ -15,7 +15,7 @@ const Logger = require('blgr'); let level = 'info'; while (args.length > 0) { - const arg = args.shift() + const arg = args.shift(); switch (arg) { case '-n': case '--network': @@ -31,7 +31,7 @@ const Logger = require('blgr'); } const logger = new Logger(level); - await logger.open + await logger.open(); const wdb = new WalletDB({ network: network, @@ -42,9 +42,8 @@ const Logger = require('blgr'); await wdb.open(); await wdb.migrateChange(); - await wdb.close() - -})().catch(err => { - console.log(err) + await wdb.close(); +})().catch((err) => { + console.log(err); process.exit(1); });