diff --git a/.eslintfiles b/.eslintfiles index a30ce329a..6afc4eee9 100644 --- a/.eslintfiles +++ b/.eslintfiles @@ -10,4 +10,5 @@ bin/hsd-cli bin/hsw-cli etc/genesis lib/ +migrate/ test/ diff --git a/lib/wallet/layout.js b/lib/wallet/layout.js index 57a9295ef..5d21d00a3 100644 --- a/lib/wallet/layout.js +++ b/lib/wallet/layout.js @@ -28,6 +28,8 @@ const bdb = require('bdb'); * o[hash][index] -> outpoint->wid map * T[hash] -> tx->wid map * t[wid]* -> txdb + * N[hash256] -> name map + * M[migration-id] -> dummy */ exports.wdb = { @@ -49,9 +51,8 @@ exports.wdb = { o: bdb.key('o', ['hash256', 'uint32']), T: bdb.key('T', ['hash256']), t: bdb.key('t', ['uint32']), - - // Name Map - N: bdb.key('N', ['hash256']) + N: bdb.key('N', ['hash256']), + M: bdb.key('M', ['uint32']) }; /* diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index aae276230..5d1f628b1 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -224,6 +224,48 @@ class Wallet extends EventEmitter { } } + /** + * Run change address migration. + * @param {Batch} b + */ + + async migrateChange(b) { + const unlock1 = await this.writeLock.lock(); + const unlock2 = await this.fundLock.lock(); + + try { + return await this._migrateChange(b); + } finally { + unlock2(); + unlock1(); + } + } + + /** + * Run change address migration (without a lock). + * @param {Batch} b + */ + + async _migrateChange(b) { + let total = 0; + + for (let i = 0; i < this.accountDepth; i++) { + const account = await this.getAccount(i); + + for (let i = 0; i < account.changeDepth + account.lookahead; i++) { + const key = account.deriveChange(i); + const path = key.toPath(); + + if (!await this.wdb.hasPath(account.wid, path.hash)) { + await this.wdb.savePath(b, account.wid, path); + total += 1; + } + } + } + + return total; + } + /** * Add a public account key to the wallet (multisig). * Saves the key in the wallet database. diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index e00ab1531..543922de5 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -208,6 +208,63 @@ class WalletDB extends EventEmitter { this.primary = wallet; } + /** + * Run change address migration. + * @param {Wallet} wallet + */ + + async migrateChange() { + if (this.options.memory) + return; + + if (await this.db.has(layout.M.encode(0))) + return; + + const {prefix} = this.options; + const bak = path.join(prefix, `wallet.${Date.now()}.bak`); + const b = this.db.batch(); + + b.put(layout.M.encode(0), null); + + if (!this.state.marked) { + await b.write(); + return; + } + + this.logger.warning('Running change address migration.'); + this.logger.warning('Backing up database to %s.', bak); + + await this.db.backup(bak); + + const wids = await this.db.keys({ + gte: layout.W.min(), + lte: layout.W.max(), + parse: key => layout.W.decode(key)[0] + }); + + let total = 0; + + for (const wid of wids) { + const wallet = await this.get(wid); + + this.logger.info('Regenerating change addresses (id=%s, wid=%d).', + wallet.id, wid); + + total += await wallet.migrateChange(b); + } + + await b.write(); + + if (total > 0) { + this.logger.warning('Regenerated %d change addresses.', total); + this.logger.warning('Rescanning...'); + + await this.scan(); + } else { + this.logger.info('All change addresses present.'); + } + } + /** * Verify network. * @returns {Promise} diff --git a/migrate/wallet0.js b/migrate/wallet0.js new file mode 100755 index 000000000..b2d65762e --- /dev/null +++ b/migrate/wallet0.js @@ -0,0 +1,49 @@ +#!/usr/bin/env node + +'use strict'; + +const WalletDB = require('../lib/wallet/walletdb'); +const path = require('path'); +const os = require('os'); +const Logger = require('blgr'); + +(async () => { + const args = process.argv.slice(2); + + let network = 'main'; + let prefix = path.join(os.homedir(), '.hsd'); + let level = 'info'; + + while (args.length > 0) { + const arg = args.shift(); + switch (arg) { + case '-n': + case '--network': + network = args.shift(); + continue; + case '-p': + case '--prefix': + prefix = args.shift(); + case '-l': + case '--log-level': + level = args.shift(); + } + } + + const logger = new Logger(level); + await logger.open(); + + const wdb = new WalletDB({ + network: network, + logger: logger, + prefix: prefix, + memory: false + }); + + await wdb.open(); + await wdb.migrateChange(); + await wdb.close(); +})().catch((err) => { + console.log(err); + process.exit(1); +}); diff --git a/test/wallet-change-test.js b/test/wallet-change-test.js new file mode 100644 index 000000000..39376570e --- /dev/null +++ b/test/wallet-change-test.js @@ -0,0 +1,155 @@ +/* eslint-env mocha */ +/* eslint prefer-arrow-callback: "off" */ + +'use strict'; + +const assert = require('bsert'); +const FullNode = require('../lib/node/fullnode'); +const Address = require('../lib/primitives/address'); +const {tmpdir} = require('os'); +const {randomBytes} = require('bcrypto/lib/random'); +const Path = require('path'); +const layouts = require('../lib/wallet/layout'); +const layout = layouts.wdb; + +const uniq = randomBytes(4).toString('hex'); +const path = Path.join(tmpdir(), `hsd-test-${uniq}`); + +const node = new FullNode({ + prefix: path, + memory: false, + network: 'regtest', + plugins: [require('../lib/wallet/plugin')] +}); + +const {wdb} = node.require('walletdb'); + +let wallet, recAddr; +const changeAddrs = []; +const manualChangeAddrs = []; +const missingChangeAddrs = []; + +async function mineBlocks(n, addr) { + addr = addr ? addr : new Address().toString('regtest'); + for (let i = 0; i < n; i++) { + const block = await node.miner.mineBlock(null, addr); + await node.chain.add(block); + } +} + +describe('Derive and save change addresses', function() { + before(async () => { + await node.ensure(); + await node.open(); + + wallet = await wdb.create(); + recAddr = await wallet.receiveAddress(); + }); + + after(async () => { + await node.close(); + }); + + it('should fund account', async () => { + await mineBlocks(2, recAddr); + + // Wallet rescan is an effective way to ensure that + // wallet and chain are synced before proceeding. + await wdb.rescan(0); + + const aliceBal = await wallet.getBalance(0); + assert(aliceBal.confirmed === 2000 * 2 * 1e6); + }); + + it('should send 20 transactions', async () => { + for (let i = 0; i < 20; i++) { + const tx = await wallet.send({outputs: [{ + address: Address.fromHash(Buffer.alloc(32, 1)), + value: 10000 + }]}); + + for (const output of tx.outputs) { + if (output.value !== 10000) + changeAddrs.push(output.address); + } + } + }); + + it('should have incremented changeDepth by 20', async () => { + const info = await wallet.getAccount(0); + assert.strictEqual(info.changeDepth, 21); + assert.strictEqual(changeAddrs.length, 20); + }); + + it('should have all change addresses saved', async () => { + for (const addr of changeAddrs) { + assert(await wallet.hasAddress(addr)); + } + }); + + it('should manually generate 20 change addresses', async () => { + for (let i = 0; i < 20; i++) { + const addr = await wallet.createChange(); + manualChangeAddrs.push(addr.getAddress()); + } + }); + + it('should have incremented changeDepth by 20', async () => { + const info = await wallet.getAccount(0); + assert.strictEqual(info.changeDepth, 41); + assert.strictEqual(manualChangeAddrs.length, 20); + }); + + it('should have all change addresses saved', async () => { + for (const addr of manualChangeAddrs) { + assert(await wallet.hasAddress(addr)); + } + }); + + it('should recreate the missing change address bug', async () => { + for (let i = 0; i < 20; i++) { + const acct = await wallet.getAccount(0); + const key = acct.deriveChange(acct.changeDepth); + acct.changeDepth += 1; + const b = wdb.db.batch(); + await wdb.saveAccount(b, acct); + await b.write(); + missingChangeAddrs.push(key.getAddress()); + } + }); + + it('should have no missing change addresses beyond lookahead', async () => { + const acct = await wallet.getAccount(0); + const lookahead = acct.lookahead; + + for (let i = 0; i < missingChangeAddrs.length; i++) { + const addr = await wallet.hasAddress(missingChangeAddrs[i]); + + if (i < lookahead) + assert(addr); + else + assert(!addr); + } + }); + + it('should migrate wallet and recover change addresses', async () => { + // Fake an old db state + await wdb.db.del(layout.M.encode(0)); + + // Run migration script + await wdb.migrateChange(); + + // Fixed + for (const addr of missingChangeAddrs) { + assert(await wallet.hasAddress(addr)); + } + + // Sanity checks + for (const addr of changeAddrs) { + assert(await wallet.hasAddress(addr)); + } + for (const addr of manualChangeAddrs) { + assert(await wallet.hasAddress(addr)); + } + }); +});