walletdb: change migration script

.eslintfiles

@@ -10,4 +10,5 @@ bin/hsd-cli
 bin/hsw-cli
 etc/genesis
 lib/
+migrate/
 test/

lib/wallet/layout.js

@@ -28,6 +28,8 @@ const bdb = require('bdb');
  *  o[hash][index] -> outpoint->wid map
  *  T[hash] -> tx->wid map
  *  t[wid]* -> txdb
+ *  N[hash256] -> name map
+ *  M[migration-id] -> dummy
  */
 
 exports.wdb = {
@@ -49,9 +51,8 @@ exports.wdb = {
   o: bdb.key('o', ['hash256', 'uint32']),
   T: bdb.key('T', ['hash256']),
   t: bdb.key('t', ['uint32']),
-
-  // Name Map
-  N: bdb.key('N', ['hash256'])
+  N: bdb.key('N', ['hash256']),
+  M: bdb.key('M', ['uint32'])
 };
 
 /*

lib/wallet/wallet.js

@@ -224,6 +224,48 @@ class Wallet extends EventEmitter {
     }
   }
 
+  /**
+   * Run change address migration.
+   * @param {Batch} b
+   */
+
+  async migrateChange(b) {
+    const unlock1 = await this.writeLock.lock();
+    const unlock2 = await this.fundLock.lock();
+
+    try {
+      return await this._migrateChange(b);
+    } finally {
+      unlock2();
+      unlock1();
+    }
+  }
+
+  /**
+   * Run change address migration (without a lock).
+   * @param {Batch} b
+   */
+
+  async _migrateChange(b) {
+    let total = 0;
+
+    for (let i = 0; i < this.accountDepth; i++) {
+      const account = await this.getAccount(i);
+
+      for (let i = 0; i < account.changeDepth + account.lookahead; i++) {
+        const key = account.deriveChange(i);
+        const path = key.toPath();
+
+        if (!await this.wdb.hasPath(account.wid, path.hash)) {
+          await this.wdb.savePath(b, account.wid, path);
+          total += 1;
+        }
+      }
+    }
+
+    return total;
+  }
+
   /**
    * Add a public account key to the wallet (multisig).
    * Saves the key in the wallet database.

lib/wallet/walletdb.js

@@ -208,6 +208,63 @@ class WalletDB extends EventEmitter {
     this.primary = wallet;
   }
 
+  /**
+   * Run change address migration.
+   * @param {Wallet} wallet
+   */
+
+  async migrateChange() {
+    if (this.options.memory)
+      return;
+
+    if (await this.db.has(layout.M.encode(0)))
+      return;
+
+    const {prefix} = this.options;
+    const bak = path.join(prefix, `wallet.${Date.now()}.bak`);
+    const b = this.db.batch();
+
+    b.put(layout.M.encode(0), null);
+
+    if (!this.state.marked) {
+      await b.write();
+      return;
+    }
+
+    this.logger.warning('Running change address migration.');
+    this.logger.warning('Backing up database to %s.', bak);
+
+    await this.db.backup(bak);
+
+    const wids = await this.db.keys({
+      gte: layout.W.min(),
+      lte: layout.W.max(),
+      parse: key => layout.W.decode(key)[0]
+    });
+
+    let total = 0;
+
+    for (const wid of wids) {
+      const wallet = await this.get(wid);
+
+      this.logger.info('Regenerating change addresses (id=%s, wid=%d).',
+                       wallet.id, wid);
+
+      total += await wallet.migrateChange(b);
+    }
+
+    await b.write();
+
+    if (total > 0) {
+      this.logger.warning('Regenerated %d change addresses.', total);
+      this.logger.warning('Rescanning...');
+
+      await this.scan();
+    } else {
+      this.logger.info('All change addresses present.');
+    }
+  }
+
   /**
    * Verify network.
    * @returns {Promise}

migrate/wallet0.js

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const WalletDB = require('../lib/wallet/walletdb');
+const path = require('path');
+const os = require('os');
+const Logger = require('blgr');
+
+(async () => {
+  const args = process.argv.slice(2);
+
+  let network = 'main';
+  let prefix = path.join(os.homedir(), '.hsd');
+  let level = 'info';
+
+  while (args.length > 0) {
+    const arg = args.shift();
+    switch (arg) {
+      case '-n':
+      case '--network':
+        network = args.shift();
+        continue;
+      case '-p':
+      case '--prefix':
+        prefix = args.shift();
+      case '-l':
+      case '--log-level':
+        level = args.shift();
+    }
+  }
+
+  const logger = new Logger(level);
+  await logger.open();
+
+  const wdb = new WalletDB({
+    network: network,
+    logger: logger,
+    prefix: prefix,
+    memory: false
+  });
+
+  await wdb.open();
+  await wdb.migrateChange();
+  await wdb.close();
+})().catch((err) => {
+  console.log(err);
+  process.exit(1);
+});

test/wallet-change-test.js

@@ -0,0 +1,155 @@
+/* eslint-env mocha */
+/* eslint prefer-arrow-callback: "off" */
+
+'use strict';
+
+const assert = require('bsert');
+const FullNode = require('../lib/node/fullnode');
+const Address = require('../lib/primitives/address');
+const {tmpdir} = require('os');
+const {randomBytes} = require('bcrypto/lib/random');
+const Path = require('path');
+const layouts = require('../lib/wallet/layout');
+const layout = layouts.wdb;
+
+const uniq = randomBytes(4).toString('hex');
+const path = Path.join(tmpdir(), `hsd-test-${uniq}`);
+
+const node = new FullNode({
+  prefix: path,
+  memory: false,
+  network: 'regtest',
+  plugins: [require('../lib/wallet/plugin')]
+});
+
+const {wdb} = node.require('walletdb');
+
+let wallet, recAddr;
+const changeAddrs = [];
+const manualChangeAddrs = [];
+const missingChangeAddrs = [];
+
+async function mineBlocks(n, addr) {
+  addr = addr ? addr : new Address().toString('regtest');
+  for (let i = 0; i < n; i++) {
+    const block = await node.miner.mineBlock(null, addr);
+    await node.chain.add(block);
+  }
+}
+
+describe('Derive and save change addresses', function() {
+  before(async () => {
+    await node.ensure();
+    await node.open();
+
+    wallet = await wdb.create();
+    recAddr = await wallet.receiveAddress();
+  });
+
+  after(async () => {
+    await node.close();
+  });
+
+  it('should fund account', async () => {
+    await mineBlocks(2, recAddr);
+
+    // Wallet rescan is an effective way to ensure that
+    // wallet and chain are synced before proceeding.
+    await wdb.rescan(0);
+
+    const aliceBal = await wallet.getBalance(0);
+    assert(aliceBal.confirmed === 2000 * 2 * 1e6);
+  });
+
+  it('should send 20 transactions', async () => {
+    for (let i = 0; i < 20; i++) {
+      const tx = await wallet.send({outputs: [{
+        address: Address.fromHash(Buffer.alloc(32, 1)),
+        value: 10000
+      }]});
+
+      for (const output of tx.outputs) {
+        if (output.value !== 10000)
+          changeAddrs.push(output.address);
+      }
+    }
+  });
+
+  it('should have incremented changeDepth by 20', async () => {
+    const info = await wallet.getAccount(0);
+    assert.strictEqual(info.changeDepth, 21);
+    assert.strictEqual(changeAddrs.length, 20);
+  });
+
+  it('should have all change addresses saved', async () => {
+    for (const addr of changeAddrs) {
+      assert(await wallet.hasAddress(addr));
+    }
+  });
+
+  it('should manually generate 20 change addresses', async () => {
+    for (let i = 0; i < 20; i++) {
+      const addr = await wallet.createChange();
+      manualChangeAddrs.push(addr.getAddress());
+    }
+  });
+
+  it('should have incremented changeDepth by 20', async () => {
+    const info = await wallet.getAccount(0);
+    assert.strictEqual(info.changeDepth, 41);
+    assert.strictEqual(manualChangeAddrs.length, 20);
+  });
+
+  it('should have all change addresses saved', async () => {
+    for (const addr of manualChangeAddrs) {
+      assert(await wallet.hasAddress(addr));
+    }
+  });
+
+  it('should recreate the missing change address bug', async () => {
+    for (let i = 0; i < 20; i++) {
+      const acct = await wallet.getAccount(0);
+      const key = acct.deriveChange(acct.changeDepth);
+      acct.changeDepth += 1;
+      const b = wdb.db.batch();
+      await wdb.saveAccount(b, acct);
+      await b.write();
+      missingChangeAddrs.push(key.getAddress());
+    }
+  });
+
+  it('should have no missing change addresses beyond lookahead', async () => {
+    const acct = await wallet.getAccount(0);
+    const lookahead = acct.lookahead;
+
+    for (let i = 0; i < missingChangeAddrs.length; i++) {
+      const addr = await wallet.hasAddress(missingChangeAddrs[i]);
+
+      if (i < lookahead)
+        assert(addr);
+      else
+        assert(!addr);
+    }
+  });
+
+  it('should migrate wallet and recover change addresses', async () => {
+    // Fake an old db state
+    await wdb.db.del(layout.M.encode(0));
+
+    // Run migration script
+    await wdb.migrateChange();
+
+    // Fixed
+    for (const addr of missingChangeAddrs) {
+      assert(await wallet.hasAddress(addr));
+    }
+
+    // Sanity checks
+    for (const addr of changeAddrs) {
+      assert(await wallet.hasAddress(addr));
+    }
+    for (const addr of manualChangeAddrs) {
+      assert(await wallet.hasAddress(addr));
+    }
+  });
+});