Merge PR #896 from 'nodech/update-bidreveal-entries'

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ## Unreleased
 
-**When upgrading to this version of hsd, you must pass `--wallet-migrate=3` when
+**When upgrading to this version of hsd, you must pass `--wallet-migrate=4` when
 you run it for the first time.**
 
 ### Primitives
@@ -72,7 +72,18 @@ process and allows parallel rescans.
     conflicting transactions)
   - Add options to `getNames` for passing `own`.
   - Rename `createAuctionTxs` to `createAuctionTXs`.
-
+  - All `bid` serializations will include `height` of the bid. (`-1` if
+    it was migrated not-owned bid)
+    - `GET /wallet/:id/auction` (`getAuctions`)
+    - `GET /wallet/:id/auction/:name` (`getAuctionByName`)
+    - `GET /wallet/:id/bid` (`getBids`)
+    - `GET /wallet/:id/bid/:name` (`getBidsByName`)
+  - All `reveal` serializations will include `bidPrevout` of the bid. (`null` if
+  it was migrated not-owned reveal)
+    - `GET /wallet/:id/auction` (`getAuctions`)
+    - `GET /wallet/:id/auction/:name` (`getAuctionByName`)
+    - `GET /wallet/:id/reveal` (`getReveals`)
+    - `GET /wallet/:id/reveal/:name` (`getRevealsByName`)
 
 ## v6.0.0
 

lib/wallet/layout.js

@@ -126,6 +126,7 @@ exports.wdb = {
  *   U[tx-hash] -> name undo record (name undo record by tx hash)
  *   i[name-hash][tx-hash][index] -> bid (BlindBid by name + tx + index)
  *   B[name-hash][tx-hash][index] -> reveal (BidReveal by name + tx + index)
+ *   E[name-hash][tx-hash][index] - bid to reveal out (by bid txhash + index)
  *   v[blind-hash] -> blind (Blind Value by blind hash)
  *   o[name-hash] -> tx hash OPEN only (tx hash by name hash)
  */
@@ -163,6 +164,7 @@ exports.txdb = {
   U: bdb.key('U', ['hash256']),
   i: bdb.key('i', ['hash256', 'hash256', 'uint32']),
   B: bdb.key('B', ['hash256', 'hash256', 'uint32']),
+  E: bdb.key('E', ['hash256', 'hash256', 'uint32']),
   v: bdb.key('v', ['hash256']),
   o: bdb.key('o', ['hash256'])
 };

lib/wallet/migrations.js

@@ -19,6 +19,8 @@ const WalletKey = require('./walletkey');
 const Path = require('./path');
 const Script = require('../script/script');
 const MapRecord = require('./records').MapRecord;
+const Outpoint = require('../primitives/outpoint');
+const TX = require('../primitives/tx');
 const AbstractMigration = require('../migrations/migration');
 const {
   MigrationResult,
@@ -27,6 +29,10 @@ const {
   oldLayout
 } = require('../migrations/migrator');
 const layouts = require('./layout');
+const wlayout = layouts.wdb;
+
+/** @typedef {import('bdb').DB} DB */
+/** @typedef {import('./walletdb')} WalletDB */
 
 /**
  * Switch to new migrations layout.
@@ -54,7 +60,7 @@ class MigrateMigrations extends AbstractMigration {
 
   /**
    * Actual migration
-   * @param {Batch} b
+   * @param {DB.Batch} b
    * @returns {Promise}
    */
 
@@ -468,6 +474,249 @@ class MigrateTXDBBalances extends AbstractMigration {
   }
 }
 
+/**
+ * Applies to WalletDB v2
+ * Migrate bid reveal entries.
+ *  - Adds height to the blind bid entries.
+ *    - NOTE: This can not be recovered if the bid is not owned by the wallet.
+ *      Wallet does not store transactions for not-owned bids.
+ *  - Add Bid Outpoint information to the reveal (BidReveal) entries.
+ *    - NOTE: This information can not be recovered for not-owned reveals.
+ *      Wallet does not store transactions for not-owned reveals.
+ *  - Add new BID -> REVEAL index. (layout.E)
+ *    - NOTE: This information can not be recovered for not-owned reveals.
+ *      Wallet does not store transactions for not-owned reveals.
+ *
+ */
+
+class MigrateBidRevealEntries extends AbstractMigration {
+  /**
+   * Create Bid Reveal Entries migration object.
+   * @param {WalletMigratorOptions} options
+   * @constructor
+   */
+
+  constructor(options) {
+    super(options);
+
+    this.options = options;
+    this.logger = options.logger.context('bid-reveal-entries-migration');
+    this.db = options.db;
+    this.ldb = options.ldb;
+    this.layout = MigrateBidRevealEntries.layout();
+  }
+
+  /**
+   * We always migrate.
+   * @returns {Promise}
+   */
+
+  async check() {
+    return types.MIGRATE;
+  }
+
+  /**
+   * Actual migration
+   * @param {DB.Batch} b
+   * @param {WalletMigrationResult} pending
+   * @returns {Promise}
+   */
+
+  async migrate(b, pending) {
+    /** @type {Number[]} */
+    const wids = await this.ldb.keys({
+      gte: wlayout.W.min(),
+      lte: wlayout.W.max(),
+      parse: key => wlayout.W.decode(key)[0]
+    });
+
+    for (const wid of wids) {
+      await this.migrateReveals(wid);
+      await this.migrateBids(wid);
+    }
+
+    this.db.writeVersion(b, 3);
+  }
+
+  /**
+   * Migrate reveals and index Bid2Reveal
+   * @param {Number} wid
+   * @returns {Promise}
+   */
+
+  async migrateReveals(wid) {
+    const txlayout = this.layout.txdb;
+    const prefix = txlayout.prefix.encode(wid);
+    const bucket = this.ldb.bucket(prefix);
+    const emptyOutpoint = new Outpoint();
+
+    const reveals = bucket.iterator({
+      gte: txlayout.B.min(),
+      lte: txlayout.B.max(),
+      values: true
+    });
+
+    for await (const {key, value} of reveals) {
+      const b = bucket.batch();
+      const [nameHash, txHash, txIndex] = txlayout.B.decode(key);
+      const nameLen = value[0];
+      const totalOld = nameLen + 1 + 13;
+      const totalNew = nameLen + 1 + 13 + 36;
+
+      // allow migration to be interrupted in the middle.
+      assert(value.length === totalOld || value.length === totalNew);
+
+      // skip if already migrated.
+      if (value.length === totalNew)
+        continue;
+
+      const owned = value[nameLen + 1 + 12];
+      const rawTXRecord = await bucket.get(txlayout.t.encode(txHash));
+      assert(owned && rawTXRecord || !owned);
+
+      // We can not index the bid link and bid2reveal index if
+      // the transaction is not owned by the wallet.
+      // But we need to put null outpoint to the reveal for serialization.
+      if (!owned) {
+        const newReveal = Buffer.concat([value, emptyOutpoint.encode()]);
+        assert(newReveal.length === totalNew);
+        b.put(key, newReveal);
+        await b.write();
+        continue;
+      }
+
+      const reader = bio.read(rawTXRecord);
+      const tx = TX.fromReader(reader);
+      assert(tx.inputs[txIndex]);
+
+      const bidPrevout = tx.inputs[txIndex].prevout;
+      const bidKey = txlayout.i.encode(
+        nameHash, bidPrevout.hash, bidPrevout.index);
+      const bidRecord = await bucket.get(bidKey);
+      // ensure bid exists.
+      assert(bidRecord);
+
+      const newReveal = Buffer.concat([value, bidPrevout.encode()]);
+      assert(newReveal.length === totalNew);
+      // update reveal with bid outpoint.
+      b.put(key, newReveal);
+      // index bid to reveal.
+      b.put(txlayout.E.encode(nameHash, bidPrevout.hash, bidPrevout.index),
+        (new Outpoint(txHash, txIndex)).encode());
+      await b.write();
+    }
+  }
+
+  /**
+   * Migrate bids, add height to the entries.
+   * @param {Number} wid
+   * @returns {Promise}
+   */
+
+  async migrateBids(wid) {
+    const txlayout = this.layout.txdb;
+    const prefix = txlayout.prefix.encode(wid);
+    const bucket = this.ldb.bucket(prefix);
+
+    const bids = bucket.iterator({
+      gte: txlayout.i.min(),
+      lte: txlayout.i.max(),
+      values: true
+    });
+
+    /**
+     * @param {Buffer} blindBid
+     * @param {Number} height
+     * @returns {Buffer}
+     */
+
+    const reencodeBlindBid = (blindBid, height) => {
+      const nameLen = blindBid[0];
+      const totalOld = nameLen + 1 + 41;
+      const totalNew = nameLen + 1 + 41 + 4;
+      assert(blindBid.length === totalOld);
+
+      const newBlindBid = Buffer.alloc(totalNew);
+      // copy everything before expected height place.
+      blindBid.copy(newBlindBid, 0, 0, totalOld - 1);
+      // copy height.
+      bio.encoding.writeU32(newBlindBid, height, totalOld - 1);
+      // copy last byte (owned flag).
+      blindBid.copy(newBlindBid, totalNew - 1, totalOld - 1);
+
+      return newBlindBid;
+    };
+
+    for await (const {key, value} of bids) {
+      const b = bucket.batch();
+      const [,txHash] = txlayout.i.decode(key);
+      const nameLen = value[0];
+      const totalNew = nameLen + 1 + 41 + 4;
+
+      // allow migration to be interrupted in the middle.
+      if (totalNew === value.length)
+        continue;
+
+      const owned = value[nameLen + 1 + 40];
+      if (!owned) {
+        const height = 0xffffffff; // -1
+        const newValue = reencodeBlindBid(value, height);
+        b.put(key, newValue);
+        await b.write();
+        continue;
+      }
+
+      const rawTXRecord = await bucket.get(txlayout.t.encode(txHash));
+      assert(rawTXRecord);
+
+      const br = bio.read(rawTXRecord);
+      TX.fromReader(br);
+      // skip mtime.
+      br.seek(4);
+
+      const hasBlock = br.readU8() === 1;
+      // We only index the bid in blocks, not in mempool.
+      assert(hasBlock);
+
+      // skip hash.
+      br.seek(32);
+      const height = br.readU32();
+      const newValue = reencodeBlindBid(value, height);
+      b.put(key, newValue);
+
+      await b.write();
+    }
+  }
+
+  static info() {
+    return {
+      name: 'Bid reveal entries migration',
+      description: 'Migrate bids and reveals to link each other.'
+    };
+  }
+
+  static layout() {
+    return {
+      wdb: {
+        V: bdb.key('V'),
+        // W[wid] -> wallet id
+        W: bdb.key('W', ['uint32'])
+      },
+      txdb: {
+        prefix: bdb.key('t', ['uint32']),
+        // t[tx-hash] -> extended tx (Read only)
+        t: bdb.key('t', ['hash256']),
+        // i[name-hash][tx-hash][index] -> txdb.BlindBid
+        i: bdb.key('i', ['hash256', 'hash256', 'uint32']),
+        // B[name-hash][tx-hash][index] -> txdb.BidReveal
+        B: bdb.key('B', ['hash256', 'hash256', 'uint32']),
+        // E[name-hash][tx-hash][index] -> bid to reveal out.
+        E: bdb.key('E', ['hash256', 'hash256', 'uint32'])
+      }
+    };
+  }
+}
+
 /**
  * Wallet migration results.
  * @alias module:blockchain.WalletMigrationResult
@@ -548,12 +797,14 @@ class WalletMigratorOptions {
     this.migrateFlag = -1;
 
     this.dbVersion = 0;
+    /** @type {WalletDB} */
     this.db = null;
+    /** @type {DB} */
     this.ldb = null;
     this.layout = layouts.wdb;
 
-    if (options)
-      this.fromOptions(options);
+    assert(options);
+    this.fromOptions(options);
   }
 
   /**
@@ -603,13 +854,15 @@ exports.migrations = {
   0: MigrateMigrations,
   1: MigrateChangeAddress,
   2: MigrateAccountLookahead,
-  3: MigrateTXDBBalances
+  3: MigrateTXDBBalances,
+  4: MigrateBidRevealEntries
 };
 
 // Expose migrations
 exports.MigrateChangeAddress = MigrateChangeAddress;
 exports.MigrateMigrations = MigrateMigrations;
 exports.MigrateAccountLookahead = MigrateAccountLookahead;
 exports.MigrateTXDBBalances = MigrateTXDBBalances;
+exports.MigrateBidRevealEntries = MigrateBidRevealEntries;
 
 module.exports = exports;