migrations: Add migration state migrations to the wallet and the chain.

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+**When upgrading to this version of hsd, you must pass `--chain-migrate=4`
+and `--wallet-migrate=6` when you run it for the first time.**
+
 ### Wallet Changes
 
 #### Wallet HTTP API

lib/blockchain/migrations.js

@@ -533,6 +533,45 @@ class MigrateTreeState extends AbstractMigration {
   }
 }
 
+class MigrateMigrationStateV1 extends AbstractMigration {
+  /**
+   * Create migration migration state
+   * @constructor
+   * @param {ChainMigratorOptions} options
+   */
+
+  constructor(options) {
+    super(options);
+
+    this.options = options;
+    this.logger = options.logger.context('chain-migration-migration-state-v1');
+    this.db = options.db;
+    this.ldb = options.ldb;
+    this.network = options.network;
+  }
+
+  async check() {
+    return types.MIGRATE;
+  }
+
+  /**
+   * @param {Batch} b
+   * @param {MigrationContext} ctx
+   * @returns {Promise}
+   */
+
+  async migrate(b, ctx) {
+    ctx.state.version = 1;
+  }
+
+  static info() {
+    return {
+      name: 'Migrate Migration State',
+      description: 'Migrate migration state to v1'
+    };
+  }
+}
+
 /**
  * Chain Migrator
  * @alias module:blockchain.ChainMigrator
@@ -550,8 +589,6 @@ class ChainMigrator extends Migrator {
     this.logger = this.options.logger.context('chain-migrations');
     this.flagError = 'Restart with `hsd --chain-migrate='
       + this.lastMigration + '`';
-
-    this._migrationsToRun = null;
   }
 
   /**
@@ -673,7 +710,8 @@ ChainMigrator.migrations = {
   0: MigrateMigrations,
   1: MigrateChainState,
   2: MigrateBlockStore,
-  3: MigrateTreeState
+  3: MigrateTreeState,
+  4: MigrateMigrationStateV1
 };
 
 // Expose migrations
@@ -681,5 +719,6 @@ ChainMigrator.MigrateChainState = MigrateChainState;
 ChainMigrator.MigrateMigrations = MigrateMigrations;
 ChainMigrator.MigrateBlockStore = MigrateBlockStore;
 ChainMigrator.MigrateTreeState = MigrateTreeState;
+ChainMigrator.MigrateMigrationStateV1 = MigrateMigrationStateV1;
 
 module.exports = ChainMigrator;

lib/migrations/migrator.js

@@ -13,6 +13,7 @@ const MigrationState = require('../migrations/state');
 
 /** @typedef {ReturnType<bdb.DB['batch']>} Batch */
 /** @typedef {import('../blockchain/chaindb')} ChainDB */
+/** @typedef {import('../wallet/walletdb')} WalletDB */
 
 const EMPTY = Buffer.alloc(0);
 
@@ -67,9 +68,9 @@ class Migrator {
     this.migrateFlag = -1;
 
     this.layout = migrationLayout;
-    /** @type {ChainDB} */
+    /** @type {ChainDB|WalletDB|null} */
     this.db = null;
-    /** @type {bdb.DB} */
+    /** @type {bdb.DB?} */
     this.ldb = null;
     this.dbVersion = 0;
 

lib/wallet/migrations.js

@@ -60,7 +60,7 @@ class MigrateMigrations extends AbstractMigration {
 
     /** @type {WalletMigratorOptions} */
     this.options = options;
-    this.logger = options.logger.context('wallet-migrations-migrate');
+    this.logger = options.logger.context('wallet-migration-migrate');
     this.db = options.db;
     this.ldb = options.ldb;
     this.layout = MigrateMigrations.layout();
@@ -136,7 +136,7 @@ class MigrateChangeAddress extends AbstractMigration {
 
     /** @type {WalletMigratorOptions} */
     this.options = options;
-    this.logger = options.logger.context('change-address-migration');
+    this.logger = options.logger.context('wallet-migration-change-address');
     this.db = options.db;
     this.ldb = options.ldb;
     this.layout = MigrateChangeAddress.layout();
@@ -381,7 +381,7 @@ class MigrateAccountLookahead extends AbstractMigration {
 
     /** @type {WalletMigratorOptions} */
     this.options = options;
-    this.logger = options.logger.context('account-lookahead-migration');
+    this.logger = options.logger.context('wallet-migration-account-lookahead');
     this.db = options.db;
     this.ldb = options.ldb;
     this.layout = MigrateAccountLookahead.layout();
@@ -492,7 +492,7 @@ class MigrateTXDBBalances extends AbstractMigration {
 
     /** @type {WalletMigratorOptions} */
     this.options = options;
-    this.logger = options.logger.context('txdb-balance-migration');
+    this.logger = options.logger.context('wallet-migration-txdb-balance');
     this.db = options.db;
     this.ldb = options.ldb;
   }
@@ -552,7 +552,7 @@ class MigrateBidRevealEntries extends AbstractMigration {
 
     /** @type {WalletMigratorOptions} */
     this.options = options;
-    this.logger = options.logger.context('bid-reveal-entries-migration');
+    this.logger = options.logger.context('wallet-migration-bid-reveal-entries');
     this.db = options.db;
     this.ldb = options.ldb;
     this.layout = MigrateBidRevealEntries.layout();
@@ -788,7 +788,8 @@ class MigrateTXCountTimeIndex extends AbstractMigration {
 
     /** @type {WalletMigratorOptions} */
     this.options = options;
-    this.logger = options.logger.context('tx-count-time-index-migration');
+    this.logger = options.logger.context(
+      'wallet-migration-tx-count-time-index');
     this.db = options.db;
     this.ldb = options.ldb;
     this.layout = MigrateTXCountTimeIndex.layout();
@@ -1306,6 +1307,51 @@ class MigrateTXCountTimeIndex extends AbstractMigration {
   }
 }
 
+class MigrateMigrationStateV1 extends AbstractMigration {
+  /**
+   * Create Migration State migration object.
+   * @param {WalletMigratorOptions} options
+   * @constructor
+   */
+
+  constructor(options) {
+    super(options);
+
+    /** @type {WalletMigratorOptions} */
+    this.options = options;
+    this.logger = options.logger.context('wallet-migration-migration-state-v1');
+    this.db = options.db;
+    this.ldb = options.ldb;
+  }
+
+  /**
+   * We always migrate.
+   * @returns {Promise<MigrationType>}
+   */
+
+  async check() {
+    return types.MIGRATE;
+  }
+
+  /**
+   * Migrate Migration State.
+   * @param {Batch} b
+   * @param {WalletMigrationContext} ctx
+   * @returns {Promise}
+   */
+
+  async migrate(b, ctx) {
+    ctx.state.version = 1;
+  }
+
+  static info() {
+    return {
+      name: 'Migrate Migration State',
+      description: 'Migrate migration state to v1'
+    };
+  }
+}
+
 /**
  * Wallet migration results.
  * @alias module:blockchain.WalletMigrationResult
@@ -1489,7 +1535,8 @@ WalletMigrator.migrations = {
   2: MigrateAccountLookahead,
   3: MigrateTXDBBalances,
   4: MigrateBidRevealEntries,
-  5: MigrateTXCountTimeIndex
+  5: MigrateTXCountTimeIndex,
+  6: MigrateMigrationStateV1
 };
 
 // Expose migrations
@@ -1499,5 +1546,6 @@ WalletMigrator.MigrateAccountLookahead = MigrateAccountLookahead;
 WalletMigrator.MigrateTXDBBalances = MigrateTXDBBalances;
 WalletMigrator.MigrateBidRevealEntries = MigrateBidRevealEntries;
 WalletMigrator.MigrateTXCountTimeIndex = MigrateTXCountTimeIndex;
+WalletMigrator.MigrateMigrationStateV1 = MigrateMigrationStateV1;
 
 module.exports = WalletMigrator;

test/chain-migration-test.js

@@ -18,13 +18,15 @@ const {
   types,
   oldLayout
 } = require('../lib/migrations/migrator');
+
+const migutils = require('./util/migrations');
 const {
   migrationError,
   writeVersion,
   getVersion,
   fillEntries,
   checkEntries
-} = require('./util/migrations');
+} = migutils;
 const common = require('./util/common');
 const {rimraf, testdir} = common;
 
@@ -629,7 +631,6 @@ describe('Chain Migrations', function() {
     for (const tcase of data.cases) {
       it(`should migrate ${tcase.description}`, async () => {
         const before = tcase.before;
-        const after = tcase.after;
         const version = tcase.dbVersion;
         const mustMigrate1 = tcase.migrate1;
         assert(typeof version === 'number');
@@ -681,7 +682,12 @@ describe('Chain Migrations', function() {
 
         if (mustMigrate1)
           assert(migrated, 'Migration 1 did not run.');
-        await checkEntries(ldb, after);
+
+        await checkEntries(ldb, {
+          before: data.before,
+          after: data.after,
+          throw: true
+        });
       });
     }
   });
@@ -749,7 +755,12 @@ describe('Chain Migrations', function() {
         ;
       }
 
-      await checkEntries(ldb, data.after);
+      await checkEntries(ldb, {
+        before: data.before,
+        after: data.after,
+        throw: true
+      });
+
       await chain.close();
     });
   });
@@ -1298,7 +1309,69 @@ describe('Chain Migrations', function() {
         ;
       }
 
-      await checkEntries(ldb, data.after);
+      await checkEntries(ldb, {
+        before: data.before,
+        after: data.after,
+        throw: true
+      });
+
+      await chain.close();
+    });
+  });
+
+  describe('Migrate Migration state v1 (data)', function() {
+    const location = testdir('migrate-migration-state-v1');
+    const data = require('./data/migrations/chain-4-migrationstate-v1.json');
+    const migrationsBAK = ChainMigrator.migrations;
+    const Migration = ChainMigrator.MigrateMigrationStateV1;
+    const store = BlockStore.create({
+      memory: true,
+      network
+    });
+
+    const chainOptions = {
+      prefix: location,
+      memory: false,
+      blocks: store,
+      logger: Logger.global,
+      network
+    };
+
+    let chain, ldb;
+    before(async () => {
+      ChainMigrator.migrations = {};
+      chain = new Chain(chainOptions);
+      ldb = chain.db.db;
+
+      await fs.mkdirp(location);
+      await store.open();
+      await chain.open();
+
+      await fillEntries(ldb, data.before);
+
+      await chain.close();
+      await store.close();
+    });
+
+    after(async () => {
+      ChainMigrator.migrations = migrationsBAK;
+      await rimraf(location);
+    });
+
+    it('should migrate', async () => {
+      ChainMigrator.migrations = {
+        0: Migration
+      };
+
+      chain.options.chainMigrate = 0;
+
+      await chain.open();
+      await checkEntries(ldb, {
+        before: data.before,
+        after: data.after,
+        logErrors: true,
+        throw: true
+      });
       await chain.close();
     });
   });

test/data/migrations/chain-4-migrationstate-v1.json

@@ -0,0 +1,9 @@
+{
+  "description": "Migration state update",
+  "before": {
+    "4d": "000000000000"
+  },
+  "after": {
+    "4d": "00000100010000"
+  }
+}

test/data/migrations/wallet-6-migrationstate-v1.json

@@ -0,0 +1,9 @@
+{
+  "description": "Migration state update",
+  "before": {
+    "4d": "000000000000"
+  },
+  "after": {
+    "4d": "00000100010000"
+  }
+}

test/wallet-migration-test.js

@@ -18,6 +18,7 @@ const {
   types,
   oldLayout
 } = require('../lib/migrations/migrator');
+const migutils = require('./util/migrations');
 const {
   migrationError,
   writeVersion,
@@ -25,7 +26,7 @@ const {
   checkVersion,
   checkEntries,
   fillEntries
-} = require('./util/migrations');
+} = migutils;
 const {rimraf, testdir} = require('./util/common');
 
 const NETWORK = 'regtest';
@@ -1059,4 +1060,51 @@ describe('Wallet Migrations', function() {
       await walletDB.close();
     });
   });
+
+  describe('Migrate Migration state v1 (data)', function() {
+    const location = testdir('wallet-migrate-migration-state-v1');
+    const data = require('./data/migrations/wallet-6-migrationstate-v1.json');
+    const migrationsBAK = WalletMigrator.migrations;
+    const Migration = WalletMigrator.MigrateMigrationStateV1;
+
+    const walletOptions = {
+      prefix: location,
+      memory: false,
+      network
+    };
+
+    let walletDB, ldb;
+    before(async () => {
+      WalletMigrator.migrations = {};
+      await fs.mkdirp(location);
+
+      walletDB = new WalletDB(walletOptions);
+      ldb = walletDB.db;
+
+      await walletDB.open();
+      await fillEntries(walletDB.db, data.before);
+      await walletDB.close();
+    });
+
+    after(async () => {
+      WalletMigrator.migrations = migrationsBAK;
+      await rimraf(location);
+    });
+
+    it('should migrate', async () => {
+      WalletMigrator.migrations = {
+        0: Migration
+      };
+
+      walletDB.options.walletMigrate = 0;
+
+      await walletDB.open();
+      await checkEntries(ldb, {
+        before: data.before,
+        after: data.after,
+        throw: true
+      });
+      await walletDB.close();
+    });
+  });
 });