wallet: separate open and connect. Add wallet-migrate-no-rescan option.

This commit is contained in:
Nodari Chkuaselidze 2023-10-16 09:46:10 +04:00
parent f11615d823
commit b53d392a83
No known key found for this signature in database
GPG key ID: B018A7BB437D1F05
9 changed files with 147 additions and 91 deletions

View file

@ -6,9 +6,21 @@
you run it for the first time.**
### Wallet Changes:
#### Configuration
Wallet now has option `wallet-migrate-no-rescan`/`migrate-no-rescan` if you
want to disable rescan when migration recommends it. It may result in the
incorrect txdb state, but can be useful if you know the issue does not affect
your wallet or is not critical.
#### Wallet API
- Add migration that recalculates txdb balances to fix any inconsistencies.
- WalletDB Now emits events for: `open`, `close`, `connect`, `disconnect`.
- WalletDB
- `open()` no longer calls `connect` and needs separate call `connect`.
- `open()` no longer calls scan, instead only rollbacks and waits for
sync to do the rescan.
- emits events for: `open`, `close`, `connect`, `disconnect`, `sync done`.
- HTTP Changes:
- All transaction creating endpoints now accept `hardFee` for specifying the
exact fee.

View file

@ -52,6 +52,7 @@ class WalletNode extends Node {
wipeNoReally: this.config.bool('wipe-no-really'),
spv: this.config.bool('spv'),
walletMigrate: this.config.uint('migrate'),
migrateNoRescan: this.config.bool('migrate-no-rescan', false),
icannlockup: this.config.bool('icannlockup', false)
});
@ -107,6 +108,7 @@ class WalletNode extends Node {
await this.openPlugins();
await this.http.open();
await this.wdb.connect();
await this.handleOpen();
this.logger.info('Wallet node is loaded.');
@ -128,6 +130,7 @@ class WalletNode extends Node {
this.rpc.wallet = null;
await this.wdb.disconnect();
await this.wdb.close();
await this.handleClose();
}

View file

@ -57,6 +57,7 @@ class Plugin extends EventEmitter {
wipeNoReally: this.config.bool('wipe-no-really'),
spv: node.spv,
walletMigrate: this.config.uint('migrate'),
migrateNoRescan: this.config.bool('migrate-no-rescan', false),
icannlockup: this.config.bool('icannlockup', false)
});
@ -90,11 +91,13 @@ class Plugin extends EventEmitter {
await this.wdb.open();
this.rpc.wallet = this.wdb.primary;
await this.http.open();
await this.wdb.connect();
}
async close() {
await this.http.close();
this.rpc.wallet = null;
await this.wdb.disconnect();
await this.wdb.close();
}
}

View file

@ -64,12 +64,18 @@ class WalletDB extends EventEmitter {
this.name = 'wallet';
this.version = 2;
this.primary = null;
// chain state.
this.hasStateCache = false;
this.state = new ChainState();
this.confirming = false;
this.height = 0;
// wallets
this.primary = null;
this.wallets = new Map();
this.depth = 0;
// guards
this.confirming = false;
this.rescanning = false;
this.filterSent = false;
@ -203,7 +209,7 @@ class WalletDB extends EventEmitter {
await this.wipe();
await this.watch();
await this.connect();
await this.loadState();
this.logger.info(
'WalletDB loaded (depth=%d, height=%d, start=%d).',
@ -224,8 +230,13 @@ class WalletDB extends EventEmitter {
this.primary = wallet;
if (migrationResult.rescan) {
this.logger.info('Rescanning...');
await this.scan(0);
if (!this.options.migrateNoRescan) {
this.logger.info('Migration rollback...');
await this.rollback(0);
} else {
this.logger.warning(
'Migration rescan skipped, state may be incorrect.');
}
}
this.logger.info('WalletDB opened.');
@ -275,7 +286,8 @@ class WalletDB extends EventEmitter {
*/
async close() {
await this.disconnect();
if (this.client.opened)
await this.disconnect();
for (const wallet of this.wallets.values()) {
await wallet.destroy();
@ -375,7 +387,7 @@ class WalletDB extends EventEmitter {
const unlock = await this.txLock.lock();
try {
this.logger.info('Resyncing from server...');
await this.syncState();
await this.syncInitState();
await this.syncFilter();
await this.syncChain();
await this.resend();
@ -384,19 +396,32 @@ class WalletDB extends EventEmitter {
}
}
/**
* Recover state from the cache.
* @returns {Promise}
*/
async loadState() {
const cache = await this.getState();
if (!cache)
return;
this.logger.info('Initialized chain state from the database.');
this.hasStateCache = true;
this.state = cache;
this.height = cache.height;
}
/**
* Initialize and write initial sync state.
* @returns {Promise}
*/
async syncState() {
const cache = await this.getState();
if (cache) {
this.state = cache;
this.height = cache.height;
return undefined;
}
async syncInitState() {
// We have recovered from the cache.
if (this.hasStateCache)
return;
this.logger.info('Initializing database state from server.');
@ -427,7 +452,7 @@ class WalletDB extends EventEmitter {
this.state = state;
this.height = state.height;
return undefined;
return;
}
/**
@ -2192,6 +2217,7 @@ class WalletDB extends EventEmitter {
await iter.each(async (key, value) => {
const [height] = layout.b.decode(key);
const block = MapRecord.decode(value);
this.logger.info('Reverting block: %d', height);
for (const wid of block.wids) {
const wallet = await this.get(wid);
@ -2501,6 +2527,7 @@ class WalletOptions {
this.spv = false;
this.wipeNoReally = false;
this.walletMigrate = -1;
this.migrateNoRescan = false;
this.icannlockup = false;
if (options)
@ -2589,6 +2616,11 @@ class WalletOptions {
this.icannlockup = options.icannlockup;
}
if (options.migrateNoRescan != null) {
assert(typeof options.migrateNoRescan === 'boolean');
this.migrateNoRescan = options.migrateNoRescan;
}
return this;
}

View file

@ -8,6 +8,7 @@ const plugin = require('../lib/wallet/plugin');
const Coin = require('../lib/primitives/coin');
const Address = require('../lib/primitives/address');
const MTX = require('../lib/primitives/mtx');
const {forEvent} = require('./util/common');
const network = Network.get('regtest');
const {
@ -25,8 +26,13 @@ describe('Mempool Covenant Reorg', function () {
let wallet, name;
before(async () => {
const wdb = node.require('walletdb').wdb;
const syncDone = forEvent(wdb, 'sync done');
await node.open();
wallet = node.get('walletdb').wdb.primary;
await syncDone;
});
after(async () => {

View file

@ -64,6 +64,7 @@ describe('Wallet Auction', function() {
await chain.open();
await miner.open();
await wdb.open();
await wdb.connect();
// Set up wallet
winner = await wdb.create();
@ -81,6 +82,7 @@ describe('Wallet Auction', function() {
});
after(async () => {
await wdb.disconnect();
await wdb.close();
await miner.close();
await chain.close();

View file

@ -7,11 +7,13 @@ const Wallet = require('../lib/wallet/wallet');
const WalletKey = require('../lib/wallet/walletkey');
const ChainEntry = require('../lib/blockchain/chainentry');
const MemWallet = require('./util/memwallet');
const {forEvent} = require('./util/common');
describe('WalletDB Events', function () {
const node = new FullNode({
memory: true,
network: 'regtest',
noDNS: true,
plugins: [require('../lib/wallet/plugin')]
});
@ -35,15 +37,31 @@ describe('WalletDB Events', function () {
await node.close();
});
it('should emit `tx` events', async () => {
const waiter = new Promise((resolve) => {
wdb.once('tx', (w, tx) => resolve([w, tx]));
});
it('should emit `open`/`close` and `connect`/`disconnect` events', async () => {
const closeEvent = forEvent(wdb, 'close');
const disconnectEvent = forEvent(wdb, 'disconnect');
await node.close();
await disconnectEvent;
await closeEvent;
const openEvent = forEvent(wdb, 'open');
const connectEvent = forEvent(wdb, 'connect');
const syncDone = forEvent(wdb, 'sync done');
await node.open();
await openEvent;
await connectEvent;
await syncDone;
wallet = await wdb.get('primary');
});
it('should emit `tx` events', async () => {
const waiter = forEvent(wdb, 'tx');
const walletReceive = await wallet.receiveAddress();
await mineBlocks(1, walletReceive);
const [w, tx] = await waiter;
const events = await waiter;
const [w, tx] = events[0].values;
assert(w);
assert(w instanceof Wallet);
@ -55,14 +73,13 @@ describe('WalletDB Events', function () {
});
it('should emit `address` events', async () => {
const waiter = new Promise((resolve) => {
wdb.once('address', (w, walletKey) => resolve([w, walletKey]));
});
const waiter = forEvent(wdb, 'address');
const walletReceive = await wallet.receiveAddress();
await mineBlocks(1, walletReceive);
const [w, walletKey] = await waiter;
const events = await waiter;
const [w, walletKey] = events[0].values;
assert(w);
assert(w instanceof Wallet);
@ -75,10 +92,7 @@ describe('WalletDB Events', function () {
describe('should emit `block connect` events', () => {
it('with a block that includes a wallet tx', async () => {
const waiter = new Promise((resolve) => {
wdb.once('block connect', (entry, txs) => resolve([entry, txs]));
});
const waiter = forEvent(wdb, 'block connect');
const walletReceive = await wallet.receiveAddress();
await wallet.send({
@ -89,7 +103,8 @@ describe('WalletDB Events', function () {
await mineBlocks(1);
const [entry, txs] = await waiter;
const events = await waiter;
const [entry, txs] = events[0].values;
assert(entry);
assert(entry instanceof ChainEntry);
@ -127,15 +142,14 @@ describe('WalletDB Events', function () {
otherTx = otherMtx.toTX();
}
const waiter = new Promise((resolve) => {
wdb.once('block connect', (entry, txs) => resolve([entry, txs]));
});
const waiter = forEvent(wdb, 'block connect');
await node.sendTX(otherTx);
await mineBlocks(1);
const [entry, txs] = await waiter;
const events = await waiter;
const [entry, txs] = events[0].values;
assert(entry);
assert(entry instanceof ChainEntry);
@ -158,14 +172,13 @@ describe('WalletDB Events', function () {
await mineBlocks(1);
// Disconnect it
const waiter = new Promise((resolve) => {
wdb.once('block disconnect', entry => resolve(entry));
});
const waiter = forEvent(wdb, 'block disconnect');
const entryToDisconnect = node.chain.tip;
await node.chain.disconnect(entryToDisconnect);
const entry = await waiter;
const events = await waiter;
const entry = events[0].values[0];
assert(entry);
assert(entry instanceof ChainEntry);
@ -173,18 +186,16 @@ describe('WalletDB Events', function () {
});
it('with a block that does not include a wallet tx', async () => {
const waiter = new Promise((resolve) => {
wdb.once('block disconnect', entry => resolve(entry));
});
const waiter = forEvent(wdb, 'block disconnect');
const entryToDisconnect = node.chain.tip;
await node.chain.disconnect(entryToDisconnect);
const entryToDisconnect = node.chain.tip;
await node.chain.disconnect(entryToDisconnect);
const events = await waiter;
const entry = events[0].values[0];
const entry = await waiter;
assert(entry);
assert(entry instanceof ChainEntry);
assert(entry.hash = entryToDisconnect.hash);
assert(entry);
assert(entry instanceof ChainEntry);
assert(entry.hash = entryToDisconnect.hash);
});
});
});

View file

@ -19,7 +19,7 @@ const {
oldLayout
} = require('../lib/migrations/migrator');
const {migrationError} = require('./util/migrations');
const {forEvent, rimraf, testdir} = require('./util/common');
const {rimraf, testdir} = require('./util/common');
const NETWORK = 'regtest';
const network = Network.get(NETWORK);
@ -82,7 +82,7 @@ describe('Wallet Migrations', function() {
return ids;
};
let walletDB, ldb, wdbOpenSync;
let walletDB, ldb;
beforeEach(async () => {
await fs.mkdirp(location);
@ -91,9 +91,7 @@ describe('Wallet Migrations', function() {
WalletMigrator.migrations = mockMigrations;
wdbOpenSync = wdbOpenSyncFn(walletDB);
await wdbOpenSync();
await walletDB.open();
});
afterEach(async () => {
@ -181,7 +179,7 @@ describe('Wallet Migrations', function() {
walletDB.options.walletMigrate = lastMigrationID;
walletDB.version = 1;
await wdbOpenSync();
await walletDB.open();
const versionData = await ldb.get(layout.V.encode());
const version = getVersion(versionData, 'wallet');
@ -244,7 +242,7 @@ describe('Wallet Migrations', function() {
network
};
let walletDB, ldb, wdbOpenSync;
let walletDB, ldb;
beforeEach(async () => {
await fs.mkdirp(location);
@ -253,8 +251,6 @@ describe('Wallet Migrations', function() {
ldb = walletDB.db;
WalletMigrator.migrations = testMigrations;
wdbOpenSync = wdbOpenSyncFn(walletDB);
});
afterEach(async () => {
@ -268,8 +264,7 @@ describe('Wallet Migrations', function() {
});
it('should initialize fresh walletdb migration state', async () => {
await wdbOpenSync();
await walletDB.open();
const rawState = await ldb.get(layout.M.encode());
const state = MigrationState.decode(rawState);
@ -281,8 +276,7 @@ describe('Wallet Migrations', function() {
});
it('should not migrate pre-old migration state w/o flag', async () => {
await wdbOpenSync();
await walletDB.open();
const b = ldb.batch();
b.del(layout.M.encode());
await b.write();
@ -308,8 +302,7 @@ describe('Wallet Migrations', function() {
});
it('should migrate pre-old migration state with flag', async () => {
await wdbOpenSync();
await walletDB.open();
const b = ldb.batch();
b.del(layout.M.encode());
writeVersion(b, 'wallet', 0);
@ -317,7 +310,7 @@ describe('Wallet Migrations', function() {
await walletDB.close();
walletDB.options.walletMigrate = 1;
await wdbOpenSync();
await walletDB.open();
const versionData = await ldb.get(layout.V.encode());
const version = getVersion(versionData, 'wallet');
@ -334,7 +327,7 @@ describe('Wallet Migrations', function() {
});
it('should not migrate from last old migration state w/o flag', async () => {
await wdbOpenSync();
await walletDB.open();
const b = ldb.batch();
b.del(layout.M.encode());
@ -363,7 +356,7 @@ describe('Wallet Migrations', function() {
});
it('should not migrate from last old migration state with flag', async () => {
await wdbOpenSync();
await walletDB.open();
const b = ldb.batch();
b.del(layout.M.encode());
@ -372,7 +365,7 @@ describe('Wallet Migrations', function() {
await walletDB.close();
walletDB.options.walletMigrate = 1;
await wdbOpenSync();
await walletDB.open();
const rawState = await ldb.get(layout.M.encode());
const state = MigrationState.decode(rawState);
@ -398,7 +391,7 @@ describe('Wallet Migrations', function() {
}
};
await wdbOpenSync();
await walletDB.open();
const b = ldb.batch();
b.del(layout.M.encode());
@ -450,7 +443,7 @@ describe('Wallet Migrations', function() {
}
};
await wdbOpenSync();
await walletDB.open();
const b = ldb.batch();
b.del(layout.M.encode());
@ -460,7 +453,7 @@ describe('Wallet Migrations', function() {
await walletDB.close();
walletDB.options.walletMigrate = 2;
await wdbOpenSync();
await walletDB.open();
assert.strictEqual(migrated1, false);
assert.strictEqual(migrated2, true);
@ -492,7 +485,7 @@ describe('Wallet Migrations', function() {
const ADD_CHANGE_DEPTH = 10;
let walletDB, ldb, wdbOpenSync;
let walletDB, ldb;
const missingAddrs = [];
before(async () => {
WalletMigrator.migrations = {};
@ -507,7 +500,6 @@ describe('Wallet Migrations', function() {
beforeEach(async () => {
walletDB = new WalletDB(walletOptions);
ldb = walletDB.db;
wdbOpenSync = wdbOpenSyncFn(walletDB);
});
afterEach(async () => {
@ -516,7 +508,7 @@ describe('Wallet Migrations', function() {
});
it('should set incorrect walletdb state', async () => {
await wdbOpenSync();
await walletDB.open();
const wallet = walletDB.primary;
const account = await wallet.getAccount(0);
@ -535,7 +527,7 @@ describe('Wallet Migrations', function() {
});
it('should have missing addresses', async () => {
await wdbOpenSync();
await walletDB.open();
const wallet = walletDB.primary;
for (const addr of missingAddrs) {
@ -568,12 +560,12 @@ describe('Wallet Migrations', function() {
it('should migrate with migrate flag', async () => {
walletDB.options.walletMigrate = 0;
let rescan = false;
walletDB.scan = () => {
rescan = true;
let rollback = false;
walletDB.rollback = () => {
rollback = true;
};
await wdbOpenSync();
await walletDB.open();
const wallet = walletDB.primary;
for (const addr of missingAddrs) {
@ -581,7 +573,7 @@ describe('Wallet Migrations', function() {
assert.strictEqual(hasAddr, true);
}
assert.strictEqual(rescan, true);
assert.strictEqual(rollback, true);
await walletDB.close();
});
@ -598,7 +590,7 @@ describe('Wallet Migrations', function() {
network: network
};
let walletDB, ldb, wdbOpenSync;
let walletDB, ldb;
before(async () => {
WalletMigrator.migrations = {};
await fs.mkdirp(location);
@ -612,7 +604,6 @@ describe('Wallet Migrations', function() {
beforeEach(async () => {
walletDB = new WalletDB(walletOptions);
ldb = walletDB.db;
wdbOpenSync = wdbOpenSyncFn(walletDB);
});
afterEach(async () => {
@ -658,7 +649,7 @@ describe('Wallet Migrations', function() {
await b.write();
};
await wdbOpenSync();
await walletDB.open();
const wallet = walletDB.primary;
await wallet.createAccount({});
@ -703,7 +694,7 @@ describe('Wallet Migrations', function() {
walletDB.options.walletMigrate = 0;
await wdbOpenSync();
await walletDB.open();
const wallet = walletDB.primary;
const wallet2 = await walletDB.get(1);
await checkLookahead(wallet, TEST_LOOKAHEAD + 0);
@ -981,11 +972,3 @@ function getVersion(data, name) {
return data.readUInt32LE(name.length);
}
function wdbOpenSyncFn(wdb) {
return async () => {
const forSync = forEvent(wdb, 'sync done');
await wdb.open();
await forSync;
};
};

View file

@ -2158,6 +2158,7 @@ describe('Wallet', function() {
before(async () => {
await wdb.open();
await wdb.connect();
wallet = await wdb.create();
recip = await wdb.create();
// rollout all names
@ -2165,6 +2166,7 @@ describe('Wallet', function() {
});
after(async () => {
await wdb.disconnect();
await wdb.close();
});
@ -2962,6 +2964,7 @@ describe('Wallet', function() {
before(async () => {
await wdb.open();
await wdb.connect();
wallet = await wdb.create();
for (let i = 0; i < 3; i++) {
@ -2971,6 +2974,7 @@ describe('Wallet', function() {
});
after(async () => {
await wdb.disconnect();
await wdb.close();
});