Skip to content

Commit d724bb5

Browse files
committed
Merge bitcoin/bitcoin#28609: wallet: Reload watchonly and solvables wallets after migration
4814e40 test: Check tx metadata is migrated to watchonly (Andrew Chow) d616d30 wallet: Reload watchonly and solvables wallets after migration (Andrew Chow) 118f2d7 wallet: Copy all tx metadata to watchonly wallet (Andrew Chow) 9af87cf test: Check that a failed wallet migration is cleaned up (Andrew Chow) Pull request description: Some incomplete/incorrect state as a result of migration can be mitigated/cleaned up by simply restarting the migrated wallets. We already do this for a wallet when it is migrated, but we do not for the new watchonly and solvables wallets that may be created. This PR introduces this behavior, in addition to creating those wallets initially without an attached chain. While implementing this, I noticed that not all `CWalletTx` metadata was being copied over to the watchonly wallet and so some data, such as time received, was being lost. This PR fixes this as a side effect of not having a chain attached to the watchonly wallet. A test has also been added. ACKs for top commit: ishaanam: light code review ACK 4814e40 ryanofsky: Code review ACK 4814e40. Just implemented the suggested orderpos, copyfrom, and path set comments since last review furszy: ACK 4814e40 Tree-SHA512: 0b992430df9f452cb252c2212df8e876613f43564fcd1dc00c6c31fa497adb84dfff6b5ef597590f9b288c5f64cb455f108fcc9b6c9d1fe9eb2c39e7f2c12a89
2 parents da8e397 + 4814e40 commit d724bb5

File tree

4 files changed

+146
-30
lines changed

4 files changed

+146
-30
lines changed

src/wallet/transaction.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,9 @@ int64_t CWalletTx::GetTxTime() const
2424
int64_t n = nTimeSmart;
2525
return n ? n : nTimeReceived;
2626
}
27+
28+
void CWalletTx::CopyFrom(const CWalletTx& _tx)
29+
{
30+
*this = _tx;
31+
}
2732
} // namespace wallet

src/wallet/transaction.h

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -334,11 +334,15 @@ class CWalletTx
334334
const uint256& GetWitnessHash() const { return tx->GetWitnessHash(); }
335335
bool IsCoinBase() const { return tx->IsCoinBase(); }
336336

337+
private:
337338
// Disable copying of CWalletTx objects to prevent bugs where instances get
338339
// copied in and out of the mapWallet map, and fields are updated in the
339340
// wrong copy.
340-
CWalletTx(CWalletTx const &) = delete;
341-
void operator=(CWalletTx const &x) = delete;
341+
CWalletTx(const CWalletTx&) = default;
342+
CWalletTx& operator=(const CWalletTx&) = default;
343+
public:
344+
// Instead have an explicit copy function
345+
void CopyFrom(const CWalletTx&);
342346
};
343347

344348
struct WalletTxOrderComparator {

src/wallet/wallet.cpp

Lines changed: 81 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3908,6 +3908,14 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error)
39083908
// Check if the transactions in the wallet are still ours. Either they belong here, or they belong in the watchonly wallet.
39093909
// We need to go through these in the tx insertion order so that lookups to spends works.
39103910
std::vector<uint256> txids_to_delete;
3911+
std::unique_ptr<WalletBatch> watchonly_batch;
3912+
if (data.watchonly_wallet) {
3913+
watchonly_batch = std::make_unique<WalletBatch>(data.watchonly_wallet->GetDatabase());
3914+
// Copy the next tx order pos to the watchonly wallet
3915+
LOCK(data.watchonly_wallet->cs_wallet);
3916+
data.watchonly_wallet->nOrderPosNext = nOrderPosNext;
3917+
watchonly_batch->WriteOrderPosNext(data.watchonly_wallet->nOrderPosNext);
3918+
}
39113919
for (const auto& [_pos, wtx] : wtxOrdered) {
39123920
if (!IsMine(*wtx->tx) && !IsFromMe(*wtx->tx)) {
39133921
// Check it is the watchonly wallet's
@@ -3916,12 +3924,20 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error)
39163924
LOCK(data.watchonly_wallet->cs_wallet);
39173925
if (data.watchonly_wallet->IsMine(*wtx->tx) || data.watchonly_wallet->IsFromMe(*wtx->tx)) {
39183926
// Add to watchonly wallet
3919-
if (!data.watchonly_wallet->AddToWallet(wtx->tx, wtx->m_state)) {
3920-
error = _("Error: Could not add watchonly tx to watchonly wallet");
3927+
const uint256& hash = wtx->GetHash();
3928+
const CWalletTx& to_copy_wtx = *wtx;
3929+
if (!data.watchonly_wallet->LoadToWallet(hash, [&](CWalletTx& ins_wtx, bool new_tx) EXCLUSIVE_LOCKS_REQUIRED(data.watchonly_wallet->cs_wallet) {
3930+
if (!new_tx) return false;
3931+
ins_wtx.SetTx(to_copy_wtx.tx);
3932+
ins_wtx.CopyFrom(to_copy_wtx);
3933+
return true;
3934+
})) {
3935+
error = strprintf(_("Error: Could not add watchonly tx %s to watchonly wallet"), wtx->GetHash().GetHex());
39213936
return false;
39223937
}
3938+
watchonly_batch->WriteTx(data.watchonly_wallet->mapWallet.at(hash));
39233939
// Mark as to remove from this wallet
3924-
txids_to_delete.push_back(wtx->GetHash());
3940+
txids_to_delete.push_back(hash);
39253941
continue;
39263942
}
39273943
}
@@ -3930,6 +3946,7 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error)
39303946
return false;
39313947
}
39323948
}
3949+
watchonly_batch.reset(); // Flush
39333950
// Do the removes
39343951
if (txids_to_delete.size() > 0) {
39353952
std::vector<uint256> deleted_txids;
@@ -4066,6 +4083,10 @@ bool DoMigration(CWallet& wallet, WalletContext& context, bilingual_str& error,
40664083
DatabaseOptions options;
40674084
options.require_existing = false;
40684085
options.require_create = true;
4086+
options.require_format = DatabaseFormat::SQLITE;
4087+
4088+
WalletContext empty_context;
4089+
empty_context.args = context.args;
40694090

40704091
// Make the wallets
40714092
options.create_flags = WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET | WALLET_FLAG_DESCRIPTORS;
@@ -4081,8 +4102,14 @@ bool DoMigration(CWallet& wallet, WalletContext& context, bilingual_str& error,
40814102
DatabaseStatus status;
40824103
std::vector<bilingual_str> warnings;
40834104
std::string wallet_name = wallet.GetName() + "_watchonly";
4084-
data->watchonly_wallet = CreateWallet(context, wallet_name, std::nullopt, options, status, error, warnings);
4085-
if (status != DatabaseStatus::SUCCESS) {
4105+
std::unique_ptr<WalletDatabase> database = MakeWalletDatabase(wallet_name, options, status, error);
4106+
if (!database) {
4107+
error = strprintf(_("Wallet file creation failed: %s"), error);
4108+
return false;
4109+
}
4110+
4111+
data->watchonly_wallet = CWallet::Create(empty_context, wallet_name, std::move(database), options.create_flags, error, warnings);
4112+
if (!data->watchonly_wallet) {
40864113
error = _("Error: Failed to create new watchonly wallet");
40874114
return false;
40884115
}
@@ -4112,8 +4139,14 @@ bool DoMigration(CWallet& wallet, WalletContext& context, bilingual_str& error,
41124139
DatabaseStatus status;
41134140
std::vector<bilingual_str> warnings;
41144141
std::string wallet_name = wallet.GetName() + "_solvables";
4115-
data->solvable_wallet = CreateWallet(context, wallet_name, std::nullopt, options, status, error, warnings);
4116-
if (status != DatabaseStatus::SUCCESS) {
4142+
std::unique_ptr<WalletDatabase> database = MakeWalletDatabase(wallet_name, options, status, error);
4143+
if (!database) {
4144+
error = strprintf(_("Wallet file creation failed: %s"), error);
4145+
return false;
4146+
}
4147+
4148+
data->solvable_wallet = CWallet::Create(empty_context, wallet_name, std::move(database), options.create_flags, error, warnings);
4149+
if (!data->solvable_wallet) {
41174150
error = _("Error: Failed to create new watchonly wallet");
41184151
return false;
41194152
}
@@ -4216,47 +4249,69 @@ util::Result<MigrationResult> MigrateLegacyToDescriptor(const std::string& walle
42164249
success = DoMigration(*local_wallet, context, error, res);
42174250
}
42184251

4252+
// In case of reloading failure, we need to remember the wallet dirs to remove
4253+
// Set is used as it may be populated with the same wallet directory paths multiple times,
4254+
// both before and after reloading. This ensures the set is complete even if one of the wallets
4255+
// fails to reload.
4256+
std::set<fs::path> wallet_dirs;
42194257
if (success) {
4220-
// Migration successful, unload the wallet locally, then reload it.
4221-
assert(local_wallet.use_count() == 1);
4222-
local_wallet.reset();
4223-
res.wallet = LoadWallet(context, wallet_name, /*load_on_start=*/std::nullopt, options, status, error, warnings);
4258+
// Migration successful, unload all wallets locally, then reload them.
4259+
const auto& reload_wallet = [&](std::shared_ptr<CWallet>& to_reload) {
4260+
assert(to_reload.use_count() == 1);
4261+
std::string name = to_reload->GetName();
4262+
wallet_dirs.insert(fs::PathFromString(to_reload->GetDatabase().Filename()).parent_path());
4263+
to_reload.reset();
4264+
to_reload = LoadWallet(context, name, /*load_on_start=*/std::nullopt, options, status, error, warnings);
4265+
return to_reload != nullptr;
4266+
};
4267+
// Reload the main wallet
4268+
success = reload_wallet(local_wallet);
4269+
res.wallet = local_wallet;
42244270
res.wallet_name = wallet_name;
4225-
} else {
4271+
if (success && res.watchonly_wallet) {
4272+
// Reload watchonly
4273+
success = reload_wallet(res.watchonly_wallet);
4274+
}
4275+
if (success && res.solvables_wallet) {
4276+
// Reload solvables
4277+
success = reload_wallet(res.solvables_wallet);
4278+
}
4279+
}
4280+
if (!success) {
42264281
// Migration failed, cleanup
42274282
// Copy the backup to the actual wallet dir
42284283
fs::path temp_backup_location = fsbridge::AbsPathJoin(GetWalletDir(), backup_filename);
42294284
fs::copy_file(backup_path, temp_backup_location, fs::copy_options::none);
42304285

4231-
// Remember this wallet's walletdir to remove after unloading
4232-
std::vector<fs::path> wallet_dirs;
4233-
wallet_dirs.emplace_back(fs::PathFromString(local_wallet->GetDatabase().Filename()).parent_path());
4234-
4235-
// Unload the wallet locally
4236-
assert(local_wallet.use_count() == 1);
4237-
local_wallet.reset();
4238-
42394286
// Make list of wallets to cleanup
42404287
std::vector<std::shared_ptr<CWallet>> created_wallets;
4288+
if (local_wallet) created_wallets.push_back(std::move(local_wallet));
42414289
if (res.watchonly_wallet) created_wallets.push_back(std::move(res.watchonly_wallet));
42424290
if (res.solvables_wallet) created_wallets.push_back(std::move(res.solvables_wallet));
42434291

42444292
// Get the directories to remove after unloading
42454293
for (std::shared_ptr<CWallet>& w : created_wallets) {
4246-
wallet_dirs.emplace_back(fs::PathFromString(w->GetDatabase().Filename()).parent_path());
4294+
wallet_dirs.emplace(fs::PathFromString(w->GetDatabase().Filename()).parent_path());
42474295
}
42484296

42494297
// Unload the wallets
42504298
for (std::shared_ptr<CWallet>& w : created_wallets) {
4251-
if (!RemoveWallet(context, w, /*load_on_start=*/false)) {
4252-
error += _("\nUnable to cleanup failed migration");
4253-
return util::Error{error};
4299+
if (w->HaveChain()) {
4300+
// Unloading for wallets that were loaded for normal use
4301+
if (!RemoveWallet(context, w, /*load_on_start=*/false)) {
4302+
error += _("\nUnable to cleanup failed migration");
4303+
return util::Error{error};
4304+
}
4305+
UnloadWallet(std::move(w));
4306+
} else {
4307+
// Unloading for wallets in local context
4308+
assert(w.use_count() == 1);
4309+
w.reset();
42544310
}
4255-
UnloadWallet(std::move(w));
42564311
}
42574312

42584313
// Delete the wallet directories
4259-
for (fs::path& dir : wallet_dirs) {
4314+
for (const fs::path& dir : wallet_dirs) {
42604315
fs::remove_all(dir);
42614316
}
42624317

test/functional/wallet_migration.py

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,15 @@
66

77
import random
88
import shutil
9+
import struct
10+
import time
11+
912
from test_framework.address import (
1013
script_to_p2sh,
1114
key_to_p2pkh,
1215
key_to_p2wpkh,
1316
)
17+
from test_framework.bdb import BTREE_MAGIC
1418
from test_framework.descriptors import descsum_create
1519
from test_framework.key import ECPubKey
1620
from test_framework.test_framework import BitcoinTestFramework
@@ -20,6 +24,7 @@
2024
assert_equal,
2125
assert_raises_rpc_error,
2226
find_vout_for_address,
27+
sha256sum_file,
2328
)
2429
from test_framework.wallet_util import (
2530
get_generate_key,
@@ -311,12 +316,17 @@ def test_other_watchonly(self):
311316
sent_watchonly_txid = send["txid"]
312317

313318
self.generate(self.nodes[0], 1)
319+
received_watchonly_tx_info = imports0.gettransaction(received_watchonly_txid, True)
320+
received_sent_watchonly_tx_info = imports0.gettransaction(received_sent_watchonly_txid, True)
314321

315322
balances = imports0.getbalances()
316323
spendable_bal = balances["mine"]["trusted"]
317324
watchonly_bal = balances["watchonly"]["trusted"]
318325
assert_equal(len(imports0.listtransactions(include_watchonly=True)), 4)
319326

327+
# Mock time forward a bit so we can check that tx metadata is preserved
328+
self.nodes[0].setmocktime(int(time.time()) + 100)
329+
320330
# Migrate
321331
imports0.migratewallet()
322332
assert_equal(imports0.getwalletinfo()["descriptors"], True)
@@ -334,8 +344,12 @@ def test_other_watchonly(self):
334344
assert_equal(watchonly_info["descriptors"], True)
335345
self.assert_is_sqlite("imports0_watchonly")
336346
assert_equal(watchonly_info["private_keys_enabled"], False)
337-
watchonly.gettransaction(received_watchonly_txid)
338-
watchonly.gettransaction(received_sent_watchonly_txid)
347+
received_migrated_watchonly_tx_info = watchonly.gettransaction(received_watchonly_txid)
348+
assert_equal(received_watchonly_tx_info["time"], received_migrated_watchonly_tx_info["time"])
349+
assert_equal(received_watchonly_tx_info["timereceived"], received_migrated_watchonly_tx_info["timereceived"])
350+
received_sent_migrated_watchonly_tx_info = watchonly.gettransaction(received_sent_watchonly_txid)
351+
assert_equal(received_sent_watchonly_tx_info["time"], received_sent_migrated_watchonly_tx_info["time"])
352+
assert_equal(received_sent_watchonly_tx_info["timereceived"], received_sent_migrated_watchonly_tx_info["timereceived"])
339353
watchonly.gettransaction(sent_watchonly_txid)
340354
assert_equal(watchonly.getbalance(), watchonly_bal)
341355
assert_raises_rpc_error(-5, "Invalid or non-wallet transaction id", watchonly.gettransaction, received_txid)
@@ -827,6 +841,43 @@ def test_hybrid_pubkey(self):
827841

828842
wallet.unloadwallet()
829843

844+
def test_failed_migration_cleanup(self):
845+
self.log.info("Test that a failed migration is cleaned up")
846+
wallet = self.create_legacy_wallet("failed")
847+
848+
# Make a copy of the wallet with the solvables wallet name so that we are unable
849+
# to create the solvables wallet when migrating, thus failing to migrate
850+
wallet.unloadwallet()
851+
solvables_path = self.nodes[0].wallets_path / "failed_solvables"
852+
shutil.copytree(self.nodes[0].wallets_path / "failed", solvables_path)
853+
original_shasum = sha256sum_file(solvables_path / "wallet.dat")
854+
855+
self.nodes[0].loadwallet("failed")
856+
857+
# Add a multisig so that a solvables wallet is created
858+
wallet.addmultisigaddress(2, [wallet.getnewaddress(), get_generate_key().pubkey])
859+
wallet.importaddress(get_generate_key().p2pkh_addr)
860+
861+
assert_raises_rpc_error(-4, "Failed to create database", wallet.migratewallet)
862+
863+
assert "failed" in self.nodes[0].listwallets()
864+
assert "failed_watchonly" not in self.nodes[0].listwallets()
865+
assert "failed_solvables" not in self.nodes[0].listwallets()
866+
867+
assert not (self.nodes[0].wallets_path / "failed_watchonly").exists()
868+
# Since the file in failed_solvables is one that we put there, migration shouldn't touch it
869+
assert solvables_path.exists()
870+
new_shasum = sha256sum_file(solvables_path / "wallet.dat")
871+
assert_equal(original_shasum, new_shasum)
872+
873+
wallet.unloadwallet()
874+
# Check the wallet we tried to migrate is still BDB
875+
with open(self.nodes[0].wallets_path / "failed" / "wallet.dat", "rb") as f:
876+
data = f.read(16)
877+
_, _, magic = struct.unpack("QII", data)
878+
assert_equal(magic, BTREE_MAGIC)
879+
880+
830881
def run_test(self):
831882
self.generate(self.nodes[0], 101)
832883

@@ -845,6 +896,7 @@ def run_test(self):
845896
self.test_migrate_raw_p2sh()
846897
self.test_conflict_txs()
847898
self.test_hybrid_pubkey()
899+
self.test_failed_migration_cleanup()
848900

849901
if __name__ == '__main__':
850902
WalletMigrationTest().main()

0 commit comments

Comments
 (0)