Skip to content

Commit 5d5b2fb

Browse files
committed
Merge #765: Fix how descriptor checksums are calculated
648282e Update docs and tests based on review comments (Steve Myers) 60057a7 Deprecate backward compatible get_checksum_bytes, get_checksum functions (Steve Myers) e2a4a58 Ensure backward compatibility of the "checksum inception" bug (志宇) fd34956 `get_checksum_bytes` now checks input data for checksum (志宇) Pull request description: ### Description Previously, the methods `get_checksum_bytes` and `get_checksum` do not check input data to see whether the input data already has a checksum. This PR does the following: * Introduce a `exclude_hash: bool` flag for `get_checksum_bytes`, that excludes the checksum portion of the original data when calculating the checksum. In addition to this, if the calculated checksum does not match the original checksum, an error is returned for extra safety. * Ensure `Wallet` is still backwards compatible with databases created with the "checksum inception" bug. ### Notes to the reviewers Thank you. ### Changelog notice Fix the "checksum inception" bug, where we may accidentally calculate the checksum of a descriptor that already has a checksum. ### Checklists #### All Submissions: * [x] I've signed all my commits * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md) * [x] I ran `cargo fmt` and `cargo clippy` before committing #### Bugfixes: * [x] This pull request breaks the existing API * [x] I've added tests to reproduce the issue which are now passing ~* [ ] I'm linking the issue being fixed by this PR~ Top commit has no ACKs. Tree-SHA512: 7ea2721dcd56459b6996e56a3ddfc3559a0c64869a08f5312a8f0f4fcb5dbef7ac7461a4ab017acde4a62fed02d8a620c402dd384323aba85736610514fcb7e1
2 parents 5720e38 + 9cb6f70 commit 5d5b2fb

File tree

5 files changed

+149
-45
lines changed

5 files changed

+149
-45
lines changed

src/blockchain/rpc.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ use crate::bitcoin::hashes::hex::ToHex;
3535
use crate::bitcoin::{Network, OutPoint, Transaction, TxOut, Txid};
3636
use crate::blockchain::*;
3737
use crate::database::{BatchDatabase, BatchOperations, DatabaseUtils};
38-
use crate::descriptor::get_checksum;
38+
use crate::descriptor::calc_checksum;
3939
use crate::error::MissingCachedScripts;
4040
use crate::{BlockTime, Error, FeeRate, KeychainKind, LocalUtxo, TransactionDetails};
4141
use bitcoin::Script;
@@ -806,7 +806,7 @@ fn is_wallet_descriptor(client: &Client) -> Result<bool, Error> {
806806

807807
fn descriptor_from_script_pubkey(script: &Script) -> String {
808808
let desc = format!("raw({})", script.to_hex());
809-
format!("{}#{}", desc, get_checksum(&desc).unwrap())
809+
format!("{}#{}", desc, calc_checksum(&desc).unwrap())
810810
}
811811

812812
/// Factory of [`RpcBlockchain`] instances, implements [`BlockchainFactory`]

src/descriptor/checksum.rs

Lines changed: 81 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,24 @@ fn poly_mod(mut c: u64, val: u64) -> u64 {
4141
c
4242
}
4343

44-
/// Computes the checksum bytes of a descriptor
45-
pub fn get_checksum_bytes(desc: &str) -> Result<[u8; 8], DescriptorError> {
44+
/// Computes the checksum bytes of a descriptor.
45+
/// `exclude_hash = true` ignores all data after the first '#' (inclusive).
46+
pub(crate) fn calc_checksum_bytes_internal(
47+
mut desc: &str,
48+
exclude_hash: bool,
49+
) -> Result<[u8; 8], DescriptorError> {
4650
let mut c = 1;
4751
let mut cls = 0;
4852
let mut clscount = 0;
4953

54+
let mut original_checksum = None;
55+
if exclude_hash {
56+
if let Some(split) = desc.split_once('#') {
57+
desc = split.0;
58+
original_checksum = Some(split.1);
59+
}
60+
}
61+
5062
for ch in desc.as_bytes() {
5163
let pos = INPUT_CHARSET
5264
.iter()
@@ -72,37 +84,96 @@ pub fn get_checksum_bytes(desc: &str) -> Result<[u8; 8], DescriptorError> {
7284
checksum[j] = CHECKSUM_CHARSET[((c >> (5 * (7 - j))) & 31) as usize];
7385
}
7486

87+
// if input data already had a checksum, check calculated checksum against original checksum
88+
if let Some(original_checksum) = original_checksum {
89+
if original_checksum.as_bytes() != checksum {
90+
return Err(DescriptorError::InvalidDescriptorChecksum);
91+
}
92+
}
93+
7594
Ok(checksum)
7695
}
7796

97+
/// Compute the checksum bytes of a descriptor, excludes any existing checksum in the descriptor string from the calculation
98+
pub fn calc_checksum_bytes(desc: &str) -> Result<[u8; 8], DescriptorError> {
99+
calc_checksum_bytes_internal(desc, true)
100+
}
101+
102+
/// Compute the checksum of a descriptor, excludes any existing checksum in the descriptor string from the calculation
103+
pub fn calc_checksum(desc: &str) -> Result<String, DescriptorError> {
104+
// unsafe is okay here as the checksum only uses bytes in `CHECKSUM_CHARSET`
105+
calc_checksum_bytes_internal(desc, true)
106+
.map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })
107+
}
108+
109+
// TODO in release 0.25.0, remove get_checksum_bytes and get_checksum
110+
// TODO in release 0.25.0, consolidate calc_checksum_bytes_internal into calc_checksum_bytes
111+
112+
/// Compute the checksum bytes of a descriptor
113+
#[deprecated(
114+
since = "0.24.0",
115+
note = "Use new `calc_checksum_bytes` function which excludes any existing checksum in the descriptor string before calculating the checksum hash bytes. See https://github.com/bitcoindevkit/bdk/pull/765."
116+
)]
117+
pub fn get_checksum_bytes(desc: &str) -> Result<[u8; 8], DescriptorError> {
118+
calc_checksum_bytes_internal(desc, false)
119+
}
120+
78121
/// Compute the checksum of a descriptor
122+
#[deprecated(
123+
since = "0.24.0",
124+
note = "Use new `calc_checksum` function which excludes any existing checksum in the descriptor string before calculating the checksum hash. See https://github.com/bitcoindevkit/bdk/pull/765."
125+
)]
79126
pub fn get_checksum(desc: &str) -> Result<String, DescriptorError> {
80127
// unsafe is okay here as the checksum only uses bytes in `CHECKSUM_CHARSET`
81-
get_checksum_bytes(desc).map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })
128+
calc_checksum_bytes_internal(desc, false)
129+
.map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })
82130
}
83131

84132
#[cfg(test)]
85133
mod test {
86134
use super::*;
87-
use crate::descriptor::get_checksum;
135+
use crate::descriptor::calc_checksum;
88136

89-
// test get_checksum() function; it should return the same value as Bitcoin Core
137+
// test calc_checksum() function; it should return the same value as Bitcoin Core
90138
#[test]
91-
fn test_get_checksum() {
139+
fn test_calc_checksum() {
92140
let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)";
93-
assert_eq!(get_checksum(desc).unwrap(), "tqz0nc62");
141+
assert_eq!(calc_checksum(desc).unwrap(), "tqz0nc62");
94142

95143
let desc = "pkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/44'/1'/0'/0/*)";
96-
assert_eq!(get_checksum(desc).unwrap(), "lasegmfs");
144+
assert_eq!(calc_checksum(desc).unwrap(), "lasegmfs");
145+
}
146+
147+
// test calc_checksum() function; it should return the same value as Bitcoin Core even if the
148+
// descriptor string includes a checksum hash
149+
#[test]
150+
fn test_calc_checksum_with_checksum_hash() {
151+
let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)#tqz0nc62";
152+
assert_eq!(calc_checksum(desc).unwrap(), "tqz0nc62");
153+
154+
let desc = "pkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/44'/1'/0'/0/*)#lasegmfs";
155+
assert_eq!(calc_checksum(desc).unwrap(), "lasegmfs");
156+
157+
let desc = "wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)#tqz0nc26";
158+
assert!(matches!(
159+
calc_checksum(desc).err(),
160+
Some(DescriptorError::InvalidDescriptorChecksum)
161+
));
162+
163+
let desc = "pkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/44'/1'/0'/0/*)#lasegmsf";
164+
assert!(matches!(
165+
calc_checksum(desc).err(),
166+
Some(DescriptorError::InvalidDescriptorChecksum)
167+
));
97168
}
98169

99170
#[test]
100-
fn test_get_checksum_invalid_character() {
171+
fn test_calc_checksum_invalid_character() {
101172
let sparkle_heart = unsafe { std::str::from_utf8_unchecked(&[240, 159, 146, 150]) };
102173
let invalid_desc = format!("wpkh(tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcL{}fjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/2/*)", sparkle_heart);
103174

104175
assert!(matches!(
105-
get_checksum(&invalid_desc).err(),
176+
calc_checksum(&invalid_desc).err(),
106177
Some(DescriptorError::InvalidDescriptorCharacter(invalid_char)) if invalid_char == sparkle_heart.as_bytes()[0]
107178
));
108179
}

src/descriptor/mod.rs

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ pub mod error;
3737
pub mod policy;
3838
pub mod template;
3939

40-
pub use self::checksum::get_checksum;
40+
pub use self::checksum::calc_checksum;
41+
use self::checksum::calc_checksum_bytes;
4142
pub use self::error::Error as DescriptorError;
4243
pub use self::policy::Policy;
4344
use self::template::DescriptorTemplateOut;
@@ -81,19 +82,15 @@ impl IntoWalletDescriptor for &str {
8182
secp: &SecpCtx,
8283
network: Network,
8384
) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> {
84-
let descriptor = if self.contains('#') {
85-
let parts: Vec<&str> = self.splitn(2, '#').collect();
86-
if !get_checksum(parts[0])
87-
.ok()
88-
.map(|computed| computed == parts[1])
89-
.unwrap_or(false)
90-
{
91-
return Err(DescriptorError::InvalidDescriptorChecksum);
85+
let descriptor = match self.split_once('#') {
86+
Some((desc, original_checksum)) => {
87+
let checksum = calc_checksum_bytes(desc)?;
88+
if original_checksum.as_bytes() != checksum {
89+
return Err(DescriptorError::InvalidDescriptorChecksum);
90+
}
91+
desc
9292
}
93-
94-
parts[0]
95-
} else {
96-
self
93+
None => self,
9794
};
9895

9996
ExtendedDescriptor::parse_descriptor(secp, descriptor)?

src/descriptor/policy.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ use crate::keys::ExtScriptContext;
6363
use crate::wallet::signer::{SignerId, SignersContainer};
6464
use crate::wallet::utils::{After, Older, SecpCtx};
6565

66-
use super::checksum::get_checksum;
66+
use super::checksum::calc_checksum;
6767
use super::error::Error;
6868
use super::XKeyUtils;
6969
use bitcoin::util::psbt::{Input as PsbtInput, PartiallySignedTransaction as Psbt};
@@ -168,7 +168,7 @@ impl SatisfiableItem {
168168

169169
/// Returns a unique id for the [`SatisfiableItem`]
170170
pub fn id(&self) -> String {
171-
get_checksum(&serde_json::to_string(self).expect("Failed to serialize a SatisfiableItem"))
171+
calc_checksum(&serde_json::to_string(self).expect("Failed to serialize a SatisfiableItem"))
172172
.expect("Failed to compute a SatisfiableItem id")
173173
}
174174
}

src/wallet/mod.rs

Lines changed: 54 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,10 @@ use utils::{check_nsequence_rbf, After, Older, SecpCtx};
5959
use crate::blockchain::{GetHeight, NoopProgress, Progress, WalletSync};
6060
use crate::database::memory::MemoryDatabase;
6161
use crate::database::{AnyDatabase, BatchDatabase, BatchOperations, DatabaseUtils, SyncTime};
62+
use crate::descriptor::checksum::calc_checksum_bytes_internal;
6263
use crate::descriptor::policy::BuildSatisfaction;
6364
use crate::descriptor::{
64-
get_checksum, into_wallet_descriptor_checked, DerivedDescriptor, DescriptorMeta,
65+
calc_checksum, into_wallet_descriptor_checked, DerivedDescriptor, DescriptorMeta,
6566
ExtendedDescriptor, ExtractPolicy, IntoWalletDescriptor, Policy, XKeyUtils,
6667
};
6768
use crate::error::{Error, MiniscriptPsbtError};
@@ -193,28 +194,27 @@ where
193194
let secp = Secp256k1::new();
194195

195196
let (descriptor, keymap) = into_wallet_descriptor_checked(descriptor, &secp, network)?;
196-
database.check_descriptor_checksum(
197+
Self::db_checksum(
198+
&mut database,
199+
&descriptor.to_string(),
197200
KeychainKind::External,
198-
get_checksum(&descriptor.to_string())?.as_bytes(),
199201
)?;
200202
let signers = Arc::new(SignersContainer::build(keymap, &descriptor, &secp));
201203
let (change_descriptor, change_signers) = match change_descriptor {
202204
Some(desc) => {
203205
let (change_descriptor, change_keymap) =
204206
into_wallet_descriptor_checked(desc, &secp, network)?;
205-
database.check_descriptor_checksum(
207+
Self::db_checksum(
208+
&mut database,
209+
&change_descriptor.to_string(),
206210
KeychainKind::Internal,
207-
get_checksum(&change_descriptor.to_string())?.as_bytes(),
208211
)?;
209212

210213
let change_signers = Arc::new(SignersContainer::build(
211214
change_keymap,
212215
&change_descriptor,
213216
&secp,
214217
));
215-
// if !parsed.same_structure(descriptor.as_ref()) {
216-
// return Err(Error::DifferentDescriptorStructure);
217-
// }
218218

219219
(Some(change_descriptor), change_signers)
220220
}
@@ -232,6 +232,19 @@ where
232232
})
233233
}
234234

235+
/// This checks the checksum within [`BatchDatabase`] twice (if needed). The first time with the
236+
/// actual checksum, and the second time with the checksum of `descriptor+checksum`. The second
237+
/// check is necessary for backwards compatibility of a checksum-inception bug.
238+
fn db_checksum(db: &mut D, desc: &str, kind: KeychainKind) -> Result<(), Error> {
239+
let checksum = calc_checksum_bytes_internal(desc, true)?;
240+
if db.check_descriptor_checksum(kind, checksum).is_ok() {
241+
return Ok(());
242+
}
243+
244+
let checksum_inception = calc_checksum_bytes_internal(desc, false)?;
245+
db.check_descriptor_checksum(kind, checksum_inception)
246+
}
247+
235248
/// Get the Bitcoin network the wallet is using.
236249
pub fn network(&self) -> Network {
237250
self.network
@@ -1781,14 +1794,14 @@ where
17811794
.into_wallet_descriptor(secp, network)?
17821795
.0
17831796
.to_string();
1784-
let mut wallet_name = get_checksum(&descriptor[..descriptor.find('#').unwrap()])?;
1797+
let mut wallet_name = calc_checksum(&descriptor[..descriptor.find('#').unwrap()])?;
17851798
if let Some(change_descriptor) = change_descriptor {
17861799
let change_descriptor = change_descriptor
17871800
.into_wallet_descriptor(secp, network)?
17881801
.0
17891802
.to_string();
17901803
wallet_name.push_str(
1791-
get_checksum(&change_descriptor[..change_descriptor.find('#').unwrap()])?.as_str(),
1804+
calc_checksum(&change_descriptor[..change_descriptor.find('#').unwrap()])?.as_str(),
17921805
);
17931806
}
17941807

@@ -1860,15 +1873,38 @@ pub(crate) mod test {
18601873
let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
18611874
let checksum = wallet.descriptor_checksum(KeychainKind::External);
18621875
assert_eq!(checksum.len(), 8);
1876+
assert_eq!(
1877+
calc_checksum(&wallet.descriptor.to_string()).unwrap(),
1878+
checksum
1879+
);
1880+
}
18631881

1864-
let raw_descriptor = wallet
1865-
.descriptor
1866-
.to_string()
1867-
.split_once('#')
1868-
.unwrap()
1869-
.0
1870-
.to_string();
1871-
assert_eq!(get_checksum(&raw_descriptor).unwrap(), checksum);
1882+
#[test]
1883+
fn test_db_checksum() {
1884+
let (wallet, _, _) = get_funded_wallet(get_test_wpkh());
1885+
let desc = wallet.descriptor.to_string();
1886+
1887+
let checksum = calc_checksum_bytes_internal(&desc, true).unwrap();
1888+
let checksum_inception = calc_checksum_bytes_internal(&desc, false).unwrap();
1889+
let checksum_invalid = [b'q'; 8];
1890+
1891+
let mut db = MemoryDatabase::new();
1892+
db.check_descriptor_checksum(KeychainKind::External, checksum)
1893+
.expect("failed to save actual checksum");
1894+
Wallet::db_checksum(&mut db, &desc, KeychainKind::External)
1895+
.expect("db that uses actual checksum should be supported");
1896+
1897+
let mut db = MemoryDatabase::new();
1898+
db.check_descriptor_checksum(KeychainKind::External, checksum_inception)
1899+
.expect("failed to save checksum inception");
1900+
Wallet::db_checksum(&mut db, &desc, KeychainKind::External)
1901+
.expect("db that uses checksum inception should be supported");
1902+
1903+
let mut db = MemoryDatabase::new();
1904+
db.check_descriptor_checksum(KeychainKind::External, checksum_invalid)
1905+
.expect("failed to save invalid checksum");
1906+
Wallet::db_checksum(&mut db, &desc, KeychainKind::External)
1907+
.expect_err("db that uses invalid checksum should fail");
18721908
}
18731909

18741910
#[test]

0 commit comments

Comments
 (0)