From 534d723059ec648e15576d371f47cfe38bd4bb34 Mon Sep 17 00:00:00 2001 From: Kris Nuttycombe Date: Sun, 16 Feb 2025 16:59:57 -0700 Subject: [PATCH] zcash_client_sqlite: Add methods for fixing broken note commitment trees. --- Cargo.lock | 6 -- Cargo.toml | 3 + zcash_client_sqlite/CHANGELOG.md | 2 + zcash_client_sqlite/src/lib.rs | 45 ++++++++++ zcash_client_sqlite/src/wallet.rs | 37 +++++++- .../src/wallet/commitment_tree.rs | 76 ++++++++++++++++- zcash_client_sqlite/src/wallet/common.rs | 85 +++++++++++++++++++ zcash_client_sqlite/src/wallet/orchard.rs | 8 +- zcash_client_sqlite/src/wallet/sapling.rs | 8 +- 9 files changed, 257 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b46413b710..6ad6a29ff9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2158,8 +2158,6 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "incrementalmerkletree" version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30821f91f0fa8660edca547918dc59812893b497d07c1144f326f07fdd94aba9" dependencies = [ "either", "proptest", @@ -2170,8 +2168,6 @@ dependencies = [ [[package]] name = "incrementalmerkletree-testing" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad20fb6cf815e76ce9b9eca74f347740ab99059fe4b5e4a002403d0441a02983" dependencies = [ "incrementalmerkletree", "proptest", @@ -4042,8 +4038,6 @@ dependencies = [ [[package]] name = "shardtree" version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "637e95dcd06bc1bb3f86ed9db1e1832a70125f32daae071ef37dcb7701b7d4fe" dependencies = [ "assert_matches", "bitflags 2.6.0", diff --git a/Cargo.toml b/Cargo.toml index e4d5b363a5..b3b4ea05b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -198,3 +198,6 @@ unexpected_cfgs = { level = "warn", check-cfg = ['cfg(zcash_unstable, values("zf orchard = { git = "https://github.com/zcash/orchard.git", rev = "b1c22c07300db22239235d16dab096e23369948f" } redjubjub = { git = "https://github.com/ZcashFoundation/redjubjub", rev = "eae848c5c14d9c795d000dd9f4c4762d1aee7ee1" } sapling = { package = "sapling-crypto", git = "https://github.com/zcash/sapling-crypto.git", rev = "6ca338532912adcd82369220faeea31aab4720c5" } +incrementalmerkletree = { path = "../../incrementalmerkletree/canon/incrementalmerkletree/" } +incrementalmerkletree-testing = { path = "../../incrementalmerkletree/canon/incrementalmerkletree-testing/" } +shardtree = { path = "../../incrementalmerkletree/canon/shardtree/" } diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index cd6a453ca1..ff43c90544 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -9,6 +9,8 @@ and this library adheres to Rust's notion of ### Added - `zcash_client_sqlite::WalletDb::from_connection` +- `zcash_client_sqlite::WalletDb::check_witnesses` +- `zcash_client_sqlite::WalletDb::queue_rescans` ### Changed - MSRV is now 1.81.0. diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index 46d4c9f568..39f7527a1e 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -39,6 +39,7 @@ use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, ShardTree}; use std::{ borrow::{Borrow, BorrowMut}, + cmp::{max, min}, collections::HashMap, convert::AsRef, fmt, @@ -140,6 +141,7 @@ pub mod wallet; use wallet::{ commitment_tree::{self, put_shard_roots}, common::spendable_notes_meta, + scanning::replace_queue_entries, SubtreeProgressEstimator, }; @@ -302,6 +304,49 @@ impl, P: consensus::Parameters + Clone> WalletDb tx.commit()?; Ok(result) } + + /// Attempts to construct a witness for each note belonging to the wallet that is believed by + /// the wallet to currently be spendable, and returns a vector of . + /// + /// This method is intended for repairing wallets that broke due to bugs in `shardtree`. + /// + /// Returns a vector of the ranges that must be rescanned in order to correct missing witness + /// data. + pub fn check_witnesses(&mut self) -> Result>, SqliteClientError> { + self.transactionally(|wdb| wallet::commitment_tree::check_witnesses(&wdb.conn.0)) + } + + /// Updates the scan queue by inserting scan ranges for the given range of block heights, with + /// the specified scanning priority. + pub fn queue_rescans( + &mut self, + rescan_ranges: NonEmpty>, + priority: ScanPriority, + ) -> Result<(), SqliteClientError> { + let query_range = rescan_ranges + .iter() + .fold(None, |acc: Option>, scan_range| { + if let Some(range) = acc { + Some(min(range.start, scan_range.start)..max(range.end, scan_range.end)) + } else { + Some(scan_range.clone()) + } + }) + .expect("rescan_ranges is nonempty"); + + self.transactionally::<_, _, SqliteClientError>(|wdb| { + replace_queue_entries( + wdb.conn.0, + &query_range, + rescan_ranges + .into_iter() + .map(|r| ScanRange::from_parts(r, priority)), + true, + ) + })?; + + Ok(()) + } } impl, P: consensus::Parameters> InputSource for WalletDb { diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 2bd486c71a..b661b4f56d 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -80,7 +80,7 @@ use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; use std::io::{self, Cursor}; use std::num::NonZeroU32; -use std::ops::RangeInclusive; +use std::ops::{Range, RangeInclusive}; use tracing::{debug, warn}; @@ -3628,6 +3628,41 @@ pub(crate) fn prune_nullifier_map( Ok(()) } +pub(crate) fn get_block_range( + conn: &rusqlite::Connection, + protocol: ShieldedProtocol, + commitment_tree_address: incrementalmerkletree::Address, +) -> Result>, SqliteClientError> { + let prefix = match protocol { + ShieldedProtocol::Sapling => "sapling", + ShieldedProtocol::Orchard => "orchard", + }; + let mut stmt = conn.prepare_cached(&format!( + "SELECT MIN(height), MAX(height) + FROM blocks + WHERE {prefix}_commitment_tree_size BETWEEN :min_tree_size AND :max_tree_size" + ))?; + + stmt.query_row( + named_params! { + ":min_tree_size": u64::from(commitment_tree_address.position_range_start()) + 1, + ":max_tree_size": u64::from(commitment_tree_address.position_range_start()) + 1, + }, + |row| { + let min_height = row + .get::<_, u32>(0) + .map(|h| BlockHeight::from_u32(h.saturating_sub(1)))?; + let max_height = row + .get::<_, u32>(1) + .map(|h| BlockHeight::from_u32(h.saturating_add(1)))?; + + Ok(min_height..max_height) + }, + ) + .optional() + .map_err(SqliteClientError::from) +} + #[cfg(any(test, feature = "test-dependencies"))] pub mod testing { use incrementalmerkletree::Position; diff --git a/zcash_client_sqlite/src/wallet/commitment_tree.rs b/zcash_client_sqlite/src/wallet/commitment_tree.rs index 6d123a6f45..495254a75e 100644 --- a/zcash_client_sqlite/src/wallet/commitment_tree.rs +++ b/zcash_client_sqlite/src/wallet/commitment_tree.rs @@ -11,17 +11,22 @@ use std::{ use incrementalmerkletree::{Address, Hashable, Level, Position, Retention}; use shardtree::{ - error::ShardTreeError, + error::{QueryError, ShardTreeError}, store::{Checkpoint, ShardStore, TreeState}, - LocatedPrunableTree, LocatedTree, PrunableTree, RetentionFlags, + LocatedPrunableTree, LocatedTree, PrunableTree, RetentionFlags, ShardTree, }; use zcash_client_backend::{ - data_api::chain::CommitmentTreeRoot, + data_api::{chain::CommitmentTreeRoot, SAPLING_SHARD_HEIGHT}, serialization::shardtree::{read_shard, write_shard}, }; use zcash_primitives::merkle_tree::HashSer; -use zcash_protocol::consensus::BlockHeight; +use zcash_protocol::{consensus::BlockHeight, ShieldedProtocol}; + +use crate::{error::SqliteClientError, PRUNING_DEPTH, SAPLING_TABLES_PREFIX}; + +#[cfg(feature = "orchard")] +use {crate::ORCHARD_TABLES_PREFIX, zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT}; /// Errors that can appear in SQLite-back [`ShardStore`] implementation operations. #[derive(Debug)] @@ -1112,6 +1117,69 @@ pub(crate) fn put_shard_roots< Ok(()) } +pub(crate) fn check_witnesses( + conn: &rusqlite::Transaction<'_>, +) -> Result>, SqliteClientError> { + let unspent_sapling_note_meta = super::sapling::select_unspent_note_meta(conn)?; + + let mut scan_ranges = vec![]; + let mut sapling_incomplete = vec![]; + let sapling_tree = + ShardTree::<_, { sapling::NOTE_COMMITMENT_TREE_DEPTH }, SAPLING_SHARD_HEIGHT>::new( + SqliteShardStore::<_, sapling::Node, SAPLING_SHARD_HEIGHT>::from_connection( + conn, + SAPLING_TABLES_PREFIX, + ) + .map_err(|e| ShardTreeError::Storage(Error::Query(e)))?, + PRUNING_DEPTH.try_into().unwrap(), + ); + for m in unspent_sapling_note_meta.iter() { + match sapling_tree.witness_at_checkpoint_depth(m.commitment_tree_position(), 0) { + Ok(_) => {} + Err(ShardTreeError::Query(QueryError::TreeIncomplete(mut addrs))) => { + sapling_incomplete.append(&mut addrs); + } + Err(other) => { + return Err(SqliteClientError::CommitmentTree(other)); + } + } + } + + for addr in sapling_incomplete { + let range = super::get_block_range(conn, ShieldedProtocol::Sapling, addr)?; + scan_ranges.extend(range.into_iter()); + } + + #[cfg(feature = "orchard")] + { + let unspent_orchard_note_meta = super::orchard::select_unspent_note_meta(conn)?; + let mut orchard_incomplete = vec![]; + let orchard_tree = ShardTree::<_, {orchard::NOTE_COMMITMENT_TREE_DEPTH as u8}, ORCHARD_SHARD_HEIGHT>::new( + SqliteShardStore::<_, orchard::tree::MerkleHashOrchard, ORCHARD_SHARD_HEIGHT>::from_connection(conn, ORCHARD_TABLES_PREFIX) + .map_err(|e| ShardTreeError::Storage(Error::Query(e)))?, + PRUNING_DEPTH.try_into().unwrap(), + ); + for m in unspent_orchard_note_meta.iter() { + match orchard_tree.witness_at_checkpoint_depth(m.commitment_tree_position(), 0) { + Ok(_) => {} + Err(ShardTreeError::Query(QueryError::TreeIncomplete(mut addrs))) => { + orchard_incomplete.append(&mut addrs); + } + Err(other) => { + return Err(SqliteClientError::CommitmentTree(other)); + } + } + } + + for addr in orchard_incomplete { + let range = super::get_block_range(conn, ShieldedProtocol::Orchard, addr)?; + scan_ranges.extend(range.into_iter()); + } + } + + Ok(scan_ranges) +} + #[cfg(test)] mod tests { use tempfile::NamedTempFile; diff --git a/zcash_client_sqlite/src/wallet/common.rs b/zcash_client_sqlite/src/wallet/common.rs index b3fdd569ce..63a1748bb8 100644 --- a/zcash_client_sqlite/src/wallet/common.rs +++ b/zcash_client_sqlite/src/wallet/common.rs @@ -1,5 +1,6 @@ //! Functions common to Sapling and Orchard support in the wallet. +use incrementalmerkletree::Position; use rusqlite::{named_params, types::Value, Connection, Row}; use std::{num::NonZeroU64, rc::Rc}; @@ -236,6 +237,90 @@ where .collect::>() } +#[allow(dead_code)] +pub(crate) struct UnspentNoteMeta { + note_id: ReceivedNoteId, + txid: TxId, + output_index: u32, + commitment_tree_position: Position, + value: Zatoshis, +} + +#[allow(dead_code)] +impl UnspentNoteMeta { + pub(crate) fn note_id(&self) -> ReceivedNoteId { + self.note_id + } + + pub(crate) fn txid(&self) -> TxId { + self.txid + } + + pub(crate) fn output_index(&self) -> u32 { + self.output_index + } + + pub(crate) fn commitment_tree_position(&self) -> Position { + self.commitment_tree_position + } + + pub(crate) fn value(&self) -> Zatoshis { + self.value + } +} + +pub(crate) fn select_unspent_note_meta( + conn: &rusqlite::Connection, + protocol: ShieldedProtocol, +) -> Result, SqliteClientError> { + let (table_prefix, index_col, _) = per_protocol_names(protocol); + let mut stmt = conn.prepare_cached(&format!(" + SELECT {table_prefix}_received_notes.id AS id, txid, {index_col}, + commitment_tree_position, value + FROM {table_prefix}_received_notes + INNER JOIN transactions + ON transactions.id_tx = {table_prefix}_received_notes.tx + WHERE value > 5000 -- FIXME #1316, allow selection of dust inputs + AND recipient_key_scope IS NOT NULL + AND nf IS NOT NULL + AND commitment_tree_position IS NOT NULL + AND {table_prefix}_received_notes.id NOT IN ( + SELECT {table_prefix}_received_note_id + FROM {table_prefix}_received_note_spends + JOIN transactions stx ON stx.id_tx = transaction_id + WHERE stx.block IS NOT NULL -- the spending tx is mined + OR stx.expiry_height IS NULL -- the spending tx will not expire + OR stx.expiry_height > :anchor_height -- the spending tx is unexpired + ) + AND NOT EXISTS ( + SELECT 1 FROM v_{table_prefix}_shard_unscanned_ranges unscanned + -- select all the unscanned ranges involving the shard containing this note + WHERE {table_prefix}_received_notes.commitment_tree_position >= unscanned.start_position + AND {table_prefix}_received_notes.commitment_tree_position < unscanned.end_position_exclusive + -- exclude unscanned ranges that start above the anchor height (they don't affect spendability) + AND unscanned.block_range_start <= :anchor_height + -- exclude unscanned ranges that end below the wallet birthday + AND unscanned.block_range_end > :wallet_birthday + ) + "))?; + + let res = stmt + .query_and_then::<_, SqliteClientError, _, _>([], |row| { + Ok(UnspentNoteMeta { + note_id: row.get("id").map(|id| ReceivedNoteId(protocol, id))?, + txid: row.get("txid").map(TxId::from_bytes)?, + output_index: row.get(index_col)?, + commitment_tree_position: row + .get::<_, u64>("commitment_tree_position") + .map(Position::from)?, + value: Zatoshis::from_nonnegative_i64(row.get("value")?)?, + }) + })? + .collect::, _>>()?; + + Ok(res) +} + pub(crate) fn spendable_notes_meta( conn: &rusqlite::Connection, protocol: ShieldedProtocol, diff --git a/zcash_client_sqlite/src/wallet/orchard.rs b/zcash_client_sqlite/src/wallet/orchard.rs index 41f9572020..d66c0f786d 100644 --- a/zcash_client_sqlite/src/wallet/orchard.rs +++ b/zcash_client_sqlite/src/wallet/orchard.rs @@ -24,7 +24,7 @@ use zip32::Scope; use crate::{error::SqliteClientError, AccountUuid, ReceivedNoteId, TxRef}; -use super::{get_account_ref, memo_repr, parse_scope, scope_code}; +use super::{common::UnspentNoteMeta, get_account_ref, memo_repr, parse_scope, scope_code}; /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedOrchardOutput { @@ -226,6 +226,12 @@ pub(crate) fn select_spendable_orchard_notes( ) } +pub(crate) fn select_unspent_note_meta( + conn: &Connection, +) -> Result, SqliteClientError> { + super::common::select_unspent_note_meta(conn, ShieldedProtocol::Orchard) +} + /// Records the specified shielded output as having been received. /// /// This implementation relies on the facts that: diff --git a/zcash_client_sqlite/src/wallet/sapling.rs b/zcash_client_sqlite/src/wallet/sapling.rs index e676b6ada1..a9783e2f83 100644 --- a/zcash_client_sqlite/src/wallet/sapling.rs +++ b/zcash_client_sqlite/src/wallet/sapling.rs @@ -24,7 +24,7 @@ use zip32::Scope; use crate::{error::SqliteClientError, AccountUuid, ReceivedNoteId, TxRef}; -use super::{get_account_ref, memo_repr, parse_scope, scope_code}; +use super::{common::UnspentNoteMeta, get_account_ref, memo_repr, parse_scope, scope_code}; /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedSaplingOutput { @@ -237,6 +237,12 @@ pub(crate) fn select_spendable_sapling_notes( ) } +pub(crate) fn select_unspent_note_meta( + conn: &Connection, +) -> Result, SqliteClientError> { + super::common::select_unspent_note_meta(conn, ShieldedProtocol::Sapling) +} + /// Retrieves the set of nullifiers for "potentially spendable" Sapling notes that the /// wallet is tracking. ///