From dbf7ea05fa9c02325c87227a48ad941befbdbc48 Mon Sep 17 00:00:00 2001
From: Oscar Pepper <oscarpepper@zingolabs.org>
Date: Mon, 23 Dec 2024 06:40:40 +0000
Subject: [PATCH 1/9] added sync status test

---
 libtonode-tests/tests/sync.rs | 50 +++++++++++++++++++++++++++++++++--
 zingo-sync/src/sync.rs        | 18 +++++++++++++
 2 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/libtonode-tests/tests/sync.rs b/libtonode-tests/tests/sync.rs
index 0bf6b79cc..88e713160 100644
--- a/libtonode-tests/tests/sync.rs
+++ b/libtonode-tests/tests/sync.rs
@@ -3,7 +3,7 @@
 use tempfile::TempDir;
 use testvectors::seeds::HOSPITAL_MUSEUM_SEED;
 use zingo_netutils::GrpcConnector;
-use zingo_sync::sync::sync;
+use zingo_sync::sync::{self, sync};
 use zingolib::{
     config::{construct_lightwalletd_uri, load_clientconfig, DEFAULT_LIGHTWALLETD_SERVER},
     get_base_address_macro,
@@ -12,7 +12,7 @@ use zingolib::{
     wallet::WalletBase,
 };
 
-#[ignore = "too slow, and flakey"]
+#[ignore = "temporary mainnet test for sync development"]
 #[tokio::test]
 async fn sync_mainnet_test() {
     rustls::crypto::ring::default_provider()
@@ -51,6 +51,52 @@ async fn sync_mainnet_test() {
     dbg!(&wallet.sync_state);
 }
 
+#[ignore = "mainnet test for large chain"]
+#[tokio::test]
+async fn sync_status() {
+    rustls::crypto::ring::default_provider()
+        .install_default()
+        .expect("Ring to work as a default");
+    tracing_subscriber::fmt().init();
+
+    let uri = construct_lightwalletd_uri(Some(DEFAULT_LIGHTWALLETD_SERVER.to_string()));
+    let temp_dir = TempDir::new().unwrap();
+    let temp_path = temp_dir.path().to_path_buf();
+    let config = load_clientconfig(
+        uri.clone(),
+        Some(temp_path),
+        zingolib::config::ChainType::Mainnet,
+        true,
+    )
+    .unwrap();
+    let lightclient = LightClient::create_from_wallet_base_async(
+        WalletBase::from_string(HOSPITAL_MUSEUM_SEED.to_string()),
+        &config,
+        2_700_000,
+        true,
+    )
+    .await
+    .unwrap();
+
+    let client = GrpcConnector::new(uri).get_client().await.unwrap();
+
+    let wallet = lightclient.wallet.clone();
+    let sync_handle = tokio::spawn(async move {
+        sync(client, &config.chain, wallet).await.unwrap();
+    });
+
+    let wallet = lightclient.wallet.clone();
+    tokio::spawn(async move {
+        loop {
+            let wallet = wallet.clone();
+            sync::sync_status(wallet).await.unwrap();
+        }
+    });
+
+    sync_handle.await.unwrap();
+}
+
+// temporary test for sync development
 #[tokio::test]
 async fn sync_test() {
     tracing_subscriber::fmt().init();
diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs
index c4ae1f432..2bfbb4a18 100644
--- a/zingo-sync/src/sync.rs
+++ b/zingo-sync/src/sync.rs
@@ -172,6 +172,24 @@ where
     Ok(())
 }
 
+/// TODO
+pub async fn sync_status<W>(wallet: Arc<Mutex<W>>) -> Result<(), SyncError>
+where
+    W: SyncWallet,
+{
+    let scan_ranges = wallet
+        .lock()
+        .await
+        .get_sync_state()
+        .unwrap()
+        .scan_ranges()
+        .clone();
+
+    dbg!(scan_ranges);
+
+    Ok(())
+}
+
 /// Returns true if sync is complete.
 ///
 /// Sync is complete when:

From 83be516cbf611a9a45a3f84815a895a4f388f3cf Mon Sep 17 00:00:00 2001
From: Oscar Pepper <oscarpepper@zingolabs.org>
Date: Mon, 23 Dec 2024 12:33:53 +0000
Subject: [PATCH 2/9] added sync status with test

---
 libtonode-tests/tests/sync.rs             |  6 ++-
 zingo-sync/src/client.rs                  |  2 +-
 zingo-sync/src/client/fetch.rs            | 12 ++---
 zingo-sync/src/primitives.rs              | 25 ++++++++--
 zingo-sync/src/scan/task.rs               |  6 +--
 zingo-sync/src/sync.rs                    | 61 +++++++++++++++++------
 zingolib/src/wallet/disk/testing/tests.rs |  2 +-
 7 files changed, 81 insertions(+), 33 deletions(-)

diff --git a/libtonode-tests/tests/sync.rs b/libtonode-tests/tests/sync.rs
index 88e713160..02368da69 100644
--- a/libtonode-tests/tests/sync.rs
+++ b/libtonode-tests/tests/sync.rs
@@ -1,5 +1,7 @@
 #![cfg(feature = "sync")]
 
+use std::time::Duration;
+
 use tempfile::TempDir;
 use testvectors::seeds::HOSPITAL_MUSEUM_SEED;
 use zingo_netutils::GrpcConnector;
@@ -89,7 +91,9 @@ async fn sync_status() {
     tokio::spawn(async move {
         loop {
             let wallet = wallet.clone();
-            sync::sync_status(wallet).await.unwrap();
+            let sync_status = sync::sync_status(wallet).await;
+            dbg!(sync_status);
+            tokio::time::sleep(Duration::from_secs(1)).await;
         }
     });
 
diff --git a/zingo-sync/src/client.rs b/zingo-sync/src/client.rs
index 04e22980b..a6baf2f65 100644
--- a/zingo-sync/src/client.rs
+++ b/zingo-sync/src/client.rs
@@ -158,7 +158,7 @@ pub async fn get_transparent_address_transactions(
 pub async fn get_mempool_transaction_stream(
     client: &mut CompactTxStreamerClient<zingo_netutils::UnderlyingService>,
 ) -> Result<tonic::Streaming<RawTransaction>, ()> {
-    tracing::info!("Fetching mempool stream");
+    tracing::debug!("Fetching mempool stream");
     let mempool_stream = fetch::get_mempool_stream(client).await.unwrap();
 
     Ok(mempool_stream)
diff --git a/zingo-sync/src/client/fetch.rs b/zingo-sync/src/client/fetch.rs
index 3b660f5e9..1362e0b04 100644
--- a/zingo-sync/src/client/fetch.rs
+++ b/zingo-sync/src/client/fetch.rs
@@ -103,29 +103,29 @@ async fn fetch_from_server(
 ) -> Result<(), ()> {
     match fetch_request {
         FetchRequest::ChainTip(sender) => {
-            tracing::info!("Fetching chain tip.");
+            tracing::debug!("Fetching chain tip.");
             let block_id = get_latest_block(client).await.unwrap();
             sender.send(block_id).unwrap();
         }
         FetchRequest::CompactBlockRange(sender, block_range) => {
-            tracing::info!("Fetching compact blocks. {:?}", &block_range);
+            tracing::debug!("Fetching compact blocks. {:?}", &block_range);
             let compact_blocks = get_block_range(client, block_range).await.unwrap();
             sender.send(compact_blocks).unwrap();
         }
         FetchRequest::TreeState(sender, block_height) => {
-            tracing::info!("Fetching tree state. {:?}", &block_height);
+            tracing::debug!("Fetching tree state. {:?}", &block_height);
             let tree_state = get_tree_state(client, block_height).await.unwrap();
             sender.send(tree_state).unwrap();
         }
         FetchRequest::Transaction(sender, txid) => {
-            tracing::info!("Fetching transaction. {:?}", txid);
+            tracing::debug!("Fetching transaction. {:?}", txid);
             let transaction = get_transaction(client, consensus_parameters, txid)
                 .await
                 .unwrap();
             sender.send(transaction).unwrap();
         }
         FetchRequest::UtxoMetadata(sender, (addresses, start_height)) => {
-            tracing::info!(
+            tracing::debug!(
                 "Fetching unspent transparent output metadata from {:?} for addresses:\n{:?}",
                 &start_height,
                 &addresses
@@ -136,7 +136,7 @@ async fn fetch_from_server(
             sender.send(utxo_metadata).unwrap();
         }
         FetchRequest::TransparentAddressTxs(sender, (address, block_range)) => {
-            tracing::info!(
+            tracing::debug!(
                 "Fetching raw transactions in block range {:?} for address {:?}",
                 &block_range,
                 &address
diff --git a/zingo-sync/src/primitives.rs b/zingo-sync/src/primitives.rs
index aab95173b..112913f0d 100644
--- a/zingo-sync/src/primitives.rs
+++ b/zingo-sync/src/primitives.rs
@@ -26,7 +26,7 @@ use crate::{
 pub type Locator = (BlockHeight, TxId);
 
 /// Encapsulates the current state of sync
-#[derive(Debug, Getters, MutGetters)]
+#[derive(Debug, Clone, Getters, MutGetters, CopyGetters, Setters)]
 #[getset(get = "pub", get_mut = "pub")]
 pub struct SyncState {
     /// A vec of block ranges with scan priorities from wallet birthday to chain tip.
@@ -34,6 +34,16 @@ pub struct SyncState {
     scan_ranges: Vec<ScanRange>,
     /// Locators for relevent transactions to the wallet.
     locators: BTreeSet<Locator>,
+    /// Fully scanned wallet height at start of sync.
+    /// Reset when sync starts.
+    #[getset(skip)]
+    #[getset(get_copy = "pub", set = "pub")]
+    sync_start_height: BlockHeight,
+    /// Total number of blocks to scan this session
+    /// Reset when sync starts.
+    #[getset(skip)]
+    #[getset(get_copy = "pub", set = "pub")]
+    total_blocks_to_scan: u32,
 }
 
 impl SyncState {
@@ -42,6 +52,8 @@ impl SyncState {
         SyncState {
             scan_ranges: Vec::new(),
             locators: BTreeSet::new(),
+            sync_start_height: 0.into(),
+            total_blocks_to_scan: 0,
         }
     }
 
@@ -70,10 +82,13 @@ impl SyncState {
     }
 }
 
-impl Default for SyncState {
-    fn default() -> Self {
-        Self::new()
-    }
+/// A snapshot of the current state of sync. Useful for displaying the status of sync to a user / consumer.
+#[derive(Debug, Clone, Getters)]
+pub struct SyncStatus {
+    pub scan_ranges: Vec<ScanRange>,
+    pub scanned_blocks: u32,
+    pub unscanned_blocks: u32,
+    pub percentage_blocks_complete: f32,
 }
 
 /// Output ID for a given pool type
diff --git a/zingo-sync/src/scan/task.rs b/zingo-sync/src/scan/task.rs
index ef3cecfdb..e4d6f8866 100644
--- a/zingo-sync/src/scan/task.rs
+++ b/zingo-sync/src/scan/task.rs
@@ -88,7 +88,7 @@ where
     ///
     /// When the worker is running it will wait for a scan task.
     pub(crate) fn spawn_worker(&mut self) {
-        tracing::info!("Spawning worker {}", self.unique_id);
+        tracing::debug!("Spawning worker {}", self.unique_id);
         let mut worker = ScanWorker::new(
             self.unique_id,
             self.consensus_parameters.clone(),
@@ -286,7 +286,7 @@ where
     }
 
     fn add_scan_task(&self, scan_task: ScanTask) -> Result<(), ()> {
-        tracing::info!("Adding scan task to worker {}:\n{:#?}", self.id, &scan_task);
+        tracing::debug!("Adding scan task to worker {}:\n{:#?}", self.id, &scan_task);
         self.scan_task_sender
             .clone()
             .unwrap()
@@ -301,7 +301,7 @@ where
     ///
     /// This should always be called in the context of the scanner as it must be also be removed from the worker pool.
     async fn shutdown(&mut self) -> Result<(), JoinError> {
-        tracing::info!("Shutting down worker {}", self.id);
+        tracing::debug!("Shutting down worker {}", self.id);
         if let Some(sender) = self.scan_task_sender.take() {
             drop(sender);
         }
diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs
index 2bfbb4a18..a6d1262b3 100644
--- a/zingo-sync/src/sync.rs
+++ b/zingo-sync/src/sync.rs
@@ -8,7 +8,7 @@ use std::time::Duration;
 use crate::client::{self, FetchRequest};
 use crate::error::SyncError;
 use crate::keys::transparent::TransparentAddressId;
-use crate::primitives::{NullifierMap, OutPointMap};
+use crate::primitives::{NullifierMap, OutPointMap, SyncState, SyncStatus};
 use crate::scan::error::{ContinuityError, ScanError};
 use crate::scan::task::Scanner;
 use crate::scan::transactions::scan_transaction;
@@ -65,7 +65,7 @@ where
     });
 
     let mut wallet_lock = wallet.lock().await;
-    let wallet_height = state::get_wallet_height(consensus_parameters, &*wallet_lock).unwrap();
+    let mut wallet_height = state::get_wallet_height(consensus_parameters, &*wallet_lock).unwrap();
     let chain_height = client::get_chain_height(fetch_request_sender.clone())
         .await
         .unwrap();
@@ -77,6 +77,7 @@ where
             );
         }
         truncate_wallet_data(&mut *wallet_lock, chain_height).unwrap();
+        wallet_height = chain_height;
     }
     let ufvks = wallet_lock.get_unified_full_viewing_keys().unwrap();
 
@@ -105,6 +106,20 @@ where
     )
     .await
     .unwrap();
+
+    let sync_state = wallet_lock.get_sync_state_mut().unwrap();
+    let total_blocks_to_scan = sync_state
+        .scan_ranges()
+        .iter()
+        .filter(|scan_range| scan_range.priority() != ScanPriority::Scanned)
+        .map(|scan_range| scan_range.block_range())
+        .fold(0, |acc, block_range| {
+            acc + u32::from(block_range.end - block_range.start)
+        });
+    sync_state.set_total_blocks_to_scan(total_blocks_to_scan);
+    let sync_start_height = sync_state.fully_scanned_height() + 1;
+    sync_state.set_sync_start_height(sync_start_height);
+
     drop(wallet_lock);
 
     // create channel for receiving scan results and launch scanner
@@ -163,6 +178,8 @@ where
         }
     }
 
+    // TODO: clear locators
+
     drop(wallet_lock);
     drop(scanner);
     drop(fetch_request_sender);
@@ -172,22 +189,34 @@ where
     Ok(())
 }
 
-/// TODO
-pub async fn sync_status<W>(wallet: Arc<Mutex<W>>) -> Result<(), SyncError>
+/// Obtains the mutex guard to the wallet and creates a [`crate::primitives::SyncStatus`] from the wallet's current
+/// [`crate::primitives::SyncState`].
+///
+/// Designed to be called during the sync process with minimal interruption.
+pub async fn sync_status<W>(wallet: Arc<Mutex<W>>) -> SyncStatus
 where
     W: SyncWallet,
 {
-    let scan_ranges = wallet
-        .lock()
-        .await
-        .get_sync_state()
-        .unwrap()
-        .scan_ranges()
-        .clone();
+    let sync_state: SyncState = wallet.lock().await.get_sync_state().unwrap().clone();
 
-    dbg!(scan_ranges);
-
-    Ok(())
+    let unscanned_blocks = sync_state
+        .scan_ranges()
+        .iter()
+        .filter(|scan_range| scan_range.priority() != ScanPriority::Scanned)
+        .map(|scan_range| scan_range.block_range())
+        .fold(0, |acc, block_range| {
+            acc + u32::from(block_range.end - block_range.start)
+        });
+    let scanned_blocks = sync_state.total_blocks_to_scan() - unscanned_blocks;
+    let percentage_blocks_complete =
+        (scanned_blocks as f32 / sync_state.total_blocks_to_scan() as f32) * 100.0;
+
+    SyncStatus {
+        scan_ranges: sync_state.scan_ranges().clone(),
+        unscanned_blocks,
+        scanned_blocks,
+        percentage_blocks_complete,
+    }
 }
 
 /// Returns true if sync is complete.
@@ -242,7 +271,7 @@ where
                 ScanPriority::Scanned,
             )
             .unwrap();
-            tracing::info!("Scan results processed.");
+            tracing::debug!("Scan results processed.");
         }
         Err(ScanError::ContinuityError(ContinuityError::HashDiscontinuity { height, .. })) => {
             tracing::info!("Re-org detected.");
@@ -289,7 +318,7 @@ async fn process_mempool_transaction<W>(
     )
     .unwrap();
 
-    tracing::info!(
+    tracing::debug!(
         "mempool received txid {} at height {}",
         transaction.txid(),
         block_height
diff --git a/zingolib/src/wallet/disk/testing/tests.rs b/zingolib/src/wallet/disk/testing/tests.rs
index 1fb3dd950..28699dd90 100644
--- a/zingolib/src/wallet/disk/testing/tests.rs
+++ b/zingolib/src/wallet/disk/testing/tests.rs
@@ -303,9 +303,9 @@ async fn reload_wallet_from_buffer() {
 #[tokio::test]
 #[cfg(feature = "sync")]
 async fn reload_wallet_from_buffer() {
-    use crate::testvectors::seeds::CHIMNEY_BETTER_SEED;
     use crate::wallet::WalletBase;
     use crate::wallet::WalletCapability;
+    use testvectors::seeds::CHIMNEY_BETTER_SEED;
 
     let mid_wallet =
         NetworkSeedVersion::Testnet(TestnetSeedVersion::ChimneyBetter(ChimneyBetterVersion::V28))

From 25ece4de4f7815eb4d150c618daeadd3c0791f3d Mon Sep 17 00:00:00 2001
From: Oscar Pepper <oscarpepper@zingolabs.org>
Date: Tue, 24 Dec 2024 08:00:40 +0000
Subject: [PATCH 3/9] set new grpc method to debug level logging

---
 zingo-sync/src/client/fetch.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/zingo-sync/src/client/fetch.rs b/zingo-sync/src/client/fetch.rs
index 9271b60d5..e7d6b64ba 100644
--- a/zingo-sync/src/client/fetch.rs
+++ b/zingo-sync/src/client/fetch.rs
@@ -113,7 +113,7 @@ async fn fetch_from_server(
             sender.send(compact_blocks).unwrap();
         }
         FetchRequest::GetSubtreeRoots(sender, start_index, shielded_protocol, max_entries) => {
-            tracing::info!(
+            tracing::debug!(
                 "Fetching subtree roots. start index: {}. shielded protocol: {}",
                 start_index,
                 shielded_protocol

From f72e996cdf1fe5130c7184ebb601be37c172712c Mon Sep 17 00:00:00 2001
From: Oscar Pepper <oscarpepper@zingolabs.org>
Date: Thu, 26 Dec 2024 08:02:55 +0000
Subject: [PATCH 4/9] fix clippy

---
 zingo-sync/src/primitives.rs | 6 ++++++
 zingo-sync/src/sync.rs       | 2 +-
 zingo-sync/src/sync/state.rs | 2 +-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/zingo-sync/src/primitives.rs b/zingo-sync/src/primitives.rs
index 52be0a11f..eff776c49 100644
--- a/zingo-sync/src/primitives.rs
+++ b/zingo-sync/src/primitives.rs
@@ -82,6 +82,12 @@ impl SyncState {
     }
 }
 
+impl Default for SyncState {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 /// A snapshot of the current state of sync. Useful for displaying the status of sync to a user / consumer.
 #[derive(Debug, Clone, Getters)]
 pub struct SyncStatus {
diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs
index fa98891d2..0ed23ac8f 100644
--- a/zingo-sync/src/sync.rs
+++ b/zingo-sync/src/sync.rs
@@ -201,7 +201,7 @@ where
         .filter(|scan_range| scan_range.priority() != ScanPriority::Scanned)
         .map(|scan_range| scan_range.block_range())
         .fold(0, |acc, block_range| {
-            acc + u32::from(block_range.end - block_range.start)
+            acc + (block_range.end - block_range.start)
         });
     let scanned_blocks = sync_state.total_blocks_to_scan() - unscanned_blocks;
     let percentage_blocks_complete =
diff --git a/zingo-sync/src/sync/state.rs b/zingo-sync/src/sync/state.rs
index a799b4d74..d29eac6fb 100644
--- a/zingo-sync/src/sync/state.rs
+++ b/zingo-sync/src/sync/state.rs
@@ -436,7 +436,7 @@ pub(super) fn set_initial_state(sync_state: &mut SyncState) {
         .filter(|scan_range| scan_range.priority() != ScanPriority::Scanned)
         .map(|scan_range| scan_range.block_range())
         .fold(0, |acc, block_range| {
-            acc + u32::from(block_range.end - block_range.start)
+            acc + (block_range.end - block_range.start)
         });
     sync_state.set_total_blocks_to_scan(total_blocks_to_scan);
     let sync_start_height = sync_state.fully_scanned_height() + 1;

From ac4d7433551efcf0c0dc4565f22c958ececf09be Mon Sep 17 00:00:00 2001
From: Oscar Pepper <oscarpepper@zingolabs.org>
Date: Tue, 7 Jan 2025 02:58:08 +0000
Subject: [PATCH 5/9] removed sync feature from libtonode

---
 libtonode-tests/Cargo.toml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/libtonode-tests/Cargo.toml b/libtonode-tests/Cargo.toml
index 8308f020d..3b8a9bf40 100644
--- a/libtonode-tests/Cargo.toml
+++ b/libtonode-tests/Cargo.toml
@@ -6,14 +6,13 @@ edition = "2021"
 [features]
 chain_generic_tests = []
 ci = ["zingolib/ci"]
-sync = ["dep:zingo-sync"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [dependencies]
 zingolib = { path = "../zingolib", features = [ "deprecations", "test-elevation" ] }
 zingo-status = { path = "../zingo-status" }
 zingo-netutils = { path = "../zingo-netutils" }
-zingo-sync = { path = "../zingo-sync", optional = true }
+zingo-sync = { path = "../zingo-sync" }
 testvectors = { path = "../testvectors" }
 
 bip0039.workspace = true

From 790f417db22d5773b227f3d7e73c19597920246e Mon Sep 17 00:00:00 2001
From: Oscar Pepper <oscarpepper@zingolabs.org>
Date: Wed, 15 Jan 2025 10:15:25 +0000
Subject: [PATCH 6/9] added outputs to initial sync state

---
 zingo-sync/src/primitives.rs          |  67 +++++++++---
 zingo-sync/src/scan.rs                | 113 ++++++++++---------
 zingo-sync/src/scan/compact_blocks.rs |  31 +++---
 zingo-sync/src/sync.rs                |  16 ++-
 zingo-sync/src/sync/state.rs          | 151 +++++++++++++++++++++++++-
 5 files changed, 283 insertions(+), 95 deletions(-)

diff --git a/zingo-sync/src/primitives.rs b/zingo-sync/src/primitives.rs
index eff776c49..749906d2b 100644
--- a/zingo-sync/src/primitives.rs
+++ b/zingo-sync/src/primitives.rs
@@ -25,6 +25,40 @@ use crate::{
 /// detections or transparent output discovery.
 pub type Locator = (BlockHeight, TxId);
 
+/// Initial sync state.
+///
+/// All fields will be reset when a new sync session starts.
+#[derive(Debug, Clone, CopyGetters, Setters)]
+#[getset(get_copy = "pub", set = "pub")]
+pub struct InitialSyncState {
+    /// One block above the fully scanned wallet height at start of sync.
+    sync_start_height: BlockHeight,
+    /// Sapling tree size of fully scanned wallet height at start of sync.
+    sync_start_sapling_tree_size: u32,
+    /// Orchard tree size of fully scanned wallet height at start of sync.
+    sync_start_orchard_tree_size: u32,
+    /// Total number of blocks to scan.
+    total_blocks_to_scan: u32,
+    /// Total number of sapling outputs to scan.
+    total_sapling_outputs_to_scan: u32,
+    /// Total number of orchard outputs to scan.
+    total_orchard_outputs_to_scan: u32,
+}
+
+impl InitialSyncState {
+    /// Create new InitialSyncState
+    pub fn new() -> Self {
+        InitialSyncState {
+            sync_start_height: 0.into(),
+            sync_start_sapling_tree_size: 0,
+            sync_start_orchard_tree_size: 0,
+            total_blocks_to_scan: 0,
+            total_sapling_outputs_to_scan: 0,
+            total_orchard_outputs_to_scan: 0,
+        }
+    }
+}
+
 /// Encapsulates the current state of sync
 #[derive(Debug, Clone, Getters, MutGetters, CopyGetters, Setters)]
 #[getset(get = "pub", get_mut = "pub")]
@@ -34,16 +68,8 @@ pub struct SyncState {
     scan_ranges: Vec<ScanRange>,
     /// Locators for relevant transactions to the wallet.
     locators: BTreeSet<Locator>,
-    /// Fully scanned wallet height at start of sync.
-    /// Reset when sync starts.
-    #[getset(skip)]
-    #[getset(get_copy = "pub", set = "pub")]
-    sync_start_height: BlockHeight,
-    /// Total number of blocks to scan this session
-    /// Reset when sync starts.
-    #[getset(skip)]
-    #[getset(get_copy = "pub", set = "pub")]
-    total_blocks_to_scan: u32,
+    /// Initial sync state.
+    initial_sync_state: InitialSyncState,
 }
 
 impl SyncState {
@@ -52,8 +78,7 @@ impl SyncState {
         SyncState {
             scan_ranges: Vec::new(),
             locators: BTreeSet::new(),
-            sync_start_height: 0.into(),
-            total_blocks_to_scan: 0,
+            initial_sync_state: InitialSyncState::new(),
         }
     }
 
@@ -171,8 +196,10 @@ pub struct WalletBlock {
     time: u32,
     #[getset(skip)]
     txids: Vec<TxId>,
-    sapling_commitment_tree_size: u32,
-    orchard_commitment_tree_size: u32,
+    sapling_initial_tree_size: u32,
+    orchard_initial_tree_size: u32,
+    sapling_final_tree_size: u32,
+    orchard_final_tree_size: u32,
 }
 
 impl WalletBlock {
@@ -182,8 +209,10 @@ impl WalletBlock {
         prev_hash: BlockHash,
         time: u32,
         txids: Vec<TxId>,
-        sapling_commitment_tree_size: u32,
-        orchard_commitment_tree_size: u32,
+        sapling_starting_tree_size: u32,
+        orchard_starting_tree_size: u32,
+        sapling_final_tree_size: u32,
+        orchard_final_tree_size: u32,
     ) -> Self {
         Self {
             block_height,
@@ -191,8 +220,10 @@ impl WalletBlock {
             prev_hash,
             time,
             txids,
-            sapling_commitment_tree_size,
-            orchard_commitment_tree_size,
+            sapling_initial_tree_size: sapling_starting_tree_size,
+            orchard_initial_tree_size: orchard_starting_tree_size,
+            sapling_final_tree_size,
+            orchard_final_tree_size,
         }
     }
 
diff --git a/zingo-sync/src/scan.rs b/zingo-sync/src/scan.rs
index 1a3c8e4b4..37115d0ab 100644
--- a/zingo-sync/src/scan.rs
+++ b/zingo-sync/src/scan.rs
@@ -50,59 +50,68 @@ impl InitialScanData {
         // gets initial tree size from previous block if available
         // otherwise, from first block if available
         // otherwise, fetches frontiers from server
-        let (sapling_initial_tree_size, orchard_initial_tree_size) =
-            if let Some(prev) = &previous_wallet_block {
-                (
-                    prev.sapling_commitment_tree_size(),
-                    prev.orchard_commitment_tree_size(),
-                )
-            } else if let Some(chain_metadata) = &first_block.chain_metadata {
-                // calculate initial tree size by subtracting number of outputs in block from the blocks final tree size
-                let sapling_output_count: u32 = first_block
-                    .vtx
-                    .iter()
-                    .map(|tx| tx.outputs.len())
-                    .sum::<usize>()
-                    .try_into()
-                    .expect("Sapling output count cannot exceed a u32");
-                let orchard_output_count: u32 = first_block
-                    .vtx
-                    .iter()
-                    .map(|tx| tx.actions.len())
-                    .sum::<usize>()
-                    .try_into()
-                    .expect("Sapling output count cannot exceed a u32");
-
-                (
-                    chain_metadata
-                        .sapling_commitment_tree_size
-                        .checked_sub(sapling_output_count)
-                        .unwrap(),
-                    chain_metadata
-                        .orchard_commitment_tree_size
-                        .checked_sub(orchard_output_count)
-                        .unwrap(),
-                )
-            } else {
-                let sapling_activation_height = consensus_parameters
-                    .activation_height(NetworkUpgrade::Sapling)
-                    .expect("should have some sapling activation height");
-
-                match first_block.height().cmp(&sapling_activation_height) {
-                    cmp::Ordering::Greater => {
-                        let frontiers =
-                            client::get_frontiers(fetch_request_sender, first_block.height() - 1)
-                                .await
-                                .unwrap();
-                        (
-                            frontiers.final_sapling_tree().tree_size() as u32,
-                            frontiers.final_orchard_tree().tree_size() as u32,
-                        )
-                    }
-                    cmp::Ordering::Equal => (0, 0),
-                    cmp::Ordering::Less => panic!("pre-sapling not supported!"),
+        let (sapling_initial_tree_size, orchard_initial_tree_size) = if let Some(prev) =
+            &previous_wallet_block
+        {
+            (
+                prev.sapling_final_tree_size(),
+                prev.orchard_final_tree_size(),
+            )
+        } else if let Some(chain_metadata) = &first_block.chain_metadata {
+            // calculate initial tree size by subtracting number of outputs in block from the blocks final tree size
+            let sapling_output_count: u32 = first_block
+                .vtx
+                .iter()
+                .map(|tx| tx.outputs.len())
+                .sum::<usize>()
+                .try_into()
+                .expect("Sapling output count cannot exceed a u32");
+            let orchard_output_count: u32 = first_block
+                .vtx
+                .iter()
+                .map(|tx| tx.actions.len())
+                .sum::<usize>()
+                .try_into()
+                .expect("Sapling output count cannot exceed a u32");
+
+            (
+                chain_metadata
+                    .sapling_commitment_tree_size
+                    .checked_sub(sapling_output_count)
+                    .unwrap(),
+                chain_metadata
+                    .orchard_commitment_tree_size
+                    .checked_sub(orchard_output_count)
+                    .unwrap(),
+            )
+        } else {
+            let sapling_activation_height = consensus_parameters
+                .activation_height(NetworkUpgrade::Sapling)
+                .expect("should have some sapling activation height");
+
+            match first_block.height().cmp(&sapling_activation_height) {
+                cmp::Ordering::Greater => {
+                    let frontiers =
+                        client::get_frontiers(fetch_request_sender, first_block.height() - 1)
+                            .await
+                            .unwrap();
+                    (
+                        frontiers
+                            .final_sapling_tree()
+                            .tree_size()
+                            .try_into()
+                            .expect("should not be more than 2^32 note commitments in the tree!"),
+                        frontiers
+                            .final_orchard_tree()
+                            .tree_size()
+                            .try_into()
+                            .expect("should not be more than 2^32 note commitments in the tree!"),
+                    )
                 }
-            };
+                cmp::Ordering::Equal => (0, 0),
+                cmp::Ordering::Less => panic!("pre-sapling not supported!"),
+            }
+        };
 
         Ok(InitialScanData {
             previous_block: previous_wallet_block,
diff --git a/zingo-sync/src/scan/compact_blocks.rs b/zingo-sync/src/scan/compact_blocks.rs
index 2d724c394..144aee17e 100644
--- a/zingo-sync/src/scan/compact_blocks.rs
+++ b/zingo-sync/src/scan/compact_blocks.rs
@@ -55,9 +55,14 @@ where
         Position::from(u64::from(initial_scan_data.sapling_initial_tree_size)),
         Position::from(u64::from(initial_scan_data.orchard_initial_tree_size)),
     );
-    let mut sapling_tree_size = initial_scan_data.sapling_initial_tree_size;
-    let mut orchard_tree_size = initial_scan_data.orchard_initial_tree_size;
+    let mut sapling_initial_tree_size;
+    let mut orchard_initial_tree_size;
+    let mut sapling_final_tree_size = initial_scan_data.sapling_initial_tree_size;
+    let mut orchard_final_tree_size = initial_scan_data.orchard_initial_tree_size;
     for block in &compact_blocks {
+        sapling_initial_tree_size = sapling_final_tree_size;
+        orchard_initial_tree_size = orchard_final_tree_size;
+
         let mut transactions = block.vtx.iter().peekable();
         while let Some(transaction) = transactions.next() {
             // collect trial decryption results by transaction
@@ -100,21 +105,21 @@ where
             );
 
             calculate_nullifiers_and_positions(
-                sapling_tree_size,
+                sapling_final_tree_size,
                 scanning_keys.sapling(),
                 &incoming_sapling_outputs,
                 &mut decrypted_note_data.sapling_nullifiers_and_positions,
             );
             calculate_nullifiers_and_positions(
-                orchard_tree_size,
+                orchard_final_tree_size,
                 scanning_keys.orchard(),
                 &incoming_orchard_outputs,
                 &mut decrypted_note_data.orchard_nullifiers_and_positions,
             );
 
-            sapling_tree_size += u32::try_from(transaction.outputs.len())
+            sapling_final_tree_size += u32::try_from(transaction.outputs.len())
                 .expect("should not be more than 2^32 outputs in a transaction");
-            orchard_tree_size += u32::try_from(transaction.actions.len())
+            orchard_final_tree_size += u32::try_from(transaction.actions.len())
                 .expect("should not be more than 2^32 outputs in a transaction");
         }
 
@@ -124,8 +129,10 @@ where
             block.prev_hash(),
             block.time,
             block.vtx.iter().map(|tx| tx.txid()).collect(),
-            sapling_tree_size,
-            orchard_tree_size,
+            sapling_initial_tree_size,
+            orchard_initial_tree_size,
+            sapling_final_tree_size,
+            orchard_final_tree_size,
         );
 
         check_tree_size(block, &wallet_block).unwrap();
@@ -203,14 +210,10 @@ fn check_continuity(
 
 fn check_tree_size(compact_block: &CompactBlock, wallet_block: &WalletBlock) -> Result<(), ()> {
     if let Some(chain_metadata) = &compact_block.chain_metadata {
-        if chain_metadata.sapling_commitment_tree_size
-            != wallet_block.sapling_commitment_tree_size()
-        {
+        if chain_metadata.sapling_commitment_tree_size != wallet_block.sapling_final_tree_size() {
             panic!("sapling tree size is incorrect!")
         }
-        if chain_metadata.orchard_commitment_tree_size
-            != wallet_block.orchard_commitment_tree_size()
-        {
+        if chain_metadata.orchard_commitment_tree_size != wallet_block.orchard_final_tree_size() {
             panic!("orchard tree size is incorrect!")
         }
     }
diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs
index 0ed23ac8f..2d1442885 100644
--- a/zingo-sync/src/sync.rs
+++ b/zingo-sync/src/sync.rs
@@ -19,7 +19,6 @@ use crate::traits::{
 
 use futures::StreamExt;
 use shardtree::{store::ShardStore, LocatedPrunableTree, RetentionFlags};
-use state::set_initial_state;
 use zcash_client_backend::proto::service::RawTransaction;
 use zcash_client_backend::{
     data_api::scanning::{ScanPriority, ScanRange},
@@ -113,7 +112,13 @@ where
     .await
     .unwrap();
 
-    set_initial_state(wallet_guard.get_sync_state_mut().unwrap());
+    state::set_initial_state(
+        consensus_parameters,
+        fetch_request_sender.clone(),
+        &mut *wallet_guard,
+        chain_height,
+    )
+    .await;
 
     update_subtree_roots(fetch_request_sender.clone(), &mut *wallet_guard).await;
 
@@ -203,9 +208,10 @@ where
         .fold(0, |acc, block_range| {
             acc + (block_range.end - block_range.start)
         });
-    let scanned_blocks = sync_state.total_blocks_to_scan() - unscanned_blocks;
-    let percentage_blocks_complete =
-        (scanned_blocks as f32 / sync_state.total_blocks_to_scan() as f32) * 100.0;
+    let scanned_blocks = sync_state.initial_sync_state().total_blocks_to_scan() - unscanned_blocks;
+    let percentage_blocks_complete = (scanned_blocks as f32
+        / sync_state.initial_sync_state().total_blocks_to_scan() as f32)
+        * 100.0;
 
     SyncStatus {
         scan_ranges: sync_state.scan_ranges().clone(),
diff --git a/zingo-sync/src/sync/state.rs b/zingo-sync/src/sync/state.rs
index d29eac6fb..93385b95f 100644
--- a/zingo-sync/src/sync/state.rs
+++ b/zingo-sync/src/sync/state.rs
@@ -2,13 +2,15 @@
 
 use std::{cmp, collections::HashMap, ops::Range};
 
+use tokio::sync::mpsc;
 use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange};
 use zcash_primitives::{
-    consensus::{self, BlockHeight},
+    consensus::{self, BlockHeight, NetworkUpgrade},
     transaction::TxId,
 };
 
 use crate::{
+    client::{self, FetchRequest},
     keys::transparent::TransparentAddressId,
     primitives::{Locator, SyncState},
     scan::task::ScanTask,
@@ -428,8 +430,114 @@ where
     }
 }
 
-/// Sets the `total_blocks_to_scan` and `sync_start_height` fields at the start of the sync process
-pub(super) fn set_initial_state(sync_state: &mut SyncState) {
+/// Sets the `initial_sync_state` field at the start of the sync session
+pub(super) async fn set_initial_state<W>(
+    consensus_parameters: &impl consensus::Parameters,
+    fetch_request_sender: mpsc::UnboundedSender<FetchRequest>,
+    wallet: &mut W,
+    chain_height: BlockHeight,
+) where
+    W: SyncWallet + SyncBlocks,
+{
+    struct ScannedRangeTreeBoundaries {
+        sapling_initial_tree_size: u32,
+        orchard_initial_tree_size: u32,
+        sapling_final_tree_size: u32,
+        orchard_final_tree_size: u32,
+    }
+
+    /// Gets `block_height` final tree sizes from wallet block if it exists, otherwise from frontiers fetched from server.
+    ///
+    /// Only used in context of setting initial sync state as can also use the compact blocks to calculate tree sizes
+    /// during scanning.
+    async fn final_tree_sizes<W>(
+        consensus_parameters: &impl consensus::Parameters,
+        fetch_request_sender: mpsc::UnboundedSender<FetchRequest>,
+        wallet: &mut W,
+        block_height: BlockHeight,
+    ) -> (u32, u32)
+    where
+        W: SyncWallet + SyncBlocks,
+    {
+        if let Ok(block) = wallet.get_wallet_block(block_height) {
+            (
+                block.sapling_final_tree_size(),
+                block.orchard_final_tree_size(),
+            )
+        } else {
+            // TODO: move this whole block into `client::get_frontiers`
+            let sapling_activation_height = consensus_parameters
+                .activation_height(NetworkUpgrade::Sapling)
+                .expect("should have some sapling activation height");
+
+            match block_height.cmp(&sapling_activation_height) {
+                cmp::Ordering::Greater => {
+                    let frontiers =
+                        client::get_frontiers(fetch_request_sender.clone(), block_height)
+                            .await
+                            .unwrap();
+                    (
+                        frontiers
+                            .final_sapling_tree()
+                            .tree_size()
+                            .try_into()
+                            .expect("should not be more than 2^32 note commitments in the tree!"),
+                        frontiers
+                            .final_orchard_tree()
+                            .tree_size()
+                            .try_into()
+                            .expect("should not be more than 2^32 note commitments in the tree!"),
+                    )
+                }
+                cmp::Ordering::Equal => (0, 0),
+                cmp::Ordering::Less => panic!("pre-sapling not supported!"),
+            }
+        }
+    }
+
+    /// Gets the initial and final tree sizes of a `scanned_range`.
+    ///
+    /// Panics if `scanned_range` boundary wallet blocks are not found in the wallet.
+    fn scanned_range_tree_boundaries<W>(
+        wallet: &mut W,
+        scanned_range: Range<BlockHeight>,
+    ) -> ScannedRangeTreeBoundaries
+    where
+        W: SyncWallet + SyncBlocks,
+    {
+        let start_block = wallet
+            .get_wallet_block(scanned_range.start)
+            .expect("scanned range boundary blocks should be retained in the wallet");
+        let end_block = wallet
+            .get_wallet_block(scanned_range.end - 1)
+            .expect("scanned range boundary blocks should be retained in the wallet");
+
+        ScannedRangeTreeBoundaries {
+            sapling_initial_tree_size: start_block.sapling_initial_tree_size(),
+            orchard_initial_tree_size: start_block.orchard_initial_tree_size(),
+            sapling_final_tree_size: end_block.sapling_final_tree_size(),
+            orchard_final_tree_size: end_block.orchard_final_tree_size(),
+        }
+    }
+
+    let fully_scanned_height = wallet.get_sync_state().unwrap().fully_scanned_height();
+
+    let (sync_start_sapling_tree_size, sync_start_orchard_tree_size) = final_tree_sizes(
+        consensus_parameters,
+        fetch_request_sender.clone(),
+        wallet,
+        fully_scanned_height,
+    )
+    .await;
+    let (chain_tip_sapling_tree_size, chain_tip_orchard_tree_size) = final_tree_sizes(
+        consensus_parameters,
+        fetch_request_sender.clone(),
+        wallet,
+        chain_height,
+    )
+    .await;
+
+    let sync_state = wallet.get_sync_state().unwrap();
     let total_blocks_to_scan = sync_state
         .scan_ranges()
         .iter()
@@ -438,7 +546,38 @@ pub(super) fn set_initial_state(sync_state: &mut SyncState) {
         .fold(0, |acc, block_range| {
             acc + (block_range.end - block_range.start)
         });
-    sync_state.set_total_blocks_to_scan(total_blocks_to_scan);
-    let sync_start_height = sync_state.fully_scanned_height() + 1;
-    sync_state.set_sync_start_height(sync_start_height);
+
+    let scanned_block_ranges = sync_state
+        .scan_ranges()
+        .iter()
+        .filter(|scan_range| {
+            scan_range.priority() == ScanPriority::Scanned
+                && scan_range.block_range().start > fully_scanned_height
+        })
+        .map(|scan_range| scan_range.block_range().clone())
+        .collect::<Vec<_>>();
+    let (scanned_sapling_outputs, scanned_orchard_outputs) = scanned_block_ranges
+        .iter()
+        .map(|block_range| scanned_range_tree_boundaries(wallet, block_range.clone()))
+        .fold((0, 0), |acc, tree_sizes| {
+            (
+                acc.0 + (tree_sizes.sapling_final_tree_size - tree_sizes.sapling_initial_tree_size),
+                acc.1 + (tree_sizes.orchard_final_tree_size - tree_sizes.orchard_initial_tree_size),
+            )
+        });
+    let total_sapling_outputs_to_scan =
+        chain_tip_sapling_tree_size - sync_start_sapling_tree_size - scanned_sapling_outputs;
+    let total_orchard_outputs_to_scan =
+        chain_tip_orchard_tree_size - sync_start_orchard_tree_size - scanned_orchard_outputs;
+
+    let initial_sync_state = wallet
+        .get_sync_state_mut()
+        .unwrap()
+        .initial_sync_state_mut();
+    initial_sync_state.set_sync_start_height(fully_scanned_height + 1);
+    initial_sync_state.set_sync_start_sapling_tree_size(sync_start_sapling_tree_size);
+    initial_sync_state.set_sync_start_orchard_tree_size(sync_start_orchard_tree_size);
+    initial_sync_state.set_total_blocks_to_scan(total_blocks_to_scan);
+    initial_sync_state.set_total_sapling_outputs_to_scan(total_sapling_outputs_to_scan);
+    initial_sync_state.set_total_orchard_outputs_to_scan(total_orchard_outputs_to_scan);
 }

From b8b2aca3b3dfe74fa5443f871594d3bd87f261e0 Mon Sep 17 00:00:00 2001
From: Oscar Pepper <oscarpepper@zingolabs.org>
Date: Thu, 16 Jan 2025 05:34:15 +0000
Subject: [PATCH 7/9] updated sync status to include outputs

---
 zingo-sync/src/primitives.rs          |  53 +++---
 zingo-sync/src/scan.rs                |   4 +-
 zingo-sync/src/scan/compact_blocks.rs |  20 ++-
 zingo-sync/src/sync.rs                |  37 +++-
 zingo-sync/src/sync/state.rs          | 239 ++++++++++++++------------
 5 files changed, 209 insertions(+), 144 deletions(-)

diff --git a/zingo-sync/src/primitives.rs b/zingo-sync/src/primitives.rs
index 749906d2b..db00bd67a 100644
--- a/zingo-sync/src/primitives.rs
+++ b/zingo-sync/src/primitives.rs
@@ -31,12 +31,10 @@ pub type Locator = (BlockHeight, TxId);
 #[derive(Debug, Clone, CopyGetters, Setters)]
 #[getset(get_copy = "pub", set = "pub")]
 pub struct InitialSyncState {
-    /// One block above the fully scanned wallet height at start of sync.
+    /// One block above the fully scanned wallet height at start of sync session.
     sync_start_height: BlockHeight,
-    /// Sapling tree size of fully scanned wallet height at start of sync.
-    sync_start_sapling_tree_size: u32,
-    /// Orchard tree size of fully scanned wallet height at start of sync.
-    sync_start_orchard_tree_size: u32,
+    /// The tree sizes of the fully scanned height and chain tip at start of sync session.
+    sync_tree_boundaries: TreeBoundaries,
     /// Total number of blocks to scan.
     total_blocks_to_scan: u32,
     /// Total number of sapling outputs to scan.
@@ -50,8 +48,12 @@ impl InitialSyncState {
     pub fn new() -> Self {
         InitialSyncState {
             sync_start_height: 0.into(),
-            sync_start_sapling_tree_size: 0,
-            sync_start_orchard_tree_size: 0,
+            sync_tree_boundaries: TreeBoundaries {
+                sapling_initial_tree_size: 0,
+                sapling_final_tree_size: 0,
+                orchard_initial_tree_size: 0,
+                orchard_final_tree_size: 0,
+            },
             total_blocks_to_scan: 0,
             total_sapling_outputs_to_scan: 0,
             total_orchard_outputs_to_scan: 0,
@@ -59,6 +61,11 @@ impl InitialSyncState {
     }
 }
 
+impl Default for InitialSyncState {
+    fn default() -> Self {
+        Self::new()
+    }
+}
 /// Encapsulates the current state of sync
 #[derive(Debug, Clone, Getters, MutGetters, CopyGetters, Setters)]
 #[getset(get = "pub", get_mut = "pub")]
@@ -113,13 +120,28 @@ impl Default for SyncState {
     }
 }
 
+#[derive(Debug, Clone, Copy)]
+pub struct TreeBoundaries {
+    pub sapling_initial_tree_size: u32,
+    pub sapling_final_tree_size: u32,
+    pub orchard_initial_tree_size: u32,
+    pub orchard_final_tree_size: u32,
+}
+
 /// A snapshot of the current state of sync. Useful for displaying the status of sync to a user / consumer.
+///
+/// `percentage_outputs_scanned` is a much more accurate indicator of sync completion than `percentage_blocks_scanned`.
 #[derive(Debug, Clone, Getters)]
 pub struct SyncStatus {
     pub scan_ranges: Vec<ScanRange>,
     pub scanned_blocks: u32,
     pub unscanned_blocks: u32,
-    pub percentage_blocks_complete: f32,
+    pub percentage_blocks_scanned: f32,
+    pub scanned_sapling_outputs: u32,
+    pub unscanned_sapling_outputs: u32,
+    pub scanned_orchard_outputs: u32,
+    pub unscanned_orchard_outputs: u32,
+    pub percentage_outputs_scanned: f32,
 }
 
 /// Output ID for a given pool type
@@ -196,10 +218,7 @@ pub struct WalletBlock {
     time: u32,
     #[getset(skip)]
     txids: Vec<TxId>,
-    sapling_initial_tree_size: u32,
-    orchard_initial_tree_size: u32,
-    sapling_final_tree_size: u32,
-    orchard_final_tree_size: u32,
+    tree_boundaries: TreeBoundaries,
 }
 
 impl WalletBlock {
@@ -209,10 +228,7 @@ impl WalletBlock {
         prev_hash: BlockHash,
         time: u32,
         txids: Vec<TxId>,
-        sapling_starting_tree_size: u32,
-        orchard_starting_tree_size: u32,
-        sapling_final_tree_size: u32,
-        orchard_final_tree_size: u32,
+        tree_boundaries: TreeBoundaries,
     ) -> Self {
         Self {
             block_height,
@@ -220,10 +236,7 @@ impl WalletBlock {
             prev_hash,
             time,
             txids,
-            sapling_initial_tree_size: sapling_starting_tree_size,
-            orchard_initial_tree_size: orchard_starting_tree_size,
-            sapling_final_tree_size,
-            orchard_final_tree_size,
+            tree_boundaries,
         }
     }
 
diff --git a/zingo-sync/src/scan.rs b/zingo-sync/src/scan.rs
index 37115d0ab..0abf75e07 100644
--- a/zingo-sync/src/scan.rs
+++ b/zingo-sync/src/scan.rs
@@ -54,8 +54,8 @@ impl InitialScanData {
             &previous_wallet_block
         {
             (
-                prev.sapling_final_tree_size(),
-                prev.orchard_final_tree_size(),
+                prev.tree_boundaries().sapling_final_tree_size,
+                prev.tree_boundaries().orchard_final_tree_size,
             )
         } else if let Some(chain_metadata) = &first_block.chain_metadata {
             // calculate initial tree size by subtracting number of outputs in block from the blocks final tree size
diff --git a/zingo-sync/src/scan/compact_blocks.rs b/zingo-sync/src/scan/compact_blocks.rs
index 144aee17e..fcd2497ff 100644
--- a/zingo-sync/src/scan/compact_blocks.rs
+++ b/zingo-sync/src/scan/compact_blocks.rs
@@ -17,7 +17,7 @@ use zcash_primitives::{
 
 use crate::{
     keys::{KeyId, ScanningKeyOps, ScanningKeys},
-    primitives::{NullifierMap, OutputId, WalletBlock},
+    primitives::{NullifierMap, OutputId, TreeBoundaries, WalletBlock},
     witness::WitnessData,
 };
 
@@ -129,10 +129,12 @@ where
             block.prev_hash(),
             block.time,
             block.vtx.iter().map(|tx| tx.txid()).collect(),
-            sapling_initial_tree_size,
-            orchard_initial_tree_size,
-            sapling_final_tree_size,
-            orchard_final_tree_size,
+            TreeBoundaries {
+                sapling_initial_tree_size,
+                sapling_final_tree_size,
+                orchard_initial_tree_size,
+                orchard_final_tree_size,
+            },
         );
 
         check_tree_size(block, &wallet_block).unwrap();
@@ -210,10 +212,14 @@ fn check_continuity(
 
 fn check_tree_size(compact_block: &CompactBlock, wallet_block: &WalletBlock) -> Result<(), ()> {
     if let Some(chain_metadata) = &compact_block.chain_metadata {
-        if chain_metadata.sapling_commitment_tree_size != wallet_block.sapling_final_tree_size() {
+        if chain_metadata.sapling_commitment_tree_size
+            != wallet_block.tree_boundaries().sapling_final_tree_size
+        {
             panic!("sapling tree size is incorrect!")
         }
-        if chain_metadata.orchard_commitment_tree_size != wallet_block.orchard_final_tree_size() {
+        if chain_metadata.orchard_commitment_tree_size
+            != wallet_block.tree_boundaries().orchard_final_tree_size
+        {
             panic!("orchard tree size is incorrect!")
         }
     }
diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs
index 2d1442885..f90c00994 100644
--- a/zingo-sync/src/sync.rs
+++ b/zingo-sync/src/sync.rs
@@ -8,7 +8,7 @@ use std::time::Duration;
 use crate::client::{self, FetchRequest};
 use crate::error::SyncError;
 use crate::keys::transparent::TransparentAddressId;
-use crate::primitives::{NullifierMap, OutPointMap, SyncState, SyncStatus};
+use crate::primitives::{NullifierMap, OutPointMap, SyncStatus};
 use crate::scan::error::{ContinuityError, ScanError};
 use crate::scan::task::Scanner;
 use crate::scan::transactions::scan_transaction;
@@ -196,9 +196,10 @@ where
 /// Designed to be called during the sync process with minimal interruption.
 pub async fn sync_status<W>(wallet: Arc<Mutex<W>>) -> SyncStatus
 where
-    W: SyncWallet,
+    W: SyncWallet + SyncBlocks,
 {
-    let sync_state: SyncState = wallet.lock().await.get_sync_state().unwrap().clone();
+    let wallet_guard = wallet.lock().await;
+    let sync_state = wallet_guard.get_sync_state().unwrap().clone();
 
     let unscanned_blocks = sync_state
         .scan_ranges()
@@ -209,15 +210,39 @@ where
             acc + (block_range.end - block_range.start)
         });
     let scanned_blocks = sync_state.initial_sync_state().total_blocks_to_scan() - unscanned_blocks;
-    let percentage_blocks_complete = (scanned_blocks as f32
+    let percentage_blocks_scanned = (scanned_blocks as f32
         / sync_state.initial_sync_state().total_blocks_to_scan() as f32)
         * 100.0;
 
+    let (unscanned_sapling_outputs, unscanned_orchard_outputs) =
+        state::calculate_unscanned_outputs(&*wallet_guard);
+    let scanned_sapling_outputs = sync_state
+        .initial_sync_state()
+        .total_sapling_outputs_to_scan()
+        - unscanned_sapling_outputs;
+    let scanned_orchard_outputs = sync_state
+        .initial_sync_state()
+        .total_orchard_outputs_to_scan()
+        - unscanned_orchard_outputs;
+    let percentage_outputs_scanned = ((scanned_sapling_outputs + scanned_orchard_outputs) as f32
+        / (sync_state
+            .initial_sync_state()
+            .total_sapling_outputs_to_scan()
+            + sync_state
+                .initial_sync_state()
+                .total_orchard_outputs_to_scan()) as f32)
+        * 100.0;
+
     SyncStatus {
         scan_ranges: sync_state.scan_ranges().clone(),
-        unscanned_blocks,
         scanned_blocks,
-        percentage_blocks_complete,
+        unscanned_blocks,
+        percentage_blocks_scanned,
+        scanned_sapling_outputs,
+        unscanned_sapling_outputs,
+        scanned_orchard_outputs,
+        unscanned_orchard_outputs,
+        percentage_outputs_scanned,
     }
 }
 
diff --git a/zingo-sync/src/sync/state.rs b/zingo-sync/src/sync/state.rs
index 93385b95f..d78ec8e3a 100644
--- a/zingo-sync/src/sync/state.rs
+++ b/zingo-sync/src/sync/state.rs
@@ -12,7 +12,7 @@ use zcash_primitives::{
 use crate::{
     client::{self, FetchRequest},
     keys::transparent::TransparentAddressId,
-    primitives::{Locator, SyncState},
+    primitives::{Locator, SyncState, TreeBoundaries},
     scan::task::ScanTask,
     traits::{SyncBlocks, SyncWallet},
 };
@@ -439,89 +439,7 @@ pub(super) async fn set_initial_state<W>(
 ) where
     W: SyncWallet + SyncBlocks,
 {
-    struct ScannedRangeTreeBoundaries {
-        sapling_initial_tree_size: u32,
-        orchard_initial_tree_size: u32,
-        sapling_final_tree_size: u32,
-        orchard_final_tree_size: u32,
-    }
-
-    /// Gets `block_height` final tree sizes from wallet block if it exists, otherwise from frontiers fetched from server.
-    ///
-    /// Only used in context of setting initial sync state as can also use the compact blocks to calculate tree sizes
-    /// during scanning.
-    async fn final_tree_sizes<W>(
-        consensus_parameters: &impl consensus::Parameters,
-        fetch_request_sender: mpsc::UnboundedSender<FetchRequest>,
-        wallet: &mut W,
-        block_height: BlockHeight,
-    ) -> (u32, u32)
-    where
-        W: SyncWallet + SyncBlocks,
-    {
-        if let Ok(block) = wallet.get_wallet_block(block_height) {
-            (
-                block.sapling_final_tree_size(),
-                block.orchard_final_tree_size(),
-            )
-        } else {
-            // TODO: move this whole block into `client::get_frontiers`
-            let sapling_activation_height = consensus_parameters
-                .activation_height(NetworkUpgrade::Sapling)
-                .expect("should have some sapling activation height");
-
-            match block_height.cmp(&sapling_activation_height) {
-                cmp::Ordering::Greater => {
-                    let frontiers =
-                        client::get_frontiers(fetch_request_sender.clone(), block_height)
-                            .await
-                            .unwrap();
-                    (
-                        frontiers
-                            .final_sapling_tree()
-                            .tree_size()
-                            .try_into()
-                            .expect("should not be more than 2^32 note commitments in the tree!"),
-                        frontiers
-                            .final_orchard_tree()
-                            .tree_size()
-                            .try_into()
-                            .expect("should not be more than 2^32 note commitments in the tree!"),
-                    )
-                }
-                cmp::Ordering::Equal => (0, 0),
-                cmp::Ordering::Less => panic!("pre-sapling not supported!"),
-            }
-        }
-    }
-
-    /// Gets the initial and final tree sizes of a `scanned_range`.
-    ///
-    /// Panics if `scanned_range` boundary wallet blocks are not found in the wallet.
-    fn scanned_range_tree_boundaries<W>(
-        wallet: &mut W,
-        scanned_range: Range<BlockHeight>,
-    ) -> ScannedRangeTreeBoundaries
-    where
-        W: SyncWallet + SyncBlocks,
-    {
-        let start_block = wallet
-            .get_wallet_block(scanned_range.start)
-            .expect("scanned range boundary blocks should be retained in the wallet");
-        let end_block = wallet
-            .get_wallet_block(scanned_range.end - 1)
-            .expect("scanned range boundary blocks should be retained in the wallet");
-
-        ScannedRangeTreeBoundaries {
-            sapling_initial_tree_size: start_block.sapling_initial_tree_size(),
-            orchard_initial_tree_size: start_block.orchard_initial_tree_size(),
-            sapling_final_tree_size: end_block.sapling_final_tree_size(),
-            orchard_final_tree_size: end_block.orchard_final_tree_size(),
-        }
-    }
-
     let fully_scanned_height = wallet.get_sync_state().unwrap().fully_scanned_height();
-
     let (sync_start_sapling_tree_size, sync_start_orchard_tree_size) = final_tree_sizes(
         consensus_parameters,
         fetch_request_sender.clone(),
@@ -537,7 +455,22 @@ pub(super) async fn set_initial_state<W>(
     )
     .await;
 
-    let sync_state = wallet.get_sync_state().unwrap();
+    let initial_sync_state = wallet
+        .get_sync_state_mut()
+        .unwrap()
+        .initial_sync_state_mut();
+    initial_sync_state.set_sync_start_height(fully_scanned_height + 1);
+    initial_sync_state.set_sync_tree_boundaries(TreeBoundaries {
+        sapling_initial_tree_size: sync_start_sapling_tree_size,
+        sapling_final_tree_size: chain_tip_sapling_tree_size,
+        orchard_initial_tree_size: sync_start_orchard_tree_size,
+        orchard_final_tree_size: chain_tip_orchard_tree_size,
+    });
+
+    let (total_sapling_outputs_to_scan, total_orchard_outputs_to_scan) =
+        calculate_unscanned_outputs(wallet);
+
+    let sync_state = wallet.get_sync_state_mut().unwrap();
     let total_blocks_to_scan = sync_state
         .scan_ranges()
         .iter()
@@ -547,37 +480,125 @@ pub(super) async fn set_initial_state<W>(
             acc + (block_range.end - block_range.start)
         });
 
-    let scanned_block_ranges = sync_state
+    let initial_sync_state = sync_state.initial_sync_state_mut();
+    initial_sync_state.set_total_blocks_to_scan(total_blocks_to_scan);
+    initial_sync_state.set_total_sapling_outputs_to_scan(total_sapling_outputs_to_scan);
+    initial_sync_state.set_total_orchard_outputs_to_scan(total_orchard_outputs_to_scan);
+}
+
+pub(super) fn calculate_unscanned_outputs<W>(wallet: &W) -> (u32, u32)
+where
+    W: SyncWallet + SyncBlocks,
+{
+    let sync_state = wallet.get_sync_state().unwrap();
+    let sync_start_height = sync_state.initial_sync_state().sync_start_height();
+
+    let nonlinear_scanned_block_ranges = sync_state
         .scan_ranges()
         .iter()
         .filter(|scan_range| {
             scan_range.priority() == ScanPriority::Scanned
-                && scan_range.block_range().start > fully_scanned_height
+                && scan_range.block_range().start >= sync_start_height
         })
         .map(|scan_range| scan_range.block_range().clone())
         .collect::<Vec<_>>();
-    let (scanned_sapling_outputs, scanned_orchard_outputs) = scanned_block_ranges
-        .iter()
-        .map(|block_range| scanned_range_tree_boundaries(wallet, block_range.clone()))
-        .fold((0, 0), |acc, tree_sizes| {
-            (
-                acc.0 + (tree_sizes.sapling_final_tree_size - tree_sizes.sapling_initial_tree_size),
-                acc.1 + (tree_sizes.orchard_final_tree_size - tree_sizes.orchard_initial_tree_size),
-            )
-        });
-    let total_sapling_outputs_to_scan =
-        chain_tip_sapling_tree_size - sync_start_sapling_tree_size - scanned_sapling_outputs;
-    let total_orchard_outputs_to_scan =
-        chain_tip_orchard_tree_size - sync_start_orchard_tree_size - scanned_orchard_outputs;
+    let (nonlinear_scanned_sapling_outputs, nonlinear_scanned_orchard_outputs) =
+        nonlinear_scanned_block_ranges
+            .iter()
+            .map(|block_range| scanned_range_tree_boundaries(wallet, block_range.clone()))
+            .fold((0, 0), |acc, tree_boundaries| {
+                (
+                    acc.0
+                        + (tree_boundaries.sapling_final_tree_size
+                            - tree_boundaries.sapling_initial_tree_size),
+                    acc.1
+                        + (tree_boundaries.orchard_final_tree_size
+                            - tree_boundaries.orchard_initial_tree_size),
+                )
+            });
+
+    let initial_sync_state = wallet.get_sync_state().unwrap().initial_sync_state();
+    let unscanned_sapling_outputs = initial_sync_state
+        .sync_tree_boundaries()
+        .sapling_final_tree_size
+        - initial_sync_state
+            .sync_tree_boundaries()
+            .sapling_initial_tree_size
+        - nonlinear_scanned_sapling_outputs;
+    let unscanned_orchard_outputs = initial_sync_state
+        .sync_tree_boundaries()
+        .orchard_final_tree_size
+        - initial_sync_state
+            .sync_tree_boundaries()
+            .orchard_initial_tree_size
+        - nonlinear_scanned_orchard_outputs;
+
+    (unscanned_sapling_outputs, unscanned_orchard_outputs)
+}
 
-    let initial_sync_state = wallet
-        .get_sync_state_mut()
-        .unwrap()
-        .initial_sync_state_mut();
-    initial_sync_state.set_sync_start_height(fully_scanned_height + 1);
-    initial_sync_state.set_sync_start_sapling_tree_size(sync_start_sapling_tree_size);
-    initial_sync_state.set_sync_start_orchard_tree_size(sync_start_orchard_tree_size);
-    initial_sync_state.set_total_blocks_to_scan(total_blocks_to_scan);
-    initial_sync_state.set_total_sapling_outputs_to_scan(total_sapling_outputs_to_scan);
-    initial_sync_state.set_total_orchard_outputs_to_scan(total_orchard_outputs_to_scan);
+/// Gets `block_height` final tree sizes from wallet block if it exists, otherwise from frontiers fetched from server.
+async fn final_tree_sizes<W>(
+    consensus_parameters: &impl consensus::Parameters,
+    fetch_request_sender: mpsc::UnboundedSender<FetchRequest>,
+    wallet: &mut W,
+    block_height: BlockHeight,
+) -> (u32, u32)
+where
+    W: SyncBlocks,
+{
+    if let Ok(block) = wallet.get_wallet_block(block_height) {
+        (
+            block.tree_boundaries().sapling_final_tree_size,
+            block.tree_boundaries().orchard_final_tree_size,
+        )
+    } else {
+        // TODO: move this whole block into `client::get_frontiers`
+        let sapling_activation_height = consensus_parameters
+            .activation_height(NetworkUpgrade::Sapling)
+            .expect("should have some sapling activation height");
+
+        match block_height.cmp(&sapling_activation_height) {
+            cmp::Ordering::Greater => {
+                let frontiers = client::get_frontiers(fetch_request_sender.clone(), block_height)
+                    .await
+                    .unwrap();
+                (
+                    frontiers
+                        .final_sapling_tree()
+                        .tree_size()
+                        .try_into()
+                        .expect("should not be more than 2^32 note commitments in the tree!"),
+                    frontiers
+                        .final_orchard_tree()
+                        .tree_size()
+                        .try_into()
+                        .expect("should not be more than 2^32 note commitments in the tree!"),
+                )
+            }
+            cmp::Ordering::Equal => (0, 0),
+            cmp::Ordering::Less => panic!("pre-sapling not supported!"),
+        }
+    }
+}
+
+/// Gets the initial and final tree sizes of a `scanned_range`.
+///
+/// Panics if `scanned_range` boundary wallet blocks are not found in the wallet.
+fn scanned_range_tree_boundaries<W>(wallet: &W, scanned_range: Range<BlockHeight>) -> TreeBoundaries
+where
+    W: SyncBlocks,
+{
+    let start_block = wallet
+        .get_wallet_block(scanned_range.start)
+        .expect("scanned range boundary blocks should be retained in the wallet");
+    let end_block = wallet
+        .get_wallet_block(scanned_range.end - 1)
+        .expect("scanned range boundary blocks should be retained in the wallet");
+
+    TreeBoundaries {
+        sapling_initial_tree_size: start_block.tree_boundaries().sapling_initial_tree_size,
+        sapling_final_tree_size: end_block.tree_boundaries().sapling_final_tree_size,
+        orchard_initial_tree_size: start_block.tree_boundaries().orchard_initial_tree_size,
+        orchard_final_tree_size: end_block.tree_boundaries().orchard_final_tree_size,
+    }
 }

From a18ec666d2fe1c09b50283e0ea75db3bc8009a07 Mon Sep 17 00:00:00 2001
From: Oscar Pepper <oscarpepper@zingolabs.org>
Date: Thu, 16 Jan 2025 05:36:35 +0000
Subject: [PATCH 8/9] format

---
 zingo-sync/src/primitives.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/zingo-sync/src/primitives.rs b/zingo-sync/src/primitives.rs
index db00bd67a..4f584f967 100644
--- a/zingo-sync/src/primitives.rs
+++ b/zingo-sync/src/primitives.rs
@@ -66,6 +66,7 @@ impl Default for InitialSyncState {
         Self::new()
     }
 }
+
 /// Encapsulates the current state of sync
 #[derive(Debug, Clone, Getters, MutGetters, CopyGetters, Setters)]
 #[getset(get = "pub", get_mut = "pub")]

From 45c95e6366e485a89b462176df0959c4c6d895e7 Mon Sep 17 00:00:00 2001
From: Oscar Pepper <oscarpepper@zingolabs.org>
Date: Thu, 16 Jan 2025 08:52:38 +0000
Subject: [PATCH 9/9] fix overflow bugs

---
 zingo-sync/src/sync.rs       | 9 ++++++---
 zingo-sync/src/sync/state.rs | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs
index f90c00994..69f5f4ada 100644
--- a/zingo-sync/src/sync.rs
+++ b/zingo-sync/src/sync.rs
@@ -209,7 +209,10 @@ where
         .fold(0, |acc, block_range| {
             acc + (block_range.end - block_range.start)
         });
-    let scanned_blocks = sync_state.initial_sync_state().total_blocks_to_scan() - unscanned_blocks;
+    let scanned_blocks = sync_state
+        .initial_sync_state()
+        .total_blocks_to_scan()
+        .saturating_sub(unscanned_blocks);
     let percentage_blocks_scanned = (scanned_blocks as f32
         / sync_state.initial_sync_state().total_blocks_to_scan() as f32)
         * 100.0;
@@ -219,11 +222,11 @@ where
     let scanned_sapling_outputs = sync_state
         .initial_sync_state()
         .total_sapling_outputs_to_scan()
-        - unscanned_sapling_outputs;
+        .saturating_sub(unscanned_sapling_outputs);
     let scanned_orchard_outputs = sync_state
         .initial_sync_state()
         .total_orchard_outputs_to_scan()
-        - unscanned_orchard_outputs;
+        .saturating_sub(unscanned_orchard_outputs);
     let percentage_outputs_scanned = ((scanned_sapling_outputs + scanned_orchard_outputs) as f32
         / (sync_state
             .initial_sync_state()
diff --git a/zingo-sync/src/sync/state.rs b/zingo-sync/src/sync/state.rs
index d78ec8e3a..783c0e4e1 100644
--- a/zingo-sync/src/sync/state.rs
+++ b/zingo-sync/src/sync/state.rs
@@ -557,7 +557,7 @@ where
             .activation_height(NetworkUpgrade::Sapling)
             .expect("should have some sapling activation height");
 
-        match block_height.cmp(&sapling_activation_height) {
+        match block_height.cmp(&(sapling_activation_height - 1)) {
             cmp::Ordering::Greater => {
                 let frontiers = client::get_frontiers(fetch_request_sender.clone(), block_height)
                     .await