Enhance tx inputs processing

.github/workflows/ci.yaml

@@ -318,7 +318,7 @@ jobs:
           path: wasm/release/spectre-wasm32-sdk-${{ env.SHORT_SHA }}.zip
 
   build-release:
-    name: Build Ubuntu Release
+    name: Build Linux Release
     runs-on: ubuntu-latest
     steps:
       - name: Checkout sources

cli/src/cli.rs

@@ -578,6 +578,16 @@ impl SpectreCli {
             }
         }
 
+        let mut watch_accounts = Vec::<(usize, Arc<dyn Account>)>::new();
+        let mut unfiltered_accounts = self.wallet.accounts(None, &guard).await?;
+
+        while let Some(account) = unfiltered_accounts.try_next().await? {
+            if account.feature().is_some() {
+                watch_accounts.push((flat_list.len(), account.clone()));
+                flat_list.push(account.clone());
+            }
+        }
+
         if flat_list.is_empty() {
             return Err(Error::NoAccounts);
         } else if autoselect && flat_list.len() == 1 {
@@ -607,6 +617,16 @@ impl SpectreCli {
                 tprintln!(self, "    {seq}: {ls_string}");
             });
 
+            if !watch_accounts.is_empty() {
+                tprintln!(self, "• watch-only");
+            }
+
+            watch_accounts.iter().for_each(|(seq, account)| {
+                let seq = style(seq.to_string()).cyan();
+                let ls_string = account.get_list_string().unwrap_or_else(|err| panic!("{err}"));
+                tprintln!(self, "    {seq}: {ls_string}");
+            });
+
             tprintln!(self);
 
             let range = if flat_list.len() > 1 { format!("[{}..{}] ", 0, flat_list.len() - 1) } else { "".to_string() };

cli/src/modules/guide.txt

@@ -1,49 +1,55 @@
-Please note - this is an alpha version of the software; not all features are currently functional.
+Before you start, you must configure the default network setting.  There are currently
+3 networks available.  `mainnet`, `testnet-10` and `testnet-11`.  If you wish to experiment,
+you should select `testnet-11` by entering `network testnet-11`
 
-For desktop or web versions of this software, you can use Ctrl+'+' or Ctrl+'-' (Command on macOS) to change the terminal font size.
+The `server` command configures the target server.  You can connect to any Rusty Spectre
+node that has wRPC enabled with `--rpclisten-borsh=0.0.0.0`. If the server setting 
+is set to 'public' the node will connect to the public node infrastructure.
 
-If using a desktop version, you can use Ctrl+M (Command on macOS) to bring up metrics.
-
-Type `help` to see the complete list of commands. Type `exit` to exit this application. On Windows, you can use `Alt+F4`, and on macOS `Command+Q` to exit.
+Both network and server values are stored in the application settings and are 
+used when running a local node or connecting to a remote node.
 
 ---
 
-Before you start, you must configure the default network settings. There is only one network currently available: `mainnet`.
-
-The `server` command configures the target server. You can connect to any Rusty Spectre node that has wRPC enabled with `--rpclisten-borsh=0.0.0.0`. If the server setting is set to 'public', the node will connect to the public node infrastructure.
+`wallet create [<name>]` Use this command to create a local wallet. The <name> argument 
+is optional (the default wallet name is "spectre") and allows you to create multiple 
+named wallets.  Only one wallet can be opened at a time. Keep in mind that a wallet can have multiple
+accounts, as such you only need one wallet, unless, for example, you want to separate wallets for 
+personal and business needs (but you can also create isolated accounts within a wallet).
 
-Both network and server values are stored in the application settings and are used when running a local node or connecting to a remote node.
-
----
+Make sure to record your mnemonic, even if working with a testnet, not to lose your
+testnet SPR.
 
-`wallet create [<name>]` - Use this command to create a local wallet. The <name> argument is optional (the default wallet name is "spectre") and allows you to create multiple named wallets. Only one wallet can be opened at a time. Keep in mind that a wallet can have multiple accounts, so you only need one wallet unless, for example, you want to separate wallets for personal and business needs (but you can also create isolated accounts within a wallet).
-
-Make sure to record your mnemonic, even if working with a testnet, to avoid losing your testnet SPR.
-
-`open <name>` - Opens the wallet (the wallet is open automatically after creation).
+`open <name>` - opens the wallet (the wallet is open automatically after creation).
 
 `list` - Lists all wallet accounts and their balances.
 
-`select <account-name>` - Selects an active account. The <account-name> can be the first few letters of the name or ID of the account.
+`select <account-name>` - Selects an active account. The <account-name> can be the first few letters of the name or id of the account.
 
 `account create bip32 [<name>]` - Allows you to create additional HD wallet accounts linked to the default private key of your wallet.
 
-`address` - Shows your selected account address. Note - you can click on the address to copy it to the clipboard. (When on mainnet, Ctrl+Click on addresses, transactions, and block hashes will open a new browser window with an explorer.)
+`address` - shows your selected account address
 
-Before you transact: The `mute` option (enabled by default) toggles mute on/off. Mute enables terminal output of internal framework events. Rust and JavaScript/TypeScript applications integrating with this platform are meant to update their state by monitoring event notifications. Mute allows you to see these events in the terminal. When mute is off, all events are displayed in the terminal. When mute is on, you can use the 'track' command to enable specific event notifications.
+Before you transact: `mute` option (enabled by default) toggles mute on/off. Mute enables terminal
+output of internal framework events.  Rust and JavaScript/TypeScript applications integrating with this platform 
+are meant to update their state by monitoring event notifications. Mute allows you to see these events in
+the terminal.  When mute is off, all events are displayed in the terminal.  When mute is on, you can use 'track'
+command to enable specific event notification.
 
-`transfer <account-name> <amount>` - Transfers from the active account to a different account. For example, `transfer p 1` will transfer 1 SPR from the selected account to an account named 'pete' (starting with the letter 'p').
+`transfer <account-name> <amount>` - Transfers from the active to a different account. For example 'transfer p 1' will transfer 1 SPR from
+the selected account to an account named 'pete' (starts with a 'p' letter)
 
-`send <address> <amount>` - Sends funds to a destination address.
+`send <address> <amount>` - Send funds to a destination address .
 
 `estimate <amount>` - Provides a fee and UTXO consumption estimate for a transaction of a given amount.
 
 `sweep` - Sweeps account UTXOs to reduce the UTXO size.
 
 `history list` - Shows previous account transactions.
 
-`history details` - Shows previous account transactions with extended information.
+`history details` - Show previous account transactions with extended information.
 
 `monitor` - A test screen environment that periodically updates account balances.
 
-`rpc` - Allows you to execute RPC methods against the node (not all methods are currently available).
+`rpc` - Allows you to execute RPC methods against the node (not all methods are currently available)
+

cli/src/wizards/account.rs

@@ -144,3 +144,62 @@ pub(crate) async fn multisig_watch(ctx: &Arc<SpectreCli>, name: Option<&str>) ->
     wallet.select(Some(&account)).await?;
     Ok(())
 }
+
+pub(crate) async fn bip32_watch(ctx: &Arc<SpectreCli>, name: Option<&str>) -> Result<()> {
+    let term = ctx.term();
+    let wallet = ctx.wallet();
+
+    let name = if let Some(name) = name {
+        Some(name.to_string())
+    } else {
+        Some(term.ask(false, "Please enter account name (optional, press <enter> to skip): ").await?.trim().to_string())
+    };
+
+    let mut xpub_keys = Vec::with_capacity(1);
+    let xpub_key = term.ask(false, "Enter extended public key: ").await?;
+    xpub_keys.push(xpub_key.trim().to_owned());
+
+    let wallet_secret = Secret::new(term.ask(true, "Enter wallet password: ").await?.trim().as_bytes().to_vec());
+    if wallet_secret.as_ref().is_empty() {
+        return Err(Error::WalletSecretRequired);
+    }
+
+    let account_create_args_bip32_watch = AccountCreateArgsBip32Watch::new(name, xpub_keys);
+    let account = wallet.create_account_bip32_watch(&wallet_secret, account_create_args_bip32_watch).await?;
+
+    tprintln!(ctx, "\naccount created: {}\n", account.get_list_string()?);
+    wallet.select(Some(&account)).await?;
+    Ok(())
+}
+
+pub(crate) async fn multisig_watch(ctx: &Arc<SpectreCli>, name: Option<&str>) -> Result<()> {
+    let term = ctx.term();
+
+    let account_name = if let Some(name) = name {
+        Some(name.to_string())
+    } else {
+        Some(term.ask(false, "Please enter account name (optional, press <enter> to skip): ").await?.trim().to_string())
+    };
+
+    let term = ctx.term();
+    let wallet = ctx.wallet();
+    let (wallet_secret, _) = ctx.ask_wallet_secret(None).await?;
+    let minimum_signatures: u16 = term.ask(false, "Enter the minimum number of signatures required: ").await?.parse()?;
+
+    let prv_key_data_args = Vec::with_capacity(0);
+
+    let answer = term.ask(false, "Enter the number of extended public keys: ").await?.trim().to_string(); //.parse()?;
+    let xpub_keys_len: usize = if answer.is_empty() { 0 } else { answer.parse()? };
+
+    let mut xpub_keys = Vec::with_capacity(xpub_keys_len);
+    for i in 1..=xpub_keys_len {
+        let xpub_key = term.ask(false, &format!("Enter extended public {i} key: ")).await?;
+        xpub_keys.push(xpub_key.trim().to_owned());
+    }
+    let account =
+        wallet.create_account_multisig(&wallet_secret, prv_key_data_args, xpub_keys, account_name, minimum_signatures).await?;
+
+    tprintln!(ctx, "\naccount created: {}\n", account.get_list_string()?);
+    wallet.select(Some(&account)).await?;
+    Ok(())
+}

components/addressmanager/Cargo.toml

@@ -27,5 +27,4 @@ thiserror.workspace = true
 tokio.workspace = true
 
 [dev-dependencies]
-statrs.workspace = true
-statest.workspace = true
+rv.workspace = true

components/addressmanager/src/lib.rs

@@ -520,8 +520,7 @@ mod address_store_with_cache {
         use spectre_database::create_temp_db;
         use spectre_database::prelude::ConnBuilder;
         use spectre_utils::networking::IpAddress;
-        use statest::ks::KSTest;
-        use statrs::distribution::Uniform;
+        use rv::{dist::Uniform, misc::ks_test as one_way_ks_test, traits::Cdf};
         use std::net::{IpAddr, Ipv6Addr};
 
         #[test]
@@ -591,10 +590,11 @@ mod address_store_with_cache {
             assert!(num_of_buckets >= 12);
 
             // Run multiple Kolmogorov–Smirnov tests to offset random noise of the random weighted iterator
-            let num_of_trials = 512;
+            let num_of_trials = 2048; // Number of trials to run the test, chosen to reduce random noise.
             let mut cul_p = 0.;
             // The target uniform distribution
-            let target_u_dist = Uniform::new(0.0, (num_of_buckets) as f64).unwrap();
+            let target_uniform_dist = Uniform::new(1.0, num_of_buckets as f64).unwrap();
+            let uniform_cdf = |x: f64| target_uniform_dist.cdf(&x);
             for _ in 0..num_of_trials {
                 // The weight sampled expected uniform distribution
                 let prioritized_address_distribution = am
@@ -603,13 +603,12 @@ mod address_store_with_cache {
                     .take(num_of_buckets)
                     .map(|addr| addr.prefix_bucket().as_u64() as f64)
                     .collect_vec();
-
-                let ks_test = KSTest::new(prioritized_address_distribution.as_slice());
-                cul_p += ks_test.ks1(&target_u_dist).0;
+                cul_p += one_way_ks_test(prioritized_address_distribution.as_slice(), uniform_cdf).1;
             }
 
             // Normalize and adjust p to test for uniformity, over average of all trials.
-            let adjusted_p = (0.5 - cul_p / num_of_trials as f64).abs();
+            // we do this to reduce the effect of random noise failing this test.
+            let adjusted_p = ((cul_p / num_of_trials as f64) - 0.5).abs();
             // Define the significance threshold.
             let significance = 0.10;
 
@@ -619,7 +618,7 @@ mod address_store_with_cache {
                 adjusted_p,
                 significance
             );
-            assert!(adjusted_p <= significance)
+            assert!(adjusted_p <= significance);
         }
     }
 }

consensus/Cargo.toml

@@ -54,11 +54,16 @@ serde_json.workspace = true
 flate2.workspace = true
 rand_distr.workspace = true
 spectre-txscript-errors.workspace = true
+spectre-addresses.workspace = true
 
 [[bench]]
 name = "hash_benchmarks"
 harness = false
 
+[[bench]]
+name = "check_scripts"
+harness = false
+
 [features]
 html_reports = []
 devnet-prealloc = ["spectre-consensus-core/devnet-prealloc"]

consensus/benches/check_scripts.rs

@@ -0,0 +1,126 @@
+use criterion::{black_box, criterion_group, criterion_main, Criterion, SamplingMode};
+use spectre_addresses::{Address, Prefix, Version};
+use spectre_consensus::processes::transaction_validator::transaction_validator_populated::{
+    check_scripts_par_iter, check_scripts_par_iter_pool, check_scripts_sequential,
+};
+use spectre_consensus_core::hashing::sighash::{calc_schnorr_signature_hash, SigHashReusedValuesUnsync};
+use spectre_consensus_core::hashing::sighash_type::SIG_HASH_ALL;
+use spectre_consensus_core::subnets::SubnetworkId;
+use spectre_consensus_core::tx::{MutableTransaction, Transaction, TransactionInput, TransactionOutpoint, UtxoEntry};
+use spectre_txscript::caches::Cache;
+use spectre_txscript::pay_to_address_script;
+use rand::{thread_rng, Rng};
+use secp256k1::Keypair;
+use std::thread::available_parallelism;
+
+// You may need to add more detailed mocks depending on your actual code.
+fn mock_tx(inputs_count: usize, non_uniq_signatures: usize) -> (Transaction, Vec<UtxoEntry>) {
+    let reused_values = SigHashReusedValuesUnsync::new();
+    let dummy_prev_out = TransactionOutpoint::new(spectre_hashes::Hash::from_u64_word(1), 1);
+    let mut tx = Transaction::new(
+        0,
+        vec![],
+        vec![],
+        0,
+        SubnetworkId::from_bytes([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),
+        0,
+        vec![],
+    );
+    let mut utxos = vec![];
+    let mut kps = vec![];
+    for _ in 0..inputs_count - non_uniq_signatures {
+        let kp = Keypair::new(secp256k1::SECP256K1, &mut thread_rng());
+        tx.inputs.push(TransactionInput { previous_outpoint: dummy_prev_out, signature_script: vec![], sequence: 0, sig_op_count: 1 });
+        let address = Address::new(Prefix::Mainnet, Version::PubKey, &kp.x_only_public_key().0.serialize());
+        utxos.push(UtxoEntry {
+            amount: thread_rng().gen::<u32>() as u64,
+            script_public_key: pay_to_address_script(&address),
+            block_daa_score: 333,
+            is_coinbase: false,
+        });
+        kps.push(kp);
+    }
+    for _ in 0..non_uniq_signatures {
+        let kp = kps.last().unwrap();
+        tx.inputs.push(TransactionInput { previous_outpoint: dummy_prev_out, signature_script: vec![], sequence: 0, sig_op_count: 1 });
+        let address = Address::new(Prefix::Mainnet, Version::PubKey, &kp.x_only_public_key().0.serialize());
+        utxos.push(UtxoEntry {
+            amount: thread_rng().gen::<u32>() as u64,
+            script_public_key: pay_to_address_script(&address),
+            block_daa_score: 444,
+            is_coinbase: false,
+        });
+    }
+    for (i, kp) in kps.iter().enumerate().take(inputs_count - non_uniq_signatures) {
+        let mut_tx = MutableTransaction::with_entries(&tx, utxos.clone());
+        let sig_hash = calc_schnorr_signature_hash(&mut_tx.as_verifiable(), i, SIG_HASH_ALL, &reused_values);
+        let msg = secp256k1::Message::from_digest_slice(sig_hash.as_bytes().as_slice()).unwrap();
+        let sig: [u8; 64] = *kp.sign_schnorr(msg).as_ref();
+        // This represents OP_DATA_65 <SIGNATURE+SIGHASH_TYPE> (since signature length is 64 bytes and SIGHASH_TYPE is one byte)
+        tx.inputs[i].signature_script = std::iter::once(65u8).chain(sig).chain([SIG_HASH_ALL.to_u8()]).collect();
+    }
+    let length = tx.inputs.len();
+    for i in (inputs_count - non_uniq_signatures)..length {
+        let kp = kps.last().unwrap();
+        let mut_tx = MutableTransaction::with_entries(&tx, utxos.clone());
+        let sig_hash = calc_schnorr_signature_hash(&mut_tx.as_verifiable(), i, SIG_HASH_ALL, &reused_values);
+        let msg = secp256k1::Message::from_digest_slice(sig_hash.as_bytes().as_slice()).unwrap();
+        let sig: [u8; 64] = *kp.sign_schnorr(msg).as_ref();
+        // This represents OP_DATA_65 <SIGNATURE+SIGHASH_TYPE> (since signature length is 64 bytes and SIGHASH_TYPE is one byte)
+        tx.inputs[i].signature_script = std::iter::once(65u8).chain(sig).chain([SIG_HASH_ALL.to_u8()]).collect();
+    }
+    (tx, utxos)
+}
+
+fn benchmark_check_scripts(c: &mut Criterion) {
+    for inputs_count in [100, 50, 25, 10, 5, 2] {
+        for non_uniq_signatures in [0, inputs_count / 2] {
+            let (tx, utxos) = mock_tx(inputs_count, non_uniq_signatures);
+            let mut group = c.benchmark_group(format!("inputs: {inputs_count}, non uniq: {non_uniq_signatures}"));
+            group.sampling_mode(SamplingMode::Flat);
+
+            group.bench_function("single_thread", |b| {
+                let tx = MutableTransaction::with_entries(&tx, utxos.clone());
+                let cache = Cache::new(inputs_count as u64);
+                b.iter(|| {
+                    cache.clear();
+                    check_scripts_sequential(black_box(&cache), black_box(&tx.as_verifiable())).unwrap();
+                })
+            });
+
+            group.bench_function("rayon par iter", |b| {
+                let tx = MutableTransaction::with_entries(tx.clone(), utxos.clone());
+                let cache = Cache::new(inputs_count as u64);
+                b.iter(|| {
+                    cache.clear();
+                    check_scripts_par_iter(black_box(&cache), black_box(&tx.as_verifiable())).unwrap();
+                })
+            });
+
+            // Iterate powers of two up to available parallelism
+            for i in (1..=(available_parallelism().unwrap().get() as f64).log2().ceil() as u32).map(|x| 2u32.pow(x) as usize) {
+                if inputs_count >= i {
+                    group.bench_function(format!("rayon, custom thread pool, thread count {i}"), |b| {
+                        let tx = MutableTransaction::with_entries(tx.clone(), utxos.clone());
+                        // Create a custom thread pool with the specified number of threads
+                        let pool = rayon::ThreadPoolBuilder::new().num_threads(i).build().unwrap();
+                        let cache = Cache::new(inputs_count as u64);
+                        b.iter(|| {
+                            cache.clear();
+                            check_scripts_par_iter_pool(black_box(&cache), black_box(&tx.as_verifiable()), black_box(&pool)).unwrap();
+                        })
+                    });
+                }
+            }
+        }
+    }
+}
+
+criterion_group! {
+    name = benches;
+    // This can be any expression that returns a `Criterion` object.
+    config = Criterion::default().with_output_color(true).measurement_time(std::time::Duration::new(20, 0));
+    targets = benchmark_check_scripts
+}
+
+criterion_main!(benches);