diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ab9f7c595f..99ba709421d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ As a minor extension, we have adopted a slightly different versioning convention - Implement the client library for the the signed entity type `CardanoDatabase` (download and prove snapshot). - Implement the client CLI commands for the signed entity type `CardanoDatabase` (snapshot list, snapshot show and download commands). + - Implement an example crate for the signed entity type `CardanoDatabase`. - Crates versions: diff --git a/Cargo.lock b/Cargo.lock index 8829a387d2b..147729a34fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -992,6 +992,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "client-cardano-database" +version = "0.0.1" +dependencies = [ + "anyhow", + "async-trait", + "clap", + "futures", + "indicatif", + "mithril-client", + "tokio", +] + [[package]] name = "client-cardano-stake-distribution" version = "0.1.9" @@ -3701,7 +3714,7 @@ dependencies = [ [[package]] name = "mithril-client" -version = "0.11.5" +version = "0.11.6" dependencies = [ "anyhow", "async-recursion", @@ -3732,7 +3745,7 @@ dependencies = [ [[package]] name = "mithril-client-cli" -version = "0.11.3" +version = "0.11.4" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index a569a123db1..0cb24601d85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ resolver = "2" members = [ "demo/protocol-demo", + "examples/client-cardano-database", "examples/client-cardano-stake-distribution", "examples/client-cardano-transaction", "examples/client-mithril-stake-distribution", diff --git a/examples/client-cardano-database/.gitignore b/examples/client-cardano-database/.gitignore new file mode 100644 index 00000000000..5ce41e82ac3 --- /dev/null +++ b/examples/client-cardano-database/.gitignore @@ -0,0 +1,3 @@ +target/ +client-cardano-database +.DS_Store diff --git a/examples/client-cardano-database/Cargo.toml b/examples/client-cardano-database/Cargo.toml new file mode 100644 index 00000000000..6588e1cfab5 --- /dev/null +++ b/examples/client-cardano-database/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "client-cardano-database" +description = "Mithril client Cardano database example" +version = "0.0.1" +authors = ["dev@iohk.io", "mithril-dev@iohk.io"] +documentation = "https://mithril.network/doc" +edition = "2021" +homepage = "https://mithril.network" +license = "Apache-2.0" +repository = "https://github.com/input-output-hk/mithril/" + +[dependencies] +anyhow = "1.0.95" +async-trait = "0.1.86" +clap = { version = "4.5.28", features = ["derive", "env"] } +futures = "0.3.31" +indicatif = "0.17.11" +mithril-client = { path = "../../mithril-client", features = ["fs", "unstable"] } +tokio = { version = "1.43.0", features = ["full"] } diff --git a/examples/client-cardano-database/README.md b/examples/client-cardano-database/README.md new file mode 100644 index 00000000000..277203c0804 --- /dev/null +++ b/examples/client-cardano-database/README.md @@ -0,0 +1,41 @@ +# Mithril client library example: Cardano database + +## Description + +This example shows how to implement a Mithril client and use its features related to the `Cardano database` type. + +In this example, the client interacts by default with a real aggregator on the network `testing-preview` to: + +- list the available snapshots +- get a single snapshot +- download and unpack snapshot archives tailored for a specific range +- verify the associated certificate and its chain +- compute a message for the retrieved artifact files +- verify that the certificate signs the computed message +- increments snapshot download statistics + +The crate [indicatif](https://docs.rs/indicatif/latest/indicatif/) is used to nicely report the progress to the console. + +## Build and run the example + +```bash +# Switch to the latest release tag +git checkout tags/$(curl -sSL https://api.github.com/repos/input-output-hk/mithril/releases/latest | jq -r '.tag_name') + +# Build from the crate directory +cargo build + +# Run from the crate directory +cargo run + +# Run with your custom network configuration +AGGREGATOR_ENDPOINT=YOUR_AGGREGATOR_ENDPOINT GENESIS_VERIFICATION_KEY=YOUR_GENESIS_VERIFICATION_KEY cargo run + +# Example with 'release-preprod' network +AGGREGATOR_ENDPOINT=https://aggregator.testing-preview.api.mithril.network/aggregator GENESIS_VERIFICATION_KEY=$(curl -s https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/testing-preview/genesis.vkey) cargo run +``` + +## Links + +- **Developer documentation**: https://docs.rs/mithril-client/latest/mithril_client/ +- **Crates.io**: https://crates.io/crates/mithril-client diff --git a/examples/client-cardano-database/src/main.rs b/examples/client-cardano-database/src/main.rs new file mode 100644 index 00000000000..1262728a6da --- /dev/null +++ b/examples/client-cardano-database/src/main.rs @@ -0,0 +1,275 @@ +//! This example shows how to implement a Mithril client and use its features. +//! +//! In this example, the client interacts by default with a real aggregator (`testing-preview`) to get the data. +//! +//! A [FeedbackReceiver] using [indicatif] is used to nicely report the progress to the console. + +use anyhow::{anyhow, Context}; +use async_trait::async_trait; +use clap::Parser; +use futures::Future; +use indicatif::{MultiProgress, ProgressBar, ProgressState, ProgressStyle}; +use mithril_client::cardano_database_client::{DownloadUnpackOptions, ImmutableFileRange}; +use std::fmt::Write; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::RwLock; + +use mithril_client::feedback::{FeedbackReceiver, MithrilEvent, MithrilEventCardanoDatabase}; +use mithril_client::{ClientBuilder, MessageBuilder, MithrilResult}; + +#[derive(Parser, Debug)] +#[command(version)] +pub struct Args { + /// Genesis verification key. + #[clap( + long, + env = "GENESIS_VERIFICATION_KEY", + default_value = "5b3132372c37332c3132342c3136312c362c3133372c3133312c3231332c3230372c3131372c3139382c38352c3137362c3139392c3136322c3234312c36382c3132332c3131392c3134352c31332c3233322c3234332c34392c3232392c322c3234392c3230352c3230352c33392c3233352c34345d" + )] + genesis_verification_key: String, + + /// Aggregator endpoint URL. + #[clap( + long, + env = "AGGREGATOR_ENDPOINT", + default_value = "https://aggregator.testing-preview.api.mithril.network/aggregator" + )] + aggregator_endpoint: String, +} + +#[tokio::main] +async fn main() -> MithrilResult<()> { + let args = Args::parse(); + let work_dir = get_temp_dir()?; + let progress_bar = indicatif::MultiProgress::new(); + let client = + ClientBuilder::aggregator(&args.aggregator_endpoint, &args.genesis_verification_key) + .add_feedback_receiver(Arc::new(IndicatifFeedbackReceiver::new(&progress_bar))) + .build()?; + + let cardano_database_snapshots = client.cardano_database().list().await?; + + let latest_hash = cardano_database_snapshots + .first() + .ok_or(anyhow!( + "No Cardano database snapshot could be listed from aggregator: '{}'", + args.aggregator_endpoint + ))? + .hash + .as_ref(); + + let cardano_database_snapshot = + client + .cardano_database() + .get(latest_hash) + .await? + .ok_or(anyhow!( + "A Cardano database should exist for hash '{latest_hash}'" + ))?; + + let unpacked_dir = work_dir.join("unpack"); + std::fs::create_dir(&unpacked_dir).unwrap(); + + let certificate = client + .certificate() + .verify_chain(&cardano_database_snapshot.certificate_hash) + .await?; + + let immutable_file_range = ImmutableFileRange::From(15000); + let download_unpack_options = DownloadUnpackOptions { + allow_override: true, + include_ancillary: false, + ..DownloadUnpackOptions::default() + }; + client + .cardano_database() + .download_unpack( + &cardano_database_snapshot, + &immutable_file_range, + &unpacked_dir, + download_unpack_options, + ) + .await?; + + println!("Computing Cardano database Merkle proof...",); + let merkle_proof = client + .cardano_database() + .compute_merkle_proof( + &certificate, + &cardano_database_snapshot, + &immutable_file_range, + &unpacked_dir, + ) + .await?; + merkle_proof + .verify() + .with_context(|| "Merkle proof verification failed")?; + + println!("Sending usage statistics to the aggregator..."); + let full_restoration = immutable_file_range == ImmutableFileRange::Full; + let include_ancillary = download_unpack_options.include_ancillary; + let number_of_immutable_files_restored = + immutable_file_range.length(cardano_database_snapshot.beacon.immutable_file_number); + if let Err(e) = client + .cardano_database() + .add_statistics( + full_restoration, + include_ancillary, + number_of_immutable_files_restored, + ) + .await + { + println!("Could not send usage statistics to the aggregator: {:?}", e); + } + + println!( + "Computing Cardano database snapshot '{}' message...", + cardano_database_snapshot.hash + ); + let message = wait_spinner( + &progress_bar, + MessageBuilder::new().compute_cardano_database_message(&certificate, &merkle_proof), + ) + .await?; + + if certificate.match_message(&message) { + println!( + "Successfully downloaded and validated Cardano database snapshot '{}'", + cardano_database_snapshot.hash + ); + + Ok(()) + } else { + Err(anyhow::anyhow!( + "Certificate and message did not match:\ncertificate_message: '{}'\n computed_message: '{}'", + certificate.signed_message, + message.compute_hash() + )) + } +} + +pub struct IndicatifFeedbackReceiver { + progress_bar: MultiProgress, + cardano_database_pb: RwLock>, + certificate_validation_pb: RwLock>, +} + +impl IndicatifFeedbackReceiver { + pub fn new(progress_bar: &MultiProgress) -> Self { + Self { + progress_bar: progress_bar.clone(), + cardano_database_pb: RwLock::new(None), + certificate_validation_pb: RwLock::new(None), + } + } +} + +#[async_trait] +impl FeedbackReceiver for IndicatifFeedbackReceiver { + async fn handle_event(&self, event: MithrilEvent) { + match event { + MithrilEvent::CardanoDatabase(cardano_database_event) => match cardano_database_event { + MithrilEventCardanoDatabase::Started { + download_id: _, + total_immutable_files, + include_ancillary, + } => { + println!("Starting download of artifact files..."); + let size = match include_ancillary { + true => 1 + total_immutable_files, + false => total_immutable_files, + }; + let pb = ProgressBar::new(size); + pb.set_style(ProgressStyle::with_template("{spinner:.green} {elapsed_precise}] [{wide_bar:.cyan/blue}] Files: {human_pos}/{human_len} ({eta})") + .unwrap() + .with_key("eta", |state: &ProgressState, w: &mut dyn Write| write!(w, "{:.1}s", state.eta().as_secs_f64()).unwrap()) + .progress_chars("#>-")); + self.progress_bar.add(pb.clone()); + let mut cardano_database_pb = self.cardano_database_pb.write().await; + *cardano_database_pb = Some(pb); + } + MithrilEventCardanoDatabase::Completed { .. } => { + let mut cardano_database_pb = self.cardano_database_pb.write().await; + if let Some(progress_bar) = cardano_database_pb.as_ref() { + progress_bar.finish_with_message("Artifact files download completed"); + } + *cardano_database_pb = None; + } + MithrilEventCardanoDatabase::ImmutableDownloadCompleted { .. } + | MithrilEventCardanoDatabase::AncillaryDownloadCompleted { .. } => { + let cardano_database_pb = self.cardano_database_pb.read().await; + if let Some(progress_bar) = cardano_database_pb.as_ref() { + progress_bar.inc(1); + } + } + _ => { + // Ignore other events + } + }, + MithrilEvent::CertificateChainValidationStarted { + certificate_chain_validation_id: _, + } => { + println!("Validating certificate chain..."); + let pb = ProgressBar::new_spinner(); + self.progress_bar.add(pb.clone()); + let mut certificate_validation_pb = self.certificate_validation_pb.write().await; + *certificate_validation_pb = Some(pb); + } + MithrilEvent::CertificateValidated { + certificate_chain_validation_id: _, + certificate_hash, + } => { + let certificate_validation_pb = self.certificate_validation_pb.read().await; + if let Some(progress_bar) = certificate_validation_pb.as_ref() { + progress_bar.set_message(format!("Certificate '{certificate_hash}' is valid")); + progress_bar.inc(1); + } + } + MithrilEvent::CertificateChainValidated { + certificate_chain_validation_id: _, + } => { + let mut certificate_validation_pb = self.certificate_validation_pb.write().await; + if let Some(progress_bar) = certificate_validation_pb.as_ref() { + progress_bar.finish_with_message("Certificate chain validated"); + } + *certificate_validation_pb = None; + } + _ => { + // Ignore other events + } + } + } +} + +fn get_temp_dir() -> MithrilResult { + let dir = std::env::temp_dir() + .join("mithril_examples") + .join("cardano_database_snapshot"); + + if dir.exists() { + std::fs::remove_dir_all(&dir).with_context(|| format!("Could not remove dir {dir:?}"))?; + } + std::fs::create_dir_all(&dir).with_context(|| format!("Could not create dir {dir:?}"))?; + + Ok(dir) +} + +async fn wait_spinner( + progress_bar: &MultiProgress, + future: impl Future>, +) -> MithrilResult { + let pb = progress_bar.add(ProgressBar::new_spinner()); + let spinner = async move { + loop { + pb.tick(); + tokio::time::sleep(Duration::from_millis(50)).await; + } + }; + + tokio::select! { + _ = spinner => Err(anyhow!("timeout")), + res = future => res, + } +} diff --git a/mithril-client-cli/Cargo.toml b/mithril-client-cli/Cargo.toml index 306ed552a02..71d3f1c78e4 100644 --- a/mithril-client-cli/Cargo.toml +++ b/mithril-client-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-client-cli" -version = "0.11.3" +version = "0.11.4" description = "A Mithril Client" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-client-cli/src/commands/cardano_db_v2/download.rs b/mithril-client-cli/src/commands/cardano_db_v2/download.rs index 809d3b17c5f..3bcbf5332fd 100644 --- a/mithril-client-cli/src/commands/cardano_db_v2/download.rs +++ b/mithril-client-cli/src/commands/cardano_db_v2/download.rs @@ -95,20 +95,6 @@ impl CardanoDbV2DownloadCommand { } } - fn number_of_immutable_files_restored( - cardano_database_snapshot: &CardanoDatabaseSnapshot, - immutable_file_range: &ImmutableFileRange, - ) -> u64 { - match immutable_file_range { - ImmutableFileRange::Full => cardano_database_snapshot.beacon.immutable_file_number, - ImmutableFileRange::From(from) => { - cardano_database_snapshot.beacon.immutable_file_number - from + 1 - } - ImmutableFileRange::Range(from, to) => to - from + 1, - ImmutableFileRange::UpTo(to) => *to, - } - } - /// Command execution pub async fn execute(&self, context: CommandContext) -> MithrilResult<()> { let params = context.config_parameters()?.add_source(self)?; @@ -305,10 +291,9 @@ impl CardanoDbV2DownloadCommand { let include_ancillary = restoration_options .download_unpack_options .include_ancillary; - let number_of_immutable_files_restored = Self::number_of_immutable_files_restored( - cardano_database_snapshot, - &restoration_options.immutable_file_range, - ); + let number_of_immutable_files_restored = restoration_options + .immutable_file_range + .length(cardano_database_snapshot.beacon.immutable_file_number); if let Err(e) = client .cardano_database() .add_statistics( @@ -587,80 +572,4 @@ mod tests { assert_eq!(range, ImmutableFileRange::UpTo(345)); } - - #[test] - fn number_of_immutable_files_restored_with_full_restoration() { - let cardano_database_snapshot = CardanoDatabaseSnapshot { - beacon: CardanoDbBeacon::new(999, 20), - ..CardanoDatabaseSnapshot::dummy() - }; - let immutable_file_range = ImmutableFileRange::Full; - - let number_of_immutable_files_restored = - CardanoDbV2DownloadCommand::number_of_immutable_files_restored( - &cardano_database_snapshot, - &immutable_file_range, - ); - - assert_eq!(number_of_immutable_files_restored, 20); - } - - #[test] - fn number_of_immutable_files_restored_with_from() { - let cardano_database_snapshot = CardanoDatabaseSnapshot { - beacon: CardanoDbBeacon::new(999, 20), - ..CardanoDatabaseSnapshot::dummy() - }; - let immutable_file_range = ImmutableFileRange::From(12); - - let number_of_immutable_files_restored = - CardanoDbV2DownloadCommand::number_of_immutable_files_restored( - &cardano_database_snapshot, - &immutable_file_range, - ); - - let expected_number_of_immutable_files_restored = 20 - 12 + 1; - assert_eq!( - number_of_immutable_files_restored, - expected_number_of_immutable_files_restored - ); - } - - #[test] - fn number_of_immutable_files_restored_with_up_to() { - let cardano_database_snapshot = CardanoDatabaseSnapshot { - beacon: CardanoDbBeacon::new(999, 20), - ..CardanoDatabaseSnapshot::dummy() - }; - let immutable_file_range = ImmutableFileRange::UpTo(14); - - let number_of_immutable_files_restored = - CardanoDbV2DownloadCommand::number_of_immutable_files_restored( - &cardano_database_snapshot, - &immutable_file_range, - ); - - assert_eq!(number_of_immutable_files_restored, 14); - } - - #[test] - fn number_of_immutable_files_restored_with_range() { - let cardano_database_snapshot = CardanoDatabaseSnapshot { - beacon: CardanoDbBeacon::new(999, 20), - ..CardanoDatabaseSnapshot::dummy() - }; - let immutable_file_range = ImmutableFileRange::Range(12, 14); - - let number_of_immutable_files_restored = - CardanoDbV2DownloadCommand::number_of_immutable_files_restored( - &cardano_database_snapshot, - &immutable_file_range, - ); - - let expected_number_of_immutable_files_restored = 14 - 12 + 1; - assert_eq!( - number_of_immutable_files_restored, - expected_number_of_immutable_files_restored - ); - } } diff --git a/mithril-client-cli/src/commands/cardano_db_v2/mod.rs b/mithril-client-cli/src/commands/cardano_db_v2/mod.rs index 03b164cc72e..33243b78def 100644 --- a/mithril-client-cli/src/commands/cardano_db_v2/mod.rs +++ b/mithril-client-cli/src/commands/cardano_db_v2/mod.rs @@ -13,6 +13,7 @@ use mithril_client::MithrilResult; /// Cardano db v2 management (alias: cdbv2) #[derive(Subcommand, Debug, Clone)] +#[command(about = "[unstable] Cardano db v2 management (alias: cdbv2)")] pub enum CardanoDbV2Commands { /// Cardano db snapshot v2 commands #[clap(subcommand)] diff --git a/mithril-client/Cargo.toml b/mithril-client/Cargo.toml index 9517dfa6b9f..5d5198015b9 100644 --- a/mithril-client/Cargo.toml +++ b/mithril-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-client" -version = "0.11.5" +version = "0.11.6" description = "Mithril client library" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-client/src/cardano_database_client/immutable_file_range.rs b/mithril-client/src/cardano_database_client/immutable_file_range.rs index 48fb799a821..321588d70f5 100644 --- a/mithril-client/src/cardano_database_client/immutable_file_range.rs +++ b/mithril-client/src/cardano_database_client/immutable_file_range.rs @@ -50,6 +50,16 @@ impl ImmutableFileRange { _ => Err(anyhow!("Invalid immutable file range: {self:?}")), } } + + /// Returns the length of the immutable file range + pub fn length(&self, last_immutable_file_number: ImmutableFileNumber) -> u64 { + match self { + ImmutableFileRange::Full => last_immutable_file_number, + ImmutableFileRange::From(from) => last_immutable_file_number - from + 1, + ImmutableFileRange::Range(from, to) => to - from + 1, + ImmutableFileRange::UpTo(to) => *to, + } + } } #[cfg(test)] @@ -123,4 +133,24 @@ mod tests { "should fail: given last immutable should be greater or equal range max bound", ); } + + #[test] + fn length() { + let last_immutable_file_number = 10; + + let immutable_file_range = ImmutableFileRange::Full; + assert_eq!( + last_immutable_file_number, + immutable_file_range.length(last_immutable_file_number) + ); + + let immutable_file_range = ImmutableFileRange::From(5); + assert_eq!(6, immutable_file_range.length(last_immutable_file_number)); + + let immutable_file_range = ImmutableFileRange::Range(5, 8); + assert_eq!(4, immutable_file_range.length(last_immutable_file_number)); + + let immutable_file_range = ImmutableFileRange::UpTo(8); + assert_eq!(8, immutable_file_range.length(last_immutable_file_number)); + } }