Feat: add example crate for incremental Cardano database

CHANGELOG.md

@@ -19,6 +19,7 @@ As a minor extension, we have adopted a slightly different versioning convention
 
   - Implement the client library for the the signed entity type `CardanoDatabase` (download and prove snapshot).
   - Implement the client CLI commands for the signed entity type `CardanoDatabase` (snapshot list, snapshot show and download commands).
+  - Implement an example crate for the signed entity type `CardanoDatabase`.
 
 - Crates versions:
 

Cargo.toml

@@ -4,6 +4,7 @@ resolver = "2"
 
 members = [
   "demo/protocol-demo",
+  "examples/client-cardano-database",
   "examples/client-cardano-stake-distribution",
   "examples/client-cardano-transaction",
   "examples/client-mithril-stake-distribution",

examples/client-cardano-database/.gitignore

@@ -0,0 +1,3 @@
+target/
+client-cardano-database
+.DS_Store

examples/client-cardano-database/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "client-cardano-database"
+description = "Mithril client Cardano database example"
+version = "0.0.1"
+authors = ["[email protected]", "[email protected]"]
+documentation = "https://mithril.network/doc"
+edition = "2021"
+homepage = "https://mithril.network"
+license = "Apache-2.0"
+repository = "https://github.com/input-output-hk/mithril/"
+
+[dependencies]
+anyhow = "1.0.95"
+async-trait = "0.1.86"
+clap = { version = "4.5.28", features = ["derive", "env"] }
+futures = "0.3.31"
+indicatif = "0.17.11"
+mithril-client = { path = "../../mithril-client", features = ["fs", "unstable"] }
+tokio = { version = "1.43.0", features = ["full"] }

examples/client-cardano-database/README.md

@@ -0,0 +1,41 @@
+# Mithril client library example: Cardano database
+
+## Description
+
+This example shows how to implement a Mithril client and use its features related to the `Cardano database` type.
+
+In this example, the client interacts by default with a real aggregator on the network `testing-preview` to:
+
+- list the available snapshots
+- get a single snapshot
+- download and unpack snapshot archives tailored for a specific range
+- verify the associated certificate and its chain
+- compute a message for the retrieved artifact files
+- verify that the certificate signs the computed message
+- increments snapshot download statistics
+
+The crate [indicatif](https://docs.rs/indicatif/latest/indicatif/) is used to nicely report the progress to the console.
+
+## Build and run the example
+
+```bash
+# Switch to the latest release tag
+git checkout tags/$(curl -sSL https://api.github.com/repos/input-output-hk/mithril/releases/latest | jq -r '.tag_name')
+
+# Build from the crate directory
+cargo build
+
+# Run from the crate directory
+cargo run
+
+# Run with your custom network configuration
+AGGREGATOR_ENDPOINT=YOUR_AGGREGATOR_ENDPOINT GENESIS_VERIFICATION_KEY=YOUR_GENESIS_VERIFICATION_KEY cargo run
+
+# Example with 'release-preprod' network
+AGGREGATOR_ENDPOINT=https://aggregator.testing-preview.api.mithril.network/aggregator GENESIS_VERIFICATION_KEY=$(curl -s https://raw.githubusercontent.com/input-output-hk/mithril/main/mithril-infra/configuration/testing-preview/genesis.vkey) cargo run
+```
+
+## Links
+
+- **Developer documentation**: https://docs.rs/mithril-client/latest/mithril_client/
+- **Crates.io**: https://crates.io/crates/mithril-client

examples/client-cardano-database/src/main.rs

@@ -0,0 +1,275 @@
+//! This example shows how to implement a Mithril client and use its features.
+//!
+//! In this example, the client interacts by default with a real aggregator (`testing-preview`) to get the data.
+//!
+//! A [FeedbackReceiver] using [indicatif] is used to nicely report the progress to the console.
+
+use anyhow::{anyhow, Context};
+use async_trait::async_trait;
+use clap::Parser;
+use futures::Future;
+use indicatif::{MultiProgress, ProgressBar, ProgressState, ProgressStyle};
+use mithril_client::cardano_database_client::{DownloadUnpackOptions, ImmutableFileRange};
+use std::fmt::Write;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::RwLock;
+
+use mithril_client::feedback::{FeedbackReceiver, MithrilEvent, MithrilEventCardanoDatabase};
+use mithril_client::{ClientBuilder, MessageBuilder, MithrilResult};
+
+#[derive(Parser, Debug)]
+#[command(version)]
+pub struct Args {
+    /// Genesis verification key.
+    #[clap(
+        long,
+        env = "GENESIS_VERIFICATION_KEY",
+        default_value = "5b3132372c37332c3132342c3136312c362c3133372c3133312c3231332c3230372c3131372c3139382c38352c3137362c3139392c3136322c3234312c36382c3132332c3131392c3134352c31332c3233322c3234332c34392c3232392c322c3234392c3230352c3230352c33392c3233352c34345d"
+    )]
+    genesis_verification_key: String,
+
+    /// Aggregator endpoint URL.
+    #[clap(
+        long,
+        env = "AGGREGATOR_ENDPOINT",
+        default_value = "https://aggregator.testing-preview.api.mithril.network/aggregator"
+    )]
+    aggregator_endpoint: String,
+}
+
+#[tokio::main]
+async fn main() -> MithrilResult<()> {
+    let args = Args::parse();
+    let work_dir = get_temp_dir()?;
+    let progress_bar = indicatif::MultiProgress::new();
+    let client =
+        ClientBuilder::aggregator(&args.aggregator_endpoint, &args.genesis_verification_key)
+            .add_feedback_receiver(Arc::new(IndicatifFeedbackReceiver::new(&progress_bar)))
+            .build()?;
+
+    let cardano_database_snapshots = client.cardano_database().list().await?;
+
+    let latest_hash = cardano_database_snapshots
+        .first()
+        .ok_or(anyhow!(
+            "No Cardano database snapshot could be listed from aggregator: '{}'",
+            args.aggregator_endpoint
+        ))?
+        .hash
+        .as_ref();
+
+    let cardano_database_snapshot =
+        client
+            .cardano_database()
+            .get(latest_hash)
+            .await?
+            .ok_or(anyhow!(
+                "A Cardano database should exist for hash '{latest_hash}'"
+            ))?;
+
+    let unpacked_dir = work_dir.join("unpack");
+    std::fs::create_dir(&unpacked_dir).unwrap();
+
+    let certificate = client
+        .certificate()
+        .verify_chain(&cardano_database_snapshot.certificate_hash)
+        .await?;
+
+    let immutable_file_range = ImmutableFileRange::From(15000);
+    let download_unpack_options = DownloadUnpackOptions {
+        allow_override: true,
+        include_ancillary: false,
+        ..DownloadUnpackOptions::default()
+    };
+    client
+        .cardano_database()
+        .download_unpack(
+            &cardano_database_snapshot,
+            &immutable_file_range,
+            &unpacked_dir,
+            download_unpack_options,
+        )
+        .await?;
+
+    println!("Computing Cardano database Merkle proof...",);
+    let merkle_proof = client
+        .cardano_database()
+        .compute_merkle_proof(
+            &certificate,
+            &cardano_database_snapshot,
+            &immutable_file_range,
+            &unpacked_dir,
+        )
+        .await?;
+    merkle_proof
+        .verify()
+        .with_context(|| "Merkle proof verification failed")?;
+
+    println!("Sending usage statistics to the aggregator...");
+    let full_restoration = immutable_file_range == ImmutableFileRange::Full;
+    let include_ancillary = download_unpack_options.include_ancillary;
+    let number_of_immutable_files_restored =
+        immutable_file_range.length(cardano_database_snapshot.beacon.immutable_file_number);
+    if let Err(e) = client
+        .cardano_database()
+        .add_statistics(
+            full_restoration,
+            include_ancillary,
+            number_of_immutable_files_restored,
+        )
+        .await
+    {
+        println!("Could not send usage statistics to the aggregator: {:?}", e);
+    }
+
+    println!(
+        "Computing Cardano database snapshot '{}' message...",
+        cardano_database_snapshot.hash
+    );
+    let message = wait_spinner(
+        &progress_bar,
+        MessageBuilder::new().compute_cardano_database_message(&certificate, &merkle_proof),
+    )
+    .await?;
+
+    if certificate.match_message(&message) {
+        println!(
+            "Successfully downloaded and validated Cardano database snapshot '{}'",
+            cardano_database_snapshot.hash
+        );
+
+        Ok(())
+    } else {
+        Err(anyhow::anyhow!(
+            "Certificate and message did not match:\ncertificate_message: '{}'\n computed_message: '{}'",
+            certificate.signed_message,
+            message.compute_hash()
+        ))
+    }
+}
+
+pub struct IndicatifFeedbackReceiver {
+    progress_bar: MultiProgress,
+    cardano_database_pb: RwLock<Option<ProgressBar>>,
+    certificate_validation_pb: RwLock<Option<ProgressBar>>,
+}
+
+impl IndicatifFeedbackReceiver {
+    pub fn new(progress_bar: &MultiProgress) -> Self {
+        Self {
+            progress_bar: progress_bar.clone(),
+            cardano_database_pb: RwLock::new(None),
+            certificate_validation_pb: RwLock::new(None),
+        }
+    }
+}
+
+#[async_trait]
+impl FeedbackReceiver for IndicatifFeedbackReceiver {
+    async fn handle_event(&self, event: MithrilEvent) {
+        match event {
+            MithrilEvent::CardanoDatabase(cardano_database_event) => match cardano_database_event {
+                MithrilEventCardanoDatabase::Started {
+                    download_id: _,
+                    total_immutable_files,
+                    include_ancillary,
+                } => {
+                    println!("Starting download of artifact files...");
+                    let size = match include_ancillary {
+                        true => 1 + total_immutable_files,
+                        false => total_immutable_files,
+                    };
+                    let pb = ProgressBar::new(size);
+                    pb.set_style(ProgressStyle::with_template("{spinner:.green} {elapsed_precise}] [{wide_bar:.cyan/blue}] Files: {human_pos}/{human_len} ({eta})")
+                        .unwrap()
+                        .with_key("eta", |state: &ProgressState, w: &mut dyn Write| write!(w, "{:.1}s", state.eta().as_secs_f64()).unwrap())
+                        .progress_chars("#>-"));
+                    self.progress_bar.add(pb.clone());
+                    let mut cardano_database_pb = self.cardano_database_pb.write().await;
+                    *cardano_database_pb = Some(pb);
+                }
+                MithrilEventCardanoDatabase::Completed { .. } => {
+                    let mut cardano_database_pb = self.cardano_database_pb.write().await;
+                    if let Some(progress_bar) = cardano_database_pb.as_ref() {
+                        progress_bar.finish_with_message("Artifact files download completed");
+                    }
+                    *cardano_database_pb = None;
+                }
+                MithrilEventCardanoDatabase::ImmutableDownloadCompleted { .. }
+                | MithrilEventCardanoDatabase::AncillaryDownloadCompleted { .. } => {
+                    let cardano_database_pb = self.cardano_database_pb.read().await;
+                    if let Some(progress_bar) = cardano_database_pb.as_ref() {
+                        progress_bar.inc(1);
+                    }
+                }
+                _ => {
+                    // Ignore other events
+                }
+            },
+            MithrilEvent::CertificateChainValidationStarted {
+                certificate_chain_validation_id: _,
+            } => {
+                println!("Validating certificate chain...");
+                let pb = ProgressBar::new_spinner();
+                self.progress_bar.add(pb.clone());
+                let mut certificate_validation_pb = self.certificate_validation_pb.write().await;
+                *certificate_validation_pb = Some(pb);
+            }
+            MithrilEvent::CertificateValidated {
+                certificate_chain_validation_id: _,
+                certificate_hash,
+            } => {
+                let certificate_validation_pb = self.certificate_validation_pb.read().await;
+                if let Some(progress_bar) = certificate_validation_pb.as_ref() {
+                    progress_bar.set_message(format!("Certificate '{certificate_hash}' is valid"));
+                    progress_bar.inc(1);
+                }
+            }
+            MithrilEvent::CertificateChainValidated {
+                certificate_chain_validation_id: _,
+            } => {
+                let mut certificate_validation_pb = self.certificate_validation_pb.write().await;
+                if let Some(progress_bar) = certificate_validation_pb.as_ref() {
+                    progress_bar.finish_with_message("Certificate chain validated");
+                }
+                *certificate_validation_pb = None;
+            }
+            _ => {
+                // Ignore other events
+            }
+        }
+    }
+}
+
+fn get_temp_dir() -> MithrilResult<PathBuf> {
+    let dir = std::env::temp_dir()
+        .join("mithril_examples")
+        .join("cardano_database_snapshot");
+
+    if dir.exists() {
+        std::fs::remove_dir_all(&dir).with_context(|| format!("Could not remove dir {dir:?}"))?;
+    }
+    std::fs::create_dir_all(&dir).with_context(|| format!("Could not create dir {dir:?}"))?;
+
+    Ok(dir)
+}
+
+async fn wait_spinner<T>(
+    progress_bar: &MultiProgress,
+    future: impl Future<Output = MithrilResult<T>>,
+) -> MithrilResult<T> {
+    let pb = progress_bar.add(ProgressBar::new_spinner());
+    let spinner = async move {
+        loop {
+            pb.tick();
+            tokio::time::sleep(Duration::from_millis(50)).await;
+        }
+    };
+
+    tokio::select! {
+        _ = spinner => Err(anyhow!("timeout")),
+        res = future => res,
+    }
+}