From d9ca40a15f96d7cb4be51dd32f8d10ce4db1ac29 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 28 Jan 2025 17:24:24 +0100 Subject: [PATCH 01/59] feat: add 'ImmutableFileRange' enum in Cardano database client --- mithril-client/src/cardano_database_client.rs | 364 ++++++++++++------ 1 file changed, 240 insertions(+), 124 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index 67d4c28c915..ac7c60305a7 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -43,12 +43,68 @@ //! # } //! ``` +#[cfg(feature = "fs")] +use anyhow::anyhow; use anyhow::Context; +#[cfg(feature = "fs")] +use std::ops::RangeInclusive; use std::sync::Arc; +#[cfg(feature = "fs")] +use mithril_common::{entities::ImmutableFileNumber, StdResult}; + use crate::aggregator_client::{AggregatorClient, AggregatorClientError, AggregatorRequest}; use crate::{CardanoDatabaseSnapshot, CardanoDatabaseSnapshotListItem, MithrilResult}; +cfg_fs! { + /// Immutable file range representation + #[derive(Debug)] + pub enum ImmutableFileRange { + /// From the first (included) to the last immutable file number (included) + Full, + + /// From a specific immutable file number (included) to the last immutable file number (included) + From(ImmutableFileNumber), + + /// From a specific immutable file number (included) to another specific immutable file number (included) + Range(ImmutableFileNumber, ImmutableFileNumber), + + /// From the first immutable file number (included) up to a specific immutable file number (included) + UpTo(ImmutableFileNumber), + } + + impl ImmutableFileRange { + /// Returns the range of immutable file numbers + pub fn to_range_inclusive( + &self, + last_immutable_file_number: ImmutableFileNumber, + ) -> StdResult> { + // TODO: first immutable file is 0 on devnet and 1 otherwise. To be handled properly + let first_immutable_file_number = 0 as ImmutableFileNumber; + let full_range = first_immutable_file_number..=last_immutable_file_number; + + match self { + ImmutableFileRange::Full => Ok(full_range), + ImmutableFileRange::From(from) if full_range.contains(from) => { + Ok(*from..=last_immutable_file_number) + } + ImmutableFileRange::Range(from, to) + if full_range.contains(from) + && full_range.contains(to) + && !(*from..=*to).is_empty() => + { + Ok(*from..=*to) + } + ImmutableFileRange::UpTo(to) if full_range.contains(to) => { + Ok(first_immutable_file_number..=*to) + } + _ => Err(anyhow!("Invalid immutable file range: {self:?}")), + } + } + } + +} + /// HTTP client for CardanoDatabase API from the Aggregator pub struct CardanoDatabaseClient { aggregator_client: Arc, @@ -102,148 +158,208 @@ impl CardanoDatabaseClient { #[cfg(test)] mod tests { - use anyhow::anyhow; - use chrono::{DateTime, Utc}; - use mithril_common::entities::{CardanoDbBeacon, CompressionAlgorithm, Epoch}; - use mockall::predicate::eq; + use super::*; - use crate::aggregator_client::MockAggregatorHTTPClient; + mod cardano_database_client { + use anyhow::anyhow; + use chrono::{DateTime, Utc}; + use mithril_common::entities::{CardanoDbBeacon, CompressionAlgorithm, Epoch}; + use mockall::predicate::eq; - use super::*; + use crate::aggregator_client::MockAggregatorHTTPClient; - fn fake_messages() -> Vec { - vec![ - CardanoDatabaseSnapshotListItem { - hash: "hash-123".to_string(), - merkle_root: "mkroot-123".to_string(), - beacon: CardanoDbBeacon { - epoch: Epoch(1), - immutable_file_number: 123, + use super::*; + + fn fake_messages() -> Vec { + vec![ + CardanoDatabaseSnapshotListItem { + hash: "hash-123".to_string(), + merkle_root: "mkroot-123".to_string(), + beacon: CardanoDbBeacon { + epoch: Epoch(1), + immutable_file_number: 123, + }, + certificate_hash: "cert-hash-123".to_string(), + total_db_size_uncompressed: 800796318, + created_at: DateTime::parse_from_rfc3339("2025-01-19T13:43:05.618857482Z") + .unwrap() + .with_timezone(&Utc), + compression_algorithm: CompressionAlgorithm::default(), + cardano_node_version: "0.0.1".to_string(), }, - certificate_hash: "cert-hash-123".to_string(), - total_db_size_uncompressed: 800796318, - created_at: DateTime::parse_from_rfc3339("2025-01-19T13:43:05.618857482Z") - .unwrap() - .with_timezone(&Utc), - compression_algorithm: CompressionAlgorithm::default(), - cardano_node_version: "0.0.1".to_string(), - }, - CardanoDatabaseSnapshotListItem { - hash: "hash-456".to_string(), - merkle_root: "mkroot-456".to_string(), - beacon: CardanoDbBeacon { - epoch: Epoch(2), - immutable_file_number: 456, + CardanoDatabaseSnapshotListItem { + hash: "hash-456".to_string(), + merkle_root: "mkroot-456".to_string(), + beacon: CardanoDbBeacon { + epoch: Epoch(2), + immutable_file_number: 456, + }, + certificate_hash: "cert-hash-456".to_string(), + total_db_size_uncompressed: 2960713808, + created_at: DateTime::parse_from_rfc3339("2025-01-27T15:22:05.618857482Z") + .unwrap() + .with_timezone(&Utc), + compression_algorithm: CompressionAlgorithm::default(), + cardano_node_version: "0.0.1".to_string(), }, - certificate_hash: "cert-hash-456".to_string(), - total_db_size_uncompressed: 2960713808, - created_at: DateTime::parse_from_rfc3339("2025-01-27T15:22:05.618857482Z") - .unwrap() - .with_timezone(&Utc), - compression_algorithm: CompressionAlgorithm::default(), - cardano_node_version: "0.0.1".to_string(), - }, - ] - } + ] + } - #[tokio::test] - async fn list_cardano_database_snapshots_returns_messages() { - let message = fake_messages(); - let mut http_client = MockAggregatorHTTPClient::new(); - http_client - .expect_get_content() - .with(eq(AggregatorRequest::ListCardanoDatabaseSnapshots)) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); - - let messages = client.list().await.unwrap(); - - assert_eq!(2, messages.len()); - assert_eq!("hash-123".to_string(), messages[0].hash); - assert_eq!("hash-456".to_string(), messages[1].hash); - } + #[tokio::test] + async fn list_cardano_database_snapshots_returns_messages() { + let message = fake_messages(); + let mut http_client = MockAggregatorHTTPClient::new(); + http_client + .expect_get_content() + .with(eq(AggregatorRequest::ListCardanoDatabaseSnapshots)) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + let client = CardanoDatabaseClient::new(Arc::new(http_client)); - #[tokio::test] - async fn list_cardano_database_snapshots_returns_error_when_invalid_json_structure_in_response() - { - let mut http_client = MockAggregatorHTTPClient::new(); - http_client - .expect_get_content() - .return_once(move |_| Ok("invalid json structure".to_string())); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); - - client - .list() - .await - .expect_err("List Cardano databases should return an error"); - } + let messages = client.list().await.unwrap(); + + assert_eq!(2, messages.len()); + assert_eq!("hash-123".to_string(), messages[0].hash); + assert_eq!("hash-456".to_string(), messages[1].hash); + } + + #[tokio::test] + async fn list_cardano_database_snapshots_returns_error_when_invalid_json_structure_in_response( + ) { + let mut http_client = MockAggregatorHTTPClient::new(); + http_client + .expect_get_content() + .return_once(move |_| Ok("invalid json structure".to_string())); + let client = CardanoDatabaseClient::new(Arc::new(http_client)); - #[tokio::test] - async fn get_cardano_database_snapshot_returns_message() { - let expected_cardano_database_snapshot = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - ..CardanoDatabaseSnapshot::dummy() - }; - let message = expected_cardano_database_snapshot.clone(); - let mut http_client = MockAggregatorHTTPClient::new(); - http_client - .expect_get_content() - .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { + client + .list() + .await + .expect_err("List Cardano databases should return an error"); + } + + #[tokio::test] + async fn get_cardano_database_snapshot_returns_message() { + let expected_cardano_database_snapshot = CardanoDatabaseSnapshot { hash: "hash-123".to_string(), - })) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); + ..CardanoDatabaseSnapshot::dummy() + }; + let message = expected_cardano_database_snapshot.clone(); + let mut http_client = MockAggregatorHTTPClient::new(); + http_client + .expect_get_content() + .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + })) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + let client = CardanoDatabaseClient::new(Arc::new(http_client)); - let cardano_database = client - .get("hash-123") - .await - .unwrap() - .expect("This test returns a Cardano database"); + let cardano_database = client + .get("hash-123") + .await + .unwrap() + .expect("This test returns a Cardano database"); - assert_eq!(expected_cardano_database_snapshot, cardano_database); - } + assert_eq!(expected_cardano_database_snapshot, cardano_database); + } - #[tokio::test] - async fn get_cardano_database_snapshot_returns_error_when_invalid_json_structure_in_response() { - let mut http_client = MockAggregatorHTTPClient::new(); - http_client - .expect_get_content() - .return_once(move |_| Ok("invalid json structure".to_string())); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); + #[tokio::test] + async fn get_cardano_database_snapshot_returns_error_when_invalid_json_structure_in_response( + ) { + let mut http_client = MockAggregatorHTTPClient::new(); + http_client + .expect_get_content() + .return_once(move |_| Ok("invalid json structure".to_string())); + let client = CardanoDatabaseClient::new(Arc::new(http_client)); - client - .get("hash-123") - .await - .expect_err("Get Cardano database should return an error"); - } + client + .get("hash-123") + .await + .expect_err("Get Cardano database should return an error"); + } + + #[tokio::test] + async fn get_cardano_database_snapshot_returns_none_when_not_found_or_remote_server_logical_error( + ) { + let mut http_client = MockAggregatorHTTPClient::new(); + http_client.expect_get_content().return_once(move |_| { + Err(AggregatorClientError::RemoteServerLogical(anyhow!( + "not found" + ))) + }); + let client = CardanoDatabaseClient::new(Arc::new(http_client)); + + let result = client.get("hash-123").await.unwrap(); - #[tokio::test] - async fn get_cardano_database_snapshot_returns_none_when_not_found_or_remote_server_logical_error( - ) { - let mut http_client = MockAggregatorHTTPClient::new(); - http_client.expect_get_content().return_once(move |_| { - Err(AggregatorClientError::RemoteServerLogical(anyhow!( - "not found" - ))) - }); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); + assert!(result.is_none()); + } - let result = client.get("hash-123").await.unwrap(); + #[tokio::test] + async fn get_cardano_database_snapshot_returns_error() { + let mut http_client = MockAggregatorHTTPClient::new(); + http_client + .expect_get_content() + .return_once(move |_| Err(AggregatorClientError::SubsystemError(anyhow!("error")))); + let client = CardanoDatabaseClient::new(Arc::new(http_client)); - assert!(result.is_none()); + client + .get("hash-123") + .await + .expect_err("Get Cardano database should return an error"); + } } - #[tokio::test] - async fn get_cardano_database_snapshot_returns_error() { - let mut http_client = MockAggregatorHTTPClient::new(); - http_client - .expect_get_content() - .return_once(move |_| Err(AggregatorClientError::SubsystemError(anyhow!("error")))); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); + cfg_fs! { + mod immutable_file_range { + use super::*; - client - .get("hash-123") - .await - .expect_err("Get Cardano database should return an error"); + #[test] + fn to_range_inclusive_with_full() { + let immutable_file_range = ImmutableFileRange::Full; + let last_immutable_file_number = 10; + + let result = immutable_file_range.to_range_inclusive(last_immutable_file_number).unwrap(); + assert_eq!(0..=10, result); + } + + #[test] + fn to_range_inclusive_with_from() { + let immutable_file_range = ImmutableFileRange::From(5); + + let last_immutable_file_number = 10; + let result = immutable_file_range.to_range_inclusive(last_immutable_file_number).unwrap(); + assert_eq!(5..=10, result); + + let last_immutable_file_number = 3; + immutable_file_range.to_range_inclusive(last_immutable_file_number).expect_err("conversion to range inlusive should fail"); + } + + #[test] + fn to_range_inclusive_with_range() { + let immutable_file_range = ImmutableFileRange::Range(5, 8); + + let last_immutable_file_number = 10; + let result = immutable_file_range.to_range_inclusive(last_immutable_file_number).unwrap(); + assert_eq!(5..=8, result); + + let last_immutable_file_number = 7; + immutable_file_range.to_range_inclusive(last_immutable_file_number).expect_err("conversion to range inlusive should fail"); + + let immutable_file_range = ImmutableFileRange::Range(10, 8); + immutable_file_range.to_range_inclusive(last_immutable_file_number).expect_err("conversion to range inlusive should fail"); + + } + + #[test] + fn to_range_inclusive_with_up_to() { + let immutable_file_range = ImmutableFileRange::UpTo(8); + + let last_immutable_file_number = 10; + let result = immutable_file_range.to_range_inclusive(last_immutable_file_number).unwrap(); + assert_eq!(0..=8, result); + + let last_immutable_file_number = 7; + immutable_file_range.to_range_inclusive(last_immutable_file_number).expect_err("conversion to range inlusive should fail"); + } + } } } From ef884a4717b6ef01d2d58ee44ae2b7e6dfde6e62 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 28 Jan 2025 18:01:39 +0100 Subject: [PATCH 02/59] feat: add 'DownloadUnpackOptions' struct in Cardano database client --- mithril-client/src/cardano_database_client.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index ac7c60305a7..985d633939e 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -103,6 +103,15 @@ cfg_fs! { } } + /// Options for downloading and unpacking a Cardano database + #[derive(Debug,Default)] + pub struct DownloadUnpackOptions { + /// Allow overriding the destination directory + pub allow_override: bool, + + /// Include ancillary files in the download + pub include_ancillary: bool + } } /// HTTP client for CardanoDatabase API from the Aggregator From bfc523b5e76d90369851b0e3ae4282f6b36522d7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 31 Jan 2025 15:39:56 +0100 Subject: [PATCH 03/59] chore: add discriminant for 'ImmutablesLocation' --- mithril-common/src/entities/cardano_database.rs | 6 +++++- mithril-common/src/entities/mod.rs | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mithril-common/src/entities/cardano_database.rs b/mithril-common/src/entities/cardano_database.rs index 4c57e96f309..4306d895d2f 100644 --- a/mithril-common/src/entities/cardano_database.rs +++ b/mithril-common/src/entities/cardano_database.rs @@ -1,6 +1,7 @@ use semver::Version; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use strum::EnumDiscriminants; use crate::entities::{CardanoDbBeacon, CompressionAlgorithm}; @@ -83,8 +84,11 @@ pub enum DigestLocation { } /// Locations of the immutable files. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[derive( + Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, EnumDiscriminants, +)] #[serde(rename_all = "snake_case", tag = "type")] +#[strum_discriminants(derive(Hash))] pub enum ImmutablesLocation { /// Cloud storage location. CloudStorage { diff --git a/mithril-common/src/entities/mod.rs b/mithril-common/src/entities/mod.rs index 6607f0501cc..d5ef53b1d94 100644 --- a/mithril-common/src/entities/mod.rs +++ b/mithril-common/src/entities/mod.rs @@ -34,8 +34,8 @@ pub use block_number::BlockNumber; pub use block_range::{BlockRange, BlockRangeLength, BlockRangesSequence}; pub use cardano_chain_point::{BlockHash, ChainPoint}; pub use cardano_database::{ - AncillaryLocation, ArtifactsLocations, CardanoDatabaseSnapshot, DigestLocation, - ImmutablesLocation, + AncillaryLocation, AncillaryLocationDiscriminants, ArtifactsLocations, CardanoDatabaseSnapshot, + DigestLocation, ImmutablesLocation, ImmutablesLocationDiscriminants, }; pub use cardano_db_beacon::CardanoDbBeacon; pub use cardano_network::CardanoNetwork; From 45b360832f460f14ef31eb37473722db0e8606e2 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 31 Jan 2025 15:41:04 +0100 Subject: [PATCH 04/59] feat: implement 'MultiFilesUri' template expansion --- mithril-common/src/entities/file_uri.rs | 59 ++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/mithril-common/src/entities/file_uri.rs b/mithril-common/src/entities/file_uri.rs index 88a870cd6e0..6639ebf9b26 100644 --- a/mithril-common/src/entities/file_uri.rs +++ b/mithril-common/src/entities/file_uri.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use anyhow::anyhow; use serde::{Deserialize, Serialize}; @@ -15,6 +15,12 @@ impl From for String { } } +/// TemplateVariable represents a variable in a template +pub type TemplateVariable = String; + +/// TemplateValue represents a value in a template +pub type TemplateValue = String; + /// [TemplateUri] represents an URI pattern used to build a file's location #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)] pub struct TemplateUri(pub String); @@ -44,6 +50,26 @@ impl MultiFilesUri { Ok(templates.into_iter().next().map(TemplateUri)) } + + /// Expand the template to a list of file URIs + pub fn expand_to_file_uris( + &self, + variables: Vec>, + ) -> StdResult> { + match self { + MultiFilesUri::Template(template) => Ok(variables + .into_iter() + .map(|variable| { + let mut file_uri = template.0.clone(); + for (key, value) in variable { + file_uri = file_uri.replace(&format!("{{{}}}", key), &value); + } + + FileUri(file_uri) + }) + .collect()), + } + } } #[cfg(test)] @@ -89,4 +115,35 @@ mod tests { "Should return an error when multiple templates are found in the file URIs", ); } + + #[test] + fn expand_multi_file_template_to_multiple_file_uris() { + let template = MultiFilesUri::Template(TemplateUri( + "http://whatever/{var1}-{var2}.tar.gz".to_string(), + )); + let variables = vec![ + HashMap::from([ + ("var1".to_string(), "00001".to_string()), + ("var2".to_string(), "abc".to_string()), + ]), + HashMap::from([ + ("var1".to_string(), "00001".to_string()), + ("var2".to_string(), "def".to_string()), + ]), + HashMap::from([ + ("var1".to_string(), "00002".to_string()), + ("var2".to_string(), "def".to_string()), + ]), + ]; + + let expanded_file_uris = template.expand_to_file_uris(variables).unwrap(); + assert_eq!( + vec![ + FileUri("http://whatever/00001-abc.tar.gz".to_string()), + FileUri("http://whatever/00001-def.tar.gz".to_string()), + FileUri("http://whatever/00002-def.tar.gz".to_string()), + ], + expanded_file_uris, + ); + } } From 2fa57278ec9218a3cb375af7b85abf5ae4e4afd0 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 31 Jan 2025 15:43:30 +0100 Subject: [PATCH 05/59] feat: scaffold file downloader module --- .../src/file_downloader/interface.rs | 116 ++++++++++++++++++ mithril-client/src/file_downloader/mod.rs | 9 ++ mithril-client/src/lib.rs | 1 + 3 files changed, 126 insertions(+) create mode 100644 mithril-client/src/file_downloader/interface.rs create mode 100644 mithril-client/src/file_downloader/mod.rs diff --git a/mithril-client/src/file_downloader/interface.rs b/mithril-client/src/file_downloader/interface.rs new file mode 100644 index 00000000000..4844a998b55 --- /dev/null +++ b/mithril-client/src/file_downloader/interface.rs @@ -0,0 +1,116 @@ +use std::{collections::HashMap, path::Path}; + +use async_trait::async_trait; + +use mithril_common::{ + entities::{CompressionAlgorithm, FileUri, ImmutableFileNumber, ImmutablesLocation}, + StdResult, +}; + +/// A file downloader URI +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum FileDownloaderUri { + /// A single file URI + FileUri(FileUri), +} + +impl FileDownloaderUri { + /// Expand the immutable locations to a list of file URIs + pub fn expand_immutable_files_location_to_file_downloader_uris( + immutable_files_location: &ImmutablesLocation, + immutable_files_range: &[ImmutableFileNumber], + ) -> StdResult> { + match immutable_files_location { + ImmutablesLocation::CloudStorage { uri } => { + let expand_variables = immutable_files_range + .iter() + .map(|immutable_file_number| { + HashMap::from([( + "immutable_file_number".to_string(), + format!("{:05}", immutable_file_number), + )]) + }) + .collect(); + let file_downloader_uris = uri + .expand_to_file_uris(expand_variables)? + .into_iter() + .map(FileDownloaderUri::FileUri); + let immutable_files_range = immutable_files_range.iter().copied(); + + Ok(immutable_files_range.zip(file_downloader_uris).collect()) + } + } + } + + /// Get the URI as a string + pub fn as_str(&self) -> &str { + match self { + FileDownloaderUri::FileUri(file_uri) => file_uri.0.as_str(), + } + } +} + +impl From for FileDownloaderUri { + fn from(file_uri: FileUri) -> Self { + Self::FileUri(file_uri) + } +} + +/// A file downloader +#[cfg_attr(test, mockall::automock)] +#[async_trait] +pub trait FileDownloader: Sync + Send { + /// Download and unpack (if necessary) a file on the disk. + /// + /// The `download_id` is a unique identifier that allow + /// [feedback receivers][crate::feedback::FeedbackReceiver] to track concurrent downloads. + async fn download_unpack( + &self, + location: &FileDownloaderUri, + target_dir: &Path, + compression_algorithm: Option, + download_id: &str, + ) -> StdResult<()>; +} + +#[cfg(test)] +mod tests { + use mithril_common::entities::{MultiFilesUri, TemplateUri}; + + use super::*; + + #[test] + fn immutable_files_location_to_file_downloader_uris() { + let immutable_files_location = ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }; + let immutable_files_range: Vec = (1..=3).collect(); + + let file_downloader_uris = + FileDownloaderUri::expand_immutable_files_location_to_file_downloader_uris( + &immutable_files_location, + &immutable_files_range, + ) + .unwrap(); + + assert_eq!( + file_downloader_uris, + vec![ + ( + 1, + FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())) + ), + ( + 2, + FileDownloaderUri::FileUri(FileUri("http://whatever/00002.tar.gz".to_string())) + ), + ( + 3, + FileDownloaderUri::FileUri(FileUri("http://whatever/00003.tar.gz".to_string())) + ), + ] + ); + } +} diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs new file mode 100644 index 00000000000..4f173d8a82e --- /dev/null +++ b/mithril-client/src/file_downloader/mod.rs @@ -0,0 +1,9 @@ +//! File downloader module. +//! +//! This module provides the necessary abstractions to download files from different sources. + +mod interface; + +#[cfg(test)] +pub use interface::MockFileDownloader; +pub use interface::{FileDownloader, FileDownloaderUri}; diff --git a/mithril-client/src/lib.rs b/mithril-client/src/lib.rs index 31e04fb0e36..84b4b7c0a33 100644 --- a/mithril-client/src/lib.rs +++ b/mithril-client/src/lib.rs @@ -98,6 +98,7 @@ pub mod mithril_stake_distribution_client; pub mod snapshot_client; cfg_fs! { pub mod snapshot_downloader; + pub mod file_downloader; } mod type_alias; From d7d98c2fdb222b6b337ec516d19210cb2e8a4f69 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 31 Jan 2025 15:44:45 +0100 Subject: [PATCH 06/59] feat: implement a file downloader resolver trait --- mithril-client/src/file_downloader/mod.rs | 4 + .../src/file_downloader/resolver.rs | 75 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 mithril-client/src/file_downloader/resolver.rs diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs index 4f173d8a82e..7762e56b572 100644 --- a/mithril-client/src/file_downloader/mod.rs +++ b/mithril-client/src/file_downloader/mod.rs @@ -3,7 +3,11 @@ //! This module provides the necessary abstractions to download files from different sources. mod interface; +mod resolver; #[cfg(test)] pub use interface::MockFileDownloader; pub use interface::{FileDownloader, FileDownloaderUri}; +#[cfg(test)] +pub use resolver::MockFileDownloaderResolver; +pub use resolver::{FileDownloaderResolver, ImmutablesFileDownloaderResolver}; diff --git a/mithril-client/src/file_downloader/resolver.rs b/mithril-client/src/file_downloader/resolver.rs new file mode 100644 index 00000000000..6526aa0730b --- /dev/null +++ b/mithril-client/src/file_downloader/resolver.rs @@ -0,0 +1,75 @@ +use std::{collections::HashMap, sync::Arc}; + +use mithril_common::entities::{ImmutablesLocation, ImmutablesLocationDiscriminants}; + +use super::FileDownloader; + +/// A file downloader resolver +#[cfg_attr(test, mockall::automock)] +pub trait FileDownloaderResolver: Sync + Send { + /// Resolve a file downloader for the given location. + fn resolve(&self, location: &L) -> Option>; +} + +/// A file downloader resolver for immutable file locations +pub struct ImmutablesFileDownloaderResolver { + file_downloaders: HashMap>, +} + +impl ImmutablesFileDownloaderResolver { + /// Constructs a new `ImmutablesFileDownloaderResolver`. + pub fn new( + file_downloaders: Vec<(ImmutablesLocationDiscriminants, Arc)>, + ) -> Self { + let file_downloaders = file_downloaders.into_iter().collect(); + + Self { file_downloaders } + } +} + +impl FileDownloaderResolver for ImmutablesFileDownloaderResolver { + fn resolve(&self, location: &ImmutablesLocation) -> Option> { + self.file_downloaders.get(&location.into()).cloned() + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use mithril_common::entities::{FileUri, MultiFilesUri, TemplateUri}; + + use crate::file_downloader::{FileDownloaderUri, MockFileDownloader}; + + use super::*; + + #[tokio::test] + async fn resolves_file_downloader() { + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .times(1) + .returning(|_, _, _, _| Ok(())); + let resolver = ImmutablesFileDownloaderResolver::new(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new(mock_file_downloader), + )]); + + let file_downloader = resolver + .resolve(&ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }) + .unwrap(); + file_downloader + .download_unpack( + &FileDownloaderUri::FileUri(FileUri("http://whatever/1.tar.gz".to_string())), + Path::new("."), + None, + "download_id", + ) + .await + .unwrap(); + } +} From 87822b605fc8947c6e26cbf11c3abdc3902a4465 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 31 Jan 2025 15:51:01 +0100 Subject: [PATCH 07/59] feat: implement immutable files download in Cardano database client --- mithril-client/src/cardano_database_client.rs | 1020 +++++++++++++++-- mithril-client/src/client.rs | 52 +- 2 files changed, 947 insertions(+), 125 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index 985d633939e..a93ac228689 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -46,14 +46,26 @@ #[cfg(feature = "fs")] use anyhow::anyhow; use anyhow::Context; +use mithril_common::entities::CompressionAlgorithm; +#[cfg(feature = "fs")] +use slog::Logger; #[cfg(feature = "fs")] use std::ops::RangeInclusive; -use std::sync::Arc; +#[cfg(feature = "fs")] +use std::path::{Path, PathBuf}; +use std::{collections::HashSet, sync::Arc}; #[cfg(feature = "fs")] +use mithril_common::{ use mithril_common::{entities::ImmutableFileNumber, StdResult}; + entities::{ImmutableFileNumber, ImmutablesLocation}, + StdResult, +}; -use crate::aggregator_client::{AggregatorClient, AggregatorClientError, AggregatorRequest}; +use crate::{ + aggregator_client::{AggregatorClient, AggregatorClientError, AggregatorRequest}, + file_downloader::{FileDownloaderResolver, FileDownloaderUri}, +}; use crate::{CardanoDatabaseSnapshot, CardanoDatabaseSnapshotListItem, MithrilResult}; cfg_fs! { @@ -117,12 +129,30 @@ cfg_fs! { /// HTTP client for CardanoDatabase API from the Aggregator pub struct CardanoDatabaseClient { aggregator_client: Arc, + #[cfg(feature = "fs")] + immutable_files_downloader_resolver: Arc>, + #[cfg(feature = "fs")] + logger: Logger, } impl CardanoDatabaseClient { /// Constructs a new `CardanoDatabase`. - pub fn new(aggregator_client: Arc) -> Self { - Self { aggregator_client } + pub fn new( + aggregator_client: Arc, + #[cfg(feature = "fs")] immutable_files_downloader_resolver: Arc< + dyn FileDownloaderResolver, + >, + #[cfg(feature = "fs")] logger: Logger, + ) -> Self { + Self { + aggregator_client, + #[cfg(feature = "fs")] + immutable_files_downloader_resolver, + #[cfg(feature = "fs")] + logger: mithril_common::logging::LoggerExtensions::new_with_component_name::( + &logger, + ), + } } /// Fetch a list of signed CardanoDatabase @@ -163,19 +193,163 @@ impl CardanoDatabaseClient { Err(e) => Err(e.into()), } } + + cfg_fs! { + /// Download and unpack the given Cardano database part data by hash. + // TODO: Add example in module documentation + pub async fn download_unpack( + &self, + hash: &str, + immutable_file_range: ImmutableFileRange, + target_dir: &std::path::Path, + download_unpack_options: DownloadUnpackOptions, + ) -> StdResult<()> { + let cardano_database_snapshot = self + .get(hash) + .await? + .ok_or_else(|| anyhow!("Cardano database snapshot not found"))?; + let compression_algorithm = cardano_database_snapshot.compression_algorithm; + let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; + let immutable_file_number_range = + immutable_file_range.to_range_inclusive(last_immutable_file_number)?; + + self.verify_can_write_to_target_dir(target_dir, download_unpack_options)?; + + let immutable_locations = cardano_database_snapshot.locations.immutables; + self.download_unpack_immutable_files( + &immutable_locations, + immutable_file_number_range, + &compression_algorithm, + &Self::immutable_files_target_dir(target_dir), + ) + .await?; + + Ok(()) + } + + fn immutable_files_target_dir(target_dir: &Path) -> PathBuf { + target_dir.join("immutable") + } + + fn volatile_target_dir(target_dir: &Path) -> PathBuf { + target_dir.join("volatile") + } + + fn ledger_target_dir(target_dir: &Path) -> PathBuf { + target_dir.join("ledger") + } + + /// Verify if the target directory is writable. + pub(crate) fn verify_can_write_to_target_dir( + &self, + target_dir: &std::path::Path, + download_unpack_options: DownloadUnpackOptions, + ) -> StdResult<()> { + if !download_unpack_options.allow_override { + if Self::immutable_files_target_dir(target_dir).exists() { + return Err(anyhow!( + "Immutable files target directory already exists in: {target_dir:?}" + )); + } + if download_unpack_options.include_ancillary { + if Self::volatile_target_dir(target_dir).exists() { + return Err(anyhow!( + "Volatile target directory already exists in: {target_dir:?}" + )); + } + if Self::ledger_target_dir(target_dir).exists() { + return Err(anyhow!( + "Ledger target directory already exists in: {target_dir:?}" + )); + } + } + } + + Ok(()) + } + + /// Download and unpack the immutable files of the given range. + /// + /// The download is attempted for each location until the full range is downloaded. + /// An error is returned if not all the files are downloaded. + // TODO: Add feedback receivers + async fn download_unpack_immutable_files( + &self, + locations: &[ImmutablesLocation], + range: RangeInclusive, + compression_algorithm: &CompressionAlgorithm, + immutable_files_target_dir: &std::path::Path, + ) -> StdResult<()> { + let mut locations_sorted = locations.to_owned(); + locations_sorted.sort(); + let mut immutable_file_numbers_to_download = + range.clone().map(|n| n.to_owned()).collect::>(); + for location in locations_sorted { + let file_downloader = self + .immutable_files_downloader_resolver + .resolve(&location) + .ok_or_else(|| { + anyhow!("Failed resolving a file downloader for location: {location:?}") + })?; + let file_downloader_uris = + FileDownloaderUri::expand_immutable_files_location_to_file_downloader_uris( + &location, + immutable_file_numbers_to_download + .clone() + .into_iter() + .collect::>() + .as_slice(), + )?; + for (immutable_file_number, file_downloader_uri) in file_downloader_uris { + let download_id = format!("{location:?}"); //TODO: check if this is the correct way to format the download_id + if file_downloader + .download_unpack( + &file_downloader_uri, + &immutable_files_target_dir, + Some(compression_algorithm.to_owned()), + &download_id, + ) + .await + .is_ok() + { + immutable_file_numbers_to_download.remove(&immutable_file_number); + } else { + slog::error!( + self.logger, + "Failed downloading and unpacking immutable files for location: {file_downloader_uri:?}" + ); + } + } + if immutable_file_numbers_to_download.is_empty() { + return Ok(()); + } + } + + Err(anyhow!( + "Failed downloading and unpacking immutable files for locations: {locations:?}" + )) + } + } } #[cfg(test)] mod tests { + use super::*; mod cardano_database_client { use anyhow::anyhow; use chrono::{DateTime, Utc}; - use mithril_common::entities::{CardanoDbBeacon, CompressionAlgorithm, Epoch}; + use mithril_common::entities::{ + CardanoDbBeacon, CompressionAlgorithm, Epoch, ImmutablesLocationDiscriminants, + }; use mockall::predicate::eq; - use crate::aggregator_client::MockAggregatorHTTPClient; + use crate::{ + aggregator_client::MockAggregatorHTTPClient, + file_downloader::{FileDownloader, ImmutablesFileDownloaderResolver}, + test_utils, + }; use super::*; @@ -214,110 +388,733 @@ mod tests { ] } - #[tokio::test] - async fn list_cardano_database_snapshots_returns_messages() { - let message = fake_messages(); - let mut http_client = MockAggregatorHTTPClient::new(); - http_client - .expect_get_content() - .with(eq(AggregatorRequest::ListCardanoDatabaseSnapshots)) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); - - let messages = client.list().await.unwrap(); - - assert_eq!(2, messages.len()); - assert_eq!("hash-123".to_string(), messages[0].hash); - assert_eq!("hash-456".to_string(), messages[1].hash); + struct CardanoDatabaseClientDependencyInjector { + http_client: MockAggregatorHTTPClient, + immutable_files_downloader_resolver: ImmutablesFileDownloaderResolver, } - #[tokio::test] - async fn list_cardano_database_snapshots_returns_error_when_invalid_json_structure_in_response( - ) { - let mut http_client = MockAggregatorHTTPClient::new(); - http_client - .expect_get_content() - .return_once(move |_| Ok("invalid json structure".to_string())); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); - - client - .list() - .await - .expect_err("List Cardano databases should return an error"); - } + impl CardanoDatabaseClientDependencyInjector { + fn new() -> Self { + Self { + http_client: MockAggregatorHTTPClient::new(), + immutable_files_downloader_resolver: ImmutablesFileDownloaderResolver::new( + vec![], + ), + } + } - #[tokio::test] - async fn get_cardano_database_snapshot_returns_message() { - let expected_cardano_database_snapshot = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - ..CardanoDatabaseSnapshot::dummy() - }; - let message = expected_cardano_database_snapshot.clone(); - let mut http_client = MockAggregatorHTTPClient::new(); - http_client - .expect_get_content() - .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - })) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); + fn with_http_client_mock_config(mut self, config: F) -> Self + where + F: FnOnce(&mut MockAggregatorHTTPClient), + { + config(&mut self.http_client); + + self + } + + fn with_immutable_file_downloaders( + self, + file_downloaders: Vec<(ImmutablesLocationDiscriminants, Arc)>, + ) -> Self { + let immutable_files_downloader_resolver = + ImmutablesFileDownloaderResolver::new(file_downloaders); - let cardano_database = client - .get("hash-123") - .await - .unwrap() - .expect("This test returns a Cardano database"); + Self { + immutable_files_downloader_resolver, + ..self + } + } - assert_eq!(expected_cardano_database_snapshot, cardano_database); + fn build_cardano_database_client(self) -> CardanoDatabaseClient { + CardanoDatabaseClient::new( + Arc::new(self.http_client), + Arc::new(self.immutable_files_downloader_resolver), + test_utils::test_logger(), + ) + } } - #[tokio::test] - async fn get_cardano_database_snapshot_returns_error_when_invalid_json_structure_in_response( - ) { - let mut http_client = MockAggregatorHTTPClient::new(); - http_client - .expect_get_content() - .return_once(move |_| Ok("invalid json structure".to_string())); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); - - client - .get("hash-123") - .await - .expect_err("Get Cardano database should return an error"); + mod list { + use super::*; + + #[tokio::test] + async fn list_cardano_database_snapshots_returns_messages() { + let message = fake_messages(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client + .expect_get_content() + .with(eq(AggregatorRequest::ListCardanoDatabaseSnapshots)) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + }) + .build_cardano_database_client(); + + let messages = client.list().await.unwrap(); + + assert_eq!(2, messages.len()); + assert_eq!("hash-123".to_string(), messages[0].hash); + assert_eq!("hash-456".to_string(), messages[1].hash); + } + + #[tokio::test] + async fn list_cardano_database_snapshots_returns_error_when_invalid_json_structure_in_response( + ) { + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client + .expect_get_content() + .return_once(move |_| Ok("invalid json structure".to_string())); + }) + .build_cardano_database_client(); + + client + .list() + .await + .expect_err("List Cardano databases should return an error"); + } } - #[tokio::test] - async fn get_cardano_database_snapshot_returns_none_when_not_found_or_remote_server_logical_error( - ) { - let mut http_client = MockAggregatorHTTPClient::new(); - http_client.expect_get_content().return_once(move |_| { - Err(AggregatorClientError::RemoteServerLogical(anyhow!( - "not found" - ))) - }); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); + mod get { + use super::*; + + #[tokio::test] + async fn get_cardano_database_snapshot_returns_message() { + let expected_cardano_database_snapshot = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshot::dummy() + }; + let message = expected_cardano_database_snapshot.clone(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client + .expect_get_content() + .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + })) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + }) + .build_cardano_database_client(); + + let cardano_database = client + .get("hash-123") + .await + .unwrap() + .expect("This test returns a Cardano database"); + + assert_eq!(expected_cardano_database_snapshot, cardano_database); + } + + #[tokio::test] + async fn get_cardano_database_snapshot_returns_error_when_invalid_json_structure_in_response( + ) { + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client + .expect_get_content() + .return_once(move |_| Ok("invalid json structure".to_string())); + }) + .build_cardano_database_client(); + + client + .get("hash-123") + .await + .expect_err("Get Cardano database should return an error"); + } + + #[tokio::test] + async fn get_cardano_database_snapshot_returns_none_when_not_found_or_remote_server_logical_error( + ) { + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client.expect_get_content().return_once(move |_| { + Err(AggregatorClientError::RemoteServerLogical(anyhow!( + "not found" + ))) + }); + }) + .build_cardano_database_client(); + + let result = client.get("hash-123").await.unwrap(); + + assert!(result.is_none()); + } - let result = client.get("hash-123").await.unwrap(); + #[tokio::test] + async fn get_cardano_database_snapshot_returns_error() { + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client.expect_get_content().return_once(move |_| { + Err(AggregatorClientError::SubsystemError(anyhow!("error"))) + }); + }) + .build_cardano_database_client(); - assert!(result.is_none()); + client + .get("hash-123") + .await + .expect_err("Get Cardano database should return an error"); + } } - #[tokio::test] - async fn get_cardano_database_snapshot_returns_error() { - let mut http_client = MockAggregatorHTTPClient::new(); - http_client - .expect_get_content() - .return_once(move |_| Err(AggregatorClientError::SubsystemError(anyhow!("error")))); - let client = CardanoDatabaseClient::new(Arc::new(http_client)); - - client - .get("hash-123") - .await - .expect_err("Get Cardano database should return an error"); + cfg_fs! { + mod download_unpack { + use std::fs; + use std::path::Path; + + use mithril_common::{ + entities::{FileUri, ImmutablesLocationDiscriminants, MultiFilesUri, TemplateUri}, + test_utils::TempDir, + }; + use mockall::predicate; + + use crate::file_downloader::MockFileDownloader; + + use super::*; + + #[tokio::test] + async fn download_unpack_fails_with_invalid_snapshot() { + let immutable_file_range = ImmutableFileRange::Range(1, 10); + let download_unpack_options = DownloadUnpackOptions::default(); + let cardano_db_snapshot_hash = &"hash-123"; + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client.expect_get_content().return_once(move |_| { + Err(AggregatorClientError::RemoteServerLogical(anyhow!( + "not found" + ))) + }); + }) + .build_cardano_database_client(); + + client + .download_unpack( + cardano_db_snapshot_hash, + immutable_file_range, + target_dir, + download_unpack_options, + ) + .await + .expect_err("download_unpack should fail"); + } + + #[tokio::test] + async fn download_unpack_fails_with_invalid_immutable_file_range() { + let immutable_file_range = ImmutableFileRange::Range(1, 0); + let download_unpack_options = DownloadUnpackOptions::default(); + let cardano_db_snapshot_hash = &"hash-123"; + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + let message = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshot::dummy() + }; + http_client + .expect_get_content() + .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + })) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + }) + .build_cardano_database_client(); + + client + .download_unpack( + cardano_db_snapshot_hash, + immutable_file_range, + target_dir, + download_unpack_options, + ) + .await + .expect_err("download_unpack should fail"); + } + + #[tokio::test] + async fn download_unpack_fails_when_immutable_files_download_fail() { + let total_immutable_files = 10; + let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); + let download_unpack_options = DownloadUnpackOptions::default(); + let cardano_db_snapshot_hash = &"hash-123"; + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + let mut message = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshot::dummy() + }; + message.locations.immutables = vec![ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }]; + http_client + .expect_get_content() + .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + })) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + }) + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .times(total_immutable_files as usize) + .returning(|_, _, _, _| Err(anyhow!("Download failed"))); + + mock_file_downloader + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack( + cardano_db_snapshot_hash, + immutable_file_range, + target_dir, + download_unpack_options, + ) + .await + .expect_err("download_unpack should fail"); + } + + #[tokio::test] + async fn download_unpack_fails_when_target_target_dir_would_be_overwritten_without_allow_override( + ) { + let immutable_file_range = ImmutableFileRange::Range(1, 10); + let download_unpack_options = DownloadUnpackOptions::default(); + let cardano_db_snapshot_hash = &"hash-123"; + let target_dir = &TempDir::new( + "cardano_database_client", + "download_unpack_fails_when_target_target_dir_would_be_overwritten_without_allow_override", + ) + .build(); + fs::create_dir_all(target_dir.join("immutable")).unwrap(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + let message = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshot::dummy() + }; + http_client + .expect_get_content() + .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + })) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + }) + .build_cardano_database_client(); + + client + .download_unpack( + cardano_db_snapshot_hash, + immutable_file_range, + target_dir, + download_unpack_options, + ) + .await + .expect_err("download_unpack should fail"); + } + + #[tokio::test] + async fn download_unpack_succeeds_with_valid_range() { + let immutable_file_range = ImmutableFileRange::Range(1, 2); + let download_unpack_options = DownloadUnpackOptions::default(); + let cardano_db_snapshot_hash = &"hash-123"; + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + let mut message = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshot::dummy() + }; + message.locations.immutables = vec![ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }]; + http_client + .expect_get_content() + .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + })) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + }) + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever/00001.tar.gz".to_string(), + ))), + eq(target_dir.join("immutable")), + eq(Some(CompressionAlgorithm::default())), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Ok(())); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever/00002.tar.gz".to_string(), + ))), + eq(target_dir.join("immutable")), + eq(Some(CompressionAlgorithm::default())), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Ok(())); + + mock_file_downloader + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack( + cardano_db_snapshot_hash, + immutable_file_range, + target_dir, + download_unpack_options, + ) + .await + .unwrap(); + } + } + + mod verify_can_write_to_target_dir { + use std::fs; + + use mithril_common::test_utils::TempDir; + + use super::*; + + #[test] + fn verify_can_write_to_target_dir_always_succeeds_with_allow_overwrite() { + let target_dir = TempDir::new( + "cardano_database_client", + "verify_can_write_to_target_dir_always_succeeds_with_allow_overwrite", + ) + .build(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .verify_can_write_to_target_dir( + &target_dir, + DownloadUnpackOptions { + allow_override: true, + include_ancillary: false, + }, + ) + .unwrap(); + + fs::create_dir_all(CardanoDatabaseClient::immutable_files_target_dir( + &target_dir, + )) + .unwrap(); + fs::create_dir_all(CardanoDatabaseClient::volatile_target_dir(&target_dir)) + .unwrap(); + fs::create_dir_all(CardanoDatabaseClient::ledger_target_dir(&target_dir)).unwrap(); + client + .verify_can_write_to_target_dir( + &target_dir, + DownloadUnpackOptions { + allow_override: true, + include_ancillary: false, + }, + ) + .unwrap(); + client + .verify_can_write_to_target_dir( + &target_dir, + DownloadUnpackOptions { + allow_override: true, + include_ancillary: true, + }, + ) + .unwrap(); + } + + #[test] + fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_immutable_target_dir( + ) { + let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_immutable_target_dir").build(); + fs::create_dir_all(CardanoDatabaseClient::immutable_files_target_dir( + &target_dir, + )) + .unwrap(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .verify_can_write_to_target_dir( + &target_dir, + DownloadUnpackOptions { + allow_override: false, + include_ancillary: false, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); + + client + .verify_can_write_to_target_dir( + &target_dir, + DownloadUnpackOptions { + allow_override: false, + include_ancillary: true, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); + } + + #[test] + fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_ledger_target_dir( + ) { + let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_ledger_target_dir").build(); + fs::create_dir_all(CardanoDatabaseClient::ledger_target_dir(&target_dir)).unwrap(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .verify_can_write_to_target_dir( + &target_dir, + DownloadUnpackOptions { + allow_override: false, + include_ancillary: true, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); + + client + .verify_can_write_to_target_dir( + &target_dir, + DownloadUnpackOptions { + allow_override: false, + include_ancillary: false, + }, + ) + .unwrap(); + } + + #[test] + fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_volatile_target_dir( + ) { + let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_volatile_target_dir").build(); + fs::create_dir_all(CardanoDatabaseClient::volatile_target_dir(&target_dir)) + .unwrap(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .verify_can_write_to_target_dir( + &target_dir, + DownloadUnpackOptions { + allow_override: false, + include_ancillary: true, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); + + client + .verify_can_write_to_target_dir( + &target_dir, + DownloadUnpackOptions { + allow_override: false, + include_ancillary: false, + }, + ) + .unwrap(); + } + } + + mod download_unpack_immutable_files { + use mithril_common::{ + entities::{FileUri, MultiFilesUri, TemplateUri}, + test_utils::TempDir, + }; + use mockall::predicate; + + use crate::file_downloader::MockFileDownloader; + + use super::*; + + #[tokio::test] + async fn download_unpack_immutable_files_fails_if_one_is_not_retrieved() { + let total_immutable_files = 2; + let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_immutable_files_succeeds", + ) + .build(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .times(1) + .returning(|_, _, _, _| Err(anyhow!("Download failed"))); + mock_file_downloader + .expect_download_unpack() + .times(1) + .returning(|_, _, _, _| Ok(())); + + mock_file_downloader + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack_immutable_files( + &[ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }], + immutable_file_range + .to_range_inclusive(total_immutable_files) + .unwrap(), + &CompressionAlgorithm::default(), + &target_dir, + ) + .await + .expect_err("download_unpack_immutable_files should fail"); + } + + #[tokio::test] + async fn download_unpack_immutable_files_succeeds_if_all_are_retrieved_with_same_location( + ) { + let total_immutable_files = 2; + let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_immutable_files_succeeds", + ) + .build(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .times(2) + .returning(|_, _, _, _| Ok(())); + + mock_file_downloader + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack_immutable_files( + &[ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever-1/{immutable_file_number}.tar.gz".to_string(), + )), + }], + immutable_file_range + .to_range_inclusive(total_immutable_files) + .unwrap(), + &CompressionAlgorithm::default(), + &target_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn download_unpack_immutable_files_succeeds_if_all_are_retrieved_with_different_locations( + ) { + let total_immutable_files = 2; + let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_immutable_files_succeeds", + ) + .build(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever-1/00001.tar.gz".to_string(), + ))), + eq(target_dir.clone()), + eq(Some(CompressionAlgorithm::default())), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Err(anyhow!("Download failed"))); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever-1/00002.tar.gz".to_string(), + ))), + eq(target_dir.clone()), + eq(Some(CompressionAlgorithm::default())), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Ok(())); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever-2/00001.tar.gz".to_string(), + ))), + eq(target_dir.clone()), + eq(Some(CompressionAlgorithm::default())), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Ok(())); + + mock_file_downloader + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack_immutable_files( + &[ + ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever-1/{immutable_file_number}.tar.gz".to_string(), + )), + }, + ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever-2/{immutable_file_number}.tar.gz".to_string(), + )), + }, + ], + immutable_file_range + .to_range_inclusive(total_immutable_files) + .unwrap(), + &CompressionAlgorithm::default(), + &target_dir, + ) + .await + .unwrap(); + } + } } - } - cfg_fs! { mod immutable_file_range { use super::*; @@ -326,7 +1123,9 @@ mod tests { let immutable_file_range = ImmutableFileRange::Full; let last_immutable_file_number = 10; - let result = immutable_file_range.to_range_inclusive(last_immutable_file_number).unwrap(); + let result = immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(); assert_eq!(0..=10, result); } @@ -335,11 +1134,15 @@ mod tests { let immutable_file_range = ImmutableFileRange::From(5); let last_immutable_file_number = 10; - let result = immutable_file_range.to_range_inclusive(last_immutable_file_number).unwrap(); + let result = immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(); assert_eq!(5..=10, result); let last_immutable_file_number = 3; - immutable_file_range.to_range_inclusive(last_immutable_file_number).expect_err("conversion to range inlusive should fail"); + immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .expect_err("conversion to range inlusive should fail"); } #[test] @@ -347,15 +1150,20 @@ mod tests { let immutable_file_range = ImmutableFileRange::Range(5, 8); let last_immutable_file_number = 10; - let result = immutable_file_range.to_range_inclusive(last_immutable_file_number).unwrap(); + let result = immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(); assert_eq!(5..=8, result); let last_immutable_file_number = 7; - immutable_file_range.to_range_inclusive(last_immutable_file_number).expect_err("conversion to range inlusive should fail"); + immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .expect_err("conversion to range inlusive should fail"); let immutable_file_range = ImmutableFileRange::Range(10, 8); - immutable_file_range.to_range_inclusive(last_immutable_file_number).expect_err("conversion to range inlusive should fail"); - + immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .expect_err("conversion to range inlusive should fail"); } #[test] @@ -363,11 +1171,15 @@ mod tests { let immutable_file_range = ImmutableFileRange::UpTo(8); let last_immutable_file_number = 10; - let result = immutable_file_range.to_range_inclusive(last_immutable_file_number).unwrap(); + let result = immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(); assert_eq!(0..=8, result); let last_immutable_file_number = 7; - immutable_file_range.to_range_inclusive(last_immutable_file_number).expect_err("conversion to range inlusive should fail"); + immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .expect_err("conversion to range inlusive should fail"); } } } diff --git a/mithril-client/src/client.rs b/mithril-client/src/client.rs index 1885e27cbfe..9fb746b75b1 100644 --- a/mithril-client/src/client.rs +++ b/mithril-client/src/client.rs @@ -18,6 +18,8 @@ use crate::certificate_client::{ CertificateClient, CertificateVerifier, MithrilCertificateVerifier, }; use crate::feedback::{FeedbackReceiver, FeedbackSender}; +#[cfg(all(feature = "fs", feature = "unstable"))] +use crate::file_downloader::ImmutablesFileDownloaderResolver; use crate::mithril_stake_distribution_client::MithrilStakeDistributionClient; use crate::snapshot_client::SnapshotClient; #[cfg(feature = "fs")] @@ -256,12 +258,20 @@ impl ClientBuilder { #[cfg(feature = "fs")] feedback_sender, #[cfg(feature = "fs")] - logger, + logger.clone(), )); + #[cfg(all(feature = "fs", feature = "unstable"))] + let immutable_file_downloader_resolver = + Arc::new(ImmutablesFileDownloaderResolver::new(Vec::new())); #[cfg(feature = "unstable")] - let cardano_database_client = - Arc::new(CardanoDatabaseClient::new(aggregator_client.clone())); + let cardano_database_client = Arc::new(CardanoDatabaseClient::new( + aggregator_client.clone(), + #[cfg(feature = "fs")] + immutable_file_downloader_resolver, + #[cfg(feature = "fs")] + logger, + )); let cardano_transaction_client = Arc::new(CardanoTransactionClient::new(aggregator_client.clone())); @@ -299,27 +309,27 @@ impl ClientBuilder { } cfg_unstable! { - /// Set the [CertificateVerifierCache] that will be used to cache certificate validation results. - /// - /// Passing a `None` value will disable the cache if any was previously set. - pub fn with_certificate_verifier_cache( - mut self, - certificate_verifier_cache: Option>, - ) -> ClientBuilder { - self.certificate_verifier_cache = certificate_verifier_cache; - self - } + /// Set the [CertificateVerifierCache] that will be used to cache certificate validation results. + /// + /// Passing a `None` value will disable the cache if any was previously set. + pub fn with_certificate_verifier_cache( + mut self, + certificate_verifier_cache: Option>, + ) -> ClientBuilder { + self.certificate_verifier_cache = certificate_verifier_cache; + self + } } cfg_fs! { - /// Set the [SnapshotDownloader] that will be used to download snapshots. - pub fn with_snapshot_downloader( - mut self, - snapshot_downloader: Arc, - ) -> ClientBuilder { - self.snapshot_downloader = Some(snapshot_downloader); - self - } + /// Set the [SnapshotDownloader] that will be used to download snapshots. + pub fn with_snapshot_downloader( + mut self, + snapshot_downloader: Arc, + ) -> ClientBuilder { + self.snapshot_downloader = Some(snapshot_downloader); + self + } } /// Set the [Logger] to use. From 3e317dde7f4951840381b747914fbb8a8bdc44c8 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 31 Jan 2025 18:05:07 +0100 Subject: [PATCH 08/59] chore: add discriminant for 'DigestLocation' --- mithril-common/src/entities/cardano_database.rs | 15 +++++++++------ mithril-common/src/entities/mod.rs | 3 ++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mithril-common/src/entities/cardano_database.rs b/mithril-common/src/entities/cardano_database.rs index 4306d895d2f..1771622cedd 100644 --- a/mithril-common/src/entities/cardano_database.rs +++ b/mithril-common/src/entities/cardano_database.rs @@ -68,19 +68,22 @@ impl CardanoDatabaseSnapshot { } /// Locations of the immutable file digests. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[derive( + Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, EnumDiscriminants, +)] #[serde(rename_all = "snake_case", tag = "type")] +#[strum_discriminants(derive(Hash))] pub enum DigestLocation { - /// Aggregator digest route location. - Aggregator { - /// URI of the aggregator digests route location. - uri: String, - }, /// Cloud storage location. CloudStorage { /// URI of the cloud storage location. uri: String, }, + /// Aggregator digest route location. + Aggregator { + /// URI of the aggregator digests route location. + uri: String, + }, } /// Locations of the immutable files. diff --git a/mithril-common/src/entities/mod.rs b/mithril-common/src/entities/mod.rs index d5ef53b1d94..aede13ff17f 100644 --- a/mithril-common/src/entities/mod.rs +++ b/mithril-common/src/entities/mod.rs @@ -35,7 +35,8 @@ pub use block_range::{BlockRange, BlockRangeLength, BlockRangesSequence}; pub use cardano_chain_point::{BlockHash, ChainPoint}; pub use cardano_database::{ AncillaryLocation, AncillaryLocationDiscriminants, ArtifactsLocations, CardanoDatabaseSnapshot, - DigestLocation, ImmutablesLocation, ImmutablesLocationDiscriminants, + DigestLocation, DigestLocationDiscriminants, ImmutablesLocation, + ImmutablesLocationDiscriminants, }; pub use cardano_db_beacon::CardanoDbBeacon; pub use cardano_network::CardanoNetwork; From a9b70cbc33bdcc68fbbebf2c7780ba9e21f81eb9 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 31 Jan 2025 18:06:33 +0100 Subject: [PATCH 09/59] feat: implement a file downloader resolver for digests --- .../src/file_downloader/interface.rs | 14 +++- mithril-client/src/file_downloader/mod.rs | 4 +- .../src/file_downloader/resolver.rs | 64 ++++++++++++++++++- 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/mithril-client/src/file_downloader/interface.rs b/mithril-client/src/file_downloader/interface.rs index 4844a998b55..c49cfb5621d 100644 --- a/mithril-client/src/file_downloader/interface.rs +++ b/mithril-client/src/file_downloader/interface.rs @@ -3,7 +3,9 @@ use std::{collections::HashMap, path::Path}; use async_trait::async_trait; use mithril_common::{ - entities::{CompressionAlgorithm, FileUri, ImmutableFileNumber, ImmutablesLocation}, + entities::{ + CompressionAlgorithm, DigestLocation, FileUri, ImmutableFileNumber, ImmutablesLocation, + }, StdResult, }; @@ -56,6 +58,16 @@ impl From for FileDownloaderUri { } } +impl From for FileDownloaderUri { + fn from(digest_location: DigestLocation) -> Self { + match digest_location { + DigestLocation::CloudStorage { uri } | DigestLocation::Aggregator { uri } => { + Self::FileUri(FileUri(uri)) + } + } + } +} + /// A file downloader #[cfg_attr(test, mockall::automock)] #[async_trait] diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs index 7762e56b572..d9cee950041 100644 --- a/mithril-client/src/file_downloader/mod.rs +++ b/mithril-client/src/file_downloader/mod.rs @@ -10,4 +10,6 @@ pub use interface::MockFileDownloader; pub use interface::{FileDownloader, FileDownloaderUri}; #[cfg(test)] pub use resolver::MockFileDownloaderResolver; -pub use resolver::{FileDownloaderResolver, ImmutablesFileDownloaderResolver}; +pub use resolver::{ + DigestFileDownloaderResolver, FileDownloaderResolver, ImmutablesFileDownloaderResolver, +}; diff --git a/mithril-client/src/file_downloader/resolver.rs b/mithril-client/src/file_downloader/resolver.rs index 6526aa0730b..64e16d4b104 100644 --- a/mithril-client/src/file_downloader/resolver.rs +++ b/mithril-client/src/file_downloader/resolver.rs @@ -1,6 +1,9 @@ use std::{collections::HashMap, sync::Arc}; -use mithril_common::entities::{ImmutablesLocation, ImmutablesLocationDiscriminants}; +use mithril_common::entities::{ + DigestLocation, DigestLocationDiscriminants, ImmutablesLocation, + ImmutablesLocationDiscriminants, +}; use super::FileDownloader; @@ -33,6 +36,28 @@ impl FileDownloaderResolver for ImmutablesFileDownloaderReso } } +/// A file downloader resolver for digests file locations +pub struct DigestFileDownloaderResolver { + file_downloaders: HashMap>, +} + +impl DigestFileDownloaderResolver { + /// Constructs a new `DigestFileDownloaderResolver`. + pub fn new( + file_downloaders: Vec<(DigestLocationDiscriminants, Arc)>, + ) -> Self { + let file_downloaders = file_downloaders.into_iter().collect(); + + Self { file_downloaders } + } +} + +impl FileDownloaderResolver for DigestFileDownloaderResolver { + fn resolve(&self, location: &DigestLocation) -> Option> { + self.file_downloaders.get(&location.into()).cloned() + } +} + #[cfg(test)] mod tests { use std::path::Path; @@ -44,7 +69,7 @@ mod tests { use super::*; #[tokio::test] - async fn resolves_file_downloader() { + async fn immutables_file_downloader_resolver() { let mut mock_file_downloader = MockFileDownloader::new(); mock_file_downloader .expect_download_unpack() @@ -72,4 +97,39 @@ mod tests { .await .unwrap(); } + + #[tokio::test] + async fn digest_file_downloader_resolver() { + let mock_file_downloader_aggregator = MockFileDownloader::new(); + let mut mock_file_downloader_cloud_storage = MockFileDownloader::new(); + mock_file_downloader_cloud_storage + .expect_download_unpack() + .times(1) + .returning(|_, _, _, _| Ok(())); + let resolver = DigestFileDownloaderResolver::new(vec![ + ( + DigestLocationDiscriminants::Aggregator, + Arc::new(mock_file_downloader_aggregator), + ), + ( + DigestLocationDiscriminants::CloudStorage, + Arc::new(mock_file_downloader_cloud_storage), + ), + ]); + + let file_downloader = resolver + .resolve(&DigestLocation::CloudStorage { + uri: "http://whatever/00001.tar.gz".to_string(), + }) + .unwrap(); + file_downloader + .download_unpack( + &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), + Path::new("."), + None, + "download_id", + ) + .await + .unwrap(); + } } From f503f59757100c18c76efa213a73651c6e981cd6 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 31 Jan 2025 18:09:21 +0100 Subject: [PATCH 10/59] feat: implement digests file download in Cardano database client --- mithril-client/src/cardano_database_client.rs | 488 +++++++++++++++--- mithril-client/src/client.rs | 7 +- 2 files changed, 427 insertions(+), 68 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index a93ac228689..122ab152aea 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -43,22 +43,27 @@ //! # } //! ``` +// TODO: reorganize the imports #[cfg(feature = "fs")] -use anyhow::anyhow; -use anyhow::Context; -use mithril_common::entities::CompressionAlgorithm; -#[cfg(feature = "fs")] -use slog::Logger; +use std::fs; #[cfg(feature = "fs")] use std::ops::RangeInclusive; #[cfg(feature = "fs")] use std::path::{Path, PathBuf}; use std::{collections::HashSet, sync::Arc}; +#[cfg(feature = "fs")] +use anyhow::anyhow; +use anyhow::Context; +#[cfg(feature = "fs")] +use slog::Logger; + #[cfg(feature = "fs")] use mithril_common::{ -use mithril_common::{entities::ImmutableFileNumber, StdResult}; - entities::{ImmutableFileNumber, ImmutablesLocation}, + entities::{ + AncillaryLocation, CompressionAlgorithm, DigestLocation, ImmutableFileNumber, + ImmutablesLocation, + }, StdResult, }; @@ -130,7 +135,9 @@ cfg_fs! { pub struct CardanoDatabaseClient { aggregator_client: Arc, #[cfg(feature = "fs")] - immutable_files_downloader_resolver: Arc>, + immutable_file_downloader_resolver: Arc>, + #[cfg(feature = "fs")] + digest_file_downloader_resolver: Arc>, #[cfg(feature = "fs")] logger: Logger, } @@ -139,15 +146,20 @@ impl CardanoDatabaseClient { /// Constructs a new `CardanoDatabase`. pub fn new( aggregator_client: Arc, - #[cfg(feature = "fs")] immutable_files_downloader_resolver: Arc< + #[cfg(feature = "fs")] immutable_file_downloader_resolver: Arc< dyn FileDownloaderResolver, >, + #[cfg(feature = "fs")] digest_file_downloader_resolver: Arc< + dyn FileDownloaderResolver, + >, #[cfg(feature = "fs")] logger: Logger, ) -> Self { Self { aggregator_client, #[cfg(feature = "fs")] - immutable_files_downloader_resolver, + immutable_file_downloader_resolver, + #[cfg(feature = "fs")] + digest_file_downloader_resolver, #[cfg(feature = "fs")] logger: mithril_common::logging::LoggerExtensions::new_with_component_name::( &logger, @@ -201,7 +213,7 @@ impl CardanoDatabaseClient { &self, hash: &str, immutable_file_range: ImmutableFileRange, - target_dir: &std::path::Path, + target_dir: &Path, download_unpack_options: DownloadUnpackOptions, ) -> StdResult<()> { let cardano_database_snapshot = self @@ -213,7 +225,11 @@ impl CardanoDatabaseClient { let immutable_file_number_range = immutable_file_range.to_range_inclusive(last_immutable_file_number)?; - self.verify_can_write_to_target_dir(target_dir, download_unpack_options)?; + self.verify_can_write_to_target_directory(target_dir, &download_unpack_options)?; + self.create_target_directory_sub_directories_if_not_exist( + target_dir, + &download_unpack_options, + )?; let immutable_locations = cardano_database_snapshot.locations.immutables; self.download_unpack_immutable_files( @@ -224,9 +240,20 @@ impl CardanoDatabaseClient { ) .await?; + let digest_locations = cardano_database_snapshot.locations.digests; + self.download_unpack_digest_file(&digest_locations, &Self::digest_target_dir()) + .await?; + Ok(()) } + fn digest_target_dir() -> PathBuf { + std::env::temp_dir() + .join("mithril") + .join("cardano_database_client") + .join("digest") + } + fn immutable_files_target_dir(target_dir: &Path) -> PathBuf { target_dir.join("immutable") } @@ -239,25 +266,36 @@ impl CardanoDatabaseClient { target_dir.join("ledger") } + fn create_directory_if_not_exists(dir: &Path) -> StdResult<()> { + if dir.exists() { + return Ok(()); + } + + fs::create_dir_all(dir).map_err(|e| anyhow!("Failed creating directory: {e}")) + } + /// Verify if the target directory is writable. - pub(crate) fn verify_can_write_to_target_dir( + fn verify_can_write_to_target_directory( &self, - target_dir: &std::path::Path, - download_unpack_options: DownloadUnpackOptions, + target_dir: &Path, + download_unpack_options: &DownloadUnpackOptions, ) -> StdResult<()> { + let immutable_files_target_dir = Self::immutable_files_target_dir(target_dir); + let volatile_target_dir = Self::volatile_target_dir(target_dir); + let ledger_target_dir = Self::ledger_target_dir(target_dir); if !download_unpack_options.allow_override { - if Self::immutable_files_target_dir(target_dir).exists() { + if immutable_files_target_dir.exists() { return Err(anyhow!( "Immutable files target directory already exists in: {target_dir:?}" )); } if download_unpack_options.include_ancillary { - if Self::volatile_target_dir(target_dir).exists() { + if volatile_target_dir.exists() { return Err(anyhow!( "Volatile target directory already exists in: {target_dir:?}" )); } - if Self::ledger_target_dir(target_dir).exists() { + if ledger_target_dir.exists() { return Err(anyhow!( "Ledger target directory already exists in: {target_dir:?}" )); @@ -268,6 +306,25 @@ impl CardanoDatabaseClient { Ok(()) } + /// Create the target directory sub-directories if they do not exist. + // TODO: is it really needed? + fn create_target_directory_sub_directories_if_not_exist( + &self, + target_dir: &Path, + download_unpack_options: &DownloadUnpackOptions, + ) -> StdResult<()> { + let immutable_files_target_dir = Self::immutable_files_target_dir(target_dir); + Self::create_directory_if_not_exists(&immutable_files_target_dir)?; + if download_unpack_options.include_ancillary { + let volatile_target_dir = Self::volatile_target_dir(target_dir); + let ledger_target_dir = Self::ledger_target_dir(target_dir); + Self::create_directory_if_not_exists(&volatile_target_dir)?; + Self::create_directory_if_not_exists(&ledger_target_dir)?; + } + + Ok(()) + } + /// Download and unpack the immutable files of the given range. /// /// The download is attempted for each location until the full range is downloaded. @@ -278,7 +335,7 @@ impl CardanoDatabaseClient { locations: &[ImmutablesLocation], range: RangeInclusive, compression_algorithm: &CompressionAlgorithm, - immutable_files_target_dir: &std::path::Path, + immutable_files_target_dir: &Path, ) -> StdResult<()> { let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); @@ -286,7 +343,7 @@ impl CardanoDatabaseClient { range.clone().map(|n| n.to_owned()).collect::>(); for location in locations_sorted { let file_downloader = self - .immutable_files_downloader_resolver + .immutable_file_downloader_resolver .resolve(&location) .ok_or_else(|| { anyhow!("Failed resolving a file downloader for location: {location:?}") @@ -302,22 +359,24 @@ impl CardanoDatabaseClient { )?; for (immutable_file_number, file_downloader_uri) in file_downloader_uris { let download_id = format!("{location:?}"); //TODO: check if this is the correct way to format the download_id - if file_downloader + let downloaded = file_downloader .download_unpack( &file_downloader_uri, - &immutable_files_target_dir, + immutable_files_target_dir, Some(compression_algorithm.to_owned()), &download_id, ) - .await - .is_ok() - { - immutable_file_numbers_to_download.remove(&immutable_file_number); - } else { - slog::error!( + .await; + match downloaded { + Ok(_) => { + immutable_file_numbers_to_download.remove(&immutable_file_number); + } + Err(e) => { + slog::error!( self.logger, - "Failed downloading and unpacking immutable files for location: {file_downloader_uri:?}" + "Failed downloading and unpacking immutable files for location {file_downloader_uri:?}"; "error" => e.to_string() ); + } } } if immutable_file_numbers_to_download.is_empty() { @@ -329,6 +388,46 @@ impl CardanoDatabaseClient { "Failed downloading and unpacking immutable files for locations: {locations:?}" )) } + + async fn download_unpack_digest_file( + &self, + locations: &[DigestLocation], + digest_file_target_dir: &Path, + ) -> StdResult<()> { + let mut locations_sorted = locations.to_owned(); + locations_sorted.sort(); + for location in locations_sorted { + let download_id = format!("{location:?}"); //TODO: check if this is the correct way to format the download_id + let file_downloader = self + .digest_file_downloader_resolver + .resolve(&location) + .ok_or_else(|| { + anyhow!("Failed resolving a file downloader for location: {location:?}") + })?; + let file_downloader_uri: FileDownloaderUri = location.into(); + let downloaded = file_downloader + .download_unpack( + &file_downloader_uri, + digest_file_target_dir, + None, + &download_id, + ) + .await; + match downloaded { + Ok(_) => return Ok(()), + Err(e) => { + slog::error!( + self.logger, + "Failed downloading and unpacking digest for location {file_downloader_uri:?}"; "error" => e.to_string() + ); + } + } + } + + Err(anyhow!( + "Failed downloading and unpacking digests for all locations" + )) + } } } @@ -341,13 +440,16 @@ mod tests { use anyhow::anyhow; use chrono::{DateTime, Utc}; use mithril_common::entities::{ - CardanoDbBeacon, CompressionAlgorithm, Epoch, ImmutablesLocationDiscriminants, + CardanoDbBeacon, CompressionAlgorithm, DigestLocationDiscriminants, Epoch, + ImmutablesLocationDiscriminants, }; use mockall::predicate::eq; use crate::{ aggregator_client::MockAggregatorHTTPClient, - file_downloader::{FileDownloader, ImmutablesFileDownloaderResolver}, + file_downloader::{ + DigestFileDownloaderResolver, FileDownloader, ImmutablesFileDownloaderResolver, + }, test_utils, }; @@ -390,16 +492,18 @@ mod tests { struct CardanoDatabaseClientDependencyInjector { http_client: MockAggregatorHTTPClient, - immutable_files_downloader_resolver: ImmutablesFileDownloaderResolver, + immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver, + digest_file_downloader_resolver: DigestFileDownloaderResolver, } impl CardanoDatabaseClientDependencyInjector { fn new() -> Self { Self { http_client: MockAggregatorHTTPClient::new(), - immutable_files_downloader_resolver: ImmutablesFileDownloaderResolver::new( + immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver::new( vec![], ), + digest_file_downloader_resolver: DigestFileDownloaderResolver::new(vec![]), } } @@ -416,11 +520,24 @@ mod tests { self, file_downloaders: Vec<(ImmutablesLocationDiscriminants, Arc)>, ) -> Self { - let immutable_files_downloader_resolver = + let immutable_file_downloader_resolver = ImmutablesFileDownloaderResolver::new(file_downloaders); Self { - immutable_files_downloader_resolver, + immutable_file_downloader_resolver, + ..self + } + } + + fn with_digest_file_downloaders( + self, + file_downloaders: Vec<(DigestLocationDiscriminants, Arc)>, + ) -> Self { + let digest_file_downloader_resolver = + DigestFileDownloaderResolver::new(file_downloaders); + + Self { + digest_file_downloader_resolver, ..self } } @@ -428,7 +545,8 @@ mod tests { fn build_cardano_database_client(self) -> CardanoDatabaseClient { CardanoDatabaseClient::new( Arc::new(self.http_client), - Arc::new(self.immutable_files_downloader_resolver), + Arc::new(self.immutable_file_downloader_resolver), + Arc::new(self.digest_file_downloader_resolver), test_utils::test_logger(), ) } @@ -636,7 +754,11 @@ mod tests { let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); let download_unpack_options = DownloadUnpackOptions::default(); let cardano_db_snapshot_hash = &"hash-123"; - let target_dir = Path::new("."); + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_fails_when_immutable_files_download_fail", + ) + .build(); let client = CardanoDatabaseClientDependencyInjector::new() .with_http_client_mock_config(|http_client| { let mut message = CardanoDatabaseSnapshot { @@ -673,7 +795,7 @@ mod tests { .download_unpack( cardano_db_snapshot_hash, immutable_file_range, - target_dir, + &target_dir, download_unpack_options, ) .await @@ -723,18 +845,25 @@ mod tests { let immutable_file_range = ImmutableFileRange::Range(1, 2); let download_unpack_options = DownloadUnpackOptions::default(); let cardano_db_snapshot_hash = &"hash-123"; - let target_dir = Path::new("."); + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_succeeds_with_valid_range", + ) + .build(); + let mut message = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshot::dummy() + }; + message.locations.immutables = vec![ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }]; + message.locations.digests = vec![DigestLocation::CloudStorage { + uri: "http://whatever/digest.txt".to_string(), + }]; let client = CardanoDatabaseClientDependencyInjector::new() .with_http_client_mock_config(|http_client| { - let mut message = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - ..CardanoDatabaseSnapshot::dummy() - }; - message.locations.immutables = vec![ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever/{immutable_file_number}.tar.gz".to_string(), - )), - }]; http_client .expect_get_content() .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { @@ -774,13 +903,33 @@ mod tests { mock_file_downloader }), )]) + .with_digest_file_downloaders(vec![( + DigestLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever/digest.txt".to_string(), + ))), + predicate::always(), + eq(None), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Ok(())); + + mock_file_downloader + }), + )]) .build_cardano_database_client(); client .download_unpack( cardano_db_snapshot_hash, immutable_file_range, - target_dir, + &target_dir, download_unpack_options, ) .await @@ -806,9 +955,9 @@ mod tests { CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); client - .verify_can_write_to_target_dir( + .verify_can_write_to_target_directory( &target_dir, - DownloadUnpackOptions { + &DownloadUnpackOptions { allow_override: true, include_ancillary: false, }, @@ -823,18 +972,18 @@ mod tests { .unwrap(); fs::create_dir_all(CardanoDatabaseClient::ledger_target_dir(&target_dir)).unwrap(); client - .verify_can_write_to_target_dir( + .verify_can_write_to_target_directory( &target_dir, - DownloadUnpackOptions { + &DownloadUnpackOptions { allow_override: true, include_ancillary: false, }, ) .unwrap(); client - .verify_can_write_to_target_dir( + .verify_can_write_to_target_directory( &target_dir, - DownloadUnpackOptions { + &DownloadUnpackOptions { allow_override: true, include_ancillary: true, }, @@ -854,9 +1003,9 @@ mod tests { CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); client - .verify_can_write_to_target_dir( + .verify_can_write_to_target_directory( &target_dir, - DownloadUnpackOptions { + &DownloadUnpackOptions { allow_override: false, include_ancillary: false, }, @@ -864,9 +1013,9 @@ mod tests { .expect_err("verify_can_write_to_target_dir should fail"); client - .verify_can_write_to_target_dir( + .verify_can_write_to_target_directory( &target_dir, - DownloadUnpackOptions { + &DownloadUnpackOptions { allow_override: false, include_ancillary: true, }, @@ -883,9 +1032,9 @@ mod tests { CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); client - .verify_can_write_to_target_dir( + .verify_can_write_to_target_directory( &target_dir, - DownloadUnpackOptions { + &DownloadUnpackOptions { allow_override: false, include_ancillary: true, }, @@ -893,9 +1042,9 @@ mod tests { .expect_err("verify_can_write_to_target_dir should fail"); client - .verify_can_write_to_target_dir( + .verify_can_write_to_target_directory( &target_dir, - DownloadUnpackOptions { + &DownloadUnpackOptions { allow_override: false, include_ancillary: false, }, @@ -913,9 +1062,9 @@ mod tests { CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); client - .verify_can_write_to_target_dir( + .verify_can_write_to_target_directory( &target_dir, - DownloadUnpackOptions { + &DownloadUnpackOptions { allow_override: false, include_ancillary: true, }, @@ -923,9 +1072,9 @@ mod tests { .expect_err("verify_can_write_to_target_dir should fail"); client - .verify_can_write_to_target_dir( + .verify_can_write_to_target_directory( &target_dir, - DownloadUnpackOptions { + &DownloadUnpackOptions { allow_override: false, include_ancillary: false, }, @@ -934,6 +1083,68 @@ mod tests { } } + mod create_target_directory_sub_directories_if_not_exist { + use mithril_common::test_utils::TempDir; + + use super::*; + + #[test] + fn create_target_directory_sub_directories_if_not_exist_without_ancillary() { + let target_dir = TempDir::new( + "cardano_database_client", + "create_target_directory_sub_directories_if_not_exist_without_ancillary", + ) + .build(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + assert!(!target_dir.join("immutable").exists()); + assert!(!target_dir.join("volatile").exists()); + assert!(!target_dir.join("ledger").exists()); + + client + .create_target_directory_sub_directories_if_not_exist( + &target_dir, + &DownloadUnpackOptions { + include_ancillary: false, + ..Default::default() + }, + ) + .unwrap(); + + assert!(target_dir.join("immutable").exists()); + assert!(!target_dir.join("volatile").exists()); + assert!(!target_dir.join("ledger").exists()); + } + + #[test] + fn create_target_directory_sub_directories_if_not_exist_with_ancillary() { + let target_dir = TempDir::new( + "cardano_database_client", + "create_target_directory_sub_directories_if_not_exist_with_ancillary", + ) + .build(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + assert!(!target_dir.join("immutable").exists()); + assert!(!target_dir.join("volatile").exists()); + assert!(!target_dir.join("ledger").exists()); + + client + .create_target_directory_sub_directories_if_not_exist( + &target_dir, + &DownloadUnpackOptions { + include_ancillary: true, + ..Default::default() + }, + ) + .unwrap(); + + assert!(target_dir.join("immutable").exists()); + assert!(target_dir.join("volatile").exists()); + assert!(target_dir.join("ledger").exists()); + } + } + mod download_unpack_immutable_files { use mithril_common::{ entities::{FileUri, MultiFilesUri, TemplateUri}, @@ -1113,6 +1324,149 @@ mod tests { .unwrap(); } } + + mod download_unpack_digest_file { + + use crate::file_downloader::MockFileDownloader; + + use super::*; + + #[tokio::test] + async fn download_unpack_digest_file_fails_if_no_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_digest_file_downloaders(vec![ + ( + DigestLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .times(1) + .returning(|_, _, _, _| Err(anyhow!("Download failed"))); + + mock_file_downloader + }), + ), + ( + DigestLocationDiscriminants::Aggregator, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .times(1) + .returning(|_, _, _, _| Err(anyhow!("Download failed"))); + + mock_file_downloader + }), + ), + ]) + .build_cardano_database_client(); + + client + .download_unpack_digest_file( + &[ + DigestLocation::CloudStorage { + uri: "http://whatever-1/digest.txt".to_string(), + }, + DigestLocation::Aggregator { + uri: "http://whatever-2/digest".to_string(), + }, + ], + &target_dir, + ) + .await + .expect_err("download_unpack_digest_file should fail"); + } + + #[tokio::test] + async fn download_unpack_digest_file_succeeds_if_at_least_one_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_digest_file_downloaders(vec![ + ( + DigestLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .times(1) + .returning(|_, _, _, _| Err(anyhow!("Download failed"))); + + mock_file_downloader + }), + ), + ( + DigestLocationDiscriminants::Aggregator, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .times(1) + .returning(|_, _, _, _| Ok(())); + + mock_file_downloader + }), + ), + ]) + .build_cardano_database_client(); + + client + .download_unpack_digest_file( + &[ + DigestLocation::CloudStorage { + uri: "http://whatever-1/digest.txt".to_string(), + }, + DigestLocation::Aggregator { + uri: "http://whatever-2/digest".to_string(), + }, + ], + &target_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn download_unpack_digest_file_succeeds_when_first_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_digest_file_downloaders(vec![ + ( + DigestLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .times(1) + .returning(|_, _, _, _| Ok(())); + + mock_file_downloader + }), + ), + ( + DigestLocationDiscriminants::Aggregator, + Arc::new(MockFileDownloader::new()), + ), + ]) + .build_cardano_database_client(); + + client + .download_unpack_digest_file( + &[ + DigestLocation::CloudStorage { + uri: "http://whatever-1/digest.txt".to_string(), + }, + DigestLocation::Aggregator { + uri: "http://whatever-2/digest".to_string(), + }, + ], + &target_dir, + ) + .await + .unwrap(); + } + } } mod immutable_file_range { diff --git a/mithril-client/src/client.rs b/mithril-client/src/client.rs index 9fb746b75b1..5e64c4fca9c 100644 --- a/mithril-client/src/client.rs +++ b/mithril-client/src/client.rs @@ -19,7 +19,7 @@ use crate::certificate_client::{ }; use crate::feedback::{FeedbackReceiver, FeedbackSender}; #[cfg(all(feature = "fs", feature = "unstable"))] -use crate::file_downloader::ImmutablesFileDownloaderResolver; +use crate::file_downloader::{DigestFileDownloaderResolver, ImmutablesFileDownloaderResolver}; use crate::mithril_stake_distribution_client::MithrilStakeDistributionClient; use crate::snapshot_client::SnapshotClient; #[cfg(feature = "fs")] @@ -264,12 +264,17 @@ impl ClientBuilder { #[cfg(all(feature = "fs", feature = "unstable"))] let immutable_file_downloader_resolver = Arc::new(ImmutablesFileDownloaderResolver::new(Vec::new())); + #[cfg(all(feature = "fs", feature = "unstable"))] + let digest_file_downloader_resolver = + Arc::new(DigestFileDownloaderResolver::new(Vec::new())); #[cfg(feature = "unstable")] let cardano_database_client = Arc::new(CardanoDatabaseClient::new( aggregator_client.clone(), #[cfg(feature = "fs")] immutable_file_downloader_resolver, #[cfg(feature = "fs")] + digest_file_downloader_resolver, + #[cfg(feature = "fs")] logger, )); From 7ba25cd24529b061f8a17b820e61c83e9ff3ae50 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 4 Feb 2025 11:45:22 +0100 Subject: [PATCH 11/59] chore: add discriminant for 'AncillaryLocation' --- mithril-common/src/entities/cardano_database.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mithril-common/src/entities/cardano_database.rs b/mithril-common/src/entities/cardano_database.rs index 1771622cedd..215244e2f15 100644 --- a/mithril-common/src/entities/cardano_database.rs +++ b/mithril-common/src/entities/cardano_database.rs @@ -101,8 +101,11 @@ pub enum ImmutablesLocation { } /// Locations of the ancillary files. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[derive( + Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, EnumDiscriminants, +)] #[serde(rename_all = "snake_case", tag = "type")] +#[strum_discriminants(derive(Hash))] pub enum AncillaryLocation { /// Cloud storage location. CloudStorage { From 57b4ee48150c2d9471ea2538bbd9d65b89792bb1 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 3 Feb 2025 12:38:55 +0100 Subject: [PATCH 12/59] feat: implement a file downloader resolver for ancillary file --- .../src/file_downloader/interface.rs | 11 +++- mithril-client/src/file_downloader/mod.rs | 3 +- .../src/file_downloader/resolver.rs | 54 ++++++++++++++++++- 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/mithril-client/src/file_downloader/interface.rs b/mithril-client/src/file_downloader/interface.rs index c49cfb5621d..7e0d268def5 100644 --- a/mithril-client/src/file_downloader/interface.rs +++ b/mithril-client/src/file_downloader/interface.rs @@ -4,7 +4,8 @@ use async_trait::async_trait; use mithril_common::{ entities::{ - CompressionAlgorithm, DigestLocation, FileUri, ImmutableFileNumber, ImmutablesLocation, + AncillaryLocation, CompressionAlgorithm, DigestLocation, FileUri, ImmutableFileNumber, + ImmutablesLocation, }, StdResult, }; @@ -58,6 +59,14 @@ impl From for FileDownloaderUri { } } +impl From for FileDownloaderUri { + fn from(digest_location: AncillaryLocation) -> Self { + match digest_location { + AncillaryLocation::CloudStorage { uri } => Self::FileUri(FileUri(uri)), + } + } +} + impl From for FileDownloaderUri { fn from(digest_location: DigestLocation) -> Self { match digest_location { diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs index d9cee950041..010463b4310 100644 --- a/mithril-client/src/file_downloader/mod.rs +++ b/mithril-client/src/file_downloader/mod.rs @@ -11,5 +11,6 @@ pub use interface::{FileDownloader, FileDownloaderUri}; #[cfg(test)] pub use resolver::MockFileDownloaderResolver; pub use resolver::{ - DigestFileDownloaderResolver, FileDownloaderResolver, ImmutablesFileDownloaderResolver, + AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloaderResolver, + ImmutablesFileDownloaderResolver, }; diff --git a/mithril-client/src/file_downloader/resolver.rs b/mithril-client/src/file_downloader/resolver.rs index 64e16d4b104..12b6c848c95 100644 --- a/mithril-client/src/file_downloader/resolver.rs +++ b/mithril-client/src/file_downloader/resolver.rs @@ -1,8 +1,8 @@ use std::{collections::HashMap, sync::Arc}; use mithril_common::entities::{ - DigestLocation, DigestLocationDiscriminants, ImmutablesLocation, - ImmutablesLocationDiscriminants, + AncillaryLocation, AncillaryLocationDiscriminants, DigestLocation, DigestLocationDiscriminants, + ImmutablesLocation, ImmutablesLocationDiscriminants, }; use super::FileDownloader; @@ -36,6 +36,28 @@ impl FileDownloaderResolver for ImmutablesFileDownloaderReso } } +/// A file downloader resolver for ancillary file locations +pub struct AncillaryFileDownloaderResolver { + file_downloaders: HashMap>, +} + +impl AncillaryFileDownloaderResolver { + /// Constructs a new `AncillaryFileDownloaderResolver`. + pub fn new( + file_downloaders: Vec<(AncillaryLocationDiscriminants, Arc)>, + ) -> Self { + let file_downloaders = file_downloaders.into_iter().collect(); + + Self { file_downloaders } + } +} + +impl FileDownloaderResolver for AncillaryFileDownloaderResolver { + fn resolve(&self, location: &AncillaryLocation) -> Option> { + self.file_downloaders.get(&location.into()).cloned() + } +} + /// A file downloader resolver for digests file locations pub struct DigestFileDownloaderResolver { file_downloaders: HashMap>, @@ -98,6 +120,34 @@ mod tests { .unwrap(); } + #[tokio::test] + async fn ancillary_file_downloader_resolver() { + let mut mock_file_downloader_cloud_storage = MockFileDownloader::new(); + mock_file_downloader_cloud_storage + .expect_download_unpack() + .times(1) + .returning(|_, _, _, _| Ok(())); + let resolver = AncillaryFileDownloaderResolver::new(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new(mock_file_downloader_cloud_storage), + )]); + + let file_downloader = resolver + .resolve(&AncillaryLocation::CloudStorage { + uri: "http://whatever/00001.tar.gz".to_string(), + }) + .unwrap(); + file_downloader + .download_unpack( + &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), + Path::new("."), + None, + "download_id", + ) + .await + .unwrap(); + } + #[tokio::test] async fn digest_file_downloader_resolver() { let mock_file_downloader_aggregator = MockFileDownloader::new(); From 203d0e71f49262e3bcf7a427100e6f1bfff0779a Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 3 Feb 2025 14:58:38 +0100 Subject: [PATCH 13/59] feat: implement ancillary file download in Cardano database client --- mithril-client/src/cardano_database_client.rs | 479 +++++++++++++----- mithril-client/src/client.rs | 6 + 2 files changed, 369 insertions(+), 116 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index 122ab152aea..cfbe3c24c4d 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -61,8 +61,8 @@ use slog::Logger; #[cfg(feature = "fs")] use mithril_common::{ entities::{ - AncillaryLocation, CompressionAlgorithm, DigestLocation, ImmutableFileNumber, - ImmutablesLocation, + AncillaryLocation, CompressionAlgorithm, DigestLocation, HexEncodedDigest, + ImmutableFileNumber, ImmutablesLocation, }, StdResult, }; @@ -137,6 +137,8 @@ pub struct CardanoDatabaseClient { #[cfg(feature = "fs")] immutable_file_downloader_resolver: Arc>, #[cfg(feature = "fs")] + ancillary_file_downloader_resolver: Arc>, + #[cfg(feature = "fs")] digest_file_downloader_resolver: Arc>, #[cfg(feature = "fs")] logger: Logger, @@ -149,6 +151,9 @@ impl CardanoDatabaseClient { #[cfg(feature = "fs")] immutable_file_downloader_resolver: Arc< dyn FileDownloaderResolver, >, + #[cfg(feature = "fs")] ancillary_file_downloader_resolver: Arc< + dyn FileDownloaderResolver, + >, #[cfg(feature = "fs")] digest_file_downloader_resolver: Arc< dyn FileDownloaderResolver, >, @@ -159,6 +164,8 @@ impl CardanoDatabaseClient { #[cfg(feature = "fs")] immutable_file_downloader_resolver, #[cfg(feature = "fs")] + ancillary_file_downloader_resolver, + #[cfg(feature = "fs")] digest_file_downloader_resolver, #[cfg(feature = "fs")] logger: mithril_common::logging::LoggerExtensions::new_with_component_name::( @@ -244,9 +251,19 @@ impl CardanoDatabaseClient { self.download_unpack_digest_file(&digest_locations, &Self::digest_target_dir()) .await?; - Ok(()) + if download_unpack_options.include_ancillary { + let ancillary_locations = cardano_database_snapshot.locations.ancillary; + self.download_unpack_ancillary_file( + &ancillary_locations, + &compression_algorithm, + target_dir, + ) + .await?; } + Ok(()) + } + fn digest_target_dir() -> PathBuf { std::env::temp_dir() .join("mithril") @@ -384,11 +401,54 @@ impl CardanoDatabaseClient { } } - Err(anyhow!( - "Failed downloading and unpacking immutable files for locations: {locations:?}" - )) + Err(anyhow!( + "Failed downloading and unpacking immutable files for locations: {locations:?}" + )) + } + + /// Download and unpack the ancillary files. + // TODO: Add feedback receivers + async fn download_unpack_ancillary_file( + &self, + locations: &[AncillaryLocation], + compression_algorithm: &CompressionAlgorithm, + ancillary_file_target_dir: &Path, + ) -> StdResult<()> { + let mut locations_sorted = locations.to_owned(); + locations_sorted.sort(); + for location in locations_sorted { + let download_id = format!("{location:?}"); //TODO: check if this is the correct way to format the download_id + let file_downloader = self + .ancillary_file_downloader_resolver + .resolve(&location) + .ok_or_else(|| { + anyhow!("Failed resolving a file downloader for location: {location:?}") + })?; + let file_downloader_uri: FileDownloaderUri = location.into(); + let downloaded = file_downloader + .download_unpack( + &file_downloader_uri, + ancillary_file_target_dir, + Some(compression_algorithm.to_owned()), + &download_id, + ) + .await; + match downloaded { + Ok(_) => return Ok(()), + Err(e) => { + slog::error!( + self.logger, + "Failed downloading and unpacking ancillaries for location {file_downloader_uri:?}"; "error" => e.to_string() + ); + } + } } + Err(anyhow!( + "Failed downloading and unpacking ancillaries for all locations" + )) + } + async fn download_unpack_digest_file( &self, locations: &[DigestLocation], @@ -440,15 +500,16 @@ mod tests { use anyhow::anyhow; use chrono::{DateTime, Utc}; use mithril_common::entities::{ - CardanoDbBeacon, CompressionAlgorithm, DigestLocationDiscriminants, Epoch, - ImmutablesLocationDiscriminants, + AncillaryLocationDiscriminants, CardanoDbBeacon, CompressionAlgorithm, + DigestLocationDiscriminants, Epoch, ImmutablesLocationDiscriminants, }; use mockall::predicate::eq; use crate::{ aggregator_client::MockAggregatorHTTPClient, file_downloader::{ - DigestFileDownloaderResolver, FileDownloader, ImmutablesFileDownloaderResolver, + AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloader, + ImmutablesFileDownloaderResolver, }, test_utils, }; @@ -493,6 +554,7 @@ mod tests { struct CardanoDatabaseClientDependencyInjector { http_client: MockAggregatorHTTPClient, immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver, + ancillary_file_downloader_resolver: AncillaryFileDownloaderResolver, digest_file_downloader_resolver: DigestFileDownloaderResolver, } @@ -503,6 +565,9 @@ mod tests { immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver::new( vec![], ), + ancillary_file_downloader_resolver: AncillaryFileDownloaderResolver::new( + vec![], + ), digest_file_downloader_resolver: DigestFileDownloaderResolver::new(vec![]), } } @@ -529,6 +594,19 @@ mod tests { } } + fn with_ancillary_file_downloaders( + self, + file_downloaders: Vec<(AncillaryLocationDiscriminants, Arc)>, + ) -> Self { + let ancillary_file_downloader_resolver = + AncillaryFileDownloaderResolver::new(file_downloaders); + + Self { + ancillary_file_downloader_resolver, + ..self + } + } + fn with_digest_file_downloaders( self, file_downloaders: Vec<(DigestLocationDiscriminants, Arc)>, @@ -546,6 +624,7 @@ mod tests { CardanoDatabaseClient::new( Arc::new(self.http_client), Arc::new(self.immutable_file_downloader_resolver), + Arc::new(self.ancillary_file_downloader_resolver), Arc::new(self.digest_file_downloader_resolver), test_utils::test_logger(), ) @@ -679,11 +758,12 @@ mod tests { use std::fs; use std::path::Path; - use mithril_common::{ - entities::{FileUri, ImmutablesLocationDiscriminants, MultiFilesUri, TemplateUri}, - test_utils::TempDir, - }; - use mockall::predicate; + use mithril_common::{ + entities::{FileUri, ImmutablesLocationDiscriminants, MultiFilesUri, TemplateUri}, + messages::ArtifactsLocationsMessagePart, + test_utils::TempDir, + }; + use mockall::predicate; use crate::file_downloader::MockFileDownloader; @@ -840,85 +920,113 @@ mod tests { .expect_err("download_unpack should fail"); } - #[tokio::test] - async fn download_unpack_succeeds_with_valid_range() { - let immutable_file_range = ImmutableFileRange::Range(1, 2); - let download_unpack_options = DownloadUnpackOptions::default(); - let cardano_db_snapshot_hash = &"hash-123"; - let target_dir = TempDir::new( - "cardano_database_client", - "download_unpack_succeeds_with_valid_range", - ) - .build(); - let mut message = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - ..CardanoDatabaseSnapshot::dummy() - }; - message.locations.immutables = vec![ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever/{immutable_file_number}.tar.gz".to_string(), - )), - }]; - message.locations.digests = vec![DigestLocation::CloudStorage { - uri: "http://whatever/digest.txt".to_string(), - }]; - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - http_client - .expect_get_content() - .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - })) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - }) - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever/00001.tar.gz".to_string(), - ))), - eq(target_dir.join("immutable")), - eq(Some(CompressionAlgorithm::default())), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Ok(())); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever/00002.tar.gz".to_string(), - ))), - eq(target_dir.join("immutable")), - eq(Some(CompressionAlgorithm::default())), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Ok(())); - - mock_file_downloader - }), - )]) - .with_digest_file_downloaders(vec![( - DigestLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever/digest.txt".to_string(), - ))), - predicate::always(), - eq(None), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Ok(())); + #[tokio::test] + async fn download_unpack_succeeds_with_valid_range() { + let immutable_file_range = ImmutableFileRange::Range(1, 2); + let download_unpack_options = DownloadUnpackOptions { + include_ancillary: true, + ..DownloadUnpackOptions::default() + }; + let cardano_db_snapshot_hash = &"hash-123"; + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_succeeds_with_valid_range", + ) + .build(); + let message = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + locations: ArtifactsLocationsMessagePart { + immutables: vec![ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }], + ancillary: vec![AncillaryLocation::CloudStorage { + uri: "http://whatever/ancillary.tar.gz".to_string(), + }], + digests: vec![DigestLocation::CloudStorage { + uri: "http://whatever/digests.json".to_string(), + }], + }, + ..CardanoDatabaseSnapshot::dummy() + }; + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client + .expect_get_content() + .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + })) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + }) + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever/00001.tar.gz".to_string(), + ))), + eq(target_dir.join("immutable")), + eq(Some(CompressionAlgorithm::default())), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Ok(())); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever/00002.tar.gz".to_string(), + ))), + eq(target_dir.join("immutable")), + eq(Some(CompressionAlgorithm::default())), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Ok(())); + + mock_file_downloader + }), + )]) + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever/ancillary.tar.gz".to_string(), + ))), + eq(target_dir.clone()), + eq(Some(CompressionAlgorithm::default())), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Ok(())); + + mock_file_downloader + }), + )]) + .with_digest_file_downloaders(vec![( + DigestLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever/digests.json".to_string(), + ))), + eq(target_dir.join("digest")), + eq(None), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Ok(())); mock_file_downloader }), @@ -1300,31 +1408,170 @@ mod tests { )]) .build_cardano_database_client(); - client - .download_unpack_immutable_files( - &[ - ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever-1/{immutable_file_number}.tar.gz".to_string(), - )), - }, - ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever-2/{immutable_file_number}.tar.gz".to_string(), - )), - }, - ], - immutable_file_range - .to_range_inclusive(total_immutable_files) - .unwrap(), - &CompressionAlgorithm::default(), - &target_dir, - ) - .await - .unwrap(); - } + client + .download_unpack_immutable_files( + &[ + ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever-1/{immutable_file_number}.tar.gz".to_string(), + )), + }, + ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever-2/{immutable_file_number}.tar.gz".to_string(), + )), + }, + ], + immutable_file_range + .to_range_inclusive(total_immutable_files) + .unwrap(), + &CompressionAlgorithm::default(), + &target_dir, + ) + .await + .unwrap(); + } + } + + mod download_unpack_ancillary_file { + + use mithril_common::entities::FileUri; + use mockall::predicate; + + use crate::file_downloader::MockFileDownloader; + + use super::*; + + #[tokio::test] + async fn download_unpack_ancillary_file_fails_if_no_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .times(1) + .returning(|_, _, _, _| Err(anyhow!("Download failed"))); + + mock_file_downloader + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack_ancillary_file( + &[AncillaryLocation::CloudStorage { + uri: "http://whatever-1/ancillary.tar.gz".to_string(), + }], + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .expect_err("download_unpack_ancillary_file should fail"); } + #[tokio::test] + async fn download_unpack_ancillary_file_succeeds_if_at_least_one_location_is_retrieved() + { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever-1/ancillary.tar.gz".to_string(), + ))), + eq(target_dir), + eq(Some(CompressionAlgorithm::default())), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Err(anyhow!("Download failed"))); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever-2/ancillary.tar.gz".to_string(), + ))), + eq(target_dir), + eq(Some(CompressionAlgorithm::default())), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Ok(())); + + mock_file_downloader + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack_ancillary_file( + &[ + AncillaryLocation::CloudStorage { + uri: "http://whatever-1/ancillary.tar.gz".to_string(), + }, + AncillaryLocation::CloudStorage { + uri: "http://whatever-2/ancillary.tar.gz".to_string(), + }, + ], + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn download_unpack_ancillary_file_succeeds_when_first_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new({ + let mut mock_file_downloader = MockFileDownloader::new(); + mock_file_downloader + .expect_download_unpack() + .with( + eq(FileDownloaderUri::FileUri(FileUri( + "http://whatever-1/ancillary.tar.gz".to_string(), + ))), + eq(target_dir), + eq(Some(CompressionAlgorithm::default())), + predicate::always(), + ) + .times(1) + .returning(|_, _, _, _| Ok(())); + + mock_file_downloader + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack_ancillary_file( + &[ + AncillaryLocation::CloudStorage { + uri: "http://whatever-1/ancillary.tar.gz".to_string(), + }, + AncillaryLocation::CloudStorage { + uri: "http://whatever-2/ancillary.tar.gz".to_string(), + }, + ], + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .unwrap(); + } + } + mod download_unpack_digest_file { use crate::file_downloader::MockFileDownloader; diff --git a/mithril-client/src/client.rs b/mithril-client/src/client.rs index 5e64c4fca9c..9b7c2a6fadb 100644 --- a/mithril-client/src/client.rs +++ b/mithril-client/src/client.rs @@ -18,6 +18,7 @@ use crate::certificate_client::{ CertificateClient, CertificateVerifier, MithrilCertificateVerifier, }; use crate::feedback::{FeedbackReceiver, FeedbackSender}; +use crate::file_downloader::AncillaryFileDownloaderResolver; #[cfg(all(feature = "fs", feature = "unstable"))] use crate::file_downloader::{DigestFileDownloaderResolver, ImmutablesFileDownloaderResolver}; use crate::mithril_stake_distribution_client::MithrilStakeDistributionClient; @@ -265,6 +266,9 @@ impl ClientBuilder { let immutable_file_downloader_resolver = Arc::new(ImmutablesFileDownloaderResolver::new(Vec::new())); #[cfg(all(feature = "fs", feature = "unstable"))] + let ancillary_file_downloader_resolver = + Arc::new(AncillaryFileDownloaderResolver::new(Vec::new())); + #[cfg(all(feature = "fs", feature = "unstable"))] let digest_file_downloader_resolver = Arc::new(DigestFileDownloaderResolver::new(Vec::new())); #[cfg(feature = "unstable")] @@ -273,6 +277,8 @@ impl ClientBuilder { #[cfg(feature = "fs")] immutable_file_downloader_resolver, #[cfg(feature = "fs")] + ancillary_file_downloader_resolver, + #[cfg(feature = "fs")] digest_file_downloader_resolver, #[cfg(feature = "fs")] logger, From c1b3f51427dcd0d1efa85953d3bcbbc18884b2ad Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 3 Feb 2025 17:07:01 +0100 Subject: [PATCH 14/59] refactor: simplify tests with 'MockFileDownloaderBuilder' in Cardano database client --- mithril-client/src/cardano_database_client.rs | 926 +++++++++--------- mithril-client/src/client.rs | 5 +- 2 files changed, 475 insertions(+), 456 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index cfbe3c24c4d..72654ca2df4 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -64,6 +64,7 @@ use mithril_common::{ AncillaryLocation, CompressionAlgorithm, DigestLocation, HexEncodedDigest, ImmutableFileNumber, ImmutablesLocation, }, + messages::CardanoDatabaseDigestListItemMessage, StdResult, }; @@ -214,7 +215,7 @@ impl CardanoDatabaseClient { } cfg_fs! { - /// Download and unpack the given Cardano database part data by hash. + /// Download and unpack the given Cardano database parts data by hash. // TODO: Add example in module documentation pub async fn download_unpack( &self, @@ -248,27 +249,24 @@ impl CardanoDatabaseClient { .await?; let digest_locations = cardano_database_snapshot.locations.digests; - self.download_unpack_digest_file(&digest_locations, &Self::digest_target_dir()) + self.download_unpack_digest_file(&digest_locations, &Self::digest_target_dir(target_dir)) .await?; - if download_unpack_options.include_ancillary { - let ancillary_locations = cardano_database_snapshot.locations.ancillary; - self.download_unpack_ancillary_file( - &ancillary_locations, - &compression_algorithm, - target_dir, - ) - .await?; - } + if download_unpack_options.include_ancillary { + let ancillary_locations = cardano_database_snapshot.locations.ancillary; + self.download_unpack_ancillary_file( + &ancillary_locations, + &compression_algorithm, + target_dir, + ) + .await?; + } - Ok(()) - } + Ok(()) + } - fn digest_target_dir() -> PathBuf { - std::env::temp_dir() - .join("mithril") - .join("cardano_database_client") - .join("digest") + fn digest_target_dir(target_dir: &Path) -> PathBuf { + target_dir.join("digest") } fn immutable_files_target_dir(target_dir: &Path) -> PathBuf { @@ -291,6 +289,19 @@ impl CardanoDatabaseClient { fs::create_dir_all(dir).map_err(|e| anyhow!("Failed creating directory: {e}")) } + fn read_files_in_directory(dir: &Path) -> StdResult> { + let mut files = vec![]; + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_file() { + files.push(path); + } + } + + Ok(files) + } + /// Verify if the target directory is writable. fn verify_can_write_to_target_directory( &self, @@ -332,6 +343,8 @@ impl CardanoDatabaseClient { ) -> StdResult<()> { let immutable_files_target_dir = Self::immutable_files_target_dir(target_dir); Self::create_directory_if_not_exists(&immutable_files_target_dir)?; + let digest_target_dir = Self::digest_target_dir(target_dir); + Self::create_directory_if_not_exists(&digest_target_dir)?; if download_unpack_options.include_ancillary { let volatile_target_dir = Self::volatile_target_dir(target_dir); let ledger_target_dir = Self::ledger_target_dir(target_dir); @@ -401,53 +414,53 @@ impl CardanoDatabaseClient { } } - Err(anyhow!( - "Failed downloading and unpacking immutable files for locations: {locations:?}" - )) - } + Err(anyhow!( + "Failed downloading and unpacking immutable files for locations: {locations:?}" + )) + } - /// Download and unpack the ancillary files. - // TODO: Add feedback receivers - async fn download_unpack_ancillary_file( - &self, - locations: &[AncillaryLocation], - compression_algorithm: &CompressionAlgorithm, - ancillary_file_target_dir: &Path, - ) -> StdResult<()> { - let mut locations_sorted = locations.to_owned(); - locations_sorted.sort(); - for location in locations_sorted { - let download_id = format!("{location:?}"); //TODO: check if this is the correct way to format the download_id - let file_downloader = self - .ancillary_file_downloader_resolver - .resolve(&location) - .ok_or_else(|| { - anyhow!("Failed resolving a file downloader for location: {location:?}") - })?; - let file_downloader_uri: FileDownloaderUri = location.into(); - let downloaded = file_downloader - .download_unpack( - &file_downloader_uri, - ancillary_file_target_dir, - Some(compression_algorithm.to_owned()), - &download_id, - ) - .await; - match downloaded { - Ok(_) => return Ok(()), - Err(e) => { - slog::error!( - self.logger, - "Failed downloading and unpacking ancillaries for location {file_downloader_uri:?}"; "error" => e.to_string() - ); + /// Download and unpack the ancillary files. + // TODO: Add feedback receivers + async fn download_unpack_ancillary_file( + &self, + locations: &[AncillaryLocation], + compression_algorithm: &CompressionAlgorithm, + ancillary_file_target_dir: &Path, + ) -> StdResult<()> { + let mut locations_sorted = locations.to_owned(); + locations_sorted.sort(); + for location in locations_sorted { + let download_id = format!("{location:?}"); //TODO: check if this is the correct way to format the download_id + let file_downloader = self + .ancillary_file_downloader_resolver + .resolve(&location) + .ok_or_else(|| { + anyhow!("Failed resolving a file downloader for location: {location:?}") + })?; + let file_downloader_uri: FileDownloaderUri = location.into(); + let downloaded = file_downloader + .download_unpack( + &file_downloader_uri, + ancillary_file_target_dir, + Some(compression_algorithm.to_owned()), + &download_id, + ) + .await; + match downloaded { + Ok(_) => return Ok(()), + Err(e) => { + slog::error!( + self.logger, + "Failed downloading and unpacking ancillaries for location {file_downloader_uri:?}"; "error" => e.to_string() + ); + } } } - } - Err(anyhow!( - "Failed downloading and unpacking ancillaries for all locations" - )) - } + Err(anyhow!( + "Failed downloading and unpacking ancillaries for all locations" + )) + } async fn download_unpack_digest_file( &self, @@ -501,15 +514,15 @@ mod tests { use chrono::{DateTime, Utc}; use mithril_common::entities::{ AncillaryLocationDiscriminants, CardanoDbBeacon, CompressionAlgorithm, - DigestLocationDiscriminants, Epoch, ImmutablesLocationDiscriminants, + DigestLocationDiscriminants, Epoch, FileUri, ImmutablesLocationDiscriminants, }; - use mockall::predicate::eq; + use mockall::predicate::{self, eq}; use crate::{ aggregator_client::MockAggregatorHTTPClient, file_downloader::{ AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloader, - ImmutablesFileDownloaderResolver, + ImmutablesFileDownloaderResolver, MockFileDownloader, }, test_utils, }; @@ -631,6 +644,134 @@ mod tests { } } + type MockFileDownloaderBuilderReturningFunc = Box< + dyn FnMut( + &FileDownloaderUri, + &Path, + Option, + &str, + ) -> StdResult<()> + + Send + + 'static, + >; + + struct MockFileDownloaderBuilder { + mock_file_downloader: Option, + times: usize, + param_file_downloader_uri: Option, + param_target_dir: Option, + param_compression_algorithm: Option>, + returning_func: Option, + } + + impl Default for MockFileDownloaderBuilder { + fn default() -> Self { + Self { + mock_file_downloader: None, + times: 1, + param_file_downloader_uri: None, + param_target_dir: None, + param_compression_algorithm: Some(Some(CompressionAlgorithm::default())), + returning_func: None, + } + } + } + + impl MockFileDownloaderBuilder { + fn from_mock(mock: MockFileDownloader) -> Self { + Self { + mock_file_downloader: Some(mock), + ..Self::default() + } + } + + fn with_success(self) -> Self { + self.with_returning(Box::new(|_, _, _, _| Ok(()))) + } + + fn with_failure(self) -> Self { + self.with_returning(Box::new(|_, _, _, _| { + Err(anyhow!("Download unpack failed")) + })) + } + + fn with_times(self, times: usize) -> Self { + let mut self_mut = self; + self_mut.times = times; + + self_mut + } + + fn with_file_uri>(self, file_uri: T) -> Self { + let mut self_mut = self; + self_mut.param_file_downloader_uri = Some(FileDownloaderUri::FileUri(FileUri( + file_uri.as_ref().to_string(), + ))); + + self_mut + } + + fn with_target_dir(self, target_dir: PathBuf) -> Self { + let mut self_mut = self; + self_mut.param_target_dir = Some(target_dir); + + self_mut + } + + fn with_compression(self, compression: Option) -> Self { + let mut self_mut = self; + self_mut.param_compression_algorithm = Some(compression); + + self_mut + } + + fn with_returning( + self, + returning_func: MockFileDownloaderBuilderReturningFunc, + ) -> Self { + let mut self_mut = self; + self_mut.returning_func = Some(returning_func); + + self_mut + } + + fn build(self) -> MockFileDownloader { + let predicate_file_downloader_uri = predicate::function(move |u| { + self.param_file_downloader_uri + .as_ref() + .map(|x| x == u) + .unwrap_or(true) + }); + let predicate_target_dir = predicate::function(move |u| { + self.param_target_dir + .as_ref() + .map(|x| x == u) + .unwrap_or(true) + }); + let predicate_compression_algorithm = predicate::function(move |u| { + self.param_compression_algorithm + .as_ref() + .map(|x| x == u) + .unwrap_or(true) + }); + let predicate_download_id = predicate::always(); + + let mut mock_file_downloader = self.mock_file_downloader.unwrap_or_default(); + mock_file_downloader + .expect_download_unpack() + .with( + predicate_file_downloader_uri, + predicate_target_dir, + predicate_compression_algorithm, + predicate_download_id, + ) + .times(self.times) + .returning(self.returning_func.unwrap()); + + mock_file_downloader + } + } + mod list { use super::*; @@ -758,14 +899,11 @@ mod tests { use std::fs; use std::path::Path; - use mithril_common::{ - entities::{FileUri, ImmutablesLocationDiscriminants, MultiFilesUri, TemplateUri}, - messages::ArtifactsLocationsMessagePart, - test_utils::TempDir, - }; - use mockall::predicate; - - use crate::file_downloader::MockFileDownloader; + use mithril_common::{ + entities::{ImmutablesLocationDiscriminants, MultiFilesUri, TemplateUri}, + messages::ArtifactsLocationsMessagePart, + test_utils::TempDir, + }; use super::*; @@ -860,13 +998,10 @@ mod tests { .with_immutable_file_downloaders(vec![( ImmutablesLocationDiscriminants::CloudStorage, Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .times(total_immutable_files as usize) - .returning(|_, _, _, _| Err(anyhow!("Download failed"))); - - mock_file_downloader + MockFileDownloaderBuilder::default() + .with_times(total_immutable_files as usize) + .with_failure() + .build() }), )]) .build_cardano_database_client(); @@ -920,115 +1055,81 @@ mod tests { .expect_err("download_unpack should fail"); } - #[tokio::test] - async fn download_unpack_succeeds_with_valid_range() { - let immutable_file_range = ImmutableFileRange::Range(1, 2); - let download_unpack_options = DownloadUnpackOptions { - include_ancillary: true, - ..DownloadUnpackOptions::default() - }; - let cardano_db_snapshot_hash = &"hash-123"; - let target_dir = TempDir::new( - "cardano_database_client", - "download_unpack_succeeds_with_valid_range", - ) - .build(); - let message = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - locations: ArtifactsLocationsMessagePart { - immutables: vec![ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever/{immutable_file_number}.tar.gz".to_string(), - )), - }], - ancillary: vec![AncillaryLocation::CloudStorage { - uri: "http://whatever/ancillary.tar.gz".to_string(), - }], - digests: vec![DigestLocation::CloudStorage { - uri: "http://whatever/digests.json".to_string(), - }], - }, - ..CardanoDatabaseSnapshot::dummy() - }; - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - http_client - .expect_get_content() - .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - })) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - }) - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever/00001.tar.gz".to_string(), - ))), - eq(target_dir.join("immutable")), - eq(Some(CompressionAlgorithm::default())), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Ok(())); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever/00002.tar.gz".to_string(), - ))), - eq(target_dir.join("immutable")), - eq(Some(CompressionAlgorithm::default())), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Ok(())); - - mock_file_downloader - }), - )]) - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever/ancillary.tar.gz".to_string(), - ))), - eq(target_dir.clone()), - eq(Some(CompressionAlgorithm::default())), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Ok(())); - - mock_file_downloader - }), - )]) - .with_digest_file_downloaders(vec![( - DigestLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever/digests.json".to_string(), - ))), - eq(target_dir.join("digest")), - eq(None), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Ok(())); - - mock_file_downloader + #[tokio::test] + async fn download_unpack_succeeds_with_valid_range() { + let immutable_file_range = ImmutableFileRange::Range(1, 2); + let download_unpack_options = DownloadUnpackOptions { + include_ancillary: true, + ..DownloadUnpackOptions::default() + }; + let cardano_db_snapshot_hash = &"hash-123"; + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_succeeds_with_valid_range", + ) + .build(); + let message = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + locations: ArtifactsLocationsMessagePart { + immutables: vec![ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }], + ancillary: vec![AncillaryLocation::CloudStorage { + uri: "http://whatever/ancillary.tar.gz".to_string(), + }], + digests: vec![DigestLocation::CloudStorage { + uri: "http://whatever/digests.json".to_string(), + }], + }, + ..CardanoDatabaseSnapshot::dummy() + }; + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client + .expect_get_content() + .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + })) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + }) + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/00001.tar.gz") + .with_target_dir(target_dir.join("immutable")) + .with_success() + .build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever/00002.tar.gz") + .with_target_dir(target_dir.join("immutable")) + .with_success() + .build() + }), + )]) + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new( + MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/ancillary.tar.gz") + .with_target_dir(target_dir.clone()) + .with_compression(Some(CompressionAlgorithm::default())) + .with_success() + .build(), + ), + )]) + .with_digest_file_downloaders(vec![( + DigestLocationDiscriminants::CloudStorage, + Arc::new({ + MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/digests.json") + .with_target_dir(target_dir.join("digest")) + .with_compression(None) + .with_success() + .build() }), )]) .build_cardano_database_client(); @@ -1205,6 +1306,7 @@ mod tests { .build(); let client = CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + assert!(!target_dir.join("digest").exists()); assert!(!target_dir.join("immutable").exists()); assert!(!target_dir.join("volatile").exists()); assert!(!target_dir.join("ledger").exists()); @@ -1219,6 +1321,7 @@ mod tests { ) .unwrap(); + assert!(target_dir.join("digest").exists()); assert!(target_dir.join("immutable").exists()); assert!(!target_dir.join("volatile").exists()); assert!(!target_dir.join("ledger").exists()); @@ -1233,6 +1336,7 @@ mod tests { .build(); let client = CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + assert!(!target_dir.join("digest").exists()); assert!(!target_dir.join("immutable").exists()); assert!(!target_dir.join("volatile").exists()); assert!(!target_dir.join("ledger").exists()); @@ -1247,6 +1351,7 @@ mod tests { ) .unwrap(); + assert!(target_dir.join("digest").exists()); assert!(target_dir.join("immutable").exists()); assert!(target_dir.join("volatile").exists()); assert!(target_dir.join("ledger").exists()); @@ -1255,12 +1360,9 @@ mod tests { mod download_unpack_immutable_files { use mithril_common::{ - entities::{FileUri, MultiFilesUri, TemplateUri}, + entities::{MultiFilesUri, TemplateUri}, test_utils::TempDir, }; - use mockall::predicate; - - use crate::file_downloader::MockFileDownloader; use super::*; @@ -1277,17 +1379,12 @@ mod tests { .with_immutable_file_downloaders(vec![( ImmutablesLocationDiscriminants::CloudStorage, Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .times(1) - .returning(|_, _, _, _| Err(anyhow!("Download failed"))); - mock_file_downloader - .expect_download_unpack() - .times(1) - .returning(|_, _, _, _| Ok(())); - - mock_file_downloader + let mock_file_downloader = + MockFileDownloaderBuilder::default().with_failure().build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_success() + .build() }), )]) .build_cardano_database_client(); @@ -1322,15 +1419,12 @@ mod tests { let client = CardanoDatabaseClientDependencyInjector::new() .with_immutable_file_downloaders(vec![( ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .times(2) - .returning(|_, _, _, _| Ok(())); - - mock_file_downloader - }), + Arc::new( + MockFileDownloaderBuilder::default() + .with_times(2) + .with_success() + .build(), + ), )]) .build_cardano_database_client(); @@ -1365,212 +1459,151 @@ mod tests { .with_immutable_file_downloaders(vec![( ImmutablesLocationDiscriminants::CloudStorage, Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever-1/00001.tar.gz".to_string(), - ))), - eq(target_dir.clone()), - eq(Some(CompressionAlgorithm::default())), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Err(anyhow!("Download failed"))); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever-1/00002.tar.gz".to_string(), - ))), - eq(target_dir.clone()), - eq(Some(CompressionAlgorithm::default())), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Ok(())); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever-2/00001.tar.gz".to_string(), - ))), - eq(target_dir.clone()), - eq(Some(CompressionAlgorithm::default())), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Ok(())); - - mock_file_downloader + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever-1/00001.tar.gz") + .with_target_dir(target_dir.clone()) + .with_failure() + .build(); + let mock_file_downloader = + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever-1/00002.tar.gz") + .with_target_dir(target_dir.clone()) + .with_success() + .build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever-2/00001.tar.gz") + .with_target_dir(target_dir.clone()) + .with_success() + .build() }), )]) .build_cardano_database_client(); - client - .download_unpack_immutable_files( - &[ - ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever-1/{immutable_file_number}.tar.gz".to_string(), - )), - }, - ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever-2/{immutable_file_number}.tar.gz".to_string(), - )), - }, - ], - immutable_file_range - .to_range_inclusive(total_immutable_files) - .unwrap(), - &CompressionAlgorithm::default(), - &target_dir, - ) - .await - .unwrap(); + client + .download_unpack_immutable_files( + &[ + ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever-1/{immutable_file_number}.tar.gz".to_string(), + )), + }, + ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever-2/{immutable_file_number}.tar.gz".to_string(), + )), + }, + ], + immutable_file_range + .to_range_inclusive(total_immutable_files) + .unwrap(), + &CompressionAlgorithm::default(), + &target_dir, + ) + .await + .unwrap(); + } } - } - mod download_unpack_ancillary_file { + mod download_unpack_ancillary_file { - use mithril_common::entities::FileUri; - use mockall::predicate; - - use crate::file_downloader::MockFileDownloader; + use super::*; - use super::*; + #[tokio::test] + async fn download_unpack_ancillary_file_fails_if_no_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new(MockFileDownloaderBuilder::default().with_failure().build()), + )]) + .build_cardano_database_client(); - #[tokio::test] - async fn download_unpack_ancillary_file_fails_if_no_location_is_retrieved() { - let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .times(1) - .returning(|_, _, _, _| Err(anyhow!("Download failed"))); - - mock_file_downloader - }), - )]) - .build_cardano_database_client(); + client + .download_unpack_ancillary_file( + &[AncillaryLocation::CloudStorage { + uri: "http://whatever-1/ancillary.tar.gz".to_string(), + }], + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .expect_err("download_unpack_ancillary_file should fail"); + } - client - .download_unpack_ancillary_file( - &[AncillaryLocation::CloudStorage { - uri: "http://whatever-1/ancillary.tar.gz".to_string(), - }], - &CompressionAlgorithm::default(), - target_dir, - ) - .await - .expect_err("download_unpack_ancillary_file should fail"); - } + #[tokio::test] + async fn download_unpack_ancillary_file_succeeds_if_at_least_one_location_is_retrieved() + { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new({ + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever-1/ancillary.tar.gz") + .with_target_dir(target_dir.to_path_buf()) + .with_failure() + .build(); - #[tokio::test] - async fn download_unpack_ancillary_file_succeeds_if_at_least_one_location_is_retrieved() - { - let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever-1/ancillary.tar.gz".to_string(), - ))), - eq(target_dir), - eq(Some(CompressionAlgorithm::default())), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Err(anyhow!("Download failed"))); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever-2/ancillary.tar.gz".to_string(), - ))), - eq(target_dir), - eq(Some(CompressionAlgorithm::default())), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Ok(())); - - mock_file_downloader - }), - )]) - .build_cardano_database_client(); + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever-2/ancillary.tar.gz") + .with_target_dir(target_dir.to_path_buf()) + .with_success() + .build() + }), + )]) + .build_cardano_database_client(); - client - .download_unpack_ancillary_file( - &[ - AncillaryLocation::CloudStorage { - uri: "http://whatever-1/ancillary.tar.gz".to_string(), - }, - AncillaryLocation::CloudStorage { - uri: "http://whatever-2/ancillary.tar.gz".to_string(), - }, - ], - &CompressionAlgorithm::default(), - target_dir, - ) - .await - .unwrap(); - } + client + .download_unpack_ancillary_file( + &[ + AncillaryLocation::CloudStorage { + uri: "http://whatever-1/ancillary.tar.gz".to_string(), + }, + AncillaryLocation::CloudStorage { + uri: "http://whatever-2/ancillary.tar.gz".to_string(), + }, + ], + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .unwrap(); + } - #[tokio::test] - async fn download_unpack_ancillary_file_succeeds_when_first_location_is_retrieved() { - let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .with( - eq(FileDownloaderUri::FileUri(FileUri( - "http://whatever-1/ancillary.tar.gz".to_string(), - ))), - eq(target_dir), - eq(Some(CompressionAlgorithm::default())), - predicate::always(), - ) - .times(1) - .returning(|_, _, _, _| Ok(())); - - mock_file_downloader - }), - )]) - .build_cardano_database_client(); + #[tokio::test] + async fn download_unpack_ancillary_file_succeeds_when_first_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new( + MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever-1/ancillary.tar.gz") + .with_target_dir(target_dir.to_path_buf()) + .with_success() + .build(), + ), + )]) + .build_cardano_database_client(); - client - .download_unpack_ancillary_file( - &[ - AncillaryLocation::CloudStorage { - uri: "http://whatever-1/ancillary.tar.gz".to_string(), - }, - AncillaryLocation::CloudStorage { - uri: "http://whatever-2/ancillary.tar.gz".to_string(), - }, - ], - &CompressionAlgorithm::default(), - target_dir, - ) - .await - .unwrap(); + client + .download_unpack_ancillary_file( + &[ + AncillaryLocation::CloudStorage { + uri: "http://whatever-1/ancillary.tar.gz".to_string(), + }, + AncillaryLocation::CloudStorage { + uri: "http://whatever-2/ancillary.tar.gz".to_string(), + }, + ], + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .unwrap(); + } } - } mod download_unpack_digest_file { @@ -1585,27 +1618,21 @@ mod tests { .with_digest_file_downloaders(vec![ ( DigestLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .times(1) - .returning(|_, _, _, _| Err(anyhow!("Download failed"))); - - mock_file_downloader - }), + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_failure() + .build(), + ), ), ( DigestLocationDiscriminants::Aggregator, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .times(1) - .returning(|_, _, _, _| Err(anyhow!("Download failed"))); - - mock_file_downloader - }), + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_failure() + .build(), + ), ), ]) .build_cardano_database_client(); @@ -1614,13 +1641,13 @@ mod tests { .download_unpack_digest_file( &[ DigestLocation::CloudStorage { - uri: "http://whatever-1/digest.txt".to_string(), + uri: "http://whatever-1/digests.json".to_string(), }, DigestLocation::Aggregator { uri: "http://whatever-2/digest".to_string(), }, ], - &target_dir, + target_dir, ) .await .expect_err("download_unpack_digest_file should fail"); @@ -1633,27 +1660,21 @@ mod tests { .with_digest_file_downloaders(vec![ ( DigestLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .times(1) - .returning(|_, _, _, _| Err(anyhow!("Download failed"))); - - mock_file_downloader - }), + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_failure() + .build(), + ), ), ( DigestLocationDiscriminants::Aggregator, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .times(1) - .returning(|_, _, _, _| Ok(())); - - mock_file_downloader - }), + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_success() + .build(), + ), ), ]) .build_cardano_database_client(); @@ -1662,13 +1683,13 @@ mod tests { .download_unpack_digest_file( &[ DigestLocation::CloudStorage { - uri: "http://whatever-1/digest.txt".to_string(), + uri: "http://whatever-1/digests.json".to_string(), }, DigestLocation::Aggregator { uri: "http://whatever-2/digest".to_string(), }, ], - &target_dir, + target_dir, ) .await .unwrap(); @@ -1681,15 +1702,12 @@ mod tests { .with_digest_file_downloaders(vec![ ( DigestLocationDiscriminants::CloudStorage, - Arc::new({ - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .times(1) - .returning(|_, _, _, _| Ok(())); - - mock_file_downloader - }), + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_success() + .build(), + ), ), ( DigestLocationDiscriminants::Aggregator, @@ -1702,13 +1720,13 @@ mod tests { .download_unpack_digest_file( &[ DigestLocation::CloudStorage { - uri: "http://whatever-1/digest.txt".to_string(), + uri: "http://whatever-1/digests.json".to_string(), }, DigestLocation::Aggregator { uri: "http://whatever-2/digest".to_string(), }, ], - &target_dir, + target_dir, ) .await .unwrap(); diff --git a/mithril-client/src/client.rs b/mithril-client/src/client.rs index 9b7c2a6fadb..90779b14be5 100644 --- a/mithril-client/src/client.rs +++ b/mithril-client/src/client.rs @@ -18,9 +18,10 @@ use crate::certificate_client::{ CertificateClient, CertificateVerifier, MithrilCertificateVerifier, }; use crate::feedback::{FeedbackReceiver, FeedbackSender}; -use crate::file_downloader::AncillaryFileDownloaderResolver; #[cfg(all(feature = "fs", feature = "unstable"))] -use crate::file_downloader::{DigestFileDownloaderResolver, ImmutablesFileDownloaderResolver}; +use crate::file_downloader::{ + AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, ImmutablesFileDownloaderResolver, +}; use crate::mithril_stake_distribution_client::MithrilStakeDistributionClient; use crate::snapshot_client::SnapshotClient; #[cfg(feature = "fs")] From 2fa55a5c390649e631882efef61e73ebc4bf1ca5 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 4 Feb 2025 16:39:16 +0100 Subject: [PATCH 15/59] refactor: extend 'ImmutableDigester' trait To compute digests for a range of immutable file numbers. --- .../digesters/cardano_immutable_digester.rs | 251 +++++++++++------- .../src/digesters/dumb_immutable_digester.rs | 42 ++- .../src/digesters/immutable_digester.rs | 133 +++++++++- .../src/digesters/immutable_file.rs | 77 +++++- mithril-common/src/digesters/mod.rs | 4 +- 5 files changed, 385 insertions(+), 122 deletions(-) diff --git a/mithril-common/src/digesters/cardano_immutable_digester.rs b/mithril-common/src/digesters/cardano_immutable_digester.rs index 0ccdd66cb59..c818bc8f2fb 100644 --- a/mithril-common/src/digesters/cardano_immutable_digester.rs +++ b/mithril-common/src/digesters/cardano_immutable_digester.rs @@ -4,18 +4,15 @@ use crate::{ cache::ImmutableFileDigestCacheProvider, ImmutableDigester, ImmutableDigesterError, ImmutableFile, }, - entities::{CardanoDbBeacon, HexEncodedDigest, ImmutableFileName, ImmutableFileNumber}, + entities::{CardanoDbBeacon, HexEncodedDigest, ImmutableFileNumber}, logging::LoggerExtensions, }; use async_trait::async_trait; use sha2::{Digest, Sha256}; use slog::{debug, info, warn, Logger}; -use std::{collections::BTreeMap, io, path::Path, sync::Arc}; +use std::{collections::BTreeMap, io, ops::RangeInclusive, path::Path, sync::Arc}; -struct ComputedImmutablesDigests { - entries: BTreeMap, - new_cached_entries: Vec, -} +use super::immutable_digester::ComputedImmutablesDigests; /// A digester working directly on a Cardano DB immutables files pub struct CardanoImmutableDigester { @@ -52,7 +49,7 @@ impl CardanoImmutableDigester { let logger = self.logger.clone(); let computed_digests = tokio::task::spawn_blocking(move || -> Result { - compute_immutables_digests(logger, cached_values) + ComputedImmutablesDigests::compute_immutables_digests(cached_values, logger) }) .await .map_err(|e| ImmutableDigesterError::DigestComputationError(e.into()))??; @@ -132,6 +129,24 @@ impl ImmutableDigester for CardanoImmutableDigester { Ok(digest) } + async fn compute_digests_for_range( + &self, + dirpath: &Path, + range: &RangeInclusive, + ) -> Result { + let immutables_to_process = list_immutable_files_to_process_for_range(dirpath, range)?; + info!(self.logger, ">> compute_digests_for_range"; "nb_of_immutables" => immutables_to_process.len()); + let computed_immutables_digests = self.process_immutables(immutables_to_process).await?; + + self.update_cache(&computed_immutables_digests).await; + + debug!( + self.logger, + "Successfully computed Digests for Cardano database"; "range" => #?range); + + Ok(computed_immutables_digests) + } + async fn compute_merkle_tree( &self, dirpath: &Path, @@ -183,37 +198,16 @@ fn list_immutable_files_to_process( } } -fn compute_immutables_digests( - logger: Logger, - entries: BTreeMap>, -) -> Result { - let mut new_cached_entries = Vec::new(); - let mut progress = Progress { - index: 0, - total: entries.len(), - }; - - let mut digests = BTreeMap::new(); - - for (ix, (entry, cache)) in entries.into_iter().enumerate() { - let hash = match cache { - None => { - new_cached_entries.push(entry.filename.clone()); - hex::encode(entry.compute_raw_hash::()?) - } - Some(digest) => digest, - }; - digests.insert(entry, hash); - - if progress.report(ix) { - info!(logger, "Hashing: {progress}"); - } - } +fn list_immutable_files_to_process_for_range( + dirpath: &Path, + range: &RangeInclusive, +) -> Result, ImmutableDigesterError> { + let immutables: Vec = ImmutableFile::list_completed_in_dir(dirpath)? + .into_iter() + .filter(|f| range.contains(&f.number)) + .collect(); - Ok(ComputedImmutablesDigests { - entries: digests, - new_cached_entries, - }) + Ok(immutables) } fn compute_beacon_hash(network: &str, cardano_db_beacon: &CardanoDbBeacon) -> String { @@ -224,28 +218,6 @@ fn compute_beacon_hash(network: &str, cardano_db_beacon: &CardanoDbBeacon) -> St hex::encode(hasher.finalize()) } -struct Progress { - index: usize, - total: usize, -} - -impl Progress { - fn report(&mut self, ix: usize) -> bool { - self.index = ix; - (20 * ix) % self.total == 0 - } - - fn percent(&self) -> f64 { - (self.index as f64 * 100.0 / self.total as f64).ceil() - } -} - -impl std::fmt::Display for Progress { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - write!(f, "{}/{} ({}%)", self.index, self.total, self.percent()) - } -} - #[cfg(test)] mod tests { use sha2::Sha256; @@ -297,32 +269,6 @@ mod tests { ); } - #[test] - fn reports_progress_every_5_percent() { - let mut progress = Progress { - index: 0, - total: 7000, - }; - - assert!(!progress.report(1)); - assert!(!progress.report(4)); - assert!(progress.report(350)); - assert!(!progress.report(351)); - } - - #[test] - fn reports_progress_when_total_lower_than_20() { - let mut progress = Progress { - index: 0, - total: 16, - }; - - assert!(progress.report(4)); - assert!(progress.report(12)); - assert!(!progress.report(3)); - assert!(!progress.report(15)); - } - #[tokio::test] async fn fail_if_no_file_in_folder() { let cardano_db = db_builder("fail_if_no_file_in_folder").build(); @@ -397,6 +343,31 @@ mod tests { ); } + #[tokio::test] + async fn can_compute_hash_of_a_hundred_immutable_file_trio() { + let cardano_db = db_builder("can_compute_hash_of_a_hundred_immutable_file_trio") + .with_immutables(&(1..=100).collect::>()) + .append_immutable_trio() + .build(); + let logger = TestLogger::stdout(); + let digester = CardanoImmutableDigester::new( + "devnet".to_string(), + Some(Arc::new(MemoryImmutableFileDigestCacheProvider::default())), + logger.clone(), + ); + let beacon = CardanoDbBeacon::new(1, 100); + + let result = digester + .compute_digest(cardano_db.get_immutable_dir(), &beacon) + .await + .expect("compute_digest must not fail"); + + assert_eq!( + "a27fd67e495c2c77e4b6b0af9925b2b0bc39656c56adfad4aaab9f20fae49122".to_string(), + result + ) + } + #[tokio::test] async fn can_compute_merkle_tree_of_a_hundred_immutable_file_trio() { let cardano_db = db_builder("can_compute_merkle_tree_of_a_hundred_immutable_file_trio") @@ -425,28 +396,30 @@ mod tests { } #[tokio::test] - async fn can_compute_hash_of_a_hundred_immutable_file_trio() { - let cardano_db = db_builder("can_compute_hash_of_a_hundred_immutable_file_trio") - .with_immutables(&(1..=100).collect::>()) - .append_immutable_trio() - .build(); + async fn can_compute_digests_for_range_of_a_hundred_immutable_file_trio() { + let immutable_range = 1..=100; + let cardano_db = + db_builder("can_compute_digests_for_range_of_a_hundred_immutable_file_trio") + .with_immutables( + &immutable_range + .clone() + .collect::>(), + ) + .append_immutable_trio() + .build(); let logger = TestLogger::stdout(); let digester = CardanoImmutableDigester::new( "devnet".to_string(), Some(Arc::new(MemoryImmutableFileDigestCacheProvider::default())), logger.clone(), ); - let beacon = CardanoDbBeacon::new(1, 100); let result = digester - .compute_digest(cardano_db.get_immutable_dir(), &beacon) + .compute_digests_for_range(cardano_db.get_immutable_dir(), &immutable_range) .await - .expect("compute_digest must not fail"); + .expect("compute_digests_for_range must not fail"); - assert_eq!( - "a27fd67e495c2c77e4b6b0af9925b2b0bc39656c56adfad4aaab9f20fae49122".to_string(), - result - ) + assert_eq!(cardano_db.get_immutable_files().len(), result.entries.len()) } #[tokio::test] @@ -521,6 +494,43 @@ mod tests { assert_eq!(expected, cached_entries); } + #[tokio::test] + async fn compute_digests_for_range_stores_digests_into_cache_provider() { + let cardano_db = db_builder("compute_digests_for_range_stores_digests_into_cache_provider") + .with_immutables(&[1, 2]) + .append_immutable_trio() + .build(); + let immutables = cardano_db.get_immutable_files().clone(); + let cache = Arc::new(MemoryImmutableFileDigestCacheProvider::default()); + let logger = TestLogger::stdout(); + let digester = CardanoImmutableDigester::new( + "devnet".to_string(), + Some(cache.clone()), + logger.clone(), + ); + let immutable_range = 1..=2; + + digester + .compute_digests_for_range(cardano_db.get_immutable_dir(), &immutable_range) + .await + .expect("compute_digests_for_range must not fail"); + + let cached_entries = cache + .get(immutables.clone()) + .await + .expect("Cache read should not fail"); + let expected: BTreeMap<_, _> = immutables + .into_iter() + .filter(|i| immutable_range.contains(&i.number)) + .map(|i| { + let digest = hex::encode(i.compute_raw_hash::().unwrap()); + (i.to_owned(), Some(digest)) + }) + .collect(); + + assert_eq!(expected, cached_entries); + } + #[tokio::test] async fn computed_digest_with_cold_or_hot_or_without_any_cache_are_equals() { let cardano_db = DummyCardanoDbBuilder::new( @@ -612,6 +622,53 @@ mod tests { ); } + #[tokio::test] + async fn computed_digests_for_range_with_cold_or_hot_or_without_any_cache_are_equals() { + let cardano_db = DummyCardanoDbBuilder::new( + "computed_digests_for_range_with_cold_or_hot_or_without_any_cache_are_equals", + ) + .with_immutables(&[1, 2, 3]) + .append_immutable_trio() + .build(); + let logger = TestLogger::stdout(); + let no_cache_digester = + CardanoImmutableDigester::new("devnet".to_string(), None, logger.clone()); + let cache_digester = CardanoImmutableDigester::new( + "devnet".to_string(), + Some(Arc::new(MemoryImmutableFileDigestCacheProvider::default())), + logger.clone(), + ); + let immutable_range = 1..=3; + + let without_cache_digests = no_cache_digester + .compute_digests_for_range(cardano_db.get_immutable_dir(), &immutable_range) + .await + .expect("compute_digests_for_range must not fail"); + + let cold_cache_digests = cache_digester + .compute_digests_for_range(cardano_db.get_immutable_dir(), &immutable_range) + .await + .expect("compute_digests_for_range must not fail"); + + let full_cache_digests = cache_digester + .compute_digests_for_range(cardano_db.get_immutable_dir(), &immutable_range) + .await + .expect("compute_digests_for_range must not fail"); + + let without_cache_entries = without_cache_digests.entries; + let cold_cache_entries = cold_cache_digests.entries; + let full_cache_entries = full_cache_digests.entries; + assert_eq!( + without_cache_entries, full_cache_entries, + "Digests for range with or without cache should be the same" + ); + + assert_eq!( + cold_cache_entries, full_cache_entries, + "Digests for range with cold or with hot cache should be the same" + ); + } + #[tokio::test] async fn hash_computation_is_quicker_with_a_full_cache() { let cardano_db = db_builder("hash_computation_is_quicker_with_a_full_cache") diff --git a/mithril-common/src/digesters/dumb_immutable_digester.rs b/mithril-common/src/digesters/dumb_immutable_digester.rs index 82b4320d7b4..b7218082768 100644 --- a/mithril-common/src/digesters/dumb_immutable_digester.rs +++ b/mithril-common/src/digesters/dumb_immutable_digester.rs @@ -1,13 +1,15 @@ -use std::path::Path; +use std::{collections::BTreeMap, ops::RangeInclusive, path::Path}; use crate::{ crypto_helper::{MKTree, MKTreeStoreInMemory}, digesters::{ImmutableDigester, ImmutableDigesterError}, - entities::CardanoDbBeacon, + entities::{CardanoDbBeacon, ImmutableFileNumber}, }; use async_trait::async_trait; use tokio::sync::RwLock; +use super::{immutable_digester::ComputedImmutablesDigests, ImmutableFile}; + /// A [ImmutableDigester] returning configurable result for testing purpose. pub struct DumbImmutableDigester { digest: RwLock, @@ -69,6 +71,42 @@ impl ImmutableDigester for DumbImmutableDigester { } } + async fn compute_digests_for_range( + &self, + dirpath: &Path, + range: &RangeInclusive, + ) -> Result { + if self.is_success { + let immutable_file_paths = range + .clone() + .flat_map(|immutable_file_number| { + vec![ + Path::new(&format!("{immutable_file_number:0>5}.chunk")).to_path_buf(), + Path::new(&format!("{immutable_file_number:0>5}.primary")).to_path_buf(), + Path::new(&format!("{immutable_file_number:0>5}.secondary")).to_path_buf(), + ] + }) + .collect::>(); + let digest = self.digest.read().await.clone(); + + Ok(ComputedImmutablesDigests::compute_immutables_digests( + BTreeMap::from_iter(immutable_file_paths.into_iter().map(|immutable_file_path| { + ( + ImmutableFile::new(immutable_file_path).unwrap(), + Some(digest.clone()), + ) + })), + slog::Logger::root(slog::Discard, slog::o!()), + )?) + } else { + Err(ImmutableDigesterError::NotEnoughImmutable { + expected_number: *range.end(), + found_number: None, + db_dir: dirpath.to_owned(), + }) + } + } + async fn compute_merkle_tree( &self, dirpath: &Path, diff --git a/mithril-common/src/digesters/immutable_digester.rs b/mithril-common/src/digesters/immutable_digester.rs index 299d0011749..e31223c9ff6 100644 --- a/mithril-common/src/digesters/immutable_digester.rs +++ b/mithril-common/src/digesters/immutable_digester.rs @@ -1,27 +1,35 @@ -use crate::{ - crypto_helper::{MKTree, MKTreeStoreInMemory}, - digesters::ImmutableFileListingError, - entities::{CardanoDbBeacon, ImmutableFileNumber}, - StdError, -}; use async_trait::async_trait; +use sha2::Sha256; +use slog::{info, Logger}; use std::{ + collections::BTreeMap, io, + ops::RangeInclusive, path::{Path, PathBuf}, }; use thiserror::Error; +use crate::{ + crypto_helper::{MKTree, MKTreeStoreInMemory}, + digesters::ImmutableFileListingError, + entities::{CardanoDbBeacon, HexEncodedDigest, ImmutableFileName, ImmutableFileNumber}, + StdError, +}; + +use super::ImmutableFile; + /// A digester than can compute the digest used for mithril signatures /// /// If you want to mock it using mockall: /// ``` /// mod test { /// use async_trait::async_trait; -/// use mithril_common::digesters::{ImmutableDigester, ImmutableDigesterError}; -/// use mithril_common::entities::CardanoDbBeacon; +/// use mithril_common::digesters::{ComputedImmutablesDigests, ImmutableDigester, ImmutableDigesterError}; +/// use mithril_common::entities::{CardanoDbBeacon, ImmutableFileNumber}; /// use mithril_common::crypto_helper::{MKTree, MKTreeStoreInMemory}; /// use anyhow::anyhow; /// use mockall::mock; +/// use std::ops::RangeInclusive; /// use std::path::Path; /// /// mock! { @@ -35,6 +43,12 @@ use thiserror::Error; /// beacon: &CardanoDbBeacon, /// ) -> Result; /// +/// async fn compute_digests_for_range( +/// &self, +/// dirpath: &Path, +/// range: &RangeInclusive, +/// ) -> Result; +/// /// async fn compute_merkle_tree( /// &self, /// dirpath: &Path, @@ -68,6 +82,13 @@ pub trait ImmutableDigester: Sync + Send { beacon: &CardanoDbBeacon, ) -> Result; + /// Compute the digests for a range of immutable files + async fn compute_digests_for_range( + &self, + dirpath: &Path, + range: &RangeInclusive, + ) -> Result; + /// Compute the digests merkle tree async fn compute_merkle_tree( &self, @@ -103,3 +124,99 @@ pub enum ImmutableDigesterError { #[error("Merkle tree computation failed")] MerkleTreeComputationError(StdError), } + +/// Computed immutables digests +pub struct ComputedImmutablesDigests { + /// A map of [ImmutableFile] to their respective digest. + pub entries: BTreeMap, + pub(super) new_cached_entries: Vec, +} + +impl ComputedImmutablesDigests { + pub(super) fn compute_immutables_digests( + entries: BTreeMap>, + logger: Logger, + ) -> Result { + let mut new_cached_entries = Vec::new(); + let mut progress = Progress { + index: 0, + total: entries.len(), + }; + + let mut digests = BTreeMap::new(); + + for (ix, (entry, cache)) in entries.into_iter().enumerate() { + let hash = match cache { + None => { + new_cached_entries.push(entry.filename.clone()); + hex::encode(entry.compute_raw_hash::()?) + } + Some(digest) => digest, + }; + digests.insert(entry, hash); + + if progress.report(ix) { + info!(logger, "Hashing: {progress}"); + } + } + + Ok(ComputedImmutablesDigests { + entries: digests, + new_cached_entries, + }) + } +} + +pub(super) struct Progress { + pub(super) index: usize, + pub(super) total: usize, +} + +impl Progress { + pub(super) fn report(&mut self, ix: usize) -> bool { + self.index = ix; + (20 * ix) % self.total == 0 + } + + pub(super) fn percent(&self) -> f64 { + (self.index as f64 * 100.0 / self.total as f64).ceil() + } +} + +impl std::fmt::Display for Progress { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + write!(f, "{}/{} ({}%)", self.index, self.total, self.percent()) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn reports_progress_every_5_percent() { + let mut progress = Progress { + index: 0, + total: 7000, + }; + + assert!(!progress.report(1)); + assert!(!progress.report(4)); + assert!(progress.report(350)); + assert!(!progress.report(351)); + } + + #[test] + fn reports_progress_when_total_lower_than_20() { + let mut progress = Progress { + index: 0, + total: 16, + }; + + assert!(progress.report(4)); + assert!(progress.report(12)); + assert!(!progress.report(3)); + assert!(!progress.report(15)); + } +} diff --git a/mithril-common/src/digesters/immutable_file.rs b/mithril-common/src/digesters/immutable_file.rs index 1e4ae0289ff..9dada02bce4 100644 --- a/mithril-common/src/digesters/immutable_file.rs +++ b/mithril-common/src/digesters/immutable_file.rs @@ -129,12 +129,7 @@ impl ImmutableFile { } /// List all [`ImmutableFile`] in a given directory. - /// - /// Important Note: It will skip the last chunk / primary / secondary trio since they're not yet - /// complete. - pub fn list_completed_in_dir( - dir: &Path, - ) -> Result, ImmutableFileListingError> { + pub fn list_all_in_dir(dir: &Path) -> Result, ImmutableFileListingError> { let immutable_dir = find_immutables_dir(dir).ok_or(MissingImmutableFolder(dir.to_path_buf()))?; let mut files: Vec = vec![]; @@ -151,6 +146,18 @@ impl ImmutableFile { } files.sort(); + Ok(files) + } + + /// List all complete [`ImmutableFile`] in a given directory. + /// + /// Important Note: It will skip the last chunk / primary / secondary trio since they're not yet + /// complete. + pub fn list_completed_in_dir( + dir: &Path, + ) -> Result, ImmutableFileListingError> { + let files = Self::list_all_in_dir(dir)?; + match files.last() { // empty list None => Ok(files), @@ -208,7 +215,7 @@ mod tests { } #[test] - fn list_immutable_file_fail_if_not_in_immutable_dir() { + fn list_completed_immutable_file_fail_if_not_in_immutable_dir() { let target_dir = get_test_dir("list_immutable_file_fail_if_not_in_immutable_dir/invalid"); let entries = vec![]; create_fake_files(&target_dir, &entries); @@ -218,7 +225,48 @@ mod tests { } #[test] - fn list_immutable_file_should_skip_last_number() { + fn list_all_immutable_file_should_not_skip_last_number() { + let target_dir = + get_test_dir("list_all_immutable_file_should_not_skip_last_number/immutable"); + let entries = vec![ + "123.chunk", + "123.primary", + "123.secondary", + "125.chunk", + "125.primary", + "125.secondary", + "0124.chunk", + "0124.primary", + "0124.secondary", + "223.chunk", + "223.primary", + "223.secondary", + "0423.chunk", + "0423.primary", + "0423.secondary", + "0424.chunk", + "0424.primary", + "0424.secondary", + "21.chunk", + "21.primary", + "21.secondary", + ]; + create_fake_files(&target_dir, &entries); + let result = ImmutableFile::list_all_in_dir(target_dir.parent().unwrap()) + .expect("ImmutableFile::list_in_dir Failed"); + + assert_eq!(result.last().unwrap().number, 424); + assert_eq!( + result.len(), + entries.len(), + "Expected to find {} files but found {}", + entries.len(), + result.len(), + ); + } + + #[test] + fn list_completed_immutable_file_should_skip_last_number() { let target_dir = get_test_dir("list_immutable_file_should_skip_last_number/immutable"); let entries = vec![ "123.chunk", @@ -251,14 +299,14 @@ mod tests { assert_eq!( result.len(), entries.len() - 3, - "Expected to find {} files since The last (chunk, primary, secondary) trio is skipped, but found {}", + "Expected to find {} files since the last (chunk, primary, secondary) trio is skipped, but found {}", entries.len() - 3, result.len(), ); } #[test] - fn list_immutable_file_should_works_in_a_empty_folder() { + fn list_completed_immutable_file_should_works_in_a_empty_folder() { let target_dir = get_test_dir("list_immutable_file_should_works_even_in_a_empty_folder/immutable"); let entries = vec![]; @@ -270,8 +318,9 @@ mod tests { } #[test] - fn immutable_order_should_be_deterministic() { - let target_dir = get_test_dir("immutable_order_should_be_deterministic/immutable"); + fn list_completed_immutable_file_order_should_be_deterministic() { + let target_dir = + get_test_dir("list_completed_immutable_file_order_should_be_deterministic/immutable"); let entries = vec![ "21.chunk", "21.primary", @@ -305,7 +354,7 @@ mod tests { } #[test] - fn list_immutable_file_should_work_with_non_immutable_files() { + fn list_completed_immutable_file_should_work_with_non_immutable_files() { let target_dir = get_test_dir("list_immutable_file_should_work_with_non_immutable_files/immutable"); let entries = vec![ @@ -328,7 +377,7 @@ mod tests { } #[test] - fn list_immutable_file_can_list_incomplete_trio() { + fn list_completed_immutable_file_can_list_incomplete_trio() { let target_dir = get_test_dir("list_immutable_file_can_list_incomplete_trio/immutable"); let entries = vec![ "21.chunk", diff --git a/mithril-common/src/digesters/mod.rs b/mithril-common/src/digesters/mod.rs index cb51a97d9cb..cd2e63957ad 100644 --- a/mithril-common/src/digesters/mod.rs +++ b/mithril-common/src/digesters/mod.rs @@ -8,7 +8,9 @@ mod immutable_file; mod immutable_file_observer; pub use cardano_immutable_digester::CardanoImmutableDigester; -pub use immutable_digester::{ImmutableDigester, ImmutableDigesterError}; +pub use immutable_digester::{ + ComputedImmutablesDigests, ImmutableDigester, ImmutableDigesterError, +}; pub use immutable_file::{ImmutableFile, ImmutableFileCreationError, ImmutableFileListingError}; pub use immutable_file_observer::{ DumbImmutableFileObserver, ImmutableFileObserver, ImmutableFileObserverError, From d42fcc94843c4b044bdea2b76202588d86ada5ab Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 4 Feb 2025 16:40:16 +0100 Subject: [PATCH 16/59] fix: add missing conversion for 'MKTreeNode' --- mithril-common/src/crypto_helper/merkle_tree.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mithril-common/src/crypto_helper/merkle_tree.rs b/mithril-common/src/crypto_helper/merkle_tree.rs index 287c243cedc..c41a75a8c33 100644 --- a/mithril-common/src/crypto_helper/merkle_tree.rs +++ b/mithril-common/src/crypto_helper/merkle_tree.rs @@ -60,6 +60,14 @@ impl From for MKTreeNode { } } +impl From<&String> for MKTreeNode { + fn from(other: &String) -> Self { + Self { + hash: other.as_str().into(), + } + } +} + impl From<&str> for MKTreeNode { fn from(other: &str) -> Self { Self { From ff6e87548856ae4e1d1e41198308a012a559dc0b Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 4 Feb 2025 16:43:18 +0100 Subject: [PATCH 17/59] feat: implement Merkle proof computation in Cardano database client --- mithril-client/src/cardano_database_client.rs | 320 +++++++++++++++++- 1 file changed, 311 insertions(+), 9 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index 72654ca2df4..77c0f09fbac 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -45,6 +45,8 @@ // TODO: reorganize the imports #[cfg(feature = "fs")] +use std::collections::BTreeMap; +#[cfg(feature = "fs")] use std::fs; #[cfg(feature = "fs")] use std::ops::RangeInclusive; @@ -56,15 +58,21 @@ use std::{collections::HashSet, sync::Arc}; use anyhow::anyhow; use anyhow::Context; #[cfg(feature = "fs")] +use serde::de::DeserializeOwned; +#[cfg(feature = "fs")] use slog::Logger; #[cfg(feature = "fs")] use mithril_common::{ + crypto_helper::{MKProof, MKTree, MKTreeNode, MKTreeStoreInMemory}, + digesters::{CardanoImmutableDigester, ImmutableDigester}, entities::{ AncillaryLocation, CompressionAlgorithm, DigestLocation, HexEncodedDigest, - ImmutableFileNumber, ImmutablesLocation, + ImmutableFileName, ImmutableFileNumber, ImmutablesLocation, + }, + messages::{ + CardanoDatabaseDigestListItemMessage, CertificateMessage, SignedEntityTypeMessagePart, }, - messages::CardanoDatabaseDigestListItemMessage, StdResult, }; @@ -198,16 +206,16 @@ impl CardanoDatabaseClient { /// Fetch the given Cardano database data with an aggregator request. /// If it cannot be found, a None is returned. - async fn fetch_with_aggregator_request( + async fn fetch_with_aggregator_request( &self, request: AggregatorRequest, - ) -> MithrilResult> { + ) -> MithrilResult> { match self.aggregator_client.get_content(request).await { Ok(content) => { - let cardano_database: CardanoDatabaseSnapshot = serde_json::from_str(&content) + let result: T = serde_json::from_str(&content) .with_context(|| "CardanoDatabase client can not deserialize artifact")?; - Ok(Some(cardano_database)) + Ok(Some(result)) } Err(AggregatorClientError::RemoteServerLogical(_)) => Ok(None), Err(e) => Err(e.into()), @@ -289,6 +297,14 @@ impl CardanoDatabaseClient { fs::create_dir_all(dir).map_err(|e| anyhow!("Failed creating directory: {e}")) } + fn delete_directory(dir: &Path) -> StdResult<()> { + if dir.exists() { + fs::remove_dir_all(dir).map_err(|e| anyhow!("Failed deleting directory: {e}"))?; + } + + Ok(()) + } + fn read_files_in_directory(dir: &Path) -> StdResult> { let mut files = vec![]; for entry in fs::read_dir(dir)? { @@ -501,6 +517,67 @@ impl CardanoDatabaseClient { "Failed downloading and unpacking digests for all locations" )) } + + fn read_digest_file( + &self, + digest_file_target_dir: &Path, + ) -> StdResult> { + let digest_files = Self::read_files_in_directory(digest_file_target_dir)?; + if digest_files.len() > 1 { + return Err(anyhow!( + "Multiple digest files found in directory: {digest_file_target_dir:?}" + )); + } + if digest_files.is_empty() { + return Err(anyhow!( + "No digest file found in directory: {digest_file_target_dir:?}" + )); + } + + let digest_file = &digest_files[0]; + let content = fs::read_to_string(digest_file) + .with_context(|| format!("Failed reading digest file: {digest_file:?}"))?; + let digest_messages: Vec = + serde_json::from_str(&content) + .with_context(|| format!("Failed deserializing digest file: {digest_file:?}"))?; + let digest_map = digest_messages + .into_iter() + .map(|message| (message.immutable_file_name, message.digest)) + .collect::>(); + + Ok(digest_map) + } + + /// Compute the Merkle proof of membership for the given immutable file range. + // TODO: Add example in module documentation + pub async fn compute_merkle_proof( + &self, + certificate: &CertificateMessage, + immutable_file_range: &ImmutableFileRange, + database_dir: &Path, + ) -> StdResult { + let network = certificate.metadata.network.clone(); + let last_immutable_file_number = match &certificate.signed_entity_type { + SignedEntityTypeMessagePart::CardanoDatabase(beacon) => beacon.immutable_file_number, + _ => return Err(anyhow!("Invalid signed entity type: {:?}",certificate.signed_entity_type)), + }; + let immutable_file_number_range = + immutable_file_range.to_range_inclusive(last_immutable_file_number)?; + + let downloaded_digests = self.read_digest_file(&Self::digest_target_dir(database_dir))?; + let merkle_tree: MKTree = + MKTree::new(&downloaded_digests.values().cloned().collect::>())?; + let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone()); + let computed_digests = immutable_digester + .compute_digests_for_range(database_dir, &immutable_file_number_range) + .await?.entries + .values() + .map(MKTreeNode::from) + .collect::>(); + Self::delete_directory(&Self::digest_target_dir(database_dir))?; + + merkle_tree.compute_proof(&computed_digests) + } } } @@ -512,9 +589,12 @@ mod tests { mod cardano_database_client { use anyhow::anyhow; use chrono::{DateTime, Utc}; - use mithril_common::entities::{ - AncillaryLocationDiscriminants, CardanoDbBeacon, CompressionAlgorithm, - DigestLocationDiscriminants, Epoch, FileUri, ImmutablesLocationDiscriminants, + use mithril_common::{ + digesters::CardanoImmutableDigester, + entities::{ + AncillaryLocationDiscriminants, CardanoDbBeacon, CompressionAlgorithm, + DigestLocationDiscriminants, Epoch, FileUri, ImmutablesLocationDiscriminants, + }, }; use mockall::predicate::{self, eq}; @@ -1732,6 +1812,228 @@ mod tests { .unwrap(); } } + + mod read_digest_file { + use std::io::Write; + + use mithril_common::test_utils::TempDir; + + use super::*; + + fn create_valid_fake_digest_file( + file_path: &Path, + digest_messages: &[CardanoDatabaseDigestListItemMessage], + ) { + let mut file = fs::File::create(file_path).unwrap(); + let digest_json = serde_json::to_string(&digest_messages).unwrap(); + file.write_all(digest_json.as_bytes()).unwrap(); + } + + fn create_invalid_fake_digest_file(file_path: &Path) { + let mut file = fs::File::create(file_path).unwrap(); + file.write_all(b"incorrect-digest").unwrap(); + } + + #[test] + fn read_digest_file_fails_when_no_digest_file() { + let target_dir = TempDir::new( + "cardano_database_client", + "read_digest_file_fails_when_no_digest_file", + ) + .build(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .read_digest_file(&target_dir) + .expect_err("read_digest_file should fail"); + } + + #[test] + fn read_digest_file_fails_when_multiple_digest_files() { + let target_dir = TempDir::new( + "cardano_database_client", + "read_digest_file_fails_when_multiple_digest_files", + ) + .build(); + create_valid_fake_digest_file(&target_dir.join("digests.json"), &[]); + create_valid_fake_digest_file(&target_dir.join("digests-2.json"), &[]); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .read_digest_file(&target_dir) + .expect_err("read_digest_file should fail"); + } + + #[test] + fn read_digest_file_fails_when_invalid_unique_digest_file() { + let target_dir = TempDir::new( + "cardano_database_client", + "read_digest_file_fails_when_invalid_unique_digest_file", + ) + .build(); + create_invalid_fake_digest_file(&target_dir.join("digests.json")); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .read_digest_file(&target_dir) + .expect_err("read_digest_file should fail"); + } + + #[test] + fn read_digest_file_succeeds_when_valid_unique_digest_file() { + let target_dir = TempDir::new( + "cardano_database_client", + "read_digest_file_succeeds_when_valid_unique_digest_file", + ) + .build(); + let digest_messages = vec![ + CardanoDatabaseDigestListItemMessage { + immutable_file_name: "00001.chunk".to_string(), + digest: "digest-1".to_string(), + }, + CardanoDatabaseDigestListItemMessage { + immutable_file_name: "00002.chunk".to_string(), + digest: "digest-2".to_string(), + }, + ]; + create_valid_fake_digest_file(&target_dir.join("digests.json"), &digest_messages); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + let digests = client.read_digest_file(&target_dir).unwrap(); + assert_eq!( + BTreeMap::from([ + ("00001.chunk".to_string(), "digest-1".to_string()), + ("00002.chunk".to_string(), "digest-2".to_string()) + ]), + digests + ) + } + } + } + + mod compute_merkle_proof { + use mithril_common::{ + digesters::{DummyCardanoDbBuilder, ImmutableDigester, ImmutableFile}, + messages::SignedEntityTypeMessagePart, + }; + + use crate::test_utils::test_logger; + + use super::*; + + async fn write_digest_file( + digest_dir: &Path, + digests: BTreeMap, + ) { + let digest_file_path = digest_dir.join("digests.json"); + if !digest_dir.exists() { + fs::create_dir_all(digest_dir).unwrap(); + } + + let immutable_digest_messages = digests + .into_iter() + .map( + |(immutable_file, digest)| CardanoDatabaseDigestListItemMessage { + immutable_file_name: immutable_file.filename, + digest, + }, + ) + .collect::>(); + serde_json::to_writer( + fs::File::create(digest_file_path).unwrap(), + &immutable_digest_messages, + ) + .unwrap(); + } + + #[tokio::test] + async fn compute_merkle_proof_fails_if_mismatching_certificate() { + let immutable_file_range = 1..=5; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let certificate = CertificateMessage { + hash: "cert-hash-123".to_string(), + signed_entity_type: SignedEntityTypeMessagePart::MithrilStakeDistribution( + Epoch(123), + ), + ..CertificateMessage::dummy() + }; + let cardano_db = DummyCardanoDbBuilder::new( + "compute_merkle_proof_fails_if_mismatching_certificate", + ) + .with_immutables(&immutable_file_range.clone().collect::>()) + .append_immutable_trio() + .build(); + let database_dir = cardano_db.get_dir(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .compute_merkle_proof( + &certificate, + &immutable_file_range_to_prove, + database_dir, + ) + .await + .expect_err("compute_merkle_proof should fail"); + } + + #[tokio::test] + async fn compute_merkle_proof_succeeds() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 5, + }; + let immutable_file_range = 1..=5; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let certificate = CertificateMessage { + hash: "cert-hash-123".to_string(), + signed_entity_type: SignedEntityTypeMessagePart::CardanoDatabase( + beacon.clone(), + ), + ..CertificateMessage::dummy() + }; + let cardano_db = DummyCardanoDbBuilder::new("compute_merkle_proof_succeeds") + .with_immutables(&immutable_file_range.clone().collect::>()) + .append_immutable_trio() + .build(); + let database_dir = cardano_db.get_dir(); + let immutable_digester = CardanoImmutableDigester::new( + certificate.metadata.network.clone(), + None, + test_logger(), + ); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + let computed_digests = immutable_digester + .compute_digests_for_range(database_dir, &immutable_file_range) + .await + .unwrap(); + write_digest_file(&database_dir.join("digest"), computed_digests.entries).await; + let merkle_tree = immutable_digester + .compute_merkle_tree(database_dir, &beacon) + .await + .unwrap(); + let expected_merkle_root = merkle_tree.compute_root().unwrap(); + + let merkle_proof = client + .compute_merkle_proof( + &certificate, + &immutable_file_range_to_prove, + database_dir, + ) + .await + .unwrap(); + let merkle_proof_root = merkle_proof.root().to_owned(); + + merkle_proof.verify().unwrap(); + assert_eq!(expected_merkle_root, merkle_proof_root); + + assert!(!database_dir.join("digest").exists()); + } } mod immutable_file_range { From 91e74c3964a5173ebde649ccc6e4517a9410f057 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 7 Feb 2025 17:35:23 +0100 Subject: [PATCH 18/59] feat: compute Cardano database messages in message service --- mithril-client/src/message.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mithril-client/src/message.rs b/mithril-client/src/message.rs index 50d2919c61e..5b340fdbbcf 100644 --- a/mithril-client/src/message.rs +++ b/mithril-client/src/message.rs @@ -10,6 +10,7 @@ use mithril_common::protocol::SignerBuilder; use mithril_common::signable_builder::CardanoStakeDistributionSignableBuilder; #[cfg(feature = "fs")] use mithril_common::{ + crypto_helper::MKProof, digesters::{CardanoImmutableDigester, ImmutableDigester}, messages::SignedEntityTypeMessagePart, }; @@ -99,6 +100,21 @@ impl MessageBuilder { Ok(message) } + + /// Compute message for a Cardano database. + pub async fn compute_cardano_database_message( + &self, + certificate: &MithrilCertificate, + merkle_proof: &MKProof, + ) -> MithrilResult { + let mut message = certificate.protocol_message.clone(); + message.set_message_part( + ProtocolMessagePartKey::CardanoDatabaseMerkleRoot, + merkle_proof.root().to_hex(), + ); + + Ok(message) + } } /// Compute message for a Mithril stake distribution. From 7e8cddd6cf6e4842242e2b7fdde57154dcdc8720 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 6 Feb 2025 15:12:13 +0100 Subject: [PATCH 19/59] refactor: 'download_unpack' function signature in Cardano database client --- mithril-client/src/cardano_database_client.rs | 155 ++++++------------ 1 file changed, 46 insertions(+), 109 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index 77c0f09fbac..def0a04a168 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -41,6 +41,7 @@ //! } //! # Ok(()) //! # } + //! ``` // TODO: reorganize the imports @@ -227,15 +228,11 @@ impl CardanoDatabaseClient { // TODO: Add example in module documentation pub async fn download_unpack( &self, - hash: &str, - immutable_file_range: ImmutableFileRange, + cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, + immutable_file_range: &ImmutableFileRange, target_dir: &Path, download_unpack_options: DownloadUnpackOptions, ) -> StdResult<()> { - let cardano_database_snapshot = self - .get(hash) - .await? - .ok_or_else(|| anyhow!("Cardano database snapshot not found"))?; let compression_algorithm = cardano_database_snapshot.compression_algorithm; let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; let immutable_file_number_range = @@ -247,23 +244,23 @@ impl CardanoDatabaseClient { &download_unpack_options, )?; - let immutable_locations = cardano_database_snapshot.locations.immutables; + let immutable_locations = &cardano_database_snapshot.locations.immutables; self.download_unpack_immutable_files( - &immutable_locations, + immutable_locations, immutable_file_number_range, &compression_algorithm, - &Self::immutable_files_target_dir(target_dir), + target_dir, ) .await?; - let digest_locations = cardano_database_snapshot.locations.digests; - self.download_unpack_digest_file(&digest_locations, &Self::digest_target_dir(target_dir)) + let digest_locations = &cardano_database_snapshot.locations.digests; + self.download_unpack_digest_file(digest_locations, &Self::digest_target_dir(target_dir)) .await?; if download_unpack_options.include_ancillary { - let ancillary_locations = cardano_database_snapshot.locations.ancillary; + let ancillary_locations = &cardano_database_snapshot.locations.ancillary; self.download_unpack_ancillary_file( - &ancillary_locations, + ancillary_locations, &compression_algorithm, target_dir, ) @@ -987,58 +984,22 @@ mod tests { use super::*; - #[tokio::test] - async fn download_unpack_fails_with_invalid_snapshot() { - let immutable_file_range = ImmutableFileRange::Range(1, 10); - let download_unpack_options = DownloadUnpackOptions::default(); - let cardano_db_snapshot_hash = &"hash-123"; - let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - http_client.expect_get_content().return_once(move |_| { - Err(AggregatorClientError::RemoteServerLogical(anyhow!( - "not found" - ))) - }); - }) - .build_cardano_database_client(); - - client - .download_unpack( - cardano_db_snapshot_hash, - immutable_file_range, - target_dir, - download_unpack_options, - ) - .await - .expect_err("download_unpack should fail"); - } - #[tokio::test] async fn download_unpack_fails_with_invalid_immutable_file_range() { let immutable_file_range = ImmutableFileRange::Range(1, 0); let download_unpack_options = DownloadUnpackOptions::default(); - let cardano_db_snapshot_hash = &"hash-123"; + let cardano_db_snapshot = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshot::dummy() + }; let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - let message = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - ..CardanoDatabaseSnapshot::dummy() - }; - http_client - .expect_get_content() - .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - })) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - }) .build_cardano_database_client(); client .download_unpack( - cardano_db_snapshot_hash, - immutable_file_range, + &cardano_db_snapshot, + &immutable_file_range, target_dir, download_unpack_options, ) @@ -1051,30 +1012,24 @@ mod tests { let total_immutable_files = 10; let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); let download_unpack_options = DownloadUnpackOptions::default(); - let cardano_db_snapshot_hash = &"hash-123"; + let cardano_db_snapshot = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + locations: ArtifactsLocationsMessagePart { + immutables: vec![ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }], + ..ArtifactsLocationsMessagePart::default() + }, + ..CardanoDatabaseSnapshot::dummy() + }; let target_dir = TempDir::new( "cardano_database_client", "download_unpack_fails_when_immutable_files_download_fail", ) .build(); let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - let mut message = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - ..CardanoDatabaseSnapshot::dummy() - }; - message.locations.immutables = vec![ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever/{immutable_file_number}.tar.gz".to_string(), - )), - }]; - http_client - .expect_get_content() - .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - })) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - }) .with_immutable_file_downloaders(vec![( ImmutablesLocationDiscriminants::CloudStorage, Arc::new({ @@ -1088,8 +1043,8 @@ mod tests { client .download_unpack( - cardano_db_snapshot_hash, - immutable_file_range, + &cardano_db_snapshot, + &immutable_file_range, &target_dir, download_unpack_options, ) @@ -1102,7 +1057,10 @@ mod tests { ) { let immutable_file_range = ImmutableFileRange::Range(1, 10); let download_unpack_options = DownloadUnpackOptions::default(); - let cardano_db_snapshot_hash = &"hash-123"; + let cardano_db_snapshot = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshot::dummy() + }; let target_dir = &TempDir::new( "cardano_database_client", "download_unpack_fails_when_target_target_dir_would_be_overwritten_without_allow_override", @@ -1110,24 +1068,12 @@ mod tests { .build(); fs::create_dir_all(target_dir.join("immutable")).unwrap(); let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - let message = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - ..CardanoDatabaseSnapshot::dummy() - }; - http_client - .expect_get_content() - .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - })) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - }) .build_cardano_database_client(); client .download_unpack( - cardano_db_snapshot_hash, - immutable_file_range, + &cardano_db_snapshot, + &immutable_file_range, target_dir, download_unpack_options, ) @@ -1142,13 +1088,7 @@ mod tests { include_ancillary: true, ..DownloadUnpackOptions::default() }; - let cardano_db_snapshot_hash = &"hash-123"; - let target_dir = TempDir::new( - "cardano_database_client", - "download_unpack_succeeds_with_valid_range", - ) - .build(); - let message = CardanoDatabaseSnapshot { + let cardano_db_snapshot = CardanoDatabaseSnapshot { hash: "hash-123".to_string(), locations: ArtifactsLocationsMessagePart { immutables: vec![ImmutablesLocation::CloudStorage { @@ -1165,27 +1105,24 @@ mod tests { }, ..CardanoDatabaseSnapshot::dummy() }; + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_succeeds_with_valid_range", + ) + .build(); let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - http_client - .expect_get_content() - .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - })) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - }) .with_immutable_file_downloaders(vec![( ImmutablesLocationDiscriminants::CloudStorage, Arc::new({ let mock_file_downloader = MockFileDownloaderBuilder::default() .with_file_uri("http://whatever/00001.tar.gz") - .with_target_dir(target_dir.join("immutable")) + .with_target_dir(target_dir.clone()) .with_success() .build(); MockFileDownloaderBuilder::from_mock(mock_file_downloader) .with_file_uri("http://whatever/00002.tar.gz") - .with_target_dir(target_dir.join("immutable")) + .with_target_dir(target_dir.clone()) .with_success() .build() }), @@ -1216,8 +1153,8 @@ mod tests { client .download_unpack( - cardano_db_snapshot_hash, - immutable_file_range, + &cardano_db_snapshot, + &immutable_file_range, &target_dir, download_unpack_options, ) From 98fdd1bf751c6ca403f7a1ab156dc89b0ac6f5af Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 5 Feb 2025 17:00:54 +0100 Subject: [PATCH 20/59] feat: add support for feedback sender in Cardano database client --- mithril-client/src/client.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mithril-client/src/client.rs b/mithril-client/src/client.rs index 90779b14be5..6d010b42ef2 100644 --- a/mithril-client/src/client.rs +++ b/mithril-client/src/client.rs @@ -258,7 +258,7 @@ impl ClientBuilder { #[cfg(feature = "fs")] snapshot_downloader, #[cfg(feature = "fs")] - feedback_sender, + feedback_sender.clone(), #[cfg(feature = "fs")] logger.clone(), )); @@ -282,6 +282,8 @@ impl ClientBuilder { #[cfg(feature = "fs")] digest_file_downloader_resolver, #[cfg(feature = "fs")] + feedback_sender, + #[cfg(feature = "fs")] logger, )); From 7785eb31f93538e4b8618f6f21145413f0047cef Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 5 Feb 2025 16:57:54 +0100 Subject: [PATCH 21/59] fix: add catchall in handling of feedback events in client CLI --- mithril-client-cli/src/utils/feedback_receiver.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mithril-client-cli/src/utils/feedback_receiver.rs b/mithril-client-cli/src/utils/feedback_receiver.rs index ef515c19a8c..dca97f9b78d 100644 --- a/mithril-client-cli/src/utils/feedback_receiver.rs +++ b/mithril-client-cli/src/utils/feedback_receiver.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use indicatif::{ProgressBar, ProgressDrawTarget, ProgressState, ProgressStyle}; -use slog::Logger; +use slog::{warn, Logger}; use std::fmt::Write; use tokio::sync::RwLock; @@ -111,6 +111,10 @@ impl FeedbackReceiver for IndicatifFeedbackReceiver { } *certificate_validation_pb = None; } + _ => { + // TODO: Handle other events from Cardano database client and remove this catchall + warn!(self.logger, "Unhandled event"; "event" => ?event); + } } } } From 0b4dbc66f52d469e1ed1625ac4212045b8dbe95f Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 5 Feb 2025 17:06:59 +0100 Subject: [PATCH 22/59] fix: add catchall in handling of feedback events in client example --- examples/client-snapshot/src/main.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/client-snapshot/src/main.rs b/examples/client-snapshot/src/main.rs index 64a94a5a978..dc18c514a62 100644 --- a/examples/client-snapshot/src/main.rs +++ b/examples/client-snapshot/src/main.rs @@ -190,6 +190,7 @@ impl FeedbackReceiver for IndicatifFeedbackReceiver { } *certificate_validation_pb = None; } + _ => panic!("Unexpected event: {:?}", event), } } } From 5139bc39720c7522b698b197cd7b650f4b86eb77 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 5 Feb 2025 17:01:23 +0100 Subject: [PATCH 23/59] feat: add feedback events for immutable downloads in Cardano database client --- mithril-client/src/cardano_database_client.rs | 85 ++++++++++++++++++- mithril-client/src/feedback.rs | 52 ++++++++++++ 2 files changed, 134 insertions(+), 3 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index def0a04a168..1bfb2307045 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -77,11 +77,13 @@ use mithril_common::{ StdResult, }; +#[cfg(feature = "fs")] +use crate::feedback::{FeedbackSender, MithrilEvent}; use crate::{ aggregator_client::{AggregatorClient, AggregatorClientError, AggregatorRequest}, file_downloader::{FileDownloaderResolver, FileDownloaderUri}, + CardanoDatabaseSnapshot, CardanoDatabaseSnapshotListItem, MithrilResult, }; -use crate::{CardanoDatabaseSnapshot, CardanoDatabaseSnapshotListItem, MithrilResult}; cfg_fs! { /// Immutable file range representation @@ -151,6 +153,8 @@ pub struct CardanoDatabaseClient { #[cfg(feature = "fs")] digest_file_downloader_resolver: Arc>, #[cfg(feature = "fs")] + feedback_sender: FeedbackSender, + #[cfg(feature = "fs")] logger: Logger, } @@ -167,6 +171,7 @@ impl CardanoDatabaseClient { #[cfg(feature = "fs")] digest_file_downloader_resolver: Arc< dyn FileDownloaderResolver, >, + #[cfg(feature = "fs")] feedback_sender: FeedbackSender, #[cfg(feature = "fs")] logger: Logger, ) -> Self { Self { @@ -178,6 +183,8 @@ impl CardanoDatabaseClient { #[cfg(feature = "fs")] digest_file_downloader_resolver, #[cfg(feature = "fs")] + feedback_sender, + #[cfg(feature = "fs")] logger: mithril_common::logging::LoggerExtensions::new_with_component_name::( &logger, ), @@ -372,7 +379,6 @@ impl CardanoDatabaseClient { /// /// The download is attempted for each location until the full range is downloaded. /// An error is returned if not all the files are downloaded. - // TODO: Add feedback receivers async fn download_unpack_immutable_files( &self, locations: &[ImmutablesLocation], @@ -401,7 +407,10 @@ impl CardanoDatabaseClient { .as_slice(), )?; for (immutable_file_number, file_downloader_uri) in file_downloader_uris { - let download_id = format!("{location:?}"); //TODO: check if this is the correct way to format the download_id + let download_id = MithrilEvent::new_snapshot_download_id(); + self.feedback_sender + .send_event(MithrilEvent::ImmutableDownloadStarted { immutable_file_number, download_id: download_id.clone()}) + .await; let downloaded = file_downloader .download_unpack( &file_downloader_uri, @@ -413,6 +422,9 @@ impl CardanoDatabaseClient { match downloaded { Ok(_) => { immutable_file_numbers_to_download.remove(&immutable_file_number); + self.feedback_sender + .send_event(MithrilEvent::ImmutableDownloadCompleted { download_id }) + .await; } Err(e) => { slog::error!( @@ -597,6 +609,7 @@ mod tests { use crate::{ aggregator_client::MockAggregatorHTTPClient, + feedback::{FeedbackReceiver, MithrilEvent, StackFeedbackReceiver}, file_downloader::{ AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloader, ImmutablesFileDownloaderResolver, MockFileDownloader, @@ -646,6 +659,7 @@ mod tests { immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver, ancillary_file_downloader_resolver: AncillaryFileDownloaderResolver, digest_file_downloader_resolver: DigestFileDownloaderResolver, + feedback_receivers: Vec>, } impl CardanoDatabaseClientDependencyInjector { @@ -659,6 +673,7 @@ mod tests { vec![], ), digest_file_downloader_resolver: DigestFileDownloaderResolver::new(vec![]), + feedback_receivers: vec![], } } @@ -710,12 +725,23 @@ mod tests { } } + fn with_feedback_receivers( + self, + feedback_receivers: &[Arc], + ) -> Self { + Self { + feedback_receivers: feedback_receivers.to_vec(), + ..self + } + } + fn build_cardano_database_client(self) -> CardanoDatabaseClient { CardanoDatabaseClient::new( Arc::new(self.http_client), Arc::new(self.immutable_file_downloader_resolver), Arc::new(self.ancillary_file_downloader_resolver), Arc::new(self.digest_file_downloader_resolver), + FeedbackSender::new(&self.feedback_receivers), test_utils::test_logger(), ) } @@ -1520,6 +1546,59 @@ mod tests { .await .unwrap(); } + + #[tokio::test] + async fn download_unpack_immutable_files_sends_feedbacks() { + let total_immutable_files = 1; + let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_immutable_files_sends_feedbacks", + ) + .build(); + let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + MockFileDownloaderBuilder::default() + .with_success() + .build() + }), + )]) + .with_feedback_receivers(&[feedback_receiver.clone()]) + .build_cardano_database_client(); + + client + .download_unpack_immutable_files( + &[ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }], + immutable_file_range + .to_range_inclusive(total_immutable_files) + .unwrap(), + &CompressionAlgorithm::default(), + &target_dir, + ) + .await + .unwrap(); + + let sent_events = feedback_receiver.stacked_events(); + let id = sent_events[0].event_id(); + let expected_events = vec![ + MithrilEvent::ImmutableDownloadStarted { + immutable_file_number: 1, + download_id: id.to_string(), + + }, + MithrilEvent::ImmutableDownloadCompleted { + download_id: id.to_string(), + }, + ]; + assert_eq!(expected_events, sent_events); + } } mod download_unpack_ancillary_file { diff --git a/mithril-client/src/feedback.rs b/mithril-client/src/feedback.rs index ab5d4b9f0a2..fae5fbf28e7 100644 --- a/mithril-client/src/feedback.rs +++ b/mithril-client/src/feedback.rs @@ -52,6 +52,7 @@ //! ``` use async_trait::async_trait; +use mithril_common::entities::ImmutableFileNumber; use serde::Serialize; use slog::{info, Logger}; use std::sync::{Arc, RwLock}; @@ -86,6 +87,27 @@ pub enum MithrilEvent { /// Unique identifier used to track this specific snapshot download download_id: String, }, + /// An immutable archive file download has started + ImmutableDownloadStarted { + /// Immutable file number downloaded + immutable_file_number: ImmutableFileNumber, + /// Unique identifier used to track this specific immutable archive file download download + download_id: String, + }, + /// An immutable archive file download is in progress + ImmutableDownloadProgress { + /// Unique identifier used to track this specific download + download_id: String, + /// Number of bytes that have been downloaded + downloaded_bytes: u64, + /// Size of the downloaded archive + size: u64, + }, + /// An immutable archive file download has completed + ImmutableDownloadCompleted { + /// Unique identifier used to track this specific immutable archive file download download + download_id: String, + }, /// A certificate chain validation has started CertificateChainValidationStarted { /// Unique identifier used to track this specific certificate chain validation @@ -118,6 +140,11 @@ impl MithrilEvent { Uuid::new_v4().to_string() } + /// Generate a random unique identifier to identify an immutable download + pub fn new_immutable_download_id() -> String { + Uuid::new_v4().to_string() + } + /// Generate a random unique identifier to identify a certificate chain validation pub fn new_certificate_chain_validation_id() -> String { Uuid::new_v4().to_string() @@ -129,6 +156,9 @@ impl MithrilEvent { MithrilEvent::SnapshotDownloadStarted { download_id, .. } => download_id, MithrilEvent::SnapshotDownloadProgress { download_id, .. } => download_id, MithrilEvent::SnapshotDownloadCompleted { download_id } => download_id, + MithrilEvent::ImmutableDownloadStarted { download_id, .. } => download_id, + MithrilEvent::ImmutableDownloadProgress { download_id, .. } => download_id, + MithrilEvent::ImmutableDownloadCompleted { download_id, .. } => download_id, MithrilEvent::CertificateChainValidationStarted { certificate_chain_validation_id, } => certificate_chain_validation_id, @@ -219,6 +249,28 @@ impl FeedbackReceiver for SlogFeedbackReceiver { MithrilEvent::SnapshotDownloadCompleted { download_id } => { info!(self.logger, "Snapshot download completed"; "download_id" => download_id); } + MithrilEvent::ImmutableDownloadStarted { + immutable_file_number, + download_id, + } => { + info!( + self.logger, "Immutable download started"; + "immutable_file_number" => immutable_file_number, "download_id" => download_id, + ); + } + MithrilEvent::ImmutableDownloadProgress { + download_id, + downloaded_bytes, + size, + } => { + info!( + self.logger, "Immutable download in progress ..."; + "downloaded_bytes" => downloaded_bytes, "size" => size, "download_id" => download_id, + ); + } + MithrilEvent::ImmutableDownloadCompleted { download_id } => { + info!(self.logger, "Immutable download completed"; "download_id" => download_id); + } MithrilEvent::CertificateChainValidationStarted { certificate_chain_validation_id, } => { From 65a766a5744494308f9879e134e9b9bfa3ac5abc Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 5 Feb 2025 18:06:37 +0100 Subject: [PATCH 24/59] refactor: 'FileUploader' uses a feedback event builder --- mithril-client/src/cardano_database_client.rs | 31 ++++++++++++++++--- .../src/file_downloader/interface.rs | 6 ++++ mithril-client/src/file_downloader/mod.rs | 2 +- .../src/file_downloader/resolver.rs | 22 ++++++++++--- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index 1bfb2307045..fe05c79ca86 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -375,6 +375,22 @@ impl CardanoDatabaseClient { Ok(()) } + fn feedback_event_builder_immutable_download(download_id: String, downloaded_bytes: u64, size: u64) -> Option { + Some(MithrilEvent::ImmutableDownloadProgress { + download_id, + downloaded_bytes, + size, + }) + } + + fn feedback_event_builder_ancillary_download(_download_id: String, _downloaded_bytes: u64, _size: u64) -> Option { + None + } + + fn feedback_event_builder_digest_download(_download_id: String, _downloaded_bytes: u64, _size: u64) -> Option { + None + } + /// Download and unpack the immutable files of the given range. /// /// The download is attempted for each location until the full range is downloaded. @@ -417,6 +433,7 @@ impl CardanoDatabaseClient { immutable_files_target_dir, Some(compression_algorithm.to_owned()), &download_id, + Self::feedback_event_builder_immutable_download, ) .await; match downloaded { @@ -469,6 +486,7 @@ impl CardanoDatabaseClient { ancillary_file_target_dir, Some(compression_algorithm.to_owned()), &download_id, + Self::feedback_event_builder_ancillary_download ) .await; match downloaded { @@ -509,6 +527,7 @@ impl CardanoDatabaseClient { digest_file_target_dir, None, &download_id, + Self::feedback_event_builder_digest_download ) .await; match downloaded { @@ -611,8 +630,9 @@ mod tests { aggregator_client::MockAggregatorHTTPClient, feedback::{FeedbackReceiver, MithrilEvent, StackFeedbackReceiver}, file_downloader::{ - AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloader, - ImmutablesFileDownloaderResolver, MockFileDownloader, + AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, + FeedbackEventBuilder, FileDownloader, ImmutablesFileDownloaderResolver, + MockFileDownloader, }, test_utils, }; @@ -753,6 +773,7 @@ mod tests { &Path, Option, &str, + FeedbackEventBuilder, ) -> StdResult<()> + Send + 'static, @@ -789,11 +810,11 @@ mod tests { } fn with_success(self) -> Self { - self.with_returning(Box::new(|_, _, _, _| Ok(()))) + self.with_returning(Box::new(|_, _, _, _, _| Ok(()))) } fn with_failure(self) -> Self { - self.with_returning(Box::new(|_, _, _, _| { + self.with_returning(Box::new(|_, _, _, _, _| { Err(anyhow!("Download unpack failed")) })) } @@ -858,6 +879,7 @@ mod tests { .unwrap_or(true) }); let predicate_download_id = predicate::always(); + let predicate_feedback_event_builder = predicate::always(); let mut mock_file_downloader = self.mock_file_downloader.unwrap_or_default(); mock_file_downloader @@ -867,6 +889,7 @@ mod tests { predicate_target_dir, predicate_compression_algorithm, predicate_download_id, + predicate_feedback_event_builder, ) .times(self.times) .returning(self.returning_func.unwrap()); diff --git a/mithril-client/src/file_downloader/interface.rs b/mithril-client/src/file_downloader/interface.rs index 7e0d268def5..ed8e84be4ef 100644 --- a/mithril-client/src/file_downloader/interface.rs +++ b/mithril-client/src/file_downloader/interface.rs @@ -10,6 +10,8 @@ use mithril_common::{ StdResult, }; +use crate::feedback::MithrilEvent; + /// A file downloader URI #[derive(Debug, PartialEq, Eq, Clone)] pub enum FileDownloaderUri { @@ -77,6 +79,9 @@ impl From for FileDownloaderUri { } } +/// A feedback event builder +pub type FeedbackEventBuilder = fn(String, u64, u64) -> Option; + /// A file downloader #[cfg_attr(test, mockall::automock)] #[async_trait] @@ -91,6 +96,7 @@ pub trait FileDownloader: Sync + Send { target_dir: &Path, compression_algorithm: Option, download_id: &str, + feedback_event_builder: FeedbackEventBuilder, ) -> StdResult<()>; } diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs index 010463b4310..1a6895b4ac0 100644 --- a/mithril-client/src/file_downloader/mod.rs +++ b/mithril-client/src/file_downloader/mod.rs @@ -7,7 +7,7 @@ mod resolver; #[cfg(test)] pub use interface::MockFileDownloader; -pub use interface::{FileDownloader, FileDownloaderUri}; +pub use interface::{FeedbackEventBuilder, FileDownloader, FileDownloaderUri}; #[cfg(test)] pub use resolver::MockFileDownloaderResolver; pub use resolver::{ diff --git a/mithril-client/src/file_downloader/resolver.rs b/mithril-client/src/file_downloader/resolver.rs index 12b6c848c95..596288fe338 100644 --- a/mithril-client/src/file_downloader/resolver.rs +++ b/mithril-client/src/file_downloader/resolver.rs @@ -86,17 +86,28 @@ mod tests { use mithril_common::entities::{FileUri, MultiFilesUri, TemplateUri}; - use crate::file_downloader::{FileDownloaderUri, MockFileDownloader}; + use crate::{ + feedback::MithrilEvent, + file_downloader::{FileDownloaderUri, MockFileDownloader}, + }; use super::*; + fn fake_feedback_event( + _download_id: String, + _downloaded_bytes: u64, + _size: u64, + ) -> Option { + None + } + #[tokio::test] async fn immutables_file_downloader_resolver() { let mut mock_file_downloader = MockFileDownloader::new(); mock_file_downloader .expect_download_unpack() .times(1) - .returning(|_, _, _, _| Ok(())); + .returning(|_, _, _, _, _| Ok(())); let resolver = ImmutablesFileDownloaderResolver::new(vec![( ImmutablesLocationDiscriminants::CloudStorage, Arc::new(mock_file_downloader), @@ -115,6 +126,7 @@ mod tests { Path::new("."), None, "download_id", + fake_feedback_event, ) .await .unwrap(); @@ -126,7 +138,7 @@ mod tests { mock_file_downloader_cloud_storage .expect_download_unpack() .times(1) - .returning(|_, _, _, _| Ok(())); + .returning(|_, _, _, _, _| Ok(())); let resolver = AncillaryFileDownloaderResolver::new(vec![( AncillaryLocationDiscriminants::CloudStorage, Arc::new(mock_file_downloader_cloud_storage), @@ -143,6 +155,7 @@ mod tests { Path::new("."), None, "download_id", + fake_feedback_event, ) .await .unwrap(); @@ -155,7 +168,7 @@ mod tests { mock_file_downloader_cloud_storage .expect_download_unpack() .times(1) - .returning(|_, _, _, _| Ok(())); + .returning(|_, _, _, _, _| Ok(())); let resolver = DigestFileDownloaderResolver::new(vec![ ( DigestLocationDiscriminants::Aggregator, @@ -178,6 +191,7 @@ mod tests { Path::new("."), None, "download_id", + fake_feedback_event, ) .await .unwrap(); From 01c8bd64fdbcae067953d8bd6f8ec326eba1dc7b Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 5 Feb 2025 18:21:12 +0100 Subject: [PATCH 25/59] feat: add feedback events for ancillary downloads in Cardano database client --- mithril-client/src/cardano_database_client.rs | 72 ++++++++++++++++--- mithril-client/src/feedback.rs | 50 ++++++++++++- 2 files changed, 109 insertions(+), 13 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index fe05c79ca86..25770e23535 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -383,8 +383,12 @@ impl CardanoDatabaseClient { }) } - fn feedback_event_builder_ancillary_download(_download_id: String, _downloaded_bytes: u64, _size: u64) -> Option { - None + fn feedback_event_builder_ancillary_download(download_id: String, downloaded_bytes: u64, size: u64) -> Option { + Some(MithrilEvent::AncillaryDownloadProgress { + download_id, + downloaded_bytes, + size, + }) } fn feedback_event_builder_digest_download(_download_id: String, _downloaded_bytes: u64, _size: u64) -> Option { @@ -462,7 +466,6 @@ impl CardanoDatabaseClient { } /// Download and unpack the ancillary files. - // TODO: Add feedback receivers async fn download_unpack_ancillary_file( &self, locations: &[AncillaryLocation], @@ -472,7 +475,10 @@ impl CardanoDatabaseClient { let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); for location in locations_sorted { - let download_id = format!("{location:?}"); //TODO: check if this is the correct way to format the download_id + let download_id = MithrilEvent::new_ancillary_download_id(); + self.feedback_sender + .send_event(MithrilEvent::AncillaryDownloadStarted { download_id: download_id.clone()}) + .await; let file_downloader = self .ancillary_file_downloader_resolver .resolve(&location) @@ -490,7 +496,12 @@ impl CardanoDatabaseClient { ) .await; match downloaded { - Ok(_) => return Ok(()), + Ok(_) => { + self.feedback_sender + .send_event(MithrilEvent::AncillaryDownloadCompleted { download_id }) + .await; + return Ok(()) + }, Err(e) => { slog::error!( self.logger, @@ -1574,11 +1585,7 @@ mod tests { async fn download_unpack_immutable_files_sends_feedbacks() { let total_immutable_files = 1; let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); - let target_dir = TempDir::new( - "cardano_database_client", - "download_unpack_immutable_files_sends_feedbacks", - ) - .build(); + let target_dir = Path::new("."); let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); let client = CardanoDatabaseClientDependencyInjector::new() .with_immutable_file_downloaders(vec![( @@ -1603,7 +1610,7 @@ mod tests { .to_range_inclusive(total_immutable_files) .unwrap(), &CompressionAlgorithm::default(), - &target_dir, + target_dir, ) .await .unwrap(); @@ -1722,6 +1729,49 @@ mod tests { .await .unwrap(); } + + #[tokio::test] + async fn download_unpack_ancillary_files_sends_feedbacks() { + let target_dir = Path::new("."); + let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new( + MockFileDownloaderBuilder::default() + .with_success() + .build(), + ), + )]) + .with_feedback_receivers(&[feedback_receiver.clone()]) + .build_cardano_database_client(); + + client + .download_unpack_ancillary_file( + &[ + AncillaryLocation::CloudStorage { + uri: "http://whatever-1/ancillary.tar.gz".to_string(), + }, + ], + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .unwrap(); + + let sent_events = feedback_receiver.stacked_events(); + let id = sent_events[0].event_id(); + let expected_events = vec![ + MithrilEvent::AncillaryDownloadStarted { + download_id: id.to_string(), + + }, + MithrilEvent::AncillaryDownloadCompleted { + download_id: id.to_string(), + }, + ]; + assert_eq!(expected_events, sent_events); + } } mod download_unpack_digest_file { diff --git a/mithril-client/src/feedback.rs b/mithril-client/src/feedback.rs index fae5fbf28e7..2425e34d741 100644 --- a/mithril-client/src/feedback.rs +++ b/mithril-client/src/feedback.rs @@ -91,7 +91,7 @@ pub enum MithrilEvent { ImmutableDownloadStarted { /// Immutable file number downloaded immutable_file_number: ImmutableFileNumber, - /// Unique identifier used to track this specific immutable archive file download download + /// Unique identifier used to track this specific immutable archive file download download_id: String, }, /// An immutable archive file download is in progress @@ -105,7 +105,26 @@ pub enum MithrilEvent { }, /// An immutable archive file download has completed ImmutableDownloadCompleted { - /// Unique identifier used to track this specific immutable archive file download download + /// Unique identifier used to track this specific immutable archive file download + download_id: String, + }, + /// An ancillary archive file download has started + AncillaryDownloadStarted { + /// Unique identifier used to track this specific ancillary archive file download + download_id: String, + }, + /// An ancillary archive file download is in progress + AncillaryDownloadProgress { + /// Unique identifier used to track this specific download + download_id: String, + /// Number of bytes that have been downloaded + downloaded_bytes: u64, + /// Size of the downloaded archive + size: u64, + }, + /// An ancillary archive file download has completed + AncillaryDownloadCompleted { + /// Unique identifier used to track this specific ancillary archive file download download_id: String, }, /// A certificate chain validation has started @@ -145,6 +164,11 @@ impl MithrilEvent { Uuid::new_v4().to_string() } + /// Generate a random unique identifier to identify an ancillary download + pub fn new_ancillary_download_id() -> String { + Uuid::new_v4().to_string() + } + /// Generate a random unique identifier to identify a certificate chain validation pub fn new_certificate_chain_validation_id() -> String { Uuid::new_v4().to_string() @@ -159,6 +183,9 @@ impl MithrilEvent { MithrilEvent::ImmutableDownloadStarted { download_id, .. } => download_id, MithrilEvent::ImmutableDownloadProgress { download_id, .. } => download_id, MithrilEvent::ImmutableDownloadCompleted { download_id, .. } => download_id, + MithrilEvent::AncillaryDownloadStarted { download_id, .. } => download_id, + MithrilEvent::AncillaryDownloadProgress { download_id, .. } => download_id, + MithrilEvent::AncillaryDownloadCompleted { download_id, .. } => download_id, MithrilEvent::CertificateChainValidationStarted { certificate_chain_validation_id, } => certificate_chain_validation_id, @@ -271,6 +298,25 @@ impl FeedbackReceiver for SlogFeedbackReceiver { MithrilEvent::ImmutableDownloadCompleted { download_id } => { info!(self.logger, "Immutable download completed"; "download_id" => download_id); } + MithrilEvent::AncillaryDownloadStarted { download_id } => { + info!( + self.logger, "Ancillary download started"; + "download_id" => download_id, + ); + } + MithrilEvent::AncillaryDownloadProgress { + download_id, + downloaded_bytes, + size, + } => { + info!( + self.logger, "Ancillary download in progress ..."; + "downloaded_bytes" => downloaded_bytes, "size" => size, "download_id" => download_id, + ); + } + MithrilEvent::AncillaryDownloadCompleted { download_id } => { + info!(self.logger, "Ancillary download completed"; "download_id" => download_id); + } MithrilEvent::CertificateChainValidationStarted { certificate_chain_validation_id, } => { From 8005d7b913408ea488e80e6fb8fe7216b4d0c6d1 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 5 Feb 2025 18:35:50 +0100 Subject: [PATCH 26/59] feat: add feedback events for digest download in Cardano database client --- mithril-client/src/cardano_database_client.rs | 65 +++++++++++++++++-- mithril-client/src/feedback.rs | 46 +++++++++++++ 2 files changed, 107 insertions(+), 4 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index 25770e23535..318ee2a1252 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -391,8 +391,12 @@ impl CardanoDatabaseClient { }) } - fn feedback_event_builder_digest_download(_download_id: String, _downloaded_bytes: u64, _size: u64) -> Option { - None + fn feedback_event_builder_digest_download(download_id: String, downloaded_bytes: u64, size: u64) -> Option { + Some(MithrilEvent::DigestDownloadProgress { + download_id, + downloaded_bytes, + size, + }) } /// Download and unpack the immutable files of the given range. @@ -524,7 +528,10 @@ impl CardanoDatabaseClient { let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); for location in locations_sorted { - let download_id = format!("{location:?}"); //TODO: check if this is the correct way to format the download_id + let download_id = MithrilEvent::new_digest_download_id(); + self.feedback_sender + .send_event(MithrilEvent::DigestDownloadStarted { download_id: download_id.clone()}) + .await; let file_downloader = self .digest_file_downloader_resolver .resolve(&location) @@ -542,7 +549,12 @@ impl CardanoDatabaseClient { ) .await; match downloaded { - Ok(_) => return Ok(()), + Ok(_) => { + self.feedback_sender + .send_event(MithrilEvent::DigestDownloadCompleted { download_id }) + .await; + return Ok(()) + }, Err(e) => { slog::error!( self.logger, @@ -1900,6 +1912,51 @@ mod tests { .await .unwrap(); } + + #[tokio::test] + async fn download_unpack_digest_file_sends_feedbacks() { + let target_dir = Path::new("."); + let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_digest_file_downloaders(vec![ + ( + DigestLocationDiscriminants::CloudStorage, + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_success() + .build(), + ), + ), + ]) + .with_feedback_receivers(&[feedback_receiver.clone()]) + .build_cardano_database_client(); + + client + .download_unpack_digest_file( + &[ + DigestLocation::CloudStorage { + uri: "http://whatever-1/digests.json".to_string(), + }, + ], + target_dir, + ) + .await + .unwrap(); + + let sent_events = feedback_receiver.stacked_events(); + let id = sent_events[0].event_id(); + let expected_events = vec![ + MithrilEvent::DigestDownloadStarted { + download_id: id.to_string(), + + }, + MithrilEvent::DigestDownloadCompleted { + download_id: id.to_string(), + }, + ]; + assert_eq!(expected_events, sent_events); + } } mod read_digest_file { diff --git a/mithril-client/src/feedback.rs b/mithril-client/src/feedback.rs index 2425e34d741..bc33bb6825e 100644 --- a/mithril-client/src/feedback.rs +++ b/mithril-client/src/feedback.rs @@ -127,6 +127,25 @@ pub enum MithrilEvent { /// Unique identifier used to track this specific ancillary archive file download download_id: String, }, + /// A digest file download has started + DigestDownloadStarted { + /// Unique identifier used to track this specific digest file download + download_id: String, + }, + /// A digest file download is in progress + DigestDownloadProgress { + /// Unique identifier used to track this specific download + download_id: String, + /// Number of bytes that have been downloaded + downloaded_bytes: u64, + /// Size of the downloaded archive + size: u64, + }, + /// A digest file download has completed + DigestDownloadCompleted { + /// Unique identifier used to track this specific digest file download + download_id: String, + }, /// A certificate chain validation has started CertificateChainValidationStarted { /// Unique identifier used to track this specific certificate chain validation @@ -169,6 +188,11 @@ impl MithrilEvent { Uuid::new_v4().to_string() } + /// Generate a random unique identifier to identify a digest download + pub fn new_digest_download_id() -> String { + Uuid::new_v4().to_string() + } + /// Generate a random unique identifier to identify a certificate chain validation pub fn new_certificate_chain_validation_id() -> String { Uuid::new_v4().to_string() @@ -186,6 +210,9 @@ impl MithrilEvent { MithrilEvent::AncillaryDownloadStarted { download_id, .. } => download_id, MithrilEvent::AncillaryDownloadProgress { download_id, .. } => download_id, MithrilEvent::AncillaryDownloadCompleted { download_id, .. } => download_id, + MithrilEvent::DigestDownloadStarted { download_id, .. } => download_id, + MithrilEvent::DigestDownloadProgress { download_id, .. } => download_id, + MithrilEvent::DigestDownloadCompleted { download_id, .. } => download_id, MithrilEvent::CertificateChainValidationStarted { certificate_chain_validation_id, } => certificate_chain_validation_id, @@ -317,6 +344,25 @@ impl FeedbackReceiver for SlogFeedbackReceiver { MithrilEvent::AncillaryDownloadCompleted { download_id } => { info!(self.logger, "Ancillary download completed"; "download_id" => download_id); } + MithrilEvent::DigestDownloadStarted { download_id } => { + info!( + self.logger, "Digest download started"; + "download_id" => download_id, + ); + } + MithrilEvent::DigestDownloadProgress { + download_id, + downloaded_bytes, + size, + } => { + info!( + self.logger, "Digest download in progress ..."; + "downloaded_bytes" => downloaded_bytes, "size" => size, "download_id" => download_id, + ); + } + MithrilEvent::DigestDownloadCompleted { download_id } => { + info!(self.logger, "Digest download completed"; "download_id" => download_id); + } MithrilEvent::CertificateChainValidationStarted { certificate_chain_validation_id, } => { From c0a403d5c5c78ddc339a04542515317afc4568b2 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 5 Feb 2025 18:59:19 +0100 Subject: [PATCH 27/59] refactor: make immutable files download more readable --- mithril-client/src/cardano_database_client.rs | 110 ++++++++++-------- 1 file changed, 63 insertions(+), 47 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index 318ee2a1252..70183d090b0 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -46,14 +46,14 @@ // TODO: reorganize the imports #[cfg(feature = "fs")] -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; #[cfg(feature = "fs")] use std::fs; #[cfg(feature = "fs")] use std::ops::RangeInclusive; #[cfg(feature = "fs")] use std::path::{Path, PathBuf}; -use std::{collections::HashSet, sync::Arc}; +use std::sync::Arc; #[cfg(feature = "fs")] use anyhow::anyhow; @@ -72,7 +72,8 @@ use mithril_common::{ ImmutableFileName, ImmutableFileNumber, ImmutablesLocation, }, messages::{ - CardanoDatabaseDigestListItemMessage, CertificateMessage, SignedEntityTypeMessagePart, + CardanoDatabaseDigestListItemMessage, CardanoDatabaseSnapshotMessage, CertificateMessage, + SignedEntityTypeMessagePart, }, StdResult, }; @@ -413,51 +414,16 @@ impl CardanoDatabaseClient { let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); let mut immutable_file_numbers_to_download = - range.clone().map(|n| n.to_owned()).collect::>(); + range.clone().map(|n| n.to_owned()).collect::>(); for location in locations_sorted { - let file_downloader = self - .immutable_file_downloader_resolver - .resolve(&location) - .ok_or_else(|| { - anyhow!("Failed resolving a file downloader for location: {location:?}") - })?; - let file_downloader_uris = - FileDownloaderUri::expand_immutable_files_location_to_file_downloader_uris( - &location, - immutable_file_numbers_to_download - .clone() - .into_iter() - .collect::>() - .as_slice(), - )?; - for (immutable_file_number, file_downloader_uri) in file_downloader_uris { - let download_id = MithrilEvent::new_snapshot_download_id(); - self.feedback_sender - .send_event(MithrilEvent::ImmutableDownloadStarted { immutable_file_number, download_id: download_id.clone()}) - .await; - let downloaded = file_downloader - .download_unpack( - &file_downloader_uri, - immutable_files_target_dir, - Some(compression_algorithm.to_owned()), - &download_id, - Self::feedback_event_builder_immutable_download, - ) - .await; - match downloaded { - Ok(_) => { - immutable_file_numbers_to_download.remove(&immutable_file_number); - self.feedback_sender - .send_event(MithrilEvent::ImmutableDownloadCompleted { download_id }) - .await; - } - Err(e) => { - slog::error!( - self.logger, - "Failed downloading and unpacking immutable files for location {file_downloader_uri:?}"; "error" => e.to_string() - ); - } - } + let immutable_files_numbers_downloaded = self.download_unpack_immutable_files_for_location( + &location, + &immutable_file_numbers_to_download, + compression_algorithm, + immutable_files_target_dir, + ).await?; + for immutable_file_number in immutable_files_numbers_downloaded { + immutable_file_numbers_to_download.remove(&immutable_file_number); } if immutable_file_numbers_to_download.is_empty() { return Ok(()); @@ -469,6 +435,56 @@ impl CardanoDatabaseClient { )) } + async fn download_unpack_immutable_files_for_location(&self, location: &ImmutablesLocation, immutable_file_numbers_to_download: &BTreeSet, compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path,) -> StdResult> { + let mut immutable_file_numbers_downloaded = BTreeSet::new(); + let file_downloader = self + .immutable_file_downloader_resolver + .resolve(location) + .ok_or_else(|| { + anyhow!("Failed resolving a file downloader for location: {location:?}") + })?; + let file_downloader_uris = + FileDownloaderUri::expand_immutable_files_location_to_file_downloader_uris( + location, + immutable_file_numbers_to_download + .clone() + .into_iter() + .collect::>() + .as_slice(), + )?; + for (immutable_file_number, file_downloader_uri) in file_downloader_uris { + let download_id = MithrilEvent::new_snapshot_download_id(); + self.feedback_sender + .send_event(MithrilEvent::ImmutableDownloadStarted { immutable_file_number, download_id: download_id.clone()}) + .await; + let downloaded = file_downloader + .download_unpack( + &file_downloader_uri, + immutable_files_target_dir, + Some(compression_algorithm.to_owned()), + &download_id, + Self::feedback_event_builder_immutable_download, + ) + .await; + match downloaded { + Ok(_) => { + immutable_file_numbers_downloaded.insert(immutable_file_number); + self.feedback_sender + .send_event(MithrilEvent::ImmutableDownloadCompleted { download_id }) + .await; + } + Err(e) => { + slog::error!( + self.logger, + "Failed downloading and unpacking immutable files for location {file_downloader_uri:?}"; "error" => e.to_string() + ); + } + } + } + + Ok(immutable_file_numbers_downloaded) + } + /// Download and unpack the ancillary files. async fn download_unpack_ancillary_file( &self, From 3666ee033b961858f4a090509bcfaf01d52a0410 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 7 Feb 2025 17:51:06 +0100 Subject: [PATCH 28/59] docs: add examples in Cardano database client --- mithril-client/src/cardano_database_client.rs | 76 ++++++++++++++++++- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index 70183d090b0..f6868384386 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -41,10 +41,80 @@ //! } //! # Ok(()) //! # } - +//! ``` +//! +//! # Download a Cardano database snapshot +//! +//! To download a partial or a full Cardano database folder the [ClientBuilder][crate::client::ClientBuilder]. +//! +//! ```no_run +//! # #[cfg(feature = "fs")] +//! # async fn run() -> mithril_client::MithrilResult<()> { +//! use mithril_client::ClientBuilder; +//! +//! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; +//! let cardano_database_snapshot = client.cardano_database().get("CARDANO_DATABASE_HASH").await?.unwrap(); +//! +//! // Note: the directory must already exist, and the user running the binary must have read/write access to it. +//! let target_directory = Path::new("/home/user/download/"); +//! let immutable_file_range = ImmutableFileRange::Range(3, 6); +//! let download_unpack_options = DownloadUnpackOptions { +//! allow_override: true, +//! include_ancillary: true, +//! }; +//! client +//! .cardano_database() +//! .download_unpack( +//! &cardano_database_snapshot, +//! &immutable_file_range, +//! &target_directory, +//! download_unpack_options, +//! ) +//! .await?; +//! # +//! # Ok(()) +//! # } +//! ``` +//! +//! # Compute a Merkle proof for a Cardano database snapshot +//! +//! To compute proof of membership of downloaded immutable files in a Cardano database folder the [ClientBuilder][crate::client::ClientBuilder]. +//! +//! ```no_run +//! # #[cfg(feature = "fs")] +//! # async fn run() -> mithril_client::MithrilResult<()> { +//! use mithril_client::ClientBuilder; +//! +//! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; +//! let cardano_database_snapshot = client.cardano_database().get("CARDANO_DATABASE_HASH").await?.unwrap(); +//! let certificate = client.certificate().verify_chain(&cardano_database_snapshot.certificate_hash).await?; +//! +//! // Note: the directory must already exist, and the user running the binary must have read/write access to it. +//! let target_directory = Path::new("/home/user/download/"); +//! let immutable_file_range = ImmutableFileRange::Full; +//! let download_unpack_options = DownloadUnpackOptions { +//! allow_override: true, +//! include_ancillary: true, +//! }; +//! client +//! .cardano_database() +//! .download_unpack( +//! &cardano_database_snapshot, +//! &immutable_file_range, +//! &target_directory, +//! download_unpack_options, +//! ) +//! .await?; +//! +//! let merkle_proof = client +//! .cardano_database() +//! .compute_merkle_proof(&certificate, &immutable_file_range, &unpacked_dir) +//! .await?; +//! # +//! # Ok(()) +//! # } //! ``` -// TODO: reorganize the imports #[cfg(feature = "fs")] use std::collections::{BTreeMap, BTreeSet}; #[cfg(feature = "fs")] @@ -233,7 +303,6 @@ impl CardanoDatabaseClient { cfg_fs! { /// Download and unpack the given Cardano database parts data by hash. - // TODO: Add example in module documentation pub async fn download_unpack( &self, cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, @@ -616,7 +685,6 @@ impl CardanoDatabaseClient { } /// Compute the Merkle proof of membership for the given immutable file range. - // TODO: Add example in module documentation pub async fn compute_merkle_proof( &self, certificate: &CertificateMessage, From c6f73afe28b354572486cf274719c9ca1bfc63d9 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 7 Feb 2025 11:44:42 +0100 Subject: [PATCH 29/59] feat: implement parallel download for immutable files download --- mithril-client/src/cardano_database_client.rs | 58 +++++++++++++------ 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index f6868384386..02a8289040e 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -132,6 +132,8 @@ use anyhow::Context; use serde::de::DeserializeOwned; #[cfg(feature = "fs")] use slog::Logger; +#[cfg(feature = "fs")] +use tokio::task::JoinSet; #[cfg(feature = "fs")] use mithril_common::{ @@ -521,31 +523,49 @@ impl CardanoDatabaseClient { .collect::>() .as_slice(), )?; + let mut join_set: JoinSet> = JoinSet::new(); for (immutable_file_number, file_downloader_uri) in file_downloader_uris { - let download_id = MithrilEvent::new_snapshot_download_id(); - self.feedback_sender - .send_event(MithrilEvent::ImmutableDownloadStarted { immutable_file_number, download_id: download_id.clone()}) - .await; - let downloaded = file_downloader - .download_unpack( - &file_downloader_uri, - immutable_files_target_dir, - Some(compression_algorithm.to_owned()), - &download_id, - Self::feedback_event_builder_immutable_download, - ) - .await; - match downloaded { - Ok(_) => { + let compression_algorithm_clone = compression_algorithm.to_owned(); + let immutable_files_target_dir_clone = immutable_files_target_dir.to_owned(); + let file_downloader_clone = file_downloader.clone(); + let feedback_receiver_clone = self.feedback_sender.clone(); + join_set.spawn(async move { + let download_id = MithrilEvent::new_snapshot_download_id(); + feedback_receiver_clone + .send_event(MithrilEvent::ImmutableDownloadStarted { immutable_file_number, download_id: download_id.clone()}) + .await; + let downloaded = file_downloader_clone + .download_unpack( + &file_downloader_uri, + &immutable_files_target_dir_clone, + Some(compression_algorithm_clone), + &download_id, + Self::feedback_event_builder_immutable_download, + ) + .await; + match downloaded { + Ok(_) => { + feedback_receiver_clone + .send_event(MithrilEvent::ImmutableDownloadCompleted { download_id }) + .await; + + Ok(immutable_file_number) + } + Err(e) => { + Err(e.context(format!("Failed downloading and unpacking immutable file {immutable_file_number} for location {file_downloader_uri:?}"))) + } + } + }); + } + while let Some(result) = join_set.join_next().await { + match result? { + Ok(immutable_file_number) => { immutable_file_numbers_downloaded.insert(immutable_file_number); - self.feedback_sender - .send_event(MithrilEvent::ImmutableDownloadCompleted { download_id }) - .await; } Err(e) => { slog::error!( self.logger, - "Failed downloading and unpacking immutable files for location {file_downloader_uri:?}"; "error" => e.to_string() + "Failed downloading and unpacking immutable files"; "error" => e.to_string() ); } } From 53a6815aee69daaf55514f5ce2d670fd49acc86b Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 11 Feb 2025 11:39:38 +0100 Subject: [PATCH 30/59] feat: implement 'RetryFileDownloader' decorator in client --- mithril-client/src/file_downloader/mod.rs | 2 + mithril-client/src/file_downloader/retry.rs | 269 ++++++++++++++++++++ 2 files changed, 271 insertions(+) create mode 100644 mithril-client/src/file_downloader/retry.rs diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs index 1a6895b4ac0..aed9a0ed320 100644 --- a/mithril-client/src/file_downloader/mod.rs +++ b/mithril-client/src/file_downloader/mod.rs @@ -4,6 +4,7 @@ mod interface; mod resolver; +mod retry; #[cfg(test)] pub use interface::MockFileDownloader; @@ -14,3 +15,4 @@ pub use resolver::{ AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloaderResolver, ImmutablesFileDownloaderResolver, }; +pub use retry::{FileDownloadRetryPolicy, RetryDownloader}; diff --git a/mithril-client/src/file_downloader/retry.rs b/mithril-client/src/file_downloader/retry.rs new file mode 100644 index 00000000000..2c451cb0264 --- /dev/null +++ b/mithril-client/src/file_downloader/retry.rs @@ -0,0 +1,269 @@ +use std::{path::Path, sync::Arc, time::Duration}; + +use async_trait::async_trait; +use mithril_common::{entities::CompressionAlgorithm, StdResult}; + +use super::{FeedbackEventBuilder, FileDownloader, FileDownloaderUri}; + +/// Policy for retrying file downloads. +#[derive(Debug, PartialEq, Clone)] +pub struct FileDownloadRetryPolicy { + /// Number of attempts to download a file. + pub attempts: usize, + /// Delay between two attempts. + pub delay_between_attempts: Duration, +} + +impl FileDownloadRetryPolicy { + /// Create a policy that never retries. + pub fn never() -> Self { + Self { + attempts: 1, + delay_between_attempts: Duration::from_secs(0), + } + } +} + +impl Default for FileDownloadRetryPolicy { + /// Create a default retry policy. + fn default() -> Self { + Self { + attempts: 3, + delay_between_attempts: Duration::from_secs(5), + } + } +} + +/// RetryDownloader is a wrapper around FileDownloader that retries downloading a file if it fails. +pub struct RetryDownloader { + /// File downloader to use. + file_downloader: Arc, + /// Number of attempts to download a file. + pub retry_policy: FileDownloadRetryPolicy, +} + +impl RetryDownloader { + /// Create a new RetryDownloader. + pub fn new( + file_downloader: Arc, + retry_policy: FileDownloadRetryPolicy, + ) -> Self { + Self { + file_downloader, + retry_policy, + } + } +} + +#[async_trait] +impl FileDownloader for RetryDownloader { + async fn download_unpack( + &self, + location: &FileDownloaderUri, + target_dir: &Path, + compression_algorithm: Option, + download_id: &str, + feedback_event_builder: FeedbackEventBuilder, + ) -> StdResult<()> { + let retry_policy = &self.retry_policy; + let mut nb_attempts = 0; + loop { + nb_attempts += 1; + match self + .file_downloader + .download_unpack( + location, + target_dir, + compression_algorithm, + download_id, + feedback_event_builder, + ) + .await + { + Ok(result) => return Ok(result), + Err(_) if nb_attempts >= retry_policy.attempts => { + return Err(anyhow::anyhow!( + "Download of location {:?} failed after {} attempts", + location, + nb_attempts + )); + } + _ => tokio::time::sleep(retry_policy.delay_between_attempts).await, + } + } + } +} + +#[cfg(test)] +mod tests { + + use std::time::Instant; + + use mithril_common::entities::FileUri; + + use crate::{feedback::MithrilEvent, file_downloader::MockFileDownloaderBuilder}; + + use super::*; + + fn fake_feedback_event( + _download_id: String, + _downloaded_bytes: u64, + _size: u64, + ) -> Option { + None + } + + #[tokio::test] + async fn download_return_the_result_of_download_without_retry() { + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/00001.tar.gz") + .with_compression(None) + .with_success() + .build(); + let retry_downloader = RetryDownloader::new( + Arc::new(mock_file_downloader), + FileDownloadRetryPolicy::never(), + ); + + retry_downloader + .download_unpack( + &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), + Path::new("."), + None, + "download_id", + fake_feedback_event, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn when_download_fails_do_not_retry_by_default() { + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/00001.tar.gz") + .with_compression(None) + .with_failure() + .build(); + let retry_downloader = RetryDownloader::new( + Arc::new(mock_file_downloader), + FileDownloadRetryPolicy::never(), + ); + + retry_downloader + .download_unpack( + &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), + Path::new("."), + None, + "download_id", + fake_feedback_event, + ) + .await + .expect_err("An error should be returned when download fails"); + } + + #[tokio::test] + async fn should_retry_if_fail() { + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/00001.tar.gz") + .with_compression(None) + .with_failure() + .with_times(2) + .build(); + let mock_file_downloader = MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever/00001.tar.gz") + .with_compression(None) + .with_times(1) + .with_success() + .build(); + let retry_downloader = RetryDownloader::new( + Arc::new(mock_file_downloader), + FileDownloadRetryPolicy { + attempts: 3, + delay_between_attempts: Duration::from_millis(10), + }, + ); + + retry_downloader + .download_unpack( + &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), + Path::new("."), + None, + "download_id", + fake_feedback_event, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn should_recall_a_failing_inner_downloader_up_to_the_limit() { + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/00001.tar.gz") + .with_compression(None) + .with_failure() + .with_times(3) + .build(); + let retry_downloader = RetryDownloader::new( + Arc::new(mock_file_downloader), + FileDownloadRetryPolicy { + attempts: 3, + delay_between_attempts: Duration::from_millis(10), + }, + ); + + retry_downloader + .download_unpack( + &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), + Path::new("."), + None, + "download_id", + fake_feedback_event, + ) + .await + .expect_err("An error should be returned when all download attempts fail"); + } + + #[tokio::test] + async fn should_delay_between_retries() { + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/00001.tar.gz") + .with_compression(None) + .with_failure() + .with_times(4) + .build(); + let delay = Duration::from_millis(50); + let retry_downloader = RetryDownloader::new( + Arc::new(mock_file_downloader), + FileDownloadRetryPolicy { + attempts: 4, + delay_between_attempts: delay, + }, + ); + + let start = Instant::now(); + retry_downloader + .download_unpack( + &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), + Path::new("."), + None, + "download_id", + fake_feedback_event, + ) + .await + .expect_err("An error should be returned when all download attempts fail"); + let duration = start.elapsed(); + + assert!( + duration >= delay * 3, + "Duration should be at least 3 times the delay ({}ms) but was {}ms", + delay.as_millis() * 3, + duration.as_millis() + ); + assert!( + duration < delay * 4, + "Duration should be less than 4 times the delay ({}ms) but was {}ms", + delay.as_millis() * 4, + duration.as_millis() + ); + } +} From 6f1bd089ff7d17070dacad0b77c66fef2521e356 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 11 Feb 2025 18:58:05 +0100 Subject: [PATCH 31/59] chore: update Makefile with unstable features for client --- mithril-client/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mithril-client/Makefile b/mithril-client/Makefile index bc268478acb..5562910930e 100644 --- a/mithril-client/Makefile +++ b/mithril-client/Makefile @@ -10,10 +10,10 @@ CARGO = cargo all: test build build: - ${CARGO} build --release --features full + ${CARGO} build --release --features full,unstable test: - ${CARGO} test --features full + ${CARGO} test --features full,unstable check: ${CARGO} check --release --all-features --all-targets From 9fa15bbaeed3b29fe4c18bf3e93232088f0629dc Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 11 Feb 2025 11:38:22 +0100 Subject: [PATCH 32/59] refactor: move 'MockFileDownloaderBuilder' to 'file_downloader' module in client --- mithril-client/src/cardano_database_client.rs | 141 +---------------- .../src/file_downloader/mock_builder.rs | 149 ++++++++++++++++++ mithril-client/src/file_downloader/mod.rs | 4 + 3 files changed, 158 insertions(+), 136 deletions(-) create mode 100644 mithril-client/src/file_downloader/mock_builder.rs diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs index 02a8289040e..51de6a88891 100644 --- a/mithril-client/src/cardano_database_client.rs +++ b/mithril-client/src/cardano_database_client.rs @@ -744,22 +744,22 @@ mod tests { mod cardano_database_client { use anyhow::anyhow; use chrono::{DateTime, Utc}; + use mockall::predicate::eq; + use mithril_common::{ digesters::CardanoImmutableDigester, entities::{ AncillaryLocationDiscriminants, CardanoDbBeacon, CompressionAlgorithm, - DigestLocationDiscriminants, Epoch, FileUri, ImmutablesLocationDiscriminants, + DigestLocationDiscriminants, Epoch, ImmutablesLocationDiscriminants, }, }; - use mockall::predicate::{self, eq}; use crate::{ aggregator_client::MockAggregatorHTTPClient, feedback::{FeedbackReceiver, MithrilEvent, StackFeedbackReceiver}, file_downloader::{ - AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, - FeedbackEventBuilder, FileDownloader, ImmutablesFileDownloaderResolver, - MockFileDownloader, + AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloader, + ImmutablesFileDownloaderResolver, MockFileDownloaderBuilder, }, test_utils, }; @@ -894,137 +894,6 @@ mod tests { } } - type MockFileDownloaderBuilderReturningFunc = Box< - dyn FnMut( - &FileDownloaderUri, - &Path, - Option, - &str, - FeedbackEventBuilder, - ) -> StdResult<()> - + Send - + 'static, - >; - - struct MockFileDownloaderBuilder { - mock_file_downloader: Option, - times: usize, - param_file_downloader_uri: Option, - param_target_dir: Option, - param_compression_algorithm: Option>, - returning_func: Option, - } - - impl Default for MockFileDownloaderBuilder { - fn default() -> Self { - Self { - mock_file_downloader: None, - times: 1, - param_file_downloader_uri: None, - param_target_dir: None, - param_compression_algorithm: Some(Some(CompressionAlgorithm::default())), - returning_func: None, - } - } - } - - impl MockFileDownloaderBuilder { - fn from_mock(mock: MockFileDownloader) -> Self { - Self { - mock_file_downloader: Some(mock), - ..Self::default() - } - } - - fn with_success(self) -> Self { - self.with_returning(Box::new(|_, _, _, _, _| Ok(()))) - } - - fn with_failure(self) -> Self { - self.with_returning(Box::new(|_, _, _, _, _| { - Err(anyhow!("Download unpack failed")) - })) - } - - fn with_times(self, times: usize) -> Self { - let mut self_mut = self; - self_mut.times = times; - - self_mut - } - - fn with_file_uri>(self, file_uri: T) -> Self { - let mut self_mut = self; - self_mut.param_file_downloader_uri = Some(FileDownloaderUri::FileUri(FileUri( - file_uri.as_ref().to_string(), - ))); - - self_mut - } - - fn with_target_dir(self, target_dir: PathBuf) -> Self { - let mut self_mut = self; - self_mut.param_target_dir = Some(target_dir); - - self_mut - } - - fn with_compression(self, compression: Option) -> Self { - let mut self_mut = self; - self_mut.param_compression_algorithm = Some(compression); - - self_mut - } - - fn with_returning( - self, - returning_func: MockFileDownloaderBuilderReturningFunc, - ) -> Self { - let mut self_mut = self; - self_mut.returning_func = Some(returning_func); - - self_mut - } - - fn build(self) -> MockFileDownloader { - let predicate_file_downloader_uri = predicate::function(move |u| { - self.param_file_downloader_uri - .as_ref() - .map(|x| x == u) - .unwrap_or(true) - }); - let predicate_target_dir = predicate::function(move |u| { - self.param_target_dir - .as_ref() - .map(|x| x == u) - .unwrap_or(true) - }); - let predicate_compression_algorithm = predicate::function(move |u| { - self.param_compression_algorithm - .as_ref() - .map(|x| x == u) - .unwrap_or(true) - }); - let predicate_download_id = predicate::always(); - let predicate_feedback_event_builder = predicate::always(); - - let mut mock_file_downloader = self.mock_file_downloader.unwrap_or_default(); - mock_file_downloader - .expect_download_unpack() - .with( - predicate_file_downloader_uri, - predicate_target_dir, - predicate_compression_algorithm, - predicate_download_id, - predicate_feedback_event_builder, - ) - .times(self.times) - .returning(self.returning_func.unwrap()); - - mock_file_downloader - } - } - mod list { use super::*; diff --git a/mithril-client/src/file_downloader/mock_builder.rs b/mithril-client/src/file_downloader/mock_builder.rs new file mode 100644 index 00000000000..af66b32d65e --- /dev/null +++ b/mithril-client/src/file_downloader/mock_builder.rs @@ -0,0 +1,149 @@ + +use anyhow::anyhow; +use mockall::predicate; +use std::path::{Path, PathBuf}; + +use mithril_common::{ + entities::{CompressionAlgorithm, FileUri}, + StdResult, +}; + +use super::{FeedbackEventBuilder, FileDownloaderUri, MockFileDownloader}; + +type MockFileDownloaderBuilderReturningFunc = Box< + dyn FnMut( + &FileDownloaderUri, + &Path, + Option, + &str, + FeedbackEventBuilder, + ) -> StdResult<()> + + Send + + 'static, +>; + +/// A mock file downloader builder +pub struct MockFileDownloaderBuilder { + mock_file_downloader: Option, + times: usize, + param_file_downloader_uri: Option, + param_target_dir: Option, + param_compression_algorithm: Option>, + returning_func: Option, +} + +impl Default for MockFileDownloaderBuilder { + fn default() -> Self { + Self { + mock_file_downloader: None, + times: 1, + param_file_downloader_uri: None, + param_target_dir: None, + param_compression_algorithm: Some(Some(CompressionAlgorithm::default())), + returning_func: None, + } + } +} + +impl MockFileDownloaderBuilder { + /// Constructs a new MockFileDownloaderBuilder from an existing MockFileDownloader. + pub fn from_mock(mock: MockFileDownloader) -> Self { + Self { + mock_file_downloader: Some(mock), + ..Self::default() + } + } + + /// The MockFileDownloader will succeed + pub fn with_success(self) -> Self { + self.with_returning(Box::new(|_, _, _, _, _| Ok(()))) + } + + /// The MockFileDownloader will fail + pub fn with_failure(self) -> Self { + self.with_returning(Box::new(|_, _, _, _, _| { + Err(anyhow!("Download unpack failed")) + })) + } + + /// The MockFileDownloader expected number of calls of download_unpack + pub fn with_times(self, times: usize) -> Self { + let mut self_mut = self; + self_mut.times = times; + + self_mut + } + + /// The MockFileDownloader expected FileDownloaderUri when download_unpack is called + pub fn with_file_uri>(self, file_uri: T) -> Self { + let mut self_mut = self; + self_mut.param_file_downloader_uri = Some(FileDownloaderUri::FileUri(FileUri( + file_uri.as_ref().to_string(), + ))); + + self_mut + } + + /// The MockFileDownloader expected target_dir when download_unpack is called + pub fn with_target_dir(self, target_dir: PathBuf) -> Self { + let mut self_mut = self; + self_mut.param_target_dir = Some(target_dir); + + self_mut + } + + /// The MockFileDownloader expected compression_algorithm when download_unpack is called + pub fn with_compression(self, compression: Option) -> Self { + let mut self_mut = self; + self_mut.param_compression_algorithm = Some(compression); + + self_mut + } + + /// The MockFileDownloader will return the result of the returning_func when download_unpack is called + pub fn with_returning(self, returning_func: MockFileDownloaderBuilderReturningFunc) -> Self { + let mut self_mut = self; + self_mut.returning_func = Some(returning_func); + + self_mut + } + + /// Builds the MockFileDownloader + pub fn build(self) -> MockFileDownloader { + let predicate_file_downloader_uri = predicate::function(move |u| { + self.param_file_downloader_uri + .as_ref() + .map(|x| x == u) + .unwrap_or(true) + }); + let predicate_target_dir = predicate::function(move |u| { + self.param_target_dir + .as_ref() + .map(|x| x == u) + .unwrap_or(true) + }); + let predicate_compression_algorithm = predicate::function(move |u| { + self.param_compression_algorithm + .as_ref() + .map(|x| x == u) + .unwrap_or(true) + }); + let predicate_download_id = predicate::always(); + let predicate_feedback_event_builder = predicate::always(); + + let mut mock_file_downloader = self.mock_file_downloader.unwrap_or_default(); + mock_file_downloader + .expect_download_unpack() + .with( + predicate_file_downloader_uri, + predicate_target_dir, + predicate_compression_algorithm, + predicate_download_id, + predicate_feedback_event_builder, + ) + .times(self.times) + .returning(self.returning_func.unwrap()); + + mock_file_downloader + } +} diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs index aed9a0ed320..d7611adacb1 100644 --- a/mithril-client/src/file_downloader/mod.rs +++ b/mithril-client/src/file_downloader/mod.rs @@ -3,6 +3,8 @@ //! This module provides the necessary abstractions to download files from different sources. mod interface; +#[cfg(test)] +mod mock_builder; mod resolver; mod retry; @@ -10,6 +12,8 @@ mod retry; pub use interface::MockFileDownloader; pub use interface::{FeedbackEventBuilder, FileDownloader, FileDownloaderUri}; #[cfg(test)] +pub use mock_builder::MockFileDownloaderBuilder; +#[cfg(test)] pub use resolver::MockFileDownloaderResolver; pub use resolver::{ AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloaderResolver, From e1ea0bbbba5ed69a9b496b03283636b745a38e15 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 11 Feb 2025 18:40:44 +0100 Subject: [PATCH 33/59] refactor: split Cardano database client in sub modules --- mithril-client/src/cardano_database_client.rs | 2226 ----------------- .../src/cardano_database_client/api.rs | 174 ++ .../download_unpack.rs | 961 +++++++ .../src/cardano_database_client/fetch.rs | 222 ++ .../immutable_file_range.rs | 122 + .../src/cardano_database_client/mod.rs | 135 + .../src/cardano_database_client/proving.rs | 633 +++++ 7 files changed, 2247 insertions(+), 2226 deletions(-) delete mode 100644 mithril-client/src/cardano_database_client.rs create mode 100644 mithril-client/src/cardano_database_client/api.rs create mode 100644 mithril-client/src/cardano_database_client/download_unpack.rs create mode 100644 mithril-client/src/cardano_database_client/fetch.rs create mode 100644 mithril-client/src/cardano_database_client/immutable_file_range.rs create mode 100644 mithril-client/src/cardano_database_client/mod.rs create mode 100644 mithril-client/src/cardano_database_client/proving.rs diff --git a/mithril-client/src/cardano_database_client.rs b/mithril-client/src/cardano_database_client.rs deleted file mode 100644 index 51de6a88891..00000000000 --- a/mithril-client/src/cardano_database_client.rs +++ /dev/null @@ -1,2226 +0,0 @@ -//! A client to retrieve Cardano databases data from an Aggregator. -//! -//! In order to do so it defines a [CardanoDatabaseClient] which exposes the following features: -//! - [get][CardanoDatabaseClient::get]: get a Cardano database data from its hash -//! - [list][CardanoDatabaseClient::list]: get the list of available Cardano database -//! -//! # Get a Cardano database -//! -//! To get a Cardano database using the [ClientBuilder][crate::client::ClientBuilder]. -//! -//! ```no_run -//! # async fn run() -> mithril_client::MithrilResult<()> { -//! use mithril_client::ClientBuilder; -//! -//! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; -//! let cardano_database = client.cardano_database().get("CARDANO_DATABASE_HASH").await?.unwrap(); -//! -//! println!( -//! "Cardano database hash={}, merkle_root={}, immutable_file_number={:?}", -//! cardano_database.hash, -//! cardano_database.merkle_root, -//! cardano_database.beacon.immutable_file_number -//! ); -//! # Ok(()) -//! # } -//! ``` -//! -//! # List available Cardano databases -//! -//! To list available Cardano databases using the [ClientBuilder][crate::client::ClientBuilder]. -//! -//! ```no_run -//! # async fn run() -> mithril_client::MithrilResult<()> { -//! use mithril_client::ClientBuilder; -//! -//! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; -//! let cardano_databases = client.cardano_database().list().await?; -//! -//! for cardano_database in cardano_databases { -//! println!("Cardano database hash={}, immutable_file_number={}", cardano_database.hash, cardano_database.beacon.immutable_file_number); -//! } -//! # Ok(()) -//! # } -//! ``` -//! -//! # Download a Cardano database snapshot -//! -//! To download a partial or a full Cardano database folder the [ClientBuilder][crate::client::ClientBuilder]. -//! -//! ```no_run -//! # #[cfg(feature = "fs")] -//! # async fn run() -> mithril_client::MithrilResult<()> { -//! use mithril_client::ClientBuilder; -//! -//! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; -//! let cardano_database_snapshot = client.cardano_database().get("CARDANO_DATABASE_HASH").await?.unwrap(); -//! -//! // Note: the directory must already exist, and the user running the binary must have read/write access to it. -//! let target_directory = Path::new("/home/user/download/"); -//! let immutable_file_range = ImmutableFileRange::Range(3, 6); -//! let download_unpack_options = DownloadUnpackOptions { -//! allow_override: true, -//! include_ancillary: true, -//! }; -//! client -//! .cardano_database() -//! .download_unpack( -//! &cardano_database_snapshot, -//! &immutable_file_range, -//! &target_directory, -//! download_unpack_options, -//! ) -//! .await?; -//! # -//! # Ok(()) -//! # } -//! ``` -//! -//! # Compute a Merkle proof for a Cardano database snapshot -//! -//! To compute proof of membership of downloaded immutable files in a Cardano database folder the [ClientBuilder][crate::client::ClientBuilder]. -//! -//! ```no_run -//! # #[cfg(feature = "fs")] -//! # async fn run() -> mithril_client::MithrilResult<()> { -//! use mithril_client::ClientBuilder; -//! -//! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; -//! let cardano_database_snapshot = client.cardano_database().get("CARDANO_DATABASE_HASH").await?.unwrap(); -//! let certificate = client.certificate().verify_chain(&cardano_database_snapshot.certificate_hash).await?; -//! -//! // Note: the directory must already exist, and the user running the binary must have read/write access to it. -//! let target_directory = Path::new("/home/user/download/"); -//! let immutable_file_range = ImmutableFileRange::Full; -//! let download_unpack_options = DownloadUnpackOptions { -//! allow_override: true, -//! include_ancillary: true, -//! }; -//! client -//! .cardano_database() -//! .download_unpack( -//! &cardano_database_snapshot, -//! &immutable_file_range, -//! &target_directory, -//! download_unpack_options, -//! ) -//! .await?; -//! -//! let merkle_proof = client -//! .cardano_database() -//! .compute_merkle_proof(&certificate, &immutable_file_range, &unpacked_dir) -//! .await?; -//! # -//! # Ok(()) -//! # } -//! ``` - -#[cfg(feature = "fs")] -use std::collections::{BTreeMap, BTreeSet}; -#[cfg(feature = "fs")] -use std::fs; -#[cfg(feature = "fs")] -use std::ops::RangeInclusive; -#[cfg(feature = "fs")] -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -#[cfg(feature = "fs")] -use anyhow::anyhow; -use anyhow::Context; -#[cfg(feature = "fs")] -use serde::de::DeserializeOwned; -#[cfg(feature = "fs")] -use slog::Logger; -#[cfg(feature = "fs")] -use tokio::task::JoinSet; - -#[cfg(feature = "fs")] -use mithril_common::{ - crypto_helper::{MKProof, MKTree, MKTreeNode, MKTreeStoreInMemory}, - digesters::{CardanoImmutableDigester, ImmutableDigester}, - entities::{ - AncillaryLocation, CompressionAlgorithm, DigestLocation, HexEncodedDigest, - ImmutableFileName, ImmutableFileNumber, ImmutablesLocation, - }, - messages::{ - CardanoDatabaseDigestListItemMessage, CardanoDatabaseSnapshotMessage, CertificateMessage, - SignedEntityTypeMessagePart, - }, - StdResult, -}; - -#[cfg(feature = "fs")] -use crate::feedback::{FeedbackSender, MithrilEvent}; -use crate::{ - aggregator_client::{AggregatorClient, AggregatorClientError, AggregatorRequest}, - file_downloader::{FileDownloaderResolver, FileDownloaderUri}, - CardanoDatabaseSnapshot, CardanoDatabaseSnapshotListItem, MithrilResult, -}; - -cfg_fs! { - /// Immutable file range representation - #[derive(Debug)] - pub enum ImmutableFileRange { - /// From the first (included) to the last immutable file number (included) - Full, - - /// From a specific immutable file number (included) to the last immutable file number (included) - From(ImmutableFileNumber), - - /// From a specific immutable file number (included) to another specific immutable file number (included) - Range(ImmutableFileNumber, ImmutableFileNumber), - - /// From the first immutable file number (included) up to a specific immutable file number (included) - UpTo(ImmutableFileNumber), - } - - impl ImmutableFileRange { - /// Returns the range of immutable file numbers - pub fn to_range_inclusive( - &self, - last_immutable_file_number: ImmutableFileNumber, - ) -> StdResult> { - // TODO: first immutable file is 0 on devnet and 1 otherwise. To be handled properly - let first_immutable_file_number = 0 as ImmutableFileNumber; - let full_range = first_immutable_file_number..=last_immutable_file_number; - - match self { - ImmutableFileRange::Full => Ok(full_range), - ImmutableFileRange::From(from) if full_range.contains(from) => { - Ok(*from..=last_immutable_file_number) - } - ImmutableFileRange::Range(from, to) - if full_range.contains(from) - && full_range.contains(to) - && !(*from..=*to).is_empty() => - { - Ok(*from..=*to) - } - ImmutableFileRange::UpTo(to) if full_range.contains(to) => { - Ok(first_immutable_file_number..=*to) - } - _ => Err(anyhow!("Invalid immutable file range: {self:?}")), - } - } - } - - /// Options for downloading and unpacking a Cardano database - #[derive(Debug,Default)] - pub struct DownloadUnpackOptions { - /// Allow overriding the destination directory - pub allow_override: bool, - - /// Include ancillary files in the download - pub include_ancillary: bool - } -} - -/// HTTP client for CardanoDatabase API from the Aggregator -pub struct CardanoDatabaseClient { - aggregator_client: Arc, - #[cfg(feature = "fs")] - immutable_file_downloader_resolver: Arc>, - #[cfg(feature = "fs")] - ancillary_file_downloader_resolver: Arc>, - #[cfg(feature = "fs")] - digest_file_downloader_resolver: Arc>, - #[cfg(feature = "fs")] - feedback_sender: FeedbackSender, - #[cfg(feature = "fs")] - logger: Logger, -} - -impl CardanoDatabaseClient { - /// Constructs a new `CardanoDatabase`. - pub fn new( - aggregator_client: Arc, - #[cfg(feature = "fs")] immutable_file_downloader_resolver: Arc< - dyn FileDownloaderResolver, - >, - #[cfg(feature = "fs")] ancillary_file_downloader_resolver: Arc< - dyn FileDownloaderResolver, - >, - #[cfg(feature = "fs")] digest_file_downloader_resolver: Arc< - dyn FileDownloaderResolver, - >, - #[cfg(feature = "fs")] feedback_sender: FeedbackSender, - #[cfg(feature = "fs")] logger: Logger, - ) -> Self { - Self { - aggregator_client, - #[cfg(feature = "fs")] - immutable_file_downloader_resolver, - #[cfg(feature = "fs")] - ancillary_file_downloader_resolver, - #[cfg(feature = "fs")] - digest_file_downloader_resolver, - #[cfg(feature = "fs")] - feedback_sender, - #[cfg(feature = "fs")] - logger: mithril_common::logging::LoggerExtensions::new_with_component_name::( - &logger, - ), - } - } - - /// Fetch a list of signed CardanoDatabase - pub async fn list(&self) -> MithrilResult> { - let response = self - .aggregator_client - .get_content(AggregatorRequest::ListCardanoDatabaseSnapshots) - .await - .with_context(|| "CardanoDatabase client can not get the artifact list")?; - let items = serde_json::from_str::>(&response) - .with_context(|| "CardanoDatabase client can not deserialize artifact list")?; - - Ok(items) - } - - /// Get the given Cardano database data by hash. - pub async fn get(&self, hash: &str) -> MithrilResult> { - self.fetch_with_aggregator_request(AggregatorRequest::GetCardanoDatabaseSnapshot { - hash: hash.to_string(), - }) - .await - } - - /// Fetch the given Cardano database data with an aggregator request. - /// If it cannot be found, a None is returned. - async fn fetch_with_aggregator_request( - &self, - request: AggregatorRequest, - ) -> MithrilResult> { - match self.aggregator_client.get_content(request).await { - Ok(content) => { - let result: T = serde_json::from_str(&content) - .with_context(|| "CardanoDatabase client can not deserialize artifact")?; - - Ok(Some(result)) - } - Err(AggregatorClientError::RemoteServerLogical(_)) => Ok(None), - Err(e) => Err(e.into()), - } - } - - cfg_fs! { - /// Download and unpack the given Cardano database parts data by hash. - pub async fn download_unpack( - &self, - cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, - immutable_file_range: &ImmutableFileRange, - target_dir: &Path, - download_unpack_options: DownloadUnpackOptions, - ) -> StdResult<()> { - let compression_algorithm = cardano_database_snapshot.compression_algorithm; - let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; - let immutable_file_number_range = - immutable_file_range.to_range_inclusive(last_immutable_file_number)?; - - self.verify_can_write_to_target_directory(target_dir, &download_unpack_options)?; - self.create_target_directory_sub_directories_if_not_exist( - target_dir, - &download_unpack_options, - )?; - - let immutable_locations = &cardano_database_snapshot.locations.immutables; - self.download_unpack_immutable_files( - immutable_locations, - immutable_file_number_range, - &compression_algorithm, - target_dir, - ) - .await?; - - let digest_locations = &cardano_database_snapshot.locations.digests; - self.download_unpack_digest_file(digest_locations, &Self::digest_target_dir(target_dir)) - .await?; - - if download_unpack_options.include_ancillary { - let ancillary_locations = &cardano_database_snapshot.locations.ancillary; - self.download_unpack_ancillary_file( - ancillary_locations, - &compression_algorithm, - target_dir, - ) - .await?; - } - - Ok(()) - } - - fn digest_target_dir(target_dir: &Path) -> PathBuf { - target_dir.join("digest") - } - - fn immutable_files_target_dir(target_dir: &Path) -> PathBuf { - target_dir.join("immutable") - } - - fn volatile_target_dir(target_dir: &Path) -> PathBuf { - target_dir.join("volatile") - } - - fn ledger_target_dir(target_dir: &Path) -> PathBuf { - target_dir.join("ledger") - } - - fn create_directory_if_not_exists(dir: &Path) -> StdResult<()> { - if dir.exists() { - return Ok(()); - } - - fs::create_dir_all(dir).map_err(|e| anyhow!("Failed creating directory: {e}")) - } - - fn delete_directory(dir: &Path) -> StdResult<()> { - if dir.exists() { - fs::remove_dir_all(dir).map_err(|e| anyhow!("Failed deleting directory: {e}"))?; - } - - Ok(()) - } - - fn read_files_in_directory(dir: &Path) -> StdResult> { - let mut files = vec![]; - for entry in fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - if path.is_file() { - files.push(path); - } - } - - Ok(files) - } - - /// Verify if the target directory is writable. - fn verify_can_write_to_target_directory( - &self, - target_dir: &Path, - download_unpack_options: &DownloadUnpackOptions, - ) -> StdResult<()> { - let immutable_files_target_dir = Self::immutable_files_target_dir(target_dir); - let volatile_target_dir = Self::volatile_target_dir(target_dir); - let ledger_target_dir = Self::ledger_target_dir(target_dir); - if !download_unpack_options.allow_override { - if immutable_files_target_dir.exists() { - return Err(anyhow!( - "Immutable files target directory already exists in: {target_dir:?}" - )); - } - if download_unpack_options.include_ancillary { - if volatile_target_dir.exists() { - return Err(anyhow!( - "Volatile target directory already exists in: {target_dir:?}" - )); - } - if ledger_target_dir.exists() { - return Err(anyhow!( - "Ledger target directory already exists in: {target_dir:?}" - )); - } - } - } - - Ok(()) - } - - /// Create the target directory sub-directories if they do not exist. - // TODO: is it really needed? - fn create_target_directory_sub_directories_if_not_exist( - &self, - target_dir: &Path, - download_unpack_options: &DownloadUnpackOptions, - ) -> StdResult<()> { - let immutable_files_target_dir = Self::immutable_files_target_dir(target_dir); - Self::create_directory_if_not_exists(&immutable_files_target_dir)?; - let digest_target_dir = Self::digest_target_dir(target_dir); - Self::create_directory_if_not_exists(&digest_target_dir)?; - if download_unpack_options.include_ancillary { - let volatile_target_dir = Self::volatile_target_dir(target_dir); - let ledger_target_dir = Self::ledger_target_dir(target_dir); - Self::create_directory_if_not_exists(&volatile_target_dir)?; - Self::create_directory_if_not_exists(&ledger_target_dir)?; - } - - Ok(()) - } - - fn feedback_event_builder_immutable_download(download_id: String, downloaded_bytes: u64, size: u64) -> Option { - Some(MithrilEvent::ImmutableDownloadProgress { - download_id, - downloaded_bytes, - size, - }) - } - - fn feedback_event_builder_ancillary_download(download_id: String, downloaded_bytes: u64, size: u64) -> Option { - Some(MithrilEvent::AncillaryDownloadProgress { - download_id, - downloaded_bytes, - size, - }) - } - - fn feedback_event_builder_digest_download(download_id: String, downloaded_bytes: u64, size: u64) -> Option { - Some(MithrilEvent::DigestDownloadProgress { - download_id, - downloaded_bytes, - size, - }) - } - - /// Download and unpack the immutable files of the given range. - /// - /// The download is attempted for each location until the full range is downloaded. - /// An error is returned if not all the files are downloaded. - async fn download_unpack_immutable_files( - &self, - locations: &[ImmutablesLocation], - range: RangeInclusive, - compression_algorithm: &CompressionAlgorithm, - immutable_files_target_dir: &Path, - ) -> StdResult<()> { - let mut locations_sorted = locations.to_owned(); - locations_sorted.sort(); - let mut immutable_file_numbers_to_download = - range.clone().map(|n| n.to_owned()).collect::>(); - for location in locations_sorted { - let immutable_files_numbers_downloaded = self.download_unpack_immutable_files_for_location( - &location, - &immutable_file_numbers_to_download, - compression_algorithm, - immutable_files_target_dir, - ).await?; - for immutable_file_number in immutable_files_numbers_downloaded { - immutable_file_numbers_to_download.remove(&immutable_file_number); - } - if immutable_file_numbers_to_download.is_empty() { - return Ok(()); - } - } - - Err(anyhow!( - "Failed downloading and unpacking immutable files for locations: {locations:?}" - )) - } - - async fn download_unpack_immutable_files_for_location(&self, location: &ImmutablesLocation, immutable_file_numbers_to_download: &BTreeSet, compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path,) -> StdResult> { - let mut immutable_file_numbers_downloaded = BTreeSet::new(); - let file_downloader = self - .immutable_file_downloader_resolver - .resolve(location) - .ok_or_else(|| { - anyhow!("Failed resolving a file downloader for location: {location:?}") - })?; - let file_downloader_uris = - FileDownloaderUri::expand_immutable_files_location_to_file_downloader_uris( - location, - immutable_file_numbers_to_download - .clone() - .into_iter() - .collect::>() - .as_slice(), - )?; - let mut join_set: JoinSet> = JoinSet::new(); - for (immutable_file_number, file_downloader_uri) in file_downloader_uris { - let compression_algorithm_clone = compression_algorithm.to_owned(); - let immutable_files_target_dir_clone = immutable_files_target_dir.to_owned(); - let file_downloader_clone = file_downloader.clone(); - let feedback_receiver_clone = self.feedback_sender.clone(); - join_set.spawn(async move { - let download_id = MithrilEvent::new_snapshot_download_id(); - feedback_receiver_clone - .send_event(MithrilEvent::ImmutableDownloadStarted { immutable_file_number, download_id: download_id.clone()}) - .await; - let downloaded = file_downloader_clone - .download_unpack( - &file_downloader_uri, - &immutable_files_target_dir_clone, - Some(compression_algorithm_clone), - &download_id, - Self::feedback_event_builder_immutable_download, - ) - .await; - match downloaded { - Ok(_) => { - feedback_receiver_clone - .send_event(MithrilEvent::ImmutableDownloadCompleted { download_id }) - .await; - - Ok(immutable_file_number) - } - Err(e) => { - Err(e.context(format!("Failed downloading and unpacking immutable file {immutable_file_number} for location {file_downloader_uri:?}"))) - } - } - }); - } - while let Some(result) = join_set.join_next().await { - match result? { - Ok(immutable_file_number) => { - immutable_file_numbers_downloaded.insert(immutable_file_number); - } - Err(e) => { - slog::error!( - self.logger, - "Failed downloading and unpacking immutable files"; "error" => e.to_string() - ); - } - } - } - - Ok(immutable_file_numbers_downloaded) - } - - /// Download and unpack the ancillary files. - async fn download_unpack_ancillary_file( - &self, - locations: &[AncillaryLocation], - compression_algorithm: &CompressionAlgorithm, - ancillary_file_target_dir: &Path, - ) -> StdResult<()> { - let mut locations_sorted = locations.to_owned(); - locations_sorted.sort(); - for location in locations_sorted { - let download_id = MithrilEvent::new_ancillary_download_id(); - self.feedback_sender - .send_event(MithrilEvent::AncillaryDownloadStarted { download_id: download_id.clone()}) - .await; - let file_downloader = self - .ancillary_file_downloader_resolver - .resolve(&location) - .ok_or_else(|| { - anyhow!("Failed resolving a file downloader for location: {location:?}") - })?; - let file_downloader_uri: FileDownloaderUri = location.into(); - let downloaded = file_downloader - .download_unpack( - &file_downloader_uri, - ancillary_file_target_dir, - Some(compression_algorithm.to_owned()), - &download_id, - Self::feedback_event_builder_ancillary_download - ) - .await; - match downloaded { - Ok(_) => { - self.feedback_sender - .send_event(MithrilEvent::AncillaryDownloadCompleted { download_id }) - .await; - return Ok(()) - }, - Err(e) => { - slog::error!( - self.logger, - "Failed downloading and unpacking ancillaries for location {file_downloader_uri:?}"; "error" => e.to_string() - ); - } - } - } - - Err(anyhow!( - "Failed downloading and unpacking ancillaries for all locations" - )) - } - - async fn download_unpack_digest_file( - &self, - locations: &[DigestLocation], - digest_file_target_dir: &Path, - ) -> StdResult<()> { - let mut locations_sorted = locations.to_owned(); - locations_sorted.sort(); - for location in locations_sorted { - let download_id = MithrilEvent::new_digest_download_id(); - self.feedback_sender - .send_event(MithrilEvent::DigestDownloadStarted { download_id: download_id.clone()}) - .await; - let file_downloader = self - .digest_file_downloader_resolver - .resolve(&location) - .ok_or_else(|| { - anyhow!("Failed resolving a file downloader for location: {location:?}") - })?; - let file_downloader_uri: FileDownloaderUri = location.into(); - let downloaded = file_downloader - .download_unpack( - &file_downloader_uri, - digest_file_target_dir, - None, - &download_id, - Self::feedback_event_builder_digest_download - ) - .await; - match downloaded { - Ok(_) => { - self.feedback_sender - .send_event(MithrilEvent::DigestDownloadCompleted { download_id }) - .await; - return Ok(()) - }, - Err(e) => { - slog::error!( - self.logger, - "Failed downloading and unpacking digest for location {file_downloader_uri:?}"; "error" => e.to_string() - ); - } - } - } - - Err(anyhow!( - "Failed downloading and unpacking digests for all locations" - )) - } - - fn read_digest_file( - &self, - digest_file_target_dir: &Path, - ) -> StdResult> { - let digest_files = Self::read_files_in_directory(digest_file_target_dir)?; - if digest_files.len() > 1 { - return Err(anyhow!( - "Multiple digest files found in directory: {digest_file_target_dir:?}" - )); - } - if digest_files.is_empty() { - return Err(anyhow!( - "No digest file found in directory: {digest_file_target_dir:?}" - )); - } - - let digest_file = &digest_files[0]; - let content = fs::read_to_string(digest_file) - .with_context(|| format!("Failed reading digest file: {digest_file:?}"))?; - let digest_messages: Vec = - serde_json::from_str(&content) - .with_context(|| format!("Failed deserializing digest file: {digest_file:?}"))?; - let digest_map = digest_messages - .into_iter() - .map(|message| (message.immutable_file_name, message.digest)) - .collect::>(); - - Ok(digest_map) - } - - /// Compute the Merkle proof of membership for the given immutable file range. - pub async fn compute_merkle_proof( - &self, - certificate: &CertificateMessage, - immutable_file_range: &ImmutableFileRange, - database_dir: &Path, - ) -> StdResult { - let network = certificate.metadata.network.clone(); - let last_immutable_file_number = match &certificate.signed_entity_type { - SignedEntityTypeMessagePart::CardanoDatabase(beacon) => beacon.immutable_file_number, - _ => return Err(anyhow!("Invalid signed entity type: {:?}",certificate.signed_entity_type)), - }; - let immutable_file_number_range = - immutable_file_range.to_range_inclusive(last_immutable_file_number)?; - - let downloaded_digests = self.read_digest_file(&Self::digest_target_dir(database_dir))?; - let merkle_tree: MKTree = - MKTree::new(&downloaded_digests.values().cloned().collect::>())?; - let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone()); - let computed_digests = immutable_digester - .compute_digests_for_range(database_dir, &immutable_file_number_range) - .await?.entries - .values() - .map(MKTreeNode::from) - .collect::>(); - Self::delete_directory(&Self::digest_target_dir(database_dir))?; - - merkle_tree.compute_proof(&computed_digests) - } - } -} - -#[cfg(test)] -mod tests { - - use super::*; - - mod cardano_database_client { - use anyhow::anyhow; - use chrono::{DateTime, Utc}; - use mockall::predicate::eq; - - use mithril_common::{ - digesters::CardanoImmutableDigester, - entities::{ - AncillaryLocationDiscriminants, CardanoDbBeacon, CompressionAlgorithm, - DigestLocationDiscriminants, Epoch, ImmutablesLocationDiscriminants, - }, - }; - - use crate::{ - aggregator_client::MockAggregatorHTTPClient, - feedback::{FeedbackReceiver, MithrilEvent, StackFeedbackReceiver}, - file_downloader::{ - AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloader, - ImmutablesFileDownloaderResolver, MockFileDownloaderBuilder, - }, - test_utils, - }; - - use super::*; - - fn fake_messages() -> Vec { - vec![ - CardanoDatabaseSnapshotListItem { - hash: "hash-123".to_string(), - merkle_root: "mkroot-123".to_string(), - beacon: CardanoDbBeacon { - epoch: Epoch(1), - immutable_file_number: 123, - }, - certificate_hash: "cert-hash-123".to_string(), - total_db_size_uncompressed: 800796318, - created_at: DateTime::parse_from_rfc3339("2025-01-19T13:43:05.618857482Z") - .unwrap() - .with_timezone(&Utc), - compression_algorithm: CompressionAlgorithm::default(), - cardano_node_version: "0.0.1".to_string(), - }, - CardanoDatabaseSnapshotListItem { - hash: "hash-456".to_string(), - merkle_root: "mkroot-456".to_string(), - beacon: CardanoDbBeacon { - epoch: Epoch(2), - immutable_file_number: 456, - }, - certificate_hash: "cert-hash-456".to_string(), - total_db_size_uncompressed: 2960713808, - created_at: DateTime::parse_from_rfc3339("2025-01-27T15:22:05.618857482Z") - .unwrap() - .with_timezone(&Utc), - compression_algorithm: CompressionAlgorithm::default(), - cardano_node_version: "0.0.1".to_string(), - }, - ] - } - - struct CardanoDatabaseClientDependencyInjector { - http_client: MockAggregatorHTTPClient, - immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver, - ancillary_file_downloader_resolver: AncillaryFileDownloaderResolver, - digest_file_downloader_resolver: DigestFileDownloaderResolver, - feedback_receivers: Vec>, - } - - impl CardanoDatabaseClientDependencyInjector { - fn new() -> Self { - Self { - http_client: MockAggregatorHTTPClient::new(), - immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver::new( - vec![], - ), - ancillary_file_downloader_resolver: AncillaryFileDownloaderResolver::new( - vec![], - ), - digest_file_downloader_resolver: DigestFileDownloaderResolver::new(vec![]), - feedback_receivers: vec![], - } - } - - fn with_http_client_mock_config(mut self, config: F) -> Self - where - F: FnOnce(&mut MockAggregatorHTTPClient), - { - config(&mut self.http_client); - - self - } - - fn with_immutable_file_downloaders( - self, - file_downloaders: Vec<(ImmutablesLocationDiscriminants, Arc)>, - ) -> Self { - let immutable_file_downloader_resolver = - ImmutablesFileDownloaderResolver::new(file_downloaders); - - Self { - immutable_file_downloader_resolver, - ..self - } - } - - fn with_ancillary_file_downloaders( - self, - file_downloaders: Vec<(AncillaryLocationDiscriminants, Arc)>, - ) -> Self { - let ancillary_file_downloader_resolver = - AncillaryFileDownloaderResolver::new(file_downloaders); - - Self { - ancillary_file_downloader_resolver, - ..self - } - } - - fn with_digest_file_downloaders( - self, - file_downloaders: Vec<(DigestLocationDiscriminants, Arc)>, - ) -> Self { - let digest_file_downloader_resolver = - DigestFileDownloaderResolver::new(file_downloaders); - - Self { - digest_file_downloader_resolver, - ..self - } - } - - fn with_feedback_receivers( - self, - feedback_receivers: &[Arc], - ) -> Self { - Self { - feedback_receivers: feedback_receivers.to_vec(), - ..self - } - } - - fn build_cardano_database_client(self) -> CardanoDatabaseClient { - CardanoDatabaseClient::new( - Arc::new(self.http_client), - Arc::new(self.immutable_file_downloader_resolver), - Arc::new(self.ancillary_file_downloader_resolver), - Arc::new(self.digest_file_downloader_resolver), - FeedbackSender::new(&self.feedback_receivers), - test_utils::test_logger(), - ) - } - } - - mod list { - use super::*; - - #[tokio::test] - async fn list_cardano_database_snapshots_returns_messages() { - let message = fake_messages(); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - http_client - .expect_get_content() - .with(eq(AggregatorRequest::ListCardanoDatabaseSnapshots)) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - }) - .build_cardano_database_client(); - - let messages = client.list().await.unwrap(); - - assert_eq!(2, messages.len()); - assert_eq!("hash-123".to_string(), messages[0].hash); - assert_eq!("hash-456".to_string(), messages[1].hash); - } - - #[tokio::test] - async fn list_cardano_database_snapshots_returns_error_when_invalid_json_structure_in_response( - ) { - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - http_client - .expect_get_content() - .return_once(move |_| Ok("invalid json structure".to_string())); - }) - .build_cardano_database_client(); - - client - .list() - .await - .expect_err("List Cardano databases should return an error"); - } - } - - mod get { - use super::*; - - #[tokio::test] - async fn get_cardano_database_snapshot_returns_message() { - let expected_cardano_database_snapshot = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - ..CardanoDatabaseSnapshot::dummy() - }; - let message = expected_cardano_database_snapshot.clone(); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - http_client - .expect_get_content() - .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - })) - .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); - }) - .build_cardano_database_client(); - - let cardano_database = client - .get("hash-123") - .await - .unwrap() - .expect("This test returns a Cardano database"); - - assert_eq!(expected_cardano_database_snapshot, cardano_database); - } - - #[tokio::test] - async fn get_cardano_database_snapshot_returns_error_when_invalid_json_structure_in_response( - ) { - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - http_client - .expect_get_content() - .return_once(move |_| Ok("invalid json structure".to_string())); - }) - .build_cardano_database_client(); - - client - .get("hash-123") - .await - .expect_err("Get Cardano database should return an error"); - } - - #[tokio::test] - async fn get_cardano_database_snapshot_returns_none_when_not_found_or_remote_server_logical_error( - ) { - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - http_client.expect_get_content().return_once(move |_| { - Err(AggregatorClientError::RemoteServerLogical(anyhow!( - "not found" - ))) - }); - }) - .build_cardano_database_client(); - - let result = client.get("hash-123").await.unwrap(); - - assert!(result.is_none()); - } - - #[tokio::test] - async fn get_cardano_database_snapshot_returns_error() { - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { - http_client.expect_get_content().return_once(move |_| { - Err(AggregatorClientError::SubsystemError(anyhow!("error"))) - }); - }) - .build_cardano_database_client(); - - client - .get("hash-123") - .await - .expect_err("Get Cardano database should return an error"); - } - } - - cfg_fs! { - mod download_unpack { - use std::fs; - use std::path::Path; - - use mithril_common::{ - entities::{ImmutablesLocationDiscriminants, MultiFilesUri, TemplateUri}, - messages::ArtifactsLocationsMessagePart, - test_utils::TempDir, - }; - - use super::*; - - #[tokio::test] - async fn download_unpack_fails_with_invalid_immutable_file_range() { - let immutable_file_range = ImmutableFileRange::Range(1, 0); - let download_unpack_options = DownloadUnpackOptions::default(); - let cardano_db_snapshot = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - ..CardanoDatabaseSnapshot::dummy() - }; - let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .build_cardano_database_client(); - - client - .download_unpack( - &cardano_db_snapshot, - &immutable_file_range, - target_dir, - download_unpack_options, - ) - .await - .expect_err("download_unpack should fail"); - } - - #[tokio::test] - async fn download_unpack_fails_when_immutable_files_download_fail() { - let total_immutable_files = 10; - let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); - let download_unpack_options = DownloadUnpackOptions::default(); - let cardano_db_snapshot = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - locations: ArtifactsLocationsMessagePart { - immutables: vec![ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever/{immutable_file_number}.tar.gz".to_string(), - )), - }], - ..ArtifactsLocationsMessagePart::default() - }, - ..CardanoDatabaseSnapshot::dummy() - }; - let target_dir = TempDir::new( - "cardano_database_client", - "download_unpack_fails_when_immutable_files_download_fail", - ) - .build(); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - MockFileDownloaderBuilder::default() - .with_times(total_immutable_files as usize) - .with_failure() - .build() - }), - )]) - .build_cardano_database_client(); - - client - .download_unpack( - &cardano_db_snapshot, - &immutable_file_range, - &target_dir, - download_unpack_options, - ) - .await - .expect_err("download_unpack should fail"); - } - - #[tokio::test] - async fn download_unpack_fails_when_target_target_dir_would_be_overwritten_without_allow_override( - ) { - let immutable_file_range = ImmutableFileRange::Range(1, 10); - let download_unpack_options = DownloadUnpackOptions::default(); - let cardano_db_snapshot = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - ..CardanoDatabaseSnapshot::dummy() - }; - let target_dir = &TempDir::new( - "cardano_database_client", - "download_unpack_fails_when_target_target_dir_would_be_overwritten_without_allow_override", - ) - .build(); - fs::create_dir_all(target_dir.join("immutable")).unwrap(); - let client = CardanoDatabaseClientDependencyInjector::new() - .build_cardano_database_client(); - - client - .download_unpack( - &cardano_db_snapshot, - &immutable_file_range, - target_dir, - download_unpack_options, - ) - .await - .expect_err("download_unpack should fail"); - } - - #[tokio::test] - async fn download_unpack_succeeds_with_valid_range() { - let immutable_file_range = ImmutableFileRange::Range(1, 2); - let download_unpack_options = DownloadUnpackOptions { - include_ancillary: true, - ..DownloadUnpackOptions::default() - }; - let cardano_db_snapshot = CardanoDatabaseSnapshot { - hash: "hash-123".to_string(), - locations: ArtifactsLocationsMessagePart { - immutables: vec![ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever/{immutable_file_number}.tar.gz".to_string(), - )), - }], - ancillary: vec![AncillaryLocation::CloudStorage { - uri: "http://whatever/ancillary.tar.gz".to_string(), - }], - digests: vec![DigestLocation::CloudStorage { - uri: "http://whatever/digests.json".to_string(), - }], - }, - ..CardanoDatabaseSnapshot::dummy() - }; - let target_dir = TempDir::new( - "cardano_database_client", - "download_unpack_succeeds_with_valid_range", - ) - .build(); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - let mock_file_downloader = MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever/00001.tar.gz") - .with_target_dir(target_dir.clone()) - .with_success() - .build(); - - MockFileDownloaderBuilder::from_mock(mock_file_downloader) - .with_file_uri("http://whatever/00002.tar.gz") - .with_target_dir(target_dir.clone()) - .with_success() - .build() - }), - )]) - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever/ancillary.tar.gz") - .with_target_dir(target_dir.clone()) - .with_compression(Some(CompressionAlgorithm::default())) - .with_success() - .build(), - ), - )]) - .with_digest_file_downloaders(vec![( - DigestLocationDiscriminants::CloudStorage, - Arc::new({ - MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever/digests.json") - .with_target_dir(target_dir.join("digest")) - .with_compression(None) - .with_success() - .build() - }), - )]) - .build_cardano_database_client(); - - client - .download_unpack( - &cardano_db_snapshot, - &immutable_file_range, - &target_dir, - download_unpack_options, - ) - .await - .unwrap(); - } - } - - mod verify_can_write_to_target_dir { - use std::fs; - - use mithril_common::test_utils::TempDir; - - use super::*; - - #[test] - fn verify_can_write_to_target_dir_always_succeeds_with_allow_overwrite() { - let target_dir = TempDir::new( - "cardano_database_client", - "verify_can_write_to_target_dir_always_succeeds_with_allow_overwrite", - ) - .build(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: true, - include_ancillary: false, - }, - ) - .unwrap(); - - fs::create_dir_all(CardanoDatabaseClient::immutable_files_target_dir( - &target_dir, - )) - .unwrap(); - fs::create_dir_all(CardanoDatabaseClient::volatile_target_dir(&target_dir)) - .unwrap(); - fs::create_dir_all(CardanoDatabaseClient::ledger_target_dir(&target_dir)).unwrap(); - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: true, - include_ancillary: false, - }, - ) - .unwrap(); - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: true, - include_ancillary: true, - }, - ) - .unwrap(); - } - - #[test] - fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_immutable_target_dir( - ) { - let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_immutable_target_dir").build(); - fs::create_dir_all(CardanoDatabaseClient::immutable_files_target_dir( - &target_dir, - )) - .unwrap(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: false, - }, - ) - .expect_err("verify_can_write_to_target_dir should fail"); - - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: true, - }, - ) - .expect_err("verify_can_write_to_target_dir should fail"); - } - - #[test] - fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_ledger_target_dir( - ) { - let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_ledger_target_dir").build(); - fs::create_dir_all(CardanoDatabaseClient::ledger_target_dir(&target_dir)).unwrap(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: true, - }, - ) - .expect_err("verify_can_write_to_target_dir should fail"); - - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: false, - }, - ) - .unwrap(); - } - - #[test] - fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_volatile_target_dir( - ) { - let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_volatile_target_dir").build(); - fs::create_dir_all(CardanoDatabaseClient::volatile_target_dir(&target_dir)) - .unwrap(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: true, - }, - ) - .expect_err("verify_can_write_to_target_dir should fail"); - - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: false, - }, - ) - .unwrap(); - } - } - - mod create_target_directory_sub_directories_if_not_exist { - use mithril_common::test_utils::TempDir; - - use super::*; - - #[test] - fn create_target_directory_sub_directories_if_not_exist_without_ancillary() { - let target_dir = TempDir::new( - "cardano_database_client", - "create_target_directory_sub_directories_if_not_exist_without_ancillary", - ) - .build(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - assert!(!target_dir.join("digest").exists()); - assert!(!target_dir.join("immutable").exists()); - assert!(!target_dir.join("volatile").exists()); - assert!(!target_dir.join("ledger").exists()); - - client - .create_target_directory_sub_directories_if_not_exist( - &target_dir, - &DownloadUnpackOptions { - include_ancillary: false, - ..Default::default() - }, - ) - .unwrap(); - - assert!(target_dir.join("digest").exists()); - assert!(target_dir.join("immutable").exists()); - assert!(!target_dir.join("volatile").exists()); - assert!(!target_dir.join("ledger").exists()); - } - - #[test] - fn create_target_directory_sub_directories_if_not_exist_with_ancillary() { - let target_dir = TempDir::new( - "cardano_database_client", - "create_target_directory_sub_directories_if_not_exist_with_ancillary", - ) - .build(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - assert!(!target_dir.join("digest").exists()); - assert!(!target_dir.join("immutable").exists()); - assert!(!target_dir.join("volatile").exists()); - assert!(!target_dir.join("ledger").exists()); - - client - .create_target_directory_sub_directories_if_not_exist( - &target_dir, - &DownloadUnpackOptions { - include_ancillary: true, - ..Default::default() - }, - ) - .unwrap(); - - assert!(target_dir.join("digest").exists()); - assert!(target_dir.join("immutable").exists()); - assert!(target_dir.join("volatile").exists()); - assert!(target_dir.join("ledger").exists()); - } - } - - mod download_unpack_immutable_files { - use mithril_common::{ - entities::{MultiFilesUri, TemplateUri}, - test_utils::TempDir, - }; - - use super::*; - - #[tokio::test] - async fn download_unpack_immutable_files_fails_if_one_is_not_retrieved() { - let total_immutable_files = 2; - let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); - let target_dir = TempDir::new( - "cardano_database_client", - "download_unpack_immutable_files_succeeds", - ) - .build(); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - let mock_file_downloader = - MockFileDownloaderBuilder::default().with_failure().build(); - - MockFileDownloaderBuilder::from_mock(mock_file_downloader) - .with_success() - .build() - }), - )]) - .build_cardano_database_client(); - - client - .download_unpack_immutable_files( - &[ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever/{immutable_file_number}.tar.gz".to_string(), - )), - }], - immutable_file_range - .to_range_inclusive(total_immutable_files) - .unwrap(), - &CompressionAlgorithm::default(), - &target_dir, - ) - .await - .expect_err("download_unpack_immutable_files should fail"); - } - - #[tokio::test] - async fn download_unpack_immutable_files_succeeds_if_all_are_retrieved_with_same_location( - ) { - let total_immutable_files = 2; - let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); - let target_dir = TempDir::new( - "cardano_database_client", - "download_unpack_immutable_files_succeeds", - ) - .build(); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_times(2) - .with_success() - .build(), - ), - )]) - .build_cardano_database_client(); - - client - .download_unpack_immutable_files( - &[ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever-1/{immutable_file_number}.tar.gz".to_string(), - )), - }], - immutable_file_range - .to_range_inclusive(total_immutable_files) - .unwrap(), - &CompressionAlgorithm::default(), - &target_dir, - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn download_unpack_immutable_files_succeeds_if_all_are_retrieved_with_different_locations( - ) { - let total_immutable_files = 2; - let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); - let target_dir = TempDir::new( - "cardano_database_client", - "download_unpack_immutable_files_succeeds", - ) - .build(); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - let mock_file_downloader = MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever-1/00001.tar.gz") - .with_target_dir(target_dir.clone()) - .with_failure() - .build(); - let mock_file_downloader = - MockFileDownloaderBuilder::from_mock(mock_file_downloader) - .with_file_uri("http://whatever-1/00002.tar.gz") - .with_target_dir(target_dir.clone()) - .with_success() - .build(); - - MockFileDownloaderBuilder::from_mock(mock_file_downloader) - .with_file_uri("http://whatever-2/00001.tar.gz") - .with_target_dir(target_dir.clone()) - .with_success() - .build() - }), - )]) - .build_cardano_database_client(); - - client - .download_unpack_immutable_files( - &[ - ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever-1/{immutable_file_number}.tar.gz".to_string(), - )), - }, - ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever-2/{immutable_file_number}.tar.gz".to_string(), - )), - }, - ], - immutable_file_range - .to_range_inclusive(total_immutable_files) - .unwrap(), - &CompressionAlgorithm::default(), - &target_dir, - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn download_unpack_immutable_files_sends_feedbacks() { - let total_immutable_files = 1; - let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); - let target_dir = Path::new("."); - let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - MockFileDownloaderBuilder::default() - .with_success() - .build() - }), - )]) - .with_feedback_receivers(&[feedback_receiver.clone()]) - .build_cardano_database_client(); - - client - .download_unpack_immutable_files( - &[ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever/{immutable_file_number}.tar.gz".to_string(), - )), - }], - immutable_file_range - .to_range_inclusive(total_immutable_files) - .unwrap(), - &CompressionAlgorithm::default(), - target_dir, - ) - .await - .unwrap(); - - let sent_events = feedback_receiver.stacked_events(); - let id = sent_events[0].event_id(); - let expected_events = vec![ - MithrilEvent::ImmutableDownloadStarted { - immutable_file_number: 1, - download_id: id.to_string(), - - }, - MithrilEvent::ImmutableDownloadCompleted { - download_id: id.to_string(), - }, - ]; - assert_eq!(expected_events, sent_events); - } - } - - mod download_unpack_ancillary_file { - - use super::*; - - #[tokio::test] - async fn download_unpack_ancillary_file_fails_if_no_location_is_retrieved() { - let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new(MockFileDownloaderBuilder::default().with_failure().build()), - )]) - .build_cardano_database_client(); - - client - .download_unpack_ancillary_file( - &[AncillaryLocation::CloudStorage { - uri: "http://whatever-1/ancillary.tar.gz".to_string(), - }], - &CompressionAlgorithm::default(), - target_dir, - ) - .await - .expect_err("download_unpack_ancillary_file should fail"); - } - - #[tokio::test] - async fn download_unpack_ancillary_file_succeeds_if_at_least_one_location_is_retrieved() - { - let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new({ - let mock_file_downloader = MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever-1/ancillary.tar.gz") - .with_target_dir(target_dir.to_path_buf()) - .with_failure() - .build(); - - MockFileDownloaderBuilder::from_mock(mock_file_downloader) - .with_file_uri("http://whatever-2/ancillary.tar.gz") - .with_target_dir(target_dir.to_path_buf()) - .with_success() - .build() - }), - )]) - .build_cardano_database_client(); - - client - .download_unpack_ancillary_file( - &[ - AncillaryLocation::CloudStorage { - uri: "http://whatever-1/ancillary.tar.gz".to_string(), - }, - AncillaryLocation::CloudStorage { - uri: "http://whatever-2/ancillary.tar.gz".to_string(), - }, - ], - &CompressionAlgorithm::default(), - target_dir, - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn download_unpack_ancillary_file_succeeds_when_first_location_is_retrieved() { - let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever-1/ancillary.tar.gz") - .with_target_dir(target_dir.to_path_buf()) - .with_success() - .build(), - ), - )]) - .build_cardano_database_client(); - - client - .download_unpack_ancillary_file( - &[ - AncillaryLocation::CloudStorage { - uri: "http://whatever-1/ancillary.tar.gz".to_string(), - }, - AncillaryLocation::CloudStorage { - uri: "http://whatever-2/ancillary.tar.gz".to_string(), - }, - ], - &CompressionAlgorithm::default(), - target_dir, - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn download_unpack_ancillary_files_sends_feedbacks() { - let target_dir = Path::new("."); - let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_success() - .build(), - ), - )]) - .with_feedback_receivers(&[feedback_receiver.clone()]) - .build_cardano_database_client(); - - client - .download_unpack_ancillary_file( - &[ - AncillaryLocation::CloudStorage { - uri: "http://whatever-1/ancillary.tar.gz".to_string(), - }, - ], - &CompressionAlgorithm::default(), - target_dir, - ) - .await - .unwrap(); - - let sent_events = feedback_receiver.stacked_events(); - let id = sent_events[0].event_id(); - let expected_events = vec![ - MithrilEvent::AncillaryDownloadStarted { - download_id: id.to_string(), - - }, - MithrilEvent::AncillaryDownloadCompleted { - download_id: id.to_string(), - }, - ]; - assert_eq!(expected_events, sent_events); - } - } - - mod download_unpack_digest_file { - - use crate::file_downloader::MockFileDownloader; - - use super::*; - - #[tokio::test] - async fn download_unpack_digest_file_fails_if_no_location_is_retrieved() { - let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_digest_file_downloaders(vec![ - ( - DigestLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_failure() - .build(), - ), - ), - ( - DigestLocationDiscriminants::Aggregator, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_failure() - .build(), - ), - ), - ]) - .build_cardano_database_client(); - - client - .download_unpack_digest_file( - &[ - DigestLocation::CloudStorage { - uri: "http://whatever-1/digests.json".to_string(), - }, - DigestLocation::Aggregator { - uri: "http://whatever-2/digest".to_string(), - }, - ], - target_dir, - ) - .await - .expect_err("download_unpack_digest_file should fail"); - } - - #[tokio::test] - async fn download_unpack_digest_file_succeeds_if_at_least_one_location_is_retrieved() { - let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_digest_file_downloaders(vec![ - ( - DigestLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_failure() - .build(), - ), - ), - ( - DigestLocationDiscriminants::Aggregator, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_success() - .build(), - ), - ), - ]) - .build_cardano_database_client(); - - client - .download_unpack_digest_file( - &[ - DigestLocation::CloudStorage { - uri: "http://whatever-1/digests.json".to_string(), - }, - DigestLocation::Aggregator { - uri: "http://whatever-2/digest".to_string(), - }, - ], - target_dir, - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn download_unpack_digest_file_succeeds_when_first_location_is_retrieved() { - let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_digest_file_downloaders(vec![ - ( - DigestLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_success() - .build(), - ), - ), - ( - DigestLocationDiscriminants::Aggregator, - Arc::new(MockFileDownloader::new()), - ), - ]) - .build_cardano_database_client(); - - client - .download_unpack_digest_file( - &[ - DigestLocation::CloudStorage { - uri: "http://whatever-1/digests.json".to_string(), - }, - DigestLocation::Aggregator { - uri: "http://whatever-2/digest".to_string(), - }, - ], - target_dir, - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn download_unpack_digest_file_sends_feedbacks() { - let target_dir = Path::new("."); - let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_digest_file_downloaders(vec![ - ( - DigestLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_success() - .build(), - ), - ), - ]) - .with_feedback_receivers(&[feedback_receiver.clone()]) - .build_cardano_database_client(); - - client - .download_unpack_digest_file( - &[ - DigestLocation::CloudStorage { - uri: "http://whatever-1/digests.json".to_string(), - }, - ], - target_dir, - ) - .await - .unwrap(); - - let sent_events = feedback_receiver.stacked_events(); - let id = sent_events[0].event_id(); - let expected_events = vec![ - MithrilEvent::DigestDownloadStarted { - download_id: id.to_string(), - - }, - MithrilEvent::DigestDownloadCompleted { - download_id: id.to_string(), - }, - ]; - assert_eq!(expected_events, sent_events); - } - } - - mod read_digest_file { - use std::io::Write; - - use mithril_common::test_utils::TempDir; - - use super::*; - - fn create_valid_fake_digest_file( - file_path: &Path, - digest_messages: &[CardanoDatabaseDigestListItemMessage], - ) { - let mut file = fs::File::create(file_path).unwrap(); - let digest_json = serde_json::to_string(&digest_messages).unwrap(); - file.write_all(digest_json.as_bytes()).unwrap(); - } - - fn create_invalid_fake_digest_file(file_path: &Path) { - let mut file = fs::File::create(file_path).unwrap(); - file.write_all(b"incorrect-digest").unwrap(); - } - - #[test] - fn read_digest_file_fails_when_no_digest_file() { - let target_dir = TempDir::new( - "cardano_database_client", - "read_digest_file_fails_when_no_digest_file", - ) - .build(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - client - .read_digest_file(&target_dir) - .expect_err("read_digest_file should fail"); - } - - #[test] - fn read_digest_file_fails_when_multiple_digest_files() { - let target_dir = TempDir::new( - "cardano_database_client", - "read_digest_file_fails_when_multiple_digest_files", - ) - .build(); - create_valid_fake_digest_file(&target_dir.join("digests.json"), &[]); - create_valid_fake_digest_file(&target_dir.join("digests-2.json"), &[]); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - client - .read_digest_file(&target_dir) - .expect_err("read_digest_file should fail"); - } - - #[test] - fn read_digest_file_fails_when_invalid_unique_digest_file() { - let target_dir = TempDir::new( - "cardano_database_client", - "read_digest_file_fails_when_invalid_unique_digest_file", - ) - .build(); - create_invalid_fake_digest_file(&target_dir.join("digests.json")); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - client - .read_digest_file(&target_dir) - .expect_err("read_digest_file should fail"); - } - - #[test] - fn read_digest_file_succeeds_when_valid_unique_digest_file() { - let target_dir = TempDir::new( - "cardano_database_client", - "read_digest_file_succeeds_when_valid_unique_digest_file", - ) - .build(); - let digest_messages = vec![ - CardanoDatabaseDigestListItemMessage { - immutable_file_name: "00001.chunk".to_string(), - digest: "digest-1".to_string(), - }, - CardanoDatabaseDigestListItemMessage { - immutable_file_name: "00002.chunk".to_string(), - digest: "digest-2".to_string(), - }, - ]; - create_valid_fake_digest_file(&target_dir.join("digests.json"), &digest_messages); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - let digests = client.read_digest_file(&target_dir).unwrap(); - assert_eq!( - BTreeMap::from([ - ("00001.chunk".to_string(), "digest-1".to_string()), - ("00002.chunk".to_string(), "digest-2".to_string()) - ]), - digests - ) - } - } - } - - mod compute_merkle_proof { - use mithril_common::{ - digesters::{DummyCardanoDbBuilder, ImmutableDigester, ImmutableFile}, - messages::SignedEntityTypeMessagePart, - }; - - use crate::test_utils::test_logger; - - use super::*; - - async fn write_digest_file( - digest_dir: &Path, - digests: BTreeMap, - ) { - let digest_file_path = digest_dir.join("digests.json"); - if !digest_dir.exists() { - fs::create_dir_all(digest_dir).unwrap(); - } - - let immutable_digest_messages = digests - .into_iter() - .map( - |(immutable_file, digest)| CardanoDatabaseDigestListItemMessage { - immutable_file_name: immutable_file.filename, - digest, - }, - ) - .collect::>(); - serde_json::to_writer( - fs::File::create(digest_file_path).unwrap(), - &immutable_digest_messages, - ) - .unwrap(); - } - - #[tokio::test] - async fn compute_merkle_proof_fails_if_mismatching_certificate() { - let immutable_file_range = 1..=5; - let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); - let certificate = CertificateMessage { - hash: "cert-hash-123".to_string(), - signed_entity_type: SignedEntityTypeMessagePart::MithrilStakeDistribution( - Epoch(123), - ), - ..CertificateMessage::dummy() - }; - let cardano_db = DummyCardanoDbBuilder::new( - "compute_merkle_proof_fails_if_mismatching_certificate", - ) - .with_immutables(&immutable_file_range.clone().collect::>()) - .append_immutable_trio() - .build(); - let database_dir = cardano_db.get_dir(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - client - .compute_merkle_proof( - &certificate, - &immutable_file_range_to_prove, - database_dir, - ) - .await - .expect_err("compute_merkle_proof should fail"); - } - - #[tokio::test] - async fn compute_merkle_proof_succeeds() { - let beacon = CardanoDbBeacon { - epoch: Epoch(123), - immutable_file_number: 5, - }; - let immutable_file_range = 1..=5; - let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); - let certificate = CertificateMessage { - hash: "cert-hash-123".to_string(), - signed_entity_type: SignedEntityTypeMessagePart::CardanoDatabase( - beacon.clone(), - ), - ..CertificateMessage::dummy() - }; - let cardano_db = DummyCardanoDbBuilder::new("compute_merkle_proof_succeeds") - .with_immutables(&immutable_file_range.clone().collect::>()) - .append_immutable_trio() - .build(); - let database_dir = cardano_db.get_dir(); - let immutable_digester = CardanoImmutableDigester::new( - certificate.metadata.network.clone(), - None, - test_logger(), - ); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - let computed_digests = immutable_digester - .compute_digests_for_range(database_dir, &immutable_file_range) - .await - .unwrap(); - write_digest_file(&database_dir.join("digest"), computed_digests.entries).await; - let merkle_tree = immutable_digester - .compute_merkle_tree(database_dir, &beacon) - .await - .unwrap(); - let expected_merkle_root = merkle_tree.compute_root().unwrap(); - - let merkle_proof = client - .compute_merkle_proof( - &certificate, - &immutable_file_range_to_prove, - database_dir, - ) - .await - .unwrap(); - let merkle_proof_root = merkle_proof.root().to_owned(); - - merkle_proof.verify().unwrap(); - assert_eq!(expected_merkle_root, merkle_proof_root); - - assert!(!database_dir.join("digest").exists()); - } - } - - mod immutable_file_range { - use super::*; - - #[test] - fn to_range_inclusive_with_full() { - let immutable_file_range = ImmutableFileRange::Full; - let last_immutable_file_number = 10; - - let result = immutable_file_range - .to_range_inclusive(last_immutable_file_number) - .unwrap(); - assert_eq!(0..=10, result); - } - - #[test] - fn to_range_inclusive_with_from() { - let immutable_file_range = ImmutableFileRange::From(5); - - let last_immutable_file_number = 10; - let result = immutable_file_range - .to_range_inclusive(last_immutable_file_number) - .unwrap(); - assert_eq!(5..=10, result); - - let last_immutable_file_number = 3; - immutable_file_range - .to_range_inclusive(last_immutable_file_number) - .expect_err("conversion to range inlusive should fail"); - } - - #[test] - fn to_range_inclusive_with_range() { - let immutable_file_range = ImmutableFileRange::Range(5, 8); - - let last_immutable_file_number = 10; - let result = immutable_file_range - .to_range_inclusive(last_immutable_file_number) - .unwrap(); - assert_eq!(5..=8, result); - - let last_immutable_file_number = 7; - immutable_file_range - .to_range_inclusive(last_immutable_file_number) - .expect_err("conversion to range inlusive should fail"); - - let immutable_file_range = ImmutableFileRange::Range(10, 8); - immutable_file_range - .to_range_inclusive(last_immutable_file_number) - .expect_err("conversion to range inlusive should fail"); - } - - #[test] - fn to_range_inclusive_with_up_to() { - let immutable_file_range = ImmutableFileRange::UpTo(8); - - let last_immutable_file_number = 10; - let result = immutable_file_range - .to_range_inclusive(last_immutable_file_number) - .unwrap(); - assert_eq!(0..=8, result); - - let last_immutable_file_number = 7; - immutable_file_range - .to_range_inclusive(last_immutable_file_number) - .expect_err("conversion to range inlusive should fail"); - } - } - } -} diff --git a/mithril-client/src/cardano_database_client/api.rs b/mithril-client/src/cardano_database_client/api.rs new file mode 100644 index 00000000000..77da0b6f8a5 --- /dev/null +++ b/mithril-client/src/cardano_database_client/api.rs @@ -0,0 +1,174 @@ +use std::sync::Arc; + +#[cfg(feature = "fs")] +use slog::Logger; + +#[cfg(feature = "fs")] +use mithril_common::entities::{AncillaryLocation, DigestLocation, ImmutablesLocation}; + +use crate::aggregator_client::AggregatorClient; +#[cfg(feature = "fs")] +use crate::feedback::FeedbackSender; +#[cfg(feature = "fs")] +use crate::file_downloader::FileDownloaderResolver; + +/// HTTP client for CardanoDatabase API from the Aggregator +pub struct CardanoDatabaseClient { + pub(super) aggregator_client: Arc, + #[cfg(feature = "fs")] + pub(super) immutable_file_downloader_resolver: + Arc>, + #[cfg(feature = "fs")] + pub(super) ancillary_file_downloader_resolver: + Arc>, + #[cfg(feature = "fs")] + pub(super) digest_file_downloader_resolver: Arc>, + #[cfg(feature = "fs")] + pub(super) feedback_sender: FeedbackSender, + #[cfg(feature = "fs")] + pub(super) logger: Logger, +} + +impl CardanoDatabaseClient { + /// Constructs a new `CardanoDatabase`. + pub fn new( + aggregator_client: Arc, + #[cfg(feature = "fs")] immutable_file_downloader_resolver: Arc< + dyn FileDownloaderResolver, + >, + #[cfg(feature = "fs")] ancillary_file_downloader_resolver: Arc< + dyn FileDownloaderResolver, + >, + #[cfg(feature = "fs")] digest_file_downloader_resolver: Arc< + dyn FileDownloaderResolver, + >, + #[cfg(feature = "fs")] feedback_sender: FeedbackSender, + #[cfg(feature = "fs")] logger: Logger, + ) -> Self { + Self { + aggregator_client, + #[cfg(feature = "fs")] + immutable_file_downloader_resolver, + #[cfg(feature = "fs")] + ancillary_file_downloader_resolver, + #[cfg(feature = "fs")] + digest_file_downloader_resolver, + #[cfg(feature = "fs")] + feedback_sender, + #[cfg(feature = "fs")] + logger: mithril_common::logging::LoggerExtensions::new_with_component_name::( + &logger, + ), + } + } +} + +#[cfg(test)] +pub(crate) mod test_dependency_injector { + use super::*; + + use mithril_common::entities::{ + AncillaryLocationDiscriminants, DigestLocationDiscriminants, + ImmutablesLocationDiscriminants, + }; + + use crate::{ + aggregator_client::MockAggregatorHTTPClient, + feedback::FeedbackReceiver, + file_downloader::{ + AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloader, + ImmutablesFileDownloaderResolver, + }, + test_utils, + }; + + /// Dependency injector for `CardanoDatabaseClient` for testing purposes. + pub(crate) struct CardanoDatabaseClientDependencyInjector { + http_client: MockAggregatorHTTPClient, + immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver, + ancillary_file_downloader_resolver: AncillaryFileDownloaderResolver, + digest_file_downloader_resolver: DigestFileDownloaderResolver, + feedback_receivers: Vec>, + } + + impl CardanoDatabaseClientDependencyInjector { + pub(crate) fn new() -> Self { + Self { + http_client: MockAggregatorHTTPClient::new(), + immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver::new(vec![]), + ancillary_file_downloader_resolver: AncillaryFileDownloaderResolver::new(vec![]), + digest_file_downloader_resolver: DigestFileDownloaderResolver::new(vec![]), + feedback_receivers: vec![], + } + } + + pub(crate) fn with_http_client_mock_config(mut self, config: F) -> Self + where + F: FnOnce(&mut MockAggregatorHTTPClient), + { + config(&mut self.http_client); + + self + } + + pub(crate) fn with_immutable_file_downloaders( + self, + file_downloaders: Vec<(ImmutablesLocationDiscriminants, Arc)>, + ) -> Self { + let immutable_file_downloader_resolver = + ImmutablesFileDownloaderResolver::new(file_downloaders); + + Self { + immutable_file_downloader_resolver, + ..self + } + } + + pub(crate) fn with_ancillary_file_downloaders( + self, + file_downloaders: Vec<(AncillaryLocationDiscriminants, Arc)>, + ) -> Self { + let ancillary_file_downloader_resolver = + AncillaryFileDownloaderResolver::new(file_downloaders); + + Self { + ancillary_file_downloader_resolver, + ..self + } + } + + pub(crate) fn with_digest_file_downloaders( + self, + file_downloaders: Vec<(DigestLocationDiscriminants, Arc)>, + ) -> Self { + let digest_file_downloader_resolver = + DigestFileDownloaderResolver::new(file_downloaders); + + Self { + digest_file_downloader_resolver, + ..self + } + } + + pub(crate) fn with_feedback_receivers( + self, + feedback_receivers: &[Arc], + ) -> Self { + Self { + feedback_receivers: feedback_receivers.to_vec(), + ..self + } + } + + pub(crate) fn build_cardano_database_client(self) -> CardanoDatabaseClient { + CardanoDatabaseClient::new( + Arc::new(self.http_client), + Arc::new(self.immutable_file_downloader_resolver), + Arc::new(self.ancillary_file_downloader_resolver), + Arc::new(self.digest_file_downloader_resolver), + FeedbackSender::new(&self.feedback_receivers), + test_utils::test_logger(), + ) + } + } +} diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs new file mode 100644 index 00000000000..6479f97982b --- /dev/null +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -0,0 +1,961 @@ +use std::collections::BTreeSet; +use std::ops::RangeInclusive; +use std::path::{Path, PathBuf}; +use tokio::task::JoinSet; + +use anyhow::anyhow; + +use mithril_common::{ + entities::{AncillaryLocation, CompressionAlgorithm, ImmutableFileNumber, ImmutablesLocation}, + messages::CardanoDatabaseSnapshotMessage, + StdResult, +}; + +use crate::feedback::MithrilEvent; +use crate::file_downloader::FileDownloaderUri; + +use super::api::CardanoDatabaseClient; +use super::immutable_file_range::ImmutableFileRange; + +/// Options for downloading and unpacking a Cardano database +#[derive(Debug, Default)] +pub struct DownloadUnpackOptions { + /// Allow overriding the destination directory + pub allow_override: bool, + + /// Include ancillary files in the download + pub include_ancillary: bool, +} + +impl CardanoDatabaseClient { + /// Download and unpack the given Cardano database parts data by hash. + pub async fn download_unpack( + &self, + cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, + immutable_file_range: &ImmutableFileRange, + target_dir: &Path, + download_unpack_options: DownloadUnpackOptions, + ) -> StdResult<()> { + let compression_algorithm = cardano_database_snapshot.compression_algorithm; + let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; + let immutable_file_number_range = + immutable_file_range.to_range_inclusive(last_immutable_file_number)?; + + self.verify_can_write_to_target_directory(target_dir, &download_unpack_options)?; + + let immutable_locations = &cardano_database_snapshot.locations.immutables; + self.download_unpack_immutable_files( + immutable_locations, + immutable_file_number_range, + &compression_algorithm, + target_dir, + ) + .await?; + + if download_unpack_options.include_ancillary { + let ancillary_locations = &cardano_database_snapshot.locations.ancillary; + self.download_unpack_ancillary_file( + ancillary_locations, + &compression_algorithm, + target_dir, + ) + .await?; + } + + Ok(()) + } + + fn immutable_files_target_dir(target_dir: &Path) -> PathBuf { + target_dir.join("immutable") + } + + fn volatile_target_dir(target_dir: &Path) -> PathBuf { + target_dir.join("volatile") + } + + fn ledger_target_dir(target_dir: &Path) -> PathBuf { + target_dir.join("ledger") + } + + /// Verify if the target directory is writable. + fn verify_can_write_to_target_directory( + &self, + target_dir: &Path, + download_unpack_options: &DownloadUnpackOptions, + ) -> StdResult<()> { + let immutable_files_target_dir = Self::immutable_files_target_dir(target_dir); + let volatile_target_dir = Self::volatile_target_dir(target_dir); + let ledger_target_dir = Self::ledger_target_dir(target_dir); + if !download_unpack_options.allow_override { + if immutable_files_target_dir.exists() { + return Err(anyhow!( + "Immutable files target directory already exists in: {target_dir:?}" + )); + } + if download_unpack_options.include_ancillary { + if volatile_target_dir.exists() { + return Err(anyhow!( + "Volatile target directory already exists in: {target_dir:?}" + )); + } + if ledger_target_dir.exists() { + return Err(anyhow!( + "Ledger target directory already exists in: {target_dir:?}" + )); + } + } + } + + Ok(()) + } + + fn feedback_event_builder_immutable_download( + download_id: String, + downloaded_bytes: u64, + size: u64, + ) -> Option { + Some(MithrilEvent::ImmutableDownloadProgress { + download_id, + downloaded_bytes, + size, + }) + } + + fn feedback_event_builder_ancillary_download( + download_id: String, + downloaded_bytes: u64, + size: u64, + ) -> Option { + Some(MithrilEvent::AncillaryDownloadProgress { + download_id, + downloaded_bytes, + size, + }) + } + + /// Download and unpack the immutable files of the given range. + /// + /// The download is attempted for each location until the full range is downloaded. + /// An error is returned if not all the files are downloaded. + async fn download_unpack_immutable_files( + &self, + locations: &[ImmutablesLocation], + range: RangeInclusive, + compression_algorithm: &CompressionAlgorithm, + immutable_files_target_dir: &Path, + ) -> StdResult<()> { + let mut locations_sorted = locations.to_owned(); + locations_sorted.sort(); + let mut immutable_file_numbers_to_download = + range.clone().map(|n| n.to_owned()).collect::>(); + for location in locations_sorted { + let immutable_files_numbers_downloaded = self + .download_unpack_immutable_files_for_location( + &location, + &immutable_file_numbers_to_download, + compression_algorithm, + immutable_files_target_dir, + ) + .await?; + for immutable_file_number in immutable_files_numbers_downloaded { + immutable_file_numbers_to_download.remove(&immutable_file_number); + } + if immutable_file_numbers_to_download.is_empty() { + return Ok(()); + } + } + + Err(anyhow!( + "Failed downloading and unpacking immutable files for immutable_file_numbers: {immutable_file_numbers_to_download:?}" + )) + } + + async fn download_unpack_immutable_files_for_location( + &self, + location: &ImmutablesLocation, + immutable_file_numbers_to_download: &BTreeSet, + compression_algorithm: &CompressionAlgorithm, + immutable_files_target_dir: &Path, + ) -> StdResult> { + let mut immutable_file_numbers_downloaded = BTreeSet::new(); + let file_downloader = self + .immutable_file_downloader_resolver + .resolve(location) + .ok_or_else(|| { + anyhow!("Failed resolving a file downloader for location: {location:?}") + })?; + let file_downloader_uris = + FileDownloaderUri::expand_immutable_files_location_to_file_downloader_uris( + location, + immutable_file_numbers_to_download + .clone() + .into_iter() + .collect::>() + .as_slice(), + )?; + let mut join_set: JoinSet> = JoinSet::new(); + for (immutable_file_number, file_downloader_uri) in file_downloader_uris { + let compression_algorithm_clone = compression_algorithm.to_owned(); + let immutable_files_target_dir_clone = immutable_files_target_dir.to_owned(); + let file_downloader_clone = file_downloader.clone(); + let feedback_receiver_clone = self.feedback_sender.clone(); + let logger_clone = self.logger.clone(); + join_set.spawn(async move { + let download_id = MithrilEvent::new_snapshot_download_id(); + feedback_receiver_clone + .send_event(MithrilEvent::ImmutableDownloadStarted { immutable_file_number, download_id: download_id.clone()}) + .await; + let downloaded = file_downloader_clone + .download_unpack( + &file_downloader_uri, + &immutable_files_target_dir_clone, + Some(compression_algorithm_clone), + &download_id, + Self::feedback_event_builder_immutable_download, + ) + .await; + match downloaded { + Ok(_) => { + feedback_receiver_clone + .send_event(MithrilEvent::ImmutableDownloadCompleted { download_id }) + .await; + + Ok(immutable_file_number) + } + Err(e) => { + slog::error!( + logger_clone, + "Failed downloading and unpacking immutable file {immutable_file_number} for location {file_downloader_uri:?}"; "error" => e.to_string() + ); + Err(e.context(format!("Failed downloading and unpacking immutable file {immutable_file_number} for location {file_downloader_uri:?}"))) + } + } + }); + } + while let Some(result) = join_set.join_next().await { + match result? { + Ok(immutable_file_number) => { + immutable_file_numbers_downloaded.insert(immutable_file_number); + } + Err(e) => { + slog::error!( + self.logger, + "Failed downloading and unpacking immutable files"; "error" => e.to_string() + ); + } + } + } + + Ok(immutable_file_numbers_downloaded) + } + + /// Download and unpack the ancillary files. + async fn download_unpack_ancillary_file( + &self, + locations: &[AncillaryLocation], + compression_algorithm: &CompressionAlgorithm, + ancillary_file_target_dir: &Path, + ) -> StdResult<()> { + let mut locations_sorted = locations.to_owned(); + locations_sorted.sort(); + for location in locations_sorted { + let download_id = MithrilEvent::new_ancillary_download_id(); + self.feedback_sender + .send_event(MithrilEvent::AncillaryDownloadStarted { + download_id: download_id.clone(), + }) + .await; + let file_downloader = self + .ancillary_file_downloader_resolver + .resolve(&location) + .ok_or_else(|| { + anyhow!("Failed resolving a file downloader for location: {location:?}") + })?; + let file_downloader_uri: FileDownloaderUri = location.into(); + let downloaded = file_downloader + .download_unpack( + &file_downloader_uri, + ancillary_file_target_dir, + Some(compression_algorithm.to_owned()), + &download_id, + Self::feedback_event_builder_ancillary_download, + ) + .await; + match downloaded { + Ok(_) => { + self.feedback_sender + .send_event(MithrilEvent::AncillaryDownloadCompleted { download_id }) + .await; + return Ok(()); + } + Err(e) => { + slog::error!( + self.logger, + "Failed downloading and unpacking ancillaries for location {file_downloader_uri:?}"; "error" => e.to_string() + ); + } + } + } + + Err(anyhow!( + "Failed downloading and unpacking ancillaries for all locations" + )) + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + use std::{fs, sync::Arc}; + + use mithril_common::{ + entities::{ + AncillaryLocationDiscriminants, ImmutablesLocationDiscriminants, MultiFilesUri, + TemplateUri, + }, + messages::{ + ArtifactsLocationsMessagePart, + CardanoDatabaseSnapshotMessage as CardanoDatabaseSnapshot, + }, + test_utils::TempDir, + }; + + use crate::cardano_database_client::CardanoDatabaseClientDependencyInjector; + use crate::feedback::StackFeedbackReceiver; + use crate::file_downloader::MockFileDownloaderBuilder; + + use super::*; + + mod download_unpack { + + use super::*; + + #[tokio::test] + async fn download_unpack_fails_with_invalid_immutable_file_range() { + let immutable_file_range = ImmutableFileRange::Range(1, 0); + let download_unpack_options = DownloadUnpackOptions::default(); + let cardano_db_snapshot = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshot::dummy() + }; + let target_dir = Path::new("."); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .download_unpack( + &cardano_db_snapshot, + &immutable_file_range, + target_dir, + download_unpack_options, + ) + .await + .expect_err("download_unpack should fail"); + } + + #[tokio::test] + async fn download_unpack_fails_when_immutable_files_download_fail() { + let total_immutable_files = 10; + let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); + let download_unpack_options = DownloadUnpackOptions::default(); + let cardano_db_snapshot = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + locations: ArtifactsLocationsMessagePart { + immutables: vec![ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }], + ..ArtifactsLocationsMessagePart::default() + }, + ..CardanoDatabaseSnapshot::dummy() + }; + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_fails_when_immutable_files_download_fail", + ) + .build(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + MockFileDownloaderBuilder::default() + .with_times(total_immutable_files as usize) + .with_failure() + .build() + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack( + &cardano_db_snapshot, + &immutable_file_range, + &target_dir, + download_unpack_options, + ) + .await + .expect_err("download_unpack should fail"); + } + + #[tokio::test] + async fn download_unpack_fails_when_target_target_dir_would_be_overwritten_without_allow_override( + ) { + let immutable_file_range = ImmutableFileRange::Range(1, 10); + let download_unpack_options = DownloadUnpackOptions::default(); + let cardano_db_snapshot = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshot::dummy() + }; + let target_dir = &TempDir::new( + "cardano_database_client", + "download_unpack_fails_when_target_target_dir_would_be_overwritten_without_allow_override", + ) + .build(); + fs::create_dir_all(target_dir.join("immutable")).unwrap(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .download_unpack( + &cardano_db_snapshot, + &immutable_file_range, + target_dir, + download_unpack_options, + ) + .await + .expect_err("download_unpack should fail"); + } + + #[tokio::test] + async fn download_unpack_succeeds_with_valid_range() { + let immutable_file_range = ImmutableFileRange::Range(1, 2); + let download_unpack_options = DownloadUnpackOptions { + include_ancillary: true, + ..DownloadUnpackOptions::default() + }; + let cardano_db_snapshot = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + locations: ArtifactsLocationsMessagePart { + immutables: vec![ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }], + ancillary: vec![AncillaryLocation::CloudStorage { + uri: "http://whatever/ancillary.tar.gz".to_string(), + }], + digests: vec![], + }, + ..CardanoDatabaseSnapshot::dummy() + }; + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_succeeds_with_valid_range", + ) + .build(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/00001.tar.gz") + .with_target_dir(target_dir.clone()) + .with_success() + .build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever/00002.tar.gz") + .with_target_dir(target_dir.clone()) + .with_success() + .build() + }), + )]) + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new( + MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/ancillary.tar.gz") + .with_target_dir(target_dir.clone()) + .with_compression(Some(CompressionAlgorithm::default())) + .with_success() + .build(), + ), + )]) + .build_cardano_database_client(); + + client + .download_unpack( + &cardano_db_snapshot, + &immutable_file_range, + &target_dir, + download_unpack_options, + ) + .await + .unwrap(); + } + } + + mod verify_can_write_to_target_dir { + + use super::*; + + #[test] + fn verify_can_write_to_target_dir_always_succeeds_with_allow_overwrite() { + let target_dir = TempDir::new( + "cardano_database_client", + "verify_can_write_to_target_dir_always_succeeds_with_allow_overwrite", + ) + .build(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: true, + include_ancillary: false, + }, + ) + .unwrap(); + + fs::create_dir_all(CardanoDatabaseClient::immutable_files_target_dir( + &target_dir, + )) + .unwrap(); + fs::create_dir_all(CardanoDatabaseClient::volatile_target_dir(&target_dir)).unwrap(); + fs::create_dir_all(CardanoDatabaseClient::ledger_target_dir(&target_dir)).unwrap(); + client + .verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: true, + include_ancillary: false, + }, + ) + .unwrap(); + client + .verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: true, + include_ancillary: true, + }, + ) + .unwrap(); + } + + #[test] + fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_immutable_target_dir( + ) { + let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_immutable_target_dir").build(); + fs::create_dir_all(CardanoDatabaseClient::immutable_files_target_dir( + &target_dir, + )) + .unwrap(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: false, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); + + client + .verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: true, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); + } + + #[test] + fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_ledger_target_dir( + ) { + let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_ledger_target_dir").build(); + fs::create_dir_all(CardanoDatabaseClient::ledger_target_dir(&target_dir)).unwrap(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: true, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); + + client + .verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: false, + }, + ) + .unwrap(); + } + + #[test] + fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_volatile_target_dir( + ) { + let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_volatile_target_dir").build(); + fs::create_dir_all(CardanoDatabaseClient::volatile_target_dir(&target_dir)).unwrap(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: true, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); + + client + .verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: false, + }, + ) + .unwrap(); + } + } + + mod download_unpack_immutable_files { + + use super::*; + + #[tokio::test] + async fn download_unpack_immutable_files_fails_if_one_is_not_retrieved() { + let total_immutable_files = 2; + let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_immutable_files_succeeds", + ) + .build(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + let mock_file_downloader = + MockFileDownloaderBuilder::default().with_failure().build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_success() + .build() + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack_immutable_files( + &[ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }], + immutable_file_range + .to_range_inclusive(total_immutable_files) + .unwrap(), + &CompressionAlgorithm::default(), + &target_dir, + ) + .await + .expect_err("download_unpack_immutable_files should fail"); + } + + #[tokio::test] + async fn download_unpack_immutable_files_succeeds_if_all_are_retrieved_with_same_location() + { + let total_immutable_files = 2; + let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_immutable_files_succeeds", + ) + .build(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new( + MockFileDownloaderBuilder::default() + .with_times(2) + .with_success() + .build(), + ), + )]) + .build_cardano_database_client(); + + client + .download_unpack_immutable_files( + &[ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever-1/{immutable_file_number}.tar.gz".to_string(), + )), + }], + immutable_file_range + .to_range_inclusive(total_immutable_files) + .unwrap(), + &CompressionAlgorithm::default(), + &target_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn download_unpack_immutable_files_succeeds_if_all_are_retrieved_with_different_locations( + ) { + let total_immutable_files = 2; + let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); + let target_dir = TempDir::new( + "cardano_database_client", + "download_unpack_immutable_files_succeeds", + ) + .build(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new({ + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever-1/00001.tar.gz") + .with_target_dir(target_dir.clone()) + .with_failure() + .build(); + let mock_file_downloader = + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever-1/00002.tar.gz") + .with_target_dir(target_dir.clone()) + .with_success() + .build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever-2/00001.tar.gz") + .with_target_dir(target_dir.clone()) + .with_success() + .build() + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack_immutable_files( + &[ + ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever-1/{immutable_file_number}.tar.gz".to_string(), + )), + }, + ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever-2/{immutable_file_number}.tar.gz".to_string(), + )), + }, + ], + immutable_file_range + .to_range_inclusive(total_immutable_files) + .unwrap(), + &CompressionAlgorithm::default(), + &target_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn download_unpack_immutable_files_sends_feedbacks() { + let total_immutable_files = 1; + let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); + let target_dir = Path::new("."); + let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_immutable_file_downloaders(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + Arc::new(MockFileDownloaderBuilder::default().with_success().build()), + )]) + .with_feedback_receivers(&[feedback_receiver.clone()]) + .build_cardano_database_client(); + + client + .download_unpack_immutable_files( + &[ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }], + immutable_file_range + .to_range_inclusive(total_immutable_files) + .unwrap(), + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .unwrap(); + + let sent_events = feedback_receiver.stacked_events(); + let id = sent_events[0].event_id(); + let expected_events = vec![ + MithrilEvent::ImmutableDownloadStarted { + immutable_file_number: 1, + download_id: id.to_string(), + }, + MithrilEvent::ImmutableDownloadCompleted { + download_id: id.to_string(), + }, + ]; + assert_eq!(expected_events, sent_events); + } + } + + mod download_unpack_ancillary_file { + + use super::*; + + #[tokio::test] + async fn download_unpack_ancillary_file_fails_if_no_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new(MockFileDownloaderBuilder::default().with_failure().build()), + )]) + .build_cardano_database_client(); + + client + .download_unpack_ancillary_file( + &[AncillaryLocation::CloudStorage { + uri: "http://whatever-1/ancillary.tar.gz".to_string(), + }], + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .expect_err("download_unpack_ancillary_file should fail"); + } + + #[tokio::test] + async fn download_unpack_ancillary_file_succeeds_if_at_least_one_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new({ + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever-1/ancillary.tar.gz") + .with_target_dir(target_dir.to_path_buf()) + .with_failure() + .build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever-2/ancillary.tar.gz") + .with_target_dir(target_dir.to_path_buf()) + .with_success() + .build() + }), + )]) + .build_cardano_database_client(); + + client + .download_unpack_ancillary_file( + &[ + AncillaryLocation::CloudStorage { + uri: "http://whatever-1/ancillary.tar.gz".to_string(), + }, + AncillaryLocation::CloudStorage { + uri: "http://whatever-2/ancillary.tar.gz".to_string(), + }, + ], + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn download_unpack_ancillary_file_succeeds_when_first_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new( + MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever-1/ancillary.tar.gz") + .with_target_dir(target_dir.to_path_buf()) + .with_success() + .build(), + ), + )]) + .build_cardano_database_client(); + + client + .download_unpack_ancillary_file( + &[ + AncillaryLocation::CloudStorage { + uri: "http://whatever-1/ancillary.tar.gz".to_string(), + }, + AncillaryLocation::CloudStorage { + uri: "http://whatever-2/ancillary.tar.gz".to_string(), + }, + ], + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn download_unpack_ancillary_files_sends_feedbacks() { + let target_dir = Path::new("."); + let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_ancillary_file_downloaders(vec![( + AncillaryLocationDiscriminants::CloudStorage, + Arc::new(MockFileDownloaderBuilder::default().with_success().build()), + )]) + .with_feedback_receivers(&[feedback_receiver.clone()]) + .build_cardano_database_client(); + + client + .download_unpack_ancillary_file( + &[AncillaryLocation::CloudStorage { + uri: "http://whatever-1/ancillary.tar.gz".to_string(), + }], + &CompressionAlgorithm::default(), + target_dir, + ) + .await + .unwrap(); + + let sent_events = feedback_receiver.stacked_events(); + let id = sent_events[0].event_id(); + let expected_events = vec![ + MithrilEvent::AncillaryDownloadStarted { + download_id: id.to_string(), + }, + MithrilEvent::AncillaryDownloadCompleted { + download_id: id.to_string(), + }, + ]; + assert_eq!(expected_events, sent_events); + } + } +} diff --git a/mithril-client/src/cardano_database_client/fetch.rs b/mithril-client/src/cardano_database_client/fetch.rs new file mode 100644 index 00000000000..9084192bcac --- /dev/null +++ b/mithril-client/src/cardano_database_client/fetch.rs @@ -0,0 +1,222 @@ +use anyhow::Context; +use serde::de::DeserializeOwned; + +use crate::{ + aggregator_client::{AggregatorClientError, AggregatorRequest}, + CardanoDatabaseSnapshot, CardanoDatabaseSnapshotListItem, MithrilResult, +}; + +use super::api::CardanoDatabaseClient; + +impl CardanoDatabaseClient { + /// Fetch a list of signed CardanoDatabase + pub async fn list(&self) -> MithrilResult> { + let response = self + .aggregator_client + .get_content(AggregatorRequest::ListCardanoDatabaseSnapshots) + .await + .with_context(|| "CardanoDatabase client can not get the artifact list")?; + let items = serde_json::from_str::>(&response) + .with_context(|| "CardanoDatabase client can not deserialize artifact list")?; + + Ok(items) + } + + /// Get the given Cardano database data by hash. + pub async fn get(&self, hash: &str) -> MithrilResult> { + self.fetch_with_aggregator_request(AggregatorRequest::GetCardanoDatabaseSnapshot { + hash: hash.to_string(), + }) + .await + } + + /// Fetch the given Cardano database data with an aggregator request. + /// If it cannot be found, a None is returned. + async fn fetch_with_aggregator_request( + &self, + request: AggregatorRequest, + ) -> MithrilResult> { + match self.aggregator_client.get_content(request).await { + Ok(content) => { + let result = serde_json::from_str(&content) + .with_context(|| "CardanoDatabase client can not deserialize artifact")?; + + Ok(Some(result)) + } + Err(AggregatorClientError::RemoteServerLogical(_)) => Ok(None), + Err(e) => Err(e.into()), + } + } +} + +#[cfg(test)] +mod tests { + + use anyhow::anyhow; + use chrono::{DateTime, Utc}; + use mockall::predicate::eq; + + use mithril_common::entities::{CardanoDbBeacon, CompressionAlgorithm, Epoch}; + + use crate::cardano_database_client::CardanoDatabaseClientDependencyInjector; + + use super::*; + + fn fake_messages() -> Vec { + vec![ + CardanoDatabaseSnapshotListItem { + hash: "hash-123".to_string(), + merkle_root: "mkroot-123".to_string(), + beacon: CardanoDbBeacon { + epoch: Epoch(1), + immutable_file_number: 123, + }, + certificate_hash: "cert-hash-123".to_string(), + total_db_size_uncompressed: 800796318, + created_at: DateTime::parse_from_rfc3339("2025-01-19T13:43:05.618857482Z") + .unwrap() + .with_timezone(&Utc), + compression_algorithm: CompressionAlgorithm::default(), + cardano_node_version: "0.0.1".to_string(), + }, + CardanoDatabaseSnapshotListItem { + hash: "hash-456".to_string(), + merkle_root: "mkroot-456".to_string(), + beacon: CardanoDbBeacon { + epoch: Epoch(2), + immutable_file_number: 456, + }, + certificate_hash: "cert-hash-456".to_string(), + total_db_size_uncompressed: 2960713808, + created_at: DateTime::parse_from_rfc3339("2025-01-27T15:22:05.618857482Z") + .unwrap() + .with_timezone(&Utc), + compression_algorithm: CompressionAlgorithm::default(), + cardano_node_version: "0.0.1".to_string(), + }, + ] + } + + mod list { + + use super::*; + + #[tokio::test] + async fn list_cardano_database_snapshots_returns_messages() { + let message = fake_messages(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client + .expect_get_content() + .with(eq(AggregatorRequest::ListCardanoDatabaseSnapshots)) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + }) + .build_cardano_database_client(); + + let messages = client.list().await.unwrap(); + + assert_eq!(2, messages.len()); + assert_eq!("hash-123".to_string(), messages[0].hash); + assert_eq!("hash-456".to_string(), messages[1].hash); + } + + #[tokio::test] + async fn list_cardano_database_snapshots_returns_error_when_invalid_json_structure_in_response( + ) { + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client + .expect_get_content() + .return_once(move |_| Ok("invalid json structure".to_string())); + }) + .build_cardano_database_client(); + + client + .list() + .await + .expect_err("List Cardano databases should return an error"); + } + } + + mod get { + use super::*; + + #[tokio::test] + async fn get_cardano_database_snapshot_returns_message() { + let expected_cardano_database_snapshot = CardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshot::dummy() + }; + let message = expected_cardano_database_snapshot.clone(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client + .expect_get_content() + .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { + hash: "hash-123".to_string(), + })) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + }) + .build_cardano_database_client(); + + let cardano_database = client + .get("hash-123") + .await + .unwrap() + .expect("This test returns a Cardano database"); + + assert_eq!(expected_cardano_database_snapshot, cardano_database); + } + + #[tokio::test] + async fn get_cardano_database_snapshot_returns_error_when_invalid_json_structure_in_response( + ) { + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client + .expect_get_content() + .return_once(move |_| Ok("invalid json structure".to_string())); + }) + .build_cardano_database_client(); + + client + .get("hash-123") + .await + .expect_err("Get Cardano database should return an error"); + } + + #[tokio::test] + async fn get_cardano_database_snapshot_returns_none_when_not_found_or_remote_server_logical_error( + ) { + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client.expect_get_content().return_once(move |_| { + Err(AggregatorClientError::RemoteServerLogical(anyhow!( + "not found" + ))) + }); + }) + .build_cardano_database_client(); + + let result = client.get("hash-123").await.unwrap(); + + assert!(result.is_none()); + } + + #[tokio::test] + async fn get_cardano_database_snapshot_returns_error() { + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_client_mock_config(|http_client| { + http_client.expect_get_content().return_once(move |_| { + Err(AggregatorClientError::SubsystemError(anyhow!("error"))) + }); + }) + .build_cardano_database_client(); + + client + .get("hash-123") + .await + .expect_err("Get Cardano database should return an error"); + } + } +} diff --git a/mithril-client/src/cardano_database_client/immutable_file_range.rs b/mithril-client/src/cardano_database_client/immutable_file_range.rs new file mode 100644 index 00000000000..8bcb7260401 --- /dev/null +++ b/mithril-client/src/cardano_database_client/immutable_file_range.rs @@ -0,0 +1,122 @@ +use std::ops::RangeInclusive; + +use anyhow::anyhow; + +use mithril_common::{entities::ImmutableFileNumber, StdResult}; + +/// Immutable file range representation +#[derive(Debug)] +pub enum ImmutableFileRange { + /// From the first (included) to the last immutable file number (included) + Full, + + /// From a specific immutable file number (included) to the last immutable file number (included) + From(ImmutableFileNumber), + + /// From a specific immutable file number (included) to another specific immutable file number (included) + Range(ImmutableFileNumber, ImmutableFileNumber), + + /// From the first immutable file number (included) up to a specific immutable file number (included) + UpTo(ImmutableFileNumber), +} + +impl ImmutableFileRange { + /// Returns the range of immutable file numbers + pub fn to_range_inclusive( + &self, + last_immutable_file_number: ImmutableFileNumber, + ) -> StdResult> { + // The immutable file numbers start from 1 on all the networks except the 'devnet' + // when it is configured with aggressive protocol parameters for fast epochs (used in the e2e tests). + // We have taken the choice to consider that the file numbers start from 1 for all the networks. + const FIRST_IMMUTABLE_FILE_NUMBER: ImmutableFileNumber = 1; + let full_range = FIRST_IMMUTABLE_FILE_NUMBER..=last_immutable_file_number; + + match self { + ImmutableFileRange::Full => Ok(full_range), + ImmutableFileRange::From(from) if full_range.contains(from) => { + Ok(*from..=last_immutable_file_number) + } + ImmutableFileRange::Range(from, to) + if full_range.contains(from) + && full_range.contains(to) + && !(*from..=*to).is_empty() => + { + Ok(*from..=*to) + } + ImmutableFileRange::UpTo(to) if full_range.contains(to) => { + Ok(FIRST_IMMUTABLE_FILE_NUMBER..=*to) + } + _ => Err(anyhow!("Invalid immutable file range: {self:?}")), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn to_range_inclusive_with_full() { + let immutable_file_range = ImmutableFileRange::Full; + let last_immutable_file_number = 10; + + let result = immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(); + assert_eq!(1..=10, result); + } + + #[test] + fn to_range_inclusive_with_from() { + let immutable_file_range = ImmutableFileRange::From(5); + + let last_immutable_file_number = 10; + let result = immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(); + assert_eq!(5..=10, result); + + let last_immutable_file_number = 3; + immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .expect_err("conversion to range inlusive should fail"); + } + + #[test] + fn to_range_inclusive_with_range() { + let immutable_file_range = ImmutableFileRange::Range(5, 8); + + let last_immutable_file_number = 10; + let result = immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(); + assert_eq!(5..=8, result); + + let last_immutable_file_number = 7; + immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .expect_err("conversion to range inlusive should fail"); + + let immutable_file_range = ImmutableFileRange::Range(10, 8); + immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .expect_err("conversion to range inlusive should fail"); + } + + #[test] + fn to_range_inclusive_with_up_to() { + let immutable_file_range = ImmutableFileRange::UpTo(8); + + let last_immutable_file_number = 10; + let result = immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(); + assert_eq!(1..=8, result); + + let last_immutable_file_number = 7; + immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .expect_err("conversion to range inlusive should fail"); + } +} diff --git a/mithril-client/src/cardano_database_client/mod.rs b/mithril-client/src/cardano_database_client/mod.rs new file mode 100644 index 00000000000..2f2389af050 --- /dev/null +++ b/mithril-client/src/cardano_database_client/mod.rs @@ -0,0 +1,135 @@ +//! A client to retrieve Cardano databases data from an Aggregator. +//! +//! In order to do so it defines a [CardanoDatabaseClient] which exposes the following features: +//! - [get][CardanoDatabaseClient::get]: get a Cardano database data from its hash +//! - [list][CardanoDatabaseClient::list]: get the list of available Cardano database +//! - [download_unpack][CardanoDatabaseClient::download_unpack]: download and unpack a Cardano database snapshot for a given immutable files range +//! - [compute_merkle_proof][CardanoDatabaseClient::compute_merkle_proof]: compute a Merkle proof for a given Cardano database snapshot and a given immutable files range +//! +//! # Get a Cardano database +//! +//! To get a Cardano database using the [ClientBuilder][crate::client::ClientBuilder]. +//! +//! ```no_run +//! # async fn run() -> mithril_client::MithrilResult<()> { +//! use mithril_client::ClientBuilder; +//! +//! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; +//! let cardano_database = client.cardano_database().get("CARDANO_DATABASE_HASH").await?.unwrap(); +//! +//! println!( +//! "Cardano database hash={}, merkle_root={}, immutable_file_number={:?}", +//! cardano_database.hash, +//! cardano_database.merkle_root, +//! cardano_database.beacon.immutable_file_number +//! ); +//! # Ok(()) +//! # } +//! ``` +//! +//! # List available Cardano databases +//! +//! To list available Cardano databases using the [ClientBuilder][crate::client::ClientBuilder]. +//! +//! ```no_run +//! # async fn run() -> mithril_client::MithrilResult<()> { +//! use mithril_client::ClientBuilder; +//! +//! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; +//! let cardano_databases = client.cardano_database().list().await?; +//! +//! for cardano_database in cardano_databases { +//! println!("Cardano database hash={}, immutable_file_number={}", cardano_database.hash, cardano_database.beacon.immutable_file_number); +//! } +//! # Ok(()) +//! # } +//! ``` +//! +//! # Download a Cardano database snapshot +//! +//! To download a partial or a full Cardano database folder the [ClientBuilder][crate::client::ClientBuilder]. +//! +//! ```no_run +//! # #[cfg(feature = "fs")] +//! # async fn run() -> mithril_client::MithrilResult<()> { +//! use mithril_client::{ClientBuilder, cardano_database_client::{ImmutableFileRange, DownloadUnpackOptions}}; +//! use std::path::Path; +//! +//! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; +//! let cardano_database_snapshot = client.cardano_database().get("CARDANO_DATABASE_HASH").await?.unwrap(); +//! +//! // Note: the directory must already exist, and the user running the binary must have read/write access to it. +//! let target_directory = Path::new("/home/user/download/"); +//! let immutable_file_range = ImmutableFileRange::Range(3, 6); +//! let download_unpack_options = DownloadUnpackOptions { +//! allow_override: true, +//! include_ancillary: true, +//! }; +//! client +//! .cardano_database() +//! .download_unpack( +//! &cardano_database_snapshot, +//! &immutable_file_range, +//! &target_directory, +//! download_unpack_options, +//! ) +//! .await?; +//! # +//! # Ok(()) +//! # } +//! ``` +//! +//! # Compute a Merkle proof for a Cardano database snapshot +//! +//! To compute proof of membership of downloaded immutable files in a Cardano database folder the [ClientBuilder][crate::client::ClientBuilder]. +//! +//! ```no_run +//! # #[cfg(feature = "fs")] +//! # async fn run() -> mithril_client::MithrilResult<()> { +//! use mithril_client::{ClientBuilder, cardano_database_client::{ImmutableFileRange, DownloadUnpackOptions}}; +//! use std::path::Path; +//! +//! let client = ClientBuilder::aggregator("YOUR_AGGREGATOR_ENDPOINT", "YOUR_GENESIS_VERIFICATION_KEY").build()?; +//! let cardano_database_snapshot = client.cardano_database().get("CARDANO_DATABASE_HASH").await?.unwrap(); +//! let certificate = client.certificate().verify_chain(&cardano_database_snapshot.certificate_hash).await?; +//! +//! // Note: the directory must already exist, and the user running the binary must have read/write access to it. +//! let target_directory = Path::new("/home/user/download/"); +//! let immutable_file_range = ImmutableFileRange::Full; +//! let download_unpack_options = DownloadUnpackOptions { +//! allow_override: true, +//! include_ancillary: true, +//! }; +//! client +//! .cardano_database() +//! .download_unpack( +//! &cardano_database_snapshot, +//! &immutable_file_range, +//! &target_directory, +//! download_unpack_options, +//! ) +//! .await?; +//! +//! let merkle_proof = client +//! .cardano_database() +//! .compute_merkle_proof(&certificate, &cardano_database_snapshot, &immutable_file_range, &target_directory) +//! .await?; +//! # +//! # Ok(()) +//! # } +//! ``` +mod api; +mod fetch; + +#[cfg(test)] +pub(crate) use api::test_dependency_injector::CardanoDatabaseClientDependencyInjector; +pub use api::CardanoDatabaseClient; + +cfg_fs! { + mod immutable_file_range; + mod download_unpack; + mod proving; + + pub use download_unpack::DownloadUnpackOptions; + pub use immutable_file_range::ImmutableFileRange; +} diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs new file mode 100644 index 00000000000..b0c6864008f --- /dev/null +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -0,0 +1,633 @@ +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, Context}; + +use mithril_common::{ + crypto_helper::{MKProof, MKTree, MKTreeNode, MKTreeStoreInMemory}, + digesters::{CardanoImmutableDigester, ImmutableDigester, ImmutableFile}, + entities::{DigestLocation, HexEncodedDigest, ImmutableFileName}, + messages::{ + CardanoDatabaseDigestListItemMessage, CardanoDatabaseSnapshotMessage, CertificateMessage, + }, + StdResult, +}; + +use crate::{feedback::MithrilEvent, file_downloader::FileDownloaderUri}; + +use super::api::CardanoDatabaseClient; +use super::immutable_file_range::ImmutableFileRange; + +impl CardanoDatabaseClient { + /// Compute the Merkle proof of membership for the given immutable file range. + pub async fn compute_merkle_proof( + &self, + certificate: &CertificateMessage, + cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, + immutable_file_range: &ImmutableFileRange, + database_dir: &Path, + ) -> StdResult { + let digest_locations = &cardano_database_snapshot.locations.digests; + self.download_unpack_digest_file(digest_locations, &Self::digest_target_dir(database_dir)) + .await?; + let network = certificate.metadata.network.clone(); + let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; + let immutable_file_number_range = + immutable_file_range.to_range_inclusive(last_immutable_file_number)?; + let downloaded_digests = self.read_digest_file(&Self::digest_target_dir(database_dir))?; + let downloaded_digests_values = downloaded_digests + .into_iter() + .filter(|(immutable_file_name, _)| { + match ImmutableFile::new(Path::new(immutable_file_name).to_path_buf()) { + Ok(immutable_file) => immutable_file.number <= last_immutable_file_number, + Err(_) => false, + } + }) + .map(|(_immutable_file_name, digest)| digest) + .collect::>(); + let merkle_tree: MKTree = MKTree::new(&downloaded_digests_values)?; + let immutable_digester = CardanoImmutableDigester::new(network, None, self.logger.clone()); + let computed_digests = immutable_digester + .compute_digests_for_range(database_dir, &immutable_file_number_range) + .await? + .entries + .values() + .map(MKTreeNode::from) + .collect::>(); + Self::delete_directory(&Self::digest_target_dir(database_dir))?; + + merkle_tree.compute_proof(&computed_digests) + } + + async fn download_unpack_digest_file( + &self, + locations: &[DigestLocation], + digest_file_target_dir: &Path, + ) -> StdResult<()> { + Self::create_directory_if_not_exists(digest_file_target_dir)?; + let mut locations_sorted = locations.to_owned(); + locations_sorted.sort(); + for location in locations_sorted { + let download_id = MithrilEvent::new_digest_download_id(); + self.feedback_sender + .send_event(MithrilEvent::DigestDownloadStarted { + download_id: download_id.clone(), + }) + .await; + let file_downloader = self + .digest_file_downloader_resolver + .resolve(&location) + .ok_or_else(|| { + anyhow!("Failed resolving a file downloader for location: {location:?}") + })?; + let file_downloader_uri: FileDownloaderUri = location.into(); + let downloaded = file_downloader + .download_unpack( + &file_downloader_uri, + digest_file_target_dir, + None, + &download_id, + Self::feedback_event_builder_digest_download, + ) + .await; + match downloaded { + Ok(_) => { + self.feedback_sender + .send_event(MithrilEvent::DigestDownloadCompleted { download_id }) + .await; + return Ok(()); + } + Err(e) => { + slog::error!( + self.logger, + "Failed downloading and unpacking digest for location {file_downloader_uri:?}"; "error" => e.to_string() + ); + } + } + } + + Err(anyhow!( + "Failed downloading and unpacking digests for all locations" + )) + } + + fn read_digest_file( + &self, + digest_file_target_dir: &Path, + ) -> StdResult> { + let digest_files = Self::read_files_in_directory(digest_file_target_dir)?; + if digest_files.len() > 1 { + return Err(anyhow!( + "Multiple digest files found in directory: {digest_file_target_dir:?}" + )); + } + if digest_files.is_empty() { + return Err(anyhow!( + "No digest file found in directory: {digest_file_target_dir:?}" + )); + } + + let digest_file = &digest_files[0]; + let content = fs::read_to_string(digest_file) + .with_context(|| format!("Failed reading digest file: {digest_file:?}"))?; + let digest_messages: Vec = + serde_json::from_str(&content) + .with_context(|| format!("Failed deserializing digest file: {digest_file:?}"))?; + let digest_map = digest_messages + .into_iter() + .map(|message| (message.immutable_file_name, message.digest)) + .collect::>(); + + Ok(digest_map) + } + + fn feedback_event_builder_digest_download( + download_id: String, + downloaded_bytes: u64, + size: u64, + ) -> Option { + Some(MithrilEvent::DigestDownloadProgress { + download_id, + downloaded_bytes, + size, + }) + } + + fn digest_target_dir(target_dir: &Path) -> PathBuf { + target_dir.join("digest") + } + + fn create_directory_if_not_exists(dir: &Path) -> StdResult<()> { + if dir.exists() { + return Ok(()); + } + + fs::create_dir_all(dir).map_err(|e| anyhow!("Failed creating directory: {e}")) + } + + fn delete_directory(dir: &Path) -> StdResult<()> { + if dir.exists() { + fs::remove_dir_all(dir).map_err(|e| anyhow!("Failed deleting directory: {e}"))?; + } + + Ok(()) + } + + fn read_files_in_directory(dir: &Path) -> StdResult> { + let mut files = vec![]; + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_file() { + files.push(path); + } + } + + Ok(files) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::fs; + use std::io::Write; + use std::path::Path; + use std::sync::Arc; + + use mithril_common::{ + digesters::{DummyCardanoDbBuilder, ImmutableDigester, ImmutableFile}, + entities::{CardanoDbBeacon, DigestLocationDiscriminants, Epoch, HexEncodedDigest}, + messages::{ArtifactsLocationsMessagePart, CardanoDatabaseDigestListItemMessage}, + test_utils::TempDir, + }; + + use crate::{ + cardano_database_client::CardanoDatabaseClientDependencyInjector, + feedback::StackFeedbackReceiver, + file_downloader::{MockFileDownloader, MockFileDownloaderBuilder}, + test_utils::test_logger, + }; + + use super::*; + + mod compute_merkle_proof { + + use std::ops::RangeInclusive; + + use mithril_common::entities::ImmutableFileNumber; + + use super::*; + + async fn create_fake_digest_artifact( + dir_name: &str, + beacon: &CardanoDbBeacon, + immutable_file_range: &RangeInclusive, + digests_offset: usize, + ) -> ( + PathBuf, + CardanoDatabaseSnapshotMessage, + CertificateMessage, + MKTree, + ) { + let cardano_database_snapshot = CardanoDatabaseSnapshotMessage { + hash: "hash-123".to_string(), + beacon: beacon.clone(), + locations: ArtifactsLocationsMessagePart { + digests: vec![DigestLocation::CloudStorage { + uri: "http://whatever/digests.json".to_string(), + }], + ..ArtifactsLocationsMessagePart::default() + }, + ..CardanoDatabaseSnapshotMessage::dummy() + }; + let certificate = CertificateMessage { + hash: "cert-hash-123".to_string(), + ..CertificateMessage::dummy() + }; + let cardano_db = DummyCardanoDbBuilder::new(dir_name) + .with_immutables(&immutable_file_range.clone().collect::>()) + .append_immutable_trio() + .build(); + let database_dir = cardano_db.get_dir(); + let immutable_digester = CardanoImmutableDigester::new( + certificate.metadata.network.to_string(), + None, + test_logger(), + ); + let computed_digests = immutable_digester + .compute_digests_for_range(database_dir, immutable_file_range) + .await + .unwrap(); + write_digest_file(&database_dir.join("digest"), &computed_digests.entries).await; + + // We remove the last digests_offset digests to simulate receiving + // a digest file with more immutable files than downloaded + for (immutable_file, _digest) in + computed_digests.entries.iter().rev().take(digests_offset) + { + fs::remove_file( + database_dir.join( + database_dir + .join("immutable") + .join(immutable_file.filename.clone()), + ), + ) + .unwrap(); + } + + let merkle_tree = immutable_digester + .compute_merkle_tree(database_dir, beacon) + .await + .unwrap(); + + ( + database_dir.to_owned(), + cardano_database_snapshot, + certificate, + merkle_tree, + ) + } + + async fn write_digest_file( + digest_dir: &Path, + digests: &BTreeMap, + ) { + let digest_file_path = digest_dir.join("digests.json"); + if !digest_dir.exists() { + fs::create_dir_all(digest_dir).unwrap(); + } + + let immutable_digest_messages = digests + .iter() + .map( + |(immutable_file, digest)| CardanoDatabaseDigestListItemMessage { + immutable_file_name: immutable_file.filename.clone(), + digest: digest.to_string(), + }, + ) + .collect::>(); + serde_json::to_writer( + fs::File::create(digest_file_path).unwrap(), + &immutable_digest_messages, + ) + .unwrap(); + } + + #[tokio::test] + async fn compute_merkle_proof_succeeds() { + let beacon = CardanoDbBeacon { + epoch: Epoch(123), + immutable_file_number: 10, + }; + let immutable_file_range = 1..=15; + let immutable_file_range_to_prove = ImmutableFileRange::Range(2, 4); + let digests_offset = 3; + let (database_dir, cardano_database_snapshot, certificate, merkle_tree) = + create_fake_digest_artifact( + "compute_merkle_proof_succeeds", + &beacon, + &immutable_file_range, + digests_offset, + ) + .await; + let expected_merkle_root = merkle_tree.compute_root().unwrap(); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_digest_file_downloaders(vec![( + DigestLocationDiscriminants::CloudStorage, + Arc::new({ + MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/digests.json") + .with_target_dir(database_dir.join("digest")) + .with_compression(None) + .with_success() + .build() + }), + )]) + .build_cardano_database_client(); + + let merkle_proof = client + .compute_merkle_proof( + &certificate, + &cardano_database_snapshot, + &immutable_file_range_to_prove, + &database_dir, + ) + .await + .unwrap(); + merkle_proof.verify().unwrap(); + + let merkle_proof_root = merkle_proof.root().to_owned(); + assert_eq!(expected_merkle_root, merkle_proof_root); + + assert!(!database_dir.join("digest").exists()); + } + } + + mod download_unpack_digest_file { + + use super::*; + + #[tokio::test] + async fn download_unpack_digest_file_fails_if_no_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_digest_file_downloaders(vec![ + ( + DigestLocationDiscriminants::CloudStorage, + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_failure() + .build(), + ), + ), + ( + DigestLocationDiscriminants::Aggregator, + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_failure() + .build(), + ), + ), + ]) + .build_cardano_database_client(); + + client + .download_unpack_digest_file( + &[ + DigestLocation::CloudStorage { + uri: "http://whatever-1/digests.json".to_string(), + }, + DigestLocation::Aggregator { + uri: "http://whatever-2/digest".to_string(), + }, + ], + target_dir, + ) + .await + .expect_err("download_unpack_digest_file should fail"); + } + + #[tokio::test] + async fn download_unpack_digest_file_succeeds_if_at_least_one_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_digest_file_downloaders(vec![ + ( + DigestLocationDiscriminants::CloudStorage, + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_failure() + .build(), + ), + ), + ( + DigestLocationDiscriminants::Aggregator, + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_success() + .build(), + ), + ), + ]) + .build_cardano_database_client(); + + client + .download_unpack_digest_file( + &[ + DigestLocation::CloudStorage { + uri: "http://whatever-1/digests.json".to_string(), + }, + DigestLocation::Aggregator { + uri: "http://whatever-2/digest".to_string(), + }, + ], + target_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn download_unpack_digest_file_succeeds_when_first_location_is_retrieved() { + let target_dir = Path::new("."); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_digest_file_downloaders(vec![ + ( + DigestLocationDiscriminants::CloudStorage, + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_success() + .build(), + ), + ), + ( + DigestLocationDiscriminants::Aggregator, + Arc::new(MockFileDownloader::new()), + ), + ]) + .build_cardano_database_client(); + + client + .download_unpack_digest_file( + &[ + DigestLocation::CloudStorage { + uri: "http://whatever-1/digests.json".to_string(), + }, + DigestLocation::Aggregator { + uri: "http://whatever-2/digest".to_string(), + }, + ], + target_dir, + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn download_unpack_digest_file_sends_feedbacks() { + let target_dir = Path::new("."); + let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_digest_file_downloaders(vec![( + DigestLocationDiscriminants::CloudStorage, + Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_success() + .build(), + ), + )]) + .with_feedback_receivers(&[feedback_receiver.clone()]) + .build_cardano_database_client(); + + client + .download_unpack_digest_file( + &[DigestLocation::CloudStorage { + uri: "http://whatever-1/digests.json".to_string(), + }], + target_dir, + ) + .await + .unwrap(); + + let sent_events = feedback_receiver.stacked_events(); + let id = sent_events[0].event_id(); + let expected_events = vec![ + MithrilEvent::DigestDownloadStarted { + download_id: id.to_string(), + }, + MithrilEvent::DigestDownloadCompleted { + download_id: id.to_string(), + }, + ]; + assert_eq!(expected_events, sent_events); + } + } + + mod read_digest_file { + + use super::*; + + fn create_valid_fake_digest_file( + file_path: &Path, + digest_messages: &[CardanoDatabaseDigestListItemMessage], + ) { + let mut file = fs::File::create(file_path).unwrap(); + let digest_json = serde_json::to_string(&digest_messages).unwrap(); + file.write_all(digest_json.as_bytes()).unwrap(); + } + + fn create_invalid_fake_digest_file(file_path: &Path) { + let mut file = fs::File::create(file_path).unwrap(); + file.write_all(b"incorrect-digest").unwrap(); + } + + #[test] + fn read_digest_file_fails_when_no_digest_file() { + let target_dir = TempDir::new( + "cardano_database_client", + "read_digest_file_fails_when_no_digest_file", + ) + .build(); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .read_digest_file(&target_dir) + .expect_err("read_digest_file should fail"); + } + + #[test] + fn read_digest_file_fails_when_multiple_digest_files() { + let target_dir = TempDir::new( + "cardano_database_client", + "read_digest_file_fails_when_multiple_digest_files", + ) + .build(); + create_valid_fake_digest_file(&target_dir.join("digests.json"), &[]); + create_valid_fake_digest_file(&target_dir.join("digests-2.json"), &[]); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .read_digest_file(&target_dir) + .expect_err("read_digest_file should fail"); + } + + #[test] + fn read_digest_file_fails_when_invalid_unique_digest_file() { + let target_dir = TempDir::new( + "cardano_database_client", + "read_digest_file_fails_when_invalid_unique_digest_file", + ) + .build(); + create_invalid_fake_digest_file(&target_dir.join("digests.json")); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + client + .read_digest_file(&target_dir) + .expect_err("read_digest_file should fail"); + } + + #[test] + fn read_digest_file_succeeds_when_valid_unique_digest_file() { + let target_dir = TempDir::new( + "cardano_database_client", + "read_digest_file_succeeds_when_valid_unique_digest_file", + ) + .build(); + let digest_messages = vec![ + CardanoDatabaseDigestListItemMessage { + immutable_file_name: "00001.chunk".to_string(), + digest: "digest-1".to_string(), + }, + CardanoDatabaseDigestListItemMessage { + immutable_file_name: "00002.chunk".to_string(), + digest: "digest-2".to_string(), + }, + ]; + create_valid_fake_digest_file(&target_dir.join("digests.json"), &digest_messages); + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + + let digests = client.read_digest_file(&target_dir).unwrap(); + assert_eq!( + BTreeMap::from([ + ("00001.chunk".to_string(), "digest-1".to_string()), + ("00002.chunk".to_string(), "digest-2".to_string()) + ]), + digests + ) + } + } +} From ad7c36a7e3ddd9004de8e06b8d5ec7a1588e9993 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 13 Feb 2025 17:01:34 +0100 Subject: [PATCH 34/59] feat: enhance concurrency level of immutable files download and unpack --- .../download_unpack.rs | 79 ++++++++++++++----- 1 file changed, 58 insertions(+), 21 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index 6479f97982b..68b02a9af4b 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -1,6 +1,7 @@ use std::collections::BTreeSet; use std::ops::RangeInclusive; use std::path::{Path, PathBuf}; +use std::sync::Arc; use tokio::task::JoinSet; use anyhow::anyhow; @@ -12,7 +13,7 @@ use mithril_common::{ }; use crate::feedback::MithrilEvent; -use crate::file_downloader::FileDownloaderUri; +use crate::file_downloader::{FileDownloader, FileDownloaderUri}; use super::api::CardanoDatabaseClient; use super::immutable_file_range::ImmutableFileRange; @@ -28,6 +29,10 @@ pub struct DownloadUnpackOptions { } impl CardanoDatabaseClient { + /// The maximum number of parallel downloads and unpacks for immutable files. + /// This could be a customizable parameter depending on level of concurrency wanted by the user. + const MAX_PARALLEL_DOWNLOAD_UNPACK: usize = 100; + /// Download and unpack the given Cardano database parts data by hash. pub async fn download_unpack( &self, @@ -170,31 +175,21 @@ impl CardanoDatabaseClient { )) } - async fn download_unpack_immutable_files_for_location( + /// Download and unpack the immutable files of the given range. + /// + /// The download is attempted for each location until the full range is downloaded. + /// An error is returned if not all the files are downloaded. + async fn batch_download_unpack_immutable_files( &self, - location: &ImmutablesLocation, - immutable_file_numbers_to_download: &BTreeSet, + file_downloader: Arc, + file_downloader_uris_chunk: Vec<(ImmutableFileNumber, FileDownloaderUri)>, compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path, ) -> StdResult> { let mut immutable_file_numbers_downloaded = BTreeSet::new(); - let file_downloader = self - .immutable_file_downloader_resolver - .resolve(location) - .ok_or_else(|| { - anyhow!("Failed resolving a file downloader for location: {location:?}") - })?; - let file_downloader_uris = - FileDownloaderUri::expand_immutable_files_location_to_file_downloader_uris( - location, - immutable_file_numbers_to_download - .clone() - .into_iter() - .collect::>() - .as_slice(), - )?; let mut join_set: JoinSet> = JoinSet::new(); - for (immutable_file_number, file_downloader_uri) in file_downloader_uris { + for (immutable_file_number, file_downloader_uri) in file_downloader_uris_chunk.into_iter() { + let file_downloader_uri_clone = file_downloader_uri.to_owned(); let compression_algorithm_clone = compression_algorithm.to_owned(); let immutable_files_target_dir_clone = immutable_files_target_dir.to_owned(); let file_downloader_clone = file_downloader.clone(); @@ -207,7 +202,7 @@ impl CardanoDatabaseClient { .await; let downloaded = file_downloader_clone .download_unpack( - &file_downloader_uri, + &file_downloader_uri_clone, &immutable_files_target_dir_clone, Some(compression_algorithm_clone), &download_id, @@ -249,6 +244,48 @@ impl CardanoDatabaseClient { Ok(immutable_file_numbers_downloaded) } + async fn download_unpack_immutable_files_for_location( + &self, + location: &ImmutablesLocation, + immutable_file_numbers_to_download: &BTreeSet, + compression_algorithm: &CompressionAlgorithm, + immutable_files_target_dir: &Path, + ) -> StdResult> { + let mut immutable_file_numbers_downloaded = BTreeSet::new(); + let file_downloader = self + .immutable_file_downloader_resolver + .resolve(location) + .ok_or_else(|| { + anyhow!("Failed resolving a file downloader for location: {location:?}") + })?; + let file_downloader_uris = + FileDownloaderUri::expand_immutable_files_location_to_file_downloader_uris( + location, + immutable_file_numbers_to_download + .clone() + .into_iter() + .collect::>() + .as_slice(), + )?; + let file_downloader_uri_chunks = file_downloader_uris + .chunks(Self::MAX_PARALLEL_DOWNLOAD_UNPACK) + .map(|x| x.to_vec()) + .collect::>(); + for file_downloader_uris_chunk in file_downloader_uri_chunks { + let immutable_file_numbers_downloaded_chunk = self + .batch_download_unpack_immutable_files( + file_downloader.clone(), + file_downloader_uris_chunk, + compression_algorithm, + immutable_files_target_dir, + ) + .await?; + immutable_file_numbers_downloaded.extend(immutable_file_numbers_downloaded_chunk); + } + + Ok(immutable_file_numbers_downloaded) + } + /// Download and unpack the ancillary files. async fn download_unpack_ancillary_file( &self, From a0dc3f36f01eddaca80c04437b3b28fc48f6cb0c Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 14 Feb 2025 15:03:40 +0100 Subject: [PATCH 35/59] refactor: use 'MithrilResult' instead of 'StdResult' in Cardano database client --- .../cardano_database_client/download_unpack.rs | 16 ++++++++-------- .../src/cardano_database_client/proving.rs | 15 +++++++-------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index 68b02a9af4b..6ca0be11cb6 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -9,11 +9,11 @@ use anyhow::anyhow; use mithril_common::{ entities::{AncillaryLocation, CompressionAlgorithm, ImmutableFileNumber, ImmutablesLocation}, messages::CardanoDatabaseSnapshotMessage, - StdResult, }; use crate::feedback::MithrilEvent; use crate::file_downloader::{FileDownloader, FileDownloaderUri}; +use crate::MithrilResult; use super::api::CardanoDatabaseClient; use super::immutable_file_range::ImmutableFileRange; @@ -40,7 +40,7 @@ impl CardanoDatabaseClient { immutable_file_range: &ImmutableFileRange, target_dir: &Path, download_unpack_options: DownloadUnpackOptions, - ) -> StdResult<()> { + ) -> MithrilResult<()> { let compression_algorithm = cardano_database_snapshot.compression_algorithm; let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; let immutable_file_number_range = @@ -87,7 +87,7 @@ impl CardanoDatabaseClient { &self, target_dir: &Path, download_unpack_options: &DownloadUnpackOptions, - ) -> StdResult<()> { + ) -> MithrilResult<()> { let immutable_files_target_dir = Self::immutable_files_target_dir(target_dir); let volatile_target_dir = Self::volatile_target_dir(target_dir); let ledger_target_dir = Self::ledger_target_dir(target_dir); @@ -148,7 +148,7 @@ impl CardanoDatabaseClient { range: RangeInclusive, compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path, - ) -> StdResult<()> { + ) -> MithrilResult<()> { let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); let mut immutable_file_numbers_to_download = @@ -185,9 +185,9 @@ impl CardanoDatabaseClient { file_downloader_uris_chunk: Vec<(ImmutableFileNumber, FileDownloaderUri)>, compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path, - ) -> StdResult> { + ) -> MithrilResult> { let mut immutable_file_numbers_downloaded = BTreeSet::new(); - let mut join_set: JoinSet> = JoinSet::new(); + let mut join_set: JoinSet> = JoinSet::new(); for (immutable_file_number, file_downloader_uri) in file_downloader_uris_chunk.into_iter() { let file_downloader_uri_clone = file_downloader_uri.to_owned(); let compression_algorithm_clone = compression_algorithm.to_owned(); @@ -250,7 +250,7 @@ impl CardanoDatabaseClient { immutable_file_numbers_to_download: &BTreeSet, compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path, - ) -> StdResult> { + ) -> MithrilResult> { let mut immutable_file_numbers_downloaded = BTreeSet::new(); let file_downloader = self .immutable_file_downloader_resolver @@ -292,7 +292,7 @@ impl CardanoDatabaseClient { locations: &[AncillaryLocation], compression_algorithm: &CompressionAlgorithm, ancillary_file_target_dir: &Path, - ) -> StdResult<()> { + ) -> MithrilResult<()> { let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); for location in locations_sorted { diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index b0c6864008f..cb0cdf7ef47 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -13,10 +13,9 @@ use mithril_common::{ messages::{ CardanoDatabaseDigestListItemMessage, CardanoDatabaseSnapshotMessage, CertificateMessage, }, - StdResult, }; -use crate::{feedback::MithrilEvent, file_downloader::FileDownloaderUri}; +use crate::{feedback::MithrilEvent, file_downloader::FileDownloaderUri, MithrilResult}; use super::api::CardanoDatabaseClient; use super::immutable_file_range::ImmutableFileRange; @@ -29,7 +28,7 @@ impl CardanoDatabaseClient { cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, immutable_file_range: &ImmutableFileRange, database_dir: &Path, - ) -> StdResult { + ) -> MithrilResult { let digest_locations = &cardano_database_snapshot.locations.digests; self.download_unpack_digest_file(digest_locations, &Self::digest_target_dir(database_dir)) .await?; @@ -66,7 +65,7 @@ impl CardanoDatabaseClient { &self, locations: &[DigestLocation], digest_file_target_dir: &Path, - ) -> StdResult<()> { + ) -> MithrilResult<()> { Self::create_directory_if_not_exists(digest_file_target_dir)?; let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); @@ -117,7 +116,7 @@ impl CardanoDatabaseClient { fn read_digest_file( &self, digest_file_target_dir: &Path, - ) -> StdResult> { + ) -> MithrilResult> { let digest_files = Self::read_files_in_directory(digest_file_target_dir)?; if digest_files.len() > 1 { return Err(anyhow!( @@ -160,7 +159,7 @@ impl CardanoDatabaseClient { target_dir.join("digest") } - fn create_directory_if_not_exists(dir: &Path) -> StdResult<()> { + fn create_directory_if_not_exists(dir: &Path) -> MithrilResult<()> { if dir.exists() { return Ok(()); } @@ -168,7 +167,7 @@ impl CardanoDatabaseClient { fs::create_dir_all(dir).map_err(|e| anyhow!("Failed creating directory: {e}")) } - fn delete_directory(dir: &Path) -> StdResult<()> { + fn delete_directory(dir: &Path) -> MithrilResult<()> { if dir.exists() { fs::remove_dir_all(dir).map_err(|e| anyhow!("Failed deleting directory: {e}"))?; } @@ -176,7 +175,7 @@ impl CardanoDatabaseClient { Ok(()) } - fn read_files_in_directory(dir: &Path) -> StdResult> { + fn read_files_in_directory(dir: &Path) -> MithrilResult> { let mut files = vec![]; for entry in fs::read_dir(dir)? { let entry = entry?; From 4ab44df48cd15f5d2f992b5b1b0384762c05e90b Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 14 Feb 2025 18:44:33 +0100 Subject: [PATCH 36/59] fix: download fails with ancillary download and last immutable file not in range --- .../download_unpack.rs | 107 +++++++++++++++++- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index 6ca0be11cb6..63105aeec64 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -45,9 +45,12 @@ impl CardanoDatabaseClient { let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; let immutable_file_number_range = immutable_file_range.to_range_inclusive(last_immutable_file_number)?; - + self.verify_download_options_compatibility( + &download_unpack_options, + &immutable_file_number_range, + last_immutable_file_number, + )?; self.verify_can_write_to_target_directory(target_dir, &download_unpack_options)?; - let immutable_locations = &cardano_database_snapshot.locations.immutables; self.download_unpack_immutable_files( immutable_locations, @@ -56,7 +59,6 @@ impl CardanoDatabaseClient { target_dir, ) .await?; - if download_unpack_options.include_ancillary { let ancillary_locations = &cardano_database_snapshot.locations.ancillary; self.download_unpack_ancillary_file( @@ -114,6 +116,24 @@ impl CardanoDatabaseClient { Ok(()) } + /// Verify if the download options are compatible with the immutable file range. + fn verify_download_options_compatibility( + &self, + download_options: &DownloadUnpackOptions, + immutable_file_range: &RangeInclusive, + last_immutable_file_number: ImmutableFileNumber, + ) -> MithrilResult<()> { + if download_options.include_ancillary + && !immutable_file_range.contains(&last_immutable_file_number) + { + return Err(anyhow!( + "The last immutable file number {last_immutable_file_number} is outside the range: {immutable_file_range:?}" + )); + } + + Ok(()) + } + fn feedback_event_builder_immutable_download( download_id: String, downloaded_bytes: u64, @@ -347,8 +367,8 @@ mod tests { use mithril_common::{ entities::{ - AncillaryLocationDiscriminants, ImmutablesLocationDiscriminants, MultiFilesUri, - TemplateUri, + AncillaryLocationDiscriminants, CardanoDbBeacon, Epoch, + ImmutablesLocationDiscriminants, MultiFilesUri, TemplateUri, }, messages::{ ArtifactsLocationsMessagePart, @@ -473,6 +493,10 @@ mod tests { }; let cardano_db_snapshot = CardanoDatabaseSnapshot { hash: "hash-123".to_string(), + beacon: CardanoDbBeacon { + immutable_file_number: 2, + epoch: Epoch(123), + }, locations: ArtifactsLocationsMessagePart { immutables: vec![ImmutablesLocation::CloudStorage { uri: MultiFilesUri::Template(TemplateUri( @@ -533,6 +557,79 @@ mod tests { } } + mod verify_download_options_compatibility { + + use super::*; + + #[test] + fn verify_download_options_compatibility_succeeds_if_without_ancillary_download() { + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + let download_options = DownloadUnpackOptions { + include_ancillary: false, + ..DownloadUnpackOptions::default() + }; + let immutable_file_range = ImmutableFileRange::Range(1, 10); + let last_immutable_file_number = 10; + + client + .verify_download_options_compatibility( + &download_options, + &immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(), + last_immutable_file_number, + ) + .unwrap(); + } + + #[test] + fn verify_download_options_compatibility_succeeds_if_with_ancillary_download_and_compatible_range( + ) { + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + let download_options = DownloadUnpackOptions { + include_ancillary: true, + ..DownloadUnpackOptions::default() + }; + let immutable_file_range = ImmutableFileRange::Range(7, 10); + let last_immutable_file_number = 10; + + client + .verify_download_options_compatibility( + &download_options, + &immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(), + last_immutable_file_number, + ) + .unwrap(); + } + + #[test] + fn verify_download_options_compatibility_fails_if_with_ancillary_download_and_incompatible_range( + ) { + let client = + CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + let download_options = DownloadUnpackOptions { + include_ancillary: true, + ..DownloadUnpackOptions::default() + }; + let immutable_file_range = ImmutableFileRange::Range(7, 10); + let last_immutable_file_number = 123; + + client + .verify_download_options_compatibility( + &download_options, + &immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(), + last_immutable_file_number, + ) + .expect_err("verify_download_options_compatibility should fail as the last immutable file number is outside the range"); + } + } + mod verify_can_write_to_target_dir { use super::*; From 610ae5cd8390874b48af5d1736f017b75a10a497 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Fri, 31 Jan 2025 15:51:55 +0100 Subject: [PATCH 37/59] feat: implement HTTP file downloader --- mithril-client/src/file_downloader/http.rs | 183 +++++++++++++++++++++ mithril-client/src/file_downloader/mod.rs | 2 + 2 files changed, 185 insertions(+) create mode 100644 mithril-client/src/file_downloader/http.rs diff --git a/mithril-client/src/file_downloader/http.rs b/mithril-client/src/file_downloader/http.rs new file mode 100644 index 00000000000..46b260d3447 --- /dev/null +++ b/mithril-client/src/file_downloader/http.rs @@ -0,0 +1,183 @@ +use std::{ + io::{BufReader, Read, Write}, + path::Path, +}; + +use anyhow::{anyhow, Context}; +use async_trait::async_trait; +use flate2::read::GzDecoder; +use flume::{Receiver, Sender}; +use futures::StreamExt; +use reqwest::{Response, StatusCode}; +use slog::{debug, Logger}; +use tar::Archive; + +use mithril_common::{logging::LoggerExtensions, StdResult}; + +use crate::common::CompressionAlgorithm; +use crate::feedback::FeedbackSender; +use crate::utils::StreamReader; + +use super::{FeedbackEventBuilder, FileDownloader, FileDownloaderUri}; + +/// A file downloader that only handles download through HTTP. +pub struct HttpFileDownloader { + http_client: reqwest::Client, + feedback_sender: FeedbackSender, + logger: Logger, +} + +impl HttpFileDownloader { + /// Constructs a new `HttpFileDownloader`. + pub fn new(feedback_sender: FeedbackSender, logger: Logger) -> StdResult { + let http_client = reqwest::ClientBuilder::new() + .build() + .with_context(|| "Building http client for HttpFileDownloader failed")?; + + Ok(Self { + http_client, + feedback_sender, + logger: logger.new_with_component_name::(), + }) + } + + async fn get(&self, location: &str) -> StdResult { + debug!(self.logger, "GET Snapshot location='{location}'."); + let request_builder = self.http_client.get(location); + let response = request_builder.send().await.with_context(|| { + format!("Cannot perform a GET for the snapshot (location='{location}')") + })?; + + match response.status() { + StatusCode::OK => Ok(response), + StatusCode::NOT_FOUND => Err(anyhow!("Location='{location} not found")), + status_code => Err(anyhow!("Unhandled error {status_code}")), + } + } + + async fn download_file( + &self, + location: &str, + sender: &Sender>, + report_progress: F, + ) -> StdResult<()> + where + F: Fn(u64) -> Fut, + Fut: std::future::Future, + { + let mut downloaded_bytes: u64 = 0; + let mut remote_stream = self.get(location).await?.bytes_stream(); + while let Some(item) = remote_stream.next().await { + let chunk = item.with_context(|| "Download: Could not read from byte stream")?; + sender.send_async(chunk.to_vec()).await.with_context(|| { + format!("Download: could not write {} bytes to stream.", chunk.len()) + })?; + downloaded_bytes += chunk.len() as u64; + report_progress(downloaded_bytes).await + } + + Ok(()) + } + + fn unpack_file( + stream: Receiver>, + compression_algorithm: Option, + unpack_dir: &Path, + download_id: &str, + ) -> StdResult<()> { + let input = StreamReader::new(stream); + match compression_algorithm { + Some(CompressionAlgorithm::Gzip) => { + let gzip_decoder = GzDecoder::new(input); + let mut file_archive = Archive::new(gzip_decoder); + file_archive.unpack(unpack_dir).with_context(|| { + format!( + "Could not unpack with 'Gzip' from streamed data to directory '{}'", + unpack_dir.display() + ) + })?; + } + Some(CompressionAlgorithm::Zstandard) => { + let zstandard_decoder = zstd::Decoder::new(input) + .with_context(|| "Unpack failed: Create Zstandard decoder error")?; + let mut file_archive = Archive::new(zstandard_decoder); + file_archive.unpack(unpack_dir).with_context(|| { + format!( + "Could not unpack with 'Zstd' from streamed data to directory '{}'", + unpack_dir.display() + ) + })?; + } + None => { + let file_path = unpack_dir.join(download_id); + if file_path.exists() { + std::fs::remove_file(file_path.clone())?; + } + let mut file = std::fs::File::create(file_path)?; + let input_buffered = BufReader::new(input); + for byte in input_buffered.bytes() { + file.write_all(&[byte?])?; + } + file.flush()?; + } + }; + + Ok(()) + } +} + +#[async_trait] +impl FileDownloader for HttpFileDownloader { + async fn download_unpack( + &self, + location: &FileDownloaderUri, + target_dir: &Path, + compression_algorithm: Option, + download_id: &str, + feedback_event: FeedbackEventBuilder, + ) -> StdResult<()> { + if !target_dir.is_dir() { + Err( + anyhow!("target path is not a directory or does not exist: `{target_dir:?}`") + .context("Download-Unpack: prerequisite error"), + )?; + } + + let (sender, receiver) = flume::bounded(32); + let dest_dir = target_dir.to_path_buf(); + let download_id_clone = download_id.to_owned(); + let unpack_thread = tokio::task::spawn_blocking(move || -> StdResult<()> { + Self::unpack_file( + receiver, + compression_algorithm, + &dest_dir, + &download_id_clone, + ) + }); + // The size will be completed with the uncompressed file size when available in the location + // (see https://github.com/input-output-hk/mithril/issues/2291) + let file_size = 0; + let report_progress = |downloaded_bytes: u64| async move { + if let Some(event) = feedback_event(download_id.to_owned(), downloaded_bytes, file_size) + { + self.feedback_sender.send_event(event).await + } + }; + self.download_file(location.as_str(), &sender, report_progress) + .await?; + drop(sender); + unpack_thread + .await + .with_context(|| { + format!( + "Unpack: panic while unpacking to dir '{}'", + target_dir.display() + ) + })? + .with_context(|| { + format!("Unpack: could not unpack to dir '{}'", target_dir.display()) + })?; + + Ok(()) + } +} diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs index d7611adacb1..bfe81157333 100644 --- a/mithril-client/src/file_downloader/mod.rs +++ b/mithril-client/src/file_downloader/mod.rs @@ -2,12 +2,14 @@ //! //! This module provides the necessary abstractions to download files from different sources. +mod http; mod interface; #[cfg(test)] mod mock_builder; mod resolver; mod retry; +pub use http::HttpFileDownloader; #[cfg(test)] pub use interface::MockFileDownloader; pub use interface::{FeedbackEventBuilder, FileDownloader, FileDownloaderUri}; From 39be0234b8cdfe1c7a94cec260e6b76f465ef57d Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Thu, 6 Feb 2025 13:08:00 +0100 Subject: [PATCH 38/59] feat: add file downloader dependency injection --- mithril-client/src/client.rs | 37 +++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/mithril-client/src/client.rs b/mithril-client/src/client.rs index 6d010b42ef2..3f8e2c28529 100644 --- a/mithril-client/src/client.rs +++ b/mithril-client/src/client.rs @@ -6,6 +6,10 @@ use std::collections::HashMap; use std::sync::Arc; use mithril_common::api_version::APIVersionProvider; +#[cfg(all(feature = "fs", feature = "unstable"))] +use mithril_common::entities::{ + AncillaryLocationDiscriminants, DigestLocationDiscriminants, ImmutablesLocationDiscriminants, +}; use crate::aggregator_client::{AggregatorClient, AggregatorHTTPClient}; #[cfg(feature = "unstable")] @@ -20,7 +24,8 @@ use crate::certificate_client::{ use crate::feedback::{FeedbackReceiver, FeedbackSender}; #[cfg(all(feature = "fs", feature = "unstable"))] use crate::file_downloader::{ - AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, ImmutablesFileDownloaderResolver, + AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloadRetryPolicy, + HttpFileDownloader, ImmutablesFileDownloaderResolver, RetryDownloader, }; use crate::mithril_stake_distribution_client::MithrilStakeDistributionClient; use crate::snapshot_client::SnapshotClient; @@ -263,15 +268,37 @@ impl ClientBuilder { logger.clone(), )); + #[cfg(all(feature = "fs", feature = "unstable"))] + let http_file_downloader = Arc::new(RetryDownloader::new( + Arc::new( + HttpFileDownloader::new(feedback_sender.clone(), logger.clone()) + .with_context(|| "Building http file downloader failed")?, + ), + FileDownloadRetryPolicy::default(), + )); #[cfg(all(feature = "fs", feature = "unstable"))] let immutable_file_downloader_resolver = - Arc::new(ImmutablesFileDownloaderResolver::new(Vec::new())); + Arc::new(ImmutablesFileDownloaderResolver::new(vec![( + ImmutablesLocationDiscriminants::CloudStorage, + http_file_downloader.clone(), + )])); #[cfg(all(feature = "fs", feature = "unstable"))] let ancillary_file_downloader_resolver = - Arc::new(AncillaryFileDownloaderResolver::new(Vec::new())); + Arc::new(AncillaryFileDownloaderResolver::new(vec![( + AncillaryLocationDiscriminants::CloudStorage, + http_file_downloader.clone(), + )])); #[cfg(all(feature = "fs", feature = "unstable"))] - let digest_file_downloader_resolver = - Arc::new(DigestFileDownloaderResolver::new(Vec::new())); + let digest_file_downloader_resolver = Arc::new(DigestFileDownloaderResolver::new(vec![ + ( + DigestLocationDiscriminants::CloudStorage, + http_file_downloader.clone(), + ), + ( + DigestLocationDiscriminants::Aggregator, + http_file_downloader.clone(), + ), + ])); #[cfg(feature = "unstable")] let cardano_database_client = Arc::new(CardanoDatabaseClient::new( aggregator_client.clone(), From b1a768195d8f2ae41cd7d6ad6f52e51163826ed5 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 17 Feb 2025 10:39:01 +0100 Subject: [PATCH 39/59] refactor: simplify Mithril event structure Co-authored-by: Damien Lachaume Co-authored-by: DJO --- .../download_unpack.rs | 79 +++++--- .../src/cardano_database_client/proving.rs | 42 +++-- mithril-client/src/feedback.rs | 176 ++++++++++++------ .../src/file_downloader/interface.rs | 119 ++++++++++++ 4 files changed, 311 insertions(+), 105 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index 63105aeec64..f6c90b906ae 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -11,7 +11,7 @@ use mithril_common::{ messages::CardanoDatabaseSnapshotMessage, }; -use crate::feedback::MithrilEvent; +use crate::feedback::{MithrilEvent, MithrilEventCardanoDatabase}; use crate::file_downloader::{FileDownloader, FileDownloaderUri}; use crate::MithrilResult; @@ -139,11 +139,13 @@ impl CardanoDatabaseClient { downloaded_bytes: u64, size: u64, ) -> Option { - Some(MithrilEvent::ImmutableDownloadProgress { - download_id, - downloaded_bytes, - size, - }) + Some(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadProgress { + download_id, + downloaded_bytes, + size, + }, + )) } fn feedback_event_builder_ancillary_download( @@ -151,11 +153,13 @@ impl CardanoDatabaseClient { downloaded_bytes: u64, size: u64, ) -> Option { - Some(MithrilEvent::AncillaryDownloadProgress { - download_id, - downloaded_bytes, - size, - }) + Some(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadProgress { + download_id, + downloaded_bytes, + size, + }, + )) } /// Download and unpack the immutable files of the given range. @@ -218,7 +222,8 @@ impl CardanoDatabaseClient { join_set.spawn(async move { let download_id = MithrilEvent::new_snapshot_download_id(); feedback_receiver_clone - .send_event(MithrilEvent::ImmutableDownloadStarted { immutable_file_number, download_id: download_id.clone()}) + .send_event(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadStarted { immutable_file_number, download_id: download_id.clone()})) .await; let downloaded = file_downloader_clone .download_unpack( @@ -232,7 +237,8 @@ impl CardanoDatabaseClient { match downloaded { Ok(_) => { feedback_receiver_clone - .send_event(MithrilEvent::ImmutableDownloadCompleted { download_id }) + .send_event(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadCompleted { immutable_file_number, download_id })) .await; Ok(immutable_file_number) @@ -318,9 +324,11 @@ impl CardanoDatabaseClient { for location in locations_sorted { let download_id = MithrilEvent::new_ancillary_download_id(); self.feedback_sender - .send_event(MithrilEvent::AncillaryDownloadStarted { - download_id: download_id.clone(), - }) + .send_event(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadStarted { + download_id: download_id.clone(), + }, + )) .await; let file_downloader = self .ancillary_file_downloader_resolver @@ -341,7 +349,9 @@ impl CardanoDatabaseClient { match downloaded { Ok(_) => { self.feedback_sender - .send_event(MithrilEvent::AncillaryDownloadCompleted { download_id }) + .send_event(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadCompleted { download_id }, + )) .await; return Ok(()); } @@ -946,13 +956,18 @@ mod tests { let sent_events = feedback_receiver.stacked_events(); let id = sent_events[0].event_id(); let expected_events = vec![ - MithrilEvent::ImmutableDownloadStarted { - immutable_file_number: 1, - download_id: id.to_string(), - }, - MithrilEvent::ImmutableDownloadCompleted { - download_id: id.to_string(), - }, + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadStarted { + immutable_file_number: 1, + download_id: id.to_string(), + }, + ), + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadCompleted { + immutable_file_number: 1, + download_id: id.to_string(), + }, + ), ]; assert_eq!(expected_events, sent_events); } @@ -1082,12 +1097,16 @@ mod tests { let sent_events = feedback_receiver.stacked_events(); let id = sent_events[0].event_id(); let expected_events = vec![ - MithrilEvent::AncillaryDownloadStarted { - download_id: id.to_string(), - }, - MithrilEvent::AncillaryDownloadCompleted { - download_id: id.to_string(), - }, + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadStarted { + download_id: id.to_string(), + }, + ), + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadCompleted { + download_id: id.to_string(), + }, + ), ]; assert_eq!(expected_events, sent_events); } diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index cb0cdf7ef47..c99af1e0a27 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -15,7 +15,11 @@ use mithril_common::{ }, }; -use crate::{feedback::MithrilEvent, file_downloader::FileDownloaderUri, MithrilResult}; +use crate::{ + feedback::{MithrilEvent, MithrilEventCardanoDatabase}, + file_downloader::FileDownloaderUri, + MithrilResult, +}; use super::api::CardanoDatabaseClient; use super::immutable_file_range::ImmutableFileRange; @@ -72,9 +76,11 @@ impl CardanoDatabaseClient { for location in locations_sorted { let download_id = MithrilEvent::new_digest_download_id(); self.feedback_sender - .send_event(MithrilEvent::DigestDownloadStarted { - download_id: download_id.clone(), - }) + .send_event(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::DigestDownloadStarted { + download_id: download_id.clone(), + }, + )) .await; let file_downloader = self .digest_file_downloader_resolver @@ -95,7 +101,9 @@ impl CardanoDatabaseClient { match downloaded { Ok(_) => { self.feedback_sender - .send_event(MithrilEvent::DigestDownloadCompleted { download_id }) + .send_event(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::DigestDownloadCompleted { download_id }, + )) .await; return Ok(()); } @@ -148,11 +156,13 @@ impl CardanoDatabaseClient { downloaded_bytes: u64, size: u64, ) -> Option { - Some(MithrilEvent::DigestDownloadProgress { - download_id, - downloaded_bytes, - size, - }) + Some(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::DigestDownloadProgress { + download_id, + downloaded_bytes, + size, + }, + )) } fn digest_target_dir(target_dir: &Path) -> PathBuf { @@ -521,12 +531,14 @@ mod tests { let sent_events = feedback_receiver.stacked_events(); let id = sent_events[0].event_id(); let expected_events = vec![ - MithrilEvent::DigestDownloadStarted { + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::DigestDownloadStarted { download_id: id.to_string(), - }, - MithrilEvent::DigestDownloadCompleted { - download_id: id.to_string(), - }, + }), + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::DigestDownloadCompleted { + download_id: id.to_string(), + }, + ), ]; assert_eq!(expected_events, sent_events); } diff --git a/mithril-client/src/feedback.rs b/mithril-client/src/feedback.rs index bc33bb6825e..2f895d15b7d 100644 --- a/mithril-client/src/feedback.rs +++ b/mithril-client/src/feedback.rs @@ -59,34 +59,11 @@ use std::sync::{Arc, RwLock}; use strum::Display; use uuid::Uuid; -/// Event that can be reported by a [FeedbackReceiver]. +/// Event that can be reported by a [FeedbackReceiver] for Cardano database related events. #[derive(Debug, Clone, Eq, PartialEq, Display, Serialize)] #[strum(serialize_all = "PascalCase")] #[serde(untagged)] -pub enum MithrilEvent { - /// A snapshot download has started - SnapshotDownloadStarted { - /// Digest of the downloaded snapshot - digest: String, - /// Unique identifier used to track this specific snapshot download - download_id: String, - /// Size of the downloaded archive - size: u64, - }, - /// A snapshot download is in progress - SnapshotDownloadProgress { - /// Unique identifier used to track this specific snapshot download - download_id: String, - /// Number of bytes that have been downloaded - downloaded_bytes: u64, - /// Size of the downloaded archive - size: u64, - }, - /// A snapshot download has completed - SnapshotDownloadCompleted { - /// Unique identifier used to track this specific snapshot download - download_id: String, - }, +pub enum MithrilEventCardanoDatabase { /// An immutable archive file download has started ImmutableDownloadStarted { /// Immutable file number downloaded @@ -96,6 +73,8 @@ pub enum MithrilEvent { }, /// An immutable archive file download is in progress ImmutableDownloadProgress { + /// Immutable file number downloaded + immutable_file_number: ImmutableFileNumber, /// Unique identifier used to track this specific download download_id: String, /// Number of bytes that have been downloaded @@ -105,6 +84,8 @@ pub enum MithrilEvent { }, /// An immutable archive file download has completed ImmutableDownloadCompleted { + /// Immutable file number downloaded + immutable_file_number: ImmutableFileNumber, /// Unique identifier used to track this specific immutable archive file download download_id: String, }, @@ -146,6 +127,40 @@ pub enum MithrilEvent { /// Unique identifier used to track this specific digest file download download_id: String, }, +} + +/// Event that can be reported by a [FeedbackReceiver]. +#[derive(Debug, Clone, Eq, PartialEq, Display, Serialize)] +#[strum(serialize_all = "PascalCase")] +#[serde(untagged)] +pub enum MithrilEvent { + /// A snapshot download has started + SnapshotDownloadStarted { + /// Digest of the downloaded snapshot + digest: String, + /// Unique identifier used to track this specific snapshot download + download_id: String, + /// Size of the downloaded archive + size: u64, + }, + /// A snapshot download is in progress + SnapshotDownloadProgress { + /// Unique identifier used to track this specific snapshot download + download_id: String, + /// Number of bytes that have been downloaded + downloaded_bytes: u64, + /// Size of the downloaded archive + size: u64, + }, + /// A snapshot download has completed + SnapshotDownloadCompleted { + /// Unique identifier used to track this specific snapshot download + download_id: String, + }, + + /// Cardano database related events + CardanoDatabase(MithrilEventCardanoDatabase), + /// A certificate chain validation has started CertificateChainValidationStarted { /// Unique identifier used to track this specific certificate chain validation @@ -204,15 +219,34 @@ impl MithrilEvent { MithrilEvent::SnapshotDownloadStarted { download_id, .. } => download_id, MithrilEvent::SnapshotDownloadProgress { download_id, .. } => download_id, MithrilEvent::SnapshotDownloadCompleted { download_id } => download_id, - MithrilEvent::ImmutableDownloadStarted { download_id, .. } => download_id, - MithrilEvent::ImmutableDownloadProgress { download_id, .. } => download_id, - MithrilEvent::ImmutableDownloadCompleted { download_id, .. } => download_id, - MithrilEvent::AncillaryDownloadStarted { download_id, .. } => download_id, - MithrilEvent::AncillaryDownloadProgress { download_id, .. } => download_id, - MithrilEvent::AncillaryDownloadCompleted { download_id, .. } => download_id, - MithrilEvent::DigestDownloadStarted { download_id, .. } => download_id, - MithrilEvent::DigestDownloadProgress { download_id, .. } => download_id, - MithrilEvent::DigestDownloadCompleted { download_id, .. } => download_id, + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadStarted { download_id, .. }, + ) => download_id, + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadProgress { download_id, .. }, + ) => download_id, + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadCompleted { download_id, .. }, + ) => download_id, + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadStarted { download_id, .. }, + ) => download_id, + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadProgress { download_id, .. }, + ) => download_id, + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadCompleted { download_id, .. }, + ) => download_id, + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::DigestDownloadStarted { + download_id, + .. + }) => download_id, + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::DigestDownloadProgress { download_id, .. }, + ) => download_id, + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::DigestDownloadCompleted { download_id, .. }, + ) => download_id, MithrilEvent::CertificateChainValidationStarted { certificate_chain_validation_id, } => certificate_chain_validation_id, @@ -303,64 +337,86 @@ impl FeedbackReceiver for SlogFeedbackReceiver { MithrilEvent::SnapshotDownloadCompleted { download_id } => { info!(self.logger, "Snapshot download completed"; "download_id" => download_id); } - MithrilEvent::ImmutableDownloadStarted { - immutable_file_number, - download_id, - } => { + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadStarted { + immutable_file_number, + download_id, + }, + ) => { info!( self.logger, "Immutable download started"; "immutable_file_number" => immutable_file_number, "download_id" => download_id, ); } - MithrilEvent::ImmutableDownloadProgress { - download_id, - downloaded_bytes, - size, - } => { + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadProgress { + immutable_file_number, + download_id, + downloaded_bytes, + size, + }, + ) => { info!( self.logger, "Immutable download in progress ..."; - "downloaded_bytes" => downloaded_bytes, "size" => size, "download_id" => download_id, + "immutable_file_number" => immutable_file_number, "downloaded_bytes" => downloaded_bytes, "size" => size, "download_id" => download_id, ); } - MithrilEvent::ImmutableDownloadCompleted { download_id } => { - info!(self.logger, "Immutable download completed"; "download_id" => download_id); + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadCompleted { + immutable_file_number, + download_id, + }, + ) => { + info!(self.logger, "Immutable download completed"; "immutable_file_number" => immutable_file_number, "download_id" => download_id); } - MithrilEvent::AncillaryDownloadStarted { download_id } => { + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadStarted { download_id }, + ) => { info!( self.logger, "Ancillary download started"; "download_id" => download_id, ); } - MithrilEvent::AncillaryDownloadProgress { - download_id, - downloaded_bytes, - size, - } => { + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadProgress { + download_id, + downloaded_bytes, + size, + }, + ) => { info!( self.logger, "Ancillary download in progress ..."; "downloaded_bytes" => downloaded_bytes, "size" => size, "download_id" => download_id, ); } - MithrilEvent::AncillaryDownloadCompleted { download_id } => { + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadCompleted { download_id }, + ) => { info!(self.logger, "Ancillary download completed"; "download_id" => download_id); } - MithrilEvent::DigestDownloadStarted { download_id } => { + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::DigestDownloadStarted { + download_id, + }) => { info!( self.logger, "Digest download started"; "download_id" => download_id, ); } - MithrilEvent::DigestDownloadProgress { - download_id, - downloaded_bytes, - size, - } => { + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::DigestDownloadProgress { + download_id, + downloaded_bytes, + size, + }, + ) => { info!( self.logger, "Digest download in progress ..."; "downloaded_bytes" => downloaded_bytes, "size" => size, "download_id" => download_id, ); } - MithrilEvent::DigestDownloadCompleted { download_id } => { + MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::DigestDownloadCompleted { download_id }, + ) => { info!(self.logger, "Digest download completed"; "download_id" => download_id); } MithrilEvent::CertificateChainValidationStarted { diff --git a/mithril-client/src/file_downloader/interface.rs b/mithril-client/src/file_downloader/interface.rs index ed8e84be4ef..252975bed04 100644 --- a/mithril-client/src/file_downloader/interface.rs +++ b/mithril-client/src/file_downloader/interface.rs @@ -82,6 +82,81 @@ impl From for FileDownloaderUri { /// A feedback event builder pub type FeedbackEventBuilder = fn(String, u64, u64) -> Option; +/// A download event +/// +/// The `download_id` is a unique identifier that allow +/// [feedback receivers][crate::feedback::FeedbackReceiver] to track concurrent downloads. +#[derive(Debug, Clone)] +pub enum DownloadEvent { + /// Immutable file download + Immutable { + /// Unique download identifier + download_id: String, + /// Immutable file number + immutable_file_number: ImmutableFileNumber, + }, + /// Ancillary file download + Ancillary { + /// Unique download identifier + download_id: String, + }, + /// Digest file download + Digest { + /// Unique download identifier + download_id: String, + }, +} + +impl DownloadEvent { + /// Get the unique download identifier + pub fn download_id(&self) -> &str { + match self { + DownloadEvent::Immutable { + immutable_file_number: _, + download_id, + } => download_id, + DownloadEvent::Ancillary { download_id } | DownloadEvent::Digest { download_id } => { + download_id + } + } + } + + /// Build a download started event + pub fn build_download_progress_event( + &self, + downloaded_bytes: u64, + total_bytes: u64, + ) -> MithrilEvent { + match self { + DownloadEvent::Immutable { + immutable_file_number, + download_id, + } => MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadProgress { + download_id: download_id.to_string(), + downloaded_bytes, + size: total_bytes, + immutable_file_number: *immutable_file_number, + }, + ), + DownloadEvent::Ancillary { download_id } => MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::AncillaryDownloadProgress { + download_id: download_id.to_string(), + downloaded_bytes, + size: total_bytes, + }, + ), + DownloadEvent::Digest { download_id } => { + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::DigestDownloadProgress { + download_id: download_id.to_string(), + downloaded_bytes, + size: total_bytes, + }) + } + } + } +} + /// A file downloader #[cfg_attr(test, mockall::automock)] #[async_trait] @@ -140,4 +215,48 @@ mod tests { ] ); } + + #[test] + fn download_event_type_builds_correct_event() { + let download_event_type = DownloadEvent::Immutable { + download_id: "download-123".to_string(), + immutable_file_number: 123, + }; + let event = download_event_type.build_download_progress_event(123, 1234); + assert_eq!( + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::ImmutableDownloadProgress { + immutable_file_number: 123, + download_id: "download-123".to_string(), + downloaded_bytes: 123, + size: 1234, + }), + event, + ); + + let download_event_type = DownloadEvent::Ancillary { + download_id: "download-123".to_string(), + }; + let event = download_event_type.build_download_progress_event(123, 1234); + assert_eq!( + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::AncillaryDownloadProgress { + download_id: "download-123".to_string(), + downloaded_bytes: 123, + size: 1234, + }), + event, + ); + + let download_event_type = DownloadEvent::Digest { + download_id: "download-123".to_string(), + }; + let event = download_event_type.build_download_progress_event(123, 1234); + assert_eq!( + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::DigestDownloadProgress { + download_id: "download-123".to_string(), + downloaded_bytes: 123, + size: 1234, + }), + event, + ); + } } From 9dd10b6f58456f8d8892f923f33a800d8bfa634c Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 17 Feb 2025 11:29:12 +0100 Subject: [PATCH 40/59] refactor: enhance FeedBackEvent handling in HTTP downloader of client Co-authored-by: Damien Lachaume Co-authored-by: DJO --- .../download_unpack.rs | 38 ++------------- .../src/cardano_database_client/proving.rs | 21 ++------- mithril-client/src/file_downloader/http.rs | 40 ++++++---------- .../src/file_downloader/interface.rs | 7 +-- .../src/file_downloader/mock_builder.rs | 16 +++---- mithril-client/src/file_downloader/mod.rs | 2 +- .../src/file_downloader/resolver.rs | 37 +++++++-------- mithril-client/src/file_downloader/retry.rs | 47 +++++++++---------- 8 files changed, 71 insertions(+), 137 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index f6c90b906ae..71a39a0805d 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -12,7 +12,7 @@ use mithril_common::{ }; use crate::feedback::{MithrilEvent, MithrilEventCardanoDatabase}; -use crate::file_downloader::{FileDownloader, FileDownloaderUri}; +use crate::file_downloader::{DownloadEvent, FileDownloader, FileDownloaderUri}; use crate::MithrilResult; use super::api::CardanoDatabaseClient; @@ -134,34 +134,6 @@ impl CardanoDatabaseClient { Ok(()) } - fn feedback_event_builder_immutable_download( - download_id: String, - downloaded_bytes: u64, - size: u64, - ) -> Option { - Some(MithrilEvent::CardanoDatabase( - MithrilEventCardanoDatabase::ImmutableDownloadProgress { - download_id, - downloaded_bytes, - size, - }, - )) - } - - fn feedback_event_builder_ancillary_download( - download_id: String, - downloaded_bytes: u64, - size: u64, - ) -> Option { - Some(MithrilEvent::CardanoDatabase( - MithrilEventCardanoDatabase::AncillaryDownloadProgress { - download_id, - downloaded_bytes, - size, - }, - )) - } - /// Download and unpack the immutable files of the given range. /// /// The download is attempted for each location until the full range is downloaded. @@ -230,8 +202,7 @@ impl CardanoDatabaseClient { &file_downloader_uri_clone, &immutable_files_target_dir_clone, Some(compression_algorithm_clone), - &download_id, - Self::feedback_event_builder_immutable_download, + DownloadEvent::Immutable{immutable_file_number, download_id: download_id.clone()}, ) .await; match downloaded { @@ -342,8 +313,9 @@ impl CardanoDatabaseClient { &file_downloader_uri, ancillary_file_target_dir, Some(compression_algorithm.to_owned()), - &download_id, - Self::feedback_event_builder_ancillary_download, + DownloadEvent::Ancillary { + download_id: download_id.clone(), + }, ) .await; match downloaded { diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index c99af1e0a27..3f8de5dbd3f 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -17,7 +17,7 @@ use mithril_common::{ use crate::{ feedback::{MithrilEvent, MithrilEventCardanoDatabase}, - file_downloader::FileDownloaderUri, + file_downloader::{DownloadEvent, FileDownloaderUri}, MithrilResult, }; @@ -94,8 +94,9 @@ impl CardanoDatabaseClient { &file_downloader_uri, digest_file_target_dir, None, - &download_id, - Self::feedback_event_builder_digest_download, + DownloadEvent::Digest { + download_id: download_id.clone(), + }, ) .await; match downloaded { @@ -151,20 +152,6 @@ impl CardanoDatabaseClient { Ok(digest_map) } - fn feedback_event_builder_digest_download( - download_id: String, - downloaded_bytes: u64, - size: u64, - ) -> Option { - Some(MithrilEvent::CardanoDatabase( - MithrilEventCardanoDatabase::DigestDownloadProgress { - download_id, - downloaded_bytes, - size, - }, - )) - } - fn digest_target_dir(target_dir: &Path) -> PathBuf { target_dir.join("digest") } diff --git a/mithril-client/src/file_downloader/http.rs b/mithril-client/src/file_downloader/http.rs index 46b260d3447..cba4d9dc16e 100644 --- a/mithril-client/src/file_downloader/http.rs +++ b/mithril-client/src/file_downloader/http.rs @@ -18,7 +18,7 @@ use crate::common::CompressionAlgorithm; use crate::feedback::FeedbackSender; use crate::utils::StreamReader; -use super::{FeedbackEventBuilder, FileDownloader, FileDownloaderUri}; +use super::{interface::DownloadEvent, FileDownloader, FileDownloaderUri}; /// A file downloader that only handles download through HTTP. pub struct HttpFileDownloader { @@ -55,25 +55,25 @@ impl HttpFileDownloader { } } - async fn download_file( + async fn download_file( &self, location: &str, sender: &Sender>, - report_progress: F, - ) -> StdResult<()> - where - F: Fn(u64) -> Fut, - Fut: std::future::Future, - { + download_event_type: DownloadEvent, + file_size: u64, + ) -> StdResult<()> { let mut downloaded_bytes: u64 = 0; let mut remote_stream = self.get(location).await?.bytes_stream(); + while let Some(item) = remote_stream.next().await { let chunk = item.with_context(|| "Download: Could not read from byte stream")?; sender.send_async(chunk.to_vec()).await.with_context(|| { format!("Download: could not write {} bytes to stream.", chunk.len()) })?; downloaded_bytes += chunk.len() as u64; - report_progress(downloaded_bytes).await + let event = + download_event_type.build_download_progress_event(downloaded_bytes, file_size); + self.feedback_sender.send_event(event).await; } Ok(()) @@ -83,7 +83,7 @@ impl HttpFileDownloader { stream: Receiver>, compression_algorithm: Option, unpack_dir: &Path, - download_id: &str, + download_id: String, ) -> StdResult<()> { let input = StreamReader::new(stream); match compression_algorithm { @@ -133,8 +133,7 @@ impl FileDownloader for HttpFileDownloader { location: &FileDownloaderUri, target_dir: &Path, compression_algorithm: Option, - download_id: &str, - feedback_event: FeedbackEventBuilder, + download_event_type: DownloadEvent, ) -> StdResult<()> { if !target_dir.is_dir() { Err( @@ -145,25 +144,14 @@ impl FileDownloader for HttpFileDownloader { let (sender, receiver) = flume::bounded(32); let dest_dir = target_dir.to_path_buf(); - let download_id_clone = download_id.to_owned(); + let download_id = download_event_type.download_id().to_owned(); let unpack_thread = tokio::task::spawn_blocking(move || -> StdResult<()> { - Self::unpack_file( - receiver, - compression_algorithm, - &dest_dir, - &download_id_clone, - ) + Self::unpack_file(receiver, compression_algorithm, &dest_dir, download_id) }); // The size will be completed with the uncompressed file size when available in the location // (see https://github.com/input-output-hk/mithril/issues/2291) let file_size = 0; - let report_progress = |downloaded_bytes: u64| async move { - if let Some(event) = feedback_event(download_id.to_owned(), downloaded_bytes, file_size) - { - self.feedback_sender.send_event(event).await - } - }; - self.download_file(location.as_str(), &sender, report_progress) + self.download_file(location.as_str(), &sender, download_event_type, file_size) .await?; drop(sender); unpack_thread diff --git a/mithril-client/src/file_downloader/interface.rs b/mithril-client/src/file_downloader/interface.rs index 252975bed04..7b0b471abdf 100644 --- a/mithril-client/src/file_downloader/interface.rs +++ b/mithril-client/src/file_downloader/interface.rs @@ -10,7 +10,7 @@ use mithril_common::{ StdResult, }; -use crate::feedback::MithrilEvent; +use crate::feedback::{MithrilEvent, MithrilEventCardanoDatabase}; /// A file downloader URI #[derive(Debug, PartialEq, Eq, Clone)] @@ -163,15 +163,12 @@ impl DownloadEvent { pub trait FileDownloader: Sync + Send { /// Download and unpack (if necessary) a file on the disk. /// - /// The `download_id` is a unique identifier that allow - /// [feedback receivers][crate::feedback::FeedbackReceiver] to track concurrent downloads. async fn download_unpack( &self, location: &FileDownloaderUri, target_dir: &Path, compression_algorithm: Option, - download_id: &str, - feedback_event_builder: FeedbackEventBuilder, + download_event_type: DownloadEvent, ) -> StdResult<()>; } diff --git a/mithril-client/src/file_downloader/mock_builder.rs b/mithril-client/src/file_downloader/mock_builder.rs index af66b32d65e..9f29106766b 100644 --- a/mithril-client/src/file_downloader/mock_builder.rs +++ b/mithril-client/src/file_downloader/mock_builder.rs @@ -1,4 +1,3 @@ - use anyhow::anyhow; use mockall::predicate; use std::path::{Path, PathBuf}; @@ -8,15 +7,14 @@ use mithril_common::{ StdResult, }; -use super::{FeedbackEventBuilder, FileDownloaderUri, MockFileDownloader}; +use super::{DownloadEvent, FileDownloaderUri, MockFileDownloader}; type MockFileDownloaderBuilderReturningFunc = Box< dyn FnMut( &FileDownloaderUri, &Path, Option, - &str, - FeedbackEventBuilder, + DownloadEvent, ) -> StdResult<()> + Send + 'static, @@ -56,12 +54,12 @@ impl MockFileDownloaderBuilder { /// The MockFileDownloader will succeed pub fn with_success(self) -> Self { - self.with_returning(Box::new(|_, _, _, _, _| Ok(()))) + self.with_returning(Box::new(|_, _, _, _| Ok(()))) } /// The MockFileDownloader will fail pub fn with_failure(self) -> Self { - self.with_returning(Box::new(|_, _, _, _, _| { + self.with_returning(Box::new(|_, _, _, _| { Err(anyhow!("Download unpack failed")) })) } @@ -128,8 +126,7 @@ impl MockFileDownloaderBuilder { .map(|x| x == u) .unwrap_or(true) }); - let predicate_download_id = predicate::always(); - let predicate_feedback_event_builder = predicate::always(); + let predicate_download_event_type = predicate::always(); let mut mock_file_downloader = self.mock_file_downloader.unwrap_or_default(); mock_file_downloader @@ -138,8 +135,7 @@ impl MockFileDownloaderBuilder { predicate_file_downloader_uri, predicate_target_dir, predicate_compression_algorithm, - predicate_download_id, - predicate_feedback_event_builder, + predicate_download_event_type, ) .times(self.times) .returning(self.returning_func.unwrap()); diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs index bfe81157333..d5d82db1e80 100644 --- a/mithril-client/src/file_downloader/mod.rs +++ b/mithril-client/src/file_downloader/mod.rs @@ -12,7 +12,7 @@ mod retry; pub use http::HttpFileDownloader; #[cfg(test)] pub use interface::MockFileDownloader; -pub use interface::{FeedbackEventBuilder, FileDownloader, FileDownloaderUri}; +pub use interface::{DownloadEvent, FeedbackEventBuilder, FileDownloader, FileDownloaderUri}; #[cfg(test)] pub use mock_builder::MockFileDownloaderBuilder; #[cfg(test)] diff --git a/mithril-client/src/file_downloader/resolver.rs b/mithril-client/src/file_downloader/resolver.rs index 596288fe338..bace010b8d8 100644 --- a/mithril-client/src/file_downloader/resolver.rs +++ b/mithril-client/src/file_downloader/resolver.rs @@ -86,28 +86,17 @@ mod tests { use mithril_common::entities::{FileUri, MultiFilesUri, TemplateUri}; - use crate::{ - feedback::MithrilEvent, - file_downloader::{FileDownloaderUri, MockFileDownloader}, - }; + use crate::file_downloader::{DownloadEvent, FileDownloaderUri, MockFileDownloader}; use super::*; - fn fake_feedback_event( - _download_id: String, - _downloaded_bytes: u64, - _size: u64, - ) -> Option { - None - } - #[tokio::test] async fn immutables_file_downloader_resolver() { let mut mock_file_downloader = MockFileDownloader::new(); mock_file_downloader .expect_download_unpack() .times(1) - .returning(|_, _, _, _, _| Ok(())); + .returning(|_, _, _, _| Ok(())); let resolver = ImmutablesFileDownloaderResolver::new(vec![( ImmutablesLocationDiscriminants::CloudStorage, Arc::new(mock_file_downloader), @@ -125,8 +114,10 @@ mod tests { &FileDownloaderUri::FileUri(FileUri("http://whatever/1.tar.gz".to_string())), Path::new("."), None, - "download_id", - fake_feedback_event, + DownloadEvent::Immutable { + download_id: "id".to_string(), + immutable_file_number: 1, + }, ) .await .unwrap(); @@ -138,7 +129,7 @@ mod tests { mock_file_downloader_cloud_storage .expect_download_unpack() .times(1) - .returning(|_, _, _, _, _| Ok(())); + .returning(|_, _, _, _| Ok(())); let resolver = AncillaryFileDownloaderResolver::new(vec![( AncillaryLocationDiscriminants::CloudStorage, Arc::new(mock_file_downloader_cloud_storage), @@ -154,8 +145,10 @@ mod tests { &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), Path::new("."), None, - "download_id", - fake_feedback_event, + DownloadEvent::Immutable { + download_id: "id".to_string(), + immutable_file_number: 1, + }, ) .await .unwrap(); @@ -168,7 +161,7 @@ mod tests { mock_file_downloader_cloud_storage .expect_download_unpack() .times(1) - .returning(|_, _, _, _, _| Ok(())); + .returning(|_, _, _, _| Ok(())); let resolver = DigestFileDownloaderResolver::new(vec![ ( DigestLocationDiscriminants::Aggregator, @@ -190,8 +183,10 @@ mod tests { &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), Path::new("."), None, - "download_id", - fake_feedback_event, + DownloadEvent::Immutable { + download_id: "id".to_string(), + immutable_file_number: 1, + }, ) .await .unwrap(); diff --git a/mithril-client/src/file_downloader/retry.rs b/mithril-client/src/file_downloader/retry.rs index 2c451cb0264..a1ee7630a35 100644 --- a/mithril-client/src/file_downloader/retry.rs +++ b/mithril-client/src/file_downloader/retry.rs @@ -3,7 +3,7 @@ use std::{path::Path, sync::Arc, time::Duration}; use async_trait::async_trait; use mithril_common::{entities::CompressionAlgorithm, StdResult}; -use super::{FeedbackEventBuilder, FileDownloader, FileDownloaderUri}; +use super::{DownloadEvent, FileDownloader, FileDownloaderUri}; /// Policy for retrying file downloads. #[derive(Debug, PartialEq, Clone)] @@ -62,8 +62,7 @@ impl FileDownloader for RetryDownloader { location: &FileDownloaderUri, target_dir: &Path, compression_algorithm: Option, - download_id: &str, - feedback_event_builder: FeedbackEventBuilder, + download_event_type: DownloadEvent, ) -> StdResult<()> { let retry_policy = &self.retry_policy; let mut nb_attempts = 0; @@ -75,8 +74,7 @@ impl FileDownloader for RetryDownloader { location, target_dir, compression_algorithm, - download_id, - feedback_event_builder, + download_event_type.clone(), ) .await { @@ -101,18 +99,10 @@ mod tests { use mithril_common::entities::FileUri; - use crate::{feedback::MithrilEvent, file_downloader::MockFileDownloaderBuilder}; + use crate::file_downloader::MockFileDownloaderBuilder; use super::*; - fn fake_feedback_event( - _download_id: String, - _downloaded_bytes: u64, - _size: u64, - ) -> Option { - None - } - #[tokio::test] async fn download_return_the_result_of_download_without_retry() { let mock_file_downloader = MockFileDownloaderBuilder::default() @@ -130,8 +120,10 @@ mod tests { &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), Path::new("."), None, - "download_id", - fake_feedback_event, + DownloadEvent::Immutable { + immutable_file_number: 1, + download_id: "download_id".to_string(), + }, ) .await .unwrap(); @@ -154,8 +146,10 @@ mod tests { &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), Path::new("."), None, - "download_id", - fake_feedback_event, + DownloadEvent::Immutable { + immutable_file_number: 1, + download_id: "download_id".to_string(), + }, ) .await .expect_err("An error should be returned when download fails"); @@ -188,8 +182,9 @@ mod tests { &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), Path::new("."), None, - "download_id", - fake_feedback_event, + DownloadEvent::Ancillary { + download_id: "download_id".to_string(), + }, ) .await .unwrap(); @@ -216,8 +211,10 @@ mod tests { &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), Path::new("."), None, - "download_id", - fake_feedback_event, + DownloadEvent::Immutable { + immutable_file_number: 1, + download_id: "download_id".to_string(), + }, ) .await .expect_err("An error should be returned when all download attempts fail"); @@ -246,8 +243,10 @@ mod tests { &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), Path::new("."), None, - "download_id", - fake_feedback_event, + DownloadEvent::Immutable { + immutable_file_number: 1, + download_id: "download_id".to_string(), + }, ) .await .expect_err("An error should be returned when all download attempts fail"); From b022b25b3bad15ad3f093d39c428f49b73aa52d0 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 17 Feb 2025 15:23:04 +0100 Subject: [PATCH 41/59] fix: add local file download to HTTP downloader --- mithril-client/src/file_downloader/http.rs | 59 ++++++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/mithril-client/src/file_downloader/http.rs b/mithril-client/src/file_downloader/http.rs index cba4d9dc16e..6fcb8b22171 100644 --- a/mithril-client/src/file_downloader/http.rs +++ b/mithril-client/src/file_downloader/http.rs @@ -8,9 +8,11 @@ use async_trait::async_trait; use flate2::read::GzDecoder; use flume::{Receiver, Sender}; use futures::StreamExt; -use reqwest::{Response, StatusCode}; +use reqwest::{Response, StatusCode, Url}; use slog::{debug, Logger}; use tar::Archive; +use tokio::fs::File; +use tokio::io::AsyncReadExt; use mithril_common::{logging::LoggerExtensions, StdResult}; @@ -55,7 +57,51 @@ impl HttpFileDownloader { } } - async fn download_file( + fn file_scheme_to_local_path(file_url: &str) -> Option { + Url::parse(file_url) + .ok() + .filter(|url| url.scheme() == "file") + .and_then(|url| url.to_file_path().ok()) + .map(|path| path.to_string_lossy().into_owned()) + } + + /// Stream the `location` directly from the local filesystem + async fn download_local_file( + &self, + local_path: &str, + sender: &Sender>, + download_event_type: DownloadEvent, + file_size: u64, + ) -> StdResult<()> { + let mut downloaded_bytes: u64 = 0; + let mut file = File::open(local_path).await?; + + loop { + // We can either allocate here each time, or clone a shared buffer into sender. + // A larger read buffer is faster, less context switches: + let mut buffer = vec![0; 16 * 1024 * 1024]; + let bytes_read = file.read(&mut buffer).await?; + if bytes_read == 0 { + break; + } + buffer.truncate(bytes_read); + sender.send_async(buffer).await.with_context(|| { + format!( + "Local file read: could not write {} bytes to stream.", + bytes_read + ) + })?; + downloaded_bytes += bytes_read as u64; + let event = + download_event_type.build_download_progress_event(downloaded_bytes, file_size); + self.feedback_sender.send_event(event).await; + } + + Ok(()) + } + + /// Stream the `location` remotely + async fn download_remote_file( &self, location: &str, sender: &Sender>, @@ -151,8 +197,13 @@ impl FileDownloader for HttpFileDownloader { // The size will be completed with the uncompressed file size when available in the location // (see https://github.com/input-output-hk/mithril/issues/2291) let file_size = 0; - self.download_file(location.as_str(), &sender, download_event_type, file_size) - .await?; + if let Some(local_path) = Self::file_scheme_to_local_path(location.as_str()) { + self.download_local_file(&local_path, &sender, download_event_type, file_size) + .await?; + } else { + self.download_remote_file(location.as_str(), &sender, download_event_type, file_size) + .await?; + } drop(sender); unpack_thread .await From a3d01d8750295b93f8ed6db499c79753e82dde53 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 17 Feb 2025 12:18:05 +0100 Subject: [PATCH 42/59] refactor: simplify resolution of digest file uploader Co-authored-by: Damien Lachaume Co-authored-by: DJO --- .../src/cardano_database_client/api.rs | 40 +++--- .../src/cardano_database_client/proving.rs | 129 ++++++------------ mithril-client/src/client.rs | 21 +-- mithril-client/src/file_downloader/mod.rs | 3 +- .../src/file_downloader/resolver.rs | 64 +-------- .../src/entities/cardano_database.rs | 5 +- mithril-common/src/entities/mod.rs | 3 +- 7 files changed, 74 insertions(+), 191 deletions(-) diff --git a/mithril-client/src/cardano_database_client/api.rs b/mithril-client/src/cardano_database_client/api.rs index 77da0b6f8a5..9f7b1c96de6 100644 --- a/mithril-client/src/cardano_database_client/api.rs +++ b/mithril-client/src/cardano_database_client/api.rs @@ -4,13 +4,13 @@ use std::sync::Arc; use slog::Logger; #[cfg(feature = "fs")] -use mithril_common::entities::{AncillaryLocation, DigestLocation, ImmutablesLocation}; +use mithril_common::entities::{AncillaryLocation, ImmutablesLocation}; use crate::aggregator_client::AggregatorClient; #[cfg(feature = "fs")] use crate::feedback::FeedbackSender; #[cfg(feature = "fs")] -use crate::file_downloader::FileDownloaderResolver; +use crate::file_downloader::{FileDownloader, FileDownloaderResolver}; /// HTTP client for CardanoDatabase API from the Aggregator pub struct CardanoDatabaseClient { @@ -22,7 +22,7 @@ pub struct CardanoDatabaseClient { pub(super) ancillary_file_downloader_resolver: Arc>, #[cfg(feature = "fs")] - pub(super) digest_file_downloader_resolver: Arc>, + pub(super) http_file_downloader: Arc, #[cfg(feature = "fs")] pub(super) feedback_sender: FeedbackSender, #[cfg(feature = "fs")] @@ -39,9 +39,7 @@ impl CardanoDatabaseClient { #[cfg(feature = "fs")] ancillary_file_downloader_resolver: Arc< dyn FileDownloaderResolver, >, - #[cfg(feature = "fs")] digest_file_downloader_resolver: Arc< - dyn FileDownloaderResolver, - >, + #[cfg(feature = "fs")] http_file_downloader: Arc, #[cfg(feature = "fs")] feedback_sender: FeedbackSender, #[cfg(feature = "fs")] logger: Logger, ) -> Self { @@ -52,7 +50,7 @@ impl CardanoDatabaseClient { #[cfg(feature = "fs")] ancillary_file_downloader_resolver, #[cfg(feature = "fs")] - digest_file_downloader_resolver, + http_file_downloader, #[cfg(feature = "fs")] feedback_sender, #[cfg(feature = "fs")] @@ -68,16 +66,15 @@ pub(crate) mod test_dependency_injector { use super::*; use mithril_common::entities::{ - AncillaryLocationDiscriminants, DigestLocationDiscriminants, - ImmutablesLocationDiscriminants, + AncillaryLocationDiscriminants, ImmutablesLocationDiscriminants, }; use crate::{ aggregator_client::MockAggregatorHTTPClient, feedback::FeedbackReceiver, file_downloader::{ - AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloader, - ImmutablesFileDownloaderResolver, + AncillaryFileDownloaderResolver, FileDownloader, ImmutablesFileDownloaderResolver, + MockFileDownloaderBuilder, }, test_utils, }; @@ -87,7 +84,7 @@ pub(crate) mod test_dependency_injector { http_client: MockAggregatorHTTPClient, immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver, ancillary_file_downloader_resolver: AncillaryFileDownloaderResolver, - digest_file_downloader_resolver: DigestFileDownloaderResolver, + http_file_downloader: Arc, feedback_receivers: Vec>, } @@ -97,7 +94,13 @@ pub(crate) mod test_dependency_injector { http_client: MockAggregatorHTTPClient::new(), immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver::new(vec![]), ancillary_file_downloader_resolver: AncillaryFileDownloaderResolver::new(vec![]), - digest_file_downloader_resolver: DigestFileDownloaderResolver::new(vec![]), + http_file_downloader: Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_success() + .with_times(0) + .build(), + ), feedback_receivers: vec![], } } @@ -137,15 +140,12 @@ pub(crate) mod test_dependency_injector { } } - pub(crate) fn with_digest_file_downloaders( + pub(crate) fn with_http_file_downloader( self, - file_downloaders: Vec<(DigestLocationDiscriminants, Arc)>, + http_file_downloader: Arc, ) -> Self { - let digest_file_downloader_resolver = - DigestFileDownloaderResolver::new(file_downloaders); - Self { - digest_file_downloader_resolver, + http_file_downloader, ..self } } @@ -165,7 +165,7 @@ pub(crate) mod test_dependency_injector { Arc::new(self.http_client), Arc::new(self.immutable_file_downloader_resolver), Arc::new(self.ancillary_file_downloader_resolver), - Arc::new(self.digest_file_downloader_resolver), + self.http_file_downloader, FeedbackSender::new(&self.feedback_receivers), test_utils::test_logger(), ) diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index 3f8de5dbd3f..e58c3edb5ed 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -82,12 +82,11 @@ impl CardanoDatabaseClient { }, )) .await; - let file_downloader = self - .digest_file_downloader_resolver - .resolve(&location) - .ok_or_else(|| { - anyhow!("Failed resolving a file downloader for location: {location:?}") - })?; + let file_downloader = match &location { + DigestLocation::CloudStorage { uri: _ } | DigestLocation::Aggregator { uri: _ } => { + self.http_file_downloader.clone() + } + }; let file_downloader_uri: FileDownloaderUri = location.into(); let downloaded = file_downloader .download_unpack( @@ -196,15 +195,14 @@ mod tests { use mithril_common::{ digesters::{DummyCardanoDbBuilder, ImmutableDigester, ImmutableFile}, - entities::{CardanoDbBeacon, DigestLocationDiscriminants, Epoch, HexEncodedDigest}, + entities::{CardanoDbBeacon, Epoch, HexEncodedDigest}, messages::{ArtifactsLocationsMessagePart, CardanoDatabaseDigestListItemMessage}, test_utils::TempDir, }; use crate::{ cardano_database_client::CardanoDatabaseClientDependencyInjector, - feedback::StackFeedbackReceiver, - file_downloader::{MockFileDownloader, MockFileDownloaderBuilder}, + feedback::StackFeedbackReceiver, file_downloader::MockFileDownloaderBuilder, test_utils::test_logger, }; @@ -332,17 +330,14 @@ mod tests { .await; let expected_merkle_root = merkle_tree.compute_root().unwrap(); let client = CardanoDatabaseClientDependencyInjector::new() - .with_digest_file_downloaders(vec![( - DigestLocationDiscriminants::CloudStorage, - Arc::new({ - MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever/digests.json") - .with_target_dir(database_dir.join("digest")) - .with_compression(None) - .with_success() - .build() - }), - )]) + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/digests.json") + .with_target_dir(database_dir.join("digest")) + .with_compression(None) + .with_success() + .build(), + )) .build_cardano_database_client(); let merkle_proof = client @@ -371,26 +366,13 @@ mod tests { async fn download_unpack_digest_file_fails_if_no_location_is_retrieved() { let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() - .with_digest_file_downloaders(vec![ - ( - DigestLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_failure() - .build(), - ), - ), - ( - DigestLocationDiscriminants::Aggregator, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_failure() - .build(), - ), - ), - ]) + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_failure() + .with_times(2) + .build(), + )) .build_cardano_database_client(); client @@ -413,26 +395,17 @@ mod tests { async fn download_unpack_digest_file_succeeds_if_at_least_one_location_is_retrieved() { let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() - .with_digest_file_downloaders(vec![ - ( - DigestLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_failure() - .build(), - ), - ), - ( - DigestLocationDiscriminants::Aggregator, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_success() - .build(), - ), - ), - ]) + .with_http_file_downloader(Arc::new({ + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_compression(None) + .with_failure() + .build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_compression(None) + .with_success() + .build() + })) .build_cardano_database_client(); client @@ -455,21 +428,12 @@ mod tests { async fn download_unpack_digest_file_succeeds_when_first_location_is_retrieved() { let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() - .with_digest_file_downloaders(vec![ - ( - DigestLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_success() - .build(), - ), - ), - ( - DigestLocationDiscriminants::Aggregator, - Arc::new(MockFileDownloader::new()), - ), - ]) + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_success() + .build(), + )) .build_cardano_database_client(); client @@ -493,15 +457,12 @@ mod tests { let target_dir = Path::new("."); let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); let client = CardanoDatabaseClientDependencyInjector::new() - .with_digest_file_downloaders(vec![( - DigestLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_compression(None) - .with_success() - .build(), - ), - )]) + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default() + .with_compression(None) + .with_success() + .build(), + )) .with_feedback_receivers(&[feedback_receiver.clone()]) .build_cardano_database_client(); diff --git a/mithril-client/src/client.rs b/mithril-client/src/client.rs index 3f8e2c28529..ecc07ad6f98 100644 --- a/mithril-client/src/client.rs +++ b/mithril-client/src/client.rs @@ -7,9 +7,7 @@ use std::sync::Arc; use mithril_common::api_version::APIVersionProvider; #[cfg(all(feature = "fs", feature = "unstable"))] -use mithril_common::entities::{ - AncillaryLocationDiscriminants, DigestLocationDiscriminants, ImmutablesLocationDiscriminants, -}; +use mithril_common::entities::{AncillaryLocationDiscriminants, ImmutablesLocationDiscriminants}; use crate::aggregator_client::{AggregatorClient, AggregatorHTTPClient}; #[cfg(feature = "unstable")] @@ -24,8 +22,8 @@ use crate::certificate_client::{ use crate::feedback::{FeedbackReceiver, FeedbackSender}; #[cfg(all(feature = "fs", feature = "unstable"))] use crate::file_downloader::{ - AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloadRetryPolicy, - HttpFileDownloader, ImmutablesFileDownloaderResolver, RetryDownloader, + AncillaryFileDownloaderResolver, FileDownloadRetryPolicy, HttpFileDownloader, + ImmutablesFileDownloaderResolver, RetryDownloader, }; use crate::mithril_stake_distribution_client::MithrilStakeDistributionClient; use crate::snapshot_client::SnapshotClient; @@ -288,17 +286,6 @@ impl ClientBuilder { AncillaryLocationDiscriminants::CloudStorage, http_file_downloader.clone(), )])); - #[cfg(all(feature = "fs", feature = "unstable"))] - let digest_file_downloader_resolver = Arc::new(DigestFileDownloaderResolver::new(vec![ - ( - DigestLocationDiscriminants::CloudStorage, - http_file_downloader.clone(), - ), - ( - DigestLocationDiscriminants::Aggregator, - http_file_downloader.clone(), - ), - ])); #[cfg(feature = "unstable")] let cardano_database_client = Arc::new(CardanoDatabaseClient::new( aggregator_client.clone(), @@ -307,7 +294,7 @@ impl ClientBuilder { #[cfg(feature = "fs")] ancillary_file_downloader_resolver, #[cfg(feature = "fs")] - digest_file_downloader_resolver, + http_file_downloader, #[cfg(feature = "fs")] feedback_sender, #[cfg(feature = "fs")] diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs index d5d82db1e80..8e4c4fbd584 100644 --- a/mithril-client/src/file_downloader/mod.rs +++ b/mithril-client/src/file_downloader/mod.rs @@ -18,7 +18,6 @@ pub use mock_builder::MockFileDownloaderBuilder; #[cfg(test)] pub use resolver::MockFileDownloaderResolver; pub use resolver::{ - AncillaryFileDownloaderResolver, DigestFileDownloaderResolver, FileDownloaderResolver, - ImmutablesFileDownloaderResolver, + AncillaryFileDownloaderResolver, FileDownloaderResolver, ImmutablesFileDownloaderResolver, }; pub use retry::{FileDownloadRetryPolicy, RetryDownloader}; diff --git a/mithril-client/src/file_downloader/resolver.rs b/mithril-client/src/file_downloader/resolver.rs index bace010b8d8..f7b41d88f40 100644 --- a/mithril-client/src/file_downloader/resolver.rs +++ b/mithril-client/src/file_downloader/resolver.rs @@ -1,8 +1,8 @@ use std::{collections::HashMap, sync::Arc}; use mithril_common::entities::{ - AncillaryLocation, AncillaryLocationDiscriminants, DigestLocation, DigestLocationDiscriminants, - ImmutablesLocation, ImmutablesLocationDiscriminants, + AncillaryLocation, AncillaryLocationDiscriminants, ImmutablesLocation, + ImmutablesLocationDiscriminants, }; use super::FileDownloader; @@ -58,28 +58,6 @@ impl FileDownloaderResolver for AncillaryFileDownloaderResolv } } -/// A file downloader resolver for digests file locations -pub struct DigestFileDownloaderResolver { - file_downloaders: HashMap>, -} - -impl DigestFileDownloaderResolver { - /// Constructs a new `DigestFileDownloaderResolver`. - pub fn new( - file_downloaders: Vec<(DigestLocationDiscriminants, Arc)>, - ) -> Self { - let file_downloaders = file_downloaders.into_iter().collect(); - - Self { file_downloaders } - } -} - -impl FileDownloaderResolver for DigestFileDownloaderResolver { - fn resolve(&self, location: &DigestLocation) -> Option> { - self.file_downloaders.get(&location.into()).cloned() - } -} - #[cfg(test)] mod tests { use std::path::Path; @@ -153,42 +131,4 @@ mod tests { .await .unwrap(); } - - #[tokio::test] - async fn digest_file_downloader_resolver() { - let mock_file_downloader_aggregator = MockFileDownloader::new(); - let mut mock_file_downloader_cloud_storage = MockFileDownloader::new(); - mock_file_downloader_cloud_storage - .expect_download_unpack() - .times(1) - .returning(|_, _, _, _| Ok(())); - let resolver = DigestFileDownloaderResolver::new(vec![ - ( - DigestLocationDiscriminants::Aggregator, - Arc::new(mock_file_downloader_aggregator), - ), - ( - DigestLocationDiscriminants::CloudStorage, - Arc::new(mock_file_downloader_cloud_storage), - ), - ]); - - let file_downloader = resolver - .resolve(&DigestLocation::CloudStorage { - uri: "http://whatever/00001.tar.gz".to_string(), - }) - .unwrap(); - file_downloader - .download_unpack( - &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), - Path::new("."), - None, - DownloadEvent::Immutable { - download_id: "id".to_string(), - immutable_file_number: 1, - }, - ) - .await - .unwrap(); - } } diff --git a/mithril-common/src/entities/cardano_database.rs b/mithril-common/src/entities/cardano_database.rs index 215244e2f15..4c7679ae6fd 100644 --- a/mithril-common/src/entities/cardano_database.rs +++ b/mithril-common/src/entities/cardano_database.rs @@ -68,11 +68,8 @@ impl CardanoDatabaseSnapshot { } /// Locations of the immutable file digests. -#[derive( - Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, EnumDiscriminants, -)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] -#[strum_discriminants(derive(Hash))] pub enum DigestLocation { /// Cloud storage location. CloudStorage { diff --git a/mithril-common/src/entities/mod.rs b/mithril-common/src/entities/mod.rs index aede13ff17f..d5ef53b1d94 100644 --- a/mithril-common/src/entities/mod.rs +++ b/mithril-common/src/entities/mod.rs @@ -35,8 +35,7 @@ pub use block_range::{BlockRange, BlockRangeLength, BlockRangesSequence}; pub use cardano_chain_point::{BlockHash, ChainPoint}; pub use cardano_database::{ AncillaryLocation, AncillaryLocationDiscriminants, ArtifactsLocations, CardanoDatabaseSnapshot, - DigestLocation, DigestLocationDiscriminants, ImmutablesLocation, - ImmutablesLocationDiscriminants, + DigestLocation, ImmutablesLocation, ImmutablesLocationDiscriminants, }; pub use cardano_db_beacon::CardanoDbBeacon; pub use cardano_network::CardanoNetwork; From 8e2ab4a3a5606a1093a4ffd6f0377b6c2f786678 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 17 Feb 2025 12:44:57 +0100 Subject: [PATCH 43/59] feat: add start/end events for Cardano database in client Co-authored-by: Damien Lachaume Co-authored-by: DJO --- .../download_unpack.rs | 33 +++++++++++-- mithril-client/src/feedback.rs | 48 +++++++++++++++---- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index 71a39a0805d..b2e887f4611 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -41,6 +41,14 @@ impl CardanoDatabaseClient { target_dir: &Path, download_unpack_options: DownloadUnpackOptions, ) -> MithrilResult<()> { + let download_id = MithrilEvent::new_snapshot_download_id(); + self.feedback_sender + .send_event(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::Started { + download_id: download_id.clone(), + }, + )) + .await; let compression_algorithm = cardano_database_snapshot.compression_algorithm; let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; let immutable_file_number_range = @@ -57,6 +65,7 @@ impl CardanoDatabaseClient { immutable_file_number_range, &compression_algorithm, target_dir, + &download_id, ) .await?; if download_unpack_options.include_ancillary { @@ -68,6 +77,13 @@ impl CardanoDatabaseClient { ) .await?; } + self.feedback_sender + .send_event(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::Completed { + download_id: download_id.clone(), + }, + )) + .await; Ok(()) } @@ -144,6 +160,7 @@ impl CardanoDatabaseClient { range: RangeInclusive, compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path, + download_id: &str, ) -> MithrilResult<()> { let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); @@ -156,6 +173,7 @@ impl CardanoDatabaseClient { &immutable_file_numbers_to_download, compression_algorithm, immutable_files_target_dir, + download_id, ) .await?; for immutable_file_number in immutable_files_numbers_downloaded { @@ -181,6 +199,7 @@ impl CardanoDatabaseClient { file_downloader_uris_chunk: Vec<(ImmutableFileNumber, FileDownloaderUri)>, compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path, + download_id: &str, ) -> MithrilResult> { let mut immutable_file_numbers_downloaded = BTreeSet::new(); let mut join_set: JoinSet> = JoinSet::new(); @@ -191,25 +210,25 @@ impl CardanoDatabaseClient { let file_downloader_clone = file_downloader.clone(); let feedback_receiver_clone = self.feedback_sender.clone(); let logger_clone = self.logger.clone(); + let download_id_clone = download_id.to_string(); join_set.spawn(async move { - let download_id = MithrilEvent::new_snapshot_download_id(); feedback_receiver_clone .send_event(MithrilEvent::CardanoDatabase( - MithrilEventCardanoDatabase::ImmutableDownloadStarted { immutable_file_number, download_id: download_id.clone()})) + MithrilEventCardanoDatabase::ImmutableDownloadStarted { immutable_file_number, download_id: download_id_clone.clone()})) .await; let downloaded = file_downloader_clone .download_unpack( &file_downloader_uri_clone, &immutable_files_target_dir_clone, Some(compression_algorithm_clone), - DownloadEvent::Immutable{immutable_file_number, download_id: download_id.clone()}, + DownloadEvent::Immutable{immutable_file_number, download_id: download_id_clone.clone()}, ) .await; match downloaded { Ok(_) => { feedback_receiver_clone .send_event(MithrilEvent::CardanoDatabase( - MithrilEventCardanoDatabase::ImmutableDownloadCompleted { immutable_file_number, download_id })) + MithrilEventCardanoDatabase::ImmutableDownloadCompleted { immutable_file_number, download_id: download_id_clone })) .await; Ok(immutable_file_number) @@ -247,6 +266,7 @@ impl CardanoDatabaseClient { immutable_file_numbers_to_download: &BTreeSet, compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path, + download_id: &str, ) -> MithrilResult> { let mut immutable_file_numbers_downloaded = BTreeSet::new(); let file_downloader = self @@ -275,6 +295,7 @@ impl CardanoDatabaseClient { file_downloader_uris_chunk, compression_algorithm, immutable_files_target_dir, + download_id, ) .await?; immutable_file_numbers_downloaded.extend(immutable_file_numbers_downloaded_chunk); @@ -792,6 +813,7 @@ mod tests { .unwrap(), &CompressionAlgorithm::default(), &target_dir, + "download_id", ) .await .expect_err("download_unpack_immutable_files should fail"); @@ -831,6 +853,7 @@ mod tests { .unwrap(), &CompressionAlgorithm::default(), &target_dir, + "download_id", ) .await .unwrap(); @@ -890,6 +913,7 @@ mod tests { .unwrap(), &CompressionAlgorithm::default(), &target_dir, + "download_id", ) .await .unwrap(); @@ -921,6 +945,7 @@ mod tests { .unwrap(), &CompressionAlgorithm::default(), target_dir, + "download_id", ) .await .unwrap(); diff --git a/mithril-client/src/feedback.rs b/mithril-client/src/feedback.rs index 2f895d15b7d..27697678f5a 100644 --- a/mithril-client/src/feedback.rs +++ b/mithril-client/src/feedback.rs @@ -64,18 +64,28 @@ use uuid::Uuid; #[strum(serialize_all = "PascalCase")] #[serde(untagged)] pub enum MithrilEventCardanoDatabase { + /// Cardano Database download sequence started + Started { + /// Unique identifier used to track a cardano database download + download_id: String, + }, + /// Cardano Database download sequence completed + Completed { + /// Unique identifier used to track a cardano database download + download_id: String, + }, /// An immutable archive file download has started ImmutableDownloadStarted { /// Immutable file number downloaded immutable_file_number: ImmutableFileNumber, - /// Unique identifier used to track this specific immutable archive file download + /// Unique identifier used to track a cardano database download download_id: String, }, /// An immutable archive file download is in progress ImmutableDownloadProgress { /// Immutable file number downloaded immutable_file_number: ImmutableFileNumber, - /// Unique identifier used to track this specific download + /// Unique identifier used to track a cardano database download download_id: String, /// Number of bytes that have been downloaded downloaded_bytes: u64, @@ -86,17 +96,17 @@ pub enum MithrilEventCardanoDatabase { ImmutableDownloadCompleted { /// Immutable file number downloaded immutable_file_number: ImmutableFileNumber, - /// Unique identifier used to track this specific immutable archive file download + /// Unique identifier used to track a cardano database download download_id: String, }, /// An ancillary archive file download has started AncillaryDownloadStarted { - /// Unique identifier used to track this specific ancillary archive file download + /// Unique identifier used to track a cardano database download download_id: String, }, /// An ancillary archive file download is in progress AncillaryDownloadProgress { - /// Unique identifier used to track this specific download + /// Unique identifier used to track a cardano database download download_id: String, /// Number of bytes that have been downloaded downloaded_bytes: u64, @@ -105,17 +115,17 @@ pub enum MithrilEventCardanoDatabase { }, /// An ancillary archive file download has completed AncillaryDownloadCompleted { - /// Unique identifier used to track this specific ancillary archive file download + /// Unique identifier used to track a cardano database download download_id: String, }, /// A digest file download has started DigestDownloadStarted { - /// Unique identifier used to track this specific digest file download + /// Unique identifier used to track a cardano database download download_id: String, }, /// A digest file download is in progress DigestDownloadProgress { - /// Unique identifier used to track this specific download + /// Unique identifier used to track a cardano database download download_id: String, /// Number of bytes that have been downloaded downloaded_bytes: u64, @@ -124,7 +134,7 @@ pub enum MithrilEventCardanoDatabase { }, /// A digest file download has completed DigestDownloadCompleted { - /// Unique identifier used to track this specific digest file download + /// Unique identifier used to track a cardano database download download_id: String, }, } @@ -219,6 +229,14 @@ impl MithrilEvent { MithrilEvent::SnapshotDownloadStarted { download_id, .. } => download_id, MithrilEvent::SnapshotDownloadProgress { download_id, .. } => download_id, MithrilEvent::SnapshotDownloadCompleted { download_id } => download_id, + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::Started { + download_id, + .. + }) => download_id, + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::Completed { + download_id, + .. + }) => download_id, MithrilEvent::CardanoDatabase( MithrilEventCardanoDatabase::ImmutableDownloadStarted { download_id, .. }, ) => download_id, @@ -337,6 +355,18 @@ impl FeedbackReceiver for SlogFeedbackReceiver { MithrilEvent::SnapshotDownloadCompleted { download_id } => { info!(self.logger, "Snapshot download completed"; "download_id" => download_id); } + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::Started { download_id }) => { + info!( + self.logger, "Cardano database download started"; "download_id" => download_id, + ); + } + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::Completed { + download_id, + }) => { + info!( + self.logger, "Cardano database download completed"; "download_id" => download_id, + ); + } MithrilEvent::CardanoDatabase( MithrilEventCardanoDatabase::ImmutableDownloadStarted { immutable_file_number, From b7d1807fc4f9ff0f839f31576074a81fd00b0ff7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 17 Feb 2025 15:11:37 +0100 Subject: [PATCH 44/59] refactor: simplify resolution of immutable and ancillary file uploaders --- .../src/cardano_database_client/api.rs | 62 +---- .../download_unpack.rs | 213 ++++++++---------- mithril-client/src/client.rs | 23 +- mithril-client/src/file_downloader/mod.rs | 6 - .../src/file_downloader/resolver.rs | 134 ----------- .../src/entities/cardano_database.rs | 11 +- mithril-common/src/entities/mod.rs | 4 +- 7 files changed, 95 insertions(+), 358 deletions(-) delete mode 100644 mithril-client/src/file_downloader/resolver.rs diff --git a/mithril-client/src/cardano_database_client/api.rs b/mithril-client/src/cardano_database_client/api.rs index 9f7b1c96de6..34a78c7e0f6 100644 --- a/mithril-client/src/cardano_database_client/api.rs +++ b/mithril-client/src/cardano_database_client/api.rs @@ -3,25 +3,16 @@ use std::sync::Arc; #[cfg(feature = "fs")] use slog::Logger; -#[cfg(feature = "fs")] -use mithril_common::entities::{AncillaryLocation, ImmutablesLocation}; - use crate::aggregator_client::AggregatorClient; #[cfg(feature = "fs")] use crate::feedback::FeedbackSender; #[cfg(feature = "fs")] -use crate::file_downloader::{FileDownloader, FileDownloaderResolver}; +use crate::file_downloader::FileDownloader; /// HTTP client for CardanoDatabase API from the Aggregator pub struct CardanoDatabaseClient { pub(super) aggregator_client: Arc, #[cfg(feature = "fs")] - pub(super) immutable_file_downloader_resolver: - Arc>, - #[cfg(feature = "fs")] - pub(super) ancillary_file_downloader_resolver: - Arc>, - #[cfg(feature = "fs")] pub(super) http_file_downloader: Arc, #[cfg(feature = "fs")] pub(super) feedback_sender: FeedbackSender, @@ -33,12 +24,6 @@ impl CardanoDatabaseClient { /// Constructs a new `CardanoDatabase`. pub fn new( aggregator_client: Arc, - #[cfg(feature = "fs")] immutable_file_downloader_resolver: Arc< - dyn FileDownloaderResolver, - >, - #[cfg(feature = "fs")] ancillary_file_downloader_resolver: Arc< - dyn FileDownloaderResolver, - >, #[cfg(feature = "fs")] http_file_downloader: Arc, #[cfg(feature = "fs")] feedback_sender: FeedbackSender, #[cfg(feature = "fs")] logger: Logger, @@ -46,10 +31,6 @@ impl CardanoDatabaseClient { Self { aggregator_client, #[cfg(feature = "fs")] - immutable_file_downloader_resolver, - #[cfg(feature = "fs")] - ancillary_file_downloader_resolver, - #[cfg(feature = "fs")] http_file_downloader, #[cfg(feature = "fs")] feedback_sender, @@ -65,25 +46,16 @@ impl CardanoDatabaseClient { pub(crate) mod test_dependency_injector { use super::*; - use mithril_common::entities::{ - AncillaryLocationDiscriminants, ImmutablesLocationDiscriminants, - }; - use crate::{ aggregator_client::MockAggregatorHTTPClient, feedback::FeedbackReceiver, - file_downloader::{ - AncillaryFileDownloaderResolver, FileDownloader, ImmutablesFileDownloaderResolver, - MockFileDownloaderBuilder, - }, + file_downloader::{FileDownloader, MockFileDownloaderBuilder}, test_utils, }; /// Dependency injector for `CardanoDatabaseClient` for testing purposes. pub(crate) struct CardanoDatabaseClientDependencyInjector { http_client: MockAggregatorHTTPClient, - immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver, - ancillary_file_downloader_resolver: AncillaryFileDownloaderResolver, http_file_downloader: Arc, feedback_receivers: Vec>, } @@ -92,8 +64,6 @@ pub(crate) mod test_dependency_injector { pub(crate) fn new() -> Self { Self { http_client: MockAggregatorHTTPClient::new(), - immutable_file_downloader_resolver: ImmutablesFileDownloaderResolver::new(vec![]), - ancillary_file_downloader_resolver: AncillaryFileDownloaderResolver::new(vec![]), http_file_downloader: Arc::new( MockFileDownloaderBuilder::default() .with_compression(None) @@ -114,32 +84,6 @@ pub(crate) mod test_dependency_injector { self } - pub(crate) fn with_immutable_file_downloaders( - self, - file_downloaders: Vec<(ImmutablesLocationDiscriminants, Arc)>, - ) -> Self { - let immutable_file_downloader_resolver = - ImmutablesFileDownloaderResolver::new(file_downloaders); - - Self { - immutable_file_downloader_resolver, - ..self - } - } - - pub(crate) fn with_ancillary_file_downloaders( - self, - file_downloaders: Vec<(AncillaryLocationDiscriminants, Arc)>, - ) -> Self { - let ancillary_file_downloader_resolver = - AncillaryFileDownloaderResolver::new(file_downloaders); - - Self { - ancillary_file_downloader_resolver, - ..self - } - } - pub(crate) fn with_http_file_downloader( self, http_file_downloader: Arc, @@ -163,8 +107,6 @@ pub(crate) mod test_dependency_injector { pub(crate) fn build_cardano_database_client(self) -> CardanoDatabaseClient { CardanoDatabaseClient::new( Arc::new(self.http_client), - Arc::new(self.immutable_file_downloader_resolver), - Arc::new(self.ancillary_file_downloader_resolver), self.http_file_downloader, FeedbackSender::new(&self.feedback_receivers), test_utils::test_logger(), diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index b2e887f4611..a226339ec86 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -269,12 +269,9 @@ impl CardanoDatabaseClient { download_id: &str, ) -> MithrilResult> { let mut immutable_file_numbers_downloaded = BTreeSet::new(); - let file_downloader = self - .immutable_file_downloader_resolver - .resolve(location) - .ok_or_else(|| { - anyhow!("Failed resolving a file downloader for location: {location:?}") - })?; + let file_downloader = match &location { + ImmutablesLocation::CloudStorage { uri: _ } => self.http_file_downloader.clone(), + }; let file_downloader_uris = FileDownloaderUri::expand_immutable_files_location_to_file_downloader_uris( location, @@ -322,12 +319,9 @@ impl CardanoDatabaseClient { }, )) .await; - let file_downloader = self - .ancillary_file_downloader_resolver - .resolve(&location) - .ok_or_else(|| { - anyhow!("Failed resolving a file downloader for location: {location:?}") - })?; + let file_downloader = match &location { + AncillaryLocation::CloudStorage { uri: _ } => self.http_file_downloader.clone(), + }; let file_downloader_uri: FileDownloaderUri = location.into(); let downloaded = file_downloader .download_unpack( @@ -369,10 +363,7 @@ mod tests { use std::{fs, sync::Arc}; use mithril_common::{ - entities::{ - AncillaryLocationDiscriminants, CardanoDbBeacon, Epoch, - ImmutablesLocationDiscriminants, MultiFilesUri, TemplateUri, - }, + entities::{CardanoDbBeacon, Epoch, MultiFilesUri, TemplateUri}, messages::{ ArtifactsLocationsMessagePart, CardanoDatabaseSnapshotMessage as CardanoDatabaseSnapshot, @@ -436,15 +427,12 @@ mod tests { ) .build(); let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - MockFileDownloaderBuilder::default() - .with_times(total_immutable_files as usize) - .with_failure() - .build() - }), - )]) + .with_http_file_downloader(Arc::new({ + MockFileDownloaderBuilder::default() + .with_times(total_immutable_files as usize) + .with_failure() + .build() + })) .build_cardano_database_client(); client @@ -519,33 +507,26 @@ mod tests { ) .build(); let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - let mock_file_downloader = MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever/00001.tar.gz") - .with_target_dir(target_dir.clone()) - .with_success() - .build(); - + .with_http_file_downloader(Arc::new({ + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever/00001.tar.gz") + .with_target_dir(target_dir.clone()) + .with_success() + .build(); + let mock_file_downloader = MockFileDownloaderBuilder::from_mock(mock_file_downloader) .with_file_uri("http://whatever/00002.tar.gz") .with_target_dir(target_dir.clone()) .with_success() - .build() - }), - )]) - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever/ancillary.tar.gz") - .with_target_dir(target_dir.clone()) - .with_compression(Some(CompressionAlgorithm::default())) - .with_success() - .build(), - ), - )]) + .build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever/ancillary.tar.gz") + .with_target_dir(target_dir.clone()) + .with_compression(Some(CompressionAlgorithm::default())) + .with_success() + .build() + })) .build_cardano_database_client(); client @@ -788,17 +769,14 @@ mod tests { ) .build(); let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - let mock_file_downloader = - MockFileDownloaderBuilder::default().with_failure().build(); - - MockFileDownloaderBuilder::from_mock(mock_file_downloader) - .with_success() - .build() - }), - )]) + .with_http_file_downloader(Arc::new({ + let mock_file_downloader = + MockFileDownloaderBuilder::default().with_failure().build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_success() + .build() + })) .build_cardano_database_client(); client @@ -830,15 +808,12 @@ mod tests { ) .build(); let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_times(2) - .with_success() - .build(), - ), - )]) + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default() + .with_times(2) + .with_success() + .build(), + )) .build_cardano_database_client(); client @@ -870,28 +845,25 @@ mod tests { ) .build(); let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new({ - let mock_file_downloader = MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever-1/00001.tar.gz") - .with_target_dir(target_dir.clone()) - .with_failure() - .build(); - let mock_file_downloader = - MockFileDownloaderBuilder::from_mock(mock_file_downloader) - .with_file_uri("http://whatever-1/00002.tar.gz") - .with_target_dir(target_dir.clone()) - .with_success() - .build(); - + .with_http_file_downloader(Arc::new({ + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever-1/00001.tar.gz") + .with_target_dir(target_dir.clone()) + .with_failure() + .build(); + let mock_file_downloader = MockFileDownloaderBuilder::from_mock(mock_file_downloader) - .with_file_uri("http://whatever-2/00001.tar.gz") + .with_file_uri("http://whatever-1/00002.tar.gz") .with_target_dir(target_dir.clone()) .with_success() - .build() - }), - )]) + .build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever-2/00001.tar.gz") + .with_target_dir(target_dir.clone()) + .with_success() + .build() + })) .build_cardano_database_client(); client @@ -926,10 +898,9 @@ mod tests { let target_dir = Path::new("."); let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); let client = CardanoDatabaseClientDependencyInjector::new() - .with_immutable_file_downloaders(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new(MockFileDownloaderBuilder::default().with_success().build()), - )]) + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default().with_success().build(), + )) .with_feedback_receivers(&[feedback_receiver.clone()]) .build_cardano_database_client(); @@ -978,10 +949,9 @@ mod tests { async fn download_unpack_ancillary_file_fails_if_no_location_is_retrieved() { let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new(MockFileDownloaderBuilder::default().with_failure().build()), - )]) + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default().with_failure().build(), + )) .build_cardano_database_client(); client @@ -1000,22 +970,19 @@ mod tests { async fn download_unpack_ancillary_file_succeeds_if_at_least_one_location_is_retrieved() { let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new({ - let mock_file_downloader = MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever-1/ancillary.tar.gz") - .with_target_dir(target_dir.to_path_buf()) - .with_failure() - .build(); - - MockFileDownloaderBuilder::from_mock(mock_file_downloader) - .with_file_uri("http://whatever-2/ancillary.tar.gz") - .with_target_dir(target_dir.to_path_buf()) - .with_success() - .build() - }), - )]) + .with_http_file_downloader(Arc::new({ + let mock_file_downloader = MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever-1/ancillary.tar.gz") + .with_target_dir(target_dir.to_path_buf()) + .with_failure() + .build(); + + MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .with_file_uri("http://whatever-2/ancillary.tar.gz") + .with_target_dir(target_dir.to_path_buf()) + .with_success() + .build() + })) .build_cardano_database_client(); client @@ -1039,16 +1006,13 @@ mod tests { async fn download_unpack_ancillary_file_succeeds_when_first_location_is_retrieved() { let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new( - MockFileDownloaderBuilder::default() - .with_file_uri("http://whatever-1/ancillary.tar.gz") - .with_target_dir(target_dir.to_path_buf()) - .with_success() - .build(), - ), - )]) + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default() + .with_file_uri("http://whatever-1/ancillary.tar.gz") + .with_target_dir(target_dir.to_path_buf()) + .with_success() + .build(), + )) .build_cardano_database_client(); client @@ -1073,10 +1037,9 @@ mod tests { let target_dir = Path::new("."); let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); let client = CardanoDatabaseClientDependencyInjector::new() - .with_ancillary_file_downloaders(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new(MockFileDownloaderBuilder::default().with_success().build()), - )]) + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default().with_success().build(), + )) .with_feedback_receivers(&[feedback_receiver.clone()]) .build_cardano_database_client(); diff --git a/mithril-client/src/client.rs b/mithril-client/src/client.rs index ecc07ad6f98..acd037a685a 100644 --- a/mithril-client/src/client.rs +++ b/mithril-client/src/client.rs @@ -6,8 +6,6 @@ use std::collections::HashMap; use std::sync::Arc; use mithril_common::api_version::APIVersionProvider; -#[cfg(all(feature = "fs", feature = "unstable"))] -use mithril_common::entities::{AncillaryLocationDiscriminants, ImmutablesLocationDiscriminants}; use crate::aggregator_client::{AggregatorClient, AggregatorHTTPClient}; #[cfg(feature = "unstable")] @@ -21,10 +19,7 @@ use crate::certificate_client::{ }; use crate::feedback::{FeedbackReceiver, FeedbackSender}; #[cfg(all(feature = "fs", feature = "unstable"))] -use crate::file_downloader::{ - AncillaryFileDownloaderResolver, FileDownloadRetryPolicy, HttpFileDownloader, - ImmutablesFileDownloaderResolver, RetryDownloader, -}; +use crate::file_downloader::{FileDownloadRetryPolicy, HttpFileDownloader, RetryDownloader}; use crate::mithril_stake_distribution_client::MithrilStakeDistributionClient; use crate::snapshot_client::SnapshotClient; #[cfg(feature = "fs")] @@ -274,26 +269,10 @@ impl ClientBuilder { ), FileDownloadRetryPolicy::default(), )); - #[cfg(all(feature = "fs", feature = "unstable"))] - let immutable_file_downloader_resolver = - Arc::new(ImmutablesFileDownloaderResolver::new(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - http_file_downloader.clone(), - )])); - #[cfg(all(feature = "fs", feature = "unstable"))] - let ancillary_file_downloader_resolver = - Arc::new(AncillaryFileDownloaderResolver::new(vec![( - AncillaryLocationDiscriminants::CloudStorage, - http_file_downloader.clone(), - )])); #[cfg(feature = "unstable")] let cardano_database_client = Arc::new(CardanoDatabaseClient::new( aggregator_client.clone(), #[cfg(feature = "fs")] - immutable_file_downloader_resolver, - #[cfg(feature = "fs")] - ancillary_file_downloader_resolver, - #[cfg(feature = "fs")] http_file_downloader, #[cfg(feature = "fs")] feedback_sender, diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs index 8e4c4fbd584..1c9a0b7f70c 100644 --- a/mithril-client/src/file_downloader/mod.rs +++ b/mithril-client/src/file_downloader/mod.rs @@ -6,7 +6,6 @@ mod http; mod interface; #[cfg(test)] mod mock_builder; -mod resolver; mod retry; pub use http::HttpFileDownloader; @@ -15,9 +14,4 @@ pub use interface::MockFileDownloader; pub use interface::{DownloadEvent, FeedbackEventBuilder, FileDownloader, FileDownloaderUri}; #[cfg(test)] pub use mock_builder::MockFileDownloaderBuilder; -#[cfg(test)] -pub use resolver::MockFileDownloaderResolver; -pub use resolver::{ - AncillaryFileDownloaderResolver, FileDownloaderResolver, ImmutablesFileDownloaderResolver, -}; pub use retry::{FileDownloadRetryPolicy, RetryDownloader}; diff --git a/mithril-client/src/file_downloader/resolver.rs b/mithril-client/src/file_downloader/resolver.rs deleted file mode 100644 index f7b41d88f40..00000000000 --- a/mithril-client/src/file_downloader/resolver.rs +++ /dev/null @@ -1,134 +0,0 @@ -use std::{collections::HashMap, sync::Arc}; - -use mithril_common::entities::{ - AncillaryLocation, AncillaryLocationDiscriminants, ImmutablesLocation, - ImmutablesLocationDiscriminants, -}; - -use super::FileDownloader; - -/// A file downloader resolver -#[cfg_attr(test, mockall::automock)] -pub trait FileDownloaderResolver: Sync + Send { - /// Resolve a file downloader for the given location. - fn resolve(&self, location: &L) -> Option>; -} - -/// A file downloader resolver for immutable file locations -pub struct ImmutablesFileDownloaderResolver { - file_downloaders: HashMap>, -} - -impl ImmutablesFileDownloaderResolver { - /// Constructs a new `ImmutablesFileDownloaderResolver`. - pub fn new( - file_downloaders: Vec<(ImmutablesLocationDiscriminants, Arc)>, - ) -> Self { - let file_downloaders = file_downloaders.into_iter().collect(); - - Self { file_downloaders } - } -} - -impl FileDownloaderResolver for ImmutablesFileDownloaderResolver { - fn resolve(&self, location: &ImmutablesLocation) -> Option> { - self.file_downloaders.get(&location.into()).cloned() - } -} - -/// A file downloader resolver for ancillary file locations -pub struct AncillaryFileDownloaderResolver { - file_downloaders: HashMap>, -} - -impl AncillaryFileDownloaderResolver { - /// Constructs a new `AncillaryFileDownloaderResolver`. - pub fn new( - file_downloaders: Vec<(AncillaryLocationDiscriminants, Arc)>, - ) -> Self { - let file_downloaders = file_downloaders.into_iter().collect(); - - Self { file_downloaders } - } -} - -impl FileDownloaderResolver for AncillaryFileDownloaderResolver { - fn resolve(&self, location: &AncillaryLocation) -> Option> { - self.file_downloaders.get(&location.into()).cloned() - } -} - -#[cfg(test)] -mod tests { - use std::path::Path; - - use mithril_common::entities::{FileUri, MultiFilesUri, TemplateUri}; - - use crate::file_downloader::{DownloadEvent, FileDownloaderUri, MockFileDownloader}; - - use super::*; - - #[tokio::test] - async fn immutables_file_downloader_resolver() { - let mut mock_file_downloader = MockFileDownloader::new(); - mock_file_downloader - .expect_download_unpack() - .times(1) - .returning(|_, _, _, _| Ok(())); - let resolver = ImmutablesFileDownloaderResolver::new(vec![( - ImmutablesLocationDiscriminants::CloudStorage, - Arc::new(mock_file_downloader), - )]); - - let file_downloader = resolver - .resolve(&ImmutablesLocation::CloudStorage { - uri: MultiFilesUri::Template(TemplateUri( - "http://whatever/{immutable_file_number}.tar.gz".to_string(), - )), - }) - .unwrap(); - file_downloader - .download_unpack( - &FileDownloaderUri::FileUri(FileUri("http://whatever/1.tar.gz".to_string())), - Path::new("."), - None, - DownloadEvent::Immutable { - download_id: "id".to_string(), - immutable_file_number: 1, - }, - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn ancillary_file_downloader_resolver() { - let mut mock_file_downloader_cloud_storage = MockFileDownloader::new(); - mock_file_downloader_cloud_storage - .expect_download_unpack() - .times(1) - .returning(|_, _, _, _| Ok(())); - let resolver = AncillaryFileDownloaderResolver::new(vec![( - AncillaryLocationDiscriminants::CloudStorage, - Arc::new(mock_file_downloader_cloud_storage), - )]); - - let file_downloader = resolver - .resolve(&AncillaryLocation::CloudStorage { - uri: "http://whatever/00001.tar.gz".to_string(), - }) - .unwrap(); - file_downloader - .download_unpack( - &FileDownloaderUri::FileUri(FileUri("http://whatever/00001.tar.gz".to_string())), - Path::new("."), - None, - DownloadEvent::Immutable { - download_id: "id".to_string(), - immutable_file_number: 1, - }, - ) - .await - .unwrap(); - } -} diff --git a/mithril-common/src/entities/cardano_database.rs b/mithril-common/src/entities/cardano_database.rs index 4c7679ae6fd..e02ce4d4a50 100644 --- a/mithril-common/src/entities/cardano_database.rs +++ b/mithril-common/src/entities/cardano_database.rs @@ -1,7 +1,6 @@ use semver::Version; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use strum::EnumDiscriminants; use crate::entities::{CardanoDbBeacon, CompressionAlgorithm}; @@ -84,11 +83,8 @@ pub enum DigestLocation { } /// Locations of the immutable files. -#[derive( - Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, EnumDiscriminants, -)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] -#[strum_discriminants(derive(Hash))] pub enum ImmutablesLocation { /// Cloud storage location. CloudStorage { @@ -98,11 +94,8 @@ pub enum ImmutablesLocation { } /// Locations of the ancillary files. -#[derive( - Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, EnumDiscriminants, -)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(rename_all = "snake_case", tag = "type")] -#[strum_discriminants(derive(Hash))] pub enum AncillaryLocation { /// Cloud storage location. CloudStorage { diff --git a/mithril-common/src/entities/mod.rs b/mithril-common/src/entities/mod.rs index d5ef53b1d94..6607f0501cc 100644 --- a/mithril-common/src/entities/mod.rs +++ b/mithril-common/src/entities/mod.rs @@ -34,8 +34,8 @@ pub use block_number::BlockNumber; pub use block_range::{BlockRange, BlockRangeLength, BlockRangesSequence}; pub use cardano_chain_point::{BlockHash, ChainPoint}; pub use cardano_database::{ - AncillaryLocation, AncillaryLocationDiscriminants, ArtifactsLocations, CardanoDatabaseSnapshot, - DigestLocation, ImmutablesLocation, ImmutablesLocationDiscriminants, + AncillaryLocation, ArtifactsLocations, CardanoDatabaseSnapshot, DigestLocation, + ImmutablesLocation, }; pub use cardano_db_beacon::CardanoDbBeacon; pub use cardano_network::CardanoNetwork; From 6a322a8e9d826f7baeea848c51c1b0696a4bdc1e Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 17 Feb 2025 16:40:12 +0100 Subject: [PATCH 45/59] refactor: remove 'SnapshotDownloader' in client library --- mithril-client/src/client.rs | 43 +--- .../src/file_downloader/interface.rs | 35 ++- mithril-client/src/lib.rs | 1 - mithril-client/src/snapshot_client.rs | 88 +++---- mithril-client/src/snapshot_downloader.rs | 233 ------------------ mithril-client/src/utils/mod.rs | 2 - mithril-client/src/utils/unpacker.rs | 51 ---- 7 files changed, 84 insertions(+), 369 deletions(-) delete mode 100644 mithril-client/src/snapshot_downloader.rs delete mode 100644 mithril-client/src/utils/unpacker.rs diff --git a/mithril-client/src/client.rs b/mithril-client/src/client.rs index acd037a685a..e4b64a31b3e 100644 --- a/mithril-client/src/client.rs +++ b/mithril-client/src/client.rs @@ -22,8 +22,6 @@ use crate::feedback::{FeedbackReceiver, FeedbackSender}; use crate::file_downloader::{FileDownloadRetryPolicy, HttpFileDownloader, RetryDownloader}; use crate::mithril_stake_distribution_client::MithrilStakeDistributionClient; use crate::snapshot_client::SnapshotClient; -#[cfg(feature = "fs")] -use crate::snapshot_downloader::{HttpSnapshotDownloader, SnapshotDownloader}; use crate::MithrilResult; #[cfg(target_family = "wasm")] @@ -138,8 +136,6 @@ pub struct ClientBuilder { certificate_verifier: Option>, #[cfg(feature = "unstable")] certificate_verifier_cache: Option>, - #[cfg(feature = "fs")] - snapshot_downloader: Option>, logger: Option, feedback_receivers: Vec>, options: ClientOptions, @@ -156,8 +152,6 @@ impl ClientBuilder { certificate_verifier: None, #[cfg(feature = "unstable")] certificate_verifier_cache: None, - #[cfg(feature = "fs")] - snapshot_downloader: None, logger: None, feedback_receivers: vec![], options: ClientOptions::default(), @@ -176,8 +170,6 @@ impl ClientBuilder { certificate_verifier: None, #[cfg(feature = "unstable")] certificate_verifier_cache: None, - #[cfg(feature = "fs")] - snapshot_downloader: None, logger: None, feedback_receivers: vec![], options: ClientOptions::default(), @@ -242,33 +234,25 @@ impl ClientBuilder { aggregator_client.clone(), )); - #[cfg(feature = "fs")] - let snapshot_downloader = match self.snapshot_downloader { - None => Arc::new( - HttpSnapshotDownloader::new(feedback_sender.clone(), logger.clone()) - .with_context(|| "Building snapshot downloader failed")?, + #[cfg(all(feature = "fs", feature = "unstable"))] + let http_file_downloader = Arc::new(RetryDownloader::new( + Arc::new( + HttpFileDownloader::new(feedback_sender.clone(), logger.clone()) + .with_context(|| "Building http file downloader failed")?, ), - Some(snapshot_downloader) => snapshot_downloader, - }; + FileDownloadRetryPolicy::default(), + )); let snapshot_client = Arc::new(SnapshotClient::new( aggregator_client.clone(), #[cfg(feature = "fs")] - snapshot_downloader, + http_file_downloader.clone(), #[cfg(feature = "fs")] feedback_sender.clone(), #[cfg(feature = "fs")] logger.clone(), )); - #[cfg(all(feature = "fs", feature = "unstable"))] - let http_file_downloader = Arc::new(RetryDownloader::new( - Arc::new( - HttpFileDownloader::new(feedback_sender.clone(), logger.clone()) - .with_context(|| "Building http file downloader failed")?, - ), - FileDownloadRetryPolicy::default(), - )); #[cfg(feature = "unstable")] let cardano_database_client = Arc::new(CardanoDatabaseClient::new( aggregator_client.clone(), @@ -328,17 +312,6 @@ impl ClientBuilder { } } - cfg_fs! { - /// Set the [SnapshotDownloader] that will be used to download snapshots. - pub fn with_snapshot_downloader( - mut self, - snapshot_downloader: Arc, - ) -> ClientBuilder { - self.snapshot_downloader = Some(snapshot_downloader); - self - } - } - /// Set the [Logger] to use. pub fn with_logger(mut self, logger: Logger) -> Self { self.logger = Some(logger); diff --git a/mithril-client/src/file_downloader/interface.rs b/mithril-client/src/file_downloader/interface.rs index 7b0b471abdf..ea78577c4bf 100644 --- a/mithril-client/src/file_downloader/interface.rs +++ b/mithril-client/src/file_downloader/interface.rs @@ -55,6 +55,12 @@ impl FileDownloaderUri { } } +impl From for FileDownloaderUri { + fn from(location: String) -> Self { + Self::FileUri(FileUri(location)) + } +} + impl From for FileDownloaderUri { fn from(file_uri: FileUri) -> Self { Self::FileUri(file_uri) @@ -105,6 +111,11 @@ pub enum DownloadEvent { /// Unique download identifier download_id: String, }, + /// Full database download + Full { + /// Unique download identifier + download_id: String, + }, } impl DownloadEvent { @@ -115,9 +126,9 @@ impl DownloadEvent { immutable_file_number: _, download_id, } => download_id, - DownloadEvent::Ancillary { download_id } | DownloadEvent::Digest { download_id } => { - download_id - } + DownloadEvent::Ancillary { download_id } + | DownloadEvent::Digest { download_id } + | DownloadEvent::Full { download_id } => download_id, } } @@ -153,6 +164,11 @@ impl DownloadEvent { size: total_bytes, }) } + DownloadEvent::Full { download_id } => MithrilEvent::SnapshotDownloadProgress { + download_id: download_id.to_string(), + downloaded_bytes, + size: total_bytes, + }, } } } @@ -255,5 +271,18 @@ mod tests { }), event, ); + + let download_event_type = DownloadEvent::Full { + download_id: "download-123".to_string(), + }; + let event = download_event_type.build_download_progress_event(123, 1234); + assert_eq!( + MithrilEvent::SnapshotDownloadProgress { + download_id: "download-123".to_string(), + downloaded_bytes: 123, + size: 1234, + }, + event, + ); } } diff --git a/mithril-client/src/lib.rs b/mithril-client/src/lib.rs index 84b4b7c0a33..762a4e2bdef 100644 --- a/mithril-client/src/lib.rs +++ b/mithril-client/src/lib.rs @@ -97,7 +97,6 @@ mod message; pub mod mithril_stake_distribution_client; pub mod snapshot_client; cfg_fs! { - pub mod snapshot_downloader; pub mod file_downloader; } diff --git a/mithril-client/src/snapshot_client.rs b/mithril-client/src/snapshot_client.rs index c9f94ecf05a..fc14410e207 100644 --- a/mithril-client/src/snapshot_client.rs +++ b/mithril-client/src/snapshot_client.rs @@ -101,7 +101,9 @@ use crate::aggregator_client::{AggregatorClient, AggregatorClientError, Aggregat #[cfg(feature = "fs")] use crate::feedback::FeedbackSender; #[cfg(feature = "fs")] -use crate::snapshot_downloader::SnapshotDownloader; +use crate::file_downloader::FileDownloader; +#[cfg(feature = "fs")] +use crate::file_downloader::{DownloadEvent, FileDownloaderUri}; use crate::{MithrilResult, Snapshot, SnapshotListItem}; /// Error for the Snapshot client @@ -122,7 +124,7 @@ pub enum SnapshotClientError { pub struct SnapshotClient { aggregator_client: Arc, #[cfg(feature = "fs")] - snapshot_downloader: Arc, + pub(super) http_file_downloader: Arc, #[cfg(feature = "fs")] feedback_sender: FeedbackSender, #[cfg(feature = "fs")] @@ -133,14 +135,14 @@ impl SnapshotClient { /// Constructs a new `SnapshotClient`. pub fn new( aggregator_client: Arc, - #[cfg(feature = "fs")] snapshot_downloader: Arc, + #[cfg(feature = "fs")] http_file_downloader: Arc, #[cfg(feature = "fs")] feedback_sender: FeedbackSender, #[cfg(feature = "fs")] logger: Logger, ) -> Self { Self { aggregator_client, #[cfg(feature = "fs")] - snapshot_downloader, + http_file_downloader, #[cfg(feature = "fs")] feedback_sender, #[cfg(feature = "fs")] @@ -196,40 +198,43 @@ impl SnapshotClient { use crate::feedback::MithrilEvent; for location in snapshot.locations.as_slice() { - if self.snapshot_downloader.probe(location).await.is_ok() { - let download_id = MithrilEvent::new_snapshot_download_id(); - self.feedback_sender - .send_event(MithrilEvent::SnapshotDownloadStarted { - digest: snapshot.digest.clone(), + let download_id = MithrilEvent::new_snapshot_download_id(); + self.feedback_sender + .send_event(MithrilEvent::SnapshotDownloadStarted { + digest: snapshot.digest.clone(), + download_id: download_id.clone(), + size: snapshot.size, + }) + .await; + let file_downloader_uri: FileDownloaderUri = location.to_owned().into(); + let file_download_outcome = match self + .http_file_downloader + .download_unpack( + &file_downloader_uri, + target_dir, + Some(snapshot.compression_algorithm), + DownloadEvent::Full { download_id: download_id.clone(), - size: snapshot.size, - }) - .await; - return match self - .snapshot_downloader - .download_unpack( - location, - target_dir, - snapshot.compression_algorithm, - &download_id, - snapshot.size, - ) - .await - { - Ok(()) => { - self.feedback_sender - .send_event(MithrilEvent::SnapshotDownloadCompleted { download_id }) - .await; - Ok(()) - } - Err(e) => { - slog::warn!( - self.logger, "Failed downloading snapshot from '{location}'"; - "error" => ?e - ); - Err(e) - } - }; + }, + ) + .await + { + Ok(()) => { + self.feedback_sender + .send_event(MithrilEvent::SnapshotDownloadCompleted { download_id }) + .await; + Ok(()) + } + Err(e) => { + slog::warn!( + self.logger, "Failed downloading snapshot from '{location}'"; + "error" => ?e + ); + Err(e) + } + }; + if file_download_outcome.is_ok() { + return Ok(()); } } @@ -261,7 +266,7 @@ mod tests_download { use crate::{ aggregator_client::MockAggregatorHTTPClient, feedback::{MithrilEvent, StackFeedbackReceiver}, - snapshot_downloader::MockHttpSnapshotDownloader, + file_downloader::MockFileDownloaderBuilder, test_utils, }; use std::path::Path; @@ -270,15 +275,10 @@ mod tests_download { #[tokio::test] async fn download_unpack_send_feedbacks() { - let mut snapshot_downloader = MockHttpSnapshotDownloader::new(); - snapshot_downloader.expect_probe().returning(|_| Ok(())); - snapshot_downloader - .expect_download_unpack() - .returning(|_, _, _, _, _| Ok(())); let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); let client = SnapshotClient::new( Arc::new(MockAggregatorHTTPClient::new()), - Arc::new(snapshot_downloader), + Arc::new(MockFileDownloaderBuilder::default().with_success().build()), FeedbackSender::new(&[feedback_receiver.clone()]), test_utils::test_logger(), ); diff --git a/mithril-client/src/snapshot_downloader.rs b/mithril-client/src/snapshot_downloader.rs deleted file mode 100644 index e031b34600d..00000000000 --- a/mithril-client/src/snapshot_downloader.rs +++ /dev/null @@ -1,233 +0,0 @@ -//! Snapshot tarball download and unpack mechanism. -//! -//! The [SnapshotDownloader] trait abstracts how to download and unpack snapshots -//! tarballs. -//! -//! Snapshots locations can be of various kinds, right now we only support HTTP -//! download (using the [HttpSnapshotDownloader]) but other types may be added in -//! the future. - -use anyhow::{anyhow, Context}; -use async_trait::async_trait; -use futures::StreamExt; -use reqwest::Url; -use reqwest::{Response, StatusCode}; -use slog::{debug, Logger}; -use std::fs; -use std::path::Path; -use tokio::fs::File; -use tokio::io::AsyncReadExt; - -use mithril_common::logging::LoggerExtensions; - -use crate::common::CompressionAlgorithm; -use crate::feedback::{FeedbackSender, MithrilEvent}; -use crate::utils::SnapshotUnpacker; -use crate::MithrilResult; - -/// API that defines a snapshot downloader -#[async_trait] -pub trait SnapshotDownloader: Sync + Send { - /// Download and unpack a snapshot archive on the disk. - /// - /// The `download_id` is a unique identifier that allow - /// [feedback receivers][crate::feedback::FeedbackReceiver] to track concurrent downloads. - /// - /// Warning: this can be a quite long operation depending on the snapshot size. - async fn download_unpack( - &self, - location: &str, - target_dir: &Path, - compression_algorithm: CompressionAlgorithm, - download_id: &str, - snapshot_size: u64, - ) -> MithrilResult<()>; - - /// Test if the given snapshot location exists. - async fn probe(&self, location: &str) -> MithrilResult<()>; -} - -/// A snapshot downloader that only handles download through HTTP. -pub struct HttpSnapshotDownloader { - http_client: reqwest::Client, - feedback_sender: FeedbackSender, - logger: Logger, -} - -impl HttpSnapshotDownloader { - /// Constructs a new `HttpSnapshotDownloader`. - pub fn new(feedback_sender: FeedbackSender, logger: Logger) -> MithrilResult { - let http_client = reqwest::ClientBuilder::new() - .build() - .with_context(|| "Building http client for HttpSnapshotDownloader failed")?; - - Ok(Self { - http_client, - feedback_sender, - logger: logger.new_with_component_name::(), - }) - } - - async fn get(&self, location: &str) -> MithrilResult { - debug!(self.logger, "GET Snapshot location='{location}'."); - let request_builder = self.http_client.get(location); - let response = request_builder.send().await.with_context(|| { - format!("Cannot perform a GET for the snapshot (location='{location}')") - })?; - - match response.status() { - StatusCode::OK => Ok(response), - StatusCode::NOT_FOUND => Err(anyhow!("Location='{location} not found")), - status_code => Err(anyhow!("Unhandled error {status_code}")), - } - } - - fn file_scheme_to_local_path(file_url: &str) -> Option { - Url::parse(file_url) - .ok() - .filter(|url| url.scheme() == "file") - .and_then(|url| url.to_file_path().ok()) - .map(|path| path.to_string_lossy().into_owned()) - } - - async fn download_local_file( - &self, - local_path: &str, - sender: &flume::Sender>, - report_progress: F, - ) -> MithrilResult<()> - where - F: Fn(u64) -> Fut, - Fut: std::future::Future, - { - // Stream the `location` directly from the local filesystem - let mut downloaded_bytes: u64 = 0; - let mut file = File::open(local_path).await?; - - loop { - // We can either allocate here each time, or clone a shared buffer into sender. - // A larger read buffer is faster, less context switches: - let mut buffer = vec![0; 16 * 1024 * 1024]; - let bytes_read = file.read(&mut buffer).await?; - if bytes_read == 0 { - break; - } - buffer.truncate(bytes_read); - sender.send_async(buffer).await.with_context(|| { - format!( - "Local file read: could not write {} bytes to stream.", - bytes_read - ) - })?; - downloaded_bytes += bytes_read as u64; - report_progress(downloaded_bytes).await - } - Ok(()) - } - - async fn download_remote_file( - &self, - location: &str, - sender: &flume::Sender>, - report_progress: F, - ) -> MithrilResult<()> - where - F: Fn(u64) -> Fut, - Fut: std::future::Future, - { - let mut downloaded_bytes: u64 = 0; - let mut remote_stream = self.get(location).await?.bytes_stream(); - while let Some(item) = remote_stream.next().await { - let chunk = item.with_context(|| "Download: Could not read from byte stream")?; - - sender.send_async(chunk.to_vec()).await.with_context(|| { - format!("Download: could not write {} bytes to stream.", chunk.len()) - })?; - - downloaded_bytes += chunk.len() as u64; - report_progress(downloaded_bytes).await - } - Ok(()) - } -} - -#[cfg_attr(test, mockall::automock)] -#[async_trait] -impl SnapshotDownloader for HttpSnapshotDownloader { - async fn download_unpack( - &self, - location: &str, - target_dir: &Path, - compression_algorithm: CompressionAlgorithm, - download_id: &str, - snapshot_size: u64, - ) -> MithrilResult<()> { - if !target_dir.is_dir() { - Err( - anyhow!("target path is not a directory or does not exist: `{target_dir:?}`") - .context("Download-Unpack: prerequisite error"), - )?; - } - let (sender, receiver) = flume::bounded(5); - - let dest_dir = target_dir.to_path_buf(); - let unpack_thread = tokio::task::spawn_blocking(move || -> MithrilResult<()> { - let unpacker = SnapshotUnpacker; - unpacker.unpack_snapshot(receiver, compression_algorithm, &dest_dir) - }); - - let report_progress = |downloaded_bytes: u64| async move { - self.feedback_sender - .send_event(MithrilEvent::SnapshotDownloadProgress { - download_id: download_id.to_owned(), - downloaded_bytes, - size: snapshot_size, - }) - .await - }; - - if let Some(local_path) = Self::file_scheme_to_local_path(location) { - self.download_local_file(&local_path, &sender, report_progress) - .await?; - } else { - self.download_remote_file(location, &sender, report_progress) - .await?; - } - - drop(sender); // Signal EOF - unpack_thread - .await - .with_context(|| { - format!( - "Unpack: panic while unpacking to dir '{}'", - target_dir.display() - ) - })? - .with_context(|| { - format!("Unpack: could not unpack to dir '{}'", target_dir.display()) - })?; - - Ok(()) - } - - async fn probe(&self, location: &str) -> MithrilResult<()> { - debug!(self.logger, "HEAD Snapshot location='{location}'."); - - if let Some(local_path) = Self::file_scheme_to_local_path(location) { - fs::metadata(local_path) - .with_context(|| format!("Local snapshot location='{location}' not found")) - .map(drop) - } else { - let request_builder = self.http_client.head(location); - let response = request_builder.send().await.with_context(|| { - format!("Cannot perform a HEAD for snapshot at location='{location}'") - })?; - - match response.status() { - StatusCode::OK => Ok(()), - StatusCode::NOT_FOUND => Err(anyhow!("Snapshot location='{location} not found")), - status_code => Err(anyhow!("Unhandled error {status_code}")), - } - } - } -} diff --git a/mithril-client/src/utils/mod.rs b/mithril-client/src/utils/mod.rs index a11b51545a1..6e7c28e70e3 100644 --- a/mithril-client/src/utils/mod.rs +++ b/mithril-client/src/utils/mod.rs @@ -3,8 +3,6 @@ cfg_fs! { mod stream_reader; - mod unpacker; pub use stream_reader::*; - pub use unpacker::*; } diff --git a/mithril-client/src/utils/unpacker.rs b/mithril-client/src/utils/unpacker.rs deleted file mode 100644 index fbd8b24de87..00000000000 --- a/mithril-client/src/utils/unpacker.rs +++ /dev/null @@ -1,51 +0,0 @@ -use anyhow::Context; -use flate2::read::GzDecoder; -use flume::Receiver; -use std::path::Path; -use tar::Archive; - -use crate::common::CompressionAlgorithm; -use crate::utils::StreamReader; -use crate::MithrilResult; - -/// Unpack a downloaded archive in a given directory. -#[derive(Default)] -pub struct SnapshotUnpacker; - -impl SnapshotUnpacker { - /// Unpack the snapshot from the given stream into the given directory. - pub fn unpack_snapshot( - &self, - stream: Receiver>, - compression_algorithm: CompressionAlgorithm, - unpack_dir: &Path, - ) -> MithrilResult<()> { - let input = StreamReader::new(stream); - - match compression_algorithm { - CompressionAlgorithm::Gzip => { - let gzip_decoder = GzDecoder::new(input); - let mut snapshot_archive = Archive::new(gzip_decoder); - snapshot_archive.unpack(unpack_dir).with_context(|| { - format!( - "Could not unpack from streamed data snapshot to directory '{}'", - unpack_dir.display() - ) - })?; - } - CompressionAlgorithm::Zstandard => { - let zstandard_decoder = zstd::Decoder::new(input) - .with_context(|| "Unpack failed: Create Zstandard decoder error")?; - let mut snapshot_archive = Archive::new(zstandard_decoder); - snapshot_archive.unpack(unpack_dir).with_context(|| { - format!( - "Could not unpack from streamed data snapshot to directory '{}'", - unpack_dir.display() - ) - })?; - } - }; - - Ok(()) - } -} From 7c1024a4eac6a0316686d000d0547f73d382f8ff Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 12:35:00 +0100 Subject: [PATCH 46/59] chore: apply review comments --- .../download_unpack.rs | 283 +++++++++--------- .../immutable_file_range.rs | 12 +- .../src/cardano_database_client/mod.rs | 3 +- .../src/cardano_database_client/proving.rs | 11 +- .../src/file_downloader/interface.rs | 3 - mithril-client/src/file_downloader/mod.rs | 2 +- mithril-client/src/file_downloader/retry.rs | 2 +- mithril-client/src/snapshot_client.rs | 8 +- .../digesters/cardano_immutable_digester.rs | 54 ++++ .../src/digesters/immutable_file.rs | 3 +- 10 files changed, 227 insertions(+), 154 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index a226339ec86..e55af6bc613 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -7,6 +7,7 @@ use tokio::task::JoinSet; use anyhow::anyhow; use mithril_common::{ + digesters::{IMMUTABLE_DIR, LEDGER_DIR, VOLATILE_DIR}, entities::{AncillaryLocation, CompressionAlgorithm, ImmutableFileNumber, ImmutablesLocation}, messages::CardanoDatabaseSnapshotMessage, }; @@ -53,12 +54,12 @@ impl CardanoDatabaseClient { let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; let immutable_file_number_range = immutable_file_range.to_range_inclusive(last_immutable_file_number)?; - self.verify_download_options_compatibility( + Self::verify_download_options_compatibility( &download_unpack_options, &immutable_file_number_range, last_immutable_file_number, )?; - self.verify_can_write_to_target_directory(target_dir, &download_unpack_options)?; + Self::verify_can_write_to_target_directory(target_dir, &download_unpack_options)?; let immutable_locations = &cardano_database_snapshot.locations.immutables; self.download_unpack_immutable_files( immutable_locations, @@ -89,20 +90,19 @@ impl CardanoDatabaseClient { } fn immutable_files_target_dir(target_dir: &Path) -> PathBuf { - target_dir.join("immutable") + target_dir.join(IMMUTABLE_DIR) } fn volatile_target_dir(target_dir: &Path) -> PathBuf { - target_dir.join("volatile") + target_dir.join(VOLATILE_DIR) } fn ledger_target_dir(target_dir: &Path) -> PathBuf { - target_dir.join("ledger") + target_dir.join(LEDGER_DIR) } /// Verify if the target directory is writable. fn verify_can_write_to_target_directory( - &self, target_dir: &Path, download_unpack_options: &DownloadUnpackOptions, ) -> MithrilResult<()> { @@ -134,7 +134,6 @@ impl CardanoDatabaseClient { /// Verify if the download options are compatible with the immutable file range. fn verify_download_options_compatibility( - &self, download_options: &DownloadUnpackOptions, immutable_file_range: &RangeInclusive, last_immutable_file_number: ImmutableFileNumber, @@ -165,7 +164,7 @@ impl CardanoDatabaseClient { let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); let mut immutable_file_numbers_to_download = - range.clone().map(|n| n.to_owned()).collect::>(); + range.map(|n| n.to_owned()).collect::>(); for location in locations_sorted { let immutable_files_numbers_downloaded = self .download_unpack_immutable_files_for_location( @@ -251,7 +250,7 @@ impl CardanoDatabaseClient { Err(e) => { slog::error!( self.logger, - "Failed downloading and unpacking immutable files"; "error" => e.to_string() + "Failed downloading and unpacking immutable files"; "error" => e.to_string(), "target_dir" => immutable_files_target_dir.display() ); } } @@ -270,7 +269,7 @@ impl CardanoDatabaseClient { ) -> MithrilResult> { let mut immutable_file_numbers_downloaded = BTreeSet::new(); let file_downloader = match &location { - ImmutablesLocation::CloudStorage { uri: _ } => self.http_file_downloader.clone(), + ImmutablesLocation::CloudStorage { .. } => self.http_file_downloader.clone(), }; let file_downloader_uris = FileDownloaderUri::expand_immutable_files_location_to_file_downloader_uris( @@ -320,9 +319,9 @@ impl CardanoDatabaseClient { )) .await; let file_downloader = match &location { - AncillaryLocation::CloudStorage { uri: _ } => self.http_file_downloader.clone(), + AncillaryLocation::CloudStorage { .. } => self.http_file_downloader.clone(), }; - let file_downloader_uri: FileDownloaderUri = location.into(); + let file_downloader_uri = location.into(); let downloaded = file_downloader .download_unpack( &file_downloader_uri, @@ -447,8 +446,8 @@ mod tests { } #[tokio::test] - async fn download_unpack_fails_when_target_target_dir_would_be_overwritten_without_allow_override( - ) { + async fn download_unpack_fails_when_target_dir_would_be_overwritten_without_allow_override() + { let immutable_file_range = ImmutableFileRange::Range(1, 10); let download_unpack_options = DownloadUnpackOptions::default(); let cardano_db_snapshot = CardanoDatabaseSnapshot { @@ -456,10 +455,10 @@ mod tests { ..CardanoDatabaseSnapshot::dummy() }; let target_dir = &TempDir::new( - "cardano_database_client", - "download_unpack_fails_when_target_target_dir_would_be_overwritten_without_allow_override", - ) - .build(); + "cardano_database_client", + "download_unpack_fails_when_target_dir_would_be_overwritten_without_allow_override", + ) + .build(); fs::create_dir_all(target_dir.join("immutable")).unwrap(); let client = CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); @@ -547,8 +546,6 @@ mod tests { #[test] fn verify_download_options_compatibility_succeeds_if_without_ancillary_download() { - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); let download_options = DownloadUnpackOptions { include_ancillary: false, ..DownloadUnpackOptions::default() @@ -556,22 +553,19 @@ mod tests { let immutable_file_range = ImmutableFileRange::Range(1, 10); let last_immutable_file_number = 10; - client - .verify_download_options_compatibility( - &download_options, - &immutable_file_range - .to_range_inclusive(last_immutable_file_number) - .unwrap(), - last_immutable_file_number, - ) - .unwrap(); + CardanoDatabaseClient::verify_download_options_compatibility( + &download_options, + &immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(), + last_immutable_file_number, + ) + .unwrap(); } #[test] fn verify_download_options_compatibility_succeeds_if_with_ancillary_download_and_compatible_range( ) { - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); let download_options = DownloadUnpackOptions { include_ancillary: true, ..DownloadUnpackOptions::default() @@ -579,22 +573,19 @@ mod tests { let immutable_file_range = ImmutableFileRange::Range(7, 10); let last_immutable_file_number = 10; - client - .verify_download_options_compatibility( - &download_options, - &immutable_file_range - .to_range_inclusive(last_immutable_file_number) - .unwrap(), - last_immutable_file_number, - ) - .unwrap(); + CardanoDatabaseClient::verify_download_options_compatibility( + &download_options, + &immutable_file_range + .to_range_inclusive(last_immutable_file_number) + .unwrap(), + last_immutable_file_number, + ) + .unwrap(); } #[test] fn verify_download_options_compatibility_fails_if_with_ancillary_download_and_incompatible_range( ) { - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); let download_options = DownloadUnpackOptions { include_ancillary: true, ..DownloadUnpackOptions::default() @@ -602,8 +593,7 @@ mod tests { let immutable_file_range = ImmutableFileRange::Range(7, 10); let last_immutable_file_number = 123; - client - .verify_download_options_compatibility( + CardanoDatabaseClient::verify_download_options_compatibility( &download_options, &immutable_file_range .to_range_inclusive(last_immutable_file_number) @@ -625,18 +615,15 @@ mod tests { "verify_can_write_to_target_dir_always_succeeds_with_allow_overwrite", ) .build(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: true, - include_ancillary: false, - }, - ) - .unwrap(); + CardanoDatabaseClient::verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: true, + include_ancillary: false, + }, + ) + .unwrap(); fs::create_dir_all(CardanoDatabaseClient::immutable_files_target_dir( &target_dir, @@ -644,24 +631,22 @@ mod tests { .unwrap(); fs::create_dir_all(CardanoDatabaseClient::volatile_target_dir(&target_dir)).unwrap(); fs::create_dir_all(CardanoDatabaseClient::ledger_target_dir(&target_dir)).unwrap(); - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: true, - include_ancillary: false, - }, - ) - .unwrap(); - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: true, - include_ancillary: true, - }, - ) - .unwrap(); + CardanoDatabaseClient::verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: true, + include_ancillary: false, + }, + ) + .unwrap(); + CardanoDatabaseClient::verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: true, + include_ancillary: true, + }, + ) + .unwrap(); } #[test] @@ -672,28 +657,24 @@ mod tests { &target_dir, )) .unwrap(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: false, - }, - ) - .expect_err("verify_can_write_to_target_dir should fail"); + CardanoDatabaseClient::verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: false, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: true, - }, - ) - .expect_err("verify_can_write_to_target_dir should fail"); + CardanoDatabaseClient::verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: true, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); } #[test] @@ -701,28 +682,24 @@ mod tests { ) { let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_ledger_target_dir").build(); fs::create_dir_all(CardanoDatabaseClient::ledger_target_dir(&target_dir)).unwrap(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: true, - }, - ) - .expect_err("verify_can_write_to_target_dir should fail"); + CardanoDatabaseClient::verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: true, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: false, - }, - ) - .unwrap(); + CardanoDatabaseClient::verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: false, + }, + ) + .unwrap(); } #[test] @@ -730,28 +707,24 @@ mod tests { ) { let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_volatile_target_dir").build(); fs::create_dir_all(CardanoDatabaseClient::volatile_target_dir(&target_dir)).unwrap(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: true, - }, - ) - .expect_err("verify_can_write_to_target_dir should fail"); + CardanoDatabaseClient::verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: true, + }, + ) + .expect_err("verify_can_write_to_target_dir should fail"); - client - .verify_can_write_to_target_directory( - &target_dir, - &DownloadUnpackOptions { - allow_override: false, - include_ancillary: false, - }, - ) - .unwrap(); + CardanoDatabaseClient::verify_can_write_to_target_directory( + &target_dir, + &DownloadUnpackOptions { + allow_override: false, + include_ancillary: false, + }, + ) + .unwrap(); } } @@ -892,7 +865,7 @@ mod tests { } #[tokio::test] - async fn download_unpack_immutable_files_sends_feedbacks() { + async fn download_unpack_immutable_files_sends_feedbacks_when_succeeds() { let total_immutable_files = 1; let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); let target_dir = Path::new("."); @@ -939,6 +912,48 @@ mod tests { ]; assert_eq!(expected_events, sent_events); } + + #[tokio::test] + async fn download_unpack_immutable_files_sends_feedbacks_when_fails() { + let total_immutable_files = 1; + let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); + let target_dir = Path::new("."); + let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); + let client = CardanoDatabaseClientDependencyInjector::new() + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default().with_failure().build(), + )) + .with_feedback_receivers(&[feedback_receiver.clone()]) + .build_cardano_database_client(); + + client + .download_unpack_immutable_files( + &[ImmutablesLocation::CloudStorage { + uri: MultiFilesUri::Template(TemplateUri( + "http://whatever/{immutable_file_number}.tar.gz".to_string(), + )), + }], + immutable_file_range + .to_range_inclusive(total_immutable_files) + .unwrap(), + &CompressionAlgorithm::default(), + target_dir, + 1, + "download_id", + ) + .await + .expect_err("download_unpack_immutable_files should fail"); + + let sent_events = feedback_receiver.stacked_events(); + let id = sent_events[0].event_id(); + let expected_events = vec![MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadStarted { + immutable_file_number: 1, + download_id: id.to_string(), + }, + )]; + assert_eq!(expected_events, sent_events); + } } mod download_unpack_ancillary_file { diff --git a/mithril-client/src/cardano_database_client/immutable_file_range.rs b/mithril-client/src/cardano_database_client/immutable_file_range.rs index 8bcb7260401..a24a75ed7d2 100644 --- a/mithril-client/src/cardano_database_client/immutable_file_range.rs +++ b/mithril-client/src/cardano_database_client/immutable_file_range.rs @@ -80,7 +80,7 @@ mod tests { let last_immutable_file_number = 3; immutable_file_range .to_range_inclusive(last_immutable_file_number) - .expect_err("conversion to range inlusive should fail"); + .expect_err("should fail: given last immutable should be greater than range start"); } #[test] @@ -96,12 +96,14 @@ mod tests { let last_immutable_file_number = 7; immutable_file_range .to_range_inclusive(last_immutable_file_number) - .expect_err("conversion to range inlusive should fail"); + .expect_err( + "should fail: given last immutable should be greater or equal range max bound", + ); let immutable_file_range = ImmutableFileRange::Range(10, 8); immutable_file_range .to_range_inclusive(last_immutable_file_number) - .expect_err("conversion to range inlusive should fail"); + .expect_err("should fail: range start should be lower than range end"); } #[test] @@ -117,6 +119,8 @@ mod tests { let last_immutable_file_number = 7; immutable_file_range .to_range_inclusive(last_immutable_file_number) - .expect_err("conversion to range inlusive should fail"); + .expect_err( + "should fail: given last immutable should be greater or equal range max bound", + ); } } diff --git a/mithril-client/src/cardano_database_client/mod.rs b/mithril-client/src/cardano_database_client/mod.rs index 2f2389af050..c2762d9a204 100644 --- a/mithril-client/src/cardano_database_client/mod.rs +++ b/mithril-client/src/cardano_database_client/mod.rs @@ -46,6 +46,7 @@ //! ``` //! //! # Download a Cardano database snapshot +//! **Note:** _Available on crate feature_ **fs** _only._ //! //! To download a partial or a full Cardano database folder the [ClientBuilder][crate::client::ClientBuilder]. //! @@ -78,8 +79,8 @@ //! # Ok(()) //! # } //! ``` -//! //! # Compute a Merkle proof for a Cardano database snapshot +//! **Note:** _Available on crate feature_ **fs** _only._ //! //! To compute proof of membership of downloaded immutable files in a Cardano database folder the [ClientBuilder][crate::client::ClientBuilder]. //! diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index e58c3edb5ed..c0cc9141c0a 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -83,7 +83,7 @@ impl CardanoDatabaseClient { )) .await; let file_downloader = match &location { - DigestLocation::CloudStorage { uri: _ } | DigestLocation::Aggregator { uri: _ } => { + DigestLocation::CloudStorage { .. } | DigestLocation::Aggregator { .. } => { self.http_file_downloader.clone() } }; @@ -363,7 +363,7 @@ mod tests { use super::*; #[tokio::test] - async fn download_unpack_digest_file_fails_if_no_location_is_retrieved() { + async fn fails_if_no_location_is_retrieved() { let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() .with_http_file_downloader(Arc::new( @@ -392,7 +392,7 @@ mod tests { } #[tokio::test] - async fn download_unpack_digest_file_succeeds_if_at_least_one_location_is_retrieved() { + async fn succeeds_if_at_least_one_location_is_retrieved() { let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() .with_http_file_downloader(Arc::new({ @@ -425,12 +425,13 @@ mod tests { } #[tokio::test] - async fn download_unpack_digest_file_succeeds_when_first_location_is_retrieved() { + async fn succeeds_when_first_location_is_retrieved() { let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() .with_http_file_downloader(Arc::new( MockFileDownloaderBuilder::default() .with_compression(None) + .with_times(1) .with_success() .build(), )) @@ -453,7 +454,7 @@ mod tests { } #[tokio::test] - async fn download_unpack_digest_file_sends_feedbacks() { + async fn sends_feedbacks() { let target_dir = Path::new("."); let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); let client = CardanoDatabaseClientDependencyInjector::new() diff --git a/mithril-client/src/file_downloader/interface.rs b/mithril-client/src/file_downloader/interface.rs index ea78577c4bf..7515563ae24 100644 --- a/mithril-client/src/file_downloader/interface.rs +++ b/mithril-client/src/file_downloader/interface.rs @@ -85,9 +85,6 @@ impl From for FileDownloaderUri { } } -/// A feedback event builder -pub type FeedbackEventBuilder = fn(String, u64, u64) -> Option; - /// A download event /// /// The `download_id` is a unique identifier that allow diff --git a/mithril-client/src/file_downloader/mod.rs b/mithril-client/src/file_downloader/mod.rs index 1c9a0b7f70c..fdc7a9e53e3 100644 --- a/mithril-client/src/file_downloader/mod.rs +++ b/mithril-client/src/file_downloader/mod.rs @@ -11,7 +11,7 @@ mod retry; pub use http::HttpFileDownloader; #[cfg(test)] pub use interface::MockFileDownloader; -pub use interface::{DownloadEvent, FeedbackEventBuilder, FileDownloader, FileDownloaderUri}; +pub use interface::{DownloadEvent, FileDownloader, FileDownloaderUri}; #[cfg(test)] pub use mock_builder::MockFileDownloaderBuilder; pub use retry::{FileDownloadRetryPolicy, RetryDownloader}; diff --git a/mithril-client/src/file_downloader/retry.rs b/mithril-client/src/file_downloader/retry.rs index a1ee7630a35..aa6f67795fc 100644 --- a/mithril-client/src/file_downloader/retry.rs +++ b/mithril-client/src/file_downloader/retry.rs @@ -39,7 +39,7 @@ pub struct RetryDownloader { /// File downloader to use. file_downloader: Arc, /// Number of attempts to download a file. - pub retry_policy: FileDownloadRetryPolicy, + retry_policy: FileDownloadRetryPolicy, } impl RetryDownloader { diff --git a/mithril-client/src/snapshot_client.rs b/mithril-client/src/snapshot_client.rs index fc14410e207..04dc4faab0c 100644 --- a/mithril-client/src/snapshot_client.rs +++ b/mithril-client/src/snapshot_client.rs @@ -101,9 +101,9 @@ use crate::aggregator_client::{AggregatorClient, AggregatorClientError, Aggregat #[cfg(feature = "fs")] use crate::feedback::FeedbackSender; #[cfg(feature = "fs")] -use crate::file_downloader::FileDownloader; +use crate::file_downloader::DownloadEvent; #[cfg(feature = "fs")] -use crate::file_downloader::{DownloadEvent, FileDownloaderUri}; +use crate::file_downloader::FileDownloader; use crate::{MithrilResult, Snapshot, SnapshotListItem}; /// Error for the Snapshot client @@ -124,7 +124,7 @@ pub enum SnapshotClientError { pub struct SnapshotClient { aggregator_client: Arc, #[cfg(feature = "fs")] - pub(super) http_file_downloader: Arc, + http_file_downloader: Arc, #[cfg(feature = "fs")] feedback_sender: FeedbackSender, #[cfg(feature = "fs")] @@ -206,7 +206,7 @@ impl SnapshotClient { size: snapshot.size, }) .await; - let file_downloader_uri: FileDownloaderUri = location.to_owned().into(); + let file_downloader_uri = location.to_owned().into(); let file_download_outcome = match self .http_file_downloader .download_unpack( diff --git a/mithril-common/src/digesters/cardano_immutable_digester.rs b/mithril-common/src/digesters/cardano_immutable_digester.rs index c818bc8f2fb..37e51fddc37 100644 --- a/mithril-common/src/digesters/cardano_immutable_digester.rs +++ b/mithril-common/src/digesters/cardano_immutable_digester.rs @@ -422,6 +422,60 @@ mod tests { assert_eq!(cardano_db.get_immutable_files().len(), result.entries.len()) } + #[tokio::test] + async fn can_compute_consistent_digests_for_range() { + let immutable_range = 1..=1; + let cardano_db = db_builder("can_compute_digests_for_range_consistently") + .with_immutables( + &immutable_range + .clone() + .collect::>(), + ) + .append_immutable_trio() + .build(); + let logger = TestLogger::stdout(); + let digester = CardanoImmutableDigester::new( + "devnet".to_string(), + Some(Arc::new(MemoryImmutableFileDigestCacheProvider::default())), + logger.clone(), + ); + + let result = digester + .compute_digests_for_range(cardano_db.get_immutable_dir(), &immutable_range) + .await + .expect("compute_digests_for_range must not fail"); + + assert_eq!( + BTreeMap::from([ + ( + ImmutableFile { + path: cardano_db.get_immutable_dir().join("00001.chunk"), + number: 1, + filename: "00001.chunk".to_string() + }, + "faebbf47077f68ef57219396ff69edc738978a3eca946ac7df1983dbf11364ec".to_string() + ), + ( + ImmutableFile { + path: cardano_db.get_immutable_dir().join("00001.primary"), + number: 1, + filename: "00001.primary".to_string() + }, + "f11bdb991fc7e72970be7d7f666e10333f92c14326d796fed8c2c041675fa826".to_string() + ), + ( + ImmutableFile { + path: cardano_db.get_immutable_dir().join("00001.secondary"), + number: 1, + filename: "00001.secondary".to_string() + }, + "b139684b968fa12ce324cce464d000de0e2c2ded0fd3e473a666410821d3fde3".to_string() + ) + ]), + result.entries + ); + } + #[tokio::test] async fn compute_digest_store_digests_into_cache_provider() { let cardano_db = db_builder("compute_digest_store_digests_into_cache_provider") diff --git a/mithril-common/src/digesters/immutable_file.rs b/mithril-common/src/digesters/immutable_file.rs index 9dada02bce4..eeb9d62dc28 100644 --- a/mithril-common/src/digesters/immutable_file.rs +++ b/mithril-common/src/digesters/immutable_file.rs @@ -256,9 +256,10 @@ mod tests { .expect("ImmutableFile::list_in_dir Failed"); assert_eq!(result.last().unwrap().number, 424); + let expected_entries_length = 21; assert_eq!( + expected_entries_length, result.len(), - entries.len(), "Expected to find {} files but found {}", entries.len(), result.len(), From 7a88e67a889301e966e29191241fe42e0fd73a70 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 12:35:26 +0100 Subject: [PATCH 47/59] refactor: create a 'utils/fs' module --- .../src/cardano_database_client/proving.rs | 36 ++--------------- mithril-client/src/utils/fs.rs | 40 +++++++++++++++++++ mithril-client/src/utils/mod.rs | 2 + 3 files changed, 46 insertions(+), 32 deletions(-) create mode 100644 mithril-client/src/utils/fs.rs diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index c0cc9141c0a..bd6da1ce82a 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -18,6 +18,7 @@ use mithril_common::{ use crate::{ feedback::{MithrilEvent, MithrilEventCardanoDatabase}, file_downloader::{DownloadEvent, FileDownloaderUri}, + utils::{create_directory_if_not_exists, delete_directory, read_files_in_directory}, MithrilResult, }; @@ -60,7 +61,7 @@ impl CardanoDatabaseClient { .values() .map(MKTreeNode::from) .collect::>(); - Self::delete_directory(&Self::digest_target_dir(database_dir))?; + delete_directory(&Self::digest_target_dir(database_dir))?; merkle_tree.compute_proof(&computed_digests) } @@ -70,7 +71,7 @@ impl CardanoDatabaseClient { locations: &[DigestLocation], digest_file_target_dir: &Path, ) -> MithrilResult<()> { - Self::create_directory_if_not_exists(digest_file_target_dir)?; + create_directory_if_not_exists(digest_file_target_dir)?; let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); for location in locations_sorted { @@ -125,7 +126,7 @@ impl CardanoDatabaseClient { &self, digest_file_target_dir: &Path, ) -> MithrilResult> { - let digest_files = Self::read_files_in_directory(digest_file_target_dir)?; + let digest_files = read_files_in_directory(digest_file_target_dir)?; if digest_files.len() > 1 { return Err(anyhow!( "Multiple digest files found in directory: {digest_file_target_dir:?}" @@ -154,35 +155,6 @@ impl CardanoDatabaseClient { fn digest_target_dir(target_dir: &Path) -> PathBuf { target_dir.join("digest") } - - fn create_directory_if_not_exists(dir: &Path) -> MithrilResult<()> { - if dir.exists() { - return Ok(()); - } - - fs::create_dir_all(dir).map_err(|e| anyhow!("Failed creating directory: {e}")) - } - - fn delete_directory(dir: &Path) -> MithrilResult<()> { - if dir.exists() { - fs::remove_dir_all(dir).map_err(|e| anyhow!("Failed deleting directory: {e}"))?; - } - - Ok(()) - } - - fn read_files_in_directory(dir: &Path) -> MithrilResult> { - let mut files = vec![]; - for entry in fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - if path.is_file() { - files.push(path); - } - } - - Ok(files) - } } #[cfg(test)] diff --git a/mithril-client/src/utils/fs.rs b/mithril-client/src/utils/fs.rs new file mode 100644 index 00000000000..2a3e6a5cc98 --- /dev/null +++ b/mithril-client/src/utils/fs.rs @@ -0,0 +1,40 @@ +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use anyhow::anyhow; + +use crate::MithrilResult; + +/// Create a directory if it does not exist +pub fn create_directory_if_not_exists(dir: &Path) -> MithrilResult<()> { + if dir.exists() { + return Ok(()); + } + + fs::create_dir_all(dir).map_err(|e| anyhow!("Failed creating directory: {e}")) +} + +/// Delete a directory if it exists +pub fn delete_directory(dir: &Path) -> MithrilResult<()> { + if dir.exists() { + fs::remove_dir_all(dir).map_err(|e| anyhow!("Failed deleting directory: {e}"))?; + } + + Ok(()) +} + +/// Read files in a directory +pub fn read_files_in_directory(dir: &Path) -> MithrilResult> { + let mut files = vec![]; + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_file() { + files.push(path); + } + } + + Ok(files) +} diff --git a/mithril-client/src/utils/mod.rs b/mithril-client/src/utils/mod.rs index 6e7c28e70e3..17859971069 100644 --- a/mithril-client/src/utils/mod.rs +++ b/mithril-client/src/utils/mod.rs @@ -3,6 +3,8 @@ cfg_fs! { mod stream_reader; + mod fs; pub use stream_reader::*; + pub use fs::*; } From fe5d68b52383d94e1111e0627d6bef4689302d11 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 13:15:50 +0100 Subject: [PATCH 48/59] refactor: enhance readability of batch immutable file download --- .../download_unpack.rs | 109 ++++++++++++------ 1 file changed, 71 insertions(+), 38 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index e55af6bc613..663e782ce47 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -1,6 +1,8 @@ use std::collections::BTreeSet; +use std::future::Future; use std::ops::RangeInclusive; use std::path::{Path, PathBuf}; +use std::pin::Pin; use std::sync::Arc; use tokio::task::JoinSet; @@ -19,6 +21,9 @@ use crate::MithrilResult; use super::api::CardanoDatabaseClient; use super::immutable_file_range::ImmutableFileRange; +/// The future type for downloading an immutable file +type DownloadImmutableFuture = dyn Future> + Send; + /// Options for downloading and unpacking a Cardano database #[derive(Debug, Default)] pub struct DownloadUnpackOptions { @@ -203,44 +208,14 @@ impl CardanoDatabaseClient { let mut immutable_file_numbers_downloaded = BTreeSet::new(); let mut join_set: JoinSet> = JoinSet::new(); for (immutable_file_number, file_downloader_uri) in file_downloader_uris_chunk.into_iter() { - let file_downloader_uri_clone = file_downloader_uri.to_owned(); - let compression_algorithm_clone = compression_algorithm.to_owned(); - let immutable_files_target_dir_clone = immutable_files_target_dir.to_owned(); - let file_downloader_clone = file_downloader.clone(); - let feedback_receiver_clone = self.feedback_sender.clone(); - let logger_clone = self.logger.clone(); - let download_id_clone = download_id.to_string(); - join_set.spawn(async move { - feedback_receiver_clone - .send_event(MithrilEvent::CardanoDatabase( - MithrilEventCardanoDatabase::ImmutableDownloadStarted { immutable_file_number, download_id: download_id_clone.clone()})) - .await; - let downloaded = file_downloader_clone - .download_unpack( - &file_downloader_uri_clone, - &immutable_files_target_dir_clone, - Some(compression_algorithm_clone), - DownloadEvent::Immutable{immutable_file_number, download_id: download_id_clone.clone()}, - ) - .await; - match downloaded { - Ok(_) => { - feedback_receiver_clone - .send_event(MithrilEvent::CardanoDatabase( - MithrilEventCardanoDatabase::ImmutableDownloadCompleted { immutable_file_number, download_id: download_id_clone })) - .await; - - Ok(immutable_file_number) - } - Err(e) => { - slog::error!( - logger_clone, - "Failed downloading and unpacking immutable file {immutable_file_number} for location {file_downloader_uri:?}"; "error" => e.to_string() - ); - Err(e.context(format!("Failed downloading and unpacking immutable file {immutable_file_number} for location {file_downloader_uri:?}"))) - } - } - }); + join_set.spawn(self.spawn_immutable_download_future( + file_downloader.clone(), + immutable_file_number, + file_downloader_uri, + compression_algorithm.to_owned(), + immutable_files_target_dir.to_path_buf(), + download_id, + )?); } while let Some(result) = join_set.join_next().await { match result? { @@ -259,6 +234,64 @@ impl CardanoDatabaseClient { Ok(immutable_file_numbers_downloaded) } + fn spawn_immutable_download_future( + &self, + file_downloader: Arc, + immutable_file_number: ImmutableFileNumber, + file_downloader_uri: FileDownloaderUri, + compression_algorithm: CompressionAlgorithm, + immutable_files_target_dir: PathBuf, + download_id: &str, + ) -> MithrilResult>> { + let feedback_receiver_clone = self.feedback_sender.clone(); + let logger_clone = self.logger.clone(); + let download_id_clone = download_id.to_string(); + let download_future = async move { + feedback_receiver_clone + .send_event(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadStarted { + immutable_file_number, + download_id: download_id_clone.clone(), + }, + )) + .await; + let downloaded = file_downloader + .download_unpack( + &file_downloader_uri, + &immutable_files_target_dir, + Some(compression_algorithm), + DownloadEvent::Immutable { + immutable_file_number, + download_id: download_id_clone.clone(), + }, + ) + .await; + match downloaded { + Ok(_) => { + feedback_receiver_clone + .send_event(MithrilEvent::CardanoDatabase( + MithrilEventCardanoDatabase::ImmutableDownloadCompleted { + immutable_file_number, + download_id: download_id_clone, + }, + )) + .await; + + Ok(immutable_file_number) + } + Err(e) => { + slog::error!( + logger_clone, + "Failed downloading and unpacking immutable file {immutable_file_number} for location {file_downloader_uri:?}"; "error" => e.to_string() + ); + Err(e.context(format!("Failed downloading and unpacking immutable file {immutable_file_number} for location {file_downloader_uri:?}"))) + } + } + }; + + Ok(Box::pin(download_future)) + } + async fn download_unpack_immutable_files_for_location( &self, location: &ImmutablesLocation, From 937d962f07ca1968ba0a11a6cbab49e4fac630e1 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 14:52:21 +0100 Subject: [PATCH 49/59] refactor: move max parallel downloads in 'DownloadUnpackOptions' --- .../download_unpack.rs | 38 ++++++++++++++++--- .../src/cardano_database_client/mod.rs | 2 + 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index 663e782ce47..4330dd3cebf 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -25,20 +25,29 @@ use super::immutable_file_range::ImmutableFileRange; type DownloadImmutableFuture = dyn Future> + Send; /// Options for downloading and unpacking a Cardano database -#[derive(Debug, Default)] +#[derive(Debug)] pub struct DownloadUnpackOptions { /// Allow overriding the destination directory pub allow_override: bool, /// Include ancillary files in the download pub include_ancillary: bool, + + /// Maximum number of parallel downloads + pub max_parallel_downloads: usize, } -impl CardanoDatabaseClient { - /// The maximum number of parallel downloads and unpacks for immutable files. - /// This could be a customizable parameter depending on level of concurrency wanted by the user. - const MAX_PARALLEL_DOWNLOAD_UNPACK: usize = 100; +impl Default for DownloadUnpackOptions { + fn default() -> Self { + Self { + allow_override: false, + include_ancillary: false, + max_parallel_downloads: 100, + } + } +} +impl CardanoDatabaseClient { /// Download and unpack the given Cardano database parts data by hash. pub async fn download_unpack( &self, @@ -71,6 +80,7 @@ impl CardanoDatabaseClient { immutable_file_number_range, &compression_algorithm, target_dir, + download_unpack_options.max_parallel_downloads, &download_id, ) .await?; @@ -164,6 +174,7 @@ impl CardanoDatabaseClient { range: RangeInclusive, compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path, + max_parallel_downloads: usize, download_id: &str, ) -> MithrilResult<()> { let mut locations_sorted = locations.to_owned(); @@ -177,6 +188,7 @@ impl CardanoDatabaseClient { &immutable_file_numbers_to_download, compression_algorithm, immutable_files_target_dir, + max_parallel_downloads, download_id, ) .await?; @@ -298,6 +310,7 @@ impl CardanoDatabaseClient { immutable_file_numbers_to_download: &BTreeSet, compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path, + max_parallel_downloads: usize, download_id: &str, ) -> MithrilResult> { let mut immutable_file_numbers_downloaded = BTreeSet::new(); @@ -314,7 +327,7 @@ impl CardanoDatabaseClient { .as_slice(), )?; let file_downloader_uri_chunks = file_downloader_uris - .chunks(Self::MAX_PARALLEL_DOWNLOAD_UNPACK) + .chunks(max_parallel_downloads) .map(|x| x.to_vec()) .collect::>(); for file_downloader_uris_chunk in file_downloader_uri_chunks { @@ -654,6 +667,7 @@ mod tests { &DownloadUnpackOptions { allow_override: true, include_ancillary: false, + ..DownloadUnpackOptions::default() }, ) .unwrap(); @@ -669,6 +683,7 @@ mod tests { &DownloadUnpackOptions { allow_override: true, include_ancillary: false, + ..DownloadUnpackOptions::default() }, ) .unwrap(); @@ -677,6 +692,7 @@ mod tests { &DownloadUnpackOptions { allow_override: true, include_ancillary: true, + ..DownloadUnpackOptions::default() }, ) .unwrap(); @@ -696,6 +712,7 @@ mod tests { &DownloadUnpackOptions { allow_override: false, include_ancillary: false, + ..DownloadUnpackOptions::default() }, ) .expect_err("verify_can_write_to_target_dir should fail"); @@ -705,6 +722,7 @@ mod tests { &DownloadUnpackOptions { allow_override: false, include_ancillary: true, + ..DownloadUnpackOptions::default() }, ) .expect_err("verify_can_write_to_target_dir should fail"); @@ -721,6 +739,7 @@ mod tests { &DownloadUnpackOptions { allow_override: false, include_ancillary: true, + ..DownloadUnpackOptions::default() }, ) .expect_err("verify_can_write_to_target_dir should fail"); @@ -730,6 +749,7 @@ mod tests { &DownloadUnpackOptions { allow_override: false, include_ancillary: false, + ..DownloadUnpackOptions::default() }, ) .unwrap(); @@ -746,6 +766,7 @@ mod tests { &DownloadUnpackOptions { allow_override: false, include_ancillary: true, + ..DownloadUnpackOptions::default() }, ) .expect_err("verify_can_write_to_target_dir should fail"); @@ -755,6 +776,7 @@ mod tests { &DownloadUnpackOptions { allow_override: false, include_ancillary: false, + ..DownloadUnpackOptions::default() }, ) .unwrap(); @@ -797,6 +819,7 @@ mod tests { .unwrap(), &CompressionAlgorithm::default(), &target_dir, + 1, "download_id", ) .await @@ -834,6 +857,7 @@ mod tests { .unwrap(), &CompressionAlgorithm::default(), &target_dir, + 1, "download_id", ) .await @@ -891,6 +915,7 @@ mod tests { .unwrap(), &CompressionAlgorithm::default(), &target_dir, + 1, "download_id", ) .await @@ -922,6 +947,7 @@ mod tests { .unwrap(), &CompressionAlgorithm::default(), target_dir, + 1, "download_id", ) .await diff --git a/mithril-client/src/cardano_database_client/mod.rs b/mithril-client/src/cardano_database_client/mod.rs index c2762d9a204..39505f0948c 100644 --- a/mithril-client/src/cardano_database_client/mod.rs +++ b/mithril-client/src/cardano_database_client/mod.rs @@ -65,6 +65,7 @@ //! let download_unpack_options = DownloadUnpackOptions { //! allow_override: true, //! include_ancillary: true, +//! ..DownloadUnpackOptions::default() //! }; //! client //! .cardano_database() @@ -100,6 +101,7 @@ //! let download_unpack_options = DownloadUnpackOptions { //! allow_override: true, //! include_ancillary: true, +//! ..DownloadUnpackOptions::default() //! }; //! client //! .cardano_database() From 357466206d81a2018327f527c4090558289dfc9e Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 15:07:02 +0100 Subject: [PATCH 50/59] refactor: simplify multiple calls for 'MockFileDownloaderBuilder' --- .../download_unpack.rs | 45 +++++++------------ .../src/cardano_database_client/proving.rs | 6 +-- .../src/file_downloader/mock_builder.rs | 9 ++++ mithril-client/src/file_downloader/retry.rs | 3 +- 4 files changed, 29 insertions(+), 34 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index 4330dd3cebf..8abf10151bb 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -553,19 +553,15 @@ mod tests { .build(); let client = CardanoDatabaseClientDependencyInjector::new() .with_http_file_downloader(Arc::new({ - let mock_file_downloader = MockFileDownloaderBuilder::default() + MockFileDownloaderBuilder::default() .with_file_uri("http://whatever/00001.tar.gz") .with_target_dir(target_dir.clone()) .with_success() - .build(); - let mock_file_downloader = - MockFileDownloaderBuilder::from_mock(mock_file_downloader) - .with_file_uri("http://whatever/00002.tar.gz") - .with_target_dir(target_dir.clone()) - .with_success() - .build(); - - MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .next_call() + .with_file_uri("http://whatever/00002.tar.gz") + .with_target_dir(target_dir.clone()) + .with_success() + .next_call() .with_file_uri("http://whatever/ancillary.tar.gz") .with_target_dir(target_dir.clone()) .with_compression(Some(CompressionAlgorithm::default())) @@ -798,10 +794,9 @@ mod tests { .build(); let client = CardanoDatabaseClientDependencyInjector::new() .with_http_file_downloader(Arc::new({ - let mock_file_downloader = - MockFileDownloaderBuilder::default().with_failure().build(); - - MockFileDownloaderBuilder::from_mock(mock_file_downloader) + MockFileDownloaderBuilder::default() + .with_failure() + .next_call() .with_success() .build() })) @@ -876,19 +871,15 @@ mod tests { .build(); let client = CardanoDatabaseClientDependencyInjector::new() .with_http_file_downloader(Arc::new({ - let mock_file_downloader = MockFileDownloaderBuilder::default() + MockFileDownloaderBuilder::default() .with_file_uri("http://whatever-1/00001.tar.gz") .with_target_dir(target_dir.clone()) .with_failure() - .build(); - let mock_file_downloader = - MockFileDownloaderBuilder::from_mock(mock_file_downloader) - .with_file_uri("http://whatever-1/00002.tar.gz") - .with_target_dir(target_dir.clone()) - .with_success() - .build(); - - MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .next_call() + .with_file_uri("http://whatever-1/00002.tar.gz") + .with_target_dir(target_dir.clone()) + .with_success() + .next_call() .with_file_uri("http://whatever-2/00001.tar.gz") .with_target_dir(target_dir.clone()) .with_success() @@ -1045,13 +1036,11 @@ mod tests { let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() .with_http_file_downloader(Arc::new({ - let mock_file_downloader = MockFileDownloaderBuilder::default() + MockFileDownloaderBuilder::default() .with_file_uri("http://whatever-1/ancillary.tar.gz") .with_target_dir(target_dir.to_path_buf()) .with_failure() - .build(); - - MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .next_call() .with_file_uri("http://whatever-2/ancillary.tar.gz") .with_target_dir(target_dir.to_path_buf()) .with_success() diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index bd6da1ce82a..69982d35633 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -368,12 +368,10 @@ mod tests { let target_dir = Path::new("."); let client = CardanoDatabaseClientDependencyInjector::new() .with_http_file_downloader(Arc::new({ - let mock_file_downloader = MockFileDownloaderBuilder::default() + MockFileDownloaderBuilder::default() .with_compression(None) .with_failure() - .build(); - - MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .next_call() .with_compression(None) .with_success() .build() diff --git a/mithril-client/src/file_downloader/mock_builder.rs b/mithril-client/src/file_downloader/mock_builder.rs index 9f29106766b..4695f9944af 100644 --- a/mithril-client/src/file_downloader/mock_builder.rs +++ b/mithril-client/src/file_downloader/mock_builder.rs @@ -142,4 +142,13 @@ impl MockFileDownloaderBuilder { mock_file_downloader } + + /// Builds the MockFileDownloader and returns a new MockFileDownloaderBuilder + /// + /// This helps building multiple expectations for the mock. + pub fn next_call(self) -> Self { + let mock_file_downloader = self.build(); + + Self::from_mock(mock_file_downloader) + } } diff --git a/mithril-client/src/file_downloader/retry.rs b/mithril-client/src/file_downloader/retry.rs index aa6f67795fc..45acbea59c7 100644 --- a/mithril-client/src/file_downloader/retry.rs +++ b/mithril-client/src/file_downloader/retry.rs @@ -162,8 +162,7 @@ mod tests { .with_compression(None) .with_failure() .with_times(2) - .build(); - let mock_file_downloader = MockFileDownloaderBuilder::from_mock(mock_file_downloader) + .next_call() .with_file_uri("http://whatever/00001.tar.gz") .with_compression(None) .with_times(1) From 6761706d482732a9ba271aa6569071350b8f1a90 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 16:38:22 +0100 Subject: [PATCH 51/59] refactor: enhance feedback events structure --- .../download_unpack.rs | 66 ++++++++++++++----- mithril-client/src/feedback.rs | 22 +++++-- 2 files changed, 67 insertions(+), 21 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index 8abf10151bb..78cacba1d6e 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -24,6 +24,17 @@ use super::immutable_file_range::ImmutableFileRange; /// The future type for downloading an immutable file type DownloadImmutableFuture = dyn Future> + Send; +/// Arguments for the immutable file download future builder +struct DownloadImmutableFutureBuilderArgs { + file_downloader: Arc, + immutable_file_number: ImmutableFileNumber, + file_downloader_uri: FileDownloaderUri, + compression_algorithm: CompressionAlgorithm, + immutable_files_target_dir: PathBuf, + download_id: String, + file_size: u64, +} + /// Options for downloading and unpacking a Cardano database #[derive(Debug)] pub struct DownloadUnpackOptions { @@ -57,17 +68,21 @@ impl CardanoDatabaseClient { download_unpack_options: DownloadUnpackOptions, ) -> MithrilResult<()> { let download_id = MithrilEvent::new_snapshot_download_id(); + let compression_algorithm = cardano_database_snapshot.compression_algorithm; + let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; + let immutable_file_number_range = + immutable_file_range.to_range_inclusive(last_immutable_file_number)?; + let immutable_file_range_length = + immutable_file_number_range.end() - immutable_file_number_range.start() + 1; self.feedback_sender .send_event(MithrilEvent::CardanoDatabase( MithrilEventCardanoDatabase::Started { download_id: download_id.clone(), + total_immutable_files: immutable_file_range_length, + include_ancillary: download_unpack_options.include_ancillary, }, )) .await; - let compression_algorithm = cardano_database_snapshot.compression_algorithm; - let last_immutable_file_number = cardano_database_snapshot.beacon.immutable_file_number; - let immutable_file_number_range = - immutable_file_range.to_range_inclusive(last_immutable_file_number)?; Self::verify_download_options_compatibility( &download_unpack_options, &immutable_file_number_range, @@ -216,17 +231,21 @@ impl CardanoDatabaseClient { compression_algorithm: &CompressionAlgorithm, immutable_files_target_dir: &Path, download_id: &str, + file_size: u64, ) -> MithrilResult> { let mut immutable_file_numbers_downloaded = BTreeSet::new(); let mut join_set: JoinSet> = JoinSet::new(); for (immutable_file_number, file_downloader_uri) in file_downloader_uris_chunk.into_iter() { join_set.spawn(self.spawn_immutable_download_future( - file_downloader.clone(), - immutable_file_number, - file_downloader_uri, - compression_algorithm.to_owned(), - immutable_files_target_dir.to_path_buf(), - download_id, + DownloadImmutableFutureBuilderArgs { + file_downloader: file_downloader.clone(), + immutable_file_number, + file_downloader_uri, + compression_algorithm: compression_algorithm.to_owned(), + immutable_files_target_dir: immutable_files_target_dir.to_path_buf(), + download_id: download_id.to_string(), + file_size, + }, )?); } while let Some(result) = join_set.join_next().await { @@ -248,22 +267,24 @@ impl CardanoDatabaseClient { fn spawn_immutable_download_future( &self, - file_downloader: Arc, - immutable_file_number: ImmutableFileNumber, - file_downloader_uri: FileDownloaderUri, - compression_algorithm: CompressionAlgorithm, - immutable_files_target_dir: PathBuf, - download_id: &str, + args: DownloadImmutableFutureBuilderArgs, ) -> MithrilResult>> { let feedback_receiver_clone = self.feedback_sender.clone(); let logger_clone = self.logger.clone(); - let download_id_clone = download_id.to_string(); + let download_id_clone = args.download_id.to_string(); + let file_downloader = args.file_downloader; + let file_downloader_uri = args.file_downloader_uri; + let compression_algorithm = args.compression_algorithm; + let immutable_files_target_dir = args.immutable_files_target_dir; + let immutable_file_number = args.immutable_file_number; + let file_size = args.file_size; let download_future = async move { feedback_receiver_clone .send_event(MithrilEvent::CardanoDatabase( MithrilEventCardanoDatabase::ImmutableDownloadStarted { immutable_file_number, download_id: download_id_clone.clone(), + size: file_size, }, )) .await; @@ -314,6 +335,9 @@ impl CardanoDatabaseClient { download_id: &str, ) -> MithrilResult> { let mut immutable_file_numbers_downloaded = BTreeSet::new(); + // The size will be completed with the uncompressed file size when available in the location + // (see https://github.com/input-output-hk/mithril/issues/2291) + let file_size = 0; let file_downloader = match &location { ImmutablesLocation::CloudStorage { .. } => self.http_file_downloader.clone(), }; @@ -338,6 +362,7 @@ impl CardanoDatabaseClient { compression_algorithm, immutable_files_target_dir, download_id, + file_size, ) .await?; immutable_file_numbers_downloaded.extend(immutable_file_numbers_downloaded_chunk); @@ -357,10 +382,14 @@ impl CardanoDatabaseClient { locations_sorted.sort(); for location in locations_sorted { let download_id = MithrilEvent::new_ancillary_download_id(); + // The size will be completed with the uncompressed file size when available in the location + // (see https://github.com/input-output-hk/mithril/issues/2291) + let file_size = 0; self.feedback_sender .send_event(MithrilEvent::CardanoDatabase( MithrilEventCardanoDatabase::AncillaryDownloadStarted { download_id: download_id.clone(), + size: file_size, }, )) .await; @@ -951,6 +980,7 @@ mod tests { MithrilEventCardanoDatabase::ImmutableDownloadStarted { immutable_file_number: 1, download_id: id.to_string(), + size: 0, }, ), MithrilEvent::CardanoDatabase( @@ -1000,6 +1030,7 @@ mod tests { MithrilEventCardanoDatabase::ImmutableDownloadStarted { immutable_file_number: 1, download_id: id.to_string(), + size: 0, }, )]; assert_eq!(expected_events, sent_events); @@ -1123,6 +1154,7 @@ mod tests { MithrilEvent::CardanoDatabase( MithrilEventCardanoDatabase::AncillaryDownloadStarted { download_id: id.to_string(), + size: 0, }, ), MithrilEvent::CardanoDatabase( diff --git a/mithril-client/src/feedback.rs b/mithril-client/src/feedback.rs index 27697678f5a..2b9125bd392 100644 --- a/mithril-client/src/feedback.rs +++ b/mithril-client/src/feedback.rs @@ -68,6 +68,10 @@ pub enum MithrilEventCardanoDatabase { Started { /// Unique identifier used to track a cardano database download download_id: String, + /// Total number of immutable files + total_immutable_files: u64, + /// Total number of ancillary files + include_ancillary: bool, }, /// Cardano Database download sequence completed Completed { @@ -80,6 +84,8 @@ pub enum MithrilEventCardanoDatabase { immutable_file_number: ImmutableFileNumber, /// Unique identifier used to track a cardano database download download_id: String, + /// Size of the downloaded archive + size: u64, }, /// An immutable archive file download is in progress ImmutableDownloadProgress { @@ -103,6 +109,8 @@ pub enum MithrilEventCardanoDatabase { AncillaryDownloadStarted { /// Unique identifier used to track a cardano database download download_id: String, + /// Size of the downloaded archive + size: u64, }, /// An ancillary archive file download is in progress AncillaryDownloadProgress { @@ -355,9 +363,13 @@ impl FeedbackReceiver for SlogFeedbackReceiver { MithrilEvent::SnapshotDownloadCompleted { download_id } => { info!(self.logger, "Snapshot download completed"; "download_id" => download_id); } - MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::Started { download_id }) => { + MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::Started { + download_id, + total_immutable_files, + include_ancillary, + }) => { info!( - self.logger, "Cardano database download started"; "download_id" => download_id, + self.logger, "Cardano database download started"; "download_id" => download_id, "total_immutable_files" => total_immutable_files, "include_ancillary" => include_ancillary, ); } MithrilEvent::CardanoDatabase(MithrilEventCardanoDatabase::Completed { @@ -371,11 +383,12 @@ impl FeedbackReceiver for SlogFeedbackReceiver { MithrilEventCardanoDatabase::ImmutableDownloadStarted { immutable_file_number, download_id, + size, }, ) => { info!( self.logger, "Immutable download started"; - "immutable_file_number" => immutable_file_number, "download_id" => download_id, + "immutable_file_number" => immutable_file_number, "download_id" => download_id, "size" => size ); } MithrilEvent::CardanoDatabase( @@ -400,11 +413,12 @@ impl FeedbackReceiver for SlogFeedbackReceiver { info!(self.logger, "Immutable download completed"; "immutable_file_number" => immutable_file_number, "download_id" => download_id); } MithrilEvent::CardanoDatabase( - MithrilEventCardanoDatabase::AncillaryDownloadStarted { download_id }, + MithrilEventCardanoDatabase::AncillaryDownloadStarted { download_id, size }, ) => { info!( self.logger, "Ancillary download started"; "download_id" => download_id, + "size" => size, ); } MithrilEvent::CardanoDatabase( From 1e20799a0556c7c8d332a23dc1172f575863eb9a Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 16:43:33 +0100 Subject: [PATCH 52/59] refactor: better handling of download id in Cardano database client --- .../cardano_database_client/download_unpack.rs | 15 +++++++++++---- .../src/cardano_database_client/proving.rs | 2 +- mithril-client/src/feedback.rs | 14 ++------------ 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index 78cacba1d6e..de2bb8d53bf 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -105,6 +105,7 @@ impl CardanoDatabaseClient { ancillary_locations, &compression_algorithm, target_dir, + &download_id, ) .await?; } @@ -377,18 +378,18 @@ impl CardanoDatabaseClient { locations: &[AncillaryLocation], compression_algorithm: &CompressionAlgorithm, ancillary_file_target_dir: &Path, + download_id: &str, ) -> MithrilResult<()> { let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); for location in locations_sorted { - let download_id = MithrilEvent::new_ancillary_download_id(); // The size will be completed with the uncompressed file size when available in the location // (see https://github.com/input-output-hk/mithril/issues/2291) let file_size = 0; self.feedback_sender .send_event(MithrilEvent::CardanoDatabase( MithrilEventCardanoDatabase::AncillaryDownloadStarted { - download_id: download_id.clone(), + download_id: download_id.to_string(), size: file_size, }, )) @@ -403,7 +404,7 @@ impl CardanoDatabaseClient { ancillary_file_target_dir, Some(compression_algorithm.to_owned()), DownloadEvent::Ancillary { - download_id: download_id.clone(), + download_id: download_id.to_string(), }, ) .await; @@ -411,7 +412,9 @@ impl CardanoDatabaseClient { Ok(_) => { self.feedback_sender .send_event(MithrilEvent::CardanoDatabase( - MithrilEventCardanoDatabase::AncillaryDownloadCompleted { download_id }, + MithrilEventCardanoDatabase::AncillaryDownloadCompleted { + download_id: download_id.to_string(), + }, )) .await; return Ok(()); @@ -1057,6 +1060,7 @@ mod tests { }], &CompressionAlgorithm::default(), target_dir, + "download_id", ) .await .expect_err("download_unpack_ancillary_file should fail"); @@ -1091,6 +1095,7 @@ mod tests { ], &CompressionAlgorithm::default(), target_dir, + "download_id", ) .await .unwrap(); @@ -1121,6 +1126,7 @@ mod tests { ], &CompressionAlgorithm::default(), target_dir, + "download_id", ) .await .unwrap(); @@ -1144,6 +1150,7 @@ mod tests { }], &CompressionAlgorithm::default(), target_dir, + "download_id", ) .await .unwrap(); diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index 69982d35633..484fb17079d 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -75,7 +75,7 @@ impl CardanoDatabaseClient { let mut locations_sorted = locations.to_owned(); locations_sorted.sort(); for location in locations_sorted { - let download_id = MithrilEvent::new_digest_download_id(); + let download_id = MithrilEvent::new_cardano_database_download_id(); self.feedback_sender .send_event(MithrilEvent::CardanoDatabase( MithrilEventCardanoDatabase::DigestDownloadStarted { diff --git a/mithril-client/src/feedback.rs b/mithril-client/src/feedback.rs index 2b9125bd392..e374097526d 100644 --- a/mithril-client/src/feedback.rs +++ b/mithril-client/src/feedback.rs @@ -211,18 +211,8 @@ impl MithrilEvent { Uuid::new_v4().to_string() } - /// Generate a random unique identifier to identify an immutable download - pub fn new_immutable_download_id() -> String { - Uuid::new_v4().to_string() - } - - /// Generate a random unique identifier to identify an ancillary download - pub fn new_ancillary_download_id() -> String { - Uuid::new_v4().to_string() - } - - /// Generate a random unique identifier to identify a digest download - pub fn new_digest_download_id() -> String { + /// Generate a random unique identifier to identify a Cardano download + pub fn new_cardano_database_download_id() -> String { Uuid::new_v4().to_string() } From 87537410685993a6efa90660aaeab95c5e355a44 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 16:55:19 +0100 Subject: [PATCH 53/59] fix: client builder untable and support custom HTTP file downloader --- mithril-client/src/client.rs | 37 +++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/mithril-client/src/client.rs b/mithril-client/src/client.rs index e4b64a31b3e..2c4804f282b 100644 --- a/mithril-client/src/client.rs +++ b/mithril-client/src/client.rs @@ -19,7 +19,9 @@ use crate::certificate_client::{ }; use crate::feedback::{FeedbackReceiver, FeedbackSender}; #[cfg(all(feature = "fs", feature = "unstable"))] -use crate::file_downloader::{FileDownloadRetryPolicy, HttpFileDownloader, RetryDownloader}; +use crate::file_downloader::{ + FileDownloadRetryPolicy, FileDownloader, HttpFileDownloader, RetryDownloader, +}; use crate::mithril_stake_distribution_client::MithrilStakeDistributionClient; use crate::snapshot_client::SnapshotClient; use crate::MithrilResult; @@ -134,6 +136,7 @@ pub struct ClientBuilder { genesis_verification_key: String, aggregator_client: Option>, certificate_verifier: Option>, + http_file_downloader: Option>, #[cfg(feature = "unstable")] certificate_verifier_cache: Option>, logger: Option, @@ -152,6 +155,7 @@ impl ClientBuilder { certificate_verifier: None, #[cfg(feature = "unstable")] certificate_verifier_cache: None, + http_file_downloader: None, logger: None, feedback_receivers: vec![], options: ClientOptions::default(), @@ -168,6 +172,7 @@ impl ClientBuilder { genesis_verification_key: genesis_verification_key.to_string(), aggregator_client: None, certificate_verifier: None, + http_file_downloader: None, #[cfg(feature = "unstable")] certificate_verifier_cache: None, logger: None, @@ -234,14 +239,17 @@ impl ClientBuilder { aggregator_client.clone(), )); - #[cfg(all(feature = "fs", feature = "unstable"))] - let http_file_downloader = Arc::new(RetryDownloader::new( - Arc::new( - HttpFileDownloader::new(feedback_sender.clone(), logger.clone()) - .with_context(|| "Building http file downloader failed")?, - ), - FileDownloadRetryPolicy::default(), - )); + #[cfg(feature = "fs")] + let http_file_downloader = match self.http_file_downloader { + None => Arc::new(RetryDownloader::new( + Arc::new( + HttpFileDownloader::new(feedback_sender.clone(), logger.clone()) + .with_context(|| "Building http file downloader failed")?, + ), + FileDownloadRetryPolicy::default(), + )), + Some(http_file_downloader) => http_file_downloader, + }; let snapshot_client = Arc::new(SnapshotClient::new( aggregator_client.clone(), @@ -312,6 +320,17 @@ impl ClientBuilder { } } + cfg_fs! { + /// Set the [FileDownloader] that will be used to download artifacts with HTTP. + pub fn with_http_file_downloader( + mut self, + http_file_downloader: Arc, + ) -> ClientBuilder { + self.http_file_downloader = Some(http_file_downloader); + self + } + } + /// Set the [Logger] to use. pub fn with_logger(mut self, logger: Logger) -> Self { self.logger = Some(logger); From 632e42142ca7532d3fdffb74576dbb38f755c546 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 17:18:56 +0100 Subject: [PATCH 54/59] fix: wrong naming for 'MockAggregatorClient' in client --- mithril-client/src/aggregator_client.rs | 2 +- .../src/cardano_database_client/api.rs | 8 +++---- .../src/cardano_stake_distribution_client.rs | 22 +++++++++---------- .../src/cardano_transaction_client.rs | 10 ++++----- mithril-client/src/certificate_client/mod.rs | 8 +++---- .../src/certificate_client/verify.rs | 6 ++--- .../src/mithril_stake_distribution_client.rs | 6 ++--- mithril-client/src/snapshot_client.rs | 4 ++-- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/mithril-client/src/aggregator_client.rs b/mithril-client/src/aggregator_client.rs index c257980f1ba..78e4f713d7f 100644 --- a/mithril-client/src/aggregator_client.rs +++ b/mithril-client/src/aggregator_client.rs @@ -189,6 +189,7 @@ impl AggregatorRequest { } /// API that defines a client for the Aggregator +#[cfg_attr(test, mockall::automock)] #[cfg_attr(target_family = "wasm", async_trait(?Send))] #[cfg_attr(not(target_family = "wasm"), async_trait)] pub trait AggregatorClient: Sync + Send { @@ -423,7 +424,6 @@ impl AggregatorHTTPClient { } } -#[cfg_attr(test, mockall::automock)] #[cfg_attr(target_family = "wasm", async_trait(?Send))] #[cfg_attr(not(target_family = "wasm"), async_trait)] impl AggregatorClient for AggregatorHTTPClient { diff --git a/mithril-client/src/cardano_database_client/api.rs b/mithril-client/src/cardano_database_client/api.rs index 34a78c7e0f6..6c32c55baf7 100644 --- a/mithril-client/src/cardano_database_client/api.rs +++ b/mithril-client/src/cardano_database_client/api.rs @@ -47,7 +47,7 @@ pub(crate) mod test_dependency_injector { use super::*; use crate::{ - aggregator_client::MockAggregatorHTTPClient, + aggregator_client::MockAggregatorClient, feedback::FeedbackReceiver, file_downloader::{FileDownloader, MockFileDownloaderBuilder}, test_utils, @@ -55,7 +55,7 @@ pub(crate) mod test_dependency_injector { /// Dependency injector for `CardanoDatabaseClient` for testing purposes. pub(crate) struct CardanoDatabaseClientDependencyInjector { - http_client: MockAggregatorHTTPClient, + http_client: MockAggregatorClient, http_file_downloader: Arc, feedback_receivers: Vec>, } @@ -63,7 +63,7 @@ pub(crate) mod test_dependency_injector { impl CardanoDatabaseClientDependencyInjector { pub(crate) fn new() -> Self { Self { - http_client: MockAggregatorHTTPClient::new(), + http_client: MockAggregatorClient::new(), http_file_downloader: Arc::new( MockFileDownloaderBuilder::default() .with_compression(None) @@ -77,7 +77,7 @@ pub(crate) mod test_dependency_injector { pub(crate) fn with_http_client_mock_config(mut self, config: F) -> Self where - F: FnOnce(&mut MockAggregatorHTTPClient), + F: FnOnce(&mut MockAggregatorClient), { config(&mut self.http_client); diff --git a/mithril-client/src/cardano_stake_distribution_client.rs b/mithril-client/src/cardano_stake_distribution_client.rs index 18cf80d4161..832ef23255f 100644 --- a/mithril-client/src/cardano_stake_distribution_client.rs +++ b/mithril-client/src/cardano_stake_distribution_client.rs @@ -144,7 +144,7 @@ mod tests { use chrono::{DateTime, Utc}; use mockall::predicate::eq; - use crate::aggregator_client::MockAggregatorHTTPClient; + use crate::aggregator_client::MockAggregatorClient; use crate::common::StakeDistribution; use super::*; @@ -173,7 +173,7 @@ mod tests { #[tokio::test] async fn list_cardano_stake_distributions_returns_messages() { let message = fake_messages(); - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client .expect_get_content() .with(eq(AggregatorRequest::ListCardanoStakeDistributions)) @@ -190,7 +190,7 @@ mod tests { #[tokio::test] async fn list_cardano_stake_distributions_returns_error_when_invalid_json_structure_in_response( ) { - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client .expect_get_content() .return_once(move |_| Ok("invalid json structure".to_string())); @@ -212,7 +212,7 @@ mod tests { stake_distribution: expected_stake_distribution.clone(), created_at: DateTime::::default(), }; - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client .expect_get_content() .with(eq(AggregatorRequest::GetCardanoStakeDistribution { @@ -238,7 +238,7 @@ mod tests { #[tokio::test] async fn get_cardano_stake_distribution_returns_error_when_invalid_json_structure_in_response() { - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client .expect_get_content() .return_once(move |_| Ok("invalid json structure".to_string())); @@ -253,7 +253,7 @@ mod tests { #[tokio::test] async fn get_cardano_stake_distribution_returns_none_when_not_found_or_remote_server_logical_error( ) { - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client.expect_get_content().return_once(move |_| { Err(AggregatorClientError::RemoteServerLogical(anyhow!( "not found" @@ -268,7 +268,7 @@ mod tests { #[tokio::test] async fn get_cardano_stake_distribution_returns_error() { - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client .expect_get_content() .return_once(move |_| Err(AggregatorClientError::SubsystemError(anyhow!("error")))); @@ -290,7 +290,7 @@ mod tests { stake_distribution: expected_stake_distribution.clone(), created_at: DateTime::::default(), }; - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client .expect_get_content() .with(eq(AggregatorRequest::GetCardanoStakeDistributionByEpoch { @@ -316,7 +316,7 @@ mod tests { #[tokio::test] async fn get_cardano_stake_distribution_by_epoch_returns_error_when_invalid_json_structure_in_response( ) { - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client .expect_get_content() .return_once(move |_| Ok("invalid json structure".to_string())); @@ -331,7 +331,7 @@ mod tests { #[tokio::test] async fn get_cardano_stake_distribution_by_epoch_returns_none_when_not_found_or_remote_server_logical_error( ) { - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client.expect_get_content().return_once(move |_| { Err(AggregatorClientError::RemoteServerLogical(anyhow!( "not found" @@ -346,7 +346,7 @@ mod tests { #[tokio::test] async fn get_cardano_stake_distribution_by_epoch_returns_error() { - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client .expect_get_content() .return_once(move |_| Err(AggregatorClientError::SubsystemError(anyhow!("error")))); diff --git a/mithril-client/src/cardano_transaction_client.rs b/mithril-client/src/cardano_transaction_client.rs index f0b4d6eff9b..946f8de68db 100644 --- a/mithril-client/src/cardano_transaction_client.rs +++ b/mithril-client/src/cardano_transaction_client.rs @@ -164,7 +164,7 @@ mod tests { use chrono::{DateTime, Utc}; use mockall::predicate::eq; - use crate::aggregator_client::{AggregatorClientError, MockAggregatorHTTPClient}; + use crate::aggregator_client::{AggregatorClientError, MockAggregatorClient}; use crate::common::{BlockNumber, Epoch}; use crate::{ CardanoTransactionSnapshot, CardanoTransactionSnapshotListItem, CardanoTransactionsProofs, @@ -201,7 +201,7 @@ mod tests { #[tokio::test] async fn get_cardano_transactions_snapshot_list() { let message = fake_messages(); - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client .expect_get_content() .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); @@ -215,7 +215,7 @@ mod tests { #[tokio::test] async fn get_cardano_transactions_snapshot() { - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); let message = CardanoTransactionSnapshot { merkle_root: "mk-123".to_string(), epoch: Epoch(1), @@ -245,7 +245,7 @@ mod tests { #[tokio::test] async fn test_get_proof_ok() { - let mut aggregator_client = MockAggregatorHTTPClient::new(); + let mut aggregator_client = MockAggregatorClient::new(); let certificate_hash = "cert-hash-123".to_string(); let set_proof = CardanoTransactionsSetProof::dummy(); let transactions_proofs = CardanoTransactionsProofs::new( @@ -277,7 +277,7 @@ mod tests { #[tokio::test] async fn test_get_proof_ko() { - let mut aggregator_client = MockAggregatorHTTPClient::new(); + let mut aggregator_client = MockAggregatorClient::new(); aggregator_client .expect_get_content() .return_once(move |_| { diff --git a/mithril-client/src/certificate_client/mod.rs b/mithril-client/src/certificate_client/mod.rs index d7f6dbd461f..add91a073d2 100644 --- a/mithril-client/src/certificate_client/mod.rs +++ b/mithril-client/src/certificate_client/mod.rs @@ -74,7 +74,7 @@ pub(crate) mod tests_utils { use mockall::predicate::eq; use std::sync::Arc; - use crate::aggregator_client::{AggregatorRequest, MockAggregatorHTTPClient}; + use crate::aggregator_client::{AggregatorRequest, MockAggregatorClient}; use crate::feedback::{FeedbackReceiver, FeedbackSender}; use crate::test_utils; @@ -82,7 +82,7 @@ pub(crate) mod tests_utils { #[derive(Default)] pub(crate) struct CertificateClientTestBuilder { - aggregator_client: MockAggregatorHTTPClient, + aggregator_client: MockAggregatorClient, genesis_verification_key: Option, feedback_receivers: Vec>, #[cfg(feature = "unstable")] @@ -92,7 +92,7 @@ pub(crate) mod tests_utils { impl CertificateClientTestBuilder { pub fn config_aggregator_client_mock( mut self, - config: impl FnOnce(&mut MockAggregatorHTTPClient), + config: impl FnOnce(&mut MockAggregatorClient), ) -> Self { config(&mut self.aggregator_client); self @@ -151,7 +151,7 @@ pub(crate) mod tests_utils { } } - impl MockAggregatorHTTPClient { + impl MockAggregatorClient { pub(crate) fn expect_certificate_chain(&mut self, certificate_chain: Vec) { for certificate in certificate_chain { let hash = certificate.hash.clone(); diff --git a/mithril-client/src/certificate_client/verify.rs b/mithril-client/src/certificate_client/verify.rs index ffaf66652b1..e5f23753e5f 100644 --- a/mithril-client/src/certificate_client/verify.rs +++ b/mithril-client/src/certificate_client/verify.rs @@ -329,7 +329,7 @@ mod tests { use mithril_common::test_utils::CertificateChainingMethod; use mockall::predicate::eq; - use crate::aggregator_client::MockAggregatorHTTPClient; + use crate::aggregator_client::MockAggregatorClient; use crate::certificate_client::verify_cache::MemoryCertificateVerifierCache; use crate::certificate_client::MockCertificateVerifierCache; use crate::test_utils; @@ -337,11 +337,11 @@ mod tests { use super::*; fn build_verifier_with_cache( - aggregator_client_mock_config: impl FnOnce(&mut MockAggregatorHTTPClient), + aggregator_client_mock_config: impl FnOnce(&mut MockAggregatorClient), genesis_verification_key: ProtocolGenesisVerificationKey, cache: Arc, ) -> MithrilCertificateVerifier { - let mut aggregator_client = MockAggregatorHTTPClient::new(); + let mut aggregator_client = MockAggregatorClient::new(); aggregator_client_mock_config(&mut aggregator_client); let genesis_verification_key: String = genesis_verification_key.try_into().unwrap(); diff --git a/mithril-client/src/mithril_stake_distribution_client.rs b/mithril-client/src/mithril_stake_distribution_client.rs index a53e0a13d71..a133a767b58 100644 --- a/mithril-client/src/mithril_stake_distribution_client.rs +++ b/mithril-client/src/mithril_stake_distribution_client.rs @@ -97,7 +97,7 @@ mod tests { use chrono::{DateTime, Utc}; use mithril_common::test_utils::fake_data; - use crate::aggregator_client::MockAggregatorHTTPClient; + use crate::aggregator_client::MockAggregatorClient; use crate::common::Epoch; use crate::MithrilSigner; @@ -127,7 +127,7 @@ mod tests { #[tokio::test] async fn get_mithril_stake_distribution_list() { let message = fake_messages(); - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); http_client .expect_get_content() .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); @@ -141,7 +141,7 @@ mod tests { #[tokio::test] async fn get_mithril_stake_distribution() { - let mut http_client = MockAggregatorHTTPClient::new(); + let mut http_client = MockAggregatorClient::new(); let message = MithrilStakeDistribution { certificate_hash: "certificate-hash-123".to_string(), epoch: Epoch(1), diff --git a/mithril-client/src/snapshot_client.rs b/mithril-client/src/snapshot_client.rs index 04dc4faab0c..7e3401d4382 100644 --- a/mithril-client/src/snapshot_client.rs +++ b/mithril-client/src/snapshot_client.rs @@ -264,7 +264,7 @@ impl SnapshotClient { #[cfg(all(test, feature = "fs"))] mod tests_download { use crate::{ - aggregator_client::MockAggregatorHTTPClient, + aggregator_client::MockAggregatorClient, feedback::{MithrilEvent, StackFeedbackReceiver}, file_downloader::MockFileDownloaderBuilder, test_utils, @@ -277,7 +277,7 @@ mod tests_download { async fn download_unpack_send_feedbacks() { let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); let client = SnapshotClient::new( - Arc::new(MockAggregatorHTTPClient::new()), + Arc::new(MockAggregatorClient::new()), Arc::new(MockFileDownloaderBuilder::default().with_success().build()), FeedbackSender::new(&[feedback_receiver.clone()]), test_utils::test_logger(), From b07fe82449c9957e00fea4b86aa1e45fa3ae32a1 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 17:21:56 +0100 Subject: [PATCH 55/59] fix: 'CardanoDatabaseClientDependencyInjector' use of 'http_client' instead of 'aggregator_client' prefix --- mithril-client/src/cardano_database_client/api.rs | 10 +++++----- mithril-client/src/cardano_database_client/fetch.rs | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mithril-client/src/cardano_database_client/api.rs b/mithril-client/src/cardano_database_client/api.rs index 6c32c55baf7..1001552cb4b 100644 --- a/mithril-client/src/cardano_database_client/api.rs +++ b/mithril-client/src/cardano_database_client/api.rs @@ -55,7 +55,7 @@ pub(crate) mod test_dependency_injector { /// Dependency injector for `CardanoDatabaseClient` for testing purposes. pub(crate) struct CardanoDatabaseClientDependencyInjector { - http_client: MockAggregatorClient, + aggregator_client: MockAggregatorClient, http_file_downloader: Arc, feedback_receivers: Vec>, } @@ -63,7 +63,7 @@ pub(crate) mod test_dependency_injector { impl CardanoDatabaseClientDependencyInjector { pub(crate) fn new() -> Self { Self { - http_client: MockAggregatorClient::new(), + aggregator_client: MockAggregatorClient::new(), http_file_downloader: Arc::new( MockFileDownloaderBuilder::default() .with_compression(None) @@ -75,11 +75,11 @@ pub(crate) mod test_dependency_injector { } } - pub(crate) fn with_http_client_mock_config(mut self, config: F) -> Self + pub(crate) fn with_aggregator_client_mock_config(mut self, config: F) -> Self where F: FnOnce(&mut MockAggregatorClient), { - config(&mut self.http_client); + config(&mut self.aggregator_client); self } @@ -106,7 +106,7 @@ pub(crate) mod test_dependency_injector { pub(crate) fn build_cardano_database_client(self) -> CardanoDatabaseClient { CardanoDatabaseClient::new( - Arc::new(self.http_client), + Arc::new(self.aggregator_client), self.http_file_downloader, FeedbackSender::new(&self.feedback_receivers), test_utils::test_logger(), diff --git a/mithril-client/src/cardano_database_client/fetch.rs b/mithril-client/src/cardano_database_client/fetch.rs index 9084192bcac..d49fa0b14c8 100644 --- a/mithril-client/src/cardano_database_client/fetch.rs +++ b/mithril-client/src/cardano_database_client/fetch.rs @@ -105,7 +105,7 @@ mod tests { async fn list_cardano_database_snapshots_returns_messages() { let message = fake_messages(); let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { + .with_aggregator_client_mock_config(|http_client| { http_client .expect_get_content() .with(eq(AggregatorRequest::ListCardanoDatabaseSnapshots)) @@ -124,7 +124,7 @@ mod tests { async fn list_cardano_database_snapshots_returns_error_when_invalid_json_structure_in_response( ) { let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { + .with_aggregator_client_mock_config(|http_client| { http_client .expect_get_content() .return_once(move |_| Ok("invalid json structure".to_string())); @@ -149,7 +149,7 @@ mod tests { }; let message = expected_cardano_database_snapshot.clone(); let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { + .with_aggregator_client_mock_config(|http_client| { http_client .expect_get_content() .with(eq(AggregatorRequest::GetCardanoDatabaseSnapshot { @@ -172,7 +172,7 @@ mod tests { async fn get_cardano_database_snapshot_returns_error_when_invalid_json_structure_in_response( ) { let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { + .with_aggregator_client_mock_config(|http_client| { http_client .expect_get_content() .return_once(move |_| Ok("invalid json structure".to_string())); @@ -189,7 +189,7 @@ mod tests { async fn get_cardano_database_snapshot_returns_none_when_not_found_or_remote_server_logical_error( ) { let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { + .with_aggregator_client_mock_config(|http_client| { http_client.expect_get_content().return_once(move |_| { Err(AggregatorClientError::RemoteServerLogical(anyhow!( "not found" @@ -206,7 +206,7 @@ mod tests { #[tokio::test] async fn get_cardano_database_snapshot_returns_error() { let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_client_mock_config(|http_client| { + .with_aggregator_client_mock_config(|http_client| { http_client.expect_get_content().return_once(move |_| { Err(AggregatorClientError::SubsystemError(anyhow!("error"))) }); From e0e225620235cfc2ebb0e92b1910e4fb8b247aad Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 18:07:05 +0100 Subject: [PATCH 56/59] refactor: use one 'impl' block for 'cardano_database_client' This will allow nice documentation for the module. --- .../src/cardano_database_client/api.rs | 124 ++++++++++-- .../download_unpack.rs | 182 ++++++++++-------- .../src/cardano_database_client/fetch.rs | 15 +- .../src/cardano_database_client/proving.rs | 142 +++++++++----- mithril-client/src/client.rs | 7 +- 5 files changed, 333 insertions(+), 137 deletions(-) diff --git a/mithril-client/src/cardano_database_client/api.rs b/mithril-client/src/cardano_database_client/api.rs index 1001552cb4b..9dcef1bb9ab 100644 --- a/mithril-client/src/cardano_database_client/api.rs +++ b/mithril-client/src/cardano_database_client/api.rs @@ -1,23 +1,37 @@ +#[cfg(feature = "fs")] +use std::path::Path; use std::sync::Arc; #[cfg(feature = "fs")] use slog::Logger; +#[cfg(feature = "fs")] +use mithril_common::{ + crypto_helper::MKProof, + messages::{CardanoDatabaseSnapshotMessage, CertificateMessage}, +}; + use crate::aggregator_client::AggregatorClient; #[cfg(feature = "fs")] use crate::feedback::FeedbackSender; #[cfg(feature = "fs")] use crate::file_downloader::FileDownloader; +use crate::{CardanoDatabaseSnapshot, CardanoDatabaseSnapshotListItem, MithrilResult}; + +use super::fetch::InternalArtifactRetriever; +#[cfg(feature = "fs")] +use super::{ + download_unpack::InternalArtifactDownloader, proving::InternalArtifactProver, + DownloadUnpackOptions, ImmutableFileRange, +}; /// HTTP client for CardanoDatabase API from the Aggregator pub struct CardanoDatabaseClient { - pub(super) aggregator_client: Arc, - #[cfg(feature = "fs")] - pub(super) http_file_downloader: Arc, + pub(super) artifact_retriever: InternalArtifactRetriever, #[cfg(feature = "fs")] - pub(super) feedback_sender: FeedbackSender, + pub(super) artifact_downloader: InternalArtifactDownloader, #[cfg(feature = "fs")] - pub(super) logger: Logger, + pub(super) artifact_prover: InternalArtifactProver, } impl CardanoDatabaseClient { @@ -28,18 +42,73 @@ impl CardanoDatabaseClient { #[cfg(feature = "fs")] feedback_sender: FeedbackSender, #[cfg(feature = "fs")] logger: Logger, ) -> Self { + #[cfg(feature = "fs")] + let logger = + mithril_common::logging::LoggerExtensions::new_with_component_name::(&logger); Self { - aggregator_client, + artifact_retriever: InternalArtifactRetriever::new(aggregator_client.clone()), #[cfg(feature = "fs")] - http_file_downloader, - #[cfg(feature = "fs")] - feedback_sender, + artifact_downloader: InternalArtifactDownloader::new( + http_file_downloader.clone(), + feedback_sender.clone(), + logger.clone(), + ), #[cfg(feature = "fs")] - logger: mithril_common::logging::LoggerExtensions::new_with_component_name::( - &logger, + artifact_prover: InternalArtifactProver::new( + http_file_downloader.clone(), + feedback_sender.clone(), + logger.clone(), ), } } + + /// Fetch a list of signed CardanoDatabase + pub async fn list(&self) -> MithrilResult> { + self.artifact_retriever.list().await + } + + /// Get the given Cardano database data by hash + pub async fn get(&self, hash: &str) -> MithrilResult> { + self.artifact_retriever.get(hash).await + } + + /// Download and unpack the given Cardano database parts data by hash. + #[cfg(feature = "fs")] + pub async fn download_unpack( + &self, + cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, + immutable_file_range: &ImmutableFileRange, + target_dir: &Path, + download_unpack_options: DownloadUnpackOptions, + ) -> MithrilResult<()> { + self.artifact_downloader + .download_unpack( + cardano_database_snapshot, + immutable_file_range, + target_dir, + download_unpack_options, + ) + .await + } + + /// Compute the Merkle proof of membership for the given immutable file range. + #[cfg(feature = "fs")] + pub async fn compute_merkle_proof( + &self, + certificate: &CertificateMessage, + cardano_database_snapshot: &CardanoDatabaseSnapshotMessage, + immutable_file_range: &ImmutableFileRange, + database_dir: &Path, + ) -> MithrilResult { + self.artifact_prover + .compute_merkle_proof( + certificate, + cardano_database_snapshot, + immutable_file_range, + database_dir, + ) + .await + } } #[cfg(test)] @@ -113,4 +182,37 @@ pub(crate) mod test_dependency_injector { ) } } + + mod tests { + use mockall::predicate; + + use crate::{aggregator_client::AggregatorRequest, feedback::StackFeedbackReceiver}; + + use super::*; + + #[test] + fn test_cardano_database_client_dependency_injector_builds() { + let _ = CardanoDatabaseClientDependencyInjector::new() + .with_aggregator_client_mock_config(|http_client| { + let message = vec![CardanoDatabaseSnapshotListItem { + hash: "hash-123".to_string(), + ..CardanoDatabaseSnapshotListItem::dummy() + }]; + http_client + .expect_get_content() + .with(predicate::eq( + AggregatorRequest::ListCardanoDatabaseSnapshots, + )) + .return_once(move |_| Ok(serde_json::to_string(&message).unwrap())); + }) + .with_http_file_downloader(Arc::new( + MockFileDownloaderBuilder::default() + .with_success() + .with_times(0) + .build(), + )) + .with_feedback_receivers(&[Arc::new(StackFeedbackReceiver::new())]) + .build_cardano_database_client(); + } + } } diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index de2bb8d53bf..ae003a315b5 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -14,11 +14,10 @@ use mithril_common::{ messages::CardanoDatabaseSnapshotMessage, }; -use crate::feedback::{MithrilEvent, MithrilEventCardanoDatabase}; +use crate::feedback::{FeedbackSender, MithrilEvent, MithrilEventCardanoDatabase}; use crate::file_downloader::{DownloadEvent, FileDownloader, FileDownloaderUri}; use crate::MithrilResult; -use super::api::CardanoDatabaseClient; use super::immutable_file_range::ImmutableFileRange; /// The future type for downloading an immutable file @@ -58,7 +57,26 @@ impl Default for DownloadUnpackOptions { } } -impl CardanoDatabaseClient { +pub struct InternalArtifactDownloader { + http_file_downloader: Arc, + feedback_sender: FeedbackSender, + logger: slog::Logger, +} + +impl InternalArtifactDownloader { + /// Constructs a new `InternalArtifactDownloader`. + pub fn new( + http_file_downloader: Arc, + feedback_sender: FeedbackSender, + logger: slog::Logger, + ) -> Self { + Self { + http_file_downloader, + feedback_sender, + logger, + } + } + /// Download and unpack the given Cardano database parts data by hash. pub async fn download_unpack( &self, @@ -373,7 +391,7 @@ impl CardanoDatabaseClient { } /// Download and unpack the ancillary files. - async fn download_unpack_ancillary_file( + pub(crate) async fn download_unpack_ancillary_file( &self, locations: &[AncillaryLocation], compression_algorithm: &CompressionAlgorithm, @@ -451,6 +469,7 @@ mod tests { use crate::cardano_database_client::CardanoDatabaseClientDependencyInjector; use crate::feedback::StackFeedbackReceiver; use crate::file_downloader::MockFileDownloaderBuilder; + use crate::test_utils; use super::*; @@ -627,7 +646,7 @@ mod tests { let immutable_file_range = ImmutableFileRange::Range(1, 10); let last_immutable_file_number = 10; - CardanoDatabaseClient::verify_download_options_compatibility( + InternalArtifactDownloader::verify_download_options_compatibility( &download_options, &immutable_file_range .to_range_inclusive(last_immutable_file_number) @@ -647,7 +666,7 @@ mod tests { let immutable_file_range = ImmutableFileRange::Range(7, 10); let last_immutable_file_number = 10; - CardanoDatabaseClient::verify_download_options_compatibility( + InternalArtifactDownloader::verify_download_options_compatibility( &download_options, &immutable_file_range .to_range_inclusive(last_immutable_file_number) @@ -667,7 +686,7 @@ mod tests { let immutable_file_range = ImmutableFileRange::Range(7, 10); let last_immutable_file_number = 123; - CardanoDatabaseClient::verify_download_options_compatibility( + InternalArtifactDownloader::verify_download_options_compatibility( &download_options, &immutable_file_range .to_range_inclusive(last_immutable_file_number) @@ -690,7 +709,7 @@ mod tests { ) .build(); - CardanoDatabaseClient::verify_can_write_to_target_directory( + InternalArtifactDownloader::verify_can_write_to_target_directory( &target_dir, &DownloadUnpackOptions { allow_override: true, @@ -700,13 +719,14 @@ mod tests { ) .unwrap(); - fs::create_dir_all(CardanoDatabaseClient::immutable_files_target_dir( + fs::create_dir_all(InternalArtifactDownloader::immutable_files_target_dir( &target_dir, )) .unwrap(); - fs::create_dir_all(CardanoDatabaseClient::volatile_target_dir(&target_dir)).unwrap(); - fs::create_dir_all(CardanoDatabaseClient::ledger_target_dir(&target_dir)).unwrap(); - CardanoDatabaseClient::verify_can_write_to_target_directory( + fs::create_dir_all(InternalArtifactDownloader::volatile_target_dir(&target_dir)) + .unwrap(); + fs::create_dir_all(InternalArtifactDownloader::ledger_target_dir(&target_dir)).unwrap(); + InternalArtifactDownloader::verify_can_write_to_target_directory( &target_dir, &DownloadUnpackOptions { allow_override: true, @@ -715,7 +735,7 @@ mod tests { }, ) .unwrap(); - CardanoDatabaseClient::verify_can_write_to_target_directory( + InternalArtifactDownloader::verify_can_write_to_target_directory( &target_dir, &DownloadUnpackOptions { allow_override: true, @@ -730,12 +750,12 @@ mod tests { fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_immutable_target_dir( ) { let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_immutable_target_dir").build(); - fs::create_dir_all(CardanoDatabaseClient::immutable_files_target_dir( + fs::create_dir_all(InternalArtifactDownloader::immutable_files_target_dir( &target_dir, )) .unwrap(); - CardanoDatabaseClient::verify_can_write_to_target_directory( + InternalArtifactDownloader::verify_can_write_to_target_directory( &target_dir, &DownloadUnpackOptions { allow_override: false, @@ -745,7 +765,7 @@ mod tests { ) .expect_err("verify_can_write_to_target_dir should fail"); - CardanoDatabaseClient::verify_can_write_to_target_directory( + InternalArtifactDownloader::verify_can_write_to_target_directory( &target_dir, &DownloadUnpackOptions { allow_override: false, @@ -760,9 +780,9 @@ mod tests { fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_ledger_target_dir( ) { let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_ledger_target_dir").build(); - fs::create_dir_all(CardanoDatabaseClient::ledger_target_dir(&target_dir)).unwrap(); + fs::create_dir_all(InternalArtifactDownloader::ledger_target_dir(&target_dir)).unwrap(); - CardanoDatabaseClient::verify_can_write_to_target_directory( + InternalArtifactDownloader::verify_can_write_to_target_directory( &target_dir, &DownloadUnpackOptions { allow_override: false, @@ -772,7 +792,7 @@ mod tests { ) .expect_err("verify_can_write_to_target_dir should fail"); - CardanoDatabaseClient::verify_can_write_to_target_directory( + InternalArtifactDownloader::verify_can_write_to_target_directory( &target_dir, &DownloadUnpackOptions { allow_override: false, @@ -787,9 +807,10 @@ mod tests { fn verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_volatile_target_dir( ) { let target_dir = TempDir::new("cardano_database_client", "verify_can_write_to_target_dir_fails_without_allow_overwrite_and_non_empty_volatile_target_dir").build(); - fs::create_dir_all(CardanoDatabaseClient::volatile_target_dir(&target_dir)).unwrap(); + fs::create_dir_all(InternalArtifactDownloader::volatile_target_dir(&target_dir)) + .unwrap(); - CardanoDatabaseClient::verify_can_write_to_target_directory( + InternalArtifactDownloader::verify_can_write_to_target_directory( &target_dir, &DownloadUnpackOptions { allow_override: false, @@ -799,7 +820,7 @@ mod tests { ) .expect_err("verify_can_write_to_target_dir should fail"); - CardanoDatabaseClient::verify_can_write_to_target_directory( + InternalArtifactDownloader::verify_can_write_to_target_directory( &target_dir, &DownloadUnpackOptions { allow_override: false, @@ -824,17 +845,19 @@ mod tests { "download_unpack_immutable_files_succeeds", ) .build(); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new({ + let artifact_downloader = InternalArtifactDownloader::new( + Arc::new( MockFileDownloaderBuilder::default() .with_failure() .next_call() .with_success() - .build() - })) - .build_cardano_database_client(); + .build(), + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); - client + artifact_downloader .download_unpack_immutable_files( &[ImmutablesLocation::CloudStorage { uri: MultiFilesUri::Template(TemplateUri( @@ -863,16 +886,18 @@ mod tests { "download_unpack_immutable_files_succeeds", ) .build(); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new( + let artifact_downloader = InternalArtifactDownloader::new( + Arc::new( MockFileDownloaderBuilder::default() .with_times(2) .with_success() .build(), - )) - .build_cardano_database_client(); + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); - client + artifact_downloader .download_unpack_immutable_files( &[ImmutablesLocation::CloudStorage { uri: MultiFilesUri::Template(TemplateUri( @@ -901,8 +926,8 @@ mod tests { "download_unpack_immutable_files_succeeds", ) .build(); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new({ + let artifact_downloader = InternalArtifactDownloader::new( + Arc::new( MockFileDownloaderBuilder::default() .with_file_uri("http://whatever-1/00001.tar.gz") .with_target_dir(target_dir.clone()) @@ -915,11 +940,13 @@ mod tests { .with_file_uri("http://whatever-2/00001.tar.gz") .with_target_dir(target_dir.clone()) .with_success() - .build() - })) - .build_cardano_database_client(); + .build(), + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); - client + artifact_downloader .download_unpack_immutable_files( &[ ImmutablesLocation::CloudStorage { @@ -951,14 +978,13 @@ mod tests { let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); let target_dir = Path::new("."); let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new( - MockFileDownloaderBuilder::default().with_success().build(), - )) - .with_feedback_receivers(&[feedback_receiver.clone()]) - .build_cardano_database_client(); + let artifact_downloader = InternalArtifactDownloader::new( + Arc::new(MockFileDownloaderBuilder::default().with_success().build()), + FeedbackSender::new(&[feedback_receiver.clone()]), + test_utils::test_logger(), + ); - client + artifact_downloader .download_unpack_immutable_files( &[ImmutablesLocation::CloudStorage { uri: MultiFilesUri::Template(TemplateUri( @@ -1002,14 +1028,13 @@ mod tests { let immutable_file_range = ImmutableFileRange::Range(1, total_immutable_files); let target_dir = Path::new("."); let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new( - MockFileDownloaderBuilder::default().with_failure().build(), - )) - .with_feedback_receivers(&[feedback_receiver.clone()]) - .build_cardano_database_client(); + let artifact_downloader = InternalArtifactDownloader::new( + Arc::new(MockFileDownloaderBuilder::default().with_failure().build()), + FeedbackSender::new(&[feedback_receiver.clone()]), + test_utils::test_logger(), + ); - client + artifact_downloader .download_unpack_immutable_files( &[ImmutablesLocation::CloudStorage { uri: MultiFilesUri::Template(TemplateUri( @@ -1047,13 +1072,13 @@ mod tests { #[tokio::test] async fn download_unpack_ancillary_file_fails_if_no_location_is_retrieved() { let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new( - MockFileDownloaderBuilder::default().with_failure().build(), - )) - .build_cardano_database_client(); + let artifact_downloader = InternalArtifactDownloader::new( + Arc::new(MockFileDownloaderBuilder::default().with_failure().build()), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); - client + artifact_downloader .download_unpack_ancillary_file( &[AncillaryLocation::CloudStorage { uri: "http://whatever-1/ancillary.tar.gz".to_string(), @@ -1069,8 +1094,8 @@ mod tests { #[tokio::test] async fn download_unpack_ancillary_file_succeeds_if_at_least_one_location_is_retrieved() { let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new({ + let artifact_downloader = InternalArtifactDownloader::new( + Arc::new( MockFileDownloaderBuilder::default() .with_file_uri("http://whatever-1/ancillary.tar.gz") .with_target_dir(target_dir.to_path_buf()) @@ -1079,11 +1104,13 @@ mod tests { .with_file_uri("http://whatever-2/ancillary.tar.gz") .with_target_dir(target_dir.to_path_buf()) .with_success() - .build() - })) - .build_cardano_database_client(); + .build(), + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); - client + artifact_downloader .download_unpack_ancillary_file( &[ AncillaryLocation::CloudStorage { @@ -1104,17 +1131,19 @@ mod tests { #[tokio::test] async fn download_unpack_ancillary_file_succeeds_when_first_location_is_retrieved() { let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new( + let artifact_downloader = InternalArtifactDownloader::new( + Arc::new( MockFileDownloaderBuilder::default() .with_file_uri("http://whatever-1/ancillary.tar.gz") .with_target_dir(target_dir.to_path_buf()) .with_success() .build(), - )) - .build_cardano_database_client(); + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); - client + artifact_downloader .download_unpack_ancillary_file( &[ AncillaryLocation::CloudStorage { @@ -1136,14 +1165,13 @@ mod tests { async fn download_unpack_ancillary_files_sends_feedbacks() { let target_dir = Path::new("."); let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new( - MockFileDownloaderBuilder::default().with_success().build(), - )) - .with_feedback_receivers(&[feedback_receiver.clone()]) - .build_cardano_database_client(); + let artifact_downloader = InternalArtifactDownloader::new( + Arc::new(MockFileDownloaderBuilder::default().with_success().build()), + FeedbackSender::new(&[feedback_receiver.clone()]), + test_utils::test_logger(), + ); - client + artifact_downloader .download_unpack_ancillary_file( &[AncillaryLocation::CloudStorage { uri: "http://whatever-1/ancillary.tar.gz".to_string(), diff --git a/mithril-client/src/cardano_database_client/fetch.rs b/mithril-client/src/cardano_database_client/fetch.rs index d49fa0b14c8..18a376d077f 100644 --- a/mithril-client/src/cardano_database_client/fetch.rs +++ b/mithril-client/src/cardano_database_client/fetch.rs @@ -1,14 +1,23 @@ +use std::sync::Arc; + use anyhow::Context; use serde::de::DeserializeOwned; use crate::{ - aggregator_client::{AggregatorClientError, AggregatorRequest}, + aggregator_client::{AggregatorClient, AggregatorClientError, AggregatorRequest}, CardanoDatabaseSnapshot, CardanoDatabaseSnapshotListItem, MithrilResult, }; -use super::api::CardanoDatabaseClient; +pub struct InternalArtifactRetriever { + pub(super) aggregator_client: Arc, +} + +impl InternalArtifactRetriever { + /// Constructs a new `InternalArtifactRetriever` + pub fn new(aggregator_client: Arc) -> Self { + Self { aggregator_client } + } -impl CardanoDatabaseClient { /// Fetch a list of signed CardanoDatabase pub async fn list(&self) -> MithrilResult> { let response = self diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index 484fb17079d..7bb4d5cefa6 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -2,6 +2,7 @@ use std::{ collections::BTreeMap, fs, path::{Path, PathBuf}, + sync::Arc, }; use anyhow::{anyhow, Context}; @@ -16,16 +17,34 @@ use mithril_common::{ }; use crate::{ - feedback::{MithrilEvent, MithrilEventCardanoDatabase}, - file_downloader::{DownloadEvent, FileDownloaderUri}, + feedback::{FeedbackSender, MithrilEvent, MithrilEventCardanoDatabase}, + file_downloader::{DownloadEvent, FileDownloader, FileDownloaderUri}, utils::{create_directory_if_not_exists, delete_directory, read_files_in_directory}, MithrilResult, }; -use super::api::CardanoDatabaseClient; use super::immutable_file_range::ImmutableFileRange; -impl CardanoDatabaseClient { +pub struct InternalArtifactProver { + http_file_downloader: Arc, + feedback_sender: FeedbackSender, + logger: slog::Logger, +} + +impl InternalArtifactProver { + /// Constructs a new `InternalArtifactProver`. + pub fn new( + http_file_downloader: Arc, + feedback_sender: FeedbackSender, + logger: slog::Logger, + ) -> Self { + Self { + http_file_downloader, + feedback_sender, + logger, + } + } + /// Compute the Merkle proof of membership for the given immutable file range. pub async fn compute_merkle_proof( &self, @@ -174,8 +193,7 @@ mod tests { use crate::{ cardano_database_client::CardanoDatabaseClientDependencyInjector, - feedback::StackFeedbackReceiver, file_downloader::MockFileDownloaderBuilder, - test_utils::test_logger, + feedback::StackFeedbackReceiver, file_downloader::MockFileDownloaderBuilder, test_utils, }; use super::*; @@ -222,7 +240,7 @@ mod tests { let immutable_digester = CardanoImmutableDigester::new( certificate.metadata.network.to_string(), None, - test_logger(), + test_utils::test_logger(), ); let computed_digests = immutable_digester .compute_digests_for_range(database_dir, immutable_file_range) @@ -337,17 +355,19 @@ mod tests { #[tokio::test] async fn fails_if_no_location_is_retrieved() { let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new( + let artifact_prover = InternalArtifactProver::new( + Arc::new( MockFileDownloaderBuilder::default() .with_compression(None) .with_failure() .with_times(2) .build(), - )) - .build_cardano_database_client(); + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); - client + artifact_prover .download_unpack_digest_file( &[ DigestLocation::CloudStorage { @@ -366,19 +386,21 @@ mod tests { #[tokio::test] async fn succeeds_if_at_least_one_location_is_retrieved() { let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new({ + let artifact_prover = InternalArtifactProver::new( + Arc::new( MockFileDownloaderBuilder::default() .with_compression(None) .with_failure() .next_call() .with_compression(None) .with_success() - .build() - })) - .build_cardano_database_client(); + .build(), + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); - client + artifact_prover .download_unpack_digest_file( &[ DigestLocation::CloudStorage { @@ -397,17 +419,19 @@ mod tests { #[tokio::test] async fn succeeds_when_first_location_is_retrieved() { let target_dir = Path::new("."); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new( + let artifact_prover = InternalArtifactProver::new( + Arc::new( MockFileDownloaderBuilder::default() .with_compression(None) .with_times(1) .with_success() .build(), - )) - .build_cardano_database_client(); + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); - client + artifact_prover .download_unpack_digest_file( &[ DigestLocation::CloudStorage { @@ -427,17 +451,18 @@ mod tests { async fn sends_feedbacks() { let target_dir = Path::new("."); let feedback_receiver = Arc::new(StackFeedbackReceiver::new()); - let client = CardanoDatabaseClientDependencyInjector::new() - .with_http_file_downloader(Arc::new( + let artifact_prover = InternalArtifactProver::new( + Arc::new( MockFileDownloaderBuilder::default() .with_compression(None) .with_success() .build(), - )) - .with_feedback_receivers(&[feedback_receiver.clone()]) - .build_cardano_database_client(); + ), + FeedbackSender::new(&[feedback_receiver.clone()]), + test_utils::test_logger(), + ); - client + artifact_prover .download_unpack_digest_file( &[DigestLocation::CloudStorage { uri: "http://whatever-1/digests.json".to_string(), @@ -488,10 +513,17 @@ mod tests { "read_digest_file_fails_when_no_digest_file", ) .build(); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - client + let artifact_prover = InternalArtifactProver::new( + Arc::new( + MockFileDownloaderBuilder::default() + .with_times(0) + .with_success() + .build(), + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); + artifact_prover .read_digest_file(&target_dir) .expect_err("read_digest_file should fail"); } @@ -505,10 +537,17 @@ mod tests { .build(); create_valid_fake_digest_file(&target_dir.join("digests.json"), &[]); create_valid_fake_digest_file(&target_dir.join("digests-2.json"), &[]); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - client + let artifact_prover = InternalArtifactProver::new( + Arc::new( + MockFileDownloaderBuilder::default() + .with_times(0) + .with_success() + .build(), + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); + artifact_prover .read_digest_file(&target_dir) .expect_err("read_digest_file should fail"); } @@ -521,10 +560,17 @@ mod tests { ) .build(); create_invalid_fake_digest_file(&target_dir.join("digests.json")); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); - - client + let artifact_prover = InternalArtifactProver::new( + Arc::new( + MockFileDownloaderBuilder::default() + .with_times(0) + .with_success() + .build(), + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); + artifact_prover .read_digest_file(&target_dir) .expect_err("read_digest_file should fail"); } @@ -547,10 +593,18 @@ mod tests { }, ]; create_valid_fake_digest_file(&target_dir.join("digests.json"), &digest_messages); - let client = - CardanoDatabaseClientDependencyInjector::new().build_cardano_database_client(); + let artifact_prover = InternalArtifactProver::new( + Arc::new( + MockFileDownloaderBuilder::default() + .with_times(0) + .with_success() + .build(), + ), + FeedbackSender::new(&[]), + test_utils::test_logger(), + ); - let digests = client.read_digest_file(&target_dir).unwrap(); + let digests = artifact_prover.read_digest_file(&target_dir).unwrap(); assert_eq!( BTreeMap::from([ ("00001.chunk".to_string(), "digest-1".to_string()), diff --git a/mithril-client/src/client.rs b/mithril-client/src/client.rs index 2c4804f282b..04dbc0ed012 100644 --- a/mithril-client/src/client.rs +++ b/mithril-client/src/client.rs @@ -18,7 +18,7 @@ use crate::certificate_client::{ CertificateClient, CertificateVerifier, MithrilCertificateVerifier, }; use crate::feedback::{FeedbackReceiver, FeedbackSender}; -#[cfg(all(feature = "fs", feature = "unstable"))] +#[cfg(feature = "fs")] use crate::file_downloader::{ FileDownloadRetryPolicy, FileDownloader, HttpFileDownloader, RetryDownloader, }; @@ -136,6 +136,7 @@ pub struct ClientBuilder { genesis_verification_key: String, aggregator_client: Option>, certificate_verifier: Option>, + #[cfg(feature = "fs")] http_file_downloader: Option>, #[cfg(feature = "unstable")] certificate_verifier_cache: Option>, @@ -153,9 +154,10 @@ impl ClientBuilder { genesis_verification_key: genesis_verification_key.to_string(), aggregator_client: None, certificate_verifier: None, + #[cfg(feature = "fs")] + http_file_downloader: None, #[cfg(feature = "unstable")] certificate_verifier_cache: None, - http_file_downloader: None, logger: None, feedback_receivers: vec![], options: ClientOptions::default(), @@ -172,6 +174,7 @@ impl ClientBuilder { genesis_verification_key: genesis_verification_key.to_string(), aggregator_client: None, certificate_verifier: None, + #[cfg(feature = "fs")] http_file_downloader: None, #[cfg(feature = "unstable")] certificate_verifier_cache: None, From b15d7a9549faf8662439bd4a7b9d4e404215184f Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Wed, 19 Feb 2025 11:10:03 +0100 Subject: [PATCH 57/59] fix: show stack trace in errors in Cardano database client --- .../src/cardano_database_client/download_unpack.rs | 6 +++--- mithril-client/src/cardano_database_client/proving.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mithril-client/src/cardano_database_client/download_unpack.rs b/mithril-client/src/cardano_database_client/download_unpack.rs index ae003a315b5..b53f5966eb7 100644 --- a/mithril-client/src/cardano_database_client/download_unpack.rs +++ b/mithril-client/src/cardano_database_client/download_unpack.rs @@ -275,7 +275,7 @@ impl InternalArtifactDownloader { Err(e) => { slog::error!( self.logger, - "Failed downloading and unpacking immutable files"; "error" => e.to_string(), "target_dir" => immutable_files_target_dir.display() + "Failed downloading and unpacking immutable files"; "error" => ?e, "target_dir" => immutable_files_target_dir.display() ); } } @@ -334,7 +334,7 @@ impl InternalArtifactDownloader { Err(e) => { slog::error!( logger_clone, - "Failed downloading and unpacking immutable file {immutable_file_number} for location {file_downloader_uri:?}"; "error" => e.to_string() + "Failed downloading and unpacking immutable file {immutable_file_number} for location {file_downloader_uri:?}"; "error" => ?e ); Err(e.context(format!("Failed downloading and unpacking immutable file {immutable_file_number} for location {file_downloader_uri:?}"))) } @@ -440,7 +440,7 @@ impl InternalArtifactDownloader { Err(e) => { slog::error!( self.logger, - "Failed downloading and unpacking ancillaries for location {file_downloader_uri:?}"; "error" => e.to_string() + "Failed downloading and unpacking ancillaries for location {file_downloader_uri:?}"; "error" => ?e ); } } diff --git a/mithril-client/src/cardano_database_client/proving.rs b/mithril-client/src/cardano_database_client/proving.rs index 7bb4d5cefa6..9e8c177d547 100644 --- a/mithril-client/src/cardano_database_client/proving.rs +++ b/mithril-client/src/cardano_database_client/proving.rs @@ -130,7 +130,7 @@ impl InternalArtifactProver { Err(e) => { slog::error!( self.logger, - "Failed downloading and unpacking digest for location {file_downloader_uri:?}"; "error" => e.to_string() + "Failed downloading and unpacking digest for location {file_downloader_uri:?}"; "error" => ?e ); } } From ba8626abbeeae340dca27d91c2161f7acfba4c44 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 18:12:52 +0100 Subject: [PATCH 58/59] docs: update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7ef9644c13..e6f0efde759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ As a minor extension, we have adopted a slightly different versioning convention - End support for **macOS x64 pre-built binaries** for the client CLI. +- **UNSTABLE** Cardano database incremental certification: + + - Implement the client library for the the signed entity type `CardanoDatabase` (download and prove snapshot). + - Crates versions: | Crate | Version | From c318e5bbbe05c63fa6a7608b324e9af92b71f5aa Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 18 Feb 2025 18:13:05 +0100 Subject: [PATCH 59/59] chore: upgrade crate versions * client-snapshot from `0.1.24` to `0.1.25` * mithril-client-cli from `0.11.0` to `0.11.1` * mithril-client from `0.11.2` to `0.11.3` * mithril-common from `0.5.3` to `0.5.4` --- Cargo.lock | 8 ++++---- examples/client-snapshot/Cargo.toml | 2 +- mithril-client-cli/Cargo.toml | 2 +- mithril-client/Cargo.toml | 2 +- mithril-common/Cargo.toml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 429121f556f..89778c45788 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1033,7 +1033,7 @@ dependencies = [ [[package]] name = "client-snapshot" -version = "0.1.24" +version = "0.1.25" dependencies = [ "anyhow", "async-trait", @@ -3701,7 +3701,7 @@ dependencies = [ [[package]] name = "mithril-client" -version = "0.11.2" +version = "0.11.3" dependencies = [ "anyhow", "async-recursion", @@ -3732,7 +3732,7 @@ dependencies = [ [[package]] name = "mithril-client-cli" -version = "0.11.0" +version = "0.11.1" dependencies = [ "anyhow", "async-trait", @@ -3777,7 +3777,7 @@ dependencies = [ [[package]] name = "mithril-common" -version = "0.5.3" +version = "0.5.4" dependencies = [ "anyhow", "async-trait", diff --git a/examples/client-snapshot/Cargo.toml b/examples/client-snapshot/Cargo.toml index bcfa11d17fc..a9a41b781ee 100644 --- a/examples/client-snapshot/Cargo.toml +++ b/examples/client-snapshot/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "client-snapshot" description = "Mithril client snapshot example" -version = "0.1.24" +version = "0.1.25" authors = ["dev@iohk.io", "mithril-dev@iohk.io"] documentation = "https://mithril.network/doc" edition = "2021" diff --git a/mithril-client-cli/Cargo.toml b/mithril-client-cli/Cargo.toml index 700368d6aef..2a630ed9c81 100644 --- a/mithril-client-cli/Cargo.toml +++ b/mithril-client-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-client-cli" -version = "0.11.0" +version = "0.11.1" description = "A Mithril Client" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-client/Cargo.toml b/mithril-client/Cargo.toml index f675d12cc2b..e9142b92aca 100644 --- a/mithril-client/Cargo.toml +++ b/mithril-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-client" -version = "0.11.2" +version = "0.11.3" description = "Mithril client library" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-common/Cargo.toml b/mithril-common/Cargo.toml index 23974b5adac..ea880cad46b 100644 --- a/mithril-common/Cargo.toml +++ b/mithril-common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-common" -version = "0.5.3" +version = "0.5.4" description = "Common types, interfaces, and utilities for Mithril nodes." authors = { workspace = true } edition = { workspace = true }