Skip to content

Commit

Permalink
feat(cli): return the files upload summary after a successful files u…
Browse files Browse the repository at this point in the history
…pload
  • Loading branch information
RolandSherwin authored and bochaco committed Apr 9, 2024
1 parent 5233739 commit 798f28e
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 19 deletions.
16 changes: 12 additions & 4 deletions sn_cli/src/files/chunk_manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ pub struct ChunkManager {
artifacts_dir: PathBuf,
files_to_chunk: Vec<(OsString, PathXorName, PathBuf)>,
chunks: BTreeMap<PathXorName, ChunkedFile>,
completed_files: Vec<(OsString, ChunkAddress)>,
completed_files: Vec<(PathBuf, OsString, ChunkAddress)>,
resumed_chunk_count: usize,
resumed_files_count: usize,
}
Expand Down Expand Up @@ -185,6 +185,7 @@ impl ChunkManager {
let completed_files = self.chunks.iter().filter_map(|(_, chunked_file)| {
if chunked_file.chunks.is_empty() {
Some((
chunked_file.file_path.clone(),
chunked_file.file_name.clone(),
chunked_file.head_chunk_address,
))
Expand Down Expand Up @@ -391,6 +392,7 @@ impl ChunkManager {
trace!("removed {path_xor:?} from chunks list");

self.completed_files.push((
chunked_file.file_path.clone(),
chunked_file.file_name.clone(),
chunked_file.head_chunk_address,
));
Expand Down Expand Up @@ -430,15 +432,21 @@ impl ChunkManager {

/// Return the filename and the file's Xor address if all their chunks has been marked as
/// completed
pub(crate) fn completed_files(&self) -> &Vec<(OsString, ChunkAddress)> {
pub(crate) fn completed_files(&self) -> &Vec<(PathBuf, OsString, ChunkAddress)> {
&self.completed_files
}

/// Return the list of Filenames that have some chunks that are yet to be marked as completed.
pub(crate) fn incomplete_files(&self) -> Vec<&OsString> {
pub(crate) fn incomplete_files(&self) -> Vec<(&PathBuf, &OsString, &ChunkAddress)> {
self.chunks
.values()
.map(|chunked_file| &chunked_file.file_name)
.map(|chunked_file| {
(
&chunked_file.file_path,
&chunked_file.file_name,
&chunked_file.head_chunk_address,
)
})
.collect()
}

Expand Down
51 changes: 36 additions & 15 deletions sn_cli/src/files/files_uploader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
// KIND, either express or implied. Please review the Licences for the specific language governing
// permissions and limitations relating to use of the SAFE Network Software.

use super::get_progress_bar;
use crate::ChunkManager;
use bytes::Bytes;
use color_eyre::{eyre::eyre, Report, Result};
Expand All @@ -16,8 +17,9 @@ use sn_client::{
transfers::{TransferError, WalletError},
Client, Error as ClientError, UploadCfg, UploadEvent, UploadSummary, Uploader,
};
use sn_protocol::storage::Chunk;
use sn_protocol::storage::{Chunk, ChunkAddress};
use std::{
ffi::OsString,
path::{Path, PathBuf},
time::{Duration, Instant},
};
Expand All @@ -26,7 +28,15 @@ use tracing::{debug, error, info, warn};
use walkdir::{DirEntry, WalkDir};
use xor_name::XorName;

use super::get_progress_bar;
/// The result of a successful files upload.
pub struct FilesUploadSummary {
/// The cost and count summary of the upload.
pub upload_summary: UploadSummary,
/// The list of completed files (FilePath, FileName, HeadChunkAddress)
pub completed_files: Vec<(PathBuf, OsString, ChunkAddress)>,
/// The list of incomplete files (FilePath, FileName, HeadChunkAddress)
pub incomplete_files: Vec<(PathBuf, OsString, ChunkAddress)>,
}

/// Combines the `Uploader` along with the `ChunkManager`
pub struct FilesUploader {
Expand Down Expand Up @@ -79,7 +89,7 @@ impl FilesUploader {
self
}

pub async fn start_upload(self) -> Result<UploadSummary> {
pub async fn start_upload(self) -> Result<FilesUploadSummary> {
let mut chunk_manager = ChunkManager::new(&self.root_dir);
let chunks_to_upload = self.get_chunks_to_upload(&mut chunk_manager).await?;
let chunks_to_upload_len = chunks_to_upload.len();
Expand Down Expand Up @@ -119,7 +129,7 @@ impl FilesUploader {
);
}

let summary = match uploader.start_upload().await {
let upload_sum = match uploader.start_upload().await {
Ok(summary) => summary,
Err(ClientError::Wallet(WalletError::Transfer(TransferError::NotEnoughBalance(
available,
Expand All @@ -132,32 +142,43 @@ impl FilesUploader {
}
Err(err) => return Err(eyre!("Failed to upload chunk batch: {err}")),
};
events_handle.await??;
let chunk_manager = events_handle.await??;

let elapsed = Self::msg_format_elapsed_time(now.elapsed());
println!(
"Among {chunks_to_upload_len} chunks, found {} already existed in network, uploaded \
the leftover {} chunks in {elapsed}",
summary.skipped_count, summary.uploaded_count,
upload_sum.skipped_count, upload_sum.uploaded_count,
);
info!(
"Among {chunks_to_upload_len} chunks, found {} already existed in network, uploaded \
the leftover {} chunks in {elapsed}",
summary.skipped_count, summary.uploaded_count,
upload_sum.skipped_count, upload_sum.uploaded_count,
);
println!("**************************************");
println!("* Payment Details *");
println!("**************************************");
println!(
"Made payment of {:?} for {} chunks",
summary.storage_cost, summary.uploaded_count
upload_sum.storage_cost, upload_sum.uploaded_count
);
println!(
"Made payment of {:?} for royalties fees",
summary.royalty_fees
upload_sum.royalty_fees
);
println!("New wallet balance: {}", summary.final_balance);

println!("New wallet balance: {}", upload_sum.final_balance);

let summary = FilesUploadSummary {
upload_summary: upload_sum,
completed_files: chunk_manager.completed_files().clone(),
incomplete_files: chunk_manager
.incomplete_files()
.into_iter()
.map(|(path, file_name, head_address)| {
(path.clone(), file_name.clone(), *head_address)
})
.collect(),
};
Ok(summary)
}

Expand Down Expand Up @@ -246,7 +267,7 @@ impl FilesUploader {
make_data_public: bool,
chunks_to_upload_len: usize,
mut upload_event_rx: Receiver<UploadEvent>,
) -> Result<JoinHandle<Result<()>>> {
) -> Result<JoinHandle<Result<ChunkManager>>> {
let progress_bar = get_progress_bar(chunks_to_upload_len as u64)?;
let handle = tokio::spawn(async move {
let mut upload_terminated_with_error = false;
Expand Down Expand Up @@ -279,7 +300,7 @@ impl FilesUploader {
if upload_terminated_with_error {
error!("Got UploadEvent::Error inside upload event loop");
} else {
for file_name in chunk_manager.incomplete_files() {
for (_, file_name, _) in chunk_manager.incomplete_files() {
if let Some(file_name) = file_name.to_str() {
println!("Unverified file \"{file_name}\", suggest to re-upload again.");
info!("Unverified {file_name}");
Expand All @@ -294,7 +315,7 @@ impl FilesUploader {
Self::print_completed_file_list(&chunk_manager);
}

Ok::<_, Report>(())
Ok::<_, Report>(chunk_manager)
});

Ok(handle)
Expand Down Expand Up @@ -323,7 +344,7 @@ impl FilesUploader {
}

fn print_completed_file_list(chunk_manager: &ChunkManager) {
for (file_name, addr) in chunk_manager.completed_files() {
for (_, file_name, addr) in chunk_manager.completed_files() {
let hex_addr = addr.to_hex();
if let Some(file_name) = file_name.to_str() {
println!("\"{file_name}\" {hex_addr}");
Expand Down

0 comments on commit 798f28e

Please sign in to comment.