diff --git a/crates/core/src/commands/check.rs b/crates/core/src/commands/check.rs index f55c91f9..38fd5bf3 100644 --- a/crates/core/src/commands/check.rs +++ b/crates/core/src/commands/check.rs @@ -136,106 +136,110 @@ pub struct CheckOptions { pub read_data_subset: ReadSubsetOption, } -impl CheckOptions { - /// Runs the `check` command - /// - /// # Type Parameters - /// - /// * `P` - The progress bar type. - /// * `S` - The state the repository is in. - /// - /// # Arguments - /// - /// * `repo` - The repository to check - /// - /// # Errors - /// - /// If the repository is corrupted - pub(crate) fn run( - self, - repo: &Repository, - trees: Vec, - ) -> RusticResult<()> { - let be = repo.dbe(); - let cache = repo.cache(); - let hot_be = &repo.be_hot; - let raw_be = repo.dbe(); - let pb = &repo.pb; - if !self.trust_cache { - if let Some(cache) = &cache { - for file_type in [FileType::Snapshot, FileType::Index] { - // list files in order to clean up the cache - // - // This lists files here and later when reading index / checking snapshots - // TODO: Only list the files once... - _ = be - .list_with_size(file_type) - .map_err(RusticErrorKind::Backend)?; - - let p = pb.progress_bytes(format!("checking {file_type:?} in cache...")); - // TODO: Make concurrency (20) customizable - check_cache_files(20, cache, raw_be, file_type, &p)?; - } +/// Runs the `check` command +/// +/// # Type Parameters +/// +/// * `P` - The progress bar type. +/// * `S` - The state the repository is in. +/// +/// # Arguments +/// +/// * `repo` - The repository to check +/// * `opts` - The check options to use +/// * `trees` - The trees to check +/// +/// # Errors +/// +/// If the repository is corrupted +/// +/// # Panics +/// +// TODO: Add panics +pub(crate) fn check_repository( + repo: &Repository, + opts: CheckOptions, + trees: Vec, +) -> RusticResult<()> { + let be = repo.dbe(); + let cache = repo.cache(); + let hot_be = &repo.be_hot; + let raw_be = repo.dbe(); + let pb = &repo.pb; + if !opts.trust_cache { + if let Some(cache) = &cache { + for file_type in [FileType::Snapshot, FileType::Index] { + // list files in order to clean up the cache + // + // This lists files here and later when reading index / checking snapshots + // TODO: Only list the files once... + _ = be + .list_with_size(file_type) + .map_err(RusticErrorKind::Backend)?; + + let p = pb.progress_bytes(format!("checking {file_type:?} in cache...")); + // TODO: Make concurrency (20) customizable + check_cache_files(20, cache, raw_be, file_type, &p)?; } } + } - if let Some(hot_be) = hot_be { - for file_type in [FileType::Snapshot, FileType::Index] { - check_hot_files(raw_be, hot_be, file_type, pb)?; - } + if let Some(hot_be) = hot_be { + for file_type in [FileType::Snapshot, FileType::Index] { + check_hot_files(raw_be, hot_be, file_type, pb)?; } + } - let index_collector = check_packs(be, hot_be, pb)?; + let index_collector = check_packs(be, hot_be, pb)?; - if let Some(cache) = &cache { - let p = pb.progress_spinner("cleaning up packs from cache..."); - let ids: Vec<_> = index_collector - .tree_packs() - .iter() - .map(|(id, size)| (**id, *size)) - .collect(); - if let Err(err) = cache.remove_not_in_list(FileType::Pack, &ids) { - warn!("Error in cache backend removing pack files: {err}"); - } - p.finish(); + if let Some(cache) = &cache { + let p = pb.progress_spinner("cleaning up packs from cache..."); + let ids: Vec<_> = index_collector + .tree_packs() + .iter() + .map(|(id, size)| (**id, *size)) + .collect(); + if let Err(err) = cache.remove_not_in_list(FileType::Pack, &ids) { + warn!("Error in cache backend removing pack files: {err}"); + } + p.finish(); - if !self.trust_cache { - let p = pb.progress_bytes("checking packs in cache..."); - // TODO: Make concurrency (5) customizable - check_cache_files(5, cache, raw_be, FileType::Pack, &p)?; - } + if !opts.trust_cache { + let p = pb.progress_bytes("checking packs in cache..."); + // TODO: Make concurrency (5) customizable + check_cache_files(5, cache, raw_be, FileType::Pack, &p)?; } + } - let index_be = GlobalIndex::new_from_index(index_collector.into_index()); + let index_be = GlobalIndex::new_from_index(index_collector.into_index()); - let packs = check_trees(be, &index_be, trees, pb)?; + let packs = check_trees(be, &index_be, trees, pb)?; - if self.read_data { - let packs = index_be - .into_index() - .into_iter() - .filter(|p| packs.contains(&p.id)); + if opts.read_data { + let packs = index_be + .into_index() + .into_iter() + .filter(|p| packs.contains(&p.id)); - let packs = self.read_data_subset.apply(packs); + let packs = opts.read_data_subset.apply(packs); - repo.warm_up_wait(packs.iter().map(|pack| pack.id))?; + repo.warm_up_wait(packs.iter().map(|pack| pack.id))?; - let total_pack_size = packs.iter().map(|pack| u64::from(pack.pack_size())).sum(); - let p = pb.progress_bytes("reading pack data..."); - p.set_length(total_pack_size); + let total_pack_size = packs.iter().map(|pack| u64::from(pack.pack_size())).sum(); + let p = pb.progress_bytes("reading pack data..."); + p.set_length(total_pack_size); - packs.into_par_iter().for_each(|pack| { - let id = pack.id; - let data = be.read_full(FileType::Pack, &id).unwrap(); - match check_pack(be, pack, data, &p) { - Ok(()) => {} - Err(err) => error!("Error reading pack {id} : {err}",), - } - }); - p.finish(); - } - Ok(()) + packs.into_par_iter().for_each(|pack| { + let id = pack.id; + let data = be.read_full(FileType::Pack, &id).unwrap(); + match check_pack(be, pack, data, &p) { + Ok(()) => {} + Err(err) => error!("Error reading pack {id} : {err}",), + } + }); + p.finish(); } + Ok(()) } /// Checks if all files in the backend are also in the hot backend diff --git a/crates/core/src/commands/init.rs b/crates/core/src/commands/init.rs index 4401c187..f932658c 100644 --- a/crates/core/src/commands/init.rs +++ b/crates/core/src/commands/init.rs @@ -7,7 +7,7 @@ use crate::{ chunker::random_poly, commands::{ config::{save_config, ConfigOptions}, - key::KeyOptions, + key::{init_key, KeyOptions}, }, crypto::aespoly1305::Key, error::{RusticErrorKind, RusticResult}, @@ -86,7 +86,7 @@ pub(crate) fn init_with_config( config: &ConfigFile, ) -> RusticResult { repo.be.create().map_err(RusticErrorKind::Backend)?; - let (key, id) = key_opts.init_key(repo, pass)?; + let (key, id) = init_key(repo, key_opts, pass)?; info!("key {id} successfully added."); save_config(repo, config.clone(), key)?; diff --git a/crates/core/src/commands/key.rs b/crates/core/src/commands/key.rs index d6d052bd..af277665 100644 --- a/crates/core/src/commands/key.rs +++ b/crates/core/src/commands/key.rs @@ -28,88 +28,94 @@ pub struct KeyOptions { pub with_created: bool, } -impl KeyOptions { - /// Add the current key to the repository. - /// - /// # Type Parameters - /// - /// * `P` - The progress bar type. - /// * `S` - The state the repository is in. - /// - /// # Arguments - /// - /// * `repo` - The repository to add the key to. - /// * `pass` - The password to encrypt the key with. - /// - /// # Errors - /// - /// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized. - /// - /// # Returns - /// - /// The id of the key. - /// - /// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError - pub(crate) fn add_key( - &self, - repo: &Repository, - pass: &str, - ) -> RusticResult { - let key = repo.dbe().key(); - self.add(repo, pass, *key) - } +/// Add the current key to the repository. +/// +/// # Type Parameters +/// +/// * `P` - The progress bar type +/// * `S` - The state the repository is in +/// +/// # Arguments +/// +/// * `repo` - The repository to add the key to +/// * `opts` - The key options to use +/// * `pass` - The password to encrypt the key with +/// +/// # Errors +/// +/// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized +/// +/// # Returns +/// +/// The id of the key. +/// +/// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError +pub(crate) fn add_current_key_to_repo( + repo: &Repository, + opts: &KeyOptions, + pass: &str, +) -> RusticResult { + let key = repo.dbe().key(); + add_key_to_repo(repo, opts, pass, *key) +} - /// Initialize a new key. - /// - /// # Type Parameters - /// - /// * `P` - The progress bar type. - /// * `S` - The state the repository is in. - /// - /// # Arguments - /// - /// * `repo` - The repository to add the key to. - /// * `pass` - The password to encrypt the key with. - /// - /// # Returns - /// - /// A tuple of the key and the id of the key. - pub(crate) fn init_key( - &self, - repo: &Repository, - pass: &str, - ) -> RusticResult<(Key, KeyId)> { - // generate key - let key = Key::new(); - Ok((key, self.add(repo, pass, key)?)) - } +/// Initialize a new key. +/// +/// # Type Parameters +/// +/// * `P` - The progress bar type +/// * `S` - The state the repository is in +/// +/// # Arguments +/// +/// * `repo` - The repository to add the key to +/// * `opts` - The key options to use +/// * `pass` - The password to encrypt the key with +/// +/// # Returns +/// +/// A tuple of the key and the id of the key. +pub(crate) fn init_key( + repo: &Repository, + opts: &KeyOptions, + pass: &str, +) -> RusticResult<(Key, KeyId)> { + // generate key + let key = Key::new(); + Ok((key, add_key_to_repo(repo, opts, pass, key)?)) +} - /// Add a key to the repository. - /// - /// # Arguments - /// - /// * `repo` - The repository to add the key to. - /// * `pass` - The password to encrypt the key with. - /// * `key` - The key to add. - /// - /// # Errors - /// - /// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized. - /// - /// # Returns - /// - /// The id of the key. - /// - /// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError - fn add(&self, repo: &Repository, pass: &str, key: Key) -> RusticResult { - let ko = self.clone(); - let keyfile = KeyFile::generate(key, &pass, ko.hostname, ko.username, ko.with_created)?; +/// Add a key to the repository. +/// +/// # Arguments +/// +/// * `repo` - The repository to add the key to +/// * `opts` - The key options to use +/// * `pass` - The password to encrypt the key with +/// * `key` - The key to add +/// +/// # Errors +/// +/// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized. +/// +/// # Returns +/// +/// The id of the key. +/// +/// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError +pub(crate) fn add_key_to_repo( + repo: &Repository, + opts: &KeyOptions, + pass: &str, + key: Key, +) -> RusticResult { + let ko = opts.clone(); + let keyfile = KeyFile::generate(key, &pass, ko.hostname, ko.username, ko.with_created)?; - let data = serde_json::to_vec(&keyfile).map_err(CommandErrorKind::FromJsonError)?; - let id = KeyId::from(hash(&data)); - repo.be - .write_bytes(FileType::Key, &id, false, data.into()) - .map_err(RusticErrorKind::Backend)?; - Ok(id) - } + let data = serde_json::to_vec(&keyfile).map_err(CommandErrorKind::FromJsonError)?; + let id = KeyId::from(hash(&data)); + repo.be + .write_bytes(FileType::Key, &id, false, data.into()) + .map_err(RusticErrorKind::Backend)?; + Ok(id) } diff --git a/crates/core/src/commands/prune.rs b/crates/core/src/commands/prune.rs index 9418ef45..d2082f9c 100644 --- a/crates/core/src/commands/prune.rs +++ b/crates/core/src/commands/prune.rs @@ -173,77 +173,15 @@ impl PruneOptions { /// /// [`CommandErrorKind::RepackUncompressedRepoV1`]: crate::error::CommandErrorKind::RepackUncompressedRepoV1 /// [`CommandErrorKind::FromOutOfRangeError`]: crate::error::CommandErrorKind::FromOutOfRangeError + #[deprecated( + since = "0.5.2", + note = "Use `PrunePlan::from_prune_options()` instead" + )] pub fn get_plan( &self, repo: &Repository, ) -> RusticResult { - let pb = &repo.pb; - let be = repo.dbe(); - - if repo.config().version < 2 && self.repack_uncompressed { - return Err(CommandErrorKind::RepackUncompressedRepoV1.into()); - } - - let mut index_files = Vec::new(); - - let p = pb.progress_counter("reading index..."); - let mut index_collector = IndexCollector::new(IndexType::OnlyTrees); - - for index in be.stream_all::(&p)? { - let (id, index) = index?; - index_collector.extend(index.packs.clone()); - // we add the trees from packs_to_delete to the index such that searching for - // used blobs doesn't abort if they are already marked for deletion - index_collector.extend(index.packs_to_delete.clone()); - - index_files.push((id, index)); - } - p.finish(); - - let (used_ids, total_size) = { - let index = GlobalIndex::new_from_index(index_collector.into_index()); - let total_size = BlobTypeMap::init(|blob_type| index.total_size(blob_type)); - let used_ids = find_used_blobs(be, &index, &self.ignore_snaps, pb)?; - (used_ids, total_size) - }; - - // list existing pack files - let p = pb.progress_spinner("getting packs from repository..."); - let existing_packs: BTreeMap<_, _> = be - .list_with_size(FileType::Pack) - .map_err(RusticErrorKind::Backend)? - .into_iter() - .map(|(id, size)| (PackId::from(id), size)) - .collect(); - p.finish(); - - let mut pruner = PrunePlan::new(used_ids, existing_packs, index_files); - pruner.count_used_blobs(); - pruner.check()?; - let repack_cacheable_only = self - .repack_cacheable_only - .unwrap_or_else(|| repo.config().is_hot == Some(true)); - let pack_sizer = - total_size.map(|tpe, size| PackSizer::from_config(repo.config(), tpe, size)); - pruner.decide_packs( - Duration::from_std(*self.keep_pack).map_err(CommandErrorKind::FromOutOfRangeError)?, - Duration::from_std(*self.keep_delete).map_err(CommandErrorKind::FromOutOfRangeError)?, - repack_cacheable_only, - self.repack_uncompressed, - self.repack_all, - &pack_sizer, - )?; - pruner.decide_repack( - &self.max_repack, - &self.max_unused, - self.repack_uncompressed || self.repack_all, - self.no_resize, - &pack_sizer, - ); - pruner.check_existing_packs()?; - pruner.filter_index_files(self.instant_delete); - - Ok(pruner) + PrunePlan::from_prune_options(repo, self) } } @@ -724,6 +662,98 @@ impl PrunePlan { } } + /// Get a `PrunePlan` from the given `PruneOptions`. + /// + /// # Type Parameters + /// + /// * `P` - The progress bar type + /// * `S` - The state the repository is in + /// + /// # Arguments + /// + /// * `repo` - The repository to get the `PrunePlan` for + /// * `opts` - The `PruneOptions` to use + /// + /// # Errors + /// + /// * [`CommandErrorKind::RepackUncompressedRepoV1`] - If `repack_uncompressed` is set and the repository is a version 1 repository + /// * [`CommandErrorKind::FromOutOfRangeError`] - If `keep_pack` or `keep_delete` is out of range + /// + /// [`CommandErrorKind::RepackUncompressedRepoV1`]: crate::error::CommandErrorKind::RepackUncompressedRepoV1 + /// [`CommandErrorKind::FromOutOfRangeError`]: crate::error::CommandErrorKind::FromOutOfRangeError + pub fn from_prune_options( + repo: &Repository, + opts: &PruneOptions, + ) -> RusticResult { + let pb = &repo.pb; + let be = repo.dbe(); + + if repo.config().version < 2 && opts.repack_uncompressed { + return Err(CommandErrorKind::RepackUncompressedRepoV1.into()); + } + + let mut index_files = Vec::new(); + + let p = pb.progress_counter("reading index..."); + let mut index_collector = IndexCollector::new(IndexType::OnlyTrees); + + for index in be.stream_all::(&p)? { + let (id, index) = index?; + index_collector.extend(index.packs.clone()); + // we add the trees from packs_to_delete to the index such that searching for + // used blobs doesn't abort if they are already marked for deletion + index_collector.extend(index.packs_to_delete.clone()); + + index_files.push((id, index)); + } + p.finish(); + + let (used_ids, total_size) = { + let index = GlobalIndex::new_from_index(index_collector.into_index()); + let total_size = BlobTypeMap::init(|blob_type| index.total_size(blob_type)); + let used_ids = find_used_blobs(be, &index, &opts.ignore_snaps, pb)?; + (used_ids, total_size) + }; + + // list existing pack files + let p = pb.progress_spinner("getting packs from repository..."); + let existing_packs: BTreeMap<_, _> = be + .list_with_size(FileType::Pack) + .map_err(RusticErrorKind::Backend)? + .into_iter() + .map(|(id, size)| (PackId::from(id), size)) + .collect(); + p.finish(); + + let mut pruner = Self::new(used_ids, existing_packs, index_files); + pruner.count_used_blobs(); + pruner.check()?; + let repack_cacheable_only = opts + .repack_cacheable_only + .unwrap_or_else(|| repo.config().is_hot == Some(true)); + let pack_sizer = + total_size.map(|tpe, size| PackSizer::from_config(repo.config(), tpe, size)); + pruner.decide_packs( + Duration::from_std(*opts.keep_pack).map_err(CommandErrorKind::FromOutOfRangeError)?, + Duration::from_std(*opts.keep_delete).map_err(CommandErrorKind::FromOutOfRangeError)?, + repack_cacheable_only, + opts.repack_uncompressed, + opts.repack_all, + &pack_sizer, + )?; + pruner.decide_repack( + &opts.max_repack, + &opts.max_unused, + opts.repack_uncompressed || opts.repack_all, + opts.no_resize, + &pack_sizer, + ); + pruner.check_existing_packs()?; + pruner.filter_index_files(opts.instant_delete); + + Ok(pruner) + } + /// This function counts the number of times a blob is used in the index files. fn count_used_blobs(&mut self) { for blob in self @@ -1120,212 +1150,245 @@ impl PrunePlan { /// TODO! In weird circumstances, should be fixed. #[allow(clippy::significant_drop_tightening)] #[allow(clippy::too_many_lines)] + #[deprecated(since = "0.5.2", note = "Use `Repository::prune()` instead.")] pub fn do_prune( self, repo: &Repository, opts: &PruneOptions, ) -> RusticResult<()> { - if repo.config().append_only == Some(true) { - return Err(CommandErrorKind::NotAllowedWithAppendOnly("prune".to_string()).into()); - } - repo.warm_up_wait(self.repack_packs().into_iter())?; - let be = repo.dbe(); - let pb = &repo.pb; - - let indexer = Indexer::new_unindexed(be.clone()).into_shared(); - - // Calculate an approximation of sizes after pruning. - // The size actually is: - // total_size_of_all_blobs + total_size_of_pack_headers + #packs * pack_overhead - // This is hard/impossible to compute because: - // - the size of blobs can change during repacking if compression is changed - // - the size of pack headers depends on whether blobs are compressed or not - // - we don't know the number of packs generated by repacking - // So, we simply use the current size of the blobs and an estimation of the pack - // header size. - - let size_after_prune = BlobTypeMap::init(|blob_type| { - self.stats.size[blob_type].total_after_prune() - + self.stats.blobs[blob_type].total_after_prune() - * u64::from(HeaderEntry::ENTRY_LEN_COMPRESSED) - }); - - let tree_repacker = Repacker::new( - be.clone(), - BlobType::Tree, - indexer.clone(), - repo.config(), - size_after_prune[BlobType::Tree], - )?; - - let data_repacker = Repacker::new( - be.clone(), - BlobType::Data, - indexer.clone(), - repo.config(), - size_after_prune[BlobType::Data], - )?; + repo.prune(opts, self) + } +} - // mark unreferenced packs for deletion - if !self.existing_packs.is_empty() { - if opts.instant_delete { - let p = pb.progress_counter("removing unindexed packs..."); - let existing_packs: Vec<_> = self.existing_packs.into_keys().collect(); - be.delete_list(true, existing_packs.iter(), p)?; - } else { - let p = - pb.progress_counter("marking unneeded unindexed pack files for deletion..."); - p.set_length(self.existing_packs.len().try_into().unwrap()); - for (id, size) in self.existing_packs { - let pack = IndexPack { - id, - size: Some(size), - time: Some(Local::now()), - blobs: Vec::new(), - }; - indexer.write().unwrap().add_remove(pack)?; - p.inc(1); - } - p.finish(); +/// Perform the pruning on the given repository. +/// +/// # Arguments +/// +/// * `repo` - The repository to prune +/// * `opts` - The options for the pruning +/// * `prune_plan` - The plan for the pruning +/// +/// # Errors +/// +/// * [`CommandErrorKind::NotAllowedWithAppendOnly`] - If the repository is in append-only mode +/// * [`CommandErrorKind::NoDecision`] - If a pack has no decision +/// +/// # Returns +/// +/// * `Ok(())` - If the pruning was successful +/// +/// # Panics +/// +/// TODO! In weird circumstances, should be fixed. +#[allow(clippy::significant_drop_tightening)] +#[allow(clippy::too_many_lines)] +pub(crate) fn prune_repository( + repo: &Repository, + opts: &PruneOptions, + prune_plan: PrunePlan, +) -> RusticResult<()> { + if repo.config().append_only == Some(true) { + return Err(CommandErrorKind::NotAllowedWithAppendOnly("prune".to_string()).into()); + } + repo.warm_up_wait(prune_plan.repack_packs().into_iter())?; + let be = repo.dbe(); + let pb = &repo.pb; + + let indexer = Indexer::new_unindexed(be.clone()).into_shared(); + + // Calculate an approximation of sizes after pruning. + // The size actually is: + // total_size_of_all_blobs + total_size_of_pack_headers + #packs * pack_overhead + // This is hard/impossible to compute because: + // - the size of blobs can change during repacking if compression is changed + // - the size of pack headers depends on whether blobs are compressed or not + // - we don't know the number of packs generated by repacking + // So, we simply use the current size of the blobs and an estimation of the pack + // header size. + + let size_after_prune = BlobTypeMap::init(|blob_type| { + prune_plan.stats.size[blob_type].total_after_prune() + + prune_plan.stats.blobs[blob_type].total_after_prune() + * u64::from(HeaderEntry::ENTRY_LEN_COMPRESSED) + }); + + let tree_repacker = Repacker::new( + be.clone(), + BlobType::Tree, + indexer.clone(), + repo.config(), + size_after_prune[BlobType::Tree], + )?; + + let data_repacker = Repacker::new( + be.clone(), + BlobType::Data, + indexer.clone(), + repo.config(), + size_after_prune[BlobType::Data], + )?; + + // mark unreferenced packs for deletion + if !prune_plan.existing_packs.is_empty() { + if opts.instant_delete { + let p = pb.progress_counter("removing unindexed packs..."); + let existing_packs: Vec<_> = prune_plan.existing_packs.into_keys().collect(); + be.delete_list(true, existing_packs.iter(), p)?; + } else { + let p = pb.progress_counter("marking unneeded unindexed pack files for deletion..."); + p.set_length(prune_plan.existing_packs.len().try_into().unwrap()); + for (id, size) in prune_plan.existing_packs { + let pack = IndexPack { + id, + size: Some(size), + time: Some(Local::now()), + blobs: Vec::new(), + }; + indexer.write().unwrap().add_remove(pack)?; + p.inc(1); } + p.finish(); } + } - // process packs by index_file - let p = match (self.index_files.is_empty(), self.stats.packs.repack > 0) { - (true, _) => { - info!("nothing to do!"); - pb.progress_hidden() - } - // TODO: Use a MultiProgressBar here - (false, true) => pb.progress_bytes("repacking // rebuilding index..."), - (false, false) => pb.progress_spinner("rebuilding index..."), - }; - - p.set_length(self.stats.size_sum().repack - self.stats.size_sum().repackrm); - - let mut indexes_remove = Vec::new(); - let tree_packs_remove = Arc::new(Mutex::new(Vec::new())); - let data_packs_remove = Arc::new(Mutex::new(Vec::new())); - - let delete_pack = |pack: PrunePack| { - // delete pack - match pack.blob_type { - BlobType::Data => data_packs_remove.lock().unwrap().push(pack.id), - BlobType::Tree => tree_packs_remove.lock().unwrap().push(pack.id), - } - }; + // process packs by index_file + let p = match ( + prune_plan.index_files.is_empty(), + prune_plan.stats.packs.repack > 0, + ) { + (true, _) => { + info!("nothing to do!"); + pb.progress_hidden() + } + // TODO: Use a MultiProgressBar here + (false, true) => pb.progress_bytes("repacking // rebuilding index..."), + (false, false) => pb.progress_spinner("rebuilding index..."), + }; + + p.set_length(prune_plan.stats.size_sum().repack - prune_plan.stats.size_sum().repackrm); + + let mut indexes_remove = Vec::new(); + let tree_packs_remove = Arc::new(Mutex::new(Vec::new())); + let data_packs_remove = Arc::new(Mutex::new(Vec::new())); + + let delete_pack = |pack: PrunePack| { + // delete pack + match pack.blob_type { + BlobType::Data => data_packs_remove.lock().unwrap().push(pack.id), + BlobType::Tree => tree_packs_remove.lock().unwrap().push(pack.id), + } + }; - let used_ids = Arc::new(Mutex::new(self.used_ids)); + let used_ids = Arc::new(Mutex::new(prune_plan.used_ids)); - let packs: Vec<_> = self - .index_files - .into_iter() - .inspect(|index| { - indexes_remove.push(index.id); - }) - .flat_map(|index| index.packs) - .collect(); - - // remove old index files early if requested - if !indexes_remove.is_empty() && opts.early_delete_index { - let p = pb.progress_counter("removing old index files..."); - be.delete_list(true, indexes_remove.iter(), p)?; - } + let packs: Vec<_> = prune_plan + .index_files + .into_iter() + .inspect(|index| { + indexes_remove.push(index.id); + }) + .flat_map(|index| index.packs) + .collect(); - // write new pack files and index files - packs - .into_par_iter() - .try_for_each(|pack| -> RusticResult<_> { - match pack.to_do { - PackToDo::Undecided => return Err(CommandErrorKind::NoDecision(pack.id).into()), - PackToDo::Keep => { - // keep pack: add to new index - let pack = pack.into_index_pack(); - indexer.write().unwrap().add(pack)?; - } - PackToDo::Repack => { - // TODO: repack in parallel - for blob in &pack.blobs { - if used_ids.lock().unwrap().remove(&blob.id).is_none() { - // don't save duplicate blobs - continue; - } + // remove old index files early if requested + if !indexes_remove.is_empty() && opts.early_delete_index { + let p = pb.progress_counter("removing old index files..."); + be.delete_list(true, indexes_remove.iter(), p)?; + } - let repacker = match blob.tpe { - BlobType::Data => &data_repacker, - BlobType::Tree => &tree_repacker, - }; - if opts.fast_repack { - repacker.add_fast(&pack.id, blob)?; - } else { - repacker.add(&pack.id, blob)?; - } - p.inc(u64::from(blob.length)); + // write new pack files and index files + packs + .into_par_iter() + .try_for_each(|pack| -> RusticResult<_> { + match pack.to_do { + PackToDo::Undecided => return Err(CommandErrorKind::NoDecision(pack.id).into()), + PackToDo::Keep => { + // keep pack: add to new index + let pack = pack.into_index_pack(); + indexer.write().unwrap().add(pack)?; + } + PackToDo::Repack => { + // TODO: repack in parallel + for blob in &pack.blobs { + if used_ids.lock().unwrap().remove(&blob.id).is_none() { + // don't save duplicate blobs + continue; } - if opts.instant_delete { - delete_pack(pack); + + let repacker = match blob.tpe { + BlobType::Data => &data_repacker, + BlobType::Tree => &tree_repacker, + }; + if opts.fast_repack { + repacker.add_fast(&pack.id, blob)?; } else { - // mark pack for removal - let pack = pack.into_index_pack_with_time(self.time); - indexer.write().unwrap().add_remove(pack)?; + repacker.add(&pack.id, blob)?; } + p.inc(u64::from(blob.length)); } - PackToDo::MarkDelete => { - if opts.instant_delete { - delete_pack(pack); - } else { - // mark pack for removal - let pack = pack.into_index_pack_with_time(self.time); - indexer.write().unwrap().add_remove(pack)?; - } + if opts.instant_delete { + delete_pack(pack); + } else { + // mark pack for removal + let pack = pack.into_index_pack_with_time(prune_plan.time); + indexer.write().unwrap().add_remove(pack)?; } - PackToDo::KeepMarked | PackToDo::KeepMarkedAndCorrect => { - if opts.instant_delete { - delete_pack(pack); - } else { - // keep pack: add to new index; keep the timestamp. - // Note the timestap shouldn't be None here, however if it is not not set, use the current time to heal the entry! - let time = pack.time.unwrap_or(self.time); - let pack = pack.into_index_pack_with_time(time); - indexer.write().unwrap().add_remove(pack)?; - } + } + PackToDo::MarkDelete => { + if opts.instant_delete { + delete_pack(pack); + } else { + // mark pack for removal + let pack = pack.into_index_pack_with_time(prune_plan.time); + indexer.write().unwrap().add_remove(pack)?; } - PackToDo::Recover => { - // recover pack: add to new index in section packs - let pack = pack.into_index_pack_with_time(self.time); - indexer.write().unwrap().add(pack)?; + } + PackToDo::KeepMarked | PackToDo::KeepMarkedAndCorrect => { + if opts.instant_delete { + delete_pack(pack); + } else { + // keep pack: add to new index; keep the timestamp. + // Note the timestamp shouldn't be None here, however if it is not not set, use the current time to heal the entry! + let time = pack.time.unwrap_or(prune_plan.time); + let pack = pack.into_index_pack_with_time(time); + indexer.write().unwrap().add_remove(pack)?; } - PackToDo::Delete => delete_pack(pack), } - Ok(()) - })?; - _ = tree_repacker.finalize()?; - _ = data_repacker.finalize()?; - indexer.write().unwrap().finalize()?; - p.finish(); - - // remove old index files first as they may reference pack files which are removed soon. - if !indexes_remove.is_empty() && !opts.early_delete_index { - let p = pb.progress_counter("removing old index files..."); - be.delete_list(true, indexes_remove.iter(), p)?; - } + PackToDo::Recover => { + // recover pack: add to new index in section packs + let pack = pack.into_index_pack_with_time(prune_plan.time); + indexer.write().unwrap().add(pack)?; + } + PackToDo::Delete => delete_pack(pack), + } + Ok(()) + })?; + _ = tree_repacker.finalize()?; + _ = data_repacker.finalize()?; + indexer.write().unwrap().finalize()?; + p.finish(); - // get variable out of Arc> - let data_packs_remove = data_packs_remove.lock().unwrap(); - if !data_packs_remove.is_empty() { - let p = pb.progress_counter("removing old data packs..."); - be.delete_list(false, data_packs_remove.iter(), p)?; - } + // remove old index files first as they may reference pack files which are removed soon. + if !indexes_remove.is_empty() && !opts.early_delete_index { + let p = pb.progress_counter("removing old index files..."); + be.delete_list(true, indexes_remove.iter(), p)?; + } - // get variable out of Arc> - let tree_packs_remove = tree_packs_remove.lock().unwrap(); - if !tree_packs_remove.is_empty() { - let p = pb.progress_counter("removing old tree packs..."); - be.delete_list(true, tree_packs_remove.iter(), p)?; - } + // get variable out of Arc> + let data_packs_remove = data_packs_remove.lock().unwrap(); + if !data_packs_remove.is_empty() { + let p = pb.progress_counter("removing old data packs..."); + be.delete_list(false, data_packs_remove.iter(), p)?; + } - Ok(()) + // get variable out of Arc> + let tree_packs_remove = tree_packs_remove.lock().unwrap(); + if !tree_packs_remove.is_empty() { + let p = pb.progress_counter("removing old tree packs..."); + be.delete_list(true, tree_packs_remove.iter(), p)?; } + + Ok(()) } /// `PackInfo` contains information about a pack which is needed to decide what to do with the pack. diff --git a/crates/core/src/commands/repair/index.rs b/crates/core/src/commands/repair/index.rs index b2bf9b62..f2e8dc56 100644 --- a/crates/core/src/commands/repair/index.rs +++ b/crates/core/src/commands/repair/index.rs @@ -27,86 +27,82 @@ pub struct RepairIndexOptions { pub read_all: bool, } -impl RepairIndexOptions { - /// Runs the `repair index` command - /// - /// # Type Parameters - /// - /// * `P` - The progress bar type - /// * `S` - The state the repository is in - /// - /// # Arguments - /// - /// * `repo` - The repository to repair - /// * `dry_run` - Whether to actually modify the repository or just print what would be done - pub(crate) fn repair( - self, - repo: &Repository, - dry_run: bool, - ) -> RusticResult<()> { - if repo.config().append_only == Some(true) { - return Err( - CommandErrorKind::NotAllowedWithAppendOnly("index repair".to_string()).into(), - ); - } +/// Runs the `repair index` command +/// +/// # Type Parameters +/// +/// * `P` - The progress bar type +/// * `S` - The state the repository is in +/// +/// # Arguments +/// +/// * `repo` - The repository to repair +/// * `dry_run` - Whether to actually modify the repository or just print what would be done +pub(crate) fn repair_index( + repo: &Repository, + opts: RepairIndexOptions, + dry_run: bool, +) -> RusticResult<()> { + if repo.config().append_only == Some(true) { + return Err(CommandErrorKind::NotAllowedWithAppendOnly("index repair".to_string()).into()); + } - let be = repo.dbe(); - let mut checker = PackChecker::new(repo)?; - - let p = repo.pb.progress_counter("reading index..."); - for index in be.stream_all::(&p)? { - let (index_id, index) = index?; - let (new_index, changed) = checker.check_pack(index, self.read_all); - match (changed, dry_run) { - (true, true) => info!("would have modified index file {index_id}"), - (true, false) => { - if !new_index.packs.is_empty() || !new_index.packs_to_delete.is_empty() { - _ = be.save_file(&new_index)?; - } - be.remove(FileType::Index, &index_id, true) - .map_err(RusticErrorKind::Backend)?; + let be = repo.dbe(); + let mut checker = PackChecker::new(repo)?; + + let p = repo.pb.progress_counter("reading index..."); + for index in be.stream_all::(&p)? { + let (index_id, index) = index?; + let (new_index, changed) = checker.check_pack(index, opts.read_all); + match (changed, dry_run) { + (true, true) => info!("would have modified index file {index_id}"), + (true, false) => { + if !new_index.packs.is_empty() || !new_index.packs_to_delete.is_empty() { + _ = be.save_file(&new_index)?; } - (false, _) => {} // nothing to do + be.remove(FileType::Index, &index_id, true) + .map_err(RusticErrorKind::Backend)?; } + (false, _) => {} // nothing to do } - p.finish(); + } + p.finish(); - let pack_read_header = checker.into_pack_to_read(); - repo.warm_up_wait(pack_read_header.iter().map(|(id, _, _)| *id))?; - - let indexer = Indexer::new(be.clone()).into_shared(); - let p = repo.pb.progress_counter("reading pack headers"); - p.set_length( - pack_read_header - .len() - .try_into() - .map_err(CommandErrorKind::ConversionFromIntFailed)?, - ); - for (id, size_hint, packsize) in pack_read_header { - debug!("reading pack {id}..."); - match PackHeader::from_file(be, id, size_hint, packsize) { - Err(err) => { - warn!("error reading pack {id} (-> removing from index): {err}"); - } - Ok(header) => { - let pack = IndexPack { - blobs: header.into_blobs(), - id, - ..Default::default() - }; - if !dry_run { - // write pack file to index - without the delete mark - indexer.write().unwrap().add_with(pack, false)?; - } + let pack_read_header = checker.into_pack_to_read(); + repo.warm_up_wait(pack_read_header.iter().map(|(id, _, _)| *id))?; + + let indexer = Indexer::new(be.clone()).into_shared(); + let p = repo.pb.progress_counter("reading pack headers"); + p.set_length( + pack_read_header + .len() + .try_into() + .map_err(CommandErrorKind::ConversionFromIntFailed)?, + ); + for (id, size_hint, packsize) in pack_read_header { + debug!("reading pack {id}..."); + match PackHeader::from_file(be, id, size_hint, packsize) { + Err(err) => { + warn!("error reading pack {id} (-> removing from index): {err}"); + } + Ok(header) => { + let pack = IndexPack { + blobs: header.into_blobs(), + id, + ..Default::default() + }; + if !dry_run { + // write pack file to index - without the delete mark + indexer.write().unwrap().add_with(pack, false)?; } } - p.inc(1); } - indexer.write().unwrap().finalize()?; - p.finish(); - - Ok(()) + p.inc(1); } + indexer.write().unwrap().finalize()?; + p.finish(); + + Ok(()) } struct PackChecker { diff --git a/crates/core/src/commands/repair/snapshots.rs b/crates/core/src/commands/repair/snapshots.rs index 8b0c0200..ebf6509c 100644 --- a/crates/core/src/commands/repair/snapshots.rs +++ b/crates/core/src/commands/repair/snapshots.rs @@ -62,231 +62,232 @@ impl Default for RepairSnapshotsOptions { // TODO: add documentation #[derive(Clone, Copy)] -enum Changed { +pub(crate) enum Changed { This, SubTree, None, } #[derive(Default)] -struct RepairState { +pub(crate) struct RepairState { replaced: BTreeMap, seen: BTreeSet, delete: Vec, } -impl RepairSnapshotsOptions { - /// Runs the `repair snapshots` command - /// - /// # Type Parameters - /// - /// * `P` - The progress bar type - /// * `S` - The type of the indexed tree. - /// - /// # Arguments - /// - /// * `repo` - The repository to repair - /// * `snapshots` - The snapshots to repair - /// * `dry_run` - Whether to actually modify the repository or just print what would be done - pub(crate) fn repair( - &self, - repo: &Repository, - snapshots: Vec, - dry_run: bool, - ) -> RusticResult<()> { - let be = repo.dbe(); - let config_file = repo.config(); +/// Runs the `repair snapshots` command +/// +/// # Type Parameters +/// +/// * `P` - The progress bar type +/// * `S` - The type of the indexed tree. +/// +/// # Arguments +/// +/// * `repo` - The repository to repair +/// * `opts` - The repair options to use +/// * `snapshots` - The snapshots to repair +/// * `dry_run` - Whether to actually modify the repository or just print what would be done +pub(crate) fn repair_snapshots( + repo: &Repository, + opts: &RepairSnapshotsOptions, + snapshots: Vec, + dry_run: bool, +) -> RusticResult<()> { + let be = repo.dbe(); + let config_file = repo.config(); - if self.delete && config_file.append_only == Some(true) { - return Err( - CommandErrorKind::NotAllowedWithAppendOnly("snapshot removal".to_string()).into(), - ); - } + if opts.delete && config_file.append_only == Some(true) { + return Err( + CommandErrorKind::NotAllowedWithAppendOnly("snapshot removal".to_string()).into(), + ); + } - let mut state = RepairState::default(); + let mut state = RepairState::default(); - let indexer = Indexer::new(be.clone()).into_shared(); - let mut packer = Packer::new( - be.clone(), - BlobType::Tree, - indexer.clone(), - config_file, - repo.index().total_size(BlobType::Tree), - )?; + let indexer = Indexer::new(be.clone()).into_shared(); + let mut packer = Packer::new( + be.clone(), + BlobType::Tree, + indexer.clone(), + config_file, + repo.index().total_size(BlobType::Tree), + )?; - for mut snap in snapshots { - let snap_id = snap.id; - info!("processing snapshot {snap_id}"); - match self.repair_tree( - repo.dbe(), - repo.index(), - &mut packer, - Some(snap.tree), - &mut state, - dry_run, - )? { - (Changed::None, _) => { - info!("snapshot {snap_id} is ok."); - } - (Changed::This, _) => { - warn!("snapshot {snap_id}: root tree is damaged -> marking for deletion!"); - state.delete.push(snap_id); + for mut snap in snapshots { + let snap_id = snap.id; + info!("processing snapshot {snap_id}"); + match repair_tree( + repo.dbe(), + opts, + repo.index(), + &mut packer, + Some(snap.tree), + &mut state, + dry_run, + )? { + (Changed::None, _) => { + info!("snapshot {snap_id} is ok."); + } + (Changed::This, _) => { + warn!("snapshot {snap_id}: root tree is damaged -> marking for deletion!"); + state.delete.push(snap_id); + } + (Changed::SubTree, id) => { + // change snapshot tree + if snap.original.is_none() { + snap.original = Some(snap.id); } - (Changed::SubTree, id) => { - // change snapshot tree - if snap.original.is_none() { - snap.original = Some(snap.id); - } - _ = snap.set_tags(self.tag.clone()); - snap.tree = id; - if dry_run { - info!("would have modified snapshot {snap_id}."); - } else { - let new_id = be.save_file(&snap)?; - info!("saved modified snapshot as {new_id}."); - } - state.delete.push(snap_id); + _ = snap.set_tags(opts.tag.clone()); + snap.tree = id; + if dry_run { + info!("would have modified snapshot {snap_id}."); + } else { + let new_id = be.save_file(&snap)?; + info!("saved modified snapshot as {new_id}."); } + state.delete.push(snap_id); } } + } - if !dry_run { - _ = packer.finalize()?; - indexer.write().unwrap().finalize()?; - } + if !dry_run { + _ = packer.finalize()?; + indexer.write().unwrap().finalize()?; + } - if self.delete { - if dry_run { - info!("would have removed {} snapshots.", state.delete.len()); - } else { - be.delete_list( - true, - state.delete.iter(), - repo.pb.progress_counter("remove defect snapshots"), - )?; - } + if opts.delete { + if dry_run { + info!("would have removed {} snapshots.", state.delete.len()); + } else { + be.delete_list( + true, + state.delete.iter(), + repo.pb.progress_counter("remove defect snapshots"), + )?; } - - Ok(()) } - /// Repairs a tree - /// - /// # Type Parameters - /// - /// * `BE` - The type of the backend. - /// - /// # Arguments - /// - /// * `be` - The backend to use - /// * `packer` - The packer to use - /// * `id` - The id of the tree to repair - /// * `replaced` - A map of already replaced trees - /// * `seen` - A set of already seen trees - /// * `dry_run` - Whether to actually modify the repository or just print what would be done - /// - /// # Returns - /// - /// A tuple containing the change status and the id of the repaired tree - fn repair_tree( - &self, - be: &impl DecryptFullBackend, - index: &impl ReadGlobalIndex, - packer: &mut Packer, - id: Option, - state: &mut RepairState, - dry_run: bool, - ) -> RusticResult<(Changed, TreeId)> { - let (tree, changed) = match id { - None => (Tree::new(), Changed::This), - Some(id) => { - if state.seen.contains(&id) { - return Ok((Changed::None, id)); - } - if let Some(r) = state.replaced.get(&id) { - return Ok(*r); - } + Ok(()) +} - let (tree, mut changed) = Tree::from_backend(be, index, id).map_or_else( - |_err| { - warn!("tree {id} could not be loaded."); - (Tree::new(), Changed::This) - }, - |tree| (tree, Changed::None), - ); +/// Repairs a tree +/// +/// # Type Parameters +/// +/// * `BE` - The type of the backend. +/// +/// # Arguments +/// +/// * `be` - The backend to use +/// * `opts` - The repair options to use +/// * `packer` - The packer to use +/// * `id` - The id of the tree to repair +/// * `replaced` - A map of already replaced trees +/// * `seen` - A set of already seen trees +/// * `dry_run` - Whether to actually modify the repository or just print what would be done +/// +/// # Returns +/// +/// A tuple containing the change status and the id of the repaired tree +pub(crate) fn repair_tree( + be: &impl DecryptFullBackend, + opts: &RepairSnapshotsOptions, + index: &impl ReadGlobalIndex, + packer: &mut Packer, + id: Option, + state: &mut RepairState, + dry_run: bool, +) -> RusticResult<(Changed, TreeId)> { + let (tree, changed) = match id { + None => (Tree::new(), Changed::This), + Some(id) => { + if state.seen.contains(&id) { + return Ok((Changed::None, id)); + } + if let Some(r) = state.replaced.get(&id) { + return Ok(*r); + } - let mut new_tree = Tree::new(); + let (tree, mut changed) = Tree::from_backend(be, index, id).map_or_else( + |_err| { + warn!("tree {id} could not be loaded."); + (Tree::new(), Changed::This) + }, + |tree| (tree, Changed::None), + ); - for mut node in tree { - match node.node_type { - NodeType::File {} => { - let mut file_changed = false; - let mut new_content = Vec::new(); - let mut new_size = 0; - for blob in node.content.take().unwrap() { - index.get_data(&blob).map_or_else( - || { - file_changed = true; - }, - |ie| { - new_content.push(blob); - new_size += u64::from(ie.data_length()); - }, - ); - } - if file_changed { - warn!("file {}: contents are missing", node.name); - node.name += &self.suffix; - changed = Changed::SubTree; - } else if new_size != node.meta.size { - info!("file {}: corrected file size", node.name); + let mut new_tree = Tree::new(); + + for mut node in tree { + match node.node_type { + NodeType::File {} => { + let mut file_changed = false; + let mut new_content = Vec::new(); + let mut new_size = 0; + for blob in node.content.take().unwrap() { + index.get_data(&blob).map_or_else( + || { + file_changed = true; + }, + |ie| { + new_content.push(blob); + new_size += u64::from(ie.data_length()); + }, + ); + } + if file_changed { + warn!("file {}: contents are missing", node.name); + node.name += &opts.suffix; + changed = Changed::SubTree; + } else if new_size != node.meta.size { + info!("file {}: corrected file size", node.name); + changed = Changed::SubTree; + } + node.content = Some(new_content); + node.meta.size = new_size; + } + NodeType::Dir {} => { + let (c, tree_id) = + repair_tree(be, opts, index, packer, node.subtree, state, dry_run)?; + match c { + Changed::None => {} + Changed::This => { + warn!("dir {}: tree is missing", node.name); + node.subtree = Some(tree_id); + node.name += &opts.suffix; changed = Changed::SubTree; } - node.content = Some(new_content); - node.meta.size = new_size; - } - NodeType::Dir {} => { - let (c, tree_id) = - self.repair_tree(be, index, packer, node.subtree, state, dry_run)?; - match c { - Changed::None => {} - Changed::This => { - warn!("dir {}: tree is missing", node.name); - node.subtree = Some(tree_id); - node.name += &self.suffix; - changed = Changed::SubTree; - } - Changed::SubTree => { - node.subtree = Some(tree_id); - changed = Changed::SubTree; - } + Changed::SubTree => { + node.subtree = Some(tree_id); + changed = Changed::SubTree; } } - _ => {} // Other types: no check needed } - new_tree.add(node); - } - if matches!(changed, Changed::None) { - _ = state.seen.insert(id); + _ => {} // Other types: no check needed } - (new_tree, changed) + new_tree.add(node); } - }; + if matches!(changed, Changed::None) { + _ = state.seen.insert(id); + } + (new_tree, changed) + } + }; - match (id, changed) { - (None, Changed::None) => panic!("this should not happen!"), - (Some(id), Changed::None) => Ok((Changed::None, id)), - (_, c) => { - // the tree has been changed => save it - let (chunk, new_id) = tree.serialize()?; - if !index.has_tree(&new_id) && !dry_run { - packer.add(chunk.into(), BlobId::from(*new_id))?; - } - if let Some(id) = id { - _ = state.replaced.insert(id, (c, new_id)); - } - Ok((c, new_id)) + match (id, changed) { + (None, Changed::None) => panic!("this should not happen!"), + (Some(id), Changed::None) => Ok((Changed::None, id)), + (_, c) => { + // the tree has been changed => save it + let (chunk, new_id) = tree.serialize()?; + if !index.has_tree(&new_id) && !dry_run { + packer.add(chunk.into(), BlobId::from(*new_id))?; + } + if let Some(id) = id { + _ = state.replaced.insert(id, (c, new_id)); } + Ok((c, new_id)) } } } diff --git a/crates/core/src/commands/restore.rs b/crates/core/src/commands/restore.rs index af44dfc1..033f9785 100644 --- a/crates/core/src/commands/restore.rs +++ b/crates/core/src/commands/restore.rs @@ -94,312 +94,318 @@ pub struct RestoreStats { pub dirs: FileDirStats, } -impl RestoreOptions { - /// Restore the repository to the given destination. - /// - /// # Type Parameters - /// - /// * `P` - The progress bar type. - /// * `S` - The type of the indexed tree. - /// - /// # Arguments - /// - /// * `file_infos` - The restore information. - /// * `repo` - The repository to restore. - /// * `node_streamer` - The node streamer to use. - /// * `dest` - The destination to restore to. - /// - /// # Errors - /// - /// If the restore failed. - pub(crate) fn restore( - self, - file_infos: RestorePlan, - repo: &Repository, - node_streamer: impl Iterator>, - dest: &LocalDestination, - ) -> RusticResult<()> { - repo.warm_up_wait(file_infos.to_packs().into_iter())?; - restore_contents(repo, dest, file_infos)?; +/// Restore the repository to the given destination. +/// +/// # Type Parameters +/// +/// * `P` - The progress bar type +/// * `S` - The type of the indexed tree +/// +/// # Arguments +/// +/// * `file_infos` - The restore information +/// * `repo` - The repository to restore +/// * `opts` - The restore options +/// * `node_streamer` - The node streamer to use +/// * `dest` - The destination to restore to +/// +/// # Errors +/// +/// If the restore failed. +pub(crate) fn restore_repository( + file_infos: RestorePlan, + repo: &Repository, + opts: RestoreOptions, + node_streamer: impl Iterator>, + dest: &LocalDestination, +) -> RusticResult<()> { + repo.warm_up_wait(file_infos.to_packs().into_iter())?; + restore_contents(repo, dest, file_infos)?; - let p = repo.pb.progress_spinner("setting metadata..."); - self.restore_metadata(node_streamer, dest)?; - p.finish(); + let p = repo.pb.progress_spinner("setting metadata..."); + restore_metadata(node_streamer, opts, dest)?; + p.finish(); - Ok(()) - } + Ok(()) +} - /// Collect restore information, scan existing files, create needed dirs and remove superfluous files - /// - /// # Type Parameters - /// - /// * `P` - The progress bar type. - /// * `S` - The type of the indexed tree. - /// - /// # Arguments - /// - /// * `repo` - The repository to restore. - /// * `node_streamer` - The node streamer to use. - /// * `dest` - The destination to restore to. - /// * `dry_run` - If true, don't actually restore anything, but only print out what would be done. - /// - /// # Errors - /// - /// * [`CommandErrorKind::ErrorCreating`] - If a directory could not be created. - /// * [`CommandErrorKind::ErrorCollecting`] - If the restore information could not be collected. - /// - /// [`CommandErrorKind::ErrorCreating`]: crate::error::CommandErrorKind::ErrorCreating - /// [`CommandErrorKind::ErrorCollecting`]: crate::error::CommandErrorKind::ErrorCollecting - #[allow(clippy::too_many_lines)] - pub(crate) fn collect_and_prepare( - self, - repo: &Repository, - mut node_streamer: impl Iterator>, - dest: &LocalDestination, - dry_run: bool, - ) -> RusticResult { - let p = repo.pb.progress_spinner("collecting file information..."); - let dest_path = dest.path(""); - - let mut stats = RestoreStats::default(); - let mut restore_infos = RestorePlan::default(); - let mut additional_existing = false; - let mut removed_dir = None; - - let mut process_existing = |entry: &DirEntry| -> RusticResult<_> { - if entry.depth() == 0 { - // don't process the root dir which should be existing - return Ok(()); - } +/// Collect restore information, scan existing files, create needed dirs and remove superfluous files +/// +/// # Type Parameters +/// +/// * `P` - The progress bar type. +/// * `S` - The type of the indexed tree. +/// +/// # Arguments +/// +/// * `repo` - The repository to restore. +/// * `node_streamer` - The node streamer to use. +/// * `dest` - The destination to restore to. +/// * `dry_run` - If true, don't actually restore anything, but only print out what would be done. +/// +/// # Errors +/// +/// * [`CommandErrorKind::ErrorCreating`] - If a directory could not be created. +/// * [`CommandErrorKind::ErrorCollecting`] - If the restore information could not be collected. +/// +/// [`CommandErrorKind::ErrorCreating`]: crate::error::CommandErrorKind::ErrorCreating +/// [`CommandErrorKind::ErrorCollecting`]: crate::error::CommandErrorKind::ErrorCollecting +#[allow(clippy::too_many_lines)] +pub(crate) fn collect_and_prepare( + repo: &Repository, + opts: RestoreOptions, + mut node_streamer: impl Iterator>, + dest: &LocalDestination, + dry_run: bool, +) -> RusticResult { + let p = repo.pb.progress_spinner("collecting file information..."); + let dest_path = dest.path(""); + + let mut stats = RestoreStats::default(); + let mut restore_infos = RestorePlan::default(); + let mut additional_existing = false; + let mut removed_dir = None; + + let mut process_existing = |entry: &DirEntry| -> RusticResult<_> { + if entry.depth() == 0 { + // don't process the root dir which should be existing + return Ok(()); + } - debug!("additional {:?}", entry.path()); - if entry.file_type().unwrap().is_dir() { - stats.dirs.additional += 1; - } else { - stats.files.additional += 1; + debug!("additional {:?}", entry.path()); + if entry.file_type().unwrap().is_dir() { + stats.dirs.additional += 1; + } else { + stats.files.additional += 1; + } + match (opts.delete, dry_run, entry.file_type().unwrap().is_dir()) { + (true, true, true) => { + info!("would have removed the additional dir: {:?}", entry.path()); } - match (self.delete, dry_run, entry.file_type().unwrap().is_dir()) { - (true, true, true) => { - info!("would have removed the additional dir: {:?}", entry.path()); - } - (true, true, false) => { - info!("would have removed the additional file: {:?}", entry.path()); - } - (true, false, true) => { - let path = entry.path(); - match &removed_dir { - Some(dir) if path.starts_with(dir) => {} - _ => match dest.remove_dir(path) { - Ok(()) => { - removed_dir = Some(path.to_path_buf()); - } - Err(err) => { - error!("error removing {path:?}: {err}"); - } - }, - } - } - (true, false, false) => { - if let Err(err) = dest.remove_file(entry.path()) { - error!("error removing {:?}: {err}", entry.path()); - } + (true, true, false) => { + info!("would have removed the additional file: {:?}", entry.path()); + } + (true, false, true) => { + let path = entry.path(); + match &removed_dir { + Some(dir) if path.starts_with(dir) => {} + _ => match dest.remove_dir(path) { + Ok(()) => { + removed_dir = Some(path.to_path_buf()); + } + Err(err) => { + error!("error removing {path:?}: {err}"); + } + }, } - (false, _, _) => { - additional_existing = true; + } + (true, false, false) => { + if let Err(err) = dest.remove_file(entry.path()) { + error!("error removing {:?}: {err}", entry.path()); } } + (false, _, _) => { + additional_existing = true; + } + } - Ok(()) - }; - - let mut process_node = |path: &PathBuf, node: &Node, exists: bool| -> RusticResult<_> { - match node.node_type { - NodeType::Dir => { - if exists { - stats.dirs.modify += 1; - trace!("existing dir {path:?}"); - } else { - stats.dirs.restore += 1; - debug!("to restore: {path:?}"); - if !dry_run { - dest.create_dir(path).map_err(|err| { - CommandErrorKind::ErrorCreating(path.clone(), Box::new(err)) - })?; - } + Ok(()) + }; + + let mut process_node = |path: &PathBuf, node: &Node, exists: bool| -> RusticResult<_> { + match node.node_type { + NodeType::Dir => { + if exists { + stats.dirs.modify += 1; + trace!("existing dir {path:?}"); + } else { + stats.dirs.restore += 1; + debug!("to restore: {path:?}"); + if !dry_run { + dest.create_dir(path).map_err(|err| { + CommandErrorKind::ErrorCreating(path.clone(), Box::new(err)) + })?; } } - NodeType::File => { - // collect blobs needed for restoring - match ( - exists, - restore_infos - .add_file(dest, node, path.clone(), repo, self.verify_existing) - .map_err(|err| { - CommandErrorKind::ErrorCollecting(path.clone(), Box::new(err)) - })?, - ) { - // Note that exists = false and Existing or Verified can happen if the file is changed between scanning the dir - // and calling add_file. So we don't care about exists but trust add_file here. - (_, AddFileResult::Existing) => { - stats.files.unchanged += 1; - trace!("identical file: {path:?}"); - } - (_, AddFileResult::Verified) => { - stats.files.verified += 1; - trace!("verified identical file: {path:?}"); - } - // TODO: The differentiation between files to modify and files to create could be done only by add_file - // Currently, add_file never returns Modify, but always New, so we differentiate based on exists - (true, AddFileResult::Modify) => { - stats.files.modify += 1; - debug!("to modify: {path:?}"); - } - (false, AddFileResult::Modify) => { - stats.files.restore += 1; - debug!("to restore: {path:?}"); - } + } + NodeType::File => { + // collect blobs needed for restoring + match ( + exists, + restore_infos + .add_file(dest, node, path.clone(), repo, opts.verify_existing) + .map_err(|err| { + CommandErrorKind::ErrorCollecting(path.clone(), Box::new(err)) + })?, + ) { + // Note that exists = false and Existing or Verified can happen if the file is changed between scanning the dir + // and calling add_file. So we don't care about exists but trust add_file here. + (_, AddFileResult::Existing) => { + stats.files.unchanged += 1; + trace!("identical file: {path:?}"); + } + (_, AddFileResult::Verified) => { + stats.files.verified += 1; + trace!("verified identical file: {path:?}"); + } + // TODO: The differentiation between files to modify and files to create could be done only by add_file + // Currently, add_file never returns Modify, but always New, so we differentiate based on exists + (true, AddFileResult::Modify) => { + stats.files.modify += 1; + debug!("to modify: {path:?}"); + } + (false, AddFileResult::Modify) => { + stats.files.restore += 1; + debug!("to restore: {path:?}"); } } - _ => {} // nothing to do for symlink, device, etc. } - Ok(()) - }; - - let mut dst_iter = WalkBuilder::new(dest_path) - .follow_links(false) - .hidden(false) - .ignore(false) - .sort_by_file_path(Path::cmp) - .build() - .filter_map(Result::ok); // TODO: print out the ignored error - let mut next_dst = dst_iter.next(); - - let mut next_node = node_streamer.next().transpose()?; - - loop { - match (&next_dst, &next_node) { - (None, None) => break, - - (Some(destination), None) => { - process_existing(destination)?; - next_dst = dst_iter.next(); - } - (Some(destination), Some((path, node))) => { - match destination.path().cmp(&dest.path(path)) { - Ordering::Less => { + _ => {} // nothing to do for symlink, device, etc. + } + Ok(()) + }; + + let mut dst_iter = WalkBuilder::new(dest_path) + .follow_links(false) + .hidden(false) + .ignore(false) + .sort_by_file_path(Path::cmp) + .build() + .filter_map(Result::ok); // TODO: print out the ignored error + let mut next_dst = dst_iter.next(); + + let mut next_node = node_streamer.next().transpose()?; + + loop { + match (&next_dst, &next_node) { + (None, None) => break, + + (Some(destination), None) => { + process_existing(destination)?; + next_dst = dst_iter.next(); + } + (Some(destination), Some((path, node))) => { + match destination.path().cmp(&dest.path(path)) { + Ordering::Less => { + process_existing(destination)?; + next_dst = dst_iter.next(); + } + Ordering::Equal => { + // process existing node + if (node.is_dir() && !destination.file_type().unwrap().is_dir()) + || (node.is_file() && !destination.metadata().unwrap().is_file()) + || node.is_special() + { + // if types do not match, first remove the existing file process_existing(destination)?; - next_dst = dst_iter.next(); - } - Ordering::Equal => { - // process existing node - if (node.is_dir() && !destination.file_type().unwrap().is_dir()) - || (node.is_file() && !destination.metadata().unwrap().is_file()) - || node.is_special() - { - // if types do not match, first remove the existing file - process_existing(destination)?; - } - process_node(path, node, true)?; - next_dst = dst_iter.next(); - next_node = node_streamer.next().transpose()?; - } - Ordering::Greater => { - process_node(path, node, false)?; - next_node = node_streamer.next().transpose()?; } + process_node(path, node, true)?; + next_dst = dst_iter.next(); + next_node = node_streamer.next().transpose()?; + } + Ordering::Greater => { + process_node(path, node, false)?; + next_node = node_streamer.next().transpose()?; } } - (None, Some((path, node))) => { - process_node(path, node, false)?; - next_node = node_streamer.next().transpose()?; - } + } + (None, Some((path, node))) => { + process_node(path, node, false)?; + next_node = node_streamer.next().transpose()?; } } + } - if additional_existing { - warn!("Note: additional entries exist in destination"); - } + if additional_existing { + warn!("Note: additional entries exist in destination"); + } - restore_infos.stats = stats; - p.finish(); + restore_infos.stats = stats; + p.finish(); - Ok(restore_infos) - } + Ok(restore_infos) +} - /// Restore the metadata of the files and directories. - /// - /// # Arguments - /// - /// * `node_streamer` - The node streamer to use. - /// * `dest` - The destination to restore to. - /// - /// # Errors - /// - /// If the restore failed. - fn restore_metadata( - self, - mut node_streamer: impl Iterator>, - dest: &LocalDestination, - ) -> RusticResult<()> { - let mut dir_stack = Vec::new(); - while let Some((path, node)) = node_streamer.next().transpose()? { - match node.node_type { - NodeType::Dir => { - // set metadata for all non-parent paths in stack - while let Some((stackpath, _)) = dir_stack.last() { - if path.starts_with(stackpath) { - break; - } - let (path, node) = dir_stack.pop().unwrap(); - self.set_metadata(dest, &path, &node); +/// Restore the metadata of the files and directories. +/// +/// # Arguments +/// +/// * `node_streamer` - The node streamer to use +/// * `opts` - The restore options to use +/// * `dest` - The destination to restore to +/// +/// # Errors +/// +/// If the restore failed. +fn restore_metadata( + mut node_streamer: impl Iterator>, + opts: RestoreOptions, + dest: &LocalDestination, +) -> RusticResult<()> { + let mut dir_stack = Vec::new(); + while let Some((path, node)) = node_streamer.next().transpose()? { + match node.node_type { + NodeType::Dir => { + // set metadata for all non-parent paths in stack + while let Some((stackpath, _)) = dir_stack.last() { + if path.starts_with(stackpath) { + break; } - // push current path to the stack - dir_stack.push((path, node)); + let (path, node) = dir_stack.pop().unwrap(); + set_metadata(dest, opts, &path, &node); } - _ => self.set_metadata(dest, &path, &node), + // push current path to the stack + dir_stack.push((path, node)); } + _ => set_metadata(dest, opts, &path, &node), } + } - // empty dir stack and set metadata - for (path, node) in dir_stack.into_iter().rev() { - self.set_metadata(dest, &path, &node); - } - - Ok(()) + // empty dir stack and set metadata + for (path, node) in dir_stack.into_iter().rev() { + set_metadata(dest, opts, &path, &node); } - /// Set the metadata of the given file or directory. - /// - /// # Arguments - /// - /// * `dest` - The destination to restore to. - /// * `path` - The path of the file or directory. - /// * `node` - The node information of the file or directory. - /// - /// # Errors - /// - /// If the metadata could not be set. - // TODO: Return a result here, introduce errors and get rid of logging. - fn set_metadata(self, dest: &LocalDestination, path: &PathBuf, node: &Node) { - debug!("setting metadata for {:?}", path); - dest.create_special(path, node) - .unwrap_or_else(|_| warn!("restore {:?}: creating special file failed.", path)); - match (self.no_ownership, self.numeric_id) { - (true, _) => {} - (false, true) => dest - .set_uid_gid(path, &node.meta) - .unwrap_or_else(|_| warn!("restore {:?}: setting UID/GID failed.", path)), - (false, false) => dest - .set_user_group(path, &node.meta) - .unwrap_or_else(|_| warn!("restore {:?}: setting User/Group failed.", path)), - } - dest.set_permission(path, node) - .unwrap_or_else(|_| warn!("restore {:?}: chmod failed.", path)); - dest.set_extended_attributes(path, &node.meta.extended_attributes) - .unwrap_or_else(|_| warn!("restore {:?}: setting extended attributes failed.", path)); - dest.set_times(path, &node.meta) - .unwrap_or_else(|_| warn!("restore {:?}: setting file times failed.", path)); + Ok(()) +} + +/// Set the metadata of the given file or directory. +/// +/// # Arguments +/// +/// * `dest` - The destination to restore to +/// * `opts` - The restore options to use +/// * `path` - The path of the file or directory +/// * `node` - The node information of the file or directory +/// +/// # Errors +/// +/// If the metadata could not be set. +// TODO: Return a result here, introduce errors and get rid of logging. +pub(crate) fn set_metadata( + dest: &LocalDestination, + opts: RestoreOptions, + path: &PathBuf, + node: &Node, +) { + debug!("setting metadata for {:?}", path); + dest.create_special(path, node) + .unwrap_or_else(|_| warn!("restore {:?}: creating special file failed.", path)); + match (opts.no_ownership, opts.numeric_id) { + (true, _) => {} + (false, true) => dest + .set_uid_gid(path, &node.meta) + .unwrap_or_else(|_| warn!("restore {:?}: setting UID/GID failed.", path)), + (false, false) => dest + .set_user_group(path, &node.meta) + .unwrap_or_else(|_| warn!("restore {:?}: setting User/Group failed.", path)), } + dest.set_permission(path, node) + .unwrap_or_else(|_| warn!("restore {:?}: chmod failed.", path)); + dest.set_extended_attributes(path, &node.meta.extended_attributes) + .unwrap_or_else(|_| warn!("restore {:?}: setting extended attributes failed.", path)); + dest.set_times(path, &node.meta) + .unwrap_or_else(|_| warn!("restore {:?}: setting file times failed.", path)); } /// [`restore_contents`] restores all files contents as described by `file_infos` diff --git a/crates/core/src/repository.rs b/crates/core/src/repository.rs index 03a7d261..d2c72400 100644 --- a/crates/core/src/repository.rs +++ b/crates/core/src/repository.rs @@ -34,18 +34,18 @@ use crate::{ commands::{ self, backup::BackupOptions, - check::CheckOptions, + check::{check_repository, CheckOptions}, config::ConfigOptions, copy::CopySnapshot, forget::{ForgetGroups, KeepOptions}, - key::KeyOptions, - prune::{PruneOptions, PrunePlan}, + key::{add_current_key_to_repo, KeyOptions}, + prune::{prune_repository, PruneOptions, PrunePlan}, repair::{ - index::{index_checked_from_collector, RepairIndexOptions}, - snapshots::RepairSnapshotsOptions, + index::{index_checked_from_collector, repair_index, RepairIndexOptions}, + snapshots::{repair_snapshots, RepairSnapshotsOptions}, }, repoinfo::{IndexInfos, RepoFileInfos}, - restore::{RestoreOptions, RestorePlan}, + restore::{collect_and_prepare, restore_repository, RestoreOptions, RestorePlan}, }, crypto::aespoly1305::Key, error::{CommandErrorKind, KeyFileErrorKind, RepositoryErrorKind, RusticErrorKind}, @@ -840,7 +840,7 @@ impl Repository { /// /// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError pub fn add_key(&self, pass: &str, opts: &KeyOptions) -> RusticResult { - opts.add_key(self, pass) + add_current_key_to_repo(self, opts, pass) } /// Update the repository config by applying the given [`ConfigOptions`] @@ -1162,7 +1162,7 @@ impl Repository { .map(|snap| snap.tree) .collect(); - opts.run(self, trees) + check_repository(self, opts, trees) } /// Check the repository and given trees for errors or inconsistencies @@ -1175,7 +1175,7 @@ impl Repository { /// // TODO: Document errors pub fn check_with_trees(&self, opts: CheckOptions, trees: Vec) -> RusticResult<()> { - opts.run(self, trees) + check_repository(self, opts, trees) } /// Get the plan about what should be pruned and/or repacked. @@ -1192,7 +1192,29 @@ impl Repository { /// /// The plan about what should be pruned and/or repacked. pub fn prune_plan(&self, opts: &PruneOptions) -> RusticResult { - opts.get_plan(self) + PrunePlan::from_prune_options(self, opts) + } + + /// Perform the pruning on the repository. + /// + /// # Arguments + /// + /// * `opts` - The options for the pruning + /// * `prune_plan` - The plan about what should be pruned and/or repacked + /// + /// # Errors + /// + /// * [`CommandErrorKind::NotAllowedWithAppendOnly`] - If the repository is in append-only mode + /// * [`CommandErrorKind::NoDecision`] - If a pack has no decision + /// + /// # Returns + /// + /// * `Ok(())` - If the pruning was successful + /// + /// # Panics + /// + pub fn prune(&self, opts: &PruneOptions, prune_plan: PrunePlan) -> RusticResult<()> { + prune_repository(self, opts, prune_plan) } /// Turn the repository into the `IndexedFull` state by reading and storing the index @@ -1365,7 +1387,7 @@ impl Repository { /// // TODO: Document errors pub fn repair_index(&self, opts: &RepairIndexOptions, dry_run: bool) -> RusticResult<()> { - opts.repair(self, dry_run) + repair_index(self, *opts, dry_run) } } @@ -1816,7 +1838,7 @@ impl Repository { node_streamer: impl Iterator>, dest: &LocalDestination, ) -> RusticResult<()> { - opts.restore(restore_infos, self, node_streamer, dest) + restore_repository(restore_infos, self, *opts, node_streamer, dest) } /// Merge the given trees. @@ -2008,7 +2030,7 @@ impl Repository { dest: &LocalDestination, dry_run: bool, ) -> RusticResult { - opts.collect_and_prepare(self, node_streamer, dest, dry_run) + collect_and_prepare(self, *opts, node_streamer, dest, dry_run) } /// Copy the given `snapshots` to `repo_dest`. @@ -2064,6 +2086,6 @@ impl Repository { snapshots: Vec, dry_run: bool, ) -> RusticResult<()> { - opts.repair(self, snapshots, dry_run) + repair_snapshots(self, opts, snapshots, dry_run) } } diff --git a/crates/core/tests/command_input.rs b/crates/core/tests/command_input.rs index 8ac2713a..8cc0c9c3 100644 --- a/crates/core/tests/command_input.rs +++ b/crates/core/tests/command_input.rs @@ -1,8 +1,11 @@ +#[cfg(not(windows))] use std::fs::File; use anyhow::Result; use rustic_core::CommandInput; use serde::{Deserialize, Serialize}; + +#[cfg(not(windows))] use tempfile::tempdir; #[test] diff --git a/crates/core/tests/integration.rs b/crates/core/tests/integration.rs index e59ce735..5593514e 100644 --- a/crates/core/tests/integration.rs +++ b/crates/core/tests/integration.rs @@ -540,7 +540,7 @@ fn test_prune( let plan = repo.prune_plan(&prune_opts)?; // TODO: Snapshot-test the plan (currently doesn't impl Serialize) // assert_ron_snapshot!("prune", plan); - plan.do_prune(&repo, &prune_opts)?; + repo.prune(&prune_opts, plan)?; // run check let check_opts = CheckOptions::default().read_data(true); @@ -549,7 +549,7 @@ fn test_prune( if !instant_delete { // re-run if we only marked pack files. As keep-delete = 0, they should be removed here let plan = repo.prune_plan(&prune_opts)?; - plan.do_prune(&repo, &prune_opts)?; + repo.prune(&prune_opts, plan)?; repo.check(check_opts)?; }