diff --git a/CHANGELOG.md b/CHANGELOG.md index 32a143900..880a95198 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,49 @@ All notable changes to this project will be documented in this file. +## [0.9.5](https://github.com/rustic-rs/rustic/compare/v0.9.4...v0.9.5) - 2024-12-02 + +### Added + +- *(commands)* More dump options ([#1339](https://github.com/rustic-rs/rustic/pull/1339)) +- shut down gracefully with ctrl+c ([#1364](https://github.com/rustic-rs/rustic/pull/1364)) +- Add --filter-jq option ([#1372](https://github.com/rustic-rs/rustic/pull/1372)) +- *(commands)* Add `mount` command ([#973](https://github.com/rustic-rs/rustic/pull/973)) +- Error messages are now much improve + ([rustic_core](https://github.com/rustic-rs/rustic_core/releases/tag/rustic_core-v0.6.0)) + +### Fixed + +- *(commands)* run backup hooks before checking source dir ([#1374](https://github.com/rustic-rs/rustic/pull/1374)) +- *(commands)* Use spawn_blocking in webdav when calling rustic_core ([#1365](https://github.com/rustic-rs/rustic/pull/1365)) +- *(forget)* Add minutely timeline + ([rustic_core](https://github.com/rustic-rs/rustic_core/releases/tag/rustic_core-v0.7.2)) +- *(init)* Prevent overwriting hot repository + ([rustic_core](https://github.com/rustic-rs/rustic_core/releases/tag/rustic_core-v0.6.0)) + +### Other + +- update snapshots to include minutely configuration options +- *(deps)* update rustic_core, bytes, and libc dependencies to latest versions +- simplify lifetime annotations in OpenFileReader and TreeIterItem implementations +- clean up whitespace and update clippy linting allowances +- *(deps)* update dependencies to latest versions +- *(deps)* update lockfile to get rid of vulnerable `url` version +- *(mount)* rename fields for clarity, add user options for mount ([#1353](https://github.com/rustic-rs/rustic/pull/1353)) +- *(deps)* update dependencies +- *(deps)* don't use rustic_core webdav feature ([#1367](https://github.com/rustic-rs/rustic/pull/1367)) +- move `webdavfs` from `rustic_core` to `rustic-rs` ([#1363](https://github.com/rustic-rs/rustic/pull/1363)) +- *(clippy)* comment out unused lints in lib.rs +- *(clippy)* apply fixes automatically +- use BTreeMap for env in global options ([#1360](https://github.com/rustic-rs/rustic/pull/1360)) +- add tiny framework for testing rustic's compat with latest restic ([#1303](https://github.com/rustic-rs/rustic/pull/1303)) +- use snapshot tests for default config, show-config and completions ([#1359](https://github.com/rustic-rs/rustic/pull/1359)) +- *(deps)* update dependencies rustic_core, rustic_backend, rustic_testing, and migrate to conflate 0.3 ([#1357](https://github.com/rustic-rs/rustic/pull/1357)) +- fix typos +- *(build)* add platform-dependent settings and remove ci flag for extra features +- clarify `--use-profile` command in config by using long form ([#1344](https://github.com/rustic-rs/rustic/pull/1344)) +- *(deps)* update core and testing crates ([#1340](https://github.com/rustic-rs/rustic/pull/1340)) + ## [0.9.4](https://github.com/rustic-rs/rustic/compare/v0.9.3...v0.9.4) - 2024-10-24 ### Added diff --git a/Cargo.lock b/Cargo.lock index 8f132822f..e0fb45770 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,6 +197,15 @@ version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.7.1" @@ -896,6 +905,16 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctrlc" +version = "3.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3" +dependencies = [ + "nix", + "windows-sys 0.59.0", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -1041,6 +1060,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "derive_destructure2" version = "0.1.3" @@ -2489,6 +2519,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.22" @@ -3844,7 +3880,7 @@ dependencies = [ [[package]] name = "rustic-rs" -version = "0.9.4" +version = "0.9.5" dependencies = [ "abscissa_core", "aho-corasick", @@ -3861,6 +3897,7 @@ dependencies = [ "conflate", "convert_case", "crossterm", + "ctrlc", "dateparser", "dav-server", "derive_more", @@ -3910,6 +3947,7 @@ dependencies = [ "toml", "tui-textarea", "warp", + "zip", ] [[package]] @@ -4463,6 +4501,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "similar" version = "2.6.0" @@ -5869,6 +5913,24 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "zip" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d52293fc86ea7cf13971b3bb81eb21683636e7ae24c729cdaf1b7c4157a352" +dependencies = [ + "arbitrary", + "chrono", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap 2.6.0", + "memchr", + "thiserror 2.0.3", + "zopfli", +] + [[package]] name = "zipsign-api" version = "0.1.2" @@ -5880,6 +5942,20 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/Cargo.toml b/Cargo.toml index e2f1a0dfc..75dda4bae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustic-rs" -version = "0.9.4" +version = "0.9.5" authors = ["the rustic-rs team"] categories = ["command-line-utilities"] documentation = "https://docs.rs/rustic-rs" @@ -103,10 +103,12 @@ clap = { version = "4", features = ["derive", "env", "wrap_help"] } clap_complete = "4" conflate = "0.3.3" convert_case = "0.6.0" +ctrlc = { version = "3.4.5", features = ["termination"] } dateparser = "0.2.1" derive_more = { version = "1", features = ["debug"] } dialoguer = "0.11.0" directories = "5" +flate2 = "1.0.34" fuse_mt = { version = "0.6", optional = true } futures = { version = "0.3.31", optional = true } gethostname = "0.5" @@ -119,6 +121,7 @@ open = "5.3.1" self_update = { version = "=0.39.0", default-features = false, optional = true, features = ["rustls", "archive-tar", "compression-flate2"] } # FIXME: Downgraded to 0.39.0 due to https://github.com/jaemk/self_update/issues/136 tar = "0.4.43" toml = "0.8" +zip = { version = "2.2.0", default-features = false, features = ["deflate", "chrono"] } # filtering jaq-core = { version = "2", optional = true } diff --git a/deny.toml b/deny.toml index 14753b951..804993957 100644 --- a/deny.toml +++ b/deny.toml @@ -106,6 +106,7 @@ allow = [ "CC0-1.0", "Zlib", "Unicode-3.0", + "BSL-1.0", ] # The confidence threshold for detecting a license from license text. # The higher the value, the more closely the license text must be to the diff --git a/src/commands.rs b/src/commands.rs index 0b242ea10..5ac4af30b 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -35,6 +35,7 @@ use std::fmt::Debug; use std::fs::File; use std::path::PathBuf; use std::str::FromStr; +use std::sync::mpsc::channel; #[cfg(feature = "mount")] use crate::commands::mount::MountCmd; @@ -64,7 +65,7 @@ use clap::builder::{ }; use convert_case::{Case, Casing}; use human_panic::setup_panic; -use log::{log, Level}; +use log::{info, log, Level}; use simplelog::{CombinedLogger, LevelFilter, TermLogger, TerminalMode, WriteLogger}; use self::find::FindCmd; @@ -179,6 +180,20 @@ impl Runnable for EntryPoint { // Set up panic hook for better error messages and logs setup_panic!(); + // Set up Ctrl-C handler + let (tx, rx) = channel(); + + ctrlc::set_handler(move || tx.send(()).expect("Could not send signal on channel.")) + .expect("Error setting Ctrl-C handler"); + + _ = std::thread::spawn(move || { + // Wait for Ctrl-C + rx.recv().expect("Could not receive from channel."); + info!("Ctrl-C received, shutting down..."); + RUSTIC_APP.shutdown(Shutdown::Graceful) + }); + + // Run the subcommand self.commands.run(); RUSTIC_APP.shutdown(Shutdown::Graceful) } diff --git a/src/commands/backup.rs b/src/commands/backup.rs index 31dca3224..86f335af8 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -184,47 +184,47 @@ impl BackupCmd { } .to_indexed_ids()?; - let config_snapshot_sources: Vec<_> = snapshot_opts - .iter() - .map(|opt| -> Result<_> { - Ok(PathList::from_iter(&opt.sources) - .sanitize() - .with_context(|| { - format!( - "error sanitizing sources=\"{:?}\" in config file", - opt.sources - ) - })? - .merge()) - }) - .filter_map(|p| match p { - Ok(paths) => Some(paths), - Err(err) => { - warn!("{err}"); - None + let hooks = config.backup.hooks.with_context("backup"); + hooks.use_with(|| -> Result<_> { + let config_snapshot_sources: Vec<_> = snapshot_opts + .iter() + .map(|opt| -> Result<_> { + Ok(PathList::from_iter(&opt.sources) + .sanitize() + .with_context(|| { + format!( + "error sanitizing sources=\"{:?}\" in config file", + opt.sources + ) + })? + .merge()) + }) + .filter_map(|p| match p { + Ok(paths) => Some(paths), + Err(err) => { + warn!("{err}"); + None + } + }) + .collect(); + + let snapshot_sources = match (self.cli_sources.is_empty(), snapshot_opts.is_empty()) { + (false, _) => { + let item = PathList::from_iter(&self.cli_sources).sanitize()?; + vec![item] } - }) - .collect(); - - let snapshot_sources = match (self.cli_sources.is_empty(), snapshot_opts.is_empty()) { - (false, _) => { - let item = PathList::from_iter(&self.cli_sources).sanitize()?; - vec![item] - } - (true, false) => { - info!("using all backup sources from config file."); - config_snapshot_sources.clone() - } - (true, true) => { - bail!("no backup source given."); + (true, false) => { + info!("using all backup sources from config file."); + config_snapshot_sources.clone() + } + (true, true) => { + bail!("no backup source given."); + } + }; + if snapshot_sources.is_empty() { + return Ok(()); } - }; - if snapshot_sources.is_empty() { - return Ok(()); - } - let hooks = config.backup.hooks.with_context("backup"); - hooks.use_with(|| -> Result<_> { let mut is_err = false; for sources in snapshot_sources { let mut opts = self.clone(); diff --git a/src/commands/dump.rs b/src/commands/dump.rs index 496edb2f5..fdf8c40c8 100644 --- a/src/commands/dump.rs +++ b/src/commands/dump.rs @@ -1,11 +1,17 @@ //! `dump` subcommand -use std::io::{Read, Write}; +use std::{ + fs::File, + io::{copy, Cursor, Read, Seek, SeekFrom, Write}, + path::PathBuf, +}; use crate::{repository::CliIndexedRepo, status_err, Application, RUSTIC_APP}; use abscissa_core::{Command, Runnable, Shutdown}; use anyhow::Result; +use derive_more::FromStr; +use flate2::{write::GzEncoder, Compression}; use log::warn; use rustic_core::{ repofile::{Node, NodeType}, @@ -13,6 +19,7 @@ use rustic_core::{ LsOptions, }; use tar::{Builder, EntryType, Header}; +use zip::{write::SimpleFileOptions, ZipWriter}; /// `dump` subcommand #[derive(clap::Parser, Command, Debug)] @@ -21,9 +28,38 @@ pub(crate) struct DumpCmd { #[clap(value_name = "SNAPSHOT[:PATH]")] snap: String, - /// Listing options - #[clap(flatten)] - ls_opts: LsOptions, + /// set archive format to use. Possible values: auto, content, tar, targz, zip. For "auto" format is dertermined by file extension (if given) or "tar" for dirs. + #[clap(long, value_name = "FORMAT", default_value = "auto")] + archive: ArchiveKind, + + /// dump output to the given file. Use this instead of redirecting stdout to a file. + #[clap(long)] + file: Option, + + /// Glob pattern to exclude/include (can be specified multiple times) + #[clap(long, help_heading = "Exclude options")] + glob: Vec, + + /// Same as --glob pattern but ignores the casing of filenames + #[clap(long, value_name = "GLOB", help_heading = "Exclude options")] + iglob: Vec, + + /// Read glob patterns to exclude/include from this file (can be specified multiple times) + #[clap(long, value_name = "FILE", help_heading = "Exclude options")] + glob_file: Vec, + + /// Same as --glob-file ignores the casing of filenames in patterns + #[clap(long, value_name = "FILE", help_heading = "Exclude options")] + iglob_file: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, FromStr)] +enum ArchiveKind { + Auto, + Content, + Tar, + TarGz, + Zip, } impl Runnable for DumpCmd { @@ -46,17 +82,77 @@ impl DumpCmd { let node = repo.node_from_snapshot_path(&self.snap, |sn| config.snapshot_filter.matches(sn))?; - let mut stdout = std::io::stdout(); - if node.is_file() { - repo.dump(&node, &mut stdout)?; + let stdout = std::io::stdout(); + + let ls_opts = LsOptions::default() + .glob(self.glob.clone()) + .glob_file(self.glob_file.clone()) + .iglob(self.iglob.clone()) + .iglob_file(self.iglob_file.clone()) + .recursive(true); + + let ext = self + .file + .as_ref() + .and_then(|f| f.extension().map(|s| s.to_string_lossy().to_string())); + + let archive = match self.archive { + ArchiveKind::Auto => match ext.as_deref() { + Some("tar") => ArchiveKind::Tar, + Some("tgz") | Some("gz") => ArchiveKind::TarGz, + Some("zip") => ArchiveKind::Zip, + _ if node.is_dir() => ArchiveKind::Tar, + _ => ArchiveKind::Content, + }, + a => a, + }; + + let mut w: Box = if let Some(file) = &self.file { + let mut file = File::create(file)?; + if archive == ArchiveKind::Zip { + // when writing zip to a file, we use the optimized writer + return write_zip_to_file(&repo, &node, &mut file, &ls_opts); + } + Box::new(file) } else { - dump_tar(&repo, &node, &mut stdout, &self.ls_opts)?; - } + Box::new(stdout) + }; + + match archive { + ArchiveKind::Content => dump_content(&repo, &node, &mut w, &ls_opts)?, + ArchiveKind::Tar => dump_tar(&repo, &node, &mut w, &ls_opts)?, + ArchiveKind::TarGz => dump_tar_gz(&repo, &node, &mut w, &ls_opts)?, + ArchiveKind::Zip => dump_zip(&repo, &node, &mut w, &ls_opts)?, + _ => {} + }; Ok(()) } } +fn dump_content( + repo: &CliIndexedRepo, + node: &Node, + w: &mut impl Write, + ls_opts: &LsOptions, +) -> Result<()> { + for item in repo.ls(node, ls_opts)? { + let (_, node) = item?; + repo.dump(&node, w)?; + } + Ok(()) +} + +fn dump_tar_gz( + repo: &CliIndexedRepo, + node: &Node, + w: &mut impl Write, + ls_opts: &LsOptions, +) -> Result<()> { + let mut w = GzEncoder::new(w, Compression::default()); + dump_tar(repo, node, &mut w, ls_opts) +} + fn dump_tar( repo: &CliIndexedRepo, node: &Node, @@ -135,6 +231,105 @@ fn dump_tar( Ok(()) } +fn dump_zip( + repo: &CliIndexedRepo, + node: &Node, + w: &mut impl Write, + ls_opts: &LsOptions, +) -> Result<()> { + let w = SeekWriter { + write: w, + cursor: Cursor::new(Vec::new()), + written: 0, + }; + let mut zip = ZipWriter::new(w); + zip.set_flush_on_finish_file(true); + write_zip_contents(repo, node, &mut zip, ls_opts)?; + let mut inner = zip.finish()?; + inner.flush()?; + Ok(()) +} + +fn write_zip_to_file( + repo: &CliIndexedRepo, + node: &Node, + file: &mut (impl Write + Seek), + ls_opts: &LsOptions, +) -> Result<()> { + let mut zip = ZipWriter::new(file); + write_zip_contents(repo, node, &mut zip, ls_opts)?; + let _ = zip.finish()?; + Ok(()) +} + +fn write_zip_contents( + repo: &CliIndexedRepo, + node: &Node, + zip: &mut ZipWriter, + ls_opts: &LsOptions, +) -> Result<()> { + for item in repo.ls(node, ls_opts)? { + let (path, node) = item?; + + let mut options = SimpleFileOptions::default(); + if let Some(mode) = node.meta.mode { + // TODO: this is some go-mapped mode, but lower bits are the standard unix mode bits -> is this ok? + options = options.unix_permissions(mode); + } + if let Some(mtime) = node.meta.mtime { + options = + options.last_modified_time(mtime.naive_local().try_into().unwrap_or_default()); + } + if node.is_file() { + zip.start_file_from_path(path, options)?; + repo.dump(&node, zip)?; + } else { + zip.add_directory_from_path(path, options)?; + } + } + Ok(()) +} + +struct SeekWriter { + write: W, + cursor: Cursor>, + written: u64, +} + +impl Read for SeekWriter { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.cursor.read(buf) + } +} + +impl Write for SeekWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.cursor.write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + _ = self.cursor.seek(SeekFrom::Start(0))?; + let n = copy(&mut self.cursor, &mut self.write)?; + _ = self.cursor.seek(SeekFrom::Start(0))?; + self.cursor.get_mut().clear(); + self.cursor.get_mut().shrink_to(1_000_000); + self.written += n; + Ok(()) + } +} + +impl Seek for SeekWriter { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + match pos { + SeekFrom::Start(n) => self.cursor.seek(SeekFrom::Start(n - self.written)), + pos => self.cursor.seek(pos), + } + } + fn stream_position(&mut self) -> std::io::Result { + Ok(self.written + self.cursor.stream_position()?) + } +} + struct OpenFileReader<'a> { repo: &'a CliIndexedRepo, open_file: OpenFile, diff --git a/tests/backup_restore.rs b/tests/backup_restore.rs index 0beaf5b66..b5803833e 100644 --- a/tests/backup_restore.rs +++ b/tests/backup_restore.rs @@ -13,6 +13,9 @@ use tempfile::{tempdir, TempDir}; use assert_cmd::Command; use predicates::prelude::{predicate, PredicateBooleanExt}; +mod repositories; +use repositories::src_snapshot; + use rustic_testing::TestResult; pub fn rustic_runner(temp_dir: &TempDir) -> TestResult { @@ -46,13 +49,13 @@ fn setup() -> TestResult { #[test] fn test_backup_and_check_passes() -> TestResult<()> { let temp_dir = setup()?; - let backup = "src/"; + let backup = src_snapshot()?.into_path().into_path(); { // Run `backup` for the first time rustic_runner(&temp_dir)? .arg("backup") - .arg(backup) + .arg(&backup) .assert() .success() .stdout(predicate::str::contains("successfully saved.")); @@ -104,16 +107,15 @@ fn test_backup_and_check_passes() -> TestResult<()> { fn test_backup_and_restore_passes() -> TestResult<()> { let temp_dir = setup()?; let restore_dir = temp_dir.path().join("restore"); - let backup = "src/"; - - // actual repository root to backup - let backup_files = std::env::current_dir()?.join(backup); + let backup_files = src_snapshot()?.into_path().into_path(); { // Run `backup` for the first time rustic_runner(&temp_dir)? .arg("backup") .arg(&backup_files) + .arg("--as-path") + .arg("/") .assert() .success() .stdout(predicate::str::contains("successfully saved.")); @@ -130,11 +132,23 @@ fn test_backup_and_restore_passes() -> TestResult<()> { } // Compare the backup and the restored directory - let compare_result = - Comparison::default().compare(&backup_files, &restore_dir.join(&backup_files))?; + let compare_result = Comparison::default().compare(&backup_files, &restore_dir)?; // no differences assert!(compare_result.is_empty()); + let dump_tar_file = restore_dir.join("test.tar"); + { + // Run `dump` + rustic_runner(&temp_dir)? + .arg("dump") + .arg("latest") + .arg("--file") + .arg(&dump_tar_file) + .assert() + .success(); + } + // TODO: compare dump output with fixture + Ok(()) } diff --git a/tests/repositories.rs b/tests/repositories.rs index f8eec7883..15ab68b3d 100644 --- a/tests/repositories.rs +++ b/tests/repositories.rs @@ -8,7 +8,7 @@ use tar::Archive; use tempfile::{tempdir, TempDir}; #[derive(Debug)] -struct TestSource(TempDir); +pub struct TestSource(TempDir); impl TestSource { pub fn new(tmp: TempDir) -> Self { @@ -57,7 +57,7 @@ fn rustic_copy_repo() -> Result { } #[fixture] -fn src_snapshot() -> Result { +pub fn src_snapshot() -> Result { let dir = tempdir()?; let path = "tests/repository-fixtures/src-snapshot.tar.gz"; open_and_unpack(path, &dir)?;