diff --git a/.github/workflows/ci-heavy.yml b/.github/workflows/ci-heavy.yml index b28c802e..1c83c994 100644 --- a/.github/workflows/ci-heavy.yml +++ b/.github/workflows/ci-heavy.yml @@ -83,7 +83,7 @@ jobs: toolchain: ${{ matrix.rust }} - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2 - name: Run Cargo Test - run: cargo +${{ matrix.rust }} test -r --all-targets --all-features --workspace --examples + run: cargo +${{ matrix.rust }} test --all-targets --all-features --workspace --examples id: run_tests env: INSTA_UPDATE: new diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03ffc549..f9f58e05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + CI: true + on: pull_request: @@ -40,6 +43,8 @@ jobs: name: Test runs-on: ${{ matrix.job.os }} strategy: + # Don't fail fast, so we can actually see all the results + fail-fast: false matrix: rust: [stable] job: @@ -64,7 +69,16 @@ jobs: toolchain: ${{ matrix.rust }} - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2 - name: Run Cargo Test - run: cargo +${{ matrix.rust }} test -r --all-targets --all-features --workspace --examples + run: cargo +${{ matrix.rust }} test --all-targets --all-features --workspace --examples + id: run_tests + env: + INSTA_UPDATE: new + - name: Upload snapshots of failed tests + if: ${{ failure() && steps.run_tests.outcome == 'failure' }} + uses: actions/upload-artifact@v3 + with: + name: failed-snapshots-${{ matrix.job.os }} + path: "**/snapshots/*.snap.new" docs: name: Build docs diff --git a/.gitignore b/.gitignore index 0e341ea3..0a0a1985 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ coverage/*.info # local repo config .cargo/config.toml + +# Generated by Tests +crates/core/tests/generated/ diff --git a/Cargo.toml b/Cargo.toml index 91accdf1..6aa1fa6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,8 +13,12 @@ rust-version = "1.72.1" [workspace.dependencies] aho-corasick = "1.1.2" +anyhow = "1.0.81" +bytes = "1.5.0" +enum-map = "2.7.3" rustic_backend = { path = "crates/backend" } rustic_core = { path = "crates/core" } +rustic_testing = { path = "crates/testing" } simplelog = "0.12.2" # dev-dependencies diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 82c21e5a..4caf6cb8 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -102,11 +102,11 @@ futures = { version = "0.3", optional = true } runtime-format = "0.1.3" # other dependencies -anyhow = "1.0.81" -bytes = "1.5.0" +anyhow = { workspace = true } +bytes = { workspace = true } bytesize = "1.3.0" chrono = { version = "0.4.35", default-features = false, features = ["clock", "serde"] } -enum-map = "2.7.3" +enum-map = { workspace = true } enum-map-derive = "0.17.0" enumset = { version = "1.1.3", features = ["serde"] } gethostname = "0.4.3" @@ -132,6 +132,8 @@ xattr = "1" [dev-dependencies] expect-test = "1.4.1" +flate2 = "1.0.28" +insta = { version = "1.36.1", features = ["redactions", "ron"] } mockall = "0.12.1" pretty_assertions = "1.4.0" quickcheck = "1.0.3" @@ -140,8 +142,10 @@ rstest = { workspace = true } rustdoc-json = "0.8.9" # We need to have rustic_backend here, because the doc-tests in lib.rs of rustic_core rustic_backend = { workspace = true } +rustic_testing = { workspace = true } rustup-toolchain = "0.1.6" simplelog = "0.12.2" +tar = "0.4.40" tempfile = { workspace = true } [lints] diff --git a/crates/core/src/backend.rs b/crates/core/src/backend.rs index c14cc6fe..50a4c911 100644 --- a/crates/core/src/backend.rs +++ b/crates/core/src/backend.rs @@ -13,9 +13,12 @@ use std::{io::Read, ops::Deref, path::PathBuf, sync::Arc}; use anyhow::Result; use bytes::Bytes; +use enum_map::Enum; use log::trace; + #[cfg(test)] -use mockall::*; +use mockall::mock; + use serde_derive::{Deserialize, Serialize}; use crate::{ @@ -34,7 +37,7 @@ pub const ALL_FILE_TYPES: [FileType; 4] = [ ]; /// Type for describing the kind of a file that can occur. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Enum)] pub enum FileType { /// Config file #[serde(rename = "config")] @@ -295,7 +298,9 @@ pub trait WriteBackend: ReadBackend { /// # Returns /// /// The result of the creation. - fn create(&self) -> Result<()>; + fn create(&self) -> Result<()> { + Ok(()) + } /// Writes bytes to the given file. /// diff --git a/crates/core/src/commands/merge.rs b/crates/core/src/commands/merge.rs index fb86412c..63cf7c32 100644 --- a/crates/core/src/commands/merge.rs +++ b/crates/core/src/commands/merge.rs @@ -39,7 +39,12 @@ pub(crate) fn merge_snapshots( ) -> RusticResult { let now = Local::now(); - let paths = PathList::from_strings(snapshots.iter().flat_map(|snap| snap.paths.iter())).merge(); + let paths = snapshots + .iter() + .flat_map(|snap| snap.paths.iter()) + .collect::() + .merge(); + snap.paths.set_paths(&paths.paths())?; // set snapshot time to time of latest snapshot to be merged diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 698763c2..f277d948 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -45,10 +45,10 @@ implement [`serde::Serialize`] and [`serde::Deserialize`]. let config_opts = ConfigOptions::default(); - let _repo = Repository::new(&repo_opts, backends.clone()).unwrap().init(&key_opts, &config_opts).unwrap(); + let _repo = Repository::new(&repo_opts, &backends.clone()).unwrap().init(&key_opts, &config_opts).unwrap(); // We could have used _repo directly, but open the repository again to show how to open it... - let repo = Repository::new(&repo_opts, backends).unwrap().open().unwrap(); + let repo = Repository::new(&repo_opts, &backends).unwrap().open().unwrap(); // Get all snapshots from the repository let snaps = repo.get_all_snapshots().unwrap(); diff --git a/crates/core/src/repofile/snapshotfile.rs b/crates/core/src/repofile/snapshotfile.rs index 95af1e61..808a32d7 100644 --- a/crates/core/src/repofile/snapshotfile.rs +++ b/crates/core/src/repofile/snapshotfile.rs @@ -1100,25 +1100,31 @@ impl Display for PathList { } } -impl PathList { - /// Create a `PathList` from `String`s. - /// - /// # Arguments - /// - /// * `source` - The `String`s to use - pub fn from_strings(source: I) -> Self - where - I: IntoIterator, - I::Item: AsRef, - { - Self( - source - .into_iter() - .map(|source| PathBuf::from(source.as_ref())) - .collect(), - ) +impl FromIterator for PathList { + fn from_iter>(iter: I) -> Self { + Self(iter.into_iter().collect()) } +} +impl<'a> FromIterator<&'a String> for PathList { + fn from_iter>(iter: I) -> Self { + Self(iter.into_iter().map(PathBuf::from).collect()) + } +} + +impl FromIterator for PathList { + fn from_iter>(iter: I) -> Self { + Self(iter.into_iter().map(PathBuf::from).collect()) + } +} + +impl<'a> FromIterator<&'a str> for PathList { + fn from_iter>(iter: I) -> Self { + Self(iter.into_iter().map(PathBuf::from).collect()) + } +} + +impl PathList { /// Create a `PathList` by parsing a Strings containing paths separated by whitspaces. /// /// # Arguments @@ -1132,7 +1138,7 @@ impl PathList { /// [`SnapshotFileErrorKind::FromSplitError`]: crate::error::SnapshotFileErrorKind::FromSplitError pub fn from_string(sources: &str) -> RusticResult { let sources = split(sources).map_err(SnapshotFileErrorKind::FromSplitError)?; - Ok(Self::from_strings(sources)) + Ok(Self::from_iter(sources)) } /// Number of paths in the `PathList`. diff --git a/crates/core/src/repository.rs b/crates/core/src/repository.rs index 5079aef4..d9261b44 100644 --- a/crates/core/src/repository.rs +++ b/crates/core/src/repository.rs @@ -1452,6 +1452,7 @@ impl Repository { /// # Arguments /// /// * `id` - The `Id` of the tree + // TODO!: This ID should be a tree ID, we should refactor it to wrap it in a TreeId type /// /// # Errors /// @@ -1475,6 +1476,7 @@ impl Repository { /// # Arguments /// /// * `root_tree` - The `Id` of the root tree + // TODO!: This ID should be a tree ID, we should refactor it to wrap it in a TreeId type /// * `path` - The path /// /// # Errors diff --git a/crates/core/tests/fixtures/backup-data.tar.gz b/crates/core/tests/fixtures/backup-data.tar.gz new file mode 100644 index 00000000..6ba5881a Binary files /dev/null and b/crates/core/tests/fixtures/backup-data.tar.gz differ diff --git a/crates/core/tests/integration.rs b/crates/core/tests/integration.rs new file mode 100644 index 00000000..f7707d64 --- /dev/null +++ b/crates/core/tests/integration.rs @@ -0,0 +1,338 @@ +//! Integration tests for the core library +//! +//! # How to update snapshots +//! +//! The CI pipeline is configured to run the tests with the `INSTA_UPDATE` environment variable set to `new`. +//! This means, it uploads the failed tests snapshots to the artifacts and you can download them and use them to update the snapshots. +//! +//! To update the snapshots, you download the artifacts and copy the files to the `tests/snapshots` directory. +//! Then you run `cargo insta review` to review the changes and accept them. +//! +//! # Redactions +//! +//! We use the `insta` crate to compare the actual output of the tests with the expected output. +//! Some data in the output changes every test run, we use insta's redactions to replace the actual values with placeholders. +//! We define the redactions inside `Settings` in the fixtures and bind them to the test functions. You can read more about +//! it [here](https://docs.rs/insta/latest/insta/struct.Settings.html). +//! +//! # Fixtures and Dependency Injection +//! +//! We use the `rstest` crate to define fixtures and dependency injection. +//! This allows us to define a set of fixtures that are used in multiple tests. +//! The fixtures are defined as functions with the `#[fixture]` attribute. +//! The tests that use the fixtures are defined as functions with the `#[rstest]` attribute. +//! The fixtures are passed as arguments to the test functions. + +use anyhow::Result; +use flate2::read::GzDecoder; +use insta::internals::{Content, ContentPath}; +use insta::{assert_ron_snapshot, Settings}; +use pretty_assertions::assert_eq; +use rstest::fixture; +use rstest::rstest; +use rustic_core::{ + repofile::SnapshotFile, BackupOptions, ConfigOptions, KeyOptions, NoProgressBars, OpenStatus, + PathList, Repository, RepositoryBackends, RepositoryOptions, +}; +use serde_derive::Serialize; + +use rustic_testing::backend::in_memory_backend::InMemoryBackend; + +use std::{ + env, + fs::File, + path::{Path, PathBuf}, + str::FromStr, + sync::Arc, +}; +// uncomment for logging output +// use simplelog::{Config, SimpleLogger}; +use tar::Archive; +use tempfile::{tempdir, TempDir}; + +type RepoOpen = Repository; + +#[derive(Debug)] +struct TestSource(TempDir); + +impl TestSource { + fn new(tmp: TempDir) -> Self { + Self(tmp) + } + + fn path_list(&self) -> PathList { + PathList::from_iter(Some(self.0.path().to_path_buf())) + } +} + +#[fixture] +fn set_up_repo() -> Result { + let be = InMemoryBackend::new(); + let be = RepositoryBackends::new(Arc::new(be), None); + let options = RepositoryOptions::default().password("test"); + let repo = Repository::new(&options, &be)?; + let key_opts = KeyOptions::default(); + let config_opts = &ConfigOptions::default(); + let repo = repo.init(&key_opts, config_opts)?; + Ok(repo) +} + +// helper func to redact options, but still keep information about some/none +#[allow(clippy::needless_pass_by_value)] // we need exactly that function signature +fn handle_option(val: Content, _: ContentPath<'_>) -> String { + if val.is_nil() { + "[none]".to_string() + } else { + "[some]".to_string() + } +} + +#[fixture] +fn insta_summary_redaction() -> Settings { + let mut settings = insta::Settings::clone_current(); + + settings.add_redaction(".tree", "[tree_id]"); + settings.add_dynamic_redaction(".program_version", |val, _| { + val.resolve_inner() + .as_str() + .map_or("[program_version]".to_string(), |v| { + v.replace(env!("CARGO_PKG_VERSION"), "[rustic_core_version]") + }) + }); + settings.add_redaction(".time", "[time]"); + settings.add_dynamic_redaction(".parent", handle_option); + settings.add_redaction(".tags", "[tags]"); + settings.add_redaction(".id", "[id]"); + settings.add_redaction(".summary.backup_start", "[backup_start]"); + settings.add_redaction(".summary.backup_end", "[backup_end]"); + settings.add_redaction(".summary.backup_duration", "[backup_duration]"); + settings.add_redaction(".summary.total_duration", "[total_duration]"); + settings.add_redaction(".summary.data_added", "[data_added]"); + settings.add_redaction(".summary.data_added_packed", "[data_added_packed]"); + settings.add_redaction( + ".summary.total_dirsize_processed", + "[total_dirsize_processed]", + ); + settings.add_redaction( + ".summary.data_added_trees_packed", + "[data_added_trees_packed]", + ); + settings.add_redaction(".summary.data_added_trees", "[data_added_trees]"); + + settings +} + +#[fixture] +fn insta_tree_redaction() -> Settings { + let mut settings = insta::Settings::clone_current(); + + settings.add_redaction(".nodes[].inode", "[inode]"); + settings.add_redaction(".nodes[].device_id", "[device_id]"); + settings.add_redaction(".nodes[].uid", "[uid]"); + settings.add_redaction(".nodes[].user", "[user]"); + settings.add_redaction(".nodes[].gid", "[gid]"); + settings.add_redaction(".nodes[].group", "[group]"); + settings.add_dynamic_redaction(".nodes[].mode", handle_option); + settings.add_dynamic_redaction(".nodes[].mtime", handle_option); + settings.add_dynamic_redaction(".nodes[].atime", handle_option); + settings.add_dynamic_redaction(".nodes[].ctime", handle_option); + + settings +} + +#[fixture] +fn tar_gz_testdata() -> Result { + let dir = tempdir()?; + let path = Path::new("tests/fixtures/backup-data.tar.gz").canonicalize()?; + let tar_gz = File::open(path)?; + let tar = GzDecoder::new(tar_gz); + let mut archive = Archive::new(tar); + archive.set_preserve_permissions(true); + archive.set_preserve_mtime(true); + archive.unpack(&dir)?; + Ok(TestSource::new(dir)) +} + +// Parts of the snapshot summary we want to test against references +// +// # Note +// +// We use a struct to avoid having to escape the field names in the snapshot +// we use insta redactions to replace the actual values with placeholders in case +// there are changes in the actual values +// Readme: https://insta.rs/docs/redactions/ +#[derive(Serialize)] +struct TestSummary<'a>(&'a SnapshotFile); + +#[rstest] +fn test_backup_with_tar_gz_passes( + tar_gz_testdata: Result, + set_up_repo: Result, + insta_summary_redaction: Settings, + insta_tree_redaction: Settings, +) -> Result<()> { + // uncomment for logging output + // SimpleLogger::init(log::LevelFilter::Debug, Config::default())?; + + // Fixtures + let (source, repo) = (tar_gz_testdata?, set_up_repo?.to_indexed_ids()?); + + let paths = &source.path_list(); + + // we use as_path to not depend on the actual tempdir + let opts = BackupOptions::default().as_path(PathBuf::from_str("test")?); + + // first backup + let first_snapshot = repo.backup(&opts, paths, SnapshotFile::default())?; + + // We can also bind to scope ( https://docs.rs/insta/latest/insta/struct.Settings.html#method.bind_to_scope ) + // But I think that can get messy with a lot of tests, also checking which settings are currently applied + // will be probably harder + insta_summary_redaction.bind(|| { + #[cfg(windows)] + assert_ron_snapshot!( + "backup-tar-summary-first-windows", + TestSummary(&first_snapshot) + ); + #[cfg(not(windows))] + assert_ron_snapshot!("backup-tar-summary-first-nix", TestSummary(&first_snapshot)); + }); + + assert_eq!(first_snapshot.parent, None); + + // tree of first backup + // re-read index + let repo = repo.to_indexed_ids()?; + let tree = repo.node_from_path(first_snapshot.tree, Path::new("test/0/tests"))?; + let tree: rustic_core::repofile::Tree = repo.get_tree(&tree.subtree.expect("Sub tree"))?; + + insta_tree_redaction.bind(|| { + #[cfg(windows)] + assert_ron_snapshot!("backup-tar-tree-windows", tree); + #[cfg(not(windows))] + assert_ron_snapshot!("backup-tar-tree-nix", tree); + }); + + // get all snapshots and check them + let all_snapshots = repo.get_all_snapshots()?; + assert_eq!(vec![first_snapshot.clone()], all_snapshots); + // save list of pack files + let packs1: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect(); + + // re-read index + let repo = repo.to_indexed_ids()?; + // second backup + let second_snapshot = repo.backup(&opts, paths, SnapshotFile::default())?; + + insta_summary_redaction.bind(|| { + #[cfg(windows)] + assert_ron_snapshot!( + "backup-tar-summary-second-windows", + TestSummary(&second_snapshot) + ); + #[cfg(not(windows))] + assert_ron_snapshot!( + "backup-tar-summary-second-nix", + TestSummary(&second_snapshot) + ); + }); + + assert_eq!(second_snapshot.parent, Some(first_snapshot.id)); + assert_eq!(first_snapshot.tree, second_snapshot.tree); + + // get all snapshots and check them + let mut all_snapshots = repo.get_all_snapshots()?; + all_snapshots.sort_unstable(); + assert_eq!(vec![first_snapshot, second_snapshot], all_snapshots); + + // pack files should be unchanged + let packs2: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect(); + assert_eq!(packs1, packs2); + Ok(()) +} + +#[rstest] +fn test_backup_dry_run_with_tar_gz_passes( + tar_gz_testdata: Result, + set_up_repo: Result, + insta_summary_redaction: Settings, + insta_tree_redaction: Settings, +) -> Result<()> { + // Fixtures + let (source, repo) = (tar_gz_testdata?, set_up_repo?.to_indexed_ids()?); + + let paths = &source.path_list(); + + // we use as_path to not depend on the actual tempdir + let opts = BackupOptions::default() + .as_path(PathBuf::from_str("test")?) + .dry_run(true); + + // dry-run backup + let snap_dry_run = repo.backup(&opts, paths, SnapshotFile::default())?; + + insta_summary_redaction.bind(|| { + #[cfg(windows)] + assert_ron_snapshot!( + "dryrun-tar-summary-first-windows", + TestSummary(&snap_dry_run) + ); + #[cfg(not(windows))] + assert_ron_snapshot!("dryrun-tar-summary-first-nix", TestSummary(&snap_dry_run)); + }); + + // check that repo is still empty + let snaps = repo.get_all_snapshots()?; + assert_eq!(snaps.len(), 0); + assert_eq!(repo.list(rustic_core::FileType::Pack)?.count(), 0); + assert_eq!(repo.list(rustic_core::FileType::Index)?.count(), 0); + + // first real backup + let opts = opts.dry_run(false); + let first_snapshot = repo.backup(&opts, paths, SnapshotFile::default())?; + assert_eq!(snap_dry_run.tree, first_snapshot.tree); + let packs: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect(); + + // tree of first backup + // re-read index + let repo = repo.to_indexed_ids()?; + let tree = repo.node_from_path(first_snapshot.tree, Path::new("test/0/tests"))?; + let tree = repo.get_tree(&tree.subtree.expect("Sub tree"))?; + + insta_tree_redaction.bind(|| { + #[cfg(windows)] + assert_ron_snapshot!("dryrun-tar-tree-windows", tree); + #[cfg(not(windows))] + assert_ron_snapshot!("dryrun-tar-tree-nix", tree); + }); + + // re-read index + let repo = repo.to_indexed_ids()?; + // second dry-run backup + let opts = opts.dry_run(true); + let snap_dry_run = repo.backup(&opts, paths, SnapshotFile::default())?; + + insta_summary_redaction.bind(|| { + #[cfg(windows)] + assert_ron_snapshot!( + "dryrun-tar-summary-second-windows", + TestSummary(&snap_dry_run) + ); + #[cfg(not(windows))] + assert_ron_snapshot!("dryrun-tar-summary-second-nix", TestSummary(&snap_dry_run)); + }); + + // check that no data has been added + let snaps = repo.get_all_snapshots()?; + assert_eq!(snaps, vec![first_snapshot]); + let packs_dry_run: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect(); + assert_eq!(packs_dry_run, packs); + + // re-read index + let repo = repo.to_indexed_ids()?; + // second real backup + let opts = opts.dry_run(false); + let second_snapshot = repo.backup(&opts, paths, SnapshotFile::default())?; + assert_eq!(snap_dry_run.tree, second_snapshot.tree); + Ok(()) +} diff --git a/crates/core/tests/snapshots/integration__backup-tar-summary-first-nix.snap b/crates/core/tests/snapshots/integration__backup-tar-summary-first-nix.snap new file mode 100644 index 00000000..6a4a9251 --- /dev/null +++ b/crates/core/tests/snapshots/integration__backup-tar-summary-first-nix.snap @@ -0,0 +1,43 @@ +--- +source: crates/core/tests/integration.rs +expression: TestSummary(&first_snapshot) +--- +TestSummary(SnapshotFile( + time: "[time]", + program_version: "rustic [rustic_core_version]", + tree: "[tree_id]", + paths: StringList([ + "test", + ]), + hostname: "", + username: "", + uid: 0, + gid: 0, + tags: "[tags]", + summary: Some(SnapshotSummary( + files_new: 73, + files_changed: 0, + files_unmodified: 0, + total_files_processed: 73, + total_bytes_processed: 1125674, + dirs_new: 6, + dirs_changed: 0, + dirs_unmodified: 0, + total_dirs_processed: 6, + total_dirsize_processed: "[total_dirsize_processed]", + data_blobs: 70, + tree_blobs: 6, + data_added: "[data_added]", + data_added_packed: "[data_added_packed]", + data_added_files: 1125653, + data_added_files_packed: 78740, + data_added_trees: "[data_added_trees]", + data_added_trees_packed: "[data_added_trees_packed]", + command: "", + backup_start: "[backup_start]", + backup_end: "[backup_end]", + backup_duration: "[backup_duration]", + total_duration: "[total_duration]", + )), + id: "[id]", +)) diff --git a/crates/core/tests/snapshots/integration__backup-tar-summary-first-windows.snap b/crates/core/tests/snapshots/integration__backup-tar-summary-first-windows.snap new file mode 100644 index 00000000..6a4a9251 --- /dev/null +++ b/crates/core/tests/snapshots/integration__backup-tar-summary-first-windows.snap @@ -0,0 +1,43 @@ +--- +source: crates/core/tests/integration.rs +expression: TestSummary(&first_snapshot) +--- +TestSummary(SnapshotFile( + time: "[time]", + program_version: "rustic [rustic_core_version]", + tree: "[tree_id]", + paths: StringList([ + "test", + ]), + hostname: "", + username: "", + uid: 0, + gid: 0, + tags: "[tags]", + summary: Some(SnapshotSummary( + files_new: 73, + files_changed: 0, + files_unmodified: 0, + total_files_processed: 73, + total_bytes_processed: 1125674, + dirs_new: 6, + dirs_changed: 0, + dirs_unmodified: 0, + total_dirs_processed: 6, + total_dirsize_processed: "[total_dirsize_processed]", + data_blobs: 70, + tree_blobs: 6, + data_added: "[data_added]", + data_added_packed: "[data_added_packed]", + data_added_files: 1125653, + data_added_files_packed: 78740, + data_added_trees: "[data_added_trees]", + data_added_trees_packed: "[data_added_trees_packed]", + command: "", + backup_start: "[backup_start]", + backup_end: "[backup_end]", + backup_duration: "[backup_duration]", + total_duration: "[total_duration]", + )), + id: "[id]", +)) diff --git a/crates/core/tests/snapshots/integration__backup-tar-summary-second-nix.snap b/crates/core/tests/snapshots/integration__backup-tar-summary-second-nix.snap new file mode 100644 index 00000000..bba6b48d --- /dev/null +++ b/crates/core/tests/snapshots/integration__backup-tar-summary-second-nix.snap @@ -0,0 +1,44 @@ +--- +source: crates/core/tests/integration.rs +expression: TestSummary(&second_snapshot) +--- +TestSummary(SnapshotFile( + time: "[time]", + program_version: "rustic [rustic_core_version]", + parent: "[some]", + tree: "[tree_id]", + paths: StringList([ + "test", + ]), + hostname: "", + username: "", + uid: 0, + gid: 0, + tags: "[tags]", + summary: Some(SnapshotSummary( + files_new: 0, + files_changed: 0, + files_unmodified: 73, + total_files_processed: 73, + total_bytes_processed: 1125682, + dirs_new: 0, + dirs_changed: 0, + dirs_unmodified: 6, + total_dirs_processed: 6, + total_dirsize_processed: "[total_dirsize_processed]", + data_blobs: 0, + tree_blobs: 0, + data_added: "[data_added]", + data_added_packed: "[data_added_packed]", + data_added_files: 0, + data_added_files_packed: 0, + data_added_trees: "[data_added_trees]", + data_added_trees_packed: "[data_added_trees_packed]", + command: "", + backup_start: "[backup_start]", + backup_end: "[backup_end]", + backup_duration: "[backup_duration]", + total_duration: "[total_duration]", + )), + id: "[id]", +)) diff --git a/crates/core/tests/snapshots/integration__backup-tar-summary-second-windows.snap b/crates/core/tests/snapshots/integration__backup-tar-summary-second-windows.snap new file mode 100644 index 00000000..b2f88d43 --- /dev/null +++ b/crates/core/tests/snapshots/integration__backup-tar-summary-second-windows.snap @@ -0,0 +1,44 @@ +--- +source: crates/core/tests/integration.rs +expression: TestSummary(&second_snapshot) +--- +TestSummary(SnapshotFile( + time: "[time]", + program_version: "rustic [rustic_core_version]", + parent: "[some]", + tree: "[tree_id]", + paths: StringList([ + "test", + ]), + hostname: "", + username: "", + uid: 0, + gid: 0, + tags: "[tags]", + summary: Some(SnapshotSummary( + files_new: 0, + files_changed: 0, + files_unmodified: 73, + total_files_processed: 73, + total_bytes_processed: 1125674, + dirs_new: 0, + dirs_changed: 0, + dirs_unmodified: 6, + total_dirs_processed: 6, + total_dirsize_processed: "[total_dirsize_processed]", + data_blobs: 0, + tree_blobs: 0, + data_added: "[data_added]", + data_added_packed: "[data_added_packed]", + data_added_files: 0, + data_added_files_packed: 0, + data_added_trees: "[data_added_trees]", + data_added_trees_packed: "[data_added_trees_packed]", + command: "", + backup_start: "[backup_start]", + backup_end: "[backup_end]", + backup_duration: "[backup_duration]", + total_duration: "[total_duration]", + )), + id: "[id]", +)) diff --git a/crates/core/tests/snapshots/integration__backup-tar-tree-nix.snap b/crates/core/tests/snapshots/integration__backup-tar-tree-nix.snap new file mode 100644 index 00000000..1d815497 --- /dev/null +++ b/crates/core/tests/snapshots/integration__backup-tar-tree-nix.snap @@ -0,0 +1,80 @@ +--- +source: crates/core/tests/integration.rs +expression: tree +--- +Tree( + nodes: [ + { + "name": "empty-file", + "type": "file", + "mode": "[some]", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "uid": "[uid]", + "gid": "[gid]", + "user": "[user]", + "group": "[group]", + "inode": "[inode]", + "device_id": "[device_id]", + "links": 1, + "content": Some([]), + }, + { + "name": "testfile", + "type": "file", + "mode": "[some]", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "uid": "[uid]", + "gid": "[gid]", + "user": "[user]", + "group": "[group]", + "inode": "[inode]", + "device_id": "[device_id]", + "size": 21, + "links": 2, + "content": Some([ + Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"), + ]), + }, + { + "name": "testfile-hardlink", + "type": "file", + "mode": "[some]", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "uid": "[uid]", + "gid": "[gid]", + "user": "[user]", + "group": "[group]", + "inode": "[inode]", + "device_id": "[device_id]", + "size": 21, + "links": 2, + "content": Some([ + Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"), + ]), + }, + { + "name": "testfile-symlink", + "type": "symlink", + "linktarget": "testfile", + "mode": "[some]", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "uid": "[uid]", + "gid": "[gid]", + "user": "[user]", + "group": "[group]", + "inode": "[inode]", + "device_id": "[device_id]", + "size": 8, + "links": 1, + "content": None, + }, + ], +) diff --git a/crates/core/tests/snapshots/integration__backup-tar-tree-windows.snap b/crates/core/tests/snapshots/integration__backup-tar-tree-windows.snap new file mode 100644 index 00000000..21c9e746 --- /dev/null +++ b/crates/core/tests/snapshots/integration__backup-tar-tree-windows.snap @@ -0,0 +1,47 @@ +--- +source: crates/core/tests/integration.rs +expression: tree +--- +Tree( + nodes: [ + { + "name": "empty-file", + "type": "file", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "content": Some([]), + }, + { + "name": "testfile", + "type": "file", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "size": 21, + "content": Some([ + Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"), + ]), + }, + { + "name": "testfile-hardlink", + "type": "file", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "size": 21, + "content": Some([ + Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"), + ]), + }, + { + "name": "testfile-symlink", + "type": "symlink", + "linktarget": "testfile", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "content": None, + }, + ], +) diff --git a/crates/core/tests/snapshots/integration__dryrun-tar-summary-first-nix.snap b/crates/core/tests/snapshots/integration__dryrun-tar-summary-first-nix.snap new file mode 100644 index 00000000..01f95b03 --- /dev/null +++ b/crates/core/tests/snapshots/integration__dryrun-tar-summary-first-nix.snap @@ -0,0 +1,42 @@ +--- +source: crates/core/tests/integration.rs +expression: TestSummary(&snap_dry_run) +--- +TestSummary(SnapshotFile( + time: "[time]", + program_version: "rustic [rustic_core_version]", + tree: "[tree_id]", + paths: StringList([ + "test", + ]), + hostname: "", + username: "", + uid: 0, + gid: 0, + tags: "[tags]", + summary: Some(SnapshotSummary( + files_new: 73, + files_changed: 0, + files_unmodified: 0, + total_files_processed: 73, + total_bytes_processed: 1125674, + dirs_new: 6, + dirs_changed: 0, + dirs_unmodified: 0, + total_dirs_processed: 6, + total_dirsize_processed: "[total_dirsize_processed]", + data_blobs: 70, + tree_blobs: 6, + data_added: "[data_added]", + data_added_packed: "[data_added_packed]", + data_added_files: 1125653, + data_added_files_packed: 78740, + data_added_trees: "[data_added_trees]", + data_added_trees_packed: "[data_added_trees_packed]", + command: "", + backup_start: "[backup_start]", + backup_end: "[backup_end]", + backup_duration: "[backup_duration]", + total_duration: "[total_duration]", + )), +)) diff --git a/crates/core/tests/snapshots/integration__dryrun-tar-summary-first-windows.snap b/crates/core/tests/snapshots/integration__dryrun-tar-summary-first-windows.snap new file mode 100644 index 00000000..01f95b03 --- /dev/null +++ b/crates/core/tests/snapshots/integration__dryrun-tar-summary-first-windows.snap @@ -0,0 +1,42 @@ +--- +source: crates/core/tests/integration.rs +expression: TestSummary(&snap_dry_run) +--- +TestSummary(SnapshotFile( + time: "[time]", + program_version: "rustic [rustic_core_version]", + tree: "[tree_id]", + paths: StringList([ + "test", + ]), + hostname: "", + username: "", + uid: 0, + gid: 0, + tags: "[tags]", + summary: Some(SnapshotSummary( + files_new: 73, + files_changed: 0, + files_unmodified: 0, + total_files_processed: 73, + total_bytes_processed: 1125674, + dirs_new: 6, + dirs_changed: 0, + dirs_unmodified: 0, + total_dirs_processed: 6, + total_dirsize_processed: "[total_dirsize_processed]", + data_blobs: 70, + tree_blobs: 6, + data_added: "[data_added]", + data_added_packed: "[data_added_packed]", + data_added_files: 1125653, + data_added_files_packed: 78740, + data_added_trees: "[data_added_trees]", + data_added_trees_packed: "[data_added_trees_packed]", + command: "", + backup_start: "[backup_start]", + backup_end: "[backup_end]", + backup_duration: "[backup_duration]", + total_duration: "[total_duration]", + )), +)) diff --git a/crates/core/tests/snapshots/integration__dryrun-tar-summary-second-nix.snap b/crates/core/tests/snapshots/integration__dryrun-tar-summary-second-nix.snap new file mode 100644 index 00000000..d75737df --- /dev/null +++ b/crates/core/tests/snapshots/integration__dryrun-tar-summary-second-nix.snap @@ -0,0 +1,43 @@ +--- +source: crates/core/tests/integration.rs +expression: TestSummary(&snap_dry_run) +--- +TestSummary(SnapshotFile( + time: "[time]", + program_version: "rustic [rustic_core_version]", + parent: "[some]", + tree: "[tree_id]", + paths: StringList([ + "test", + ]), + hostname: "", + username: "", + uid: 0, + gid: 0, + tags: "[tags]", + summary: Some(SnapshotSummary( + files_new: 0, + files_changed: 0, + files_unmodified: 73, + total_files_processed: 73, + total_bytes_processed: 1125682, + dirs_new: 0, + dirs_changed: 0, + dirs_unmodified: 6, + total_dirs_processed: 6, + total_dirsize_processed: "[total_dirsize_processed]", + data_blobs: 0, + tree_blobs: 0, + data_added: "[data_added]", + data_added_packed: "[data_added_packed]", + data_added_files: 0, + data_added_files_packed: 0, + data_added_trees: "[data_added_trees]", + data_added_trees_packed: "[data_added_trees_packed]", + command: "", + backup_start: "[backup_start]", + backup_end: "[backup_end]", + backup_duration: "[backup_duration]", + total_duration: "[total_duration]", + )), +)) diff --git a/crates/core/tests/snapshots/integration__dryrun-tar-summary-second-windows.snap b/crates/core/tests/snapshots/integration__dryrun-tar-summary-second-windows.snap new file mode 100644 index 00000000..b6169e59 --- /dev/null +++ b/crates/core/tests/snapshots/integration__dryrun-tar-summary-second-windows.snap @@ -0,0 +1,43 @@ +--- +source: crates/core/tests/integration.rs +expression: TestSummary(&snap_dry_run) +--- +TestSummary(SnapshotFile( + time: "[time]", + program_version: "rustic [rustic_core_version]", + parent: "[some]", + tree: "[tree_id]", + paths: StringList([ + "test", + ]), + hostname: "", + username: "", + uid: 0, + gid: 0, + tags: "[tags]", + summary: Some(SnapshotSummary( + files_new: 0, + files_changed: 0, + files_unmodified: 73, + total_files_processed: 73, + total_bytes_processed: 1125674, + dirs_new: 0, + dirs_changed: 0, + dirs_unmodified: 6, + total_dirs_processed: 6, + total_dirsize_processed: "[total_dirsize_processed]", + data_blobs: 0, + tree_blobs: 0, + data_added: "[data_added]", + data_added_packed: "[data_added_packed]", + data_added_files: 0, + data_added_files_packed: 0, + data_added_trees: "[data_added_trees]", + data_added_trees_packed: "[data_added_trees_packed]", + command: "", + backup_start: "[backup_start]", + backup_end: "[backup_end]", + backup_duration: "[backup_duration]", + total_duration: "[total_duration]", + )), +)) diff --git a/crates/core/tests/snapshots/integration__dryrun-tar-tree-nix.snap b/crates/core/tests/snapshots/integration__dryrun-tar-tree-nix.snap new file mode 100644 index 00000000..1d815497 --- /dev/null +++ b/crates/core/tests/snapshots/integration__dryrun-tar-tree-nix.snap @@ -0,0 +1,80 @@ +--- +source: crates/core/tests/integration.rs +expression: tree +--- +Tree( + nodes: [ + { + "name": "empty-file", + "type": "file", + "mode": "[some]", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "uid": "[uid]", + "gid": "[gid]", + "user": "[user]", + "group": "[group]", + "inode": "[inode]", + "device_id": "[device_id]", + "links": 1, + "content": Some([]), + }, + { + "name": "testfile", + "type": "file", + "mode": "[some]", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "uid": "[uid]", + "gid": "[gid]", + "user": "[user]", + "group": "[group]", + "inode": "[inode]", + "device_id": "[device_id]", + "size": 21, + "links": 2, + "content": Some([ + Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"), + ]), + }, + { + "name": "testfile-hardlink", + "type": "file", + "mode": "[some]", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "uid": "[uid]", + "gid": "[gid]", + "user": "[user]", + "group": "[group]", + "inode": "[inode]", + "device_id": "[device_id]", + "size": 21, + "links": 2, + "content": Some([ + Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"), + ]), + }, + { + "name": "testfile-symlink", + "type": "symlink", + "linktarget": "testfile", + "mode": "[some]", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "uid": "[uid]", + "gid": "[gid]", + "user": "[user]", + "group": "[group]", + "inode": "[inode]", + "device_id": "[device_id]", + "size": 8, + "links": 1, + "content": None, + }, + ], +) diff --git a/crates/core/tests/snapshots/integration__dryrun-tar-tree-windows.snap b/crates/core/tests/snapshots/integration__dryrun-tar-tree-windows.snap new file mode 100644 index 00000000..21c9e746 --- /dev/null +++ b/crates/core/tests/snapshots/integration__dryrun-tar-tree-windows.snap @@ -0,0 +1,47 @@ +--- +source: crates/core/tests/integration.rs +expression: tree +--- +Tree( + nodes: [ + { + "name": "empty-file", + "type": "file", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "content": Some([]), + }, + { + "name": "testfile", + "type": "file", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "size": 21, + "content": Some([ + Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"), + ]), + }, + { + "name": "testfile-hardlink", + "type": "file", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "size": 21, + "content": Some([ + Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"), + ]), + }, + { + "name": "testfile-symlink", + "type": "symlink", + "linktarget": "testfile", + "mtime": "[some]", + "atime": "[some]", + "ctime": "[some]", + "content": None, + }, + ], +) diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index 35d865ad..e39f9724 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -6,7 +6,11 @@ publish = false [dependencies] aho-corasick = { workspace = true } +anyhow = { workspace = true } +bytes = { workspace = true } +enum-map = { workspace = true } once_cell = "1.19.0" +rustic_core = { workspace = true } tempfile = { workspace = true } [lints] diff --git a/crates/testing/src/backend.rs b/crates/testing/src/backend.rs new file mode 100644 index 00000000..1c7ca4d1 --- /dev/null +++ b/crates/testing/src/backend.rs @@ -0,0 +1,81 @@ +/// In-memory backend to be used for testing +pub mod in_memory_backend { + use std::{collections::BTreeMap, sync::RwLock}; + + use anyhow::{bail, Result}; + use bytes::Bytes; + use enum_map::EnumMap; + + use rustic_core::{FileType, Id, ReadBackend, WriteBackend}; + + #[derive(Debug)] + /// In-Memory backend to be used for testing + pub struct InMemoryBackend(RwLock>>); + + impl InMemoryBackend { + /// Create a new (empty) `InMemoryBackend` + #[must_use] + pub fn new() -> Self { + Self(RwLock::new(EnumMap::from_fn(|_| BTreeMap::new()))) + } + } + + impl Default for InMemoryBackend { + fn default() -> Self { + Self::new() + } + } + + impl ReadBackend for InMemoryBackend { + fn location(&self) -> String { + "test".to_string() + } + + fn list_with_size(&self, tpe: FileType) -> Result> { + Ok(self.0.read().unwrap()[tpe] + .iter() + .map(|(id, byte)| { + ( + *id, + u32::try_from(byte.len()).expect("byte length is too large"), + ) + }) + .collect()) + } + + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + Ok(self.0.read().unwrap()[tpe][id].clone()) + } + + fn read_partial( + &self, + tpe: FileType, + id: &Id, + _cacheable: bool, + offset: u32, + length: u32, + ) -> Result { + Ok(self.0.read().unwrap()[tpe][id].slice(offset as usize..(offset + length) as usize)) + } + } + + impl WriteBackend for InMemoryBackend { + fn create(&self) -> Result<()> { + Ok(()) + } + + fn write_bytes(&self, tpe: FileType, id: &Id, _cacheable: bool, buf: Bytes) -> Result<()> { + if self.0.write().unwrap()[tpe].insert(*id, buf).is_some() { + bail!("id {id} already exists"); + } + Ok(()) + } + + fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> { + if self.0.write().unwrap()[tpe].remove(id).is_none() { + bail!("id {id} doesn't exists"); + } + Ok(()) + } + } +} diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs index 7a04e793..741f75dc 100644 --- a/crates/testing/src/lib.rs +++ b/crates/testing/src/lib.rs @@ -1,5 +1,8 @@ //! Testing utilities for the `rustic` ecosystem. +/// Backends to be used solely for testing. +pub mod backend; + use aho_corasick::{AhoCorasick, PatternID}; use std::{error::Error, ffi::OsStr}; use tempfile::NamedTempFile;