test: add backup integration tests using snapshots (#175)

This PR adds some integration tests (mostly inspired by restic's
integration tests). We use `insta` for "snapshotting" internal state and
then compare against it depending on which operating system we are on.

---------

Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com>
Co-authored-by: simonsan <14062932+simonsan@users.noreply.github.com>

.github/workflows/ci-heavy.yml

@@ -83,7 +83,7 @@ jobs:
           toolchain: ${{ matrix.rust }}
       - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2
       - name: Run Cargo Test
-        run: cargo +${{ matrix.rust }} test -r --all-targets --all-features --workspace --examples
+        run: cargo +${{ matrix.rust }} test --all-targets --all-features --workspace --examples
         id: run_tests
         env:
           INSTA_UPDATE: new

.github/workflows/ci.yml

@@ -4,6 +4,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  CI: true
+
 on:
   pull_request:
 
@@ -40,6 +43,8 @@ jobs:
     name: Test
     runs-on: ${{ matrix.job.os }}
     strategy:
+      # Don't fail fast, so we can actually see all the results
+      fail-fast: false
       matrix:
         rust: [stable]
         job:
@@ -64,7 +69,16 @@ jobs:
           toolchain: ${{ matrix.rust }}
       - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2
       - name: Run Cargo Test
-        run: cargo +${{ matrix.rust }} test -r --all-targets --all-features --workspace --examples
+        run: cargo +${{ matrix.rust }} test --all-targets --all-features --workspace --examples
+        id: run_tests
+        env:
+          INSTA_UPDATE: new
+      - name: Upload snapshots of failed tests
+        if: ${{ failure() && steps.run_tests.outcome == 'failure' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: failed-snapshots-${{ matrix.job.os }}
+          path: "**/snapshots/*.snap.new"
 
   docs:
     name: Build docs

.gitignore

@@ -15,3 +15,6 @@ coverage/*.info
 
 # local repo config
 .cargo/config.toml
+
+# Generated by Tests
+crates/core/tests/generated/

Cargo.toml

@@ -13,8 +13,12 @@ rust-version = "1.72.1"
 
 [workspace.dependencies]
 aho-corasick = "1.1.2"
+anyhow = "1.0.81"
+bytes = "1.5.0"
+enum-map = "2.7.3"
 rustic_backend = { path = "crates/backend" }
 rustic_core = { path = "crates/core" }
+rustic_testing = { path = "crates/testing" }
 simplelog = "0.12.2"
 
 # dev-dependencies

crates/core/Cargo.toml

@@ -102,11 +102,11 @@ futures = { version = "0.3", optional = true }
 runtime-format = "0.1.3"
 
 # other dependencies
-anyhow = "1.0.81"
-bytes = "1.5.0"
+anyhow = { workspace = true }
+bytes = { workspace = true }
 bytesize = "1.3.0"
 chrono = { version = "0.4.35", default-features = false, features = ["clock", "serde"] }
-enum-map = "2.7.3"
+enum-map = { workspace = true }
 enum-map-derive = "0.17.0"
 enumset = { version = "1.1.3", features = ["serde"] }
 gethostname = "0.4.3"
@@ -132,6 +132,8 @@ xattr = "1"
 
 [dev-dependencies]
 expect-test = "1.4.1"
+flate2 = "1.0.28"
+insta = { version = "1.36.1", features = ["redactions", "ron"] }
 mockall = "0.12.1"
 pretty_assertions = "1.4.0"
 quickcheck = "1.0.3"
@@ -140,8 +142,10 @@ rstest = { workspace = true }
 rustdoc-json = "0.8.9"
 # We need to have rustic_backend here, because the doc-tests in lib.rs of rustic_core
 rustic_backend = { workspace = true }
+rustic_testing = { workspace = true }
 rustup-toolchain = "0.1.6"
 simplelog = "0.12.2"
+tar = "0.4.40"
 tempfile = { workspace = true }
 
 [lints]

crates/core/src/backend.rs

@@ -13,9 +13,12 @@ use std::{io::Read, ops::Deref, path::PathBuf, sync::Arc};
 
 use anyhow::Result;
 use bytes::Bytes;
+use enum_map::Enum;
 use log::trace;
+
 #[cfg(test)]
-use mockall::*;
+use mockall::mock;
+
 use serde_derive::{Deserialize, Serialize};
 
 use crate::{
@@ -34,7 +37,7 @@ pub const ALL_FILE_TYPES: [FileType; 4] = [
 ];
 
 /// Type for describing the kind of a file that can occur.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Enum)]
 pub enum FileType {
     /// Config file
     #[serde(rename = "config")]
@@ -295,7 +298,9 @@ pub trait WriteBackend: ReadBackend {
     /// # Returns
     ///
     /// The result of the creation.
-    fn create(&self) -> Result<()>;
+    fn create(&self) -> Result<()> {
+        Ok(())
+    }
 
     /// Writes bytes to the given file.
     ///

crates/core/src/commands/merge.rs

@@ -39,7 +39,12 @@ pub(crate) fn merge_snapshots<P: ProgressBars, S: IndexedTree>(
 ) -> RusticResult<SnapshotFile> {
     let now = Local::now();
 
-    let paths = PathList::from_strings(snapshots.iter().flat_map(|snap| snap.paths.iter())).merge();
+    let paths = snapshots
+        .iter()
+        .flat_map(|snap| snap.paths.iter())
+        .collect::<PathList>()
+        .merge();
+
     snap.paths.set_paths(&paths.paths())?;
 
     // set snapshot time to time of latest snapshot to be merged

crates/core/src/lib.rs

@@ -45,10 +45,10 @@ implement [`serde::Serialize`] and [`serde::Deserialize`].
 
     let config_opts = ConfigOptions::default();
 
-    let _repo = Repository::new(&repo_opts, backends.clone()).unwrap().init(&key_opts, &config_opts).unwrap();
+    let _repo = Repository::new(&repo_opts, &backends.clone()).unwrap().init(&key_opts, &config_opts).unwrap();
 
     // We could have used _repo directly, but open the repository again to show how to open it...
-    let repo = Repository::new(&repo_opts, backends).unwrap().open().unwrap();
+    let repo = Repository::new(&repo_opts, &backends).unwrap().open().unwrap();
 
     // Get all snapshots from the repository
     let snaps = repo.get_all_snapshots().unwrap();

crates/core/src/repofile/snapshotfile.rs

@@ -1100,25 +1100,31 @@ impl Display for PathList {
     }
 }
 
-impl PathList {
-    /// Create a `PathList` from `String`s.
-    ///
-    /// # Arguments
-    ///
-    /// * `source` - The `String`s to use
-    pub fn from_strings<I>(source: I) -> Self
-    where
-        I: IntoIterator,
-        I::Item: AsRef<str>,
-    {
-        Self(
-            source
-                .into_iter()
-                .map(|source| PathBuf::from(source.as_ref()))
-                .collect(),
-        )
+impl FromIterator<PathBuf> for PathList {
+    fn from_iter<I: IntoIterator<Item = PathBuf>>(iter: I) -> Self {
+        Self(iter.into_iter().collect())
     }
+}
 
+impl<'a> FromIterator<&'a String> for PathList {
+    fn from_iter<I: IntoIterator<Item = &'a String>>(iter: I) -> Self {
+        Self(iter.into_iter().map(PathBuf::from).collect())
+    }
+}
+
+impl FromIterator<String> for PathList {
+    fn from_iter<I: IntoIterator<Item = String>>(iter: I) -> Self {
+        Self(iter.into_iter().map(PathBuf::from).collect())
+    }
+}
+
+impl<'a> FromIterator<&'a str> for PathList {
+    fn from_iter<I: IntoIterator<Item = &'a str>>(iter: I) -> Self {
+        Self(iter.into_iter().map(PathBuf::from).collect())
+    }
+}
+
+impl PathList {
     /// Create a `PathList` by parsing a Strings containing paths separated by whitspaces.
     ///
     /// # Arguments
@@ -1132,7 +1138,7 @@ impl PathList {
     /// [`SnapshotFileErrorKind::FromSplitError`]: crate::error::SnapshotFileErrorKind::FromSplitError
     pub fn from_string(sources: &str) -> RusticResult<Self> {
         let sources = split(sources).map_err(SnapshotFileErrorKind::FromSplitError)?;
-        Ok(Self::from_strings(sources))
+        Ok(Self::from_iter(sources))
     }
 
     /// Number of paths in the `PathList`.

crates/core/src/repository.rs

@@ -1452,6 +1452,7 @@ impl<P, S: IndexedTree> Repository<P, S> {
     /// # Arguments
     ///
     /// * `id` - The `Id` of the tree
+    // TODO!: This ID should be a tree ID, we should refactor it to wrap it in a TreeId type
     ///
     /// # Errors
     ///
@@ -1475,6 +1476,7 @@ impl<P, S: IndexedTree> Repository<P, S> {
     /// # Arguments
     ///
     /// * `root_tree` - The `Id` of the root tree
+    // TODO!: This ID should be a tree ID, we should refactor it to wrap it in a TreeId type
     /// * `path` - The path
     ///
     /// # Errors

crates/core/tests/integration.rs

@@ -0,0 +1,338 @@
+//! Integration tests for the core library
+//!
+//! # How to update snapshots
+//!
+//! The CI pipeline is configured to run the tests with the `INSTA_UPDATE` environment variable set to `new`.
+//! This means, it uploads the failed tests snapshots to the artifacts and you can download them and use them to update the snapshots.
+//!
+//! To update the snapshots, you download the artifacts and copy the files to the `tests/snapshots` directory.
+//! Then you run `cargo insta review` to review the changes and accept them.
+//!
+//! # Redactions
+//!
+//! We use the `insta` crate to compare the actual output of the tests with the expected output.
+//! Some data in the output changes every test run, we use insta's redactions to replace the actual values with placeholders.
+//! We define the redactions inside `Settings` in the fixtures and bind them to the test functions. You can read more about
+//! it [here](https://docs.rs/insta/latest/insta/struct.Settings.html).
+//!
+//! # Fixtures and Dependency Injection
+//!
+//! We use the `rstest` crate to define fixtures and dependency injection.
+//! This allows us to define a set of fixtures that are used in multiple tests.
+//! The fixtures are defined as functions with the `#[fixture]` attribute.
+//! The tests that use the fixtures are defined as functions with the `#[rstest]` attribute.
+//! The fixtures are passed as arguments to the test functions.
+
+use anyhow::Result;
+use flate2::read::GzDecoder;
+use insta::internals::{Content, ContentPath};
+use insta::{assert_ron_snapshot, Settings};
+use pretty_assertions::assert_eq;
+use rstest::fixture;
+use rstest::rstest;
+use rustic_core::{
+    repofile::SnapshotFile, BackupOptions, ConfigOptions, KeyOptions, NoProgressBars, OpenStatus,
+    PathList, Repository, RepositoryBackends, RepositoryOptions,
+};
+use serde_derive::Serialize;
+
+use rustic_testing::backend::in_memory_backend::InMemoryBackend;
+
+use std::{
+    env,
+    fs::File,
+    path::{Path, PathBuf},
+    str::FromStr,
+    sync::Arc,
+};
+// uncomment for logging output
+// use simplelog::{Config, SimpleLogger};
+use tar::Archive;
+use tempfile::{tempdir, TempDir};
+
+type RepoOpen = Repository<NoProgressBars, OpenStatus>;
+
+#[derive(Debug)]
+struct TestSource(TempDir);
+
+impl TestSource {
+    fn new(tmp: TempDir) -> Self {
+        Self(tmp)
+    }
+
+    fn path_list(&self) -> PathList {
+        PathList::from_iter(Some(self.0.path().to_path_buf()))
+    }
+}
+
+#[fixture]
+fn set_up_repo() -> Result<RepoOpen> {
+    let be = InMemoryBackend::new();
+    let be = RepositoryBackends::new(Arc::new(be), None);
+    let options = RepositoryOptions::default().password("test");
+    let repo = Repository::new(&options, &be)?;
+    let key_opts = KeyOptions::default();
+    let config_opts = &ConfigOptions::default();
+    let repo = repo.init(&key_opts, config_opts)?;
+    Ok(repo)
+}
+
+// helper func to redact options, but still keep information about some/none
+#[allow(clippy::needless_pass_by_value)] // we need exactly that function signature
+fn handle_option(val: Content, _: ContentPath<'_>) -> String {
+    if val.is_nil() {
+        "[none]".to_string()
+    } else {
+        "[some]".to_string()
+    }
+}
+
+#[fixture]
+fn insta_summary_redaction() -> Settings {
+    let mut settings = insta::Settings::clone_current();
+
+    settings.add_redaction(".tree", "[tree_id]");
+    settings.add_dynamic_redaction(".program_version", |val, _| {
+        val.resolve_inner()
+            .as_str()
+            .map_or("[program_version]".to_string(), |v| {
+                v.replace(env!("CARGO_PKG_VERSION"), "[rustic_core_version]")
+            })
+    });
+    settings.add_redaction(".time", "[time]");
+    settings.add_dynamic_redaction(".parent", handle_option);
+    settings.add_redaction(".tags", "[tags]");
+    settings.add_redaction(".id", "[id]");
+    settings.add_redaction(".summary.backup_start", "[backup_start]");
+    settings.add_redaction(".summary.backup_end", "[backup_end]");
+    settings.add_redaction(".summary.backup_duration", "[backup_duration]");
+    settings.add_redaction(".summary.total_duration", "[total_duration]");
+    settings.add_redaction(".summary.data_added", "[data_added]");
+    settings.add_redaction(".summary.data_added_packed", "[data_added_packed]");
+    settings.add_redaction(
+        ".summary.total_dirsize_processed",
+        "[total_dirsize_processed]",
+    );
+    settings.add_redaction(
+        ".summary.data_added_trees_packed",
+        "[data_added_trees_packed]",
+    );
+    settings.add_redaction(".summary.data_added_trees", "[data_added_trees]");
+
+    settings
+}
+
+#[fixture]
+fn insta_tree_redaction() -> Settings {
+    let mut settings = insta::Settings::clone_current();
+
+    settings.add_redaction(".nodes[].inode", "[inode]");
+    settings.add_redaction(".nodes[].device_id", "[device_id]");
+    settings.add_redaction(".nodes[].uid", "[uid]");
+    settings.add_redaction(".nodes[].user", "[user]");
+    settings.add_redaction(".nodes[].gid", "[gid]");
+    settings.add_redaction(".nodes[].group", "[group]");
+    settings.add_dynamic_redaction(".nodes[].mode", handle_option);
+    settings.add_dynamic_redaction(".nodes[].mtime", handle_option);
+    settings.add_dynamic_redaction(".nodes[].atime", handle_option);
+    settings.add_dynamic_redaction(".nodes[].ctime", handle_option);
+
+    settings
+}
+
+#[fixture]
+fn tar_gz_testdata() -> Result<TestSource> {
+    let dir = tempdir()?;
+    let path = Path::new("tests/fixtures/backup-data.tar.gz").canonicalize()?;
+    let tar_gz = File::open(path)?;
+    let tar = GzDecoder::new(tar_gz);
+    let mut archive = Archive::new(tar);
+    archive.set_preserve_permissions(true);
+    archive.set_preserve_mtime(true);
+    archive.unpack(&dir)?;
+    Ok(TestSource::new(dir))
+}
+
+// Parts of the snapshot summary we want to test against references
+//
+// # Note
+//
+// We use a struct to avoid having to escape the field names in the snapshot
+// we use insta redactions to replace the actual values with placeholders in case
+// there are changes in the actual values
+// Readme: https://insta.rs/docs/redactions/
+#[derive(Serialize)]
+struct TestSummary<'a>(&'a SnapshotFile);
+
+#[rstest]
+fn test_backup_with_tar_gz_passes(
+    tar_gz_testdata: Result<TestSource>,
+    set_up_repo: Result<RepoOpen>,
+    insta_summary_redaction: Settings,
+    insta_tree_redaction: Settings,
+) -> Result<()> {
+    // uncomment for logging output
+    // SimpleLogger::init(log::LevelFilter::Debug, Config::default())?;
+
+    // Fixtures
+    let (source, repo) = (tar_gz_testdata?, set_up_repo?.to_indexed_ids()?);
+
+    let paths = &source.path_list();
+
+    // we use as_path to not depend on the actual tempdir
+    let opts = BackupOptions::default().as_path(PathBuf::from_str("test")?);
+
+    // first backup
+    let first_snapshot = repo.backup(&opts, paths, SnapshotFile::default())?;
+
+    // We can also bind to scope ( https://docs.rs/insta/latest/insta/struct.Settings.html#method.bind_to_scope )
+    // But I think that can get messy with a lot of tests, also checking which settings are currently applied
+    // will be probably harder
+    insta_summary_redaction.bind(|| {
+        #[cfg(windows)]
+        assert_ron_snapshot!(
+            "backup-tar-summary-first-windows",
+            TestSummary(&first_snapshot)
+        );
+        #[cfg(not(windows))]
+        assert_ron_snapshot!("backup-tar-summary-first-nix", TestSummary(&first_snapshot));
+    });
+
+    assert_eq!(first_snapshot.parent, None);
+
+    // tree of first backup
+    // re-read index
+    let repo = repo.to_indexed_ids()?;
+    let tree = repo.node_from_path(first_snapshot.tree, Path::new("test/0/tests"))?;
+    let tree: rustic_core::repofile::Tree = repo.get_tree(&tree.subtree.expect("Sub tree"))?;
+
+    insta_tree_redaction.bind(|| {
+        #[cfg(windows)]
+        assert_ron_snapshot!("backup-tar-tree-windows", tree);
+        #[cfg(not(windows))]
+        assert_ron_snapshot!("backup-tar-tree-nix", tree);
+    });
+
+    // get all snapshots and check them
+    let all_snapshots = repo.get_all_snapshots()?;
+    assert_eq!(vec![first_snapshot.clone()], all_snapshots);
+    // save list of pack files
+    let packs1: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect();
+
+    // re-read index
+    let repo = repo.to_indexed_ids()?;
+    // second backup
+    let second_snapshot = repo.backup(&opts, paths, SnapshotFile::default())?;
+
+    insta_summary_redaction.bind(|| {
+        #[cfg(windows)]
+        assert_ron_snapshot!(
+            "backup-tar-summary-second-windows",
+            TestSummary(&second_snapshot)
+        );
+        #[cfg(not(windows))]
+        assert_ron_snapshot!(
+            "backup-tar-summary-second-nix",
+            TestSummary(&second_snapshot)
+        );
+    });
+
+    assert_eq!(second_snapshot.parent, Some(first_snapshot.id));
+    assert_eq!(first_snapshot.tree, second_snapshot.tree);
+
+    // get all snapshots and check them
+    let mut all_snapshots = repo.get_all_snapshots()?;
+    all_snapshots.sort_unstable();
+    assert_eq!(vec![first_snapshot, second_snapshot], all_snapshots);
+
+    // pack files should be unchanged
+    let packs2: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect();
+    assert_eq!(packs1, packs2);
+    Ok(())
+}
+
+#[rstest]
+fn test_backup_dry_run_with_tar_gz_passes(
+    tar_gz_testdata: Result<TestSource>,
+    set_up_repo: Result<RepoOpen>,
+    insta_summary_redaction: Settings,
+    insta_tree_redaction: Settings,
+) -> Result<()> {
+    // Fixtures
+    let (source, repo) = (tar_gz_testdata?, set_up_repo?.to_indexed_ids()?);
+
+    let paths = &source.path_list();
+
+    // we use as_path to not depend on the actual tempdir
+    let opts = BackupOptions::default()
+        .as_path(PathBuf::from_str("test")?)
+        .dry_run(true);
+
+    // dry-run backup
+    let snap_dry_run = repo.backup(&opts, paths, SnapshotFile::default())?;
+
+    insta_summary_redaction.bind(|| {
+        #[cfg(windows)]
+        assert_ron_snapshot!(
+            "dryrun-tar-summary-first-windows",
+            TestSummary(&snap_dry_run)
+        );
+        #[cfg(not(windows))]
+        assert_ron_snapshot!("dryrun-tar-summary-first-nix", TestSummary(&snap_dry_run));
+    });
+
+    // check that repo is still empty
+    let snaps = repo.get_all_snapshots()?;
+    assert_eq!(snaps.len(), 0);
+    assert_eq!(repo.list(rustic_core::FileType::Pack)?.count(), 0);
+    assert_eq!(repo.list(rustic_core::FileType::Index)?.count(), 0);
+
+    // first real backup
+    let opts = opts.dry_run(false);
+    let first_snapshot = repo.backup(&opts, paths, SnapshotFile::default())?;
+    assert_eq!(snap_dry_run.tree, first_snapshot.tree);
+    let packs: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect();
+
+    // tree of first backup
+    // re-read index
+    let repo = repo.to_indexed_ids()?;
+    let tree = repo.node_from_path(first_snapshot.tree, Path::new("test/0/tests"))?;
+    let tree = repo.get_tree(&tree.subtree.expect("Sub tree"))?;
+
+    insta_tree_redaction.bind(|| {
+        #[cfg(windows)]
+        assert_ron_snapshot!("dryrun-tar-tree-windows", tree);
+        #[cfg(not(windows))]
+        assert_ron_snapshot!("dryrun-tar-tree-nix", tree);
+    });
+
+    // re-read index
+    let repo = repo.to_indexed_ids()?;
+    // second dry-run backup
+    let opts = opts.dry_run(true);
+    let snap_dry_run = repo.backup(&opts, paths, SnapshotFile::default())?;
+
+    insta_summary_redaction.bind(|| {
+        #[cfg(windows)]
+        assert_ron_snapshot!(
+            "dryrun-tar-summary-second-windows",
+            TestSummary(&snap_dry_run)
+        );
+        #[cfg(not(windows))]
+        assert_ron_snapshot!("dryrun-tar-summary-second-nix", TestSummary(&snap_dry_run));
+    });
+
+    // check that no data has been added
+    let snaps = repo.get_all_snapshots()?;
+    assert_eq!(snaps, vec![first_snapshot]);
+    let packs_dry_run: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect();
+    assert_eq!(packs_dry_run, packs);
+
+    // re-read index
+    let repo = repo.to_indexed_ids()?;
+    // second real backup
+    let opts = opts.dry_run(false);
+    let second_snapshot = repo.backup(&opts, paths, SnapshotFile::default())?;
+    assert_eq!(snap_dry_run.tree, second_snapshot.tree);
+    Ok(())
+}

crates/core/tests/snapshots/integration__backup-tar-summary-first-nix.snap

@@ -0,0 +1,43 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSummary(&first_snapshot)
+---
+TestSummary(SnapshotFile(
+  time: "[time]",
+  program_version: "rustic [rustic_core_version]",
+  tree: "[tree_id]",
+  paths: StringList([
+    "test",
+  ]),
+  hostname: "",
+  username: "",
+  uid: 0,
+  gid: 0,
+  tags: "[tags]",
+  summary: Some(SnapshotSummary(
+    files_new: 73,
+    files_changed: 0,
+    files_unmodified: 0,
+    total_files_processed: 73,
+    total_bytes_processed: 1125674,
+    dirs_new: 6,
+    dirs_changed: 0,
+    dirs_unmodified: 0,
+    total_dirs_processed: 6,
+    total_dirsize_processed: "[total_dirsize_processed]",
+    data_blobs: 70,
+    tree_blobs: 6,
+    data_added: "[data_added]",
+    data_added_packed: "[data_added_packed]",
+    data_added_files: 1125653,
+    data_added_files_packed: 78740,
+    data_added_trees: "[data_added_trees]",
+    data_added_trees_packed: "[data_added_trees_packed]",
+    command: "",
+    backup_start: "[backup_start]",
+    backup_end: "[backup_end]",
+    backup_duration: "[backup_duration]",
+    total_duration: "[total_duration]",
+  )),
+  id: "[id]",
+))

crates/core/tests/snapshots/integration__backup-tar-summary-first-windows.snap

@@ -0,0 +1,43 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSummary(&first_snapshot)
+---
+TestSummary(SnapshotFile(
+  time: "[time]",
+  program_version: "rustic [rustic_core_version]",
+  tree: "[tree_id]",
+  paths: StringList([
+    "test",
+  ]),
+  hostname: "",
+  username: "",
+  uid: 0,
+  gid: 0,
+  tags: "[tags]",
+  summary: Some(SnapshotSummary(
+    files_new: 73,
+    files_changed: 0,
+    files_unmodified: 0,
+    total_files_processed: 73,
+    total_bytes_processed: 1125674,
+    dirs_new: 6,
+    dirs_changed: 0,
+    dirs_unmodified: 0,
+    total_dirs_processed: 6,
+    total_dirsize_processed: "[total_dirsize_processed]",
+    data_blobs: 70,
+    tree_blobs: 6,
+    data_added: "[data_added]",
+    data_added_packed: "[data_added_packed]",
+    data_added_files: 1125653,
+    data_added_files_packed: 78740,
+    data_added_trees: "[data_added_trees]",
+    data_added_trees_packed: "[data_added_trees_packed]",
+    command: "",
+    backup_start: "[backup_start]",
+    backup_end: "[backup_end]",
+    backup_duration: "[backup_duration]",
+    total_duration: "[total_duration]",
+  )),
+  id: "[id]",
+))

crates/core/tests/snapshots/integration__backup-tar-summary-second-nix.snap

@@ -0,0 +1,44 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSummary(&second_snapshot)
+---
+TestSummary(SnapshotFile(
+  time: "[time]",
+  program_version: "rustic [rustic_core_version]",
+  parent: "[some]",
+  tree: "[tree_id]",
+  paths: StringList([
+    "test",
+  ]),
+  hostname: "",
+  username: "",
+  uid: 0,
+  gid: 0,
+  tags: "[tags]",
+  summary: Some(SnapshotSummary(
+    files_new: 0,
+    files_changed: 0,
+    files_unmodified: 73,
+    total_files_processed: 73,
+    total_bytes_processed: 1125682,
+    dirs_new: 0,
+    dirs_changed: 0,
+    dirs_unmodified: 6,
+    total_dirs_processed: 6,
+    total_dirsize_processed: "[total_dirsize_processed]",
+    data_blobs: 0,
+    tree_blobs: 0,
+    data_added: "[data_added]",
+    data_added_packed: "[data_added_packed]",
+    data_added_files: 0,
+    data_added_files_packed: 0,
+    data_added_trees: "[data_added_trees]",
+    data_added_trees_packed: "[data_added_trees_packed]",
+    command: "",
+    backup_start: "[backup_start]",
+    backup_end: "[backup_end]",
+    backup_duration: "[backup_duration]",
+    total_duration: "[total_duration]",
+  )),
+  id: "[id]",
+))

crates/core/tests/snapshots/integration__backup-tar-summary-second-windows.snap

@@ -0,0 +1,44 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSummary(&second_snapshot)
+---
+TestSummary(SnapshotFile(
+  time: "[time]",
+  program_version: "rustic [rustic_core_version]",
+  parent: "[some]",
+  tree: "[tree_id]",
+  paths: StringList([
+    "test",
+  ]),
+  hostname: "",
+  username: "",
+  uid: 0,
+  gid: 0,
+  tags: "[tags]",
+  summary: Some(SnapshotSummary(
+    files_new: 0,
+    files_changed: 0,
+    files_unmodified: 73,
+    total_files_processed: 73,
+    total_bytes_processed: 1125674,
+    dirs_new: 0,
+    dirs_changed: 0,
+    dirs_unmodified: 6,
+    total_dirs_processed: 6,
+    total_dirsize_processed: "[total_dirsize_processed]",
+    data_blobs: 0,
+    tree_blobs: 0,
+    data_added: "[data_added]",
+    data_added_packed: "[data_added_packed]",
+    data_added_files: 0,
+    data_added_files_packed: 0,
+    data_added_trees: "[data_added_trees]",
+    data_added_trees_packed: "[data_added_trees_packed]",
+    command: "",
+    backup_start: "[backup_start]",
+    backup_end: "[backup_end]",
+    backup_duration: "[backup_duration]",
+    total_duration: "[total_duration]",
+  )),
+  id: "[id]",
+))

crates/core/tests/snapshots/integration__backup-tar-tree-nix.snap

@@ -0,0 +1,80 @@
+---
+source: crates/core/tests/integration.rs
+expression: tree
+---
+Tree(
+  nodes: [
+    {
+      "name": "empty-file",
+      "type": "file",
+      "mode": "[some]",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "uid": "[uid]",
+      "gid": "[gid]",
+      "user": "[user]",
+      "group": "[group]",
+      "inode": "[inode]",
+      "device_id": "[device_id]",
+      "links": 1,
+      "content": Some([]),
+    },
+    {
+      "name": "testfile",
+      "type": "file",
+      "mode": "[some]",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "uid": "[uid]",
+      "gid": "[gid]",
+      "user": "[user]",
+      "group": "[group]",
+      "inode": "[inode]",
+      "device_id": "[device_id]",
+      "size": 21,
+      "links": 2,
+      "content": Some([
+        Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"),
+      ]),
+    },
+    {
+      "name": "testfile-hardlink",
+      "type": "file",
+      "mode": "[some]",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "uid": "[uid]",
+      "gid": "[gid]",
+      "user": "[user]",
+      "group": "[group]",
+      "inode": "[inode]",
+      "device_id": "[device_id]",
+      "size": 21,
+      "links": 2,
+      "content": Some([
+        Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"),
+      ]),
+    },
+    {
+      "name": "testfile-symlink",
+      "type": "symlink",
+      "linktarget": "testfile",
+      "mode": "[some]",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "uid": "[uid]",
+      "gid": "[gid]",
+      "user": "[user]",
+      "group": "[group]",
+      "inode": "[inode]",
+      "device_id": "[device_id]",
+      "size": 8,
+      "links": 1,
+      "content": None,
+    },
+  ],
+)

crates/core/tests/snapshots/integration__backup-tar-tree-windows.snap

@@ -0,0 +1,47 @@
+---
+source: crates/core/tests/integration.rs
+expression: tree
+---
+Tree(
+  nodes: [
+    {
+      "name": "empty-file",
+      "type": "file",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "content": Some([]),
+    },
+    {
+      "name": "testfile",
+      "type": "file",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "size": 21,
+      "content": Some([
+        Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"),
+      ]),
+    },
+    {
+      "name": "testfile-hardlink",
+      "type": "file",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "size": 21,
+      "content": Some([
+        Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"),
+      ]),
+    },
+    {
+      "name": "testfile-symlink",
+      "type": "symlink",
+      "linktarget": "testfile",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "content": None,
+    },
+  ],
+)

crates/core/tests/snapshots/integration__dryrun-tar-summary-first-nix.snap

@@ -0,0 +1,42 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSummary(&snap_dry_run)
+---
+TestSummary(SnapshotFile(
+  time: "[time]",
+  program_version: "rustic [rustic_core_version]",
+  tree: "[tree_id]",
+  paths: StringList([
+    "test",
+  ]),
+  hostname: "",
+  username: "",
+  uid: 0,
+  gid: 0,
+  tags: "[tags]",
+  summary: Some(SnapshotSummary(
+    files_new: 73,
+    files_changed: 0,
+    files_unmodified: 0,
+    total_files_processed: 73,
+    total_bytes_processed: 1125674,
+    dirs_new: 6,
+    dirs_changed: 0,
+    dirs_unmodified: 0,
+    total_dirs_processed: 6,
+    total_dirsize_processed: "[total_dirsize_processed]",
+    data_blobs: 70,
+    tree_blobs: 6,
+    data_added: "[data_added]",
+    data_added_packed: "[data_added_packed]",
+    data_added_files: 1125653,
+    data_added_files_packed: 78740,
+    data_added_trees: "[data_added_trees]",
+    data_added_trees_packed: "[data_added_trees_packed]",
+    command: "",
+    backup_start: "[backup_start]",
+    backup_end: "[backup_end]",
+    backup_duration: "[backup_duration]",
+    total_duration: "[total_duration]",
+  )),
+))

crates/core/tests/snapshots/integration__dryrun-tar-summary-first-windows.snap

@@ -0,0 +1,42 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSummary(&snap_dry_run)
+---
+TestSummary(SnapshotFile(
+  time: "[time]",
+  program_version: "rustic [rustic_core_version]",
+  tree: "[tree_id]",
+  paths: StringList([
+    "test",
+  ]),
+  hostname: "",
+  username: "",
+  uid: 0,
+  gid: 0,
+  tags: "[tags]",
+  summary: Some(SnapshotSummary(
+    files_new: 73,
+    files_changed: 0,
+    files_unmodified: 0,
+    total_files_processed: 73,
+    total_bytes_processed: 1125674,
+    dirs_new: 6,
+    dirs_changed: 0,
+    dirs_unmodified: 0,
+    total_dirs_processed: 6,
+    total_dirsize_processed: "[total_dirsize_processed]",
+    data_blobs: 70,
+    tree_blobs: 6,
+    data_added: "[data_added]",
+    data_added_packed: "[data_added_packed]",
+    data_added_files: 1125653,
+    data_added_files_packed: 78740,
+    data_added_trees: "[data_added_trees]",
+    data_added_trees_packed: "[data_added_trees_packed]",
+    command: "",
+    backup_start: "[backup_start]",
+    backup_end: "[backup_end]",
+    backup_duration: "[backup_duration]",
+    total_duration: "[total_duration]",
+  )),
+))

crates/core/tests/snapshots/integration__dryrun-tar-summary-second-nix.snap

@@ -0,0 +1,43 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSummary(&snap_dry_run)
+---
+TestSummary(SnapshotFile(
+  time: "[time]",
+  program_version: "rustic [rustic_core_version]",
+  parent: "[some]",
+  tree: "[tree_id]",
+  paths: StringList([
+    "test",
+  ]),
+  hostname: "",
+  username: "",
+  uid: 0,
+  gid: 0,
+  tags: "[tags]",
+  summary: Some(SnapshotSummary(
+    files_new: 0,
+    files_changed: 0,
+    files_unmodified: 73,
+    total_files_processed: 73,
+    total_bytes_processed: 1125682,
+    dirs_new: 0,
+    dirs_changed: 0,
+    dirs_unmodified: 6,
+    total_dirs_processed: 6,
+    total_dirsize_processed: "[total_dirsize_processed]",
+    data_blobs: 0,
+    tree_blobs: 0,
+    data_added: "[data_added]",
+    data_added_packed: "[data_added_packed]",
+    data_added_files: 0,
+    data_added_files_packed: 0,
+    data_added_trees: "[data_added_trees]",
+    data_added_trees_packed: "[data_added_trees_packed]",
+    command: "",
+    backup_start: "[backup_start]",
+    backup_end: "[backup_end]",
+    backup_duration: "[backup_duration]",
+    total_duration: "[total_duration]",
+  )),
+))

crates/core/tests/snapshots/integration__dryrun-tar-summary-second-windows.snap

@@ -0,0 +1,43 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSummary(&snap_dry_run)
+---
+TestSummary(SnapshotFile(
+  time: "[time]",
+  program_version: "rustic [rustic_core_version]",
+  parent: "[some]",
+  tree: "[tree_id]",
+  paths: StringList([
+    "test",
+  ]),
+  hostname: "",
+  username: "",
+  uid: 0,
+  gid: 0,
+  tags: "[tags]",
+  summary: Some(SnapshotSummary(
+    files_new: 0,
+    files_changed: 0,
+    files_unmodified: 73,
+    total_files_processed: 73,
+    total_bytes_processed: 1125674,
+    dirs_new: 0,
+    dirs_changed: 0,
+    dirs_unmodified: 6,
+    total_dirs_processed: 6,
+    total_dirsize_processed: "[total_dirsize_processed]",
+    data_blobs: 0,
+    tree_blobs: 0,
+    data_added: "[data_added]",
+    data_added_packed: "[data_added_packed]",
+    data_added_files: 0,
+    data_added_files_packed: 0,
+    data_added_trees: "[data_added_trees]",
+    data_added_trees_packed: "[data_added_trees_packed]",
+    command: "",
+    backup_start: "[backup_start]",
+    backup_end: "[backup_end]",
+    backup_duration: "[backup_duration]",
+    total_duration: "[total_duration]",
+  )),
+))

crates/core/tests/snapshots/integration__dryrun-tar-tree-nix.snap

@@ -0,0 +1,80 @@
+---
+source: crates/core/tests/integration.rs
+expression: tree
+---
+Tree(
+  nodes: [
+    {
+      "name": "empty-file",
+      "type": "file",
+      "mode": "[some]",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "uid": "[uid]",
+      "gid": "[gid]",
+      "user": "[user]",
+      "group": "[group]",
+      "inode": "[inode]",
+      "device_id": "[device_id]",
+      "links": 1,
+      "content": Some([]),
+    },
+    {
+      "name": "testfile",
+      "type": "file",
+      "mode": "[some]",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "uid": "[uid]",
+      "gid": "[gid]",
+      "user": "[user]",
+      "group": "[group]",
+      "inode": "[inode]",
+      "device_id": "[device_id]",
+      "size": 21,
+      "links": 2,
+      "content": Some([
+        Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"),
+      ]),
+    },
+    {
+      "name": "testfile-hardlink",
+      "type": "file",
+      "mode": "[some]",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "uid": "[uid]",
+      "gid": "[gid]",
+      "user": "[user]",
+      "group": "[group]",
+      "inode": "[inode]",
+      "device_id": "[device_id]",
+      "size": 21,
+      "links": 2,
+      "content": Some([
+        Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"),
+      ]),
+    },
+    {
+      "name": "testfile-symlink",
+      "type": "symlink",
+      "linktarget": "testfile",
+      "mode": "[some]",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "uid": "[uid]",
+      "gid": "[gid]",
+      "user": "[user]",
+      "group": "[group]",
+      "inode": "[inode]",
+      "device_id": "[device_id]",
+      "size": 8,
+      "links": 1,
+      "content": None,
+    },
+  ],
+)

crates/core/tests/snapshots/integration__dryrun-tar-tree-windows.snap

@@ -0,0 +1,47 @@
+---
+source: crates/core/tests/integration.rs
+expression: tree
+---
+Tree(
+  nodes: [
+    {
+      "name": "empty-file",
+      "type": "file",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "content": Some([]),
+    },
+    {
+      "name": "testfile",
+      "type": "file",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "size": 21,
+      "content": Some([
+        Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"),
+      ]),
+    },
+    {
+      "name": "testfile-hardlink",
+      "type": "file",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "size": 21,
+      "content": Some([
+        Id("649b8b471e7d7bc175eec758a7006ac693c434c8297c07db15286788c837154a"),
+      ]),
+    },
+    {
+      "name": "testfile-symlink",
+      "type": "symlink",
+      "linktarget": "testfile",
+      "mtime": "[some]",
+      "atime": "[some]",
+      "ctime": "[some]",
+      "content": None,
+    },
+  ],
+)

crates/testing/Cargo.toml

@@ -6,7 +6,11 @@ publish = false
 
 [dependencies]
 aho-corasick = { workspace = true }
+anyhow = { workspace = true }
+bytes = { workspace = true }
+enum-map = { workspace = true }
 once_cell = "1.19.0"
+rustic_core = { workspace = true }
 tempfile = { workspace = true }
 
 [lints]

crates/testing/src/backend.rs

@@ -0,0 +1,81 @@
+/// In-memory backend to be used for testing
+pub mod in_memory_backend {
+    use std::{collections::BTreeMap, sync::RwLock};
+
+    use anyhow::{bail, Result};
+    use bytes::Bytes;
+    use enum_map::EnumMap;
+
+    use rustic_core::{FileType, Id, ReadBackend, WriteBackend};
+
+    #[derive(Debug)]
+    /// In-Memory backend to be used for testing
+    pub struct InMemoryBackend(RwLock<EnumMap<FileType, BTreeMap<Id, Bytes>>>);
+
+    impl InMemoryBackend {
+        /// Create a new (empty) `InMemoryBackend`
+        #[must_use]
+        pub fn new() -> Self {
+            Self(RwLock::new(EnumMap::from_fn(|_| BTreeMap::new())))
+        }
+    }
+
+    impl Default for InMemoryBackend {
+        fn default() -> Self {
+            Self::new()
+        }
+    }
+
+    impl ReadBackend for InMemoryBackend {
+        fn location(&self) -> String {
+            "test".to_string()
+        }
+
+        fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
+            Ok(self.0.read().unwrap()[tpe]
+                .iter()
+                .map(|(id, byte)| {
+                    (
+                        *id,
+                        u32::try_from(byte.len()).expect("byte length is too large"),
+                    )
+                })
+                .collect())
+        }
+
+        fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
+            Ok(self.0.read().unwrap()[tpe][id].clone())
+        }
+
+        fn read_partial(
+            &self,
+            tpe: FileType,
+            id: &Id,
+            _cacheable: bool,
+            offset: u32,
+            length: u32,
+        ) -> Result<Bytes> {
+            Ok(self.0.read().unwrap()[tpe][id].slice(offset as usize..(offset + length) as usize))
+        }
+    }
+
+    impl WriteBackend for InMemoryBackend {
+        fn create(&self) -> Result<()> {
+            Ok(())
+        }
+
+        fn write_bytes(&self, tpe: FileType, id: &Id, _cacheable: bool, buf: Bytes) -> Result<()> {
+            if self.0.write().unwrap()[tpe].insert(*id, buf).is_some() {
+                bail!("id {id} already exists");
+            }
+            Ok(())
+        }
+
+        fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> {
+            if self.0.write().unwrap()[tpe].remove(id).is_none() {
+                bail!("id {id} doesn't exists");
+            }
+            Ok(())
+        }
+    }
+}

crates/testing/src/lib.rs

@@ -1,5 +1,8 @@
 //! Testing utilities for the `rustic` ecosystem.
 
+/// Backends to be used solely for testing.
+pub mod backend;
+
 use aho_corasick::{AhoCorasick, PatternID};
 use std::{error::Error, ffi::OsStr};
 use tempfile::NamedTempFile;