tests: Add integration tests

crates/core/Cargo.toml

@@ -132,6 +132,8 @@ xattr = "1"
 
 [dev-dependencies]
 expect-test = "1.4.1"
+flate2 = "1.0.28"
+insta = "1.36.1"
 pretty_assertions = "1.4.0"
 public-api = "0.33.1"
 quickcheck = "1.0.3"
@@ -142,4 +144,5 @@ rustdoc-json = "0.8.9"
 rustic_backend = { workspace = true }
 rustup-toolchain = "0.1.6"
 simplelog = "0.12.1"
+tar = "0.4.40"
 tempfile = { workspace = true }

crates/core/src/backend.rs

@@ -13,6 +13,7 @@ use std::{io::Read, ops::Deref, path::PathBuf, sync::Arc};
 
 use anyhow::Result;
 use bytes::Bytes;
+use enum_map::Enum;
 use log::trace;
 use serde_derive::{Deserialize, Serialize};
 
@@ -32,7 +33,7 @@ pub const ALL_FILE_TYPES: [FileType; 4] = [
 ];
 
 /// Type for describing the kind of a file that can occur.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Enum)]
 pub enum FileType {
     /// Config file
     #[serde(rename = "config")]
@@ -284,7 +285,9 @@ impl<T: ReadBackend> FindInBackend for T {}
 /// This trait is implemented by all backends that can write data.
 pub trait WriteBackend: ReadBackend {
     /// Creates a new backend.
-    fn create(&self) -> Result<()>;
+    fn create(&self) -> Result<()> {
+        Ok(())
+    }
 
     /// Writes bytes to the given file.
     ///
@@ -475,3 +478,79 @@ impl RepositoryBackends {
         self.repo_hot.clone()
     }
 }
+
+pub mod in_memory_backend {
+    use std::{collections::BTreeMap, sync::RwLock};
+
+    use anyhow::{bail, Result};
+    use bytes::Bytes;
+    use enum_map::EnumMap;
+
+    use super::{ReadBackend, WriteBackend};
+    use crate::{FileType, Id};
+
+    #[derive(Debug)]
+    /// In-Memory backend to be used for testing
+    pub struct InMemoryBackend(RwLock<EnumMap<FileType, BTreeMap<Id, Bytes>>>);
+
+    impl InMemoryBackend {
+        /// Create a new (empty) `InMemoryBackend`
+        pub fn new() -> Self {
+            Self(RwLock::new(EnumMap::from_fn(|_| BTreeMap::new())))
+        }
+    }
+
+    impl Default for InMemoryBackend {
+        fn default() -> Self {
+            Self::new()
+        }
+    }
+
+    impl ReadBackend for InMemoryBackend {
+        fn location(&self) -> String {
+            "test".to_string()
+        }
+
+        fn list_with_size(&self, tpe: FileType) -> Result<Vec<(Id, u32)>> {
+            Ok(self.0.read().unwrap()[tpe]
+                .iter()
+                .map(|(id, byte)| (*id, byte.len() as u32))
+                .collect())
+        }
+
+        fn read_full(&self, tpe: FileType, id: &Id) -> Result<Bytes> {
+            Ok(self.0.read().unwrap()[tpe][id].clone())
+        }
+
+        fn read_partial(
+            &self,
+            tpe: FileType,
+            id: &Id,
+            _cacheable: bool,
+            offset: u32,
+            length: u32,
+        ) -> Result<Bytes> {
+            Ok(self.0.read().unwrap()[tpe][id].slice(offset as usize..(offset + length) as usize))
+        }
+    }
+
+    impl WriteBackend for InMemoryBackend {
+        fn create(&self) -> Result<()> {
+            Ok(())
+        }
+
+        fn write_bytes(&self, tpe: FileType, id: &Id, _cacheable: bool, buf: Bytes) -> Result<()> {
+            if self.0.write().unwrap()[tpe].insert(*id, buf).is_some() {
+                bail!("id {id} already exists");
+            }
+            Ok(())
+        }
+
+        fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> {
+            if self.0.write().unwrap()[tpe].remove(id).is_none() {
+                bail!("id {id} doesn't exists");
+            }
+            Ok(())
+        }
+    }
+}

crates/core/src/lib.rs

@@ -183,6 +183,7 @@ pub use crate::{
     backend::{
         decrypt::{compression_level_range, max_compression_level},
         ignore::{LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions},
+        in_memory_backend::InMemoryBackend,
         local_destination::LocalDestination,
         node::last_modified_node,
         FileType, ReadBackend, ReadSource, ReadSourceEntry, ReadSourceOpen, RepositoryBackends,

crates/core/tests/integration.rs

@@ -0,0 +1,162 @@
+use std::{env, fs::File, path::Path, str::FromStr, sync::Arc};
+
+use anyhow::Result;
+use flate2::read::GzDecoder;
+use insta::assert_debug_snapshot;
+use pretty_assertions::assert_eq;
+use rustic_core::{
+    repofile::SnapshotFile, BackupOptions, ConfigOptions, InMemoryBackend, KeyOptions,
+    NoProgressBars, OpenStatus, PathList, Repository, RepositoryBackends, RepositoryOptions,
+    StringList,
+};
+use tar::Archive;
+use tempfile::{tempdir, TempDir};
+
+fn set_up_repo() -> Result<Repository<NoProgressBars, OpenStatus>> {
+    let be = InMemoryBackend::new();
+    let be = RepositoryBackends::new(Arc::new(be), None);
+    let options = RepositoryOptions::default().password("test");
+    let repo = Repository::new(&options, be)?;
+    let key_opts = KeyOptions::default();
+    let config_opts = &ConfigOptions::default();
+    let repo = repo.init(&key_opts, config_opts)?;
+    Ok(repo)
+}
+
+struct TestSource(TempDir);
+
+impl TestSource {
+    fn new(tmp: TempDir) -> Self {
+        Self(tmp)
+    }
+
+    fn paths(&self) -> Result<PathList> {
+        Ok(PathList::from_string(self.0.path().to_str().unwrap())?)
+    }
+
+    fn strings(&self) -> Result<StringList> {
+        Ok(StringList::from_str(self.0.path().to_str().unwrap())?)
+    }
+}
+
+fn set_up_testdata(path: impl AsRef<Path>) -> Result<TestSource> {
+    let dir = tempdir()?;
+    let path = Path::new("tests/testdata").join(path);
+    let tar_gz = File::open(path)?;
+    let tar = GzDecoder::new(tar_gz);
+    let mut archive = Archive::new(tar);
+    archive.set_preserve_permissions(true);
+    archive.set_preserve_mtime(true);
+    archive.unpack(&dir)?;
+    Ok(TestSource::new(dir))
+}
+
+// Parts of the snapshot summary we want to test against references
+struct TestSummary<'a>(&'a SnapshotFile);
+
+impl<'a> std::fmt::Debug for TestSummary<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let s = self.0.summary.as_ref().unwrap();
+        // leave out info we expect to change:
+        // Ids, times, tree sizes (as used uid/username is saved in trees)
+        let mut b = f.debug_struct("TestSnap");
+        b.field("files_new", &s.files_new);
+        b.field("files_changed", &s.files_changed);
+        b.field("files_unmodified", &s.files_unmodified);
+        b.field("total_files_processed", &s.total_files_processed);
+        b.field("total_bytes_processed", &s.total_bytes_processed);
+        b.field("dirs_new", &s.dirs_new);
+        b.field("dirs_changed", &s.dirs_changed);
+        b.field("dirs_unmodified", &s.dirs_unmodified);
+        b.field("total_dirs_processed", &s.total_dirs_processed);
+        b.field("data_blobs", &s.data_blobs);
+        b.field("tree_blobs", &s.tree_blobs);
+        b.field("data_added_files", &s.data_added_files);
+        b.field("data_added_files_packed", &s.data_added_files_packed);
+        b.finish()
+    }
+}
+
+#[test]
+fn backup() -> Result<()> {
+    // SimpleLogger::init(log::LevelFilter::Debug, Config::default())?;
+    let source = set_up_testdata("backup-data.tar.gz")?;
+    let paths = &source.paths()?;
+    let repo = set_up_repo()?.to_indexed_ids()?;
+    let opts = BackupOptions::default();
+
+    // first backup
+    let snap1 = repo.backup(&opts, paths, SnapshotFile::default())?;
+    assert_debug_snapshot!(TestSummary(&snap1));
+    assert_eq!(snap1.parent, None);
+    assert_eq!(snap1.paths, source.strings()?);
+
+    // get all snapshots and check them
+    let snaps = repo.get_all_snapshots()?;
+    assert_eq!(vec![snap1.clone()], snaps);
+    // save list of pack files
+    let packs1: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect();
+
+    // re-read index
+    let repo = repo.to_indexed_ids()?;
+    // second backup
+    let snap2 = repo.backup(&opts, paths, SnapshotFile::default())?;
+    assert_debug_snapshot!(TestSummary(&snap2));
+    assert_eq!(snap2.parent, Some(snap1.id));
+    assert_eq!(snap1.tree, snap2.tree);
+
+    // get all snapshots and check them
+    let mut snaps = repo.get_all_snapshots()?;
+    snaps.sort_unstable();
+    assert_eq!(vec![snap1, snap2], snaps);
+
+    // pack files should be unchanged
+    let packs2: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect();
+    assert_eq!(packs1, packs2);
+    Ok(())
+}
+
+#[test]
+fn backup_dry_run() -> Result<()> {
+    let source = &set_up_testdata("backup-data.tar.gz")?;
+    let paths = &source.paths()?;
+    let repo = set_up_repo()?.to_indexed_ids()?;
+    let opts = BackupOptions::default().dry_run(true);
+
+    // dry-run backup
+    let snap_dry_run = repo.backup(&opts, paths, SnapshotFile::default())?;
+    assert_debug_snapshot!(TestSummary(&snap_dry_run));
+    // check that repo is still empty
+    let snaps = repo.get_all_snapshots()?;
+    assert_eq!(snaps.len(), 0);
+    let packs: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect();
+    assert_eq!(packs.len(), 0);
+    let indexes: Vec<_> = repo.list(rustic_core::FileType::Index)?.collect();
+    assert_eq!(indexes.len(), 0);
+
+    // first real backup
+    let opts = opts.dry_run(false);
+    let snap1 = repo.backup(&opts, paths, SnapshotFile::default())?;
+    assert_eq!(snap_dry_run.tree, snap1.tree);
+    let packs: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect();
+
+    // re-read index
+    let repo = repo.to_indexed_ids()?;
+    // second dry-run backup
+    let opts = opts.dry_run(true);
+    let snap_dry_run = repo.backup(&opts, paths, SnapshotFile::default())?;
+    assert_debug_snapshot!(TestSummary(&snap_dry_run));
+    // check that no data has been added
+    let snaps = repo.get_all_snapshots()?;
+    assert_eq!(snaps, vec![snap1.clone()]);
+    let packs_dry_run: Vec<_> = repo.list(rustic_core::FileType::Pack)?.collect();
+    assert_eq!(packs_dry_run, packs);
+
+    // re-read index
+    let repo = repo.to_indexed_ids()?;
+    // second real backup
+    let opts = opts.dry_run(false);
+    let snap2 = repo.backup(&opts, paths, SnapshotFile::default())?;
+    assert_eq!(snap_dry_run.tree, snap2.tree);
+    Ok(())
+}

crates/core/tests/snapshots/integration__backup-2.snap

@@ -0,0 +1,19 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSnap(&snap2)
+---
+TestSnap {
+    files_new: 0,
+    files_changed: 0,
+    files_unmodified: 73,
+    total_files_processed: 73,
+    total_bytes_processed: 1125682,
+    dirs_new: 0,
+    dirs_changed: 0,
+    dirs_unmodified: 7,
+    total_dirs_processed: 7,
+    data_blobs: 0,
+    tree_blobs: 0,
+    data_added_files: 0,
+    data_added_files_packed: 0,
+}

crates/core/tests/snapshots/integration__backup.snap

@@ -0,0 +1,19 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSnap(&snap1)
+---
+TestSnap {
+    files_new: 73,
+    files_changed: 0,
+    files_unmodified: 0,
+    total_files_processed: 73,
+    total_bytes_processed: 1125674,
+    dirs_new: 7,
+    dirs_changed: 0,
+    dirs_unmodified: 0,
+    total_dirs_processed: 7,
+    data_blobs: 70,
+    tree_blobs: 7,
+    data_added_files: 1125653,
+    data_added_files_packed: 78740,
+}

crates/core/tests/snapshots/integration__backup_dry_run-2.snap

@@ -0,0 +1,19 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSnap(&snap_dry_run)
+---
+TestSnap {
+    files_new: 0,
+    files_changed: 0,
+    files_unmodified: 73,
+    total_files_processed: 73,
+    total_bytes_processed: 1125682,
+    dirs_new: 0,
+    dirs_changed: 0,
+    dirs_unmodified: 7,
+    total_dirs_processed: 7,
+    data_blobs: 0,
+    tree_blobs: 0,
+    data_added_files: 0,
+    data_added_files_packed: 0,
+}

crates/core/tests/snapshots/integration__backup_dry_run.snap

@@ -0,0 +1,19 @@
+---
+source: crates/core/tests/integration.rs
+expression: TestSnap(&snap_dry_run)
+---
+TestSnap {
+    files_new: 73,
+    files_changed: 0,
+    files_unmodified: 0,
+    total_files_processed: 73,
+    total_bytes_processed: 1125674,
+    dirs_new: 7,
+    dirs_changed: 0,
+    dirs_unmodified: 0,
+    total_dirs_processed: 7,
+    data_blobs: 70,
+    tree_blobs: 7,
+    data_added_files: 1125653,
+    data_added_files_packed: 78740,
+}