From 310f5a13d85e97d09b54f8cd769f7771acca9f06 Mon Sep 17 00:00:00 2001 From: aawsome <37850842+aawsome@users.noreply.github.com> Date: Wed, 17 Jan 2024 09:58:00 +0100 Subject: [PATCH] refactor backend (#73) * refactor backend usage * remove unneeded Downcast * error handling; add opendal backend * deactivate opendal:sftp for windows * fix most clippy lints * fix windows builds * fix clippy lint * update MSRV (as required by ordered-multimap, needed by opendal->rsa) * opendal - more services, add logging and retry * some code clean-ups * clean-up code and comments * clippy hints and s3 convenience backend * correct path * temporarily ignore rsa advisory until there is a workaround * fix: import and typo Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: unused import on win Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: deactivate security warning for rsa for cargo-audit and cargo-deny and annotate with fixme Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * style: dprint fmt Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * ci: test also examples Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: run cargo-msrv and update msrv to 1.71.1 Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: check and merge imports Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor: move RestErrorKind to error.rs Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor: move RcloneErrorKind to error.rs Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * style: fmt Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: apply clippy fixes Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: borrowing for tree streamer Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: borrowing for path list Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: close name Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: clippy Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: clippy values_mut iter Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * style: cargo fmt Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: readd error to docs that was removed Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: use `new` -> Self for HotColdBackend Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: lifetimes in public api Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: add docs and remove dbg! Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: log error in warm up Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * docs: add docs Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * feat(choose): implement trait for choosing backends Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor: move backend crate out of rustic_core, first try Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor: move tests into workspace crate Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor: choose Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor: move backup example and update to new api Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * WIP: stash Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor: init backends Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: cleanup Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor(backends): use RepositoryBackends struct Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: cleanup Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * doc: tiny doc for RepositoryBackends Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor: implement test cases Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor: rework examples and move them to workspace examples Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: remove examples from manifest.include and thus packaging as they are now outside of the package in workspace Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: remove opendal dep from core crate Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: cleanup errors Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix(api): fix export of overwrite for mergable struct fields Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * docs: document FindInBackend Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor: move errors to error.rs Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: ignore lint for unused imports as overwrite is not being recognized as being used Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: update package details Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: cleanup imports Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * style: dprint fmt Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * ci: Check for workspace crates's MSRV Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * ci: update actions to be used on workspaces or especially on both workspace crates Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * feat(s3): implement extra s3 backend, delegated read- and writebackend impl of opendal Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: remove glob import Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: using features and fixing stuff so tests run through on them with `cargo +stable hack test --feature-powerset --workspace` Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: remove cyclic dependency Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * style: dprint fmt Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: set default features to include all backends Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: change back delimiter to `:` Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * refactor: impl AsRef in Backend::new() * chore: implement to_inner() on S3Backend, but keep impl Write/Read backend for now * chore(deps): update dependencies Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: rename backends to backend to account for crate name Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: add back cfg in backend/util.rs for parsing the provider string Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: add missing attribute Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * chore: use clearer naming Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: use locahost Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: add match case for unc only Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: test with respecting features as some things don't need to work and won't work on *nix Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: allow unused imports in tests Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * fix: test with respecting platforms Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> * use options_hot and options_cold * simplify s3 backend impl * add evaluate_password to RepositoryOptions * don't panic when repository is missing * rclone backend: Remove Arc * expose less in rustic_core crate * clean-up dependencies for rustic_core * clean-up dependencies for rustic_backend * don't validate repo path (-> init command) * fix clippy lint * fix tokio dependency * style: fmt Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> --------- Signed-off-by: simonsan <14062932+simonsan@users.noreply.github.com> Co-authored-by: simonsan <14062932+simonsan@users.noreply.github.com> --- .cargo/audit.toml | 6 + .github/workflows/audit.yml | 1 + .github/workflows/careful.yml | 86 +- .github/workflows/ci.yml | 5 +- .github/workflows/msrv.yml | 5 +- Cargo.toml | 148 +-- README.md | 2 +- crates/backend/Cargo.toml | 97 ++ crates/backend/src/choose.rs | 211 ++++ crates/backend/src/error.rs | 130 ++ crates/backend/src/lib.rs | 28 + crates/backend/src/local.rs | 393 ++++++ crates/backend/src/opendal.rs | 225 ++++ crates/backend/src/opendal/s3.rs | 99 ++ {src/backend => crates/backend/src}/rclone.rs | 174 +-- {src/backend => crates/backend/src}/rest.rs | 134 +-- crates/backend/src/util.rs | 207 ++++ crates/core/Cargo.toml | 136 +++ {src => crates/core/src}/archiver.rs | 37 +- .../core/src}/archiver/file_archiver.rs | 13 +- {src => crates/core/src}/archiver/parent.rs | 34 +- {src => crates/core/src}/archiver/tree.rs | 2 +- .../core/src}/archiver/tree_archiver.rs | 15 +- {src => crates/core/src}/backend.rs | 206 +++- {src => crates/core/src}/backend/cache.rs | 104 +- {src => crates/core/src}/backend/decrypt.rs | 127 +- {src => crates/core/src}/backend/dry_run.rs | 24 +- crates/core/src/backend/hotcold.rs | 101 ++ {src => crates/core/src}/backend/ignore.rs | 0 crates/core/src/backend/local_destination.rs | 647 ++++++++++ {src => crates/core/src}/backend/node.rs | 3 + {src => crates/core/src}/backend/stdin.rs | 9 +- crates/core/src/backend/warm_up.rs | 77 ++ {src => crates/core/src}/blob.rs | 0 {src => crates/core/src}/blob/packer.rs | 33 +- {src => crates/core/src}/blob/tree.rs | 85 +- {src => crates/core/src}/cdc.rs | 0 {src => crates/core/src}/cdc/LICENSE.txt | 0 {src => crates/core/src}/cdc/README.md | 0 {src => crates/core/src}/cdc/polynom.rs | 0 {src => crates/core/src}/cdc/rolling_hash.rs | 0 {src => crates/core/src}/chunker.rs | 0 {src => crates/core/src}/commands.rs | 2 + {src => crates/core/src}/commands/backup.rs | 24 +- {src => crates/core/src}/commands/cat.rs | 23 +- {src => crates/core/src}/commands/check.rs | 44 +- {src => crates/core/src}/commands/config.rs | 21 +- {src => crates/core/src}/commands/copy.rs | 11 +- {src => crates/core/src}/commands/dump.rs | 6 +- {src => crates/core/src}/commands/forget.rs | 9 +- {src => crates/core/src}/commands/init.rs | 10 +- {src => crates/core/src}/commands/key.rs | 13 +- {src => crates/core/src}/commands/merge.rs | 6 +- {src => crates/core/src}/commands/prune.rs | 34 +- {src => crates/core/src}/commands/repair.rs | 0 .../core/src}/commands/repair/index.rs | 11 +- .../core/src}/commands/repair/snapshots.rs | 61 +- {src => crates/core/src}/commands/repoinfo.rs | 8 +- {src => crates/core/src}/commands/restore.rs | 79 +- .../core/src}/commands/snapshots.rs | 6 +- {src => crates/core/src}/crypto.rs | 2 +- .../core/src}/crypto/aespoly1305.rs | 0 {src => crates/core/src}/crypto/hasher.rs | 0 {src => crates/core/src}/error.rs | 149 +-- {src => crates/core/src}/id.rs | 3 + {src => crates/core/src}/index.rs | 106 +- .../core/src}/index/binarysorted.rs | 2 +- {src => crates/core/src}/index/indexer.rs | 0 {src => crates/core/src}/lib.rs | 28 +- {src => crates/core/src}/progress.rs | 0 {src => crates/core/src}/repofile.rs | 0 .../core/src}/repofile/configfile.rs | 0 .../core/src}/repofile/indexfile.rs | 6 +- {src => crates/core/src}/repofile/keyfile.rs | 8 +- {src => crates/core/src}/repofile/packfile.rs | 9 +- .../core/src}/repofile/snapshotfile.rs | 35 +- {src => crates/core/src}/repository.rs | 352 +++--- .../core/src}/repository/warm_up.rs | 35 +- {tests => crates/core/tests}/public_api.rs | 0 .../core/tests}/public_api_fixtures/.gitkeep | 0 .../public_api_fixtures/public-api_linux.txt | 0 .../public_api_fixtures/public-api_macos.txt | 0 .../public_api_fixtures/public-api_win.txt | 0 deny.toml | 6 +- examples/backup/Cargo.toml | 12 + examples/{ => backup/examples}/backup.rs | 18 +- examples/backup/src/lib.rs | 1 + examples/check/Cargo.toml | 12 + examples/{ => check/examples}/check.rs | 12 +- examples/check/src/lib.rs | 1 + examples/config/Cargo.toml | 12 + examples/{ => config/examples}/config.rs | 12 +- examples/config/src/lib.rs | 1 + examples/copy/Cargo.toml | 12 + examples/{ => copy/examples}/copy.rs | 27 +- examples/copy/src/lib.rs | 1 + examples/forget/Cargo.toml | 12 + examples/{ => forget/examples}/forget.rs | 13 +- examples/forget/src/lib.rs | 1 + examples/init/Cargo.toml | 12 + examples/{ => init/examples}/init.rs | 12 +- examples/init/src/lib.rs | 1 + examples/key/Cargo.toml | 12 + examples/{ => key/examples}/key.rs | 13 +- examples/key/src/lib.rs | 1 + examples/ls/Cargo.toml | 12 + examples/{ => ls/examples}/ls.rs | 15 +- examples/ls/src/lib.rs | 1 + examples/merge/Cargo.toml | 12 + examples/{ => merge/examples}/merge.rs | 15 +- examples/merge/src/lib.rs | 1 + examples/prune/Cargo.toml | 12 + examples/{ => prune/examples}/prune.rs | 13 +- examples/prune/src/lib.rs | 1 + examples/restore/Cargo.toml | 12 + examples/{ => restore/examples}/restore.rs | 14 +- examples/restore/src/lib.rs | 1 + examples/tag/Cargo.toml | 12 + examples/{ => tag/examples}/tag.rs | 12 +- examples/tag/src/lib.rs | 1 + src/backend/choose.rs | 197 --- src/backend/hotcold.rs | 98 -- src/backend/local.rs | 1062 ----------------- 123 files changed, 3741 insertions(+), 2661 deletions(-) create mode 100644 .cargo/audit.toml create mode 100644 crates/backend/Cargo.toml create mode 100644 crates/backend/src/choose.rs create mode 100644 crates/backend/src/error.rs create mode 100644 crates/backend/src/lib.rs create mode 100644 crates/backend/src/local.rs create mode 100644 crates/backend/src/opendal.rs create mode 100644 crates/backend/src/opendal/s3.rs rename {src/backend => crates/backend/src}/rclone.rs (56%) rename {src/backend => crates/backend/src}/rest.rs (75%) create mode 100644 crates/backend/src/util.rs create mode 100644 crates/core/Cargo.toml rename {src => crates/core/src}/archiver.rs (87%) rename {src => crates/core/src}/archiver/file_archiver.rs (94%) rename {src => crates/core/src}/archiver/parent.rs (92%) rename {src => crates/core/src}/archiver/tree.rs (98%) rename {src => crates/core/src}/archiver/tree_archiver.rs (95%) rename {src => crates/core/src}/backend.rs (60%) rename {src => crates/core/src}/backend/cache.rs (82%) rename {src => crates/core/src}/backend/decrypt.rs (78%) rename {src => crates/core/src}/backend/dry_run.rs (85%) create mode 100644 crates/core/src/backend/hotcold.rs rename {src => crates/core/src}/backend/ignore.rs (100%) create mode 100644 crates/core/src/backend/local_destination.rs rename {src => crates/core/src}/backend/node.rs (99%) rename {src => crates/core/src}/backend/stdin.rs (89%) create mode 100644 crates/core/src/backend/warm_up.rs rename {src => crates/core/src}/blob.rs (100%) rename {src => crates/core/src}/blob/packer.rs (97%) rename {src => crates/core/src}/blob/tree.rs (91%) rename {src => crates/core/src}/cdc.rs (100%) rename {src => crates/core/src}/cdc/LICENSE.txt (100%) rename {src => crates/core/src}/cdc/README.md (100%) rename {src => crates/core/src}/cdc/polynom.rs (100%) rename {src => crates/core/src}/cdc/rolling_hash.rs (100%) rename {src => crates/core/src}/chunker.rs (100%) rename {src => crates/core/src}/commands.rs (88%) rename {src => crates/core/src}/commands/backup.rs (94%) rename {src => crates/core/src}/commands/cat.rs (77%) rename {src => crates/core/src}/commands/check.rs (93%) rename {src => crates/core/src}/commands/config.rs (96%) rename {src => crates/core/src}/commands/copy.rs (96%) rename {src => crates/core/src}/commands/dump.rs (89%) rename {src => crates/core/src}/commands/forget.rs (98%) rename {src => crates/core/src}/commands/init.rs (92%) rename {src => crates/core/src}/commands/key.rs (92%) rename {src => crates/core/src}/commands/merge.rs (95%) rename {src => crates/core/src}/commands/prune.rs (98%) rename {src => crates/core/src}/commands/repair.rs (100%) rename {src => crates/core/src}/commands/repair/index.rs (93%) rename {src => crates/core/src}/commands/repair/snapshots.rs (86%) rename {src => crates/core/src}/commands/repoinfo.rs (97%) rename {src => crates/core/src}/commands/restore.rs (92%) rename {src => crates/core/src}/commands/snapshots.rs (93%) rename {src => crates/core/src}/crypto.rs (90%) rename {src => crates/core/src}/crypto/aespoly1305.rs (100%) rename {src => crates/core/src}/crypto/hasher.rs (100%) rename {src => crates/core/src}/error.rs (85%) rename {src => crates/core/src}/id.rs (98%) rename {src => crates/core/src}/index.rs (79%) rename {src => crates/core/src}/index/binarysorted.rs (99%) rename {src => crates/core/src}/index/indexer.rs (100%) rename {src => crates/core/src}/lib.rs (91%) rename {src => crates/core/src}/progress.rs (100%) rename {src => crates/core/src}/repofile.rs (100%) rename {src => crates/core/src}/repofile/configfile.rs (100%) rename {src => crates/core/src}/repofile/indexfile.rs (97%) rename {src => crates/core/src}/repofile/keyfile.rs (97%) rename {src => crates/core/src}/repofile/packfile.rs (98%) rename {src => crates/core/src}/repofile/snapshotfile.rs (96%) rename {src => crates/core/src}/repository.rs (84%) rename {src => crates/core/src}/repository/warm_up.rs (84%) rename {tests => crates/core/tests}/public_api.rs (100%) rename {tests => crates/core/tests}/public_api_fixtures/.gitkeep (100%) rename {tests => crates/core/tests}/public_api_fixtures/public-api_linux.txt (100%) rename {tests => crates/core/tests}/public_api_fixtures/public-api_macos.txt (100%) rename {tests => crates/core/tests}/public_api_fixtures/public-api_win.txt (100%) create mode 100644 examples/backup/Cargo.toml rename examples/{ => backup/examples}/backup.rs (63%) create mode 100644 examples/backup/src/lib.rs create mode 100644 examples/check/Cargo.toml rename examples/{ => check/examples}/check.rs (66%) create mode 100644 examples/check/src/lib.rs create mode 100644 examples/config/Cargo.toml rename examples/{ => config/examples}/config.rs (68%) create mode 100644 examples/config/src/lib.rs create mode 100644 examples/copy/Cargo.toml rename examples/{ => copy/examples}/copy.rs (53%) create mode 100644 examples/copy/src/lib.rs create mode 100644 examples/forget/Cargo.toml rename examples/{ => forget/examples}/forget.rs (74%) create mode 100644 examples/forget/src/lib.rs create mode 100644 examples/init/Cargo.toml rename examples/{ => init/examples}/init.rs (64%) create mode 100644 examples/init/src/lib.rs create mode 100644 examples/key/Cargo.toml rename examples/{ => key/examples}/key.rs (65%) create mode 100644 examples/key/src/lib.rs create mode 100644 examples/ls/Cargo.toml rename examples/{ => ls/examples}/ls.rs (70%) create mode 100644 examples/ls/src/lib.rs create mode 100644 examples/merge/Cargo.toml rename examples/{ => merge/examples}/merge.rs (70%) create mode 100644 examples/merge/src/lib.rs create mode 100644 examples/prune/Cargo.toml rename examples/{ => prune/examples}/prune.rs (71%) create mode 100644 examples/prune/src/lib.rs create mode 100644 examples/restore/Cargo.toml rename examples/{ => restore/examples}/restore.rs (80%) create mode 100644 examples/restore/src/lib.rs create mode 100644 examples/tag/Cargo.toml rename examples/{ => tag/examples}/tag.rs (78%) create mode 100644 examples/tag/src/lib.rs delete mode 100644 src/backend/choose.rs delete mode 100644 src/backend/hotcold.rs delete mode 100644 src/backend/local.rs diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 00000000..0b118933 --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,6 @@ +[advisories] +ignore = [ + # FIXME!: See https://github.com/RustCrypto/RSA/issues/19#issuecomment-1822995643. + # There is no workaround available yet. + "RUSTSEC-2023-0071", +] diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index e0d9d57e..e043c870 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -28,6 +28,7 @@ jobs: - uses: rustsec/audit-check@dd51754d4e59da7395a4cd9b593f0ff2d61a9b95 # v1.4.1 with: token: ${{ secrets.GITHUB_TOKEN }} + ignore: RUSTSEC-2023-0071 # rsa thingy, ignored for now cargo-deny: if: ${{ github.repository_owner == 'rustic-rs' }} diff --git a/.github/workflows/careful.yml b/.github/workflows/careful.yml index be391686..7b2d7492 100644 --- a/.github/workflows/careful.yml +++ b/.github/workflows/careful.yml @@ -41,59 +41,61 @@ jobs: - name: Run Cargo Careful run: cargo +${{ matrix.rust }} careful test - miri: - name: Miri Test - runs-on: ${{ matrix.job.os }} - strategy: - fail-fast: false - matrix: - rust: [nightly] # runs on nightly only - job: - - os: macos-latest - - os: ubuntu-latest - - os: windows-latest - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - if: github.event_name != 'pull_request' - with: - fetch-depth: 0 + # TODO: don't run miri for now, due to addition of workspace + #1 crates and we'll need to figure out if we want to run miri + # miri: + # name: Miri Test + # runs-on: ${{ matrix.job.os }} + # strategy: + # fail-fast: false + # matrix: + # rust: [nightly] # runs on nightly only + # job: + # - os: macos-latest + # - os: ubuntu-latest + # - os: windows-latest + # steps: + # - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + # if: github.event_name != 'pull_request' + # with: + # fetch-depth: 0 - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - if: github.event_name == 'pull_request' - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 + # - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + # if: github.event_name == 'pull_request' + # with: + # ref: ${{ github.event.pull_request.head.sha }} + # fetch-depth: 0 - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@1482605bfc5719782e1267fd0c0cc350fe7646b8 # v1 - with: - toolchain: ${{ matrix.rust }} - components: miri - - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 # v2 + # - name: Install Rust toolchain + # uses: dtolnay/rust-toolchain@1482605bfc5719782e1267fd0c0cc350fe7646b8 # v1 + # with: + # toolchain: ${{ matrix.rust }} + # components: miri + # - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 # v2 - - name: Run Cargo Clean - run: cargo +${{ matrix.rust }} clean # miri needs clean builds + # - name: Run Cargo Clean + # run: cargo +${{ matrix.rust }} clean # miri needs clean builds - - name: Patch Cargo.toml - shell: bash - run: | - # Account for sha256_compress not being interpreted by miri - # https://github.com/rust-lang/miri/issues/3066 - sed -i -e 's/^sha2 = { version.*/sha2 = "0"/g' ./Cargo.toml - - name: Run Cargo Miri Setup - run: cargo +${{ matrix.rust }} miri setup # keep output clean + # - name: Patch Cargo.toml + # shell: bash + # run: | + # # Account for sha256_compress not being interpreted by miri + # # https://github.com/rust-lang/miri/issues/3066 + # sed -i -e 's/^sha2 = { version.*/sha2 = "0"/g' ./Cargo.toml + # - name: Run Cargo Miri Setup + # run: cargo +${{ matrix.rust }} miri setup # keep output clean - - name: Run Cargo Miri - env: - MIRIFLAGS: -Zmiri-disable-isolation - run: cargo +${{ matrix.rust }} miri test -- --nocapture + # - name: Run Cargo Miri + # env: + # MIRIFLAGS: -Zmiri-disable-isolation + # run: cargo +${{ matrix.rust }} miri test -- --nocapture result: name: Result (Careful CI) runs-on: ubuntu-latest needs: - careful - - miri + # - miri # FIXME: don't run miri for now, due to addition of workspace steps: - name: Mark the job as successful run: exit 0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eea2c0c0..fbfa8aed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: toolchain: ${{ matrix.rust }} - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 # v2 - name: Run Cargo Test - run: cargo +${{ matrix.rust }} test -r --all-targets --all-features --workspace + run: cargo +${{ matrix.rust }} test -r --all-targets --all-features --workspace --examples docs: name: Build docs @@ -109,6 +109,7 @@ jobs: strategy: matrix: rust: [stable, beta, nightly] + crate: [rustic_core, rustic_backend] # if we use a workspace, we also check all examples/* job: - os: macos-latest - os: ubuntu-latest @@ -135,7 +136,7 @@ jobs: tool: cargo-hack - uses: Swatinem/rust-cache@a95ba195448af2da9b00fb742d14ffaaf3c21f43 # v2 - name: Run Cargo Hack - run: cargo +${{ matrix.rust }} hack check --feature-powerset --no-dev-deps + run: cargo +${{ matrix.rust }} hack check --feature-powerset --no-dev-deps -p ${{ matrix.crate }} result: name: Result (CI) diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml index d554ada4..2f6ccd72 100644 --- a/.github/workflows/msrv.yml +++ b/.github/workflows/msrv.yml @@ -18,6 +18,9 @@ jobs: msrv: name: Check MSRV runs-on: ubuntu-latest + strategy: + matrix: + crate: [rustic_core, rustic_backend] steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install cargo-hack @@ -26,7 +29,7 @@ jobs: tool: cargo-hack - name: Run Cargo Hack - run: cargo hack check --rust-version + run: cargo hack check --rust-version -p ${{ matrix.crate }} result: name: Result (MSRV) diff --git a/Cargo.toml b/Cargo.toml index 83494736..8a227f05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,148 +1,14 @@ -[package] -name = "rustic_core" -version = "0.1.2" -authors = ["Alexander Weiss"] -categories = [ - "Algorithms", - "Compression", - "Cryptography", - "Data structures", - "Filesystem", -] -documentation = "https://docs.rs/rustic_core" -edition = "2021" -homepage = "https://rustic.cli.rs/" -include = ["src/**/*", "LICENSE-*", "README.md", "examples/**/*"] -keywords = ["backup", "restic", "deduplication", "encryption", "library"] -license = "Apache-2.0 OR MIT" -publish = true -readme = "README.md" -repository = "https://github.com/rustic-rs/rustic_core" +[workspace] +members = ["crates/core", "crates/backend", "examples/*"] resolver = "2" -rust-version = "1.70.0" -description = """ -rustic_core - library for fast, encrypted, deduplicated backups that powers rustic-rs -""" -[lib] -path = "src/lib.rs" -name = "rustic_core" -test = true -doctest = true -bench = true -doc = true -harness = true -edition = "2021" +[workspace.package] +rust-version = "1.71.1" -[features] -default = [] -cli = ["merge", "clap"] -merge = ["dep:merge"] -clap = ["dep:clap"] - -[package.metadata.docs.rs] -all-features = true -rustdoc-args = ["--document-private-items", "--generate-link-to-definition"] - -[dependencies] -# errors -displaydoc = "0.2.4" -thiserror = "1.0.56" - -# macros -derivative = "2.2.0" -derive_more = "0.99.17" -derive_setters = "0.1.6" - -# logging -log = "0.4.20" - -# parallelize -crossbeam-channel = "0.5.10" -pariter = "0.5.1" -rayon = "1.8.0" - -# crypto -aes256ctr_poly1305aes = "0.2.0" -rand = "0.8.5" -scrypt = { version = "0.11.0", default-features = false } - -# chunker / packer -integer-sqrt = "0.1.5" - -# serialization -binrw = "0.13.3" -hex = { version = "0.4.3", features = ["serde"] } -serde = { version = "1.0.195" } -serde-aux = "4.3.1" -serde_derive = "1.0.195" -serde_json = "1.0.111" -serde_with = { version = "3.4.0", features = ["base64"] } - -# other dependencies -bytes = "1.5.0" -chrono = { version = "0.4.31", default-features = false, features = ["clock", "serde"] } -enum-map = "2.7.3" -enum-map-derive = "0.17.0" -zstd = "0.13.0" - -# local backend -aho-corasick = "1.1.2" -cached = { version = "0.47.0", default-features = false, features = ["proc_macro"] } -filetime = "0.2.23" -ignore = "0.4.22" -nix = { version = "0.27.1", default-features = false, features = ["user", "fs"] } -walkdir = "2.4.0" - -# rest backend -backoff = "0.4.0" -reqwest = { version = "0.11.23", default-features = false, features = ["json", "rustls-tls-native-roots", "stream", "blocking"] } -url = "2.5.0" - -# rclone backend -# semver = "1.0.18" # FIXME: unused, remove? - -# cache -cachedir = "0.3.1" -dirs = "5.0.1" - -# cli -clap = { version = "4.4.13", optional = true, features = ["derive", "env", "wrap_help"] } - -bytesize = "1.3.0" -directories = "5.0.1" -dunce = "1.0.4" -gethostname = "0.4.3" -humantime = "2.1.0" -itertools = "0.12.0" -merge = { version = "0.1.0", optional = true } -path-dedot = "3.1.1" -shell-words = "1.1.0" - -[target.'cfg(not(windows))'.dependencies] -sha2 = { version = "0.10", features = ["asm"] } - -[target.'cfg(all(windows, not(target_env="gnu")))'.dependencies] -# unfortunately, the asm extensions do not build on MSVC, see https://github.com/RustCrypto/asm-hashes/issues/17 -sha2 = "0.10" - -[target.'cfg(all(windows, target_env="gnu"))'.dependencies] -sha2 = { version = "0.10", features = ["asm"] } - -[target.'cfg(not(any(windows, target_os="openbsd")))'.dependencies] -xattr = "1" - -[dev-dependencies] -expect-test = "1.4.1" -pretty_assertions = "1.4.0" -public-api = "0.33.1" -quickcheck = "1.0.3" -quickcheck_macros = "1.0.0" -rstest = "0.18.2" -rustdoc-json = "0.8.8" -rustup-toolchain = "0.1.6" +[workspace.dependencies] +rustic_backend = { path = "crates/backend" } +rustic_core = { path = "crates/core" } simplelog = "0.12.1" -tempfile = "3.9.0" # see: https://nnethercote.github.io/perf-book/build-configuration.html [profile.dev] diff --git a/README.md b/README.md index be79047c..0dec3660 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ Please make sure, that you read the ## Minimum Rust version policy -This crate's minimum supported `rustc` version is `1.68.2`. +This crate's minimum supported `rustc` version is `1.71.1`. The current policy is that the minimum Rust version required to use this crate can be increased in minor version updates. For example, if `crate 1.0` requires diff --git a/crates/backend/Cargo.toml b/crates/backend/Cargo.toml new file mode 100644 index 00000000..cdd6d09e --- /dev/null +++ b/crates/backend/Cargo.toml @@ -0,0 +1,97 @@ +[package] +name = "rustic_backend" +version = "0.1.0" +authors = ["the rustic-rs team"] +categories = ["Algorithms", "Data structures", "Filesystem"] +documentation = "https://docs.rs/rustic_backend" +edition = "2021" +homepage = "https://rustic.cli.rs/" +include = ["src/**/*", "LICENSE-*", "README.md"] +keywords = ["backup", "restic", "deduplication", "encryption", "library"] +license = "Apache-2.0 OR MIT" +publish = true +readme = "README.md" +repository = "https://github.com/rustic-rs/rustic_core/tree/main/crates/backend" +resolver = "2" +rust-version = { workspace = true } +description = """ +rustic_backend - library for supporting various backends in rustic-rs +""" + +[lib] +path = "src/lib.rs" +name = "rustic_backend" +test = true +doctest = true +bench = true +doc = true +harness = true +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[features] +default = ["opendal", "s3", "rest", "rclone"] +cli = ["merge", "clap"] +merge = ["dep:merge"] +clap = ["dep:clap"] +s3 = ["opendal"] +opendal = ["dep:opendal", "dep:rayon", "dep:tokio", "tokio/rt-multi-thread"] +rest = ["dep:reqwest", "dep:backoff"] +rclone = ["rest", "dep:rand"] + +[dependencies] +# core +rustic_core = { path = "../core" } + +# errors +anyhow = "1.0.79" +displaydoc = "0.2.4" +thiserror = "1.0.56" + +# logging +log = "0.4.20" + +# other dependencies +bytes = "1.5.0" +derive_setters = "0.1.6" +humantime = "2.1.0" +itertools = "0.12.0" +strum = "0.25" +strum_macros = "0.25" + +# general / backend choosing +hex = { version = "0.4.3", features = ["serde"] } +serde = { version = "1.0.195" } +url = "2.5.0" + +# cli support +clap = { version = "4.4.14", optional = true, features = ["derive", "env", "wrap_help"] } +merge = { version = "0.1.0", optional = true } + +# local backend +aho-corasick = "1.1.2" +shell-words = "1.1.0" +walkdir = "2.4.0" + +# rest backend +backoff = { version = "0.4.0", optional = true } +reqwest = { version = "0.11.23", default-features = false, features = ["json", "rustls-tls-native-roots", "stream", "blocking"], optional = true } + +# rclone backend +rand = { version = "0.8.5", optional = true } + +# opendal backend +rayon = { version = "1.8.0", optional = true } +tokio = { version = "1.35.1", optional = true, default-features = false } + +[target.'cfg(not(windows))'.dependencies] +# opendal backend - sftp is not supported on windows, see https://github.com/apache/incubator-opendal/issues/2963 +opendal = { version = "0.44.1", features = ["services-b2", "services-sftp", "services-swift"], optional = true } + +[target.'cfg(windows)'.dependencies] +# opendal backend +opendal = { version = "0.44.1", features = ["services-b2", "services-swift"], optional = true } + +[dev-dependencies] +rstest = "0.18.2" diff --git a/crates/backend/src/choose.rs b/crates/backend/src/choose.rs new file mode 100644 index 00000000..82717f98 --- /dev/null +++ b/crates/backend/src/choose.rs @@ -0,0 +1,211 @@ +//! This module contains the trait [`BackendChoice`] and the function [`get_backend`] to choose a backend from a given url. +use anyhow::{anyhow, Result}; +use derive_setters::Setters; +use std::{collections::HashMap, sync::Arc}; +use strum_macros::{Display, EnumString}; + +#[allow(unused_imports)] +use rustic_core::{RepositoryBackends, WriteBackend}; + +use crate::{ + error::BackendAccessErrorKind, + local::LocalBackend, + util::{location_to_type_and_path, BackendLocation}, +}; + +#[cfg(feature = "s3")] +use crate::opendal::s3::S3Backend; + +#[cfg(feature = "opendal")] +use crate::opendal::OpenDALBackend; + +#[cfg(feature = "rclone")] +use crate::rclone::RcloneBackend; + +#[cfg(feature = "rest")] +use crate::rest::RestBackend; + +/// Options for a backend. +#[cfg_attr(feature = "clap", derive(clap::Parser))] +#[cfg_attr(feature = "merge", derive(merge::Merge))] +#[derive(Clone, Default, Debug, serde::Deserialize, serde::Serialize, Setters)] +#[serde(default, rename_all = "kebab-case", deny_unknown_fields)] +#[setters(into, strip_option)] +pub struct BackendOptions { + /// Repository to use + #[cfg_attr( + feature = "clap", + clap(short, long, global = true, alias = "repo", env = "RUSTIC_REPOSITORY") + )] + pub repository: Option, + + /// Repository to use as hot storage + #[cfg_attr( + feature = "clap", + clap(long, global = true, alias = "repository_hot", env = "RUSTIC_REPO_HOT") + )] + pub repo_hot: Option, + + /// Other options for this repository (hot and cold part) + #[cfg_attr(feature = "clap", clap(skip))] + #[cfg_attr(feature = "merge", merge(strategy = overwrite))] + pub options: HashMap, + + /// Other options for the hot repository + #[cfg_attr(feature = "clap", clap(skip))] + #[cfg_attr(feature = "merge", merge(strategy = overwrite))] + pub options_hot: HashMap, + + /// Other options for the cold repository + #[cfg_attr(feature = "clap", clap(skip))] + #[cfg_attr(feature = "merge", merge(strategy = overwrite))] + pub options_cold: HashMap, +} + +/// Overwrite the left value with the right value +/// +/// This is used for merging [`RepositoryOptions`] and [`ConfigOptions`] +/// +/// # Arguments +/// +/// * `left` - The left value +/// * `right` - The right value +#[cfg(feature = "merge")] +pub fn overwrite(left: &mut T, right: T) { + *left = right; +} + +impl BackendOptions { + pub fn to_backends(&self) -> Result { + let mut options = self.options.clone(); + options.extend(self.options_cold.clone()); + let be = self + .get_backend(self.repository.clone(), options)? + .ok_or(anyhow!("Should be able to initialize main backend."))?; + let mut options = self.options.clone(); + options.extend(self.options_hot.clone()); + let be_hot = self.get_backend(self.repo_hot.clone(), options)?; + + Ok(RepositoryBackends::new(be, be_hot)) + } + + fn get_backend( + &self, + repo_string: Option, + options: HashMap, + ) -> Result>> { + repo_string + .map(|string| { + let (be_type, location) = location_to_type_and_path(&string)?; + be_type.to_backend(location, options.into()).map_err(|err| { + BackendAccessErrorKind::BackendLoadError(be_type.to_string(), err).into() + }) + }) + .transpose() + } +} + +/// Trait which can be implemented to choose a backend from a backend type, a backend path and options given as `HashMap`. +pub trait BackendChoice { + /// Init backend from a path and options. + /// + /// # Arguments + /// + /// * `path` - The path to create that points to the backend. + /// * `options` - additional options for creating the backend + /// + /// # Errors + /// + /// * [`BackendAccessErrorKind::BackendNotSupported`] - If the backend is not supported. + /// + /// [`BackendAccessErrorKind::BackendNotSupported`]: crate::error::BackendAccessErrorKind::BackendNotSupported + fn to_backend( + &self, + location: BackendLocation, + options: Option>, + ) -> Result>; +} + +/// The supported backend types. +/// +/// Currently supported types are "local", "rclone", "rest", "opendal", "s3" +/// +/// # Notes +/// +/// If the url is a windows path, the type will be "local". +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, EnumString, Display)] +pub enum SupportedBackend { + /// A local backend + #[strum(serialize = "local", to_string = "Local Backend")] + Local, + + #[cfg(feature = "rclone")] + /// A rclone backend + #[strum(serialize = "rclone", to_string = "rclone Backend")] + Rclone, + + #[cfg(feature = "rest")] + /// A REST backend + #[strum(serialize = "rest", to_string = "REST Backend")] + Rest, + + #[cfg(feature = "opendal")] + /// An openDAL backend (general) + #[strum(serialize = "opendal", to_string = "openDAL Backend")] + OpenDAL, + + #[cfg(feature = "s3")] + /// An openDAL S3 backend + #[strum(serialize = "s3", to_string = "S3 Backend")] + S3, +} + +impl BackendChoice for SupportedBackend { + fn to_backend( + &self, + location: BackendLocation, + options: Option>, + ) -> Result> { + let options = options.unwrap_or_default(); + + Ok(match self { + Self::Local => Arc::new(LocalBackend::new(location, options)?), + #[cfg(feature = "rclone")] + Self::Rclone => Arc::new(RcloneBackend::new(location, options)?), + #[cfg(feature = "rest")] + Self::Rest => Arc::new(RestBackend::new(location, options)?), + #[cfg(feature = "opendal")] + Self::OpenDAL => Arc::new(OpenDALBackend::new(location, options)?), + #[cfg(feature = "s3")] + Self::S3 => Arc::new(S3Backend::new(location, options)?), + }) + } +} + +#[cfg(test)] +mod tests { + + use rstest::rstest; + + use super::*; + + #[rstest] + #[case("local", SupportedBackend::Local)] + #[cfg(feature = "rclone")] + #[case("rclone", SupportedBackend::Rclone)] + #[cfg(feature = "rest")] + #[case("rest", SupportedBackend::Rest)] + #[cfg(feature = "opendal")] + #[case("opendal", SupportedBackend::OpenDAL)] + #[cfg(feature = "s3")] + #[case("s3", SupportedBackend::S3)] + fn test_try_from_is_ok(#[case] input: &str, #[case] expected: SupportedBackend) { + assert_eq!(SupportedBackend::try_from(input).unwrap(), expected); + } + + #[test] + fn test_try_from_unknown_is_err() { + assert!(SupportedBackend::try_from("unknown").is_err()); + } +} diff --git a/crates/backend/src/error.rs b/crates/backend/src/error.rs new file mode 100644 index 00000000..0ddc7f41 --- /dev/null +++ b/crates/backend/src/error.rs @@ -0,0 +1,130 @@ +use std::{ + num::{ParseIntError, TryFromIntError}, + process::ExitStatus, + str::Utf8Error, +}; + +use displaydoc::Display; +use thiserror::Error; + +/// [`BackendAccessErrorKind`] describes the errors that can be returned by the various Backends +#[derive(Error, Debug, Display)] +pub enum BackendAccessErrorKind { + /// backend {0:?} is not supported! + BackendNotSupported(String), + /// backend {0} cannot be loaded: {1:?} + BackendLoadError(String, anyhow::Error), + /// no suitable id found for {0} + NoSuitableIdFound(String), + /// id {0} is not unique + IdNotUnique(String), + /// {0:?} + #[error(transparent)] + FromIoError(#[from] std::io::Error), + /// {0:?} + #[error(transparent)] + FromTryIntError(#[from] TryFromIntError), + #[cfg(feature = "rest")] + /// backoff failed: {0:?} + BackoffError(#[from] backoff::Error), + /// parsing failed for url: `{0:?}` + UrlParsingFailed(#[from] url::ParseError), + /// creating data in backend failed + CreatingDataOnBackendFailed, + /// writing bytes to backend failed + WritingBytesToBackendFailed, + /// removing data from backend failed + RemovingDataFromBackendFailed, + /// failed to list files on Backend + ListingFilesOnBackendFailed, +} + +/// [`RcloneErrorKind`] describes the errors that can be returned by a backend provider +#[derive(Error, Debug, Display)] +pub enum RcloneErrorKind { + /// 'rclone version' doesn't give any output + NoOutputForRcloneVersion, + /// cannot get stdout of rclone + NoStdOutForRclone, + /// rclone exited with `{0:?}` + RCloneExitWithBadStatus(ExitStatus), + /// url must start with http:\/\/! url: {0:?} + UrlNotStartingWithHttp(String), + /// StdIo Error: `{0:?}` + #[error(transparent)] + FromIoError(#[from] std::io::Error), + /// utf8 error: `{0:?}` + #[error(transparent)] + FromUtf8Error(#[from] Utf8Error), + /// `{0:?}` + #[error(transparent)] + FromParseIntError(#[from] ParseIntError), +} + +/// [`RestErrorKind`] describes the errors that can be returned while dealing with the REST API +#[derive(Error, Debug, Display)] +pub enum RestErrorKind { + /// value `{0:?}` not supported for option retry! + NotSupportedForRetry(String), + /// parsing failed for url: `{0:?}` + UrlParsingFailed(#[from] url::ParseError), + #[cfg(feature = "rest")] + /// requesting resource failed: `{0:?}` + RequestingResourceFailed(#[from] reqwest::Error), + /// couldn't parse duration in humantime library: `{0:?}` + CouldNotParseDuration(#[from] humantime::DurationError), + #[cfg(feature = "rest")] + /// backoff failed: {0:?} + BackoffError(#[from] backoff::Error), + #[cfg(feature = "rest")] + /// Failed to build HTTP client: `{0:?}` + BuildingClientFailed(reqwest::Error), + /// joining URL failed on: {0:?} + JoiningUrlFailed(url::ParseError), +} + +/// [`LocalBackendErrorKind`] describes the errors that can be returned by an action on the filesystem in Backends +#[derive(Error, Debug, Display)] +pub enum LocalBackendErrorKind { + /// directory creation failed: `{0:?}` + DirectoryCreationFailed(#[from] std::io::Error), + /// querying metadata failed: `{0:?}` + QueryingMetadataFailed(std::io::Error), + /// querying WalkDir metadata failed: `{0:?}` + QueryingWalkDirMetadataFailed(walkdir::Error), + /// executtion of command failed: `{0:?}` + CommandExecutionFailed(std::io::Error), + /// command was not successful for filename {file_name}, type {file_type}, id {id}: {status} + CommandNotSuccessful { + file_name: String, + file_type: String, + id: String, + status: ExitStatus, + }, + /// error building automaton `{0:?}` + FromAhoCorasick(#[from] aho_corasick::BuildError), + /// {0:?} + FromSplitError(#[from] shell_words::ParseError), + /// {0:?} + #[error(transparent)] + FromTryIntError(#[from] TryFromIntError), + /// {0:?} + #[error(transparent)] + FromWalkdirError(#[from] walkdir::Error), + /// removing file failed: `{0:?}` + FileRemovalFailed(std::io::Error), + /// opening file failed: `{0:?}` + OpeningFileFailed(std::io::Error), + /// setting file length failed: `{0:?}` + SettingFileLengthFailed(std::io::Error), + /// can't jump to position in file: `{0:?}` + CouldNotSeekToPositionInFile(std::io::Error), + /// couldn't write to buffer: `{0:?}` + CouldNotWriteToBuffer(std::io::Error), + /// reading file contents failed: `{0:?}` + ReadingContentsOfFileFailed(std::io::Error), + /// reading exact length of file contents failed: `{0:?}` + ReadingExactLengthOfFileFailed(std::io::Error), + /// failed to sync OS Metadata to disk: `{0:?}` + SyncingOfOsMetadataFailed(std::io::Error), +} diff --git a/crates/backend/src/lib.rs b/crates/backend/src/lib.rs new file mode 100644 index 00000000..cc0d3cfb --- /dev/null +++ b/crates/backend/src/lib.rs @@ -0,0 +1,28 @@ +pub mod choose; +pub mod error; +pub mod local; +#[cfg(feature = "opendal")] +pub mod opendal; +#[cfg(feature = "rclone")] +pub mod rclone; +#[cfg(feature = "rest")] +pub mod rest; +pub mod util; + +// rustic_backend Public API +pub use crate::{ + choose::{BackendOptions, SupportedBackend}, + local::LocalBackend, +}; + +#[cfg(feature = "s3")] +pub use crate::opendal::s3::S3Backend; + +#[cfg(feature = "opendal")] +pub use crate::opendal::OpenDALBackend; + +#[cfg(feature = "rclone")] +pub use crate::rclone::RcloneBackend; + +#[cfg(feature = "rest")] +pub use crate::rest::RestBackend; diff --git a/crates/backend/src/local.rs b/crates/backend/src/local.rs new file mode 100644 index 00000000..32ed149b --- /dev/null +++ b/crates/backend/src/local.rs @@ -0,0 +1,393 @@ +use std::{ + fs::{self, File}, + io::{Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + process::Command, +}; + +use aho_corasick::AhoCorasick; +use anyhow::Result; +use bytes::Bytes; +use log::{debug, trace, warn}; +use shell_words::split; +use walkdir::WalkDir; + +use rustic_core::{FileType, Id, ReadBackend, WriteBackend, ALL_FILE_TYPES}; + +use crate::error::LocalBackendErrorKind; + +#[derive(Clone, Debug)] +pub struct LocalBackend { + /// The base path of the backend. + path: PathBuf, + /// The command to call after a file was created. + post_create_command: Option, + /// The command to call after a file was deleted. + post_delete_command: Option, +} + +impl LocalBackend { + /// Create a new [`LocalBackend`] + /// + /// # Arguments + /// + /// * `path` - The base path of the backend + /// + /// # Errors + /// + /// * [`LocalBackendErrorKind::DirectoryCreationFailed`] - If the directory could not be created. + /// + /// [`LocalBackendErrorKind::DirectoryCreationFailed`]: LocalBackendErrorKind::DirectoryCreationFailed + pub fn new( + path: impl AsRef, + options: impl IntoIterator, + ) -> Result { + let path = path.as_ref().into(); + fs::create_dir_all(&path).map_err(LocalBackendErrorKind::DirectoryCreationFailed)?; + let mut post_create_command = None; + let mut post_delete_command = None; + for (option, value) in options { + match option.as_str() { + "post-create-command" => { + post_create_command = Some(value); + } + "post-delete-command" => { + post_delete_command = Some(value); + } + opt => { + warn!("Option {opt} is not supported! Ignoring it."); + } + } + } + Ok(Self { + path, + post_create_command, + post_delete_command, + }) + } + + /// Path to the given file type and id. + /// + /// If the file type is `FileType::Pack`, the id will be used to determine the subdirectory. + /// + /// # Arguments + /// + /// * `tpe` - The type of the file. + /// * `id` - The id of the file. + /// + /// # Returns + /// + /// The path to the file. + fn path(&self, tpe: FileType, id: &Id) -> PathBuf { + let hex_id = id.to_hex(); + match tpe { + FileType::Config => self.path.join("config"), + FileType::Pack => self.path.join("data").join(&hex_id[0..2]).join(hex_id), + _ => self.path.join(tpe.dirname()).join(hex_id), + } + } + + /// Call the given command. + /// + /// # Arguments + /// + /// * `tpe` - The type of the file. + /// * `id` - The id of the file. + /// * `filename` - The path to the file. + /// * `command` - The command to call. + /// + /// # Errors + /// + /// * [`LocalBackendErrorKind::FromAhoCorasick`] - If the patterns could not be compiled. + /// * [`LocalBackendErrorKind::FromSplitError`] - If the command could not be parsed. + /// * [`LocalBackendErrorKind::CommandExecutionFailed`] - If the command could not be executed. + /// * [`LocalBackendErrorKind::CommandNotSuccessful`] - If the command was not successful. + /// + /// # Notes + /// + /// The following placeholders are supported: + /// * `%file` - The path to the file. + /// * `%type` - The type of the file. + /// * `%id` - The id of the file. + /// + /// [`LocalBackendErrorKind::FromAhoCorasick`]: LocalBackendErrorKind::FromAhoCorasick + /// [`LocalBackendErrorKind::FromSplitError`]: LocalBackendErrorKind::FromSplitError + /// [`LocalBackendErrorKind::CommandExecutionFailed`]: LocalBackendErrorKind::CommandExecutionFailed + /// [`LocalBackendErrorKind::CommandNotSuccessful`]: LocalBackendErrorKind::CommandNotSuccessful + fn call_command(tpe: FileType, id: &Id, filename: &Path, command: &str) -> Result<()> { + let id = id.to_hex(); + let patterns = &["%file", "%type", "%id"]; + let ac = AhoCorasick::new(patterns).map_err(LocalBackendErrorKind::FromAhoCorasick)?; + let replace_with = &[filename.to_str().unwrap(), tpe.dirname(), id.as_str()]; + let actual_command = ac.replace_all(command, replace_with); + debug!("calling {actual_command}..."); + let commands = split(&actual_command).map_err(LocalBackendErrorKind::FromSplitError)?; + let status = Command::new(&commands[0]) + .args(&commands[1..]) + .status() + .map_err(LocalBackendErrorKind::CommandExecutionFailed)?; + if !status.success() { + return Err(LocalBackendErrorKind::CommandNotSuccessful { + file_name: replace_with[0].to_owned(), + file_type: replace_with[1].to_owned(), + id: replace_with[2].to_owned(), + status, + } + .into()); + } + Ok(()) + } +} + +impl ReadBackend for LocalBackend { + /// Returns the location of the backend. + /// + /// This is `local:`. + fn location(&self) -> String { + let mut location = "local:".to_string(); + location.push_str(&self.path.to_string_lossy()); + location + } + + /// Lists all files of the given type. + /// + /// # Arguments + /// + /// * `tpe` - The type of the files to list. + /// + /// # Notes + /// + /// If the file type is `FileType::Config`, this will return a list with a single default id. + fn list(&self, tpe: FileType) -> Result> { + trace!("listing tpe: {tpe:?}"); + if tpe == FileType::Config { + return Ok(if self.path.join("config").exists() { + vec![Id::default()] + } else { + Vec::new() + }); + } + + let walker = WalkDir::new(self.path.join(tpe.dirname())) + .into_iter() + .filter_map(walkdir::Result::ok) + .filter(|e| e.file_type().is_file()) + .map(|e| Id::from_hex(&e.file_name().to_string_lossy())) + .filter_map(std::result::Result::ok); + Ok(walker.collect()) + } + + /// Lists all files with their size of the given type. + /// + /// # Arguments + /// + /// * `tpe` - The type of the files to list. + /// + /// # Errors + /// + /// * [`LocalBackendErrorKind::QueryingMetadataFailed`] - If the metadata of the file could not be queried. + /// * [`LocalBackendErrorKind::FromTryIntError`] - If the length of the file could not be converted to u32. + /// * [`LocalBackendErrorKind::QueryingWalkDirMetadataFailed`] - If the metadata of the file could not be queried. + /// + /// [`LocalBackendErrorKind::QueryingMetadataFailed`]: LocalBackendErrorKind::QueryingMetadataFailed + /// [`LocalBackendErrorKind::FromTryIntError`]: LocalBackendErrorKind::FromTryIntError + /// [`LocalBackendErrorKind::QueryingWalkDirMetadataFailed`]: LocalBackendErrorKind::QueryingWalkDirMetadataFailed + fn list_with_size(&self, tpe: FileType) -> Result> { + trace!("listing tpe: {tpe:?}"); + let path = self.path.join(tpe.dirname()); + + if tpe == FileType::Config { + return Ok(if path.exists() { + vec![( + Id::default(), + path.metadata() + .map_err(LocalBackendErrorKind::QueryingMetadataFailed)? + .len() + .try_into() + .map_err(LocalBackendErrorKind::FromTryIntError)?, + )] + } else { + Vec::new() + }); + } + + let walker = WalkDir::new(path) + .into_iter() + .filter_map(walkdir::Result::ok) + .filter(|e| e.file_type().is_file()) + .map(|e| -> Result<_> { + Ok(( + Id::from_hex(&e.file_name().to_string_lossy())?, + e.metadata() + .map_err(LocalBackendErrorKind::QueryingWalkDirMetadataFailed)? + .len() + .try_into() + .map_err(LocalBackendErrorKind::FromTryIntError)?, + )) + }) + .filter_map(Result::ok); + + Ok(walker.collect()) + } + + /// Reads full data of the given file. + /// + /// # Arguments + /// + /// * `tpe` - The type of the file. + /// * `id` - The id of the file. + /// + /// # Errors + /// + /// * [`LocalBackendErrorKind::ReadingContentsOfFileFailed`] - If the file could not be read. + /// + /// [`LocalBackendErrorKind::ReadingContentsOfFileFailed`]: LocalBackendErrorKind::ReadingContentsOfFileFailed + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + trace!("reading tpe: {tpe:?}, id: {id}"); + Ok(fs::read(self.path(tpe, id)) + .map_err(LocalBackendErrorKind::ReadingContentsOfFileFailed)? + .into()) + } + + /// Reads partial data of the given file. + /// + /// # Arguments + /// + /// * `tpe` - The type of the file. + /// * `id` - The id of the file. + /// * `cacheable` - Whether the file is cacheable. + /// * `offset` - The offset to read from. + /// * `length` - The length to read. + /// + /// # Errors + /// + /// * [`LocalBackendErrorKind::OpeningFileFailed`] - If the file could not be opened. + /// * [`LocalBackendErrorKind::CouldNotSeekToPositionInFile`] - If the file could not be seeked to the given position. + /// * [`LocalBackendErrorKind::FromTryIntError`] - If the length of the file could not be converted to u32. + /// * [`LocalBackendErrorKind::ReadingExactLengthOfFileFailed`] - If the length of the file could not be read. + /// + /// [`LocalBackendErrorKind::OpeningFileFailed`]: LocalBackendErrorKind::OpeningFileFailed + /// [`LocalBackendErrorKind::CouldNotSeekToPositionInFile`]: LocalBackendErrorKind::CouldNotSeekToPositionInFile + /// [`LocalBackendErrorKind::FromTryIntError`]: LocalBackendErrorKind::FromTryIntError + /// [`LocalBackendErrorKind::ReadingExactLengthOfFileFailed`]: LocalBackendErrorKind::ReadingExactLengthOfFileFailed + fn read_partial( + &self, + tpe: FileType, + id: &Id, + _cacheable: bool, + offset: u32, + length: u32, + ) -> Result { + trace!("reading tpe: {tpe:?}, id: {id}, offset: {offset}, length: {length}"); + let mut file = + File::open(self.path(tpe, id)).map_err(LocalBackendErrorKind::OpeningFileFailed)?; + _ = file + .seek(SeekFrom::Start(offset.into())) + .map_err(LocalBackendErrorKind::CouldNotSeekToPositionInFile)?; + let mut vec = vec![ + 0; + length + .try_into() + .map_err(LocalBackendErrorKind::FromTryIntError)? + ]; + file.read_exact(&mut vec) + .map_err(LocalBackendErrorKind::ReadingExactLengthOfFileFailed)?; + Ok(vec.into()) + } +} + +impl WriteBackend for LocalBackend { + /// Create a repository on the backend. + /// + /// # Errors + /// + /// * [`LocalBackendErrorKind::DirectoryCreationFailed`] - If the directory could not be created. + /// + /// [`LocalBackendErrorKind::DirectoryCreationFailed`]: LocalBackendErrorKind::DirectoryCreationFailed + fn create(&self) -> Result<()> { + trace!("creating repo at {:?}", self.path); + + for tpe in ALL_FILE_TYPES { + fs::create_dir_all(self.path.join(tpe.dirname())) + .map_err(LocalBackendErrorKind::DirectoryCreationFailed)?; + } + for i in 0u8..=255 { + fs::create_dir_all(self.path.join("data").join(hex::encode([i]))) + .map_err(LocalBackendErrorKind::DirectoryCreationFailed)?; + } + Ok(()) + } + + /// Write the given bytes to the given file. + /// + /// # Arguments + /// + /// * `tpe` - The type of the file. + /// * `id` - The id of the file. + /// * `cacheable` - Whether the file is cacheable. + /// * `buf` - The bytes to write. + /// + /// # Errors + /// + /// * [`LocalBackendErrorKind::OpeningFileFailed`] - If the file could not be opened. + /// * [`LocalBackendErrorKind::FromTryIntError`] - If the length of the bytes could not be converted to u64. + /// * [`LocalBackendErrorKind::SettingFileLengthFailed`] - If the length of the file could not be set. + /// * [`LocalBackendErrorKind::CouldNotWriteToBuffer`] - If the bytes could not be written to the file. + /// * [`LocalBackendErrorKind::SyncingOfOsMetadataFailed`] - If the metadata of the file could not be synced. + /// + /// [`LocalBackendErrorKind::OpeningFileFailed`]: LocalBackendErrorKind::OpeningFileFailed + /// [`LocalBackendErrorKind::FromTryIntError`]: LocalBackendErrorKind::FromTryIntError + /// [`LocalBackendErrorKind::SettingFileLengthFailed`]: LocalBackendErrorKind::SettingFileLengthFailed + /// [`LocalBackendErrorKind::CouldNotWriteToBuffer`]: LocalBackendErrorKind::CouldNotWriteToBuffer + /// [`LocalBackendErrorKind::SyncingOfOsMetadataFailed`]: LocalBackendErrorKind::SyncingOfOsMetadataFailed + fn write_bytes(&self, tpe: FileType, id: &Id, _cacheable: bool, buf: Bytes) -> Result<()> { + trace!("writing tpe: {:?}, id: {}", &tpe, &id); + let filename = self.path(tpe, id); + let mut file = fs::OpenOptions::new() + .create(true) + .write(true) + .open(&filename) + .map_err(LocalBackendErrorKind::OpeningFileFailed)?; + file.set_len( + buf.len() + .try_into() + .map_err(LocalBackendErrorKind::FromTryIntError)?, + ) + .map_err(LocalBackendErrorKind::SettingFileLengthFailed)?; + file.write_all(&buf) + .map_err(LocalBackendErrorKind::CouldNotWriteToBuffer)?; + file.sync_all() + .map_err(LocalBackendErrorKind::SyncingOfOsMetadataFailed)?; + if let Some(command) = &self.post_create_command { + if let Err(err) = Self::call_command(tpe, id, &filename, command) { + warn!("post-create: {err}"); + } + } + Ok(()) + } + + /// Remove the given file. + /// + /// # Arguments + /// + /// * `tpe` - The type of the file. + /// * `id` - The id of the file. + /// * `cacheable` - Whether the file is cacheable. + /// + /// # Errors + /// + /// * [`LocalBackendErrorKind::FileRemovalFailed`] - If the file could not be removed. + /// + /// [`LocalBackendErrorKind::FileRemovalFailed`]: LocalBackendErrorKind::FileRemovalFailed + fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> { + trace!("removing tpe: {:?}, id: {}", &tpe, &id); + let filename = self.path(tpe, id); + fs::remove_file(&filename).map_err(LocalBackendErrorKind::FileRemovalFailed)?; + if let Some(command) = &self.post_delete_command { + if let Err(err) = Self::call_command(tpe, id, &filename, command) { + warn!("post-delete: {err}"); + } + } + Ok(()) + } +} diff --git a/crates/backend/src/opendal.rs b/crates/backend/src/opendal.rs new file mode 100644 index 00000000..3d92a88e --- /dev/null +++ b/crates/backend/src/opendal.rs @@ -0,0 +1,225 @@ +#[cfg(feature = "s3")] +pub mod s3; + +use std::{collections::HashMap, path::PathBuf, str::FromStr, sync::OnceLock}; + +use anyhow::Result; +use bytes::Bytes; +use log::trace; +use opendal::{ + layers::{BlockingLayer, LoggingLayer, RetryLayer}, + BlockingOperator, ErrorKind, Metakey, Operator, Scheme, +}; +use rayon::prelude::{IntoParallelIterator, ParallelIterator}; +use tokio::runtime::Runtime; + +use rustic_core::{FileType, Id, ReadBackend, WriteBackend, ALL_FILE_TYPES}; + +mod consts { + /// Default number of retries + pub(super) const DEFAULT_RETRY: usize = 5; +} + +#[derive(Clone, Debug)] +pub struct OpenDALBackend { + operator: BlockingOperator, +} + +fn runtime() -> &'static Runtime { + static RUNTIME: OnceLock = OnceLock::new(); + RUNTIME.get_or_init(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() + }) +} + +impl OpenDALBackend { + /// Create a new openDAL backend. + /// + /// # Arguments + /// + /// * `path` - The path to the OpenDAL backend. + /// * `options` - Additional options for the OpenDAL backend. + pub fn new(path: impl AsRef, options: HashMap) -> Result { + let max_retries = match options.get("retry").map(std::string::String::as_str) { + Some("false" | "off") => 0, + None | Some("default") => consts::DEFAULT_RETRY, + Some(value) => usize::from_str(value)?, + }; + + let schema = Scheme::from_str(path.as_ref())?; + let _guard = runtime().enter(); + let operator = Operator::via_map(schema, options)? + .layer(RetryLayer::new().with_max_times(max_retries).with_jitter()) + .layer(LoggingLayer::default()) + .layer(BlockingLayer::create()?) + .blocking(); + Ok(Self { operator }) + } + + fn path(&self, tpe: FileType, id: &Id) -> String { + let hex_id = id.to_hex(); + match tpe { + FileType::Config => PathBuf::from("config"), + FileType::Pack => PathBuf::from("data").join(&hex_id[0..2]).join(hex_id), + _ => PathBuf::from(tpe.dirname()).join(hex_id), + } + .to_string_lossy() + .to_string() + } +} + +impl ReadBackend for OpenDALBackend { + /// Returns the location of the backend. + /// + /// This is `local:`. + fn location(&self) -> String { + let mut location = "opendal:".to_string(); + location.push_str(self.operator.info().name()); + location + } + + /// Lists all files of the given type. + /// + /// # Arguments + /// + /// * `tpe` - The type of the files to list. + /// + /// # Notes + /// + /// If the file type is `FileType::Config`, this will return a list with a single default id. + fn list(&self, tpe: FileType) -> Result> { + trace!("listing tpe: {tpe:?}"); + if tpe == FileType::Config { + return Ok(if self.operator.is_exist("config")? { + vec![Id::default()] + } else { + Vec::new() + }); + } + + Ok(self + .operator + .list_with(&(tpe.dirname().to_string() + "/")) + .recursive(true) + .call()? + .into_iter() + .filter(|e| e.metadata().is_file()) + .map(|e| Id::from_hex(e.name())) + .filter_map(Result::ok) + .collect()) + } + + /// Lists all files with their size of the given type. + /// + /// # Arguments + /// + /// * `tpe` - The type of the files to list. + /// + fn list_with_size(&self, tpe: FileType) -> Result> { + trace!("listing tpe: {tpe:?}"); + if tpe == FileType::Config { + return match self.operator.stat("config") { + Ok(entry) => Ok(vec![(Id::default(), entry.content_length().try_into()?)]), + Err(err) if err.kind() == ErrorKind::NotFound => Ok(Vec::new()), + Err(err) => Err(err.into()), + }; + } + + Ok(self + .operator + .list_with(&(tpe.dirname().to_string() + "/")) + .recursive(true) + .metakey(Metakey::ContentLength) + .call()? + .into_iter() + .filter(|e| e.metadata().is_file()) + .map(|e| -> Result<(Id, u32)> { + Ok(( + Id::from_hex(e.name())?, + e.metadata().content_length().try_into()?, + )) + }) + .filter_map(Result::ok) + .collect()) + } + + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + trace!("reading tpe: {tpe:?}, id: {id}"); + + Ok(self.operator.read(&self.path(tpe, id))?.into()) + } + + fn read_partial( + &self, + tpe: FileType, + id: &Id, + _cacheable: bool, + offset: u32, + length: u32, + ) -> Result { + trace!("reading tpe: {tpe:?}, id: {id}, offset: {offset}, length: {length}"); + let range = u64::from(offset)..u64::from(offset + length); + Ok(self + .operator + .read_with(&self.path(tpe, id)) + .range(range) + .call()? + .into()) + } +} + +impl WriteBackend for OpenDALBackend { + /// Create a repository on the backend. + fn create(&self) -> Result<()> { + trace!("creating repo at {:?}", self.location()); + + for tpe in ALL_FILE_TYPES { + self.operator + .create_dir(&(tpe.dirname().to_string() + "/"))?; + } + // creating 256 dirs can be slow on remote backends, hence we parallelize it. + (0u8..=255).into_par_iter().try_for_each(|i| { + self.operator.create_dir( + &(PathBuf::from("data") + .join(hex::encode([i])) + .to_string_lossy() + .to_string() + + "/"), + ) + })?; + + Ok(()) + } + + /// Write the given bytes to the given file. + /// + /// # Arguments + /// + /// * `tpe` - The type of the file. + /// * `id` - The id of the file. + /// * `cacheable` - Whether the file is cacheable. + /// * `buf` - The bytes to write. + fn write_bytes(&self, tpe: FileType, id: &Id, _cacheable: bool, buf: Bytes) -> Result<()> { + trace!("writing tpe: {:?}, id: {}", &tpe, &id); + let filename = self.path(tpe, id); + self.operator.write(&filename, buf)?; + Ok(()) + } + + /// Remove the given file. + /// + /// # Arguments + /// + /// * `tpe` - The type of the file. + /// * `id` - The id of the file. + /// * `cacheable` - Whether the file is cacheable. + fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> { + trace!("removing tpe: {:?}, id: {}", &tpe, &id); + let filename = self.path(tpe, id); + self.operator.delete(&filename)?; + Ok(()) + } +} diff --git a/crates/backend/src/opendal/s3.rs b/crates/backend/src/opendal/s3.rs new file mode 100644 index 00000000..ea067897 --- /dev/null +++ b/crates/backend/src/opendal/s3.rs @@ -0,0 +1,99 @@ +use std::collections::HashMap; + +use anyhow::Result; +use itertools::Itertools; +use url::{self, Url}; + +use crate::opendal::OpenDALBackend; +use bytes::Bytes; +use rustic_core::{FileType, Id, ReadBackend, WriteBackend}; + +#[derive(Clone, Debug)] +pub struct S3Backend(OpenDALBackend); + +impl ReadBackend for S3Backend { + fn location(&self) -> String { + self.0.location() + } + + fn list(&self, tpe: FileType) -> Result> { + self.0.list(tpe) + } + + fn list_with_size(&self, tpe: FileType) -> Result> { + self.0.list_with_size(tpe) + } + + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + self.0.read_full(tpe, id) + } + + fn read_partial( + &self, + tpe: FileType, + id: &Id, + cacheable: bool, + offset: u32, + length: u32, + ) -> Result { + self.0.read_partial(tpe, id, cacheable, offset, length) + } +} + +impl WriteBackend for S3Backend { + fn create(&self) -> Result<()> { + self.0.create() + } + + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + self.0.write_bytes(tpe, id, cacheable, buf) + } + + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + self.0.remove(tpe, id, cacheable) + } +} + +impl S3Backend { + /// Create a new S3 backend. + /// + /// # Arguments + /// + /// * `path` - The path to the s3 bucket + /// * `options` - Additional options for the s3 backend + /// + /// # Notes + /// + /// The path should be something like "`https://s3.amazonaws.com/bucket/my/repopath`" + pub fn new(path: impl AsRef, mut options: HashMap) -> Result { + let mut url = Url::parse(path.as_ref())?; + if let Some(mut path_segments) = url.path_segments() { + if let Some(bucket) = path_segments.next() { + let _ = options.insert("bucket".to_string(), bucket.to_string()); + } + let root = path_segments.join("/"); + if !root.is_empty() { + let _ = options.insert("root".to_string(), root); + } + } + if url.has_host() { + if url.scheme().is_empty() { + url.set_scheme("https") + .expect("could not set scheme to https"); + } + url.set_path(""); + url.set_query(None); + url.set_fragment(None); + let _ = options.insert("endpoint".to_string(), url.to_string()); + } + _ = options + .entry("region".to_string()) + .or_insert_with(|| "auto".to_string()); + + Ok(Self(OpenDALBackend::new("s3", options)?)) + } + + pub fn to_inner(self) -> OpenDALBackend { + self.0 + } +} diff --git a/src/backend/rclone.rs b/crates/backend/src/rclone.rs similarity index 56% rename from src/backend/rclone.rs rename to crates/backend/src/rclone.rs index 205bd1f1..6237a2d0 100644 --- a/src/backend/rclone.rs +++ b/crates/backend/src/rclone.rs @@ -1,10 +1,9 @@ use std::{ io::{BufRead, BufReader}, process::{Child, Command, Stdio}, - str, - sync::Arc, }; +use anyhow::Result; use bytes::Bytes; use log::{debug, info, warn}; use rand::{ @@ -12,80 +11,74 @@ use rand::{ thread_rng, }; -use crate::{ - backend::{rest::RestBackend, FileType, ReadBackend, WriteBackend}, - error::{ProviderErrorKind, RusticResult}, - id::Id, -}; +use crate::{error::RcloneErrorKind, rest::RestBackend}; + +use rustic_core::{FileType, Id, ReadBackend, WriteBackend}; pub(super) mod constants { /// The string to search for in the rclone output. pub(super) const SEARCHSTRING: &str = "Serving restic REST API on "; } -/// `ChildToKill` is a wrapper around a `Child` process that kills the child when it is dropped. -#[derive(Debug)] -struct ChildToKill(Child); - -impl Drop for ChildToKill { - /// Kill the child process. - fn drop(&mut self) { - debug!("killing rclone."); - self.0.kill().unwrap(); - } -} - /// `RcloneBackend` is a backend that uses rclone to access a remote backend. -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct RcloneBackend { /// The REST backend. rest: RestBackend, /// The url of the backend. url: String, /// The child data contains the child process and is used to kill the child process when the backend is dropped. - _child_data: Arc, + child: Child, +} + +impl Drop for RcloneBackend { + /// Kill the child process. + fn drop(&mut self) { + debug!("killing rclone."); + self.child.kill().unwrap(); + } } /// Get the rclone version. /// /// # Errors /// -/// * [`ProviderErrorKind::FromIoError`] - If the rclone version could not be determined. -/// * [`ProviderErrorKind::FromUtf8Error`] - If the rclone version could not be determined. -/// * [`ProviderErrorKind::NoOutputForRcloneVersion`] - If the rclone version could not be determined. -/// * [`ProviderErrorKind::FromParseIntError`] - If the rclone version could not be determined. +/// * [`RcloneErrorKind::FromIoError`] - If the rclone version could not be determined. +/// * [`RcloneErrorKind::FromUtf8Error`] - If the rclone version could not be determined. +/// * [`RcloneErrorKind::NoOutputForRcloneVersion`] - If the rclone version could not be determined. +/// * [`RcloneErrorKind::FromParseIntError`] - If the rclone version could not be determined. /// /// # Returns /// /// The rclone version as a tuple of (major, minor, patch). /// -/// [`ProviderErrorKind::FromIoError`]: crate::error::ProviderErrorKind::FromIoError -/// [`ProviderErrorKind::FromUtf8Error`]: crate::error::ProviderErrorKind::FromUtf8Error -/// [`ProviderErrorKind::NoOutputForRcloneVersion`]: crate::error::ProviderErrorKind::NoOutputForRcloneVersion -/// [`ProviderErrorKind::FromParseIntError`]: crate::error::ProviderErrorKind::FromParseIntError -fn rclone_version() -> RusticResult<(i32, i32, i32)> { +/// [`RcloneErrorKind::FromIoError`]: RcloneErrorKind::FromIoError +/// [`RcloneErrorKind::FromUtf8Error`]: RcloneErrorKind::FromUtf8Error +/// [`RcloneErrorKind::NoOutputForRcloneVersion`]: RcloneErrorKind::NoOutputForRcloneVersion +/// [`RcloneErrorKind::FromParseIntError`]: RcloneErrorKind::FromParseIntError +fn rclone_version() -> Result<(i32, i32, i32)> { let rclone_version_output = Command::new("rclone") .arg("version") .output() - .map_err(ProviderErrorKind::FromIoError)? + .map_err(RcloneErrorKind::FromIoError)? .stdout; - let rclone_version = str::from_utf8(&rclone_version_output) - .map_err(ProviderErrorKind::FromUtf8Error)? + let rclone_version = std::str::from_utf8(&rclone_version_output) + .map_err(RcloneErrorKind::FromUtf8Error)? .lines() .next() - .ok_or_else(|| ProviderErrorKind::NoOutputForRcloneVersion)? + .ok_or_else(|| RcloneErrorKind::NoOutputForRcloneVersion)? .trim_start_matches(|c: char| !c.is_numeric()); let versions: Vec<&str> = rclone_version.split(&['.', '-', ' '][..]).collect(); let major = versions[0] .parse::() - .map_err(ProviderErrorKind::FromParseIntError)?; + .map_err(RcloneErrorKind::FromParseIntError)?; let minor = versions[1] .parse::() - .map_err(ProviderErrorKind::FromParseIntError)?; + .map_err(RcloneErrorKind::FromParseIntError)?; let patch = versions[2] .parse::() - .map_err(ProviderErrorKind::FromParseIntError)?; + .map_err(RcloneErrorKind::FromParseIntError)?; Ok((major, minor, patch)) } @@ -98,20 +91,19 @@ impl RcloneBackend { /// /// # Errors /// - /// * [`ProviderErrorKind::FromIoError`] - If the rclone version could not be determined. - /// * [`ProviderErrorKind::NoStdOutForRclone`] - If the rclone version could not be determined. - /// * [`ProviderErrorKind::RCloneExitWithBadStatus`] - If rclone exited with a bad status. - /// * [`ProviderErrorKind::UrlNotStartingWithHttp`] - If the URL does not start with `http`. - /// * [`RestErrorKind::UrlParsingFailed`] - If the URL could not be parsed. - /// * [`RestErrorKind::BuildingClientFailed`] - If the client could not be built. - /// - /// [`ProviderErrorKind::FromIoError`]: crate::error::ProviderErrorKind::FromIoError - /// [`ProviderErrorKind::NoStdOutForRclone`]: crate::error::ProviderErrorKind::NoStdOutForRclone - /// [`ProviderErrorKind::RCloneExitWithBadStatus`]: crate::error::ProviderErrorKind::RCloneExitWithBadStatus - /// [`ProviderErrorKind::UrlNotStartingWithHttp`]: crate::error::ProviderErrorKind::UrlNotStartingWithHttp - /// [`RestErrorKind::UrlParsingFailed`]: crate::error::RestErrorKind::UrlParsingFailed - /// [`RestErrorKind::BuildingClientFailed`]: crate::error::RestErrorKind::BuildingClientFailed - pub fn new(url: &str) -> RusticResult { + /// * [`RcloneErrorKind::FromIoError`] - If the rclone version could not be determined. + /// * [`RcloneErrorKind::NoStdOutForRclone`] - If the rclone version could not be determined. + /// * [`RcloneErrorKind::RCloneExitWithBadStatus`] - If rclone exited with a bad status. + /// * [`RcloneErrorKind::UrlNotStartingWithHttp`] - If the URL does not start with `http`. + /// + /// [`RcloneErrorKind::FromIoError`]: RcloneErrorKind::FromIoError + /// [`RcloneErrorKind::NoStdOutForRclone`]: RcloneErrorKind::NoStdOutForRclone + /// [`RcloneErrorKind::RCloneExitWithBadStatus`]: RcloneErrorKind::RCloneExitWithBadStatus + /// [`RcloneErrorKind::UrlNotStartingWithHttp`]: RcloneErrorKind::UrlNotStartingWithHttp + pub fn new( + url: impl AsRef, + options: impl IntoIterator, + ) -> Result { match rclone_version() { Ok((major, minor, patch)) => { if major @@ -136,7 +128,7 @@ impl RcloneBackend { let user = Alphanumeric.sample_string(&mut thread_rng(), 12); let password = Alphanumeric.sample_string(&mut thread_rng(), 12); - let args = ["serve", "restic", url, "--addr", "localhost:0"]; + let args = ["serve", "restic", url.as_ref(), "--addr", "localhost:0"]; debug!("starting rclone with args {args:?}"); let mut child = Command::new("rclone") @@ -145,22 +137,22 @@ impl RcloneBackend { .args(args) .stderr(Stdio::piped()) .spawn() - .map_err(ProviderErrorKind::FromIoError)?; + .map_err(RcloneErrorKind::FromIoError)?; let mut stderr = BufReader::new( child .stderr .take() - .ok_or_else(|| ProviderErrorKind::NoStdOutForRclone)?, + .ok_or_else(|| RcloneErrorKind::NoStdOutForRclone)?, ); let rest_url = loop { - if let Some(status) = child.try_wait().map_err(ProviderErrorKind::FromIoError)? { - return Err(ProviderErrorKind::RCloneExitWithBadStatus(status).into()); + if let Some(status) = child.try_wait().map_err(RcloneErrorKind::FromIoError)? { + return Err(RcloneErrorKind::RCloneExitWithBadStatus(status).into()); } let mut line = String::new(); _ = stderr .read_line(&mut line) - .map_err(ProviderErrorKind::FromIoError)?; + .map_err(RcloneErrorKind::FromIoError)?; match line.find(constants::SEARCHSTRING) { Some(result) => { if let Some(url) = line.get(result + constants::SEARCHSTRING.len()..) { @@ -185,17 +177,17 @@ impl RcloneBackend { }); if !rest_url.starts_with("http://") { - return Err(ProviderErrorKind::UrlNotStartingWithHttp(rest_url).into()); + return Err(RcloneErrorKind::UrlNotStartingWithHttp(rest_url).into()); } let rest_url = "http://".to_string() + user.as_str() + ":" + password.as_str() + "@" + &rest_url[7..]; - debug!("using REST backend with url {url}."); - let rest = RestBackend::new(&rest_url)?; + debug!("using REST backend with url {}.", url.as_ref()); + let rest = RestBackend::new(rest_url, options)?; Ok(Self { - _child_data: Arc::new(ChildToKill(child)), - url: url.to_string(), + child, + url: String::from(url.as_ref()), rest, }) } @@ -209,30 +201,14 @@ impl ReadBackend for RcloneBackend { location } - /// Sets an option of the backend. - /// - /// # Arguments - /// - /// * `option` - The option to set. - /// * `value` - The value to set the option to. - /// - /// # Errors - /// - /// If the option is not supported. - fn set_option(&mut self, option: &str, value: &str) -> RusticResult<()> { - self.rest.set_option(option, value) - } - /// Returns the size of the given file. /// /// # Arguments /// /// * `tpe` - The type of the file. /// - /// # Errors - /// /// If the size could not be determined. - fn list_with_size(&self, tpe: FileType) -> RusticResult> { + fn list_with_size(&self, tpe: FileType) -> Result> { self.rest.list_with_size(tpe) } @@ -243,16 +219,10 @@ impl ReadBackend for RcloneBackend { /// * `tpe` - The type of the file. /// * `id` - The id of the file. /// - /// # Errors - /// - /// * [`RestErrorKind::BackoffError`] - If the backoff failed. - /// /// # Returns /// /// The data read. - /// - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError - fn read_full(&self, tpe: FileType, id: &Id) -> RusticResult { + fn read_full(&self, tpe: FileType, id: &Id) -> Result { self.rest.read_full(tpe, id) } @@ -266,15 +236,9 @@ impl ReadBackend for RcloneBackend { /// * `offset` - The offset to read from. /// * `length` - The length to read. /// - /// # Errors - /// - /// * [`RestErrorKind::BackoffError`] - If the backoff failed. - /// /// # Returns /// /// The data read. - /// - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError fn read_partial( &self, tpe: FileType, @@ -282,20 +246,14 @@ impl ReadBackend for RcloneBackend { cacheable: bool, offset: u32, length: u32, - ) -> RusticResult { + ) -> Result { self.rest.read_partial(tpe, id, cacheable, offset, length) } } impl WriteBackend for RcloneBackend { /// Creates a new file. - /// - /// # Errors - /// - /// * [`RestErrorKind::BackoffError`] - If the backoff failed. - /// - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError - fn create(&self) -> RusticResult<()> { + fn create(&self) -> Result<()> { self.rest.create() } @@ -307,13 +265,7 @@ impl WriteBackend for RcloneBackend { /// * `id` - The id of the file. /// * `cacheable` - Whether the data should be cached. /// * `buf` - The data to write. - /// - /// # Errors - /// - /// * [`RestErrorKind::BackoffError`] - If the backoff failed. - /// - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError - fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> RusticResult<()> { + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { self.rest.write_bytes(tpe, id, cacheable, buf) } @@ -324,13 +276,7 @@ impl WriteBackend for RcloneBackend { /// * `tpe` - The type of the file. /// * `id` - The id of the file. /// * `cacheable` - Whether the file is cacheable. - /// - /// # Errors - /// - /// * [`RestErrorKind::BackoffError`] - If the backoff failed. - /// - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError - fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> RusticResult<()> { + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { self.rest.remove(tpe, id, cacheable) } } diff --git a/src/backend/rest.rs b/crates/backend/src/rest.rs similarity index 75% rename from src/backend/rest.rs rename to crates/backend/src/rest.rs index 1a29331b..0423fec1 100644 --- a/src/backend/rest.rs +++ b/crates/backend/src/rest.rs @@ -1,7 +1,8 @@ use std::str::FromStr; use std::time::Duration; -use backoff::{backoff::Backoff, Error, ExponentialBackoff, ExponentialBackoffBuilder}; +use anyhow::Result; +use backoff::{backoff::Backoff, ExponentialBackoff, ExponentialBackoffBuilder}; use bytes::Bytes; use log::{trace, warn}; use reqwest::{ @@ -9,13 +10,11 @@ use reqwest::{ header::{HeaderMap, HeaderValue}, Url, }; -use serde_derive::Deserialize; +use serde::Deserialize; -use crate::{ - backend::{FileType, ReadBackend, WriteBackend}, - error::{RestErrorKind, RusticResult}, - id::Id, -}; +use crate::error::RestErrorKind; + +use rustic_core::{FileType, Id, ReadBackend, WriteBackend}; mod consts { /// Default number of retries @@ -25,7 +24,7 @@ mod consts { // trait CheckError to add user-defined method check_error on Response pub(crate) trait CheckError { /// Check reqwest Response for error and treat errors as permanent or transient - fn check_error(self) -> Result>; + fn check_error(self) -> Result>; } impl CheckError for Response { @@ -38,12 +37,14 @@ impl CheckError for Response { /// # Returns /// /// The response if it is not an error - fn check_error(self) -> Result> { + fn check_error(self) -> Result> { match self.error_for_status() { Ok(t) => Ok(t), // Note: status() always give Some(_) as it is called from a Response - Err(err) if err.status().unwrap().is_client_error() => Err(Error::Permanent(err)), - Err(err) => Err(Error::Transient { + Err(err) if err.status().unwrap().is_client_error() => { + Err(backoff::Error::Permanent(err)) + } + Err(err) => Err(backoff::Error::Transient { err, retry_after: None, }), @@ -129,9 +130,13 @@ impl RestBackend { /// * [`RestErrorKind::UrlParsingFailed`] - If the url could not be parsed. /// * [`RestErrorKind::BuildingClientFailed`] - If the client could not be built. /// - /// [`RestErrorKind::UrlParsingFailed`]: crate::error::RestErrorKind::UrlParsingFailed - /// [`RestErrorKind::BuildingClientFailed`]: crate::error::RestErrorKind::BuildingClientFailed - pub fn new(url: &str) -> RusticResult { + /// [`RestErrorKind::UrlParsingFailed`]: RestErrorKind::UrlParsingFailed + /// [`RestErrorKind::BuildingClientFailed`]: RestErrorKind::BuildingClientFailed + pub fn new( + url: impl AsRef, + options: impl IntoIterator, + ) -> Result { + let url = url.as_ref(); let url = if url.ends_with('/') { Url::parse(url).map_err(RestErrorKind::UrlParsingFailed)? } else { @@ -144,16 +149,38 @@ impl RestBackend { let mut headers = HeaderMap::new(); _ = headers.insert("User-Agent", HeaderValue::from_static("rustic")); - let client = ClientBuilder::new() + let mut client = ClientBuilder::new() .default_headers(headers) .timeout(Duration::from_secs(600)) // set default timeout to 10 minutes (we can have *large* packfiles) .build() .map_err(RestErrorKind::BuildingClientFailed)?; + let mut backoff = LimitRetryBackoff::default(); + + for (option, value) in options { + if option == "retry" { + let max_retries = match value.as_str() { + "false" | "off" => 0, + "default" => consts::DEFAULT_RETRY, + _ => usize::from_str(&value) + .map_err(|_| RestErrorKind::NotSupportedForRetry(value))?, + }; + backoff.max_retries = max_retries; + } else if option == "timeout" { + let timeout = match humantime::Duration::from_str(&value) { + Ok(val) => val, + Err(e) => return Err(RestErrorKind::CouldNotParseDuration(e).into()), + }; + client = match ClientBuilder::new().timeout(*timeout).build() { + Ok(val) => val, + Err(err) => return Err(RestErrorKind::BuildingClientFailed(err).into()), + }; + } + } Ok(Self { url, client, - backoff: LimitRetryBackoff::default(), + backoff, }) } @@ -167,7 +194,7 @@ impl RestBackend { /// # Errors /// /// If the url could not be created. - fn url(&self, tpe: FileType, id: &Id) -> RusticResult { + fn url(&self, tpe: FileType, id: &Id) -> Result { let id_path = if tpe == FileType::Config { "config".to_string() } else { @@ -196,44 +223,6 @@ impl ReadBackend for RestBackend { location } - /// Sets an option of the backend. - /// - /// # Arguments - /// - /// * `option` - The option to set. - /// * `value` - The value to set the option to. - /// - /// # Errors - /// - /// If the option is not supported. - /// - /// # Notes - /// - /// Currently supported options: - /// * `retry` - The number of retries to use for transient errors. Default is 5. Set to 0 to disable retries. - /// * `timeout` - The timeout to use for requests. Default is 10 minutes. Format is described in [humantime](https://docs.rs/humantime/2.1.0/humantime/fn.parse_duration.html). - fn set_option(&mut self, option: &str, value: &str) -> RusticResult<()> { - if option == "retry" { - let max_retries = match value { - "false" | "off" => 0, - "default" => consts::DEFAULT_RETRY, - _ => usize::from_str(value) - .map_err(|_| RestErrorKind::NotSupportedForRetry(value.into()))?, - }; - self.backoff.max_retries = max_retries; - } else if option == "timeout" { - let timeout = match humantime::Duration::from_str(value) { - Ok(val) => val, - Err(e) => return Err(RestErrorKind::CouldNotParseDuration(e).into()), - }; - self.client = match ClientBuilder::new().timeout(*timeout).build() { - Ok(val) => val, - Err(err) => return Err(RestErrorKind::BuildingClientFailed(err).into()), - }; - } - Ok(()) - } - /// Returns a list of all files of a given type with their size. /// /// # Arguments @@ -243,8 +232,6 @@ impl ReadBackend for RestBackend { /// # Errors /// /// * [`RestErrorKind::JoiningUrlFailed`] - If the url could not be created. - /// * [`RestErrorKind::BackoffError`] - If the backoff failed. - /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string /// /// # Notes /// @@ -254,10 +241,8 @@ impl ReadBackend for RestBackend { /// /// A vector of tuples containing the id and size of the files. /// - /// [`RestErrorKind::JoiningUrlFailed`]: crate::error::RestErrorKind::JoiningUrlFailed - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError - /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError - fn list_with_size(&self, tpe: FileType) -> RusticResult> { + /// [`RestErrorKind::JoiningUrlFailed`]: RestErrorKind::JoiningUrlFailed + fn list_with_size(&self, tpe: FileType) -> Result> { trace!("listing tpe: {tpe:?}"); let url = if tpe == FileType::Config { self.url @@ -326,8 +311,8 @@ impl ReadBackend for RestBackend { /// * [`reqwest::Error`] - If the request failed. /// * [`RestErrorKind::BackoffError`] - If the backoff failed. /// - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError - fn read_full(&self, tpe: FileType, id: &Id) -> RusticResult { + /// [`RestErrorKind::BackoffError`]: RestErrorKind::BackoffError + fn read_full(&self, tpe: FileType, id: &Id) -> Result { trace!("reading tpe: {tpe:?}, id: {id}"); let url = self.url(tpe, id)?; Ok(backoff::retry_notify( @@ -359,7 +344,7 @@ impl ReadBackend for RestBackend { /// /// * [`RestErrorKind::BackoffError`] - If the backoff failed. /// - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError + /// [`RestErrorKind::BackoffError`]: RestErrorKind::BackoffError fn read_partial( &self, tpe: FileType, @@ -367,7 +352,7 @@ impl ReadBackend for RestBackend { _cacheable: bool, offset: u32, length: u32, - ) -> RusticResult { + ) -> Result { trace!("reading tpe: {tpe:?}, id: {id}, offset: {offset}, length: {length}"); let offset2 = offset + length - 1; let header_value = format!("bytes={offset}-{offset2}"); @@ -396,8 +381,8 @@ impl WriteBackend for RestBackend { /// /// * [`RestErrorKind::BackoffError`] - If the backoff failed. /// - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError - fn create(&self) -> RusticResult<()> { + /// [`RestErrorKind::BackoffError`]: RestErrorKind::BackoffError + fn create(&self) -> Result<()> { let url = self .url .join("?create=true") @@ -426,15 +411,8 @@ impl WriteBackend for RestBackend { /// /// * [`RestErrorKind::BackoffError`] - If the backoff failed. /// - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError - // TODO: If the file is not cacheable, the bytes could be written to a temporary file and then moved to the final location. - fn write_bytes( - &self, - tpe: FileType, - id: &Id, - _cacheable: bool, - buf: Bytes, - ) -> RusticResult<()> { + /// [`RestErrorKind::BackoffError`]: RestErrorKind::BackoffError + fn write_bytes(&self, tpe: FileType, id: &Id, _cacheable: bool, buf: Bytes) -> Result<()> { trace!("writing tpe: {:?}, id: {}", &tpe, &id); let req_builder = self.client.post(self.url(tpe, id)?).body(buf); Ok(backoff::retry_notify( @@ -461,8 +439,8 @@ impl WriteBackend for RestBackend { /// /// * [`RestErrorKind::BackoffError`] - If the backoff failed. /// - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError - fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> RusticResult<()> { + /// [`RestErrorKind::BackoffError`]: RestErrorKind::BackoffError + fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> Result<()> { trace!("removing tpe: {:?}, id: {}", &tpe, &id); let url = self.url(tpe, id)?; Ok(backoff::retry_notify( diff --git a/crates/backend/src/util.rs b/crates/backend/src/util.rs new file mode 100644 index 00000000..9f764149 --- /dev/null +++ b/crates/backend/src/util.rs @@ -0,0 +1,207 @@ +use crate::SupportedBackend; +use anyhow::Result; + +#[derive(PartialEq, Debug)] +pub struct BackendLocation(String); + +impl std::ops::Deref for BackendLocation { + type Target = String; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl AsRef for BackendLocation { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for BackendLocation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0)?; + Ok(()) + } +} + +/// Splits the given url into the backend type and the path. +/// +/// # Arguments +/// +/// * `url` - The url to split. +/// +/// # Returns +/// +/// A tuple with the backend type and the path. +/// +/// # Notes +/// +/// If the url is a windows path, the type will be "local". +pub fn location_to_type_and_path( + raw_location: &str, +) -> Result<(SupportedBackend, BackendLocation)> { + match raw_location.split_once(':') { + #[cfg(windows)] + Some((drive_letter, _)) if drive_letter.len() == 1 && !raw_location.contains('/') => Ok(( + SupportedBackend::Local, + BackendLocation(raw_location.to_string()), + )), + #[cfg(windows)] + Some((scheme, path)) if scheme.contains('\\') || path.contains('\\') => Ok(( + SupportedBackend::Local, + BackendLocation(raw_location.to_string()), + )), + Some((scheme, path)) => Ok(( + SupportedBackend::try_from(scheme)?, + BackendLocation(path.to_string()), + )), + None => Ok(( + SupportedBackend::Local, + BackendLocation(raw_location.to_string()), + )), + } +} + +#[cfg(test)] +mod tests { + + #[allow(unused_imports)] + use rstest::rstest; + + #[allow(unused_imports)] + use super::*; + + #[rstest] + #[cfg(not(windows))] + #[case("local:/tmp/repo", (SupportedBackend::Local, BackendLocation::try_from("/tmp/repo").unwrap()))] + #[cfg(not(windows))] + #[case("/tmp/repo", (SupportedBackend::Local, BackendLocation::try_from("/tmp/repo").unwrap()))] + #[cfg(feature = "rclone")] + #[case( + "rclone:remote:/tmp/repo", + (SupportedBackend::Rclone, + BackendLocation::try_from("remote:/tmp/repo").unwrap()) + )] + #[cfg(feature = "rest")] + #[case( + "rest:https://example.com/tmp/repo", + (SupportedBackend::Rest, + BackendLocation::try_from("https://example.com/tmp/repo").unwrap()) + )] + #[cfg(feature = "opendal")] + #[case( + "opendal:https://example.com/tmp/repo", + (SupportedBackend::OpenDAL, + BackendLocation::try_from("https://example.com/tmp/repo").unwrap()) + )] + #[cfg(feature = "s3")] + #[case( + "s3:https://example.com/tmp/repo", + (SupportedBackend::S3, + BackendLocation::try_from("https://example.com/tmp/repo").unwrap()) + )] + #[cfg(windows)] + #[case( + r#"C:\tmp\repo"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"C:\tmp\repo"#).unwrap()) + )] + #[should_panic] + #[cfg(windows)] + #[case( + r#"C:/tmp/repo"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"C:/tmp/repo"#).unwrap()) + )] + #[cfg(windows)] + #[case( + r#"\\.\C:\Test\repo"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"\\.\C:\Test\repo"#).unwrap()) + )] + #[cfg(windows)] + #[case( + r#"\\?\C:\Test\repo"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"\\?\C:\Test\repo"#).unwrap()) + )] + #[cfg(windows)] + #[case( + r#"\\.\Volume{b75e2c83-0000-0000-0000-602f00000000}\Test\repo"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"\\.\Volume{b75e2c83-0000-0000-0000-602f00000000}\Test\repo"#).unwrap()) + )] + #[cfg(windows)] + #[case( + r#"\\?\Volume{b75e2c83-0000-0000-0000-602f00000000}\Test\repo"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"\\?\Volume{b75e2c83-0000-0000-0000-602f00000000}\Test\repo"#).unwrap()) + )] + #[cfg(windows)] + #[case( + r#"\\Server2\Share\Test\repo"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"\\Server2\Share\Test\repo"#).unwrap()) + )] + #[cfg(windows)] + #[case( + r#"\\?\UNC\Server\Share\Test\repo"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"\\?\UNC\Server\Share\Test\repo"#).unwrap()) + )] + #[cfg(windows)] + #[case( + r#"C:\Projects\apilibrary\"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"C:\Projects\apilibrary\"#).unwrap()) + )] + // A relative path from the current directory of the C: drive. + #[cfg(windows)] + #[case( + r#"C:Projects\apilibrary\"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"C:Projects\apilibrary\"#).unwrap()) + )] + // A relative path from the root of the current drive. + #[cfg(windows)] + #[case( + r#"\Program Files\Custom Utilities\rustic\Repositories\"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"\Program Files\Custom Utilities\rustic\Repositories\"#).unwrap()) + )] + #[should_panic] + #[cfg(windows)] + #[case( + r#"..\Publications\TravelBrochures\"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"..\Publications\TravelBrochures\"#).unwrap()) + )] + #[should_panic] + #[cfg(windows)] + #[case( + r#"2023\repos\"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"2023\repos\"#).unwrap()) + )] + // The root directory of the C: drive on localhost. + #[cfg(windows)] + #[case( + r#"\\localhost\C$\"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"\\localhost\C$\"#).unwrap()) + )] + #[cfg(windows)] + #[case( + r#"\\127.0.0.1\c$\temp\repo\"#, + (SupportedBackend::Local, + BackendLocation::try_from(r#"\\127.0.0.1\c$\temp\repo\"#).unwrap()) + )] + fn test_location_to_type_and_path_is_ok( + #[case] url: &str, + #[case] expected: (SupportedBackend, BackendLocation), + ) { + // Check https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats + assert_eq!(location_to_type_and_path(url).unwrap(), expected); + } +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml new file mode 100644 index 00000000..c810f330 --- /dev/null +++ b/crates/core/Cargo.toml @@ -0,0 +1,136 @@ +[package] +name = "rustic_core" +version = "0.1.2" +authors = ["Alexander Weiss"] +categories = [ + "Algorithms", + "Compression", + "Cryptography", + "Data structures", + "Filesystem", +] +documentation = "https://docs.rs/rustic_core" +edition = "2021" +homepage = "https://rustic.cli.rs/" +include = ["src/**/*", "LICENSE-*", "README.md"] +keywords = ["backup", "restic", "deduplication", "encryption", "library"] +license = "Apache-2.0 OR MIT" +publish = true +readme = "README.md" +repository = "https://github.com/rustic-rs/rustic_core" +resolver = "2" +rust-version = { workspace = true } +description = """ +rustic_core - library for fast, encrypted, deduplicated backups that powers rustic-rs +""" + +[lib] +path = "src/lib.rs" +name = "rustic_core" +test = true +doctest = true +bench = true +doc = true +harness = true +edition = "2021" + +[features] +default = [] +cli = ["merge", "clap"] +merge = ["dep:merge"] +clap = ["dep:clap"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--document-private-items", "--generate-link-to-definition"] + +[dependencies] +# errors +displaydoc = "0.2.4" +thiserror = "1.0.56" + +# macros +derivative = "2.2.0" +derive_more = "0.99.17" +derive_setters = "0.1.6" + +# logging +log = "0.4.20" + +# parallelize +crossbeam-channel = "0.5.11" +pariter = "0.5.1" +rayon = "1.8.0" + +# crypto +aes256ctr_poly1305aes = "0.2.0" +rand = "0.8.5" +scrypt = { version = "0.11.0", default-features = false } + +# serialization / packing +binrw = "0.13.3" +hex = { version = "0.4.3", features = ["serde"] } +integer-sqrt = "0.1.5" +serde = { version = "1.0.195" } +serde-aux = "4.3.1" +serde_derive = "1.0.195" +serde_json = "1.0.111" +serde_with = { version = "3.4.0", features = ["base64"] } + +# local source/destination +cached = { version = "0.47.0", default-features = false, features = ["proc_macro"] } +dunce = "1.0.4" +filetime = "0.2.23" +ignore = "0.4.22" +nix = { version = "0.27.1", default-features = false, features = ["user", "fs"] } +path-dedot = "3.1.1" +shell-words = "1.1.0" +walkdir = "2.4.0" + +# cache +cachedir = "0.3.1" +dirs = "5.0.1" + +# cli support +clap = { version = "4.4.14", optional = true, features = ["derive", "env", "wrap_help"] } +merge = { version = "0.1.0", optional = true } + +# other dependencies +anyhow = "1.0.79" +bytes = "1.5.0" +bytesize = "1.3.0" +chrono = { version = "0.4.31", default-features = false, features = ["clock", "serde"] } +enum-map = "2.7.3" +enum-map-derive = "0.17.0" +gethostname = "0.4.3" +humantime = "2.1.0" +itertools = "0.12.0" +zstd = "0.13.0" + +[target.'cfg(not(windows))'.dependencies] +sha2 = { version = "0.10", features = ["asm"] } + +[target.'cfg(all(windows, not(target_env="gnu")))'.dependencies] +# unfortunately, the asm extensions do not build on MSVC, see https://github.com/RustCrypto/asm-hashes/issues/17 +sha2 = "0.10" + +[target.'cfg(all(windows, target_env="gnu"))'.dependencies] +sha2 = { version = "0.10", features = ["asm"] } + +[target.'cfg(not(any(windows, target_os="openbsd")))'.dependencies] +# for local source/destination +xattr = "1" + +[dev-dependencies] +expect-test = "1.4.1" +pretty_assertions = "1.4.0" +public-api = "0.33.1" +quickcheck = "1.0.3" +quickcheck_macros = "1.0.0" +rstest = "0.18.2" +rustdoc-json = "0.8.8" +# We need to have rustic_backend here, because the doc-tests in lib.rs of rustic_core +rustic_backend = { workspace = true } +rustup-toolchain = "0.1.6" +simplelog = "0.12.1" +tempfile = "3.9.0" diff --git a/src/archiver.rs b/crates/core/src/archiver.rs similarity index 87% rename from src/archiver.rs rename to crates/core/src/archiver.rs index 5d4df208..db136b87 100644 --- a/src/archiver.rs +++ b/crates/core/src/archiver.rs @@ -14,9 +14,9 @@ use crate::{ file_archiver::FileArchiver, parent::Parent, tree::TreeIterator, tree_archiver::TreeArchiver, }, - backend::{decrypt::DecryptWriteBackend, ReadSource, ReadSourceEntry}, + backend::{decrypt::DecryptFullBackend, ReadSource, ReadSourceEntry}, blob::BlobType, - index::{indexer::Indexer, indexer::SharedIndexer, IndexedBackend}, + index::{indexer::Indexer, indexer::SharedIndexer, ReadGlobalIndex}, repofile::{configfile::ConfigFile, snapshotfile::SnapshotFile}, Progress, RusticResult, }; @@ -29,12 +29,12 @@ use crate::{ /// * `BE` - The backend type. /// * `I` - The index to read from. #[allow(missing_debug_implementations)] -pub struct Archiver { +pub struct Archiver<'a, BE: DecryptFullBackend, I: ReadGlobalIndex> { /// The `FileArchiver` is responsible for archiving files. - file_archiver: FileArchiver, + file_archiver: FileArchiver<'a, BE, I>, /// The `TreeArchiver` is responsible for archiving trees. - tree_archiver: TreeArchiver, + tree_archiver: TreeArchiver<'a, BE, I>, /// The parent snapshot to use. parent: Parent, @@ -45,11 +45,14 @@ pub struct Archiver { /// The backend to write to. be: BE, + /// The backend to write to. + index: &'a I, + /// The SnapshotFile to write to. snap: SnapshotFile, } -impl Archiver { +impl<'a, BE: DecryptFullBackend, I: ReadGlobalIndex> Archiver<'a, BE, I> { /// Creates a new `Archiver`. /// /// # Arguments @@ -69,7 +72,7 @@ impl Archiver { /// [`PackerErrorKind::IntConversionFailed`]: crate::error::PackerErrorKind::IntConversionFailed pub fn new( be: BE, - index: I, + index: &'a I, config: &ConfigFile, parent: Parent, mut snap: SnapshotFile, @@ -78,7 +81,7 @@ impl Archiver { let mut summary = snap.summary.take().unwrap_or_default(); summary.backup_start = Local::now(); - let file_archiver = FileArchiver::new(be.clone(), index.clone(), indexer.clone(), config)?; + let file_archiver = FileArchiver::new(be.clone(), index, indexer.clone(), config)?; let tree_archiver = TreeArchiver::new(be.clone(), index, indexer.clone(), config, summary)?; Ok(Self { file_archiver, @@ -86,6 +89,7 @@ impl Archiver { parent, indexer, be, + index, snap, }) } @@ -118,7 +122,6 @@ impl Archiver { /// [`SnapshotFileErrorKind::OutOfRange`]: crate::error::SnapshotFileErrorKind::OutOfRange pub fn archive( mut self, - index: &I, src: R, backup_path: &Path, as_path: Option<&PathBuf>, @@ -170,13 +173,15 @@ impl Archiver { scope(|scope| -> RusticResult<_> { // use parent snapshot - iter.filter_map(|item| match self.parent.process(index, item) { - Ok(item) => Some(item), - Err(err) => { - warn!("ignoring error reading parent snapshot: {err:?}"); - None - } - }) + iter.filter_map( + |item| match self.parent.process(&self.be, self.index, item) { + Ok(item) => Some(item), + Err(err) => { + warn!("ignoring error reading parent snapshot: {err:?}"); + None + } + }, + ) // archive files in parallel .parallel_map_scoped(scope, |item| self.file_archiver.process(item, p)) .readahead_scoped(scope) diff --git a/src/archiver/file_archiver.rs b/crates/core/src/archiver/file_archiver.rs similarity index 94% rename from src/archiver/file_archiver.rs rename to crates/core/src/archiver/file_archiver.rs index 232f5b59..4b0d96b7 100644 --- a/src/archiver/file_archiver.rs +++ b/crates/core/src/archiver/file_archiver.rs @@ -18,9 +18,8 @@ use crate::{ cdc::rolling_hash::Rabin64, chunker::ChunkIter, crypto::hasher::hash, - error::ArchiverErrorKind, - error::RusticResult, - index::{indexer::SharedIndexer, IndexedBackend}, + error::{ArchiverErrorKind, RusticResult}, + index::{indexer::SharedIndexer, ReadGlobalIndex}, progress::Progress, repofile::configfile::ConfigFile, }; @@ -33,13 +32,13 @@ use crate::{ /// * `BE` - The backend type. /// * `I` - The index to read from. #[derive(Clone)] -pub(crate) struct FileArchiver { - index: I, +pub(crate) struct FileArchiver<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> { + index: &'a I, data_packer: Packer, rabin: Rabin64, } -impl FileArchiver { +impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> FileArchiver<'a, BE, I> { /// Creates a new `FileArchiver`. /// /// # Type Parameters @@ -63,7 +62,7 @@ impl FileArchiver { /// [`PackerErrorKind::IntConversionFailed`]: crate::error::PackerErrorKind::IntConversionFailed pub(crate) fn new( be: BE, - index: I, + index: &'a I, indexer: SharedIndexer, config: &ConfigFile, ) -> RusticResult { diff --git a/src/archiver/parent.rs b/crates/core/src/archiver/parent.rs similarity index 92% rename from src/archiver/parent.rs rename to crates/core/src/archiver/parent.rs index d7289a7b..46c181f8 100644 --- a/src/archiver/parent.rs +++ b/crates/core/src/archiver/parent.rs @@ -6,8 +6,12 @@ use std::{ use log::warn; use crate::{ - archiver::tree::TreeType, backend::node::Node, blob::tree::Tree, error::ArchiverErrorKind, - error::RusticResult, id::Id, index::IndexedBackend, + archiver::tree::TreeType, + backend::{decrypt::DecryptReadBackend, node::Node}, + blob::tree::Tree, + error::{ArchiverErrorKind, RusticResult}, + id::Id, + index::ReadGlobalIndex, }; /// The `ItemWithParent` is a `TreeType` wrapping the result of a parent search and a type `O`. @@ -85,14 +89,15 @@ impl Parent { /// * `tree_id` - The tree id of the parent tree. /// * `ignore_ctime` - Ignore ctime when comparing nodes. /// * `ignore_inode` - Ignore inode number when comparing nodes. - pub(crate) fn new( - be: &BE, + pub(crate) fn new( + be: &impl DecryptReadBackend, + index: &impl ReadGlobalIndex, tree_id: Option, ignore_ctime: bool, ignore_inode: bool, ) -> Self { // if tree_id is given, try to load tree from backend. - let tree = tree_id.and_then(|tree_id| match Tree::from_backend(be, tree_id) { + let tree = tree_id.and_then(|tree_id| match Tree::from_backend(be, index, tree_id) { Ok(tree) => Some(tree), Err(err) => { warn!("ignoring error when loading parent tree {tree_id}: {err}"); @@ -184,14 +189,19 @@ impl Parent { /// /// * `be` - The backend to read from. /// * `name` - The name of the parent node. - fn set_dir(&mut self, be: &BE, name: &OsStr) { + fn set_dir( + &mut self, + be: &impl DecryptReadBackend, + index: &impl ReadGlobalIndex, + name: &OsStr, + ) { let tree = self.p_node(name).and_then(|p_node| { p_node.subtree.map_or_else( || { warn!("ignoring parent node {}: is no tree!", p_node.name); None }, - |tree_id| match Tree::from_backend(be, tree_id) { + |tree_id| match Tree::from_backend(be, index, tree_id) { Ok(tree) => Some(tree), Err(err) => { warn!("ignoring error when loading parent tree {tree_id}: {err}"); @@ -246,9 +256,10 @@ impl Parent { /// * [`ArchiverErrorKind::TreeStackEmpty`] - If the tree stack is empty. /// /// [`ArchiverErrorKind::TreeStackEmpty`]: crate::error::ArchiverErrorKind::TreeStackEmpty - pub(crate) fn process( + pub(crate) fn process( &mut self, - be: &BE, + be: &impl DecryptReadBackend, + index: &impl ReadGlobalIndex, item: TreeType, ) -> RusticResult> { let result = match item { @@ -256,7 +267,7 @@ impl Parent { let parent_result = self .is_parent(&node, &tree) .map(|node| node.subtree.unwrap()); - self.set_dir(be, &tree); + self.set_dir(be, index, &tree); TreeType::NewTree((path, node, parent_result)) } TreeType::EndTree => { @@ -264,11 +275,10 @@ impl Parent { TreeType::EndTree } TreeType::Other((path, mut node, open)) => { - let be = be.clone(); let parent = self.is_parent(&node, &node.name()); let parent = match parent { ParentResult::Matched(p_node) => { - if p_node.content.iter().flatten().all(|id| be.has_data(id)) { + if p_node.content.iter().flatten().all(|id| index.has_data(id)) { node.content = Some(p_node.content.iter().flatten().copied().collect()); ParentResult::Matched(()) } else { diff --git a/src/archiver/tree.rs b/crates/core/src/archiver/tree.rs similarity index 98% rename from src/archiver/tree.rs rename to crates/core/src/archiver/tree.rs index 7e7cd043..cc5c4297 100644 --- a/src/archiver/tree.rs +++ b/crates/core/src/archiver/tree.rs @@ -1,7 +1,7 @@ use std::{ffi::OsString, path::PathBuf}; use crate::{ - backend::{node::Metadata, node::Node, node::NodeType}, + backend::node::{Metadata, Node, NodeType}, blob::tree::comp_to_osstr, }; diff --git a/src/archiver/tree_archiver.rs b/crates/core/src/archiver/tree_archiver.rs similarity index 95% rename from src/archiver/tree_archiver.rs rename to crates/core/src/archiver/tree_archiver.rs index ee9441b7..b6728d5b 100644 --- a/src/archiver/tree_archiver.rs +++ b/crates/core/src/archiver/tree_archiver.rs @@ -7,10 +7,9 @@ use crate::{ archiver::{parent::ParentResult, tree::TreeType}, backend::{decrypt::DecryptWriteBackend, node::Node}, blob::{packer::Packer, tree::Tree, BlobType}, - error::ArchiverErrorKind, - error::RusticResult, + error::{ArchiverErrorKind, RusticResult}, id::Id, - index::{indexer::SharedIndexer, IndexedBackend}, + index::{indexer::SharedIndexer, ReadGlobalIndex}, repofile::{configfile::ConfigFile, snapshotfile::SnapshotSummary}, }; @@ -24,20 +23,20 @@ pub(crate) type TreeItem = TreeType<(ParentResult<()>, u64), ParentResult>; /// * `I` - The index to read from. /// // TODO: Add documentation -pub(crate) struct TreeArchiver { +pub(crate) struct TreeArchiver<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> { /// The current tree. tree: Tree, /// The stack of trees. stack: Vec<(PathBuf, Node, ParentResult, Tree)>, /// The index to read from. - index: I, + index: &'a I, /// The packer to write to. tree_packer: Packer, /// The summary of the snapshot. summary: SnapshotSummary, } -impl TreeArchiver { +impl<'a, BE: DecryptWriteBackend, I: ReadGlobalIndex> TreeArchiver<'a, BE, I> { /// Creates a new `TreeArchiver`. /// /// # Type Parameters @@ -62,7 +61,7 @@ impl TreeArchiver { /// [`PackerErrorKind::IntConversionFailed`]: crate::error::PackerErrorKind::IntConversionFailed pub(crate) fn new( be: BE, - index: I, + index: &'a I, indexer: SharedIndexer, config: &ConfigFile, summary: SnapshotSummary, @@ -135,7 +134,7 @@ impl TreeArchiver { fn add_file(&mut self, path: &Path, node: Node, parent: &ParentResult<()>, size: u64) { let filename = path.join(node.name()); match parent { - ParentResult::Matched(_) => { + ParentResult::Matched(()) => { debug!("unchanged file: {:?}", filename); self.summary.files_unmodified += 1; } diff --git a/src/backend.rs b/crates/core/src/backend.rs similarity index 60% rename from src/backend.rs rename to crates/core/src/backend.rs index ee14deae..d31fcfbf 100644 --- a/src/backend.rs +++ b/crates/core/src/backend.rs @@ -1,22 +1,28 @@ +//! Module for backend related functionality. + pub(crate) mod cache; -pub(crate) mod choose; pub(crate) mod decrypt; pub(crate) mod dry_run; pub(crate) mod hotcold; pub(crate) mod ignore; -pub(crate) mod local; +pub(crate) mod local_destination; pub(crate) mod node; -pub(crate) mod rclone; -pub(crate) mod rest; pub(crate) mod stdin; +pub(crate) mod warm_up; -use std::{io::Read, path::PathBuf}; +use std::{io::Read, ops::Deref, path::PathBuf, sync::Arc}; +use anyhow::Result; use bytes::Bytes; use log::trace; use serde_derive::{Deserialize, Serialize}; -use crate::{backend::node::Node, error::BackendErrorKind, error::RusticResult, id::Id}; +use crate::{ + backend::node::Node, + error::{BackendAccessErrorKind, RusticErrorKind}, + id::Id, + RusticResult, +}; /// All [`FileType`]s which are located in separated directories pub const ALL_FILE_TYPES: [FileType; 4] = [ @@ -47,7 +53,8 @@ pub enum FileType { } impl FileType { - const fn dirname(self) -> &'static str { + /// Returns the directory name of the file type. + pub const fn dirname(self) -> &'static str { match self { Self::Config => "config", Self::Snapshot => "snapshots", @@ -69,22 +76,10 @@ impl FileType { /// Trait for backends that can read. /// /// This trait is implemented by all backends that can read data. -pub trait ReadBackend: Clone + Send + Sync + 'static { +pub trait ReadBackend: Send + Sync + 'static { /// Returns the location of the backend. fn location(&self) -> String; - /// Sets an option of the backend. - /// - /// # Arguments - /// - /// * `option` - The option to set. - /// * `value` - The value to set the option to. - /// - /// # Errors - /// - /// If the option is not supported. - fn set_option(&mut self, option: &str, value: &str) -> RusticResult<()>; - /// Lists all files with their size of the given type. /// /// # Arguments @@ -94,7 +89,7 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { /// # Errors /// /// If the files could not be listed. - fn list_with_size(&self, tpe: FileType) -> RusticResult>; + fn list_with_size(&self, tpe: FileType) -> Result>; /// Lists all files of the given type. /// @@ -105,7 +100,7 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { /// # Errors /// /// If the files could not be listed. - fn list(&self, tpe: FileType) -> RusticResult> { + fn list(&self, tpe: FileType) -> Result> { Ok(self .list_with_size(tpe)? .into_iter() @@ -123,7 +118,7 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { /// # Errors /// /// If the file could not be read. - fn read_full(&self, tpe: FileType, id: &Id) -> RusticResult; + fn read_full(&self, tpe: FileType, id: &Id) -> Result; /// Reads partial data of the given file. /// @@ -145,8 +140,36 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { cacheable: bool, offset: u32, length: u32, - ) -> RusticResult; + ) -> Result; + + /// Specify if the backend needs a warming-up of files before accessing them. + fn needs_warm_up(&self) -> bool { + false + } + + /// Warm-up the given file. + /// + /// # Arguments + /// + /// * `tpe` - The type of the file. + /// * `id` - The id of the file. + /// + /// # Errors + /// + /// If the file could not be read. + fn warm_up(&self, _tpe: FileType, _id: &Id) -> Result<()> { + Ok(()) + } +} +/// Trait for Searching in a backend. +/// +/// This trait is implemented by all backends that can be searched in. +/// +/// # Note +/// +/// This trait is used to find the id of a snapshot that contains a given file name. +pub trait FindInBackend: ReadBackend { /// Finds the id of the file starting with the given string. /// /// # Type Parameters @@ -160,16 +183,15 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { /// /// # Errors /// - /// * [`BackendErrorKind::NoSuitableIdFound`] - If no id could be found. - /// * [`BackendErrorKind::IdNotUnique`] - If the id is not unique. + /// * [`BackendAccessErrorKind::NoSuitableIdFound`] - If no id could be found. + /// * [`BackendAccessErrorKind::IdNotUnique`] - If the id is not unique. /// /// # Note /// - /// This function is used to find the id of a snapshot or index file. - /// The id of a snapshot or index file is the id of the first pack file. + /// This function is used to find the id of a snapshot. /// - /// [`BackendErrorKind::NoSuitableIdFound`]: crate::error::BackendErrorKind::NoSuitableIdFound - /// [`BackendErrorKind::IdNotUnique`]: crate::error::BackendErrorKind::IdNotUnique + /// [`BackendAccessErrorKind::NoSuitableIdFound`]: crate::error::BackendAccessErrorKind::NoSuitableIdFound + /// [`BackendAccessErrorKind::IdNotUnique`]: crate::error::BackendAccessErrorKind::IdNotUnique fn find_starts_with>(&self, tpe: FileType, vec: &[T]) -> RusticResult> { #[derive(Clone, Copy, PartialEq, Eq)] enum MapResult { @@ -178,7 +200,7 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { NonUnique, } let mut results = vec![MapResult::None; vec.len()]; - for id in self.list(tpe)? { + for id in self.list(tpe).map_err(RusticErrorKind::Backend)? { let id_hex = id.to_hex(); for (i, v) in vec.iter().enumerate() { if id_hex.starts_with(v.as_ref()) { @@ -196,11 +218,12 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { .enumerate() .map(|(i, id)| match id { MapResult::Some(id) => Ok(id), - MapResult::None => { - Err(BackendErrorKind::NoSuitableIdFound((vec[i]).as_ref().to_string()).into()) - } + MapResult::None => Err(BackendAccessErrorKind::NoSuitableIdFound( + (vec[i]).as_ref().to_string(), + ) + .into()), MapResult::NonUnique => { - Err(BackendErrorKind::IdNotUnique((vec[i]).as_ref().to_string()).into()) + Err(BackendAccessErrorKind::IdNotUnique((vec[i]).as_ref().to_string()).into()) } }) .collect() @@ -216,12 +239,12 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { /// # Errors /// /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string - /// * [`BackendErrorKind::NoSuitableIdFound`] - If no id could be found. - /// * [`BackendErrorKind::IdNotUnique`] - If the id is not unique. + /// * [`BackendAccessErrorKind::NoSuitableIdFound`] - If no id could be found. + /// * [`BackendAccessErrorKind::IdNotUnique`] - If the id is not unique. /// /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError - /// [`BackendErrorKind::NoSuitableIdFound`]: crate::error::BackendErrorKind::NoSuitableIdFound - /// [`BackendErrorKind::IdNotUnique`]: crate::error::BackendErrorKind::IdNotUnique + /// [`BackendAccessErrorKind::NoSuitableIdFound`]: crate::error::BackendAccessErrorKind::NoSuitableIdFound + /// [`BackendAccessErrorKind::IdNotUnique`]: crate::error::BackendAccessErrorKind::IdNotUnique fn find_id(&self, tpe: FileType, id: &str) -> RusticResult { Ok(self.find_ids(tpe, &[id.to_string()])?.remove(0)) } @@ -240,12 +263,12 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { /// # Errors /// /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string - /// * [`BackendErrorKind::NoSuitableIdFound`] - If no id could be found. - /// * [`BackendErrorKind::IdNotUnique`] - If the id is not unique. + /// * [`BackendAccessErrorKind::NoSuitableIdFound`] - If no id could be found. + /// * [`BackendAccessErrorKind::IdNotUnique`] - If the id is not unique. /// /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError - /// [`BackendErrorKind::NoSuitableIdFound`]: crate::error::BackendErrorKind::NoSuitableIdFound - /// [`BackendErrorKind::IdNotUnique`]: crate::error::BackendErrorKind::IdNotUnique + /// [`BackendAccessErrorKind::NoSuitableIdFound`]: crate::error::BackendAccessErrorKind::NoSuitableIdFound + /// [`BackendAccessErrorKind::IdNotUnique`]: crate::error::BackendAccessErrorKind::IdNotUnique fn find_ids>(&self, tpe: FileType, ids: &[T]) -> RusticResult> { ids.iter() .map(|id| Id::from_hex(id.as_ref())) @@ -256,11 +279,13 @@ pub trait ReadBackend: Clone + Send + Sync + 'static { } } +impl FindInBackend for T {} + /// Trait for backends that can write. /// This trait is implemented by all backends that can write data. pub trait WriteBackend: ReadBackend { /// Creates a new backend. - fn create(&self) -> RusticResult<()>; + fn create(&self) -> Result<()>; /// Writes bytes to the given file. /// @@ -268,9 +293,9 @@ pub trait WriteBackend: ReadBackend { /// /// * `tpe` - The type of the file. /// * `id` - The id of the file. - /// * `cacheable` - Whether the data should be cached. + /// * `cacheable` - Whether the data can be cached. /// * `buf` - The data to write. - fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> RusticResult<()>; + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()>; /// Removes the given file. /// @@ -279,7 +304,51 @@ pub trait WriteBackend: ReadBackend { /// * `tpe` - The type of the file. /// * `id` - The id of the file. /// * `cacheable` - Whether the file is cacheable. - fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> RusticResult<()>; + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()>; +} + +impl WriteBackend for Arc { + fn create(&self) -> Result<()> { + self.deref().create() + } + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + self.deref().write_bytes(tpe, id, cacheable, buf) + } + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + self.deref().remove(tpe, id, cacheable) + } +} + +impl ReadBackend for Arc { + fn location(&self) -> String { + self.deref().location() + } + fn list_with_size(&self, tpe: FileType) -> Result> { + self.deref().list_with_size(tpe) + } + fn list(&self, tpe: FileType) -> Result> { + self.deref().list(tpe) + } + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + self.deref().read_full(tpe, id) + } + fn read_partial( + &self, + tpe: FileType, + id: &Id, + cacheable: bool, + offset: u32, + length: u32, + ) -> Result { + self.deref() + .read_partial(tpe, id, cacheable, offset, length) + } +} + +impl std::fmt::Debug for dyn WriteBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "WriteBackend{{{}}}", self.location()) + } } /// Information about an entry to be able to open it. @@ -302,6 +371,7 @@ pub struct ReadSourceEntry { /// Trait for backends that can read and open sources. /// This trait is implemented by all backends that can read data and open from a source. pub trait ReadSourceOpen { + /// The type of the reader. type Reader: Read + Send + 'static; /// Opens the source. @@ -312,7 +382,10 @@ pub trait ReadSourceOpen { /// /// This trait is implemented by all backends that can read data from a source. pub trait ReadSource { + /// The type of the open information. type Open: ReadSourceOpen; + + /// The type of the iterator over the entries. type Iter: Iterator>>; /// Returns the size of the source. @@ -363,3 +436,42 @@ pub trait WriteSource: Clone { /// * `data` - The data to write. fn write_at>(&self, path: P, offset: u64, data: Bytes); } + +/// The backends a repository can be initialized and operated on +/// +/// # Note +/// +/// This struct is used to initialize a [`Repository`]. +#[derive(Debug, Clone)] +pub struct RepositoryBackends { + /// The main repository of this [`RepositoryBackends`]. + repository: Arc, + + /// The hot repository of this [`RepositoryBackends`]. + repo_hot: Option>, +} + +impl RepositoryBackends { + /// Creates a new [`RepositoryBackends`]. + /// + /// # Arguments + /// + /// * `repository` - The main repository of this [`RepositoryBackends`]. + /// * `repo_hot` - The hot repository of this [`RepositoryBackends`]. + pub fn new(repository: Arc, repo_hot: Option>) -> Self { + Self { + repository, + repo_hot, + } + } + + /// Returns the repository of this [`RepositoryBackends`]. + pub fn repository(&self) -> Arc { + self.repository.clone() + } + + /// Returns the hot repository of this [`RepositoryBackends`]. + pub fn repo_hot(&self) -> Option> { + self.repo_hot.clone() + } +} diff --git a/src/backend/cache.rs b/crates/core/src/backend/cache.rs similarity index 82% rename from src/backend/cache.rs rename to crates/core/src/backend/cache.rs index 3c218403..aaf99a6d 100644 --- a/src/backend/cache.rs +++ b/crates/core/src/backend/cache.rs @@ -3,8 +3,10 @@ use std::{ fs::{self, File}, io::{Read, Seek, SeekFrom, Write}, path::PathBuf, + sync::Arc, }; +use anyhow::Result; use bytes::Bytes; use dirs::cache_dir; use log::{trace, warn}; @@ -12,8 +14,7 @@ use walkdir::WalkDir; use crate::{ backend::{FileType, ReadBackend, WriteBackend}, - error::CacheBackendErrorKind, - error::RusticResult, + error::{CacheBackendErrorKind, RusticResult}, id::Id, }; @@ -26,40 +27,30 @@ use crate::{ /// /// * `BE` - The backend to cache. #[derive(Clone, Debug)] -pub struct CachedBackend { +pub struct CachedBackend { /// The backend to cache. - be: BE, + be: Arc, /// The cache. - cache: Option, + cache: Cache, } -impl CachedBackend { +impl CachedBackend { /// Create a new [`CachedBackend`] from a given backend. /// /// # Type Parameters /// /// * `BE` - The backend to cache. - pub fn new(be: BE, cache: Option) -> Self { - Self { be, cache } + pub fn new_cache(be: Arc, cache: Cache) -> Arc { + Arc::new(Self { be, cache }) } } -impl ReadBackend for CachedBackend { +impl ReadBackend for CachedBackend { /// Returns the location of the backend as a String. fn location(&self) -> String { self.be.location() } - /// Sets an option of the backend. - /// - /// # Arguments - /// - /// * `option` - The option to set. - /// * `value` - The value to set the option to. - fn set_option(&mut self, option: &str, value: &str) -> RusticResult<()> { - self.be.set_option(option, value) - } - /// Lists all files with their size of the given type. /// /// # Arguments @@ -73,13 +64,11 @@ impl ReadBackend for CachedBackend { /// # Returns /// /// A vector of tuples containing the id and size of the files. - fn list_with_size(&self, tpe: FileType) -> RusticResult> { + fn list_with_size(&self, tpe: FileType) -> Result> { let list = self.be.list_with_size(tpe)?; - if let Some(cache) = &self.cache { - if tpe.is_cacheable() { - cache.remove_not_in_list(tpe, &list)?; - } + if tpe.is_cacheable() { + self.cache.remove_not_in_list(tpe, &list)?; } Ok(list) @@ -101,20 +90,21 @@ impl ReadBackend for CachedBackend { /// The data read. /// /// [`CacheBackendErrorKind::FromIoError`]: crate::error::CacheBackendErrorKind::FromIoError - fn read_full(&self, tpe: FileType, id: &Id) -> RusticResult { - match (&self.cache, tpe.is_cacheable()) { - (None, _) | (Some(_), false) => self.be.read_full(tpe, id), - (Some(cache), true) => cache.read_full(tpe, id).map_or_else( + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + if tpe.is_cacheable() { + self.cache.read_full(tpe, id).map_or_else( |err| { warn!("Error in cache backend: {err}"); let res = self.be.read_full(tpe, id); if let Ok(data) = &res { - _ = cache.write_bytes(tpe, id, data.clone()); + _ = self.cache.write_bytes(tpe, id, data.clone()); } res }, Ok, - ), + ) + } else { + self.be.read_full(tpe, id) } } @@ -144,39 +134,41 @@ impl ReadBackend for CachedBackend { cacheable: bool, offset: u32, length: u32, - ) -> RusticResult { - match (&self.cache, cacheable || tpe.is_cacheable()) { - (None, _) | (Some(_), false) => { - self.be.read_partial(tpe, id, cacheable, offset, length) - } - (Some(cache), true) => { - cache.read_partial(tpe, id, offset, length).map_or_else( + ) -> Result { + if cacheable || tpe.is_cacheable() { + self.cache + .read_partial(tpe, id, offset, length) + .map_or_else( |err| { warn!("Error in cache backend: {err}"); + // read full file, save to cache and return partial content match self.be.read_full(tpe, id) { - // read full file, save to cache and return partial content from cache - // TODO: - Do not read to memory, but use a Reader - // - Don't read from cache, but use the right part of the read content Ok(data) => { - if cache.write_bytes(tpe, id, data).is_ok() { - cache.read_partial(tpe, id, offset, length) - } else { - self.be.read_partial(tpe, id, false, offset, length) - } + let range = offset as usize..(offset + length) as usize; + _ = self.cache.write_bytes(tpe, id, data.clone()); + Ok(Bytes::copy_from_slice(&data.slice(range))) } error => error, } }, Ok, ) - } + } else { + self.be.read_partial(tpe, id, cacheable, offset, length) } } + fn needs_warm_up(&self) -> bool { + self.be.needs_warm_up() + } + + fn warm_up(&self, tpe: FileType, id: &Id) -> Result<()> { + self.be.warm_up(tpe, id) + } } -impl WriteBackend for CachedBackend { +impl WriteBackend for CachedBackend { /// Creates the backend. - fn create(&self) -> RusticResult<()> { + fn create(&self) -> Result<()> { self.be.create() } @@ -190,11 +182,9 @@ impl WriteBackend for CachedBackend { /// * `id` - The id of the file. /// * `cacheable` - Whether the file is cacheable. /// * `buf` - The data to write. - fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> RusticResult<()> { - if let Some(cache) = &self.cache { - if cacheable || tpe.is_cacheable() { - _ = cache.write_bytes(tpe, id, buf.clone()); - } + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + if cacheable || tpe.is_cacheable() { + _ = self.cache.write_bytes(tpe, id, buf.clone()); } self.be.write_bytes(tpe, id, cacheable, buf) } @@ -207,11 +197,9 @@ impl WriteBackend for CachedBackend { /// /// * `tpe` - The type of the file. /// * `id` - The id of the file. - fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> RusticResult<()> { - if let Some(cache) = &self.cache { - if cacheable || tpe.is_cacheable() { - _ = cache.remove(tpe, id); - } + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + if cacheable || tpe.is_cacheable() { + _ = self.cache.remove(tpe, id); } self.be.remove(tpe, id, cacheable) } diff --git a/src/backend/decrypt.rs b/crates/core/src/backend/decrypt.rs similarity index 78% rename from src/backend/decrypt.rs rename to crates/core/src/backend/decrypt.rs index a00bb3dc..0e959f76 100644 --- a/src/backend/decrypt.rs +++ b/crates/core/src/backend/decrypt.rs @@ -1,5 +1,6 @@ -use std::num::NonZeroU32; +use std::{num::NonZeroU32, sync::Arc}; +use anyhow::Result; use bytes::Bytes; use crossbeam_channel::{unbounded, Receiver}; use rayon::prelude::*; @@ -8,16 +9,15 @@ use zstd::stream::{copy_encode, decode_all}; pub use zstd::compression_level_range; /// The maximum compression level allowed by zstd +#[must_use] pub fn max_compression_level() -> i32 { *compression_level_range().end() } use crate::{ - backend::FileType, - backend::ReadBackend, - backend::WriteBackend, + backend::{FileType, ReadBackend, WriteBackend}, crypto::{hasher::hash, CryptoKey}, - error::CryptBackendErrorKind, + error::{CryptBackendErrorKind, RusticErrorKind}, id::Id, repofile::RepoFile, Progress, RusticResult, @@ -31,7 +31,7 @@ pub trait DecryptFullBackend: DecryptWriteBackend + DecryptReadBackend {} impl DecryptFullBackend for T {} -pub trait DecryptReadBackend: ReadBackend { +pub trait DecryptReadBackend: ReadBackend + Clone + 'static { /// Decrypts the given data. /// /// # Arguments @@ -109,7 +109,9 @@ pub trait DecryptReadBackend: ReadBackend { uncompressed_length: Option, ) -> RusticResult { self.read_encrypted_from_partial( - &self.read_partial(tpe, id, cacheable, offset, length)?, + &self + .read_partial(tpe, id, cacheable, offset, length) + .map_err(RusticErrorKind::Backend)?, uncompressed_length, ) } @@ -142,7 +144,7 @@ pub trait DecryptReadBackend: ReadBackend { &self, p: &impl Progress, ) -> RusticResult>> { - let list = self.list(F::TYPE)?; + let list = self.list(F::TYPE).map_err(RusticErrorKind::Backend)?; self.stream_list(list, p) } @@ -174,7 +176,7 @@ pub trait DecryptReadBackend: ReadBackend { } } -pub trait DecryptWriteBackend: WriteBackend { +pub trait DecryptWriteBackend: WriteBackend + Clone + 'static { /// The type of the key. type Key: CryptoKey; @@ -194,9 +196,30 @@ pub trait DecryptWriteBackend: WriteBackend { /// /// # Returns /// - /// The id of the data. (TODO: Check if this is correct) + /// The hash of the written data. fn hash_write_full(&self, tpe: FileType, data: &[u8]) -> RusticResult; + /// Writes the given data to the backend without compression and returns the id of the data. + /// + /// # Arguments + /// + /// * `tpe` - The type of the file. + /// * `data` - The data to write. + /// + /// # Errors + /// + /// If the data could not be written. + /// + /// # Returns + /// + /// The hash of the written data. + fn hash_write_full_uncompressed(&self, tpe: FileType, data: &[u8]) -> RusticResult { + let data = self.key().encrypt_data(data)?; + let id = hash(&data); + self.write_bytes(tpe, &id, false, data.into()) + .map_err(RusticErrorKind::Backend)?; + Ok(id) + } /// Saves the given file. /// /// # Arguments @@ -218,6 +241,27 @@ pub trait DecryptWriteBackend: WriteBackend { self.hash_write_full(F::TYPE, &data) } + /// Saves the given file uncompressed. + /// + /// # Arguments + /// + /// * `file` - The file to save. + /// + /// # Errors + /// + /// * [`CryptBackendErrorKind::SerializingToJsonByteVectorFailed`] - If the file could not be serialized to json. + /// + /// # Returns + /// + /// The id of the file. + /// + /// [`CryptBackendErrorKind::SerializingToJsonByteVectorFailed`]: crate::error::CryptBackendErrorKind::SerializingToJsonByteVectorFailed + fn save_file_uncompressed(&self, file: &F) -> RusticResult { + let data = serde_json::to_vec(file) + .map_err(CryptBackendErrorKind::SerializingToJsonByteVectorFailed)?; + self.hash_write_full_uncompressed(F::TYPE, &data) + } + /// Saves the given list of files. /// /// # Arguments @@ -281,24 +325,22 @@ pub trait DecryptWriteBackend: WriteBackend { /// /// # Type Parameters /// -/// * `R` - The type of the backend to decrypt. /// * `C` - The type of the key to decrypt the backend with. -#[derive(Clone, Debug)] -pub struct DecryptBackend { +#[derive(Debug, Clone)] +pub struct DecryptBackend { /// The backend to decrypt. - backend: R, + be: Arc, /// The key to decrypt the backend with. key: C, /// The compression level to use for zstd. zstd: Option, } -impl DecryptBackend { +impl DecryptBackend { /// Creates a new decrypt backend. /// /// # Type Parameters /// - /// * `R` - The type of the backend to decrypt. /// * `C` - The type of the key to decrypt the backend with. /// /// # Arguments @@ -309,16 +351,16 @@ impl DecryptBackend { /// # Returns /// /// The new decrypt backend. - pub fn new(be: &R, key: C) -> Self { + pub fn new(be: Arc, key: C) -> Self { Self { - backend: be.clone(), + be, key, zstd: None, } } } -impl DecryptWriteBackend for DecryptBackend { +impl DecryptWriteBackend for DecryptBackend { /// The type of the key. type Key = C; @@ -354,7 +396,8 @@ impl DecryptWriteBackend for DecryptBackend None => self.key().encrypt_data(data)?, }; let id = hash(&data); - self.write_bytes(tpe, &id, false, data.into())?; + self.write_bytes(tpe, &id, false, data.into()) + .map_err(RusticErrorKind::Backend)?; Ok(id) } @@ -368,7 +411,7 @@ impl DecryptWriteBackend for DecryptBackend } } -impl DecryptReadBackend for DecryptBackend { +impl DecryptReadBackend for DecryptBackend { /// Decrypts the given data. /// /// # Arguments @@ -397,7 +440,8 @@ impl DecryptReadBackend for DecryptBackend { /// [`CryptBackendErrorKind::DecryptionNotSupportedForBackend`]: crate::error::CryptBackendErrorKind::DecryptionNotSupportedForBackend /// [`CryptBackendErrorKind::DecodingZstdCompressedDataFailed`]: crate::error::CryptBackendErrorKind::DecodingZstdCompressedDataFailed fn read_encrypted_full(&self, tpe: FileType, id: &Id) -> RusticResult { - let decrypted = self.decrypt(&self.read_full(tpe, id)?)?; + let decrypted = + self.decrypt(&self.read_full(tpe, id).map_err(RusticErrorKind::Backend)?)?; Ok(match decrypted.first() { Some(b'{' | b'[') => decrypted, // not compressed Some(2) => decode_all(&decrypted[1..]) @@ -408,25 +452,21 @@ impl DecryptReadBackend for DecryptBackend { } } -impl ReadBackend for DecryptBackend { +impl ReadBackend for DecryptBackend { fn location(&self) -> String { - self.backend.location() - } - - fn set_option(&mut self, option: &str, value: &str) -> RusticResult<()> { - self.backend.set_option(option, value) + self.be.location() } - fn list(&self, tpe: FileType) -> RusticResult> { - self.backend.list(tpe) + fn list(&self, tpe: FileType) -> Result> { + self.be.list(tpe) } - fn list_with_size(&self, tpe: FileType) -> RusticResult> { - self.backend.list_with_size(tpe) + fn list_with_size(&self, tpe: FileType) -> Result> { + self.be.list_with_size(tpe) } - fn read_full(&self, tpe: FileType, id: &Id) -> RusticResult { - self.backend.read_full(tpe, id) + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + self.be.read_full(tpe, id) } fn read_partial( @@ -436,22 +476,21 @@ impl ReadBackend for DecryptBackend { cacheable: bool, offset: u32, length: u32, - ) -> RusticResult { - self.backend - .read_partial(tpe, id, cacheable, offset, length) + ) -> Result { + self.be.read_partial(tpe, id, cacheable, offset, length) } } -impl WriteBackend for DecryptBackend { - fn create(&self) -> RusticResult<()> { - self.backend.create() +impl WriteBackend for DecryptBackend { + fn create(&self) -> Result<()> { + self.be.create() } - fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> RusticResult<()> { - self.backend.write_bytes(tpe, id, cacheable, buf) + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + self.be.write_bytes(tpe, id, cacheable, buf) } - fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> RusticResult<()> { - self.backend.remove(tpe, id, cacheable) + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + self.be.remove(tpe, id, cacheable) } } diff --git a/src/backend/dry_run.rs b/crates/core/src/backend/dry_run.rs similarity index 85% rename from src/backend/dry_run.rs rename to crates/core/src/backend/dry_run.rs index 5b5ce5d4..34eeca1f 100644 --- a/src/backend/dry_run.rs +++ b/crates/core/src/backend/dry_run.rs @@ -1,12 +1,13 @@ +use anyhow::Result; use bytes::Bytes; use zstd::decode_all; use crate::{ backend::{ - decrypt::DecryptFullBackend, decrypt::DecryptReadBackend, decrypt::DecryptWriteBackend, + decrypt::{DecryptFullBackend, DecryptReadBackend, DecryptWriteBackend}, FileType, ReadBackend, WriteBackend, }, - error::{CryptBackendErrorKind, RusticResult}, + error::{CryptBackendErrorKind, RusticErrorKind, RusticResult}, id::Id, }; @@ -59,7 +60,8 @@ impl DecryptReadBackend for DryRunBackend { /// [`CryptBackendErrorKind::DecryptionNotSupportedForBackend`]: crate::error::CryptBackendErrorKind::DecryptionNotSupportedForBackend /// [`CryptBackendErrorKind::DecodingZstdCompressedDataFailed`]: crate::error::CryptBackendErrorKind::DecodingZstdCompressedDataFailed fn read_encrypted_full(&self, tpe: FileType, id: &Id) -> RusticResult { - let decrypted = self.decrypt(&self.read_full(tpe, id)?)?; + let decrypted = + self.decrypt(&self.read_full(tpe, id).map_err(RusticErrorKind::Backend)?)?; Ok(match decrypted.first() { Some(b'{' | b'[') => decrypted, // not compressed Some(2) => decode_all(&decrypted[1..]) @@ -75,15 +77,11 @@ impl ReadBackend for DryRunBackend { self.be.location() } - fn set_option(&mut self, option: &str, value: &str) -> RusticResult<()> { - self.be.set_option(option, value) - } - - fn list_with_size(&self, tpe: FileType) -> RusticResult> { + fn list_with_size(&self, tpe: FileType) -> Result> { self.be.list_with_size(tpe) } - fn read_full(&self, tpe: FileType, id: &Id) -> RusticResult { + fn read_full(&self, tpe: FileType, id: &Id) -> Result { self.be.read_full(tpe, id) } @@ -94,7 +92,7 @@ impl ReadBackend for DryRunBackend { cacheable: bool, offset: u32, length: u32, - ) -> RusticResult { + ) -> Result { self.be.read_partial(tpe, id, cacheable, offset, length) } } @@ -122,7 +120,7 @@ impl DecryptWriteBackend for DryRunBackend { } impl WriteBackend for DryRunBackend { - fn create(&self) -> RusticResult<()> { + fn create(&self) -> Result<()> { if self.dry_run { Ok(()) } else { @@ -130,7 +128,7 @@ impl WriteBackend for DryRunBackend { } } - fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> RusticResult<()> { + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { if self.dry_run { Ok(()) } else { @@ -138,7 +136,7 @@ impl WriteBackend for DryRunBackend { } } - fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> RusticResult<()> { + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { if self.dry_run { Ok(()) } else { diff --git a/crates/core/src/backend/hotcold.rs b/crates/core/src/backend/hotcold.rs new file mode 100644 index 00000000..75a1f750 --- /dev/null +++ b/crates/core/src/backend/hotcold.rs @@ -0,0 +1,101 @@ +use std::sync::Arc; + +use anyhow::Result; +use bytes::Bytes; + +use crate::{ + backend::{FileType, ReadBackend, WriteBackend}, + id::Id, +}; + +/// A hot/cold backend implementation. +/// +/// # Type Parameters +/// +/// * `BE` - The backend to use. +#[derive(Clone, Debug)] +pub struct HotColdBackend { + /// The backend to use. + be: Arc, + /// The backend to use for hot files. + be_hot: Arc, +} + +impl HotColdBackend { + /// Creates a new `HotColdBackend`. + /// + /// # Type Parameters + /// + /// * `BE` - The backend to use. + /// + /// # Arguments + /// + /// * `be` - The backend to use. + /// * `hot_be` - The backend to use for hot files. + pub fn new(be: BE, hot_be: BE) -> Self { + Self { + be: Arc::new(be), + be_hot: Arc::new(hot_be), + } + } +} + +impl ReadBackend for HotColdBackend { + fn location(&self) -> String { + self.be.location() + } + + fn list_with_size(&self, tpe: FileType) -> Result> { + self.be.list_with_size(tpe) + } + + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + self.be_hot.read_full(tpe, id) + } + + fn read_partial( + &self, + tpe: FileType, + id: &Id, + cacheable: bool, + offset: u32, + length: u32, + ) -> Result { + if cacheable || tpe != FileType::Pack { + self.be_hot.read_partial(tpe, id, cacheable, offset, length) + } else { + self.be.read_partial(tpe, id, cacheable, offset, length) + } + } + + fn needs_warm_up(&self) -> bool { + self.be.needs_warm_up() + } + + fn warm_up(&self, tpe: FileType, id: &Id) -> Result<()> { + self.be.warm_up(tpe, id) + } +} + +impl WriteBackend for HotColdBackend { + fn create(&self) -> Result<()> { + self.be.create()?; + self.be_hot.create() + } + + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + if tpe != FileType::Config && (cacheable || tpe != FileType::Pack) { + self.be_hot.write_bytes(tpe, id, cacheable, buf.clone())?; + } + self.be.write_bytes(tpe, id, cacheable, buf) + } + + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + // First remove cold file + self.be.remove(tpe, id, cacheable)?; + if cacheable || tpe != FileType::Pack { + self.be_hot.remove(tpe, id, cacheable)?; + } + Ok(()) + } +} diff --git a/src/backend/ignore.rs b/crates/core/src/backend/ignore.rs similarity index 100% rename from src/backend/ignore.rs rename to crates/core/src/backend/ignore.rs diff --git a/crates/core/src/backend/local_destination.rs b/crates/core/src/backend/local_destination.rs new file mode 100644 index 00000000..965767e9 --- /dev/null +++ b/crates/core/src/backend/local_destination.rs @@ -0,0 +1,647 @@ +#[cfg(not(windows))] +use std::os::unix::fs::{symlink, PermissionsExt}; + +use std::{ + fs::{self, File, OpenOptions}, + io::{Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, +}; + +use bytes::Bytes; +#[allow(unused_imports)] +use cached::proc_macro::cached; +use filetime::{set_symlink_file_times, FileTime}; +#[cfg(not(windows))] +use log::warn; +#[cfg(not(windows))] +use nix::sys::stat::{mknod, Mode, SFlag}; +#[cfg(not(windows))] +use nix::unistd::{fchownat, FchownatFlags, Gid, Group, Uid, User}; + +#[cfg(not(windows))] +use crate::backend::ignore::mapper::map_mode_from_go; +#[cfg(not(windows))] +use crate::backend::node::NodeType; +use crate::{ + backend::node::{ExtendedAttribute, Metadata, Node}, + error::LocalDestinationErrorKind, + RusticResult, +}; + +#[derive(Clone, Debug)] +/// Local destination, used when restoring. +pub struct LocalDestination { + /// The base path of the destination. + path: PathBuf, + /// Whether we expect a single file as destination. + is_file: bool, +} + +// Helper function to cache mapping user name -> uid +#[cfg(not(windows))] +#[cached] +fn uid_from_name(name: String) -> Option { + User::from_name(&name).unwrap().map(|u| u.uid) +} + +// Helper function to cache mapping group name -> gid +#[cfg(not(windows))] +#[cached] +fn gid_from_name(name: String) -> Option { + Group::from_name(&name).unwrap().map(|g| g.gid) +} + +impl LocalDestination { + /// Create a new [`LocalDestination`] + /// + /// # Arguments + /// + /// * `path` - The base path of the destination + /// * `create` - If `create` is true, create the base path if it doesn't exist. + /// * `expect_file` - Whether we expect a single file as destination. + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::DirectoryCreationFailed`] - If the directory could not be created. + /// + /// [`LocalDestinationErrorKind::DirectoryCreationFailed`]: crate::error::LocalDestinationErrorKind::DirectoryCreationFailed + // TODO: We should use `impl Into` here. we even use it in the body! + pub fn new(path: &str, create: bool, expect_file: bool) -> RusticResult { + let is_dir = path.ends_with('/'); + let path: PathBuf = path.into(); + let is_file = path.is_file() || (!path.is_dir() && !is_dir && expect_file); + + if create { + if is_file { + if let Some(path) = path.parent() { + fs::create_dir_all(path) + .map_err(LocalDestinationErrorKind::DirectoryCreationFailed)?; + } + } else { + fs::create_dir_all(&path) + .map_err(LocalDestinationErrorKind::DirectoryCreationFailed)?; + } + } + + Ok(Self { path, is_file }) + } + + /// Path to the given item (relative to the base path) + /// + /// # Arguments + /// + /// * `item` - The item to get the path for + /// + /// # Returns + /// + /// The path to the item. + /// + /// # Notes + /// + /// * If the destination is a file, this will return the base path. + /// * If the destination is a directory, this will return the base path joined with the item. + pub(crate) fn path(&self, item: impl AsRef) -> PathBuf { + if self.is_file { + self.path.clone() + } else { + self.path.join(item) + } + } + + /// Remove the given directory (relative to the base path) + /// + /// # Arguments + /// + /// * `dirname` - The directory to remove + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::DirectoryRemovalFailed`] - If the directory could not be removed. + /// + /// # Notes + /// + /// This will remove the directory recursively. + /// + /// [`LocalDestinationErrorKind::DirectoryRemovalFailed`]: crate::error::LocalDestinationErrorKind::DirectoryRemovalFailed + pub fn remove_dir(&self, dirname: impl AsRef) -> RusticResult<()> { + Ok(fs::remove_dir_all(dirname) + .map_err(LocalDestinationErrorKind::DirectoryRemovalFailed)?) + } + + /// Remove the given file (relative to the base path) + /// + /// # Arguments + /// + /// * `filename` - The file to remove + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::FileRemovalFailed`] - If the file could not be removed. + /// + /// # Notes + /// + /// This will remove the file. + /// + /// * If the file is a symlink, the symlink will be removed, not the file it points to. + /// * If the file is a directory or device, this will fail. + /// + /// [`LocalDestinationErrorKind::FileRemovalFailed`]: crate::error::LocalDestinationErrorKind::FileRemovalFailed + pub fn remove_file(&self, filename: impl AsRef) -> RusticResult<()> { + Ok(fs::remove_file(filename).map_err(LocalDestinationErrorKind::FileRemovalFailed)?) + } + + /// Create the given directory (relative to the base path) + /// + /// # Arguments + /// + /// * `item` - The directory to create + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::DirectoryCreationFailed`] - If the directory could not be created. + /// + /// # Notes + /// + /// This will create the directory structure recursively. + /// + /// [`LocalDestinationErrorKind::DirectoryCreationFailed`]: crate::error::LocalDestinationErrorKind::DirectoryCreationFailed + pub fn create_dir(&self, item: impl AsRef) -> RusticResult<()> { + let dirname = self.path.join(item); + fs::create_dir_all(dirname).map_err(LocalDestinationErrorKind::DirectoryCreationFailed)?; + Ok(()) + } + + /// Set changed and modified times for `item` (relative to the base path) utilizing the file metadata + /// + /// # Arguments + /// + /// * `item` - The item to set the times for + /// * `meta` - The metadata to get the times from + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::SettingTimeMetadataFailed`] - If the times could not be set + /// + /// [`LocalDestinationErrorKind::SettingTimeMetadataFailed`]: crate::error::LocalDestinationErrorKind::SettingTimeMetadataFailed + pub fn set_times(&self, item: impl AsRef, meta: &Metadata) -> RusticResult<()> { + let filename = self.path(item); + if let Some(mtime) = meta.mtime { + let atime = meta.atime.unwrap_or(mtime); + set_symlink_file_times( + filename, + FileTime::from_system_time(atime.into()), + FileTime::from_system_time(mtime.into()), + ) + .map_err(LocalDestinationErrorKind::SettingTimeMetadataFailed)?; + } + + Ok(()) + } + + #[cfg(windows)] + // TODO: Windows support + /// Set user/group for `item` (relative to the base path) utilizing the file metadata + /// + /// # Arguments + /// + /// * `item` - The item to set the user/group for + /// * `meta` - The metadata to get the user/group from + /// + /// # Errors + /// + /// If the user/group could not be set. + pub fn set_user_group(&self, _item: impl AsRef, _meta: &Metadata) -> RusticResult<()> { + // https://learn.microsoft.com/en-us/windows/win32/fileio/file-security-and-access-rights + // https://microsoft.github.io/windows-docs-rs/doc/windows/Win32/Security/struct.SECURITY_ATTRIBUTES.html + // https://microsoft.github.io/windows-docs-rs/doc/windows/Win32/Storage/FileSystem/struct.CREATEFILE2_EXTENDED_PARAMETERS.html#structfield.lpSecurityAttributes + Ok(()) + } + + #[cfg(not(windows))] + /// Set user/group for `item` (relative to the base path) utilizing the file metadata + /// + /// # Arguments + /// + /// * `item` - The item to set the user/group for + /// * `meta` - The metadata to get the user/group from + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::FromErrnoError`] - If the user/group could not be set. + /// + /// [`LocalDestinationErrorKind::FromErrnoError`]: crate::error::LocalDestinationErrorKind::FromErrnoError + pub fn set_user_group(&self, item: impl AsRef, meta: &Metadata) -> RusticResult<()> { + let filename = self.path(item); + + let user = meta.user.clone().and_then(uid_from_name); + // use uid from user if valid, else from saved uid (if saved) + let uid = user.or_else(|| meta.uid.map(Uid::from_raw)); + + let group = meta.group.clone().and_then(gid_from_name); + // use gid from group if valid, else from saved gid (if saved) + let gid = group.or_else(|| meta.gid.map(Gid::from_raw)); + + fchownat(None, &filename, uid, gid, FchownatFlags::NoFollowSymlink) + .map_err(LocalDestinationErrorKind::FromErrnoError)?; + Ok(()) + } + + #[cfg(windows)] + // TODO: Windows support + /// Set uid/gid for `item` (relative to the base path) utilizing the file metadata + /// + /// # Arguments + /// + /// * `item` - The item to set the uid/gid for + /// * `meta` - The metadata to get the uid/gid from + /// + /// # Errors + /// + /// If the uid/gid could not be set. + pub fn set_uid_gid(&self, _item: impl AsRef, _meta: &Metadata) -> RusticResult<()> { + Ok(()) + } + + #[cfg(not(windows))] + /// Set uid/gid for `item` (relative to the base path) utilizing the file metadata + /// + /// # Arguments + /// + /// * `item` - The item to set the uid/gid for + /// * `meta` - The metadata to get the uid/gid from + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::FromErrnoError`] - If the uid/gid could not be set. + /// + /// [`LocalDestinationErrorKind::FromErrnoError`]: crate::error::LocalDestinationErrorKind::FromErrnoError + pub fn set_uid_gid(&self, item: impl AsRef, meta: &Metadata) -> RusticResult<()> { + let filename = self.path(item); + + let uid = meta.uid.map(Uid::from_raw); + let gid = meta.gid.map(Gid::from_raw); + + fchownat(None, &filename, uid, gid, FchownatFlags::NoFollowSymlink) + .map_err(LocalDestinationErrorKind::FromErrnoError)?; + Ok(()) + } + + #[cfg(windows)] + // TODO: Windows support + /// Set permissions for `item` (relative to the base path) from `node` + /// + /// # Arguments + /// + /// * `item` - The item to set the permissions for + /// * `node` - The node to get the permissions from + /// + /// # Errors + /// + /// If the permissions could not be set. + pub fn set_permission(&self, _item: impl AsRef, _node: &Node) -> RusticResult<()> { + Ok(()) + } + + #[cfg(not(windows))] + /// Set permissions for `item` (relative to the base path) from `node` + /// + /// # Arguments + /// + /// * `item` - The item to set the permissions for + /// * `node` - The node to get the permissions from + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::SettingFilePermissionsFailed`] - If the permissions could not be set. + /// + /// [`LocalDestinationErrorKind::SettingFilePermissionsFailed`]: crate::error::LocalDestinationErrorKind::SettingFilePermissionsFailed + pub fn set_permission(&self, item: impl AsRef, node: &Node) -> RusticResult<()> { + if node.is_symlink() { + return Ok(()); + } + + let filename = self.path(item); + + if let Some(mode) = node.meta.mode { + let mode = map_mode_from_go(mode); + std::fs::set_permissions(filename, fs::Permissions::from_mode(mode)) + .map_err(LocalDestinationErrorKind::SettingFilePermissionsFailed)?; + } + Ok(()) + } + + #[cfg(any(windows, target_os = "openbsd"))] + // TODO: Windows support + // TODO: openbsd support + /// Set extended attributes for `item` (relative to the base path) + /// + /// # Arguments + /// + /// * `item` - The item to set the extended attributes for + /// * `extended_attributes` - The extended attributes to set + /// + /// # Errors + /// + /// If the extended attributes could not be set. + pub fn set_extended_attributes( + &self, + _item: impl AsRef, + _extended_attributes: &[ExtendedAttribute], + ) -> RusticResult<()> { + Ok(()) + } + + #[cfg(not(any(windows, target_os = "openbsd")))] + /// Set extended attributes for `item` (relative to the base path) + /// + /// # Arguments + /// + /// * `item` - The item to set the extended attributes for + /// * `extended_attributes` - The extended attributes to set + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::ListingXattrsFailed`] - If listing the extended attributes failed. + /// * [`LocalDestinationErrorKind::GettingXattrFailed`] - If getting an extended attribute failed. + /// * [`LocalDestinationErrorKind::SettingXattrFailed`] - If setting an extended attribute failed. + /// + /// [`LocalDestinationErrorKind::ListingXattrsFailed`]: crate::error::LocalDestinationErrorKind::ListingXattrsFailed + /// [`LocalDestinationErrorKind::GettingXattrFailed`]: crate::error::LocalDestinationErrorKind::GettingXattrFailed + /// [`LocalDestinationErrorKind::SettingXattrFailed`]: crate::error::LocalDestinationErrorKind::SettingXattrFailed + pub fn set_extended_attributes( + &self, + item: impl AsRef, + extended_attributes: &[ExtendedAttribute], + ) -> RusticResult<()> { + let filename = self.path(item); + let mut done = vec![false; extended_attributes.len()]; + + for curr_name in xattr::list(&filename) + .map_err(|err| LocalDestinationErrorKind::ListingXattrsFailed(err, filename.clone()))? + { + match extended_attributes.iter().enumerate().find( + |(_, ExtendedAttribute { name, .. })| name == curr_name.to_string_lossy().as_ref(), + ) { + Some((index, ExtendedAttribute { name, value })) => { + let curr_value = xattr::get(&filename, name) + .map_err(|err| LocalDestinationErrorKind::GettingXattrFailed { + name: name.clone(), + filename: filename.clone(), + source: err, + })? + .unwrap(); + if value != &curr_value { + xattr::set(&filename, name, value).map_err(|err| { + LocalDestinationErrorKind::SettingXattrFailed { + name: name.clone(), + filename: filename.clone(), + source: err, + } + })?; + } + done[index] = true; + } + None => { + if let Err(err) = xattr::remove(&filename, &curr_name) { + warn!("error removing xattr {curr_name:?} on {filename:?}: {err}"); + } + } + } + } + + for (index, ExtendedAttribute { name, value }) in extended_attributes.iter().enumerate() { + if !done[index] { + xattr::set(&filename, name, value).map_err(|err| { + LocalDestinationErrorKind::SettingXattrFailed { + name: name.clone(), + filename: filename.clone(), + source: err, + } + })?; + } + } + + Ok(()) + } + + /// Set length of `item` (relative to the base path) + /// + /// # Arguments + /// + /// * `item` - The item to set the length for + /// * `size` - The size to set the length to + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::FileDoesNotHaveParent`] - If the file does not have a parent. + /// * [`LocalDestinationErrorKind::DirectoryCreationFailed`] - If the directory could not be created. + /// * [`LocalDestinationErrorKind::OpeningFileFailed`] - If the file could not be opened. + /// * [`LocalDestinationErrorKind::SettingFileLengthFailed`] - If the length of the file could not be set. + /// + /// # Notes + /// + /// If the file exists, truncate it to the given length. (TODO: check if this is correct) + /// If it doesn't exist, create a new (empty) one with given length. + /// + /// [`LocalDestinationErrorKind::FileDoesNotHaveParent`]: crate::error::LocalDestinationErrorKind::FileDoesNotHaveParent + /// [`LocalDestinationErrorKind::DirectoryCreationFailed`]: crate::error::LocalDestinationErrorKind::DirectoryCreationFailed + /// [`LocalDestinationErrorKind::OpeningFileFailed`]: crate::error::LocalDestinationErrorKind::OpeningFileFailed + /// [`LocalDestinationErrorKind::SettingFileLengthFailed`]: crate::error::LocalDestinationErrorKind::SettingFileLengthFailed + pub fn set_length(&self, item: impl AsRef, size: u64) -> RusticResult<()> { + let filename = self.path(item); + let dir = filename + .parent() + .ok_or_else(|| LocalDestinationErrorKind::FileDoesNotHaveParent(filename.clone()))?; + fs::create_dir_all(dir).map_err(LocalDestinationErrorKind::DirectoryCreationFailed)?; + + OpenOptions::new() + .create(true) + .write(true) + .open(filename) + .map_err(LocalDestinationErrorKind::OpeningFileFailed)? + .set_len(size) + .map_err(LocalDestinationErrorKind::SettingFileLengthFailed)?; + Ok(()) + } + + #[cfg(windows)] + // TODO: Windows support + /// Create a special file (relative to the base path) + pub fn create_special(&self, _item: impl AsRef, _node: &Node) -> RusticResult<()> { + Ok(()) + } + + #[cfg(not(windows))] + /// Create a special file (relative to the base path) + /// + /// # Arguments + /// + /// * `item` - The item to create + /// * `node` - The node to get the type from + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::SymlinkingFailed`] - If the symlink could not be created. + /// * [`LocalDestinationErrorKind::FromTryIntError`] - If the device could not be converted to the correct type. + /// * [`LocalDestinationErrorKind::FromErrnoError`] - If the device could not be created. + /// + /// [`LocalDestinationErrorKind::SymlinkingFailed`]: crate::error::LocalDestinationErrorKind::SymlinkingFailed + /// [`LocalDestinationErrorKind::FromTryIntError`]: crate::error::LocalDestinationErrorKind::FromTryIntError + /// [`LocalDestinationErrorKind::FromErrnoError`]: crate::error::LocalDestinationErrorKind::FromErrnoError + pub fn create_special(&self, item: impl AsRef, node: &Node) -> RusticResult<()> { + let filename = self.path(item); + + match &node.node_type { + NodeType::Symlink { .. } => { + let linktarget = node.node_type.to_link(); + symlink(linktarget, &filename).map_err(|err| { + LocalDestinationErrorKind::SymlinkingFailed { + linktarget: linktarget.to_path_buf(), + filename, + source: err, + } + })?; + } + NodeType::Dev { device } => { + #[cfg(not(any( + target_os = "macos", + target_os = "openbsd", + target_os = "freebsd" + )))] + let device = *device; + #[cfg(any(target_os = "macos", target_os = "openbsd"))] + let device = + i32::try_from(*device).map_err(LocalDestinationErrorKind::FromTryIntError)?; + #[cfg(target_os = "freebsd")] + let device = + u32::try_from(*device).map_err(LocalDestinationErrorKind::FromTryIntError)?; + mknod(&filename, SFlag::S_IFBLK, Mode::empty(), device) + .map_err(LocalDestinationErrorKind::FromErrnoError)?; + } + NodeType::Chardev { device } => { + #[cfg(not(any( + target_os = "macos", + target_os = "openbsd", + target_os = "freebsd" + )))] + let device = *device; + #[cfg(any(target_os = "macos", target_os = "openbsd"))] + let device = + i32::try_from(*device).map_err(LocalDestinationErrorKind::FromTryIntError)?; + #[cfg(target_os = "freebsd")] + let device = + u32::try_from(*device).map_err(LocalDestinationErrorKind::FromTryIntError)?; + mknod(&filename, SFlag::S_IFCHR, Mode::empty(), device) + .map_err(LocalDestinationErrorKind::FromErrnoError)?; + } + NodeType::Fifo => { + mknod(&filename, SFlag::S_IFIFO, Mode::empty(), 0) + .map_err(LocalDestinationErrorKind::FromErrnoError)?; + } + NodeType::Socket => { + mknod(&filename, SFlag::S_IFSOCK, Mode::empty(), 0) + .map_err(LocalDestinationErrorKind::FromErrnoError)?; + } + _ => {} + } + Ok(()) + } + + /// Read the given item (relative to the base path) + /// + /// # Arguments + /// + /// * `item` - The item to read + /// * `offset` - The offset to read from + /// * `length` - The length to read + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::OpeningFileFailed`] - If the file could not be opened. + /// * [`LocalDestinationErrorKind::CouldNotSeekToPositionInFile`] - If the file could not be seeked to the given position. + /// * [`LocalDestinationErrorKind::FromTryIntError`] - If the length of the file could not be converted to u32. + /// * [`LocalDestinationErrorKind::ReadingExactLengthOfFileFailed`] - If the length of the file could not be read. + /// + /// [`LocalDestinationErrorKind::OpeningFileFailed`]: crate::error::LocalDestinationErrorKind::OpeningFileFailed + /// [`LocalDestinationErrorKind::CouldNotSeekToPositionInFile`]: crate::error::LocalDestinationErrorKind::CouldNotSeekToPositionInFile + /// [`LocalDestinationErrorKind::FromTryIntError`]: crate::error::LocalDestinationErrorKind::FromTryIntError + /// [`LocalDestinationErrorKind::ReadingExactLengthOfFileFailed`]: crate::error::LocalDestinationErrorKind::ReadingExactLengthOfFileFailed + pub fn read_at(&self, item: impl AsRef, offset: u64, length: u64) -> RusticResult { + let filename = self.path(item); + let mut file = + File::open(filename).map_err(LocalDestinationErrorKind::OpeningFileFailed)?; + _ = file + .seek(SeekFrom::Start(offset)) + .map_err(LocalDestinationErrorKind::CouldNotSeekToPositionInFile)?; + let mut vec = vec![ + 0; + length + .try_into() + .map_err(LocalDestinationErrorKind::FromTryIntError)? + ]; + file.read_exact(&mut vec) + .map_err(LocalDestinationErrorKind::ReadingExactLengthOfFileFailed)?; + Ok(vec.into()) + } + + /// Check if a matching file exists. + /// + /// # Arguments + /// + /// * `item` - The item to check + /// * `size` - The size to check + /// + /// # Returns + /// + /// If a file exists and size matches, this returns a `File` open for reading. + /// In all other cases, returns `None` + pub fn get_matching_file(&self, item: impl AsRef, size: u64) -> Option { + let filename = self.path(item); + fs::symlink_metadata(&filename).map_or_else( + |_| None, + |meta| { + if meta.is_file() && meta.len() == size { + File::open(&filename).ok() + } else { + None + } + }, + ) + } + + /// Write `data` to given item (relative to the base path) at `offset` + /// + /// # Arguments + /// + /// * `item` - The item to write to + /// * `offset` - The offset to write at + /// * `data` - The data to write + /// + /// # Errors + /// + /// * [`LocalDestinationErrorKind::OpeningFileFailed`] - If the file could not be opened. + /// * [`LocalDestinationErrorKind::CouldNotSeekToPositionInFile`] - If the file could not be seeked to the given position. + /// * [`LocalDestinationErrorKind::CouldNotWriteToBuffer`] - If the bytes could not be written to the file. + /// + /// # Notes + /// + /// This will create the file if it doesn't exist. + /// + /// [`LocalDestinationErrorKind::OpeningFileFailed`]: crate::error::LocalDestinationErrorKind::OpeningFileFailed + /// [`LocalDestinationErrorKind::CouldNotSeekToPositionInFile`]: crate::error::LocalDestinationErrorKind::CouldNotSeekToPositionInFile + /// [`LocalDestinationErrorKind::CouldNotWriteToBuffer`]: crate::error::LocalDestinationErrorKind::CouldNotWriteToBuffer + pub fn write_at(&self, item: impl AsRef, offset: u64, data: &[u8]) -> RusticResult<()> { + let filename = self.path(item); + let mut file = fs::OpenOptions::new() + .create(true) + .write(true) + .open(filename) + .map_err(LocalDestinationErrorKind::OpeningFileFailed)?; + _ = file + .seek(SeekFrom::Start(offset)) + .map_err(LocalDestinationErrorKind::CouldNotSeekToPositionInFile)?; + file.write_all(data) + .map_err(LocalDestinationErrorKind::CouldNotWriteToBuffer)?; + Ok(()) + } +} diff --git a/src/backend/node.rs b/crates/core/src/backend/node.rs similarity index 99% rename from src/backend/node.rs rename to crates/core/src/backend/node.rs index ea29c084..ddf8afe2 100644 --- a/src/backend/node.rs +++ b/crates/core/src/backend/node.rs @@ -128,6 +128,7 @@ impl NodeType { // Windows doesn't support non-unicode link targets, so we assume unicode here. // TODO: Test and check this! /// Get a [`NodeType`] from a linktarget path + #[must_use] pub fn from_link(target: &Path) -> Self { Self::Symlink { linktarget: target.as_os_str().to_string_lossy().to_string(), @@ -163,6 +164,7 @@ impl NodeType { /// * If the link target is not valid unicode // TODO: Implement non-unicode link targets correctly for windows #[cfg(windows)] + #[must_use] pub fn to_link(&self) -> &Path { match self { Self::Symlink { linktarget, .. } => Path::new(linktarget), @@ -329,6 +331,7 @@ impl Node { /// # Returns /// /// The ordering of the two nodes +#[must_use] pub fn last_modified_node(n1: &Node, n2: &Node) -> Ordering { n1.meta.mtime.cmp(&n2.meta.mtime) } diff --git a/src/backend/stdin.rs b/crates/core/src/backend/stdin.rs similarity index 89% rename from src/backend/stdin.rs rename to crates/core/src/backend/stdin.rs index 5467016c..1eeac37d 100644 --- a/src/backend/stdin.rs +++ b/crates/core/src/backend/stdin.rs @@ -2,7 +2,8 @@ use std::{io::stdin, path::PathBuf}; use crate::{ backend::{ - node::Metadata, node::Node, node::NodeType, ReadSource, ReadSourceEntry, ReadSourceOpen, + node::{Metadata, Node, NodeType}, + ReadSource, ReadSourceEntry, ReadSourceOpen, }, error::RusticResult, }; @@ -18,11 +19,11 @@ pub struct StdinSource { impl StdinSource { /// Creates a new `StdinSource`. - pub const fn new(path: PathBuf) -> RusticResult { - Ok(Self { + pub const fn new(path: PathBuf) -> Self { + Self { finished: false, path, - }) + } } } diff --git a/crates/core/src/backend/warm_up.rs b/crates/core/src/backend/warm_up.rs new file mode 100644 index 00000000..87e9e009 --- /dev/null +++ b/crates/core/src/backend/warm_up.rs @@ -0,0 +1,77 @@ +use std::sync::Arc; + +use anyhow::Result; +use bytes::Bytes; + +use crate::{ + backend::{FileType, ReadBackend, WriteBackend}, + id::Id, +}; + +/// A backend which warms up files by simply accessing them. +#[derive(Clone, Debug)] +pub struct WarmUpAccessBackend { + /// The backend to use. + be: Arc, +} + +impl WarmUpAccessBackend { + /// Creates a new `WarmUpAccessBackend`. + /// + /// # Arguments + /// + /// * `be` - The backend to use. + pub fn new_warm_up(be: Arc) -> Arc { + Arc::new(Self { be }) + } +} + +impl ReadBackend for WarmUpAccessBackend { + fn location(&self) -> String { + self.be.location() + } + + fn list_with_size(&self, tpe: FileType) -> Result> { + self.be.list_with_size(tpe) + } + + fn read_full(&self, tpe: FileType, id: &Id) -> Result { + self.be.read_full(tpe, id) + } + + fn read_partial( + &self, + tpe: FileType, + id: &Id, + cacheable: bool, + offset: u32, + length: u32, + ) -> Result { + self.be.read_partial(tpe, id, cacheable, offset, length) + } + + fn needs_warm_up(&self) -> bool { + true + } + + fn warm_up(&self, tpe: FileType, id: &Id) -> Result<()> { + // warm up files by accessing them - error is ignored as we expect this to error out! + _ = self.be.read_partial(tpe, id, false, 0, 1); + Ok(()) + } +} + +impl WriteBackend for WarmUpAccessBackend { + fn create(&self) -> Result<()> { + self.be.create() + } + + fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> Result<()> { + self.be.write_bytes(tpe, id, cacheable, buf) + } + + fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> Result<()> { + // First remove cold file + self.be.remove(tpe, id, cacheable) + } +} diff --git a/src/blob.rs b/crates/core/src/blob.rs similarity index 100% rename from src/blob.rs rename to crates/core/src/blob.rs diff --git a/src/blob/packer.rs b/crates/core/src/blob/packer.rs similarity index 97% rename from src/blob/packer.rs rename to crates/core/src/blob/packer.rs index f8818f5c..ea659bff 100644 --- a/src/blob/packer.rs +++ b/crates/core/src/blob/packer.rs @@ -16,13 +16,14 @@ use crate::{ backend::{decrypt::DecryptFullBackend, decrypt::DecryptWriteBackend, FileType}, blob::BlobType, crypto::{hasher::hash, CryptoKey}, - error::PackerErrorKind, - error::RusticResult, + error::{PackerErrorKind, RusticErrorKind, RusticResult}, id::Id, index::indexer::SharedIndexer, repofile::{ - configfile::ConfigFile, indexfile::IndexBlob, indexfile::IndexPack, - packfile::PackHeaderLength, packfile::PackHeaderRef, snapshotfile::SnapshotSummary, + configfile::ConfigFile, + indexfile::{IndexBlob, IndexPack}, + packfile::{PackHeaderLength, PackHeaderRef}, + snapshotfile::SnapshotSummary, }, }; @@ -171,7 +172,7 @@ impl Packer { config: &ConfigFile, total_size: u64, ) -> RusticResult { - let key = be.key().clone(); + let key = *be.key(); let raw_packer = Arc::new(RwLock::new(RawPacker::new( be, blob_type, @@ -230,7 +231,7 @@ impl Packer { .unwrap() .add_raw(&data, &id, data_len, ul, size_limit) }) - .and_then(|_| raw_packer.write().unwrap().finalize()); + .and_then(|()| raw_packer.write().unwrap().finalize()); _ = finish_tx.send(status); }) .unwrap(); @@ -613,7 +614,8 @@ impl FileWriterHandle { let (file, id, mut index) = load; index.id = id; self.be - .write_bytes(FileType::Pack, &id, self.cacheable, file)?; + .write_bytes(FileType::Pack, &id, self.cacheable, file) + .map_err(RusticErrorKind::Backend)?; index.time = Some(Local::now()); Ok(index) } @@ -769,13 +771,16 @@ impl Repacker { /// If the blob could not be added /// If reading the blob from the backend fails pub fn add_fast(&self, pack_id: &Id, blob: &IndexBlob) -> RusticResult<()> { - let data = self.be.read_partial( - FileType::Pack, - pack_id, - blob.tpe.is_cacheable(), - blob.offset, - blob.length, - )?; + let data = self + .be + .read_partial( + FileType::Pack, + pack_id, + blob.tpe.is_cacheable(), + blob.offset, + blob.length, + ) + .map_err(RusticErrorKind::Backend)?; self.packer.add_raw( &data, &blob.id, diff --git a/src/blob/tree.rs b/crates/core/src/blob/tree.rs similarity index 91% rename from src/blob/tree.rs rename to crates/core/src/blob/tree.rs index e3e06ec3..7f7e048d 100644 --- a/src/blob/tree.rs +++ b/crates/core/src/blob/tree.rs @@ -17,12 +17,14 @@ use serde::{Deserialize, Deserializer}; use serde_derive::Serialize; use crate::{ - backend::{node::Metadata, node::Node, node::NodeType}, + backend::{ + decrypt::DecryptReadBackend, + node::{Metadata, Node, NodeType}, + }, crypto::hasher::hash, - error::RusticResult, - error::TreeErrorKind, + error::{RusticResult, TreeErrorKind}, id::Id, - index::IndexedBackend, + index::ReadGlobalIndex, progress::Progress, repofile::snapshotfile::SnapshotSummary, }; @@ -101,11 +103,15 @@ impl Tree { /// /// [`TreeErrorKind::BlobIdNotFound`]: crate::error::TreeErrorKind::BlobIdNotFound /// [`TreeErrorKind::DeserializingTreeFailed`]: crate::error::TreeErrorKind::DeserializingTreeFailed - pub(crate) fn from_backend(be: &impl IndexedBackend, id: Id) -> RusticResult { - let data = be + pub(crate) fn from_backend( + be: &impl DecryptReadBackend, + index: &impl ReadGlobalIndex, + id: Id, + ) -> RusticResult { + let data = index .get_tree(&id) .ok_or_else(|| TreeErrorKind::BlobIdNotFound(id))? - .read_data(be.be())?; + .read_data(be)?; Ok(serde_json::from_slice(&data).map_err(TreeErrorKind::DeserializingTreeFailed)?) } @@ -128,7 +134,8 @@ impl Tree { /// [`TreeErrorKind::PathNotFound`]: crate::error::TreeErrorKind::PathNotFound /// [`TreeErrorKind::PathIsNotUtf8Conform`]: crate::error::TreeErrorKind::PathIsNotUtf8Conform pub(crate) fn node_from_path( - be: &impl IndexedBackend, + be: &impl DecryptReadBackend, + index: &impl ReadGlobalIndex, id: Id, path: &Path, ) -> RusticResult { @@ -140,7 +147,7 @@ impl Tree { let id = node .subtree .ok_or_else(|| TreeErrorKind::NotADirectory(p.clone()))?; - let tree = Self::from_backend(be, id)?; + let tree = Self::from_backend(be, index, id)?; node = tree .nodes .into_iter() @@ -231,9 +238,10 @@ pub struct TreeStreamerOptions { /// [`NodeStreamer`] recursively streams all nodes of a given tree including all subtrees in-order #[derive(Debug, Clone)] -pub struct NodeStreamer +pub struct NodeStreamer<'a, BE, I> where - BE: IndexedBackend, + BE: DecryptReadBackend, + I: ReadGlobalIndex, { /// The open iterators for subtrees open_iterators: Vec>, @@ -243,15 +251,18 @@ where path: PathBuf, /// The backend to read from be: BE, + /// index + index: &'a I, /// The glob overrides overrides: Option, /// Whether to stream recursively recursive: bool, } -impl NodeStreamer +impl<'a, BE, I> NodeStreamer<'a, BE, I> where - BE: IndexedBackend, + BE: DecryptReadBackend, + I: ReadGlobalIndex, { /// Creates a new `NodeStreamer`. /// @@ -268,8 +279,8 @@ where /// [`TreeErrorKind::BlobIdNotFound`]: crate::error::TreeErrorKind::BlobIdNotFound /// [`TreeErrorKind::DeserializingTreeFailed`]: crate::error::TreeErrorKind::DeserializingTreeFailed #[allow(unused)] - pub fn new(be: BE, node: &Node) -> RusticResult { - Self::new_streamer(be, node, None, true) + pub fn new(be: BE, index: &'a I, node: &Node) -> RusticResult { + Self::new_streamer(be, index, node, None, true) } /// Creates a new `NodeStreamer`. @@ -290,12 +301,13 @@ where /// [`TreeErrorKind::DeserializingTreeFailed`]: crate::error::TreeErrorKind::DeserializingTreeFailed fn new_streamer( be: BE, + index: &'a I, node: &Node, overrides: Option, recursive: bool, ) -> RusticResult { let inner = if node.is_dir() { - Tree::from_backend(&be, node.subtree.unwrap())? + Tree::from_backend(&be, index, node.subtree.unwrap())? .nodes .into_iter() } else { @@ -306,6 +318,7 @@ where open_iterators: Vec::new(), path: PathBuf::new(), be, + index, overrides, recursive, }) @@ -326,7 +339,12 @@ where /// /// [`TreeErrorKind::BuildingNodeStreamerFailed`]: crate::error::TreeErrorKind::BuildingNodeStreamerFailed /// [`TreeErrorKind::ReadingFileStringFromGlobsFailed`]: crate::error::TreeErrorKind::ReadingFileStringFromGlobsFailed - pub fn new_with_glob(be: BE, node: &Node, opts: &TreeStreamerOptions) -> RusticResult { + pub fn new_with_glob( + be: BE, + index: &'a I, + node: &Node, + opts: &TreeStreamerOptions, + ) -> RusticResult { let mut override_builder = OverrideBuilder::new(""); for g in &opts.glob { @@ -369,14 +387,15 @@ where .build() .map_err(TreeErrorKind::BuildingNodeStreamerFailed)?; - Self::new_streamer(be, node, Some(overrides), opts.recursive) + Self::new_streamer(be, index, node, Some(overrides), opts.recursive) } } // TODO: This is not parallel at the moment... -impl Iterator for NodeStreamer +impl<'a, BE, I> Iterator for NodeStreamer<'a, BE, I> where - BE: IndexedBackend, + BE: DecryptReadBackend, + I: ReadGlobalIndex, { type Item = NodeStreamItem; @@ -389,7 +408,7 @@ where if let Some(id) = node.subtree { self.path.push(node.name()); let be = self.be.clone(); - let tree = match Tree::from_backend(&be, id) { + let tree = match Tree::from_backend(&be, self.index, id) { Ok(tree) => tree, Err(err) => return Some(Err(err)), }; @@ -458,7 +477,12 @@ impl TreeStreamerOnce

{ /// * [`TreeErrorKind::SendingCrossbeamMessageFailed`] - If sending the message fails. /// /// [`TreeErrorKind::SendingCrossbeamMessageFailed`]: crate::error::TreeErrorKind::SendingCrossbeamMessageFailed - pub fn new(be: BE, ids: Vec, p: P) -> RusticResult { + pub fn new( + be: &BE, + index: &I, + ids: Vec, + p: P, + ) -> RusticResult { p.set_length(ids.len() as u64); let (out_tx, out_rx) = bounded(constants::MAX_TREE_LOADER); @@ -466,12 +490,13 @@ impl TreeStreamerOnce

{ for _ in 0..constants::MAX_TREE_LOADER { let be = be.clone(); + let index = index.clone(); let in_rx = in_rx.clone(); let out_tx = out_tx.clone(); let _join_handle = std::thread::spawn(move || { for (path, id, count) in in_rx { out_tx - .send(Tree::from_backend(&be, id).map(|tree| (path, tree, count))) + .send(Tree::from_backend(&be, &index, id).map(|tree| (path, tree, count))) .unwrap(); } }); @@ -581,7 +606,8 @@ impl Iterator for TreeStreamerOnce

{ /// // TODO!: add errors pub(crate) fn merge_trees( - be: &impl IndexedBackend, + be: &impl DecryptReadBackend, + index: &impl ReadGlobalIndex, trees: &[Id], cmp: &impl Fn(&Node, &Node) -> Ordering, save: &impl Fn(Tree) -> RusticResult<(Id, u64)>, @@ -608,7 +634,7 @@ pub(crate) fn merge_trees( let mut tree_iters: Vec<_> = trees .iter() - .map(|id| Tree::from_backend(be, *id).map(std::iter::IntoIterator::into_iter)) + .map(|id| Tree::from_backend(be, index, *id).map(std::iter::IntoIterator::into_iter)) .collect::>()?; // fill Heap with first elements from all trees @@ -643,14 +669,14 @@ pub(crate) fn merge_trees( // Add node to nodes list nodes.push(node); // no node left to proceed, merge nodes and quit - tree.add(merge_nodes(be, nodes, cmp, save, summary)?); + tree.add(merge_nodes(be, index, nodes, cmp, save, summary)?); break; } Some(SortedNode(new_node, new_num)) if node.name != new_node.name => { // Add node to nodes list nodes.push(node); // next node has other name; merge present nodes - tree.add(merge_nodes(be, nodes, cmp, save, summary)?); + tree.add(merge_nodes(be, index, nodes, cmp, save, summary)?); nodes = Vec::new(); // use this node as new node (node, num) = (new_node, new_num); @@ -688,7 +714,8 @@ pub(crate) fn merge_trees( /// // TODO: add errors pub(crate) fn merge_nodes( - be: &impl IndexedBackend, + be: &impl DecryptReadBackend, + index: &impl ReadGlobalIndex, nodes: Vec, cmp: &impl Fn(&Node, &Node) -> Ordering, save: &impl Fn(Tree) -> RusticResult<(Id, u64)>, @@ -704,7 +731,7 @@ pub(crate) fn merge_nodes( // if this is a dir, merge with all other dirs if node.is_dir() { - node.subtree = Some(merge_trees(be, &trees, cmp, save, summary)?); + node.subtree = Some(merge_trees(be, index, &trees, cmp, save, summary)?); } else { summary.files_unmodified += 1; summary.total_files_processed += 1; diff --git a/src/cdc.rs b/crates/core/src/cdc.rs similarity index 100% rename from src/cdc.rs rename to crates/core/src/cdc.rs diff --git a/src/cdc/LICENSE.txt b/crates/core/src/cdc/LICENSE.txt similarity index 100% rename from src/cdc/LICENSE.txt rename to crates/core/src/cdc/LICENSE.txt diff --git a/src/cdc/README.md b/crates/core/src/cdc/README.md similarity index 100% rename from src/cdc/README.md rename to crates/core/src/cdc/README.md diff --git a/src/cdc/polynom.rs b/crates/core/src/cdc/polynom.rs similarity index 100% rename from src/cdc/polynom.rs rename to crates/core/src/cdc/polynom.rs diff --git a/src/cdc/rolling_hash.rs b/crates/core/src/cdc/rolling_hash.rs similarity index 100% rename from src/cdc/rolling_hash.rs rename to crates/core/src/cdc/rolling_hash.rs diff --git a/src/chunker.rs b/crates/core/src/chunker.rs similarity index 100% rename from src/chunker.rs rename to crates/core/src/chunker.rs diff --git a/src/commands.rs b/crates/core/src/commands.rs similarity index 88% rename from src/commands.rs rename to crates/core/src/commands.rs index 7e08e174..e4d15a12 100644 --- a/src/commands.rs +++ b/crates/core/src/commands.rs @@ -1,3 +1,5 @@ +//! The commands that can be run by the CLI. + pub mod backup; /// The `cat` command. pub mod cat; diff --git a/src/commands/backup.rs b/crates/core/src/commands/backup.rs similarity index 94% rename from src/commands/backup.rs rename to crates/core/src/commands/backup.rs index 4a474d0e..8f3d6ff3 100644 --- a/src/commands/backup.rs +++ b/crates/core/src/commands/backup.rs @@ -10,13 +10,18 @@ use serde_with::{serde_as, DisplayFromStr}; use crate::{ archiver::{parent::Parent, Archiver}, - backend::ignore::{LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions}, - backend::{dry_run::DryRunBackend, stdin::StdinSource}, + backend::{ + dry_run::DryRunBackend, + ignore::{LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions}, + stdin::StdinSource, + }, error::RusticResult, id::Id, progress::ProgressBars, - repofile::snapshotfile::{SnapshotGroup, SnapshotGroupCriterion}, - repofile::{PathList, SnapshotFile}, + repofile::{ + snapshotfile::{SnapshotGroup, SnapshotGroupCriterion}, + PathList, SnapshotFile, + }, repository::{IndexedIds, IndexedTree, Repository}, }; @@ -107,6 +112,7 @@ impl ParentOptions { ( parent_id, Parent::new( + repo.dbe(), repo.index(), parent_tree, self.ignore_ctime, @@ -191,12 +197,12 @@ pub struct BackupOptions { pub(crate) fn backup( repo: &Repository, opts: &BackupOptions, - source: PathList, + source: &PathList, mut snap: SnapshotFile, ) -> RusticResult { let index = repo.index(); - let backup_stdin = source == PathList::from_string("-")?; + let backup_stdin = *source == PathList::from_string("-")?; let backup_path = if backup_stdin { vec![PathBuf::from(&opts.stdin_filename)] } else { @@ -227,14 +233,13 @@ pub(crate) fn backup( let be = DryRunBackend::new(repo.dbe().clone(), opts.dry_run); info!("starting to backup {source}..."); - let archiver = Archiver::new(be, index.clone(), repo.config(), parent, snap)?; + let archiver = Archiver::new(be, index, repo.config(), parent, snap)?; let p = repo.pb.progress_bytes("determining size..."); let snap = if backup_stdin { let path = &backup_path[0]; - let src = StdinSource::new(path.clone())?; + let src = StdinSource::new(path.clone()); archiver.archive( - repo.index(), src, path, as_path.as_ref(), @@ -248,7 +253,6 @@ pub(crate) fn backup( &backup_path, )?; archiver.archive( - repo.index(), src, &backup_path[0], as_path.as_ref(), diff --git a/src/commands/cat.rs b/crates/core/src/commands/cat.rs similarity index 77% rename from src/commands/cat.rs rename to crates/core/src/commands/cat.rs index e7b0c0be..a4d523ca 100644 --- a/src/commands/cat.rs +++ b/crates/core/src/commands/cat.rs @@ -3,12 +3,11 @@ use std::path::Path; use bytes::Bytes; use crate::{ - backend::{decrypt::DecryptReadBackend, FileType, ReadBackend}, + backend::{decrypt::DecryptReadBackend, FileType, FindInBackend}, blob::{tree::Tree, BlobType}, - error::CommandErrorKind, - error::RusticResult, + error::{CommandErrorKind, RusticResult}, id::Id, - index::IndexedBackend, + index::ReadIndex, progress::ProgressBars, repofile::SnapshotFile, repository::{IndexedFull, IndexedTree, Open, Repository}, @@ -30,16 +29,16 @@ use crate::{ /// # Errors /// /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string -/// * [`BackendErrorKind::NoSuitableIdFound`] - If no id could be found. -/// * [`BackendErrorKind::IdNotUnique`] - If the id is not unique. +/// * [`BackendAccessErrorKind::NoSuitableIdFound`] - If no id could be found. +/// * [`BackendAccessErrorKind::IdNotUnique`] - If the id is not unique. /// /// # Returns /// /// The data read. /// /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError -/// [`BackendErrorKind::NoSuitableIdFound`]: crate::error::BackendErrorKind::NoSuitableIdFound -/// [`BackendErrorKind::IdNotUnique`]: crate::error::BackendErrorKind::IdNotUnique +/// [`BackendAccessErrorKind::NoSuitableIdFound`]: crate::error::BackendAccessErrorKind::NoSuitableIdFound +/// [`BackendAccessErrorKind::IdNotUnique`]: crate::error::BackendAccessErrorKind::IdNotUnique pub(crate) fn cat_file( repo: &Repository, tpe: FileType, @@ -74,7 +73,7 @@ pub(crate) fn cat_blob( id: &str, ) -> RusticResult { let id = Id::from_hex(id)?; - let data = repo.index().blob_from_backend(tpe, &id)?; + let data = repo.index().blob_from_backend(repo.dbe(), tpe, &id)?; Ok(data) } @@ -113,10 +112,12 @@ pub(crate) fn cat_tree( sn_filter, &repo.pb.progress_counter("getting snapshot..."), )?; - let node = Tree::node_from_path(repo.index(), snap.tree, Path::new(path))?; + let node = Tree::node_from_path(repo.dbe(), repo.index(), snap.tree, Path::new(path))?; let id = node .subtree .ok_or_else(|| CommandErrorKind::PathIsNoDir(path.to_string()))?; - let data = repo.index().blob_from_backend(BlobType::Tree, &id)?; + let data = repo + .index() + .blob_from_backend(repo.dbe(), BlobType::Tree, &id)?; Ok(data) } diff --git a/src/commands/check.rs b/crates/core/src/commands/check.rs similarity index 93% rename from src/commands/check.rs rename to crates/core/src/commands/check.rs index 493d768e..bbd76d5f 100644 --- a/src/commands/check.rs +++ b/crates/core/src/commands/check.rs @@ -12,14 +12,13 @@ use crate::{ backend::{cache::Cache, decrypt::DecryptReadBackend, node::NodeType, FileType, ReadBackend}, blob::{tree::TreeStreamerOnce, BlobType}, crypto::hasher::hash, - error::RusticResult, + error::{RusticErrorKind, RusticResult}, id::Id, index::{ binarysorted::{IndexCollector, IndexType}, - IndexBackend, IndexedBackend, + GlobalIndex, ReadGlobalIndex, }, - progress::Progress, - progress::ProgressBars, + progress::{Progress, ProgressBars}, repofile::{IndexFile, IndexPack, PackHeader, PackHeaderLength, PackHeaderRef, SnapshotFile}, repository::{Open, Repository}, }; @@ -57,7 +56,7 @@ impl CheckOptions { let be = repo.dbe(); let cache = repo.cache(); let hot_be = &repo.be_hot; - let raw_be = &repo.be; + let raw_be = repo.dbe(); let pb = &repo.pb; if !self.trust_cache { if let Some(cache) = &cache { @@ -66,7 +65,9 @@ impl CheckOptions { // // This lists files here and later when reading index / checking snapshots // TODO: Only list the files once... - _ = be.list_with_size(file_type)?; + _ = be + .list_with_size(file_type) + .map_err(RusticErrorKind::Backend)?; let p = pb.progress_bytes(format!("checking {file_type:?} in cache...")); // TODO: Make concurrency (20) customizable @@ -106,9 +107,9 @@ impl CheckOptions { .map(|(_, size)| u64::from(*size)) .sum::(); - let index_be = IndexBackend::new_from_index(be, index_collector.into_index()); + let index_be = GlobalIndex::new_from_index(index_collector.into_index()); - check_snapshots(&index_be, pb)?; + check_snapshots(be, &index_be, pb)?; if self.read_data { let p = pb.progress_bytes("reading pack data..."); @@ -118,10 +119,10 @@ impl CheckOptions { .into_index() .into_iter() .par_bridge() - .for_each_with((be.clone(), p.clone()), |(be, p), pack| { + .for_each(|pack| { let id = pack.id; let data = be.read_full(FileType::Pack, &id).unwrap(); - match check_pack(be, pack, data, p) { + match check_pack(be, pack, data, &p) { Ok(()) => {} Err(err) => error!("Error reading pack {id} : {err}",), } @@ -152,11 +153,14 @@ fn check_hot_files( ) -> RusticResult<()> { let p = pb.progress_spinner(format!("checking {file_type:?} in hot repo...")); let mut files = be - .list_with_size(file_type)? + .list_with_size(file_type) + .map_err(RusticErrorKind::Backend)? .into_iter() .collect::>(); - let files_hot = be_hot.list_with_size(file_type)?; + let files_hot = be_hot + .list_with_size(file_type) + .map_err(RusticErrorKind::Backend)?; for (id, size_hot) in files_hot { match files.remove(&id) { @@ -339,7 +343,10 @@ fn check_packs( /// /// If a pack is missing or has a different size fn check_packs_list(be: &impl ReadBackend, mut packs: HashMap) -> RusticResult<()> { - for (id, size) in be.list_with_size(FileType::Pack)? { + for (id, size) in be + .list_with_size(FileType::Pack) + .map_err(RusticErrorKind::Backend)? + { match packs.remove(&id) { None => warn!("pack {id} not referenced in index. Can be a parallel backup job. To repair: 'rustic repair index'."), Some(index_size) if index_size != size => { @@ -365,10 +372,13 @@ fn check_packs_list(be: &impl ReadBackend, mut packs: HashMap) -> Rusti /// # Errors /// /// If a snapshot or tree is missing or has a different size -fn check_snapshots(index: &impl IndexedBackend, pb: &impl ProgressBars) -> RusticResult<()> { +fn check_snapshots( + be: &impl DecryptReadBackend, + index: &impl ReadGlobalIndex, + pb: &impl ProgressBars, +) -> RusticResult<()> { let p = pb.progress_counter("reading snapshots..."); - let snap_trees: Vec<_> = index - .be() + let snap_trees: Vec<_> = be .stream_all::(&p)? .iter() .map_ok(|(_, snap)| snap.tree) @@ -376,7 +386,7 @@ fn check_snapshots(index: &impl IndexedBackend, pb: &impl ProgressBars) -> Rusti p.finish(); let p = pb.progress_counter("checking trees..."); - let mut tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p)?; + let mut tree_streamer = TreeStreamerOnce::new(be, index, snap_trees, p)?; while let Some(item) = tree_streamer.next().transpose()? { let (path, tree) = item; for node in tree.nodes { diff --git a/src/commands/config.rs b/crates/core/src/commands/config.rs similarity index 96% rename from src/commands/config.rs rename to crates/core/src/commands/config.rs index d7e44af7..0777cd8b 100644 --- a/src/commands/config.rs +++ b/crates/core/src/commands/config.rs @@ -4,9 +4,8 @@ use derive_setters::Setters; use crate::{ backend::decrypt::{DecryptBackend, DecryptWriteBackend}, - crypto::aespoly1305::Key, - error::CommandErrorKind, - error::RusticResult, + crypto::CryptoKey, + error::{CommandErrorKind, RusticResult}, repofile::ConfigFile, repository::{Open, Repository}, }; @@ -55,7 +54,7 @@ pub(crate) fn apply_config( if &new_config == repo.config() { Ok(false) } else { - save_config(repo, new_config, *repo.key())?; + save_config(repo, new_config, *repo.dbe().key())?; Ok(true) } } @@ -81,22 +80,18 @@ pub(crate) fn apply_config( pub(crate) fn save_config( repo: &Repository, mut new_config: ConfigFile, - key: Key, + key: impl CryptoKey, ) -> RusticResult<()> { new_config.is_hot = None; - // don't compress the config file - let mut dbe = DecryptBackend::new(&repo.be, key); - dbe.set_zstd(None); + let dbe = DecryptBackend::new(repo.be.clone(), key); // for hot/cold backend, this only saves the config to the cold repo. - _ = dbe.save_file(&new_config)?; + _ = dbe.save_file_uncompressed(&new_config)?; if let Some(hot_be) = repo.be_hot.clone() { // save config to hot repo - let mut dbe = DecryptBackend::new(&hot_be, key); - // don't compress the config file - dbe.set_zstd(None); + let dbe = DecryptBackend::new(hot_be, key); new_config.is_hot = Some(true); - _ = dbe.save_file(&new_config)?; + _ = dbe.save_file_uncompressed(&new_config)?; } Ok(()) } diff --git a/src/commands/copy.rs b/crates/core/src/commands/copy.rs similarity index 96% rename from src/commands/copy.rs rename to crates/core/src/commands/copy.rs index d8ba8942..4a80ac59 100644 --- a/src/commands/copy.rs +++ b/crates/core/src/commands/copy.rs @@ -7,7 +7,7 @@ use crate::{ backend::{decrypt::DecryptWriteBackend, node::NodeType}, blob::{packer::Packer, tree::TreeStreamerOnce, BlobType}, error::RusticResult, - index::{indexer::Indexer, IndexedBackend, ReadIndex}, + index::{indexer::Indexer, ReadIndex}, progress::ProgressBars, repofile::SnapshotFile, repository::{IndexedFull, IndexedIds, IndexedTree, Open, Repository}, @@ -51,6 +51,7 @@ pub(crate) fn copy<'a, Q, R: IndexedFull, P: ProgressBars, S: IndexedIds>( .map(|sn| (sn.tree, SnapshotFile::clear_ids(sn))) .unzip(); + let be = repo.dbe(); let index = repo.index(); let index_dest = repo_dest.index(); let indexer = Indexer::new(be_dest.clone()).into_shared(); @@ -77,13 +78,13 @@ pub(crate) fn copy<'a, Q, R: IndexedFull, P: ProgressBars, S: IndexedIds>( .try_for_each(|id| -> RusticResult<_> { trace!("copy tree blob {id}"); if !index_dest.has_tree(id) { - let data = index.get_tree(id).unwrap().read_data(index.be())?; + let data = index.get_tree(id).unwrap().read_data(be)?; tree_packer.add(data, *id)?; } Ok(()) })?; - let tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p)?; + let tree_streamer = TreeStreamerOnce::new(be, index, snap_trees, p)?; tree_streamer .par_bridge() .try_for_each(|item| -> RusticResult<_> { @@ -95,7 +96,7 @@ pub(crate) fn copy<'a, Q, R: IndexedFull, P: ProgressBars, S: IndexedIds>( |id| -> RusticResult<_> { trace!("copy data blob {id}"); if !index_dest.has_data(id) { - let data = index.get_data(id).unwrap().read_data(index.be())?; + let data = index.get_data(id).unwrap().read_data(be)?; data_packer.add(data, *id)?; } Ok(()) @@ -107,7 +108,7 @@ pub(crate) fn copy<'a, Q, R: IndexedFull, P: ProgressBars, S: IndexedIds>( let id = node.subtree.unwrap(); trace!("copy tree blob {id}"); if !index_dest.has_tree(&id) { - let data = index.get_tree(&id).unwrap().read_data(index.be())?; + let data = index.get_tree(&id).unwrap().read_data(be)?; tree_packer.add(data, id)?; } } diff --git a/src/commands/dump.rs b/crates/core/src/commands/dump.rs similarity index 89% rename from src/commands/dump.rs rename to crates/core/src/commands/dump.rs index 80940a99..4255160b 100644 --- a/src/commands/dump.rs +++ b/crates/core/src/commands/dump.rs @@ -4,7 +4,7 @@ use crate::{ backend::node::{Node, NodeType}, blob::BlobType, error::{CommandErrorKind, RusticResult}, - index::IndexedBackend, + index::ReadIndex, repository::{IndexedFull, IndexedTree, Repository}, }; @@ -37,7 +37,9 @@ pub(crate) fn dump( for id in node.content.as_ref().unwrap() { // TODO: cache blobs which are needed later - let data = repo.index().blob_from_backend(BlobType::Data, id)?; + let data = repo + .index() + .blob_from_backend(repo.dbe(), BlobType::Data, id)?; w.write_all(&data)?; } Ok(()) diff --git a/src/commands/forget.rs b/crates/core/src/commands/forget.rs similarity index 98% rename from src/commands/forget.rs rename to crates/core/src/commands/forget.rs index 93f5dee1..8405b76e 100644 --- a/src/commands/forget.rs +++ b/crates/core/src/commands/forget.rs @@ -10,8 +10,10 @@ use crate::{ error::RusticResult, id::Id, progress::ProgressBars, - repofile::snapshotfile::{SnapshotGroup, SnapshotGroupCriterion}, - repofile::{SnapshotFile, StringList}, + repofile::{ + snapshotfile::{SnapshotGroup, SnapshotGroupCriterion}, + SnapshotFile, StringList, + }, repository::{Open, Repository}, }; @@ -43,6 +45,7 @@ pub struct ForgetSnapshot { impl ForgetGroups { /// Turn `ForgetGroups` into the list of all snapshot IDs to remove. + #[must_use] pub fn into_forget_ids(self) -> Vec { self.0 .into_iter() @@ -279,7 +282,7 @@ pub struct KeepOptions { /// assert_eq!(left, "60s".parse::().unwrap().into()); /// ``` #[cfg(feature = "merge")] -fn overwrite_zero_duration(left: &mut humantime::Duration, right: humantime::Duration) { +pub fn overwrite_zero_duration(left: &mut humantime::Duration, right: humantime::Duration) { if *left == std::time::Duration::ZERO.into() { *left = right; } diff --git a/src/commands/init.rs b/crates/core/src/commands/init.rs similarity index 92% rename from src/commands/init.rs rename to crates/core/src/commands/init.rs index ed2bb56b..6e198591 100644 --- a/src/commands/init.rs +++ b/crates/core/src/commands/init.rs @@ -5,10 +5,12 @@ use log::info; use crate::{ backend::WriteBackend, chunker::random_poly, - commands::config::{save_config, ConfigOptions}, - commands::key::KeyOptions, + commands::{ + config::{save_config, ConfigOptions}, + key::KeyOptions, + }, crypto::aespoly1305::Key, - error::RusticResult, + error::{RusticErrorKind, RusticResult}, id::Id, repofile::ConfigFile, repository::Repository, @@ -83,7 +85,7 @@ pub(crate) fn init_with_config( key_opts: &KeyOptions, config: &ConfigFile, ) -> RusticResult { - repo.be.create()?; + repo.be.create().map_err(RusticErrorKind::Backend)?; let (key, id) = key_opts.init_key(repo, pass)?; info!("key {id} successfully added."); save_config(repo, config.clone(), key)?; diff --git a/src/commands/key.rs b/crates/core/src/commands/key.rs similarity index 92% rename from src/commands/key.rs rename to crates/core/src/commands/key.rs index 5bf3eda8..4a4d195a 100644 --- a/src/commands/key.rs +++ b/crates/core/src/commands/key.rs @@ -2,11 +2,9 @@ use derive_setters::Setters; use crate::{ - backend::{FileType, WriteBackend}, - crypto::aespoly1305::Key, - crypto::hasher::hash, - error::CommandErrorKind, - error::RusticResult, + backend::{decrypt::DecryptWriteBackend, FileType, WriteBackend}, + crypto::{aespoly1305::Key, hasher::hash}, + error::{CommandErrorKind, RusticErrorKind, RusticResult}, id::Id, repofile::KeyFile, repository::{Open, Repository}, @@ -57,7 +55,7 @@ impl KeyOptions { repo: &Repository, pass: &str, ) -> RusticResult { - let key = repo.key(); + let key = repo.dbe().key(); self.add(repo, pass, *key) } @@ -110,7 +108,8 @@ impl KeyOptions { let data = serde_json::to_vec(&keyfile).map_err(CommandErrorKind::FromJsonError)?; let id = hash(&data); repo.be - .write_bytes(FileType::Key, &id, false, data.into())?; + .write_bytes(FileType::Key, &id, false, data.into()) + .map_err(RusticErrorKind::Backend)?; Ok(id) } } diff --git a/src/commands/merge.rs b/crates/core/src/commands/merge.rs similarity index 95% rename from src/commands/merge.rs rename to crates/core/src/commands/merge.rs index 0ecb9fd2..53e8c627 100644 --- a/src/commands/merge.rs +++ b/crates/core/src/commands/merge.rs @@ -11,8 +11,7 @@ use crate::{ tree::{self, Tree}, BlobType, }, - error::CommandErrorKind, - error::RusticResult, + error::{CommandErrorKind, RusticResult}, id::Id, index::{indexer::Indexer, ReadIndex}, progress::{Progress, ProgressBars}, @@ -91,6 +90,7 @@ pub(crate) fn merge_trees( cmp: &impl Fn(&Node, &Node) -> Ordering, summary: &mut SnapshotSummary, ) -> RusticResult { + let be = repo.dbe(); let index = repo.index(); let indexer = Indexer::new(repo.dbe().clone()).into_shared(); let packer = Packer::new( @@ -110,7 +110,7 @@ pub(crate) fn merge_trees( }; let p = repo.pb.progress_spinner("merging snapshots..."); - let tree_merged = tree::merge_trees(index, trees, cmp, &save, summary)?; + let tree_merged = tree::merge_trees(be, index, trees, cmp, &save, summary)?; let stats = packer.finalize()?; indexer.write().unwrap().finalize()?; p.finish(); diff --git a/src/commands/prune.rs b/crates/core/src/commands/prune.rs similarity index 98% rename from src/commands/prune.rs rename to crates/core/src/commands/prune.rs index ea0ecf46..8e8aed96 100644 --- a/src/commands/prune.rs +++ b/crates/core/src/commands/prune.rs @@ -30,13 +30,12 @@ use crate::{ tree::TreeStreamerOnce, BlobType, BlobTypeMap, Initialize, }, - error::CommandErrorKind, - error::RusticResult, + error::{CommandErrorKind, RusticErrorKind, RusticResult}, id::Id, index::{ binarysorted::{IndexCollector, IndexType}, indexer::Indexer, - IndexBackend, IndexedBackend, ReadIndex, + GlobalIndex, ReadGlobalIndex, ReadIndex, }, progress::{Progress, ProgressBars}, repofile::{HeaderEntry, IndexBlob, IndexFile, IndexPack, SnapshotFile}, @@ -199,17 +198,19 @@ impl PruneOptions { p.finish(); let (used_ids, total_size) = { - let index = index_collector.into_index(); + let index = GlobalIndex::new_from_index(index_collector.into_index()); let total_size = BlobTypeMap::init(|blob_type| index.total_size(blob_type)); - let indexed_be = IndexBackend::new_from_index(&be.clone(), index); - let used_ids = find_used_blobs(&indexed_be, &self.ignore_snaps, pb)?; + let used_ids = find_used_blobs(be, &index, &self.ignore_snaps, pb)?; (used_ids, total_size) }; // list existing pack files let p = pb.progress_spinner("getting packs from repository..."); - let existing_packs: HashMap<_, _> = - be.list_with_size(FileType::Pack)?.into_iter().collect(); + let existing_packs: HashMap<_, _> = be + .list_with_size(FileType::Pack) + .map_err(RusticErrorKind::Backend)? + .into_iter() + .collect(); p.finish(); let mut pruner = PrunePlan::new(used_ids, existing_packs, index_files); @@ -357,6 +358,7 @@ pub struct PruneStats { impl PruneStats { /// Compute statistics about blobs of all types + #[must_use] pub fn blobs_sum(&self) -> SizeStats { self.blobs .values() @@ -364,6 +366,7 @@ impl PruneStats { } /// Compute total size statistics for blobs of all types + #[must_use] pub fn size_sum(&self) -> SizeStats { self.size .values() @@ -981,6 +984,7 @@ impl PrunePlan { } /// Get the list of packs-to-repack from the [`PrunePlan`]. + #[must_use] pub fn repack_packs(&self) -> Vec { self.index_files .iter() @@ -1331,21 +1335,21 @@ impl PackInfo { /// // TODO!: add errors! fn find_used_blobs( - index: &impl IndexedBackend, + be: &impl DecryptReadBackend, + index: &impl ReadGlobalIndex, ignore_snaps: &[Id], pb: &impl ProgressBars, ) -> RusticResult> { let ignore_snaps: HashSet<_> = ignore_snaps.iter().collect(); let p = pb.progress_counter("reading snapshots..."); - let list = index - .be() - .list(FileType::Snapshot)? + let list = be + .list(FileType::Snapshot) + .map_err(RusticErrorKind::Backend)? .into_iter() .filter(|id| !ignore_snaps.contains(id)) .collect(); - let snap_trees: Vec<_> = index - .be() + let snap_trees: Vec<_> = be .stream_list::(list, &p)? .into_iter() .map_ok(|(_, snap)| snap.tree) @@ -1355,7 +1359,7 @@ fn find_used_blobs( let mut ids: HashMap<_, _> = snap_trees.iter().map(|id| (*id, 0)).collect(); let p = pb.progress_counter("finding used blobs..."); - let mut tree_streamer = TreeStreamerOnce::new(index.clone(), snap_trees, p)?; + let mut tree_streamer = TreeStreamerOnce::new(be, index, snap_trees, p)?; while let Some(item) = tree_streamer.next().transpose()? { let (_, tree) = item; for node in tree.nodes { diff --git a/src/commands/repair.rs b/crates/core/src/commands/repair.rs similarity index 100% rename from src/commands/repair.rs rename to crates/core/src/commands/repair.rs diff --git a/src/commands/repair/index.rs b/crates/core/src/commands/repair/index.rs similarity index 93% rename from src/commands/repair/index.rs rename to crates/core/src/commands/repair/index.rs index 134a7ddf..89d21f86 100644 --- a/src/commands/repair/index.rs +++ b/crates/core/src/commands/repair/index.rs @@ -9,7 +9,7 @@ use crate::{ decrypt::{DecryptReadBackend, DecryptWriteBackend}, FileType, ReadBackend, WriteBackend, }, - error::{CommandErrorKind, RusticResult}, + error::{CommandErrorKind, RusticErrorKind, RusticResult}, index::indexer::Indexer, progress::{Progress, ProgressBars}, repofile::{IndexFile, IndexPack, PackHeader, PackHeaderRef}, @@ -46,7 +46,11 @@ impl RepairIndexOptions { ) -> RusticResult<()> { let be = repo.dbe(); let p = repo.pb.progress_spinner("listing packs..."); - let mut packs: HashMap<_, _> = be.list_with_size(FileType::Pack)?.into_iter().collect(); + let mut packs: HashMap<_, _> = be + .list_with_size(FileType::Pack) + .map_err(RusticErrorKind::Backend)? + .into_iter() + .collect(); p.finish(); let mut pack_read_header = Vec::new(); @@ -100,7 +104,8 @@ impl RepairIndexOptions { if !new_index.packs.is_empty() || !new_index.packs_to_delete.is_empty() { _ = be.save_file(&new_index)?; } - be.remove(FileType::Index, &index_id, true)?; + be.remove(FileType::Index, &index_id, true) + .map_err(RusticErrorKind::Backend)?; } (false, _) => {} // nothing to do } diff --git a/src/commands/repair/snapshots.rs b/crates/core/src/commands/repair/snapshots.rs similarity index 86% rename from src/commands/repair/snapshots.rs rename to crates/core/src/commands/repair/snapshots.rs index 5ff8c5a4..50fbbd7d 100644 --- a/src/commands/repair/snapshots.rs +++ b/crates/core/src/commands/repair/snapshots.rs @@ -5,11 +5,15 @@ use log::{info, warn}; use std::collections::{HashMap, HashSet}; use crate::{ - backend::{decrypt::DecryptWriteBackend, node::NodeType, FileType}, + backend::{ + decrypt::{DecryptFullBackend, DecryptWriteBackend}, + node::NodeType, + FileType, + }, blob::{packer::Packer, tree::Tree, BlobType}, error::RusticResult, id::Id, - index::{indexer::Indexer, IndexedBackend, ReadIndex}, + index::{indexer::Indexer, ReadGlobalIndex, ReadIndex}, progress::ProgressBars, repofile::{SnapshotFile, StringList}, repository::{IndexedFull, IndexedTree, Repository}, @@ -61,6 +65,13 @@ enum Changed { None, } +#[derive(Default)] +struct RepairState { + replaced: HashMap, + seen: HashSet, + delete: Vec, +} + impl RepairSnapshotsOptions { /// Runs the `repair snapshots` command /// @@ -83,9 +94,7 @@ impl RepairSnapshotsOptions { let be = repo.dbe(); let config_file = repo.config(); - let mut replaced = HashMap::new(); - let mut seen = HashSet::new(); - let mut delete = Vec::new(); + let mut state = RepairState::default(); let indexer = Indexer::new(be.clone()).into_shared(); let mut packer = Packer::new( @@ -100,11 +109,11 @@ impl RepairSnapshotsOptions { let snap_id = snap.id; info!("processing snapshot {snap_id}"); match self.repair_tree( + repo.dbe(), repo.index(), &mut packer, Some(snap.tree), - &mut replaced, - &mut seen, + &mut state, dry_run, )? { (Changed::None, _) => { @@ -112,7 +121,7 @@ impl RepairSnapshotsOptions { } (Changed::This, _) => { warn!("snapshot {snap_id}: root tree is damaged -> marking for deletion!"); - delete.push(snap_id); + state.delete.push(snap_id); } (Changed::SubTree, id) => { // change snapshot tree @@ -127,7 +136,7 @@ impl RepairSnapshotsOptions { let new_id = be.save_file(&snap)?; info!("saved modified snapshot as {new_id}."); } - delete.push(snap_id); + state.delete.push(snap_id); } } } @@ -139,12 +148,12 @@ impl RepairSnapshotsOptions { if self.delete { if dry_run { - info!("would have removed {} snapshots.", delete.len()); + info!("would have removed {} snapshots.", state.delete.len()); } else { be.delete_list( FileType::Snapshot, true, - delete.iter(), + state.delete.iter(), repo.pb.progress_counter("remove defect snapshots"), )?; } @@ -173,24 +182,24 @@ impl RepairSnapshotsOptions { /// A tuple containing the change status and the id of the repaired tree fn repair_tree( &self, - be: &impl IndexedBackend, + be: &impl DecryptFullBackend, + index: &impl ReadGlobalIndex, packer: &mut Packer, id: Option, - replaced: &mut HashMap, - seen: &mut HashSet, + state: &mut RepairState, dry_run: bool, ) -> RusticResult<(Changed, Id)> { let (tree, changed) = match id { None => (Tree::new(), Changed::This), Some(id) => { - if seen.contains(&id) { + if state.seen.contains(&id) { return Ok((Changed::None, id)); } - if let Some(r) = replaced.get(&id) { + if let Some(r) = state.replaced.get(&id) { return Ok(*r); } - let (tree, mut changed) = Tree::from_backend(be, id).map_or_else( + let (tree, mut changed) = Tree::from_backend(be, index, id).map_or_else( |_err| { warn!("tree {id} could not be loaded."); (Tree::new(), Changed::This) @@ -207,7 +216,7 @@ impl RepairSnapshotsOptions { let mut new_content = Vec::new(); let mut new_size = 0; for blob in node.content.take().unwrap() { - be.get_data(&blob).map_or_else( + index.get_data(&blob).map_or_else( || { file_changed = true; }, @@ -229,14 +238,8 @@ impl RepairSnapshotsOptions { node.meta.size = new_size; } NodeType::Dir {} => { - let (c, tree_id) = self.repair_tree( - be, - packer, - node.subtree, - replaced, - seen, - dry_run, - )?; + let (c, tree_id) = + self.repair_tree(be, index, packer, node.subtree, state, dry_run)?; match c { Changed::None => {} Changed::This => { @@ -256,7 +259,7 @@ impl RepairSnapshotsOptions { new_tree.add(node); } if matches!(changed, Changed::None) { - _ = seen.insert(id); + _ = state.seen.insert(id); } (new_tree, changed) } @@ -268,11 +271,11 @@ impl RepairSnapshotsOptions { (_, c) => { // the tree has been changed => save it let (chunk, new_id) = tree.serialize()?; - if !be.has_tree(&new_id) && !dry_run { + if !index.has_tree(&new_id) && !dry_run { packer.add(chunk.into(), new_id)?; } if let Some(id) = id { - _ = replaced.insert(id, (c, new_id)); + _ = state.replaced.insert(id, (c, new_id)); } Ok((c, new_id)) } diff --git a/src/commands/repoinfo.rs b/crates/core/src/commands/repoinfo.rs similarity index 97% rename from src/commands/repoinfo.rs rename to crates/core/src/commands/repoinfo.rs index 73f241a1..00252ddf 100644 --- a/src/commands/repoinfo.rs +++ b/crates/core/src/commands/repoinfo.rs @@ -3,7 +3,7 @@ use serde_derive::{Deserialize, Serialize}; use crate::{ backend::{decrypt::DecryptReadBackend, FileType, ReadBackend, ALL_FILE_TYPES}, blob::{BlobType, BlobTypeMap}, - error::RusticResult, + error::{RusticErrorKind, RusticResult}, index::IndexEntry, progress::{Progress, ProgressBars}, repofile::indexfile::{IndexFile, IndexPack}, @@ -103,14 +103,14 @@ impl PackInfo { pub(crate) fn collect_index_infos( repo: &Repository, ) -> RusticResult { - let mut blob_info = BlobTypeMap::<()>::default().map(|blob_type, _| BlobInfo { + let mut blob_info = BlobTypeMap::<()>::default().map(|blob_type, ()| BlobInfo { blob_type, count: 0, size: 0, data_size: 0, }); let mut blob_info_delete = blob_info; - let mut pack_info = BlobTypeMap::<()>::default().map(|blob_type, _| PackInfo { + let mut pack_info = BlobTypeMap::<()>::default().map(|blob_type, ()| PackInfo { blob_type, count: 0, min_size: None, @@ -185,7 +185,7 @@ pub struct RepoFileInfo { pub(crate) fn collect_file_info(be: &impl ReadBackend) -> RusticResult> { let mut files = Vec::with_capacity(ALL_FILE_TYPES.len()); for tpe in ALL_FILE_TYPES { - let list = be.list_with_size(tpe)?; + let list = be.list_with_size(tpe).map_err(RusticErrorKind::Backend)?; let count = list.len() as u64; let size = list.iter().map(|f| u64::from(f.1)).sum(); files.push(RepoFileInfo { tpe, count, size }); diff --git a/src/commands/restore.rs b/crates/core/src/commands/restore.rs similarity index 92% rename from src/commands/restore.rs rename to crates/core/src/commands/restore.rs index 9f3396f7..59f2af9c 100644 --- a/src/commands/restore.rs +++ b/crates/core/src/commands/restore.rs @@ -19,13 +19,12 @@ use rayon::ThreadPoolBuilder; use crate::{ backend::{ decrypt::DecryptReadBackend, - local::LocalDestination, + local_destination::LocalDestination, node::{Node, NodeType}, FileType, ReadBackend, }, blob::BlobType, - error::CommandErrorKind, - error::RusticResult, + error::{CommandErrorKind, RusticResult}, id::Id, progress::{Progress, ProgressBars}, repository::{IndexedFull, IndexedTree, Open, Repository}, @@ -218,7 +217,7 @@ impl RestoreOptions { debug!("to restore: {path:?}"); if !dry_run { dest.create_dir(path).map_err(|err| { - CommandErrorKind::ErrorCreating(path.to_path_buf(), Box::new(err)) + CommandErrorKind::ErrorCreating(path.clone(), Box::new(err)) })?; } } @@ -230,7 +229,7 @@ impl RestoreOptions { restore_infos .add_file(dest, node, path.clone(), repo, self.verify_existing) .map_err(|err| { - CommandErrorKind::ErrorCollecting(path.to_path_buf(), Box::new(err)) + CommandErrorKind::ErrorCollecting(path.clone(), Box::new(err)) })?, ) { // Note that exists = false and Existing or Verified can happen if the file is changed between scanning the dir @@ -275,33 +274,35 @@ impl RestoreOptions { match (&next_dst, &next_node) { (None, None) => break, - (Some(dst), None) => { - process_existing(dst)?; + (Some(destination), None) => { + process_existing(destination)?; next_dst = dst_iter.next(); } - (Some(dst), Some((path, node))) => match dst.path().cmp(&dest.path(path)) { - Ordering::Less => { - process_existing(dst)?; - next_dst = dst_iter.next(); - } - Ordering::Equal => { - // process existing node - if (node.is_dir() && !dst.file_type().unwrap().is_dir()) - || (node.is_file() && !dst.metadata().unwrap().is_file()) - || node.is_special() - { - // if types do not match, first remove the existing file - process_existing(dst)?; + (Some(destination), Some((path, node))) => { + match destination.path().cmp(&dest.path(path)) { + Ordering::Less => { + process_existing(destination)?; + next_dst = dst_iter.next(); + } + Ordering::Equal => { + // process existing node + if (node.is_dir() && !destination.file_type().unwrap().is_dir()) + || (node.is_file() && !destination.metadata().unwrap().is_file()) + || node.is_special() + { + // if types do not match, first remove the existing file + process_existing(destination)?; + } + process_node(path, node, true)?; + next_dst = dst_iter.next(); + next_node = node_streamer.next().transpose()?; + } + Ordering::Greater => { + process_node(path, node, false)?; + next_node = node_streamer.next().transpose()?; } - process_node(path, node, true)?; - next_dst = dst_iter.next(); - next_node = node_streamer.next().transpose()?; - } - Ordering::Greater => { - process_node(path, node, false)?; - next_node = node_streamer.next().transpose()?; } - }, + } (None, Some((path, node))) => { process_node(path, node, false)?; next_node = node_streamer.next().transpose()?; @@ -436,9 +437,8 @@ fn restore_contents( for (i, size) in file_lengths.iter().enumerate() { if *size == 0 { let path = &filenames[i]; - dest.set_length(path, *size).map_err(|err| { - CommandErrorKind::ErrorSettingLength(path.to_path_buf(), Box::new(err)) - })?; + dest.set_length(path, *size) + .map_err(|err| CommandErrorKind::ErrorSettingLength(path.clone(), Box::new(err)))?; } } @@ -522,7 +522,7 @@ fn restore_contents( dest.set_length(path, filesize) .map_err(|err| { CommandErrorKind::ErrorSettingLength( - path.to_path_buf(), + path.clone(), Box::new(err), ) }) @@ -585,7 +585,7 @@ impl BlobLocation { self.uncompressed_length .map_or( self.length - 32, // crypto overhead - |length| length.get(), + std::num::NonZeroU32::get, ) .into() } @@ -643,7 +643,11 @@ impl RestorePlan { // Empty files which exists with correct size should always return Ok(Existing)! if file.meta.size == 0 { - if let Some(meta) = open_file.as_ref().map(|f| f.metadata()).transpose()? { + if let Some(meta) = open_file + .as_ref() + .map(std::fs::File::metadata) + .transpose()? + { if meta.len() == 0 { // Empty file exists return Ok(AddFileResult::Existing); @@ -652,7 +656,11 @@ impl RestorePlan { } if !ignore_mtime { - if let Some(meta) = open_file.as_ref().map(|f| f.metadata()).transpose()? { + if let Some(meta) = open_file + .as_ref() + .map(std::fs::File::metadata) + .transpose()? + { // TODO: This is the same logic as in backend/ignore.rs => consollidate! let mtime = meta .modified() @@ -713,6 +721,7 @@ impl RestorePlan { /// Get a list of all pack files needed to perform the restore /// /// This can be used e.g. to warm-up those pack files before doing the atual restore. + #[must_use] pub fn to_packs(&self) -> Vec { self.r .iter() diff --git a/src/commands/snapshots.rs b/crates/core/src/commands/snapshots.rs similarity index 93% rename from src/commands/snapshots.rs rename to crates/core/src/commands/snapshots.rs index 7ffb8b6b..37b11e0c 100644 --- a/src/commands/snapshots.rs +++ b/crates/core/src/commands/snapshots.rs @@ -3,8 +3,10 @@ use crate::{ error::RusticResult, progress::ProgressBars, - repofile::snapshotfile::{SnapshotGroup, SnapshotGroupCriterion}, - repofile::SnapshotFile, + repofile::{ + snapshotfile::{SnapshotGroup, SnapshotGroupCriterion}, + SnapshotFile, + }, repository::{Open, Repository}, }; diff --git a/src/crypto.rs b/crates/core/src/crypto.rs similarity index 90% rename from src/crypto.rs rename to crates/core/src/crypto.rs index 6019db07..89e88554 100644 --- a/src/crypto.rs +++ b/crates/core/src/crypto.rs @@ -4,7 +4,7 @@ pub(crate) mod aespoly1305; pub(crate) mod hasher; /// A trait for encrypting and decrypting data. -pub trait CryptoKey: Clone + Sized + Send + Sync + 'static { +pub trait CryptoKey: Clone + Copy + Sized + Send + Sync + 'static { /// Decrypt the given data. /// /// # Arguments diff --git a/src/crypto/aespoly1305.rs b/crates/core/src/crypto/aespoly1305.rs similarity index 100% rename from src/crypto/aespoly1305.rs rename to crates/core/src/crypto/aespoly1305.rs diff --git a/src/crypto/hasher.rs b/crates/core/src/crypto/hasher.rs similarity index 100% rename from src/crypto/hasher.rs rename to crates/core/src/crypto/hasher.rs diff --git a/src/error.rs b/crates/core/src/error.rs similarity index 85% rename from src/error.rs rename to crates/core/src/error.rs index 40fbc044..6a942ad9 100644 --- a/src/error.rs +++ b/crates/core/src/error.rs @@ -9,7 +9,6 @@ use std::{ num::{ParseIntError, TryFromIntError}, ops::RangeInclusive, path::{PathBuf, StripPrefixError}, - process::ExitStatus, str::Utf8Error, time::SystemTimeError, }; @@ -49,6 +48,17 @@ impl RusticError { RusticErrorKind::Repository(RepositoryErrorKind::IncorrectPassword) ) } + + /// Get the corresponding backend error, if error is caused by the backend. + /// + /// Returns `anyhow::Error`; you need to cast this to the real backend error type + pub fn backend_error(&self) -> Option<&anyhow::Error> { + if let RusticErrorKind::Backend(error) = &self.0 { + Some(error) + } else { + None + } + } } /// [`RusticErrorKind`] describes the errors that can happen while executing a high-level command. @@ -83,9 +93,13 @@ pub enum RusticErrorKind { #[error(transparent)] Index(#[from] IndexErrorKind), - /// [`BackendErrorKind`] describes the errors that can be returned by the various Backends + /// describes the errors that can be returned by the various Backends + #[error(transparent)] + Backend(#[from] anyhow::Error), + + /// [`BackendAccessErrorKind`] describes the errors that can be returned by accessing the various Backends #[error(transparent)] - Backend(#[from] BackendErrorKind), + BackendAccess(#[from] BackendAccessErrorKind), /// [`ConfigFileErrorKind`] describes the errors that can be returned for `ConfigFile`s #[error(transparent)] @@ -127,22 +141,14 @@ pub enum RusticErrorKind { #[error(transparent)] Ignore(#[from] IgnoreErrorKind), - /// [`LocalErrorKind`] describes the errors that can be returned by an action on the filesystem in Backends + /// [`LocalDestinationErrorKind`] describes the errors that can be returned by an action on the local filesystem as Destination #[error(transparent)] - Local(#[from] LocalErrorKind), + LocalDestination(#[from] LocalDestinationErrorKind), /// [`NodeErrorKind`] describes the errors that can be returned by an action utilizing a node in Backends #[error(transparent)] Node(#[from] NodeErrorKind), - /// [`ProviderErrorKind`] describes the errors that can be returned by a backend provider - #[error(transparent)] - Provider(#[from] ProviderErrorKind), - - /// [`RestErrorKind`] describes the errors that can be returned while dealing with the REST API - #[error(transparent)] - Rest(#[from] RestErrorKind), - /// [`StdInErrorKind`] describes the errors that can be returned while dealing IO from CLI #[error(transparent)] StdIn(#[from] StdInErrorKind), @@ -290,6 +296,8 @@ pub enum RepositoryErrorKind { ConfigFileExists, /// did not find id {0} in index IdNotFound(Id), + /// no suitable backend type found + NoBackendTypeGiven, } /// [`IndexErrorKind`] describes the errors that can be returned by processing Indizes @@ -305,30 +313,17 @@ pub enum IndexErrorKind { CouldNotGetElapsedTimeFromSystemTime(#[from] SystemTimeError), } -/// [`BackendErrorKind`] describes the errors that can be returned by the various Backends +/// [`BackendAccessErrorKind`] describes the errors that can be returned by the various Backends #[derive(Error, Debug, Display)] -pub enum BackendErrorKind { +pub enum BackendAccessErrorKind { + /// backend {0:?} is not supported! + BackendNotSupported(String), + /// backend {0} cannot be loaded: {1:?} + BackendLoadError(String, anyhow::Error), /// no suitable id found for {0} NoSuitableIdFound(String), /// id {0} is not unique IdNotUnique(String), - /// backend {0:?} is not supported! - BackendNotSupported(String), - /// Rest API threw an error: `{0:?}` - RestApiError(#[from] RestErrorKind), - /// building REST client failed: `{0:?}` - BuildingRestClientFailed(#[from] reqwest::Error), - /// fully reading from Backend failed - FullyReadingFromBackendFailed, - /// setting option on Backend failed - SettingOptionOnBackendFailed, - /// partially reading from Backend data failed - PartiallyReadingFromBackendDataFailed, - /// listing with size failed - ListingWithSizeFailed, - /// {0:?} - #[error(transparent)] - FromBackendCacheError(#[from] CacheBackendErrorKind), /// {0:?} #[error(transparent)] FromIoError(#[from] std::io::Error), @@ -337,10 +332,7 @@ pub enum BackendErrorKind { FromTryIntError(#[from] TryFromIntError), /// {0:?} #[error(transparent)] - FromLocalError(#[from] LocalErrorKind), - /// {0:?} - #[error(transparent)] - FromProviderError(#[from] ProviderErrorKind), + FromLocalError(#[from] LocalDestinationErrorKind), /// {0:?} #[error(transparent)] FromIdError(#[from] IdErrorKind), @@ -350,10 +342,6 @@ pub enum BackendErrorKind { /// {0:?} #[error(transparent)] FromBackendDecryptionError(#[from] CryptBackendErrorKind), - /// backoff failed: {0:?} - BackoffError(#[from] backoff::Error), - /// parsing failed for url: `{0:?}` - UrlParsingFailed(#[from] url::ParseError), /// generic Ignore error: `{0:?}` GenericError(#[from] ignore::Error), /// creating data in backend failed @@ -412,7 +400,7 @@ pub enum PackFileErrorKind { /// pack size computed from header doesn't match real pack isch! Computed: {size_computed}, real: {size_real} HeaderPackSizeComputedDoesNotMatchRealPackFile { size_real: u32, size_computed: u32 }, /// partially reading the pack header from packfile failed: `{0:?}` - ListingKeyFilesFailed(#[from] BackendErrorKind), + ListingKeyFilesFailed(#[from] BackendAccessErrorKind), /// decrypting from binary failed BinaryDecryptionFailed, /// Partial read of PackFile failed @@ -480,7 +468,7 @@ pub enum PackerErrorKind { /// couldn't create binary representation for pack header: `{0:?}` CouldNotCreateBinaryRepresentationForHeader(#[from] PackFileErrorKind), /// failed to write bytes in backend: `{0:?}` - WritingBytesFailedInBackend(#[from] BackendErrorKind), + WritingBytesFailedInBackend(#[from] BackendAccessErrorKind), /// failed to write bytes for PackFile: `{0:?}` WritingBytesFailedForPackFile(PackFileErrorKind), /// failed to read partially encrypted data: `{0:?}` @@ -601,30 +589,13 @@ pub enum IgnoreErrorKind { TargetIsNotValidUnicode { file: PathBuf, target: PathBuf }, } -/// [`LocalErrorKind`] describes the errors that can be returned by an action on the filesystem in Backends +/// [`LocalDestinationErrorKind`] describes the errors that can be returned by an action on the filesystem in Backends #[derive(Error, Debug, Display)] -pub enum LocalErrorKind { +pub enum LocalDestinationErrorKind { /// directory creation failed: `{0:?}` DirectoryCreationFailed(#[from] std::io::Error), - /// querying metadata failed: `{0:?}` - QueryingMetadataFailed(std::io::Error), - /// querying WalkDir metadata failed: `{0:?}` - QueryingWalkDirMetadataFailed(walkdir::Error), - /// executtion of command failed: `{0:?}` - CommandExecutionFailed(std::io::Error), - /// command was not successful for filename {file_name}, type {file_type}, id {id}: {status} - CommandNotSuccessful { - file_name: String, - file_type: String, - id: String, - status: ExitStatus, - }, /// file `{0:?}` should have a parent FileDoesNotHaveParent(PathBuf), - /// error building automaton `{0:?}` - FromAhoCorasick(#[from] aho_corasick::BuildError), - /// {0:?} - FromSplitError(#[from] shell_words::ParseError), /// {0:?} #[error(transparent)] FromTryIntError(#[from] TryFromIntError), @@ -671,12 +642,8 @@ pub enum LocalErrorKind { CouldNotSeekToPositionInFile(std::io::Error), /// couldn't write to buffer: `{0:?}` CouldNotWriteToBuffer(std::io::Error), - /// reading file contents failed: `{0:?}` - ReadingContentsOfFileFailed(std::io::Error), /// reading exact length of file contents failed: `{0:?}` ReadingExactLengthOfFileFailed(std::io::Error), - /// failed to sync OS Metadata to disk: `{0:?}` - SyncingOfOsMetadataFailed(std::io::Error), /// setting file permissions failed: `{0:?}` #[cfg(not(windows))] SettingFilePermissionsFailed(std::io::Error), @@ -706,50 +673,6 @@ pub enum NodeErrorKind { UnrecognizedEscape, } -/// [`ProviderErrorKind`] describes the errors that can be returned by a backend provider -#[derive(Error, Debug, Display)] -pub enum ProviderErrorKind { - /// 'rclone version' doesn't give any output - NoOutputForRcloneVersion, - /// cannot get stdout of rclone - NoStdOutForRclone, - /// rclone exited with `{0:?}` - RCloneExitWithBadStatus(ExitStatus), - /// url must start with http:\/\/! url: {0:?} - UrlNotStartingWithHttp(String), - /// StdIo Error: `{0:?}` - #[error(transparent)] - FromIoError(#[from] std::io::Error), - /// utf8 error: `{0:?}` - #[error(transparent)] - FromUtf8Error(#[from] Utf8Error), - /// `{0:?}` - #[error(transparent)] - FromRestError(#[from] RestErrorKind), - /// `{0:?}` - #[error(transparent)] - FromParseIntError(#[from] ParseIntError), -} - -/// [`RestErrorKind`] describes the errors that can be returned while dealing with the REST API -#[derive(Error, Debug, Display)] -pub enum RestErrorKind { - /// value `{0:?}` not supported for option retry! - NotSupportedForRetry(String), - /// parsing failed for url: `{0:?}` - UrlParsingFailed(#[from] url::ParseError), - /// requesting resource failed: `{0:?}` - RequestingResourceFailed(#[from] reqwest::Error), - /// couldn't parse duration in humantime library: `{0:?}` - CouldNotParseDuration(#[from] humantime::DurationError), - /// backoff failed: {0:?} - BackoffError(#[from] backoff::Error), - /// Failed to build HTTP client: `{0:?}` - BuildingClientFailed(reqwest::Error), - /// joining URL failed on: {0:?} - JoiningUrlFailed(url::ParseError), -} - /// [`StdInErrorKind`] describes the errors that can be returned while dealing IO from CLI #[derive(Error, Debug, Display)] pub enum StdInErrorKind { @@ -767,7 +690,7 @@ pub enum ArchiverErrorKind { /// option should contain a value, but contained `None` UnpackingTreeTypeOptionalFailed, /// couldn't get size for archive: `{0:?}` - CouldNotGetSizeForArchive(#[from] BackendErrorKind), + CouldNotGetSizeForArchive(#[from] BackendAccessErrorKind), /// couldn't determine size for item in Archiver CouldNotDetermineSize, /// failed to save index: `{0:?}` @@ -802,7 +725,7 @@ impl RusticErrorMarker for PolynomialErrorKind {} impl RusticErrorMarker for IdErrorKind {} impl RusticErrorMarker for RepositoryErrorKind {} impl RusticErrorMarker for IndexErrorKind {} -impl RusticErrorMarker for BackendErrorKind {} +impl RusticErrorMarker for BackendAccessErrorKind {} impl RusticErrorMarker for ConfigFileErrorKind {} impl RusticErrorMarker for KeyFileErrorKind {} impl RusticErrorMarker for PackFileErrorKind {} @@ -813,10 +736,8 @@ impl RusticErrorMarker for TreeErrorKind {} impl RusticErrorMarker for CacheBackendErrorKind {} impl RusticErrorMarker for CryptBackendErrorKind {} impl RusticErrorMarker for IgnoreErrorKind {} -impl RusticErrorMarker for LocalErrorKind {} +impl RusticErrorMarker for LocalDestinationErrorKind {} impl RusticErrorMarker for NodeErrorKind {} -impl RusticErrorMarker for ProviderErrorKind {} -impl RusticErrorMarker for RestErrorKind {} impl RusticErrorMarker for StdInErrorKind {} impl RusticErrorMarker for ArchiverErrorKind {} impl RusticErrorMarker for CommandErrorKind {} diff --git a/src/id.rs b/crates/core/src/id.rs similarity index 98% rename from src/id.rs rename to crates/core/src/id.rs index 9ec7053d..9e77c11b 100644 --- a/src/id.rs +++ b/crates/core/src/id.rs @@ -1,3 +1,5 @@ +//! The `Id` type and related functions + use std::{fmt, io::Read, ops::Deref, path::Path}; use binrw::{BinRead, BinWrite}; @@ -147,6 +149,7 @@ impl HexId { const EMPTY: Self = Self([b'0'; constants::HEX_LEN]); /// Get the string representation of a [`HexId`] + #[must_use] pub fn as_str(&self) -> &str { // This is only ever filled with hex chars, which are ascii std::str::from_utf8(&self.0).unwrap() diff --git a/src/index.rs b/crates/core/src/index.rs similarity index 79% rename from src/index.rs rename to crates/core/src/index.rs index 9bd9c1cf..c73f95ac 100644 --- a/src/index.rs +++ b/crates/core/src/index.rs @@ -6,11 +6,12 @@ use derive_more::Constructor; use crate::{ backend::{decrypt::DecryptReadBackend, FileType}, blob::BlobType, - error::{IndexErrorKind, RusticResult}, + error::IndexErrorKind, id::Id, index::binarysorted::{Index, IndexCollector, IndexType}, progress::Progress, repofile::indexfile::{IndexBlob, IndexFile}, + RusticResult, }; pub(crate) mod binarysorted; @@ -162,15 +163,6 @@ pub trait ReadIndex { fn has_data(&self, id: &Id) -> bool { self.has(BlobType::Data, id) } -} - -/// A trait for backends with an index -pub trait IndexedBackend: ReadIndex + Clone + Sync + Send + 'static { - /// The backend type - type Backend: DecryptReadBackend; - - /// Get a reference to the backend - fn be(&self) -> &Self::Backend; /// Get a blob from the backend /// @@ -181,28 +173,33 @@ pub trait IndexedBackend: ReadIndex + Clone + Sync + Send + 'static { /// /// # Errors /// - /// If the blob could not be found in the backend - /// - /// # Returns + /// * [`IndexErrorKind::BlobInIndexNotFound`] - If the blob could not be found in the index /// - /// The data of the blob - fn blob_from_backend(&self, tpe: BlobType, id: &Id) -> RusticResult; + /// [`IndexErrorKind::BlobInIndexNotFound`]: crate::error::IndexErrorKind::BlobInIndexNotFound + fn blob_from_backend( + &self, + be: &impl DecryptReadBackend, + tpe: BlobType, + id: &Id, + ) -> RusticResult { + self.get_id(tpe, id).map_or_else( + || Err(IndexErrorKind::BlobInIndexNotFound.into()), + |ie| ie.read_data(be), + ) + } } -/// A backend with an index -/// -/// # Type Parameters -/// -/// * `BE` - The backend type +/// A trait for a global index +pub trait ReadGlobalIndex: ReadIndex + Clone + Sync + Send + 'static {} + +/// A global index #[derive(Clone, Debug)] -pub struct IndexBackend { - /// The backend to read from. - be: BE, +pub struct GlobalIndex { /// The atomic reference counted, sharable index. index: Arc, } -impl ReadIndex for IndexBackend { +impl ReadIndex for GlobalIndex { /// Get an [`IndexEntry`] from the index /// /// # Arguments @@ -241,8 +238,8 @@ impl ReadIndex for IndexBackend { } } -impl IndexBackend { - /// Create a new [`IndexBackend`] from an [`Index`] +impl GlobalIndex { + /// Create a new [`GlobalIndex`] from an [`Index`] /// /// # Type Parameters /// @@ -252,18 +249,13 @@ impl IndexBackend { /// /// * `be` - The backend to read from /// * `index` - The index to use - pub fn new_from_index(be: &BE, index: Index) -> Self { + pub fn new_from_index(index: Index) -> Self { Self { - be: be.clone(), index: Arc::new(index), } } - /// Create a new [`IndexBackend`] from an [`IndexCollector`] - /// - /// # Type Parameters - /// - /// * `BE` - The backend type + /// Create a new [`GlobalIndex`] from an [`IndexCollector`] /// /// # Arguments /// @@ -275,7 +267,7 @@ impl IndexBackend { /// /// If the index could not be read fn new_from_collector( - be: &BE, + be: &impl DecryptReadBackend, p: &impl Progress, mut collector: IndexCollector, ) -> RusticResult { @@ -286,28 +278,20 @@ impl IndexBackend { p.finish(); - Ok(Self::new_from_index(be, collector.into_index())) + Ok(Self::new_from_index(collector.into_index())) } - /// Create a new [`IndexBackend`] - /// - /// # Type Parameters - /// - /// * `BE` - The backend type + /// Create a new [`GlobalIndex`] /// /// # Arguments /// /// * `be` - The backend to read from /// * `p` - The progress tracker - pub fn new(be: &BE, p: &impl Progress) -> RusticResult { + pub fn new(be: &impl DecryptReadBackend, p: &impl Progress) -> RusticResult { Self::new_from_collector(be, p, IndexCollector::new(IndexType::Full)) } - /// Create a new [`IndexBackend`] with only full trees - /// - /// # Type Parameters - /// - /// * `BE` - The backend type + /// Create a new [`GlobalIndex`] with only full trees /// /// # Arguments /// @@ -317,7 +301,7 @@ impl IndexBackend { /// # Errors /// /// If the index could not be read - pub fn only_full_trees(be: &BE, p: &impl Progress) -> RusticResult { + pub fn only_full_trees(be: &impl DecryptReadBackend, p: &impl Progress) -> RusticResult { Self::new_from_collector(be, p, IndexCollector::new(IndexType::DataIds)) } @@ -335,30 +319,4 @@ impl IndexBackend { } } -impl IndexedBackend for IndexBackend { - type Backend = BE; - - /// Get a reference to the backend - fn be(&self) -> &Self::Backend { - &self.be - } - - /// Get a blob from the backend - /// - /// # Arguments - /// - /// * `tpe` - The type of the blob - /// * `id` - The id of the blob - /// - /// # Errors - /// - /// * [`IndexErrorKind::BlobInIndexNotFound`] - If the blob could not be found in the index - /// - /// [`IndexErrorKind::BlobInIndexNotFound`]: crate::error::IndexErrorKind::BlobInIndexNotFound - fn blob_from_backend(&self, tpe: BlobType, id: &Id) -> RusticResult { - self.get_id(tpe, id).map_or_else( - || Err(IndexErrorKind::BlobInIndexNotFound.into()), - |ie| ie.read_data(self.be()), - ) - } -} +impl ReadGlobalIndex for GlobalIndex {} diff --git a/src/index/binarysorted.rs b/crates/core/src/index/binarysorted.rs similarity index 99% rename from src/index/binarysorted.rs rename to crates/core/src/index/binarysorted.rs index 531b2bc0..feb02d52 100644 --- a/src/index/binarysorted.rs +++ b/crates/core/src/index/binarysorted.rs @@ -205,7 +205,7 @@ impl IntoIterator for Index { // Turns Collector into an iterator yielding PackIndex by sorting the entries by pack. fn into_iter(mut self) -> Self::IntoIter { - for (_, tc) in self.0.iter_mut() { + for tc in self.0.values_mut() { if let EntriesVariants::FullEntries(entries) = &mut tc.entries { entries.par_sort_unstable_by(|e1, e2| e1.pack_idx.cmp(&e2.pack_idx)); } diff --git a/src/index/indexer.rs b/crates/core/src/index/indexer.rs similarity index 100% rename from src/index/indexer.rs rename to crates/core/src/index/indexer.rs diff --git a/src/lib.rs b/crates/core/src/lib.rs similarity index 91% rename from src/lib.rs rename to crates/core/src/lib.rs index 04bda03e..8683b553 100644 --- a/src/lib.rs +++ b/crates/core/src/lib.rs @@ -23,20 +23,32 @@ implement [`serde::Serialize`] and [`serde::Deserialize`]. # Example - initialize a repository, backup to it and get snapshots -``` - use rustic_core::{BackupOptions, ConfigOptions, KeyOptions, PathList, Repository, RepositoryOptions, SnapshotOptions}; +```rust + use rustic_backend::BackendOptions; + use rustic_core::{BackupOptions, ConfigOptions, KeyOptions, PathList, + Repository, RepositoryOptions, SnapshotOptions + }; // Initialize the repository in a temporary dir let repo_dir = tempfile::tempdir().unwrap(); + let repo_opts = RepositoryOptions::default() - .repository(repo_dir.path().to_str().unwrap()) .password("test"); + + // Initialize Backends + let backends = BackendOptions::default() + .repository(repo_dir.path().to_str().unwrap()) + .to_backends() + .unwrap(); + let key_opts = KeyOptions::default(); + let config_opts = ConfigOptions::default(); - let _repo = Repository::new(&repo_opts).unwrap().init(&key_opts, &config_opts).unwrap(); + + let _repo = Repository::new(&repo_opts, backends.clone()).unwrap().init(&key_opts, &config_opts).unwrap(); // We could have used _repo directly, but open the repository again to show how to open it... - let repo = Repository::new(&repo_opts).unwrap().open().unwrap(); + let repo = Repository::new(&repo_opts, backends).unwrap().open().unwrap(); // Get all snapshots from the repository let snaps = repo.get_all_snapshots().unwrap(); @@ -56,7 +68,7 @@ implement [`serde::Serialize`] and [`serde::Deserialize`]. let source = PathList::from_string("src").unwrap().sanitize().unwrap(); // run the backup and return the snapshot pointing to the backup'ed data. - let snap = repo.backup(&backup_opts, source, snap).unwrap(); + let snap = repo.backup(&backup_opts, &source, snap).unwrap(); // assert_eq!(&snap.paths, ["src"]); // Get all snapshots from the repository @@ -159,9 +171,9 @@ pub use crate::{ backend::{ decrypt::{compression_level_range, max_compression_level}, ignore::{LocalSource, LocalSourceFilterOptions, LocalSourceSaveOptions}, - local::LocalDestination, + local_destination::LocalDestination, node::last_modified_node, - ReadSourceEntry, + FileType, ReadBackend, ReadSourceEntry, RepositoryBackends, WriteBackend, ALL_FILE_TYPES, }, blob::tree::TreeStreamerOptions as LsOptions, commands::{ diff --git a/src/progress.rs b/crates/core/src/progress.rs similarity index 100% rename from src/progress.rs rename to crates/core/src/progress.rs diff --git a/src/repofile.rs b/crates/core/src/repofile.rs similarity index 100% rename from src/repofile.rs rename to crates/core/src/repofile.rs diff --git a/src/repofile/configfile.rs b/crates/core/src/repofile/configfile.rs similarity index 100% rename from src/repofile/configfile.rs rename to crates/core/src/repofile/configfile.rs diff --git a/src/repofile/indexfile.rs b/crates/core/src/repofile/indexfile.rs similarity index 97% rename from src/repofile/indexfile.rs rename to crates/core/src/repofile/indexfile.rs index fdc63a48..99edd938 100644 --- a/src/repofile/indexfile.rs +++ b/crates/core/src/repofile/indexfile.rs @@ -5,8 +5,10 @@ use chrono::{DateTime, Local}; use serde_derive::{Deserialize, Serialize}; use crate::{ - backend::FileType, blob::BlobType, id::Id, repofile::packfile::PackHeaderRef, - repofile::RepoFile, + backend::FileType, + blob::BlobType, + id::Id, + repofile::{packfile::PackHeaderRef, RepoFile}, }; /// Index files describe index information about multiple `pack` files. diff --git a/src/repofile/keyfile.rs b/crates/core/src/repofile/keyfile.rs similarity index 97% rename from src/repofile/keyfile.rs rename to crates/core/src/repofile/keyfile.rs index 67be9cd0..24c2946b 100644 --- a/src/repofile/keyfile.rs +++ b/crates/core/src/repofile/keyfile.rs @@ -7,7 +7,7 @@ use serde_with::{base64::Base64, serde_as}; use crate::{ backend::{FileType, ReadBackend}, crypto::{aespoly1305::Key, CryptoKey}, - error::{KeyFileErrorKind, RusticResult}, + error::{KeyFileErrorKind, RusticErrorKind, RusticResult}, id::Id, }; @@ -199,7 +199,9 @@ impl KeyFile { /// /// The [`KeyFile`] read from the backend fn from_backend(be: &B, id: &Id) -> RusticResult { - let data = be.read_full(FileType::Key, id)?; + let data = be + .read_full(FileType::Key, id) + .map_err(RusticErrorKind::Backend)?; Ok( serde_json::from_slice(&data) .map_err(KeyFileErrorKind::DeserializingFromSliceFailed)?, @@ -328,7 +330,7 @@ pub(crate) fn find_key_in_backend( if let Some(id) = hint { key_from_backend(be, id, passwd) } else { - for id in be.list(FileType::Key)? { + for id in be.list(FileType::Key).map_err(RusticErrorKind::Backend)? { if let Ok(key) = key_from_backend(be, &id, passwd) { return Ok(key); } diff --git a/src/repofile/packfile.rs b/crates/core/src/repofile/packfile.rs similarity index 98% rename from src/repofile/packfile.rs rename to crates/core/src/repofile/packfile.rs index 6a5b71c2..a34c1d59 100644 --- a/src/repofile/packfile.rs +++ b/crates/core/src/repofile/packfile.rs @@ -6,7 +6,7 @@ use log::trace; use crate::{ backend::{decrypt::DecryptReadBackend, FileType}, blob::BlobType, - error::PackFileErrorKind, + error::{PackFileErrorKind, RusticErrorKind}, id::Id, repofile::indexfile::{IndexBlob, IndexPack}, RusticResult, @@ -271,7 +271,9 @@ impl PackHeader { // read (guessed) header + length field let read_size = size_guess + constants::LENGTH_LEN; let offset = pack_size - read_size; - let mut data = be.read_partial(FileType::Pack, &id, false, offset, read_size)?; + let mut data = be + .read_partial(FileType::Pack, &id, false, offset, read_size) + .map_err(RusticErrorKind::Backend)?; // get header length from the file let size_real = @@ -293,7 +295,8 @@ impl PackHeader { } else { // size_guess was too small; we have to read again let offset = pack_size - size_real - constants::LENGTH_LEN; - be.read_partial(FileType::Pack, &id, false, offset, size_real)? + be.read_partial(FileType::Pack, &id, false, offset, size_real) + .map_err(RusticErrorKind::Backend)? }; let header = Self::from_binary(&be.decrypt(&data)?)?; diff --git a/src/repofile/snapshotfile.rs b/crates/core/src/repofile/snapshotfile.rs similarity index 96% rename from src/repofile/snapshotfile.rs rename to crates/core/src/repofile/snapshotfile.rs index 0b55f7fc..10a8170b 100644 --- a/src/repofile/snapshotfile.rs +++ b/crates/core/src/repofile/snapshotfile.rs @@ -18,9 +18,8 @@ use serde_with::{serde_as, DisplayFromStr}; use shell_words::split; use crate::{ - backend::{decrypt::DecryptReadBackend, FileType}, - error::SnapshotFileErrorKind, - error::{RusticError, RusticResult}, + backend::{decrypt::DecryptReadBackend, FileType, FindInBackend}, + error::{RusticError, RusticResult, SnapshotFileErrorKind}, id::Id, progress::Progress, repofile::RepoFile, @@ -365,7 +364,7 @@ impl SnapshotFile { .collect::>() .join(" ") }, - |command| command.clone(), + std::clone::Clone::clone, ); let mut snap = Self { @@ -428,12 +427,12 @@ impl SnapshotFile { /// # Errors /// /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string - /// * [`BackendErrorKind::NoSuitableIdFound`] - If no id could be found. - /// * [`BackendErrorKind::IdNotUnique`] - If the id is not unique. + /// * [`BackendAccessErrorKind::NoSuitableIdFound`] - If no id could be found. + /// * [`BackendAccessErrorKind::IdNotUnique`] - If the id is not unique. /// /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError - /// [`BackendErrorKind::NoSuitableIdFound`]: crate::error::BackendErrorKind::NoSuitableIdFound - /// [`BackendErrorKind::IdNotUnique`]: crate::error::BackendErrorKind::IdNotUnique + /// [`BackendAccessErrorKind::NoSuitableIdFound`]: crate::error::BackendAccessErrorKind::NoSuitableIdFound + /// [`BackendAccessErrorKind::IdNotUnique`]: crate::error::BackendAccessErrorKind::IdNotUnique pub(crate) fn from_str( be: &B, string: &str, @@ -495,12 +494,12 @@ impl SnapshotFile { /// /// # Errors /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string - /// * [`BackendErrorKind::NoSuitableIdFound`] - If no id could be found. - /// * [`BackendErrorKind::IdNotUnique`] - If the id is not unique. + /// * [`BackendAccessErrorKind::NoSuitableIdFound`] - If no id could be found. + /// * [`BackendAccessErrorKind::IdNotUnique`] - If the id is not unique. /// /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError - /// [`BackendErrorKind::NoSuitableIdFound`]: crate::error::BackendErrorKind::NoSuitableIdFound - /// [`BackendErrorKind::IdNotUnique`]: crate::error::BackendErrorKind::IdNotUnique + /// [`BackendAccessErrorKind::NoSuitableIdFound`]: crate::error::BackendAccessErrorKind::NoSuitableIdFound + /// [`BackendAccessErrorKind::IdNotUnique`]: crate::error::BackendAccessErrorKind::IdNotUnique pub(crate) fn from_id(be: &B, id: &str) -> RusticResult { info!("getting snapshot..."); let id = be.find_id(FileType::Snapshot, id)?; @@ -518,12 +517,12 @@ impl SnapshotFile { /// # Errors /// /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string - /// * [`BackendErrorKind::NoSuitableIdFound`] - If no id could be found. - /// * [`BackendErrorKind::IdNotUnique`] - If the id is not unique. + /// * [`BackendAccessErrorKind::NoSuitableIdFound`] - If no id could be found. + /// * [`BackendAccessErrorKind::IdNotUnique`] - If the id is not unique. /// /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError - /// [`BackendErrorKind::NoSuitableIdFound`]: crate::error::BackendErrorKind::NoSuitableIdFound - /// [`BackendErrorKind::IdNotUnique`]: crate::error::BackendErrorKind::IdNotUnique + /// [`BackendAccessErrorKind::NoSuitableIdFound`]: crate::error::BackendAccessErrorKind::NoSuitableIdFound + /// [`BackendAccessErrorKind::IdNotUnique`]: crate::error::BackendAccessErrorKind::IdNotUnique pub(crate) fn from_ids>( be: &B, ids: &[T], @@ -902,6 +901,7 @@ impl SnapshotGroup { /// /// * `sn` - The [`SnapshotFile`] to extract the [`SnapshotGroup`] from /// * `crit` - The [`SnapshotGroupCriterion`] to use + #[must_use] pub fn from_snapshot(sn: &SnapshotFile, crit: SnapshotGroupCriterion) -> Self { Self { hostname: crit.hostname.then(|| sn.hostname.clone()), @@ -944,6 +944,7 @@ impl StringList { /// # Arguments /// /// * `s` - The String to check + #[must_use] pub fn contains(&self, s: &str) -> bool { self.0.iter().any(|m| m == s) } @@ -953,6 +954,7 @@ impl StringList { /// # Arguments /// /// * `sl` - The [`StringList`] to check + #[must_use] pub fn contains_all(&self, sl: &Self) -> bool { sl.0.iter().all(|s| self.contains(s)) } @@ -1147,6 +1149,7 @@ impl PathList { } /// Sort paths and filters out subpaths of already existing paths. + #[must_use] pub fn merge(self) -> Self { let mut paths = self.0; // sort paths diff --git a/src/repository.rs b/crates/core/src/repository.rs similarity index 84% rename from src/repository.rs rename to crates/core/src/repository.rs index a97f3c88..718390dd 100644 --- a/src/repository.rs +++ b/crates/core/src/repository.rs @@ -1,10 +1,12 @@ +mod warm_up; + use std::{ cmp::Ordering, - collections::HashMap, fs::File, io::{BufRead, BufReader, Write}, path::{Path, PathBuf}, process::{Command, Stdio}, + sync::Arc, }; use bytes::Bytes; @@ -15,14 +17,13 @@ use shell_words::split; use crate::{ backend::{ - cache::Cache, - cache::CachedBackend, - choose::ChooseBackend, - decrypt::{DecryptBackend, DecryptFullBackend, DecryptReadBackend, DecryptWriteBackend}, + cache::{Cache, CachedBackend}, + decrypt::{DecryptBackend, DecryptReadBackend, DecryptWriteBackend}, hotcold::HotColdBackend, - local::LocalDestination, + local_destination::LocalDestination, node::Node, - FileType, ReadBackend, + warm_up::WarmUpAccessBackend, + FileType, ReadBackend, WriteBackend, }, blob::{ tree::{NodeStreamer, TreeStreamerOptions as LsOptions}, @@ -42,21 +43,19 @@ use crate::{ restore::{RestoreOptions, RestorePlan}, }, crypto::aespoly1305::Key, - error::RusticResult, error::{KeyFileErrorKind, RepositoryErrorKind, RusticErrorKind}, id::Id, - index::{IndexBackend, IndexEntry, IndexedBackend, ReadIndex}, + index::{GlobalIndex, IndexEntry, ReadGlobalIndex, ReadIndex}, progress::{NoProgressBars, ProgressBars}, repofile::{ keyfile::find_key_in_backend, snapshotfile::{SnapshotGroup, SnapshotGroupCriterion}, ConfigFile, PathList, RepoFile, SnapshotFile, SnapshotSummary, Tree, }, + repository::{warm_up::warm_up, warm_up::warm_up_wait}, + RepositoryBackends, RusticResult, }; -mod warm_up; -use warm_up::{warm_up, warm_up_wait}; - /// Options for using and opening a [`Repository`] #[serde_as] #[cfg_attr(feature = "clap", derive(clap::Parser))] @@ -65,20 +64,6 @@ use warm_up::{warm_up, warm_up_wait}; #[serde(default, rename_all = "kebab-case", deny_unknown_fields)] #[setters(into, strip_option)] pub struct RepositoryOptions { - /// Repository to use - #[cfg_attr( - feature = "clap", - clap(short, long, global = true, alias = "repo", env = "RUSTIC_REPOSITORY") - )] - pub repository: Option, - - /// Repository to use as hot storage - #[cfg_attr( - feature = "clap", - clap(long, global = true, alias = "repository_hot", env = "RUSTIC_REPO_HOT") - )] - pub repo_hot: Option, - /// Password of the repository /// /// # Warning @@ -143,50 +128,67 @@ pub struct RepositoryOptions { #[cfg_attr(feature = "clap", clap(long, global = true, value_name = "DURATION"))] #[serde_as(as = "Option")] pub warm_up_wait: Option, - - /// Other options for this repository - #[cfg_attr(feature = "clap", clap(skip))] - #[cfg_attr(feature = "merge", merge(strategy = overwrite))] - pub options: HashMap, -} - -/// Overwrite the left value with the right value -/// -/// This is used for merging [`RepositoryOptions`] and [`ConfigOptions`] -/// -/// # Arguments -/// -/// * `left` - The left value -/// * `right` - The right value -#[cfg(feature = "merge")] -pub(crate) fn overwrite(left: &mut T, right: T) { - *left = right; } impl RepositoryOptions { - /// Create a [`Repository`] using the given repository options + /// Evaluates the password given by the repository options /// /// # Errors /// - /// * [`RepositoryErrorKind::NoRepositoryGiven`] - If no repository is given - /// * [`RepositoryErrorKind::NoIDSpecified`] - If the warm-up command does not contain `%id` - /// * [`BackendErrorKind::BackendNotSupported`] - If the backend is not supported. - /// * [`LocalErrorKind::DirectoryCreationFailed`] - If the directory could not be created. - /// * [`RestErrorKind::UrlParsingFailed`] - If the url could not be parsed. - /// * [`RestErrorKind::BuildingClientFailed`] - If the client could not be built. + /// * [`RepositoryErrorKind::OpeningPasswordFileFailed`] - If opening the password file failed + /// * [`RepositoryErrorKind::ReadingPasswordFromReaderFailed`] - If reading the password failed + /// * [`RepositoryErrorKind::FromSplitError`] - If splitting the password command failed + /// * [`RepositoryErrorKind::PasswordCommandParsingFailed`] - If parsing the password command failed + /// * [`RepositoryErrorKind::ReadingPasswordFromCommandFailed`] - If reading the password from the command failed /// /// # Returns /// - /// The repository without progress bars + /// The password or `None` if no password is given /// - /// [`RepositoryErrorKind::NoRepositoryGiven`]: crate::error::RepositoryErrorKind::NoRepositoryGiven - /// [`RepositoryErrorKind::NoIDSpecified`]: crate::error::RepositoryErrorKind::NoIDSpecified - /// [`BackendErrorKind::BackendNotSupported`]: crate::error::BackendErrorKind::BackendNotSupported - /// [`LocalErrorKind::DirectoryCreationFailed`]: crate::error::LocalErrorKind::DirectoryCreationFailed - /// [`RestErrorKind::UrlParsingFailed`]: crate::error::RestErrorKind::UrlParsingFailed - /// [`RestErrorKind::BuildingClientFailed`]: crate::error::RestErrorKind::BuildingClientFailed - pub fn to_repository(&self) -> RusticResult> { - Repository::new(self) + /// [`RepositoryErrorKind::OpeningPasswordFileFailed`]: crate::error::RepositoryErrorKind::OpeningPasswordFileFailed + /// [`RepositoryErrorKind::ReadingPasswordFromReaderFailed`]: crate::error::RepositoryErrorKind::ReadingPasswordFromReaderFailed + /// [`RepositoryErrorKind::FromSplitError`]: crate::error::RepositoryErrorKind::FromSplitError + /// [`RepositoryErrorKind::PasswordCommandParsingFailed`]: crate::error::RepositoryErrorKind::PasswordCommandParsingFailed + /// [`RepositoryErrorKind::ReadingPasswordFromCommandFailed`]: crate::error::RepositoryErrorKind::ReadingPasswordFromCommandFailed + pub fn evaluate_password(&self) -> RusticResult> { + match (&self.password, &self.password_file, &self.password_command) { + (Some(pwd), _, _) => Ok(Some(pwd.clone())), + (_, Some(file), _) => { + let mut file = BufReader::new( + File::open(file).map_err(RepositoryErrorKind::OpeningPasswordFileFailed)?, + ); + Ok(Some(read_password_from_reader(&mut file)?)) + } + (_, _, Some(command)) => { + let commands = split(command).map_err(RepositoryErrorKind::FromSplitError)?; + debug!("commands: {commands:?}"); + let command = Command::new(&commands[0]) + .args(&commands[1..]) + .stdout(Stdio::piped()) + .spawn()?; + let Ok(output) = command.wait_with_output() else { + return Err(RepositoryErrorKind::PasswordCommandParsingFailed.into()); + }; + if !output.status.success() { + #[allow(clippy::option_if_let_else)] + let s = match output.status.code() { + Some(c) => format!("exited with status code {c}"), + None => "was terminated".into(), + }; + error!("password-command {s}"); + return Err(RepositoryErrorKind::ReadingPasswordFromCommandFailed.into()); + } + + let mut pwd = BufReader::new(&*output.stdout); + Ok(Some(match read_password_from_reader(&mut pwd) { + Ok(val) => val, + Err(_) => { + return Err(RepositoryErrorKind::ReadingPasswordFromCommandFailed.into()) + } + })) + } + (None, None, None) => Ok(None), + } } } @@ -236,10 +238,10 @@ pub struct Repository { pub name: String, /// The HotColdBackend to use for this repository - pub(crate) be: HotColdBackend, + pub(crate) be: Arc, - /// The Backende to use for hot files - pub(crate) be_hot: Option, + /// The Backend to use for hot files + pub(crate) be_hot: Option>, /// The options used for this repository opts: RepositoryOptions, @@ -257,24 +259,16 @@ impl Repository { /// # Arguments /// /// * `opts` - The options to use for the repository + /// * `backends` - The backends to create/access a repository on /// /// # Errors /// /// * [`RepositoryErrorKind::NoRepositoryGiven`] - If no repository is given /// * [`RepositoryErrorKind::NoIDSpecified`] - If the warm-up command does not contain `%id` - /// * [`BackendErrorKind::BackendNotSupported`] - If the backend is not supported. - /// * [`LocalErrorKind::DirectoryCreationFailed`] - If the directory could not be created. - /// * [`RestErrorKind::UrlParsingFailed`] - If the url could not be parsed. - /// * [`RestErrorKind::BuildingClientFailed`] - If the client could not be built. + /// * [`BackendAccessErrorKind::BackendLoadError`] - If the specified backend cannot be loaded, e.g. is not supported /// - /// [`RepositoryErrorKind::NoRepositoryGiven`]: crate::error::RepositoryErrorKind::NoRepositoryGiven - /// [`RepositoryErrorKind::NoIDSpecified`]: crate::error::RepositoryErrorKind::NoIDSpecified - /// [`BackendErrorKind::BackendNotSupported`]: crate::error::BackendErrorKind::BackendNotSupported - /// [`LocalErrorKind::DirectoryCreationFailed`]: crate::error::LocalErrorKind::DirectoryCreationFailed - /// [`RestErrorKind::UrlParsingFailed`]: crate::error::RestErrorKind::UrlParsingFailed - /// [`RestErrorKind::BuildingClientFailed`]: crate::error::RestErrorKind::BuildingClientFailed - pub fn new(opts: &RepositoryOptions) -> RusticResult { - Self::new_with_progress(opts, NoProgressBars {}) + pub fn new(opts: &RepositoryOptions, backends: RepositoryBackends) -> RusticResult { + Self::new_with_progress(opts, backends, NoProgressBars {}) } } @@ -288,28 +282,25 @@ impl

Repository { /// # Arguments /// /// * `opts` - The options to use for the repository + /// * `backends` - The backends to create/access a repository on /// * `pb` - The progress bars to use /// /// # Errors /// /// * [`RepositoryErrorKind::NoRepositoryGiven`] - If no repository is given /// * [`RepositoryErrorKind::NoIDSpecified`] - If the warm-up command does not contain `%id` - /// * [`BackendErrorKind::BackendNotSupported`] - If the backend is not supported. - /// * [`LocalErrorKind::DirectoryCreationFailed`] - If the directory could not be created. - /// * [`RestErrorKind::UrlParsingFailed`] - If the url could not be parsed. - /// * [`RestErrorKind::BuildingClientFailed`] - If the client could not be built. + /// * [`BackendAccessErrorKind::BackendLoadError`] - If the specified backend cannot be loaded, e.g. is not supported /// /// [`RepositoryErrorKind::NoRepositoryGiven`]: crate::error::RepositoryErrorKind::NoRepositoryGiven /// [`RepositoryErrorKind::NoIDSpecified`]: crate::error::RepositoryErrorKind::NoIDSpecified - /// [`BackendErrorKind::BackendNotSupported`]: crate::error::BackendErrorKind::BackendNotSupported - /// [`LocalErrorKind::DirectoryCreationFailed`]: crate::error::LocalErrorKind::DirectoryCreationFailed - /// [`RestErrorKind::UrlParsingFailed`]: crate::error::RestErrorKind::UrlParsingFailed - /// [`RestErrorKind::BuildingClientFailed`]: crate::error::RestErrorKind::BuildingClientFailed - pub fn new_with_progress(opts: &RepositoryOptions, pb: P) -> RusticResult { - let be = match &opts.repository { - Some(repo) => ChooseBackend::from_url(repo)?, - None => return Err(RepositoryErrorKind::NoRepositoryGiven.into()), - }; + /// [`BackendAccessErrorKind::BackendLoadError`]: crate::error::BackendAccessErrorKind::BackendLoadError + pub fn new_with_progress( + opts: &RepositoryOptions, + backends: RepositoryBackends, + pb: P, + ) -> RusticResult { + let mut be = backends.repository(); + let be_hot = backends.repo_hot(); if let Some(command) = &opts.warm_up_command { if !command.contains("%id") { @@ -318,18 +309,13 @@ impl

Repository { info!("using warm-up command {command}"); } - let be_hot = opts - .repo_hot - .as_ref() - .map(|repo| ChooseBackend::from_url(repo)) - .transpose()?; - - let mut be = HotColdBackend::new(be, be_hot.clone()); - for (opt, value) in &opts.options { - be.set_option(opt, value)?; + if opts.warm_up { + be = WarmUpAccessBackend::new_warm_up(be); } + let mut name = be.location(); if let Some(be_hot) = &be_hot { + be = Arc::new(HotColdBackend::new(be, be_hot.clone())); name.push('#'); name.push_str(&be_hot.location()); } @@ -366,48 +352,7 @@ impl Repository { /// [`RepositoryErrorKind::PasswordCommandParsingFailed`]: crate::error::RepositoryErrorKind::PasswordCommandParsingFailed /// [`RepositoryErrorKind::ReadingPasswordFromCommandFailed`]: crate::error::RepositoryErrorKind::ReadingPasswordFromCommandFailed pub fn password(&self) -> RusticResult> { - match ( - &self.opts.password, - &self.opts.password_file, - &self.opts.password_command, - ) { - (Some(pwd), _, _) => Ok(Some(pwd.clone())), - (_, Some(file), _) => { - let mut file = BufReader::new( - File::open(file).map_err(RepositoryErrorKind::OpeningPasswordFileFailed)?, - ); - Ok(Some(read_password_from_reader(&mut file)?)) - } - (_, _, Some(command)) => { - let commands = split(command).map_err(RepositoryErrorKind::FromSplitError)?; - debug!("commands: {commands:?}"); - let command = Command::new(&commands[0]) - .args(&commands[1..]) - .stdout(Stdio::piped()) - .spawn()?; - let Ok(output) = command.wait_with_output() else { - return Err(RepositoryErrorKind::PasswordCommandParsingFailed.into()); - }; - if !output.status.success() { - #[allow(clippy::option_if_let_else)] - let s = match output.status.code() { - Some(c) => format!("exited with status code {c}"), - None => "was terminated".into(), - }; - error!("password-command {s}"); - return Err(RepositoryErrorKind::ReadingPasswordFromCommandFailed.into()); - } - - let mut pwd = BufReader::new(&*output.stdout); - Ok(Some(match read_password_from_reader(&mut pwd) { - Ok(val) => val, - Err(_) => { - return Err(RepositoryErrorKind::ReadingPasswordFromCommandFailed.into()) - } - })) - } - (None, None, None) => Ok(None), - } + self.opts.evaluate_password() } /// Returns the Id of the config file @@ -509,9 +454,14 @@ impl Repository { ))?; if let Some(be_hot) = &self.be_hot { - let mut keys = self.be.list_with_size(FileType::Key)?; + let mut keys = self + .be + .list_with_size(FileType::Key) + .map_err(RusticErrorKind::Backend)?; keys.sort_unstable_by_key(|key| key.0); - let mut hot_keys = be_hot.list_with_size(FileType::Key)?; + let mut hot_keys = be_hot + .list_with_size(FileType::Key) + .map_err(RusticErrorKind::Backend)?; hot_keys.sort_unstable_by_key(|key| key.0); if keys != hot_keys { return Err(RepositoryErrorKind::KeysDontMatchForRepositories(self.name).into()); @@ -527,7 +477,7 @@ impl Repository { } })?; info!("repository {}: password is correct.", self.name); - let dbe = DecryptBackend::new(&self.be, key); + let dbe = DecryptBackend::new(self.be.clone(), key); let config: ConfigFile = dbe.get_file(&config_id)?; self.open_raw(key, config) } @@ -649,30 +599,29 @@ impl Repository { /// /// [`RepositoryErrorKind::HotRepositoryFlagMissing`]: crate::error::RepositoryErrorKind::HotRepositoryFlagMissing /// [`RepositoryErrorKind::IsNotHotRepository`]: crate::error::RepositoryErrorKind::IsNotHotRepository - fn open_raw(self, key: Key, config: ConfigFile) -> RusticResult> { + fn open_raw(mut self, key: Key, config: ConfigFile) -> RusticResult> { match (config.is_hot == Some(true), self.be_hot.is_some()) { (true, false) => return Err(RepositoryErrorKind::HotRepositoryFlagMissing.into()), (false, true) => return Err(RepositoryErrorKind::IsNotHotRepository.into()), _ => {} } + let cache = (!self.opts.no_cache) .then(|| Cache::new(config.id, self.opts.cache_dir.clone()).ok()) .flatten(); - cache.as_ref().map_or_else( - || info!("using no cache"), - |cache| info!("using cache at {}", cache.location()), - ); - let be_cached = CachedBackend::new(self.be.clone(), cache.clone()); - let mut dbe = DecryptBackend::new(&be_cached, key); + + if let Some(cache) = &cache { + self.be = CachedBackend::new_cache(self.be.clone(), cache.clone()); + info!("using cache at {}", cache.location()); + } else { + info!("using no cache"); + } + + let mut dbe = DecryptBackend::new(self.be.clone(), key); let zstd = config.zstd()?; dbe.set_zstd(zstd); - let open = OpenStatus { - key, - dbe, - cache, - config, - }; + let open = OpenStatus { cache, dbe, config }; Ok(Repository { name: self.name, @@ -690,7 +639,11 @@ impl Repository { /// /// * `tpe` - The type of the files to list pub fn list(&self, tpe: FileType) -> RusticResult> { - Ok(self.be.list(tpe)?.into_iter()) + Ok(self + .be + .list(tpe) + .map_err(RusticErrorKind::Backend)? + .into_iter()) } } @@ -736,38 +689,24 @@ impl Repository { /// A repository which is open, i.e. the password has been checked and the decryption key is available. pub trait Open { - /// The [`DecryptBackend`] used by this repository - type DBE: DecryptFullBackend; - - /// Get the decryption key - fn key(&self) -> &Key; - /// Get the cache fn cache(&self) -> Option<&Cache>; /// Get the [`DecryptBackend`] - fn dbe(&self) -> &Self::DBE; + fn dbe(&self) -> &DecryptBackend; /// Get the [`ConfigFile`] fn config(&self) -> &ConfigFile; } impl Open for Repository { - /// The [`DecryptBackend`] used by this repository - type DBE = S::DBE; - - /// Get the decryption key - fn key(&self) -> &Key { - self.status.key() - } - /// Get the cache fn cache(&self) -> Option<&Cache> { self.status.cache() } /// Get the [`DecryptBackend`] - fn dbe(&self) -> &Self::DBE { + fn dbe(&self) -> &DecryptBackend { self.status.dbe() } @@ -777,35 +716,25 @@ impl Open for Repository { } } -#[derive(Debug)] /// Open Status: This repository is open, i.e. the password has been checked and the decryption key is available. +#[derive(Debug)] pub struct OpenStatus { - /// The decryption key - key: Key, /// The cache cache: Option, /// The [`DecryptBackend`] - dbe: DecryptBackend>, Key>, + dbe: DecryptBackend, /// The [`ConfigFile`] config: ConfigFile, } impl Open for OpenStatus { - /// The [`DecryptBackend`] used by this repository - type DBE = DecryptBackend>, Key>; - - /// Get the decryption key - fn key(&self) -> &Key { - &self.key - } - /// Get the cache fn cache(&self) -> Option<&Cache> { self.cache.as_ref() } /// Get the [`DecryptBackend`] - fn dbe(&self) -> &Self::DBE { + fn dbe(&self) -> &DecryptBackend { &self.dbe } @@ -826,12 +755,12 @@ impl Repository { /// # Errors /// /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string - /// * [`BackendErrorKind::NoSuitableIdFound`] - If no id could be found. - /// * [`BackendErrorKind::IdNotUnique`] - If the id is not unique. + /// * [`BackendAccessErrorKind::NoSuitableIdFound`] - If no id could be found. + /// * [`BackendAccessErrorKind::IdNotUnique`] - If the id is not unique. /// /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError - /// [`BackendErrorKind::NoSuitableIdFound`]: crate::error::BackendErrorKind::NoSuitableIdFound - /// [`BackendErrorKind::IdNotUnique`]: crate::error::BackendErrorKind::IdNotUnique + /// [`BackendAccessErrorKind::NoSuitableIdFound`]: crate::error::BackendAccessErrorKind::NoSuitableIdFound + /// [`BackendAccessErrorKind::IdNotUnique`]: crate::error::BackendAccessErrorKind::IdNotUnique pub fn cat_file(&self, tpe: FileType, id: &str) -> RusticResult { commands::cat::cat_file(self, tpe, id) } @@ -887,7 +816,7 @@ impl Repository { } // TODO: add documentation! - pub(crate) fn dbe(&self) -> &S::DBE { + pub(crate) fn dbe(&self) -> &DecryptBackend { self.status.dbe() } } @@ -924,8 +853,8 @@ impl Repository { /// # Errors /// /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string - /// * [`BackendErrorKind::NoSuitableIdFound`] - If no id could be found. - /// * [`BackendErrorKind::IdNotUnique`] - If the id is not unique. + /// * [`BackendAccessErrorKind::NoSuitableIdFound`] - If no id could be found. + /// * [`BackendAccessErrorKind::IdNotUnique`] - If the id is not unique. /// /// # Returns /// @@ -933,8 +862,8 @@ impl Repository { /// If `id` is "latest", return the latest snapshot respecting the giving filter. /// /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError - /// [`BackendErrorKind::NoSuitableIdFound`]: crate::error::BackendErrorKind::NoSuitableIdFound - /// [`BackendErrorKind::IdNotUnique`]: crate::error::BackendErrorKind::IdNotUnique + /// [`BackendAccessErrorKind::NoSuitableIdFound`]: crate::error::BackendAccessErrorKind::NoSuitableIdFound + /// [`BackendAccessErrorKind::IdNotUnique`]: crate::error::BackendAccessErrorKind::IdNotUnique pub fn get_snapshot_from_str( &self, id: &str, @@ -1078,7 +1007,7 @@ impl Repository { /// /// This saves the full index in memory which can be quite memory-consuming! pub fn to_indexed(self) -> RusticResult>> { - let index = IndexBackend::new(self.dbe(), &self.pb.progress_counter(""))?; + let index = GlobalIndex::new(self.dbe(), &self.pb.progress_counter(""))?; let status = IndexedStatus { open: self.status, index, @@ -1099,7 +1028,7 @@ impl Repository { /// This saves only the `Id`s for data blobs. Therefore, not all operations are possible on the repository. /// However, operations which add data are fully functional. pub fn to_indexed_ids(self) -> RusticResult>> { - let index = IndexBackend::only_full_trees(self.dbe(), &self.pb.progress_counter(""))?; + let index = GlobalIndex::only_full_trees(self.dbe(), &self.pb.progress_counter(""))?; let status = IndexedStatus { open: self.status, index, @@ -1148,7 +1077,7 @@ impl Repository { /// A repository which is indexed such that all tree blobs are contained in the index. pub trait IndexedTree: Open { - type I: IndexedBackend; + type I: ReadGlobalIndex; fn index(&self) -> &Self::I; } @@ -1177,7 +1106,7 @@ pub struct IndexedStatus { /// The open status open: S, /// The index backend - index: IndexBackend, + index: GlobalIndex, /// The marker for the type of index marker: std::marker::PhantomData, } @@ -1189,7 +1118,7 @@ pub struct IdIndex {} pub struct FullIndex {} impl IndexedTree for IndexedStatus { - type I = IndexBackend; + type I = GlobalIndex; fn index(&self) -> &Self::I { &self.index @@ -1201,15 +1130,10 @@ impl IndexedIds for IndexedStatus {} impl IndexedFull for IndexedStatus {} impl Open for IndexedStatus { - type DBE = S::DBE; - - fn key(&self) -> &Key { - self.open.key() - } fn cache(&self) -> Option<&Cache> { self.open.cache() } - fn dbe(&self) -> &Self::DBE { + fn dbe(&self) -> &DecryptBackend { self.open.dbe() } fn config(&self) -> &ConfigFile { @@ -1252,12 +1176,12 @@ impl Repository { /// # Errors /// /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string - /// * [`BackendErrorKind::NoSuitableIdFound`] - If no id could be found. - /// * [`BackendErrorKind::IdNotUnique`] - If the id is not unique. + /// * [`BackendAccessErrorKind::NoSuitableIdFound`] - If no id could be found. + /// * [`BackendAccessErrorKind::IdNotUnique`] - If the id is not unique. /// /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError - /// [`BackendErrorKind::NoSuitableIdFound`]: crate::error::BackendErrorKind::NoSuitableIdFound - /// [`BackendErrorKind::IdNotUnique`]: crate::error::BackendErrorKind::IdNotUnique + /// [`BackendAccessErrorKind::NoSuitableIdFound`]: crate::error::BackendAccessErrorKind::NoSuitableIdFound + /// [`BackendAccessErrorKind::IdNotUnique`]: crate::error::BackendAccessErrorKind::IdNotUnique pub fn node_from_snapshot_path( &self, snap_path: &str, @@ -1268,7 +1192,7 @@ impl Repository { let p = &self.pb.progress_counter("getting snapshot..."); let snap = SnapshotFile::from_str(self.dbe(), id, filter, p)?; - Tree::node_from_path(self.index(), snap.tree, Path::new(path)) + Tree::node_from_path(self.dbe(), self.index(), snap.tree, Path::new(path)) } /// Get a [`Node`] from a [`SnapshotFile`] and a `path` @@ -1284,7 +1208,7 @@ impl Repository { snap: &SnapshotFile, path: &str, ) -> RusticResult { - Tree::node_from_path(self.index(), snap.tree, Path::new(path)) + Tree::node_from_path(self.dbe(), self.index(), snap.tree, Path::new(path)) } /// Reads a raw tree from a "SNAP\[:PATH\]" syntax @@ -1322,8 +1246,8 @@ impl Repository { &self, node: &Node, ls_opts: &LsOptions, - ) -> RusticResult> + Clone> { - NodeStreamer::new_with_glob(self.index().clone(), node, ls_opts) + ) -> RusticResult> + Clone + '_> { + NodeStreamer::new_with_glob(self.dbe().clone(), self.index(), node, ls_opts) } /// Restore a given [`RestorePlan`] to a local destination @@ -1408,7 +1332,7 @@ impl Repository { pub fn backup( &self, opts: &BackupOptions, - source: PathList, + source: &PathList, snap: SnapshotFile, ) -> RusticResult { commands::backup::backup(self, opts, source, snap) diff --git a/src/repository/warm_up.rs b/crates/core/src/repository/warm_up.rs similarity index 84% rename from src/repository/warm_up.rs rename to crates/core/src/repository/warm_up.rs index af88234e..2cc886fe 100644 --- a/src/repository/warm_up.rs +++ b/crates/core/src/repository/warm_up.rs @@ -1,7 +1,7 @@ use std::process::Command; use std::thread::sleep; -use log::{debug, warn}; +use log::{debug, error, warn}; use rayon::ThreadPoolBuilder; use shell_words::split; @@ -65,8 +65,8 @@ pub(crate) fn warm_up( ) -> RusticResult<()> { if let Some(command) = &repo.opts.warm_up_command { warm_up_command(packs, command, &repo.pb)?; - } else if repo.opts.warm_up { - warm_up_access(repo, packs)?; + } else if repo.be.needs_warm_up() { + warm_up_repo(repo, packs)?; } Ok(()) } @@ -104,7 +104,7 @@ fn warm_up_command( Ok(()) } -/// Warm up the repository using access. +/// Warm up the repository. /// /// # Arguments /// @@ -116,33 +116,32 @@ fn warm_up_command( /// * [`RepositoryErrorKind::FromThreadPoolbilderError`] - If the thread pool could not be created. /// /// [`RepositoryErrorKind::FromThreadPoolbilderError`]: crate::error::RepositoryErrorKind::FromThreadPoolbilderError -fn warm_up_access( +fn warm_up_repo( repo: &Repository, packs: impl ExactSizeIterator, ) -> RusticResult<()> { - let mut be = repo.be.clone(); - be.set_option("retry", "false")?; - - let p = repo.pb.progress_counter("warming up packs..."); - p.set_length(packs.len() as u64); + let progress_bar = repo.pb.progress_counter("warming up packs..."); + progress_bar.set_length(packs.len() as u64); let pool = ThreadPoolBuilder::new() .num_threads(constants::MAX_READER_THREADS_NUM) .build() .map_err(RepositoryErrorKind::FromThreadPoolbilderError)?; - let p = &p; - let be = &be; - pool.in_place_scope(|s| { + let progress_bar_ref = &progress_bar; + let backend = &repo.be; + pool.in_place_scope(|scope| { for pack in packs { - s.spawn(move |_| { - // ignore errors as they are expected from the warm-up - _ = be.read_partial(FileType::Pack, &pack, false, 0, 1); - p.inc(1); + scope.spawn(move |_| { + if let Err(e) = backend.warm_up(FileType::Pack, &pack) { + // FIXME: Use error handling + error!("warm-up failed for pack {pack:?}. {e}"); + }; + progress_bar_ref.inc(1); }); } }); - p.finish(); + progress_bar_ref.finish(); Ok(()) } diff --git a/tests/public_api.rs b/crates/core/tests/public_api.rs similarity index 100% rename from tests/public_api.rs rename to crates/core/tests/public_api.rs diff --git a/tests/public_api_fixtures/.gitkeep b/crates/core/tests/public_api_fixtures/.gitkeep similarity index 100% rename from tests/public_api_fixtures/.gitkeep rename to crates/core/tests/public_api_fixtures/.gitkeep diff --git a/tests/public_api_fixtures/public-api_linux.txt b/crates/core/tests/public_api_fixtures/public-api_linux.txt similarity index 100% rename from tests/public_api_fixtures/public-api_linux.txt rename to crates/core/tests/public_api_fixtures/public-api_linux.txt diff --git a/tests/public_api_fixtures/public-api_macos.txt b/crates/core/tests/public_api_fixtures/public-api_macos.txt similarity index 100% rename from tests/public_api_fixtures/public-api_macos.txt rename to crates/core/tests/public_api_fixtures/public-api_macos.txt diff --git a/tests/public_api_fixtures/public-api_win.txt b/crates/core/tests/public_api_fixtures/public-api_win.txt similarity index 100% rename from tests/public_api_fixtures/public-api_win.txt rename to crates/core/tests/public_api_fixtures/public-api_win.txt diff --git a/deny.toml b/deny.toml index 37aceb1a..7d395c0e 100644 --- a/deny.toml +++ b/deny.toml @@ -76,9 +76,9 @@ notice = "warn" # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ - - - # "RUSTSEC-0000-0000", + # FIXME!: See https://github.com/RustCrypto/RSA/issues/19#issuecomment-1822995643. + # There is no workaround available yet. + "RUSTSEC-2023-0071", ] # Threshold for security vulnerabilities, any vulnerability with a CVSS score # lower than the range specified will be ignored. Note that ignored advisories diff --git a/examples/backup/Cargo.toml b/examples/backup/Cargo.toml new file mode 100644 index 00000000..6fd3a9fd --- /dev/null +++ b/examples/backup/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "backup" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/backup.rs b/examples/backup/examples/backup.rs similarity index 63% rename from examples/backup.rs rename to examples/backup/examples/backup.rs index 7689cfa3..91b49fbb 100644 --- a/examples/backup.rs +++ b/examples/backup/examples/backup.rs @@ -1,4 +1,5 @@ //! `backup` example +use rustic_backend::BackendOptions; use rustic_core::{BackupOptions, PathList, Repository, RepositoryOptions, SnapshotOptions}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -7,11 +8,18 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); - // Open repository - let repo_opts = RepositoryOptions::default() + // Initialize Backends + let backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); - let repo = Repository::new(&repo_opts)?.open()?.to_indexed_ids()?; + .repo_hot("/tmp/repo2") + .to_backends()?; + + // Open repository + let repo_opts = RepositoryOptions::default().password("test"); + + let repo = Repository::new(&repo_opts, backends)? + .open()? + .to_indexed_ids()?; let backup_opts = BackupOptions::default(); let source = PathList::from_string(".")?.sanitize()?; @@ -20,7 +28,7 @@ fn main() -> Result<(), Box> { .to_snapshot()?; // Create snapshot - let snap = repo.backup(&backup_opts, source, snap)?; + let snap = repo.backup(&backup_opts, &source, snap)?; println!("successfully created snapshot:\n{snap:#?}"); Ok(()) diff --git a/examples/backup/src/lib.rs b/examples/backup/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/backup/src/lib.rs @@ -0,0 +1 @@ + diff --git a/examples/check/Cargo.toml b/examples/check/Cargo.toml new file mode 100644 index 00000000..dcd95fdb --- /dev/null +++ b/examples/check/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "check" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/check.rs b/examples/check/examples/check.rs similarity index 66% rename from examples/check.rs rename to examples/check/examples/check.rs index 26b1bc0f..c09bc27d 100644 --- a/examples/check.rs +++ b/examples/check/examples/check.rs @@ -1,4 +1,5 @@ //! `check` example +use rustic_backend::BackendOptions; use rustic_core::{CheckOptions, Repository, RepositoryOptions}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -7,11 +8,14 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); - // Open repository - let repo_opts = RepositoryOptions::default() + // Initialize Backends + let backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); - let repo = Repository::new(&repo_opts)?.open()?; + .to_backends()?; + + // Open repository + let repo_opts = RepositoryOptions::default().password("test"); + let repo = Repository::new(&repo_opts, backends)?.open()?; // Check repository with standard options but omitting cache checks let opts = CheckOptions::default().trust_cache(true); diff --git a/examples/check/src/lib.rs b/examples/check/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/check/src/lib.rs @@ -0,0 +1 @@ + diff --git a/examples/config/Cargo.toml b/examples/config/Cargo.toml new file mode 100644 index 00000000..ede60b9f --- /dev/null +++ b/examples/config/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "config" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/config.rs b/examples/config/examples/config.rs similarity index 68% rename from examples/config.rs rename to examples/config/examples/config.rs index f40125bb..dd4eff98 100644 --- a/examples/config.rs +++ b/examples/config/examples/config.rs @@ -1,4 +1,5 @@ //! `config` example +use rustic_backend::BackendOptions; use rustic_core::{max_compression_level, ConfigOptions, Repository, RepositoryOptions}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -7,11 +8,14 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); - // Open repository - let repo_opts = RepositoryOptions::default() + // Initialize Backends + let backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); - let repo = Repository::new(&repo_opts)?.open()?; + .to_backends()?; + + // Open repository + let repo_opts = RepositoryOptions::default().password("test"); + let repo = Repository::new(&repo_opts, backends)?.open()?; // Set Config, e.g. Compression level let config_opts = ConfigOptions::default().set_compression(max_compression_level()); diff --git a/examples/config/src/lib.rs b/examples/config/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/config/src/lib.rs @@ -0,0 +1 @@ + diff --git a/examples/copy/Cargo.toml b/examples/copy/Cargo.toml new file mode 100644 index 00000000..fda3dcab --- /dev/null +++ b/examples/copy/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "copy" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/copy.rs b/examples/copy/examples/copy.rs similarity index 53% rename from examples/copy.rs rename to examples/copy/examples/copy.rs index e045878d..c4ce3281 100644 --- a/examples/copy.rs +++ b/examples/copy/examples/copy.rs @@ -1,4 +1,5 @@ //! `copy` example +use rustic_backend::BackendOptions; use rustic_core::{CopySnapshot, Repository, RepositoryOptions}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -7,16 +8,28 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); + // Initialize src backends + let src_backends = BackendOptions::default() + .repository("/tmp/repo") + .to_backends()?; + // Open repository - let src_repo_opts = RepositoryOptions::default() + let src_repo_opts = RepositoryOptions::default().password("test"); + + let src_repo = Repository::new(&src_repo_opts, src_backends)? + .open()? + .to_indexed()?; + + // Initialize dst backends + let dst_backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); - let src_repo = Repository::new(&src_repo_opts)?.open()?.to_indexed()?; + .to_backends()?; + + let dst_repo_opts = RepositoryOptions::default().password("test"); - let dst_repo_opts = RepositoryOptions::default() - .repository("tmp/repo") - .password("test"); - let dst_repo = Repository::new(&dst_repo_opts)?.open()?.to_indexed_ids()?; + let dst_repo = Repository::new(&dst_repo_opts, dst_backends)? + .open()? + .to_indexed_ids()?; // get snapshots which are missing in dst_repo let snapshots = src_repo.get_all_snapshots()?; diff --git a/examples/copy/src/lib.rs b/examples/copy/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/copy/src/lib.rs @@ -0,0 +1 @@ + diff --git a/examples/forget/Cargo.toml b/examples/forget/Cargo.toml new file mode 100644 index 00000000..150baae0 --- /dev/null +++ b/examples/forget/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "forget" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/forget.rs b/examples/forget/examples/forget.rs similarity index 74% rename from examples/forget.rs rename to examples/forget/examples/forget.rs index 3645358d..2c72f204 100644 --- a/examples/forget.rs +++ b/examples/forget/examples/forget.rs @@ -1,4 +1,5 @@ //! `forget` example +use rustic_backend::BackendOptions; use rustic_core::{KeepOptions, Repository, RepositoryOptions, SnapshotGroupCriterion}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -7,11 +8,15 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); - // Open repository - let repo_opts = RepositoryOptions::default() + // Initialize Backends + let backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); - let repo = Repository::new(&repo_opts)?.open()?; + .to_backends()?; + + // Open repository + let repo_opts = RepositoryOptions::default().password("test"); + + let repo = Repository::new(&repo_opts, backends)?.open()?; // Check repository with standard options let group_by = SnapshotGroupCriterion::default(); diff --git a/examples/forget/src/lib.rs b/examples/forget/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/forget/src/lib.rs @@ -0,0 +1 @@ + diff --git a/examples/init/Cargo.toml b/examples/init/Cargo.toml new file mode 100644 index 00000000..f101ce68 --- /dev/null +++ b/examples/init/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "init" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/init.rs b/examples/init/examples/init.rs similarity index 64% rename from examples/init.rs rename to examples/init/examples/init.rs index 0b04632e..ce355a02 100644 --- a/examples/init.rs +++ b/examples/init/examples/init.rs @@ -1,4 +1,5 @@ //! `init` example +use rustic_backend::BackendOptions; use rustic_core::{ConfigOptions, KeyOptions, Repository, RepositoryOptions}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -7,13 +8,16 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); - // Init repository - let repo_opts = RepositoryOptions::default() + // Initialize Backends + let backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); + .to_backends()?; + + // Init repository + let repo_opts = RepositoryOptions::default().password("test"); let key_opts = KeyOptions::default(); let config_opts = ConfigOptions::default(); - let _repo = Repository::new(&repo_opts)?.init(&key_opts, &config_opts)?; + let _repo = Repository::new(&repo_opts, backends)?.init(&key_opts, &config_opts)?; // -> use _repo for any operation on an open repository Ok(()) diff --git a/examples/init/src/lib.rs b/examples/init/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/init/src/lib.rs @@ -0,0 +1 @@ + diff --git a/examples/key/Cargo.toml b/examples/key/Cargo.toml new file mode 100644 index 00000000..88c72738 --- /dev/null +++ b/examples/key/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "key" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/key.rs b/examples/key/examples/key.rs similarity index 65% rename from examples/key.rs rename to examples/key/examples/key.rs index 4ef11f74..900dbd96 100644 --- a/examples/key.rs +++ b/examples/key/examples/key.rs @@ -1,4 +1,5 @@ //! `key` example +use rustic_backend::BackendOptions; use rustic_core::{KeyOptions, Repository, RepositoryOptions}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -7,11 +8,15 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); - // Open repository - let repo_opts = RepositoryOptions::default() + // Initialize Backends + let backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); - let repo = Repository::new(&repo_opts)?.open()?; + .to_backends()?; + + // Open repository + let repo_opts = RepositoryOptions::default().password("test"); + + let repo = Repository::new(&repo_opts, backends)?.open()?; // Add a new key with the given password let key_opts = KeyOptions::default(); diff --git a/examples/key/src/lib.rs b/examples/key/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/key/src/lib.rs @@ -0,0 +1 @@ + diff --git a/examples/ls/Cargo.toml b/examples/ls/Cargo.toml new file mode 100644 index 00000000..b4f9fe03 --- /dev/null +++ b/examples/ls/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ls" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/ls.rs b/examples/ls/examples/ls.rs similarity index 70% rename from examples/ls.rs rename to examples/ls/examples/ls.rs index ab156181..b070c066 100644 --- a/examples/ls.rs +++ b/examples/ls/examples/ls.rs @@ -1,4 +1,5 @@ //! `ls` example +use rustic_backend::BackendOptions; use rustic_core::{LsOptions, Repository, RepositoryOptions}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -7,11 +8,17 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); - // Open repository - let repo_opts = RepositoryOptions::default() + // Initialize Backends + let backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); - let repo = Repository::new(&repo_opts)?.open()?.to_indexed()?; + .to_backends()?; + + // Open repository + let repo_opts = RepositoryOptions::default().password("test"); + + let repo = Repository::new(&repo_opts, backends)? + .open()? + .to_indexed()?; // use latest snapshot without filtering snapshots let node = repo.node_from_snapshot_path("latest", |_| true)?; diff --git a/examples/ls/src/lib.rs b/examples/ls/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/ls/src/lib.rs @@ -0,0 +1 @@ + diff --git a/examples/merge/Cargo.toml b/examples/merge/Cargo.toml new file mode 100644 index 00000000..99e75505 --- /dev/null +++ b/examples/merge/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "merge" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/merge.rs b/examples/merge/examples/merge.rs similarity index 70% rename from examples/merge.rs rename to examples/merge/examples/merge.rs index 13dd4443..cbb31128 100644 --- a/examples/merge.rs +++ b/examples/merge/examples/merge.rs @@ -1,4 +1,5 @@ //! `merge` example +use rustic_backend::BackendOptions; use rustic_core::{last_modified_node, repofile::SnapshotFile, Repository, RepositoryOptions}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -7,11 +8,17 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); - // Open repository - let repo_opts = RepositoryOptions::default() + // Initialize Backends + let backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); - let repo = Repository::new(&repo_opts)?.open()?.to_indexed_ids()?; + .to_backends()?; + + // Open repository + let repo_opts = RepositoryOptions::default().password("test"); + + let repo = Repository::new(&repo_opts, backends)? + .open()? + .to_indexed_ids()?; // Merge all snapshots using the latest entry for duplicate entries let snaps = repo.get_all_snapshots()?; diff --git a/examples/merge/src/lib.rs b/examples/merge/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/merge/src/lib.rs @@ -0,0 +1 @@ + diff --git a/examples/prune/Cargo.toml b/examples/prune/Cargo.toml new file mode 100644 index 00000000..a70554bb --- /dev/null +++ b/examples/prune/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "prune" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/prune.rs b/examples/prune/examples/prune.rs similarity index 71% rename from examples/prune.rs rename to examples/prune/examples/prune.rs index e6d0c497..970bc5a9 100644 --- a/examples/prune.rs +++ b/examples/prune/examples/prune.rs @@ -1,4 +1,5 @@ //! `prune` example +use rustic_backend::BackendOptions; use rustic_core::{PruneOptions, Repository, RepositoryOptions}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -7,11 +8,15 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); - // Open repository - let repo_opts = RepositoryOptions::default() + // Initialize Backends + let backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); - let repo = Repository::new(&repo_opts)?.open()?; + .to_backends()?; + + // Open repository + let repo_opts = RepositoryOptions::default().password("test"); + + let repo = Repository::new(&repo_opts, backends)?.open()?; let prune_opts = PruneOptions::default(); let prune_plan = repo.prune_plan(&prune_opts)?; diff --git a/examples/prune/src/lib.rs b/examples/prune/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/prune/src/lib.rs @@ -0,0 +1 @@ + diff --git a/examples/restore/Cargo.toml b/examples/restore/Cargo.toml new file mode 100644 index 00000000..ba716211 --- /dev/null +++ b/examples/restore/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "restore" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/restore.rs b/examples/restore/examples/restore.rs similarity index 80% rename from examples/restore.rs rename to examples/restore/examples/restore.rs index 085e5d3e..ec96d551 100644 --- a/examples/restore.rs +++ b/examples/restore/examples/restore.rs @@ -1,4 +1,5 @@ //! `restore` example +use rustic_backend::BackendOptions; use rustic_core::{LocalDestination, LsOptions, Repository, RepositoryOptions, RestoreOptions}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -7,11 +8,16 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); - // Open repository - let repo_opts = RepositoryOptions::default() + // Initialize Backends + let backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); - let repo = Repository::new(&repo_opts)?.open()?.to_indexed()?; + .to_backends()?; + + // Open repository + let repo_opts = RepositoryOptions::default().password("test"); + let repo = Repository::new(&repo_opts, backends)? + .open()? + .to_indexed()?; // use latest snapshot without filtering snapshots let node = repo.node_from_snapshot_path("latest", |_| true)?; diff --git a/examples/restore/src/lib.rs b/examples/restore/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/restore/src/lib.rs @@ -0,0 +1 @@ + diff --git a/examples/tag/Cargo.toml b/examples/tag/Cargo.toml new file mode 100644 index 00000000..c350d5dd --- /dev/null +++ b/examples/tag/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "tag" +version = "0.1.0" +edition = "2021" +publish = false + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustic_backend = { workspace = true } +rustic_core = { workspace = true } +simplelog = { workspace = true } diff --git a/examples/tag.rs b/examples/tag/examples/tag.rs similarity index 78% rename from examples/tag.rs rename to examples/tag/examples/tag.rs index 0c308da5..3cc21393 100644 --- a/examples/tag.rs +++ b/examples/tag/examples/tag.rs @@ -1,4 +1,5 @@ //! `tag` example +use rustic_backend::BackendOptions; use rustic_core::{Repository, RepositoryOptions, StringList}; use simplelog::{Config, LevelFilter, SimpleLogger}; use std::error::Error; @@ -8,11 +9,14 @@ fn main() -> Result<(), Box> { // Display info logs let _ = SimpleLogger::init(LevelFilter::Info, Config::default()); - // Open repository - let repo_opts = RepositoryOptions::default() + // Initialize Backends + let backends = BackendOptions::default() .repository("/tmp/repo") - .password("test"); - let repo = Repository::new(&repo_opts)?.open()?; + .to_backends()?; + + // Open repository + let repo_opts = RepositoryOptions::default().password("test"); + let repo = Repository::new(&repo_opts, backends)?.open()?; // Set tag "test" to all snapshots, filtering out unchanged (i.e. tag was already preset) snapshots let snaps = repo.get_all_snapshots()?; diff --git a/examples/tag/src/lib.rs b/examples/tag/src/lib.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/examples/tag/src/lib.rs @@ -0,0 +1 @@ + diff --git a/src/backend/choose.rs b/src/backend/choose.rs deleted file mode 100644 index d09f54b8..00000000 --- a/src/backend/choose.rs +++ /dev/null @@ -1,197 +0,0 @@ -use bytes::Bytes; - -use crate::{ - backend::{ - local::LocalBackend, rclone::RcloneBackend, rest::RestBackend, FileType, ReadBackend, - WriteBackend, - }, - error::{BackendErrorKind, RusticResult}, - id::Id, -}; - -/// Backend helper that chooses the correct backend based on the url. -#[derive(Clone, Debug)] -pub enum ChooseBackend { - /// Local backend. - Local(LocalBackend), - /// REST backend. - Rest(RestBackend), - /// Rclone backend. - Rclone(RcloneBackend), -} - -impl ChooseBackend { - /// Create a new [`ChooseBackend`] from a given url. - /// - /// # Arguments - /// - /// * `url` - The url to create the [`ChooseBackend`] from. - /// - /// # Errors - /// - /// * [`BackendErrorKind::BackendNotSupported`] - If the backend is not supported. - /// * [`LocalErrorKind::DirectoryCreationFailed`] - If the directory could not be created. - /// * [`RestErrorKind::UrlParsingFailed`] - If the url could not be parsed. - /// * [`RestErrorKind::BuildingClientFailed`] - If the client could not be built. - /// - /// [`BackendErrorKind::BackendNotSupported`]: crate::error::BackendErrorKind::BackendNotSupported - /// [`LocalErrorKind::DirectoryCreationFailed`]: crate::error::LocalErrorKind::DirectoryCreationFailed - /// [`RestErrorKind::UrlParsingFailed`]: crate::error::RestErrorKind::UrlParsingFailed - /// [`RestErrorKind::BuildingClientFailed`]: crate::error::RestErrorKind::BuildingClientFailed - pub fn from_url(url: &str) -> RusticResult { - Ok(match url.split_once(':') { - #[cfg(windows)] - Some((drive, _)) if drive.len() == 1 => Self::Local(LocalBackend::new(url)?), - Some(("rclone", path)) => Self::Rclone(RcloneBackend::new(path)?), - Some(("rest", path)) => Self::Rest(RestBackend::new(path)?), - Some(("local", path)) => Self::Local(LocalBackend::new(path)?), - Some((backend, _)) => { - return Err(BackendErrorKind::BackendNotSupported(backend.to_owned()).into()) - } - None => Self::Local(LocalBackend::new(url)?), - }) - } -} - -impl ReadBackend for ChooseBackend { - /// Returns the location of the backend. - fn location(&self) -> String { - match self { - Self::Local(local) => local.location(), - Self::Rest(rest) => rest.location(), - Self::Rclone(rclone) => rclone.location(), - } - } - - /// Sets an option of the backend. - /// - /// # Arguments - /// - /// * `option` - The option to set. - /// * `value` - The value to set the option to. - fn set_option(&mut self, option: &str, value: &str) -> RusticResult<()> { - match self { - Self::Local(local) => local.set_option(option, value), - Self::Rest(rest) => rest.set_option(option, value), - Self::Rclone(rclone) => rclone.set_option(option, value), - } - } - - /// Lists all files with their size of the given type. - /// - /// # Arguments - /// - /// * `tpe` - The type of the files to list. - /// - /// # Errors - /// - /// If the backend does not support listing files. - /// - /// # Returns - /// - /// A vector of tuples containing the id and size of the files. - fn list_with_size(&self, tpe: FileType) -> RusticResult> { - match self { - Self::Local(local) => local.list_with_size(tpe), - Self::Rest(rest) => rest.list_with_size(tpe), - Self::Rclone(rclone) => rclone.list_with_size(tpe), - } - } - - /// Reads full data of the given file. - /// - /// # Arguments - /// - /// * `tpe` - The type of the file. - /// * `id` - The id of the file. - /// - /// # Errors - /// - /// * [`LocalErrorKind::ReadingContentsOfFileFailed`] - If the file could not be read. - /// * [`reqwest::Error`] - If the request failed. - /// * [`RestErrorKind::BackoffError`] - If the backoff failed. - /// - /// # Returns - /// - /// The data read. - /// - /// [`RestErrorKind::BackoffError`]: crate::error::RestErrorKind::BackoffError - /// [`LocalErrorKind::ReadingContentsOfFileFailed`]: crate::error::LocalErrorKind::ReadingContentsOfFileFailed - fn read_full(&self, tpe: FileType, id: &Id) -> RusticResult { - match self { - Self::Local(local) => local.read_full(tpe, id), - Self::Rest(rest) => rest.read_full(tpe, id), - Self::Rclone(rclone) => rclone.read_full(tpe, id), - } - } - - /// Reads partial data of the given file. - /// - /// # Arguments - /// - /// * `tpe` - The type of the file. - /// * `id` - The id of the file. - /// * `cacheable` - Whether the file is cacheable. - /// * `offset` - The offset to read from. - /// * `length` - The length to read. - /// - /// # Returns - /// - /// The data read. - fn read_partial( - &self, - tpe: FileType, - id: &Id, - cacheable: bool, - offset: u32, - length: u32, - ) -> RusticResult { - match self { - Self::Local(local) => local.read_partial(tpe, id, cacheable, offset, length), - Self::Rest(rest) => rest.read_partial(tpe, id, cacheable, offset, length), - Self::Rclone(rclone) => rclone.read_partial(tpe, id, cacheable, offset, length), - } - } -} - -impl WriteBackend for ChooseBackend { - /// Creates the backend. - fn create(&self) -> RusticResult<()> { - match self { - Self::Local(local) => local.create(), - Self::Rest(rest) => rest.create(), - Self::Rclone(rclone) => rclone.create(), - } - } - - /// Writes the given data to the given file. - /// - /// # Arguments - /// - /// * `tpe` - The type of the file. - /// * `id` - The id of the file. - /// * `cacheable` - Whether the file is cacheable. - /// * `buf` - The data to write. - fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> RusticResult<()> { - match self { - Self::Local(local) => local.write_bytes(tpe, id, cacheable, buf), - Self::Rest(rest) => rest.write_bytes(tpe, id, cacheable, buf), - Self::Rclone(rclone) => rclone.write_bytes(tpe, id, cacheable, buf), - } - } - - /// Removes the given file. - /// - /// # Arguments - /// - /// * `tpe` - The type of the file. - /// * `id` - The id of the file. - /// * `cacheable` - Whether the file is cacheable. - fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> RusticResult<()> { - match self { - Self::Local(local) => local.remove(tpe, id, cacheable), - Self::Rest(rest) => rest.remove(tpe, id, cacheable), - Self::Rclone(rclone) => rclone.remove(tpe, id, cacheable), - } - } -} diff --git a/src/backend/hotcold.rs b/src/backend/hotcold.rs deleted file mode 100644 index 2f3455c2..00000000 --- a/src/backend/hotcold.rs +++ /dev/null @@ -1,98 +0,0 @@ -use bytes::Bytes; - -use crate::{backend::FileType, backend::ReadBackend, backend::WriteBackend, id::Id, RusticResult}; - -/// A hot/cold backend implementation. -/// -/// # Type Parameters -/// -/// * `BE` - The backend to use. -#[derive(Clone, Debug)] -pub struct HotColdBackend { - /// The backend to use. - be: BE, - /// The backend to use for hot files. - hot_be: Option, -} - -impl HotColdBackend { - /// Creates a new `HotColdBackend`. - /// - /// # Type Parameters - /// - /// * `BE` - The backend to use. - /// - /// # Arguments - /// - /// * `be` - The backend to use. - /// * `hot_be` - The backend to use for hot files. - pub fn new(be: BE, hot_be: Option) -> Self { - Self { be, hot_be } - } -} - -impl ReadBackend for HotColdBackend { - fn location(&self) -> String { - self.be.location() - } - - fn set_option(&mut self, option: &str, value: &str) -> RusticResult<()> { - self.be.set_option(option, value) - } - - fn list_with_size(&self, tpe: FileType) -> RusticResult> { - self.be.list_with_size(tpe) - } - - fn read_full(&self, tpe: FileType, id: &Id) -> RusticResult { - self.hot_be - .as_ref() - .map_or_else(|| self.be.read_full(tpe, id), |be| be.read_full(tpe, id)) - } - - fn read_partial( - &self, - tpe: FileType, - id: &Id, - cacheable: bool, - offset: u32, - length: u32, - ) -> RusticResult { - match (&self.hot_be, cacheable || tpe != FileType::Pack) { - (None, _) | (Some(_), false) => { - self.be.read_partial(tpe, id, cacheable, offset, length) - } - (Some(be), true) => be.read_partial(tpe, id, cacheable, offset, length), - } - } -} - -impl WriteBackend for HotColdBackend { - fn create(&self) -> RusticResult<()> { - self.be.create()?; - if let Some(be) = &self.hot_be { - be.create()?; - } - Ok(()) - } - - fn write_bytes(&self, tpe: FileType, id: &Id, cacheable: bool, buf: Bytes) -> RusticResult<()> { - if let Some(be) = &self.hot_be { - if tpe != FileType::Config && (cacheable || tpe != FileType::Pack) { - be.write_bytes(tpe, id, cacheable, buf.clone())?; - } - } - self.be.write_bytes(tpe, id, cacheable, buf) - } - - fn remove(&self, tpe: FileType, id: &Id, cacheable: bool) -> RusticResult<()> { - // First remove cold file - self.be.remove(tpe, id, cacheable)?; - if let Some(be) = &self.hot_be { - if cacheable || tpe != FileType::Pack { - be.remove(tpe, id, cacheable)?; - } - } - Ok(()) - } -} diff --git a/src/backend/local.rs b/src/backend/local.rs deleted file mode 100644 index 36c2a488..00000000 --- a/src/backend/local.rs +++ /dev/null @@ -1,1062 +0,0 @@ -#[cfg(not(windows))] -use std::os::unix::fs::{symlink, PermissionsExt}; - -use std::{ - fs::{self, File, OpenOptions}, - io::{Read, Seek, SeekFrom, Write}, - path::{Path, PathBuf}, - process::Command, -}; - -use aho_corasick::AhoCorasick; -use bytes::Bytes; -#[allow(unused_imports)] -use cached::proc_macro::cached; -use filetime::{set_symlink_file_times, FileTime}; -use log::{debug, trace, warn}; -#[cfg(not(windows))] -use nix::sys::stat::{mknod, Mode, SFlag}; -#[cfg(not(windows))] -use nix::unistd::{fchownat, FchownatFlags, Gid, Group, Uid, User}; -use shell_words::split; -use walkdir::WalkDir; - -#[cfg(not(windows))] -use crate::backend::ignore::mapper::map_mode_from_go; -#[cfg(not(windows))] -use crate::backend::node::NodeType; - -use crate::{ - backend::{ - node::{ExtendedAttribute, Metadata, Node}, - FileType, ReadBackend, WriteBackend, ALL_FILE_TYPES, - }, - error::{LocalErrorKind, RusticResult}, - id::Id, -}; - -/// Local backend, used when backing up. -/// -/// This backend is used when backing up to a local directory. -/// It will create a directory structure like this: -/// -/// ```text -/// / -/// ├── config -/// ├── data -/// │ ├── 00 -/// │ │ └── -/// │ ├── 01 -/// │ │ └── -/// │ └── ... -/// ├── index -/// │ └── -/// ├── keys -/// │ └── -/// ├── snapshots -/// │ └── -/// └── ... -/// ``` -/// -/// The `data` directory will contain all data files, split into 256 subdirectories. -/// The `config` directory will contain the config file. -/// The `index` directory will contain the index file. -/// The `keys` directory will contain the keys file. -/// The `snapshots` directory will contain the snapshots file. -/// All other directories will contain the pack files. -#[derive(Clone, Debug)] -pub struct LocalBackend { - /// The base path of the backend. - path: PathBuf, - /// The command to call after a file was created. - post_create_command: Option, - /// The command to call after a file was deleted. - post_delete_command: Option, -} - -impl LocalBackend { - /// Create a new [`LocalBackend`] - /// - /// # Arguments - /// - /// * `path` - The base path of the backend - /// - /// # Errors - /// - /// * [`LocalErrorKind::DirectoryCreationFailed`] - If the directory could not be created. - /// - /// [`LocalErrorKind::DirectoryCreationFailed`]: crate::error::LocalErrorKind::DirectoryCreationFailed - // TODO: We should use `impl Into` here. we even use it in the body! - pub fn new(path: &str) -> RusticResult { - let path = path.into(); - fs::create_dir_all(&path).map_err(LocalErrorKind::DirectoryCreationFailed)?; - Ok(Self { - path, - post_create_command: None, - post_delete_command: None, - }) - } - - /// Path to the given file type and id. - /// - /// If the file type is `FileType::Pack`, the id will be used to determine the subdirectory. - /// - /// # Arguments - /// - /// * `tpe` - The type of the file. - /// * `id` - The id of the file. - /// - /// # Returns - /// - /// The path to the file. - fn path(&self, tpe: FileType, id: &Id) -> PathBuf { - let hex_id = id.to_hex(); - match tpe { - FileType::Config => self.path.join("config"), - FileType::Pack => self.path.join("data").join(&hex_id[0..2]).join(hex_id), - _ => self.path.join(tpe.dirname()).join(hex_id), - } - } - - /// Call the given command. - /// - /// # Arguments - /// - /// * `tpe` - The type of the file. - /// * `id` - The id of the file. - /// * `filename` - The path to the file. - /// * `command` - The command to call. - /// - /// # Errors - /// - /// * [`LocalErrorKind::FromAhoCorasick`] - If the patterns could not be compiled. - /// * [`LocalErrorKind::FromSplitError`] - If the command could not be parsed. - /// * [`LocalErrorKind::CommandExecutionFailed`] - If the command could not be executed. - /// * [`LocalErrorKind::CommandNotSuccessful`] - If the command was not successful. - /// - /// # Notes - /// - /// The following placeholders are supported: - /// * `%file` - The path to the file. - /// * `%type` - The type of the file. - /// * `%id` - The id of the file. - /// - /// [`LocalErrorKind::FromAhoCorasick`]: crate::error::LocalErrorKind::FromAhoCorasick - /// [`LocalErrorKind::FromSplitError`]: crate::error::LocalErrorKind::FromSplitError - /// [`LocalErrorKind::CommandExecutionFailed`]: crate::error::LocalErrorKind::CommandExecutionFailed - /// [`LocalErrorKind::CommandNotSuccessful`]: crate::error::LocalErrorKind::CommandNotSuccessful - fn call_command(tpe: FileType, id: &Id, filename: &Path, command: &str) -> RusticResult<()> { - let id = id.to_hex(); - let patterns = &["%file", "%type", "%id"]; - let ac = AhoCorasick::new(patterns).map_err(LocalErrorKind::FromAhoCorasick)?; - let replace_with = &[filename.to_str().unwrap(), tpe.dirname(), id.as_str()]; - let actual_command = ac.replace_all(command, replace_with); - debug!("calling {actual_command}..."); - let commands = split(&actual_command).map_err(LocalErrorKind::FromSplitError)?; - let status = Command::new(&commands[0]) - .args(&commands[1..]) - .status() - .map_err(LocalErrorKind::CommandExecutionFailed)?; - if !status.success() { - return Err(LocalErrorKind::CommandNotSuccessful { - file_name: replace_with[0].to_owned(), - file_type: replace_with[1].to_owned(), - id: replace_with[2].to_owned(), - status, - } - .into()); - } - Ok(()) - } -} - -impl ReadBackend for LocalBackend { - /// Returns the location of the backend. - /// - /// This is `local:`. - fn location(&self) -> String { - let mut location = "local:".to_string(); - location.push_str(&self.path.to_string_lossy()); - location - } - - /// Sets an option of the backend. - /// - /// # Arguments - /// - /// * `option` - The option to set. - /// * `value` - The value to set the option to. - /// - /// # Notes - /// - /// The following options are supported: - /// * `post-create-command` - The command to call after a file was created. - /// * `post-delete-command` - The command to call after a file was deleted. - fn set_option(&mut self, option: &str, value: &str) -> RusticResult<()> { - match option { - "post-create-command" => { - self.post_create_command = Some(value.to_string()); - } - "post-delete-command" => { - self.post_delete_command = Some(value.to_string()); - } - opt => { - warn!("Option {opt} is not supported! Ignoring it."); - } - } - Ok(()) - } - - /// Lists all files of the given type. - /// - /// # Arguments - /// - /// * `tpe` - The type of the files to list. - /// - /// # Errors - /// - /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string - /// - /// # Notes - /// - /// If the file type is `FileType::Config`, this will return a list with a single default id. - /// - /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError - fn list(&self, tpe: FileType) -> RusticResult> { - trace!("listing tpe: {tpe:?}"); - if tpe == FileType::Config { - return Ok(if self.path.join("config").exists() { - vec![Id::default()] - } else { - Vec::new() - }); - } - - let walker = WalkDir::new(self.path.join(tpe.dirname())) - .into_iter() - .filter_map(walkdir::Result::ok) - .filter(|e| e.file_type().is_file()) - .map(|e| Id::from_hex(&e.file_name().to_string_lossy())) - .filter_map(std::result::Result::ok); - Ok(walker.collect()) - } - - /// Lists all files with their size of the given type. - /// - /// # Arguments - /// - /// * `tpe` - The type of the files to list. - /// - /// # Errors - /// - /// * [`LocalErrorKind::QueryingMetadataFailed`] - If the metadata of the file could not be queried. - /// * [`LocalErrorKind::FromTryIntError`] - If the length of the file could not be converted to u32. - /// * [`LocalErrorKind::QueryingWalkDirMetadataFailed`] - If the metadata of the file could not be queried. - /// * [`IdErrorKind::HexError`] - If the string is not a valid hexadecimal string - /// - /// [`LocalErrorKind::QueryingMetadataFailed`]: crate::error::LocalErrorKind::QueryingMetadataFailed - /// [`LocalErrorKind::FromTryIntError`]: crate::error::LocalErrorKind::FromTryIntError - /// [`LocalErrorKind::QueryingWalkDirMetadataFailed`]: crate::error::LocalErrorKind::QueryingWalkDirMetadataFailed - /// [`IdErrorKind::HexError`]: crate::error::IdErrorKind::HexError - fn list_with_size(&self, tpe: FileType) -> RusticResult> { - trace!("listing tpe: {tpe:?}"); - let path = self.path.join(tpe.dirname()); - - if tpe == FileType::Config { - return Ok(if path.exists() { - vec![( - Id::default(), - path.metadata() - .map_err(LocalErrorKind::QueryingMetadataFailed)? - .len() - .try_into() - .map_err(LocalErrorKind::FromTryIntError)?, - )] - } else { - Vec::new() - }); - } - - let walker = WalkDir::new(path) - .into_iter() - .filter_map(walkdir::Result::ok) - .filter(|e| e.file_type().is_file()) - .map(|e| -> RusticResult<_> { - Ok(( - Id::from_hex(&e.file_name().to_string_lossy())?, - e.metadata() - .map_err(LocalErrorKind::QueryingWalkDirMetadataFailed)? - .len() - .try_into() - .map_err(LocalErrorKind::FromTryIntError)?, - )) - }) - .filter_map(RusticResult::ok); - - Ok(walker.collect()) - } - - /// Reads full data of the given file. - /// - /// # Arguments - /// - /// * `tpe` - The type of the file. - /// * `id` - The id of the file. - /// - /// # Errors - /// - /// * [`LocalErrorKind::ReadingContentsOfFileFailed`] - If the file could not be read. - /// - /// [`LocalErrorKind::ReadingContentsOfFileFailed`]: crate::error::LocalErrorKind::ReadingContentsOfFileFailed - fn read_full(&self, tpe: FileType, id: &Id) -> RusticResult { - trace!("reading tpe: {tpe:?}, id: {id}"); - Ok(fs::read(self.path(tpe, id)) - .map_err(LocalErrorKind::ReadingContentsOfFileFailed)? - .into()) - } - - /// Reads partial data of the given file. - /// - /// # Arguments - /// - /// * `tpe` - The type of the file. - /// * `id` - The id of the file. - /// * `cacheable` - Whether the file is cacheable. - /// * `offset` - The offset to read from. - /// * `length` - The length to read. - /// - /// # Errors - /// - /// * [`LocalErrorKind::OpeningFileFailed`] - If the file could not be opened. - /// * [`LocalErrorKind::CouldNotSeekToPositionInFile`] - If the file could not be seeked to the given position. - /// * [`LocalErrorKind::FromTryIntError`] - If the length of the file could not be converted to u32. - /// * [`LocalErrorKind::ReadingExactLengthOfFileFailed`] - If the length of the file could not be read. - /// - /// [`LocalErrorKind::OpeningFileFailed`]: crate::error::LocalErrorKind::OpeningFileFailed - /// [`LocalErrorKind::CouldNotSeekToPositionInFile`]: crate::error::LocalErrorKind::CouldNotSeekToPositionInFile - /// [`LocalErrorKind::FromTryIntError`]: crate::error::LocalErrorKind::FromTryIntError - /// [`LocalErrorKind::ReadingExactLengthOfFileFailed`]: crate::error::LocalErrorKind::ReadingExactLengthOfFileFailed - fn read_partial( - &self, - tpe: FileType, - id: &Id, - _cacheable: bool, - offset: u32, - length: u32, - ) -> RusticResult { - trace!("reading tpe: {tpe:?}, id: {id}, offset: {offset}, length: {length}"); - let mut file = File::open(self.path(tpe, id)).map_err(LocalErrorKind::OpeningFileFailed)?; - _ = file - .seek(SeekFrom::Start(offset.into())) - .map_err(LocalErrorKind::CouldNotSeekToPositionInFile)?; - let mut vec = vec![0; length.try_into().map_err(LocalErrorKind::FromTryIntError)?]; - file.read_exact(&mut vec) - .map_err(LocalErrorKind::ReadingExactLengthOfFileFailed)?; - Ok(vec.into()) - } -} - -impl WriteBackend for LocalBackend { - /// Create a repository on the backend. - /// - /// # Errors - /// - /// * [`LocalErrorKind::DirectoryCreationFailed`] - If the directory could not be created. - /// - /// [`LocalErrorKind::DirectoryCreationFailed`]: crate::error::LocalErrorKind::DirectoryCreationFailed - fn create(&self) -> RusticResult<()> { - trace!("creating repo at {:?}", self.path); - - for tpe in ALL_FILE_TYPES { - fs::create_dir_all(self.path.join(tpe.dirname())) - .map_err(LocalErrorKind::DirectoryCreationFailed)?; - } - for i in 0u8..=255 { - fs::create_dir_all(self.path.join("data").join(hex::encode([i]))) - .map_err(LocalErrorKind::DirectoryCreationFailed)?; - } - Ok(()) - } - - /// Write the given bytes to the given file. - /// - /// # Arguments - /// - /// * `tpe` - The type of the file. - /// * `id` - The id of the file. - /// * `cacheable` - Whether the file is cacheable. - /// * `buf` - The bytes to write. - /// - /// # Errors - /// - /// * [`LocalErrorKind::OpeningFileFailed`] - If the file could not be opened. - /// * [`LocalErrorKind::FromTryIntError`] - If the length of the bytes could not be converted to u64. - /// * [`LocalErrorKind::SettingFileLengthFailed`] - If the length of the file could not be set. - /// * [`LocalErrorKind::CouldNotWriteToBuffer`] - If the bytes could not be written to the file. - /// * [`LocalErrorKind::SyncingOfOsMetadataFailed`] - If the metadata of the file could not be synced. - /// - /// [`LocalErrorKind::OpeningFileFailed`]: crate::error::LocalErrorKind::OpeningFileFailed - /// [`LocalErrorKind::FromTryIntError`]: crate::error::LocalErrorKind::FromTryIntError - /// [`LocalErrorKind::SettingFileLengthFailed`]: crate::error::LocalErrorKind::SettingFileLengthFailed - /// [`LocalErrorKind::CouldNotWriteToBuffer`]: crate::error::LocalErrorKind::CouldNotWriteToBuffer - /// [`LocalErrorKind::SyncingOfOsMetadataFailed`]: crate::error::LocalErrorKind::SyncingOfOsMetadataFailed - fn write_bytes( - &self, - tpe: FileType, - id: &Id, - _cacheable: bool, - buf: Bytes, - ) -> RusticResult<()> { - trace!("writing tpe: {:?}, id: {}", &tpe, &id); - let filename = self.path(tpe, id); - let mut file = fs::OpenOptions::new() - .create(true) - .write(true) - .open(&filename) - .map_err(LocalErrorKind::OpeningFileFailed)?; - file.set_len( - buf.len() - .try_into() - .map_err(LocalErrorKind::FromTryIntError)?, - ) - .map_err(LocalErrorKind::SettingFileLengthFailed)?; - file.write_all(&buf) - .map_err(LocalErrorKind::CouldNotWriteToBuffer)?; - file.sync_all() - .map_err(LocalErrorKind::SyncingOfOsMetadataFailed)?; - if let Some(command) = &self.post_create_command { - if let Err(err) = Self::call_command(tpe, id, &filename, command) { - warn!("post-create: {err}"); - } - } - Ok(()) - } - - /// Remove the given file. - /// - /// # Arguments - /// - /// * `tpe` - The type of the file. - /// * `id` - The id of the file. - /// * `cacheable` - Whether the file is cacheable. - /// - /// # Errors - /// - /// * [`LocalErrorKind::FileRemovalFailed`] - If the file could not be removed. - /// - /// [`LocalErrorKind::FileRemovalFailed`]: crate::error::LocalErrorKind::FileRemovalFailed - fn remove(&self, tpe: FileType, id: &Id, _cacheable: bool) -> RusticResult<()> { - trace!("removing tpe: {:?}, id: {}", &tpe, &id); - let filename = self.path(tpe, id); - fs::remove_file(&filename).map_err(LocalErrorKind::FileRemovalFailed)?; - if let Some(command) = &self.post_delete_command { - if let Err(err) = Self::call_command(tpe, id, &filename, command) { - warn!("post-delete: {err}"); - } - } - Ok(()) - } -} - -#[derive(Clone, Debug)] -/// Local destination, used when restoring. -pub struct LocalDestination { - /// The base path of the destination. - path: PathBuf, - /// Whether we expect a single file as destination. - is_file: bool, -} - -// Helper function to cache mapping user name -> uid -#[cfg(not(windows))] -#[cached] -fn uid_from_name(name: String) -> Option { - User::from_name(&name).unwrap().map(|u| u.uid) -} - -// Helper function to cache mapping group name -> gid -#[cfg(not(windows))] -#[cached] -fn gid_from_name(name: String) -> Option { - Group::from_name(&name).unwrap().map(|g| g.gid) -} - -impl LocalDestination { - /// Create a new [`LocalDestination`] - /// - /// # Arguments - /// - /// * `path` - The base path of the destination - /// * `create` - If `create` is true, create the base path if it doesn't exist. - /// * `expect_file` - Whether we expect a single file as destination. - /// - /// # Errors - /// - /// * [`LocalErrorKind::DirectoryCreationFailed`] - If the directory could not be created. - /// - /// [`LocalErrorKind::DirectoryCreationFailed`]: crate::error::LocalErrorKind::DirectoryCreationFailed - // TODO: We should use `impl Into` here. we even use it in the body! - pub fn new(path: &str, create: bool, expect_file: bool) -> RusticResult { - let is_dir = path.ends_with('/'); - let path: PathBuf = path.into(); - let is_file = path.is_file() || (!path.is_dir() && !is_dir && expect_file); - - if create { - if is_file { - if let Some(path) = path.parent() { - fs::create_dir_all(path).map_err(LocalErrorKind::DirectoryCreationFailed)?; - } - } else { - fs::create_dir_all(&path).map_err(LocalErrorKind::DirectoryCreationFailed)?; - } - } - - Ok(Self { path, is_file }) - } - - /// Path to the given item (relative to the base path) - /// - /// # Arguments - /// - /// * `item` - The item to get the path for - /// - /// # Returns - /// - /// The path to the item. - /// - /// # Notes - /// - /// * If the destination is a file, this will return the base path. - /// * If the destination is a directory, this will return the base path joined with the item. - pub(crate) fn path(&self, item: impl AsRef) -> PathBuf { - if self.is_file { - self.path.clone() - } else { - self.path.join(item) - } - } - - /// Remove the given directory (relative to the base path) - /// - /// # Arguments - /// - /// * `dirname` - The directory to remove - /// - /// # Errors - /// - /// * [`LocalErrorKind::DirectoryRemovalFailed`] - If the directory could not be removed. - /// - /// # Notes - /// - /// This will remove the directory recursively. - /// - /// [`LocalErrorKind::DirectoryRemovalFailed`]: crate::error::LocalErrorKind::DirectoryRemovalFailed - pub fn remove_dir(&self, dirname: impl AsRef) -> RusticResult<()> { - Ok(fs::remove_dir_all(dirname).map_err(LocalErrorKind::DirectoryRemovalFailed)?) - } - - /// Remove the given file (relative to the base path) - /// - /// # Arguments - /// - /// * `filename` - The file to remove - /// - /// # Errors - /// - /// * [`LocalErrorKind::FileRemovalFailed`] - If the file could not be removed. - /// - /// # Notes - /// - /// This will remove the file. - /// - /// * If the file is a symlink, the symlink will be removed, not the file it points to. - /// * If the file is a directory or device, this will fail. - /// - /// [`LocalErrorKind::FileRemovalFailed`]: crate::error::LocalErrorKind::FileRemovalFailed - pub fn remove_file(&self, filename: impl AsRef) -> RusticResult<()> { - Ok(fs::remove_file(filename).map_err(LocalErrorKind::FileRemovalFailed)?) - } - - /// Create the given directory (relative to the base path) - /// - /// # Arguments - /// - /// * `item` - The directory to create - /// - /// # Errors - /// - /// * [`LocalErrorKind::DirectoryCreationFailed`] - If the directory could not be created. - /// - /// # Notes - /// - /// This will create the directory structure recursively. - /// - /// [`LocalErrorKind::DirectoryCreationFailed`]: crate::error::LocalErrorKind::DirectoryCreationFailed - pub fn create_dir(&self, item: impl AsRef) -> RusticResult<()> { - let dirname = self.path.join(item); - fs::create_dir_all(dirname).map_err(LocalErrorKind::DirectoryCreationFailed)?; - Ok(()) - } - - /// Set changed and modified times for `item` (relative to the base path) utilizing the file metadata - /// - /// # Arguments - /// - /// * `item` - The item to set the times for - /// * `meta` - The metadata to get the times from - /// - /// # Errors - /// - /// * [`LocalErrorKind::SettingTimeMetadataFailed`] - If the times could not be set - /// - /// [`LocalErrorKind::SettingTimeMetadataFailed`]: crate::error::LocalErrorKind::SettingTimeMetadataFailed - pub fn set_times(&self, item: impl AsRef, meta: &Metadata) -> RusticResult<()> { - let filename = self.path(item); - if let Some(mtime) = meta.mtime { - let atime = meta.atime.unwrap_or(mtime); - set_symlink_file_times( - filename, - FileTime::from_system_time(atime.into()), - FileTime::from_system_time(mtime.into()), - ) - .map_err(LocalErrorKind::SettingTimeMetadataFailed)?; - } - - Ok(()) - } - - #[cfg(windows)] - // TODO: Windows support - /// Set user/group for `item` (relative to the base path) utilizing the file metadata - /// - /// # Arguments - /// - /// * `item` - The item to set the user/group for - /// * `meta` - The metadata to get the user/group from - /// - /// # Errors - /// - /// If the user/group could not be set. - pub fn set_user_group(&self, _item: impl AsRef, _meta: &Metadata) -> RusticResult<()> { - // https://learn.microsoft.com/en-us/windows/win32/fileio/file-security-and-access-rights - // https://microsoft.github.io/windows-docs-rs/doc/windows/Win32/Security/struct.SECURITY_ATTRIBUTES.html - // https://microsoft.github.io/windows-docs-rs/doc/windows/Win32/Storage/FileSystem/struct.CREATEFILE2_EXTENDED_PARAMETERS.html#structfield.lpSecurityAttributes - Ok(()) - } - - #[cfg(not(windows))] - /// Set user/group for `item` (relative to the base path) utilizing the file metadata - /// - /// # Arguments - /// - /// * `item` - The item to set the user/group for - /// * `meta` - The metadata to get the user/group from - /// - /// # Errors - /// - /// * [`LocalErrorKind::FromErrnoError`] - If the user/group could not be set. - /// - /// [`LocalErrorKind::FromErrnoError`]: crate::error::LocalErrorKind::FromErrnoError - pub fn set_user_group(&self, item: impl AsRef, meta: &Metadata) -> RusticResult<()> { - let filename = self.path(item); - - let user = meta.user.clone().and_then(uid_from_name); - // use uid from user if valid, else from saved uid (if saved) - let uid = user.or_else(|| meta.uid.map(Uid::from_raw)); - - let group = meta.group.clone().and_then(gid_from_name); - // use gid from group if valid, else from saved gid (if saved) - let gid = group.or_else(|| meta.gid.map(Gid::from_raw)); - - fchownat(None, &filename, uid, gid, FchownatFlags::NoFollowSymlink) - .map_err(LocalErrorKind::FromErrnoError)?; - Ok(()) - } - - #[cfg(windows)] - // TODO: Windows support - /// Set uid/gid for `item` (relative to the base path) utilizing the file metadata - /// - /// # Arguments - /// - /// * `item` - The item to set the uid/gid for - /// * `meta` - The metadata to get the uid/gid from - /// - /// # Errors - /// - /// If the uid/gid could not be set. - pub fn set_uid_gid(&self, _item: impl AsRef, _meta: &Metadata) -> RusticResult<()> { - Ok(()) - } - - #[cfg(not(windows))] - /// Set uid/gid for `item` (relative to the base path) utilizing the file metadata - /// - /// # Arguments - /// - /// * `item` - The item to set the uid/gid for - /// * `meta` - The metadata to get the uid/gid from - /// - /// # Errors - /// - /// * [`LocalErrorKind::FromErrnoError`] - If the uid/gid could not be set. - /// - /// [`LocalErrorKind::FromErrnoError`]: crate::error::LocalErrorKind::FromErrnoError - pub fn set_uid_gid(&self, item: impl AsRef, meta: &Metadata) -> RusticResult<()> { - let filename = self.path(item); - - let uid = meta.uid.map(Uid::from_raw); - let gid = meta.gid.map(Gid::from_raw); - - fchownat(None, &filename, uid, gid, FchownatFlags::NoFollowSymlink) - .map_err(LocalErrorKind::FromErrnoError)?; - Ok(()) - } - - #[cfg(windows)] - // TODO: Windows support - /// Set permissions for `item` (relative to the base path) from `node` - /// - /// # Arguments - /// - /// * `item` - The item to set the permissions for - /// * `node` - The node to get the permissions from - /// - /// # Errors - /// - /// If the permissions could not be set. - pub fn set_permission(&self, _item: impl AsRef, _node: &Node) -> RusticResult<()> { - Ok(()) - } - - #[cfg(not(windows))] - /// Set permissions for `item` (relative to the base path) from `node` - /// - /// # Arguments - /// - /// * `item` - The item to set the permissions for - /// * `node` - The node to get the permissions from - /// - /// # Errors - /// - /// * [`LocalErrorKind::SettingFilePermissionsFailed`] - If the permissions could not be set. - /// - /// [`LocalErrorKind::SettingFilePermissionsFailed`]: crate::error::LocalErrorKind::SettingFilePermissionsFailed - pub fn set_permission(&self, item: impl AsRef, node: &Node) -> RusticResult<()> { - if node.is_symlink() { - return Ok(()); - } - - let filename = self.path(item); - - if let Some(mode) = node.meta.mode { - let mode = map_mode_from_go(mode); - std::fs::set_permissions(filename, fs::Permissions::from_mode(mode)) - .map_err(LocalErrorKind::SettingFilePermissionsFailed)?; - } - Ok(()) - } - - #[cfg(any(windows, target_os = "openbsd"))] - // TODO: Windows support - // TODO: openbsd support - /// Set extended attributes for `item` (relative to the base path) - /// - /// # Arguments - /// - /// * `item` - The item to set the extended attributes for - /// * `extended_attributes` - The extended attributes to set - /// - /// # Errors - /// - /// If the extended attributes could not be set. - pub fn set_extended_attributes( - &self, - _item: impl AsRef, - _extended_attributes: &[ExtendedAttribute], - ) -> RusticResult<()> { - Ok(()) - } - - #[cfg(not(any(windows, target_os = "openbsd")))] - /// Set extended attributes for `item` (relative to the base path) - /// - /// # Arguments - /// - /// * `item` - The item to set the extended attributes for - /// * `extended_attributes` - The extended attributes to set - /// - /// # Errors - /// - /// * [`LocalErrorKind::ListingXattrsFailed`] - If listing the extended attributes failed. - /// * [`LocalErrorKind::GettingXattrFailed`] - If getting an extended attribute failed. - /// * [`LocalErrorKind::SettingXattrFailed`] - If setting an extended attribute failed. - /// - /// [`LocalErrorKind::ListingXattrsFailed`]: crate::error::LocalErrorKind::ListingXattrsFailed - /// [`LocalErrorKind::GettingXattrFailed`]: crate::error::LocalErrorKind::GettingXattrFailed - /// [`LocalErrorKind::SettingXattrFailed`]: crate::error::LocalErrorKind::SettingXattrFailed - pub fn set_extended_attributes( - &self, - item: impl AsRef, - extended_attributes: &[ExtendedAttribute], - ) -> RusticResult<()> { - let filename = self.path(item); - let mut done = vec![false; extended_attributes.len()]; - - for curr_name in xattr::list(&filename) - .map_err(|err| LocalErrorKind::ListingXattrsFailed(err, filename.clone()))? - { - match extended_attributes.iter().enumerate().find( - |(_, ExtendedAttribute { name, .. })| name == curr_name.to_string_lossy().as_ref(), - ) { - Some((index, ExtendedAttribute { name, value })) => { - let curr_value = xattr::get(&filename, name) - .map_err(|err| LocalErrorKind::GettingXattrFailed { - name: name.clone(), - filename: filename.clone(), - source: err, - })? - .unwrap(); - if value != &curr_value { - xattr::set(&filename, name, value).map_err(|err| { - LocalErrorKind::SettingXattrFailed { - name: name.clone(), - filename: filename.clone(), - source: err, - } - })?; - } - done[index] = true; - } - None => { - if let Err(err) = xattr::remove(&filename, &curr_name) { - warn!("error removing xattr {curr_name:?} on {filename:?}: {err}"); - } - } - } - } - - for (index, ExtendedAttribute { name, value }) in extended_attributes.iter().enumerate() { - if !done[index] { - xattr::set(&filename, name, value).map_err(|err| { - LocalErrorKind::SettingXattrFailed { - name: name.clone(), - filename: filename.clone(), - source: err, - } - })?; - } - } - - Ok(()) - } - - /// Set length of `item` (relative to the base path) - /// - /// # Arguments - /// - /// * `item` - The item to set the length for - /// * `size` - The size to set the length to - /// - /// # Errors - /// - /// * [`LocalErrorKind::FileDoesNotHaveParent`] - If the file does not have a parent. - /// * [`LocalErrorKind::DirectoryCreationFailed`] - If the directory could not be created. - /// * [`LocalErrorKind::OpeningFileFailed`] - If the file could not be opened. - /// * [`LocalErrorKind::SettingFileLengthFailed`] - If the length of the file could not be set. - /// - /// # Notes - /// - /// If the file exists, truncate it to the given length. (TODO: check if this is correct) - /// If it doesn't exist, create a new (empty) one with given length. - /// - /// [`LocalErrorKind::FileDoesNotHaveParent`]: crate::error::LocalErrorKind::FileDoesNotHaveParent - /// [`LocalErrorKind::DirectoryCreationFailed`]: crate::error::LocalErrorKind::DirectoryCreationFailed - /// [`LocalErrorKind::OpeningFileFailed`]: crate::error::LocalErrorKind::OpeningFileFailed - /// [`LocalErrorKind::SettingFileLengthFailed`]: crate::error::LocalErrorKind::SettingFileLengthFailed - pub fn set_length(&self, item: impl AsRef, size: u64) -> RusticResult<()> { - let filename = self.path(item); - let dir = filename - .parent() - .ok_or_else(|| LocalErrorKind::FileDoesNotHaveParent(filename.clone()))?; - fs::create_dir_all(dir).map_err(LocalErrorKind::DirectoryCreationFailed)?; - - OpenOptions::new() - .create(true) - .write(true) - .open(filename) - .map_err(LocalErrorKind::OpeningFileFailed)? - .set_len(size) - .map_err(LocalErrorKind::SettingFileLengthFailed)?; - Ok(()) - } - - #[cfg(windows)] - // TODO: Windows support - /// Create a special file (relative to the base path) - pub fn create_special(&self, _item: impl AsRef, _node: &Node) -> RusticResult<()> { - Ok(()) - } - - #[cfg(not(windows))] - /// Create a special file (relative to the base path) - /// - /// # Arguments - /// - /// * `item` - The item to create - /// * `node` - The node to get the type from - /// - /// # Errors - /// - /// * [`LocalErrorKind::SymlinkingFailed`] - If the symlink could not be created. - /// * [`LocalErrorKind::FromTryIntError`] - If the device could not be converted to the correct type. - /// * [`LocalErrorKind::FromErrnoError`] - If the device could not be created. - /// - /// [`LocalErrorKind::SymlinkingFailed`]: crate::error::LocalErrorKind::SymlinkingFailed - /// [`LocalErrorKind::FromTryIntError`]: crate::error::LocalErrorKind::FromTryIntError - /// [`LocalErrorKind::FromErrnoError`]: crate::error::LocalErrorKind::FromErrnoError - pub fn create_special(&self, item: impl AsRef, node: &Node) -> RusticResult<()> { - let filename = self.path(item); - - match &node.node_type { - NodeType::Symlink { .. } => { - let linktarget = node.node_type.to_link(); - symlink(linktarget, &filename).map_err(|err| LocalErrorKind::SymlinkingFailed { - linktarget: linktarget.to_path_buf(), - filename, - source: err, - })?; - } - NodeType::Dev { device } => { - #[cfg(not(any( - target_os = "macos", - target_os = "openbsd", - target_os = "freebsd" - )))] - let device = *device; - #[cfg(any(target_os = "macos", target_os = "openbsd"))] - let device = i32::try_from(*device).map_err(LocalErrorKind::FromTryIntError)?; - #[cfg(target_os = "freebsd")] - let device = u32::try_from(*device).map_err(LocalErrorKind::FromTryIntError)?; - mknod(&filename, SFlag::S_IFBLK, Mode::empty(), device) - .map_err(LocalErrorKind::FromErrnoError)?; - } - NodeType::Chardev { device } => { - #[cfg(not(any( - target_os = "macos", - target_os = "openbsd", - target_os = "freebsd" - )))] - let device = *device; - #[cfg(any(target_os = "macos", target_os = "openbsd"))] - let device = i32::try_from(*device).map_err(LocalErrorKind::FromTryIntError)?; - #[cfg(target_os = "freebsd")] - let device = u32::try_from(*device).map_err(LocalErrorKind::FromTryIntError)?; - mknod(&filename, SFlag::S_IFCHR, Mode::empty(), device) - .map_err(LocalErrorKind::FromErrnoError)?; - } - NodeType::Fifo => { - mknod(&filename, SFlag::S_IFIFO, Mode::empty(), 0) - .map_err(LocalErrorKind::FromErrnoError)?; - } - NodeType::Socket => { - mknod(&filename, SFlag::S_IFSOCK, Mode::empty(), 0) - .map_err(LocalErrorKind::FromErrnoError)?; - } - _ => {} - } - Ok(()) - } - - /// Read the given item (relative to the base path) - /// - /// # Arguments - /// - /// * `item` - The item to read - /// * `offset` - The offset to read from - /// * `length` - The length to read - /// - /// # Errors - /// - /// * [`LocalErrorKind::OpeningFileFailed`] - If the file could not be opened. - /// * [`LocalErrorKind::CouldNotSeekToPositionInFile`] - If the file could not be seeked to the given position. - /// * [`LocalErrorKind::FromTryIntError`] - If the length of the file could not be converted to u32. - /// * [`LocalErrorKind::ReadingExactLengthOfFileFailed`] - If the length of the file could not be read. - /// - /// [`LocalErrorKind::OpeningFileFailed`]: crate::error::LocalErrorKind::OpeningFileFailed - /// [`LocalErrorKind::CouldNotSeekToPositionInFile`]: crate::error::LocalErrorKind::CouldNotSeekToPositionInFile - /// [`LocalErrorKind::FromTryIntError`]: crate::error::LocalErrorKind::FromTryIntError - /// [`LocalErrorKind::ReadingExactLengthOfFileFailed`]: crate::error::LocalErrorKind::ReadingExactLengthOfFileFailed - pub fn read_at(&self, item: impl AsRef, offset: u64, length: u64) -> RusticResult { - let filename = self.path(item); - let mut file = File::open(filename).map_err(LocalErrorKind::OpeningFileFailed)?; - _ = file - .seek(SeekFrom::Start(offset)) - .map_err(LocalErrorKind::CouldNotSeekToPositionInFile)?; - let mut vec = vec![0; length.try_into().map_err(LocalErrorKind::FromTryIntError)?]; - file.read_exact(&mut vec) - .map_err(LocalErrorKind::ReadingExactLengthOfFileFailed)?; - Ok(vec.into()) - } - - /// Check if a matching file exists. - /// - /// # Arguments - /// - /// * `item` - The item to check - /// * `size` - The size to check - /// - /// # Returns - /// - /// If a file exists and size matches, this returns a `File` open for reading. - /// In all other cases, returns `None` - pub fn get_matching_file(&self, item: impl AsRef, size: u64) -> Option { - let filename = self.path(item); - fs::symlink_metadata(&filename).map_or_else( - |_| None, - |meta| { - if meta.is_file() && meta.len() == size { - File::open(&filename).ok() - } else { - None - } - }, - ) - } - - /// Write `data` to given item (relative to the base path) at `offset` - /// - /// # Arguments - /// - /// * `item` - The item to write to - /// * `offset` - The offset to write at - /// * `data` - The data to write - /// - /// # Errors - /// - /// * [`LocalErrorKind::OpeningFileFailed`] - If the file could not be opened. - /// * [`LocalErrorKind::CouldNotSeekToPositionInFile`] - If the file could not be seeked to the given position. - /// * [`LocalErrorKind::CouldNotWriteToBuffer`] - If the bytes could not be written to the file. - /// - /// # Notes - /// - /// This will create the file if it doesn't exist. - /// - /// [`LocalErrorKind::OpeningFileFailed`]: crate::error::LocalErrorKind::OpeningFileFailed - /// [`LocalErrorKind::CouldNotSeekToPositionInFile`]: crate::error::LocalErrorKind::CouldNotSeekToPositionInFile - /// [`LocalErrorKind::CouldNotWriteToBuffer`]: crate::error::LocalErrorKind::CouldNotWriteToBuffer - pub fn write_at(&self, item: impl AsRef, offset: u64, data: &[u8]) -> RusticResult<()> { - let filename = self.path(item); - let mut file = fs::OpenOptions::new() - .create(true) - .write(true) - .open(filename) - .map_err(LocalErrorKind::OpeningFileFailed)?; - _ = file - .seek(SeekFrom::Start(offset)) - .map_err(LocalErrorKind::CouldNotSeekToPositionInFile)?; - file.write_all(data) - .map_err(LocalErrorKind::CouldNotWriteToBuffer)?; - Ok(()) - } -}