From 56419f299d971927abbd75cbd3e9453796b18779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Hlusi=C4=8Dka?= Date: Mon, 5 Aug 2024 00:50:31 +0200 Subject: [PATCH] Add `registry.toml` to the source directory template assets --- Cargo.lock | 18 +-- Cargo.toml | 4 +- .../source-directory-template/registry.toml | 29 ++++ src/assets.rs | 41 +++++- src/owned/registry.rs | 127 ++++++++++-------- tests/test.rs | 72 +++++++++- 6 files changed, 220 insertions(+), 71 deletions(-) create mode 100644 assets/source-directory-template/registry.toml diff --git a/Cargo.lock b/Cargo.lock index 5cef944..94ec1bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1437,6 +1437,7 @@ dependencies = [ "ed25519-dalek", "futures", "include_dir", + "itertools", "rrr", "serde", "serde_bytes", @@ -1445,6 +1446,7 @@ dependencies = [ "thiserror", "tokio", "toml", + "toml_edit", "tracing", "tracing-error", "tracing-subscriber", @@ -1551,9 +1553,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" dependencies = [ "serde", ] @@ -1861,18 +1863,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.15" +version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" +checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ "indexmap 2.2.6", "serde", @@ -2260,9 +2262,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.13" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 0b0bb5d..ab044f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,13 +24,15 @@ derive_more = "0.99.17" ed25519-dalek = { version = "2.1.1", features = ["rand_core", "pem"] } futures = "0.3.30" include_dir = { version = "0.7.4", features = ["nightly"] } -rrr = { git = "https://github.com/recursive-record-registry/rrr.git", rev = "8c1a37007cbded8c144b6ff566656b104a7d0add" } +itertools = "0.13.0" +rrr = { git = "https://github.com/recursive-record-registry/rrr.git", rev = "c5258b43eb1d98a0a8b676d86d6f93b21fb489e2" } serde = { version = "1.0.203", features = ["derive"] } serde_bytes = "0.11.14" serde_with = "3.8.1" thiserror = "1.0.62" tokio = { version = "1.37", features = ["full"] } toml = { version = "0.8.14", features = ["preserve_order"] } +toml_edit = { version = "0.22.20", features = ["serde"] } tracing = "0.1.40" # Dependencies of the executable binary clap = { version = "4.5", features = ["derive"], optional = true } diff --git a/assets/source-directory-template/registry.toml b/assets/source-directory-template/registry.toml new file mode 100644 index 0000000..764b117 --- /dev/null +++ b/assets/source-directory-template/registry.toml @@ -0,0 +1,29 @@ +root_record_path = "root" +staging_directory_path = "target/staging" +revisions_directory_path = "target/revisions" +published_directory_path = "target/published" +signing_key_paths = ['keys/key_ed25519.pem'] + +[hash] +output_length_in_bytes = 32 + +[hash.algorithm.argon2] +variant = "argon2id" +m_cost = 1024 +t_cost = 1024 +p_cost = 1 + +[kdf] +file_name_length_in_bytes = 8 +file_tag_length_in_bytes = 32 +succession_nonce_length_in_bytes = 32 +root_predecessor_nonce = "PLACEHOLDER" + +[kdf.algorithm.hkdf] +prf = "sha256" + +[default_record_parameters.splitting_strategy.fill] + +[default_record_parameters.encryption] +algorithm = "Aes256Gcm" +segment_padding_to_bytes = 1024 diff --git a/src/assets.rs b/src/assets.rs index 462c621..ead9069 100644 --- a/src/assets.rs +++ b/src/assets.rs @@ -1,4 +1,43 @@ -use include_dir::{include_dir, Dir}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; + +use futures::{future::BoxFuture, FutureExt}; +use include_dir::{include_dir, Dir, DirEntry}; +use rrr::utils::fd_lock::{FileLock, WriteLock}; +use tokio::io::AsyncWriteExt; pub const SOURCE_DIRECTORY_TEMPLATE: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/assets/source-directory-template"); + +pub(crate) fn extract_with_locks<'a>( + dir: &'a Dir<'_>, + base_path: impl AsRef + Send + Sync + 'a, + lock_map: &'a mut HashMap, +) -> BoxFuture<'a, std::io::Result<()>> { + async move { + let base_path = base_path.as_ref(); + + for entry in dir.entries() { + let path = base_path.join(entry.path()); + + match entry { + DirEntry::Dir(d) => { + tokio::fs::create_dir_all(&path).await?; + extract_with_locks(d, base_path, lock_map).await?; + } + DirEntry::File(f) => { + if let Some(lock) = lock_map.get_mut(entry.path()) { + lock.file_mut().write_all(f.contents()).await?; + } else { + tokio::fs::write(path, f.contents()).await?; + } + } + } + } + + Ok(()) + } + .boxed() +} diff --git a/src/owned/registry.rs b/src/owned/registry.rs index 73857aa..51bb461 100644 --- a/src/owned/registry.rs +++ b/src/owned/registry.rs @@ -1,33 +1,33 @@ use aes_gcm::aead::OsRng; use color_eyre::Result; use ed25519_dalek::pkcs8::{spki::der::pem::LineEnding, DecodePrivateKey, EncodePrivateKey}; -use rrr::crypto::kdf::hkdf::HkdfParams; -use rrr::crypto::kdf::KdfAlgorithm; -use rrr::crypto::password_hash::{argon2::Argon2Params, PasswordHashAlgorithm}; +use itertools::Itertools; use rrr::crypto::signature::{SigningKey, SigningKeyEd25519}; use rrr::registry::{RegistryConfig, RegistryConfigHash, RegistryConfigKdf}; use rrr::utils::fd_lock::{FileLock, FileLockType, ReadLock, WriteLock}; use rrr::utils::serde::Secret; -use rrr::{crypto::encryption::EncryptionAlgorithm, record::RecordKey}; +use rrr::record::RecordKey; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::io::SeekFrom; use std::{ fmt::Debug, ops::{Deref, DerefMut}, path::{Path, PathBuf}, }; use tokio::fs::OpenOptions; +use tokio::io::AsyncSeekExt; use tokio::{ fs::File, io::{AsyncReadExt, AsyncWriteExt}, }; +use toml_edit::DocumentMut; use crate::assets; use crate::error::Error; -use crate::record::{ - OwnedRecordConfigEncryption, OwnedRecordConfigParameters, OwnedRecordConfigParametersUnresolved, -}; +use crate::record::OwnedRecordConfigParametersUnresolved; -use super::record::{OwnedRecord, SplittingStrategy}; +use super::record::OwnedRecord; /// Represents a registry with cryptographic credentials for editing. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -68,6 +68,8 @@ pub struct OwnedRegistry { } impl OwnedRegistry { + const FILE_NAME_CONFIG: &str = "registry.toml"; + pub async fn load(directory_path: impl Into) -> Result { let directory_path = directory_path.into(); let config_path = Self::get_config_path_from_registry_directory_path(&directory_path); @@ -117,6 +119,7 @@ impl OwnedRegistry { pub async fn save_config(&mut self) -> Result<()> { let config_string = toml::to_string_pretty(&self.config)?; + self.file_lock.file_mut().seek(SeekFrom::Start(0)).await?; self.file_lock .file_mut() .write_all(config_string.as_bytes()) @@ -126,7 +129,7 @@ impl OwnedRegistry { } fn get_config_path_from_registry_directory_path(directory_path: impl AsRef) -> PathBuf { - directory_path.as_ref().join("registry.toml") + directory_path.as_ref().join(Self::FILE_NAME_CONFIG) } fn get_config_path(&self) -> PathBuf { @@ -221,13 +224,11 @@ impl OwnedRegistry { Err(error) => return Err(error.into()), } - let signing_keys_directory_relative = PathBuf::from("keys"); - let signing_keys_directory_absolute = directory_path.join(&signing_keys_directory_relative); let config_path = Self::get_config_path_from_registry_directory_path(&directory_path); // TODO: Unify with save_config tokio::fs::create_dir_all(&directory_path).await?; - let file_lock = { + let mut file_lock = { let open_options = { let mut open_options = OpenOptions::new(); open_options.read(true); @@ -239,66 +240,74 @@ impl OwnedRegistry { }; WriteLock::lock(&config_path, &open_options).await? }; - tokio::task::spawn_blocking({ - let directory_path = directory_path.clone(); - move || assets::SOURCE_DIRECTORY_TEMPLATE.extract(directory_path) - }) - .await??; - tokio::fs::create_dir(&signing_keys_directory_absolute).await?; + + { + let mut lock_map = HashMap::new(); + lock_map.insert(PathBuf::from(Self::FILE_NAME_CONFIG), &mut file_lock); + assets::extract_with_locks( + &assets::SOURCE_DIRECTORY_TEMPLATE, + &directory_path, + &mut lock_map, + ) + .await?; + } let mut csprng = OsRng; - let signing_keys = vec![SigningKey::Ed25519(Secret(SigningKeyEd25519( - ed25519_dalek::SigningKey::generate(&mut csprng), - )))]; - let signing_key_paths = { - let mut signing_key_paths = Vec::new(); - - for signing_key in &signing_keys { - let signing_key_path_relative = signing_keys_directory_relative - .join(format!("key_{}.pem", signing_key.key_type_name())); - let signing_key_path_absolute = directory_path.join(&signing_key_path_relative); - let pem = signing_key.to_pkcs8_pem(LineEnding::default()).unwrap(); - let mut file = File::create_new(&signing_key_path_absolute).await?; - - file.write_all(pem.as_bytes()).await?; - signing_key_paths.push(signing_key_path_relative); - } - signing_key_paths + // Patch registry config. + let config = { + let mut config_string = String::new(); + file_lock.file_mut().seek(SeekFrom::Start(0)).await?; + file_lock + .file_mut() + .read_to_string(&mut config_string) + .await?; + let mut config_doc = config_string.parse::()?; + let root_predecessor_nonce = + RegistryConfigKdf::generate_random_root_predecessor_nonce(csprng, None); + let root_predecessor_nonce_string = + format!("{:02x}", root_predecessor_nonce.iter().format("")); + config_doc["kdf"]["root_predecessor_nonce"] = + toml_edit::value(root_predecessor_nonce_string); + config_string = config_doc.to_string(); + + // Parse patched config + let config = toml::from_str::(&config_string)?; + + // Store patched config + file_lock.file_mut().seek(SeekFrom::Start(0)).await?; + file_lock + .file_mut() + .write_all(config_string.as_bytes()) + .await?; + + config }; - let config = OwnedRegistryConfig { - hash: RegistryConfigHash { - algorithm: PasswordHashAlgorithm::Argon2(Argon2Params::default()), - output_length_in_bytes: Default::default(), - }, - kdf: RegistryConfigKdf::builder() - .with_algorithm(KdfAlgorithm::Hkdf(HkdfParams::default())) - .build_with_random_root_predecessor_nonce(csprng)?, - default_record_parameters: OwnedRecordConfigParameters { - splitting_strategy: SplittingStrategy::Fill {}, - encryption: Some(OwnedRecordConfigEncryption { - algorithm: EncryptionAlgorithm::Aes256Gcm, - segment_padding_to_bytes: 1024, // 1 KiB - }), - } - .into(), - staging_directory_path: PathBuf::from("target/staging"), - revisions_directory_path: PathBuf::from("target/revisions"), - published_directory_path: PathBuf::from("target/published"), - root_record_path: PathBuf::from("root"), - signing_key_paths, + // Generate signing keys. + let signing_keys = { + let signing_key = SigningKey::Ed25519(Secret(SigningKeyEd25519( + ed25519_dalek::SigningKey::generate(&mut csprng), + ))); + let signing_keys_directory_relative = PathBuf::from("keys"); + let signing_key_path_relative = signing_keys_directory_relative + .join(format!("key_{}.pem", signing_key.key_type_name())); + let signing_key_path_absolute = directory_path.join(&signing_key_path_relative); + let pem = signing_key.to_pkcs8_pem(LineEnding::default()).unwrap(); + let mut file = File::create_new(&signing_key_path_absolute).await?; + + file.write_all(pem.as_bytes()).await?; + + vec![signing_key] }; - let mut registry = Self { + let registry = Self { directory_path, config, signing_keys, file_lock, }; - registry.save_config().await?; - Ok(registry) } diff --git a/tests/test.rs b/tests/test.rs index 50b25d5..ce91538 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -1,4 +1,17 @@ -use rrr_make::registry::OwnedRegistry; +use std::path::PathBuf; + +use rrr::{ + crypto::{ + encryption::EncryptionAlgorithm, + kdf::{hkdf::HkdfParams, KdfAlgorithm}, + password_hash::{argon2::Argon2Params, PasswordHashAlgorithm}, + }, + registry::{RegistryConfigHash, RegistryConfigKdf}, +}; +use rrr_make::{ + record::{OwnedRecordConfigEncryption, OwnedRecordConfigParameters, SplittingStrategy}, + registry::{OwnedRegistry, OwnedRegistryConfig}, +}; use tempfile::tempdir; use tracing_test::traced_test; @@ -27,10 +40,65 @@ async fn owned_registry() { .is_empty()); } +#[tokio::test] +#[traced_test] +async fn new_registry_config() { + let registry_dir = tempdir().unwrap(); + + dbg!(®istry_dir); + + let generated_registry = OwnedRegistry::generate(registry_dir.path(), false) + .await + .unwrap() + .lock_read() + .await + .unwrap(); + let generated_config = &generated_registry.config; + let expected_config = OwnedRegistryConfig { + hash: RegistryConfigHash { + algorithm: PasswordHashAlgorithm::Argon2(Argon2Params::default()), + output_length_in_bytes: Default::default(), + }, + kdf: RegistryConfigKdf::builder() + .with_algorithm(KdfAlgorithm::Hkdf(HkdfParams::default())) + .build( + generated_config + .kdf + .get_root_record_predecessor_nonce() + .clone(), + ) + .unwrap(), + default_record_parameters: OwnedRecordConfigParameters { + splitting_strategy: SplittingStrategy::Fill {}, + encryption: Some(OwnedRecordConfigEncryption { + algorithm: EncryptionAlgorithm::Aes256Gcm, + segment_padding_to_bytes: 1024, // 1 KiB + }), + } + .into(), + staging_directory_path: PathBuf::from("target/staging"), + revisions_directory_path: PathBuf::from("target/revisions"), + published_directory_path: PathBuf::from("target/published"), + root_record_path: PathBuf::from("root"), + signing_key_paths: vec![PathBuf::from("keys/key_ed25519.pem")], + }; + + println!( + "\ngenerated config:\n{}", + toml::to_string_pretty(generated_config).unwrap() + ); + println!( + "\nexpected config:\n{}", + toml::to_string_pretty(&expected_config).unwrap() + ); + + assert_eq!(generated_config, &expected_config); +} + #[cfg(feature = "cmd")] #[tokio::test] #[traced_test] -async fn new_registry() { +async fn commands_new_make() { use rrr_make::cmd::Command; let registry_dir = tempdir().unwrap();