diff --git a/Cargo.lock b/Cargo.lock index b2fa37e..5cef944 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1417,6 +1417,7 @@ dependencies = [ "serde", "serde_with", "sha2", + "test-strategy", "thiserror", "tokio", "tokio-util", @@ -1432,6 +1433,7 @@ dependencies = [ "chrono", "clap", "color-eyre", + "derive_more", "ed25519-dalek", "futures", "include_dir", @@ -1665,6 +1667,29 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.71", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.71", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1705,6 +1730,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "test-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf41af45e3f54cc184831d629d41d5b2bda8297e29c81add7ae4f362ed5e01b" +dependencies = [ + "proc-macro2", + "quote", + "structmeta", + "syn 2.0.71", +] + [[package]] name = "thiserror" version = "1.0.62" diff --git a/Cargo.toml b/Cargo.toml index 5545a6b..c2cbf20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,10 +20,11 @@ cmd = ["clap", "tracing-error", "tracing-subscriber"] aes-gcm = { version = "0.10.3", features = ["std"] } chrono = { version = "0.4", features = ["arbitrary"] } color-eyre = "0.6.3" +derive_more = "0.99.17" ed25519-dalek = { version = "2.1.1", features = ["rand_core", "pem"] } futures = "0.3.30" include_dir = { version = "0.7.4", features = ["nightly"] } -rrr = { git = "https://github.com/recursive-record-registry/rrr.git", rev = "782db4752ee74fc56199151e5b261f33b32cfbfa" } +rrr = { git = "https://github.com/recursive-record-registry/rrr.git", rev = "826ae06afaf769a64bdb215ba507e69f78fc1a0e" } serde = { version = "1.0.203", features = ["derive"] } serde_bytes = "0.11.14" serde_with = "3.8.1" diff --git a/README.md b/README.md index d230f1f..6c53444 100644 --- a/README.md +++ b/README.md @@ -16,3 +16,10 @@ Launch it by running the following: # On Unix target/release/rrr-make ``` + +## TODO +* [ ] Better error reporting for incomplete record parameters +* [ ] Output registry staging + * [ ] Checking whether the stored record is identical to the to-be-written one + * [ ] `published` directory + * [ ] `revisions` directory diff --git a/src/cmd/mod.rs b/src/cmd/mod.rs index 465a98a..f6a2237 100644 --- a/src/cmd/mod.rs +++ b/src/cmd/mod.rs @@ -21,12 +21,12 @@ pub enum Command { /// Path to a source directory. #[arg(short, long, default_value = ".")] input_directory: PathBuf, - /// Path to a directory in which to put the RRR registry. - #[arg(short, long, default_value = "target")] - output_directory: PathBuf, /// Force existing files to be overwritten. #[arg(short, long, default_value = "false")] force: bool, + /// Whether a new revision should be created in the published directory. + #[arg(long, default_value = "false")] + publish: bool, }, } @@ -39,13 +39,13 @@ impl Command { } Command::Make { input_directory, - output_directory, force, + publish, } => { let input_registry = OwnedRegistry::load(input_directory).await?; let input_root_record = input_registry.load_root_record().await?; let mut output_registry = Registry::create( - output_directory, + input_registry.get_staging_directory_path(), RegistryConfig::from(&input_registry), force, ) diff --git a/src/lib.rs b/src/lib.rs index d3be86e..f4191ad 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,8 +3,9 @@ use futures::{future::BoxFuture, FutureExt}; use record::OwnedRecord; use registry::OwnedRegistry; use rrr::{ - crypto::encryption::EncryptionAlgorithm, - record::{Record, RecordKey, RecordMetadata, RecordName, SuccessionNonce}, + record::{ + segment::SegmentEncryption, Record, RecordKey, RecordMetadata, RecordName, SuccessionNonce, + }, registry::{Registry, WriteLock}, utils::serde::BytesOrAscii, }; @@ -13,6 +14,7 @@ use tokio::io::AsyncReadExt; pub mod assets; pub mod error; pub mod owned; +pub mod util; #[cfg(feature = "cmd")] pub mod cmd; @@ -61,10 +63,16 @@ pub fn make_recursive<'a>( &input_registry.signing_keys, &hashed_key, &output_record, - 0.into(), // TODO - 0, // TODO - &[], // TODO - Some(&EncryptionAlgorithm::Aes256Gcm), // TODO + 0.into(), // TODO + 0, // TODO + &[], // TODO + input_record + .config + .parameters + .encryption + .as_ref() + .map(SegmentEncryption::from) + .as_ref(), force, ) .await?; diff --git a/src/owned/record.rs b/src/owned/record.rs index 5f768c1..2201369 100644 --- a/src/owned/record.rs +++ b/src/owned/record.rs @@ -1,9 +1,10 @@ use chrono::{DateTime, Utc}; use color_eyre::{ - eyre::{bail, OptionExt}, + eyre::{bail, eyre, OptionExt}, Result, }; use futures::future::{BoxFuture, FutureExt}; +use rrr::{crypto::encryption::EncryptionAlgorithm, record::segment::SegmentEncryption}; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; use std::{ @@ -15,17 +16,175 @@ use std::{ }; use tokio::io::{AsyncRead, AsyncWriteExt}; -use crate::error::Error; +use crate::{error::Error, registry::OwnedRegistryConfig, util::serde::DoubleOption}; -#[derive(Debug, Serialize, Deserialize)] +pub trait Unresolved: Sized + Default + From { + type Resolved: Sized; + + fn or(self, fallback: Self) -> Self; + fn resolve(self) -> Result; +} + +#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct OwnedRecordConfigEncryptionUnresolved { + pub algorithm: Option, + pub segment_padding_to_bytes: Option, +} + +impl Unresolved for OwnedRecordConfigEncryptionUnresolved { + type Resolved = OwnedRecordConfigEncryption; + + fn or(self, fallback: Self) -> Self { + Self { + algorithm: self.algorithm.or(fallback.algorithm), + segment_padding_to_bytes: self + .segment_padding_to_bytes + .or(fallback.segment_padding_to_bytes), + } + } + + fn resolve(self) -> Result { + if let Self { + algorithm: Some(algorithm), + segment_padding_to_bytes: Some(segment_padding_to_bytes), + } = self + { + Ok(Self::Resolved { + algorithm, + segment_padding_to_bytes, + }) + } else { + Err(self) + } + } +} + +impl From for OwnedRecordConfigEncryptionUnresolved { + fn from(value: OwnedRecordConfigEncryption) -> Self { + Self { + algorithm: Some(value.algorithm), + segment_padding_to_bytes: Some(value.segment_padding_to_bytes), + } + } +} + +#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct OwnedRecordConfigParametersUnresolved { + pub encryption: DoubleOption, +} + +impl Unresolved for OwnedRecordConfigParametersUnresolved { + type Resolved = OwnedRecordConfigParameters; + + fn or(self, fallback: Self) -> Self { + Self { + encryption: self.encryption.or(fallback.encryption), + } + } + + fn resolve(self) -> Result { + if let Self { + encryption: Some(encryption), + } = self + { + match Option::from(encryption) + .map(Unresolved::resolve) + .transpose() + { + Ok(resolved) => Ok(Self::Resolved { + encryption: resolved, + }), + Err(unresolved) => Err(Self { + encryption: Some(Some(unresolved).into()), + }), + } + } else { + Err(self) + } + } +} + +impl From for OwnedRecordConfigParametersUnresolved { + fn from(value: OwnedRecordConfigParameters) -> Self { + Self { + encryption: Some( + value + .encryption + .map(OwnedRecordConfigEncryptionUnresolved::from) + .into(), + ), + } + } +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct OwnedRecordConfigUnresolved { + pub name: ByteBuf, + pub metadata: OwnedRecordMetadata, + #[serde(flatten)] + pub parameters: OwnedRecordConfigParametersUnresolved, +} + +impl OwnedRecordConfigUnresolved { + pub fn try_resolve_with( + self, + parameters: OwnedRecordConfigParametersUnresolved, + ) -> Result { + match self.parameters.or(parameters).resolve() { + Ok(resolved) => Ok(OwnedRecordConfig { + name: self.name, + metadata: self.metadata, + parameters: resolved, + }), + Err(unresolved) => Err(Self { + name: self.name, + metadata: self.metadata, + parameters: unresolved, + }), + } + } +} + +impl From for OwnedRecordConfigUnresolved { + fn from(value: OwnedRecordConfig) -> Self { + Self { + name: value.name, + metadata: value.metadata, + parameters: value.parameters.into(), + } + } +} + +impl From<&OwnedRecordConfigEncryption> for SegmentEncryption { + fn from(value: &OwnedRecordConfigEncryption) -> Self { + Self { + algorithm: value.algorithm, + padding_to_bytes: value.segment_padding_to_bytes, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct OwnedRecordMetadata { pub created_at: Option, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct OwnedRecordConfigEncryption { + pub algorithm: EncryptionAlgorithm, + pub segment_padding_to_bytes: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct OwnedRecordConfigParameters { + pub encryption: Option, +} + +#[derive(Clone, Debug)] pub struct OwnedRecordConfig { pub name: ByteBuf, pub metadata: OwnedRecordMetadata, + pub parameters: OwnedRecordConfigParameters, } #[derive(Debug)] @@ -37,10 +196,15 @@ pub struct OwnedRecord { impl OwnedRecord { pub fn load_from_directory<'a>( + registry_config: &'a OwnedRegistryConfig, directory_path: impl AsRef + Send + Sync + 'a, ) -> BoxFuture<'a, Result> { async move { - let config = Self::load_config(&directory_path).await?; + let config_unresolved = Self::load_config(&directory_path).await?; + let config = config_unresolved + .try_resolve_with(registry_config.default_record_parameters.clone() /* TODO: cloning seems excessive */) + .map_err(|_| eyre!("incomplete record parameters"))?; + dbg!(&config); let mut successive_records_stream = tokio::fs::read_dir(&directory_path).await?; let mut successive_records = Vec::new(); let mut successive_record_names = HashSet::new(); @@ -48,8 +212,11 @@ impl OwnedRecord { while let Some(entry) = successive_records_stream.next_entry().await? { if entry.metadata().await?.is_dir() { let successive_record_directory = entry.path(); - let successive_record = - OwnedRecord::load_from_directory(&successive_record_directory).await?; + let successive_record = OwnedRecord::load_from_directory( + registry_config, + &successive_record_directory, + ) + .await?; let successive_record_name_unique = successive_record_names.insert(successive_record.config.name.clone()); @@ -77,7 +244,8 @@ impl OwnedRecord { pub async fn save(&self) -> Result<()> { tokio::fs::create_dir_all(&self.directory_path).await?; - let config_string = toml::to_string_pretty(&self.config)?; + let config_string = + toml::to_string_pretty(&OwnedRecordConfigUnresolved::from(self.config.clone()))?; let mut config_file = tokio::fs::OpenOptions::new() .create_new(true) .write(true) @@ -89,14 +257,16 @@ impl OwnedRecord { Ok(()) } - pub async fn load_config(directory_path: impl AsRef) -> Result { + pub async fn load_config( + directory_path: impl AsRef, + ) -> Result { match tokio::fs::read_to_string(Self::get_config_path_from_record_directory_path( &directory_path, )) .await { Ok(config_string) => { - toml::from_str::(&config_string).map_err(Into::into) + toml::from_str::(&config_string).map_err(Into::into) } Err(error) if error.kind() == std::io::ErrorKind::NotFound => { let file_name = directory_path.as_ref().file_name().ok_or_else(|| { @@ -118,11 +288,12 @@ impl OwnedRecord { let created_at_chrono = DateTime::::from(created_at_system); let created_at = toml::value::Datetime::from_str(&created_at_chrono.to_rfc3339()).unwrap(); - Ok(OwnedRecordConfig { + Ok(OwnedRecordConfigUnresolved { name: ByteBuf::from(file_name_utf8.as_bytes()), metadata: OwnedRecordMetadata { created_at: Some(created_at), }, + parameters: Default::default(), }) } Err(error) => Err(error.into()), diff --git a/src/owned/registry.rs b/src/owned/registry.rs index 330c761..04de5b5 100644 --- a/src/owned/registry.rs +++ b/src/owned/registry.rs @@ -5,13 +5,10 @@ use rrr::crypto::kdf::hkdf::HkdfParams; use rrr::crypto::kdf::KdfAlgorithm; use rrr::crypto::password_hash::{argon2::Argon2Params, PasswordHashAlgorithm}; use rrr::crypto::signature::{SigningKey, SigningKeyEd25519}; -use rrr::registry::{ - RegistryConfig, RegistryConfigHash, RegistryConfigKdf, -}; +use rrr::registry::{RegistryConfig, RegistryConfigHash, RegistryConfigKdf}; use rrr::utils::serde::Secret; use rrr::{crypto::encryption::EncryptionAlgorithm, record::RecordKey}; use serde::{Deserialize, Serialize}; -use serde_with::serde_as; use std::{ fmt::Debug, ops::{Deref, DerefMut}, @@ -24,17 +21,28 @@ use tokio::{ use crate::assets; use crate::error::Error; +use crate::record::{ + OwnedRecordConfigEncryption, OwnedRecordConfigParameters, + OwnedRecordConfigParametersUnresolved, +}; use super::record::OwnedRecord; /// Represents a registry with cryptographic credentials for editing. -#[serde_as] #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct OwnedRegistryConfig { pub hash: RegistryConfigHash, pub kdf: RegistryConfigKdf, - pub encryption_algorithm: EncryptionAlgorithm, + pub default_record_parameters: OwnedRecordConfigParametersUnresolved, pub root_record_path: PathBuf, + /// This is where the resulting registry is generated, every time the `make` subcommand is executed. + pub staging_directory_path: PathBuf, + /// This directory contains all of the published record fragments, separated to directories according + /// to the revision they were published in. + pub revisions_directory_path: PathBuf, + /// Path to a directory where the accumulation of all published revisions is stored. + /// This directory contains all the published data of the registry, and can be browsed. + pub published_directory_path: PathBuf, /// Paths to files with signing keys. /// These paths are relative to the directory containing the registry config. pub signing_key_paths: Vec, @@ -165,7 +173,16 @@ impl OwnedRegistry { kdf: RegistryConfigKdf::builder() .with_algorithm(KdfAlgorithm::Hkdf(HkdfParams::default())) .build_with_random_root_predecessor_nonce(csprng)?, - encryption_algorithm: EncryptionAlgorithm::Aes256Gcm, + default_record_parameters: OwnedRecordConfigParameters { + encryption: Some(OwnedRecordConfigEncryption { + algorithm: EncryptionAlgorithm::Aes256Gcm, + segment_padding_to_bytes: 1024, // 1 KiB + }), + } + .into(), + staging_directory_path: PathBuf::from("target/staging"), + revisions_directory_path: PathBuf::from("target/revisions"), + published_directory_path: PathBuf::from("target/published"), root_record_path: PathBuf::from("root"), signing_key_paths, }; @@ -217,12 +234,24 @@ impl OwnedRegistry { Self::get_key_path_from_record_directory_path(&self.directory_path, key_path) } + pub fn get_staging_directory_path(&self) -> PathBuf { + self.directory_path.join(&self.staging_directory_path) + } + + pub fn get_revisions_directory_path(&self) -> PathBuf { + self.directory_path.join(&self.revisions_directory_path) + } + + pub fn get_published_directory_path(&self) -> PathBuf { + self.directory_path.join(&self.published_directory_path) + } + fn get_root_record_path(&self) -> PathBuf { self.directory_path.join(&self.root_record_path) } pub async fn load_root_record(&self) -> Result { - OwnedRecord::load_from_directory(self.get_root_record_path()).await + OwnedRecord::load_from_directory(&self.config, self.get_root_record_path()).await } } diff --git a/src/util/mod.rs b/src/util/mod.rs new file mode 100644 index 0000000..82b2d09 --- /dev/null +++ b/src/util/mod.rs @@ -0,0 +1 @@ +pub mod serde; diff --git a/src/util/serde.rs b/src/util/serde.rs new file mode 100644 index 0000000..2631659 --- /dev/null +++ b/src/util/serde.rs @@ -0,0 +1,110 @@ + +use ::serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum ExplicitOption { + None(ExplicitNone), + Some(T), +} + +impl From> for ExplicitOption { + fn from(value: Option) -> Self { + match value { + None => Self::default(), + Some(inner) => Self::Some(inner), + } + } +} + +impl From> for Option { + fn from(value: ExplicitOption) -> Self { + match value { + ExplicitOption::None(_) => None, + ExplicitOption::Some(inner) => Some(inner), + } + } +} + +impl Default for ExplicitOption { + fn default() -> Self { + Self::None(ExplicitNone::new()) + } +} + +impl Clone for ExplicitOption +where + T: Clone, +{ + fn clone(&self) -> Self { + match self { + Self::None(_) => Self::default(), + Self::Some(inner) => Self::Some(inner.clone()), + } + } +} + +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct ExplicitNone { + none: bool, +} + +impl ExplicitNone { + fn new() -> Self { + Self { none: true } + } +} + +/// Used to disambiguate between an unspecified field and a `null` field. +pub type DoubleOption = Option>; + +#[cfg(test)] +mod tests { + use std::fmt::Debug; + + use serde::{Deserialize, Serialize}; + + use crate::util::serde::ExplicitNone; + + use super::ExplicitOption; + + fn validate_serde_of Deserialize<'a>>(original: T) { + let serialized = toml::to_string(&original).unwrap(); + + dbg!(&original); + dbg!(&serialized); + + let deserialized = toml::from_str::(&serialized).unwrap(); + + assert_eq!(deserialized, original); + } + + #[test] + fn explicit_option_toml() { + validate_serde_of(ExplicitNone::new()); + assert_eq!( + &toml::to_string(&ExplicitNone::new()).unwrap(), + "none = true\n" + ); + + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] + struct Struct { + hello: String, + } + + validate_serde_of(ExplicitOption::::default()); + validate_serde_of(ExplicitOption::Some(Struct { + hello: "World".to_string(), + })); + + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] + enum Enum { + Hello(u64), + World(String), + } + + validate_serde_of(ExplicitOption::::default()); + validate_serde_of(ExplicitOption::Some(Enum::Hello(0))); + validate_serde_of(ExplicitOption::Some(Enum::World("World".to_string()))); + } +}