Merge pull request #2660 from b-zee/feat-versioned-archive

feat: add version field to archive and metadata

autonomi/src/client/high_level/files/archive_private.rs

@@ -36,6 +36,14 @@ pub struct PrivateArchive {
     map: BTreeMap<PathBuf, (DataMapChunk, Metadata)>,
 }
 
+/// This type essentially wraps archive in version marker. E.g. in JSON format:
+/// `{ "V0": { "map": <xxx> } }`
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum PrivateArchiveVersioned {
+    V0(PrivateArchive),
+}
+
 impl PrivateArchive {
     /// Create a new emtpy local archive
     /// Note that this does not upload the archive to the network
@@ -101,14 +109,17 @@ impl PrivateArchive {
 
     /// Deserialize from bytes.
     pub fn from_bytes(data: Bytes) -> Result<PrivateArchive, rmp_serde::decode::Error> {
-        let root: PrivateArchive = rmp_serde::from_slice(&data[..])?;
+        let root: PrivateArchiveVersioned = rmp_serde::from_slice(&data[..])?;
+        // Currently we have only `V0`. If we add `V1`, then we need an upgrade/migration path here.
+        let PrivateArchiveVersioned::V0(root) = root;
 
         Ok(root)
     }
 
     /// Serialize to bytes.
     pub fn to_bytes(&self) -> Result<Bytes, rmp_serde::encode::Error> {
-        let root_serialized = rmp_serde::to_vec(&self)?;
+        let versioned = PrivateArchiveVersioned::V0(self.clone());
+        let root_serialized = rmp_serde::to_vec_named(&versioned)?;
         let root_serialized = Bytes::from(root_serialized);
 
         Ok(root_serialized)

autonomi/src/client/high_level/files/archive_public.rs

@@ -39,6 +39,14 @@ pub struct PublicArchive {
     map: BTreeMap<PathBuf, (DataAddr, Metadata)>,
 }
 
+/// This type essentially wraps archive in version marker. E.g. in JSON format:
+/// `{ "V0": { "map": <xxx> } }`
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum PublicArchiveVersioned {
+    V0(PublicArchive),
+}
+
 impl PublicArchive {
     /// Create a new emtpy local archive
     /// Note that this does not upload the archive to the network
@@ -100,14 +108,17 @@ impl PublicArchive {
 
     /// Deserialize from bytes.
     pub fn from_bytes(data: Bytes) -> Result<PublicArchive, rmp_serde::decode::Error> {
-        let root: PublicArchive = rmp_serde::from_slice(&data[..])?;
+        let root: PublicArchiveVersioned = rmp_serde::from_slice(&data[..])?;
+        // Currently we have only `V0`. If we add `V1`, then we need an upgrade/migration path here.
+        let PublicArchiveVersioned::V0(root) = root;
 
         Ok(root)
     }
 
     /// Serialize to bytes.
     pub fn to_bytes(&self) -> Result<Bytes, rmp_serde::encode::Error> {
-        let root_serialized = rmp_serde::to_vec(&self)?;
+        let versioned = PublicArchiveVersioned::V0(self.clone());
+        let root_serialized = rmp_serde::to_vec_named(&versioned)?;
         let root_serialized = Bytes::from(root_serialized);
 
         Ok(root_serialized)
@@ -182,3 +193,93 @@ impl Client {
         result
     }
 }
+
+#[cfg(test)]
+mod test {
+    use std::str::FromStr;
+
+    use super::*;
+
+    #[test]
+    fn compatibility() {
+        // In the future we'll have an extra variant.
+        #[derive(Serialize, Deserialize)]
+        #[non_exhaustive]
+        pub enum FuturePublicArchiveVersioned {
+            V0(PublicArchive),
+            V1(PublicArchive),
+            #[serde(other)]
+            Unsupported,
+        }
+
+        let mut arch = PublicArchive::new();
+        arch.add_file(
+            PathBuf::from_str("hello_world").unwrap(),
+            DataAddr::random(&mut rand::thread_rng()),
+            Metadata::new_with_size(1),
+        );
+        let arch_serialized = arch.to_bytes().unwrap();
+
+        // Create archive, forward compatible (still the same V0 version).
+        let future_arch = FuturePublicArchiveVersioned::V0(arch.clone());
+        let future_arch_serialized = rmp_serde::to_vec_named(&future_arch).unwrap();
+
+        // Let's see if we can deserialize a (forward compatible) archive arriving to us from the future
+        let _ = PublicArchive::from_bytes(Bytes::from(future_arch_serialized)).unwrap();
+
+        // Let's see if we can deserialize an old archive from the future
+        let _: FuturePublicArchiveVersioned = rmp_serde::from_slice(&arch_serialized[..]).unwrap();
+
+        // Now we break forward compatibility by introducing a new version not supported by the old code.
+        let future_arch = FuturePublicArchiveVersioned::V1(arch.clone());
+        let future_arch_serialized = rmp_serde::to_vec_named(&future_arch).unwrap();
+        // The old archive will not be able to decode this.
+        assert!(PublicArchive::from_bytes(Bytes::from(future_arch_serialized)).is_err());
+
+        // Now we prove backwards compatibility. Our old V0 archive will still be decoded by our new archive wrapper as V0.
+        let versioned_arch = PublicArchiveVersioned::V0(arch.clone()); // 'Old' archive wrapper
+        let versioned_arch_serialized = rmp_serde::to_vec_named(&versioned_arch).unwrap();
+        let _: FuturePublicArchiveVersioned = // Into 'new' wrapper
+            rmp_serde::from_slice(&versioned_arch_serialized[..]).unwrap();
+    }
+
+    #[test]
+    fn forward_compatibility() {
+        // What we do here is we create a new `Metadata` and use that in the `Archive` structs.
+
+        /// A version `1.1` which is non-breaking (`1.0` is forward compatible with `1.1`).
+        #[derive(Debug, Default, Serialize, Deserialize)]
+        pub struct MetadataV1p1 {
+            created: u64,
+            modified: u64,
+            size: u64,
+            extra: Option<String>,
+            accessed: Option<u64>, // Added field
+        }
+        #[derive(Debug, Default, Serialize, Deserialize)]
+        pub struct PublicArchiveV1p1 {
+            map: BTreeMap<PathBuf, (DataAddr, MetadataV1p1)>,
+        }
+        #[derive(Debug, Serialize, Deserialize)]
+        pub enum PublicArchiveVersionedV1p1 {
+            V0(PublicArchiveV1p1),
+        }
+
+        let mut arch_p1 = PublicArchiveV1p1::default();
+        arch_p1.map.insert(
+            PathBuf::from_str("hello_world").unwrap(),
+            (
+                DataAddr::random(&mut rand::thread_rng()),
+                MetadataV1p1 {
+                    accessed: Some(1),
+                    ..Default::default()
+                },
+            ),
+        );
+        let arch_p1_ser =
+            rmp_serde::to_vec_named(&PublicArchiveVersionedV1p1::V0(arch_p1)).unwrap();
+
+        // Our old data structure should be forward compatible with the new one.
+        assert!(PublicArchive::from_bytes(Bytes::from(arch_p1_ser)).is_ok());
+    }
+}

autonomi/src/client/high_level/files/fs_public.rs

@@ -197,6 +197,7 @@ pub(crate) fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata {
                 created: 0,
                 modified: 0,
                 size: 0,
+                extra: None,
             };
         }
     };
@@ -226,5 +227,6 @@ pub(crate) fn metadata_from_entry(entry: &walkdir::DirEntry) -> Metadata {
         created,
         modified,
         size: fs_metadata.len(),
+        extra: None,
     }
 }

autonomi/src/client/high_level/files/mod.rs

@@ -39,6 +39,9 @@ pub struct Metadata {
     pub modified: u64,
     /// File size in bytes
     pub size: u64,
+
+    /// Optional extra metadata with undefined structure, e.g. JSON.
+    pub extra: Option<String>,
 }
 
 impl Metadata {
@@ -53,6 +56,7 @@ impl Metadata {
             created: now,
             modified: now,
             size,
+            extra: None,
         }
     }
 }