Adding changes needed by DDB PR (#49)

huggingface · Oct 18, 2024 · e160cba · e160cba
1 parent 7e41603
commit e160cba
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 4 deletions.
diff --git a/cas_types/src/key.rs b/cas_types/src/key.rs
@@ -38,7 +38,7 @@ impl FromStr for Key {
     }
 }
 
-mod hex {
+pub mod hex {
     pub mod serde {
         use merklehash::MerkleHash;
         use serde::de::{self, Visitor};

diff --git a/hf_xet/Cargo.lock b/hf_xet/Cargo.lock
diff --git a/mdb_shard/src/shard_format.rs b/mdb_shard/src/shard_format.rs
@@ -926,7 +926,7 @@ impl MDBShardInfo {
     /// Returns a list of chunk hashes for the global dedup service.
     /// The chunk hashes are either multiple of 'hash_filter_modulues',
     /// or the hash of the first chunk of a file present in the shard.
-    pub fn read_cas_chunks_for_global_dedup<R: Read + Seek>(
+    pub fn filter_cas_chunks_for_global_dedup<R: Read + Seek>(
         reader: &mut R,
     ) -> Result<Vec<MerkleHash>> {
         let mut ret = Vec::new();

diff --git a/shard_client/src/http_shard_client.rs b/shard_client/src/http_shard_client.rs
@@ -306,7 +306,7 @@ mod test {
         }
 
         // test chunk dedup lookup
-        let chunks = MDBShardInfo::read_cas_chunks_for_global_dedup(&mut reader)?;
+        let chunks = MDBShardInfo::filter_cas_chunks_for_global_dedup(&mut reader)?;
         for chunk in chunks {
             let expected = shard_hash;
             let result = client

diff --git a/shard_client/src/local_shard_client.rs b/shard_client/src/local_shard_client.rs
@@ -73,7 +73,7 @@ impl RegistrationClient for LocalShardClient {
         // Add dedup info to the global dedup table.
         let mut shard_reader = Cursor::new(shard_data);
 
-        let chunk_hashes = MDBShardInfo::read_cas_chunks_for_global_dedup(&mut shard_reader)?;
+        let chunk_hashes = MDBShardInfo::filter_cas_chunks_for_global_dedup(&mut shard_reader)?;
 
         self.global_dedup
             .batch_add(&chunk_hashes, hash, prefix, salt)