feat(autonomi): parallel chunk copy for encrypt

Testing on my local machine saved about 18s from 70s of total encrypt time with a 10GiB file in-memory.
maidsafe · Dec 6, 2024 · 1742772 · 1742772
1 parent 53cbf8d
commit 1742772
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 2 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/autonomi/Cargo.toml b/autonomi/Cargo.toml
@@ -46,6 +46,7 @@ hex = "~0.4.3"
 libp2p = { git = "https://github.com/maqi/rust-libp2p.git", branch = "kad_0.46.2" }
 pyo3 = { version = "0.20", optional = true, features = ["extension-module", "abi3-py38"] }
 rand = "0.8.5"
+rayon = "1.8.0"
 rmp-serde = "1.1.1"
 self_encryption = "~0.30.0"
 serde = { version = "1.0.133", features = ["derive", "rc"] }

diff --git a/autonomi/src/self_encryption.rs b/autonomi/src/self_encryption.rs
@@ -8,6 +8,7 @@
 
 use ant_protocol::storage::Chunk;
 use bytes::{BufMut, Bytes, BytesMut};
+use rayon::prelude::*;
 use self_encryption::{DataMap, MAX_CHUNK_SIZE};
 use serde::{Deserialize, Serialize};
 use tracing::debug;
@@ -36,8 +37,8 @@ pub(crate) fn encrypt(data: Bytes) -> Result<(Chunk, Vec<Chunk>), Error> {
 
     // Transform `EncryptedChunk` into `Chunk`
     let chunks: Vec<Chunk> = chunks
-        .into_iter()
-        .map(|c| Chunk::new(c.content))
+        .into_par_iter()
+        .map(|c| Chunk::new(c.content.clone()))
         .chain(additional_chunks)
         .collect();