Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(autonomi): restructure modules and private/public naming #2499

Merged
merged 14 commits into from
Dec 6, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor(autonomi): rename PrivateDataAccess to
DataMapChunk
b-zee committed Dec 6, 2024
commit ca218073d102c9773b5ee7ad9f8e008f654f4818
4 changes: 2 additions & 2 deletions autonomi/README_PYTHON.md
Original file line number Diff line number Diff line change
@@ -43,7 +43,7 @@ print(f"Retrieved: {retrieved.decode()}")
- `data_put_public(data: bytes, payment: PaymentOption)`: Upload data
- `data_get_public(addr: str)`: Download data
- `data_put(data: bytes, payment: PaymentOption)`: Store private data
- `data_get(access: PrivateDataAccess)`: Retrieve private data
- `data_get(access: DataMapChunk)`: Retrieve private data
- `register_generate_key()`: Generate register key

- `Wallet`: Ethereum wallet management
@@ -56,7 +56,7 @@ print(f"Retrieved: {retrieved.decode()}")

#### Private Data

- `PrivateDataAccess`: Handle private data storage
- `DataMapChunk`: Handle private data storage
- `from_hex(hex: str)`: Create from hex string
- `to_hex()`: Convert to hex string
- `address()`: Get short reference address
2 changes: 1 addition & 1 deletion autonomi/examples/autonomi_private_encryption.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from autonomi_client import (
Client, Wallet, PaymentOption, PrivateDataAccess,
Client, Wallet, PaymentOption, DataMapChunk,
encrypt, hash_to_short_string
)
import json
4 changes: 2 additions & 2 deletions autonomi/python/autonomi_client/__init__.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
from .autonomi_client import Client, Wallet, PaymentOption, VaultSecretKey, UserData, PrivateDataAccess, encrypt
from .autonomi_client import Client, Wallet, PaymentOption, VaultSecretKey, UserData, DataMapChunk, encrypt

__all__ = [
"Client",
"Wallet",
"PaymentOption",
"VaultSecretKey",
"UserData",
"PrivateDataAccess",
"DataMapChunk",
"encrypt"
]
23 changes: 12 additions & 11 deletions autonomi/src/client/data/mod.rs
Original file line number Diff line number Diff line change
@@ -127,9 +127,9 @@ pub enum CostError {

/// Private data on the network can be accessed with this
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct PrivateDataAccess(Chunk);
pub struct DataMapChunk(Chunk);

impl PrivateDataAccess {
impl DataMapChunk {
pub fn to_hex(&self) -> String {
hex::encode(self.0.value())
}
@@ -139,7 +139,7 @@ impl PrivateDataAccess {
Ok(Self(Chunk::new(Bytes::from(data))))
}

/// Get a private address for [`PrivateDataAccess`]. Note that this is not a network address, it is only used for refering to private data client side.
/// Get a private address for [`DataMapChunk`]. Note that this is not a network address, it is only used for refering to private data client side.
pub fn address(&self) -> String {
hash_to_short_string(&self.to_hex())
}
@@ -153,8 +153,8 @@ fn hash_to_short_string(input: &str) -> String {
}

impl Client {
/// Fetch a blob of private data from the network
pub async fn data_get(&self, data_map: PrivateDataAccess) -> Result<Bytes, GetError> {
/// Fetch a blob of (private) data from the network
pub async fn data_get(&self, data_map: DataMapChunk) -> Result<Bytes, GetError> {
info!(
"Fetching private data from Data Map {:?}",
data_map.0.address()
@@ -166,13 +166,14 @@ impl Client {
}

/// Upload a piece of private data to the network. This data will be self-encrypted.
/// Returns the [`PrivateDataAccess`] containing the map to the encrypted chunks.
/// This data is private and only accessible with the [`PrivateDataAccess`].
/// The [`DataMapChunk`] is not uploaded to the network, keeping the data private.
///
/// Returns the [`DataMapChunk`] containing the map to the encrypted chunks.
pub async fn data_put(
&self,
data: Bytes,
payment_option: PaymentOption,
) -> Result<PrivateDataAccess, PutError> {
) -> Result<DataMapChunk, PutError> {
let now = ant_networking::target_arch::Instant::now();
let (data_map_chunk, chunks) = encrypt(data)?;
debug!("Encryption took: {:.2?}", now.elapsed());
@@ -220,7 +221,7 @@ impl Client {
}
}

Ok(PrivateDataAccess(data_map_chunk))
Ok(DataMapChunk(data_map_chunk))
}
}

@@ -230,9 +231,9 @@ mod tests {

#[test]
fn test_hex() {
let data_map = PrivateDataAccess(Chunk::new(Bytes::from_static(b"hello")));
let data_map = DataMapChunk(Chunk::new(Bytes::from_static(b"hello")));
let hex = data_map.to_hex();
let data_map2 = PrivateDataAccess::from_hex(&hex).expect("Failed to decode hex");
let data_map2 = DataMapChunk::from_hex(&hex).expect("Failed to decode hex");
assert_eq!(data_map, data_map2);
}
}
19 changes: 10 additions & 9 deletions autonomi/src/client/files/archive.rs
Original file line number Diff line number Diff line change
@@ -15,7 +15,7 @@ use ant_networking::target_arch::{Duration, SystemTime, UNIX_EPOCH};

use crate::{
client::{
data::{GetError, PrivateDataAccess, PutError},
data::{DataMapChunk, GetError, PutError},
payment::PaymentOption,
},
Client,
@@ -24,9 +24,10 @@ use bytes::Bytes;
use serde::{Deserialize, Serialize};
use thiserror::Error;

/// The address of a private archive
/// Contains the [`PrivateDataAccess`] leading to the [`PrivateArchive`] data
pub type PrivateArchiveAccess = PrivateDataAccess;
/// The address of a private archive.
///
/// Contains the [`DataMapChunk`] leading to the [`PrivateArchive`] data
pub type PrivateArchiveAccess = DataMapChunk;

#[derive(Error, Debug, PartialEq, Eq)]
pub enum RenameError {
@@ -68,7 +69,7 @@ impl Metadata {
/// Using archives is useful for uploading entire directories to the network, only needing to keep track of a single address.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
pub struct PrivateArchive {
map: HashMap<PathBuf, (PrivateDataAccess, Metadata)>,
map: HashMap<PathBuf, (DataMapChunk, Metadata)>,
}

impl PrivateArchive {
@@ -99,7 +100,7 @@ impl PrivateArchive {

/// Add a file to a local archive
/// Note that this does not upload the archive to the network
pub fn add_file(&mut self, path: PathBuf, data_map: PrivateDataAccess, meta: Metadata) {
pub fn add_file(&mut self, path: PathBuf, data_map: DataMapChunk, meta: Metadata) {
self.map.insert(path.clone(), (data_map, meta));
debug!("Added a new file to the archive, path: {:?}", path);
}
@@ -113,7 +114,7 @@ impl PrivateArchive {
}

/// List all data addresses of the files in the archive
pub fn addresses(&self) -> Vec<PrivateDataAccess> {
pub fn addresses(&self) -> Vec<DataMapChunk> {
self.map
.values()
.map(|(data_map, _)| data_map.clone())
@@ -122,14 +123,14 @@ impl PrivateArchive {

/// Iterate over the archive items
/// Returns an iterator over (PathBuf, SecretDataMap, Metadata)
pub fn iter(&self) -> impl Iterator<Item = (&PathBuf, &PrivateDataAccess, &Metadata)> {
pub fn iter(&self) -> impl Iterator<Item = (&PathBuf, &DataMapChunk, &Metadata)> {
self.map
.iter()
.map(|(path, (data_map, meta))| (path, data_map, meta))
}

/// Get the underlying map
pub fn map(&self) -> &HashMap<PathBuf, (PrivateDataAccess, Metadata)> {
pub fn map(&self) -> &HashMap<PathBuf, (DataMapChunk, Metadata)> {
&self.map
}

8 changes: 4 additions & 4 deletions autonomi/src/client/files/fs.rs
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@
// KIND, either express or implied. Please review the Licences for the specific language governing
// permissions and limitations relating to use of the SAFE Network Software.

use crate::client::data::{CostError, GetError, PrivateDataAccess, PutError};
use crate::client::data::{CostError, DataMapChunk, GetError, PutError};
use crate::client::utils::process_tasks_with_max_concurrency;
use crate::client::Client;
use ant_evm::EvmWallet;
@@ -85,7 +85,7 @@ impl Client {
/// Download a private file from network to local file system
pub async fn file_download(
&self,
data_access: PrivateDataAccess,
data_access: DataMapChunk,
to_dest: PathBuf,
) -> Result<(), DownloadError> {
let data = self.data_get(data_access).await?;
@@ -171,12 +171,12 @@ impl Client {
}

/// Upload a private file to the network.
/// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`PrivateDataAccess`] (pointing to the datamap)
/// Reads file, splits into chunks, uploads chunks, uploads datamap, returns [`DataMapChunk`] (pointing to the datamap)
async fn file_upload(
&self,
path: PathBuf,
wallet: &EvmWallet,
) -> Result<PrivateDataAccess, UploadError> {
) -> Result<DataMapChunk, UploadError> {
info!("Uploading file: {path:?}");
#[cfg(feature = "loud")]
println!("Uploading file: {path:?}");
12 changes: 6 additions & 6 deletions autonomi/src/client/wasm.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use super::address::{addr_to_str, str_to_addr};
#[cfg(feature = "vault")]
use super::vault::UserData;
use crate::client::data_private::PrivateDataAccess;
use crate::client::data_private::DataMapChunk;
use crate::client::payment::Receipt;
use ant_protocol::storage::Chunk;
use libp2p::Multiaddr;
@@ -107,7 +107,7 @@ impl JsClient {

/// Upload private data to the network.
///
/// Returns the `PrivateDataAccess` chunk of the data.
/// Returns the `DataMapChunk` chunk of the data.
#[wasm_bindgen(js_name = putPrivateData)]
pub async fn put_private_data(
&self,
@@ -124,7 +124,7 @@ impl JsClient {
/// Upload private data to the network.
/// Uses a `Receipt` as payment.
///
/// Returns the `PrivateDataAccess` chunk of the data.
/// Returns the `DataMapChunk` chunk of the data.
#[wasm_bindgen(js_name = putPrivateDataWithReceipt)]
pub async fn put_private_data_with_receipt(
&self,
@@ -151,7 +151,7 @@ impl JsClient {
/// Fetch the data from the network.
#[wasm_bindgen(js_name = getPrivateData)]
pub async fn get_private_data(&self, private_data_access: JsValue) -> Result<Vec<u8>, JsError> {
let private_data_access: PrivateDataAccess =
let private_data_access: DataMapChunk =
serde_wasm_bindgen::from_value(private_data_access)?;
let data = self.0.data_get(private_data_access).await?;

@@ -278,7 +278,7 @@ mod archive_private {
use super::*;
use crate::client::archive::Metadata;
use crate::client::archive_private::{PrivateArchive, PrivateArchiveAccess};
use crate::client::data_private::PrivateDataAccess;
use crate::client::data_private::DataMapChunk;
use crate::client::payment::Receipt;
use std::path::PathBuf;
use wasm_bindgen::{JsError, JsValue};
@@ -304,7 +304,7 @@ mod archive_private {
metadata: JsValue,
) -> Result<(), JsError> {
let path = PathBuf::from(path);
let data_map: PrivateDataAccess = serde_wasm_bindgen::from_value(data_map)?;
let data_map: DataMapChunk = serde_wasm_bindgen::from_value(data_map)?;
let metadata: Metadata = serde_wasm_bindgen::from_value(metadata)?;
self.0.add_file(path, data_map, metadata);

20 changes: 10 additions & 10 deletions autonomi/src/python.rs
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
#![allow(non_local_definitions)]

use crate::client::{
data::PrivateDataAccess,
data::DataMapChunk,
files::{archive::PrivateArchiveAccess, archive_public::ArchiveAddr},
payment::PaymentOption as RustPaymentOption,
vault::{UserData, VaultSecretKey},
@@ -38,7 +38,7 @@ impl PyClient {
Ok(Self { inner: client })
}

fn data_put(&self, data: Vec<u8>, payment: &PyPaymentOption) -> PyResult<PyPrivateDataAccess> {
fn data_put(&self, data: Vec<u8>, payment: &PyPaymentOption) -> PyResult<PyDataMapChunk> {
let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime");
let access = rt
.block_on(
@@ -49,10 +49,10 @@ impl PyClient {
pyo3::exceptions::PyValueError::new_err(format!("Failed to put private data: {e}"))
})?;

Ok(PyPrivateDataAccess { inner: access })
Ok(PyDataMapChunk { inner: access })
}

fn data_get(&self, access: &PyPrivateDataAccess) -> PyResult<Vec<u8>> {
fn data_get(&self, access: &PyDataMapChunk) -> PyResult<Vec<u8>> {
let rt = tokio::runtime::Runtime::new().expect("Could not start tokio runtime");
let data = rt
.block_on(self.inner.data_get(access.inner.clone()))
@@ -294,17 +294,17 @@ impl PyUserData {
}
}

#[pyclass(name = "PrivateDataAccess")]
#[pyclass(name = "DataMapChunk")]
#[derive(Clone)]
pub(crate) struct PyPrivateDataAccess {
inner: PrivateDataAccess,
pub(crate) struct PyDataMapChunk {
inner: DataMapChunk,
}

#[pymethods]
impl PyPrivateDataAccess {
impl PyDataMapChunk {
#[staticmethod]
fn from_hex(hex: &str) -> PyResult<Self> {
PrivateDataAccess::from_hex(hex)
DataMapChunk::from_hex(hex)
.map(|access| Self { inner: access })
.map_err(|e| pyo3::exceptions::PyValueError::new_err(format!("Invalid hex: {e}")))
}
@@ -342,7 +342,7 @@ fn autonomi_client_module(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
m.add_class::<PyPaymentOption>()?;
m.add_class::<PyVaultSecretKey>()?;
m.add_class::<PyUserData>()?;
m.add_class::<PyPrivateDataAccess>()?;
m.add_class::<PyDataMapChunk>()?;
m.add_function(wrap_pyfunction!(encrypt, m)?)?;
Ok(())
}