From b91351ab999c26dadc790a2bd81890afcdc93818 Mon Sep 17 00:00:00 2001 From: Ulf Lilleengen Date: Wed, 24 Apr 2024 15:09:56 +0200 Subject: [PATCH 1/4] Allow configuring max chunk size Changes the magic since it breaks existing on-disk format --- Cargo.toml | 6 ++++++ build.rs | 1 + gen_config.py | 1 + src/config.rs | 8 ++++++++ src/page.rs | 4 +--- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 03b9cd8..adf6418 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -189,4 +189,10 @@ branching-factor-2 = [] # Default branching-factor-3 = [] branching-factor-4 = [] +max-chunk-size-256 = [] +max-chunk-size-512 = [] +max-chunk-size-1024 = [] +max-chunk-size-2048 = [] +max-chunk-size-4096 = [] # Default + # END AUTOGENERATED CONFIG FEATURES diff --git a/build.rs b/build.rs index 2712960..ae8bec2 100644 --- a/build.rs +++ b/build.rs @@ -14,6 +14,7 @@ static CONFIGS: &[(&str, usize)] = &[ ("MAX_VALUE_SIZE", 1024), ("SCRATCH_PAGE_COUNT", 4), ("BRANCHING_FACTOR", 2), + ("MAX_CHUNK_SIZE", 4096), // END AUTOGENERATED CONFIG FEATURES ]; diff --git a/gen_config.py b/gen_config.py index 8af10f1..8cd6c10 100644 --- a/gen_config.py +++ b/gen_config.py @@ -46,6 +46,7 @@ def feature(name, default, min=None, max=None, pow2=None, vals=None, factors=[]) feature("scratch_page_count", default=4, min=0, max=65536, pow2=True) feature("branching_factor", default=2, min=2, max=4) +feature("max_chunk_size", default=4096, vals=[256, 512, 1024, 2048, 4096]) # ========= Update Cargo.toml diff --git a/src/config.rs b/src/config.rs index b470bd0..2c0e66c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -90,6 +90,14 @@ pub const ERASE_VALUE: u8 = match raw::ERASE_VALUE { _ => core::panic!("invalid ERASE_VALUE"), }; +/// Maximum chunk size supported. +/// +/// The chunk size controls how big chunks of data is read and written from/to flash. A low +/// value reduces the memory usage of EKV at the expense of more reads/writes. +/// +/// Default: 4096 +pub const MAX_CHUNK_SIZE: usize = raw::MAX_CHUNK_SIZE; + pub(crate) const MAX_HEADER_SIZE: usize = { let a = size_of::(); let b = size_of::(); diff --git a/src/page.rs b/src/page.rs index 334f5f5..6db1423 100644 --- a/src/page.rs +++ b/src/page.rs @@ -6,7 +6,7 @@ use crate::errors::Error; use crate::flash::Flash; use crate::types::PageID; -const CHUNK_MAGIC: u16 = 0x58A4; +const CHUNK_MAGIC: u16 = 0x59B4; #[derive(Clone, Copy, PartialEq, Eq, Debug)] #[repr(C)] @@ -53,8 +53,6 @@ pub unsafe trait Header: Sized { const MAGIC: u32; } -pub const MAX_CHUNK_SIZE: usize = PAGE_SIZE - PageHeader::SIZE - ChunkHeader::SIZE; - async fn write_header(flash: &mut F, page_id: PageID, header: H) -> Result<(), F::Error> { assert!(size_of::() <= MAX_HEADER_SIZE); let mut buf = [0u8; PageHeader::SIZE + MAX_HEADER_SIZE]; From c3a5ebbe9c1b3f94b1bd62f5ddba6284d8555751 Mon Sep 17 00:00:00 2001 From: Ulf Lilleengen Date: Wed, 24 Apr 2024 18:09:55 +0200 Subject: [PATCH 2/4] change additional magic --- src/file.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/file.rs b/src/file.rs index 0f013b6..628a8b8 100644 --- a/src/file.rs +++ b/src/file.rs @@ -28,7 +28,7 @@ pub struct MetaHeader { } unsafe impl page::Header for MetaHeader { - const MAGIC: u32 = 0x1d81accb; + const MAGIC: u32 = 0x1d81bccc; } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -48,7 +48,7 @@ pub struct DataHeader { } unsafe impl page::Header for DataHeader { - const MAGIC: u32 = 0x7fccf25b; + const MAGIC: u32 = 0x7fcbf25c; } #[derive(Clone, Copy, PartialEq, Eq, Debug)] From 648c2f27795c13a4835ad24365440848533884b7 Mon Sep 17 00:00:00 2001 From: Ulf Lilleengen Date: Wed, 24 Apr 2024 18:12:00 +0200 Subject: [PATCH 3/4] set max chunk size --- src/page.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/page.rs b/src/page.rs index 6db1423..f09d3d1 100644 --- a/src/page.rs +++ b/src/page.rs @@ -1,7 +1,7 @@ use core::marker::PhantomData; use core::mem::size_of; -use crate::config::*; +use crate::config::{ALIGN, PAGE_SIZE, self, MAX_HEADER_SIZE, ERASE_VALUE}; use crate::errors::Error; use crate::flash::Flash; use crate::types::PageID; @@ -53,6 +53,12 @@ pub unsafe trait Header: Sized { const MAGIC: u32; } +const MAX_CHUNK_SIZE: usize = if config::MAX_CHUNK_SIZE > (PAGE_SIZE - PageHeader::SIZE - ChunkHeader::SIZE) { + PAGE_SIZE - PageHeader::SIZE - ChunkHeader::SIZE +} else { + config::MAX_CHUNK_SIZE +}; + async fn write_header(flash: &mut F, page_id: PageID, header: H) -> Result<(), F::Error> { assert!(size_of::() <= MAX_HEADER_SIZE); let mut buf = [0u8; PageHeader::SIZE + MAX_HEADER_SIZE]; From 804bc21a600d2d041de519315bb8cb6b688b753e Mon Sep 17 00:00:00 2001 From: Ulf Lilleengen Date: Wed, 24 Apr 2024 18:13:11 +0200 Subject: [PATCH 4/4] rustfm --- src/page.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/page.rs b/src/page.rs index f09d3d1..9c29301 100644 --- a/src/page.rs +++ b/src/page.rs @@ -1,7 +1,7 @@ use core::marker::PhantomData; use core::mem::size_of; -use crate::config::{ALIGN, PAGE_SIZE, self, MAX_HEADER_SIZE, ERASE_VALUE}; +use crate::config::{self, ALIGN, ERASE_VALUE, MAX_HEADER_SIZE, PAGE_SIZE}; use crate::errors::Error; use crate::flash::Flash; use crate::types::PageID;