config: add set_mapsize

.github/workflows/build-and-test.yml

@@ -29,8 +29,8 @@ jobs:
         if: ${{ inputs.additional-setup != '' }}
         run: ${{ inputs.additional-setup }}
 
-      - name: Run Tests
-        run: cargo test --release
-
       - name: Build
         run: cargo build --release
+
+      - name: Run Tests
+        run: cargo test --release

Cargo.toml

@@ -20,6 +20,7 @@ bindgen = []
 [dependencies]
 flatbuffers = "23.5.26"
 libc = "0.2.151"
+thiserror = "2.0.3"
 tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"

src/config.rs

@@ -1,5 +1,6 @@
 use crate::bindings;
 
+#[derive(Copy, Clone)]
 pub struct Config {
     pub config: bindings::ndb_config,
 }
@@ -30,12 +31,12 @@ impl Config {
     }
 
     //
-    pub fn set_flags(&mut self, flags: i32) -> &mut Self {
+    pub fn set_flags(mut self, flags: i32) -> Self {
         self.config.flags = flags;
         self
     }
 
-    pub fn skip_validation(&mut self, skip: bool) -> &mut Self {
+    pub fn skip_validation(mut self, skip: bool) -> Self {
         let skip_note_verify: i32 = 1 << 1;
 
         if skip {
@@ -47,7 +48,12 @@ impl Config {
         self
     }
 
-    pub fn set_ingester_threads(&mut self, threads: i32) -> &mut Self {
+    pub fn set_mapsize(mut self, bytes: usize) -> Self {
+        self.config.mapsize = bytes;
+        self
+    }
+
+    pub fn set_ingester_threads(mut self, threads: i32) -> Self {
         self.config.ingester_threads = threads;
         self
     }

src/error.rs

@@ -1,27 +1,46 @@
-use std::fmt;
+use thiserror::Error;
 
-#[derive(Debug, Eq, PartialEq)]
+/// Main error type
+#[derive(Debug, Error)]
 pub enum Error {
+    #[error("Database open failed")]
     DbOpenFailed,
+
+    #[error("Not found")]
     NotFound,
+
+    #[error("Decode error")]
     DecodeError,
+
+    #[error("Query failed")]
     QueryError,
+
+    #[error("Note process failed")]
     NoteProcessFailed,
+
+    #[error("Transaction failed")]
     TransactionFailed,
+
+    #[error("Subscription failed")]
     SubscriptionError,
+
+    #[error("Buffer overflow")]
     BufferOverflow,
-    Filter(FilterError),
-}
 
-impl Error {
-    pub fn filter(ferr: FilterError) -> Self {
-        Error::Filter(ferr)
-    }
+    #[error("Filter error: {0}")]
+    Filter(#[from] FilterError),
+
+    #[error("IO error: {0}")]
+    IO(#[from] std::io::Error),
 }
 
-#[derive(Debug, Eq, PartialEq)]
+/// Filter-specific error type
+#[derive(Debug, Error, Eq, PartialEq)]
 pub enum FilterError {
+    #[error("Field already exists")]
     FieldAlreadyExists,
+
+    #[error("Field already started")]
     FieldAlreadyStarted,
 }
 
@@ -34,30 +53,3 @@ impl FilterError {
         Error::Filter(FilterError::FieldAlreadyStarted)
     }
 }
-
-impl fmt::Display for FilterError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            FilterError::FieldAlreadyExists => write!(f, "field already exists"),
-            FilterError::FieldAlreadyStarted => write!(f, "field already started"),
-        }
-    }
-}
-
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Error::DbOpenFailed => write!(f, "Open failed"),
-            Error::NotFound => write!(f, "Not found"),
-            Error::QueryError => write!(f, "Query failed"),
-            Error::DecodeError => write!(f, "Decode error"),
-            Error::NoteProcessFailed => write!(f, "Note process failed"),
-            Error::TransactionFailed => write!(f, "Transaction failed"),
-            Error::SubscriptionError => write!(f, "Subscription failed"),
-            Error::BufferOverflow => write!(f, "Buffer overflow"),
-            Error::Filter(filter_err) => write!(f, "Filter: {filter_err}"),
-        }
-    }
-}
-
-impl std::error::Error for Error {}

src/filter.rs

@@ -401,7 +401,7 @@ impl FilterBuilder {
         let r =
             unsafe { bindings::ndb_filter_start_tag_field(self.as_mut_ptr(), tag as u8 as c_char) };
         if r == 0 {
-            return Err(Error::filter(FilterError::FieldAlreadyStarted));
+            return Err(FilterError::FieldAlreadyStarted.into());
         }
         Ok(())
     }

src/ndb.rs

@@ -10,6 +10,7 @@ use std::os::raw::c_int;
 use std::path::Path;
 use std::sync::Arc;
 use tokio::task; // Make sure to import the task module
+use tracing::debug;
 
 #[derive(Debug)]
 struct NdbRef {
@@ -51,7 +52,28 @@ impl Ndb {
             let _ = fs::create_dir_all(path);
         }
 
-        let result = unsafe { bindings::ndb_init(&mut ndb, db_dir_cstr.as_ptr(), config.as_ptr()) };
+        let min_mapsize = 1024 * 1024 * 512;
+        let mut mapsize = config.config.mapsize;
+        let mut config = *config;
+
+        let result = loop {
+            let result =
+                unsafe { bindings::ndb_init(&mut ndb, db_dir_cstr.as_ptr(), config.as_ptr()) };
+
+            if result == 0 {
+                mapsize /= 2;
+                config = config.set_mapsize(mapsize);
+                debug!("ndb init failed, reducing mapsize to {}", mapsize);
+
+                if mapsize > min_mapsize {
+                    continue;
+                } else {
+                    break 0;
+                }
+            } else {
+                break result;
+            }
+        };
 
         if result == 0 {
             return Err(Error::DbOpenFailed);
@@ -448,4 +470,43 @@ mod tests {
             assert_eq!(note.kind(), 1);
         }
     }
+
+    #[test]
+    #[cfg(target_os = "windows")]
+    fn test_windows_large_mapsize() {
+        use std::{fs, path::Path};
+
+        let db = "target/testdbs/windows_large_mapsize";
+        test_util::cleanup_db(&db);
+
+        {
+            // 32 TiB should be way too big for CI
+            let config =
+                Config::new().set_mapsize(1024usize * 1024usize * 1024usize * 1024usize * 32usize);
+
+            // in this case, nostrdb should try to keep resizing to
+            // smaller mapsizes until success
+
+            let ndb = Ndb::new(db, &config);
+
+            assert!(ndb.is_ok());
+        }
+
+        let file_len = fs::metadata(Path::new(db).join("data.mdb"))
+            .expect("metadata")
+            .len();
+
+        assert!(file_len > 0);
+
+        if cfg!(target_os = "windows") {
+            // on windows the default mapsize will be 1MB when we fail
+            // to open it
+            assert_ne!(file_len, 1048576);
+        } else {
+            assert!(file_len < 1024u64 * 1024u64);
+        }
+
+        // we should definitely clean this up... especially on windows
+        test_util::cleanup_db(&db);
+    }
 }

src/note.rs

@@ -509,7 +509,7 @@ mod tests {
             let err = ndb
                 .get_note_by_id(&mut txn, &[0; 32])
                 .expect_err("not found");
-            assert!(err == Error::NotFound);
+            assert!(matches!(err, Error::NotFound));
         }
 
         test_util::cleanup_db(db);

src/transaction.rs

@@ -69,7 +69,7 @@ mod tests {
             {
                 let _txn = Transaction::new(&ndb).expect("txn1 failed");
                 let txn2 = Transaction::new(&ndb).expect_err("tx2");
-                assert!(txn2 == Error::TransactionFailed);
+                assert!(matches!(txn2, Error::TransactionFailed));
             }
 
             {