Merge pull request #263 from tpunder/mdb_maxkeysize_0

Add the "mdb_maxkeysize_0" feature which sets "-DMDB_MAXKEYSIZE=0" when compiling lmdb

heed/Cargo.toml

@@ -90,6 +90,26 @@ mdb_idl_logn_14 = ["lmdb-master-sys/mdb_idl_logn_14"]
 mdb_idl_logn_15 = ["lmdb-master-sys/mdb_idl_logn_15"]
 mdb_idl_logn_16 = ["lmdb-master-sys/mdb_idl_logn_16"]
 
+# Setting this enables you to use keys longer than 511 bytes. The exact limit
+# is computed by LMDB at compile time. You can find the exact value by calling
+# Env::max_key_size(). This value varies by architecture.
+#
+# Example max key sizes:
+#   - Apple M1 (ARM64): 8126 bytes
+#   - Apple Intel (AMD64): 1982 bytes
+#   - Linux Intel (AMD64): 1982 bytes
+#
+# Setting this also enables you to use values larger than 511 bytes when using
+# a Database with the DatabaseFlags::DUP_SORT flag.
+#
+# This builds LMDB with the -DMDB_MAXKEYSIZE=0 option.
+#
+# Note: If you are moving database files between architectures then your longest
+# stored key must fit within the smallest limit of all architectures used. For
+# example, if you are moving databases between Apple M1 and Apple Intel
+# computers then you need to keep your keys within the smaller 1982 byte limit.
+longer-keys = ["lmdb-master-sys/longer-keys"]
+
 [[example]]
 name = "rmp-serde"
 required-features = ["serde-rmp"]

heed/src/database.rs

@@ -2657,4 +2657,26 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    #[cfg(feature = "longer-keys")]
+    fn longer_keys() -> Result<()> {
+        let dir = tempfile::tempdir()?;
+        let env = unsafe { EnvOpenOptions::new().open(dir.path())? };
+        let mut txn = env.write_txn()?;
+        let db = env.create_database::<Bytes, Bytes>(&mut txn, None)?;
+
+        // Try storing a key larger than 511 bytes (the default if MDB_MAXKEYSIZE is not set)
+        let long_key = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut pharetra sit amet aliquam. Sit amet nisl purus in mollis nunc. Eget egestas purus viverra accumsan in nisl nisi scelerisque. Duis ultricies lacus sed turpis tincidunt. Sem nulla pharetra diam sit. Leo vel orci porta non pulvinar. Erat pellentesque adipiscing commodo elit at imperdiet dui. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla. Diam donec adipiscing tristique risus nec feugiat. In fermentum et sollicitudin ac orci. Ut sem nulla pharetra diam sit amet. Aliquam purus sit amet luctus venenatis lectus. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan. Urna duis convallis convallis tellus id interdum velit laoreet id. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed. Tincidunt arcu non sodales neque. Habitant morbi tristique senectus et netus et malesuada fames.";
+
+        assert_eq!(db.get(&txn, long_key).unwrap(), None);
+
+        db.put(&mut txn, long_key, b"hi").unwrap();
+        assert_eq!(db.get(&txn, long_key).unwrap(), Some(&b"hi"[..]));
+
+        db.put(&mut txn, long_key, b"bye").unwrap();
+        assert_eq!(db.get(&txn, long_key).unwrap(), Some(&b"bye"[..]));
+
+        Ok(())
+    }
 }

heed/src/env.rs

@@ -901,6 +901,14 @@ impl Env {
         mdb_result(unsafe { ffi::mdb_env_set_mapsize(self.env_mut_ptr(), new_size) })
             .map_err(Into::into)
     }
+
+    /// Get the maximum size of keys and MDB_DUPSORT data we can write.
+    ///
+    /// Depends on the compile-time constant MDB_MAXKEYSIZE. Default 511
+    pub fn max_key_size(&self) -> usize {
+        let maxsize: i32 = unsafe { ffi::mdb_env_get_maxkeysize(self.env_mut_ptr()) };
+        maxsize as usize
+    }
 }
 
 /// Contains information about the environment.
@@ -1251,4 +1259,21 @@ mod tests {
             env.prepare_for_closing().wait();
         }
     }
+
+    #[test]
+    fn max_key_size() {
+        let dir = tempfile::tempdir().unwrap();
+        let env = unsafe { EnvOpenOptions::new().open(dir.path().join(dir.path())).unwrap() };
+        let maxkeysize = env.max_key_size();
+
+        eprintln!("maxkeysize: {}", maxkeysize);
+
+        if cfg!(feature = "longer-keys") {
+            // Should be larger than the default of 511
+            assert!(maxkeysize > 511);
+        } else {
+            // Should be the default of 511
+            assert_eq!(maxkeysize, 511);
+        }
+    }
 }

heed/src/mdb/lmdb_error.rs

@@ -53,6 +53,14 @@ pub enum Error {
     /// Transaction cannot recover - it must be aborted.
     BadTxn,
     /// Unsupported size of key/DB name/data, or wrong DUP_FIXED size.
+    ///
+    /// Common causes of this error:
+    ///   - You tried to store a zero-length key
+    ///   - You tried to store a key longer than the max allowed key (511 bytes by default)
+    ///   - You are using [DUP_SORT](crate::DatabaseFlags::DUP_SORT) and trying to store a
+    ///     value longer than the max allowed key size (511 bytes by default)
+    ///
+    /// In the last two cases you can enable the `longer-keys` feature to increase the max allowed key size.
     BadValSize,
     /// The specified DBI was changed unexpectedly.
     BadDbi,

heed/src/mdb/lmdb_ffi.rs

@@ -3,11 +3,12 @@ use std::ptr;
 pub use ffi::{
     mdb_cursor_close, mdb_cursor_del, mdb_cursor_get, mdb_cursor_open, mdb_cursor_put,
     mdb_dbi_open, mdb_del, mdb_drop, mdb_env_close, mdb_env_copyfd2, mdb_env_create,
-    mdb_env_get_fd, mdb_env_get_flags, mdb_env_info, mdb_env_open, mdb_env_set_flags,
-    mdb_env_set_mapsize, mdb_env_set_maxdbs, mdb_env_set_maxreaders, mdb_env_stat, mdb_env_sync,
-    mdb_filehandle_t, mdb_get, mdb_put, mdb_reader_check, mdb_set_compare, mdb_stat, mdb_txn_abort,
-    mdb_txn_begin, mdb_txn_commit, mdb_version, MDB_cursor, MDB_dbi, MDB_env, MDB_stat, MDB_txn,
-    MDB_val, MDB_CP_COMPACT, MDB_CURRENT, MDB_RDONLY, MDB_RESERVE,
+    mdb_env_get_fd, mdb_env_get_flags, mdb_env_get_maxkeysize, mdb_env_info, mdb_env_open,
+    mdb_env_set_flags, mdb_env_set_mapsize, mdb_env_set_maxdbs, mdb_env_set_maxreaders,
+    mdb_env_stat, mdb_env_sync, mdb_filehandle_t, mdb_get, mdb_put, mdb_reader_check,
+    mdb_set_compare, mdb_stat, mdb_txn_abort, mdb_txn_begin, mdb_txn_commit, mdb_version,
+    MDB_cursor, MDB_dbi, MDB_env, MDB_stat, MDB_txn, MDB_val, MDB_CP_COMPACT, MDB_CURRENT,
+    MDB_RDONLY, MDB_RESERVE,
 };
 use lmdb_master_sys as ffi;
 

lmdb-master-sys/Cargo.toml

@@ -60,3 +60,23 @@ mdb_idl_logn_13 = []
 mdb_idl_logn_14 = []
 mdb_idl_logn_15 = []
 mdb_idl_logn_16 = []
+
+# Setting this enables you to use keys longer than 511 bytes. The exact limit
+# is computed by LMDB at compile time. You can find the exact value by calling
+# Env::max_key_size(). This value varies by architecture.
+#
+# Example max key sizes:
+#   - Apple M1 (ARM64): 8126 bytes
+#   - Apple Intel (AMD64): 1982 bytes
+#   - Linux Intel (AMD64): 1982 bytes
+#
+# Setting this also enables you to use values larger than 511 bytes when using
+# a Database with the DatabaseFlags::DUP_SORT flag.
+#
+# This builds LMDB with the -DMDB_MAXKEYSIZE=0 option.
+#
+# Note: If you are moving database files between architectures then your longest
+# stored key must fit within the smallest limit of all architectures used. For
+# example, if you are moving databases between Apple M1 and Apple Intel
+# computers then you need to keep your keys within the smaller 1982 byte limit.
+longer-keys = []

lmdb-master-sys/build.rs

@@ -147,5 +147,9 @@ fn main() {
         builder.flag("-fsanitize=fuzzer-no-link");
     }
 
+    if cfg!(feature = "longer-keys") {
+        builder.define("MDB_MAXKEYSIZE", "0");
+    }
+
     builder.compile("liblmdb.a")
 }

lmdb-master-sys/tests/simple.rs

@@ -80,6 +80,17 @@ fn test_simple(env_path: &str) {
 
         E!(mdb_txn_begin(env, ptr::null_mut(), 0, &mut txn));
         E!(mdb_put(txn, dbi, &mut key, &mut data, 0));
+
+        if cfg!(feature = "longer-keys") {
+            // Try storing a key larger than 511 bytes (the default if MDB_MAXKEYSIZE is not set)
+            let sval = cstr!("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut pharetra sit amet aliquam. Sit amet nisl purus in mollis nunc. Eget egestas purus viverra accumsan in nisl nisi scelerisque. Duis ultricies lacus sed turpis tincidunt. Sem nulla pharetra diam sit. Leo vel orci porta non pulvinar. Erat pellentesque adipiscing commodo elit at imperdiet dui. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla. Diam donec adipiscing tristique risus nec feugiat. In fermentum et sollicitudin ac orci. Ut sem nulla pharetra diam sit amet. Aliquam purus sit amet luctus venenatis lectus. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan. Urna duis convallis convallis tellus id interdum velit laoreet id. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed. Tincidunt arcu non sodales neque. Habitant morbi tristique senectus et netus et malesuada fames.").as_ptr() as *mut c_void;
+
+            key.mv_size = 952;
+            key.mv_data = sval;
+
+            E!(mdb_put(txn, dbi, &mut key, &mut data, 0));
+        }
+
         E!(mdb_txn_commit(txn));
     }