From d925a5a66e2dead4c1af97d406abe79e0dab3fdf Mon Sep 17 00:00:00 2001 From: Tim Underwood Date: Fri, 24 May 2024 14:42:32 -0700 Subject: [PATCH 1/2] Add "longer-keys" feature which sets "-DMDB_MAXKEYSIZE=0" when compiling LMDB By default (and for backwards compatibility) LMDB only allows keys up to 511 bytes in length. If you want larger keys then you need to set "-DMDB_MAXKEYSIZE=0" at compile time. This limit also applies to values when using the MDB_DUPSORT flag on a database. --- heed/Cargo.toml | 20 ++++++++++++++++++++ heed/src/database.rs | 22 ++++++++++++++++++++++ heed/src/mdb/lmdb_error.rs | 8 ++++++++ lmdb-master-sys/Cargo.toml | 20 ++++++++++++++++++++ lmdb-master-sys/build.rs | 4 ++++ lmdb-master-sys/tests/simple.rs | 11 +++++++++++ 6 files changed, 85 insertions(+) diff --git a/heed/Cargo.toml b/heed/Cargo.toml index 9b09dcae..445a8467 100644 --- a/heed/Cargo.toml +++ b/heed/Cargo.toml @@ -90,6 +90,26 @@ mdb_idl_logn_14 = ["lmdb-master-sys/mdb_idl_logn_14"] mdb_idl_logn_15 = ["lmdb-master-sys/mdb_idl_logn_15"] mdb_idl_logn_16 = ["lmdb-master-sys/mdb_idl_logn_16"] +# Setting this enables you to use keys longer than 511 bytes. The exact limit +# is computed by LMDB at compile time. You can find the exact value by calling +# Env::max_key_size(). This value varies by architecture. +# +# Example max key sizes: +# - Apple M1 (ARM64): 8126 bytes +# - Apple Intel (AMD64): 1982 bytes +# - Linux Intel (AMD64): 1982 bytes +# +# Setting this also enables you to use values larger than 511 bytes when using +# a Database with the DatabaseFlags::DUP_SORT flag. +# +# This builds LMDB with the -DMDB_MAXKEYSIZE=0 option. +# +# Note: If you are moving database files between architectures then your longest +# stored key must fit within the smallest limit of all architectures used. For +# example, if you are moving databases between Apple M1 and Apple Intel +# computers then you need to keep your keys within the smaller 1982 byte limit. +longer-keys = ["lmdb-master-sys/longer-keys"] + [[example]] name = "rmp-serde" required-features = ["serde-rmp"] diff --git a/heed/src/database.rs b/heed/src/database.rs index d40d6eda..3817103f 100644 --- a/heed/src/database.rs +++ b/heed/src/database.rs @@ -2657,4 +2657,26 @@ mod tests { Ok(()) } + + #[test] + #[cfg(feature = "longer-keys")] + fn longer_keys() -> Result<()> { + let dir = tempfile::tempdir()?; + let env = unsafe { EnvOpenOptions::new().open(dir.path())? }; + let mut txn = env.write_txn()?; + let db = env.create_database::(&mut txn, None)?; + + // Try storing a key larger than 511 bytes (the default if MDB_MAXKEYSIZE is not set) + let long_key = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut pharetra sit amet aliquam. Sit amet nisl purus in mollis nunc. Eget egestas purus viverra accumsan in nisl nisi scelerisque. Duis ultricies lacus sed turpis tincidunt. Sem nulla pharetra diam sit. Leo vel orci porta non pulvinar. Erat pellentesque adipiscing commodo elit at imperdiet dui. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla. Diam donec adipiscing tristique risus nec feugiat. In fermentum et sollicitudin ac orci. Ut sem nulla pharetra diam sit amet. Aliquam purus sit amet luctus venenatis lectus. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan. Urna duis convallis convallis tellus id interdum velit laoreet id. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed. Tincidunt arcu non sodales neque. Habitant morbi tristique senectus et netus et malesuada fames."; + + assert_eq!(db.get(&txn, long_key).unwrap(), None); + + db.put(&mut txn, long_key, b"hi").unwrap(); + assert_eq!(db.get(&txn, long_key).unwrap(), Some(&b"hi"[..])); + + db.put(&mut txn, long_key, b"bye").unwrap(); + assert_eq!(db.get(&txn, long_key).unwrap(), Some(&b"bye"[..])); + + Ok(()) + } } diff --git a/heed/src/mdb/lmdb_error.rs b/heed/src/mdb/lmdb_error.rs index e21e3d4b..36b62605 100644 --- a/heed/src/mdb/lmdb_error.rs +++ b/heed/src/mdb/lmdb_error.rs @@ -53,6 +53,14 @@ pub enum Error { /// Transaction cannot recover - it must be aborted. BadTxn, /// Unsupported size of key/DB name/data, or wrong DUP_FIXED size. + /// + /// Common causes of this error: + /// - You tried to store a zero-length key + /// - You tried to store a key longer than the max allowed key (511 bytes by default) + /// - You are using [DUP_SORT](crate::DatabaseFlags::DUP_SORT) and trying to store a + /// value longer than the max allowed key size (511 bytes by default) + /// + /// In the last two cases you can enable the `longer-keys` feature to increase the max allowed key size. BadValSize, /// The specified DBI was changed unexpectedly. BadDbi, diff --git a/lmdb-master-sys/Cargo.toml b/lmdb-master-sys/Cargo.toml index d79d14c7..a8e88533 100644 --- a/lmdb-master-sys/Cargo.toml +++ b/lmdb-master-sys/Cargo.toml @@ -60,3 +60,23 @@ mdb_idl_logn_13 = [] mdb_idl_logn_14 = [] mdb_idl_logn_15 = [] mdb_idl_logn_16 = [] + +# Setting this enables you to use keys longer than 511 bytes. The exact limit +# is computed by LMDB at compile time. You can find the exact value by calling +# Env::max_key_size(). This value varies by architecture. +# +# Example max key sizes: +# - Apple M1 (ARM64): 8126 bytes +# - Apple Intel (AMD64): 1982 bytes +# - Linux Intel (AMD64): 1982 bytes +# +# Setting this also enables you to use values larger than 511 bytes when using +# a Database with the DatabaseFlags::DUP_SORT flag. +# +# This builds LMDB with the -DMDB_MAXKEYSIZE=0 option. +# +# Note: If you are moving database files between architectures then your longest +# stored key must fit within the smallest limit of all architectures used. For +# example, if you are moving databases between Apple M1 and Apple Intel +# computers then you need to keep your keys within the smaller 1982 byte limit. +longer-keys = [] diff --git a/lmdb-master-sys/build.rs b/lmdb-master-sys/build.rs index d3e7d18d..7a1dedd9 100644 --- a/lmdb-master-sys/build.rs +++ b/lmdb-master-sys/build.rs @@ -147,5 +147,9 @@ fn main() { builder.flag("-fsanitize=fuzzer-no-link"); } + if cfg!(feature = "longer-keys") { + builder.define("MDB_MAXKEYSIZE", "0"); + } + builder.compile("liblmdb.a") } diff --git a/lmdb-master-sys/tests/simple.rs b/lmdb-master-sys/tests/simple.rs index b00fbec5..2def5968 100644 --- a/lmdb-master-sys/tests/simple.rs +++ b/lmdb-master-sys/tests/simple.rs @@ -80,6 +80,17 @@ fn test_simple(env_path: &str) { E!(mdb_txn_begin(env, ptr::null_mut(), 0, &mut txn)); E!(mdb_put(txn, dbi, &mut key, &mut data, 0)); + + if cfg!(feature = "longer-keys") { + // Try storing a key larger than 511 bytes (the default if MDB_MAXKEYSIZE is not set) + let sval = cstr!("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut pharetra sit amet aliquam. Sit amet nisl purus in mollis nunc. Eget egestas purus viverra accumsan in nisl nisi scelerisque. Duis ultricies lacus sed turpis tincidunt. Sem nulla pharetra diam sit. Leo vel orci porta non pulvinar. Erat pellentesque adipiscing commodo elit at imperdiet dui. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla. Diam donec adipiscing tristique risus nec feugiat. In fermentum et sollicitudin ac orci. Ut sem nulla pharetra diam sit amet. Aliquam purus sit amet luctus venenatis lectus. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan. Urna duis convallis convallis tellus id interdum velit laoreet id. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed. Tincidunt arcu non sodales neque. Habitant morbi tristique senectus et netus et malesuada fames.").as_ptr() as *mut c_void; + + key.mv_size = 952; + key.mv_data = sval; + + E!(mdb_put(txn, dbi, &mut key, &mut data, 0)); + } + E!(mdb_txn_commit(txn)); } From 92a8cd1570498ebbe48db3750aedb76a8359d3f7 Mon Sep 17 00:00:00 2001 From: Tim Underwood Date: Tue, 28 May 2024 10:23:16 -0700 Subject: [PATCH 2/2] Add Env::max_key_size() method to determine the max supported key length. This calls the LMDB mdb_env_get_maxkeysize function. --- heed/src/env.rs | 25 +++++++++++++++++++++++++ heed/src/mdb/lmdb_ffi.rs | 11 ++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/heed/src/env.rs b/heed/src/env.rs index 21fbcf68..360035f3 100644 --- a/heed/src/env.rs +++ b/heed/src/env.rs @@ -901,6 +901,14 @@ impl Env { mdb_result(unsafe { ffi::mdb_env_set_mapsize(self.env_mut_ptr(), new_size) }) .map_err(Into::into) } + + /// Get the maximum size of keys and MDB_DUPSORT data we can write. + /// + /// Depends on the compile-time constant MDB_MAXKEYSIZE. Default 511 + pub fn max_key_size(&self) -> usize { + let maxsize: i32 = unsafe { ffi::mdb_env_get_maxkeysize(self.env_mut_ptr()) }; + maxsize as usize + } } /// Contains information about the environment. @@ -1251,4 +1259,21 @@ mod tests { env.prepare_for_closing().wait(); } } + + #[test] + fn max_key_size() { + let dir = tempfile::tempdir().unwrap(); + let env = unsafe { EnvOpenOptions::new().open(dir.path().join(dir.path())).unwrap() }; + let maxkeysize = env.max_key_size(); + + eprintln!("maxkeysize: {}", maxkeysize); + + if cfg!(feature = "longer-keys") { + // Should be larger than the default of 511 + assert!(maxkeysize > 511); + } else { + // Should be the default of 511 + assert_eq!(maxkeysize, 511); + } + } } diff --git a/heed/src/mdb/lmdb_ffi.rs b/heed/src/mdb/lmdb_ffi.rs index 1aa03d56..966a53e0 100644 --- a/heed/src/mdb/lmdb_ffi.rs +++ b/heed/src/mdb/lmdb_ffi.rs @@ -3,11 +3,12 @@ use std::ptr; pub use ffi::{ mdb_cursor_close, mdb_cursor_del, mdb_cursor_get, mdb_cursor_open, mdb_cursor_put, mdb_dbi_open, mdb_del, mdb_drop, mdb_env_close, mdb_env_copyfd2, mdb_env_create, - mdb_env_get_fd, mdb_env_get_flags, mdb_env_info, mdb_env_open, mdb_env_set_flags, - mdb_env_set_mapsize, mdb_env_set_maxdbs, mdb_env_set_maxreaders, mdb_env_stat, mdb_env_sync, - mdb_filehandle_t, mdb_get, mdb_put, mdb_reader_check, mdb_set_compare, mdb_stat, mdb_txn_abort, - mdb_txn_begin, mdb_txn_commit, mdb_version, MDB_cursor, MDB_dbi, MDB_env, MDB_stat, MDB_txn, - MDB_val, MDB_CP_COMPACT, MDB_CURRENT, MDB_RDONLY, MDB_RESERVE, + mdb_env_get_fd, mdb_env_get_flags, mdb_env_get_maxkeysize, mdb_env_info, mdb_env_open, + mdb_env_set_flags, mdb_env_set_mapsize, mdb_env_set_maxdbs, mdb_env_set_maxreaders, + mdb_env_stat, mdb_env_sync, mdb_filehandle_t, mdb_get, mdb_put, mdb_reader_check, + mdb_set_compare, mdb_stat, mdb_txn_abort, mdb_txn_begin, mdb_txn_commit, mdb_version, + MDB_cursor, MDB_dbi, MDB_env, MDB_stat, MDB_txn, MDB_val, MDB_CP_COMPACT, MDB_CURRENT, + MDB_RDONLY, MDB_RESERVE, }; use lmdb_master_sys as ffi;