Dynamic

.github/workflows/ci.yml

@@ -17,6 +17,8 @@ on:
       - 'LICENSE-*'
       - '**.md'
       - '**.txt'
+  workflow_dispatch:
+  schedule: [cron: "0 1 */7 * *"]
 
 env:
   CARGO_TERM_COLOR: always
@@ -85,6 +87,8 @@ jobs:
           - wasm32-unknown-unknown
           - wasm32-unknown-emscripten
           - wasm32-wasip1
+          - wasm32-wasip1-threads
+          - wasm32-wasip2
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -103,13 +107,7 @@ jobs:
       - name: cargo build --target ${{ matrix.target }}
         run: |
           rustup target add ${{ matrix.target }}
-          cargo build --target ${{ matrix.target }}
-        if: matrix.target != 'wasm32-unknown-unknown' && matrix.target != 'wasm32-unknown-emscripten' && matrix.target != 'wasm32-wasip1'
-      - name: cargo build --target ${{ matrix.target }}
-        run: |
-          rustup target add ${{ matrix.target }}
-          cargo build --target ${{ matrix.target }} --no-default-features --features alloc
-        if: matrix.target == 'wasm32-unknown-unknown' || matrix.target == 'wasm32-unknown-emscripten' || matrix.target == 'wasm32-wasip1'
+          cargo build --target ${{ matrix.target }} --no-default-features --features bounded,unbounded,std
 
   build:
     name: build
@@ -176,13 +174,13 @@ jobs:
         path: ~/.cargo
         key: ${{ runner.os }}-coverage-dotcargo
     - name: Run test (Unix)
-      run: RUSTFLAGS="--cfg all_orderwal_tests" cargo test --no-default-features --features alloc
+      run: RUSTFLAGS="--cfg all_orderwal_tests" cargo test --no-default-features --features bounded --lib
       if: matrix.os != 'windows-latest'
     - name: Run test (Windows)
       shell: pwsh
       run: |
         $env:RUSTFLAGS="--cfg all_orderwal_tests"
-        cargo test --no-default-features --features alloc
+        cargo test --no-default-features --features bounded --lib
       if: matrix.os == 'windows-latest'
 
   test:
@@ -258,8 +256,8 @@ jobs:
         run: ci/sanitizer_generic.sh
         if: matrix.os != 'ubuntu-latest'
 
-  miri-tb:
-    name: miri-tb-${{ matrix.target }}-${{ matrix.cfg }}
+  miri-tb-bounded:
+    name: miri-tb-bounded-${{ matrix.target }}-${{ matrix.cfg }}
     strategy:
       matrix:
         os:
@@ -272,14 +270,14 @@ jobs:
           - x86_64-apple-darwin
           - aarch64-apple-darwin
         cfg:
-          - swmr_insert
-          - swmr_iters
-          - swmr_get
-          - swmr_constructor
-          - swmr_multiple_version_insert
-          - swmr_multiple_version_iters
-          - swmr_multiple_version_get
-          - swmr_multiple_version_constructor
+          - generic_insert
+          - generic_iters
+          - generic_get
+          - generic_constructor
+          - dynamic_insert
+          - dynamic_iters
+          - dynamic_get
+          - dynamic_constructor
         # Exclude invalid combinations
         exclude:
           - os: ubuntu-latest
@@ -309,46 +307,42 @@ jobs:
         run: cargo install cargo-hack
       - name: Miri
         run: |
-          bash ci/miri_tb.sh ${{ matrix.target }} ${{ matrix.cfg }}
+          bash ci/miri_tb_bounded.sh ${{ matrix.target }} ${{ matrix.cfg }} 
 
-  # miri-sb:
-  #   name: miri-sb-${{ matrix.target }}-${{ matrix.cfg }}
+  # miri-tb-unbounded:
+  #   name: miri-tb-unbounded-${{ matrix.target }}-${{ matrix.cfg }}
   #   strategy:
   #     matrix:
   #       os:
   #         - ubuntu-latest
-  #         - macos-latest
+  #         # - macos-latest
   #       target:
   #         - x86_64-unknown-linux-gnu
   #         - i686-unknown-linux-gnu
   #         - powerpc64-unknown-linux-gnu
   #         - x86_64-apple-darwin
   #         - aarch64-apple-darwin
   #       cfg:
-  #         - unsync_insert
-  #         - unsync_iters
-  #         - unsync_get
-  #         - unsync_constructor
-  #         - swmr_insert
-  #         - swmr_iters
-  #         - swmr_get
-  #         - swmr_constructor
-  #         - swmr_insert
-  #         - swmr_iters
-  #         - swmr_get
-  #         - swmr_constructor
+  #         - generic_insert
+  #         - generic_iters
+  #         - generic_get
+  #         - generic_constructor
+  #         - dynamic_insert
+  #         - dynamic_iters
+  #         - dynamic_get
+  #         - dynamic_constructor
   #       # Exclude invalid combinations
   #       exclude:
   #         - os: ubuntu-latest
   #           target: x86_64-apple-darwin
   #         - os: ubuntu-latest
   #           target: aarch64-apple-darwin
-  #         - os: macos-latest
-  #           target: x86_64-unknown-linux-gnu
-  #         - os: macos-latest
-  #           target: i686-unknown-linux-gnu
-  #         - os: macos-latest
-  #           target: powerpc64-unknown-linux-gnu
+  #         # - os: macos-latest
+  #         #   target: x86_64-unknown-linux-gnu
+  #         # - os: macos-latest
+  #         #   target: i686-unknown-linux-gnu
+  #         # - os: macos-latest
+  #         #   target: powerpc64-unknown-linux-gnu
   #   runs-on: ${{ matrix.os }}
   #   steps:
   #     - uses: actions/checkout@v4
@@ -366,7 +360,7 @@ jobs:
   #       run: cargo install cargo-hack
   #     - name: Miri
   #       run: |
-  #         bash ci/miri_sb.sh ${{ matrix.target }} ${{ matrix.cfg }}
+  #         bash ci/miri_tb_unbounded.sh ${{ matrix.target }} ${{ matrix.cfg }}
 
   # valgrind
   valgrind:

CHANGELOG.md

@@ -1,4 +1,11 @@
-# Rleases
+# Releases
+
+## 0.6.0 (Dec 28th, 2024)
+
+- Refactor the project to support both dynamic key-value types and generic key-value types
+- Allow users to implement their own memtable
+- Support `range_remove`, `range_set` and `range_unset` operations
+- Make the project comfortable with `-Z miri-strict-provenance`
 
 ## 0.5.0 (Oct 27th, 2024)
 
@@ -9,8 +16,6 @@
 
 ## 0.4.0 (Sep 30th, 2024)
 
-FEATURES
-
 - Support `K: ?Sized` and `V: ?Sized` for `OrderWal`.
 - Use `flush_header_and_range` instead of `flush_range` when insertion.
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "orderwal"
-version = "0.5.1"
+version = "0.6.0"
 edition = "2021"
 repository = "https://github.com/al8n/orderwal"
 homepage = "https://github.com/al8n/orderwal"
@@ -17,10 +17,13 @@ name = "foo"
 harness = false
 
 [features]
-default = ["memmap"]
-alloc = ["rarena-allocator/alloc", "skl/alloc", "dbutils/alloc"]
-std = ["rarena-allocator/default", "crossbeam-skiplist/default", "crossbeam-skiplist-mvcc/default", "bitflags/std", "dbutils/default", "among/default", "skl/std"]
-memmap = ["std", "rarena-allocator/memmap", "skl/memmap"]
+default = ["bounded", "unbounded", "memmap"]
+alloc = ["rarena-allocator/alloc", "skl?/alloc", "dbutils/alloc"]
+std = ["rarena-allocator/default", "crossbeam-skiplist-mvcc?/default", "either/default", "bitflags/std", "dbutils/default", "among/default", "skl?/std"]
+memmap = ["std", "rarena-allocator/memmap", "skl?/memmap"]
+
+bounded = ["skl", "alloc"]
+unbounded = ["crossbeam-skiplist-mvcc", "std"]
 
 xxhash3 = ["dbutils/xxhash3", "std"]
 xxhash64 = ["dbutils/xxhash64", "std"]
@@ -30,16 +33,20 @@ tracing = ["dep:tracing", "dbutils/tracing"]
 [dependencies]
 among = { version = "0.1", default-features = false, features = ["either"] }
 bitflags = { version = "2", default-features = false }
-dbutils = { version = "0.9", default-features = false, features = ["crc32fast"] }
+dbutils = { version = "0.12", default-features = false, features = ["crc32fast", "triomphe01"] }
+# dbutils = { version = "0.12", path = "../layer0/dbutils", default-features = false, features = ["crc32fast", "triomphe01"] }
 derive-where = "1"
+either = { version = "1", default-features = false }
 ref-cast = "1"
 rarena-allocator = { version = "0.4", default-features = false }
-crossbeam-skiplist = { version = "0.1", default-features = false, package = "crossbeam-skiplist-pr1132", optional = true }
-crossbeam-skiplist-mvcc = { version = "0.2", optional = true }
-skl = { version = "0.19", default-features = false, features = ["alloc"] }
+crossbeam-skiplist-mvcc = { version = "0.4", optional = true }
+skl = { version = "0.22", default-features = false, optional = true }
+# skl = { version = "0.22", optional = true, default-features = false, path = "../skl-rs" }
 paste = "1"
-
+zerocopy = { version = "0.8", features = ["derive"] }
 tracing = { version = "0.1", default-features = false, optional = true }
+triomphe = "0.1"
+
 
 [dev-dependencies]
 criterion = "0.5"
@@ -66,14 +73,14 @@ rust_2018_idioms = "warn"
 single_use_lifetimes = "warn"
 unexpected_cfgs = { level = "warn", check-cfg = [
   'cfg(all_orderwal_tests)',
-  'cfg(test_swmr_constructor)',
-  'cfg(test_swmr_insert)',
-  'cfg(test_swmr_iters)',
-  'cfg(test_swmr_get)',
-  'cfg(test_swmr_multiple_version_constructor)',
-  'cfg(test_swmr_multiple_version_insert)',
-  'cfg(test_swmr_multiple_version_iters)',
-  'cfg(test_swmr_multiple_version_get)',
+  'cfg(test_generic_constructor)',
+  'cfg(test_generic_insert)',
+  'cfg(test_generic_iters)',
+  'cfg(test_generic_get)',
+  'cfg(test_dynamic_constructor)',
+  'cfg(test_dynamic_insert)',
+  'cfg(test_dynamic_iters)',
+  'cfg(test_dynamic_get)',
 ] }
 
 [[example]]
@@ -82,11 +89,16 @@ path = "examples/zero_cost.rs"
 required-features = ["memmap"]
 
 [[example]]
-name = "multiple_version"
-path = "examples/multiple_version.rs"
+name = "basic"
+path = "examples/basic.rs"
 required-features = ["memmap"]
 
 [[example]]
 name = "generic_not_sized"
 path = "examples/generic_not_sized.rs"
 required-features = ["memmap"]
+
+[[example]]
+name = "descend"
+path = "examples/descend.rs"
+required-features = ["memmap"]

README.md

@@ -27,25 +27,31 @@ English | [简体中文][zh-cn-url]
 
 ## Installation
 
+> [!TIP]
+> 
+> - `bounded` feature will enable a memory table implementation based on ARENA-style [`SkipMap`](https://github.com/al8n/skl)
+> 
+> - `unbouned` feature (cannot compile without `std`) will enable a memory table implementation based on linked-style [`SkipMap`](https://github.com/al8n/crossbeam-skiplist-mvcc)
+
 - Default (with on-disk support)
 
   ```toml
   [dependencies]
-  orderwal = "0.5"
+  orderwal = "0.6"
   ```
 
-- `std` only (without on-disk support)
+- `std` (without on-disk support)
 
   ```toml
   [dependencies]
-  orderwal = { version = "0.5", default-features = false, features = ["std"] } 
+  orderwal = { version = "0.6", default-features = false, features = ["std", "bounded", "unbounded"] } 
   ```
 
 - `no-std` (`alloc` required)
 
   ```toml
   [dependencies]
-  orderwal = { version = "0.5", default-features = false, features = ["alloc"] }
+  orderwal = { version = "0.6", default-features = false, features = ["alloc", "bounded"] }
   ```
 
 ## Example

ci/miri_tb.sh → ci/miri_tb_bounded.sh

@@ -25,8 +25,6 @@ rustup override set nightly
 cargo miri setup
 
 # Zmiri-ignore-leaks needed because of https://github.com/crossbeam-rs/crossbeam/issues/579
-export MIRIFLAGS="-Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-tree-borrows -Zmiri-ignore-leaks"
+export MIRIFLAGS="-Zmiri-strict-provenance -Zmiri-disable-isolation -Zmiri-symbolic-alignment-check -Zmiri-tree-borrows -Zmiri-ignore-leaks"
 export RUSTFLAGS="--cfg test_$CONFIG_FLAGS"
-
-cargo miri test --tests --target $TARGET --lib 2>&1 | ts -i '%.s  '
-
+cargo miri test --tests --target $TARGET --lib --no-default-features --features bounded,std 2>&1 | ts -i '%.s  '

ci/miri_tb_unbounded.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -euxo pipefail
+IFS=$'\n\t'
+
+# We need 'ts' for the per-line timing
+sudo apt-get -y install moreutils
+echo
+
+# Check if TARGET and CONFIG_FLAGS are provided, otherwise panic
+if [ -z "$1" ]; then
+  echo "Error: TARGET is not provided"
+  exit 1
+fi
+
+if [ -z "$2" ]; then
+  echo "Error: CONFIG_FLAGS are not provided"
+  exit 1
+fi
+
+TARGET=$1
+CONFIG_FLAGS=$2
+
+rustup toolchain install nightly --component miri
+rustup override set nightly
+cargo miri setup
+
+# Zmiri-ignore-leaks needed because of https://github.com/crossbeam-rs/crossbeam/issues/579
+# -Zmiri-strict-provenance (crossbeam-epoch is not compatible with this flag) 
+RUSTFLAGS="--cfg test_$CONFIG_FLAGS" \
+MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-symbolic-alignment-check -Zmiri-tree-borrows -Zmiri-ignore-leaks" \
+cargo miri test --tests --target $TARGET --lib --no-default-features --features unbounded,std 2>&1 | ts -i '%.s  '
+