add mmap backend supports

.github/workflows/ci.yml

@@ -174,9 +174,18 @@ jobs:
         working-directory: integration
 
       # Run with valgrind
-      - name: Run valgrind test-fixed
-        run: valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all ./target/debug/test-fixed
+      - name: Run valgrind test-vec
+        run: valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all ./target/debug/test-vec
         working-directory: integration
+
+      - name: Run valgrind test-mmap
+        run: valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all ./target/debug/test-mmap
+        working-directory: integration
+
+      - name: Run valgrind test-mmap-anon
+        run: valgrind --error-exitcode=1 --leak-check=full --show-leak-kinds=all ./target/debug/test-mmap-anon
+        working-directory: integration
+
 
   miri:
     name: miri

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "skl"
-version = "0.3.1"
+version = "0.4.0"
 edition = "2021"
 repository = "https://github.com/al8n/skl-rs"
 description = "A lock-free thread-safe concurrent ARENA based skiplist implementation which helps develop MVCC memtable for LSM-Tree. Inspired by Dgraph's badger https://github.com/dgraph-io/badger/tree/main/skl."
@@ -22,6 +22,7 @@ name = "loom"
 [features]
 default = ["std"]
 alloc = []
+mmap = ["memmapix", "fs4", "std"]
 std = ["bytes/std", "rand/std", "rand/std_rng", "inline_more"]
 inline_more = []
 js = ["getrandom/js"]
@@ -30,16 +31,19 @@ js = ["getrandom/js"]
 loom = "0.7"
 
 [dependencies]
-bytes = { version = "1.4", default-features = false }
+bytes = { version = "1.5", default-features = false }
 crossbeam-utils = { version = "0.8", default-features = false }
+fs4 = { version = "0.6", optional = true }
 getrandom = { version = "0.2", optional = true }
+memmapix = { version = "0.7", optional = true }
 rand = { version = "0.8", default-features = false, features = ["getrandom"] }
 viewit = "0.1.5"
 
 [dev-dependencies]
 criterion = "0.5"
 tempfile = "3"
 parking_lot = "0.12"
+paste = "1"
 
 [profile.bench]
 opt-level = 3

README.md

@@ -29,12 +29,12 @@ skl = "0.3"
 ## Example
 
 ```rust
-use skl::Skiplist;
+use skl::SkipMap;
 use std::sync::Arc;
 
 fn main() {
     const N: usize = 1000;
-    let l = Arc::new(Skiplist::new(1 << 20));
+    let l = Arc::new(SkipMap::new(1 << 20));
     let wg = Arc::new(());
     for i in 0..N {
         let w = wg.clone();
@@ -97,6 +97,13 @@ fn main() {
 
 - [x] make the crate test cases pass `cargo miri`
 - [ ] make the crate test cases pass `cargo loom`
+- [ ] Implement
+  - [ ] `std::iter::Iterator`
+  - [ ] `get_or_insert`
+  - [ ] `remove`
+  - [ ] `contains`
+  - [ ] change signature from `insert(k, v)` => `insert(k, v) -> Option<ValueRef>`
+  - [ ] mmap backend (currently is vector backend)
 
 #### License
 

benches/bench.rs

@@ -21,7 +21,7 @@ fn fixed_map_round(l: Arc<Mutex<HashMap<Key, Value>>>, case: &(Key, bool), exp:
   }
 }
 
-fn fixed_skiplist_round(l: &Skiplist, case: &(Key, bool), exp: &Value) {
+fn fixed_skiplist_round(l: &SkipMap, case: &(Key, bool), exp: &Value) {
   if case.1 {
     if let Some(v) = l.get(case.0.as_key_ref()) {
       assert_eq!(v.value(), exp.value());
@@ -48,7 +48,7 @@ fn random_key(rng: &mut ThreadRng) -> Key {
 fn bench_read_write_fixed_skiplist_frac(b: &mut Bencher<'_>, frac: &usize) {
   let frac = *frac;
   let value = Value::from(Bytes::from_static(b"00123"));
-  let list = Arc::new(Skiplist::new(512 << 20));
+  let list = Arc::new(SkipMap::new(512 << 20));
   let l = list.clone();
   let stop = Arc::new(AtomicBool::new(false));
   let s = stop.clone();
@@ -170,7 +170,7 @@ fn bench_write_fixed_map(c: &mut Criterion) {
 }
 
 fn bench_write_fixed_skiplist(c: &mut Criterion) {
-  let list = Arc::new(Skiplist::new(512 << 21));
+  let list = Arc::new(SkipMap::new(512 << 21));
   let value = Value::from(Bytes::from_static(b"00123"));
   let l = list.clone();
   let stop = Arc::new(AtomicBool::new(false));

ci/miri.sh

@@ -5,7 +5,7 @@ rustup toolchain install nightly --component miri
 rustup override set nightly
 cargo miri setup
 
-export MIRIFLAGS="-Zmiri-strict-provenance -Zmiri-disable-isolation"
+export MIRIFLAGS="-Zmiri-strict-provenance -Zmiri-disable-isolation -Zmiri-symbolic-alignment-check"
 
 cargo miri test --all-features --target x86_64-unknown-linux-gnu
 cargo miri test --all-features --target aarch64-unknown-linux-gnu

integration/Cargo.toml

@@ -8,4 +8,5 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-skl = { path = "../"}
+skl = { path = "../", features = ["mmap"] }
+tempfile = "3"

integration/src/bin/test-mmap-anon.rs

@@ -0,0 +1,54 @@
+use integration::{big_value, key, new_value};
+use skl::*;
+use std::sync::Arc;
+
+fn main() {
+  {
+    const N: usize = 1000;
+    let l = Arc::new(SkipMap::mmap_anon(1 << 20));
+    for i in 0..N {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        l.insert(key(i), new_value(i));
+        drop(l);
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+    for i in 0..N {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        assert_eq!(
+          l.get(key(i).as_key_ref()).unwrap(),
+          new_value(i).as_value_ref(),
+          "broken: {i}"
+        );
+        drop(l);
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+  }
+
+  {
+    const N2: usize = 100;
+    let l = Arc::new(SkipMap::mmap_anon(120 << 20));
+    for i in 0..N2 {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        l.insert(key(i), big_value(i));
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+    assert_eq!(N2, l.len());
+    for i in 0..N2 {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        assert_eq!(
+          l.get(key(i).as_key_ref()).unwrap(),
+          big_value(i).as_value_ref(),
+          "broken: {i}"
+        );
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+  }
+}

integration/src/bin/test-mmap.rs

@@ -0,0 +1,54 @@
+use integration::{big_value, key, new_value};
+use skl::*;
+use std::sync::Arc;
+
+fn main() {
+  {
+    const N: usize = 1000;
+    let l = Arc::new(SkipMap::mmap(1 << 20, tempfile::tempfile().unwrap(), true));
+    for i in 0..N {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        l.insert(key(i), new_value(i));
+        drop(l);
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+    for i in 0..N {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        assert_eq!(
+          l.get(key(i).as_key_ref()).unwrap(),
+          new_value(i).as_value_ref(),
+          "broken: {i}"
+        );
+        drop(l);
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+  }
+
+  {
+    const N2: usize = 100;
+    let l = Arc::new(SkipMap::mmap(120 << 20, tempfile::tempfile().unwrap(), true));
+    for i in 0..N2 {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        l.insert(key(i), big_value(i));
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+    assert_eq!(N2, l.len());
+    for i in 0..N2 {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        assert_eq!(
+          l.get(key(i).as_key_ref()).unwrap(),
+          big_value(i).as_value_ref(),
+          "broken: {i}"
+        );
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+  }
+}

integration/src/bin/test-vec.rs

@@ -0,0 +1,54 @@
+use integration::{big_value, key, new_value};
+use skl::*;
+use std::sync::Arc;
+
+fn main() {
+  {
+    const N: usize = 1000;
+    let l = Arc::new(SkipMap::new(1 << 20));
+    for i in 0..N {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        l.insert(key(i), new_value(i));
+        drop(l);
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+    for i in 0..N {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        assert_eq!(
+          l.get(key(i).as_key_ref()).unwrap(),
+          new_value(i).as_value_ref(),
+          "broken: {i}"
+        );
+        drop(l);
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+  }
+
+  {
+    const N2: usize = 100;
+    let l = Arc::new(SkipMap::new(120 << 20));
+    for i in 0..N2 {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        l.insert(key(i), big_value(i));
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+    assert_eq!(N2, l.len());
+    for i in 0..N2 {
+      let l = l.clone();
+      std::thread::spawn(move || {
+        assert_eq!(
+          l.get(key(i).as_key_ref()).unwrap(),
+          big_value(i).as_value_ref(),
+          "broken: {i}"
+        );
+      });
+    }
+    while Arc::strong_count(&l) > 1 {}
+  }
+}

src/lib.rs

@@ -12,7 +12,7 @@ extern crate alloc;
 extern crate std;
 
 mod skl;
-pub use crate::skl::{Skiplist, SkiplistIterator, UniSkiplistIterator};
+pub use crate::skl::{SkipMap, SkipMapIterator, UniSkipMapIterator};
 
 mod value;
 pub use value::*;