Merge pull request #122 from fjall-rs/2.8.0

2.8.0

Author: marvin-j97

.gitignore

@@ -14,8 +14,5 @@ Cargo.lock
 *.pdb
 
 .lsm.data
-.data
-/old_*
 .test*
-.block_index_test
 .bench

Cargo.toml

@@ -2,7 +2,7 @@
 name = "lsm-tree"
 description = "A K.I.S.S. implementation of log-structured merge trees (LSM-trees/LSMTs)"
 license = "MIT OR Apache-2.0"
-version = "2.7.6"
+version = "2.8.0"
 edition = "2021"
 rust-version = "1.75.0"
 readme = "README.md"
@@ -37,7 +37,7 @@ quick_cache = { version = "0.6.5", default-features = false, features = [] }
 rustc-hash = "2.0.0"
 self_cell = "1.0.4"
 tempfile = "3.12.0"
-value-log = { version = "=1.7.3", default-features = false, features = [] }
+value-log = { version = "~1.8", default-features = false, features = [] }
 varint-rs = "2.2.0"
 xxhash-rust = { version = "0.8.12", features = ["xxh3"] }
 
@@ -101,3 +101,9 @@ name = "fd_table"
 harness = false
 path = "benches/fd_table.rs"
 required-features = []
+
+[[bench]]
+name = "partition_point"
+harness = false
+path = "benches/partition_point.rs"
+required-features = []

README.md

@@ -19,7 +19,7 @@ A K.I.S.S. implementation of log-structured merge trees (LSM-trees/LSMTs) in Rus
 This is the most feature-rich LSM-tree implementation in Rust! It features:
 
 - Thread-safe BTreeMap-like API
-- 100% safe & stable Rust
+- [99.9% safe](./UNSAFE.md) & stable Rust
 - Block-based tables with compression support
 - Range & prefix searching with forward and reverse iteration
 - Size-tiered, (concurrent) Leveled and FIFO compaction 

UNSAFE.md

@@ -0,0 +1,5 @@
+# Unsafe usage
+
+Currently, the project itself only uses one **1** unsafe block (ignoring dependencies which are tested themselves separately):
+
+- https://github.com/fjall-rs/lsm-tree/blob/2d8686e873369bd9c4ff2b562ed988c1cea38331/src/binary_search.rs#L23-L25

benches/partition_point.rs

@@ -0,0 +1,30 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use lsm_tree::binary_search::partition_point;
+use rand::Rng;
+
+fn bench_partition_point(c: &mut Criterion) {
+    let mut group = c.benchmark_group("partition_point");
+
+    let mut rng = rand::rng();
+
+    for item_count in [10, 100, 1_000, 10_000, 100_000, 1_000_000] {
+        let items = (0..item_count).collect::<Vec<_>>();
+
+        group.bench_function(format!("native {item_count}"), |b| {
+            b.iter(|| {
+                let needle = rng.random_range(0..item_count);
+                items.partition_point(|&x| x <= needle)
+            })
+        });
+
+        group.bench_function(format!("rewrite {item_count}"), |b| {
+            b.iter(|| {
+                let needle = rng.random_range(0..item_count);
+                partition_point(&items, |&x| x <= needle)
+            })
+        });
+    }
+}
+
+criterion_group!(benches, bench_partition_point);
+criterion_main!(benches);

fuzz/.gitignore

@@ -1 +1,2 @@
 corpus
+artifacts

fuzz/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "lsm-tree-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2021"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+lsm-tree = { path = ".." }
+
+[[bin]]
+name = "partition_point"
+path = "fuzz_targets/partition_point.rs"
+test = false
+doc = false
+bench = false

fuzz/fuzz_targets/partition_point.rs

@@ -0,0 +1,21 @@
+#![no_main]
+use libfuzzer_sys::{
+    arbitrary::{Arbitrary, Unstructured},
+    fuzz_target,
+};
+use lsm_tree::binary_search::partition_point;
+
+fuzz_target!(|data: &[u8]| {
+    let mut unstructured = Unstructured::new(data);
+
+    if let Ok(mut items) = <Vec<u8> as Arbitrary>::arbitrary(&mut unstructured) {
+        items.sort();
+        items.dedup();
+
+        let mut index = <u8 as Arbitrary>::arbitrary(&mut unstructured).unwrap();
+
+        let idx = partition_point(&items, |&x| x < index);
+        let std_pp_idx = items.partition_point(|&x| x < index);
+        assert_eq!(std_pp_idx, idx);
+    }
+});

src/abstract.rs

@@ -18,6 +18,22 @@ pub type RangeItem = crate::Result<KvPair>;
 #[allow(clippy::module_name_repetitions)]
 #[enum_dispatch]
 pub trait AbstractTree {
+    /// Ingests a sorted stream of key-value pairs into the tree.
+    ///
+    /// Can only be called on a new fresh, empty tree.
+    ///
+    /// # Errors
+    ///
+    /// Will return `Err` if an IO error occurs.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the tree is **not** initially empty.
+    ///
+    /// Will panic if the input iterator is not sorted in ascending order.
+    #[doc(hidden)]
+    fn ingest(&self, iter: impl Iterator<Item = (UserKey, UserValue)>) -> crate::Result<()>;
+
     /// Performs major compaction, blocking the caller until it's done.
     ///
     /// # Errors
@@ -478,7 +494,6 @@ pub trait AbstractTree {
     fn snapshot(&self, seqno: SeqNo) -> Snapshot;
 
     /// Opens a snapshot of this partition with a given sequence number
-    #[must_use]
     fn snapshot_at(&self, seqno: SeqNo) -> Snapshot {
         self.snapshot(seqno)
     }

src/binary_search.rs

@@ -0,0 +1,91 @@
+// Copyright (c) 2024-present, fjall-rs
+// This source code is licensed under both the Apache 2.0 and MIT License
+// (found in the LICENSE-* files in the repository)
+
+/// Returns the index of the partition point according to the given predicate
+/// (the index of the first element of the second partition).
+///
+/// This seems to be faster than std's `partition_point`: <https://github.com/rust-lang/rust/issues/138796>
+pub fn partition_point<T, F>(slice: &[T], pred: F) -> usize
+where
+    F: Fn(&T) -> bool,
+{
+    let mut left = 0;
+    let mut right = slice.len();
+
+    if right == 0 {
+        return 0;
+    }
+
+    while left < right {
+        let mid = (left + right) / 2;
+
+        // SAFETY: See https://github.com/rust-lang/rust/blob/ebf0cf75d368c035f4c7e7246d203bd469ee4a51/library/core/src/slice/mod.rs#L2834-L2836
+        #[warn(unsafe_code)]
+        let item = unsafe { slice.get_unchecked(mid) };
+
+        if pred(item) {
+            left = mid + 1;
+        } else {
+            right = mid;
+        }
+    }
+
+    left
+}
+
+#[cfg(test)]
+mod tests {
+    use super::partition_point;
+    use test_log::test;
+
+    #[test]
+    fn binary_search_first() {
+        let items = [1, 2, 3, 4, 5];
+        let idx = partition_point(&items, |&x| x < 1);
+        assert_eq!(0, idx);
+
+        let std_pp_idx = items.partition_point(|&x| x < 1);
+        assert_eq!(std_pp_idx, idx);
+    }
+
+    #[test]
+    fn binary_search_last() {
+        let items = [1, 2, 3, 4, 5];
+        let idx = partition_point(&items, |&x| x < 5);
+        assert_eq!(4, idx);
+
+        let std_pp_idx = items.partition_point(|&x| x < 5);
+        assert_eq!(std_pp_idx, idx);
+    }
+
+    #[test]
+    fn binary_search_middle() {
+        let items = [1, 2, 3, 4, 5];
+        let idx = partition_point(&items, |&x| x < 3);
+        assert_eq!(2, idx);
+
+        let std_pp_idx = items.partition_point(|&x| x < 3);
+        assert_eq!(std_pp_idx, idx);
+    }
+
+    #[test]
+    fn binary_search_none() {
+        let items = [1, 2, 3, 4, 5];
+        let idx = partition_point(&items, |&x| x < 10);
+        assert_eq!(5, idx);
+
+        let std_pp_idx = items.partition_point(|&x| x < 10);
+        assert_eq!(std_pp_idx, idx);
+    }
+
+    #[test]
+    fn binary_search_empty() {
+        let items: [i32; 0] = [];
+        let idx = partition_point(&items, |&x| x < 10);
+        assert_eq!(0, idx);
+
+        let std_pp_idx = items.partition_point(|&x| x < 10);
+        assert_eq!(std_pp_idx, idx);
+    }
+}