0.7.0

.github/workflows/test.yml

@@ -41,14 +41,14 @@ jobs:
           workspaces: >
             . -> target
             examples/kv -> target
-      - name: Build
-        run: cargo build -v
+      - name: Install cargo-all-features
+        run: cargo install cargo-all-features
       - name: Format
         run: cargo fmt --all -- --check
       - name: Clippy
         run: cargo clippy
       - name: Run tests
-        run: cargo test -v -- --nocapture
+        run: cargo test-all-features -v -- --nocapture
         env:
           RUST_LOG: debug
       - name: Build & test LSM examples

.gitignore

@@ -16,4 +16,6 @@ Cargo.lock
 .lsm.data
 .data
 /old_*
-.test
+.test*
+segment_history.jsonl
+.block_index_test

Cargo.toml

@@ -19,28 +19,26 @@ path = "src/lib.rs"
 [features]
 default = []
 bloom = ["dep:seahash"]
-segment_history = []
+segment_history = ["dep:serde", "dep:serde_json"]
 
 [dependencies]
 byteorder = "1.5.0"
-chrono = "0.4.38"
 crc32fast = "1.4.0"
 crossbeam-skiplist = "0.1.3"
 double-ended-peekable = "0.1.0"
-fs_extra = "1.3.0"
 guardian = "1.1.0"
 log = "0.4.21"
 lz4_flex = "0.11.3"
 path-absolutize = "3.1.1"
 quick_cache = { version = "0.5.1", default-features = false, features = [] }
-rand = "0.8.5"
 seahash = { version = "4.1.0", optional = true }
-serde = { version = "1.0.200", features = ["derive", "rc"] }
-serde_json = "1.0.116"
+serde = { version = "1.0.200", features = ["derive", "rc"], optional = true }
+serde_json = { version = "1.0.116", optional = true }
 tempfile = "3.10.1"
 
 [dev-dependencies]
 criterion = { version = "0.5.1", features = ["html_reports"] }
+fs_extra = "1.3.0"
 nanoid = "0.4.0"
 test-log = "0.2.16"
 

benches/lsmt.rs

@@ -5,6 +5,28 @@ use lsm_tree::{
 use nanoid::nanoid;
 use std::{io::Write, sync::Arc};
 
+fn iterate_level_manifest(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Iterate level manifest");
+
+    for segment_count in [0, 1, 5, 10, 20, 50, 100, 250, 500, 1_000] {
+        let folder = tempfile::tempdir().unwrap();
+        let tree = Config::new(folder).block_size(1_024).open().unwrap();
+
+        for x in 0..segment_count {
+            tree.insert("a", "b", x as u64);
+            tree.flush_active_memtable().unwrap();
+        }
+
+        group.bench_function(&format!("iterate {segment_count} segments"), |b| {
+            let levels = tree.levels.read().unwrap();
+
+            b.iter(|| {
+                assert_eq!(levels.iter().count(), segment_count);
+            });
+        });
+    }
+}
+
 fn memtable_get_upper_bound(c: &mut Criterion) {
     let memtable = MemTable::default();
 
@@ -24,6 +46,66 @@ fn memtable_get_upper_bound(c: &mut Criterion) {
     });
 }
 
+fn tli_find_item(c: &mut Criterion) {
+    use lsm_tree::segment::block_index::{
+        block_handle::KeyedBlockHandle, top_level::TopLevelIndex,
+    };
+
+    let mut group = c.benchmark_group("TLI find item");
+
+    for item_count in [10u64, 100, 1_000, 10_000, 100_000, 1_000_000] {
+        let items = {
+            let mut items = Vec::with_capacity(item_count as usize);
+
+            for x in 0..item_count {
+                items.push(KeyedBlockHandle {
+                    start_key: x.to_be_bytes().into(),
+                    offset: x,
+                    size: 0,
+                });
+            }
+
+            items
+        };
+
+        let index = TopLevelIndex::from_boxed_slice(items.into());
+
+        group.bench_function(
+            format!("TLI get_next_block_handle ({item_count} items)"),
+            |b| {
+                let key = (item_count / 10 * 6).to_be_bytes();
+                let expected: Arc<[u8]> = (item_count / 10 * 6 + 1).to_be_bytes().into();
+
+                let block = index.get_lowest_block_containing_item(&key).unwrap();
+
+                b.iter(|| {
+                    assert_eq!(
+                        expected,
+                        index.get_next_block_handle(block.offset).unwrap().start_key
+                    );
+                })
+            },
+        );
+
+        group.bench_function(
+            format!("TLI get_block_containing_item ({item_count} items)"),
+            |b| {
+                let key = (item_count / 10 * 6).to_be_bytes();
+
+                b.iter(|| {
+                    assert_eq!(
+                        key,
+                        &*index
+                            .get_lowest_block_containing_item(&key)
+                            .unwrap()
+                            .start_key
+                    );
+                })
+            },
+        );
+    }
+}
+
 fn value_block_size(c: &mut Criterion) {
     let mut group = c.benchmark_group("ValueBlock::size");
 
@@ -49,6 +131,30 @@ fn value_block_size(c: &mut Criterion) {
     }
 }
 
+fn value_block_size_find(c: &mut Criterion) {
+    use lsm_tree::segment::block_index::{block_handle::KeyedBlockHandle, IndexBlock};
+
+    let mut group = c.benchmark_group("Find item in BlockHandleBlock");
+
+    // NOTE: Anything above 1000 is unlikely
+    for item_count in [10, 100, 500, 1_000] {
+        group.bench_function(format!("{item_count} items"), |b| {
+            let items = (0u64..item_count)
+                .map(|x| KeyedBlockHandle {
+                    start_key: x.to_be_bytes().into(),
+                    offset: 56,
+                    size: 635,
+                })
+                .collect();
+
+            let block = IndexBlock { items, crc: 0 };
+            let key = &0u64.to_be_bytes();
+
+            b.iter(|| block.get_lowest_block_containing_item(key))
+        });
+    }
+}
+
 fn load_block_from_disk(c: &mut Criterion) {
     let mut group = c.benchmark_group("Load block from disk");
 
@@ -102,7 +208,7 @@ fn load_block_from_disk(c: &mut Criterion) {
     }
 }
 
-fn file_descriptor(c: &mut Criterion) {
+fn file_descriptor_table(c: &mut Criterion) {
     use std::fs::File;
 
     let file = tempfile::NamedTempFile::new().unwrap();
@@ -115,9 +221,9 @@ fn file_descriptor(c: &mut Criterion) {
         });
     });
 
-    let id: Arc<str> = Arc::from("file");
+    let id = (0, 523).into();
     let descriptor_table = lsm_tree::descriptor_table::FileDescriptorTable::new(1, 1);
-    descriptor_table.insert(file.path(), id.clone());
+    descriptor_table.insert(file.path(), id);
 
     group.bench_function("descriptor table", |b: &mut criterion::Bencher<'_>| {
         b.iter(|| {
@@ -250,13 +356,15 @@ fn tree_get_pairs(c: &mut Criterion) {
 
 criterion_group!(
     benches,
+    tli_find_item,
     memtable_get_upper_bound,
+    value_block_size_find,
     value_block_size,
     load_block_from_disk,
-    file_descriptor,
+    file_descriptor_table,
     bloom_filter_construction,
     bloom_filter_contains,
     tree_get_pairs,
-    // first_kv_disjoint
+    iterate_level_manifest,
 );
 criterion_main!(benches);

src/bit_array.rs

@@ -20,7 +20,7 @@ fn set_bit(byte: u8, idx: usize, value: bool) -> u8 {
 }
 
 /// Fixed-size bit array
-#[derive(Debug)]
+#[derive(Debug, Eq, PartialEq)]
 pub struct BitArray(Box<[u8]>);
 
 impl BitArray {

src/block_cache.rs

@@ -1,12 +1,10 @@
-use crate::segment::block_index::block_handle::BlockHandle;
-use crate::segment::{block::ValueBlock, block_index::BlockHandleBlock};
-use crate::{
-    either::{
-        Either,
-        Either::{Left, Right},
-    },
-    value::UserKey,
+use crate::either::{
+    Either,
+    Either::{Left, Right},
 };
+use crate::segment::block_index::block_handle::KeyedBlockHandle;
+use crate::segment::id::GlobalSegmentId;
+use crate::segment::{block::ValueBlock, block_index::IndexBlock};
 use quick_cache::Weighter;
 use quick_cache::{sync::Cache, Equivalent};
 use std::sync::Arc;
@@ -17,30 +15,30 @@ enum BlockTag {
     Index = 1,
 }
 
-type Item = Either<Arc<ValueBlock>, Arc<BlockHandleBlock>>;
+type Item = Either<Arc<ValueBlock>, Arc<IndexBlock>>;
 
-// (Type (disk or index), Segment ID, Block key)
+// (Type (disk or index), Segment ID, Block offset)
 #[derive(Eq, std::hash::Hash, PartialEq)]
-struct CacheKey((BlockTag, Arc<str>, UserKey));
+struct CacheKey((BlockTag, GlobalSegmentId, u64));
 
-impl From<(BlockTag, Arc<str>, UserKey)> for CacheKey {
-    fn from(value: (BlockTag, Arc<str>, UserKey)) -> Self {
+impl From<(BlockTag, GlobalSegmentId, u64)> for CacheKey {
+    fn from(value: (BlockTag, GlobalSegmentId, u64)) -> Self {
         Self(value)
     }
 }
 
 impl std::ops::Deref for CacheKey {
-    type Target = (BlockTag, Arc<str>, UserKey);
+    type Target = (BlockTag, GlobalSegmentId, u64);
 
     fn deref(&self) -> &Self::Target {
         &self.0
     }
 }
 
-impl Equivalent<CacheKey> for (BlockTag, &str, &UserKey) {
+impl Equivalent<CacheKey> for (BlockTag, GlobalSegmentId, &u64) {
     fn equivalent(&self, key: &CacheKey) -> bool {
         let inner = &**key;
-        self.0 == inner.0 && self.1 == &*inner.1 && self.2 == &inner.2
+        self.0 == inner.0 && self.1 == inner.1 && self.2 == &inner.2
     }
 }
 
@@ -56,7 +54,7 @@ impl Weighter<CacheKey, Item> for BlockWeighter {
             Either::Right(block) => block
                 .items
                 .iter()
-                .map(|x| x.start_key.len() + std::mem::size_of::<BlockHandle>())
+                .map(|x| x.start_key.len() + std::mem::size_of::<KeyedBlockHandle>())
                 .sum::<usize>() as u32,
         }
     }
@@ -120,42 +118,51 @@ impl BlockCache {
     }
 
     #[doc(hidden)]
-    pub fn insert_disk_block(&self, segment_id: Arc<str>, key: UserKey, value: Arc<ValueBlock>) {
+    pub fn insert_disk_block(
+        &self,
+        segment_id: GlobalSegmentId,
+        offset: u64,
+        value: Arc<ValueBlock>,
+    ) {
         if self.capacity > 0 {
             self.data
-                .insert((BlockTag::Data, segment_id, key).into(), Left(value));
+                .insert((BlockTag::Data, segment_id, offset).into(), Left(value));
         }
     }
 
     #[doc(hidden)]
-    pub fn insert_block_handle_block(
+    pub fn insert_index_block(
         &self,
-        segment_id: Arc<str>,
-        key: UserKey,
-        value: Arc<BlockHandleBlock>,
+        segment_id: GlobalSegmentId,
+        offset: u64,
+        value: Arc<IndexBlock>,
     ) {
         if self.capacity > 0 {
             self.data
-                .insert((BlockTag::Index, segment_id, key).into(), Right(value));
+                .insert((BlockTag::Index, segment_id, offset).into(), Right(value));
         }
     }
 
     #[doc(hidden)]
     #[must_use]
-    pub fn get_disk_block(&self, segment_id: &str, key: &UserKey) -> Option<Arc<ValueBlock>> {
-        let key = (BlockTag::Data, segment_id, key);
+    pub fn get_disk_block(
+        &self,
+        segment_id: GlobalSegmentId,
+        offset: u64,
+    ) -> Option<Arc<ValueBlock>> {
+        let key = (BlockTag::Data, segment_id, &offset);
         let item = self.data.get(&key)?;
         Some(item.left().clone())
     }
 
     #[doc(hidden)]
     #[must_use]
-    pub fn get_block_handle_block(
+    pub fn get_index_block(
         &self,
-        segment_id: &str,
-        key: &UserKey,
-    ) -> Option<Arc<BlockHandleBlock>> {
-        let key = (BlockTag::Index, segment_id, key);
+        segment_id: GlobalSegmentId,
+        offset: u64,
+    ) -> Option<Arc<IndexBlock>> {
+        let key = (BlockTag::Index, segment_id, &offset);
         let item = self.data.get(&key)?;
         Some(item.right().clone())
     }