Preparation for open sourcing (#15)

General preparation for open sourcing

Author: GSGerritsen

LICENSE.txt renamed to LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) Reference Genomics, Inc.
+Copyright (c) 2022 One Codex, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -2,21 +2,44 @@
 
 ![ci](https://github.com/onecodex/mmap-bitvec/workflows/ci/badge.svg)
 
-mmap-bitvec is a library for working with mmap-backed bit-vectors and some simple
-data structures derived from bit-vectors.
+`mmap-bitvec` is a library for working with bit-vectors backed by memory-mapped files. Included is a simple Bloom filter built on top of such bit-vectors.
 
-## Benchmarks
-
-To run benchmarks you need to download a bfield.mmap file, I used `s3://refgenomics-datafiles/dbs/mg_targeted_loci_20160517/bfield.mmap` in
-the root of the repo and then run `cargo +nightly bench`.
-
-## Example
+## Examples
 
+Using a memory-mapped bit-vector:
 ```rust
-    let mut b = BitVec::from_memory(128).unwrap();
+    // Build an in-memory bit-vector with a capacity of 128 bits.
+    use mmap_bitvec::{MmapBitVec, BitVector};
+
+    let mut bitvec = MmapBitVec::from_memory(128).unwrap();
+    bitvec.set(2, true);
+    assert!(bitvec.get(2));
+    assert_eq!(bitvec.get_range(0..8), 0b00100000);
+
+    // Write the bit-vector to disk, passing in `None` where an optional magic bytes arg can be set
+    let dir = tempfile::tempdir().unwrap();
+    bitvec.save_to_disk(dir.path().join("test"), None, &[])
+        .unwrap();
+    let f = MmapBitVec::open(dir.path().join("test"), None, false).unwrap();
+    assert_eq!(f.get(2), true);
+```
 
-    b.set(2, true);
-    assert!(b.get(2));
-    assert_eq!(b.get_range(0..8), 0b00100000);
+Using the Bloom filter:
+```rust,no_run
+    use mmap_bitvec::{BloomFilter};
+    // Create a Bloom filter with a capacity of 100 bits that uses 2 hash functions on each insert.
+    let mut filter = BloomFilter::new(Some("./test.bloom"), 100, 2).unwrap();
+    let (a, b) = (1, 2);
+    assert!(!filter.contains(a));
+    assert!(!filter.contains(b));
+
+    filter.insert(b);
+    assert!(!filter.contains(a));
+    assert!(filter.contains(b));
 ```
 
+## License
+
+This project is licensed under the [MIT license].
+
+[MIT license]: https://github.com/onecodex/mmap-bitvec/blob/master/LICENSE

benches/benchmark.rs

@@ -7,16 +7,15 @@ use std::mem::transmute;
 use std::ops::Range;
 
 use memmap2::{MmapMut, MmapOptions};
-use mmap_bitvec::{combinatorial::rank, BitVector, MmapBitVec};
+use mmap_bitvec::{BitVector, MmapBitVec};
 
 use criterion::{criterion_group, criterion_main, Criterion};
 
 type BitVecSlice = u64;
+
 const BIT_VEC_SLICE_SIZE: u8 = 64;
-const FILENAME: &str = "./bfield.mmap";
+const FILENAME: &str = "./data/bfield.mmap";
 
-// we could use an RNG, but I want to make sure everything is
-// as comparable as possible
 fn next_random(n: usize) -> usize {
     // https://en.wikipedia.org/wiki/Xorshift
     let mut x = n as u32;
@@ -40,8 +39,6 @@ fn get_range_simplified(mmap: &MmapMut, size: usize, l: usize) -> BitVecSlice {
     v >>= 7 - ((l + 63) & 7);
 
     if l < size - BIT_VEC_SLICE_SIZE as usize {
-        // really nasty/unsafe, but we're just reading a u64/u128 out instead of doing it
-        // byte-wise --- also does not work with legacy mode!!!
         unsafe {
             let lg_ptr: *const BitVecSlice = transmute(ptr.offset(byte_idx_st as isize));
             v |= (*lg_ptr).to_be() << (l & 7) >> (BIT_VEC_SLICE_SIZE - new_size);
@@ -82,8 +79,6 @@ fn get_range(mmap: &MmapMut, size: usize, r: Range<usize>) -> BitVecSlice {
     v >>= 7 - ((r.end - 1) & 7);
 
     if r.start < size - BIT_VEC_SLICE_SIZE as usize {
-        // really nasty/unsafe, but we're just reading a u64/u128 out instead of doing it
-        // byte-wise --- also does not work with legacy mode!!!
         unsafe {
             let lg_ptr: *const BitVecSlice = transmute(ptr.offset(byte_idx_st as isize));
             v |= (*lg_ptr).to_be() << (r.start & 7) >> (BIT_VEC_SLICE_SIZE - new_size);
@@ -114,7 +109,7 @@ fn bench_get_range_simplified() {
 
     let mut r = 0;
     let mut i = 1;
-    for _ in 0..100000 {
+    for _ in 0..100_000 {
         let l = i % (size - 64);
         r += get_range_simplified(&mmap, size, l).count_ones();
         i = next_random(i);
@@ -132,7 +127,7 @@ fn bench_get_range() {
 
     let mut r = 0;
     let mut i = 1;
-    for _ in 0..100000 {
+    for _ in 0..100_000 {
         let l = i % (size - 64);
         r += get_range(&mmap, size, l..l + 64).count_ones();
         i = next_random(i);
@@ -143,15 +138,15 @@ fn bench_get_range_actual() {
     let bitvec = MmapBitVec::open_no_header(FILENAME, 0).unwrap();
     let mut r = 0;
     let mut i = 1;
-    for _ in 0..100000 {
+    for _ in 0..100_000 {
         let l = i % (bitvec.size() - 64);
         r += bitvec.get_range(l..l + 64).count_ones();
         i = next_random(i);
     }
 }
 
 fn bench_save_to_disk(bv: &MmapBitVec) {
-    bv.save_to_disk("hello.tmp", [0, 1], &[]).unwrap();
+    bv.save_to_disk("hello.tmp", Some([0, 1]), &[]).unwrap();
 }
 
 fn criterion_benchmark(c: &mut Criterion) {
@@ -161,12 +156,14 @@ fn criterion_benchmark(c: &mut Criterion) {
     });
     c.bench_function("get_range", |b| b.iter(|| bench_get_range()));
     c.bench_function("save_to_disk", |b| {
-        let mut bitvec = MmapBitVec::from_memory(1_000_000_000).unwrap();
-        for i in 0..1_000_000_000 {
+        let mut bitvec = MmapBitVec::from_memory(100_000_000).unwrap();
+        for i in 0..100_000_000 {
             bitvec.set(i, true);
         }
         b.iter(|| bench_save_to_disk(&bitvec))
     });
+    // clean up temp file
+    std::fs::remove_file("./hello.tmp").unwrap();
 }
 
 criterion_group!(benches, criterion_benchmark);

src/bitvec.rs

@@ -1,6 +1,6 @@
 use std::ops::Range;
 
-/// A nasic bitvector trait that we implement for mmap
+/// A basic bitvector trait that we implement for mmap
 pub trait BitVector {
     /// Get the value at bit `i`
     fn get(&self, i: usize) -> bool;
@@ -14,7 +14,7 @@ pub trait BitVector {
         r.fold(0, |a, x| a + if self.get(x) { 1 } else { 0 })
     }
 
-    /// Returns the position of the n-th bit set
+    /// Returns the position of the n-th set bit
     fn select(&self, n: usize, start: usize) -> Option<usize> {
         let mut bits_left = n;
 
@@ -30,7 +30,10 @@ pub trait BitVector {
         None
     }
 
-    /// Return all the bits in the given range as a u128
+    /// Return all the bits in the given range as a `u128`. The input range `r` must span `<= 128`,
+    /// as the result is bitpacked into a `u128`.
+    ///
+    /// For example, an input range of `(0, 7)` will set the first 8 bits of the returned `u128` to the result of `self.get(0, 1, ... 7)`.
     fn get_range(&self, r: Range<usize>) -> u128 {
         if r.end - r.start > 128 {
             panic!("Range too large (>128)")
@@ -49,7 +52,7 @@ pub trait BitVector {
         bvs
     }
 
-    /// Sets all the bits in the given range from the given u128
+    /// Sets all the bits in the given range from the given `u128`
     fn set_range(&mut self, r: Range<usize>, x: u128) {
         let mut cur = x;
         for i in r.rev() {

src/bloom.rs

@@ -7,9 +7,9 @@ use murmurhash3::murmurhash3_x64_128;
 use crate::bitvec::BitVector;
 use crate::mmap_bitvec::MmapBitVec;
 
-/// Newtype for murmur hashing
-/// we don't want to use murmurhash3::Murmur3Hasher b/c it makes copies of the
-/// bytes to be hashed with every single `hash` call
+/// Newtype for murmur hashing.
+/// We don't use murmurhash3::Murmur3Hasher because it makes copies of the
+/// bytes to be hashed on every `hash` call
 #[derive(Default)]
 pub struct MurmurHasher(u64, u64);
 
@@ -30,7 +30,7 @@ impl Hasher for MurmurHasher {
         let hash = murmurhash3_x64_128(bytes, self.0);
         *self = MurmurHasher(hash.0, hash.1);
     }
-    // have to provide this to fulfill the trait requirements
+
     fn finish(&self) -> u64 {
         self.0
     }
@@ -43,9 +43,9 @@ pub struct BloomFilter {
 }
 
 impl BloomFilter {
-    /// Creates a new Bloom filter (or opens an existing one, if the file
-    /// already exists) of a given size (bits) and with a given number of
-    /// hash functions for each insert (n_hashes). If a filename is not
+    /// Creates a new `BloomFilter` (or opens an existing one, if the file
+    /// already exists) of a given size (in bits) and with a given number of
+    /// hash functions for each insert (`n_hashes`). If a filename is not
     /// passed, the Bloom filter will be created in memory.
     pub fn new<P>(filename: Option<P>, bits: usize, n_hashes: u8) -> Result<Self, io::Error>
     where
@@ -57,7 +57,7 @@ impl BloomFilter {
                 if Path::exists(filename.as_ref()) {
                     MmapBitVec::open(&filename, Some(b"!!"), false)?
                 } else {
-                    MmapBitVec::create(&filename, bits, *b"!!", &header)?
+                    MmapBitVec::create(&filename, bits, Some(*b"!!"), &header)?
                 }
             }
             None => MmapBitVec::from_memory(bits)?,
@@ -68,7 +68,7 @@ impl BloomFilter {
         })
     }
 
-    /// Insert an item into the bloom filter.
+    /// Insert an item into the Bloom filter.
     pub fn insert<H>(&mut self, item: H)
     where
         H: Hash,
@@ -82,7 +82,7 @@ impl BloomFilter {
         }
     }
 
-    /// Check if an item is in the bloom filter already.
+    /// Check if an item is in the Bloom filter already.
     pub fn contains<H>(&self, item: H) -> bool
     where
         H: Hash,