Move combinatorial fns to mmap-bitvec and assume markers are 128 bits

Author: Roderick Bovee

Cargo.toml

@@ -1,19 +1,18 @@
 [package]
 name = "bfield"
-version = "0.1.3"
+version = "0.2.0"
 authors = ["Roderick Bovee <roderick@onecodex.com>"]
 edition = "2018"
 
 [dependencies]
 bincode = "0.9.2"
-mmap-bitvec = { git="ssh://git@github.com/onecodex/mmap-bitvec.git", tag="v0.1.1" }
+mmap-bitvec = { git="ssh://git@github.com/onecodex/mmap-bitvec.git", tag="v0.3.0" }
 murmurhash3 = "0.0.5"
 serde = "1.0.15"
 serde_derive = "1.0.15"
 serde_json = { version = "1.0.4", optional = true}
 
 [features]
-marker_lookup = []
+marker_lookup = ["mmap-bitvec/rank_lookup"]
 prefetching = []
 legacy = ["mmap-bitvec/backward_bytes", "serde_json"]
-wide_markers = ["mmap-bitvec/u128"]

src/bfield.rs

@@ -1,11 +1,11 @@
 use std::io;
 use std::path::Path;
 
+use mmap_bitvec::combinatorial::rank;
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 
 use crate::bfield_member::{BFieldLookup, BFieldMember, BFieldVal};
-use crate::marker::to_marker;
 
 pub struct BField<T> {
     members: Vec<BFieldMember<T>>,
@@ -59,7 +59,7 @@ impl<'a, T: Clone + DeserializeOwned + Serialize> BField<T> {
 
         // Initialize our marker table, so we don't
         // have any race conditions across threads
-        let _ = to_marker(0, n_marker_bits);
+        let _ = rank(0, n_marker_bits);
 
         Ok(BField {
             members,

src/bfield_member.rs

@@ -7,15 +7,14 @@ use std::io;
 use std::path::Path;
 
 use bincode::{deserialize, serialize, Infinite};
-use mmap_bitvec::{BitVecSlice, BitVector, MmapBitVec};
+use mmap_bitvec::combinatorial::{rank, unrank};
+use mmap_bitvec::{BitVector, MmapBitVec};
 use murmurhash3::murmurhash3_x64_128;
 use serde::de::DeserializeOwned;
 use serde::Serialize;
 #[cfg(feature = "legacy")]
 use serde_json;
 
-use crate::marker::{from_marker, to_marker};
-
 #[derive(Debug, Deserialize, Serialize)]
 pub(crate) struct BFieldParams<T> {
     n_hashes: u8,      // k
@@ -59,7 +58,7 @@ impl<T: Clone + DeserializeOwned + Serialize> BFieldMember<T> {
         };
 
         let header: Vec<u8> = serialize(&bf_params, Infinite).unwrap();
-        let bv = MmapBitVec::create(filename, size, &BF_MAGIC, &header)?;
+        let bv = MmapBitVec::create(filename, size, BF_MAGIC, &header)?;
 
         Ok(BFieldMember {
             bitvec: bv,
@@ -136,11 +135,11 @@ impl<T: Clone + DeserializeOwned + Serialize> BFieldMember<T> {
         // TODO: need to do a check that `value` < allowable range based on
         // self.params.marker_width and self.params.n_marker_bits
         let k = self.params.n_marker_bits;
-        self.insert_raw(key, to_marker(value, k));
+        self.insert_raw(key, rank(value as usize, k));
     }
 
     #[inline]
-    fn insert_raw(&mut self, key: &[u8], marker: BitVecSlice) {
+    fn insert_raw(&mut self, key: &[u8], marker: u128) {
         let marker_width = self.params.marker_width as usize;
         let hash = murmurhash3_x64_128(key, 0);
         let aligned_marker = align_bits(marker, marker_width);
@@ -160,7 +159,7 @@ impl<T: Clone + DeserializeOwned + Serialize> BFieldMember<T> {
     /// the correct value; `false` if masking occured or if it was already
     /// indeterminate.
     pub fn mask_or_insert(&mut self, key: &[u8], value: BFieldVal) -> bool {
-        let correct_marker = to_marker(value, self.params.n_marker_bits);
+        let correct_marker = rank(value as usize, self.params.n_marker_bits);
         let k = u32::from(self.params.n_marker_bits);
         let existing_marker = self.get_raw(key, k);
 
@@ -198,16 +197,16 @@ impl<T: Clone + DeserializeOwned + Serialize> BFieldMember<T> {
         let putative_marker = self.get_raw(key, k);
         match putative_marker.count_ones().cmp(&k) {
             Ordering::Greater => BFieldLookup::Indeterminate,
-            Ordering::Equal => BFieldLookup::Some(from_marker(putative_marker)),
+            Ordering::Equal => BFieldLookup::Some(unrank(putative_marker) as u32),
             Ordering::Less => BFieldLookup::None,
         }
     }
 
     #[inline]
-    fn get_raw(&self, key: &[u8], k: u32) -> BitVecSlice {
+    fn get_raw(&self, key: &[u8], k: u32) -> u128 {
         let marker_width = self.params.marker_width as usize;
         let hash = murmurhash3_x64_128(key, 0);
-        let mut merged_marker = BitVecSlice::max_value();
+        let mut merged_marker = u128::max_value();
         let mut positions: [usize; 16] = [0; 16]; // support up to 16 hashes
         for marker_ix in 0usize..self.params.n_hashes as usize {
             let pos = marker_pos(hash, marker_ix, self.bitvec.size(), marker_width);
@@ -217,7 +216,8 @@ impl<T: Clone + DeserializeOwned + Serialize> BFieldMember<T> {
             if cfg!(feature = "prefetching") {
                 unsafe {
                     let byte_idx_st = (pos >> 3) as usize;
-                    let ptr: *const u8 = self.bitvec.mmap.as_ptr().offset(byte_idx_st as isize);
+                    #[allow(unused_variables)]
+                    let ptr: *const u8 = self.bitvec.mmap.as_ptr().add(byte_idx_st);
                     #[cfg(feature = "prefetching")]
                     intrinsics::prefetch_read_data(ptr, 3);
                 }
@@ -247,17 +247,17 @@ impl<T: Clone + DeserializeOwned + Serialize> BFieldMember<T> {
 
 #[cfg(not(feature = "legacy"))]
 #[inline]
-fn align_bits(b: BitVecSlice, _: usize) -> BitVecSlice {
+fn align_bits(b: u128, _: usize) -> u128 {
     // everything is normal if we're not in legacy mode (this is a noop)
     b
 }
 
 #[cfg(feature = "legacy")]
 #[inline]
-fn align_bits(b: BitVecSlice, len: usize) -> BitVecSlice {
+fn align_bits(b: u128, len: usize) -> u128 {
     // we need to reverse the bits (everything is backwards at both the byte
     // and the marker level in the existing nim implementation)
-    let mut new_b = 0 as BitVecSlice;
+    let mut new_b = 0u128;
     for i in 0..len {
         new_b |= (b & (1 << (len - i - 1))) >> (len - i - 1) << i;
     }

src/lib.rs

@@ -6,8 +6,7 @@ extern crate serde_derive;
 
 mod bfield;
 mod bfield_member;
-mod marker;
 
-pub use crate::marker::choose;
 pub use crate::bfield::BField;
 pub use crate::bfield_member::BFieldVal;
+pub use mmap_bitvec::combinatorial::choose;