feat: refactors dataset

Author: Volodymyr Orlov

src/dataset/boston.rs

@@ -15,61 +15,21 @@
 //! | Petal width | Numerical | No |
 //! | Class | Nominal | Yes |
 //!
+use crate::dataset::deserialize_data;
 use crate::dataset::Dataset;
 
 /// Get dataset
 pub fn load_dataset() -> Dataset<f32, f32> {
-    let x = vec![
-        5.1, 3.5, 1.4, 0.2, 4.9, 3.0, 1.4, 0.2, 4.7, 3.2, 1.3, 0.2, 4.6, 3.1, 1.5, 0.2, 5.0, 3.6,
-        1.4, 0.2, 5.4, 3.9, 1.7, 0.4, 4.6, 3.4, 1.4, 0.3, 5.0, 3.4, 1.5, 0.2, 4.4, 2.9, 1.4, 0.2,
-        4.9, 3.1, 1.5, 0.1, 5.4, 3.7, 1.5, 0.2, 4.8, 3.4, 1.6, 0.2, 4.8, 3.0, 1.4, 0.1, 4.3, 3.0,
-        1.1, 0.1, 5.8, 4.0, 1.2, 0.2, 5.7, 4.4, 1.5, 0.4, 5.4, 3.9, 1.3, 0.4, 5.1, 3.5, 1.4, 0.3,
-        5.7, 3.8, 1.7, 0.3, 5.1, 3.8, 1.5, 0.3, 5.4, 3.4, 1.7, 0.2, 5.1, 3.7, 1.5, 0.4, 4.6, 3.6,
-        1.0, 0.2, 5.1, 3.3, 1.7, 0.5, 4.8, 3.4, 1.9, 0.2, 5.0, 3.0, 1.6, 0.2, 5.0, 3.4, 1.6, 0.4,
-        5.2, 3.5, 1.5, 0.2, 5.2, 3.4, 1.4, 0.2, 4.7, 3.2, 1.6, 0.2, 4.8, 3.1, 1.6, 0.2, 5.4, 3.4,
-        1.5, 0.4, 5.2, 4.1, 1.5, 0.1, 5.5, 4.2, 1.4, 0.2, 4.9, 3.1, 1.5, 0.1, 5.0, 3.2, 1.2, 0.2,
-        5.5, 3.5, 1.3, 0.2, 4.9, 3.1, 1.5, 0.1, 4.4, 3.0, 1.3, 0.2, 5.1, 3.4, 1.5, 0.2, 5.0, 3.5,
-        1.3, 0.3, 4.5, 2.3, 1.3, 0.3, 4.4, 3.2, 1.3, 0.2, 5.0, 3.5, 1.6, 0.6, 5.1, 3.8, 1.9, 0.4,
-        4.8, 3.0, 1.4, 0.3, 5.1, 3.8, 1.6, 0.2, 4.6, 3.2, 1.4, 0.2, 5.3, 3.7, 1.5, 0.2, 5.0, 3.3,
-        1.4, 0.2, 7.0, 3.2, 4.7, 1.4, 6.4, 3.2, 4.5, 1.5, 6.9, 3.1, 4.9, 1.5, 5.5, 2.3, 4.0, 1.3,
-        6.5, 2.8, 4.6, 1.5, 5.7, 2.8, 4.5, 1.3, 6.3, 3.3, 4.7, 1.6, 4.9, 2.4, 3.3, 1.0, 6.6, 2.9,
-        4.6, 1.3, 5.2, 2.7, 3.9, 1.4, 5.0, 2.0, 3.5, 1.0, 5.9, 3.0, 4.2, 1.5, 6.0, 2.2, 4.0, 1.0,
-        6.1, 2.9, 4.7, 1.4, 5.6, 2.9, 3.6, 1.3, 6.7, 3.1, 4.4, 1.4, 5.6, 3.0, 4.5, 1.5, 5.8, 2.7,
-        4.1, 1.0, 6.2, 2.2, 4.5, 1.5, 5.6, 2.5, 3.9, 1.1, 5.9, 3.2, 4.8, 1.8, 6.1, 2.8, 4.0, 1.3,
-        6.3, 2.5, 4.9, 1.5, 6.1, 2.8, 4.7, 1.2, 6.4, 2.9, 4.3, 1.3, 6.6, 3.0, 4.4, 1.4, 6.8, 2.8,
-        4.8, 1.4, 6.7, 3.0, 5.0, 1.7, 6.0, 2.9, 4.5, 1.5, 5.7, 2.6, 3.5, 1.0, 5.5, 2.4, 3.8, 1.1,
-        5.5, 2.4, 3.7, 1.0, 5.8, 2.7, 3.9, 1.2, 6.0, 2.7, 5.1, 1.6, 5.4, 3.0, 4.5, 1.5, 6.0, 3.4,
-        4.5, 1.6, 6.7, 3.1, 4.7, 1.5, 6.3, 2.3, 4.4, 1.3, 5.6, 3.0, 4.1, 1.3, 5.5, 2.5, 4.0, 1.3,
-        5.5, 2.6, 4.4, 1.2, 6.1, 3.0, 4.6, 1.4, 5.8, 2.6, 4.0, 1.2, 5.0, 2.3, 3.3, 1.0, 5.6, 2.7,
-        4.2, 1.3, 5.7, 3.0, 4.2, 1.2, 5.7, 2.9, 4.2, 1.3, 6.2, 2.9, 4.3, 1.3, 5.1, 2.5, 3.0, 1.1,
-        5.7, 2.8, 4.1, 1.3, 6.3, 3.3, 6.0, 2.5, 5.8, 2.7, 5.1, 1.9, 7.1, 3.0, 5.9, 2.1, 6.3, 2.9,
-        5.6, 1.8, 6.5, 3.0, 5.8, 2.2, 7.6, 3.0, 6.6, 2.1, 4.9, 2.5, 4.5, 1.7, 7.3, 2.9, 6.3, 1.8,
-        6.7, 2.5, 5.8, 1.8, 7.2, 3.6, 6.1, 2.5, 6.5, 3.2, 5.1, 2.0, 6.4, 2.7, 5.3, 1.9, 6.8, 3.0,
-        5.5, 2.1, 5.7, 2.5, 5.0, 2.0, 5.8, 2.8, 5.1, 2.4, 6.4, 3.2, 5.3, 2.3, 6.5, 3.0, 5.5, 1.8,
-        7.7, 3.8, 6.7, 2.2, 7.7, 2.6, 6.9, 2.3, 6.0, 2.2, 5.0, 1.5, 6.9, 3.2, 5.7, 2.3, 5.6, 2.8,
-        4.9, 2.0, 7.7, 2.8, 6.7, 2.0, 6.3, 2.7, 4.9, 1.8, 6.7, 3.3, 5.7, 2.1, 7.2, 3.2, 6.0, 1.8,
-        6.2, 2.8, 4.8, 1.8, 6.1, 3.0, 4.9, 1.8, 6.4, 2.8, 5.6, 2.1, 7.2, 3.0, 5.8, 1.6, 7.4, 2.8,
-        6.1, 1.9, 7.9, 3.8, 6.4, 2.0, 6.4, 2.8, 5.6, 2.2, 6.3, 2.8, 5.1, 1.5, 6.1, 2.6, 5.6, 1.4,
-        7.7, 3.0, 6.1, 2.3, 6.3, 3.4, 5.6, 2.4, 6.4, 3.1, 5.5, 1.8, 6.0, 3.0, 4.8, 1.8, 6.9, 3.1,
-        5.4, 2.1, 6.7, 3.1, 5.6, 2.4, 6.9, 3.1, 5.1, 2.3, 5.8, 2.7, 5.1, 1.9, 6.8, 3.2, 5.9, 2.3,
-        6.7, 3.3, 5.7, 2.5, 6.7, 3.0, 5.2, 2.3, 6.3, 2.5, 5.0, 1.9, 6.5, 3.0, 5.2, 2.0, 6.2, 3.4,
-        5.4, 2.3, 5.9, 3.0, 5.1, 1.8,
-    ];
-
-    let setosa = std::iter::repeat(0f32).take(50);
-    let versicolor = std::iter::repeat(1f32).take(50);
-    let virginica = std::iter::repeat(2f32).take(50);
-    let y = setosa
-        .chain(versicolor)
-        .chain(virginica)
-        .collect::<Vec<f32>>();
-    let shape = (150, 4);
+    let (x, y, num_samples, num_features) = match deserialize_data(std::include_bytes!("iris.xy")) {
+        Err(why) => panic!("Can't deserialize iris.xy. {}", why),
+        Ok((x, y, num_samples, num_features)) => (x, y, num_samples, num_features),
+    };
 
     Dataset {
         data: x,
         target: y,
-        num_samples: shape.0,
-        num_features: shape.1,
+        num_samples: num_samples,
+        num_features: num_features,
         feature_names: vec![
             "sepal length (cm)",
             "sepal width (cm)",
@@ -90,8 +50,17 @@ pub fn load_dataset() -> Dataset<f32, f32> {
 #[cfg(test)]
 mod tests {
 
+    use super::super::*;
     use super::*;
 
+    #[test]
+    #[ignore]
+    fn refresh_iris_dataset() {
+        // run this test to generate iris.xy file.
+        let dataset = load_dataset();
+        assert!(serialize_data(&dataset, "iris.xy").is_ok());
+    }
+
     #[test]
     fn iris_dataset() {
         let dataset = load_dataset();

src/dataset/boston.xy

@@ -1,12 +1,16 @@
 //! Datasets
 //!
 //! In this module you will find small datasets that are used in SmartCore for demonstration purpose mostly.
-
 pub mod boston;
 pub mod breast_cancer;
 pub mod diabetes;
 pub mod iris;
 
+use crate::math::num::RealNumber;
+use std::fs::File;
+use std::io;
+use std::io::prelude::*;
+
 /// Dataset
 pub struct Dataset<X, Y> {
     /// data in one-dimensional array.
@@ -42,6 +46,67 @@ impl<X, Y> Dataset<X, Y> {
     }
 }
 
+#[allow(dead_code)]
+pub(crate) fn serialize_data<X: RealNumber, Y: RealNumber>(
+    dataset: &Dataset<X, Y>,
+    filename: &str,
+) -> Result<(), io::Error> {
+    match File::create(filename) {
+        Ok(mut file) => {
+            file.write(&dataset.num_features.to_le_bytes())?;
+            file.write(&dataset.num_samples.to_le_bytes())?;
+            let x: Vec<u8> = dataset
+                .data
+                .iter()
+                .map(|v| *v)
+                .flat_map(|f| f.to_f32_bits().to_le_bytes().to_vec().into_iter())
+                .collect();
+            file.write_all(&x)?;
+            let y: Vec<u8> = dataset
+                .target
+                .iter()
+                .map(|v| *v)
+                .flat_map(|f| f.to_f32_bits().to_le_bytes().to_vec().into_iter())
+                .collect();
+            file.write_all(&y)?;
+        }
+        Err(why) => panic!("couldn't create {}: {}", filename, why),
+    }
+    Ok(())
+}
+
+pub(crate) fn deserialize_data(
+    bytes: &[u8],
+) -> Result<(Vec<f32>, Vec<f32>, usize, usize), io::Error> {
+    // read the same file back into a Vec of bytes
+    let (num_samples, num_features) = {
+        let mut buffer = [0u8; 8];
+        buffer.copy_from_slice(&bytes[0..8]);
+        let num_features = usize::from_le_bytes(buffer);
+        buffer.copy_from_slice(&bytes[8..16]);
+        let num_samples = usize::from_le_bytes(buffer);
+        (num_samples, num_features)
+    };
+
+    let mut x = Vec::with_capacity(num_samples * num_features);
+    let mut y = Vec::with_capacity(num_samples);
+
+    let mut buffer = [0u8; 4];
+    let mut c = 16;
+    for _ in 0..(num_samples * num_features) {
+        buffer.copy_from_slice(&bytes[c..(c + 4)]);
+        x.push(f32::from_bits(u32::from_le_bytes(buffer)));
+        c += 4;
+    }
+
+    for _ in 0..(num_samples) {
+        buffer.copy_from_slice(&bytes[c..(c + 4)]);
+        y.push(f32::from_bits(u32::from_le_bytes(buffer)));
+    }
+
+    Ok((x, y, num_samples, num_features))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/dataset/breast_cancer.rs

@@ -32,6 +32,9 @@ pub trait RealNumber: Float + FromPrimitive + Debug + Display + Copy + Sum + Pro
     fn square(self) -> Self {
         self * self
     }
+
+    /// Raw transmutation to u64
+    fn to_f32_bits(self) -> u32;
 }
 
 impl RealNumber for f64 {
@@ -69,6 +72,10 @@ impl RealNumber for f64 {
     fn half() -> Self {
         0.5f64
     }
+
+    fn to_f32_bits(self) -> u32 {
+        self.to_bits() as u32
+    }
 }
 
 impl RealNumber for f32 {
@@ -106,6 +113,10 @@ impl RealNumber for f32 {
     fn half() -> Self {
         0.5f32
     }
+
+    fn to_f32_bits(self) -> u32 {
+        self.to_bits()
+    }
 }
 
 #[cfg(test)]