Complete grid search params (#166)

* grid search draft

* hyperparam search for linear estimators

* grid search for ensembles

* support grid search for more algos

* grid search for unsupervised algos

* minor cleanup

Author: montanalow

src/cluster/dbscan.rs

@@ -109,6 +109,103 @@ impl<T: RealNumber, D: Distance<Vec<T>, T>> DBSCANParameters<T, D> {
     }
 }
 
+/// DBSCAN grid search parameters
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[derive(Debug, Clone)]
+pub struct DBSCANSearchParameters<T: RealNumber, D: Distance<Vec<T>, T>> {
+    /// a function that defines a distance between each pair of point in training data.
+    /// This function should extend [`Distance`](../../math/distance/trait.Distance.html) trait.
+    /// See [`Distances`](../../math/distance/struct.Distances.html) for a list of available functions.
+    pub distance: Vec<D>,
+    /// The number of samples (or total weight) in a neighborhood for a point to be considered as a core point.
+    pub min_samples: Vec<usize>,
+    /// The maximum distance between two samples for one to be considered as in the neighborhood of the other.
+    pub eps: Vec<T>,
+    /// KNN algorithm to use.
+    pub algorithm: Vec<KNNAlgorithmName>,
+}
+
+/// DBSCAN grid search iterator
+pub struct DBSCANSearchParametersIterator<T: RealNumber, D: Distance<Vec<T>, T>> {
+    dbscan_search_parameters: DBSCANSearchParameters<T, D>,
+    current_distance: usize,
+    current_min_samples: usize,
+    current_eps: usize,
+    current_algorithm: usize,
+}
+
+impl<T: RealNumber, D: Distance<Vec<T>, T>> IntoIterator for DBSCANSearchParameters<T, D> {
+    type Item = DBSCANParameters<T, D>;
+    type IntoIter = DBSCANSearchParametersIterator<T, D>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        DBSCANSearchParametersIterator {
+            dbscan_search_parameters: self,
+            current_distance: 0,
+            current_min_samples: 0,
+            current_eps: 0,
+            current_algorithm: 0,
+        }
+    }
+}
+
+impl<T: RealNumber, D: Distance<Vec<T>, T>> Iterator for DBSCANSearchParametersIterator<T, D> {
+    type Item = DBSCANParameters<T, D>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current_distance == self.dbscan_search_parameters.distance.len()
+            && self.current_min_samples == self.dbscan_search_parameters.min_samples.len()
+            && self.current_eps == self.dbscan_search_parameters.eps.len()
+            && self.current_algorithm == self.dbscan_search_parameters.algorithm.len()
+        {
+            return None;
+        }
+
+        let next = DBSCANParameters {
+            distance: self.dbscan_search_parameters.distance[self.current_distance].clone(),
+            min_samples: self.dbscan_search_parameters.min_samples[self.current_min_samples],
+            eps: self.dbscan_search_parameters.eps[self.current_eps],
+            algorithm: self.dbscan_search_parameters.algorithm[self.current_algorithm].clone(),
+        };
+
+        if self.current_distance + 1 < self.dbscan_search_parameters.distance.len() {
+            self.current_distance += 1;
+        } else if self.current_min_samples + 1 < self.dbscan_search_parameters.min_samples.len() {
+            self.current_distance = 0;
+            self.current_min_samples += 1;
+        } else if self.current_eps + 1 < self.dbscan_search_parameters.eps.len() {
+            self.current_distance = 0;
+            self.current_min_samples = 0;
+            self.current_eps += 1;
+        } else if self.current_algorithm + 1 < self.dbscan_search_parameters.algorithm.len() {
+            self.current_distance = 0;
+            self.current_min_samples = 0;
+            self.current_eps = 0;
+            self.current_algorithm += 1;
+        } else {
+            self.current_distance += 1;
+            self.current_min_samples += 1;
+            self.current_eps += 1;
+            self.current_algorithm += 1;
+        }
+
+        Some(next)
+    }
+}
+
+impl<T: RealNumber> Default for DBSCANSearchParameters<T, Euclidian> {
+    fn default() -> Self {
+        let default_params = DBSCANParameters::default();
+
+        DBSCANSearchParameters {
+            distance: vec![default_params.distance],
+            min_samples: vec![default_params.min_samples],
+            eps: vec![default_params.eps],
+            algorithm: vec![default_params.algorithm],
+        }
+    }
+}
+
 impl<T: RealNumber, D: Distance<Vec<T>, T>> PartialEq for DBSCAN<T, D> {
     fn eq(&self, other: &Self) -> bool {
         self.cluster_labels.len() == other.cluster_labels.len()
@@ -268,6 +365,29 @@ mod tests {
     #[cfg(feature = "serde")]
     use crate::math::distance::euclidian::Euclidian;
 
+    #[test]
+    fn search_parameters() {
+        let parameters = DBSCANSearchParameters {
+            min_samples: vec![10, 100],
+            eps: vec![1., 2.],
+            ..Default::default()
+        };
+        let mut iter = parameters.into_iter();
+        let next = iter.next().unwrap();
+        assert_eq!(next.min_samples, 10);
+        assert_eq!(next.eps, 1.);
+        let next = iter.next().unwrap();
+        assert_eq!(next.min_samples, 100);
+        assert_eq!(next.eps, 1.);
+        let next = iter.next().unwrap();
+        assert_eq!(next.min_samples, 10);
+        assert_eq!(next.eps, 2.);
+        let next = iter.next().unwrap();
+        assert_eq!(next.min_samples, 100);
+        assert_eq!(next.eps, 2.);
+        assert!(iter.next().is_none());
+    }
+
     #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
     #[test]
     fn fit_predict_dbscan() {

src/cluster/kmeans.rs

@@ -132,6 +132,76 @@ impl Default for KMeansParameters {
     }
 }
 
+/// KMeans grid search parameters
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[derive(Debug, Clone)]
+pub struct KMeansSearchParameters {
+    /// Number of clusters.
+    pub k: Vec<usize>,
+    /// Maximum number of iterations of the k-means algorithm for a single run.
+    pub max_iter: Vec<usize>,
+}
+
+/// KMeans grid search iterator
+pub struct KMeansSearchParametersIterator {
+    kmeans_search_parameters: KMeansSearchParameters,
+    current_k: usize,
+    current_max_iter: usize,
+}
+
+impl IntoIterator for KMeansSearchParameters {
+    type Item = KMeansParameters;
+    type IntoIter = KMeansSearchParametersIterator;
+
+    fn into_iter(self) -> Self::IntoIter {
+        KMeansSearchParametersIterator {
+            kmeans_search_parameters: self,
+            current_k: 0,
+            current_max_iter: 0,
+        }
+    }
+}
+
+impl Iterator for KMeansSearchParametersIterator {
+    type Item = KMeansParameters;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current_k == self.kmeans_search_parameters.k.len()
+            && self.current_max_iter == self.kmeans_search_parameters.max_iter.len()
+        {
+            return None;
+        }
+
+        let next = KMeansParameters {
+            k: self.kmeans_search_parameters.k[self.current_k],
+            max_iter: self.kmeans_search_parameters.max_iter[self.current_max_iter],
+        };
+
+        if self.current_k + 1 < self.kmeans_search_parameters.k.len() {
+            self.current_k += 1;
+        } else if self.current_max_iter + 1 < self.kmeans_search_parameters.max_iter.len() {
+            self.current_k = 0;
+            self.current_max_iter += 1;
+        } else {
+            self.current_k += 1;
+            self.current_max_iter += 1;
+        }
+
+        Some(next)
+    }
+}
+
+impl Default for KMeansSearchParameters {
+    fn default() -> Self {
+        let default_params = KMeansParameters::default();
+
+        KMeansSearchParameters {
+            k: vec![default_params.k],
+            max_iter: vec![default_params.max_iter],
+        }
+    }
+}
+
 impl<T: RealNumber + Sum, M: Matrix<T>> UnsupervisedEstimator<M, KMeansParameters> for KMeans<T> {
     fn fit(x: &M, parameters: KMeansParameters) -> Result<Self, Failed> {
         KMeans::fit(x, parameters)
@@ -313,6 +383,29 @@ mod tests {
         );
     }
 
+    #[test]
+    fn search_parameters() {
+        let parameters = KMeansSearchParameters {
+            k: vec![2, 4],
+            max_iter: vec![10, 100],
+            ..Default::default()
+        };
+        let mut iter = parameters.into_iter();
+        let next = iter.next().unwrap();
+        assert_eq!(next.k, 2);
+        assert_eq!(next.max_iter, 10);
+        let next = iter.next().unwrap();
+        assert_eq!(next.k, 4);
+        assert_eq!(next.max_iter, 10);
+        let next = iter.next().unwrap();
+        assert_eq!(next.k, 2);
+        assert_eq!(next.max_iter, 100);
+        let next = iter.next().unwrap();
+        assert_eq!(next.k, 4);
+        assert_eq!(next.max_iter, 100);
+        assert!(iter.next().is_none());
+    }
+
     #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
     #[test]
     fn fit_predict_iris() {

src/decomposition/pca.rs

@@ -116,6 +116,81 @@ impl Default for PCAParameters {
     }
 }
 
+/// PCA grid search parameters
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[derive(Debug, Clone)]
+pub struct PCASearchParameters {
+    /// Number of components to keep.
+    pub n_components: Vec<usize>,
+    /// By default, covariance matrix is used to compute principal components.
+    /// Enable this flag if you want to use correlation matrix instead.
+    pub use_correlation_matrix: Vec<bool>,
+}
+
+/// PCA grid search iterator
+pub struct PCASearchParametersIterator {
+    pca_search_parameters: PCASearchParameters,
+    current_k: usize,
+    current_use_correlation_matrix: usize,
+}
+
+impl IntoIterator for PCASearchParameters {
+    type Item = PCAParameters;
+    type IntoIter = PCASearchParametersIterator;
+
+    fn into_iter(self) -> Self::IntoIter {
+        PCASearchParametersIterator {
+            pca_search_parameters: self,
+            current_k: 0,
+            current_use_correlation_matrix: 0,
+        }
+    }
+}
+
+impl Iterator for PCASearchParametersIterator {
+    type Item = PCAParameters;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current_k == self.pca_search_parameters.n_components.len()
+            && self.current_use_correlation_matrix
+                == self.pca_search_parameters.use_correlation_matrix.len()
+        {
+            return None;
+        }
+
+        let next = PCAParameters {
+            n_components: self.pca_search_parameters.n_components[self.current_k],
+            use_correlation_matrix: self.pca_search_parameters.use_correlation_matrix
+                [self.current_use_correlation_matrix],
+        };
+
+        if self.current_k + 1 < self.pca_search_parameters.n_components.len() {
+            self.current_k += 1;
+        } else if self.current_use_correlation_matrix + 1
+            < self.pca_search_parameters.use_correlation_matrix.len()
+        {
+            self.current_k = 0;
+            self.current_use_correlation_matrix += 1;
+        } else {
+            self.current_k += 1;
+            self.current_use_correlation_matrix += 1;
+        }
+
+        Some(next)
+    }
+}
+
+impl Default for PCASearchParameters {
+    fn default() -> Self {
+        let default_params = PCAParameters::default();
+
+        PCASearchParameters {
+            n_components: vec![default_params.n_components],
+            use_correlation_matrix: vec![default_params.use_correlation_matrix],
+        }
+    }
+}
+
 impl<T: RealNumber, M: Matrix<T>> UnsupervisedEstimator<M, PCAParameters> for PCA<T, M> {
     fn fit(x: &M, parameters: PCAParameters) -> Result<Self, Failed> {
         PCA::fit(x, parameters)
@@ -271,6 +346,29 @@ mod tests {
     use super::*;
     use crate::linalg::naive::dense_matrix::*;
 
+    #[test]
+    fn search_parameters() {
+        let parameters = PCASearchParameters {
+            n_components: vec![2, 4],
+            use_correlation_matrix: vec![true, false],
+            ..Default::default()
+        };
+        let mut iter = parameters.into_iter();
+        let next = iter.next().unwrap();
+        assert_eq!(next.n_components, 2);
+        assert_eq!(next.use_correlation_matrix, true);
+        let next = iter.next().unwrap();
+        assert_eq!(next.n_components, 4);
+        assert_eq!(next.use_correlation_matrix, true);
+        let next = iter.next().unwrap();
+        assert_eq!(next.n_components, 2);
+        assert_eq!(next.use_correlation_matrix, false);
+        let next = iter.next().unwrap();
+        assert_eq!(next.n_components, 4);
+        assert_eq!(next.use_correlation_matrix, false);
+        assert!(iter.next().is_none());
+    }
+
     fn us_arrests_data() -> DenseMatrix<f64> {
         DenseMatrix::from_2d_array(&[
             &[13.2, 236.0, 58.0, 21.2],