Merge pull request #35 from smartcorelib/lasso

LASSO

Author: VolodymyrOrlov

src/linalg/high_order.rs

@@ -0,0 +1,28 @@
+//! In this module you will find composite of matrix operations that are used elsewhere
+//! for improved efficiency.
+
+use crate::linalg::BaseMatrix;
+use crate::math::num::RealNumber;
+
+/// High order matrix operations.
+pub trait HighOrderOperations<T: RealNumber>: BaseMatrix<T> {
+    /// Y = AB
+    /// ```
+    /// use smartcore::linalg::naive::dense_matrix::*;
+    /// use smartcore::linalg::high_order::HighOrderOperations;
+    ///
+    /// let a = DenseMatrix::from_2d_array(&[&[1., 2.], &[3., 4.], &[5., 6.]]);
+    /// let b = DenseMatrix::from_2d_array(&[&[5., 6.], &[7., 8.], &[9., 10.]]);
+    /// let expected = DenseMatrix::from_2d_array(&[&[71., 80.], &[92., 104.]]);
+    ///
+    /// assert_eq!(a.ab(true, &b, false), expected);
+    /// ```
+    fn ab(&self, a_transpose: bool, b: &Self, b_transpose: bool) -> Self {
+        match (a_transpose, b_transpose) {
+            (true, true) => b.matmul(self).transpose(),
+            (false, true) => self.matmul(&b.transpose()),
+            (true, false) => self.transpose().matmul(b),
+            (false, false) => self.matmul(b),
+        }
+    }
+}

src/linalg/mod.rs

@@ -36,6 +36,7 @@
 pub mod cholesky;
 /// The matrix is represented in terms of its eigenvalues and eigenvectors.
 pub mod evd;
+pub mod high_order;
 /// Factors a matrix as the product of a lower triangular matrix and an upper triangular matrix.
 pub mod lu;
 /// Dense matrix with column-major order that wraps [Vec](https://doc.rust-lang.org/std/vec/struct.Vec.html).
@@ -59,6 +60,7 @@ use std::ops::Range;
 use crate::math::num::RealNumber;
 use cholesky::CholeskyDecomposableMatrix;
 use evd::EVDDecomposableMatrix;
+use high_order::HighOrderOperations;
 use lu::LUDecomposableMatrix;
 use qr::QRDecomposableMatrix;
 use stats::MatrixStats;
@@ -134,6 +136,66 @@ pub trait BaseVector<T: RealNumber>: Clone + Debug {
     /// Subtract `x` from single element of the vector, write result to original vector.
     fn sub_element_mut(&mut self, pos: usize, x: T);
 
+    /// Subtract scalar
+    fn sub_scalar_mut(&mut self, x: T) -> &Self {
+        for i in 0..self.len() {
+            self.set(i, self.get(i) - x);
+        }
+        self
+    }
+
+    /// Subtract scalar
+    fn add_scalar_mut(&mut self, x: T) -> &Self {
+        for i in 0..self.len() {
+            self.set(i, self.get(i) + x);
+        }
+        self
+    }
+
+    /// Subtract scalar
+    fn mul_scalar_mut(&mut self, x: T) -> &Self {
+        for i in 0..self.len() {
+            self.set(i, self.get(i) * x);
+        }
+        self
+    }
+
+    /// Subtract scalar
+    fn div_scalar_mut(&mut self, x: T) -> &Self {
+        for i in 0..self.len() {
+            self.set(i, self.get(i) / x);
+        }
+        self
+    }
+
+    /// Add vectors, element-wise
+    fn add_scalar(&self, x: T) -> Self {
+        let mut r = self.clone();
+        r.add_scalar_mut(x);
+        r
+    }
+
+    /// Subtract vectors, element-wise
+    fn sub_scalar(&self, x: T) -> Self {
+        let mut r = self.clone();
+        r.sub_scalar_mut(x);
+        r
+    }
+
+    /// Multiply vectors, element-wise
+    fn mul_scalar(&self, x: T) -> Self {
+        let mut r = self.clone();
+        r.mul_scalar_mut(x);
+        r
+    }
+
+    /// Divide vectors, element-wise
+    fn div_scalar(&self, x: T) -> Self {
+        let mut r = self.clone();
+        r.div_scalar_mut(x);
+        r
+    }
+
     /// Add vectors, element-wise, overriding original vector with result.
     fn add_mut(&mut self, other: &Self) -> &Self;
 
@@ -557,6 +619,7 @@ pub trait Matrix<T: RealNumber>:
     + LUDecomposableMatrix<T>
     + CholeskyDecomposableMatrix<T>
     + MatrixStats<T>
+    + HighOrderOperations<T>
     + PartialEq
     + Display
 {

src/linalg/naive/dense_matrix.rs

@@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::linalg::cholesky::CholeskyDecomposableMatrix;
 use crate::linalg::evd::EVDDecomposableMatrix;
+use crate::linalg::high_order::HighOrderOperations;
 use crate::linalg::lu::LUDecomposableMatrix;
 use crate::linalg::qr::QRDecomposableMatrix;
 use crate::linalg::stats::MatrixStats;
@@ -444,6 +445,38 @@ impl<T: RealNumber> LUDecomposableMatrix<T> for DenseMatrix<T> {}
 
 impl<T: RealNumber> CholeskyDecomposableMatrix<T> for DenseMatrix<T> {}
 
+impl<T: RealNumber> HighOrderOperations<T> for DenseMatrix<T> {
+    fn ab(&self, a_transpose: bool, b: &Self, b_transpose: bool) -> Self {
+        if !a_transpose && !b_transpose {
+            self.matmul(b)
+        } else {
+            let (d1, d2, d3, d4) = match (a_transpose, b_transpose) {
+                (true, false) => (self.nrows, self.ncols, b.ncols, b.nrows),
+                (false, true) => (self.ncols, self.nrows, b.nrows, b.ncols),
+                _ => (self.nrows, self.ncols, b.nrows, b.ncols),
+            };
+            if d1 != d4 {
+                panic!("Can not multiply {}x{} by {}x{} matrices", d2, d1, d4, d3);
+            }
+            let mut result = Self::zeros(d2, d3);
+            for r in 0..d2 {
+                for c in 0..d3 {
+                    let mut s = T::zero();
+                    for i in 0..d1 {
+                        match (a_transpose, b_transpose) {
+                            (true, false) => s += self.get(i, r) * b.get(i, c),
+                            (false, true) => s += self.get(r, i) * b.get(c, i),
+                            _ => s += self.get(i, r) * b.get(c, i),
+                        }
+                    }
+                    result.set(r, c, s);
+                }
+            }
+            result
+        }
+    }
+}
+
 impl<T: RealNumber> MatrixStats<T> for DenseMatrix<T> {}
 
 impl<T: RealNumber> Matrix<T> for DenseMatrix<T> {}
@@ -625,8 +658,8 @@ impl<T: RealNumber> BaseMatrix<T> for DenseMatrix<T> {
     }
 
     fn dot(&self, other: &Self) -> T {
-        if self.nrows != 1 && other.nrows != 1 {
-            panic!("A and B should both be 1-dimentional vectors.");
+        if (self.nrows != 1 && other.nrows != 1) && (self.ncols != 1 && other.ncols != 1) {
+            panic!("A and B should both be either a row or a column vector.");
         }
         if self.nrows * self.ncols != other.nrows * other.ncols {
             panic!("A and B should have the same size");
@@ -1120,6 +1153,29 @@ mod tests {
         assert_eq!(result, expected);
     }
 
+    #[test]
+    fn ab() {
+        let a = DenseMatrix::from_2d_array(&[&[1., 2., 3.], &[4., 5., 6.]]);
+        let b = DenseMatrix::from_2d_array(&[&[5., 6.], &[7., 8.], &[9., 10.]]);
+        let c = DenseMatrix::from_2d_array(&[&[1., 2.], &[3., 4.], &[5., 6.]]);
+        assert_eq!(
+            a.ab(false, &b, false),
+            DenseMatrix::from_2d_array(&[&[46., 52.], &[109., 124.]])
+        );
+        assert_eq!(
+            c.ab(true, &b, false),
+            DenseMatrix::from_2d_array(&[&[71., 80.], &[92., 104.]])
+        );
+        assert_eq!(
+            b.ab(false, &c, true),
+            DenseMatrix::from_2d_array(&[&[17., 39., 61.], &[23., 53., 83.,], &[29., 67., 105.]])
+        );
+        assert_eq!(
+            a.ab(true, &b, true),
+            DenseMatrix::from_2d_array(&[&[29., 39., 49.], &[40., 54., 68.,], &[51., 69., 87.]])
+        );
+    }
+
     #[test]
     fn dot() {
         let a = DenseMatrix::from_array(1, 3, &[1., 2., 3.]);

src/linalg/nalgebra_bindings.rs

@@ -44,6 +44,7 @@ use nalgebra::{DMatrix, Dynamic, Matrix, MatrixMN, RowDVector, Scalar, VecStorag
 
 use crate::linalg::cholesky::CholeskyDecomposableMatrix;
 use crate::linalg::evd::EVDDecomposableMatrix;
+use crate::linalg::high_order::HighOrderOperations;
 use crate::linalg::lu::LUDecomposableMatrix;
 use crate::linalg::qr::QRDecomposableMatrix;
 use crate::linalg::stats::MatrixStats;
@@ -553,6 +554,11 @@ impl<T: RealNumber + Scalar + AddAssign + SubAssign + MulAssign + DivAssign + Su
 {
 }
 
+impl<T: RealNumber + Scalar + AddAssign + SubAssign + MulAssign + DivAssign + Sum + 'static>
+    HighOrderOperations<T> for Matrix<T, Dynamic, Dynamic, VecStorage<T, Dynamic, Dynamic>>
+{
+}
+
 impl<T: RealNumber + Scalar + AddAssign + SubAssign + MulAssign + DivAssign + Sum + 'static>
     SmartCoreMatrix<T> for Matrix<T, Dynamic, Dynamic, VecStorage<T, Dynamic, Dynamic>>
 {

src/linalg/ndarray_bindings.rs

@@ -51,6 +51,7 @@ use ndarray::{s, stack, Array, ArrayBase, Axis, Ix1, Ix2, OwnedRepr};
 
 use crate::linalg::cholesky::CholeskyDecomposableMatrix;
 use crate::linalg::evd::EVDDecomposableMatrix;
+use crate::linalg::high_order::HighOrderOperations;
 use crate::linalg::lu::LUDecomposableMatrix;
 use crate::linalg::qr::QRDecomposableMatrix;
 use crate::linalg::stats::MatrixStats;
@@ -502,6 +503,11 @@ impl<T: RealNumber + ScalarOperand + AddAssign + SubAssign + MulAssign + DivAssi
 {
 }
 
+impl<T: RealNumber + ScalarOperand + AddAssign + SubAssign + MulAssign + DivAssign + Sum>
+    HighOrderOperations<T> for ArrayBase<OwnedRepr<T>, Ix2>
+{
+}
+
 impl<T: RealNumber + ScalarOperand + AddAssign + SubAssign + MulAssign + DivAssign + Sum> Matrix<T>
     for ArrayBase<OwnedRepr<T>, Ix2>
 {

src/linear/bg_solver.rs

@@ -0,0 +1,146 @@
+//! This is a generic solver for Ax = b type of equation
+//!
+//! for more information take a look at [this Wikipedia article](https://en.wikipedia.org/wiki/Biconjugate_gradient_method)
+//! and [this paper](https://www.cs.cmu.edu/~quake-papers/painless-conjugate-gradient.pdf)
+use crate::error::Failed;
+use crate::linalg::Matrix;
+use crate::math::num::RealNumber;
+
+pub trait BiconjugateGradientSolver<T: RealNumber, M: Matrix<T>> {
+    fn solve_mut(&self, a: &M, b: &M, x: &mut M, tol: T, max_iter: usize) -> Result<T, Failed> {
+        if tol <= T::zero() {
+            return Err(Failed::fit("tolerance shoud be > 0"));
+        }
+
+        if max_iter == 0 {
+            return Err(Failed::fit("maximum number of iterations should be > 0"));
+        }
+
+        let (n, _) = b.shape();
+
+        let mut r = M::zeros(n, 1);
+        let mut rr = M::zeros(n, 1);
+        let mut z = M::zeros(n, 1);
+        let mut zz = M::zeros(n, 1);
+
+        self.mat_vec_mul(a, x, &mut r);
+
+        for j in 0..n {
+            r.set(j, 0, b.get(j, 0) - r.get(j, 0));
+            rr.set(j, 0, r.get(j, 0));
+        }
+
+        let bnrm = b.norm(T::two());
+        self.solve_preconditioner(a, &r, &mut z);
+
+        let mut p = M::zeros(n, 1);
+        let mut pp = M::zeros(n, 1);
+        let mut bkden = T::zero();
+        let mut err = T::zero();
+
+        for iter in 1..max_iter {
+            let mut bknum = T::zero();
+
+            self.solve_preconditioner(a, &rr, &mut zz);
+            for j in 0..n {
+                bknum += z.get(j, 0) * rr.get(j, 0);
+            }
+            if iter == 1 {
+                for j in 0..n {
+                    p.set(j, 0, z.get(j, 0));
+                    pp.set(j, 0, zz.get(j, 0));
+                }
+            } else {
+                let bk = bknum / bkden;
+                for j in 0..n {
+                    p.set(j, 0, bk * p.get(j, 0) + z.get(j, 0));
+                    pp.set(j, 0, bk * pp.get(j, 0) + zz.get(j, 0));
+                }
+            }
+            bkden = bknum;
+            self.mat_vec_mul(a, &p, &mut z);
+            let mut akden = T::zero();
+            for j in 0..n {
+                akden += z.get(j, 0) * pp.get(j, 0);
+            }
+            let ak = bknum / akden;
+            self.mat_t_vec_mul(a, &pp, &mut zz);
+            for j in 0..n {
+                x.set(j, 0, x.get(j, 0) + ak * p.get(j, 0));
+                r.set(j, 0, r.get(j, 0) - ak * z.get(j, 0));
+                rr.set(j, 0, rr.get(j, 0) - ak * zz.get(j, 0));
+            }
+            self.solve_preconditioner(a, &r, &mut z);
+            err = r.norm(T::two()) / bnrm;
+
+            if err <= tol {
+                break;
+            }
+        }
+
+        Ok(err)
+    }
+
+    fn solve_preconditioner(&self, a: &M, b: &M, x: &mut M) {
+        let diag = Self::diag(a);
+        let n = diag.len();
+
+        for i in 0..n {
+            if diag[i] != T::zero() {
+                x.set(i, 0, b.get(i, 0) / diag[i]);
+            } else {
+                x.set(i, 0, b.get(i, 0));
+            }
+        }
+    }
+
+    // y = Ax
+    fn mat_vec_mul(&self, a: &M, x: &M, y: &mut M) {
+        y.copy_from(&a.matmul(x));
+    }
+
+    // y = Atx
+    fn mat_t_vec_mul(&self, a: &M, x: &M, y: &mut M) {
+        y.copy_from(&a.ab(true, x, false));
+    }
+
+    fn diag(a: &M) -> Vec<T> {
+        let (nrows, ncols) = a.shape();
+        let n = nrows.min(ncols);
+
+        let mut d = Vec::with_capacity(n);
+        for i in 0..n {
+            d.push(a.get(i, i));
+        }
+
+        d
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::linalg::naive::dense_matrix::*;
+
+    pub struct BGSolver {}
+
+    impl<T: RealNumber, M: Matrix<T>> BiconjugateGradientSolver<T, M> for BGSolver {}
+
+    #[test]
+    fn bg_solver() {
+        let a = DenseMatrix::from_2d_array(&[&[25., 15., -5.], &[15., 18., 0.], &[-5., 0., 11.]]);
+        let b = DenseMatrix::from_2d_array(&[&[40., 51., 28.]]);
+        let expected = DenseMatrix::from_2d_array(&[&[1.0, 2.0, 3.0]]);
+
+        let mut x = DenseMatrix::zeros(3, 1);
+
+        let solver = BGSolver {};
+
+        let err: f64 = solver
+            .solve_mut(&a, &b.transpose(), &mut x, 1e-6, 6)
+            .unwrap();
+
+        assert!(x.transpose().approximate_eq(&expected, 1e-4));
+        assert!((err - 0.0).abs() < 1e-4);
+    }
+}