Skip to content

Commit cceb2f0

Browse files
Volodymyr OrlovVolodymyr Orlov
authored andcommitted
feat: lasso documentation
1 parent a27c29b commit cceb2f0

File tree

4 files changed

+86
-34
lines changed

4 files changed

+86
-34
lines changed

src/linalg/naive/dense_matrix.rs

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -187,9 +187,7 @@ impl<T: RealNumber> BaseVector<T> for Vec<T> {
187187
);
188188
}
189189

190-
for i in 0..self.len() {
191-
self[i] = other[i];
192-
}
190+
self[..].clone_from_slice(&other[..]);
193191
}
194192
}
195193

@@ -929,9 +927,7 @@ impl<T: RealNumber> BaseMatrix<T> for DenseMatrix<T> {
929927
);
930928
}
931929

932-
for i in 0..self.values.len() {
933-
self.values[i] = other.values[i];
934-
}
930+
self.values[..].clone_from_slice(&other.values[..]);
935931
}
936932

937933
fn abs_mut(&mut self) -> &Self {
@@ -1066,6 +1062,14 @@ mod tests {
10661062
assert_eq!(32.0, BaseVector::dot(&v1, &v2));
10671063
}
10681064

1065+
#[test]
1066+
fn vec_copy_from() {
1067+
let mut v1 = vec![1., 2., 3.];
1068+
let v2 = vec![4., 5., 6.];
1069+
v1.copy_from(&v2);
1070+
assert_eq!(v1, v2);
1071+
}
1072+
10691073
#[test]
10701074
fn vec_approximate_eq() {
10711075
let a = vec![1., 2., 3.];
@@ -1199,6 +1203,14 @@ mod tests {
11991203
assert_eq!(a.dot(&b), 32.);
12001204
}
12011205

1206+
#[test]
1207+
fn copy_from() {
1208+
let mut a = DenseMatrix::from_2d_array(&[&[1., 2.], &[3., 4.], &[5., 6.]]);
1209+
let b = DenseMatrix::from_2d_array(&[&[7., 8.], &[9., 10.], &[11., 12.]]);
1210+
a.copy_from(&b);
1211+
assert_eq!(a, b);
1212+
}
1213+
12021214
#[test]
12031215
fn slice() {
12041216
let m = DenseMatrix::from_2d_array(&[

src/linear/elasticnet.rs renamed to src/linear/elastic_net.rs

Lines changed: 59 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,51 @@
1+
#![allow(clippy::needless_range_loop)]
12
//! # Elastic Net
23
//!
4+
//! Elastic net is an extension of [linear regression](../linear_regression/index.html) that adds regularization penalties to the loss function during training.
5+
//! Just like in ordinary linear regression you assume a linear relationship between input variables and the target variable.
6+
//! Unlike linear regression elastic net adds regularization penalties to the loss function during training.
7+
//! In particular, the elastic net coefficient estimates \\(\beta\\) are the values that minimize
8+
//!
9+
//! \\[L(\alpha, \beta) = \vert \boldsymbol{y} - \boldsymbol{X}\beta\vert^2 + \lambda_1 \vert \beta \vert^2 + \lambda_2 \vert \beta \vert_1\\]
10+
//!
11+
//! where \\(\lambda_1 = \\alpha l_{1r}\\), \\(\lambda_2 = \\alpha (1 - l_{1r})\\) and \\(l_{1r}\\) is the l1 ratio, elastic net mixing parameter.
12+
//!
13+
//! In essense, elastic net combines both the [L1](../lasso/index.html) and [L2](../ridge_regression/index.html) penalties during training,
14+
//! which can result in better performance than a model with either one or the other penalty on some problems.
15+
//! The elastic net is particularly useful when the number of predictors (p) is much bigger than the number of observations (n).
16+
//!
17+
//! Example:
18+
//!
19+
//! ```
20+
//! use smartcore::linalg::naive::dense_matrix::*;
21+
//! use smartcore::linear::elastic_net::*;
22+
//!
23+
//! // Longley dataset (https://www.statsmodels.org/stable/datasets/generated/longley.html)
24+
//! let x = DenseMatrix::from_2d_array(&[
25+
//! &[234.289, 235.6, 159.0, 107.608, 1947., 60.323],
26+
//! &[259.426, 232.5, 145.6, 108.632, 1948., 61.122],
27+
//! &[258.054, 368.2, 161.6, 109.773, 1949., 60.171],
28+
//! &[284.599, 335.1, 165.0, 110.929, 1950., 61.187],
29+
//! &[328.975, 209.9, 309.9, 112.075, 1951., 63.221],
30+
//! &[346.999, 193.2, 359.4, 113.270, 1952., 63.639],
31+
//! &[365.385, 187.0, 354.7, 115.094, 1953., 64.989],
32+
//! &[363.112, 357.8, 335.0, 116.219, 1954., 63.761],
33+
//! &[397.469, 290.4, 304.8, 117.388, 1955., 66.019],
34+
//! &[419.180, 282.2, 285.7, 118.734, 1956., 67.857],
35+
//! &[442.769, 293.6, 279.8, 120.445, 1957., 68.169],
36+
//! &[444.546, 468.1, 263.7, 121.950, 1958., 66.513],
37+
//! &[482.704, 381.3, 255.2, 123.366, 1959., 68.655],
38+
//! &[502.601, 393.1, 251.4, 125.368, 1960., 69.564],
39+
//! &[518.173, 480.6, 257.2, 127.852, 1961., 69.331],
40+
//! &[554.894, 400.7, 282.7, 130.081, 1962., 70.551],
41+
//! ]);
42+
//!
43+
//! let y: Vec<f64> = vec![83.0, 88.5, 88.2, 89.5, 96.2, 98.1, 99.0,
44+
//! 100.0, 101.2, 104.6, 108.4, 110.8, 112.6, 114.2, 115.7, 116.9];
45+
//!
46+
//! let y_hat = ElasticNet::fit(&x, &y, Default::default()).
47+
//! and_then(|lr| lr.predict(&x)).unwrap();
48+
//! ```
349
//!
450
//! ## References:
551
//!
@@ -19,17 +65,24 @@ use crate::math::num::RealNumber;
1965

2066
use crate::linear::lasso_optimizer::InteriorPointOptimizer;
2167

22-
/// Ridge Regression parameters
68+
/// Elastic net parameters
2369
#[derive(Serialize, Deserialize, Debug)]
2470
pub struct ElasticNetParameters<T: RealNumber> {
71+
/// Regularization parameter.
2572
pub alpha: T,
73+
/// The elastic net mixing parameter, with 0 <= l1_ratio <= 1.
74+
/// For l1_ratio = 0 the penalty is an L2 penalty.
75+
/// For l1_ratio = 1 it is an L1 penalty. For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2.
2676
pub l1_ratio: T,
77+
/// If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the standard deviation.
2778
pub normalize: bool,
79+
/// The tolerance for the optimization
2880
pub tol: T,
81+
/// The maximum number of iterations
2982
pub max_iter: usize,
3083
}
3184

32-
/// Ridge regression
85+
/// Elastic net
3386
#[derive(Serialize, Deserialize, Debug)]
3487
pub struct ElasticNet<T: RealNumber, M: Matrix<T>> {
3588
coefficients: M,
@@ -56,7 +109,7 @@ impl<T: RealNumber, M: Matrix<T>> PartialEq for ElasticNet<T, M> {
56109
}
57110

58111
impl<T: RealNumber, M: Matrix<T>> ElasticNet<T, M> {
59-
/// Fits ridge regression to your data.
112+
/// Fits elastic net regression to your data.
60113
/// * `x` - _NxM_ matrix with _N_ observations and _M_ features in each observation.
61114
/// * `y` - target values
62115
/// * `parameters` - other parameters, use `Default::default()` to set parameters to default values.
@@ -81,7 +134,7 @@ impl<T: RealNumber, M: Matrix<T>> ElasticNet<T, M> {
81134
let (w, b) = if parameters.normalize {
82135
let (scaled_x, col_mean, col_std) = Self::rescale_x(x)?;
83136

84-
let (x, y, gamma) = Self::augment_X_and_y(&scaled_x, y, l2_reg);
137+
let (x, y, gamma) = Self::augment_x_and_y(&scaled_x, y, l2_reg);
85138

86139
let mut optimizer = InteriorPointOptimizer::new(&x, p);
87140

@@ -102,7 +155,7 @@ impl<T: RealNumber, M: Matrix<T>> ElasticNet<T, M> {
102155

103156
(w, b)
104157
} else {
105-
let (x, y, gamma) = Self::augment_X_and_y(x, y, l2_reg);
158+
let (x, y, gamma) = Self::augment_x_and_y(x, y, l2_reg);
106159

107160
let mut optimizer = InteriorPointOptimizer::new(&x, p);
108161

@@ -159,7 +212,7 @@ impl<T: RealNumber, M: Matrix<T>> ElasticNet<T, M> {
159212
Ok((scaled_x, col_mean, col_std))
160213
}
161214

162-
fn augment_X_and_y(x: &M, y: &M::RowVector, l2_reg: T) -> (M, M::RowVector, T) {
215+
fn augment_x_and_y(x: &M, y: &M::RowVector, l2_reg: T) -> (M, M::RowVector, T) {
163216
let (n, p) = x.shape();
164217

165218
let gamma = T::one() / (T::one() + l2_reg).sqrt();

src/linear/lasso.rs

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -105,18 +105,15 @@ impl<T: RealNumber, M: Matrix<T>> Lasso<T, M> {
105105
return Err(Failed::fit("Number of rows in X should = len(y)"));
106106
}
107107

108+
let l1_reg = parameters.alpha * T::from_usize(n).unwrap();
109+
108110
let (w, b) = if parameters.normalize {
109111
let (scaled_x, col_mean, col_std) = Self::rescale_x(x)?;
110112

111113
let mut optimizer = InteriorPointOptimizer::new(&scaled_x, p);
112114

113-
let mut w = optimizer.optimize(
114-
&scaled_x,
115-
y,
116-
parameters.alpha,
117-
parameters.max_iter,
118-
parameters.tol,
119-
)?;
115+
let mut w =
116+
optimizer.optimize(&scaled_x, y, l1_reg, parameters.max_iter, parameters.tol)?;
120117

121118
for (j, col_std_j) in col_std.iter().enumerate().take(p) {
122119
w.set(j, 0, w.get(j, 0) / *col_std_j);
@@ -133,8 +130,7 @@ impl<T: RealNumber, M: Matrix<T>> Lasso<T, M> {
133130
} else {
134131
let mut optimizer = InteriorPointOptimizer::new(x, p);
135132

136-
let w =
137-
optimizer.optimize(x, y, parameters.alpha, parameters.max_iter, parameters.tol)?;
133+
let w = optimizer.optimize(x, y, l1_reg, parameters.max_iter, parameters.tol)?;
138134

139135
(w, y.mean())
140136
};
@@ -215,18 +211,9 @@ mod tests {
215211
114.2, 115.7, 116.9,
216212
];
217213

218-
let y_hat = Lasso::fit(
219-
&x,
220-
&y,
221-
LassoParameters {
222-
alpha: 0.1,
223-
normalize: true,
224-
tol: 1e-4,
225-
max_iter: 1000,
226-
},
227-
)
228-
.and_then(|lr| lr.predict(&x))
229-
.unwrap();
214+
let y_hat = Lasso::fit(&x, &y, Default::default())
215+
.and_then(|lr| lr.predict(&x))
216+
.unwrap();
230217

231218
assert!(mean_absolute_error(&y_hat, &y) < 2.0);
232219

src/linear/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
//! <script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
2222
2323
pub(crate) mod bg_solver;
24-
pub mod elasticnet;
24+
pub mod elastic_net;
2525
pub mod lasso;
2626
pub(crate) mod lasso_optimizer;
2727
pub mod linear_regression;

0 commit comments

Comments
 (0)