1
+ #![ allow( clippy:: needless_range_loop) ]
1
2
//! # Elastic Net
2
3
//!
4
+ //! Elastic net is an extension of [linear regression](../linear_regression/index.html) that adds regularization penalties to the loss function during training.
5
+ //! Just like in ordinary linear regression you assume a linear relationship between input variables and the target variable.
6
+ //! Unlike linear regression elastic net adds regularization penalties to the loss function during training.
7
+ //! In particular, the elastic net coefficient estimates \\(\beta\\) are the values that minimize
8
+ //!
9
+ //! \\[L(\alpha, \beta) = \vert \boldsymbol{y} - \boldsymbol{X}\beta\vert^2 + \lambda_1 \vert \beta \vert^2 + \lambda_2 \vert \beta \vert_1\\]
10
+ //!
11
+ //! where \\(\lambda_1 = \\alpha l_{1r}\\), \\(\lambda_2 = \\alpha (1 - l_{1r})\\) and \\(l_{1r}\\) is the l1 ratio, elastic net mixing parameter.
12
+ //!
13
+ //! In essense, elastic net combines both the [L1](../lasso/index.html) and [L2](../ridge_regression/index.html) penalties during training,
14
+ //! which can result in better performance than a model with either one or the other penalty on some problems.
15
+ //! The elastic net is particularly useful when the number of predictors (p) is much bigger than the number of observations (n).
16
+ //!
17
+ //! Example:
18
+ //!
19
+ //! ```
20
+ //! use smartcore::linalg::naive::dense_matrix::*;
21
+ //! use smartcore::linear::elastic_net::*;
22
+ //!
23
+ //! // Longley dataset (https://www.statsmodels.org/stable/datasets/generated/longley.html)
24
+ //! let x = DenseMatrix::from_2d_array(&[
25
+ //! &[234.289, 235.6, 159.0, 107.608, 1947., 60.323],
26
+ //! &[259.426, 232.5, 145.6, 108.632, 1948., 61.122],
27
+ //! &[258.054, 368.2, 161.6, 109.773, 1949., 60.171],
28
+ //! &[284.599, 335.1, 165.0, 110.929, 1950., 61.187],
29
+ //! &[328.975, 209.9, 309.9, 112.075, 1951., 63.221],
30
+ //! &[346.999, 193.2, 359.4, 113.270, 1952., 63.639],
31
+ //! &[365.385, 187.0, 354.7, 115.094, 1953., 64.989],
32
+ //! &[363.112, 357.8, 335.0, 116.219, 1954., 63.761],
33
+ //! &[397.469, 290.4, 304.8, 117.388, 1955., 66.019],
34
+ //! &[419.180, 282.2, 285.7, 118.734, 1956., 67.857],
35
+ //! &[442.769, 293.6, 279.8, 120.445, 1957., 68.169],
36
+ //! &[444.546, 468.1, 263.7, 121.950, 1958., 66.513],
37
+ //! &[482.704, 381.3, 255.2, 123.366, 1959., 68.655],
38
+ //! &[502.601, 393.1, 251.4, 125.368, 1960., 69.564],
39
+ //! &[518.173, 480.6, 257.2, 127.852, 1961., 69.331],
40
+ //! &[554.894, 400.7, 282.7, 130.081, 1962., 70.551],
41
+ //! ]);
42
+ //!
43
+ //! let y: Vec<f64> = vec![83.0, 88.5, 88.2, 89.5, 96.2, 98.1, 99.0,
44
+ //! 100.0, 101.2, 104.6, 108.4, 110.8, 112.6, 114.2, 115.7, 116.9];
45
+ //!
46
+ //! let y_hat = ElasticNet::fit(&x, &y, Default::default()).
47
+ //! and_then(|lr| lr.predict(&x)).unwrap();
48
+ //! ```
3
49
//!
4
50
//! ## References:
5
51
//!
@@ -19,17 +65,24 @@ use crate::math::num::RealNumber;
19
65
20
66
use crate :: linear:: lasso_optimizer:: InteriorPointOptimizer ;
21
67
22
- /// Ridge Regression parameters
68
+ /// Elastic net parameters
23
69
#[ derive( Serialize , Deserialize , Debug ) ]
24
70
pub struct ElasticNetParameters < T : RealNumber > {
71
+ /// Regularization parameter.
25
72
pub alpha : T ,
73
+ /// The elastic net mixing parameter, with 0 <= l1_ratio <= 1.
74
+ /// For l1_ratio = 0 the penalty is an L2 penalty.
75
+ /// For l1_ratio = 1 it is an L1 penalty. For 0 < l1_ratio < 1, the penalty is a combination of L1 and L2.
26
76
pub l1_ratio : T ,
77
+ /// If True, the regressors X will be normalized before regression by subtracting the mean and dividing by the standard deviation.
27
78
pub normalize : bool ,
79
+ /// The tolerance for the optimization
28
80
pub tol : T ,
81
+ /// The maximum number of iterations
29
82
pub max_iter : usize ,
30
83
}
31
84
32
- /// Ridge regression
85
+ /// Elastic net
33
86
#[ derive( Serialize , Deserialize , Debug ) ]
34
87
pub struct ElasticNet < T : RealNumber , M : Matrix < T > > {
35
88
coefficients : M ,
@@ -56,7 +109,7 @@ impl<T: RealNumber, M: Matrix<T>> PartialEq for ElasticNet<T, M> {
56
109
}
57
110
58
111
impl < T : RealNumber , M : Matrix < T > > ElasticNet < T , M > {
59
- /// Fits ridge regression to your data.
112
+ /// Fits elastic net regression to your data.
60
113
/// * `x` - _NxM_ matrix with _N_ observations and _M_ features in each observation.
61
114
/// * `y` - target values
62
115
/// * `parameters` - other parameters, use `Default::default()` to set parameters to default values.
@@ -81,7 +134,7 @@ impl<T: RealNumber, M: Matrix<T>> ElasticNet<T, M> {
81
134
let ( w, b) = if parameters. normalize {
82
135
let ( scaled_x, col_mean, col_std) = Self :: rescale_x ( x) ?;
83
136
84
- let ( x, y, gamma) = Self :: augment_X_and_y ( & scaled_x, y, l2_reg) ;
137
+ let ( x, y, gamma) = Self :: augment_x_and_y ( & scaled_x, y, l2_reg) ;
85
138
86
139
let mut optimizer = InteriorPointOptimizer :: new ( & x, p) ;
87
140
@@ -102,7 +155,7 @@ impl<T: RealNumber, M: Matrix<T>> ElasticNet<T, M> {
102
155
103
156
( w, b)
104
157
} else {
105
- let ( x, y, gamma) = Self :: augment_X_and_y ( x, y, l2_reg) ;
158
+ let ( x, y, gamma) = Self :: augment_x_and_y ( x, y, l2_reg) ;
106
159
107
160
let mut optimizer = InteriorPointOptimizer :: new ( & x, p) ;
108
161
@@ -159,7 +212,7 @@ impl<T: RealNumber, M: Matrix<T>> ElasticNet<T, M> {
159
212
Ok ( ( scaled_x, col_mean, col_std) )
160
213
}
161
214
162
- fn augment_X_and_y ( x : & M , y : & M :: RowVector , l2_reg : T ) -> ( M , M :: RowVector , T ) {
215
+ fn augment_x_and_y ( x : & M , y : & M :: RowVector , l2_reg : T ) -> ( M , M :: RowVector , T ) {
163
216
let ( n, p) = x. shape ( ) ;
164
217
165
218
let gamma = T :: one ( ) / ( T :: one ( ) + l2_reg) . sqrt ( ) ;
0 commit comments