Cleanup (#2)

* Clean up utils.

* Better docstrings and RST files.

* Better latex formatting.

Author: wwkong

docs/conf.py

@@ -21,9 +21,9 @@
 author = 'Weiwei Kong'
 
 # The full version, including alpha/beta/rc tags
-release = '0.1a1'
+release = '0.2.1'
 # The short version
-version = '0.1a1'
+version = '0.2.1'
 
 
 # -- General configuration ---------------------------------------------------

docs/pages/examples.rst

@@ -38,7 +38,7 @@ Unconstrained Problems
 
 This subsection considers unconstrained composite optimization problems.
 
-:scpt:`src.examples.unconstrained.basic_convex_qp`
+:scpt:`examples/unconstrained/basic_convex_qp.m`
 
 This example solves the convex univariate optimization problem
 
@@ -47,7 +47,7 @@ This example solves the convex univariate optimization problem
     \underset{x}{\text{minimize}}\quad  & \frac{1}{2}x^{2}-x+\frac{1}{2} \\
     \text{subject to}\quad & x\in\mathbb{R}.
 
-:scpt:`src.examples.unconstrained.nonconvex_qp`
+:scpt:`examples/unconstrained/nonconvex_qp`
 
 This example solves the nonconvex quadratic programming problem
 
@@ -62,7 +62,7 @@ where $\Delta^n$ is the unit simplex given by
     
     \Delta^{n}:=\left\{ x\in\mathbb{R}^{n}:\sum_{i=1}^{n}x_{i}=1,0\leq x\leq1\right\}.
 
-:scpt:`src.examples.unconstrained.nonconvex_qsdp`
+:scpt:`examples/unconstrained/nonconvex_qsdp.m`
 
 This example solves the nonconvex quadratic semidefinite programming problem
 
@@ -77,7 +77,7 @@ where $\mathbb{S}^{n}_{+}$ is the collection of positive semidefinite matrices a
 
     P^{n}:=\left\{ X\in\mathbb{S}^{n}_{+}: {\rm tr}\, X = 1\right\}.
 
-:scpt:`src.examples.unconstrained.nonconvex_svm`
+:scpt:`examples/unconstrained/nonconvex_svm.m`
 
 This example solves the nonconvex support vector machine problem
 
@@ -111,7 +111,7 @@ under a saddle-point termination criterion based on the one given in **[3]**. Mo
     \partial\left[-\Phi(x,\cdot)\right](y)
     \end{array}\right),\quad\|v\|\leq\rho_{x},\quad\|w\|\leq\rho_{y}.
 
-:scpt:`src.examples.minmax.nonconvex_minmax_qp`
+:scpt:`examples/minmax/nonconvex_minmax_qp.m`
 
 This example solves the nonconvex minmax quadratic programming problem
 
@@ -131,7 +131,7 @@ where $\Delta^n$ is the unit simplex given by
 
     \Delta^{n}:=\left\{ x\in\mathbb{R}^{n}:\sum_{i=1}^{n}x_{i}=1,0\leq x\leq1\right\}.
 
-:scpt:`src.examples.minmax.nonconvex_power_control`
+:scpt:`examples/minmax/nonconvex_power_control.m`
 
 This example solves the nonconvex power control problem
 
@@ -149,7 +149,7 @@ where $f_{k,n}$, $B_x$, and $B_y$ are given by
     B_x & := \left\{X\in \mathbb{R}^{K\times N} : 0 \leq X \leq R \right\}, \\
     B_y & := \left\{y\in \mathbb{R}^{N} : 0 \leq y \leq \frac{N}{2} \right\}.
 
-:scpt:`src.examples.minmax.nonconvex_robust_regression`
+:scpt:`examples/minmax/nonconvex_robust_regression.m`
 
 This example solves the robust regression problem 
 
@@ -173,7 +173,7 @@ Spectral Problems
 -----------------
 This subsection considers spectral optimization problems where $f_s = f_{1,s} + f_{2,s}^{\cal V} \circ \sigma$ and $f_n = f_{n}^{\cal V} \circ \sigma$ for absolutely symmetric functions $f_{2,s}^{\cal V}$ and $f_{n}^{\cal V}$.
 
-:scpt:`src.examples.spectral.nonconvex_spectral_mc`
+:scpt:`examples/spectral/nonconvex_spectral_mc.m`
 
 This example solves the spectral matrix completion problem
 
@@ -184,7 +184,7 @@ This example solves the spectral matrix completion problem
 
 where ${\cal R}_{\mu}$ is an absolutely symmetric regularization function, $\sigma(\cdot)$ is the function that maps a matrix to its vector of singular values, and $B_R$ is the Euclidean ball of radius $R$, i.e., $B_R := \{X \in \mathbb{R}^{p\times q} : \|X\|_F \leq R\}$.
 
-:scpt:`src.examples.spectral.nonconvex_spectral_bmc`
+:scpt:`examples/spectral/nonconvex_spectral_bmc.m`
 
 This example solves the spectral blockwise matrix completion problem
 
@@ -200,7 +200,7 @@ Linearly-Set Constrained Problems
 
 This subsection considers linearly-set constrained composite optimization problems where $g(x)=Ax$ for a linear operator $A$ and $S$ is a closed convex set.
 
-:scpt:`src.examples.constrained.lin_constr_nonconvex_qp`
+:scpt:`examples/constrained/lin_constr_nonconvex_qp.m`
 
 This example solves the linearly-constrained nonconvex quadratic programming problem
 
@@ -215,7 +215,7 @@ where $\Delta^n$ is the unit simplex given by
 
     \Delta^{n}:=\left\{ x\in\mathbb{R}^{n}:\sum_{i=1}^{n}x_{i}=1,0\leq x\leq1\right\}.
 
-:scpt:`src.examples.constrained.nonconvex_lin_constr_qsdp`
+:scpt:`examples/constrained/nonconvex_lin_constr_qsdp.m`
 
 This example solves the linearly-constrained nonconvex quadratic semidefinite programming problem
 
@@ -230,7 +230,7 @@ where $\mathbb{S}^{n}_{+}$ is the collection of positive semidefinite matrices a
 
     P^{n}:=\left\{ X\in\mathbb{S}^{n}_{+}: {\rm tr}\, X = 1\right\}.
 
-:scpt:`src.examples.constrained.nonconvex_sparse_pca`
+:scpt:`examples/constrained/nonconvex_sparse_pca.m`
 
 This example solves the nonconvex sparse principal component analysis problem
 
@@ -246,7 +246,7 @@ where ${\cal F}^k$ is the $k$-Fantope given by
 
     {\cal F}^k:=\left\{ X\in\mathbb{S}^{n}_{+}: 0 \preceq X \preceq I, {\rm tr}\, X = k\right\}.
 
-:scpt:`src.examples.constrained.nonconvex_bounded_mc`
+:scpt:`examples/constrained.nonconvex_bounded_mc.m`
 
 This example solves the nonconvex bounded matrix completion problem
 

docs/pages/frameworks.rst

@@ -11,8 +11,12 @@ For simplicity, we only list the parameter inputs that affect the behavior of th
 
 .. automodule:: src.frameworks
 
-.. autofunction:: penalty(solver, oracle, params)
+.. autofunction:: AIDAL(~, oracle, params)
 
-.. autofunction:: IAPIAL(~, oracle, params)
+.. autofunction:: IAIPAL(~, oracle, params)
 
 .. autofunction:: iALM(~, oracle, params)
+
+.. autofunction:: penalty(solver, oracle, params)
+
+.. autofunction:: sProxALM(~, oracle, params)

docs/pages/introduction.rst

@@ -11,20 +11,20 @@ Currently, the following problems classes are supported:
     - Linearly Set Constrained Composite Optimization
     - Nonconvex-Concave Min-Max Optimization
     - Spectral Composite Optimization
-    - Convex Cone Constrained Composite Optimization (WIP)
+    - Convex Cone Constrained Composite Optimization
 
 Instances of the above classes include *semidefinite programming*, *convex programming*, *cone programming*, *linear and quadratic programmin*, and *nonconvex programming*.
 
 The components of NC-OPT can be split into the following categories.
 
-:mod:`src.oracles`
-    A class that abstracts the idea of a first-order oracle at a point. It contains any one of the following oracles: *function value*, *function gradient*, and *proximal oracle*.
+:mod:`oracles`
+    Classes that abstract the idea of a first-order oracle at a point. It contains one or more of the following oracles: *function value*, *function gradient*, and *proximal oracle*.
 
-:mod:`src.solvers`
+:mod:`solvers`
     A collection of composite optimzation solvers that solve the problem associated with a particular first-order oracle. Some examples include: *the composite gradient method*,  *the accelerated composite gradient method*, and *the accelerated inexact proximal point method*.
 
-:mod:`src.frameworks`
+:mod:`frameworks`
     A collection of constrained composite optimization frameworks that use a solver to solve a constrained composite optimization model. Some examples include: *the quadratic penalty framework*, *the augmented Lagrangian framework*, and *the dampened augmented Lagrangian framework*.
 
-:mod:`src.models`
+:mod:`models`
     A class that abstracts the idea of a composite optimization model. It contains properties that describe various aspects of the model, including: *objective function*, *constraints*, *solver*, *framework*, *tolerances*, and *logging*.

docs/pages/solvers.rst

@@ -41,6 +41,6 @@ Spectral NCO Solvers
 
 These solvers are used for spectral nonconvex composite optimization problems.
 
-.. autofunction:: IA_ICG(spectral_oracle, params)
+.. autofunction:: DA_ICG(spectral_oracle, params)
 
-.. autofunction:: DA_ICG(spectral_oracle, params)
+.. autofunction:: IA_ICG(spectral_oracle, params)

src/frameworks/AIDAL.m

@@ -2,31 +2,26 @@
 % Copyright © 2021 Weiwei "William" Kong
 
 function [model, history] = AIDAL(~, oracle, params)
-  % An accelerated inexact dampened augmeneted Lagrangian (AIDAL) framework for solving a nonconvex composite optimization problem
-  % with linear constraints
-  % 
-  % Note:
-  % 
-  %   Based on the paper:
-  %
-  %     ?????
-  %
-  % Arguments:
-  % 
-  %   oracle (Oracle): The oracle underlying the optimization problem.
-  % 
-  %   params (struct): Contains instructions on how to call the framework.
-  %   
-  %   ??? add defaults here ???
-  % 
-  % Returns:
-  %   
-  %   A pair of structs containing model and history related outputs of the solved problem associated with the oracle and input
-  %   parameters.
-  %
-
-  % Global constants.
-  MIN_PENALTY_CONST = 1;
+% An accelerated inexact dampened augmeneted Lagrangian (AIDAL) framework for solving a nonconvex composite optimization problem
+% with linear constraints
+%
+% Note:
+%
+%   Based on the paper:
+%
+%   **TBD**
+%
+% Arguments:
+%
+%   oracle (Oracle): The oracle underlying the optimization problem.
+%
+%   params (struct): Contains instructions on how to call the framework.
+%
+% Returns:
+%
+%   A pair of structs containing model and history related outputs of the solved problem associated with the oracle and input
+%   parameters.
+%
 
   % Timer start.
   t_start = tic;
@@ -107,7 +102,7 @@
   z0 = params.x0;
   p0 = zeros(size(params.constr_fn(z0)));
   p00 = p0;
-  c = max([MIN_PENALTY_CONST, L / K_constr ^ 2]);
+  c = params.c0;
   c0 = c;
   params_acg = params;
   params_acg.mu = 1 - lambda * m;
@@ -215,6 +210,9 @@
 % Fills in parameters that were not set as input.
 function params = set_default_params(params)
 
+  % Global constants.
+  MIN_PENALTY_CONST = 1;
+
   % Overwrite if necessary.
   if (~isfield(params, 'i_logging')) 
     params.i_logging = false;
@@ -234,6 +232,9 @@
   if (~isfield(params, 'theta'))
     params.theta = 0.01;
   end
+  if (~isfield(params, 'c0'))
+    params.c0 = max([MIN_PENALTY_CONST, L / K_constr ^ 2]);
+  end  
   if (~isfield(params, 'chi'))
     theta = params.theta;
     params.chi = theta ^ 2 / (2 * (2 - theta) * (1 - theta));

src/frameworks/IAIPAL.m

@@ -2,15 +2,18 @@
 % Copyright © 2021 Weiwei "William" Kong
 
 function [model, history] = IAIPAL(~, oracle, params)
-% An inexact accelerated proximal augmeneted Lagrangian (IAPIAL) framework for solving a nonconvex composite optimization problem
-% with convex cone constraints.
+% An inner accelerated proximal inexact augmeneted Lagrangian (IAPIAL) framework for solving a nonconvex composite optimization 
+% problem with convex (linear or nonliner) cone constraints.
 % 
 % Note:
 % 
-%   Based on the paper:
+%   Based on the papers:
 %
-%   Kong, W., Melo, J. G., & Monteiro, R. D. (2020). Iteration-complexity of a proximal augmented Lagrangian method for solving
-%   nonconvex composite optimization problems with nonlinear convex constraints. *arXiv preprint arXiv:2008.07080*\.
+%   **[1]** Melo, J. G., & Monteiro, R. D. (2020). Iteration-complexity of an inner accelerated inexact proximal augmented Lagrangian 
+%   method based on the classical Lagrangian function and a full Lagrange multiplier update. *arXiv preprint arXiv:2008.00562*\.
+%
+%   **[2]** Kong, W., Melo, J. G., & Monteiro, R. D. (2020). Iteration-complexity of a proximal augmented Lagrangian method for 
+%   solving nonconvex composite optimization problems with nonlinear convex constraints. *arXiv preprint arXiv:2008.07080*\.
 %
 % Arguments:
 % 
@@ -22,6 +25,7 @@
 %   
 %   A pair of structs containing model and history related outputs of the solved problem associated with the oracle and input
 %   parameters.
+%
 
   % Timer start.
   t_start = tic;

src/frameworks/iALM.m

@@ -3,7 +3,7 @@
 
 function [model, history] = iALM(~, oracle, params)
 % An inexact augmented Lagrangian method (iALM) for solving a nonconvex composite optimization model with nonlinear equality
-% constraints, i.e. g(x) = 0.
+% constraints, i.e. $g(x) = 0$.
 % 
 % Note:
 % 

src/models/CompModel.m

@@ -9,26 +9,29 @@
 %   The following properties are necessary before the ``optimize()`` method can be called to solve the model: either {``f_s``,
 %   ``grad_f_s``} or ``oracle``, ``L``, ``x0``, and ``solver``. 
 %
-% Key Attributes:
+% Attributes:
 %
 %   f_s (function handle): A one argument function that, when evaluated at a point $x$, outputs $f_s(x)$. Defaults to ``None``.
 %
 %   grad_f_s (function handle): A one argument function that, when evaluated at a point $x$, outputs $\nabla f_s(x)$. Defaults
 %     to ``None``.
 %
-%   f_n (function handle): A one argument function that, when evaluated at a point $x$, outputs $f_n(x)$. Defaults to ``@(x)
-%     zeros(size(x))``.
+%   f_n (function handle): A one argument function that, when evaluated at a point $x$, outputs $f_n(x)$. Defaults to 
+%     ``@(x) zeros(size(x))``.
 %
-%   prox_f_n (function handle): A two argument function that, when evaluated at $\{x,\lambda\}$, outputs $${\rm prox}_{\lambda
-%     f_n}(x) := {\rm argmin}_u \left\{\lambda f_n(u) + \frac{1}{2}\|u-x\|^2\right\}.$$ Defaults to ``@(x, lam) x``.
+%   prox_f_n (function handle): A two argument function that, when evaluated at $\{x,\lambda\}$, outputs 
+%     $${\rm prox}_{\lambda f_n}(x) := {\rm argmin}_u \left\{\lambda f_n(u) + \frac{1}{2}\|u-x\|^2\right\}.$$ 
+%     Defaults to ``@(x, lam) x``.
 %
 %   L (double): A **required** Lipschitz constant of $\nabla f_s$. Defaults to ``None``.
 %
-%   M (double): An **optional** upper curvature constant of $\nabla f_s$, i.e., a constant satisfying $$f_s(z) - f_s(u) - \langle
-%     \nabla f_s(u), z - u \rangle \leq \frac{M}{2}\|u-z\|^2 \quad \forall u,z \in {\rm dom}\, f_n.$$ Defaults to ``None``.
+%   M (double): An **optional** upper curvature constant of $\nabla f_s$, i.e., a constant satisfying 
+%     $$f_s(z) - f_s(u) - \langle\nabla f_s(u), z - u \rangle \leq \frac{M}{2}\|u-z\|^2 \quad \forall u,z \in {\rm dom}\, f_n.$$ 
+%     Defaults to ``None``.
 %
-%   m (double): An **optional** lower curvature constant of $\nabla f_s$, i.e. a constant satisfying $$f_s(z) - f_s(u) - \langle
-%     \nabla f_s(u), z - u \rangle \geq -\frac{m}{2}\|u-z\|^2 \quad \forall u,z \in {\rm dom}\, f_n.$$ Defaults to ``None``.
+%   m (double): An **optional** lower curvature constant of $\nabla f_s$, i.e. a constant satisfying 
+%     $$f_s(z) - f_s(u) - \langle\nabla f_s(u), z - u \rangle \geq -\frac{m}{2}\|u-z\|^2 \quad \forall u,z \in {\rm dom}\, f_n.$$ 
+%     Defaults to ``None``.
 %
 %   x0 (double vector): A **required** starting point of the solver. Defaults to ``None``.
 %
@@ -70,7 +73,7 @@
 %   x (double vector): The stationary point returned by the solver. Defaults to ``None``. Cannot be set by the user.
 %
 %   v (double vector): The stationary residual returned by the solver. Defaults to ``None``. Cannot be set by the user.
-
+%
   %% CONSTRUCTORS
   methods
     function obj = CompModel(varargin)

src/models/ConstrCompModel.m

@@ -9,7 +9,7 @@
 %   The following (non-inherited) properties are necessary before the ``optimize()`` method can be called to solve the model:
 %   ``framework``, ``constr_fn``, ``grad_constr_fn``, ``set_projector``, and ``K_constr``.
 % 
-% Key Attributes:
+% Attributes:
 %
 %   framework (function handle): A **required** function handle to a framework that solves constrained composite optimization
 %     problems (see src/frameworks). Defaults to ``None``.