Stochastic Gradient Descent. (#44)

* Initial sketch of a stochastic gradient descent.
* refactor variable names
* adds a Nesterov mode for the gradient
* Adds a Momentum and an Average Direction Update Rule
* refactor Processor to be renamed/unified to the DirectionUpdateRule.
* introduces a tutorial for the stochastic gradient descent

Author: kellertuer

Project.toml

@@ -1,7 +1,7 @@
 name = "Manopt"
 uuid = "0fc0a36d-df90-57f3-8f93-d78a9fc72bb5"
 authors = ["Ronny Bergmann <manopt@ronnybergmann.net>"]
-version = "0.2.12"
+version = "0.2.13"
 
 [deps]
 ColorSchemes = "35d6a980-a343-548e-a6ea-1d62b119f2f4"

docs/make.jl

@@ -5,10 +5,15 @@ tutorialsInputPath = joinpath(@__DIR__, "..", "src/tutorials")
 tutorialsRelativePath = "tutorials/"
 tutorialsOutputPath = joinpath(@__DIR__, "src/" * tutorialsRelativePath)
 tutorials = [
-    "MeanAndMedian", "BezierCurves", "GradientOfSecondOrderDifference", "JacobiFields"
+    "MeanAndMedian",
+    "StochasticGradientDescent",
+    "BezierCurves",
+    "GradientOfSecondOrderDifference",
+    "JacobiFields",
 ]
 menuEntries = [
     "get Started: Optimize!",
+    "do stochastic gradient descent",
     "work with Bézier curves",
     "see the gradient of \$d_2\$",
     "use Jacobi Fields",
@@ -39,6 +44,7 @@ makedocs(;
             "Gradient Descent" => "solvers/gradient_descent.md",
             "Nelder–Mead" => "solvers/NelderMead.md",
             "Particle Swarm Optimization" => "solvers/particle_swarm.md",
+            "Stochastic Gradient Descent" => "solvers/stochastic_gradient_descent.md",
             "Subgradient method" => "solvers/subgradient.md",
             "Steihaug-Toint TCG Method" =>
                 "solvers/truncated_conjugate_gradient_descent.md",
@@ -49,7 +55,7 @@ makedocs(;
             "Bézier curves" => "functions/bezier.md",
             "Cost functions" => "functions/costs.md",
             "Differentials" => "functions/differentials.md",
-            "Adjoint Differentials" => "functions/adjoint_differentials.md",
+            "Adjoint Differentials" => "functions/adjointdifferentials.md",
             "Gradients" => "functions/gradients.md",
             "Jacobi Fields" => "functions/Jacobi_fields.md",
             "Proximal Maps" => "functions/proximal_maps.md",

docs/src/assets/images/tutorials/centerAndLargeData.asy

@@ -2,5 +2,5 @@
 
 ```@autodocs
 Modules = [Manopt]
-Pages   = ["adjointdifferentials.jl"]
+Pages   = ["adjoint_differentials.jl"]
 ```

docs/src/assets/images/tutorials/centerAndLargeData.png

@@ -2,5 +2,5 @@
 
 ```@autodocs
 Modules = [Manopt]
-Pages   = ["differentials.jl"]
+Pages   = ["functions/differentials.jl"]
 ```

docs/src/functions/adjoint_differentials.md renamed to docs/src/functions/adjointdifferentials.md

@@ -107,16 +107,6 @@ Problem
 get_cost
 ```
 
-For any algorithm that involves a cyclic evalutaion, e.g.
-[`cyclic_proximal_point`](@ref), one can specify the [`EvalOrder`](@ref) as
-
-```@docs
-EvalOrder
-LinearEvalOrder
-RandomEvalOrder
-FixedRandomEvalOrder
-```
-
 ### Cost based problem
 
 ```@docs
@@ -127,7 +117,9 @@ CostProblem
 
 ```@docs
 GradientProblem
+StochasticGradientProblem
 get_gradient
+get_gradients
 ```
 
 ### Subgradient based problem

docs/src/functions/differentials.md

@@ -17,10 +17,11 @@ ConjugateGradientDescentOptions
 
 ## Available Coefficients
 
+The update rules act as [`DirectionUpdateRule`](@ref), which internally always first evaluate the gradient itself.
+
 ```@docs
 ConjugateDescentCoefficient
 DaiYuanCoefficient
-DirectionUpdateRule
 FletcherReevesCoefficient
 HagerZhangCoefficient
 HeestenesStiefelCoefficient

docs/src/plans/index.md

@@ -11,9 +11,22 @@ CurrentModule = Manopt
 ## Options
 
 ```@docs
+AbstractGradientDescentOptions
 GradientDescentOptions
 ```
 
+## Direction Update Rules
+
+A field of the options is the `direction`, a [`DirectionUpdateRule`](@ref), which by default [`Gradient`](@ref) just evaluates the gradient but can be enhanced for example to
+
+```@docs
+DirectionUpdateRule
+Gradient
+MomentumGradient
+AverageGradient
+Nesterov
+```
+
 ## Debug Actions
 
 ```@docs

docs/src/solvers/conjugate_gradient_descent.md

@@ -0,0 +1,24 @@
+# [Gradient Descent](@id StochasticGradientDescentSolver)
+
+```@meta
+CurrentModule = Manopt
+```
+
+```@docs
+stochastic_gradient_descent
+```
+
+## Options
+
+```@docs
+StochasticGradientDescentOptions
+```
+
+Additionally, the options share a [`DirectionUpdateRule`](@ref),
+so you can also apply [`MomentumGradient`](@ref) and [`AverageGradient`](@ref) here.
+The most inner one should always be.
+
+```@docs
+AbstractStochasticGradientProcessor
+StochasticGradient
+```

docs/src/solvers/gradient_descent.md

@@ -29,7 +29,7 @@ comparison_data = joinpath(
 f = artificial_SPD_image2(32)
 if export_original
     fn = joinpath(results_folder, experiment_name * "-orig.asy")
-    asymptote_export_SPD(fn; data=f, scaleAxes=(7.5, 7.5, 7.5))
+    asymptote_export_SPD(fn; data=f, scale_axes=(7.5, 7.5, 7.5))
     render_asymptote(fn; render=asy_render_detail)
 end
 sC = StopAfterIteration(400)
@@ -95,7 +95,7 @@ if export_result
         results_folder,
         experiment_name * "img-result-$(numIter)-α$(replace(string(α), "." => "-")).asy",
     )
-    asymptote_export_SPD(fn; data=y, scaleAxes=(7.5, 7.5, 7.5))
+    asymptote_export_SPD(fn; data=y, scale_axes=(7.5, 7.5, 7.5))
     render_asymptote(fn; render=asy_render_detail)
 end
 if export_table

docs/src/solvers/stochastic_gradient_descent.md

@@ -35,7 +35,7 @@ f = artificial_SPD_image2(32)
 pixelM = SymmetricPositiveDefinite(3)
 if export_orig
     orig_filename = joinpath(results_folder, experiment_name * "-orig.asy")
-    asymptote_export_SPD(orig_filename; data=f, scaleAxes=(7.5, 7.5, 7.5))
+    asymptote_export_SPD(orig_filename; data=f, scale_axes=(7.5, 7.5, 7.5))
     render_asymptote(orig_filename; render=asy_render_detail)
 end
 
@@ -79,7 +79,7 @@ yRec = get_record(o)
 # Results
 if export_result
     result_filename = joinpath(results_folder, experiment_name * "-result.asy")
-    asymptote_export_SPD(result_filename; data=y, scaleAxes=(7.5, 7.5, 7.5))
+    asymptote_export_SPD(result_filename; data=y, scale_axes=(7.5, 7.5, 7.5))
     render_asymptote(result_filename; render=asy_render_detail)
 end
 if export_table

examples/ChambollePock/SPD_Image_CP_vs_CPPA.jl

@@ -38,7 +38,7 @@ comparison_data = joinpath(
 f = artificial_SPD_image2(32)
 if export_orig
     fn = joinpath(results_folder, experiment_name * "-orig.asy")
-    asymptote_export_SPD(fn; data=f, scaleAxes=(7.5, 7.5, 7.5))
+    asymptote_export_SPD(fn; data=f, scale_axes=(7.5, 7.5, 7.5))
     render_asymptote(fn; render=asy_render_detail)
 end
 sC = StopAfterIteration(400)
@@ -91,7 +91,7 @@ if export_result
         results_folder,
         experiment_name * "img-result-$(numIter)-α$(replace(string(α), "." => "-")).asy",
     )
-    asymptote_export_SPD(fn; data=y, scaleAxes=(7.5, 7.5, 7.5))
+    asymptote_export_SPD(fn; data=y, scale_axes=(7.5, 7.5, 7.5))
     render_asymptote(fn; render=asy_render_detail)
 end
 if export_table

examples/CyclicProximalPoint/SPD_Image_CPPA.jl

@@ -18,7 +18,7 @@ asy_export = true #export data and results to asyExport
 
 curve_samples = [range(0, 3; length=101)...] # sample curve for the gradient
 curve_samples_plot = [range(0, 3; length=201)...] # sample curve for asy exports
-cameraPosition = (-1.0, -0.7, 0.3)
+camera_position = (-1.0, -0.7, 0.3)
 curveColor = RGBA{Float64}(colorant"#000000")
 sColor = RGBA{Float64}(colorant"#BBBBBB")
 dColor = RGBA{Float64}(colorant"#EE7733") # data Color: Tol Vibrant Orange
@@ -38,7 +38,7 @@ if asy_export
         joinpath(results_folder, experiment_name * "-orig.asy");
         curves=[cP],
         points=[get_bezier_junctions(M, B), get_bezier_inner_points(M, B)],
-        tVectors=[[
+        tangent_vectors=[[
             Tuple(a)
             for
             a in
@@ -47,10 +47,10 @@ if asy_export
         colors=Dict(
             :curves => [curveColor], :points => [dColor, bColor], :tvectors => [ξColor]
         ),
-        cameraPosition=cameraPosition,
-        arrowHeadSize=10.0,
-        lineWidths=[1.5, 1.5],
-        dotSize=4.0,
+        camera_position=camera_position,
+        arrow_head_size=10.0,
+        line_widths=[1.5, 1.5],
+        dot_size=4.0,
     )
     render_asymptote(joinpath(results_folder, experiment_name * "-orig.asy"); render=4)
 end
@@ -97,7 +97,7 @@ if asy_export
         joinpath(results_folder, experiment_name * "-result.asy");
         curves=[res_curve, cP],
         points=[get_bezier_junctions(M, B_opt), get_bezier_inner_points(M, B_opt)],
-        tVectors=[[
+        tangent_vectors=[[
             Tuple(a)
             for
             a in zip(
@@ -110,10 +110,10 @@ if asy_export
             :points => [dColor, bColor],
             :tvectors => [ξColor],
         ),
-        cameraPosition=cameraPosition,
-        arrowHeadSize=10.0,
-        lineWidths=[1.5, 0.75, 1.5],
-        dotSize=4.0,
+        camera_position=camera_position,
+        arrow_head_size=10.0,
+        line_widths=[1.5, 0.75, 1.5],
+        dot_size=4.0,
     )
     render_asymptote(joinpath(results_folder, experiment_name * "-result.asy"); render=4)
 end

examples/DouglasRachford/SPD_Image_DR_vs_CPPA.jl

@@ -33,7 +33,7 @@ colors = RGBA.(get.(Ref(viridis), range(0.0, 1.0; length=length(λRange))))
 curve_samples = [range(0, 3; length=101)...] # sample curve for the gradient
 curve_samples_plot = [range(0, 3; length=201)...] # sample curve for asy exports
 
-cameraPosition = (-1.0, -0.7, 0.3)
+camera_position = (-1.0, -0.7, 0.3)
 curveColor = RGBA{Float64}(colorant"#AAAAAA")
 M = Sphere(2)
 B = artificial_S2_composite_bezier_curve()
@@ -90,9 +90,9 @@ if asy_export
             colors=Dict(
                 :curves => [colors[i], curveColor], :points => [colors[i], colors[i]]
             ),
-            cameraPosition=cameraPosition,
-            lineWidths=[1.0, 0.5],
-            dotSize=2.0,
+            camera_position=camera_position,
+            line_widths=[1.0, 0.5],
+            dot_size=2.0,
         )
         render_asymptote(s * "-$(@sprintf "%04.0f" i)-result.asy"; render=render_detail)
     end
@@ -103,8 +103,8 @@ if asy_export_summary
         s * "-Summary-result.asy";
         curves=[cP, resulting_curves...],
         colors=Dict(:curves => [curveColor, colors...]),
-        cameraPosition=cameraPosition,
-        lineWidths=[0.75, [1.5 for i in eachindex(λRange)]...],
+        camera_position=camera_position,
+        line_widths=[0.75, [1.5 for i in eachindex(λRange)]...],
     )
     render_asymptote(s * "-Summary-result.asy"; render=render_detail)
 end

examples/GradientDescent/S2_Bezier_Approximation.jl

@@ -16,7 +16,7 @@ asy_export = true #export data and results to asyExport
 curve_samples = [range(0, 3; length=101)...] # sample curve for the gradient
 curve_samples_plot = [range(0, 3; length=201)...] # sample curve for asy exports
 
-cameraPosition = (-1.0, -0.7, 0.3)
+camera_position = (-1.0, -0.7, 0.3)
 curveColor = RGBA{Float64}(colorant"#000000")
 sColor = RGBA{Float64}(colorant"#BBBBBB")
 dColor = RGBA{Float64}(colorant"#EE7733") # data Color: Tol Vibrant Orange
@@ -35,7 +35,7 @@ if asy_export
         joinpath(results_folder, experiment_name * "-orig.asy");
         curves=[cP],
         points=[get_bezier_junctions(M, B), get_bezier_inner_points(M, B)],
-        tVectors=[[
+        tangent_vectors=[[
             Tuple(a)
             for
             a in
@@ -44,10 +44,10 @@ if asy_export
         colors=Dict(
             :curves => [curveColor], :points => [dColor, bColor], :tvectors => [ξColor]
         ),
-        cameraPosition=cameraPosition,
-        arrowHeadSize=10.0,
-        lineWidths=[1.5, 1.5],
-        dotSize=4.0,
+        camera_position=camera_position,
+        arrow_head_size=10.0,
+        line_widths=[1.5, 1.5],
+        dot_size=4.0,
     )
     render_asymptote(joinpath(results_folder, experiment_name * "-orig.asy"); render=4)
 end
@@ -87,7 +87,7 @@ if asy_export
         joinpath(results_folder, experiment_name * "-result.asy");
         curves=[de_casteljau(M, B_opt, curve_samples_plot), cP],
         points=[get_bezier_junctions(M, B_opt), get_bezier_inner_points(M, B_opt)],
-        tVectors=[[
+        tangent_vectors=[[
             Tuple(a)
             for
             a in zip(
@@ -100,10 +100,10 @@ if asy_export
             :points => [dColor, bColor],
             :tvectors => [ξColor],
         ),
-        cameraPosition=cameraPosition,
-        arrowHeadSize=10.0,
-        lineWidths=[1.5, 0.75, 1.5],
-        dotSize=4.0,
+        camera_position=camera_position,
+        arrow_head_size=10.0,
+        line_widths=[1.5, 0.75, 1.5],
+        dot_size=4.0,
     )
     render_asymptote(joinpath(results_folder, experiment_name * "-result.asy"); render=4)
 end