From 64741d991b8f129943c62f70e478a7565a26e3df Mon Sep 17 00:00:00 2001 From: Stuart Daines Date: Thu, 3 Aug 2023 14:28:07 +0100 Subject: [PATCH 1/3] Add a project_region! option for Newton method If using `method=:newton` with `linesearch=LineSearches.Static()` (the default linesearch), an additional argument `project_region!` (default value `(x)->(nothing)`) can be used to define a function to modify the proposed new `x` value after each Newton iteration. For example, `project_region! = (x) -> (x .= max.(x, 1e-80))` will enforce a constraint `x[i] >= 1e-80`. --- README.md | 6 ++++++ src/nlsolve/nlsolve.jl | 3 ++- src/solvers/newton.jl | 7 +++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7e41d4b..83e7a67 100644 --- a/README.md +++ b/README.md @@ -307,6 +307,12 @@ By default, no linesearch is performed. **Note:** it is assumed that a passed linesearch function will at least update the solution vector and evaluate the function at the new point. +Added in sjdaines fork: if `linesearch=LineSearches.Static()` (the default), an additional argument +`project_region!` (default value `(x)->(nothing)`) can be used to define a function to modify the proposed new +`x` value after the Newton iteration. eg `project_region! = (x)->(x .= max.(x, 1e-80))` will enforce a constraint +`x[i] >= 1e-80`. + + ## Anderson acceleration This method is selected with `method = :anderson`. diff --git a/src/nlsolve/nlsolve.jl b/src/nlsolve/nlsolve.jl index 9bc06e8..19ef5b0 100644 --- a/src/nlsolve/nlsolve.jl +++ b/src/nlsolve/nlsolve.jl @@ -9,6 +9,7 @@ function nlsolve(df::Union{NonDifferentiable, OnceDifferentiable}, extended_trace::Bool = false, linesearch = LineSearches.Static(), linsolve=(x, A, b) -> copyto!(x, A\b), + project_region! = (x) -> (nothing), factor::Real = one(real(eltype(initial_x))), autoscale::Bool = true, m::Integer = 10, @@ -21,7 +22,7 @@ function nlsolve(df::Union{NonDifferentiable, OnceDifferentiable}, end if method == :newton newton(df, initial_x, xtol, ftol, iterations, - store_trace, show_trace, extended_trace, linesearch; linsolve=linsolve) + store_trace, show_trace, extended_trace, linesearch; linsolve=linsolve, project_region! = project_region!) elseif method == :trust_region trust_region(df, initial_x, xtol, ftol, iterations, store_trace, show_trace, extended_trace, factor, diff --git a/src/solvers/newton.jl b/src/solvers/newton.jl index d9a424b..91b6eaf 100644 --- a/src/solvers/newton.jl +++ b/src/solvers/newton.jl @@ -42,6 +42,7 @@ function newton_(df::OnceDifferentiable, extended_trace::Bool, linesearch, linsolve, + project_region!, cache = NewtonCache(df)) where T n = length(initial_x) copyto!(cache.x, initial_x) @@ -111,6 +112,7 @@ function newton_(df::OnceDifferentiable, if linesearch isa Static x_ls .= cache.x .+ cache.p + project_region!(x_ls) value_jacobian!(df, x_ls) alpha, ϕalpha = one(real(T)), value(dfo) else @@ -142,6 +144,7 @@ function newton(df::OnceDifferentiable, extended_trace::Bool, linesearch, cache = NewtonCache(df); - linsolve=(x, A, b) -> copyto!(x, A\b)) where T - newton_(df, initial_x, convert(real(T), xtol), convert(real(T), ftol), iterations, store_trace, show_trace, extended_trace, linesearch, linsolve, cache) + linsolve=(x, A, b) -> copyto!(x, A\b), + project_region! = (x) -> (nothing)) where T + newton_(df, initial_x, convert(real(T), xtol), convert(real(T), ftol), iterations, store_trace, show_trace, extended_trace, linesearch, linsolve, project_region!, cache) end From c3a182ca4b8e3cb4306f387cf4e22cd6b3b9bb70 Mon Sep 17 00:00:00 2001 From: Daines Date: Wed, 11 Oct 2023 20:08:21 +0100 Subject: [PATCH 2/3] Generalize project_region! to apply_step! If `method = :newton` and `linesearch=LineSearches.Static()` (the default), an additional argument `apply_step!` (default value `(x, x_old, newton_step)->(x .= x_old .+ newton_step)`) can be used to define a function to modify the proposed new `x` value after the Newton iteration. eg `apply_step! = (x, x_old, newton_step)->(x .= x_old .+ newton_step; x .= max.(x, 1e-80))` will enforce a constraint `x[i] >= 1e-80`. --- README.md | 10 +++++----- src/nlsolve/nlsolve.jl | 4 ++-- src/solvers/newton.jl | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 83e7a67..47bb929 100644 --- a/README.md +++ b/README.md @@ -307,11 +307,11 @@ By default, no linesearch is performed. **Note:** it is assumed that a passed linesearch function will at least update the solution vector and evaluate the function at the new point. -Added in sjdaines fork: if `linesearch=LineSearches.Static()` (the default), an additional argument -`project_region!` (default value `(x)->(nothing)`) can be used to define a function to modify the proposed new -`x` value after the Newton iteration. eg `project_region! = (x)->(x .= max.(x, 1e-80))` will enforce a constraint -`x[i] >= 1e-80`. - +If `method = :newton` and `linesearch=LineSearches.Static()` (the default), an additional parameter +`apply_step!` (default value `(x, x_old, newton_step)->(x .= x_old .+ newton_step)`) can be used to +define a problem-specific function to update the `x` value after the Newton iteration. +For example, `apply_step! = (x, x_old, newton_step)->(x .= x_old .+ newton_step; x .= max.(x, 1e-80))` +will enforce a constraint `x[i] >= 1e-80`. ## Anderson acceleration diff --git a/src/nlsolve/nlsolve.jl b/src/nlsolve/nlsolve.jl index 19ef5b0..2ef6971 100644 --- a/src/nlsolve/nlsolve.jl +++ b/src/nlsolve/nlsolve.jl @@ -9,7 +9,7 @@ function nlsolve(df::Union{NonDifferentiable, OnceDifferentiable}, extended_trace::Bool = false, linesearch = LineSearches.Static(), linsolve=(x, A, b) -> copyto!(x, A\b), - project_region! = (x) -> (nothing), + apply_step! = (x, x_old, newton_step)->(x .= x_old .+ newton_step), factor::Real = one(real(eltype(initial_x))), autoscale::Bool = true, m::Integer = 10, @@ -22,7 +22,7 @@ function nlsolve(df::Union{NonDifferentiable, OnceDifferentiable}, end if method == :newton newton(df, initial_x, xtol, ftol, iterations, - store_trace, show_trace, extended_trace, linesearch; linsolve=linsolve, project_region! = project_region!) + store_trace, show_trace, extended_trace, linesearch; linsolve=linsolve, apply_step! =apply_step!) elseif method == :trust_region trust_region(df, initial_x, xtol, ftol, iterations, store_trace, show_trace, extended_trace, factor, diff --git a/src/solvers/newton.jl b/src/solvers/newton.jl index 91b6eaf..ecfad5a 100644 --- a/src/solvers/newton.jl +++ b/src/solvers/newton.jl @@ -42,7 +42,7 @@ function newton_(df::OnceDifferentiable, extended_trace::Bool, linesearch, linsolve, - project_region!, + apply_step!, cache = NewtonCache(df)) where T n = length(initial_x) copyto!(cache.x, initial_x) @@ -111,8 +111,8 @@ function newton_(df::OnceDifferentiable, if linesearch isa Static - x_ls .= cache.x .+ cache.p - project_region!(x_ls) + # default is x_ls .= cache.x .+ cache.p + apply_step!(x_ls, cache.x, cache.p) value_jacobian!(df, x_ls) alpha, ϕalpha = one(real(T)), value(dfo) else @@ -145,6 +145,6 @@ function newton(df::OnceDifferentiable, linesearch, cache = NewtonCache(df); linsolve=(x, A, b) -> copyto!(x, A\b), - project_region! = (x) -> (nothing)) where T - newton_(df, initial_x, convert(real(T), xtol), convert(real(T), ftol), iterations, store_trace, show_trace, extended_trace, linesearch, linsolve, project_region!, cache) + apply_step! = (x, x_old, newton_step)->(x .= x_old .+ newton_step)) where T + newton_(df, initial_x, convert(real(T), xtol), convert(real(T), ftol), iterations, store_trace, show_trace, extended_trace, linesearch, linsolve, apply_step!, cache) end From b475fd008dc2a022a11eff4b9ae2d3e88d35af07 Mon Sep 17 00:00:00 2001 From: Stuart Daines Date: Fri, 13 Dec 2024 10:34:16 +0000 Subject: [PATCH 3/3] Add always_step option to Newton solver From updated README.md: If the optional parameter `always_step = false` (the default), no Newton iteration is performed if the solution is already converged for the supplied initial `x` values, and the supplied initial `x` is returned in the `zero` field of `SolverResults`. If `always_step = true`, a Newton step is performed before testing for convergence. --- README.md | 7 ++++++- src/nlsolve/nlsolve.jl | 3 ++- src/solvers/newton.jl | 13 ++++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 47bb929..32be5fd 100644 --- a/README.md +++ b/README.md @@ -307,12 +307,17 @@ By default, no linesearch is performed. **Note:** it is assumed that a passed linesearch function will at least update the solution vector and evaluate the function at the new point. -If `method = :newton` and `linesearch=LineSearches.Static()` (the default), an additional parameter +If `linesearch=LineSearches.Static()` (the default), an additional parameter `apply_step!` (default value `(x, x_old, newton_step)->(x .= x_old .+ newton_step)`) can be used to define a problem-specific function to update the `x` value after the Newton iteration. For example, `apply_step! = (x, x_old, newton_step)->(x .= x_old .+ newton_step; x .= max.(x, 1e-80))` will enforce a constraint `x[i] >= 1e-80`. +If the optional parameter `always_step = false` (the default), no Newton iteration is performed +if the solution is already converged for the supplied initial `x` values, and the supplied initial `x` is returned +in the `zero` field of `SolverResults`. If `always_step = true`, a Newton step is performed +before testing for convergence. + ## Anderson acceleration This method is selected with `method = :anderson`. diff --git a/src/nlsolve/nlsolve.jl b/src/nlsolve/nlsolve.jl index 2ef6971..2d1ca43 100644 --- a/src/nlsolve/nlsolve.jl +++ b/src/nlsolve/nlsolve.jl @@ -10,6 +10,7 @@ function nlsolve(df::Union{NonDifferentiable, OnceDifferentiable}, linesearch = LineSearches.Static(), linsolve=(x, A, b) -> copyto!(x, A\b), apply_step! = (x, x_old, newton_step)->(x .= x_old .+ newton_step), + always_step = false, factor::Real = one(real(eltype(initial_x))), autoscale::Bool = true, m::Integer = 10, @@ -22,7 +23,7 @@ function nlsolve(df::Union{NonDifferentiable, OnceDifferentiable}, end if method == :newton newton(df, initial_x, xtol, ftol, iterations, - store_trace, show_trace, extended_trace, linesearch; linsolve=linsolve, apply_step! =apply_step!) + store_trace, show_trace, extended_trace, linesearch; linsolve=linsolve, apply_step! =apply_step!, always_step=always_step) elseif method == :trust_region trust_region(df, initial_x, xtol, ftol, iterations, store_trace, show_trace, extended_trace, factor, diff --git a/src/solvers/newton.jl b/src/solvers/newton.jl index ecfad5a..4f16a78 100644 --- a/src/solvers/newton.jl +++ b/src/solvers/newton.jl @@ -43,6 +43,7 @@ function newton_(df::OnceDifferentiable, linesearch, linsolve, apply_step!, + always_step, cache = NewtonCache(df)) where T n = length(initial_x) copyto!(cache.x, initial_x) @@ -53,7 +54,12 @@ function newton_(df::OnceDifferentiable, x_converged, f_converged = assess_convergence(initial_x, cache.xold, value(df), NaN, ftol) stopped = any(isnan, cache.x) || any(isnan, value(df)) ? true : false - converged = x_converged || f_converged + if always_step + converged = false + else + converged = x_converged || f_converged + end + x_ls = copy(cache.x) tr = SolverTrace() tracing = store_trace || show_trace || extended_trace @@ -145,6 +151,7 @@ function newton(df::OnceDifferentiable, linesearch, cache = NewtonCache(df); linsolve=(x, A, b) -> copyto!(x, A\b), - apply_step! = (x, x_old, newton_step)->(x .= x_old .+ newton_step)) where T - newton_(df, initial_x, convert(real(T), xtol), convert(real(T), ftol), iterations, store_trace, show_trace, extended_trace, linesearch, linsolve, apply_step!, cache) + apply_step! = (x, x_old, newton_step)->(x .= x_old .+ newton_step), + always_step::Bool) where T + newton_(df, initial_x, convert(real(T), xtol), convert(real(T), ftol), iterations, store_trace, show_trace, extended_trace, linesearch, linsolve, apply_step!, always_step, cache) end