From 6f08a89c94af4c0ec0886fc2d38efc40d238af24 Mon Sep 17 00:00:00 2001
From: Jutho Haegeman <jutho.haegeman@ugent.be>
Date: Fri, 26 Jun 2020 00:28:15 +0200
Subject: [PATCH] add both function value and normgrad to history

---
 src/cg.jl           | 5 ++++-
 src/gd.jl           | 9 ++++++---
 src/lbfgs.jl        | 5 ++++-
 src/linesearches.jl | 3 ++-
 4 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/cg.jl b/src/cg.jl
index 0d0d42b..7bedce0 100644
--- a/src/cg.jl
+++ b/src/cg.jl
@@ -25,6 +25,7 @@ function optimize(fg, x, alg::ConjugateGradient;
     numfg = 1
     innergg = inner(x, g, g)
     normgrad = sqrt(innergg)
+    fhistory = [f]
     normgradhistory = [normgrad]
 
     # compute here once to define initial value of α in scale-invariant way
@@ -71,6 +72,7 @@ function optimize(fg, x, alg::ConjugateGradient;
         x, f, g = finalize!(x, f, g, numiter)
         innergg = inner(x, g, g)
         normgrad = sqrt(innergg)
+        push!(fhistory, f)
         push!(normgradhistory, normgrad)
 
         # check stopping criteria and print info
@@ -102,7 +104,8 @@ function optimize(fg, x, alg::ConjugateGradient;
                             f, normgrad)
         end
     end
-    return x, f, g, numfg, normgradhistory
+    history = [fhistory normgradhistory]
+    return x, f, g, numfg, history
 end
 
 struct HagerZhang{T<:Real} <: CGFlavor
diff --git a/src/gd.jl b/src/gd.jl
index 41cba0e..eefd265 100644
--- a/src/gd.jl
+++ b/src/gd.jl
@@ -20,6 +20,7 @@ function optimize(fg, x, alg::GradientDescent;
     numfg = 1
     innergg = inner(x, g, g)
     normgrad = sqrt(innergg)
+    fhistory = [f]
     normgradhistory = [normgrad]
 
     # compute here once to define initial value of α in scale-invariant way
@@ -46,6 +47,7 @@ function optimize(fg, x, alg::GradientDescent;
         x, f, g = finalize!(x, f, g, numiter)
         innergg = inner(x, g, g)
         normgrad = sqrt(innergg)
+        push!(fhistory, f)
         push!(normgradhistory, normgrad)
 
         # check stopping criteria and print info
@@ -53,8 +55,8 @@ function optimize(fg, x, alg::GradientDescent;
             break
         end
         verbosity >= 2 &&
-            @info @sprintf("GD: iter %4d: f = %.12f, ‖∇f‖ = %.4e, step size = %.2e",
-                            numiter, f, normgrad, α)
+            @info @sprintf("GD: iter %4d: f = %.12f, ‖∇f‖ = %.4e, α = %.2e, nfg = %d",
+                            numiter, f, normgrad, α, nfg)
 
         # increase α for next step
         α = 2*α
@@ -68,5 +70,6 @@ function optimize(fg, x, alg::GradientDescent;
                             f, normgrad)
         end
     end
-    return x, f, g, numfg, normgradhistory
+    history = [fhistory normgradhistory]
+    return x, f, g, numfg, history
 end
diff --git a/src/lbfgs.jl b/src/lbfgs.jl
index e9c9a1a..02078b4 100644
--- a/src/lbfgs.jl
+++ b/src/lbfgs.jl
@@ -22,6 +22,7 @@ function optimize(fg, x, alg::LBFGS;
     numfg = 1
     innergg = inner(x, g, g)
     normgrad = sqrt(innergg)
+    fhistory = [f]
     normgradhistory = [normgrad]
 
     TangentType = typeof(g)
@@ -64,6 +65,7 @@ function optimize(fg, x, alg::LBFGS;
         x, f, g = finalize!(x, f, g, numiter)
         innergg = inner(x, g, g)
         normgrad = sqrt(innergg)
+        push!(fhistory, f)
         push!(normgradhistory, normgrad)
 
         # check stopping criteria and print info
@@ -140,7 +142,8 @@ function optimize(fg, x, alg::LBFGS;
                             f, normgrad)
         end
     end
-    return x, f, g, numfg, normgradhistory
+    history = [fhistory normgradhistory]
+    return x, f, g, numfg, history
 end
 
 mutable struct LBFGSInverseHessian{TangentType,ScalarType}
diff --git a/src/linesearches.jl b/src/linesearches.jl
index b3a87dd..e870571 100644
--- a/src/linesearches.jl
+++ b/src/linesearches.jl
@@ -103,7 +103,8 @@ function bisect(iter::HagerZhangLineSearchIterator, a::LineSearchPoint, b::LineS
     numfg = 0
     while true
         if b.α - a.α < eps()
-            error(@sprintf("Linesearch bisection failure: [a, b] = [%.2e, %.2e], b-a = %.2e, dϕᵃ = %.2e, dϕᵇ = %.2e, (ϕᵇ - ϕᵃ)/(b-a) = %.2e", a.α, b.α, b.α - a.α, a.dϕ, b.dϕ, (b.ϕ - a.ϕ)/(b.α - a.α)))
+            @warn @sprintf("Linesearch bisection failure: [a, b] = [%.2e, %.2e], b-a = %.2e, dϕᵃ = %.2e, dϕᵇ = %.2e, (ϕᵇ - ϕᵃ)/(b-a) = %.2e", a.α, b.α, b.α - a.α, a.dϕ, b.dϕ, (b.ϕ - a.ϕ)/(b.α - a.α))
+            return a, b, numfg
         end
         αc = (1 - θ) * a.α + θ * b.α
         c = takestep(iter, αc)