From 4fe1bf219c2c612358a9b497a14597c22b0e093a Mon Sep 17 00:00:00 2001 From: Hendrych Date: Wed, 19 Feb 2025 15:30:14 +0100 Subject: [PATCH 1/5] Replace default linesearch with Secant. --- src/afw.jl | 2 +- src/blended_cg.jl | 4 ++-- src/blended_pairwise.jl | 2 +- src/block_coordinate_algorithms.jl | 2 +- src/fw_algorithms.jl | 4 ++-- src/pairwise.jl | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/afw.jl b/src/afw.jl index 14fefdbbc..5dbbe1c79 100644 --- a/src/afw.jl +++ b/src/afw.jl @@ -12,7 +12,7 @@ function away_frank_wolfe( grad!, lmo, x0; - line_search::LineSearchMethod=Adaptive(), + line_search::LineSearchMethod=Secant(), lazy_tolerance=2.0, epsilon=1e-7, away_steps=true, diff --git a/src/blended_cg.jl b/src/blended_cg.jl index f239b5ca9..73680736a 100644 --- a/src/blended_cg.jl +++ b/src/blended_cg.jl @@ -14,8 +14,8 @@ function blended_conditional_gradient( grad!, lmo, x0; - line_search::LineSearchMethod=Adaptive(), - line_search_inner::LineSearchMethod=Adaptive(), + line_search::LineSearchMethod=Secant(), + line_search_inner::LineSearchMethod=Secant(), hessian=nothing, epsilon=1e-7, max_iteration=10000, diff --git a/src/blended_pairwise.jl b/src/blended_pairwise.jl index d6cc39451..a04a446aa 100644 --- a/src/blended_pairwise.jl +++ b/src/blended_pairwise.jl @@ -10,7 +10,7 @@ function blended_pairwise_conditional_gradient( grad!, lmo, x0; - line_search::LineSearchMethod=Adaptive(), + line_search::LineSearchMethod=Secant(), epsilon=1e-7, max_iteration=10000, print_iter=1000, diff --git a/src/block_coordinate_algorithms.jl b/src/block_coordinate_algorithms.jl index cabe0527e..627e504ea 100644 --- a/src/block_coordinate_algorithms.jl +++ b/src/block_coordinate_algorithms.jl @@ -476,7 +476,7 @@ function block_coordinate_frank_wolfe( lmo::ProductLMO{N}, x0::BlockVector; update_order::BlockCoordinateUpdateOrder=CyclicUpdate(), - line_search::LS=Adaptive(), + line_search::LS=Secant(), update_step::US=FrankWolfeStep(), momentum=nothing, epsilon=1e-7, diff --git a/src/fw_algorithms.jl b/src/fw_algorithms.jl index c2f8c731a..40affa24d 100644 --- a/src/fw_algorithms.jl +++ b/src/fw_algorithms.jl @@ -15,7 +15,7 @@ function frank_wolfe( grad!, lmo, x0; - line_search::LineSearchMethod=Adaptive(), + line_search::LineSearchMethod=Secant(), momentum=nothing, epsilon=1e-7, max_iteration=10000, @@ -254,7 +254,7 @@ function lazified_conditional_gradient( grad!, lmo_base, x0; - line_search::LineSearchMethod=Adaptive(), + line_search::LineSearchMethod=Secant(), lazy_tolerance=2.0, cache_size=Inf, greedy_lazy=false, diff --git a/src/pairwise.jl b/src/pairwise.jl index 1083a92ac..426320595 100644 --- a/src/pairwise.jl +++ b/src/pairwise.jl @@ -13,7 +13,7 @@ function pairwise_frank_wolfe( grad!, lmo, x0; - line_search::LineSearchMethod=Adaptive(), + line_search::LineSearchMethod=Secant(), lazy_tolerance=2.0, epsilon=1e-7, lazy=false, From ddc4abb36bb09d0f4740ba19f26529865f981a36 Mon Sep 17 00:00:00 2001 From: Hendrych Date: Wed, 19 Feb 2025 15:44:58 +0100 Subject: [PATCH 2/5] Set Adaptive as fallback for Secant. --- src/linesearch.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/linesearch.jl b/src/linesearch.jl index 497f14695..e671f4565 100644 --- a/src/linesearch.jl +++ b/src/linesearch.jl @@ -380,10 +380,11 @@ struct Secant{F,LSM<:LineSearchMethod} <: LineSearchMethod end function Secant(limit_num_steps, tol) - return Secant(Backtracking(), false, limit_num_steps, tol, x -> true) + #return Secant(Backtracking(), false, limit_num_steps, tol, x -> true) + return Secant(Adaptive(), false, limit_num_steps, tol, x -> true) end -function Secant(;inner_ls=Backtracking(), safe=false, limit_num_steps=40, tol=1e-8, domain_oracle=(x -> true)) +function Secant(;inner_ls=Adaptive(), safe=false, limit_num_steps=40, tol=1e-8, domain_oracle=(x -> true)) return Secant(inner_ls, safe, limit_num_steps, tol, domain_oracle) end From 545c82c815d2658617334c3aecbd4b2cb3bfe92f Mon Sep 17 00:00:00 2001 From: Deborah Hendrych Date: Tue, 25 Feb 2025 10:40:21 +0100 Subject: [PATCH 3/5] Set up counter. --- test/trajectory_tests/away_step_cg.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/trajectory_tests/away_step_cg.jl b/test/trajectory_tests/away_step_cg.jl index 309d36274..065c8c060 100644 --- a/test/trajectory_tests/away_step_cg.jl +++ b/test/trajectory_tests/away_step_cg.jl @@ -508,7 +508,7 @@ const xp = [ @test norm(res3[1] - x_true3) <= 5e-5 @test res3[3] ≈ primal_true3 - @test res3[5][end][1] <= 451 + @test res3[5][end][1] <= 1203 x_true4 = [ 0.12350364160905855, @@ -634,6 +634,6 @@ const xp = [ @test norm(res4[1] - x_true4) <= 5e-5 @test res4[3] ≈ primal_true4 - @test res4[5][end][1] <= 451 + @test res4[5][end][1] <= 1203 end From 7da01e97006b7d15fafed3f784dc0e211bb88693 Mon Sep 17 00:00:00 2001 From: Deborah Hendrych Date: Tue, 25 Feb 2025 10:40:34 +0100 Subject: [PATCH 4/5] Set up counter and adapt solution. --- .../approximate_caratheodory.jl | 208 +++++++++--------- 1 file changed, 104 insertions(+), 104 deletions(-) diff --git a/test/trajectory_tests/approximate_caratheodory.jl b/test/trajectory_tests/approximate_caratheodory.jl index 13c604989..0de4f9d54 100644 --- a/test/trajectory_tests/approximate_caratheodory.jl +++ b/test/trajectory_tests/approximate_caratheodory.jl @@ -278,7 +278,7 @@ using LinearAlgebra max_iteration=k, line_search=FrankWolfe.AdaptiveZerothOrder(), print_iter=k / 10, - verbose=false, + verbose=true, memory_mode=FrankWolfe.OutplaceEmphasis(), trajectory=true, ) @@ -291,122 +291,122 @@ using LinearAlgebra max_iteration=2k, line_search=FrankWolfe.Adaptive(), print_iter=k / 10, - verbose=false, + verbose=true, memory_mode=FrankWolfe.OutplaceEmphasis(), trajectory=true, ) x_true = [ - 0.0110488779224248954979881176541312015615403652191162109375, - 0.01116382439959395615758364073144548456184566020965576171875, - 0.01132266603788188719104113033608882687985897064208984375, - 0.01096697959160035373837871475188876502215862274169921875, - 0.01099748235218585799832791138896936899982392787933349609375, - 0.01090658273640816855465374146660906262695789337158203125, - 0.01100728641192584740526871911470152554102241992950439453125, - 0.01113317498199122533575344817791119567118585109710693359375, - 0.01129240465802835817477056679081215406768023967742919921875, - 0.01066971268420941788834799268670394667424261569976806640625, - 0.01073806129575670918752106075544361374340951442718505859375, - 0.01081623406552431089500121430546641931869089603424072265625, - 0.01090669919495564944844634425180629477836191654205322265625, - 0.011012761038243910893807964157531387172639369964599609375, - 0.011138933094013821201162528495842707343399524688720703125, - 0.01129152502908182219287791525630382238887250423431396484375, - 0.0106825195127519152749062669727209140546619892120361328125, - 0.01075709593593467301719801554327204939909279346466064453125, - 0.010841556533814002138971233080155798234045505523681640625, - 0.0109381334615336230087212499029192258603870868682861328125, - 0.0110670496344356487916638087654064293019473552703857421875, - 0.0113528003501498948868420058033734676428139209747314453125, - 0.010755333351140705655524243411491625010967254638671875, - 0.01048383730090962639991403193562291562557220458984375, - 0.01067948351180760153955606739373251912184059619903564453125, - 0.01090231617566354838100295410185935907065868377685546875, - 0.011158644863763077237361898141898564063012599945068359375, - 0.01145699293377413878480819420246916706673800945281982421875, - 0.0101981244628182483868972241225492325611412525177001953125, - 0.01036021643924461659025393345245902310125529766082763671875, - 0.01054198990165635708982083684759345487691462039947509765625, - 0.01074709040997723179244882629745916347019374370574951171875, - 0.010980253194607388078640752837600302882492542266845703125, - 0.01124773117307073695692043457938780193217098712921142578125, - 0.01155792943935683714240525432614958845078945159912109375, - 0.0102594043125243845893113103784344275481998920440673828125, - 0.01043029476421276045827735146076520322822034358978271484375, - 0.0106219729796506283381329893700240063481032848358154296875, - 0.0108381932769191467735847567155360593460500240325927734375, - 0.01108380957646447860509564264930304489098489284515380859375, - 0.01136520571619974682986420333463684073649346828460693359375, - 0.01015188403354872447026391313329440890811383724212646484375, - 0.01031140103979160359271016744742155424319207668304443359375, - 0.01048962535578343464870432200086725060828030109405517578125, - 0.01068961614661836749540224644761110539548099040985107421875, - 0.0109152507282073325811655450934267719276249408721923828125, - 0.01117152676702971858535562432734877802431583404541015625, - 0.01146500004615019947806775491017106105573475360870361328125, - 0.0102103010985286381251402332281941198743879795074462890625, - 0.0103773954918663681434853884866242879070341587066650390625, - 0.0105642139757310508929588621640505152754485607147216796875, - 0.010773965466789041378614655286583001725375652313232421875, - 0.01101070848304721259969252145083373761735856533050537109375, - 0.01127966475382357995627113922409989754669368267059326171875, - 0.01010692369146876622154618274862514226697385311126708984375, - 0.0102628472748964548466599211451466544531285762786865234375, - 0.0104367298381483307456729647810789174400269985198974609375, - 0.0106312766291480863267704393138046725653111934661865234375, - 0.01084984936168993142902028381513446220196783542633056640625, - 0.01109669576874410847067142782407245249487459659576416015625, - 0.0113772771187743877707720940861690905876457691192626953125, - 0.01016309273690228949516001222264094394631683826446533203125, - 0.01032593705535572552178802396838364074937999248504638671875, - 0.01050767080610974339716090497631739708594977855682373046875, - 0.010711151044073143057122621257803984917700290679931640625, - 0.010939927274482348640294304686904069967567920684814453125, - 0.011198484026182613237931917637979495339095592498779296875, - 0.01006344361601584948273657715844819904305040836334228515625, - 0.0102154242798425699823017254175283596850931644439697265625, - 0.0103847197150773624951813900452179950661957263946533203125, - 0.0105737916639839467369821335296364850364625453948974609375, - 0.01078565122675433952947887661366621614433825016021728515625, - 0.01102404130098686395322626907500307424925267696380615234375, - 0.01129369392894255010040271969273817376233637332916259765625, - 0.01011769773639966361888919976763645536266267299652099609375, - 0.0102761657076108035846484511921516968868672847747802734375, - 0.0104528084497103428140984959782144869677722454071044921875, - 0.0106502317386623530925948699632499483413994312286376953125, - 0.0108716260264075381680726195554598234593868255615234375, - 0.01112096145948445434503693007854963070712983608245849609375, - 0.0120102566865593306244530680260140798054635524749755859375, - 0.00932655184389477413808844374898399109952151775360107421875, - 0.00826079786559208613383464836488201399333775043487548828125, - 0.00916835118748905719687769533265964128077030181884765625, - 0.0101832309647222028770041646339450380764901638031005859375, - 0.01132195055394567655138171602402508142404258251190185546875, - 0.012605048988675919552360227271492476575076580047607421875, - 0.00972619723519543323553282476723325089551508426666259765625, - 0.0086199687394420236585812489238378475420176982879638671875, - 0.0095702153731971155437019405098908464424312114715576171875, - 0.0106337321108964551197306747098991763778030872344970703125, - 0.01182819416640477905300343763883574865758419036865234375, - 0.01317564671500810573323558827496526646427810192108154296875, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086126771722998540781190968118607997894287109375, + 0.0107935205086125661499973915624650544486939907073974609375, + 0.010793520508612812480730980269072460941970348358154296875, + 0.0082491592577794252416634179780885460786521434783935546875, + 0.0091554500430683742606685626697071711532771587371826171875, + 0.0101689076486805307641958506792434491217136383056640625, + 0.0113060143892506036189526952284722938202321529388427734375, + 0.01258726853086838438910977089335574419237673282623291015625, + 0.00971248924824500044217234773213931475766003131866455078125, + 0.00860784740372220504023292875217521213926374912261962890625, + 0.0095567688004197272333417600975735695101320743560791015625, + 0.0106187909950250895796397543335842783562839031219482421875, + 0.011811556112865738421557892934288247488439083099365234375, + 0.01315706586383530267425623350163732538931071758270263671875, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 ] - primal_true = 0.01079492075856676147135835760962987221028689269130597783892595739691365095379305 + primal_true = 0.01077968911106035588615645830682523782565520615359182770140112868746551529584854 @test norm(res4[1] - x_true) ≈ 0 atol = 1e-6 @test res4[3] ≈ primal_true @test res4[5][end][1] == 101 @test res4_adaptive[3] ≈ 0.01 atol=1e-8 - @test res4_adaptive[5][end][1] == 138 + @test res4_adaptive[5][end][1] == 122 end From 2921e67df25b79c8d688c4e5ffe452aba522abe8 Mon Sep 17 00:00:00 2001 From: dhendryc <92737336+dhendryc@users.noreply.github.com> Date: Tue, 25 Feb 2025 11:44:36 +0100 Subject: [PATCH 5/5] Update linesearch.jl --- src/linesearch.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/linesearch.jl b/src/linesearch.jl index e671f4565..815d74bee 100644 --- a/src/linesearch.jl +++ b/src/linesearch.jl @@ -380,7 +380,6 @@ struct Secant{F,LSM<:LineSearchMethod} <: LineSearchMethod end function Secant(limit_num_steps, tol) - #return Secant(Backtracking(), false, limit_num_steps, tol, x -> true) return Secant(Adaptive(), false, limit_num_steps, tol, x -> true) end