From 29709d412948b24fe98c5531cd33d76a7766afc0 Mon Sep 17 00:00:00 2001 From: tomchor Date: Thu, 23 May 2024 17:30:26 -0700 Subject: [PATCH 01/14] reset model clock and skip time_step if next_actuation_time is really tiny --- src/Simulations/run.jl | 18 ++++++++++++++++-- src/Utils/schedules.jl | 6 ++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/Simulations/run.jl b/src/Simulations/run.jl index 56c46de757..a3cf50b414 100644 --- a/src/Simulations/run.jl +++ b/src/Simulations/run.jl @@ -7,7 +7,7 @@ using Oceananigans: AbstractModel, run_diagnostic!, write_output! import Oceananigans: initialize! import Oceananigans.OutputWriters: checkpoint_path, set! import Oceananigans.TimeSteppers: time_step! -import Oceananigans.Utils: aligned_time_step +import Oceananigans.Utils: aligned_time_step, next_actuation_time # Simulations are for running @@ -21,6 +21,15 @@ function collect_scheduled_activities(sim) return tuple(writers..., callbacks...) end +function next_actuation_time(sim::Simulation) + activities = collect_scheduled_activities(sim) + nearest_next_actuation_time = Inf + for activity in activities + nearest_next_actuation_time = min(nearest_next_actuation_time, next_actuation_time(activity.schedule)) + end + return nearest_next_actuation_time +end + function schedule_aligned_Δt(sim, aligned_Δt) clock = sim.model.clock activities = collect_scheduled_activities(sim) @@ -131,7 +140,12 @@ function time_step!(sim::Simulation) else # business as usual... Δt = aligned_time_step(sim, sim.Δt) - time_step!(sim.model, Δt, callbacks=model_callbacks) + if Δt > (sim.Δt / 1e10) + time_step!(sim.model, Δt, callbacks=model_callbacks) + else + println("Skipping aligned time step, which is of ", Δt) + sim.model.clock.time = next_actuation_time(sim) + end end # Callbacks and callback-like things diff --git a/src/Utils/schedules.jl b/src/Utils/schedules.jl index bf710a835a..62e988325a 100644 --- a/src/Utils/schedules.jl +++ b/src/Utils/schedules.jl @@ -61,9 +61,11 @@ function (schedule::TimeInterval)(model) end end +next_actuation_time(schedule::TimeInterval) = schedule.previous_actuation_time + schedule.interval +next_actuation_time(schedule::IterationInterval) = Inf + function aligned_time_step(schedule::TimeInterval, clock, Δt) - next_actuation_time = schedule.previous_actuation_time + schedule.interval - return min(Δt, next_actuation_time - clock.time) + return min(Δt, next_actuation_time(schedule::TimeInterval) - clock.time) end ##### From ed130eee59c8662e793c26972465e7cda63c5f60 Mon Sep 17 00:00:00 2001 From: tomchor Date: Fri, 24 May 2024 13:34:11 -0700 Subject: [PATCH 02/14] move next_actuation_time definition to after IterationInterval is defined --- src/Utils/schedules.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Utils/schedules.jl b/src/Utils/schedules.jl index 62e988325a..bdf72b302c 100644 --- a/src/Utils/schedules.jl +++ b/src/Utils/schedules.jl @@ -62,7 +62,6 @@ function (schedule::TimeInterval)(model) end next_actuation_time(schedule::TimeInterval) = schedule.previous_actuation_time + schedule.interval -next_actuation_time(schedule::IterationInterval) = Inf function aligned_time_step(schedule::TimeInterval, clock, Δt) return min(Δt, next_actuation_time(schedule::TimeInterval) - clock.time) @@ -92,6 +91,8 @@ IterationInterval(interval; offset=0) = IterationInterval(interval, offset) (schedule::IterationInterval)(model) = (model.clock.iteration - schedule.offset) % schedule.interval == 0 +next_actuation_time(schedule::IterationInterval) = Inf + ##### ##### WallTimeInterval ##### From d6b0ca4f5757f464badfb38a68d70fed54b773de Mon Sep 17 00:00:00 2001 From: tomaschor Date: Mon, 18 Nov 2024 08:23:48 -0300 Subject: [PATCH 03/14] bugfix --- src/Utils/schedules.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Utils/schedules.jl b/src/Utils/schedules.jl index e7bd406f36..23f2e2bab4 100644 --- a/src/Utils/schedules.jl +++ b/src/Utils/schedules.jl @@ -78,8 +78,6 @@ function (schedule::TimeInterval)(model) end end -next_actuation_time(schedule::TimeInterval) = schedule.previous_actuation_time + schedule.interval - function schedule_aligned_time_step(schedule::TimeInterval, clock, Δt) t★ = next_actuation_time(schedule) t = clock.time From 218d8034d2243e0aeed8d12bf6819dc6055b8c4d Mon Sep 17 00:00:00 2001 From: tomaschor Date: Fri, 13 Dec 2024 14:58:22 -0500 Subject: [PATCH 04/14] add minimum_relative_step to Simulation --- src/Simulations/simulation.jl | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/Simulations/simulation.jl b/src/Simulations/simulation.jl index bd6a0fbbe1..c05364f633 100644 --- a/src/Simulations/simulation.jl +++ b/src/Simulations/simulation.jl @@ -9,18 +9,19 @@ import Oceananigans.TimeSteppers: reset! default_progress(simulation) = nothing mutable struct Simulation{ML, DT, ST, DI, OW, CB} - model :: ML - Δt :: DT - stop_iteration :: Float64 - stop_time :: ST - wall_time_limit :: Float64 - diagnostics :: DI - output_writers :: OW - callbacks :: CB - run_wall_time :: Float64 - running :: Bool - initialized :: Bool - verbose :: Bool + model :: ML + Δt :: DT + stop_iteration :: Float64 + stop_time :: ST + wall_time_limit :: Float64 + diagnostics :: DI + output_writers :: OW + callbacks :: CB + run_wall_time :: Float64 + running :: Bool + initialized :: Bool + verbose :: Bool + minimum_relative_step :: Float64 end """ @@ -49,7 +50,8 @@ function Simulation(model; Δt, verbose = true, stop_iteration = Inf, stop_time = Inf, - wall_time_limit = Inf) + wall_time_limit = Inf, + minimum_relative_step = 0) if stop_iteration == Inf && stop_time == Inf && wall_time_limit == Inf @warn "This simulation will run forever as stop iteration = stop time " * @@ -88,7 +90,8 @@ function Simulation(model; Δt, 0.0, false, false, - verbose) + verbose, + Float64(minimum_relative_step)) end function Base.show(io::IO, s::Simulation) @@ -100,6 +103,7 @@ function Base.show(io::IO, s::Simulation) "├── Stop time: $(prettytime(s.stop_time))", "\n", "├── Stop iteration: $(s.stop_iteration)", "\n", "├── Wall time limit: $(s.wall_time_limit)", "\n", + "├── Minimum relative step: $(s.minimum_relative_step)", "\n", "├── Callbacks: $(ordered_dict_show(s.callbacks, "│"))", "\n", "├── Output writers: $(ordered_dict_show(s.output_writers, "│"))", "\n", "└── Diagnostics: $(ordered_dict_show(s.diagnostics, "│"))") From e4aabf3760dd2974b887a52c356051e2d9af1667 Mon Sep 17 00:00:00 2001 From: tomaschor Date: Sat, 14 Dec 2024 14:37:12 -0500 Subject: [PATCH 05/14] apply logic --- src/Simulations/run.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Simulations/run.jl b/src/Simulations/run.jl index e902c8419c..55e774c68b 100644 --- a/src/Simulations/run.jl +++ b/src/Simulations/run.jl @@ -140,11 +140,12 @@ function time_step!(sim::Simulation) else # business as usual... Δt = aligned_time_step(sim, sim.Δt) - if Δt > (sim.Δt / 1e10) - time_step!(sim.model, Δt, callbacks=model_callbacks) + if Δt < sim.minimum_relative_step * sim.Δt + next_time = next_actuation_time(sim) + @warn "Reseting clock to $next_time and skipping aligned time step Δt = $Δt" + sim.model.clock.time = next_time else - println("Skipping aligned time step, which is of ", Δt) - sim.model.clock.time = next_actuation_time(sim) + time_step!(sim.model, Δt, callbacks=model_callbacks) end end From 8c8285f94aa0ed90f7a0a46811028fe3891d48e3 Mon Sep 17 00:00:00 2001 From: tomaschor Date: Sat, 14 Dec 2024 15:11:19 -0500 Subject: [PATCH 06/14] bump minor version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index b7de074bd9..c41dd2bfc0 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Oceananigans" uuid = "9e8cae18-63c1-5223-a75c-80ca9d6e9a09" authors = ["Climate Modeling Alliance and contributors"] -version = "0.95.2" +version = "0.95.3" [deps] Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" From 36a2197ec23bc7b4979fd713e6ed14a49262a3b4 Mon Sep 17 00:00:00 2001 From: Tomas Chor Date: Sat, 14 Dec 2024 12:13:25 -0800 Subject: [PATCH 07/14] Update src/Simulations/simulation.jl Co-authored-by: Gregory L. Wagner --- src/Simulations/simulation.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Simulations/simulation.jl b/src/Simulations/simulation.jl index c05364f633..d1981d9f1b 100644 --- a/src/Simulations/simulation.jl +++ b/src/Simulations/simulation.jl @@ -103,7 +103,7 @@ function Base.show(io::IO, s::Simulation) "├── Stop time: $(prettytime(s.stop_time))", "\n", "├── Stop iteration: $(s.stop_iteration)", "\n", "├── Wall time limit: $(s.wall_time_limit)", "\n", - "├── Minimum relative step: $(s.minimum_relative_step)", "\n", + "├── Minimum relative step: ", prettysummary(s.minimum_relative_step), "\n", "├── Callbacks: $(ordered_dict_show(s.callbacks, "│"))", "\n", "├── Output writers: $(ordered_dict_show(s.output_writers, "│"))", "\n", "└── Diagnostics: $(ordered_dict_show(s.diagnostics, "│"))") From 5b062d6a4782a4df0d203bb865816514cc4bdd10 Mon Sep 17 00:00:00 2001 From: tomaschor Date: Sat, 14 Dec 2024 15:19:49 -0500 Subject: [PATCH 08/14] added minimum_time_step to docstring --- src/Simulations/simulation.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Simulations/simulation.jl b/src/Simulations/simulation.jl index d1981d9f1b..4aa61b9e63 100644 --- a/src/Simulations/simulation.jl +++ b/src/Simulations/simulation.jl @@ -45,6 +45,9 @@ Keyword arguments - `wall_time_limit`: Stop the simulation if it's been running for longer than this many seconds of wall clock time. +- `minimum_relative_step`: time steps smaller than `Δt * minimum_relative_step` will be skipped. + This avoids extremely high values when writing the pressure to disk. + Default value is 0. See github.com/CliMA/Oceananigans.jl/issues/3593 for details. """ function Simulation(model; Δt, verbose = true, From 44efaab71cf63f0b975f4592097455cafcc2c78c Mon Sep 17 00:00:00 2001 From: tomaschor Date: Sat, 14 Dec 2024 22:14:42 -0500 Subject: [PATCH 09/14] better logic --- src/Simulations/run.jl | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/Simulations/run.jl b/src/Simulations/run.jl index 55e774c68b..229e22d7b7 100644 --- a/src/Simulations/run.jl +++ b/src/Simulations/run.jl @@ -27,6 +27,10 @@ function next_actuation_time(sim::Simulation) for activity in activities nearest_next_actuation_time = min(nearest_next_actuation_time, next_actuation_time(activity.schedule)) end + + # Align nearest_next_actuation_time with simulation stop_time + nearest_next_actuation_time = min(nearest_next_actuation_time, sim.stop_time) + return nearest_next_actuation_time end @@ -111,11 +115,24 @@ end const ModelCallsite = Union{TendencyCallsite, UpdateStateCallsite} + +function time_step_or_skip!(sim) + model_callbacks = Tuple(cb for cb in values(sim.callbacks) if cb.callsite isa ModelCallsite) + Δt = aligned_time_step(sim, sim.Δt) + if Δt < sim.minimum_relative_step * sim.Δt + next_time = next_actuation_time(sim) + @warn "Reseting clock to $next_time and skipping aligned time step Δt = $Δt" + sim.model.clock.time = next_time + else + time_step!(sim.model, Δt, callbacks=model_callbacks) + end +end + + """ Step `sim`ulation forward by one time step. """ function time_step!(sim::Simulation) start_time_step = time_ns() - model_callbacks = Tuple(cb for cb in values(sim.callbacks) if cb.callsite isa ModelCallsite) if !(sim.initialized) # execute initialization step initialize!(sim) @@ -127,8 +144,7 @@ function time_step!(sim::Simulation) start_time = time_ns() end - Δt = aligned_time_step(sim, sim.Δt) - time_step!(sim.model, Δt, callbacks=model_callbacks) + time_step_or_skip!(sim) if sim.verbose elapsed_initial_step_time = prettytime(1e-9 * (time_ns() - start_time)) @@ -139,14 +155,7 @@ function time_step!(sim::Simulation) end else # business as usual... - Δt = aligned_time_step(sim, sim.Δt) - if Δt < sim.minimum_relative_step * sim.Δt - next_time = next_actuation_time(sim) - @warn "Reseting clock to $next_time and skipping aligned time step Δt = $Δt" - sim.model.clock.time = next_time - else - time_step!(sim.model, Δt, callbacks=model_callbacks) - end + time_step_or_skip!(sim) end # Callbacks and callback-like things From 80870de12b39ec1a8cb48293081014912d16bb12 Mon Sep 17 00:00:00 2001 From: tomaschor Date: Sat, 14 Dec 2024 22:27:36 -0500 Subject: [PATCH 10/14] add test --- test/test_simulations.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/test_simulations.jl b/test/test_simulations.jl index ee384cf298..3e650b91bb 100644 --- a/test/test_simulations.jl +++ b/test/test_simulations.jl @@ -153,9 +153,17 @@ function run_basic_simulation_tests(arch) simulation.callbacks[:tester] = Callback(capture_call_time, schedule, parameters=called_at) run!(simulation) - @show called_at @test all(called_at .≈ 0.0:schedule.interval:simulation.stop_time) + + # Test that minimum_relative_step is running correctly + final_time = 1.0 + 1e-11 + simulation = Simulation(model, Δt=1, stop_time=final_time, minimum_relative_step=1e-10) + run!(simulation) + + @test simulation.model.clock.time == final_time + @test simulation.model.clock.iteration == 1 + return nothing end From 4fee2b6011def9cd331121eb931cb5415edfc571 Mon Sep 17 00:00:00 2001 From: tomaschor Date: Sun, 15 Dec 2024 10:09:04 -0500 Subject: [PATCH 11/14] fix test --- test/test_simulations.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_simulations.jl b/test/test_simulations.jl index 3e650b91bb..6164153475 100644 --- a/test/test_simulations.jl +++ b/test/test_simulations.jl @@ -153,11 +153,14 @@ function run_basic_simulation_tests(arch) simulation.callbacks[:tester] = Callback(capture_call_time, schedule, parameters=called_at) run!(simulation) + @show called_at @test all(called_at .≈ 0.0:schedule.interval:simulation.stop_time) # Test that minimum_relative_step is running correctly final_time = 1.0 + 1e-11 + model.clock.time = 0 + model.clock.iteration = 0 simulation = Simulation(model, Δt=1, stop_time=final_time, minimum_relative_step=1e-10) run!(simulation) From eaa200ace0d67797fa0ba0f028ab182d2739b9a5 Mon Sep 17 00:00:00 2001 From: tomaschor Date: Sun, 15 Dec 2024 16:17:30 -0500 Subject: [PATCH 12/14] update docstring call signature --- src/Simulations/simulation.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Simulations/simulation.jl b/src/Simulations/simulation.jl index 4aa61b9e63..f4446a172b 100644 --- a/src/Simulations/simulation.jl +++ b/src/Simulations/simulation.jl @@ -29,7 +29,8 @@ end verbose = true, stop_iteration = Inf, stop_time = Inf, - wall_time_limit = Inf) + wall_time_limit = Inf, + minimum_relative_step = 0) Construct a `Simulation` for a `model` with time step `Δt`. From ab1ebad9ed0f512386869645ef622562b2de2e72 Mon Sep 17 00:00:00 2001 From: tomaschor Date: Mon, 16 Dec 2024 07:57:54 -0500 Subject: [PATCH 13/14] don't use next_actuation_time for setting next_time --- src/Simulations/run.jl | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/Simulations/run.jl b/src/Simulations/run.jl index 229e22d7b7..2b8bde1c91 100644 --- a/src/Simulations/run.jl +++ b/src/Simulations/run.jl @@ -7,7 +7,7 @@ using Oceananigans: AbstractModel, run_diagnostic!, write_output! import Oceananigans: initialize! import Oceananigans.OutputWriters: checkpoint_path, set! import Oceananigans.TimeSteppers: time_step! -import Oceananigans.Utils: schedule_aligned_time_step, next_actuation_time +import Oceananigans.Utils: schedule_aligned_time_step # Simulations are for running @@ -21,19 +21,6 @@ function collect_scheduled_activities(sim) return tuple(writers..., callbacks...) end -function next_actuation_time(sim::Simulation) - activities = collect_scheduled_activities(sim) - nearest_next_actuation_time = Inf - for activity in activities - nearest_next_actuation_time = min(nearest_next_actuation_time, next_actuation_time(activity.schedule)) - end - - # Align nearest_next_actuation_time with simulation stop_time - nearest_next_actuation_time = min(nearest_next_actuation_time, sim.stop_time) - - return nearest_next_actuation_time -end - function schedule_aligned_time_step(sim, aligned_Δt) clock = sim.model.clock activities = collect_scheduled_activities(sim) @@ -120,7 +107,7 @@ function time_step_or_skip!(sim) model_callbacks = Tuple(cb for cb in values(sim.callbacks) if cb.callsite isa ModelCallsite) Δt = aligned_time_step(sim, sim.Δt) if Δt < sim.minimum_relative_step * sim.Δt - next_time = next_actuation_time(sim) + next_time = sim.model.clock.time + Δt @warn "Reseting clock to $next_time and skipping aligned time step Δt = $Δt" sim.model.clock.time = next_time else From c006efddbd8703481a643155d4382c4f3d13a8a3 Mon Sep 17 00:00:00 2001 From: Tomas Chor Date: Mon, 16 Dec 2024 06:30:51 -0800 Subject: [PATCH 14/14] Update src/Simulations/run.jl Co-authored-by: Simone Silvestri --- src/Simulations/run.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Simulations/run.jl b/src/Simulations/run.jl index 2b8bde1c91..5d5a63f2e6 100644 --- a/src/Simulations/run.jl +++ b/src/Simulations/run.jl @@ -108,7 +108,7 @@ function time_step_or_skip!(sim) Δt = aligned_time_step(sim, sim.Δt) if Δt < sim.minimum_relative_step * sim.Δt next_time = sim.model.clock.time + Δt - @warn "Reseting clock to $next_time and skipping aligned time step Δt = $Δt" + @warn "Resetting clock to $next_time and skipping time step of size Δt = $Δt" sim.model.clock.time = next_time else time_step!(sim.model, Δt, callbacks=model_callbacks)