Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

+Add a restart registry lock & fix OBC call order #1441

Merged
merged 2 commits into from
Jul 30, 2021
Merged

+Add a restart registry lock & fix OBC call order #1441

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
afe75e8
+Add a restart registry lock & fix OBC call order
Hallberg-NOAA Jul 8, 2021
321ced4
Merge branch 'dev/gfdl' into restart_lock
adcroft Jul 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 34 additions & 30 deletions src/core/MOM.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ module MOM
use MOM_io, only : slasher, file_exists, MOM_read_data
use MOM_obsolete_params, only : find_obsolete_params
use MOM_restart, only : register_restart_field, register_restart_pair
use MOM_restart, only : query_initialized, save_restart
use MOM_restart, only : query_initialized, save_restart, restart_registry_lock
use MOM_restart, only : restart_init, is_new_run, determine_is_new_run, MOM_restart_CS
use MOM_spatial_means, only : global_mass_integral
use MOM_time_manager, only : time_type, real_to_time, time_type_to_real, operator(+)
Expand Down Expand Up @@ -2145,8 +2145,7 @@ subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, &
if (associated(OBC_in)) then
! TODO: General OBC index rotations is not yet supported.
if (modulo(turns, 4) /= 1) &
call MOM_error(FATAL, "OBC index rotation of 180 and 270 degrees is " &
// "not yet unsupported.")
call MOM_error(FATAL, "OBC index rotation of 180 and 270 degrees is not yet supported.")
allocate(CS%OBC)
call rotate_OBC_config(OBC_in, dG_in, CS%OBC, dG, turns)
endif
Expand All @@ -2167,8 +2166,6 @@ subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, &

call MOM_timing_init(CS)

if (associated(CS%OBC)) call call_OBC_register(param_file, CS%update_OBC_CSp, US, CS%OBC)

call tracer_registry_init(param_file, CS%tracer_Reg)

! Allocate and initialize space for the primary time-varying MOM variables.
Expand Down Expand Up @@ -2222,21 +2219,8 @@ subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, &
flux_scale=conv2salt, convergence_units='kg m-2 s-1', &
convergence_scale=0.001*GV%H_to_kg_m2, CMOR_tendprefix="osalt", diag_form=2)
endif
! NOTE: register_temp_salt_segments includes allocation of tracer fields
! along segments. Bit reproducibility requires that MOM_initialize_state
! be called on the input index map, so we must setup both OBC and OBC_in.
!
! XXX: This call on OBC_in allocates the tracer fields on the unrotated
! grid, but also incorrectly stores a pointer to a tracer_type for the
! rotated registry (e.g. segment%tr_reg%Tr(n)%Tr) from CS%tracer_reg.
!
! While incorrect and potentially dangerous, it does not seem that this
! pointer is used during initialization, so we leave it for now.
if (CS%rotate_index .and. associated(OBC_in)) &
call register_temp_salt_segments(GV, OBC_in, CS%tracer_Reg, param_file)
if (associated(CS%OBC)) &
call register_temp_salt_segments(GV, CS%OBC, CS%tracer_Reg, param_file)
endif

if (use_frazil) then
allocate(CS%tv%frazil(isd:ied,jsd:jed)) ; CS%tv%frazil(:,:) = 0.0
endif
Expand Down Expand Up @@ -2329,11 +2313,38 @@ subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, &
call mixedlayer_restrat_register_restarts(dG%HI, param_file, &
CS%mixedlayer_restrat_CSp, restart_CSp)

if (associated(CS%OBC)) &
if (CS%rotate_index .and. associated(OBC_in) .and. use_temperature) then
! NOTE: register_temp_salt_segments includes allocation of tracer fields
! along segments. Bit reproducibility requires that MOM_initialize_state
! be called on the input index map, so we must setup both OBC and OBC_in.
!
! XXX: This call on OBC_in allocates the tracer fields on the unrotated
! grid, but also incorrectly stores a pointer to a tracer_type for the
! rotated registry (e.g. segment%tr_reg%Tr(n)%Tr) from CS%tracer_reg.
!
! While incorrect and potentially dangerous, it does not seem that this
! pointer is used during initialization, so we leave it for now.
call register_temp_salt_segments(GV, OBC_in, CS%tracer_Reg, param_file)
endif

if (associated(CS%OBC)) then
! Set up remaining information about open boundary conditions that is needed for OBCs.
call call_OBC_register(param_file, CS%update_OBC_CSp, US, CS%OBC, CS%tracer_Reg)
!### Package specific changes to OBCs need to go here?

! This is the equivalent to 2 calls to register_segment_tracer (per segment), which
! could occur with the call to update_OBC_data or after the main initialization.
if (use_temperature) &
call register_temp_salt_segments(GV, CS%OBC, CS%tracer_Reg, param_file)

! This needs the number of tracers and to have called any code that sets whether
! reservoirs are used.
call open_boundary_register_restarts(dg%HI, GV, CS%OBC, CS%tracer_Reg, &
param_file, restart_CSp, use_temperature)
endif

call callTree_waypoint("restart registration complete (initialize_MOM)")
call restart_registry_lock(restart_CSp)

! Shift from using the temporary dynamic grid type to using the final
! (potentially static) ocean-specific grid type.
Expand Down Expand Up @@ -2431,7 +2442,7 @@ subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, &
turns, CS%u, CS%v, CS%h, CS%T, CS%S)

if (associated(sponge_in_CSp)) then
! TODO: Implementation and testing of non-ALE spong rotation
! TODO: Implementation and testing of non-ALE sponge rotation
call MOM_error(FATAL, "Index rotation of non-ALE sponge is not yet implemented.")
endif

Expand Down Expand Up @@ -2471,19 +2482,11 @@ subroutine initialize_MOM(Time, Time_init, param_file, dirs, CS, restart_CSp, &
endif

if (use_ice_shelf .and. CS%debug) &
call hchksum(CS%frac_shelf_h, "MOM:frac_shelf_h", G%HI, &
haloshift=0)
call hchksum(CS%frac_shelf_h, "MOM:frac_shelf_h", G%HI, haloshift=0)

call cpu_clock_end(id_clock_MOM_init)
call callTree_waypoint("returned from MOM_initialize_state() (initialize_MOM)")

! ! Need this after MOM_initialize_state for DOME OBC stuff.
! if (associated(CS%OBC)) &
! call open_boundary_register_restarts(G%HI, GV, CS%OBC, CS%tracer_Reg, &
! param_file, restart_CSp, use_temperature)

! call callTree_waypoint("restart registration complete (initialize_MOM)")

! From this point, there may be pointers being set, so the final grid type
! that will persist throughout the run has to be used.

Expand Down Expand Up @@ -2838,6 +2841,7 @@ subroutine finish_MOM_initialization(Time, dirs, CS, restart_CSp)
if (CS%write_IC) then
allocate(restart_CSp_tmp)
restart_CSp_tmp = restart_CSp
call restart_registry_lock(restart_CSp_tmp, unlocked=.true.)
allocate(z_interface(SZI_(G),SZJ_(G),SZK_(GV)+1))
call find_eta(CS%h, CS%tv, G, GV, US, z_interface, eta_to_m=1.0)
call register_restart_field(z_interface, "eta", .true., restart_CSp_tmp, &
Expand Down
26 changes: 24 additions & 2 deletions src/core/MOM_boundary_update.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ module MOM_boundary_update
use MOM_tracer_registry, only : tracer_registry_type
use MOM_variables, only : thermo_var_ptrs
use MOM_verticalGrid, only : verticalGrid_type
use DOME_initialization, only : register_DOME_OBC
use tidal_bay_initialization, only : tidal_bay_set_OBC_data, register_tidal_bay_OBC
use tidal_bay_initialization, only : tidal_bay_OBC_end, tidal_bay_OBC_CS
use Kelvin_initialization, only : Kelvin_set_OBC_data, register_Kelvin_OBC
Expand Down Expand Up @@ -58,13 +59,15 @@ module MOM_boundary_update
!> The following subroutines and associated definitions provide the
!! machinery to register and call the subroutines that initialize
!! open boundary conditions.
subroutine call_OBC_register(param_file, CS, US, OBC)
subroutine call_OBC_register(param_file, CS, US, OBC, tr_Reg)
type(param_file_type), intent(in) :: param_file !< Parameter file to parse
type(update_OBC_CS), pointer :: CS !< Control structure for OBCs
type(unit_scale_type), intent(in) :: US !< A dimensional unit scaling type
type(ocean_OBC_type), pointer :: OBC !< Open boundary structure
type(tracer_registry_type), pointer :: tr_Reg !< Tracer registry.

! Local variables
character(len=200) :: config
character(len=40) :: mdl = "MOM_boundary_update" ! This module's name.
! This include declares and sets the variable "version".
# include "version_variable.h"
Expand All @@ -91,10 +94,29 @@ subroutine call_OBC_register(param_file, CS, US, OBC)
call get_param(param_file, mdl, "USE_DYED_CHANNEL_OBC", CS%use_dyed_channel, &
"If true, use the dyed channel open boundary.", &
default=.false.)
call get_param(param_file, mdl, "OBC_USER_CONFIG", config, &
"A string that sets how the user code is invoked to set open boundary data: \n"//&
" DOME - specified inflow on northern boundary\n"//&
" dyed_channel - supercritical with dye on the inflow boundary\n"//&
" dyed_obcs - circle_obcs with dyes on the open boundaries\n"//&
" Kelvin - barotropic Kelvin wave forcing on the western boundary\n"//&
" shelfwave - Flather with shelf wave forcing on western boundary\n"//&
" supercritical - now only needed here for the allocations\n"//&
" tidal_bay - Flather with tidal forcing on eastern boundary\n"//&
" USER - user specified", default="none", do_not_log=.true.)

if (CS%use_files) CS%use_files = &
register_file_OBC(param_file, CS%file_OBC_CSp, US, &
OBC%OBC_Reg)

if (trim(config) == "DOME") then
call register_DOME_OBC(param_file, US, OBC, tr_Reg)
! elseif (trim(config) == "tidal_bay") then
! elseif (trim(config) == "Kelvin") then
! elseif (trim(config) == "shelfwave") then
! elseif (trim(config) == "dyed_channel") then
endif

if (CS%use_tidal_bay) CS%use_tidal_bay = &
register_tidal_bay_OBC(param_file, CS%tidal_bay_OBC_CSp, US, &
OBC%OBC_Reg)
Expand Down Expand Up @@ -150,7 +172,7 @@ end subroutine OBC_register_end

!> \namespace mom_boundary_update
!! This module updates the open boundary arrays when time-varying.
!! It caused a circular dependency with the tidal_bay setup when
!! It caused a circular dependency with the tidal_bay and other setups when in
!! MOM_open_boundary.
!!
!! A small fragment of the grid is shown below:
Expand Down
26 changes: 15 additions & 11 deletions src/core/MOM_open_boundary.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,8 +342,6 @@ module MOM_open_boundary
integer :: id_clock_pass !< A CPU time clock

character(len=40) :: mdl = "MOM_open_boundary" !< This module's name.
! This include declares and sets the variable "version".
#include "version_variable.h"

contains

Expand All @@ -359,6 +357,7 @@ subroutine open_boundary_config(G, US, param_file, OBC)
type(unit_scale_type), intent(in) :: US !< A dimensional unit scaling type
type(param_file_type), intent(in) :: param_file !< Parameter file handle
type(ocean_OBC_type), pointer :: OBC !< Open boundary control structure

! Local variables
integer :: l ! For looping over segments
logical :: debug_OBC, debug, mask_outside, reentrant_x, reentrant_y
Expand All @@ -370,6 +369,9 @@ subroutine open_boundary_config(G, US, param_file, OBC)
logical :: answers_2018, default_2018_answers
logical :: check_reconstruction, check_remapping, force_bounds_in_subcell
character(len=32) :: remappingScheme
! This include declares and sets the variable "version".
# include "version_variable.h"

allocate(OBC)

call get_param(param_file, mdl, "OBC_NUMBER_OF_SEGMENTS", OBC%number_of_segments, &
Expand Down Expand Up @@ -4433,8 +4435,8 @@ subroutine register_OBC(name, param_file, Reg)
Reg%OB(nobc)%name = name

if (Reg%locked) call MOM_error(FATAL, &
"MOM register_tracer was called for variable "//trim(Reg%OB(nobc)%name)//&
" with a locked tracer registry.")
"MOM register_OBC was called for OBC "//trim(Reg%OB(nobc)%name)//&
" with a locked OBC registry.")

end subroutine register_OBC

Expand All @@ -4445,15 +4447,15 @@ subroutine OBC_registry_init(param_file, Reg)

integer, save :: init_calls = 0

#include "version_variable.h"
# include "version_variable.h"
character(len=40) :: mdl = "MOM_open_boundary" ! This module's name.
character(len=256) :: mesg ! Message for error messages.

if (.not.associated(Reg)) then ; allocate(Reg)
else ; return ; endif

! Read all relevant parameters and write them to the model log.
! call log_version(param_file, mdl,s version, "")
! call log_version(param_file, mdl, version, "")

init_calls = init_calls + 1
if (init_calls > 1) then
Expand Down Expand Up @@ -4503,7 +4505,7 @@ subroutine segment_tracer_registry_init(param_file, segment)
integer, save :: init_calls = 0

! This include declares and sets the variable "version".
#include "version_variable.h"
# include "version_variable.h"
character(len=40) :: mdl = "segment_tracer_registry_init" ! This routine's name.
character(len=256) :: mesg ! Message for error messages.

Expand All @@ -4527,6 +4529,8 @@ subroutine segment_tracer_registry_init(param_file, segment)

end subroutine segment_tracer_registry_init

!> Register a tracer array that is active on an OBC segment, potentially also specifing how the
!! tracer inflow values are specified.
subroutine register_segment_tracer(tr_ptr, param_file, GV, segment, &
OBC_scalar, OBC_array)
type(verticalGrid_type), intent(in) :: GV !< ocean vertical grid structure
Expand All @@ -4537,7 +4541,7 @@ subroutine register_segment_tracer(tr_ptr, param_file, GV, segment, &
!! but it also means that any updates to this
!! structure in the calling module will be
!! available subsequently to the tracer registry.
type(param_file_type), intent(in) :: param_file !< file to parse for model parameter values
type(param_file_type), intent(in) :: param_file !< file to parse for model parameter values
type(OBC_segment_type), intent(inout) :: segment !< current segment data structure
real, optional, intent(in) :: OBC_scalar !< If present, use scalar value for segment tracer
!! inflow concentration.
Expand All @@ -4555,8 +4559,8 @@ subroutine register_segment_tracer(tr_ptr, param_file, GV, segment, &

if (segment%tr_Reg%ntseg>=MAX_FIELDS_) then
write(mesg,'("Increase MAX_FIELDS_ in MOM_memory.h to at least ",I3," to allow for &
&all the tracers being registered via register_tracer.")') segment%tr_Reg%ntseg+1
call MOM_error(FATAL,"MOM register_tracer: "//mesg)
&all the tracers being registered via register_segment_tracer.")') segment%tr_Reg%ntseg+1
call MOM_error(FATAL,"MOM register_segment_tracer: "//mesg)
endif
segment%tr_Reg%ntseg = segment%tr_Reg%ntseg + 1
ntseg = segment%tr_Reg%ntseg
Expand All @@ -4570,7 +4574,7 @@ subroutine register_segment_tracer(tr_ptr, param_file, GV, segment, &
segment%tr_Reg%Tr(ntseg)%name = tr_ptr%name

if (segment%tr_Reg%locked) call MOM_error(FATAL, &
"MOM register_tracer was called for variable "//trim(segment%tr_Reg%Tr(ntseg)%name)//&
"MOM register_segment_tracer was called for variable "//trim(segment%tr_Reg%Tr(ntseg)%name)//&
" with a locked tracer registry.")

if (present(OBC_scalar)) segment%tr_Reg%Tr(ntseg)%OBC_inflow_conc = OBC_scalar ! initialize tracer value later
Expand Down
Loading