Skip to content

Commit

Permalink
Add support for JRA55do (#843)
Browse files Browse the repository at this point in the history
* updating paths for local nrlssc builds

* Add jra55do forcing option

* Updated env.nrlssc_gnu for new local directory structure

* Added JRA55do to file names. Added comments for each variable name at top of JRA55do_???_files subroutine

* Make JRA55 forcing to use common subroutines. Search atm_data_type for specific cases

* remove extraneous 'i' variable in JRA55_files

* Changed JRA55 filename JRA55_grid instead of grid at end of filename

* Add jra55do tests to base_suite and quick_suite. This is done via set_nml options.

* Update forcing implementation to provide a little more flexibility for
JRA55, JRA55do, and ncar bulk atm forcing files.

* Update documentation

* update Onyx port

* Update forcing documentation

Initial port to derecho_intel

* clean up blank spaces

---------

Co-authored-by: daveh150 <david.hebert@nrlssc.navy.mil>
  • Loading branch information
apcraig and daveh150 committed Jul 13, 2023
1 parent 766ff8d commit f9d3002
Show file tree
Hide file tree
Showing 22 changed files with 368 additions and 148 deletions.
245 changes: 144 additions & 101 deletions cicecore/cicedyn/general/ice_forcing.F90

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -753,7 +753,7 @@ subroutine ice_write_hist (ns)
!-----------------------------------------------------------------
! write time variable
!-----------------------------------------------------------------

ltime2 = timesecs/secday ! hist_time_axis = 'end' (default)

! Some coupled models require the time axis "stamp" to be in the middle
Expand All @@ -762,7 +762,7 @@ subroutine ice_write_hist (ns)
if (trim(hist_time_axis) == "begin" ) ltime2 = time_beg(ns)
if (trim(hist_time_axis) == "middle") ltime2 = p5*(time_beg(ns)+time_end(ns))
endif

status = nf90_inq_varid(ncid,'time',varid)
if (status /= nf90_noerr) call abort_ice(subname// &
'ERROR: getting time varid')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -715,7 +715,7 @@ subroutine ice_write_hist (ns)
if (trim(hist_time_axis) == "begin" ) ltime2 = time_beg(ns)
if (trim(hist_time_axis) == "middle") ltime2 = p5*(time_beg(ns)+time_end(ns))
endif

status = pio_inq_varid(File,'time',varid)
status = pio_put_var(File,varid,(/1/),ltime2)

Expand Down
15 changes: 15 additions & 0 deletions configuration/scripts/cice.batch.csh
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,21 @@ cat >> ${jobfile} << EOFB
#PBS -l walltime=${batchtime}
EOFB

else if (${ICE_MACHINE} =~ derecho*) then
cat >> ${jobfile} << EOFB
#PBS -q ${queue}
#PBS -l job_priority=regular
#PBS -N ${ICE_CASENAME}
#PBS -A ${acct}
#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}
#PBS -l walltime=${batchtime}
#PBS -j oe
#PBS -W umask=022
#PBS -o ${ICE_CASEDIR}
###PBS -M username@domain.com
###PBS -m be
EOFB

else if (${ICE_MACHINE} =~ gust*) then
cat >> ${jobfile} << EOFB
#PBS -q ${queue}
Expand Down
12 changes: 12 additions & 0 deletions configuration/scripts/cice.launch.csh
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,18 @@ mpiexec_mpt -np ${ntasks} omplace ./cice >&! \$ICE_RUNLOG_FILE
EOFR
endif

#=======
else if (${ICE_MACHCOMP} =~ derecho*) then
if (${ICE_COMMDIR} =~ serial*) then
cat >> ${jobfile} << EOFR
./cice >&! \$ICE_RUNLOG_FILE
EOFR
else
cat >> ${jobfile} << EOFR
mpiexec --cpu-bind depth -n ${ntasks} -ppn ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
EOFR
endif

#=======
else if (${ICE_MACHCOMP} =~ gust*) then
if (${ICE_COMMDIR} =~ serial*) then
Expand Down
69 changes: 69 additions & 0 deletions configuration/scripts/machines/Macros.derecho_intel
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#==============================================================================
# Makefile macros for NCAR cheyenne, intel compiler
#==============================================================================

CPP := fpp
CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
CFLAGS := -c -O2 -fp-model precise -march=core-avx2

FIXEDFLAGS := -fixed -132
FREEFLAGS := -free
FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -march=core-avx2
FFLAGS_NOOPT:= -O0

ifeq ($(ICE_BLDDEBUG), true)
FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
else
FFLAGS += -O2
endif

SCC := icx
SFC := ifort
MPICC := mpicc
MPIFC := mpif90

ifeq ($(ICE_COMMDIR), mpi)
FC := $(MPIFC)
CC := $(MPICC)
else
FC := $(SFC)
CC := $(SCC)
endif
LD:= $(FC)

NETCDF_PATH := $(NETCDF)

#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs

PNETCDF_PATH := $(PNETCDF)
#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib

INCLDIR := $(INCLDIR)

LIB_NETCDF := $(NETCDF)/lib
#LIB_PNETCDF := $(PNETCDF_PATH)/lib
#LIB_MPI := $(IMPILIBDIR)

#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl
SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff

ifeq ($(ICE_THREADED), true)
LDFLAGS += -qopenmp
CFLAGS += -qopenmp
FFLAGS += -qopenmp
endif

#ifeq ($(ICE_IOTYPE), pio1)
# LIB_PIO := $(PIO_LIBDIR)
# SLIBS := $(SLIBS) -L$(LIB_PIO) -lpio
#endif

ifeq ($(ICE_IOTYPE), pio2)
# CPPDEFS := $(CPPDEFS) -DGPTL
# LIB_PIO := $(PIO_LIBDIR)
# SLIBS := $(SLIBS) -L$(LIB_PIO) -lpiof -lpioc -lgptl
SLIBS := $(SLIBS) -lpiof -lpioc
endif

4 changes: 2 additions & 2 deletions configuration/scripts/machines/Macros.onyx_intel
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@

CPP := fpp
CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
CFLAGS := -c -O2 -fp-model precise -xHost
CFLAGS := -c -O2 -fp-model precise

FIXEDFLAGS := -132
FREEFLAGS := -FR
FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost
FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback
FFLAGS_NOOPT:= -O0

ifeq ($(ICE_BLDDEBUG), true)
Expand Down
70 changes: 70 additions & 0 deletions configuration/scripts/machines/env.derecho_intel
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/bin/csh -f

set inp = "undefined"
if ($#argv == 1) then
set inp = $1
endif

if ("$inp" != "-nomodules") then

source ${MODULESHOME}/init/csh

module --force purge
module load ncarenv/23.06
module load craype
module load intel/2023.0.0
module load ncarcompilers
module load cray-mpich/8.1.25
#module load hdf5/1.12.2
module load netcdf-mpi/4.9.2
module load cray-libsci/23.02.1.1

if ($?ICE_IOTYPE) then
if ($ICE_IOTYPE =~ pio*) then
module load parallel-netcdf/1.12.3
if ($ICE_IOTYPE == "pio1") then
module load parallelio/1.10.1
else
module load parallelio/2.6.0
endif
endif
endif

if ($?ICE_BFBTYPE) then
if ($ICE_BFBTYPE =~ qcchk*) then
module load conda
# conda env create -f ../../configuration/scripts/tests/qctest.yml
conda activate qctest
endif
endif

# For perftools with mpiexec
# module load perftools-base
# module load perftools
#setenv PALS_TRANSFER FALSE

endif

limit coredumpsize unlimited
limit stacksize unlimited
setenv PALS_QUIET TRUE

# May be needed for OpenMP memory
setenv OMP_STACKSIZE 64M
# OMP runtime diagnostics
#setenv OMP_DISPLAY_ENV TRUE

setenv ICE_MACHINE_MACHNAME derecho
setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11"
setenv ICE_MACHINE_ENVNAME intel
setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, cray-mpich 2.25, netcdf-mpi4.9.2, pnetcdf1.12.3, pio2.6.0"
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/CICE_RUNS
setenv ICE_MACHINE_INPUTDATA /glade/p/cesm/pcwg_dev
setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/CICE_BASELINE
setenv ICE_MACHINE_SUBMIT "qsub"
setenv ICE_MACHINE_ACCT P00000000
setenv ICE_MACHINE_QUEUE "main"
setenv ICE_MACHINE_TPNODE 128
setenv ICE_MACHINE_BLDTHRDS 8
setenv ICE_MACHINE_QSTAT "qstat "
10 changes: 5 additions & 5 deletions configuration/scripts/machines/env.nrlssc_gnu
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ setenv ICE_MACHINE_MACHINFO "nrlssc"
setenv ICE_MACHINE_ENVNAME gnu
setenv ICE_MACHINE_ENVINFO "gnu"
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR /u/data/hebert/CICE_RUNS
setenv ICE_MACHINE_INPUTDATA /u/data/hebert/CICE_RUNS
setenv ICE_MACHINE_BASELINE /u/data/hebert/CICE_BASELINE
setenv ICE_MACHINE_WKDIR /u/hebert/data/CICE_RUNS
setenv ICE_MACHINE_INPUTDATA /u/hebert/data/
setenv ICE_MACHINE_BASELINE /u/hebert/data/CICE_BASELINE
setenv ICE_MACHINE_SUBMIT "qsub "
setenv ICE_MACHINE_ACCT P00000000
setenv ICE_MACHINE_QUEUE "standard"
setenv ICE_MACHINE_TPNODE 20 # tasks per node
setenv ICE_MACHINE_BLDTHRDS 1
setenv ICE_MACHINE_TPNODE 28 # tasks per node
setenv ICE_MACHINE_BLDTHRDS 8
setenv ICE_MACHINE_QSTAT "qstat "
12 changes: 6 additions & 6 deletions configuration/scripts/machines/env.onyx_cray
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,23 @@ module unload PrgEnv-cray
module unload PrgEnv-gnu
module unload PrgEnv-intel
module unload PrgEnv-pgi
module load PrgEnv-cray/6.0.9
module load PrgEnv-cray/6.0.10

module unload cce
module load cce/11.0.2
module load cce/14.0.3

module unload cray-mpich
module unload cray-mpich2
module load cray-mpich/7.7.16
module load cray-mpich/7.7.20

module unload netcdf
module unload cray-netcdf
module unload cray-hdf5
module unload cray-hdf5-parallel
module unload cray-netcdf-hdf5parallel
module unload cray-parallel-netcdf
module load cray-netcdf/4.7.4.0
module load cray-hdf5/1.12.0.0
module load cray-netcdf/4.8.1.3
module load cray-hdf5/1.12.1.3

module unload cray-libsci
module unload craype-hugepages2M
Expand All @@ -46,7 +46,7 @@ endif
setenv ICE_MACHINE_MACHNAME onyx
setenv ICE_MACHINE_MACHINFO "Cray XC40/50 Xeon E5-2699v4 Broadwell"
setenv ICE_MACHINE_ENVNAME cray
setenv ICE_MACHINE_ENVINFO "Cray cce/11.0.2, cray-mpich/7.7.16, netcdf/4.7.4.0"
setenv ICE_MACHINE_ENVINFO "Cray cce/14.0.3, cray-mpich/7.7.20, netcdf/4.8.1.3"
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium
Expand Down
12 changes: 6 additions & 6 deletions configuration/scripts/machines/env.onyx_gnu
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,23 @@ module unload PrgEnv-cray
module unload PrgEnv-gnu
module unload PrgEnv-intel
module unload PrgEnv-pgi
module load PrgEnv-gnu/6.0.9
module load PrgEnv-gnu/6.0.10

module unload gcc
module load gcc/10.2.0
module load gcc/12.1.0

module unload cray-mpich
module unload cray-mpich2
module load cray-mpich/7.7.16
module load cray-mpich/7.7.20

module unload netcdf
module unload cray-netcdf
module unload cray-hdf5
module unload cray-hdf5-parallel
module unload cray-netcdf-hdf5parallel
module unload cray-parallel-netcdf
module load cray-netcdf/4.7.4.0
module load cray-hdf5/1.12.0.0
module load cray-netcdf/4.8.1.3
module load cray-hdf5/1.12.1.3

module unload cray-libsci
module unload craype-hugepages2M
Expand All @@ -46,7 +46,7 @@ endif
setenv ICE_MACHINE_MACHNAME onyx
setenv ICE_MACHINE_MACHINFO "Cray XC40/50 Xeon E5-2699v4 Broadwell"
setenv ICE_MACHINE_ENVNAME gnu
setenv ICE_MACHINE_ENVINFO "GNU Fortran (GCC) 10.2.0, cray-mpich/7.7.16, netcdf/4.7.4.0"
setenv ICE_MACHINE_ENVINFO "GNU Fortran (GCC) 12.1.0, cray-mpich/7.7.20, netcdf/4.8.1.3"
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium
Expand Down
12 changes: 6 additions & 6 deletions configuration/scripts/machines/env.onyx_intel
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,23 @@ module unload PrgEnv-cray
module unload PrgEnv-gnu
module unload PrgEnv-intel
module unload PrgEnv-pgi
module load PrgEnv-intel/6.0.9
module load PrgEnv-intel/6.0.10

module unload intel
module load intel/19.1.3.304
module load intel/2021.4.0

module unload cray-mpich
module unload cray-mpich2
module load cray-mpich/7.7.16
module load cray-mpich/7.7.20

module unload netcdf
module unload cray-netcdf
module unload cray-hdf5
module unload cray-hdf5-parallel
module unload cray-netcdf-hdf5parallel
module unload cray-parallel-netcdf
module load cray-netcdf/4.7.4.0
module load cray-hdf5/1.12.0.0
module load cray-netcdf/4.8.1.3
module load cray-hdf5/1.12.1.3

module unload cray-libsci
module unload craype-hugepages2M
Expand All @@ -46,7 +46,7 @@ endif
setenv ICE_MACHINE_MACHNAME onyx
setenv ICE_MACHINE_MACHINFO "Cray XC40/50 Xeon E5-2699v4 Broadwell"
setenv ICE_MACHINE_ENVNAME intel
setenv ICE_MACHINE_ENVINFO "ifort 19.1.3.304, cray-mpich/7.7.16, netcdf/4.7.4.0"
setenv ICE_MACHINE_ENVINFO "ifort 2021.4.0, cray-mpich/7.7.20, netcdf/4.8.1.3"
setenv ICE_MACHINE_MAKE gmake
setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium
Expand Down
4 changes: 2 additions & 2 deletions configuration/scripts/options/set_nml.gx1
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ maskhalo_remap = .true.
maskhalo_bound = .true.
fyear_init = 2005
atm_data_format = 'nc'
atm_data_type = 'JRA55_gx1'
atm_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx1/JRA55'
atm_data_type = 'JRA55'
atm_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx1'
precip_units = 'mks'
ocn_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx1/CESM/MONTHLY'
bgc_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx1/WOA/MONTHLY'
Expand Down
4 changes: 2 additions & 2 deletions configuration/scripts/options/set_nml.gx3
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ kmt_file = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/gx3/kmt_gx3.bin'
bathymetry_file = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/gx3/global_gx3.bathy.nc'
fyear_init = 2005
atm_data_format = 'nc'
atm_data_type = 'JRA55_gx3'
atm_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx3/JRA55'
atm_data_type = 'JRA55'
atm_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx3'
precip_units = 'mks'
ocn_data_format = 'bin'
ocn_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx3/'
2 changes: 1 addition & 1 deletion configuration/scripts/options/set_nml.gx3ncarbulk
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ use_restart_time = .true.
fyear_init = 1997
atm_data_format = 'bin'
atm_data_type = 'ncar'
atm_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx3/NCAR_bulk'
atm_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx3'
precip_units = 'mm_per_month'

2 changes: 2 additions & 0 deletions configuration/scripts/options/set_nml.jra55
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
atm_data_format = 'nc'
atm_data_type = 'JRA55'
2 changes: 2 additions & 0 deletions configuration/scripts/options/set_nml.jra55do
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
atm_data_format = 'nc'
atm_data_type = 'JRA55do'
Loading

0 comments on commit f9d3002

Please sign in to comment.