CICE-Consortium · eclare108213 · Jul 13, 2023 · May 23, 2023 · May 23, 2023 · May 25, 2023
diff --git a/cicecore/cicedyn/general/ice_forcing.F90 b/cicecore/cicedyn/general/ice_forcing.F90
diff --git a/cicecore/cicedyn/infrastructure/io/io_netcdf/ice_history_write.F90 b/cicecore/cicedyn/infrastructure/io/io_netcdf/ice_history_write.F90
@@ -753,7 +753,7 @@ subroutine ice_write_hist (ns)
       !-----------------------------------------------------------------
       ! write time variable
       !-----------------------------------------------------------------
-       
+
         ltime2 = timesecs/secday ! hist_time_axis = 'end' (default)
 
         ! Some coupled models require the time axis "stamp" to be in the middle
@@ -762,7 +762,7 @@ subroutine ice_write_hist (ns)
            if (trim(hist_time_axis) == "begin" ) ltime2 = time_beg(ns)
            if (trim(hist_time_axis) == "middle") ltime2 = p5*(time_beg(ns)+time_end(ns))
         endif
-       
+
         status = nf90_inq_varid(ncid,'time',varid)
         if (status /= nf90_noerr) call abort_ice(subname// &
                       'ERROR: getting time varid')

diff --git a/cicecore/cicedyn/infrastructure/io/io_pio2/ice_history_write.F90 b/cicecore/cicedyn/infrastructure/io/io_pio2/ice_history_write.F90
@@ -715,7 +715,7 @@ subroutine ice_write_hist (ns)
            if (trim(hist_time_axis) == "begin" ) ltime2 = time_beg(ns)
            if (trim(hist_time_axis) == "middle") ltime2 = p5*(time_beg(ns)+time_end(ns))
         endif
-       
+
         status = pio_inq_varid(File,'time',varid)
         status = pio_put_var(File,varid,(/1/),ltime2)
 

diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh
@@ -33,6 +33,21 @@ cat >> ${jobfile} << EOFB
 #PBS -l walltime=${batchtime}
 EOFB
 
+else if (${ICE_MACHINE} =~ derecho*) then
+cat >> ${jobfile} << EOFB
+#PBS -q ${queue}
+#PBS -l job_priority=regular
+#PBS -N ${ICE_CASENAME}
+#PBS -A ${acct}
+#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}
+#PBS -l walltime=${batchtime}
+#PBS -j oe
+#PBS -W umask=022
+#PBS -o ${ICE_CASEDIR}
+###PBS -M username@domain.com
+###PBS -m be
+EOFB
+
 else if (${ICE_MACHINE} =~ gust*) then
 cat >> ${jobfile} << EOFB
 #PBS -q ${queue}

diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh
@@ -22,6 +22,18 @@ mpiexec_mpt -np ${ntasks} omplace ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 endif
 
+#=======
+else if (${ICE_MACHCOMP} =~ derecho*) then
+if (${ICE_COMMDIR} =~ serial*) then
+cat >> ${jobfile} << EOFR
+./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+else
+cat >> ${jobfile} << EOFR
+mpiexec --cpu-bind depth -n ${ntasks} -ppn ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+endif
+
 #=======
 else if (${ICE_MACHCOMP} =~ gust*) then
 if (${ICE_COMMDIR} =~ serial*) then

diff --git a/configuration/scripts/machines/Macros.derecho_intel b/configuration/scripts/machines/Macros.derecho_intel
@@ -0,0 +1,69 @@
+#==============================================================================
+# Makefile macros for NCAR cheyenne, intel compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise -march=core-avx2
+
+FIXEDFLAGS := -fixed -132
+FREEFLAGS  := -free
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -march=core-avx2
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
+else
+  FFLAGS     += -O2
+endif
+
+SCC   := icx
+SFC   := ifort
+MPICC := mpicc
+MPIFC := mpif90
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+NETCDF_PATH := $(NETCDF)
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs
+
+PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib
+
+INCLDIR := $(INCLDIR)
+
+LIB_NETCDF := $(NETCDF)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true)
+   LDFLAGS += -qopenmp
+   CFLAGS += -qopenmp
+   FFLAGS += -qopenmp
+endif
+
+#ifeq ($(ICE_IOTYPE), pio1)
+#   LIB_PIO := $(PIO_LIBDIR)
+#   SLIBS   := $(SLIBS) -L$(LIB_PIO) -lpio
+#endif
+
+ifeq ($(ICE_IOTYPE), pio2)
+#   CPPDEFS := $(CPPDEFS) -DGPTL
+#   LIB_PIO := $(PIO_LIBDIR)
+#   SLIBS   := $(SLIBS) -L$(LIB_PIO) -lpiof -lpioc -lgptl
+   SLIBS   := $(SLIBS) -lpiof -lpioc
+endif
+
diff --git a/configuration/scripts/machines/Macros.onyx_intel b/configuration/scripts/machines/Macros.onyx_intel
@@ -4,11 +4,11 @@
 
 CPP        := fpp
 CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
-CFLAGS     := -c -O2 -fp-model precise   -xHost
+CFLAGS     := -c -O2 -fp-model precise
 
 FIXEDFLAGS := -132
 FREEFLAGS  := -FR
-FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback   -xHost
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback
 FFLAGS_NOOPT:= -O0
 
 ifeq ($(ICE_BLDDEBUG), true)

diff --git a/configuration/scripts/machines/env.derecho_intel b/configuration/scripts/machines/env.derecho_intel
@@ -0,0 +1,70 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module --force purge
+module load ncarenv/23.06
+module load craype
+module load intel/2023.0.0
+module load ncarcompilers
+module load cray-mpich/8.1.25
+#module load hdf5/1.12.2
+module load netcdf-mpi/4.9.2
+module load cray-libsci/23.02.1.1
+
+if ($?ICE_IOTYPE) then
+if ($ICE_IOTYPE =~ pio*) then
+  module load parallel-netcdf/1.12.3
+  if ($ICE_IOTYPE == "pio1") then
+    module load parallelio/1.10.1
+  else
+    module load parallelio/2.6.0
+  endif
+endif
+endif
+
+if ($?ICE_BFBTYPE) then
+if ($ICE_BFBTYPE =~ qcchk*) then
+  module load conda
+# conda env create -f ../../configuration/scripts/tests/qctest.yml
+  conda activate qctest
+endif
+endif
+
+# For perftools with mpiexec
+# module load perftools-base
+# module load perftools
+#setenv PALS_TRANSFER FALSE
+
+endif
+
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv PALS_QUIET TRUE
+
+# May be needed for OpenMP memory
+setenv OMP_STACKSIZE 64M
+# OMP runtime diagnostics
+#setenv OMP_DISPLAY_ENV TRUE
+
+setenv ICE_MACHINE_MACHNAME derecho
+setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11"
+setenv ICE_MACHINE_ENVNAME intel
+setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, cray-mpich 2.25, netcdf-mpi4.9.2, pnetcdf1.12.3, pio2.6.0"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /glade/p/cesm/pcwg_dev
+setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_TPNODE 128
+setenv ICE_MACHINE_BLDTHRDS 8
+setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.nrlssc_gnu b/configuration/scripts/machines/env.nrlssc_gnu
@@ -5,12 +5,12 @@ setenv ICE_MACHINE_MACHINFO "nrlssc"
 setenv ICE_MACHINE_ENVNAME gnu
 setenv ICE_MACHINE_ENVINFO "gnu"
 setenv ICE_MACHINE_MAKE gmake
-setenv ICE_MACHINE_WKDIR /u/data/hebert/CICE_RUNS
-setenv ICE_MACHINE_INPUTDATA /u/data/hebert/CICE_RUNS
-setenv ICE_MACHINE_BASELINE /u/data/hebert/CICE_BASELINE
+setenv ICE_MACHINE_WKDIR /u/hebert/data/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /u/hebert/data/
+setenv ICE_MACHINE_BASELINE /u/hebert/data/CICE_BASELINE
 setenv ICE_MACHINE_SUBMIT "qsub "
 setenv ICE_MACHINE_ACCT P00000000
 setenv ICE_MACHINE_QUEUE "standard"
-setenv ICE_MACHINE_TPNODE 20    # tasks per node
-setenv ICE_MACHINE_BLDTHRDS 1
+setenv ICE_MACHINE_TPNODE 28    # tasks per node
+setenv ICE_MACHINE_BLDTHRDS 8
 setenv ICE_MACHINE_QSTAT "qstat "
diff --git a/configuration/scripts/machines/env.onyx_cray b/configuration/scripts/machines/env.onyx_cray
@@ -13,23 +13,23 @@ module unload PrgEnv-cray
 module unload PrgEnv-gnu
 module unload PrgEnv-intel
 module unload PrgEnv-pgi
-module load PrgEnv-cray/6.0.9
+module load PrgEnv-cray/6.0.10
 
 module unload cce
-module load cce/11.0.2
+module load cce/14.0.3
 
 module unload cray-mpich
 module unload cray-mpich2
-module load cray-mpich/7.7.16
+module load cray-mpich/7.7.20
 
 module unload netcdf
 module unload cray-netcdf
 module unload cray-hdf5
 module unload cray-hdf5-parallel
 module unload cray-netcdf-hdf5parallel
 module unload cray-parallel-netcdf
-module load cray-netcdf/4.7.4.0
-module load cray-hdf5/1.12.0.0
+module load cray-netcdf/4.8.1.3
+module load cray-hdf5/1.12.1.3
 
 module unload cray-libsci
 module unload craype-hugepages2M
@@ -46,7 +46,7 @@ endif
 setenv ICE_MACHINE_MACHNAME onyx
 setenv ICE_MACHINE_MACHINFO "Cray XC40/50 Xeon E5-2699v4 Broadwell"
 setenv ICE_MACHINE_ENVNAME cray
-setenv ICE_MACHINE_ENVINFO "Cray cce/11.0.2, cray-mpich/7.7.16, netcdf/4.7.4.0"
+setenv ICE_MACHINE_ENVINFO "Cray cce/14.0.3, cray-mpich/7.7.20, netcdf/4.8.1.3"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
 setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium

diff --git a/configuration/scripts/machines/env.onyx_gnu b/configuration/scripts/machines/env.onyx_gnu
@@ -13,23 +13,23 @@ module unload PrgEnv-cray
 module unload PrgEnv-gnu
 module unload PrgEnv-intel
 module unload PrgEnv-pgi
-module load PrgEnv-gnu/6.0.9
+module load PrgEnv-gnu/6.0.10
 
 module unload gcc
-module load gcc/10.2.0
+module load gcc/12.1.0
 
 module unload cray-mpich
 module unload cray-mpich2
-module load cray-mpich/7.7.16
+module load cray-mpich/7.7.20
 
 module unload netcdf
 module unload cray-netcdf
 module unload cray-hdf5
 module unload cray-hdf5-parallel
 module unload cray-netcdf-hdf5parallel
 module unload cray-parallel-netcdf
-module load cray-netcdf/4.7.4.0
-module load cray-hdf5/1.12.0.0
+module load cray-netcdf/4.8.1.3
+module load cray-hdf5/1.12.1.3
 
 module unload cray-libsci
 module unload craype-hugepages2M
@@ -46,7 +46,7 @@ endif
 setenv ICE_MACHINE_MACHNAME onyx
 setenv ICE_MACHINE_MACHINFO "Cray XC40/50 Xeon E5-2699v4 Broadwell"
 setenv ICE_MACHINE_ENVNAME gnu
-setenv ICE_MACHINE_ENVINFO "GNU Fortran (GCC) 10.2.0, cray-mpich/7.7.16, netcdf/4.7.4.0"
+setenv ICE_MACHINE_ENVINFO "GNU Fortran (GCC) 12.1.0, cray-mpich/7.7.20, netcdf/4.8.1.3"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
 setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium

diff --git a/configuration/scripts/machines/env.onyx_intel b/configuration/scripts/machines/env.onyx_intel
@@ -13,23 +13,23 @@ module unload PrgEnv-cray
 module unload PrgEnv-gnu
 module unload PrgEnv-intel
 module unload PrgEnv-pgi
-module load PrgEnv-intel/6.0.9
+module load PrgEnv-intel/6.0.10
 
 module unload intel
-module load intel/19.1.3.304
+module load intel/2021.4.0
 
 module unload cray-mpich
 module unload cray-mpich2
-module load cray-mpich/7.7.16
+module load cray-mpich/7.7.20
 
 module unload netcdf
 module unload cray-netcdf
 module unload cray-hdf5
 module unload cray-hdf5-parallel
 module unload cray-netcdf-hdf5parallel
 module unload cray-parallel-netcdf
-module load cray-netcdf/4.7.4.0
-module load cray-hdf5/1.12.0.0
+module load cray-netcdf/4.8.1.3
+module load cray-hdf5/1.12.1.3
 
 module unload cray-libsci
 module unload craype-hugepages2M
@@ -46,7 +46,7 @@ endif
 setenv ICE_MACHINE_MACHNAME onyx
 setenv ICE_MACHINE_MACHINFO "Cray XC40/50 Xeon E5-2699v4 Broadwell"
 setenv ICE_MACHINE_ENVNAME intel
-setenv ICE_MACHINE_ENVINFO "ifort 19.1.3.304, cray-mpich/7.7.16, netcdf/4.7.4.0"
+setenv ICE_MACHINE_ENVINFO "ifort 2021.4.0, cray-mpich/7.7.20, netcdf/4.8.1.3"
 setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS
 setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium

diff --git a/configuration/scripts/options/set_nml.gx1 b/configuration/scripts/options/set_nml.gx1
@@ -14,8 +14,8 @@ maskhalo_remap = .true.
 maskhalo_bound = .true.
 fyear_init      = 2005
 atm_data_format = 'nc'
-atm_data_type   = 'JRA55_gx1'
-atm_data_dir    = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx1/JRA55'
+atm_data_type   = 'JRA55'
+atm_data_dir    = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx1'
 precip_units    = 'mks'
 ocn_data_dir    = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx1/CESM/MONTHLY'
 bgc_data_dir    = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx1/WOA/MONTHLY'

diff --git a/configuration/scripts/options/set_nml.gx3 b/configuration/scripts/options/set_nml.gx3
@@ -11,8 +11,8 @@ kmt_file    = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/gx3/kmt_gx3.bin'
 bathymetry_file = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/gx3/global_gx3.bathy.nc'
 fyear_init      = 2005
 atm_data_format = 'nc'
-atm_data_type   = 'JRA55_gx3'
-atm_data_dir    = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx3/JRA55'
+atm_data_type   = 'JRA55'
+atm_data_dir    = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx3'
 precip_units    = 'mks'
 ocn_data_format = 'bin'
 ocn_data_dir    = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx3/'
diff --git a/configuration/scripts/options/set_nml.gx3ncarbulk b/configuration/scripts/options/set_nml.gx3ncarbulk
@@ -4,6 +4,6 @@ use_restart_time = .true.
 fyear_init      = 1997
 atm_data_format = 'bin'
 atm_data_type   = 'ncar'
-atm_data_dir    = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx3/NCAR_bulk'
+atm_data_dir    = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx3'
 precip_units    = 'mm_per_month'
 
diff --git a/configuration/scripts/options/set_nml.jra55 b/configuration/scripts/options/set_nml.jra55
@@ -0,0 +1,2 @@
+atm_data_format = 'nc'
+atm_data_type   = 'JRA55'
diff --git a/configuration/scripts/options/set_nml.jra55do b/configuration/scripts/options/set_nml.jra55do
@@ -0,0 +1,2 @@
+atm_data_format = 'nc'
+atm_data_type   = 'JRA55do'