Add support for Gaea C5 (includes PR #1977: new ccpp SDFs added to support RRFS multiphysics ensemble and add tob in ocean output)(Includes PR #1997)

cmake/configure_gaea-c5.intel.cmake

@@ -0,0 +1,4 @@
+set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
+set(MOM6_Extra_FORTRAN_FLAGS "-xsse2")
+set(HYCOM_Extra_FORTRAN_FLAGS "-xSSE4.2")
+set(HYCOM_Extra_C_FLAGS "-xSSE4.2")

cmake/configure_gaea_c5.intel.cmake

@@ -0,0 +1,4 @@
+set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
+set(MOM6_Extra_FORTRAN_FLAGS "-xsse2")
+set(HYCOM_Extra_FORTRAN_FLAGS "-xSSE4.2")
+set(HYCOM_Extra_C_FLAGS "-xSSE4.2")

modulefiles/ufs_gaea-c5.intel.lua

@@ -0,0 +1,32 @@
+help([[
+  This module loads libraries required for building and running UFS Weather Model 
+  on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0.
+]])
+
+whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C5 ]===])
+
+load("PrgEnv-intel/8.3.3")
+load("intel-classic/2023.1.0")
+load("cray-mpich/8.1.25")
+load("python/3.9.12")
+
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+stack_python_ver=os.getenv("stack_python_ver") or "3.10.8"
+load(pathJoin("stack-python", stack_python_ver))
+
+load("ufs_common")
+
+unload("darshan-runtime")
+
+setenv("CC","cc")
+setenv("CXX","CC")
+setenv("FC","ftn")
+setenv("CMAKE_Platform","gaea-c5.intel")

tests/compile.sh

@@ -57,9 +57,12 @@ set +x
 if [[ $MACHINE_ID == macosx ]] || [[ $MACHINE_ID == linux ]]; then
   source $PATHTR/modulefiles/ufs_${MACHINE_ID}.${RT_COMPILER}
 else
-  # Activate lua environment for gaea
+  # Activate lua environment for gaea c4
   if [[ $MACHINE_ID == gaea ]]; then
     source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh
+  # Activate lua environment for gaea c5
+  elif [[ $MACHINE_ID == gaea-c5 ]]; then
+    source /lustre/f2/dev/role.epic/contrib/Lmod_init_C5.sh
   fi
   # Load fv3 module
   module use $PATHTR/modulefiles

tests/default_vars.sh

@@ -229,6 +229,19 @@ elif [[ $MACHINE_ID = gaea ]]; then
   INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24
   WAV_tasks_atmw_gdas=264
 
+elif [[ $MACHINE_ID = gaea-c5 ]]; then
+
+  TPN=128
+
+  INPES_dflt=3 ; JNPES_dflt=8
+  INPES_thrd=3 ; JNPES_thrd=4
+  INPES_c384=6 ; JNPES_c384=8  ; THRD_c384=1
+  INPES_c768=8 ; JNPES_c768=16 ; THRD_c768=2
+
+  THRD_cpl_atmw_gdas=3
+  INPES_cpl_atmw_gdas=6; JNPES_cpl_atmw_gdas=8; WPG_cpl_atmw_gdas=24
+  WAV_tasks_atmw_gdas=264
+
 elif [[ $MACHINE_ID = cheyenne ]]; then
 
   TPN=36

tests/detect_machine.sh

@@ -43,6 +43,15 @@ case $(hostname -f) in
   gaea15.ncrc.gov)         MACHINE_ID=gaea ;; ### gaea15
   gaea16.ncrc.gov)         MACHINE_ID=gaea ;; ### gaea16
 
+  gaea51.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea51
+  gaea52.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea52
+  gaea53.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea53
+  gaea54.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea54
+  gaea55.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea55
+  gaea56.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea56
+  gaea57.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea57
+  gaea58.ncrc.gov)         MACHINE_ID=gaea-c5 ;; ### gaea58
+
   hfe01)                   MACHINE_ID=hera ;; ### hera01
   hfe02)                   MACHINE_ID=hera ;; ### hera02
   hfe03)                   MACHINE_ID=hera ;; ### hera03

tests/fv3_conf/compile_slurm.IN_gaea-c5

@@ -0,0 +1,21 @@
+#!/bin/bash -l
+#SBATCH -e err
+#SBATCH -o out
+#SBATCH --account=@[ACCNR]
+##SBATCH --qos=@[QUEUE]
+#SBATCH --clusters=es
+#SBATCH --partition=eslogin_c5
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=8
+#SBATCH --time=180
+#SBATCH --job-name="@[JBNME]"
+
+set -eux
+
+echo -n " $( date +%s )," >  job_timestamp.txt
+echo "Compile started:  " `date`
+
+@[PATHRT]/compile.sh @[MACHINE_ID] "@[MAKE_OPT]" @[COMPILE_NR] @[RT_COMPILER]
+
+echo "Compile ended:    " `date`
+echo -n " $( date +%s )," >> job_timestamp.txt

tests/fv3_conf/fv3_slurm.IN_gaea-c5

@@ -0,0 +1,38 @@
+#!/bin/bash -l
+#SBATCH -e err
+#SBATCH -o out
+#SBATCH --job-name="@[JBNME]"
+#SBATCH --account=@[ACCNR]
+#SBATCH --qos=@[QUEUE]
+#SBATCH --clusters=c5
+#SBATCH --partition=batch
+#SBATCH --nodes=@[NODES]
+#SBATCH --ntasks-per-node=@[TPN]
+#SBATCH --time=@[WLCLK]
+
+set -eux
+echo -n " $( date +%s )," >  job_timestamp.txt
+
+set +x
+MACHINE_ID=gaea-c5
+source ./module-setup.sh
+module use --prepend $PWD/modulefiles
+module load modules.fv3
+module list
+set -x
+
+echo "Model started:  " `date`
+
+export OMP_NUM_THREADS=@[THRD]
+export OMP_STACKSIZE=1024M
+export NC_BLKSZ=1M
+export ESMF_RUNTIME_PROFILE=ON
+export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
+
+# Avoid job errors because of filesystem synchronization delays
+sync && sleep 1
+
+srun --label -n @[TASKS] ./fv3.exe
+
+echo "Model ended:    " `date`
+echo -n " $( date +%s )," >> job_timestamp.txt