Merge pull request ufs-community#107 from climbfuji/gsl_develop_updat…

…e_from_develop_20211025 Update gsl/develop from develop 2021/10/25
climbfuji · Oct 27, 2021 · 80a374a · 80a374a
2 parents 1ea2a8e + 51b8ef8
commit 80a374a
Show file tree

Hide file tree

Showing 30 changed files with 2,653 additions and 2,925 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -9,7 +9,7 @@
 [submodule "WW3"]
   path = WW3
   url = https://github.com/NOAA-EMC/WW3
-  branch = develop
+  branch = dev/ufs-weather-model
 [submodule "stochastic_physics"]
   path = stochastic_physics
   url = https://github.com/noaa-psd/stochastic_physics

diff --git a/FV3 b/FV3
diff --git a/NEMS b/NEMS
diff --git a/WW3 b/WW3
diff --git a/cmake/configure_expanse.intel.cmake b/cmake/configure_expanse.intel.cmake
@@ -0,0 +1,2 @@
+set(INLINE_POST     ON  CACHE BOOL "Enable inline post" FORCE)
+set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
diff --git a/modulefiles/ufs_expanse.intel b/modulefiles/ufs_expanse.intel
@@ -0,0 +1,32 @@
+#%Module
+
+proc ModulesHelp {} {
+  puts stderr "\tcit - loads modules required for building and running UFS Model on Expanse/Intel"
+}
+
+module-whatis "loads UFS Model prerequisites for Expanse/Intel"
+
+module load slurm/expanse/20.02.3
+module load cpu/0.15.4
+module load intel/19.1.1.217
+module load intel-mpi/2019.8.254
+module load python/3.8.5
+module load cmake/3.18.2
+
+module use module use /expanse/lustre/scratch/domh/temp_project/hpc-stack-20210929/modulefiles/stack
+
+module load hpc/1.2.0
+
+module load hpc-intel/19.1.1.217
+module load hpc-intel-mpi/2019.8.254
+
+module load jasper/2.0.25
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load ufs_common
+
+setenv CC mpiicc
+setenv CXX mpiicpc
+setenv FC mpiifort
+setenv CMAKE_Platform expanse.intel
diff --git a/modulefiles/ufs_expanse.intel_debug b/modulefiles/ufs_expanse.intel_debug
@@ -0,0 +1,32 @@
+#%Module
+
+proc ModulesHelp {} {
+  puts stderr "\tcit - loads modules required for building and running UFS Model on Expanse/Intel"
+}
+
+module-whatis "loads UFS Model prerequisites for Expanse/Intel"
+
+module load slurm/expanse/20.02.3
+module load cpu/0.15.4
+module load intel/19.1.1.217
+module load intel-mpi/2019.8.254
+module load python/3.8.5
+module load cmake/3.18.2
+
+module use module use /expanse/lustre/scratch/domh/temp_project/hpc-stack-20210929/modulefiles/stack
+
+module load hpc/1.2.0
+
+module load hpc-intel/19.1.1.217
+module load hpc-intel-mpi/2019.8.254
+
+module load jasper/2.0.25
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load ufs_common_debug
+
+setenv CC mpiicc
+setenv CXX mpiicpc
+setenv FC mpiifort
+setenv CMAKE_Platform expanse.intel
diff --git a/stochastic_physics b/stochastic_physics
diff --git a/tests/RegressionTests_cheyenne.gnu.log b/tests/RegressionTests_cheyenne.gnu.log
diff --git a/tests/RegressionTests_cheyenne.intel.log b/tests/RegressionTests_cheyenne.intel.log
diff --git a/tests/RegressionTests_gaea.intel.log b/tests/RegressionTests_gaea.intel.log
diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log
diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log
diff --git a/tests/RegressionTests_jet.intel.log b/tests/RegressionTests_jet.intel.log
diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log
diff --git a/tests/RegressionTests_wcoss_cray.log b/tests/RegressionTests_wcoss_cray.log
diff --git a/tests/RegressionTests_wcoss_dell_p3.log b/tests/RegressionTests_wcoss_dell_p3.log
diff --git a/tests/ci/repo_check.sh b/tests/ci/repo_check.sh
@@ -36,7 +36,7 @@ cice[branch]='emc/develop'
 cice[dir]='CICE-interface/CICE'
 
 ww3[repo]='https://github.com/NOAA-EMC/WW3'
-ww3[branch]='develop'
+ww3[branch]='dev/ufs-weather-model'
 ww3[dir]='WW3'
 
 stoch[repo]='https://github.com/noaa-psd/stochastic_physics'

diff --git a/tests/default_vars.sh b/tests/default_vars.sh
@@ -389,6 +389,40 @@ elif [[ $MACHINE_ID = stampede.* ]]; then
   MPB_cdeps_025="0 39"; APB_cdeps_025="0 39"
   OPB_cdeps_025="40 159"; IPB_cdeps_025="160 207"
 
+elif [[ $MACHINE_ID = expanse.* ]]; then
+
+  TASKS_dflt=150 ; TPN_dflt=64 ; INPES_dflt=3 ; JNPES_dflt=8
+  TASKS_thrd=78  ; TPN_thrd=64 ; INPES_thrd=3 ; JNPES_thrd=4
+  TASKS_stretch=48 ; TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4
+
+  TASKS_cpl_dflt=192; TPN_cpl_dflt=64; INPES_cpl_dflt=3; JNPES_cpl_dflt=8
+  THRD_cpl_dflt=1; WPG_cpl_dflt=6;  MPB_cpl_dflt="0 143"; APB_cpl_dflt="0 149"
+  OPB_cpl_dflt="150 179"; IPB_cpl_dflt="180 191"
+
+  TASKS_cpl_dflt_wwav=384; TPN_cpl_dflt_wwav=64; INPES_cpl_dflt_wwav=3; JNPES_cpl_dflt_wwav=8
+  THRD_cpl_dflt_wwav=1; WPG_cpl_dflt_wwav=6;  MPB_cpl_dflt_wwav="0 143"; APB_cpl_dflt_wwav="0 149"
+  OPB_cpl_dflt_wwav="150 179"; IPB_cpl_dflt_wwav="180 191"; WPB_cpl_dflt_wwav="192 383"
+
+  TASKS_cpl_thrd=120; TPN_cpl_thrd=64; INPES_cpl_thrd=3; JNPES_cpl_thrd=4
+  THRD_cpl_thrd=2; WPG_cpl_thrd=6;  MPB_cpl_thrd="0 77";  APB_cpl_thrd="0 77"
+  OPB_cpl_thrd="78 107";  IPB_cpl_thrd="108 119"
+
+  TASKS_cpl_bmrk=480; TPN_cpl_bmrk=64; INPES_cpl_bmrk=6; JNPES_cpl_bmrk=8
+  THRD_cpl_bmrk=1; WPG_cpl_bmrk=24; MPB_cpl_bmrk="0 287"; APB_cpl_bmrk="0 311"
+  OPB_cpl_bmrk="312 431"; IPB_cpl_bmrk="432 479"
+
+  TASKS_cpl_wwav=640; TPN_cpl_wwav=64; INPES_cpl_wwav=6; JNPES_cpl_wwav=8
+  THRD_cpl_wwav=2; WPG_cpl_wwav=24; MPB_cpl_wwav="0 287"; APB_cpl_wwav="0 311"
+  OPB_cpl_wwav="312 431"; IPB_cpl_wwav="432 479"; WPB_cpl_wwav="480 639"
+
+  TASKS_cpl_c192=288; TPN_cpl_c192=64; INPES_cpl_c192=4; JNPES_cpl_c192=8
+  THRD_cpl_c192=1; WPG_cpl_c192=12;  MPB_cpl_c192="0 191"; APB_cpl_c192="0 203"
+  OPB_cpl_c192="204 263"; IPB_cpl_c192="264 287"
+
+  TASKS_cpl_c384=318; TPN_cpl_c384=64; INPES_cpl_c384=3; JNPES_cpl_c384=8
+  THRD_cpl_c384=1; WPG_cpl_c384=6;  MPB_cpl_c384="0 143"; APB_cpl_c384="0 149"
+  OPB_cpl_c384="150 269"; IPB_cpl_c384="270 317"
+
 else
 
   echo "Unknown MACHINE_ID ${MACHINE_ID}"
@@ -549,6 +583,7 @@ export H2O_PHYS=.false.
 export CPL=.false.
 export CPLCHM=.false.
 export CPLFLX=.false.
+export CPLICE=.false.
 export CPLWAV=.false.
 export CPLWAV2ATM=.false.
 export DAYS=1
@@ -792,6 +827,7 @@ export KTHERM=2
 export TFREEZE_OPTION=mushy
 
 export CPLFLX=.true.
+export CPLICE=.true.
 export CPL=.true.
 export MIN_SEAICE=1.0e-11
 

diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh
@@ -104,13 +104,17 @@ case $(hostname -f) in
   login2.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede2
   login3.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede3
   login4.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede4
+
+  login01.expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse1
+  login02.expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse2
 esac
 
 # Overwrite auto-detect with RT_MACHINE if set
 MACHINE_ID=${RT_MACHINE:-${MACHINE_ID}}
 
 # Append compiler
-if [ $MACHINE_ID = orion ] || [ $MACHINE_ID = hera ] || [ $MACHINE_ID = cheyenne ] || [ $MACHINE_ID = jet ] || [ $MACHINE_ID = gaea ] || [ $MACHINE_ID = stampede ] || [ $MACHINE_ID = s4 ] ; then
+if [ $MACHINE_ID = orion ] || [ $MACHINE_ID = hera ] || [ $MACHINE_ID = cheyenne ] || [ $MACHINE_ID = jet ] || \
+   [ $MACHINE_ID = gaea ] || [ $MACHINE_ID = stampede ] || [ $MACHINE_ID = s4] || [ $MACHINE_ID = expanse ] ; then
     MACHINE_ID=${MACHINE_ID}.${RT_COMPILER}
 fi
 

diff --git a/tests/fv3_conf/fv3_slurm.IN_expanse b/tests/fv3_conf/fv3_slurm.IN_expanse
@@ -0,0 +1,32 @@
+#!/bin/bash
+#SBATCH --job-name="@[JBNME]"
+#SBATCH -o out
+#SBATCH -e err
+#SBATCH -A @[ACCNR]
+#SBATCH -p @[QUEUE]
+#SBATCH --nodes=@[NODES]
+#SBATCH --ntasks-per-node=@[TPN]
+#SBATCH -t @[WLCLK]
+
+set -eux
+echo -n " $( date +%s )," >  job_timestamp.txt
+
+set +x
+source ./module-setup.sh
+module use $( pwd -P )
+module load modules.fv3
+module list
+
+set -x
+
+ulimit -s unlimited
+
+echo "Model started: "`date`
+
+export OMP_STACK_SIZE=512M
+export OMP_NUM_THREADS=@[THRD]
+export I_MPI_PMI_LIBRARY=/cm/shared/apps/slurm/current/lib64/libpmi.so
+srun -n @[TASKS] ./fv3.exe
+
+echo "Model ended:    " `date`
+echo -n " $( date +%s )," >> job_timestamp.txt
diff --git a/tests/fv3_conf/gsd_sar_run.IN b/tests/fv3_conf/gsd_sar_run.IN
diff --git a/tests/parm/cpld_control.nml.IN b/tests/parm/cpld_control.nml.IN
@@ -178,6 +178,7 @@ deflate_level=1
   cplwav       = @[CPLWAV]
   cplwav2atm   = @[CPLWAV2ATM]
   cplflx       = @[CPLFLX]
+  cplice       = @[CPLICE]
   min_seaice   = @[MIN_SEAICE]
   use_cice_alb = @[USE_CICE_ALB]
   frac_grid    = @[FRAC_GRID]

diff --git a/tests/parm/gsd_sar-model_configure.IN b/tests/parm/gsd_sar-model_configure.IN