change mpiexec to srun (#117)

ufs-community · Jun 20, 2024 · 0bd5f01 · 0bd5f01
1 parent 3391683
commit 0bd5f01
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 13 deletions.
diff --git a/parm/land_analysis_hera.yaml b/parm/land_analysis_hera.yaml
@@ -16,7 +16,7 @@ workflow:
     EXP_NAME: "LETKF"
     EXP_BASEDIR: "/scratch2/NAGAPE/epic/{USER}/landda_test"
     JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/jedi_v7"
-    WARMSTART_DIR: "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/DATA_RESTART"
+    WARMSTART_DIR: "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/inputs/DATA_RESTART"
     FORCING: "gswp3"   # "gswp3" or "era5"
     RES: "96"
     FCSTHR: "24"

diff --git a/parm/land_analysis_orion.yaml b/parm/land_analysis_orion.yaml
@@ -16,7 +16,7 @@ workflow:
     EXP_NAME: "LETKF"
     EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test"
     JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_stack1.6"
-    WARMSTART_DIR: "/work/noaa/epic/UFS_Land-DA_Dev/DATA_RESTART"
+    WARMSTART_DIR: "/work/noaa/epic/UFS_Land-DA_Dev/inputs/DATA_RESTART"
     FORCING: "gswp3"   # "gswp3" or "era5"
     RES: "96"
     FCSTHR: "24"

diff --git a/scripts/exlandda_analysis.sh b/scripts/exlandda_analysis.sh
@@ -17,7 +17,20 @@ FILEDATE=${YYYY}${MM}${DD}.${HH}0000
 JEDI_STATICDIR=${JEDI_INSTALL}/jedi-bundle/fv3-jedi/test/Data
 JEDI_EXECDIR=${JEDI_INSTALL}/build/bin
 
-MPIEXEC=`which mpiexec`
+case $MACHINE in
+  "hera")
+    RUN_CMD="srun"
+    ;;
+  "orion")
+    RUN_CMD="srun"
+    ;;
+  "hercules")
+    RUN_CMD="srun"
+    ;;
+  *)
+    RUN_CMD=`which mpiexec`
+    ;;
+esac
 
 #SNOWDEPTHVAR=snwdph
 YAML_DA=construct
@@ -169,7 +182,7 @@ echo 'do_landDA: calling fv3-jedi'
 if [[ $do_DA == "YES" ]]; then
   export pgm="fv3jedi_letkf.x"
   . prep_step
-  ${MPIEXEC} -n $NPROC_JEDI ${JEDI_EXECDIR}/$pgm letkf_land.yaml >>$pgmout 2>errfile
+  ${RUN_CMD} -n $NPROC_JEDI ${JEDI_EXECDIR}/$pgm letkf_land.yaml >>$pgmout 2>errfile
   export err=$?; err_chk
   cp errfile errfile_jedi_letkf
   if [[ $err != 0 ]]; then
@@ -180,7 +193,7 @@ fi
 if [[ $do_HOFX == "YES" ]]; then
   export pgm="fv3jedi_letkf.x"
   . prep_step
-  ${MPIEXEC} -n $NPROC_JEDI ${JEDI_EXECDIR}/$pgm hofx_land.yaml >>$pgmout 2>errfile
+  ${RUN_CMD} -n $NPROC_JEDI ${JEDI_EXECDIR}/$pgm hofx_land.yaml >>$pgmout 2>errfile
   export err=$?; err_chk
   cp errfile errfile_jedi_hofx
   if [[ $err != 0 ]]; then
@@ -213,7 +226,7 @@ EOF
     export pgm="apply_incr.exe"
     . prep_step
     # (n=6) -> this is fixed, at one task per tile (with minor code change, could run on a single proc). 
-    ${MPIEXEC} -n 6 ${EXEClandda}/$pgm >>$pgmout 2>errfile
+    ${RUN_CMD} -n 6 ${EXEClandda}/$pgm >>$pgmout 2>errfile
     export err=$?; err_chk
     cp errfile errfile_apply_incr
     if [[ $err != 0 ]]; then

diff --git a/scripts/exlandda_forecast.sh b/scripts/exlandda_forecast.sh
@@ -14,7 +14,20 @@ nMM=${NTIME:4:2}
 nDD=${NTIME:6:2}
 nHH=${NTIME:8:2}
 
-MPIEXEC=`which mpiexec`
+case $MACHINE in
+  "hera")
+    RUN_CMD="srun"
+    ;;
+  "orion")
+    RUN_CMD="srun"
+    ;;
+  "hercules")
+    RUN_CMD="srun"
+    ;;
+  *)
+    RUN_CMD=`which mpiexec`
+    ;;
+esac
 
 #  convert back to UFS tile, run model (all members)
 if [[ ${ATMOS_FORC} == "gswp3" ]]; then  
@@ -34,9 +47,9 @@ if [[ ${ATMOS_FORC} == "gswp3" ]]; then
   source ${PATHRT}/tests/$TEST_NAME_RST
   source ${PATHRT}/atparse.bash
 
-  BL_DATE=20230816
-  RTPWD=${RTPWD:-${FIXlandda}/NEMSfv3gfs/develop-${BL_DATE}/INTEL/${TEST_NAME}}
-  INPUTDATA_ROOT=${INPUTDATA_ROOT:-${FIXlandda}/NEMSfv3gfs/input-data-20221101}
+#  BL_DATE=20230816
+  RTPWD=${RTPWD:-${FIXlandda}/test_base/${TEST_NAME}_intel}
+  INPUTDATA_ROOT=${INPUTDATA_ROOT:-${FIXlandda}/UFS_WM}
 
   echo "RTPWD= $RTPWD"
   echo "INPUTDATA_ROOT= $INPUTDATA_ROOT"
@@ -163,7 +176,7 @@ if [[ ${ATMOS_FORC} == "gswp3" ]]; then
   echo "Start ufs-cdeps-land model run with TASKS: ${TASKS}"
   export pgm="ufs_model"
   . prep_step
-  ${MPIEXEC} -n ${TASKS} ${EXEClandda}/$pgm >>$pgmout 2>errfile
+  ${RUN_CMD} -n ${TASKS} ${EXEClandda}/$pgm >>$pgmout 2>errfile
   export err=$?; err_chk
   cp errfile errfile_ufs_model
   if [[ $err != 0 ]]; then

diff --git a/scripts/exlandda_post_anal.sh b/scripts/exlandda_post_anal.sh
@@ -20,7 +20,20 @@ FREQ=$((${FCSTHR}*3600))
 RDD=$((${FCSTHR}/24))
 RHH=$((${FCSTHR}%24))
 
-MPIEXEC=`which mpiexec`
+case $MACHINE in
+  "hera")
+    RUN_CMD="srun"
+    ;;
+  "orion")
+    RUN_CMD="srun"
+    ;;
+  "hercules")
+    RUN_CMD="srun"
+    ;;
+  *)
+    RUN_CMD=`which mpiexec`
+    ;;
+esac
 
 FILEDATE=${YYYY}${MM}${DD}.${HH}0000
 for itile in {1..6}
@@ -77,7 +90,7 @@ if [[ ${ATMOS_FORC} == "era5" ]]; then
 
   export pgm="ufsLand.exe"
   . prep_step
-  ${MPIEXEC} -n 1 ${EXEClandda}/$pgm >>$pgmout 2>errfile
+  ${RUN_CMD} -n 1 ${EXEClandda}/$pgm >>$pgmout 2>errfile
   export err=$?; err_chk
   cp errfile errfile_ufsLand
   if [[ $err != 0 ]]; then