Compute petlist bounds for each subcomponent from number of tasks. Up…

…date CICE (ufs-community#1200) * Instead of specifying petlist bounds (lower and upper bound), compute them using the number of tasks for each component. The function that computes petlist bounds assumes that the mediator always runs on the same tasks as ATM component, and that the Chemistry component also always runs on the same tasks as ATM. * Update CICE submodule (@DeniseWorthen) Co-authored-by: Minsuk Ji <57227195+MinsukJi-NOAA@users.noreply.github.com>
DeniseWorthen · May 10, 2022 · 38f4f6a · 38f4f6a
1 parent 0ee3448
commit 38f4f6a
Show file tree

Hide file tree

Showing 79 changed files with 2,931 additions and 2,731 deletions.
diff --git a/CICE-interface/CICE b/CICE-interface/CICE
diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log
diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log
diff --git a/tests/RegressionTests_jet.intel.log b/tests/RegressionTests_jet.intel.log
diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log
diff --git a/tests/RegressionTests_wcoss_cray.log b/tests/RegressionTests_wcoss_cray.log
diff --git a/tests/RegressionTests_wcoss_dell_p3.log b/tests/RegressionTests_wcoss_dell_p3.log
diff --git a/tests/compile.sh b/tests/compile.sh
@@ -47,7 +47,7 @@ BUILD_DIR=$(pwd)/build_${BUILD_NAME}
 if [[ $MACHINE_ID == cheyenne.* ]] ; then
     BUILD_JOBS=${BUILD_JOBS:-3}
 elif [[ $MACHINE_ID == wcoss_dell_p3 ]] ; then
-    BUILD_JOBS=${BUILD_JOBS:-4}
+    BUILD_JOBS=${BUILD_JOBS:-2}
     source $PATHTR/tests/module-setup.sh
 fi
 

diff --git a/tests/default_vars.sh b/tests/default_vars.sh
diff --git a/tests/opnReqTests/dcp.sh b/tests/opnReqTests/dcp.sh
@@ -14,22 +14,22 @@ elif [[ $application == 'regional' ]]; then
   if [[ $CI_TEST == 'true' ]]; then
     INPES=10
     JNPES=3
-    TASKS=$((INPES*JNPES + WRITE_GROUP*WRTTASK_PER_GROUP))
+    NTILES=1
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP))
     NODES=$(((TASKS+TPN-1)/TPN))
   else
     INPES=5
     JNPES=12
+    NTILES=1
   fi
 elif [[ $application == 'cpld' ]]; then
   if [[ $CI_TEST == 'true' ]]; then
     INPES=3
     JNPES=1
-    NPROC_ICE=6
-    med_petlist_bounds="0 17"
-    atm_petlist_bounds="0 23"
-    ocn_petlist_bounds="24 33"
-    ice_petlist_bounds="34 39"
-    TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP + 10 + 6))
+    OCN_tasks=10
+    ICE_tasks=6
+    NPROC_ICE=$ICE_tasks
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP + OCN_tasks + ICE_tasks))
   else
     temp=$INPES
     INPES=$JNPES

diff --git a/tests/opnReqTests/fhz.sh b/tests/opnReqTests/fhz.sh
@@ -11,7 +11,7 @@ if [[ $application == 'global' ]]; then
                                    | sed -E "s/GFSPRS.GrbF24 ?//g" \
                                    | sed -e "s/^ *//" -e "s/ *$//")
 elif [[ $application == 'cpld' ]]; then
-  if [[ $TEST_NAME == 'cpld_control_c96_p8' ]] || [[ $TEST_NAME == 'cpld_control_p8' ]]; then
+  if [[ $TEST_NAME =~ 'cpld_control_c96_p8' ]] || [[ $TEST_NAME =~ 'cpld_control_p8' ]] || [[ $TEST_NAME =~ 'cpld_control_c96_noaero_p8' ]]; then
     FHZERO=3
     LIST_FILES=$(echo -n $LIST_FILES | sed -E "s/sfcf024.tile[1-6].nc ?//g" \
                                      | sed -E "s/atmf024.tile[1-6].nc ?//g" \

diff --git a/tests/opnReqTests/mpi.sh b/tests/opnReqTests/mpi.sh
@@ -10,7 +10,7 @@ if [[ $application == 'global' ]]; then
   fi
   WRITE_GROUP=2
   WRTTASK_PER_GROUP=12
-  TASKS=$(( INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP ))
+  TASKS=$(( INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP ))
   NODES=$(((TASKS+TPN-1)/TPN))
 elif [[ $application == 'regional' ]]; then
   echo "Regional application not yet implemented for mpi"

diff --git a/tests/opnReqTests/std.sh b/tests/opnReqTests/std.sh
@@ -6,27 +6,26 @@ if [[ $application == 'global' ]]; then
     JNPES=2
     WRITE_GROUP=1
     WRTTASK_PER_GROUP=12
-    TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP))
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP))
   fi
   RESTART_N=$(( FHMAX/2 ))
   RESTART_INTERVAL="${RESTART_N} -1"
 elif [[ $application == 'regional' ]]; then
   if [[ $CI_TEST == 'true' ]]; then
     INPES=4
     JNPES=6
+    NTILES=1
     WRTTASK_PER_GROUP=8
-    TASKS=$((INPES*JNPES + WRITE_GROUP*WRTTASK_PER_GROUP))
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP))
   fi
 elif [[ $application == 'cpld' ]]; then
   if [ $CI_TEST == 'true' ]; then
     INPES=2
     JNPES=2
-    NPROC_ICE=6
-    med_petlist_bounds="0 23"
-    atm_petlist_bounds="0 29"
-    ocn_petlist_bounds="30 39"
-    ice_petlist_bounds="40 45"
-    TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP + 10 + 6))
+    OCN_tasks=10
+    ICE_tasks=6
+    NPROC_ICE=$ICE_tasks
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP + OCN_tasks + ICE_tasks))
   fi
   RESTART_N=$(( FHMAX/2 ))
   RESTART_INTERVAL="${RESTART_N} -1"

diff --git a/tests/opnReqTests/thr.sh b/tests/opnReqTests/thr.sh
@@ -5,46 +5,42 @@ THRD=2
 TPN=$((TPN/THRD))
 if [[ $application == 'global' ]]; then
   JNPES=$((JNPES/THRD))
-  TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP))
+  TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP))
   NODES=$(((TASKS+TPN-1)/TPN))
 elif [[ $application == 'regional' ]]; then
   if [[ $CI_TEST == 'true' ]]; then
     INPES=4
     JNPES=4
-    TASKS=$((INPES*JNPES + WRITE_GROUP*WRTTASK_PER_GROUP))
+    NTILES=1
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP))
   fi
   NODES=$(((TASKS+TPN-1)/TPN))
 elif [[ $application == 'cpld' ]]; then
   if [[ $CI_TEST != 'true' ]]; then
-    if [[ $TEST_NAME == 'cpld_control_c96_p8' ]]; then
+    if [[ $TEST_NAME =~ 'cpld_control_c96_p8' ]]; then
       INPES=3
       JNPES=4
-      med_petlist_bounds="0 71"
-      chm_petlist_bounds="0 71"
-      atm_petlist_bounds="0 77"
-      ocn_petlist_bounds="78 107"
-      ice_petlist_bounds="108 119"
-      TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP + 30 + 12))
+      OCN_tasks=30
+      ICE_tasks=12
+      NPROC_ICE=$ICE_tasks
+      TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP + OCN_tasks + ICE_tasks))
       NODES=$(((TASKS+TPN-1)/TPN))
-    elif [[ $TEST_NAME == 'cpld_control_c96_noaero_p8' ]]; then
+    elif [[ $TEST_NAME =~ 'cpld_control_c96_noaero_p8' ]]; then
       INPES=3
       JNPES=4
-      med_petlist_bounds="0 71"
-      atm_petlist_bounds="0 77"
-      ocn_petlist_bounds="78 107"
-      ice_petlist_bounds="108 119"
-      TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP + 30 + 12))
+      OCN_tasks=30
+      ICE_tasks=12
+      NPROC_ICE=$ICE_tasks
+      TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP + OCN_tasks + ICE_tasks))
       NODES=$(((TASKS+TPN-1)/TPN))
-    elif [[ $TEST_NAME == 'cpld_control_p8' ]]; then
+    elif [[ $TEST_NAME =~ 'cpld_control_p8' ]]; then
       INPES=3
       JNPES=4
-      med_petlist_bounds="0 71"
-      chm_petlist_bounds="0 71"
-      atm_petlist_bounds="0 77"
-      ocn_petlist_bounds="78 97"
-      ice_petlist_bounds="98 107"
-      wav_petlist_bounds="108 119"
-      TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP + 20 + 10 + 12))
+      OCN_tasks=20
+      ICE_tasks=10
+      WAV_tasks=12
+      NPROC_ICE=$ICE_tasks
+      TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP + OCN_tasks + ICE_tasks + WAV_tasks))
       NODES=$(((TASKS+TPN-1)/TPN))
     elif [[ $TEST_NAME == 'cpld_bmark_p8' ]]; then
       NODES=$(((TASKS+TPN-1)/TPN))

diff --git a/tests/opnReqTests/wrt_env.sh b/tests/opnReqTests/wrt_env.sh
@@ -8,15 +8,13 @@ export RESTART_N=${RESTART_N:-}
 export RESTART_INTERVAL="${RESTART_INTERVAL:-}"
 export INPES=${INPES}
 export JNPES=${JNPES}
+export NTILES=${NTILES:-}
+export OCN_tasks=${OCN_tasks:-}
+export ICE_tasks=${ICE_tasks:-}
+export WAV_tasks=${WAV_tasks:-}
 export WRITE_GROUP=${WRITE_GROUP}
 export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP}
 export NPROC_ICE=${NPROC_ICE:-}
-export med_petlist_bounds="${med_petlist_bounds:-}"
-export chm_petlist_bounds="${chm_petlist_bounds:-}"
-export atm_petlist_bounds="${atm_petlist_bounds:-}"
-export ocn_petlist_bounds="${ocn_petlist_bounds:-}"
-export ice_petlist_bounds="${ice_petlist_bounds:-}"
-export wav_petlist_bounds="${wav_petlist_bounds:-}"
 export THRD=${THRD}
 export TASKS=${TASKS}
 export TPN=${TPN}

diff --git a/tests/rt.sh b/tests/rt.sh
@@ -777,6 +777,7 @@ EOF
     (
       source ${PATHRT}/tests/$TEST_NAME
 
+      TPN=$(( TPN / THRD ))
       NODES=$(( TASKS / TPN ))
       if (( NODES * TPN < TASKS )); then
         NODES=$(( NODES + 1 ))

diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh
@@ -548,7 +548,7 @@ ecflow_run() {
       module load ecflow
       echo "Using special Jet ECFLOW start procedure"
       MYCOMM="bash -l -c \"module load ecflow && ${ECFLOW_START} -d ${RUNDIR_ROOT}/ecflow_server\""
-      ssh $ECF_HOST "${MYCOMM}" 
+      ssh $ECF_HOST "${MYCOMM}"
     else
       ${ECFLOW_START} -p ${ECF_PORT} -d ${RUNDIR_ROOT}/ecflow_server
     fi

diff --git a/tests/run_compile.sh b/tests/run_compile.sh
@@ -89,7 +89,7 @@ cat ${RUNDIR}/job_timestamp.txt >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
 # End compile job
 ################################################################################
 
-echo " $( date +%s )" >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
+echo " $( date +%s ), 1" >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
 
 elapsed=$SECONDS
 echo "Elapsed time $elapsed seconds. Compile ${COMPILE_NR}"
diff --git a/tests/run_test.sh b/tests/run_test.sh
@@ -23,6 +23,60 @@ write_fail_test() {
   exit 1
 }
 
+function compute_petbounds() {
+
+  # each test MUST define ${COMPONENT}_tasks variable for all components it is using
+  # and MUST NOT define those that it's not using or set the value to 0.
+
+  # ATM is a special case since it is running on the sum of compute and io tasks.
+  # CHM component and mediator are running on ATM compute tasks only.
+
+  local n=0
+  unset atm_petlist_bounds ocn_petlist_bounds ice_petlist_bounds wav_petlist_bounds chm_petlist_bounds med_petlist_bounds
+
+  # ATM
+  ATM_io_tasks=${ATM_io_tasks:-0}
+  if [[ $((ATM_compute_tasks + ATM_io_tasks)) -gt 0 ]]; then
+     atm_petlist_bounds="${n} $((n + ATM_compute_tasks + ATM_io_tasks -1))"
+     n=$((n + ATM_compute_tasks + ATM_io_tasks))
+  fi
+
+  # OCN
+  if [[ ${OCN_tasks:-0} -gt 0 ]]; then
+     ocn_petlist_bounds="${n} $((n + OCN_tasks - 1))"
+     n=$((n + OCN_tasks))
+  fi
+
+  # ICE
+  if [[ ${ICE_tasks:-0} -gt 0 ]]; then
+     ice_petlist_bounds="${n} $((n + ICE_tasks - 1))"
+     n=$((n + ICE_tasks))
+  fi
+
+  # WAV
+  if [[ ${WAV_tasks:-0} -gt 0 ]]; then
+     wav_petlist_bounds="${n} $((n + WAV_tasks - 1))"
+     n=$((n + WAV_tasks))
+  fi
+
+  # CHM
+  chm_petlist_bounds="0 $((ATM_compute_tasks - 1))"
+
+  # MED
+  med_petlist_bounds="0 $((ATM_compute_tasks - 1))"
+
+  UFS_tasks=${n}
+
+  echo "ATM_petlist_bounds: ${atm_petlist_bounds:-}"
+  echo "OCN_petlist_bounds: ${ocn_petlist_bounds:-}"
+  echo "ICE_petlist_bounds: ${ice_petlist_bounds:-}"
+  echo "WAV_petlist_bounds: ${wav_petlist_bounds:-}"
+  echo "CHM_petlist_bounds: ${chm_petlist_bounds:-}"
+  echo "MED_petlist_bounds: ${med_petlist_bounds:-}"
+  echo "UFS_tasks         : ${UFS_tasks:-}"
+
+}
+
 if [[ $# != 5 ]]; then
   echo "Usage: $0 PATHRT RUNDIR_ROOT TEST_NAME TEST_NR COMPILE_NR"
   exit 1
@@ -107,8 +161,25 @@ fi
 
 atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure
 
+if [[ $DATM_CDEPS = 'false' ]]; then
+  if [[ ${ATM_compute_tasks:-0} -eq 0 ]]; then
+    ATM_compute_tasks=$((INPES * JNPES * NTILES))
+  fi
+  if [[ $QUILTING = '.true.' ]]; then
+    ATM_io_tasks=$((WRITE_GROUP * WRTTASK_PER_GROUP))
+  fi
+fi
+
+compute_petbounds
+
 atparse < ${PATHRT}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure
 
+# remove after all tests pass
+if [[ $TASKS -ne $UFS_tasks ]]; then
+   echo "$TASKS -ne $UFS_tasks "
+  exit 1
+fi
+
 if [[ "Q${INPUT_NEST02_NML:-}" != Q ]] ; then
     INPES_NEST=$INPES_NEST02; JNPES_NEST=$JNPES_NEST02
     NPX_NEST=$NPX_NEST02; NPY_NEST=$NPY_NEST02
@@ -213,26 +284,20 @@ if [[ $DOCN_CDEPS = 'true' ]]; then
   atparse < ${PATHRT}/parm/${DOCN_STREAM_CONFIGURE:-docn.streams.IN} > docn.streams
 fi
 
+TPN=$(( TPN / THRD ))
+if (( TASKS < TPN )); then
+  TPN=${TASKS}
+fi
+NODES=$(( TASKS / TPN ))
+if (( NODES * TPN < TASKS )); then
+  NODES=$(( NODES + 1 ))
+fi
+
 if [[ $SCHEDULER = 'pbs' ]]; then
-  NODES=$(( TASKS / TPN ))
-  if (( NODES * TPN < TASKS )); then
-    NODES=$(( NODES + 1 ))
-  fi
   atparse < $PATHRT/fv3_conf/fv3_qsub.IN > job_card
 elif [[ $SCHEDULER = 'slurm' ]]; then
-  NODES=$(( TASKS / TPN ))
-  if (( NODES * TPN < TASKS )); then
-    NODES=$(( NODES + 1 ))
-  fi
   atparse < $PATHRT/fv3_conf/fv3_slurm.IN > job_card
 elif [[ $SCHEDULER = 'lsf' ]]; then
-  if (( TASKS < TPN )); then
-    TPN=${TASKS}
-  fi
-  NODES=$(( TASKS / TPN ))
-  if (( NODES * TPN < TASKS )); then
-    NODES=$(( NODES + 1 ))
-  fi
   atparse < $PATHRT/fv3_conf/fv3_bsub.IN > job_card
 fi
 
@@ -281,7 +346,7 @@ fi
 # End test
 ################################################################################
 
-echo " $( date +%s )" >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
+echo " $( date +%s ), ${NODES}" >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
 
 ################################################################################
 # Remove RUN_DIRs if they are no longer needed by other tests

diff --git a/tests/tests/control_2threads b/tests/tests/control_2threads
@@ -65,7 +65,6 @@ export IOVR=3
 
 export THRD=2
 export TASKS=$TASKS_thrd
-export TPN=$TPN_thrd
 export INPES=$INPES_thrd
 export JNPES=$JNPES_thrd
 export WRTTASK_PER_GROUP=6

diff --git a/tests/tests/control_2threads_debug b/tests/tests/control_2threads_debug
@@ -30,7 +30,6 @@ export OUTPUT_FH="0 1"
 
 export THRD=2
 export TASKS=$TASKS_thrd
-export TPN=$TPN_thrd
 export INPES=$INPES_thrd
 export JNPES=$JNPES_thrd
 export WRTTASK_PER_GROUP=6

diff --git a/tests/tests/control_2threads_p8 b/tests/tests/control_2threads_p8
@@ -62,7 +62,6 @@ export WRITE_DOPOST=.true.
 
 export THRD=2
 export TASKS=$TASKS_thrd
-export TPN=$TPN_thrd
 export INPES=$INPES_thrd
 export JNPES=$JNPES_thrd
 export WRTTASK_PER_GROUP=6

diff --git a/tests/tests/control_atm_aerosols b/tests/tests/control_atm_aerosols
@@ -73,9 +73,7 @@ export DNATS=3
 export CPL=.true.
 export CPLCHM=.true.
 export atm_model='fv3'
-export atm_petlist_bounds="0 149"
 export chm_model='gocart'
-export chm_petlist_bounds="0 143"
 export coupling_interval_sec=${DT_ATMOS}
 export NEMS_CONFIGURE="nems.configure.atm_aerosols.IN"
 

diff --git a/tests/tests/control_atmwav b/tests/tests/control_atmwav
@@ -70,14 +70,12 @@ export WW3RSTDTHR=3
 export DT_2_RST="$(printf "%02d" $(( ${WW3RSTDTHR}*3600 )))"
 
 export TASKS=$TASKS_cpl_atmw
-export TPN=$TPN_cpl_atmw
 export INPES=$INPES_cpl_atmw
 export JNPES=$JNPES_cpl_atmw
 export THRD=$THRD_cpl_atmw
 export WRTTASK_PER_GROUP=$WPG_cpl_atmw
 
-export atm_petlist_bounds=$APB_cpl_atmw
-export wav_petlist_bounds=$WPB_cpl_atmw
+WAV_tasks=${WAV_tasks_atmw}
 
 export CPL=.true.
 export CPLWAV=.true.

diff --git a/tests/tests/control_c384 b/tests/tests/control_c384
@@ -19,7 +19,7 @@ export LIST_FILES="sfcf000.nc \
 
 export_fv3
 export TASKS=${TASKS_c384}
-export TPN=${TPN_c384}
+export THRD=${THRD_c384}
 export INPES=${INPES_c384}
 export JNPES=${JNPES_c384}
 export WRITE_GROUP=1