Merge branch 'ufs-community:develop' into develop

tests/compile.sh

@@ -47,7 +47,7 @@ BUILD_DIR=$(pwd)/build_${BUILD_NAME}
 if [[ $MACHINE_ID == cheyenne.* ]] ; then
     BUILD_JOBS=${BUILD_JOBS:-3}
 elif [[ $MACHINE_ID == wcoss_dell_p3 ]] ; then
-    BUILD_JOBS=${BUILD_JOBS:-4}
+    BUILD_JOBS=${BUILD_JOBS:-2}
     source $PATHTR/tests/module-setup.sh
 fi
 

tests/opnReqTests/dcp.sh

@@ -14,22 +14,22 @@ elif [[ $application == 'regional' ]]; then
   if [[ $CI_TEST == 'true' ]]; then
     INPES=10
     JNPES=3
-    TASKS=$((INPES*JNPES + WRITE_GROUP*WRTTASK_PER_GROUP))
+    NTILES=1
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP))
     NODES=$(((TASKS+TPN-1)/TPN))
   else
     INPES=5
     JNPES=12
+    NTILES=1
   fi
 elif [[ $application == 'cpld' ]]; then
   if [[ $CI_TEST == 'true' ]]; then
     INPES=3
     JNPES=1
-    NPROC_ICE=6
-    med_petlist_bounds="0 17"
-    atm_petlist_bounds="0 23"
-    ocn_petlist_bounds="24 33"
-    ice_petlist_bounds="34 39"
-    TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP + 10 + 6))
+    OCN_tasks=10
+    ICE_tasks=6
+    NPROC_ICE=$ICE_tasks
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP + OCN_tasks + ICE_tasks))
   else
     temp=$INPES
     INPES=$JNPES

tests/opnReqTests/fhz.sh

@@ -11,7 +11,7 @@ if [[ $application == 'global' ]]; then
                                    | sed -E "s/GFSPRS.GrbF24 ?//g" \
                                    | sed -e "s/^ *//" -e "s/ *$//")
 elif [[ $application == 'cpld' ]]; then
-  if [[ $TEST_NAME == 'cpld_control_c96_p8' ]] || [[ $TEST_NAME == 'cpld_control_p8' ]]; then
+  if [[ $TEST_NAME =~ 'cpld_control_c96_p8' ]] || [[ $TEST_NAME =~ 'cpld_control_p8' ]] || [[ $TEST_NAME =~ 'cpld_control_c96_noaero_p8' ]]; then
     FHZERO=3
     LIST_FILES=$(echo -n $LIST_FILES | sed -E "s/sfcf024.tile[1-6].nc ?//g" \
                                      | sed -E "s/atmf024.tile[1-6].nc ?//g" \

tests/opnReqTests/mpi.sh

@@ -10,7 +10,7 @@ if [[ $application == 'global' ]]; then
   fi
   WRITE_GROUP=2
   WRTTASK_PER_GROUP=12
-  TASKS=$(( INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP ))
+  TASKS=$(( INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP ))
   NODES=$(((TASKS+TPN-1)/TPN))
 elif [[ $application == 'regional' ]]; then
   echo "Regional application not yet implemented for mpi"

tests/opnReqTests/std.sh

@@ -6,27 +6,26 @@ if [[ $application == 'global' ]]; then
     JNPES=2
     WRITE_GROUP=1
     WRTTASK_PER_GROUP=12
-    TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP))
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP))
   fi
   RESTART_N=$(( FHMAX/2 ))
   RESTART_INTERVAL="${RESTART_N} -1"
 elif [[ $application == 'regional' ]]; then
   if [[ $CI_TEST == 'true' ]]; then
     INPES=4
     JNPES=6
+    NTILES=1
     WRTTASK_PER_GROUP=8
-    TASKS=$((INPES*JNPES + WRITE_GROUP*WRTTASK_PER_GROUP))
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP))
   fi
 elif [[ $application == 'cpld' ]]; then
   if [ $CI_TEST == 'true' ]; then
     INPES=2
     JNPES=2
-    NPROC_ICE=6
-    med_petlist_bounds="0 23"
-    atm_petlist_bounds="0 29"
-    ocn_petlist_bounds="30 39"
-    ice_petlist_bounds="40 45"
-    TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP + 10 + 6))
+    OCN_tasks=10
+    ICE_tasks=6
+    NPROC_ICE=$ICE_tasks
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP + OCN_tasks + ICE_tasks))
   fi
   RESTART_N=$(( FHMAX/2 ))
   RESTART_INTERVAL="${RESTART_N} -1"

tests/opnReqTests/thr.sh

@@ -5,46 +5,42 @@ THRD=2
 TPN=$((TPN/THRD))
 if [[ $application == 'global' ]]; then
   JNPES=$((JNPES/THRD))
-  TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP))
+  TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP))
   NODES=$(((TASKS+TPN-1)/TPN))
 elif [[ $application == 'regional' ]]; then
   if [[ $CI_TEST == 'true' ]]; then
     INPES=4
     JNPES=4
-    TASKS=$((INPES*JNPES + WRITE_GROUP*WRTTASK_PER_GROUP))
+    NTILES=1
+    TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP))
   fi
   NODES=$(((TASKS+TPN-1)/TPN))
 elif [[ $application == 'cpld' ]]; then
   if [[ $CI_TEST != 'true' ]]; then
-    if [[ $TEST_NAME == 'cpld_control_c96_p8' ]]; then
+    if [[ $TEST_NAME =~ 'cpld_control_c96_p8' ]]; then
       INPES=3
       JNPES=4
-      med_petlist_bounds="0 71"
-      chm_petlist_bounds="0 71"
-      atm_petlist_bounds="0 77"
-      ocn_petlist_bounds="78 107"
-      ice_petlist_bounds="108 119"
-      TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP + 30 + 12))
+      OCN_tasks=30
+      ICE_tasks=12
+      NPROC_ICE=$ICE_tasks
+      TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP + OCN_tasks + ICE_tasks))
       NODES=$(((TASKS+TPN-1)/TPN))
-    elif [[ $TEST_NAME == 'cpld_control_c96_noaero_p8' ]]; then
+    elif [[ $TEST_NAME =~ 'cpld_control_c96_noaero_p8' ]]; then
       INPES=3
       JNPES=4
-      med_petlist_bounds="0 71"
-      atm_petlist_bounds="0 77"
-      ocn_petlist_bounds="78 107"
-      ice_petlist_bounds="108 119"
-      TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP + 30 + 12))
+      OCN_tasks=30
+      ICE_tasks=12
+      NPROC_ICE=$ICE_tasks
+      TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP + OCN_tasks + ICE_tasks))
       NODES=$(((TASKS+TPN-1)/TPN))
-    elif [[ $TEST_NAME == 'cpld_control_p8' ]]; then
+    elif [[ $TEST_NAME =~ 'cpld_control_p8' ]]; then
       INPES=3
       JNPES=4
-      med_petlist_bounds="0 71"
-      chm_petlist_bounds="0 71"
-      atm_petlist_bounds="0 77"
-      ocn_petlist_bounds="78 97"
-      ice_petlist_bounds="98 107"
-      wav_petlist_bounds="108 119"
-      TASKS=$((INPES*JNPES*6 + WRITE_GROUP*WRTTASK_PER_GROUP + 20 + 10 + 12))
+      OCN_tasks=20
+      ICE_tasks=10
+      WAV_tasks=12
+      NPROC_ICE=$ICE_tasks
+      TASKS=$((INPES*JNPES*NTILES + WRITE_GROUP*WRTTASK_PER_GROUP + OCN_tasks + ICE_tasks + WAV_tasks))
       NODES=$(((TASKS+TPN-1)/TPN))
     elif [[ $TEST_NAME == 'cpld_bmark_p8' ]]; then
       NODES=$(((TASKS+TPN-1)/TPN))

tests/opnReqTests/wrt_env.sh

@@ -8,15 +8,13 @@ export RESTART_N=${RESTART_N:-}
 export RESTART_INTERVAL="${RESTART_INTERVAL:-}"
 export INPES=${INPES}
 export JNPES=${JNPES}
+export NTILES=${NTILES:-}
+export OCN_tasks=${OCN_tasks:-}
+export ICE_tasks=${ICE_tasks:-}
+export WAV_tasks=${WAV_tasks:-}
 export WRITE_GROUP=${WRITE_GROUP}
 export WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP}
 export NPROC_ICE=${NPROC_ICE:-}
-export med_petlist_bounds="${med_petlist_bounds:-}"
-export chm_petlist_bounds="${chm_petlist_bounds:-}"
-export atm_petlist_bounds="${atm_petlist_bounds:-}"
-export ocn_petlist_bounds="${ocn_petlist_bounds:-}"
-export ice_petlist_bounds="${ice_petlist_bounds:-}"
-export wav_petlist_bounds="${wav_petlist_bounds:-}"
 export THRD=${THRD}
 export TASKS=${TASKS}
 export TPN=${TPN}

tests/rt.sh

@@ -777,6 +777,7 @@ EOF
     (
       source ${PATHRT}/tests/$TEST_NAME
 
+      TPN=$(( TPN / THRD ))
       NODES=$(( TASKS / TPN ))
       if (( NODES * TPN < TASKS )); then
         NODES=$(( NODES + 1 ))

tests/rt_utils.sh

@@ -548,7 +548,7 @@ ecflow_run() {
       module load ecflow
       echo "Using special Jet ECFLOW start procedure"
       MYCOMM="bash -l -c \"module load ecflow && ${ECFLOW_START} -d ${RUNDIR_ROOT}/ecflow_server\""
-      ssh $ECF_HOST "${MYCOMM}" 
+      ssh $ECF_HOST "${MYCOMM}"
     else
       ${ECFLOW_START} -p ${ECF_PORT} -d ${RUNDIR_ROOT}/ecflow_server
     fi

tests/run_compile.sh

@@ -89,7 +89,7 @@ cat ${RUNDIR}/job_timestamp.txt >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
 # End compile job
 ################################################################################
 
-echo " $( date +%s )" >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
+echo " $( date +%s ), 1" >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
 
 elapsed=$SECONDS
 echo "Elapsed time $elapsed seconds. Compile ${COMPILE_NR}"

tests/run_test.sh

@@ -23,6 +23,60 @@ write_fail_test() {
   exit 1
 }
 
+function compute_petbounds() {
+
+  # each test MUST define ${COMPONENT}_tasks variable for all components it is using
+  # and MUST NOT define those that it's not using or set the value to 0.
+
+  # ATM is a special case since it is running on the sum of compute and io tasks.
+  # CHM component and mediator are running on ATM compute tasks only.
+
+  local n=0
+  unset atm_petlist_bounds ocn_petlist_bounds ice_petlist_bounds wav_petlist_bounds chm_petlist_bounds med_petlist_bounds
+
+  # ATM
+  ATM_io_tasks=${ATM_io_tasks:-0}
+  if [[ $((ATM_compute_tasks + ATM_io_tasks)) -gt 0 ]]; then
+     atm_petlist_bounds="${n} $((n + ATM_compute_tasks + ATM_io_tasks -1))"
+     n=$((n + ATM_compute_tasks + ATM_io_tasks))
+  fi
+
+  # OCN
+  if [[ ${OCN_tasks:-0} -gt 0 ]]; then
+     ocn_petlist_bounds="${n} $((n + OCN_tasks - 1))"
+     n=$((n + OCN_tasks))
+  fi
+
+  # ICE
+  if [[ ${ICE_tasks:-0} -gt 0 ]]; then
+     ice_petlist_bounds="${n} $((n + ICE_tasks - 1))"
+     n=$((n + ICE_tasks))
+  fi
+
+  # WAV
+  if [[ ${WAV_tasks:-0} -gt 0 ]]; then
+     wav_petlist_bounds="${n} $((n + WAV_tasks - 1))"
+     n=$((n + WAV_tasks))
+  fi
+
+  # CHM
+  chm_petlist_bounds="0 $((ATM_compute_tasks - 1))"
+
+  # MED
+  med_petlist_bounds="0 $((ATM_compute_tasks - 1))"
+
+  UFS_tasks=${n}
+
+  echo "ATM_petlist_bounds: ${atm_petlist_bounds:-}"
+  echo "OCN_petlist_bounds: ${ocn_petlist_bounds:-}"
+  echo "ICE_petlist_bounds: ${ice_petlist_bounds:-}"
+  echo "WAV_petlist_bounds: ${wav_petlist_bounds:-}"
+  echo "CHM_petlist_bounds: ${chm_petlist_bounds:-}"
+  echo "MED_petlist_bounds: ${med_petlist_bounds:-}"
+  echo "UFS_tasks         : ${UFS_tasks:-}"
+
+}
+
 if [[ $# != 5 ]]; then
   echo "Usage: $0 PATHRT RUNDIR_ROOT TEST_NAME TEST_NR COMPILE_NR"
   exit 1
@@ -107,8 +161,25 @@ fi
 
 atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure
 
+if [[ $DATM_CDEPS = 'false' ]]; then
+  if [[ ${ATM_compute_tasks:-0} -eq 0 ]]; then
+    ATM_compute_tasks=$((INPES * JNPES * NTILES))
+  fi
+  if [[ $QUILTING = '.true.' ]]; then
+    ATM_io_tasks=$((WRITE_GROUP * WRTTASK_PER_GROUP))
+  fi
+fi
+
+compute_petbounds
+
 atparse < ${PATHRT}/parm/${NEMS_CONFIGURE:-nems.configure} > nems.configure
 
+# remove after all tests pass
+if [[ $TASKS -ne $UFS_tasks ]]; then
+   echo "$TASKS -ne $UFS_tasks "
+  exit 1
+fi
+
 if [[ "Q${INPUT_NEST02_NML:-}" != Q ]] ; then
     INPES_NEST=$INPES_NEST02; JNPES_NEST=$JNPES_NEST02
     NPX_NEST=$NPX_NEST02; NPY_NEST=$NPY_NEST02
@@ -213,26 +284,20 @@ if [[ $DOCN_CDEPS = 'true' ]]; then
   atparse < ${PATHRT}/parm/${DOCN_STREAM_CONFIGURE:-docn.streams.IN} > docn.streams
 fi
 
+TPN=$(( TPN / THRD ))
+if (( TASKS < TPN )); then
+  TPN=${TASKS}
+fi
+NODES=$(( TASKS / TPN ))
+if (( NODES * TPN < TASKS )); then
+  NODES=$(( NODES + 1 ))
+fi
+
 if [[ $SCHEDULER = 'pbs' ]]; then
-  NODES=$(( TASKS / TPN ))
-  if (( NODES * TPN < TASKS )); then
-    NODES=$(( NODES + 1 ))
-  fi
   atparse < $PATHRT/fv3_conf/fv3_qsub.IN > job_card
 elif [[ $SCHEDULER = 'slurm' ]]; then
-  NODES=$(( TASKS / TPN ))
-  if (( NODES * TPN < TASKS )); then
-    NODES=$(( NODES + 1 ))
-  fi
   atparse < $PATHRT/fv3_conf/fv3_slurm.IN > job_card
 elif [[ $SCHEDULER = 'lsf' ]]; then
-  if (( TASKS < TPN )); then
-    TPN=${TASKS}
-  fi
-  NODES=$(( TASKS / TPN ))
-  if (( NODES * TPN < TASKS )); then
-    NODES=$(( NODES + 1 ))
-  fi
   atparse < $PATHRT/fv3_conf/fv3_bsub.IN > job_card
 fi
 
@@ -281,7 +346,7 @@ fi
 # End test
 ################################################################################
 
-echo " $( date +%s )" >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
+echo " $( date +%s ), ${NODES}" >> ${LOG_DIR}/job_${JOB_NR}_timestamp.txt
 
 ################################################################################
 # Remove RUN_DIRs if they are no longer needed by other tests

tests/tests/control_2threads

@@ -65,7 +65,6 @@ export IOVR=3
 
 export THRD=2
 export TASKS=$TASKS_thrd
-export TPN=$TPN_thrd
 export INPES=$INPES_thrd
 export JNPES=$JNPES_thrd
 export WRTTASK_PER_GROUP=6

tests/tests/control_2threads_debug

@@ -30,7 +30,6 @@ export OUTPUT_FH="0 1"
 
 export THRD=2
 export TASKS=$TASKS_thrd
-export TPN=$TPN_thrd
 export INPES=$INPES_thrd
 export JNPES=$JNPES_thrd
 export WRTTASK_PER_GROUP=6

tests/tests/control_2threads_p8

@@ -62,7 +62,6 @@ export WRITE_DOPOST=.true.
 
 export THRD=2
 export TASKS=$TASKS_thrd
-export TPN=$TPN_thrd
 export INPES=$INPES_thrd
 export JNPES=$JNPES_thrd
 export WRTTASK_PER_GROUP=6

tests/tests/control_atm_aerosols

@@ -73,9 +73,7 @@ export DNATS=3
 export CPL=.true.
 export CPLCHM=.true.
 export atm_model='fv3'
-export atm_petlist_bounds="0 149"
 export chm_model='gocart'
-export chm_petlist_bounds="0 143"
 export coupling_interval_sec=${DT_ATMOS}
 export NEMS_CONFIGURE="nems.configure.atm_aerosols.IN"
 

tests/tests/control_atmwav

@@ -70,14 +70,12 @@ export WW3RSTDTHR=3
 export DT_2_RST="$(printf "%02d" $(( ${WW3RSTDTHR}*3600 )))"
 
 export TASKS=$TASKS_cpl_atmw
-export TPN=$TPN_cpl_atmw
 export INPES=$INPES_cpl_atmw
 export JNPES=$JNPES_cpl_atmw
 export THRD=$THRD_cpl_atmw
 export WRTTASK_PER_GROUP=$WPG_cpl_atmw
 
-export atm_petlist_bounds=$APB_cpl_atmw
-export wav_petlist_bounds=$WPB_cpl_atmw
+WAV_tasks=${WAV_tasks_atmw}
 
 export CPL=.true.
 export CPLWAV=.true.

tests/tests/control_c384

@@ -19,7 +19,7 @@ export LIST_FILES="sfcf000.nc \
 
 export_fv3
 export TASKS=${TASKS_c384}
-export TPN=${TPN_c384}
+export THRD=${THRD_c384}
 export INPES=${INPES_c384}
 export JNPES=${JNPES_c384}
 export WRITE_GROUP=1