Regression test bug fixes for running on Jet or with Rocoto (#981)

* Bug fixes for running on jet or with rocoto 1. Correctly detect Jet when deciding build job runtimes 2. Do not delete out and err files in compile & test jobs 3. Correct dependencies for WW3 in Rocoto 4. Do not hard-code ACCNR to h-nems This PR also contains fv3/ccpp physics (CCPP PR#812) updates that implements a vertically-integrated formula to properly compute tracer column mass and ensure mass conservation for all tracers except TKE. See issue NCAR/ccpp-physics#811 for details
ufs-community · Jan 5, 2022 · 0f047b2 · 0f047b2
1 parent 58ab0e2
commit 0f047b2
Show file tree

Hide file tree

Showing 14 changed files with 3,451 additions and 3,354 deletions.
diff --git a/FV3 b/FV3
diff --git a/tests/RegressionTests_cheyenne.gnu.log b/tests/RegressionTests_cheyenne.gnu.log
diff --git a/tests/RegressionTests_cheyenne.intel.log b/tests/RegressionTests_cheyenne.intel.log
diff --git a/tests/RegressionTests_gaea.intel.log b/tests/RegressionTests_gaea.intel.log
diff --git a/tests/RegressionTests_hera.gnu.log b/tests/RegressionTests_hera.gnu.log
diff --git a/tests/RegressionTests_hera.intel.log b/tests/RegressionTests_hera.intel.log
diff --git a/tests/RegressionTests_jet.intel.log b/tests/RegressionTests_jet.intel.log
diff --git a/tests/RegressionTests_orion.intel.log b/tests/RegressionTests_orion.intel.log
diff --git a/tests/RegressionTests_wcoss_cray.log b/tests/RegressionTests_wcoss_cray.log
diff --git a/tests/RegressionTests_wcoss_dell_p3.log b/tests/RegressionTests_wcoss_dell_p3.log
diff --git a/tests/rt.sh b/tests/rt.sh
@@ -300,12 +300,12 @@ elif [[ $MACHINE_ID = jet.* ]]; then
 
   QUEUE=batch
   COMPILE_QUEUE=batch
-  ACCNR=h-nems
+  ACCNR=${ACCNR:-h-nems}
   PARTITION=xjet
   DISKNM=/lfs4/HFIP/h-nems/emc.nemspara/RT
-  dprefix=/lfs4/HFIP/h-nems/$USER
-  STMP=$dprefix/RT_BASELINE
-  PTMP=$dprefix/RT_RUNDIRS
+  dprefix=${dprefix:-/lfs4/HFIP/$ACCNR/$USER}
+  STMP=${STMP:-$dprefix/RT_BASELINE}
+  PTMP=${PTMP:-$dprefix/RT_RUNDIRS}
 
   SCHEDULER=slurm
   cp fv3_conf/fv3_slurm.IN_jet fv3_conf/fv3_slurm.IN
@@ -467,7 +467,7 @@ if [[ $TESTS_FILE =~ '35d' ]] || [[ $TESTS_FILE =~ 'weekly' ]]; then
   TEST_35D=true
 fi
 
-BL_DATE=20211230
+BL_DATE=20220103
 if [[ $MACHINE_ID = hera.* ]] || [[ $MACHINE_ID = orion.* ]] || [[ $MACHINE_ID = cheyenne.* ]] || [[ $MACHINE_ID = gaea.* ]] || [[ $MACHINE_ID = jet.* ]] || [[ $MACHINE_ID = s4.* ]]; then
   RTPWD=${RTPWD:-$DISKNM/NEMSfv3gfs/develop-${BL_DATE}/${RT_COMPILER^^}}
 else

diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh
@@ -385,8 +385,10 @@ rocoto_create_compile_task() {
 
   # serialize WW3 builds. FIXME
   DEP_STRING=""
-  if [[ ${MAKE_OPT^^} =~ "WW3=Y" && ${COMPILE_PREV_WW3_NR} != '' ]]; then
-    DEP_STRING="<dependency><taskdep task=\"compile_${COMPILE_PREV_WW3_NR}\"/></dependency>"
+  if [[ ${COMPILE_PREV_WW3_NR} != '' ]]; then
+      if [[ ${MAKE_OPT^^} =~ "-DAPP=ATMW" ]] || [[ ${MAKE_OPT^^} =~ "-DAPP=S2SW" ]] || [[ ${MAKE_OPT^^} =~ "-DAPP=HAFSW" ]] || [[ ${MAKE_OPT^^} =~ "-DAPP=HAFS-ALL" ]] ; then
+          DEP_STRING="<dependency><taskdep task=\"compile_${COMPILE_PREV_WW3_NR}\"/></dependency>"
+      fi
   fi
 
   NATIVE=""
@@ -401,7 +403,7 @@ rocoto_create_compile_task() {
     BUILD_CORES=24
     NATIVE="<exclusive></exclusive> <envar><name>PATHTR</name><value>&PATHTR;</value></envar>"
   fi
-  if [[ ${MACHINE_ID} == jet ]]; then
+  if [[ ${MACHINE_ID} == jet.* ]]; then
     BUILD_WALLTIME="01:00:00"
   fi
   if [[ ${MACHINE_ID} == orion.* ]]; then
@@ -421,8 +423,7 @@ rocoto_create_compile_task() {
     <partition>${PARTITION}</partition>
     <cores>${BUILD_CORES}</cores>
     <walltime>${BUILD_WALLTIME}</walltime>
-    <stdout>&RUNDIR_ROOT;/compile_${COMPILE_NR}/out</stdout>
-    <stderr>&RUNDIR_ROOT;/compile_${COMPILE_NR}/err</stderr>
+    <join>&RUNDIR_ROOT;/compile_${COMPILE_NR}.log</join>
     ${NATIVE}
   </task>
 EOF
@@ -459,8 +460,8 @@ rocoto_create_run_task() {
       <partition>${PARTITION}</partition>
       <nodes>${NODES}:ppn=${TPN}</nodes>
       <walltime>00:${WLCLK}:00</walltime>
-      <stdout>&RUNDIR_ROOT;/${TEST_NAME}${RT_SUFFIX}/out</stdout>
-      <stderr>&RUNDIR_ROOT;/${TEST_NAME}${RT_SUFFIX}/err</stderr>
+      <stdout>&RUNDIR_ROOT;/${TEST_NAME}${RT_SUFFIX}.out</stdout>
+      <stderr>&RUNDIR_ROOT;/${TEST_NAME}${RT_SUFFIX}.err</stderr>
       ${NATIVE}
     </task>
 EOF

diff --git a/tests/run_compile.sh b/tests/run_compile.sh
@@ -75,7 +75,10 @@ if [[ $ROCOTO = 'false' ]]; then
   submit_and_wait job_card
 else
   chmod u+x job_card
-  ./job_card
+  ( ./job_card 2>&1 1>&3 3>&- | tee err ) 3>&1 1>&2 | tee out
+  # The above shell redirection copies stdout to "out" and stderr to "err"
+  # while still sending them to stdout and stderr. It does this without
+  # relying on bash-specific extensions or non-standard OS features.
 fi
 
 ls -l ${PATHTR}/tests/fv3_${COMPILE_NR}.exe

diff --git a/tests/run_test.sh b/tests/run_test.sh
@@ -238,7 +238,10 @@ else
     submit_and_wait job_card
   else
     chmod u+x job_card
-    ./job_card
+    ( ./job_card 2>&1 1>&3 3>&- | tee err ) 3>&1 1>&2 | tee out
+    # The above shell redirection copies stdout to "out" and stderr to "err"
+    # while still sending them to stdout and stderr. It does this without
+    # relying on bash-specific extensions or non-standard OS features.
   fi
 
 fi