enhance run_jedi_exe ctest error checking and optimize job configurat…

…on (#223)
NOAA-EMC · Feb 4, 2023 · 7a75fb5 · 7a75fb5
1 parent 70b7d65
commit 7a75fb5
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 11 deletions.
diff --git a/test/atm/global-workflow/run_jedi_exe_3dhofx.sh b/test/atm/global-workflow/run_jedi_exe_3dhofx.sh
@@ -33,8 +33,8 @@ config:
   obs_list: ${srcdir}/parm/atm/obs/lists/gdas_prototype_3d.yaml
   gdas_fix_root: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix
   atm: true
-  layout_x: 1
-  layout_y: 1
+  layout_x: 3
+  layout_y: 2
   atm_window_length: PT6H
   valid_time: 2021-08-01T00:00:00Z
   dump: gdas
@@ -47,8 +47,8 @@ job options:
   queue: debug
   partition: hera
   walltime: '30:00'
-  ntasks: 6
-  ntasks-per-node: 2
+  ntasks: 36
+  ntasks-per-node: 9
   modulepath: ${srcdir}/modulefiles
 EOF
 
@@ -76,11 +76,15 @@ rc=1
 n=1
 while [ $n -le $nloop ]; do
     count=$(cat GDASApp.o$jobid | grep "OOPS_STATS Run end" | wc -l)
-    echo "n = $n   count = $count"
     if [ $count -gt 0 ]; then
 	rc=0
 	break
     fi
+    count=$(cat GDASApp.o$jobid | grep "srun: error" | wc -l)
+    if [ $count -gt 0 ]; then
+        rc=9
+        break
+    fi
     sleep 10
     n=$((n+1))
 done

diff --git a/test/atm/global-workflow/run_jedi_exe_3dvar.sh b/test/atm/global-workflow/run_jedi_exe_3dvar.sh
@@ -62,7 +62,7 @@ job options:
   modulepath: ${srcdir}/modulefiles
 EOF
 
-if [ -e stdout.txt]; then
+if [ -e stdout.txt ]; then
     rm -f stdout.txt
 fi
 
@@ -89,11 +89,15 @@ rc=1
 n=1
 while [ $n -le $nloop ]; do
     count=$(cat GDASApp.o$jobid | grep "OOPS_STATS Run end" | wc -l)
-    echo "n = $n   count = $count"
     if [ $count -gt 0 ]; then
 	rc=0
 	break
     fi
+    count=$(cat GDASApp.o$jobid | grep "srun: error" | wc -l)
+    if [ $count -gt 0 ]; then
+        rc=9
+        break
+    fi
     sleep 10
     n=$((n+1))
 done

diff --git a/test/atm/global-workflow/run_jedi_exe_letkf.sh b/test/atm/global-workflow/run_jedi_exe_letkf.sh
@@ -44,8 +44,8 @@ config:
   obs_list: ${srcdir}/parm/atm/obs/lists/lgetkf_prototype.yaml
   gdas_fix_root: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix
   atm: true
-  layout_x: 1
-  layout_y: 1
+  layout_x: 3
+  layout_y: 2
   atm_window_length: PT6H
   valid_time: 2021-12-21T06:00:00Z
   dump: gdas
@@ -63,7 +63,7 @@ job options:
   queue: debug
   partition: hera
   walltime: '30:00'
-  ntasks: 6
+  ntasks: 36
   modulepath: ${srcdir}/modulefiles
 EOF
 
@@ -91,11 +91,15 @@ rc=1
 n=1
 while [ $n -le $nloop ]; do
     count=$(cat GDASApp.o$jobid | grep "OOPS_STATS Run end" | wc -l)
-    echo "n = $n   count = $count"
     if [ $count -gt 0 ]; then
 	rc=0
 	break
     fi
+    count=$(cat GDASApp.o$jobid | grep "srun: error" | wc -l)
+    if [ $count -gt 0 ]; then
+        rc=9
+        break
+    fi
     sleep 10
     n=$((n+1))
 done