Fix separate threading for gfs and gdas forecasts.

Refs NOAA-EMC#610
DavidHuber-NOAA · Jan 31, 2022 · 7dcebf7 · 7dcebf7
1 parent d3028b9
commit 7dcebf7
Show file tree

Hide file tree

Showing 7 changed files with 28 additions and 9 deletions.
diff --git a/env/HERA.env b/env/HERA.env
@@ -130,14 +130,17 @@ elif [ $step = "eupd" ]; then
 
 elif [ $step = "fcst" ]; then
 
+
+    if [[ $CDUMP == "gfs" ]]; then
+        npe_fcst=$npe_fcst_gfs
+        npe_node_fcst=$npe_node_fcst_gfs
+    fi
+
     nth_max=$(($npe_node_max / $npe_node_fcst))
 
     export NTHREADS_FV3=${nth_fv3:-$nth_max}
     [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
     export cores_per_node=$npe_node_max
-    if [[ $CDUMP == "gfs" ]]; then
-        npe_fcst=$npe_fcst_gfs
-    fi
     export APRUN_FV3="$launcher -n $npe_fcst"
 
     export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}

diff --git a/env/JET.env b/env/JET.env
@@ -99,6 +99,11 @@ elif [ $step = "eupd" ]; then
 
 elif [ $step = "fcst" ]; then
 
+    if [[ $CDUMP == "gfs" ]]; then
+        npe_fcst=$npe_fcst_gfs
+        npe_node_fcst=$npe_node_fcst_gfs
+    fi
+
     nth_max=$(($npe_node_max / $npe_node_fcst))
 
     export NTHREADS_FV3=${nth_fv3:-$nth_max}

diff --git a/env/ORION.env b/env/ORION.env
@@ -126,14 +126,16 @@ elif [ $step = "eupd" ]; then
 
 elif [ $step = "fcst" ]; then
 
+    if [[ $CDUMP == "gfs" ]]; then
+        npe_fcst=$npe_fcst_gfs
+        npe_node_fcst=$npe_node_fcst_gfs
+    fi
+
     nth_max=$(($npe_node_max / $npe_node_fcst))
 
     export NTHREADS_FV3=${nth_fv3:-$nth_max}
     [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
     export cores_per_node=$npe_node_max
-    if [[ $CDUMP == "gfs" ]]; then
-        npe_fcst=$npe_fcst_gfs
-    fi
     export APRUN_FV3="$launcher -n $npe_fcst"
 
     export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}

diff --git a/parm/config/config.fcst b/parm/config/config.fcst
@@ -9,7 +9,6 @@ echo "BEGIN: config.fcst"
 
 # Source model specific information that is resolution dependent
 . $EXPDIR/config.fv3 $CASE
-[[ "$CDUMP" == "gfs" ]] && export nth_fv3=$nth_fv3_gfs
 
 # Turn off waves if not used for this CDUMP
 case $WAVE_CDUMP in

diff --git a/parm/config/config.resources b/parm/config/config.resources
@@ -209,8 +209,11 @@ elif [ $step = "fcst" ]; then
         NTASKS_TOT=$ATMPETS
 
         export nth_fcst=${nth_fv3:-2}
+        export nth_fcst_gfs=${nth_fv3_gfs:-2}
 
         export npe_node_fcst=$(echo "$npe_node_max / $nth_fcst" | bc)
+        export npe_node_fcst_gfs=$(echo "$npe_node_max / $nth_fcst_gfs" | bc)
+
         if [[ "$machine" == "WCOSS_C" ]]; then export memory_fcst="1024M"; fi
 
         if [[ $DO_WAVE == "YES" ]]; then

diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh
@@ -122,7 +122,11 @@ FV3_GFS_predet(){
   #file and the value of npe_node_fcst is not correctly defined when using more than
   #one thread and sets NTHREADS_FV3=1 even when the number of threads is appropraitely >1
   #NTHREADS_FV3=${NTHREADS_FV3:-${NTHREADS_FCST:-${nth_fv3:-1}}}
-  NTHREADS_FV3=${nth_fv3:-1}
+  if [[ $CDUMP == "gdas" ]]; then
+     NTHREADS_FV3=${nth_fv3:-1}
+  else
+     NTHREADS_FV3=${nth_fv3_gfs:-1}
+  fi
   cores_per_node=${cores_per_node:-${npe_node_fcst:-24}}
   ntiles=${ntiles:-6}
   if [ $MEMBER -lt 0 ]; then

diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py
@@ -259,7 +259,10 @@ def get_resources(machine, cfg, task, reservation, cdump='gdas'):
         ppn = cfg[f'npe_node_{ltask}']
 
     if machine in [ 'WCOSS_DELL_P3', 'HERA', 'ORION', 'JET' ]:
-        threads = cfg[f'nth_{ltask}']
+        if cdump in ['gfs'] and f'nth_{task}_gfs' in cfg.keys():
+            threads = cfg[f'nth_{ltask}_gfs']
+        else:
+            threads = cfg[f'nth_{ltask}']
 
     nodes = np.int(np.ceil(np.float(tasks) / np.float(ppn)))