Skip to content

Commit

Permalink
Consolidate fcst resources and reorg configs
Browse files Browse the repository at this point in the history 
Resource settings for the forecast job were previously scattered
between multiple files. These settings are now consolidated in
config.resources.

The total number of PEs is now calculated properly if different
coupled models are included.

The config.fcst script is updated to properly source component
scripts if that component is on. New scripts for those components
were created where needed, and settings were moved into the
appropriate scripts (though it may be appropriate to relocate
additional settings in the future). The special load of the wave
config in the j-job is removed as it is no longer needed.

CDUMP assignment had to be moved up in the j-job to occur before
the config files are sourced so the correct settings are used.

Undefined DO switches for stochastic components are added to
prevent syntax errors downstream.

Settings for do_ugwp, do_tofd, and launch_level are removed from
the forecast config, as they are now set in the forecast script
based on the CCPP suite.

Refs: NOAA-EMC#416
  • Loading branch information
@WalterKolczynski-NOAA
WalterKolczynski-NOAA committed Aug 19, 2021
1 parent 68385d9 commit 107e99f
Show file tree
Hide file tree
Showing 7 changed files with 125 additions and 94 deletions.
21 changes: 6 additions & 15 deletions jobs/JGLOBAL_FORECAST
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ date
if [ $RUN_ENVIR = "emc" ]; then
#--------------------------------

##############################################
# Set variables used in the exglobal script
##############################################
export CDATE=${CDATE:-${PDY}${cyc}}
export CDUMP=${CDUMP:-${RUN:-"gfs"}}

#############################
# Source relevant config files
#############################
Expand All @@ -20,15 +26,6 @@ for config in $configs; do
status=$?
[[ $status -ne 0 ]] && exit $status
done
# Source additional configs
if [ ${DO_WAVE:-"NO"} = "YES" ]; then
configs="wave"
for config in $configs; do
. $config_path/config.$config
status=$?
[[ $status -ne 0 ]] && exit $status
done
fi

##########################################
# Source machine runtime environment
Expand Down Expand Up @@ -65,12 +62,6 @@ export pgmout="OUTPUT.${pid}"
export pgmerr=errfile


##############################################
# Set variables used in the exglobal script
##############################################
export CDATE=${CDATE:-${PDY}${cyc}}
export CDUMP=${CDUMP:-${RUN:-"gfs"}}

if [ $RUN_ENVIR = "nco" ]; then
export ROTDIR=${COMROOT:?}/$NET/$envir
export RSTDIR=${GESROOT:?}/$envir
Expand Down
Empty file added parm/config/config.aero
View file
Empty file.
82 changes: 23 additions & 59 deletions parm/config/config.fcst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,26 @@

echo "BEGIN: config.fcst"

# set -eu

# Source model specific information that is resolution dependent
. $EXPDIR/config.fv3 $CASE
[[ "$CDUMP" == "gfs" ]] && export nth_fv3=$nth_fv3_gfs

# Turn off waves if not used for this CDUMP
case $WAVE_CDUMP in
both | $CDUMP ) ;; # Don't change
*) DO_WAVE="NO" ;; # Turn waves off
esac

# Source component configs if necessary
for component in WAVE OCN ICE AERO; do
control="DO_${component}"
if [[ ${!control} == "YES" ]]; then
. $EXPDIR/config.$(echo "$component" | awk '{ print tolower($1) }')
fi
done

# Get task specific resources
. $EXPDIR/config.resources fcst

Expand Down Expand Up @@ -57,58 +73,13 @@ fi
# COUPLING COMPONENTS
export use_coldstart=".false."

#######################################################################
# CICE parameters
export NX_GLB="1440"
export NY_GLB="1080"
export MESHICE="mesh.mx025.nc"

#######################################################################

export FORECASTSH="$HOMEgfs/scripts/exglobal_forecast.sh"
export FCSTEXECDIR="$HOMEgfs/exec"
export FCSTEXEC="ufs_model"

export npe_fv3=$npe_fcst

if [[ "$CDUMP" == "gfs" ]] ; then
export npe_fv3=$npe_fcst_gfs
export layout_x=$layout_x_gfs
export layout_y=$layout_y_gfs
export WRITE_GROUP=$WRITE_GROUP_GFS
export WRTTASK_PER_GROUP=$WRTTASK_PER_GROUP_GFS
fi

if [ $QUILTING = ".true." ]; then
export npe_fv3=$(echo " $npe_fv3 + $WRITE_GROUP * $WRTTASK_PER_GROUP" | bc)
export npe_fcst=$npe_fv3
export npe_fcst_gfs=$(echo " $npe_fcst_gfs + $WRITE_GROUP_GFS * $WRTTASK_PER_GROUP_GFS" | bc)
export ATMPETS=$(echo " $layout_x * $layout_y * 6 + $WRITE_GROUP * $WRTTASK_PER_GROUP" | bc)
else
export ATMPETS=$(echo " $layout_x * $layout_y * 6" |bc)
fi

if [ $DO_WAVE = "YES" ] ; then
export npe_fcst=$((npe_fcst + npe_wav))
if [ "$WAVE_CDUMP" = "gfs" -o "$WAVE_CDUMP" = "both" ]; then
export npe_fcst_gfs=$((npe_fcst_gfs + npe_wav_gfs))
if [ "$CDUMP" = "gfs" ]; then npe_wav=$npe_wav_gfs ; fi
fi
fi

# Additional PE Mapping
export MEDPETS=$(echo " $layout_x * $layout_y * 6" |bc)
export OCNPETS="220"
export ICEPETS="80"
export WAVPETS="160"
export CHMPETS="1700"

if [ "$CDUMP" = "gfs" ]; then
export ntasks_fcst=$npe_fcst_gfs
else
export ntasks_fcst=$npe_fcst
fi

#######################################################################
# Model configuration
export TYPE="nh"
Expand All @@ -120,11 +91,7 @@ export h2o_phys=".true."
# Options of stratosphere O3 physics reaction coefficients
export new_o3forc="YES"

# do_ugwp=T: use unified CGWD and OGWD, and turbulent orographic form drag (TOFD)
# do_ugwp=F: use unified CGWD but old OGWD, TOFD is not uded.
export do_ugwp=".false."
export do_tofd=".true."
export launch_level=$(echo "$LEVS/2.35" |bc)
# export launch_level=$(echo "$LEVS/2.35" |bc)

# Sponge layer settings
export tau=10.0
Expand Down Expand Up @@ -216,6 +183,12 @@ else

fi

# Stochastic physics
DO_SPPT=${DO_SPPT:-"NO"}
DO_SKEB=${DO_SKEB:-"NO"}
DO_SHUM=${DO_SHUM:-"NO"}
DO_CA=${DO_CA:-"NO"}

#coupling settings
export FRAC_GRID=".true."
export cplmode="nems_frac"
Expand Down Expand Up @@ -352,15 +325,6 @@ elif [[ $DO_COUPLED = "YES" ]] ; then # coupled model
export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_cpl"
fi

case "$CASE" in
"C48") export OCNRES=100;;
"C96") export OCNRES=100;;
"C192") export OCNRES=050;;
"C384") export OCNRES=025;;
"C768") export OCNRES=025;;
*) export OCNRES=025;;
esac

# Remember config.efcs will over-ride these values for ensemble forecasts
# if these variables are re-defined there.
# Otherwise, the ensemble forecast will inherit from config.fcst
Expand Down
14 changes: 0 additions & 14 deletions parm/config/config.fv3
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,6 @@ case $case_in in
export layout_y=2
export layout_x_gfs=3
export layout_y_gfs=2
export npe_wav=16
export npe_wav_gfs=16
export nth_fv3=1
export nth_fv3_gfs=1
export cdmbgwd="0.071,2.1,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
Expand All @@ -64,8 +62,6 @@ case $case_in in
export layout_y=4
export layout_x_gfs=6
export layout_y_gfs=4
export npe_wav=40
export npe_wav_gfs=40
export nth_fv3=1
export nth_fv3_gfs=1
export cdmbgwd="0.14,1.8,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
Expand All @@ -82,8 +78,6 @@ case $case_in in
export layout_y=6
export layout_x_gfs=4
export layout_y_gfs=6
export npe_wav=140
export npe_wav_gfs=140
export nth_fv3=2
export nth_fv3_gfs=2
export cdmbgwd="0.23,1.5,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
Expand All @@ -99,8 +93,6 @@ case $case_in in
export layout_y=8
export layout_x_gfs=6
export layout_y_gfs=8
export npe_wav=140
export npe_wav_gfs=140
export nth_fv3=2
export nth_fv3_gfs=2
export cdmbgwd="1.1,0.72,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
Expand All @@ -116,8 +108,6 @@ case $case_in in
export layout_y=12
export layout_x_gfs=16
export layout_y_gfs=12
export npe_wav=140
export npe_wav_gfs=140
export nth_fv3=4
export nth_fv3_gfs=4
export cdmbgwd="4.0,0.15,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
Expand All @@ -133,8 +123,6 @@ case $case_in in
export layout_y=16
export layout_x_gfs=8
export layout_y_gfs=16
export npe_wav=140
export npe_wav_gfs=140
export nth_fv3=4
export nth_fv3_gfs=4
export cdmbgwd="4.0,0.10,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
Expand All @@ -150,8 +138,6 @@ case $case_in in
export layout_y=32
export layout_x_gfs=16
export layout_y_gfs=32
export npe_wav=140
export npe_wav_gfs=140
export nth_fv3=4
export nth_fv3_gfs=4
export cdmbgwd="4.0,0.05,1.0,1.0" # mountain blocking, ogwd, cgwd, cgwd src scaling
Expand Down
5 changes: 5 additions & 0 deletions parm/config/config.ice
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/ksh -x

export NX_GLB="1440"
export NY_GLB="1080"
export MESHICE="mesh.mx025.nc"
10 changes: 10 additions & 0 deletions parm/config/config.ocn
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/ksh -x

case "$CASE" in
"C48") export OCNRES=100;;
"C96") export OCNRES=100;;
"C192") export OCNRES=050;;
"C384") export OCNRES=025;;
"C768") export OCNRES=025;;
*) export OCNRES=025;;
esac
87 changes: 81 additions & 6 deletions parm/config/config.resources
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,12 +181,87 @@ elif [ $step = "fcst" ]; then
else
export wtime_fcst_gfs="03:00:00"
fi
export npe_fcst=$(echo "$layout_x * $layout_y * 6" | bc)
export npe_fcst_gfs=$(echo "$layout_x_gfs * $layout_y_gfs * 6" | bc)
export nth_fcst=${nth_fv3:-2}
if [[ "$CASE" == "C384" ]]; then nth_fcst=4; fi
export npe_node_fcst=$(echo "$npe_node_max / $nth_fcst" | bc)
if [[ "$machine" == "WCOSS_C" ]]; then export memory_fcst="1024M"; fi

# During workflow creation, we need resources for all CDUMPs and CDUMP is undefined
CDUMP_LIST=${CDUMP:-"gfs gdas"}
for CDUMP in $CDUMP_LIST; do
if [[ "$CDUMP" == "gfs" ]]; then
suffix="_gfs"
export layout_x=$layout_x_gfs
export layout_y=$layout_y_gfs
export WRITE_GROUP=$WRITE_GROUP_GFS
export WRTTASK_PER_GROUP=$WRTTASK_PER_GROUP_GFS
fi

(( ATMPETS = layout_x * layout_y * 6 ))

# Mediator only uses the atm model PETS
export MEDPETS=$ATMPETS

# If using in-line post, add the write tasks to the ATMPETS
if [[ $QUILTING == ".true." ]]; then
(( ATMPETS = ATMPETS + WRITE_GROUP * WRTTASK_PER_GROUP ))
fi
export ATMPETS
NTASKS_TOT=$ATMPETS

export nth_fcst=${nth_fv3:-2}

if [[ "$CASE" == "C384" ]]; then nth_fcst=4; fi
export npe_node_fcst=$(echo "$npe_node_max / $nth_fcst" | bc)
if [[ "$machine" == "WCOSS_C" ]]; then export memory_fcst="1024M"; fi

if [[ $DO_WAVE == "YES" ]]; then
case $waveGRD in
'gnh_10m aoc_9km gsh_15m') export WAVPETS=140 ;;
'gwes_30m') export WAVPETS=160 ;;
*)
echo "FATAL: Number of PEs not defined for wave grid '$waveGRD'"
echo " Please add an entry to config.resources within fcst for this grid"
exit 3
esac
(( NTASKS_TOT = NTASKS_TOT + WAVPETS ))
fi

if [[ $DO_OCN == "YES" ]]; then
case $OCNRES in
# Except for 025, these are guesses for now
100) export OCNPETS=20 ;;
050) export OCNPETS=60 ;;
025) export OCNPETS=225 ;;
*)
echo "FATAL: Number of PEs not defined for ocean resolution '$OCNRES'"
echo " Please add an entry to config.resources within fcst for this resolution"
exit 3
esac
(( NTASKS_TOT = NTASKS_TOT + OCNPETS ))
fi

if [[ $DO_ICE == "YES" ]]; then
case $ICERES in
# Except for 025, these are guesses for now
100) export ICEPETS=10 ;;
050) export ICEPETS=30 ;;
025) export ICEPETS=80 ;;
*)
echo "FATAL: Number of PEs not defined for ice resolution '$ICERES'"
echo " Please add an entry to config.resources within fcst for this resolution"
exit 3
esac
(( NTASKS_TOT = NTASKS_TOT + ICEPETS ))
fi

if [[ $DO_AERO == "YES" ]]; then
# Aerosol model runs on same PETS as ATM model
export CHMPETS=$MEDPETS
fi

if [[ $CDUMP == "gfs" ]]; then
export npe_fcst_gfs=$NTASKS_TOT
else
export npe_fcst=$NTASKS_TOT
fi
done

elif [ $step = "ocnpost" ]; then

Expand Down

0 comments on commit 107e99f

Please sign in to comment.