WCOSS2 run support and script/config updates (#1030)

- WCOSS2 port changes requested by NCO in the move of operations to WCOSS2; mainly within scripts.
- Adding WCOSS2 run support by creating WCOSS2.env, wcoss2.yaml hosts file for rocoto, and config updates.
- Also some cleanup that was done in the GFS operations port to WCOSS2.

Refs #419

env/HERA.env

@@ -86,7 +86,7 @@ elif [ $step = "aeroanlrun" ]; then
     [[ $NTHREADS_AEROANL -gt $nth_max ]] && export NTHREADS_AEROANL=$nth_max
     export APRUN_AEROANL="$launcher -n $npe_aeroanlrun"
 
-elif [ $step = "anal" ]; then
+elif [ $step = "anal" -o $step = "analcalc" ]; then
 
     export MKL_NUM_THREADS=4
     export MKL_CBWR=AUTO

env/ORION.env

@@ -85,7 +85,7 @@ elif [ $step = "aeroanlrun" ]; then
     [[ $NTHREADS_AEROANL -gt $nth_max ]] && export NTHREADS_AEROANL=$nth_max
     export APRUN_AEROANL="$launcher -n $npe_aeroanlrun"
 
-elif [ $step = "anal" ]; then
+elif [ $step = "anal" -o $step = "analcalc" ]; then
 
     export MKL_NUM_THREADS=4
     export MKL_CBWR=AUTO

env/WCOSS2.env

@@ -0,0 +1,319 @@
+#! /usr/bin/env bash
+
+if [ $# -ne 1 ]; then
+
+    echo "Must specify an input argument to set runtime environment variables!"
+    echo "argument can be any one of the following:"
+    echo "atmanalrun atmensanalrun"
+    echo "aeroanlrun"
+    echo "anal sfcanl fcst post vrfy metp"
+    echo "eobs eupd ecen esfc efcs epos"
+    echo "postsnd awips gempak"
+    exit 1
+
+fi
+
+step=$1
+
+# WCOSS2 information
+export launcher="mpiexec -l"
+export mpmd="--cpu-bind verbose,core cfp"
+
+export npe_node_max=128
+
+export job=${PBS_JOBNAME:-$step}
+export jobid=${job}.${PBS_JOBID:-$$}
+
+if [ $step = "prep" -o $step = "prepbufr" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_prep))
+
+    export POE=${POE:-"YES"}
+    export BACK=${BACK:-"off"}
+    export sys_tp="wcoss2"
+    export launcher_PREP="mpiexec"
+
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" -o $step = "wavepostbndpnt" -o $step = "wavepostbndpntbll"  -o $step = "wavepostpnt" ]; then
+
+    if [ $step = "waveprep" -a $CDUMP = "gfs" ]; then NTASKS=$NTASKS_gfs ; fi
+    export wavempexec="$launcher -np"
+    export wave_mpmd=${mpmd}
+
+elif [ $step = "atmanalrun" ]; then
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="$launcher -np \$ncmd $mpmd"
+
+    nth_max=$(($npe_node_max / $npe_node_atmanalrun))
+
+    export NTHREADS_ATMANAL=${nth_atmanalrun:-$nth_max}
+    [[ $NTHREADS_ATMANAL -gt $nth_max ]] && export NTHREADS_ATMANAL=$nth_max
+    export APRUN_ATMANAL="$launcher -n $npe_atmanalrun"
+
+elif [ $step = "atmensanalrun" ]; then
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="$launcher -np \$ncmd $mpmd"
+
+    nth_max=$(($npe_node_max / $npe_node_atmensanalrun))
+
+    export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-$nth_max}
+    [[ $NTHREADS_ATMENSANAL -gt $nth_max ]] && export NTHREADS_ATMENSANAL=$nth_max
+    export APRUN_ATMENSANAL="$launcher -n $npe_atmensanalrun"
+
+elif [ $step = "aeroanlrun" ]; then
+
+    export APRUNCFP="$launcher -np \$ncmd $mpmd"
+
+    nth_max=$(($npe_node_max / $npe_node_aeroanlrun))
+
+    export NTHREADS_AEROANL=${nth_aeroanlrun:-$nth_max}
+    [[ $NTHREADS_AEROANL -gt $nth_max ]] && export NTHREADS_AEROANL=$nth_max
+    export APRUN_AEROANL="$launcher -n $npe_aeroanlrun"
+
+elif [ $step = "anal" -o $step = "analcalc" ]; then
+
+    export OMP_PLACES=cores
+    export OMP_STACKSIZE=1G
+    export FI_OFI_RXM_SAR_LIMIT=3145728
+
+    if [ $step = "analcalc" ]; then
+      export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
+    fi
+
+    nth_max=$(($npe_node_max / $npe_node_anal))
+
+    export NTHREADS_GSI=${nth_anal:-$nth_max}
+    [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max
+    export APRUN_GSI="$launcher -n ${npe_gsi:-$npe_anal} -ppn $npe_node_anal --cpu-bind depth --depth $NTHREADS_GSI"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max
+    export APRUN_CALCINC="$launcher \$ncmd"
+
+    export NTHREADS_CYCLE=${nth_cycle:-14}
+    [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max
+    npe_cycle=${ntiles:-6}
+    export APRUN_CYCLE="$launcher -n $npe_cycle -ppn $npe_node_cycle --cpu-bind depth --depth $NTHREADS_CYCLE"
+
+    export NTHREADS_GAUSFCANL=1
+    npe_gausfcanl=${npe_gausfcanl:-1}
+    export APRUN_GAUSFCANL="$launcher -n $npe_gausfcanl"
+
+    export NTHREADS_CHGRES=${nth_echgres:-14}
+    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
+    export APRUN_CHGRES=""
+
+    export CFP_MP=${CFP_MP:-"NO"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="$launcher -np \$ncmd $mpmd"
+
+elif [ $step = "sfcanl" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_sfcanl))
+
+    export NTHREADS_CYCLE=${nth_sfcanl:-14}
+    [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max
+    npe_sfcanl=${ntiles:-6}
+    export APRUN_CYCLE="$launcher -n $npe_sfcanl"
+
+elif [ $step = "gldas" ]; then
+
+    export USE_CFP="YES"
+    export CFP_MP="NO"
+
+    nth_max=$(($npe_node_max / $npe_node_gldas))
+
+    export NTHREADS_GLDAS=${nth_gldas:-$nth_max}
+    [[ $NTHREADS_GLDAS -gt $nth_max ]] && export NTHREADS_GLDAS=$nth_max
+    export APRUN_GLDAS="$launcher -n $npe_gldas -ppn $npe_node_gldas --cpu-bind depth --depth $NTHREADS_GLDAS"
+
+    export NTHREADS_GAUSSIAN=${nth_gaussian:-1}
+    [[ $NTHREADS_GAUSSIAN -gt $nth_max ]] && export NTHREADS_GAUSSIAN=$nth_max
+    export APRUN_GAUSSIAN="$launcher -n $npe_gaussian -ppn $npe_node_gaussian --cpu-bind depth --depth $NTHREADS_GAUSSIAN"
+
+    # Must run data processing with exactly the number of tasks as time
+    # periods being processed.
+    export USE_CFP=${USE_CFP:-"YES"}
+    npe_gldas_data_proc=$(($gldas_spinup_hours + 12))
+    export APRUN_GLDAS_DATA_PROC="$launcher -np $npe_gldas_data_proc $mpmd"
+
+elif [ $step = "eobs" ]; then
+
+    export OMP_PLACES=cores
+    export OMP_STACKSIZE=1G
+    export FI_OFI_RXM_SAR_LIMIT=3145728
+
+    nth_max=$(($npe_node_max / $npe_node_eobs))
+
+    export NTHREADS_GSI=${nth_eobs:-$nth_max}
+    [[ $NTHREADS_GSI -gt $nth_max ]] && export NTHREADS_GSI=$nth_max
+    export APRUN_GSI="$launcher -n ${npe_gsi:-$npe_eobs} -ppn $npe_node_eobs --cpu-bind depth --depth $NTHREADS_GSI"
+
+    export CFP_MP=${CFP_MP:-"NO"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="$launcher -np \$ncmd $mpmd"
+
+elif [ $step = "eupd" ]; then
+
+    export OMP_PLACES=cores
+    export OMP_STACKSIZE=2G
+    export MPICH_COLL_OPT_OFF=1
+    export FI_OFI_RXM_SAR_LIMIT=3145728
+
+    nth_max=$(($npe_node_max / $npe_node_eupd))
+
+    export NTHREADS_ENKF=${nth_eupd:-$nth_max}
+    [[ $NTHREADS_ENKF -gt $nth_max ]] && export NTHREADS_ENKF=$nth_max
+    export APRUN_ENKF="$launcher -n ${npe_enkf:-$npe_eupd} -ppn $npe_node_eupd --cpu-bind depth --depth $NTHREADS_ENKF"
+
+    export CFP_MP=${CFP_MP:-"NO"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="$launcher -np \$ncmd $mpmd"
+
+elif [ $step = "fcst" ]; then
+
+    export OMP_PLACES=cores
+    export OMP_STACKSIZE=2048M
+    export FI_OFI_RXM_RX_SIZE=40000
+    export FI_OFI_RXM_TX_SIZE=40000
+
+    #PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs
+    if [[ $CDUMP == "gfs" ]]; then
+        npe_fcst=$npe_fcst_gfs
+        npe_node_fcst=$npe_node_fcst_gfs
+        nth_fv3=$nth_fv3_gfs
+    fi
+
+    nth_max=$(($npe_node_max / $npe_node_fcst))
+
+    export NTHREADS_FV3=${nth_fv3:-$nth_max}
+    [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
+    export cores_per_node=$npe_node_max
+    export APRUN_FV3="$launcher -n $npe_fcst -ppn $npe_node_fcst --cpu-bind depth --depth $NTHREADS_FV3"
+
+    export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
+    [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max
+    export APRUN_REGRID_NEMSIO="$launcher -n $LEVS"
+
+    export NTHREADS_REMAP=${nth_remap:-2}
+    [[ $NTHREADS_REMAP -gt $nth_max ]] && export NTHREADS_REMAP=$nth_max
+    export APRUN_REMAP="$launcher -n ${npe_remap:-$npe_fcst}"
+    export I_MPI_DAPL_UD="enable"
+
+elif [ $step = "efcs" ]; then
+
+    export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
+    export FI_OFI_RXM_SAR_LIMIT=3145728
+    export FI_OFI_RXM_RX_SIZE=40000
+    export FI_OFI_RXM_TX_SIZE=40000
+
+    nth_max=$(($npe_node_max / $npe_node_efcs))
+
+    export NTHREADS_FV3=${nth_efcs:-$nth_max}
+    [[ $NTHREADS_FV3 -gt $nth_max ]] && export NTHREADS_FV3=$nth_max
+    export cores_per_node=$npe_node_max
+    export APRUN_FV3="$launcher -n ${npe_fv3:-$npe_efcs} -ppn $npe_node_efcs --cpu-bind depth --depth $NTHREADS_FV3"
+
+    export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
+    [[ $NTHREADS_REGRID_NEMSIO -gt $nth_max ]] && export NTHREADS_REGRID_NEMSIO=$nth_max
+    export APRUN_REGRID_NEMSIO="$launcher -n $LEVS"
+
+elif [ $step = "post" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_post))
+
+    export NTHREADS_NP=${nth_np:-1}
+    [[ $NTHREADS_NP -gt $nth_max ]] && export NTHREADS_NP=$nth_max
+    export APRUN_NP="$launcher -n ${npe_np:-$npe_post} -ppn $npe_node_post --cpu-bind depth --depth $NTHREADS_NP"
+
+    export NTHREADS_DWN=${nth_dwn:-1}
+    [[ $NTHREADS_DWN -gt $nth_max ]] && export NTHREADS_DWN=$nth_max
+    export APRUN_DWN="$launcher -np $npe_dwn $mpmd"
+
+elif [ $step = "ecen" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_ecen))
+
+    export NTHREADS_ECEN=${nth_ecen:-$nth_max}
+    [[ $NTHREADS_ECEN -gt $nth_max ]] && export NTHREADS_ECEN=$nth_max
+    export APRUN_ECEN="$launcher -n $npe_ecen -ppn $npe_node_ecen --cpu-bind depth --depth $NTHREADS_ECEN"
+
+    export NTHREADS_CHGRES=${nth_chgres:-14}
+    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ $NTHREADS_CALCINC -gt $nth_max ]] && export NTHREADS_CALCINC=$nth_max
+    export APRUN_CALCINC="$launcher -n $npe_ecen"
+
+    export NTHREADS_CYCLE=${nth_cycle:-14}
+    [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max
+    export APRUN_CYCLE="$launcher -n $npe_ecen -ppn $npe_node_cycle --cpu-bind depth --depth $NTHREADS_CYCLE"
+
+elif [ $step = "esfc" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_esfc))
+
+    export NTHREADS_ESFC=${nth_esfc:-$nth_max}
+    [[ $NTHREADS_ESFC -gt $nth_max ]] && export NTHREADS_ESFC=$nth_max
+    export APRUN_ESFC="$launcher -n $npe_esfc -ppn $npe_node_esfc --cpu-bind depth --depth $NTHREADS_ESFC"
+
+    export NTHREADS_CYCLE=${nth_cycle:-14}
+    [[ $NTHREADS_CYCLE -gt $npe_node_max ]] && export NTHREADS_CYCLE=$npe_node_max
+    export APRUN_CYCLE="$launcher -n $npe_esfc -ppn $npe_node_cycle --cpu-bind depth --depth $NTHREADS_CYCLE"
+
+elif [ $step = "epos" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_epos))
+
+    export NTHREADS_EPOS=${nth_epos:-$nth_max}
+    [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max
+    export APRUN_EPOS="$launcher -n $npe_epos -ppn $npe_node_epos --cpu-bind depth --depth $NTHREADS_EPOS"
+
+elif [ $step = "init" ]; then
+
+    export APRUN="$launcher"
+
+elif [ $step = "postsnd" ]; then
+
+    export MPICH_MPIIO_HINTS_DISPLAY=1
+    export OMP_NUM_THREADS=1
+
+    nth_max=$(($npe_node_max / $npe_node_postsnd))
+
+    export NTHREADS_POSTSND=${nth_postsnd:-1}
+    [[ $NTHREADS_POSTSND -gt $nth_max ]] && export NTHREADS_POSTSND=$nth_max
+    export APRUN_POSTSND="$launcher -n $npe_postsnd --depth=$NTHREADS_POSTSND --cpu-bind depth"
+
+    export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
+    [[ $NTHREADS_POSTSNDCFP -gt $nth_max ]] && export NTHREADS_POSTSNDCFP=$nth_max
+    export APRUN_POSTSNDCFP="$launcher -np $npe_postsndcfp $mpmd"
+
+elif [ $step = "awips" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_awips))
+
+    export NTHREADS_AWIPS=${nth_awips:-2}
+    [[ $NTHREADS_AWIPS -gt $nth_max ]] && export NTHREADS_AWIPS=$nth_max
+    export APRUN_AWIPSCFP="$launcher -np $npe_awips $mpmd"
+
+elif [ $step = "gempak" ]; then
+
+    nth_max=$(($npe_node_max / $npe_node_gempak))
+
+    export NTHREADS_GEMPAK=${nth_gempak:-1}
+    [[ $NTHREADS_GEMPAK -gt $nth_max ]] && export NTHREADS_GEMPAK=$nth_max
+    export APRUN_GEMPAKCFP="$launcher -np $npe_gempak $mpmd"
+
+elif [ $step = "waveawipsbulls" ]; then
+
+    unset PERL5LIB
+
+elif [ $step = "wafsgrib2" -o $step = "wafsgrib20p25" ]; then
+
+    export USE_CFP=${USE_CFP:-"YES"}
+
+fi

jobs/JGDAS_ATMOS_ANALYSIS_DIAG

@@ -9,7 +9,7 @@ export RUN_ENVIR=${RUN_ENVIR:-"nco"}
 #############################
 export EXPDIR=${EXPDIR:-$HOMEgfs/parm/config}
 configs="base anal analdiag"
-config_path=${EXPDIR:-$NWROOT/gfs.${gfs_ver}/parm/config}
+config_path=${EXPDIR:-$PACKAGEROOT/gfs.${gfs_ver}/parm/config}
 for config in $configs; do
     . $config_path/config.$config
     status=$?

jobs/JGDAS_ATMOS_CHGRES_FORENKF

@@ -9,7 +9,7 @@ export RUN_ENVIR=${RUN_ENVIR:-"nco"}
 #############################
 export EXPDIR=${EXPDIR:-$HOMEgfs/parm/config}
 configs="base anal echgres"
-config_path=${EXPDIR:-$NWROOT/gfs.${gfs_ver}/parm/config}
+config_path=${EXPDIR:-$PACKAGEROOT/gfs.${gfs_ver}/parm/config}
 for config in $configs; do
     . $config_path/config.$config
     status=$?