Skip to content

Commit

Permalink
Add dependence to GSI analysis to wait for RRFSE ensemble forecast at…
Browse files Browse the repository at this point in the history
… certain cycles. (NOAA-EMC#384)
  • Loading branch information
hu5970 committed May 12, 2022
1 parent 9b8862f commit 8c8f9fc
Show file tree
Hide file tree
Showing 7 changed files with 277 additions and 11 deletions.
4 changes: 2 additions & 2 deletions scripts/exregional_run_analysis.sh
Original file line number Diff line number Diff line change
Expand Up @@ -223,8 +223,8 @@ if [[ ${regional_ensemble_option:-1} -eq 5 ]]; then
dynvarfile=${bkpathmem}/${restart_prefix}fv_core.res.tile1.nc
tracerfile=${bkpathmem}/${restart_prefix}fv_tracer.res.tile1.nc
if [ -r "${dynvarfile}" ] && [ -r "${tracerfile}" ] ; then
ln_vrfy ${bkpathmem}/${restart_prefix}fv_core.res.tile1.nc fv3SAR${ens_nstarthr}_ens_mem${memcharv0}-fv3_dynvars
ln_vrfy ${bkpathmem}/${restart_prefix}fv_tracer.res.tile1.nc fv3SAR${ens_nstarthr}_ens_mem${memcharv0}-fv3_tracer
ln_vrfy -snf ${bkpathmem}/${restart_prefix}fv_core.res.tile1.nc fv3SAR${ens_nstarthr}_ens_mem${memcharv0}-fv3_dynvars
ln_vrfy -snf ${bkpathmem}/${restart_prefix}fv_tracer.res.tile1.nc fv3SAR${ens_nstarthr}_ens_mem${memcharv0}-fv3_tracer
(( ifound += 1 ))
else
print_info_msg "Error: cannot find ensemble files: ${dynvarfile} ${tracerfile} "
Expand Down
216 changes: 216 additions & 0 deletions ush/config.sh.RRFS_CONUS_3km.2022HWT
Original file line number Diff line number Diff line change
@@ -0,0 +1,216 @@
MACHINE="jet"
ACCOUNT="nrtrr"
#RESERVATION="rrfsdet"
EXPT_BASEDIR="YourOwnSpace"
EXPT_SUBDIR="RRFS_CONUS_3km"

if [[ -n $RESERVATION ]] ; then
ACCOUNT=rtrr
SERVICE_ACCOUNT=rtrr
PARTITION_DEFAULT=kjet
PARTITION_FCST=kjet
PARTITION_GRAPHICS=kjet
PARTITION_ANALYSIS=kjet
PARTITION_WGRIB2=kjet

QUEUE_DEFAULT="rth"
QUEUE_FCST="rth"
QUEUE_ANALYSIS="rth"
QUEUE_WGRIB2="rt"
QUEUE_GRAPHICS="rt"
QUEUE_HPSS="rt"
fi

if [[ $MACHINE == "hera" ]] ; then
ACCOUNT="wrfruc"
PARTITION_DEFAULT=""
PARTITION_FCST=""
QUEUE_ANALYSIS="batch"
QUEUE_WGRIB2="batch"
QUEUE_GRAPHICS="batch"
fi

if [[ $MACHINE == "orion" ]] ; then
ACCOUNT=ztrr
QUEUE_ANALYSIS="batch"
fi

if [[ $MACHINE == "wcoss_dell_p3" ]] ; then
ACCOUNT=HRRR-T2O
QUEUE_DEFAULT="dev"
QUEUE_ANALYSIS="dev"
QUEUE_FCST="dev"
QUEUE_HPSS="dev"
QUEUE_WGRIB2="dev"
QUEUE_GRAPHICS="dev"
fi

VERBOSE="TRUE"

RUN_ENVIR="nco"

PREDEF_GRID_NAME=RRFS_CONUS_3km
ADDNL_OUTPUT_GRIDS=()

TILE_LABELS="CONUS REGIONS"
TILE_SETS="full NE,NC,NW,SE,SC,SW"

DO_DACYCLE="TRUE"
DO_SURFACE_CYCLE="TRUE"
DO_SPINUP="TRUE"
DO_RETRO="FALSE"
DO_NONVAR_CLDANAL="TRUE"
DO_REFL2TTEN="FALSE"
RADARREFL_TIMELEVEL=(0)
FH_DFI_RADAR="0.0,0.25,0.5"
DO_SOIL_ADJUST="TRUE"
DO_RADDA="FALSE"

QUILTING="TRUE"
CCPP_PHYS_SUITE="FV3_HRRR"

EXTRN_MDL_ICS_OFFSET_HRS="3"
LBC_SPEC_INTVL_HRS="1"
EXTRN_MDL_LBCS_OFFSET_HRS="6"
BOUNDARY_LEN_HRS="48"

# avaialble retro period:
# 20210511-20210531; 20210718-20210801
DATE_FIRST_CYCL="20220427"
DATE_LAST_CYCL="20220630"
CYCL_HRS=( "00" "12" )
CYCL_HRS_SPINSTART=("03" "15")
CYCL_HRS_PRODSTART=("09" "21")
CYCLEMONTH="4,5,6"
CYCLEDAY="*"

PREEXISTING_DIR_METHOD="upgrade" # "rename"
INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}0300 ${DATE_LAST_CYCL}2300 12:00:00"
BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
PROD_CYCLEDEF="${DATE_FIRST_CYCL}0900 ${DATE_LAST_CYCL}2300 01:00:00"
POSTPROC_CYCLEDEF="${DATE_FIRST_CYCL}0900 ${DATE_LAST_CYCL}2300 01:00:00"
POSTPROC_LONG_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 12:00:00"
#ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}0700 ${DATE_LAST_CYCL}2300 24:00:00"
if [[ $DO_SPINUP == "TRUE" ]] ; then
SPINUP_CYCLEDEF="00 03-08,15-20 ${CYCLEDAY} ${CYCLEMONTH} 2022 *"
fi

FCST_LEN_HRS="3"
FCST_LEN_HRS_SPINUP="1"
POSTPROC_LEN_HRS="3"
POSTPROC_LONG_LEN_HRS="36"
#FCST_LEN_HRS_CYCLES=(48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18)
for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=3; done
for i in {0..23..12}; do FCST_LEN_HRS_CYCLES[$i]=36; done
DA_CYCLE_INTERV="1"
RESTART_INTERVAL="1 2 3"

USE_RRFSE_ENS="TRUE"
CYCL_HRS_HYB_FV3LAM_ENS=("19" "20" "21" "22" "23" "00")

SST_update_hour=01
GVF_update_hour=04
SNOWICE_update_hour=00

DT_ATMOS=60
NCORES_RUN_ANAL=360
NCORES_RUN_OBSERVER=80
HYBENSMEM_NMIN=66
HALO_BLEND=20
IO_LAYOUT_Y=1
PPN_RUN_REF2TTEN=1
PPN_RUN_NONVARCLDANL=${IO_LAYOUT_Y}
regional_ensemble_option=5

MAXTRIES_MAKE_ICS="2"
MAXTRIES_MAKE_LBCS="2"
MAXTRIES_RUN_PREPSTART="2"
MAXTRIES_RUN_FCST="2"
MAXTRIES_ANAL_GSI="2"
MAXTRIES_CLDANL_NONVAR="2"

WTIME_RUN_FCST="03:45:00"
WTIME_MAKE_LBCS="04:00:00"
WTIME_RUN_ANAL="00:50:00"

START_TIME_SPINUP="02:10:00"
START_TIME_PROD="08:20:00"
START_TIME_LATE_ANALYSIS="08:40:00"

PRINT_DIFF_PGR="TRUE"

if [[ -n $RESERVATION ]] ; then
NNODES_MAKE_ICS="3"
PPN_MAKE_ICS="20"
NNODES_MAKE_LBCS="3"
PPN_MAKE_LBCS="20"
NNODES_RUN_POST="1"
PPN_RUN_POST="40"
fi

EXTRN_MDL_NAME_ICS="FV3GFS"
EXTRN_MDL_NAME_LBCS="FV3GFS"

FV3GFS_FILE_FMT_ICS="grib2"
FV3GFS_FILE_FMT_LBCS="grib2"

envir="para"

NET="RRFS_CONUS"
TAG="RRFS_CONUS_3km"

USE_CUSTOM_POST_CONFIG_FILE="TRUE"
TESTBED_FIELDS_FN="testbed_fields_bgdawp.txt"
CUSTOM_POST_CONFIG_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/postxconfig-NT-fv3lam_rrfs.txt"
CUSTOM_POST_PARAMS_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/params_grib2_tbl_new"
ARCHIVEDIR="/1year/BMC/wrfruc/rrfs_dev1"
NCARG_ROOT="/apps/ncl/6.5.0-CentOS6.10_64bit_nodap_gnu447"
NCL_HOME="/home/rtrr/RRFS/graphics"
NCL_REGION="conus"
MODEL="RRFS_conus_3km"

#
# In NCO mode, the following don't need to be explicitly set to "FALSE"
# in this configuration file because the experiment generation script
# will do this (along with printing out an informational message).
#
#RUN_TASK_MAKE_GRID="FALSE"
#RUN_TASK_MAKE_OROG="FALSE"
#RUN_TASK_MAKE_SFC_CLIMO="FALSE"

if [[ $MACHINE == "wcoss_dell_p3" ]] ; then
LAYOUT_X="42"
LAYOUT_Y="40"
PPN_MAKE_ICS="14"
PPN_MAKE_LBCS="14"
PPN_RUN_ANAL="28"
PPN_RUN_FCST="28"
PPN_RUN_POST="28"
FV3GFS_FILE_FMT_ICS="netcdf"
FV3GFS_FILE_FMT_LBCS="netcdf"
fi

RUN="RRFS_conus_3km"
COMINgfs=""

. set_rrfs_config.sh

STMP="/lfs4/BMC/nrtrr/NCO_dirs/stmp" # Path to directory STMP that mostly contains input files.
PTMP="/lfs4/BMC/nrtrr/NCO_dirs/ptmp" # Path to directory STMP that mostly contains input files.
NWGES="/lfs4/BMC/nrtrr/NCO_dirs/nwges" # Path to directory NWGES that save boundary, cold initial, restart files
if [[ ${regional_ensemble_option} == "5" ]]; then
RRFSE_NWGES="/lfs4/BMC/wrfruc/RRFSE/NCO_dirs/nwges" # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid.
NUM_ENS_MEMBERS=30 # FV3LAM ensemble size for GSI hybrid analysis
fi


if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then
NUM_ENS_MEMBERS=9
# DO_SPPT=TRUE
# SPPT_MAG=0.5
# DO_ENSCONTROL="TRUE"
DO_GSIOBSERVER="TRUE"
DO_ENKFUPDATE="TRUE"
ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens"
fi

19 changes: 19 additions & 0 deletions ush/config_defaults.sh
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,9 @@ WFLOW_LAUNCH_LOG_FN="log.launch_FV3LAM_wflow"
# POSTPROC_LONG_LEN_HRS:
# The length of long post process, in integer hours.
#
# CYCL_HRS_HYB_FV3LAM_ENS:
# An array containing the hours of the day at which the GSI hybrid using FV3LAM ensemeble.
#
#-----------------------------------------------------------------------
#
DATE_FIRST_CYCL="YYYYMMDD"
Expand All @@ -505,6 +508,7 @@ FCST_LEN_HRS_SPINUP="1"
FCST_LEN_HRS_CYCLES=( )
DA_CYCLE_INTERV="3"
RESTART_INTERVAL="3,6"
CYCL_HRS_HYB_FV3LAM_ENS=( "99" )

#-----------------------------------------------------------------------
#
Expand Down Expand Up @@ -1620,6 +1624,17 @@ WTIME_RUN_REF2TTEN="00:20:00"
WTIME_RUN_NONVARCLDANL="00:20:00"
WTIME_RUN_BUFRSND="00:45:00"
WTIME_SAVE_RESTART="00:15:00"
#
# Start times.
#
START_TIME_SPINUP="01:10:00"
START_TIME_PROD="02:20:00"
START_TIME_CONVENTIONAL_SPINUP="00:40:00"
START_TIME_LATE_ANALYSIS="01:40:00"
START_TIME_CONVENTIONAL="00:40:00"
START_TIME_NSSLMOSIAC="00:45:00"
START_TIME_LIGHTNINGNC="00:45:00"

#
# Memory.
#
Expand Down Expand Up @@ -1821,6 +1836,9 @@ DO_ENS_GRAPHICS="FALSE"
#
# DO_BUFRSND:
# Decide whether or not to run EMC BUFR sounding
#
# USE_RRFSE_ENS:
# Use rrfse ensemble for hybrid analysis
#-----------------------------------------------------------------------
DO_DACYCLE="FALSE"
DO_SURFACE_CYCLE="FALSE"
Expand All @@ -1829,6 +1847,7 @@ DO_SOIL_ADJUST="FALSE"
DO_UPDATE_BC="FALSE"
DO_RADDA="FALSE"
DO_BUFRSND="FALSE"
USE_RRFSE_ENS="FALSE"
#
#-----------------------------------------------------------------------
#
Expand Down
12 changes: 12 additions & 0 deletions ush/generate_FV3LAM_wflow.sh
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,16 @@ settings="\
'wtime_run_bufrsnd': ${WTIME_RUN_BUFRSND}
'wtime_save_restart': ${WTIME_SAVE_RESTART}
#
# start time for each task.
#
'start_time_spinup': ${START_TIME_SPINUP}
'start_time_prod': ${START_TIME_PROD}
'start_time_conventional_spinup': ${START_TIME_CONVENTIONAL_SPINUP}
'start_time_late_analysis': ${START_TIME_LATE_ANALYSIS}
'start_time_conventional': ${START_TIME_CONVENTIONAL}
'start_time_nsslmosiac': ${START_TIME_NSSLMOSIAC}
'start_time_lightningnc': ${START_TIME_LIGHTNINGNC}
#
# Maximum memory for each task.
#
'memo_run_processbufr': ${MEMO_RUN_PROCESSBUFR}
Expand Down Expand Up @@ -386,6 +396,7 @@ settings="\
'cycl_hrs': [ $( printf "\'%s\', " "${CYCL_HRS[@]}" ) ]
'cycl_hrs_spinstart': [ $( printf "\'%s\', " "${CYCL_HRS_SPINSTART[@]}" ) ]
'cycl_hrs_prodstart': [ $( printf "\'%s\', " "${CYCL_HRS_PRODSTART[@]}" ) ]
'cycl_hrs_hyb_fv3lam_ens': [ $( printf "\'%s\', " "${CYCL_HRS_HYB_FV3LAM_ENS[@]}" ) ]
'restart_hrs_prod': ${RESTART_INTERVAL}
'cycl_freq': !!str 12:00:00
'at_start_cycledef': ${AT_START_CYCLEDEF}
Expand Down Expand Up @@ -433,6 +444,7 @@ settings="\
'regional_ensemble_option': ${regional_ensemble_option}
'radar_ref_thinning': ${RADAR_REF_THINNING}
'ensctrl_stmp': ${ENSCTRL_STMP}
'use_rrfse_ens': ${USE_RRFSE_ENS}
#
# graphics related parameters
#
Expand Down
1 change: 1 addition & 0 deletions ush/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,7 @@ optionList[8]=DO_UPDATE_BC
optionList[9]=DO_RADDA
optionList[10]=DO_RECENTER
optionList[11]=DO_BUFRSND
optionList[12]=USE_RRFSE_ENS

obs_number=${#optionList[@]}
for (( i=0; i<${obs_number}; i++ ));
Expand Down
35 changes: 26 additions & 9 deletions ush/templates/FV3LAM_wflow.xml
Original file line number Diff line number Diff line change
Expand Up @@ -162,13 +162,13 @@ tasks; and the "FCST" type is used for the RUN_FCST_TN task.
<!ENTITY DEADLINE_SAVE_RESTART "24:00:00">
{% endif %}

<!ENTITY START_TIME_SPINUP "01:10:00">
<!ENTITY START_TIME_PROD "02:20:00">
<!ENTITY START_TIME_CONVENTIONAL_SPINUP "00:40:00">
<!ENTITY START_TIME_LATE_ANALYSIS "01:40:00">
<!ENTITY START_TIME_CONVENTIONAL "00:40:00">
<!ENTITY START_TIME_NSSLMOSIAC "00:45:00">
<!ENTITY START_TIME_LIGHTNINGNC "00:45:00">
<!ENTITY START_TIME_SPINUP "{{ start_time_spinup }}">
<!ENTITY START_TIME_PROD "{{ start_time_prod }}">
<!ENTITY START_TIME_CONVENTIONAL_SPINUP "{{ start_time_conventional_spinup }}">
<!ENTITY START_TIME_LATE_ANALYSIS "{{ start_time_late_analysis }}">
<!ENTITY START_TIME_CONVENTIONAL "{{ start_time_conventional }}">
<!ENTITY START_TIME_NSSLMOSIAC "{{ start_time_nsslmosiac }}">
<!ENTITY START_TIME_LIGHTNINGNC "{{ start_time_lightningnc }}">

{%- if do_retro %}
<!ENTITY WALL_LIMIT_PRE ''>
Expand Down Expand Up @@ -960,6 +960,25 @@ MODULES_RUN_TASK_FP script.
<and>
<timedep><cyclestr offset="&START_TIME_CONVENTIONAL;">@Y@m@d@H@M00</cyclestr></timedep>
<taskdep task="&PREP_CYC_TN;_#type#{{ uscore_ensmem_name }}"/>
{%- if use_rrfse_ens %}
<or>
<and>
<or>
{%- for h in cycl_hrs_hyb_fv3lam_ens %}
<streq><left>{{ h }}</left><right><cyclestr>@H</cyclestr></right></streq>
{%- endfor %}
</or>
{% for h in range(1, num_ens_members+1) %}
<datadep age="00:00:01:00"><cyclestr offset="-{{ da_cycle_interval_hrs }}:00:00">&RRFSE_FG_ROOT;/@Y@m@d@H/mem{{ "%04d" % h }}/fcst_fv3lam/RESTART/</cyclestr><cyclestr>@Y@m@d.@H0000.coupler.res</cyclestr></datadep>
{%- endfor %}
</and>
<and>
{%- for h in cycl_hrs_hyb_fv3lam_ens %}
<strneq><left>{{ h }}</left><right><cyclestr>@H</cyclestr></right></strneq>
{%- endfor %}
</and>
</or>
{%- endif %}
</and>
</dependency>

Expand Down Expand Up @@ -1193,9 +1212,7 @@ MODULES_RUN_TASK_FP script.
<envar><name>TMMARK</name><value>tm00</value></envar>

<dependency>
<or>
<datadep age="01:30"><cyclestr>&CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/logf#fhr#</cyclestr></datadep>
</or>
</dependency>

</task>
Expand Down
1 change: 1 addition & 0 deletions ush/valid_param_vals.sh
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ valid_vals_DO_NONVAR_CLDANAL=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no
valid_vals_DO_REFL2TTEN=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")
valid_vals_DO_NLDN_LGHT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")
valid_vals_DO_RADDA=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")
valid_vals_USE_RRFSE_ENS=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")
valid_vals_USE_CUSTOM_POST_CONFIG_FILE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")
valid_vals_DO_SHUM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")
valid_vals_DO_SPPT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")
Expand Down

0 comments on commit 8c8f9fc

Please sign in to comment.