From d7b7bad1469cce3abe749b52a18ff038bbe1563b Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Mon, 16 Sep 2024 21:51:37 -0500 Subject: [PATCH 01/25] Replace gfs_cyc with interval To facilitate longer and more flexible GFS cadences, the `gfs_cyc` variable is replaced with a specified interval. Up front, this is reflected in a change in the arguments for setup_exp to: ``` --interval ``` Where `n_hours` is the interval (in hours) between gfs forecasts. `n_hours` must be a multiple of 6. If 0, no gfs will be run (only gdas; only valid for cycled mode). The default value is 6 (every cycle). In cycled mode, there is an additional argument to control which cycle will be the first gfs cycle: ``` ---sdate_gfs ``` The default if not provided is `--idate` + 6h (first full cycle). As part of this change, some of the validation of the dates has been added. `--edate` has also been made optional and defaults to `--idate` if not provided. During `config.base` template-filling, `INTERVAL_GFS` (renamed from `STEP_GFS`) is defined as `--interval` and `SDATE_GFS as `--sdate_gfs`. Some changes were necessary to the gfs verification (metp) job, as `gfs_cyc` was being used downstream by verif-global. That has been removed, and instead workflow will be responsible for only running metp on the correct cycles. This also removes "do nothing" metp tasks that exit immediately, because only the last GFS cycle in a day would actually process verification. Now, metp has its own cycledef and always runs at 18z, regardless of whether gfs is running at 18z or not. This is simplier than trying to determine the last gfs cycle of a day when it could change from day to day. To facilitate this change, support for the undocumented rocoto dependency tag `taskvalid` is added, as the metp task needs to know whether the cycle has a gfsarch task or not. metp will trigger on gfsarch completing (as before), or gdasarch completing if there is no gfsarch. metp tasks are no longer generated for forecast-only, as the pgbanl files (copied of the 1p00 pgbanl files) are not generated for f-o anyway. If metp is needed for f-o, additional work will be needed. Additionally, a couple EE2 issues with the metp job are resolved (even though it is not run in ops): - verif-global update replaced `$CDUMP` with `$RUN` - `$DATAROOT` is no longer redefined in the metp job Depends on NOAA-EMC/EMC_verif-global#137 Resolves #260 Refs #1299 --- gempak/ush/gfs_meta_comp.sh | 2 +- gempak/ush/gfs_meta_mar_comp.sh | 2 +- jobs/JGFS_ATMOS_VERIFICATION | 6 -- parm/config/gefs/config.base | 8 +-- parm/config/gfs/config.base | 9 +-- parm/config/gfs/config.wave | 2 +- scripts/exgfs_aero_init_aerosol.py | 16 +++--- sorc/verif-global.fd | 2 +- workflow/applications/applications.py | 17 +----- workflow/applications/gefs.py | 1 - workflow/applications/gfs_cycled.py | 50 +---------------- workflow/applications/gfs_forecast_only.py | 8 +-- workflow/rocoto/gefs_xml.py | 12 ++-- workflow/rocoto/gfs_cycled_xml.py | 25 ++++++--- workflow/rocoto/gfs_forecast_only_xml.py | 11 ++-- workflow/rocoto/gfs_tasks.py | 65 +++++++++++----------- workflow/rocoto/rocoto.py | 22 ++++++++ workflow/rocoto/tasks.py | 3 +- workflow/setup_expt.py | 49 +++++++++++++--- 19 files changed, 150 insertions(+), 160 deletions(-) diff --git a/gempak/ush/gfs_meta_comp.sh b/gempak/ush/gfs_meta_comp.sh index 36d18d8659..38c15a60c7 100755 --- a/gempak/ush/gfs_meta_comp.sh +++ b/gempak/ush/gfs_meta_comp.sh @@ -24,7 +24,7 @@ device="nc | ${metaname}" export COMIN="gfs.multi" mkdir "${COMIN}" -for cycle in $(seq -f "%02g" -s ' ' 0 "${STEP_GFS}" "${cyc}"); do +for cycle in $(seq -f "%02g" -s ' ' 0 "${INTERVAL_GFS}" "${cyc}"); do YMD=${PDY} HH=${cycle} GRID="1p00" declare_from_tmpl gempak_dir:COM_ATMOS_GEMPAK_TMPL for file_in in "${gempak_dir}/gfs_1p00_${PDY}${cycle}f"*; do file_out="${COMIN}/$(basename "${file_in}")" diff --git a/gempak/ush/gfs_meta_mar_comp.sh b/gempak/ush/gfs_meta_mar_comp.sh index d25fc0dc9a..91f8a48876 100755 --- a/gempak/ush/gfs_meta_mar_comp.sh +++ b/gempak/ush/gfs_meta_mar_comp.sh @@ -15,7 +15,7 @@ cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl export COMIN="gfs.multi" mkdir -p "${COMIN}" -for cycle in $(seq -f "%02g" -s ' ' 0 "${STEP_GFS}" "${cyc}"); do +for cycle in $(seq -f "%02g" -s ' ' 0 "${INTERVAL_GFS}" "${cyc}"); do YMD=${PDY} HH=${cycle} GRID="1p00" declare_from_tmpl gempak_dir:COM_ATMOS_GEMPAK_TMPL for file_in in "${gempak_dir}/gfs_1p00_${PDY}${cycle}f"*; do file_out="${COMIN}/$(basename "${file_in}")" diff --git a/jobs/JGFS_ATMOS_VERIFICATION b/jobs/JGFS_ATMOS_VERIFICATION index 48133364e5..fde0d73b1e 100755 --- a/jobs/JGFS_ATMOS_VERIFICATION +++ b/jobs/JGFS_ATMOS_VERIFICATION @@ -16,12 +16,6 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "metp" -c "base metp" ## METPCASE : METplus verification use case (g2g1 | g2o1 | pcp1) ############################################################### -# TODO: This should not be permitted as DATAROOT is set at the job-card level. -# TODO: DATAROOT is being used as DATA in metp jobs. This should be rectified in metp. -# TODO: The temporary directory is DATA and is created at the top of the J-Job. -# TODO: remove this line -export DATAROOT=${DATA} - VDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${VRFYBACK_HRS} hours") export VDATE=${VDATE:0:8} diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base index a0bd8b3bd1..756c14ce39 100644 --- a/parm/config/gefs/config.base +++ b/parm/config/gefs/config.base @@ -227,7 +227,8 @@ export FHOUT_OCN=3 export FHOUT_ICE=3 # GFS cycle info -export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles. +export INTERVAL_GFS=@INTERVAL_GFS@ # Frequency of GFS forecast +export SDATE_GFS=@SDATE_GFS@ # GFS output and frequency export FHMIN_GFS=0 @@ -246,11 +247,6 @@ export FHOUT_WAV=3 export FHMAX_HF_WAV=120 export FHOUT_HF_WAV=1 export FHMAX_WAV=${FHMAX_GFS} -if (( gfs_cyc != 0 )); then - export STEP_GFS=$(( 24 / gfs_cyc )) -else - export STEP_GFS="0" -fi export ILPOST=1 # gempak output frequency up to F120 export FHMIN_ENKF=${FHMIN_GFS} diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 7fa8245057..ccb05abe88 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -283,7 +283,8 @@ export FHOUT_ICE=3 export EUPD_CYC="@EUPD_CYC@" # GFS cycle info -export gfs_cyc=@gfs_cyc@ # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles. +export INTERVAL_GFS=@INTERVAL_GFS@ # Frequency of GFS forecast +export SDATE_GFS=@SDATE_GFS@ # GFS output and frequency export FHMIN_GFS=0 @@ -302,11 +303,7 @@ export FHMAX_HF_WAV=120 export FHOUT_HF_WAV=1 export FHMAX_WAV=${FHMAX:-9} export FHMAX_WAV_GFS=${FHMAX_GFS} -if (( gfs_cyc != 0 )); then - export STEP_GFS=$(( 24 / gfs_cyc )) -else - export STEP_GFS="0" -fi + # TODO: Change gempak to use standard out variables (#2348) export ILPOST=${FHOUT_HF_GFS} # gempak output frequency up to F120 if (( FHMAX_HF_GFS < 120 )); then diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave index db4eb9f708..ea68508547 100644 --- a/parm/config/gfs/config.wave +++ b/parm/config/gfs/config.wave @@ -117,7 +117,7 @@ if [[ "${RUN}" == "gdas" ]]; then export WAVNCYC=4 export WAVHCYC=${assim_freq:-6} export FHMAX_WAV_CUR=48 # RTOFS forecasts only out to 8 days -elif [[ ${gfs_cyc} -ne 0 ]]; then +elif (( INTERVAL_GFS > 0 )); then export WAVHCYC=${assim_freq:-6} export FHMAX_WAV_CUR=192 # RTOFS forecasts only out to 8 days else diff --git a/scripts/exgfs_aero_init_aerosol.py b/scripts/exgfs_aero_init_aerosol.py index aed6b88647..bc4e495e42 100755 --- a/scripts/exgfs_aero_init_aerosol.py +++ b/scripts/exgfs_aero_init_aerosol.py @@ -11,13 +11,13 @@ --------- This script requires the following environment variables be set beforehand: -CDATE: Initial time in YYYYMMDDHH format -STEP_GFS: Forecast cadence (frequency) in hours -FHMAX_GFS: Forecast length in hours -RUN: Forecast phase (gfs or gdas). Currently always expected to be gfs. -ROTDIR: Rotating (COM) directory -USHgfs: Path to global-workflow `ush` directory -PARMgfs: Path to global-workflow `parm` directory +CDATE: Initial time in YYYYMMDDHH format +INTERVAL_GFS: Forecast cadence (frequency) in hours +FHMAX_GFS: Forecast length in hours +RUN: Forecast phase (gfs or gdas). Currently always expected to be gfs. +ROTDIR: Rotating (COM) directory +USHgfs: Path to global-workflow `ush` directory +PARMgfs: Path to global-workflow `parm` directory Additionally, the following data files are used: @@ -66,7 +66,7 @@ def main() -> None: # Read in environment variables and make sure they exist cdate = get_env_var("CDATE") - incr = int(get_env_var('STEP_GFS')) + incr = int(get_env_var('INTERVAL_GFS')) fcst_length = int(get_env_var('FHMAX_GFS')) run = get_env_var("RUN") rot_dir = get_env_var("ROTDIR") diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index e7e6bc4358..8f0428b441 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit e7e6bc43584e0b8911819b8f875cc8ee747db76d +Subproject commit 8f0428b4415136b6f24f6bbd1b01c8e761c2b22f diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index a694129e38..ecd320d708 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -68,6 +68,7 @@ def __init__(self, conf: Configuration) -> None: self.nens = base.get('NMEM_ENS', 0) self.fcst_segments = base.get('FCST_SEGMENTS', None) + self.interval_gfs = to_timedelta(f"{base.get('INTERVAL_GFS')}H") if not AppConfig.is_monotonic(self.fcst_segments): raise ValueError(f'Forecast segments do not increase monotonically: {",".join(self.fcst_segments)}') @@ -109,9 +110,6 @@ def _init_finalize(self, conf: Configuration): # Save base in the internal state since it is often needed base = self.configs['_no_run']['base'] - # Get more configuration options into the class attributes - self.gfs_cyc = base.get('gfs_cyc') - # Get task names for the application self.task_names = self.get_task_names() @@ -199,19 +197,6 @@ def get_task_names(self, run="_no_run") -> Dict[str, List[str]]: ''' pass - @staticmethod - def get_gfs_interval(gfs_cyc: int) -> timedelta: - """ - return interval in hours based on gfs_cyc - """ - - gfs_internal_map = {'1': '24H', '2': '12H', '4': '6H'} - - try: - return to_timedelta(gfs_internal_map[str(gfs_cyc)]) - except KeyError: - raise KeyError(f'Invalid gfs_cyc = {gfs_cyc}') - @staticmethod def is_monotonic(test_list: List, check_decreasing: bool = False) -> bool: """ diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index c15786a2e8..9d1d5c3dc4 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -42,7 +42,6 @@ def _get_app_configs(self): def _update_base(base_in): base_out = base_in.copy() - base_out['INTERVAL_GFS'] = AppConfig.get_gfs_interval(base_in['gfs_cyc']) base_out['RUN'] = 'gefs' return base_out diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 19f4dd607b..da78166ede 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -128,7 +128,7 @@ def _get_app_configs(self): @staticmethod def _update_base(base_in): - return GFSCycledAppConfig.get_gfs_cyc_dates(base_in) + return base_in def get_task_names(self): """ @@ -297,7 +297,7 @@ def get_task_names(self): tasks['enkfgdas'] = enkfgdas_tasks # Add RUN=gfs tasks if running early cycle - if self.gfs_cyc > 0: + if self.interval_gfs > to_timedelta("0H"): tasks['gfs'] = gfs_tasks if self.do_hybvar and 'gfs' in self.eupd_runs: @@ -307,49 +307,3 @@ def get_task_names(self): tasks['enkfgfs'] = enkfgfs_tasks return tasks - - @staticmethod - def get_gfs_cyc_dates(base: Dict[str, Any]) -> Dict[str, Any]: - """ - Generate GFS dates from experiment dates and gfs_cyc choice - """ - - base_out = base.copy() - - sdate = base['SDATE'] - edate = base['EDATE'] - base_out['INTERVAL'] = to_timedelta(f"{base['assim_freq']}H") - - # Set GFS cycling dates - gfs_cyc = base['gfs_cyc'] - if gfs_cyc != 0: - interval_gfs = AppConfig.get_gfs_interval(gfs_cyc) - hrinc = 0 - hrdet = 0 - if gfs_cyc == 1: - hrinc = 24 - sdate.hour - hrdet = edate.hour - elif gfs_cyc == 2: - if sdate.hour in [0, 12]: - hrinc = 12 - elif sdate.hour in [6, 18]: - hrinc = 6 - if edate.hour in [6, 18]: - hrdet = 6 - elif gfs_cyc == 4: - hrinc = 6 - sdate_gfs = sdate + timedelta(hours=hrinc) - edate_gfs = edate - timedelta(hours=hrdet) - if sdate_gfs > edate: - print('W A R N I N G!') - print('Starting date for GFS cycles is after Ending date of experiment') - print(f'SDATE = {sdate.strftime("%Y%m%d%H")}, EDATE = {edate.strftime("%Y%m%d%H")}') - print(f'SDATE_GFS = {sdate_gfs.strftime("%Y%m%d%H")}, EDATE_GFS = {edate_gfs.strftime("%Y%m%d%H")}') - gfs_cyc = 0 - - base_out['gfs_cyc'] = gfs_cyc - base_out['SDATE_GFS'] = sdate_gfs - base_out['EDATE_GFS'] = edate_gfs - base_out['INTERVAL_GFS'] = interval_gfs - - return base_out diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 93551ac0cc..3b120054c6 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -42,9 +42,6 @@ def _get_app_configs(self): if self.do_genesis_fsu: configs += ['genesis_fsu'] - if self.do_metp: - configs += ['metp'] - if self.do_bufrsnd: configs += ['postsnd'] @@ -78,7 +75,7 @@ def _get_app_configs(self): def _update_base(base_in): base_out = base_in.copy() - base_out['INTERVAL_GFS'] = AppConfig.get_gfs_interval(base_in['gfs_cyc']) + base_out['RUN'] = 'gfs' return base_out @@ -122,9 +119,6 @@ def get_task_names(self): if self.do_genesis_fsu: tasks += ['genesis_fsu'] - if self.do_metp: - tasks += ['metp'] - if self.do_bufrsnd: tasks += ['postsnd'] diff --git a/workflow/rocoto/gefs_xml.py b/workflow/rocoto/gefs_xml.py index b25a73fa6c..18959f9da8 100644 --- a/workflow/rocoto/gefs_xml.py +++ b/workflow/rocoto/gefs_xml.py @@ -14,19 +14,19 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: super().__init__(app_config, rocoto_config) def get_cycledefs(self): - sdate = self._base['SDATE'] + sdate = self._base['SDATE_GFS'] edate = self._base['EDATE'] - interval = self._base.get('INTERVAL_GFS', to_timedelta('24H')) + interval = self.interval_gfs sdate_str = sdate.strftime("%Y%m%d%H%M") edate_str = edate.strftime("%Y%m%d%H%M") interval_str = timedelta_to_HMS(interval) strings = [] strings.append(f'\t{sdate_str} {edate_str} {interval_str}') - sdate = sdate + interval - if sdate <= edate: - sdate_str = sdate.strftime("%Y%m%d%H%M") - strings.append(f'\t{sdate_str} {edate_str} {interval_str}') + date2 = sdate + interval + if date2 <= edate: + date2_str = date2.strftime("%Y%m%d%H%M") + strings.append(f'\t{date2_str} {edate_str} {interval_str}') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_cycled_xml.py b/workflow/rocoto/gfs_cycled_xml.py index afd663c337..b2c60fa124 100644 --- a/workflow/rocoto/gfs_cycled_xml.py +++ b/workflow/rocoto/gfs_cycled_xml.py @@ -24,19 +24,30 @@ def get_cycledefs(self): sdate_str = sdate.strftime("%Y%m%d%H%M") strings.append(f'\t{sdate_str} {edate_str} {interval_str}') - if self._app_config.gfs_cyc != 0: + interval_gfs = self._app_config.interval_gfs + + if interval_gfs > to_timedelta("0H"): sdate_gfs = self._base['SDATE_GFS'] - edate_gfs = self._base['EDATE_GFS'] - interval_gfs = self._base['INTERVAL_GFS'] + edate_gfs = edate sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") edate_gfs_str = edate_gfs.strftime("%Y%m%d%H%M") interval_gfs_str = timedelta_to_HMS(interval_gfs) strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') - sdate_gfs = sdate_gfs + interval_gfs - sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") - if sdate_gfs <= edate_gfs: - strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') + date2_gfs = sdate_gfs + interval_gfs + date2_gfs_str = date2_gfs.strftime("%Y%m%d%H%M") + if date2_gfs <= edate_gfs: + strings.append(f'\t{date2_gfs_str} {edate_gfs_str} {interval_gfs_str}') + + if self._base['DO_METP']: + # Run verification at 18z, no matter what + sdate_metp = sdate_gfs.replace(hour=18) + edate_metp = edate_gfs.replace(hour=18) + interval_metp = to_timedelta('24H') + sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") + edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") + interval_metp_str = timedelta_to_HMS(interval_metp) + strings.append(f'\t{sdate_metp_str} {edate_metp_str} {interval_metp_str}') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_forecast_only_xml.py b/workflow/rocoto/gfs_forecast_only_xml.py index cf53e685e9..b143b7335a 100644 --- a/workflow/rocoto/gfs_forecast_only_xml.py +++ b/workflow/rocoto/gfs_forecast_only_xml.py @@ -12,15 +12,16 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: super().__init__(app_config, rocoto_config) def get_cycledefs(self): - sdate = self._base['SDATE'] + sdate = self._base['SDATE_GFS'] edate = self._base['EDATE'] - interval = self._base.get('INTERVAL_GFS', to_timedelta('24H')) + interval = self._app_config.interval_gfs strings = [] strings.append(f'\t{sdate.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') - sdate = sdate + interval - if sdate <= edate: - strings.append(f'\t{sdate.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') + date2 = sdate + interval + date2_str = date2.strftime("%Y%m%d%H%M") + if date2 <= edate: + strings.append(f'\t{date2.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 6b9d6358c6..911a3305de 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -39,7 +39,6 @@ def stage_ic(self): def prep(self): dump_suffix = self._base["DUMP_SUFFIX"] - gfs_cyc = self._base["gfs_cyc"] dmpdir = self._base["DMPDIR"] atm_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'RUN': 'gdas'}) dump_path = self._template_to_rocoto_cycstring(self._base["COM_OBSDMP_TMPL"], @@ -48,10 +47,10 @@ def prep(self): gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_runs else False deps = [] - dep_dict = {'type': 'metatask', 'name': 'gdasatmos_prod', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'gdasatmos_prod', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) data = f'{atm_hist_path}/gdas.t@Hz.atmf009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) data = f'{dump_path}/{self.run}.t@Hz.updated.status.tm00.bufr_d' dep_dict = {'type': 'data', 'data': data} @@ -59,7 +58,7 @@ def prep(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) cycledef = self.run - if self.run in ['gfs'] and gfs_enkf and gfs_cyc != 4: + if self.run in ['gfs'] and gfs_enkf and self.interval_gfs == 6: cycledef = 'gdas' resources = self.get_resource('prep') @@ -89,7 +88,7 @@ def waveinit(self): dep_dict = {'type': 'task', 'name': f'{self.run}prep'} deps.append(rocoto.add_dependency(dep_dict)) if self.run in ['gdas']: - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) @@ -189,7 +188,7 @@ def anal(self): dep_dict = {'type': 'task', 'name': f'{self.run}prep'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar: - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: @@ -255,7 +254,7 @@ def analcalc(self): dep_dict = {'type': 'task', 'name': f'{self.run}sfcanl'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar and self.run in ['gdas']: - dep_dict = {'type': 'task', 'name': 'enkfgdasechgres', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': 'enkfgdasechgres', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -330,17 +329,17 @@ def atmanlinit(self): dep_dict = {'type': 'task', 'name': f'{self.run}prepatmiodaobs'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_hybvar: - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: dependencies = rocoto.create_dependency(dep=deps) - gfs_cyc = self._base["gfs_cyc"] + interval_gfs = self._base["INTERVAL_GFS"] gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_runs else False cycledef = self.run - if self.run in ['gfs'] and gfs_enkf and gfs_cyc != 4: + if self.run in ['gfs'] and gfs_enkf and interval_gfs == 6: cycledef = 'gdas' resources = self.get_resource('atmanlinit') @@ -482,7 +481,7 @@ def aeroanlgenb(self): def aeroanlinit(self): deps = [] - dep_dict = {'type': 'task', 'name': 'gdasaeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': 'gdasaeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}prep'} deps.append(rocoto.add_dependency(dep_dict)) @@ -514,7 +513,7 @@ def aeroanlvar(self): deps = [] dep_dict = { 'type': 'task', 'name': f'gdasaeroanlgenb', - 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", + 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}", } deps.append(rocoto.add_dependency(dep_dict)) dep_dict = { @@ -618,7 +617,7 @@ def esnowrecen(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}snowanl'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': f'{self.run}epmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': f'{self.run}epmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -644,7 +643,7 @@ def prepoceanobs(self): deps = [] data = f'{ocean_hist_path}/gdas.ocean.t@Hz.inst.f009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -671,7 +670,7 @@ def marinebmat(self): deps = [] data = f'{ocean_hist_path}/gdas.ocean.t@Hz.inst.f009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -699,7 +698,7 @@ def marineanlinit(self): deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}marinebmat'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'gdasfcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'gdasfcst', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -878,9 +877,9 @@ def _fcst_forecast_only(self): # Calculate offset based on RUN = gfs | gdas interval = None if self.run in ['gfs']: - interval = self._base['INTERVAL_GFS'] + interval = to_timedelta(f"{self._base['INTERVAL_GFS']}H") elif self.run in ['gdas']: - interval = self._base['INTERVAL'] + interval = self._base['assim_freq'] offset = timedelta_to_HMS(-interval) deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}aerosol_init'} @@ -1835,14 +1834,18 @@ def metp(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}arch'} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + if self.run in "gfs": + deps2 = [] + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}arch', 'condition': 'not'} + deps2.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'gdasarch'} + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='and', dep=deps2)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) metpenvars = self.envars.copy() - if self.app_config.mode in ['cycled']: - metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS').strftime("%Y%m%d%H"), - 'EDATE_GFS': self._base.get('EDATE_GFS').strftime("%Y%m%d%H")} - elif self.app_config.mode in ['forecast-only']: - metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE').strftime("%Y%m%d%H")} + metpenvar_dict = {'SDATE_GFS': self._base.get('SDATE_GFS').strftime("%Y%m%d%H"), + 'EDATE_GFS': self._base.get('EDATE').strftime("%Y%m%d%H")} metpenvar_dict['METPCASE'] = '#metpcase#' for key, value in metpenvar_dict.items(): metpenvars.append(rocoto.create_envar(name=key, value=str(value))) @@ -1858,7 +1861,7 @@ def metp(self): 'resources': resources, 'dependency': dependencies, 'envars': metpenvars, - 'cycledef': self.run.replace('enkf', ''), + 'cycledef': 'metp', 'command': f'{self.HOMEgfs}/jobs/rocoto/metp.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2358,7 +2361,7 @@ def eobs(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}prep'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2468,7 +2471,7 @@ def atmensanlinit(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf","")}prepatmiodaobs'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2495,7 +2498,7 @@ def atmensanlobs(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlinit'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2521,7 +2524,7 @@ def atmensanlsol(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlobs'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2547,7 +2550,7 @@ def atmensanlletkf(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlinit'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -2576,7 +2579,7 @@ def atmensanlfv3inc(self): else: dep_dict = {'type': 'task', 'name': f'{self.run}atmensanlletkf'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdasepmn', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/rocoto.py b/workflow/rocoto/rocoto.py index 2a20820da8..7bacf99829 100644 --- a/workflow/rocoto/rocoto.py +++ b/workflow/rocoto/rocoto.py @@ -183,6 +183,7 @@ def add_dependency(dep_dict: Dict[str, Any]) -> str: 'metatask': _add_task_tag, 'data': _add_data_tag, 'cycleexist': _add_cycle_tag, + 'taskvalid': _add_taskvalid_tag, 'streq': _add_streq_tag, 'strneq': _add_streq_tag, 'sh': _add_sh_tag} @@ -296,6 +297,27 @@ def _add_cycle_tag(dep_dict: Dict[str, Any]) -> str: return string +def _add_taskvalid_tag(dep_dict: Dict[str, Any]) -> str: + """ + create a validtask tag + :param dep_dict: dependency key-value parameters + :type dep_dict: dict + :return: Rocoto validtask dependency + :rtype: str + """ + + dep_type = dep_dict.get('type', None) + dep_name = dep_dict.get('name', None) + + if dep_name is None: + msg = f'a {dep_type} name is necessary for {dep_type} dependency' + raise KeyError(msg) + + string = f'<{dep_type} task="{dep_name}"/>' + + return string + + def _add_streq_tag(dep_dict: Dict[str, Any]) -> str: """ create a simple string comparison tag diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 8a32827377..92ceea73aa 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -57,7 +57,8 @@ def __init__(self, app_config: AppConfig, run: str) -> None: self.nmem = int(self._base['NMEM_ENS_GFS']) else: self.nmem = int(self._base['NMEM_ENS']) - self._base['cycle_interval'] = to_timedelta(f'{self._base["assim_freq"]}H') + self._base['interval_gdas'] = to_timedelta(f'{self._base["assim_freq"]}H') + self._base['interval_gfs'] = to_timedelta(f'{self._base["INTERVAL_GFS"]}H') self.n_tiles = 6 # TODO - this needs to be elsewhere diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index e213394e20..e8f4bdc596 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -8,7 +8,7 @@ import glob import shutil import warnings -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, SUPPRESS, ArgumentTypeError from hosts import Host @@ -116,7 +116,8 @@ def edit_baseconfig(host, inputs, yaml_dict): "@COMROOT@": inputs.comroot, "@EXP_WARM_START@": is_warm_start, "@MODE@": inputs.mode, - "@gfs_cyc@": inputs.gfs_cyc, + "@INTERVAL_GFS@": inputs.interval, + "@SDATE_GFS@": datetime_to_YMDH(inputs.sdate_gfs), "@APP@": inputs.app, "@NMEM_ENS@": getattr(inputs, 'nens', 0) } @@ -186,6 +187,19 @@ def input_args(*argv): ufs_apps = ['ATM', 'ATMA', 'ATMW', 'S2S', 'S2SA', 'S2SW', 'S2SWA'] + def _validate_interval(interval_str): + err_msg = f'must be a non-negative integer multiple of 6 ({interval_str} given)' + try: + interval = int(interval_str) + except ValueError: + raise ArgumentTypeError(err_msg) + + # This assumes the gdas frequency (assim_freq) is 6h + # If this changes, the modulus needs to as well + if interval < 0 or interval % 6 != 0: + raise ArgumentTypeError(err_msg) + return interval + def _common_args(parser): parser.add_argument('--pslot', help='parallel experiment name', type=str, required=False, default='test') @@ -199,7 +213,8 @@ def _common_args(parser): type=str, required=False, default=os.getenv('HOME')) parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', required=True, type=lambda dd: to_datetime(dd)) - parser.add_argument('--edate', help='end date experiment', required=True, type=lambda dd: to_datetime(dd)) + parser.add_argument('--edate', help='end date experiment', required=False, type=lambda dd: to_datetime(dd)) + parser.add_argument('--interval', help='frequency of forecast (in hours); must be a multiple of 6', type=_validate_interval, required=False, default=6) parser.add_argument('--icsdir', help='full path to user initial condition directory', type=str, required=False, default='') parser.add_argument('--overwrite', help='overwrite previously created experiment (if it exists)', action='store_true', required=False) @@ -219,8 +234,7 @@ def _gfs_args(parser): def _gfs_cycled_args(parser): parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') - parser.add_argument('--gfs_cyc', help='cycles to run forecast', type=int, - choices=[0, 1, 2, 4], default=1, required=False) + parser.add_argument('--sdate_gfs', help='date to start GFS', type=lambda dd: to_datetime(dd), required=False, default=None) return parser def _gfs_or_gefs_ensemble_args(parser): @@ -233,8 +247,6 @@ def _gfs_or_gefs_ensemble_args(parser): def _gfs_or_gefs_forecast_args(parser): parser.add_argument('--app', help='UFS application', type=str, choices=ufs_apps, required=False, default='ATM') - parser.add_argument('--gfs_cyc', help='Number of forecasts per day', type=int, - choices=[1, 2, 4], default=1, required=False) return parser def _gefs_args(parser): @@ -291,7 +303,28 @@ def _gefs_args(parser): for subp in [gefsforecasts]: subp = _gefs_args(subp) - return parser.parse_args(list(*argv) if len(argv) else None) + inputs = parser.parse_args(list(*argv) if len(argv) else None) + + # Validate dates + if inputs.edate is None: + inputs.edate = inputs.idate + + if inputs.edate < inputs.idate: + raise ArgumentTypeError(f'edate ({inputs.edate}) cannot be before idate ({inputs.idate})') + + # For forecast-only, GFS starts in the first cycle + if not hasattr(inputs, 'sdate_gfs'): + inputs.sdate_gfs = inputs.idate + + # For cycled, GFS starts after the half-cycle + if inputs.sdate_gfs is None: + inputs.sdate_gfs = inputs.idate + to_timedelta("6H") + + if inputs.interval > 0: + if inputs.sdate_gfs < inputs.idate or inputs.sdate_gfs > inputs.edate: + raise ArgumentTypeError(f'sdate_gfs ({inputs.sdate_gfs}) must be between idate ({inputs.idate}) and edate ({inputs.edate})') + + return inputs def query_and_clean(dirname, force_clean=False): From b2d26b35115ec0b3e7c44959f9b86de20ef99072 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Mon, 16 Sep 2024 23:06:15 -0500 Subject: [PATCH 02/25] Update CI tests for gfs_cyc to interval change --- ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml | 2 +- ci/cases/pr/C48_S2SWA_gefs.yaml | 2 +- ci/cases/pr/C48mx500_3DVarAOWCDA.yaml | 2 +- ci/cases/pr/C96C48_hybatmDA.yaml | 2 +- ci/cases/pr/C96C48_hybatmaerosnowDA.yaml | 2 +- ci/cases/pr/C96C48_ufs_hybatmDA.yaml | 2 +- ci/cases/pr/C96_atm3DVar.yaml | 2 +- ci/cases/pr/C96_atm3DVar_extended.yaml | 2 +- ci/cases/weekly/C384C192_hybatmda.yaml | 2 +- ci/cases/weekly/C384_atm3DVar.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml index d97c9567e9..99ba7c3661 100644 --- a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml +++ b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml @@ -8,7 +8,7 @@ arguments: resdetatmos: 384 resdetocean: 0.25 nens: 0 - gfs_cyc: 4 + interval: 6 start: cold comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml index a924b416c3..9cc230d35c 100644 --- a/ci/cases/pr/C48_S2SWA_gefs.yaml +++ b/ci/cases/pr/C48_S2SWA_gefs.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 5.0 resensatmos: 48 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index e1b76f0db8..2de5fea7ff 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2021032412 edate: 2021032418 nens: 0 - gfs_cyc: 0 + interval: 0 start: warm yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_hybatmDA.yaml b/ci/cases/pr/C96C48_hybatmDA.yaml index 7617e39217..b527903d69 100644 --- a/ci/cases/pr/C96C48_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_hybatmDA.yaml @@ -14,6 +14,6 @@ arguments: idate: 2021122018 edate: 2021122106 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml index 7387e55b24..be5ad32238 100644 --- a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml +++ b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2021122012 edate: 2021122100 nens: 2 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index b1566d77a0..41a8baa725 100644 --- a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -13,7 +13,7 @@ arguments: idate: 2024022318 edate: 2024022406 nens: 2 - gfs_cyc: 1 + interval: 24 start: warm yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml index e9e6c2b31c..fc09beeacf 100644 --- a/ci/cases/pr/C96_atm3DVar.yaml +++ b/ci/cases/pr/C96_atm3DVar.yaml @@ -12,7 +12,7 @@ arguments: idate: 2021122018 edate: 2021122106 nens: 0 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96_atm3DVar_extended.yaml b/ci/cases/pr/C96_atm3DVar_extended.yaml index cdf69f04e0..8ab67a750e 100644 --- a/ci/cases/pr/C96_atm3DVar_extended.yaml +++ b/ci/cases/pr/C96_atm3DVar_extended.yaml @@ -12,7 +12,7 @@ arguments: idate: 2021122018 edate: 2021122118 nens: 0 - gfs_cyc: 4 + interval: 6 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml diff --git a/ci/cases/weekly/C384C192_hybatmda.yaml b/ci/cases/weekly/C384C192_hybatmda.yaml index 131ada95d5..a04ac84e82 100644 --- a/ci/cases/weekly/C384C192_hybatmda.yaml +++ b/ci/cases/weekly/C384C192_hybatmda.yaml @@ -14,6 +14,6 @@ arguments: idate: 2023040118 edate: 2023040200 nens: 2 - gfs_cyc: 1 + interval: 6 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/weekly/C384_atm3DVar.yaml b/ci/cases/weekly/C384_atm3DVar.yaml index 40487f3b47..1a14059ab1 100644 --- a/ci/cases/weekly/C384_atm3DVar.yaml +++ b/ci/cases/weekly/C384_atm3DVar.yaml @@ -14,6 +14,6 @@ arguments: idate: 2023040118 edate: 2023040200 nens: 0 - gfs_cyc: 1 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml From 3f827681e7cefb05259ae06cd9c49f827f08fd6a Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Mon, 16 Sep 2024 23:23:01 -0500 Subject: [PATCH 03/25] Update documentation for change to interval from gfs_cyc --- docs/source/jobs.rst | 2 +- docs/source/setup.rst | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/source/jobs.rst b/docs/source/jobs.rst index 0e3700bf20..2cdecb01de 100644 --- a/docs/source/jobs.rst +++ b/docs/source/jobs.rst @@ -8,7 +8,7 @@ GFS Configuration The sequence of jobs that are run for an end-to-end (analysis+forecast+post processing+verification) GFS configuration using the Global Workflow is shown above. The system utilizes a collection of scripts that perform the tasks for each step. -For any cycle the system consists of two suites -- the "gdas" suite which provides the initial guess fields, and the "gfs" suite which creates the initial conditions and forecast of the system. As with the operational system, the gdas runs for each cycle (00, 06, 12, and 18 UTC), however, to save time and space in experiments, the gfs (right side of the diagram) is initially setup to run for only the 00 UTC cycle (See the "run GFS this cycle?" portion of the diagram). The option to run the GFS for all four cycles is available (see the ``gfs_cyc`` variable in configuration file). +For any cycle the system consists of two suites -- the "gdas" suite which provides the initial guess fields, and the "gfs" suite which creates the initial conditions and forecast of the system. An experimental run is different from operations in the following ways: diff --git a/docs/source/setup.rst b/docs/source/setup.rst index 1715899927..e7d5323e40 100644 --- a/docs/source/setup.rst +++ b/docs/source/setup.rst @@ -32,7 +32,7 @@ The following command examples include variables for reference but users should :: cd workflow - ./setup_expt.py gfs forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] + ./setup_expt.py gfs forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--interval $INTERVAL_GFS] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR] where: @@ -51,12 +51,12 @@ where: * ``$START`` is the start type (warm or cold [default]) * ``$IDATE`` is the initial start date of your run (first cycle CDATE, YYYYMMDDCC) - * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete + * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete [default: $IDATE] * ``$PSLOT`` is the name of your experiment [default: test] * ``$CONFIGDIR`` is the path to the ``/config`` folder under the copy of the system you're using [default: $TOP_OF_CLONE/parm/config/] * ``$RESDETATMOS`` is the resolution of the atmosphere component of the system (i.e. 768 for C768) [default: 384] * ``$RESDETOCEAN`` is the resolution of the ocean component of the system (i.e. 0.25 for 1/4 degree) [default: 0.; determined based on atmosphere resolution] - * ``$GFS_CYC`` is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles) + * ``$INTERVAL_GFS`` is the forecast interval in hours [default: 6] * ``$COMROOT`` is the path to your experiment output directory. Your ``ROTDIR`` (rotating com directory) will be created using ``COMROOT`` and ``PSLOT``. [default: $HOME (but do not use default due to limited space in home directories normally, provide a path to a larger scratch space)] * ``$EXPDIR`` is the path to your experiment directory where your configs will be placed and where you will find your workflow monitoring files (i.e. rocoto database and xml file). DO NOT include PSLOT folder at end of path, it will be built for you. [default: $HOME] @@ -67,7 +67,7 @@ Atm-only: :: cd workflow - ./setup_expt.py gfs forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdetatmos 384 --gfs_cyc 4 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir + ./setup_expt.py gfs forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdetatmos 384 --interval 6 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir Coupled: @@ -144,7 +144,8 @@ The following command examples include variables for reference but users should :: cd workflow - ./setup_expt.py gfs cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] + ./setup_expt.py gfs cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] + [--interval $INTERVAL_GFS] [--sdate_gfs $SDATE_GFS] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--resensatmos $RESENSATMOS] [--nens $NENS] [--run $RUN] [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR] [--icsdir $ICSDIR] @@ -163,9 +164,10 @@ where: - S2SWA: atm-ocean-ice-wave-aerosols * ``$IDATE`` is the initial start date of your run (first cycle CDATE, YYYYMMDDCC) - * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete + * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete [default: $IDATE] * ``$START`` is the start type (warm or cold [default]) - * ``$GFS_CYC`` is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles) + * ``$INTERVAL_GFS`` is the forecast interval in hours [default: 6] + * ``$SDATE_GFS`` cycle to begin GFS forecast [default: $IDATE + 6] * ``$RESDETATMOS`` is the resolution of the atmosphere component of the deterministic forecast [default: 384] * ``$RESDETOCEAN`` is the resolution of the ocean component of the deterministic forecast [default: 0.; determined based on atmosphere resolution] * ``$RESENSATMOS`` is the resolution of the atmosphere component of the ensemble forecast [default: 192] @@ -184,7 +186,7 @@ Example: :: cd workflow - ./setup_expt.py gfs cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdetatmos 384 --resensatmos 192 --nens 80 --gfs_cyc 4 + ./setup_expt.py gfs cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdetatmos 384 --resensatmos 192 --nens 80 --interval 6 Example ``setup_expt.py`` on Orion: From ed9f426ade600eb8d588c85f6b54d9ecf68fda0a Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Mon, 16 Sep 2024 23:26:29 -0500 Subject: [PATCH 04/25] Fix overindented line in setup_expt --- workflow/setup_expt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index e8f4bdc596..c6e4e59011 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -314,7 +314,7 @@ def _gefs_args(parser): # For forecast-only, GFS starts in the first cycle if not hasattr(inputs, 'sdate_gfs'): - inputs.sdate_gfs = inputs.idate + inputs.sdate_gfs = inputs.idate # For cycled, GFS starts after the half-cycle if inputs.sdate_gfs is None: From 40cb68a1a5d44c74aec8b93dc6d313466ea3484f Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Mon, 16 Sep 2024 23:31:04 -0500 Subject: [PATCH 05/25] Revert inadvertent change to C384C192_hybatmda test --- ci/cases/weekly/C384C192_hybatmda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cases/weekly/C384C192_hybatmda.yaml b/ci/cases/weekly/C384C192_hybatmda.yaml index a04ac84e82..6053f73124 100644 --- a/ci/cases/weekly/C384C192_hybatmda.yaml +++ b/ci/cases/weekly/C384C192_hybatmda.yaml @@ -14,6 +14,6 @@ arguments: idate: 2023040118 edate: 2023040200 nens: 2 - interval: 6 + interval: 24 start: cold yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml From cad3f646a7d1605594fac3823817401a858ecc14 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Mon, 16 Sep 2024 23:37:34 -0500 Subject: [PATCH 06/25] Remove unnecessary declaration --- workflow/rocoto/gfs_forecast_only_xml.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/rocoto/gfs_forecast_only_xml.py b/workflow/rocoto/gfs_forecast_only_xml.py index b143b7335a..dbfbbc9508 100644 --- a/workflow/rocoto/gfs_forecast_only_xml.py +++ b/workflow/rocoto/gfs_forecast_only_xml.py @@ -19,7 +19,6 @@ def get_cycledefs(self): strings.append(f'\t{sdate.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') date2 = sdate + interval - date2_str = date2.strftime("%Y%m%d%H%M") if date2 <= edate: strings.append(f'\t{date2.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') From 911b377472f2995c844e63f1fe7bff9cc58df7dc Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Mon, 16 Sep 2024 23:40:18 -0500 Subject: [PATCH 07/25] Revert inadvertent cycledef change for prep and atmanlinit --- workflow/rocoto/gfs_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 911a3305de..1084cde3b9 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -58,7 +58,7 @@ def prep(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) cycledef = self.run - if self.run in ['gfs'] and gfs_enkf and self.interval_gfs == 6: + if self.run in ['gfs'] and gfs_enkf and self.interval_gfs != 6: cycledef = 'gdas' resources = self.get_resource('prep') @@ -339,7 +339,7 @@ def atmanlinit(self): gfs_enkf = True if self.app_config.do_hybvar and 'gfs' in self.app_config.eupd_runs else False cycledef = self.run - if self.run in ['gfs'] and gfs_enkf and interval_gfs == 6: + if self.run in ['gfs'] and gfs_enkf and interval_gfs != 6: cycledef = 'gdas' resources = self.get_resource('atmanlinit') From 19313796355ceba2631b6701513a37568996fe88 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Mon, 16 Sep 2024 23:45:59 -0500 Subject: [PATCH 08/25] Fix interval issue with GEFS XML generation --- workflow/rocoto/gefs_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_xml.py b/workflow/rocoto/gefs_xml.py index 18959f9da8..a5dfd5140e 100644 --- a/workflow/rocoto/gefs_xml.py +++ b/workflow/rocoto/gefs_xml.py @@ -16,7 +16,7 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: def get_cycledefs(self): sdate = self._base['SDATE_GFS'] edate = self._base['EDATE'] - interval = self.interval_gfs + interval = self._app_config.interval_gfs sdate_str = sdate.strftime("%Y%m%d%H%M") edate_str = edate.strftime("%Y%m%d%H%M") interval_str = timedelta_to_HMS(interval) From 1c951ad0eb6634154933f2c86a96c8f2aa1dfc9c Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Tue, 17 Sep 2024 11:32:19 -0500 Subject: [PATCH 09/25] Bugfixes for verif-global --- sorc/verif-global.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index 8f0428b441..b2ee80cac7 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit 8f0428b4415136b6f24f6bbd1b01c8e761c2b22f +Subproject commit b2ee80cac7921a3016fa5a857cc58acfccc4baea From 97e8605484497a6749b0174634650732d8182a9c Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Fri, 27 Sep 2024 19:53:56 -0500 Subject: [PATCH 10/25] Correct metp cycledefs To avoid running metp on days where there is no gfs, the cycledefs are adjusted somewhat. First, if the interval is >= 24h, the metp cycledef will be identical to gfs. If the interval is < 24h, it remains 18z every day (except the last). Second, a last_gfs is added so metp will run on for the last gfs cycle even if it there is no gdas cycle for 18z that day. This required computing the real gfs end date to use as the last cycle. --- workflow/rocoto/gfs_cycled_xml.py | 24 ++++++++++++++++-------- workflow/rocoto/gfs_tasks.py | 2 +- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/workflow/rocoto/gfs_cycled_xml.py b/workflow/rocoto/gfs_cycled_xml.py index b2c60fa124..d697c36817 100644 --- a/workflow/rocoto/gfs_cycled_xml.py +++ b/workflow/rocoto/gfs_cycled_xml.py @@ -28,7 +28,7 @@ def get_cycledefs(self): if interval_gfs > to_timedelta("0H"): sdate_gfs = self._base['SDATE_GFS'] - edate_gfs = edate + edate_gfs = sdate_gfs + (edate - sdate_gfs)//interval_gfs * interval_gfs sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") edate_gfs_str = edate_gfs.strftime("%Y%m%d%H%M") interval_gfs_str = timedelta_to_HMS(interval_gfs) @@ -40,14 +40,22 @@ def get_cycledefs(self): strings.append(f'\t{date2_gfs_str} {edate_gfs_str} {interval_gfs_str}') if self._base['DO_METP']: - # Run verification at 18z, no matter what - sdate_metp = sdate_gfs.replace(hour=18) - edate_metp = edate_gfs.replace(hour=18) - interval_metp = to_timedelta('24H') - sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") - edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") - interval_metp_str = timedelta_to_HMS(interval_metp) + if interval_gfs < to_timedelta('24H'): + # Run verification at 18z, no matter what if there is more than one gfs per day + sdate_metp = sdate_gfs.replace(hour=18) + edate_metp = (edate_gfs - to_timedelta('24H')).replace(hour=18) + interval_metp = to_timedelta('24H') + sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") + edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") + interval_metp_str = timedelta_to_HMS(interval_metp) + else: + # Use same cycledef as gfs if there is no more than one per day + sdate_metp_str = sdate_gfs.strftime("%Y%m%d%H%M") + edate_metp_str = edate_gfs.strftime("%Y%m%d%H%M") + interval_metp_str = timedelta_to_HMS(interval_gfs) + strings.append(f'\t{sdate_metp_str} {edate_metp_str} {interval_metp_str}') + strings.append(f'\t{edate_gfs_str} {edate_gfs_str} 24:00:00') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 1084cde3b9..5aeccf1a13 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1861,7 +1861,7 @@ def metp(self): 'resources': resources, 'dependency': dependencies, 'envars': metpenvars, - 'cycledef': 'metp', + 'cycledef': 'metp,last_gfs', 'command': f'{self.HOMEgfs}/jobs/rocoto/metp.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', From 98207f684d17e994991be3c549a47ce4bf55cde9 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Fri, 27 Sep 2024 20:12:32 -0500 Subject: [PATCH 11/25] Fix missing whitespace around operator --- workflow/rocoto/gfs_cycled_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_cycled_xml.py b/workflow/rocoto/gfs_cycled_xml.py index d697c36817..2852551e41 100644 --- a/workflow/rocoto/gfs_cycled_xml.py +++ b/workflow/rocoto/gfs_cycled_xml.py @@ -28,7 +28,7 @@ def get_cycledefs(self): if interval_gfs > to_timedelta("0H"): sdate_gfs = self._base['SDATE_GFS'] - edate_gfs = sdate_gfs + (edate - sdate_gfs)//interval_gfs * interval_gfs + edate_gfs = sdate_gfs + ((edate - sdate_gfs) // interval_gfs) * interval_gfs sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") edate_gfs_str = edate_gfs.strftime("%Y%m%d%H%M") interval_gfs_str = timedelta_to_HMS(interval_gfs) From bfc2907a662bb27d10f4c13c2141a62394256379 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Tue, 1 Oct 2024 00:02:34 -0500 Subject: [PATCH 12/25] Fix cleanup dependency when metp is not run that cycle --- workflow/rocoto/gfs_tasks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 5aeccf1a13..bd35a22ef0 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2334,8 +2334,12 @@ def cleanup(self): deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_metp and self.run in ['gfs']: + deps2 = [] + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}metp', 'condition': 'not'} + deps2.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': f'{self.run}metp'} - deps.append(rocoto.add_dependency(dep_dict)) + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='or', dep=deps2)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From 4ceec90f681d3bafc49ffc84f20e695138f97a3d Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Tue, 1 Oct 2024 23:03:40 -0500 Subject: [PATCH 13/25] Fix cleanup dependency for forecast-only --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index bd35a22ef0..01a33a6d33 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2333,7 +2333,7 @@ def cleanup(self): dep_dict = {'type': 'task', 'name': f'{self.run}npoess_pgrb2_0p5deg'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_metp and self.run in ['gfs']: + if self.app_config.mode in ['cycled'] and self.app_config.do_metp and self.run in ['gfs']: deps2 = [] dep_dict = {'type': 'taskvalid', 'name': f'{self.run}metp', 'condition': 'not'} deps2.append(rocoto.add_dependency(dep_dict)) From ec512b98861f0530ac0c8509293186f108c9fc0e Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 2 Oct 2024 16:55:24 -0500 Subject: [PATCH 14/25] Re-enable metp for forecast-only Turns metp back on for forecast-only mode. The metp cycledef follows the same practice of cycled, where if the interval is under 24, it will run every 18z. But if the interval is 24+, the cycledef will follow the gfs cycledef. Since there is no gdasarch task to key off of when gfsarch does not exist for the 18z cycle, the dependency is now more complicated and progressively works backwards to find a cycle that exists, then uses the gfsarch for that cycle. --- workflow/applications/gfs_forecast_only.py | 6 ++++ workflow/rocoto/gfs_forecast_only_xml.py | 35 +++++++++++++++++----- workflow/rocoto/gfs_tasks.py | 27 +++++++++++------ 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 3b120054c6..fb1d2cdb8f 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -42,6 +42,9 @@ def _get_app_configs(self): if self.do_genesis_fsu: configs += ['genesis_fsu'] + if self.do_metp: + configs += ['metp'] + if self.do_bufrsnd: configs += ['postsnd'] @@ -119,6 +122,9 @@ def get_task_names(self): if self.do_genesis_fsu: tasks += ['genesis_fsu'] + if self.do_metp: + tasks += ['metp'] + if self.do_bufrsnd: tasks += ['postsnd'] diff --git a/workflow/rocoto/gfs_forecast_only_xml.py b/workflow/rocoto/gfs_forecast_only_xml.py index dbfbbc9508..9ed76e2543 100644 --- a/workflow/rocoto/gfs_forecast_only_xml.py +++ b/workflow/rocoto/gfs_forecast_only_xml.py @@ -12,15 +12,36 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: super().__init__(app_config, rocoto_config) def get_cycledefs(self): - sdate = self._base['SDATE_GFS'] + sdate_gfs = self._base['SDATE_GFS'] edate = self._base['EDATE'] - interval = self._app_config.interval_gfs + interval_gfs = self._app_config.interval_gfs + edate_gfs = sdate_gfs + ((edate - sdate_gfs) // interval_gfs) * interval_gfs strings = [] - strings.append(f'\t{sdate.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') - - date2 = sdate + interval - if date2 <= edate: - strings.append(f'\t{date2.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval)}') + sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") + edate_gfs_str = edate_gfs.strftime("%Y%m%d%H%M") + strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {timedelta_to_HMS(interval_gfs)}') + + date2 = sdate_gfs + interval_gfs + if date2 <= edate_gfs: + strings.append(f'\t{date2.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval_gfs)}') + + if self._base['DO_METP']: + if interval_gfs < to_timedelta('24H'): + # Run verification at 18z, no matter what if there is more than one gfs per day + sdate_metp = sdate_gfs.replace(hour=18) + edate_metp = (edate_gfs - to_timedelta('24H')).replace(hour=18) + interval_metp = to_timedelta('24H') + sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") + edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") + interval_metp_str = timedelta_to_HMS(interval_metp) + else: + # Use same cycledef as gfs if there is no more than one per day + sdate_metp_str = sdate.strftime("%Y%m%d%H%M") + edate_metp_str = edate.strftime("%Y%m%d%H%M") + interval_metp_str = timedelta_to_HMS(interval_gfs) + + strings.append(f'\t{sdate_metp_str} {edate_metp_str} {interval_metp_str}') + strings.append(f'\t{edate_gfs_str} {edate_gfs_str} 24:00:00') strings.append('') strings.append('') diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 01a33a6d33..57b9f4daa5 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1,6 +1,6 @@ from applications.applications import AppConfig from rocoto.tasks import Tasks -from wxflow import timedelta_to_HMS +from wxflow import timedelta_to_HMS, to_timedelta import rocoto.rocoto as rocoto import numpy as np @@ -58,7 +58,7 @@ def prep(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) cycledef = self.run - if self.run in ['gfs'] and gfs_enkf and self.interval_gfs != 6: + if self.run in ['gfs'] and gfs_enkf and self.app_config.interval_gfs != 6: cycledef = 'gdas' resources = self.get_resource('prep') @@ -1834,13 +1834,22 @@ def metp(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.run}arch'} deps.append(rocoto.add_dependency(dep_dict)) - if self.run in "gfs": - deps2 = [] - dep_dict = {'type': 'taskvalid', 'name': f'{self.run}arch', 'condition': 'not'} - deps2.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'gdasarch'} - deps2.append(rocoto.add_dependency(dep_dict)) - deps.append(rocoto.create_dependency(dep_condition='and', dep=deps2)) + if self.app_config.interval_gfs < to_timedelta('24H'): + n_lookback = self.app_config.interval_gfs // to_timedelta('6H') + for lookback in range(1,n_lookback+1): + deps2 = [] + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}arch', 'condition': 'not'} + deps2.append(rocoto.add_dependency(dep_dict)) + for lookback2 in range(1,lookback): + offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback2}H')) + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': offset} + deps2.append(rocoto.add_dependency(dep_dict)) + + offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback}H')) + dep_dict = {'type': 'task', 'name': f'{self.run}arch', 'offset': offset} + deps2.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.create_dependency(dep_condition='and', dep=deps2)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) metpenvars = self.envars.copy() From 76b38632118e9b69b4c93d56f2b2526fcc3b065f Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 2 Oct 2024 17:06:51 -0500 Subject: [PATCH 15/25] Make metp tasks parallel again --- workflow/rocoto/gfs_tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 57b9f4daa5..a35913be8f 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1878,7 +1878,6 @@ def metp(self): } metatask_dict = {'task_name': f'{self.run}metp', - 'is_serial': True, 'task_dict': task_dict, 'var_dict': var_dict, } From a75175101dae0ce8cd761299d20460ba878a38d0 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 2 Oct 2024 17:19:15 -0500 Subject: [PATCH 16/25] Fix pynorms --- workflow/rocoto/gfs_forecast_only_xml.py | 3 ++- workflow/rocoto/gfs_tasks.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/gfs_forecast_only_xml.py b/workflow/rocoto/gfs_forecast_only_xml.py index 9ed76e2543..12f7ab4b5a 100644 --- a/workflow/rocoto/gfs_forecast_only_xml.py +++ b/workflow/rocoto/gfs_forecast_only_xml.py @@ -23,7 +23,8 @@ def get_cycledefs(self): date2 = sdate_gfs + interval_gfs if date2 <= edate_gfs: - strings.append(f'\t{date2.strftime("%Y%m%d%H%M")} {edate.strftime("%Y%m%d%H%M")} {timedelta_to_HMS(interval_gfs)}') + date2_gfs_str = date2_gfs.strftime("%Y%m%d%H%M") + strings.append(f'\t{date2_gfs_str} {edate_gfs_str} {timedelta_to_HMS(interval_gfs)}') if self._base['DO_METP']: if interval_gfs < to_timedelta('24H'): diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index a35913be8f..e60ccd0edd 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1836,11 +1836,11 @@ def metp(self): deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.interval_gfs < to_timedelta('24H'): n_lookback = self.app_config.interval_gfs // to_timedelta('6H') - for lookback in range(1,n_lookback+1): + for lookback in range(1, n_lookback + 1): deps2 = [] dep_dict = {'type': 'taskvalid', 'name': f'{self.run}arch', 'condition': 'not'} deps2.append(rocoto.add_dependency(dep_dict)) - for lookback2 in range(1,lookback): + for lookback2 in range(1, lookback): offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback2}H')) dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': offset} deps2.append(rocoto.add_dependency(dep_dict)) From 0c600afaf8ab231e10de2ad4a7247a209c4dcaf3 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 2 Oct 2024 19:12:08 -0500 Subject: [PATCH 17/25] Fix metp forecast-only cycledef --- workflow/rocoto/gfs_cycled_xml.py | 6 +++--- workflow/rocoto/gfs_forecast_only_xml.py | 17 ++++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/workflow/rocoto/gfs_cycled_xml.py b/workflow/rocoto/gfs_cycled_xml.py index 2852551e41..1d3458a8d9 100644 --- a/workflow/rocoto/gfs_cycled_xml.py +++ b/workflow/rocoto/gfs_cycled_xml.py @@ -50,9 +50,9 @@ def get_cycledefs(self): interval_metp_str = timedelta_to_HMS(interval_metp) else: # Use same cycledef as gfs if there is no more than one per day - sdate_metp_str = sdate_gfs.strftime("%Y%m%d%H%M") - edate_metp_str = edate_gfs.strftime("%Y%m%d%H%M") - interval_metp_str = timedelta_to_HMS(interval_gfs) + sdate_metp_str = sdate_gfs_str + edate_metp_str = edate_gfs_str + interval_metp_str = interval_gfs_str strings.append(f'\t{sdate_metp_str} {edate_metp_str} {interval_metp_str}') strings.append(f'\t{edate_gfs_str} {edate_gfs_str} 24:00:00') diff --git a/workflow/rocoto/gfs_forecast_only_xml.py b/workflow/rocoto/gfs_forecast_only_xml.py index 12f7ab4b5a..a4d5b0878b 100644 --- a/workflow/rocoto/gfs_forecast_only_xml.py +++ b/workflow/rocoto/gfs_forecast_only_xml.py @@ -13,36 +13,35 @@ def __init__(self, app_config: AppConfig, rocoto_config: Dict) -> None: def get_cycledefs(self): sdate_gfs = self._base['SDATE_GFS'] - edate = self._base['EDATE'] + edate_gfs = self._base['EDATE'] interval_gfs = self._app_config.interval_gfs - edate_gfs = sdate_gfs + ((edate - sdate_gfs) // interval_gfs) * interval_gfs strings = [] sdate_gfs_str = sdate_gfs.strftime("%Y%m%d%H%M") edate_gfs_str = edate_gfs.strftime("%Y%m%d%H%M") - strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {timedelta_to_HMS(interval_gfs)}') + interval_gfs_str = timedelta_to_HMS(interval_gfs) + strings.append(f'\t{sdate_gfs_str} {edate_gfs_str} {interval_gfs_str}') date2 = sdate_gfs + interval_gfs if date2 <= edate_gfs: date2_gfs_str = date2_gfs.strftime("%Y%m%d%H%M") - strings.append(f'\t{date2_gfs_str} {edate_gfs_str} {timedelta_to_HMS(interval_gfs)}') + strings.append(f'\t{date2_gfs_str} {edate_gfs_str} {interval_gfs_str}') if self._base['DO_METP']: if interval_gfs < to_timedelta('24H'): # Run verification at 18z, no matter what if there is more than one gfs per day sdate_metp = sdate_gfs.replace(hour=18) - edate_metp = (edate_gfs - to_timedelta('24H')).replace(hour=18) + edate_metp = edate_gfs.replace(hour=18) interval_metp = to_timedelta('24H') sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") interval_metp_str = timedelta_to_HMS(interval_metp) else: # Use same cycledef as gfs if there is no more than one per day - sdate_metp_str = sdate.strftime("%Y%m%d%H%M") - edate_metp_str = edate.strftime("%Y%m%d%H%M") - interval_metp_str = timedelta_to_HMS(interval_gfs) + sdate_metp_str = sdate_gfs_str + edate_metp_str = edate_gfs_str + interval_metp_str = interval_gfs_str strings.append(f'\t{sdate_metp_str} {edate_metp_str} {interval_metp_str}') - strings.append(f'\t{edate_gfs_str} {edate_gfs_str} 24:00:00') strings.append('') strings.append('') From f1b46b722257457f2eae3ba6d42bce70e3fcfce5 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 9 Oct 2024 13:00:34 -0500 Subject: [PATCH 18/25] Update new replay case for interval --- ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml index 1475e81ea0..6c807ff4d6 100644 --- a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml +++ b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml @@ -9,7 +9,7 @@ arguments: resdetocean: 1.0 resensatmos: 96 nens: 2 - gfs_cyc: 1 + interval: 6 start: warm comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR From 30c373b812470c620eae167c56d0955963596f9d Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Thu, 10 Oct 2024 14:19:55 -0500 Subject: [PATCH 19/25] Update metp dependencies for new task names --- workflow/rocoto/gfs_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index d38430eafb..86abd80c7b 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1838,7 +1838,7 @@ def metp(self): n_lookback = self.app_config.interval_gfs // to_timedelta('6H') for lookback in range(1, n_lookback + 1): deps2 = [] - dep_dict = {'type': 'taskvalid', 'name': f'{self.run}arch', 'condition': 'not'} + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_arch', 'condition': 'not'} deps2.append(rocoto.add_dependency(dep_dict)) for lookback2 in range(1, lookback): offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback2}H')) @@ -1846,7 +1846,7 @@ def metp(self): deps2.append(rocoto.add_dependency(dep_dict)) offset = timedelta_to_HMS(-to_timedelta(f'{6*lookback}H')) - dep_dict = {'type': 'task', 'name': f'{self.run}arch', 'offset': offset} + dep_dict = {'type': 'task', 'name': f'{self.run}_arch', 'offset': offset} deps2.append(rocoto.add_dependency(dep_dict)) deps.append(rocoto.create_dependency(dep_condition='and', dep=deps2)) From 93993914dc9f4abf813abc03b64934befdeb7548 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Thu, 10 Oct 2024 20:48:00 -0500 Subject: [PATCH 20/25] Fix extract vars dependency --- workflow/rocoto/gefs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index 955f631c8e..e9338c90df 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -472,7 +472,7 @@ def wavepostpnt(self): def extractvars(self): deps = [] if self.app_config.do_wave: - dep_dict = {'type': 'task', 'name': 'wave_post_grid_mem#member#'} + dep_dict = {'type': 'task', 'name': 'gefs_wave_post_grid_mem#member#'} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_ocean: dep_dict = {'type': 'metatask', 'name': 'gefs_ocean_prod_#member#'} From c258f4af4f0c2c951d2cb0f7f91c22ea488a06e0 Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Fri, 11 Oct 2024 13:09:34 -0500 Subject: [PATCH 21/25] Fix gfs_cleanup dependency While `taskvalid` does not complain about using a metatask name, using one will always return as not valid. --- workflow/rocoto/gfs_tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 86abd80c7b..ea64d2fa32 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -2341,9 +2341,10 @@ def cleanup(self): dep_dict = {'type': 'task', 'name': f'{self.run}_npoess_pgrb2_0p5deg'} deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.mode in ['cycled'] and self.app_config.do_metp and self.run in ['gfs']: + if self.app_config.do_metp and self.run in ['gfs']: deps2 = [] - dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_metp', 'condition': 'not'} + # taskvalid only handles regular tasks, so just check the first metp job exists + dep_dict = {'type': 'taskvalid', 'name': f'{self.run}_metpg2g1', 'condition': 'not'} deps2.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'metatask', 'name': f'{self.run}_metp'} deps2.append(rocoto.add_dependency(dep_dict)) From c9e3174a38d54fee7e965ff0568ab76d5fdda10b Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Fri, 11 Oct 2024 13:27:29 -0500 Subject: [PATCH 22/25] Make metp serial again metp is using non-standard DATA directories and the same for all tasks, so when one task remove the directory, that interferes with other jobs using it. --- workflow/rocoto/gfs_tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index ea64d2fa32..82dfb9f1d4 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -1878,6 +1878,7 @@ def metp(self): } metatask_dict = {'task_name': f'{self.run}_metp', + 'is_serial': True, 'task_dict': task_dict, 'var_dict': var_dict, } From ead6a5ef93313c4152a726df2e83cd55ca32db0a Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 15 Oct 2024 13:02:04 -0500 Subject: [PATCH 23/25] Run the metp jobs with one task each --- parm/config/gfs/config.resources | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 5024bc8873..d512f1f885 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -988,8 +988,8 @@ case ${step} in threads_per_task=1 walltime_gdas="03:00:00" walltime_gfs="06:00:00" - ntasks=4 - tasks_per_node=4 + ntasks=1 + tasks_per_node=1 export memory="80G" ;; From d54bc650f5e0a96a1385688135fd78c04bc99afb Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Wed, 16 Oct 2024 13:17:54 -0500 Subject: [PATCH 24/25] Fix cycled metp cycledef --- workflow/rocoto/gfs_cycled_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_cycled_xml.py b/workflow/rocoto/gfs_cycled_xml.py index 1d3458a8d9..eef77ba7fc 100644 --- a/workflow/rocoto/gfs_cycled_xml.py +++ b/workflow/rocoto/gfs_cycled_xml.py @@ -43,7 +43,7 @@ def get_cycledefs(self): if interval_gfs < to_timedelta('24H'): # Run verification at 18z, no matter what if there is more than one gfs per day sdate_metp = sdate_gfs.replace(hour=18) - edate_metp = (edate_gfs - to_timedelta('24H')).replace(hour=18) + edate_metp = edate_gfs.replace(hour=18) interval_metp = to_timedelta('24H') sdate_metp_str = sdate_metp.strftime("%Y%m%d%H%M") edate_metp_str = edate_metp.strftime("%Y%m%d%H%M") From e1976381af1393ac68b9dc64b82694549c8f135a Mon Sep 17 00:00:00 2001 From: "Walter.Kolczynski" Date: Thu, 17 Oct 2024 01:07:06 -0500 Subject: [PATCH 25/25] Turn off gefs replay test on WCOSS The UFS model executable is not built for the structured wave grid used by GEFS on WCOSS, so GEFS tests should not run on WCOSS. --- ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml index 6c807ff4d6..7118dde53f 100644 --- a/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml +++ b/ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml @@ -17,3 +17,6 @@ arguments: edate: 2020110100 yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_replay_ci.yaml icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610 + +skip_ci_on_hosts: + - wcoss2