From 51633b5257e6d75f364dae95eca2e3c00df055c6 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 7 Feb 2024 18:14:50 +0000 Subject: [PATCH 1/7] initial commit of ci yamls for JEDI hybatmDA case (#2294) --- ci/cases/pr/C48C48_ufs_hybatmDA.yaml | 22 ++++++++++++++++++++ ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml | 14 +++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 ci/cases/pr/C48C48_ufs_hybatmDA.yaml create mode 100644 ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml diff --git a/ci/cases/pr/C48C48_ufs_hybatmDA.yaml b/ci/cases/pr/C48C48_ufs_hybatmDA.yaml new file mode 100644 index 0000000000..1dd2a00c01 --- /dev/null +++ b/ci/cases/pr/C48C48_ufs_hybatmDA.yaml @@ -0,0 +1,22 @@ +experiment: + system: gfs + mode: cycled + +arguments: + pslot: {{ 'pslot' | getenv }} + app: ATM + resdetatmos: 48 + resensatmos: 48 + comroot: {{ 'RUNTESTS' | getenv }}/COMROOT + expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + idate: 2021032312 + edate: 2021032400 + nens: 2 + gfs_cyc: 1 + start: cold + yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults_ci.yaml + +skip_ci_on_hosts: + - orion + - hercules diff --git a/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml b/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml new file mode 100644 index 0000000000..c072c391b3 --- /dev/null +++ b/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml @@ -0,0 +1,14 @@ +defaults: + !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml +base: + DOIAU: "NO" + DO_JEDIATMVAR: "YES" + DO_JEDIATMENS: "YES" + ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} +esfc: + DONST: "NO" +nsst: + NST_MODEL: "1" +sfcanl: + DONST: "NO" + \ No newline at end of file From a41a0dba9a5b212ec2b36741839fb7d0cfffcc64 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 8 Feb 2024 18:14:14 +0000 Subject: [PATCH 2/7] updates to populate EXPDIR and ROTDIR for JEDI hybatmDA ci (#2294) --- ci/cases/pr/C48C48_ufs_hybatmDA.yaml | 6 +++--- parm/config/gfs/config.esfc | 5 +++++ parm/config/gfs/config.nsst | 5 +++++ parm/config/gfs/config.sfcanl | 5 +++++ workflow/setup_expt.py | 2 +- 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ci/cases/pr/C48C48_ufs_hybatmDA.yaml b/ci/cases/pr/C48C48_ufs_hybatmDA.yaml index 1dd2a00c01..b0f7a2908c 100644 --- a/ci/cases/pr/C48C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C48C48_ufs_hybatmDA.yaml @@ -9,13 +9,13 @@ arguments: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48C48 idate: 2021032312 edate: 2021032400 nens: 2 gfs_cyc: 1 - start: cold - yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults_ci.yaml + start: warm + yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml skip_ci_on_hosts: - orion diff --git a/parm/config/gfs/config.esfc b/parm/config/gfs/config.esfc index 2bb3d48bb4..7a5bce4681 100644 --- a/parm/config/gfs/config.esfc +++ b/parm/config/gfs/config.esfc @@ -16,4 +16,9 @@ if [ $DOIAU_ENKF = "YES" ]; then export DOSFCANL_ENKF="NO" fi +# Turn off NST in JEDIATMENS +if [ $DO_JEDIATMENS = "YES" ]; then + export DONST="NO" +fi + echo "END: config.esfc" diff --git a/parm/config/gfs/config.nsst b/parm/config/gfs/config.nsst index db4367b2c0..a85e87b47f 100644 --- a/parm/config/gfs/config.nsst +++ b/parm/config/gfs/config.nsst @@ -10,6 +10,11 @@ echo "BEGIN: config.nsst" # nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, 1 = ON but uncoupled, 2 = ON and coupled export NST_MODEL=2 +# Set NST_MODEL for JEDIATMVAR or JEDIATMENS +if [[ $DO_JEDIATMVAR = "YES" || $DO_JEDIATMENS = "YES" ]]; then + export NST_MODEL=1 +fi + # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON, export NST_SPINUP=0 cdate="${PDY}${cyc}" diff --git a/parm/config/gfs/config.sfcanl b/parm/config/gfs/config.sfcanl index 9592fb77c9..aa08f5a692 100644 --- a/parm/config/gfs/config.sfcanl +++ b/parm/config/gfs/config.sfcanl @@ -8,4 +8,9 @@ echo "BEGIN: config.sfcanl" # Get task specific resources . $EXPDIR/config.resources sfcanl +# Turn off NST in JEDIATMVAR +if [ $DO_JEDIATMVAR = "YES" ]; then + export DONST="NO" +fi + echo "END: config.sfcanl" diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 2bc41854d8..06af4d3d2b 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -219,7 +219,7 @@ def link_files_from_src_to_dst(src_dir, dst_dir): src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_anl_dir) dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_anl_dir) makedirs_if_missing(dst_dir) - for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat']: + for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat', 'atms_n20.satbias.nc4', 'atms_n20.satbias_cov.nc4', 'atms_n20.tlapse.txt' ]: fname = f'{inputs.cdump}.t{idatestr[8:]}z.{ftype}' src_file = os.path.join(src_dir, fname) if os.path.exists(src_file): From 38414ac980fae723bff79ae0d38b8e41cc4355af Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 9 Feb 2024 13:48:08 +0000 Subject: [PATCH 3/7] turn off monitor jobs for JEDIATMVAR, adjust JEDIATM resources, fix pynorm error (#2294) --- parm/config/gfs/config.base.emc.dyn | 7 +++++++ parm/config/gfs/config.resources | 20 +++++++++++--------- workflow/setup_expt.py | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn index 6e1e8339f0..022b765180 100644 --- a/parm/config/gfs/config.base.emc.dyn +++ b/parm/config/gfs/config.base.emc.dyn @@ -396,4 +396,11 @@ export FITSARC="YES" export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} +# The monitor jobs are not yet supported for JEDIATMVAR +if [[ ${DO_JEDIATMVAR} = "NO" ]]; then + export DO_VERFOZN="NO" # Ozone data assimilation monitoring + export DO_VERFRAD="NO" # Radiance data assimilation monitoring + export DO_VMINMON="NO" # GSI minimization monitoring +fi + echo "END: config.base" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index b746a4b32a..e3c5ff3185 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -91,7 +91,7 @@ case ${step} in ;; "prepatmiodaobs") - export wtime_prepatmiodaobs="00:10:00" + export wtime_prepatmiodaobs="00:30:00" export npe_prepatmiodaobs=1 export nth_prepatmiodaobs=1 export npe_node_prepatmiodaobs=$(( npe_node_max / nth_prepatmiodaobs )) @@ -196,8 +196,8 @@ case ${step} in "atmanlinit") # make below case dependent later - export layout_x=1 - export layout_y=1 + export layout_x=8 + export layout_y=8 export layout_gsib_x=$(( layout_x * 3 )) export layout_gsib_y=$(( layout_y * 2 )) @@ -212,8 +212,8 @@ case ${step} in "atmanlrun") # make below case dependent later - export layout_x=1 - export layout_y=1 + export layout_x=8 + export layout_y=8 export wtime_atmanlrun="00:30:00" export npe_atmanlrun=$(( layout_x * layout_y * 6 )) @@ -221,6 +221,7 @@ case ${step} in export nth_atmanlrun=1 export nth_atmanlrun_gfs=${nth_atmanlrun} export npe_node_atmanlrun=$(( npe_node_max / nth_atmanlrun )) + export memory_atmanlrun="96GB" export is_exclusive=True ;; @@ -833,8 +834,8 @@ case ${step} in "atmensanlinit") # make below case dependent later - export layout_x=1 - export layout_y=1 + export layout_x=8 + export layout_y=8 export wtime_atmensanlinit="00:10:00" export npe_atmensanlinit=1 @@ -845,8 +846,8 @@ case ${step} in "atmensanlrun") # make below case dependent later - export layout_x=1 - export layout_y=1 + export layout_x=8 + export layout_y=8 export wtime_atmensanlrun="00:30:00" export npe_atmensanlrun=$(( layout_x * layout_y * 6 )) @@ -854,6 +855,7 @@ case ${step} in export nth_atmensanlrun=1 export nth_atmensanlrun_gfs=${nth_atmensanlrun} export npe_node_atmensanlrun=$(( npe_node_max / nth_atmensanlrun )) + export memory_atmensanlrun="96GB" export is_exclusive=True ;; diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 7152403767..430ee5220a 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -219,7 +219,7 @@ def link_files_from_src_to_dst(src_dir, dst_dir): src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_anl_dir) dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_anl_dir) makedirs_if_missing(dst_dir) - for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat', 'atms_n20.satbias.nc4', 'atms_n20.satbias_cov.nc4', 'atms_n20.tlapse.txt' ]: + for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat', 'atms_n20.satbias.nc4', 'atms_n20.satbias_cov.nc4', 'atms_n20.tlapse.txt']: fname = f'{inputs.cdump}.t{idatestr[8:]}z.{ftype}' src_file = os.path.join(src_dir, fname) if os.path.exists(src_file): From 91b930886e332e53e98f5c281447a65cf89d30f0 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 13 Feb 2024 18:24:05 +0000 Subject: [PATCH 4/7] extend stage_ic to gdas for cycled parallels (#2294) --- ci/cases/pr/C48C48_ufs_hybatmDA.yaml | 1 - parm/config/gfs/config.base.emc.dyn | 2 +- parm/config/gfs/config.stage_ic | 9 +++++ scripts/exglobal_stage_ic.sh | 55 +++++++++++++++++++++++++++ ush/python/pygfs/task/atm_analysis.py | 2 +- workflow/applications/gfs_cycled.py | 5 ++- workflow/rocoto/gfs_tasks.py | 29 +++++++++----- 7 files changed, 88 insertions(+), 15 deletions(-) diff --git a/ci/cases/pr/C48C48_ufs_hybatmDA.yaml b/ci/cases/pr/C48C48_ufs_hybatmDA.yaml index b0f7a2908c..5ebedd5e69 100644 --- a/ci/cases/pr/C48C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C48C48_ufs_hybatmDA.yaml @@ -9,7 +9,6 @@ arguments: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48C48 idate: 2021032312 edate: 2021032400 nens: 2 diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn index 4ed5865be0..a3db3290d9 100644 --- a/parm/config/gfs/config.base.emc.dyn +++ b/parm/config/gfs/config.base.emc.dyn @@ -397,7 +397,7 @@ export FHMAX_FITS=132 [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]] && export FHMAX_FITS=${FHMAX_GFS} # The monitor jobs are not yet supported for JEDIATMVAR -if [[ ${DO_JEDIATMVAR} = "NO" ]]; then +if [[ ${DO_JEDIATMVAR} = "YES" ]]; then export DO_VERFOZN="NO" # Ozone data assimilation monitoring export DO_VERFRAD="NO" # Radiance data assimilation monitoring export DO_VMINMON="NO" # GSI minimization monitoring diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index 7f3956af4d..2583f831ac 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -7,6 +7,8 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic +# Stage ICs for coupled system +if [[ "${DO_COUPLED}" = "YES" ]]; then case "${CASE}" in "C48" | "C96") export CPL_ATMIC="workflow_${CASE}_refactored" @@ -37,5 +39,12 @@ case "${CASE}" in exit 1 ;; esac +fi # End block for coupled ICs + +# Stage ICs for cycled runs +if [[ "$MODE" = "cycled" ]]; then + export CPL_ATMIC="C48C48" + export CPL_ATMDAIC="C48C48" +fi echo "END: config.stage_ic" diff --git a/scripts/exglobal_stage_ic.sh b/scripts/exglobal_stage_ic.sh index 58b37f3114..e08e4c892c 100755 --- a/scripts/exglobal_stage_ic.sh +++ b/scripts/exglobal_stage_ic.sh @@ -25,6 +25,8 @@ error_message() { echo "FATAL ERROR: Unable to copy ${1} to ${2} (Error code ${3})" } +if [[ "${MODE}" == "forecast-only" ]]; then + ############################################################### for MEMDIR in "${MEMDIR_ARRAY[@]}"; do @@ -152,6 +154,59 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do +# Stage files for MODE="cycled" +else + # Stage deterministic ICs from previous cycle + RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_TOP:COM_TOP_TMPL + COM_TOP_BASE=$(dirname $COM_TOP) + [[ ! -d "${COM_TOP_BASE}" ]] && mkdir -p "${COM_TOP_BASE}" + [[ -d "${COM_TOP_BASE}/${gcyc}" ]] && rm -rf "${COM_TOP_BASE}/${gcyc}" + src="${BASE_CPLIC}/${CPL_ATMIC:-}/${RUN}.${gPDY}/${gcyc}" + tgt="${COM_TOP_BASE}/" + ${NLN} "${src}" "${tgt}" + rc=$? + ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" + err=$((err + rc)) + + # Stage ensemble ICs from previous cycle + if [[ "${DOHYBVAR}" == "YES" ]]; then + RUN=enkf${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_TOP:COM_TOP_TMPL + COM_TOP_BASE=$(dirname $COM_TOP) + [[ ! -d "${COM_TOP_BASE}" ]] && mkdir -p "${COM_TOP_BASE}" + [[ -d "${COM_TOP_BASE}/${gcyc}" ]] && rm -rf "${COM_TOP_BASE}/${gcyc}" + src="${BASE_CPLIC}/${CPL_ATMIC:-}/enkf${RUN}.${gPDY}/${gcyc}" + tgt="${COM_TOP_BASE}/" + ${NLN} "${src}" "${tgt}" + rc=$? + ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" + err=$((err + rc)) + fi + + # Stage bias correction files + RUN=${rCDUMP} YMD=${PDY} HH=${cyc} generate_com COM_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL + [[ ! -d "${COM_ATMOS_ANALYSIS}" ]] && mkdir -p "${COM_ATMOS_ANALYSIS}" + src="${BASE_CPLIC}/${CPL_ATMIC:-}/${RUN}.${PDY}/${cyc}/analysis/atmos" + tgt="${COM_ATMOS_ANALYSIS}/" + if [[ "${DO_JEDIATMVAR}" == "YES" || "${DO_JEDIATMENS}" == "YES" ]]; then + flist="radbcor" + else + flist="abias abias_air abias_pc radstat" + fi + for ftype in $flist; do + file=${rCDUMP}.t${cyc}z.$ftype + ${NCP} "${src}/$file" "${tgt}" + rc=$? + ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" + err=$((err + rc)) + done + if [[ "${DO_JEDIATMVAR}" == "YES" || "${DO_JEDIATMENS}" == "YES" ]]; then + cd ${tgt} + tar -xvf ${rCDUMP}.t${cyc}z.radbcor + cd ${DATA} + fi + +fi + ############################################################### # Check for errors and exit if any of the above failed if [[ "${err}" -ne 0 ]]; then diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index da41574fc9..6aed0533c6 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -94,7 +94,7 @@ def initialize(self: Analysis) -> None: 'NMEM_ENS', 'DATA', 'current_cycle', 'ntiles'] for key in keys: localconf[key] = self.task_config[key] - localconf.RUN = 'enkf' + self.task_config.RUN + localconf.RUN = 'enkfgdas' localconf.dirname = 'ens' FileHandler(self.get_fv3ens_dict(localconf)).sync() diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 6dd0342a78..7ad892bb68 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -35,7 +35,7 @@ def _get_app_configs(self): Returns the config_files that are involved in the cycled app """ - configs = ['prep'] + configs = ['stage_ic', 'prep'] if self.do_jediatmvar: configs += ['prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlfinal'] @@ -166,7 +166,8 @@ def get_task_names(self): hybrid_after_eupd_tasks += ['ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] # Collect all "gdas" cycle tasks - gdas_tasks = gdas_gfs_common_tasks_before_fcst.copy() + gdas_tasks = ['stage_ic'] + gdas_tasks += gdas_gfs_common_tasks_before_fcst.copy() if not self.do_jediatmvar: gdas_tasks += ['analdiag'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 9102c74e35..727145815c 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -24,14 +24,22 @@ def stage_ic(self): # Atm ICs if self.app_config.do_atm: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/atmos" - for file in ['gfs_ctrl.nc'] + \ - [f'{datatype}_data.tile{tile}.nc' - for datatype in ['gfs', 'sfc'] - for tile in range(1, self.n_tiles + 1)]: + if self.cdump in 'gdas': + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/gdas.@Y@m@d/@H/model_data/atmos/restart" + PDY = self._base.get('SDATE').strftime("%Y%m%d%H")[0:8] + cyc = self._base.get('SDATE').strftime("%Y%m%d%H")[8:10] + file = PDY + '.' + cyc + '0000.fv_core.res.nc' data = f"{prefix}/{file}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + else: + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/atmos" + for file in ['gfs_ctrl.nc'] + \ + [f'{datatype}_data.tile{tile}.nc' + for datatype in ['gfs', 'sfc'] + for tile in range(1, self.n_tiles + 1)]: + data = f"{prefix}/{file}" + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) else: # data-atmosphere # TODO - need more information about how these forcings are stored prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_DATM']}/@Y@m@d@H" @@ -70,13 +78,14 @@ def stage_ic(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + cycledef = 'gdas_half' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('stage_ic') task_name = f'{self.cdump}stage_ic' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.cdump, + 'cycledef': cycledef, 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -833,7 +842,7 @@ def _fcst_cycled(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) if self.cdump in ['gdas']: - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': f'{self.cdump}stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) @@ -2535,7 +2544,7 @@ def efcs(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}esfc'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'task', 'name': f'gdasstage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) From c102c6f55aeb0be8a0fc0da75881395f25281647 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 13 Feb 2024 20:28:23 +0000 Subject: [PATCH 5/7] correct shellcheck errors (#2294) --- parm/config/gfs/config.esfc | 2 +- parm/config/gfs/config.nsst | 2 +- parm/config/gfs/config.sfcanl | 2 +- parm/config/gfs/config.stage_ic | 2 +- scripts/exglobal_stage_ic.sh | 20 ++++++++++---------- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/parm/config/gfs/config.esfc b/parm/config/gfs/config.esfc index 7a5bce4681..7c32313758 100644 --- a/parm/config/gfs/config.esfc +++ b/parm/config/gfs/config.esfc @@ -17,7 +17,7 @@ if [ $DOIAU_ENKF = "YES" ]; then fi # Turn off NST in JEDIATMENS -if [ $DO_JEDIATMENS = "YES" ]; then +if [[ "${DO_JEDIATMENS}" == "YES" ]]; then export DONST="NO" fi diff --git a/parm/config/gfs/config.nsst b/parm/config/gfs/config.nsst index a85e87b47f..7bda81f058 100644 --- a/parm/config/gfs/config.nsst +++ b/parm/config/gfs/config.nsst @@ -11,7 +11,7 @@ echo "BEGIN: config.nsst" export NST_MODEL=2 # Set NST_MODEL for JEDIATMVAR or JEDIATMENS -if [[ $DO_JEDIATMVAR = "YES" || $DO_JEDIATMENS = "YES" ]]; then +if [[ "${DO_JEDIATMVAR}" == "YES" || "${DO_JEDIATMENS}" == "YES" ]]; then export NST_MODEL=1 fi diff --git a/parm/config/gfs/config.sfcanl b/parm/config/gfs/config.sfcanl index aa08f5a692..e2fde8992a 100644 --- a/parm/config/gfs/config.sfcanl +++ b/parm/config/gfs/config.sfcanl @@ -9,7 +9,7 @@ echo "BEGIN: config.sfcanl" . $EXPDIR/config.resources sfcanl # Turn off NST in JEDIATMVAR -if [ $DO_JEDIATMVAR = "YES" ]; then +if [[ "${DO_JEDIATMVAR}" == "YES" ]]; then export DONST="NO" fi diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index 2583f831ac..a99fa88bf6 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -42,7 +42,7 @@ esac fi # End block for coupled ICs # Stage ICs for cycled runs -if [[ "$MODE" = "cycled" ]]; then +if [[ "${MODE}" == "cycled" ]]; then export CPL_ATMIC="C48C48" export CPL_ATMDAIC="C48C48" fi diff --git a/scripts/exglobal_stage_ic.sh b/scripts/exglobal_stage_ic.sh index e08e4c892c..17c2e0d8dc 100755 --- a/scripts/exglobal_stage_ic.sh +++ b/scripts/exglobal_stage_ic.sh @@ -158,9 +158,9 @@ done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do else # Stage deterministic ICs from previous cycle RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_TOP:COM_TOP_TMPL - COM_TOP_BASE=$(dirname $COM_TOP) + COM_TOP_BASE=$(dirname "${COM_TOP}") [[ ! -d "${COM_TOP_BASE}" ]] && mkdir -p "${COM_TOP_BASE}" - [[ -d "${COM_TOP_BASE}/${gcyc}" ]] && rm -rf "${COM_TOP_BASE}/${gcyc}" + [[ -d "${COM_TOP_BASE}/${gcyc}" ]] && rm -f "${COM_TOP_BASE:-}/${gcyc:-}" src="${BASE_CPLIC}/${CPL_ATMIC:-}/${RUN}.${gPDY}/${gcyc}" tgt="${COM_TOP_BASE}/" ${NLN} "${src}" "${tgt}" @@ -171,9 +171,9 @@ else # Stage ensemble ICs from previous cycle if [[ "${DOHYBVAR}" == "YES" ]]; then RUN=enkf${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_TOP:COM_TOP_TMPL - COM_TOP_BASE=$(dirname $COM_TOP) + COM_TOP_BASE=$(dirname "${COM_TOP}") [[ ! -d "${COM_TOP_BASE}" ]] && mkdir -p "${COM_TOP_BASE}" - [[ -d "${COM_TOP_BASE}/${gcyc}" ]] && rm -rf "${COM_TOP_BASE}/${gcyc}" + [[ -d "${COM_TOP_BASE}/${gcyc}" ]] && rm -rf "${COM_TOP_BASE:-}/${gcyc:-}" src="${BASE_CPLIC}/${CPL_ATMIC:-}/enkf${RUN}.${gPDY}/${gcyc}" tgt="${COM_TOP_BASE}/" ${NLN} "${src}" "${tgt}" @@ -192,17 +192,17 @@ else else flist="abias abias_air abias_pc radstat" fi - for ftype in $flist; do - file=${rCDUMP}.t${cyc}z.$ftype - ${NCP} "${src}/$file" "${tgt}" + for ftype in ${flist}; do + file=${rCDUMP}.t${cyc}z.${ftype} + ${NCP} "${src}"/"${file}" "${tgt}" rc=$? ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" err=$((err + rc)) done if [[ "${DO_JEDIATMVAR}" == "YES" || "${DO_JEDIATMENS}" == "YES" ]]; then - cd ${tgt} - tar -xvf ${rCDUMP}.t${cyc}z.radbcor - cd ${DATA} + cd "${tgt}" || exit + tar -xvf "${rCDUMP}".t"${cyc}"z.radbcor + cd "${DATA}" || exit fi fi From ea200a4006a21d77491dd7c4dc5ee7f35bc8531d Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Wed, 14 Feb 2024 21:25:16 +0000 Subject: [PATCH 6/7] revert changes to extend stage_ic to gdas for cycled parallels (#2294) --- parm/config/gfs/config.stage_ic | 9 ----- scripts/exglobal_stage_ic.sh | 55 ----------------------------- workflow/applications/gfs_cycled.py | 5 ++- workflow/rocoto/gfs_tasks.py | 29 ++++++--------- workflow/setup_expt.py | 2 +- 5 files changed, 13 insertions(+), 87 deletions(-) diff --git a/parm/config/gfs/config.stage_ic b/parm/config/gfs/config.stage_ic index a99fa88bf6..7f3956af4d 100644 --- a/parm/config/gfs/config.stage_ic +++ b/parm/config/gfs/config.stage_ic @@ -7,8 +7,6 @@ echo "BEGIN: config.stage_ic" # Get task specific resources source "${EXPDIR}/config.resources" stage_ic -# Stage ICs for coupled system -if [[ "${DO_COUPLED}" = "YES" ]]; then case "${CASE}" in "C48" | "C96") export CPL_ATMIC="workflow_${CASE}_refactored" @@ -39,12 +37,5 @@ case "${CASE}" in exit 1 ;; esac -fi # End block for coupled ICs - -# Stage ICs for cycled runs -if [[ "${MODE}" == "cycled" ]]; then - export CPL_ATMIC="C48C48" - export CPL_ATMDAIC="C48C48" -fi echo "END: config.stage_ic" diff --git a/scripts/exglobal_stage_ic.sh b/scripts/exglobal_stage_ic.sh index 17c2e0d8dc..58b37f3114 100755 --- a/scripts/exglobal_stage_ic.sh +++ b/scripts/exglobal_stage_ic.sh @@ -25,8 +25,6 @@ error_message() { echo "FATAL ERROR: Unable to copy ${1} to ${2} (Error code ${3})" } -if [[ "${MODE}" == "forecast-only" ]]; then - ############################################################### for MEMDIR in "${MEMDIR_ARRAY[@]}"; do @@ -154,59 +152,6 @@ for MEMDIR in "${MEMDIR_ARRAY[@]}"; do done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do -# Stage files for MODE="cycled" -else - # Stage deterministic ICs from previous cycle - RUN=${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_TOP:COM_TOP_TMPL - COM_TOP_BASE=$(dirname "${COM_TOP}") - [[ ! -d "${COM_TOP_BASE}" ]] && mkdir -p "${COM_TOP_BASE}" - [[ -d "${COM_TOP_BASE}/${gcyc}" ]] && rm -f "${COM_TOP_BASE:-}/${gcyc:-}" - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${RUN}.${gPDY}/${gcyc}" - tgt="${COM_TOP_BASE}/" - ${NLN} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - - # Stage ensemble ICs from previous cycle - if [[ "${DOHYBVAR}" == "YES" ]]; then - RUN=enkf${rCDUMP} YMD=${gPDY} HH=${gcyc} generate_com COM_TOP:COM_TOP_TMPL - COM_TOP_BASE=$(dirname "${COM_TOP}") - [[ ! -d "${COM_TOP_BASE}" ]] && mkdir -p "${COM_TOP_BASE}" - [[ -d "${COM_TOP_BASE}/${gcyc}" ]] && rm -rf "${COM_TOP_BASE:-}/${gcyc:-}" - src="${BASE_CPLIC}/${CPL_ATMIC:-}/enkf${RUN}.${gPDY}/${gcyc}" - tgt="${COM_TOP_BASE}/" - ${NLN} "${src}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - fi - - # Stage bias correction files - RUN=${rCDUMP} YMD=${PDY} HH=${cyc} generate_com COM_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL - [[ ! -d "${COM_ATMOS_ANALYSIS}" ]] && mkdir -p "${COM_ATMOS_ANALYSIS}" - src="${BASE_CPLIC}/${CPL_ATMIC:-}/${RUN}.${PDY}/${cyc}/analysis/atmos" - tgt="${COM_ATMOS_ANALYSIS}/" - if [[ "${DO_JEDIATMVAR}" == "YES" || "${DO_JEDIATMENS}" == "YES" ]]; then - flist="radbcor" - else - flist="abias abias_air abias_pc radstat" - fi - for ftype in ${flist}; do - file=${rCDUMP}.t${cyc}z.${ftype} - ${NCP} "${src}"/"${file}" "${tgt}" - rc=$? - ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}" - err=$((err + rc)) - done - if [[ "${DO_JEDIATMVAR}" == "YES" || "${DO_JEDIATMENS}" == "YES" ]]; then - cd "${tgt}" || exit - tar -xvf "${rCDUMP}".t"${cyc}"z.radbcor - cd "${DATA}" || exit - fi - -fi - ############################################################### # Check for errors and exit if any of the above failed if [[ "${err}" -ne 0 ]]; then diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index a70abd14a2..040fc090cb 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -35,7 +35,7 @@ def _get_app_configs(self): Returns the config_files that are involved in the cycled app """ - configs = ['stage_ic', 'prep'] + configs = ['prep'] if self.do_jediatmvar: configs += ['prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlfinal'] @@ -166,8 +166,7 @@ def get_task_names(self): hybrid_after_eupd_tasks += ['ecen', 'esfc', 'efcs', 'epos', 'earc', 'cleanup'] # Collect all "gdas" cycle tasks - gdas_tasks = ['stage_ic'] - gdas_tasks += gdas_gfs_common_tasks_before_fcst.copy() + gdas_tasks = gdas_gfs_common_tasks_before_fcst.copy() if not self.do_jediatmvar: gdas_tasks += ['analdiag'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 35362d8fe5..83623f42d2 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -24,22 +24,14 @@ def stage_ic(self): # Atm ICs if self.app_config.do_atm: - if self.cdump in 'gdas': - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/gdas.@Y@m@d/@H/model_data/atmos/restart" - PDY = self._base.get('SDATE').strftime("%Y%m%d%H")[0:8] - cyc = self._base.get('SDATE').strftime("%Y%m%d%H")[8:10] - file = PDY + '.' + cyc + '0000.fv_core.res.nc' + prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/atmos" + for file in ['gfs_ctrl.nc'] + \ + [f'{datatype}_data.tile{tile}.nc' + for datatype in ['gfs', 'sfc'] + for tile in range(1, self.n_tiles + 1)]: data = f"{prefix}/{file}" - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} - else: - prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_ATMIC']}/@Y@m@d@H/atmos" - for file in ['gfs_ctrl.nc'] + \ - [f'{datatype}_data.tile{tile}.nc' - for datatype in ['gfs', 'sfc'] - for tile in range(1, self.n_tiles + 1)]: - data = f"{prefix}/{file}" - dep_dict = {'type': 'data', 'data': data} - deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) else: # data-atmosphere # TODO - need more information about how these forcings are stored prefix = f"{cpl_ic['BASE_CPLIC']}/{cpl_ic['CPL_DATM']}/@Y@m@d@H" @@ -78,14 +70,13 @@ def stage_ic(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - cycledef = 'gdas_half' if self.cdump in ['gdas'] else self.cdump resources = self.get_resource('stage_ic') task_name = f'{self.cdump}stage_ic' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': cycledef, + 'cycledef': self.cdump, 'command': f'{self.HOMEgfs}/jobs/rocoto/stage_ic.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -842,7 +833,7 @@ def _fcst_cycled(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) if self.cdump in ['gdas']: - dep_dict = {'type': 'task', 'name': f'{self.cdump}stage_ic'} + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) @@ -2540,7 +2531,7 @@ def efcs(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}esfc'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - dep_dict = {'type': 'task', 'name': f'gdasstage_ic'} + dep_dict = {'type': 'cycleexist', 'condition': 'not', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='or', dep=dependencies) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 430ee5220a..3eeb584f46 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -219,7 +219,7 @@ def link_files_from_src_to_dst(src_dir, dst_dir): src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_anl_dir) dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_anl_dir) makedirs_if_missing(dst_dir) - for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat', 'atms_n20.satbias.nc4', 'atms_n20.satbias_cov.nc4', 'atms_n20.tlapse.txt']: + for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat']: fname = f'{inputs.cdump}.t{idatestr[8:]}z.{ftype}' src_file = os.path.join(src_dir, fname) if os.path.exists(src_file): From a78ac19a5c1ed24e907d5c12353ee0e4d5c30c16 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 16 Feb 2024 14:17:57 +0000 Subject: [PATCH 7/7] skip C48C48_ufs_hybatmDA ci test on hera (#2294) --- ci/cases/pr/C48C48_ufs_hybatmDA.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/cases/pr/C48C48_ufs_hybatmDA.yaml b/ci/cases/pr/C48C48_ufs_hybatmDA.yaml index 5ebedd5e69..7d3644b1af 100644 --- a/ci/cases/pr/C48C48_ufs_hybatmDA.yaml +++ b/ci/cases/pr/C48C48_ufs_hybatmDA.yaml @@ -17,5 +17,6 @@ arguments: yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml skip_ci_on_hosts: + - hera - orion - hercules