From b957f4dd9a199712f9bab63563053148577d9ce9 Mon Sep 17 00:00:00 2001 From: Edward Snyder Date: Mon, 10 Apr 2023 14:18:23 +0000 Subject: [PATCH 1/7] added nemsio filenames and secondary nemsio archive path --- parm/data_locations.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/parm/data_locations.yml b/parm/data_locations.yml index 8d19240f00..3b57850925 100644 --- a/parm/data_locations.yml +++ b/parm/data_locations.yml @@ -128,6 +128,7 @@ GDAS: - /NCEPPROD/5year/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} archive_internal_dir: - ./enkfgdas.{yyyymmdd}/{hh}/atmos/mem{mem:03d} + - ./enkfgdas.{yyyymmdd}/{hh}/mem{mem:03d} archive_file_names: nemsio: anl: @@ -147,8 +148,10 @@ GDAS: nemsio: anl: - gdas.t{hh}z.atmf{fcst_hr:03d}.nemsio + - gdas.t{hh}z.sfcf{fcst_hr:03d}.nemsio fcst: - - gdas.t{hh}.sfcf{fcst_hr:03d}.nemsio + - gdas.t{hh}z.atmf{fcst_hr:03d}.nemsio + - gdas.t{hh}z.sfcf{fcst_hr:03d}.nemsio netcdf: anl: - gdas.t{hh}z.atmf{fcst_hr:03d}.nc From cbaf7b7692151df6b76365d7872b498ea6ab1b48 Mon Sep 17 00:00:00 2001 From: Edward Snyder Date: Mon, 10 Apr 2023 14:18:53 +0000 Subject: [PATCH 2/7] added nemsio logic --- scripts/exregional_make_ics.sh | 8 ++++++-- scripts/exregional_make_lbcs.sh | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 25c4a03f95..29f3817ba0 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -391,10 +391,14 @@ case "${EXTRN_MDL_NAME_ICS}" in ;; "GDAS") + if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then + input_type="gaussian_nemsio" + elif [ "${FV3GFS_FILE_FMT_ICS}" = "netcdf" ]; then + input_type="gaussian_netcdf" + fi + external_model="GFS" tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" - external_model="GFS" - input_type="gaussian_netcdf" convert_nst=False fn_atm="${EXTRN_MDL_FNS[0]}" fn_sfc="${EXTRN_MDL_FNS[1]}" diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index bb43a22714..f7fdad6b55 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -302,10 +302,14 @@ case "${EXTRN_MDL_NAME_LBCS}" in ;; "GDAS") + if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then + input_type="gaussian_nemsio" + elif [ "${FV3GFS_FILE_FMT_LBCS}" = "netcdf" ]; then + input_type="gaussian_netcdf" + fi tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" external_model="GFS" - input_type="gaussian_netcdf" fn_atm="${EXTRN_MDL_FNS[0]}" ;; From f129276139b0ffd988b7aad98738672d8ff00dc9 Mon Sep 17 00:00:00 2001 From: Edward Snyder Date: Mon, 10 Apr 2023 14:19:57 +0000 Subject: [PATCH 3/7] fixed bug in archive file loop --- ush/retrieve_data.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 081bfb138c..fce2a11fde 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -68,7 +68,6 @@ def clean_up_output_dir(expected_subdir, local_archive, output_path, source_path # If an archive exists on disk, remove it if os.path.exists(local_archive): os.remove(local_archive) - return unavailable @@ -587,12 +586,14 @@ def hpss_requested_files(cla, file_names, store_specs, members=-1, ens_group=-1) # additional files are reported as unavailable, then # something has gone wrong. unavailable = set.union(*unavailable.values()) - + # Report only the files that are truly unavailable + # Break loop if unexpected file was found and if files are found + # A successful file found returns 'set()' which != expected file path variable if not expected == unavailable: return unavailable - expected - - return {} + + return unavailable def load_str(arg): From 30427e81c829a07cedd6833d4abaa2e8e681b85e Mon Sep 17 00:00:00 2001 From: Edward Snyder Date: Mon, 10 Apr 2023 15:10:14 +0000 Subject: [PATCH 4/7] cleaned up comments --- ush/retrieve_data.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index fce2a11fde..8f873e087f 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -587,12 +587,13 @@ def hpss_requested_files(cla, file_names, store_specs, members=-1, ens_group=-1) # something has gone wrong. unavailable = set.union(*unavailable.values()) - # Report only the files that are truly unavailable - # Break loop if unexpected file was found and if files are found - # A successful file found returns 'set()' which != expected file path variable + # Break loop if unexpected files were found or if files were found + # A successful file found does not equal the expected file list and + # returns an empty set function. if not expected == unavailable: return unavailable - expected + # Returns unavailable files if the expected list equals the unavailable list return unavailable From 7d97cac92a6504f78377c00d0d3a2971d3fdbcad Mon Sep 17 00:00:00 2001 From: Edward Snyder Date: Mon, 10 Apr 2023 15:14:43 +0000 Subject: [PATCH 5/7] organized variables --- scripts/exregional_make_lbcs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index f7fdad6b55..e6dd529aa7 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -306,10 +306,10 @@ case "${EXTRN_MDL_NAME_LBCS}" in input_type="gaussian_nemsio" elif [ "${FV3GFS_FILE_FMT_LBCS}" = "netcdf" ]; then input_type="gaussian_netcdf" - fi + fi + external_model="GFS" tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]" - external_model="GFS" fn_atm="${EXTRN_MDL_FNS[0]}" ;; From f4be27fddb2f71ad464b30e7a5e92064c37bdd4d Mon Sep 17 00:00:00 2001 From: Edward Snyder Date: Mon, 10 Apr 2023 21:55:37 +0000 Subject: [PATCH 6/7] added new WE2E file for GDAS nemsio --- ..._fmt_nemsio_2020062900_ensemble_2mems.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_GDAS_lbcs_GDAS_fmt_nemsio_2020062900_ensemble_2mems.yaml diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_GDAS_lbcs_GDAS_fmt_nemsio_2020062900_ensemble_2mems.yaml b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_GDAS_lbcs_GDAS_fmt_nemsio_2020062900_ensemble_2mems.yaml new file mode 100644 index 0000000000..c5de75f145 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_GDAS_lbcs_GDAS_fmt_nemsio_2020062900_ensemble_2mems.yaml @@ -0,0 +1,27 @@ +metadata: + description: |- + This test checks the capability of the workflow to retrieve from NOAA + HPSS nemsio-formatted output files generated by GDAS. +user: + RUN_ENVIR: community +platform: + EXTRN_MDL_DATA_STORES: hpss +workflow: + CCPP_PHYS_SUITE: FV3_HRRR + PREDEF_GRID_NAME: RRFS_CONUS_3km + DATE_FIRST_CYCL: '2020062900' + DATE_LAST_CYCL: '2020062900' + FCST_LEN_HRS: 6 + PREEXISTING_DIR_METHOD: rename +task_get_extrn_ics: + EXTRN_MDL_NAME_ICS: GDAS + EXTRN_MDL_ICS_OFFSET_HRS: 6 + FV3GFS_FILE_FMT_ICS: nemsio +task_get_extrn_lbcs: + EXTRN_MDL_NAME_LBCS: GDAS + LBC_SPEC_INTVL_HRS: 6 + EXTRN_MDL_LBCS_OFFSET_HRS: 0 + FV3GFS_FILE_FMT_LBCS: nemsio +global: + DO_ENSEMBLE: true + NUM_ENS_MEMBERS: 2 From 0bb0c2e569377741530c0a13c7dd077d59233a23 Mon Sep 17 00:00:00 2001 From: Edward Snyder Date: Mon, 17 Apr 2023 19:26:38 +0000 Subject: [PATCH 7/7] revert back to original logic --- ush/retrieve_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 8f873e087f..d0ba405017 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -593,8 +593,8 @@ def hpss_requested_files(cla, file_names, store_specs, members=-1, ens_group=-1) if not expected == unavailable: return unavailable - expected - # Returns unavailable files if the expected list equals the unavailable list - return unavailable + # If this loop has completed successfully without returning early, then all files have been found + return {} def load_str(arg):