[develop] Add GDAS nemsio data type for initial conditions

parm/data_locations.yml

@@ -128,6 +128,7 @@ GDAS:
       - /NCEPPROD/5year/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd}
     archive_internal_dir:
       - ./enkfgdas.{yyyymmdd}/{hh}/atmos/mem{mem:03d}
+      - ./enkfgdas.{yyyymmdd}/{hh}/mem{mem:03d}
     archive_file_names:
       nemsio:
         anl:
@@ -147,8 +148,10 @@ GDAS:
       nemsio:
         anl:
           - gdas.t{hh}z.atmf{fcst_hr:03d}.nemsio
+          - gdas.t{hh}z.sfcf{fcst_hr:03d}.nemsio
         fcst:
-          - gdas.t{hh}.sfcf{fcst_hr:03d}.nemsio
+          - gdas.t{hh}z.atmf{fcst_hr:03d}.nemsio
+          - gdas.t{hh}z.sfcf{fcst_hr:03d}.nemsio
       netcdf:
         anl:
           - gdas.t{hh}z.atmf{fcst_hr:03d}.nc

scripts/exregional_make_ics.sh

@@ -391,10 +391,14 @@ case "${EXTRN_MDL_NAME_ICS}" in
   ;;
 
 "GDAS")
+  if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then
+    input_type="gaussian_nemsio"
+  elif [ "${FV3GFS_FILE_FMT_ICS}" = "netcdf" ]; then
+    input_type="gaussian_netcdf"
+  fi
+  external_model="GFS"
   tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]"
   tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]"
-  external_model="GFS"
-  input_type="gaussian_netcdf"
   convert_nst=False
   fn_atm="${EXTRN_MDL_FNS[0]}"
   fn_sfc="${EXTRN_MDL_FNS[1]}"

scripts/exregional_make_lbcs.sh

@@ -302,10 +302,14 @@ case "${EXTRN_MDL_NAME_LBCS}" in
   ;;
 
 "GDAS")
+  if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then
+    input_type="gaussian_nemsio"
+  elif [ "${FV3GFS_FILE_FMT_LBCS}" = "netcdf" ]; then
+    input_type="gaussian_netcdf"
+  fi 
+  external_model="GFS" 
   tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]"
   tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]"
-  external_model="GFS"
-  input_type="gaussian_netcdf"
   fn_atm="${EXTRN_MDL_FNS[0]}"
   ;;
 

tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_GDAS_lbcs_GDAS_fmt_nemsio_2020062900_ensemble_2mems.yaml

@@ -0,0 +1,27 @@
+metadata:
+  description: |-
+    This test checks the capability of the workflow to retrieve from NOAA
+    HPSS nemsio-formatted output files generated by GDAS.
+user:
+  RUN_ENVIR: community
+platform:
+  EXTRN_MDL_DATA_STORES: hpss
+workflow:
+  CCPP_PHYS_SUITE: FV3_HRRR
+  PREDEF_GRID_NAME: RRFS_CONUS_3km
+  DATE_FIRST_CYCL: '2020062900'
+  DATE_LAST_CYCL: '2020062900'
+  FCST_LEN_HRS: 6
+  PREEXISTING_DIR_METHOD: rename
+task_get_extrn_ics:
+  EXTRN_MDL_NAME_ICS: GDAS
+  EXTRN_MDL_ICS_OFFSET_HRS: 6
+  FV3GFS_FILE_FMT_ICS: nemsio
+task_get_extrn_lbcs:
+  EXTRN_MDL_NAME_LBCS: GDAS
+  LBC_SPEC_INTVL_HRS: 6
+  EXTRN_MDL_LBCS_OFFSET_HRS: 0
+  FV3GFS_FILE_FMT_LBCS: nemsio
+global:
+  DO_ENSEMBLE: true
+  NUM_ENS_MEMBERS: 2

ush/retrieve_data.py

@@ -68,7 +68,6 @@ def clean_up_output_dir(expected_subdir, local_archive, output_path, source_path
     # If an archive exists on disk, remove it
     if os.path.exists(local_archive):
         os.remove(local_archive)
-
     return unavailable
 
 
@@ -587,11 +586,14 @@ def hpss_requested_files(cla, file_names, store_specs, members=-1, ens_group=-1)
             # additional files are reported as unavailable, then
             # something has gone wrong.
             unavailable = set.union(*unavailable.values())
-
-        # Report only the files that are truly unavailable
+
+        # Break loop if unexpected files were found or if files were found
+        # A successful file found does not equal the expected file list and 
+        # returns an empty set function.
         if not expected == unavailable:
             return unavailable - expected
-
+
+    # If this loop has completed successfully without returning early, then all files have been found
     return {}