[develop] Introduce test cases from ufs-case-studies platform thru WE…

…2E (#822)

The UFS Case Studies Platform provides a set of cases that reveal the forecast challenges of NOAA's operational Global Forecast System (GFS). Here we introduce one of these cases, the 2020 Cold Air Damming, into the UFS SRW App through the WE2E testing framework. A yaml config file is added and moderate modifications are done for exregional_get_extrn_mdl_files.sh. This new function allows users to run any test cases from UFS Case Studies Platform directly through the WE2E framework without need of additional steps (e.g. download ICS/LBCS data from platform first). Users can still modify the yaml file to suit their needs (e.g. increase fcst time, play with different grid resolution or CCPP suite).

Additionally, we added CCPP-SCM user and technical guide as a reference in Section 8.2 for users who are interested in running single column model.

parm/wflow/coldstart.yaml

@@ -28,6 +28,7 @@ task_get_extrn_ics:
   join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
   memory: 2G
   nnodes: 1
+  native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}'
   partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'
   ppn: 1
   queue: '&QUEUE_HPSS;'
@@ -55,6 +56,7 @@ task_get_extrn_lbcs:
   join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
   memory: 2G
   nnodes: 1
+  native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}'
   partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'
   ppn: 1
   queue: '&QUEUE_HPSS;'

scripts/exregional_get_extrn_mdl_files.sh

@@ -62,7 +62,8 @@ if [ "${ICS_OR_LBCS}" = "ICS" ]; then
   fi
   fcst_hrs=${TIME_OFFSET_HRS}
   file_names=${EXTRN_MDL_FILES_ICS[@]}
-  if [ ${EXTRN_MDL_NAME} = FV3GFS ] || [ "${EXTRN_MDL_NAME}" == "GDAS" ] ; then
+  if [ ${EXTRN_MDL_NAME} = FV3GFS ] || [ "${EXTRN_MDL_NAME}" == "GDAS" ] \
+     || [ ${EXTRN_MDL_NAME} == "UFS-CASE-STUDY" ] ; then
     file_fmt=$FV3GFS_FILE_FMT_ICS
   fi
   input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_ICS:-$EXTRN_MDL_SYSBASEDIR_ICS}
@@ -91,7 +92,8 @@ elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then
 
   fcst_hrs="${first_time} ${last_time} ${LBC_SPEC_INTVL_HRS}"
   file_names=${EXTRN_MDL_FILES_LBCS[@]}
-  if [ ${EXTRN_MDL_NAME} = FV3GFS ] || [ "${EXTRN_MDL_NAME}" == "GDAS" ] ; then
+  if [ ${EXTRN_MDL_NAME} = FV3GFS ] || [ "${EXTRN_MDL_NAME}" == "GDAS" ] \
+     || [ ${EXTRN_MDL_NAME} == "UFS-CASE-STUDY" ] ; then
     file_fmt=$FV3GFS_FILE_FMT_LBCS
   fi
   input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS}
@@ -272,6 +274,30 @@ fi
 #
 #-----------------------------------------------------------------------
 #
+# unzip UFS-CASE-STUDY ICS/LBCS files
+#
+#-----------------------------------------------------------------------
+#
+if [ "${EXTRN_MDL_NAME}" = "UFS-CASE-STUDY" ]; then
+    # Look for filenames, if they exist, unzip them
+    base_path="${EXTRN_MDL_STAGING_DIR}${mem_dir}"
+    for filename in ${base_path}/*.tar.gz; do
+        printf "unzip file: ${filename}\n"
+        tar -zxvf ${filename} --directory ${base_path}
+    done
+    # check file naming issue
+    for filename in ${base_path}/*.nemsio; do
+        filename=$(basename -- "${filename}")
+        len=`echo $filename | wc -c`
+        if [ "${filename:4:4}" != "t${hh}z" ]; then
+            printf "rename ${filename} to ${filename:0:4}t${hh}z.${filename:4:${len}} \n"
+           mv ${base_path}/${filename} ${base_path}/${filename:0:4}t${hh}z.${filename:4:${len}}
+        fi
+    done
+fi
+#
+#-----------------------------------------------------------------------
+#
 # Restore the shell options saved at the beginning of this script/function.
 #
 #-----------------------------------------------------------------------

scripts/exregional_make_ics.sh

@@ -131,6 +131,7 @@ case "${CCPP_PHYS_SUITE}" in
       varmap_file="GSDphys_var_map.txt"
     elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ] || \
          [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \
+         [ "${EXTRN_MDL_NAME_ICS}" = "UFS-CASE-STUDY" ] || \
          [ "${EXTRN_MDL_NAME_ICS}" = "GEFS" ] || \
          [ "${EXTRN_MDL_NAME_ICS}" = "GDAS" ] || \
          [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then
@@ -391,6 +392,26 @@ case "${EXTRN_MDL_NAME_ICS}" in
   tg3_from_soil=False
   ;;
 
+"UFS-CASE-STUDY")
+  hh="${EXTRN_MDL_CDATE:8:2}"
+  if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then
+    external_model="UFS-CASE-STUDY"
+    input_type="gaussian_nemsio"
+    tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]"
+    tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]"
+    fn_atm="gfs.t${hh}z.atmanl.nemsio"
+    fn_sfc="gfs.t${hh}z.sfcanl.nemsio"
+    convert_nst=True
+  fi
+  vgtyp_from_climo=True
+  sotyp_from_climo=True
+  vgfrc_from_climo=True
+  minmax_vgfrc_from_climo=True
+  lai_from_climo=True
+  tg3_from_soil=False
+  unset hh
+  ;;
+
 "GDAS")
   if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then
     input_type="gaussian_nemsio"

scripts/exregional_make_lbcs.sh

@@ -129,6 +129,7 @@ case "${CCPP_PHYS_SUITE}" in
       varmap_file="GSDphys_var_map.txt"
     elif [ "${EXTRN_MDL_NAME_LBCS}" = "NAM" ] || \
          [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \
+         [ "${EXTRN_MDL_NAME_LBCS}" = "UFS-CASE-STUDY" ] || \
          [ "${EXTRN_MDL_NAME_LBCS}" = "GEFS" ] || \
          [ "${EXTRN_MDL_NAME_LBCS}" = "GDAS" ] || \
          [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then
@@ -293,6 +294,15 @@ case "${EXTRN_MDL_NAME_LBCS}" in
   fi
   ;;
 
+"UFS-CASE-STUDY")
+  if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then
+    external_model="UFS-CASE-STUDY"
+    input_type="gaussian_nemsio"     # For FV3GFS data on a Gaussian grid in nemsio format.
+    tracers_input="[\"spfh\",\"clwmr\",\"o3mr\",\"icmr\",\"rwmr\",\"snmr\",\"grle\"]"
+    tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]"
+  fi
+  ;;
+
 "GDAS")
   if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then
     input_type="gaussian_nemsio"
@@ -389,6 +399,14 @@ for (( ii=0; ii<${num_fhrs}; ii=ii+bcgrpnum10 )); do
       fn_atm="${EXTRN_MDL_FNS[$i]}"
     fi
     ;;
+  "UFS-CASE-STUDY")
+    if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then
+      hh="${EXTRN_MDL_CDATE:8:2}"
+      fhr_str=$(printf "%03d" ${fhr})
+      fn_atm="gfs.t${hh}z.atmf${fhr_str}.nemsio"
+      unset hh fhr_str
+    fi
+    ;;
   "GDAS")
     fn_atm="${EXTRN_MDL_FNS[0][$i]}"
     ;;

tests/WE2E/machine_suites/comprehensive.orion

@@ -56,3 +56,4 @@ nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR
 pregen_grid_orog_sfc_climo
 specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS
 specify_template_filenames
+2020_CAD

tests/WE2E/machine_suites/coverage.orion

@@ -9,3 +9,4 @@ grid_RRFS_CONUScompact_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
 grid_RRFS_CONUScompact_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16
 grid_SUBCONUS_Ind_3km_ics_FV3GFS_lbcs_FV3GFS_suite_WoFS_v0
 nco
+2020_CAD

tests/WE2E/test_configs/ufs_case_studies/config.2020_CAD.yaml

@@ -0,0 +1,42 @@
+metadata:
+  description: |-
+    This test is to ensure that the workflow running in community mode
+    completes successfully on the RRFS_CONUS_13km grid using the GFS_v16
+    physics suite with ICs and LBCs derived from the UFS-CASE-STUDIES platforms for 2020 Cold Air Damming case.
+    #Note for runnning this test on Cheyenne: please modify this config as follows
+    #task_get_extrn_ics:
+    #  EXTRN_MDL_NAME_ICS: FV3GFS
+    #  FV3GFS_FILE_FMT_ICS: nemsio
+    #  USE_USER_STAGED_EXTRN_FILES: true
+    #task_get_extrn_lbcs:
+    #  EXTRN_MDL_NAME_LBCS: FV3GFS
+    #  LBC_SPEC_INTVL_HRS: 3
+    #  FV3GFS_FILE_FMT_LBCS: nemsio
+    #  USE_USER_STAGED_EXTRN_FILES: true
+user:
+  RUN_ENVIR: community
+platform:
+  EXTRN_MDL_DATA_STORES: aws
+workflow:
+  CCPP_PHYS_SUITE: FV3_GFS_v16 
+  PREDEF_GRID_NAME: RRFS_CONUS_13km
+  DATE_FIRST_CYCL: '2020020312'
+  DATE_LAST_CYCL: '2020020312'
+  FCST_LEN_HRS: 6
+  PREEXISTING_DIR_METHOD: rename
+task_get_extrn_ics:
+  EXTRN_MDL_NAME_ICS: UFS-CASE-STUDY
+  FV3GFS_FILE_FMT_ICS: nemsio
+task_get_extrn_lbcs:
+  EXTRN_MDL_NAME_LBCS: UFS-CASE-STUDY
+  LBC_SPEC_INTVL_HRS: 3
+  FV3GFS_FILE_FMT_LBCS: nemsio
+rocoto:
+  tasks:
+    task_get_extrn_ics:
+      walltime: 06:00:00
+    task_get_extrn_lbcs:
+      walltime: 06:00:00
+    metatask_run_ensemble:
+      task_make_lbcs_mem#mem#:
+        walltime: 06:00:00

ush/machine/gaea.yaml

@@ -19,6 +19,7 @@ platform:
   RUN_CMD_SERIAL: time
   RUN_CMD_UTILS: srun --export=ALL --mpi=pmi2 -n $nprocs
   SCHED_NATIVE_CMD: -M c3 --export=NONE
+  SCHED_NATIVE_CMD_HPSS: -M es --export=NONE
   PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
   TEST_EXTRN_MDL_SOURCE_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data
   TEST_PREGEN_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen

ush/valid_param_vals.yaml

@@ -47,8 +47,8 @@ valid_vals_CCPP_PHYS_SUITE: [
 "FV3_RAP"
 ] 
 valid_vals_GFDLgrid_NUM_CELLS: [48, 96, 192, 384, 768, 1152, 3072]
-valid_vals_EXTRN_MDL_NAME_ICS: ["GSMGFS", "FV3GFS", "GEFS", "GDAS", "RAP", "HRRR", "NAM"]
-valid_vals_EXTRN_MDL_NAME_LBCS: ["GSMGFS", "FV3GFS", "GEFS", "GDAS", "RAP", "HRRR", "NAM"]
+valid_vals_EXTRN_MDL_NAME_ICS: ["GSMGFS", "FV3GFS", "UFS-CASE-STUDY", "GEFS", "GDAS", "RAP", "HRRR", "NAM"]
+valid_vals_EXTRN_MDL_NAME_LBCS: ["GSMGFS", "FV3GFS", "UFS-CASE-STUDY", "GEFS", "GDAS", "RAP", "HRRR", "NAM"]
 valid_vals_USE_USER_STAGED_EXTRN_FILES: [True, False]
 valid_vals_FV3GFS_FILE_FMT_ICS: ["nemsio", "grib2", "netcdf"]
 valid_vals_FV3GFS_FILE_FMT_LBCS: ["nemsio", "grib2", "netcdf"]