[develop] Introduce DA data preprocessing to workflow

.github/workflows/python_linter.yaml

@@ -34,4 +34,4 @@ jobs:
       - name: Lint the test directory
         run: |
           export PYTHONPATH=${PWD}/ush
-          pylint --ignore-imports=yes tests/test_python/
+          pylint --min-similarity-lines=15 --ignore-imports=yes tests/test_python/

jobs/JREGIONAL_GET_DA_OBS

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+#
+#-----------------------------------------------------------------------
+#
+# Source the variable definitions file and the bash utility functions.
+#
+#-----------------------------------------------------------------------
+#
+. $USHdir/source_util_funcs.sh
+source_config_for_task "task_run_anl|task_run_enkf" ${GLOBAL_VAR_DEFNS_FP}
+. $USHdir/job_preamble.sh
+#
+#-----------------------------------------------------------------------
+#
+# Save current shell options (in a global array).  Then set new options
+# for this script/function.
+#
+#-----------------------------------------------------------------------
+#
+{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1
+#
+#-----------------------------------------------------------------------
+#
+# Get the full path to the file in which this script/function is located 
+# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in
+# which the file is located (scrfunc_dir).
+#
+#-----------------------------------------------------------------------
+#
+scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" )
+scrfunc_fn=$( basename "${scrfunc_fp}" )
+scrfunc_dir=$( dirname "${scrfunc_fp}" )
+#
+#-----------------------------------------------------------------------
+#
+# Print message indicating entry into script.
+#
+#-----------------------------------------------------------------------
+#
+print_info_msg "
+========================================================================
+Entering script:  \"${scrfunc_fn}\"
+In directory:     \"${scrfunc_dir}\"
+
+This script retrieves observation data for RRFS data assimilation tasks.
+========================================================================"
+
+#
+#-----------------------------------------------------------------------
+#
+# Create the directory where the GSI obs files should be stored
+#
+#-----------------------------------------------------------------------
+#
+export DATA="${COMIN}/obs"
+mkdir_vrfy -p "${DATA}"
+
+# Set needed date/time variables
+export START_DATE=$(echo "${PDY} ${cyc}")
+export YYYYMMDDHH=$(date +%Y%m%d%H -d "${START_DATE}")
+
+#
+#-----------------------------------------------------------------------
+#
+# Call the ex-script for this J-job
+#
+#-----------------------------------------------------------------------
+#
+$SCRIPTSdir/exregional_get_da_obs.sh || \
+print_err_msg_exit "\
+Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
+#
+#-----------------------------------------------------------------------
+#
+# Run job postamble.
+#
+#-----------------------------------------------------------------------
+#
+job_postamble
+#
+#-----------------------------------------------------------------------
+#
+# Restore the shell options saved at the beginning of this script/function.
+#
+#-----------------------------------------------------------------------
+#
+{ restore_shell_opts; } > /dev/null 2>&1
+

jobs/JREGIONAL_GET_EXTRN_MDL_FILES

@@ -216,7 +216,7 @@ esac
 #
 #-----------------------------------------------------------------------
 #
-# Create the directory where the exetrnal model files should be stored
+# Create the directory where the external model files should be stored
 #
 #-----------------------------------------------------------------------
 #

jobs/JREGIONAL_PROCESS_BUFR → jobs/JREGIONAL_PROCESS_BUFROBS

@@ -29,7 +29,7 @@
 #-----------------------------------------------------------------------
 #
 . $USHdir/source_util_funcs.sh
-source_config_for_task "task_process_bufr" ${GLOBAL_VAR_DEFNS_FP}
+source_config_for_task "task_process_bufrobs" ${GLOBAL_VAR_DEFNS_FP}
 . $USHdir/job_preamble.sh "TRUE"
 #
 #-----------------------------------------------------------------------
@@ -75,11 +75,16 @@ the specified cycle.
 #-----------------------------------------------------------------------
 #
 if [ ${CYCLE_TYPE} == "spinup" ]; then
-  DATA="${DATA:-${COMIN}/process_bufr_spinup}"
+  DATA="${DATA:-${COMIN}/process_bufrobs_spinup}"
 else
-  DATA="${DATA:-${COMIN}/process_bufr}"
+  DATA="${DATA:-${COMIN}/process_bufrobs}"
 fi
 mkdir_vrfy -p ${DATA}
+
+# Set needed date/time variables
+export START_DATE=$(echo "${PDY} ${cyc}")
+export YYYYMMDDHH=$(date +%Y%m%d%H -d "${START_DATE}")
+
 #
 #-----------------------------------------------------------------------
 #
@@ -88,7 +93,7 @@ mkdir_vrfy -p ${DATA}
 #
 #-----------------------------------------------------------------------
 #
-$SCRIPTSdir/exregional_process_bufr.sh || print_err_msg_exit "\
+$SCRIPTSdir/exregional_process_bufrobs.sh || print_err_msg_exit "\
 Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_PROCESS_LIGHTNING

@@ -16,7 +16,7 @@
 #-----------------------------------------------------------------------
 #
 . ${USHdir}/source_util_funcs.sh
-source_config_for_task "task_process_lightning" ${GLOBAL_VAR_DEFNS_FP}
+source_config_for_task "task_process_bufrobs" ${GLOBAL_VAR_DEFNS_FP}
 . ${USHdir}/job_preamble.sh "TRUE"
 #
 #-----------------------------------------------------------------------

modulefiles/tasks/hera/get_da_obs.local.lua

@@ -0,0 +1,2 @@
+load("hpss")
+load("python_srw")

modulefiles/tasks/jet/get_da_obs.local.lua

@@ -0,0 +1,2 @@
+load("hpss")
+load("python_srw")

parm/data_locations.yml

@@ -322,10 +322,12 @@ RAP_obs:
     archive_format: zip
     archive_path:
       - /BMC/fdr/Permanent/{yyyy}/{mm}/{dd}/data/grids/rap/obs
+      - /BMC/fdr/Permanent/{yyyy}/{mm}/{dd}/data/grids/rap/prepbufr
     archive_internal_dir:
-      - ./
+      - ""
     archive_file_names:
       - "{yyyymmddhh}00.zip"
+      - "{yyyymmddhh}00.zip"
     file_names:
       obs:
         - "{yyyymmddhh}.rap.t{hh}z.prepbufr.tm00"
@@ -358,6 +360,13 @@ RAP_obs:
         - "{yyyymmddhh}.rap.t{hh}z.satwnd.tm00.bufr_d"
         - "{yyyymmddhh}.rap.t{hh}z.sevasr.tm00.bufr_d"
         - "{yyyymmddhh}.rap.t{hh}z.ssmisu.tm00.bufr_d"
+        - "{yyyymmddhh}.rap_e.t{hh}z.prepbufr.tm00"
+        - "{yyyymmddhh}.rap_e.t{hh}z.1bamua.tm00.bufr_d"
+        - "{yyyymmddhh}.rap_e.t{hh}z.1bhrs4.tm00.bufr_d"
+        - "{yyyymmddhh}.rap_e.t{hh}z.1bmhs.tm00.bufr_d"
+        - "{yyyymmddhh}.rap_e.t{hh}z.lgycld.tm00.bufr_d"
+        - "{yyyymmddhh}.rap_e.t{hh}z.nexrad.tm00.bufr_d"
+        - "{yyyymmddhh}.rap_e.t{hh}z.satwnd.tm00.bufr_d"
   aws:
     protocol: download
     url: https://noaa-rap-pds.s3.amazonaws.com/rap.{yyyymmdd}

parm/wflow/coldstart.yaml

@@ -213,4 +213,3 @@ metatask_run_ensemble:
           taskdep:
             attrs:
               task: aqm_lbcs
-

parm/wflow/da_data_preproc.yaml

@@ -0,0 +1,102 @@
+# This group contains all the tasks needed for preprocessing tasks for
+# RRFS DA.
+
+default_data_preproc_task: &default_preproc
+  account: '&ACCOUNT;'
+  attrs:
+    cycledefs: #cycledefs_type#
+    maxtries: '1'
+  envars: &default_envars
+    GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;'
+    USHdir: '&USHdir;'
+    PDY: !cycstr "@Y@m@d"
+    cyc: !cycstr "@H"
+    subcyc: !cycstr "@M"
+    LOGDIR: !cycstr "&LOGDIR;"
+    CYCLE_TYPE: '#cycle_type#'
+  native: '{{ platform.SCHED_NATIVE_CMD }}'
+  nodes: '{{ nnodes }}:ppn={{ ppn }}'
+  nnodes: 1
+  nodesize: "&NCORES_PER_NODE;"
+  ppn: 1
+  partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}'
+  queue: '&QUEUE_DEFAULT;'
+  walltime: 00:25:00
+
+
+task_get_da_obs:
+  <<: *default_preproc
+  command: '&LOAD_MODULES_RUN_TASK_FP; "get_da_obs" "&JOBSdir;/JREGIONAL_GET_DA_OBS"'
+  attrs:
+    cycledefs: forecast
+    maxtries: '1'
+  join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
+  partition: '&PARTITION_HPSS;'
+  queue: '&QUEUE_HPSS;'
+  dependency:
+    timedep: '<cyclestr offset="&START_TIME_CONVENTIONAL;">@Y@m@d@H@M00</cyclestr>'
+
+metatask_process_obs_cycle_type:
+  var:
+    cycledefs_type: forecast,long_forecast
+    cycle_type: prod
+
+  task_process_radarref_#cycle_type#:
+    <<: *default_preproc
+    command: '&LOAD_MODULES_RUN_TASK_FP; "process_obs" "&JOBSdir;/JREGIONAL_PROCESS_RADARREF"'
+    ppn: 24
+    join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
+    dependency:
+      or:
+        taskdep:
+          attrs:
+            task: get_da_obs
+        and:
+          not:
+            taskvalid:
+              attrs:
+                task: get_da_obs
+          streq:
+            left: do_real_time
+            right: '{% if workflow.DO_REAL_TIME %}do_real_time{% endif %}'
+          timedep: '<cyclestr offset="&START_TIME_NSSLMOSAIC;">@Y@m@d@H@M00</cyclestr>'
+
+  task_process_lightning_#cycle_type#:
+    <<: *default_preproc
+    command: '&LOAD_MODULES_RUN_TASK_FP; "process_obs" "&JOBSdir;/JREGIONAL_PROCESS_LIGHTNING"'
+    join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
+    dependency:
+      or:
+        taskdep:
+          attrs:
+            task: get_da_obs
+        and:
+          not:
+            taskvalid:
+              attrs:
+                task: get_da_obs
+          streq:
+            left: do_real_time
+            right: '{% if workflow.DO_REAL_TIME %}do_real_time{% endif %}'
+          timedep: '<cyclestr offset="&START_TIME_LIGHTNING;">@Y@m@d@H@M00</cyclestr>'
+
+
+  task_process_bufrobs_#cycle_type#:
+    <<: *default_preproc
+    command: '&LOAD_MODULES_RUN_TASK_FP; "process_obs" "&JOBSdir;/JREGIONAL_PROCESS_BUFROBS"'
+    join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
+    dependency:
+      or:
+        taskdep:
+          attrs:
+            task: get_da_obs
+        and:
+          not:
+            taskvalid:
+              attrs:
+                task: get_da_obs
+          streq:
+            left: do_real_time
+            right: '{% if workflow.DO_REAL_TIME %}do_real_time{% endif %}'
+          timedep: '<cyclestr offset="&START_TIME_CONVENTIONAL;">@Y@m@d@H@M00</cyclestr>'
+