Merge remote-tracking branch 'origin/develop' into feature/version-files

* origin/develop:
  Initial blocks in place for forecast refactor work (NOAA-EMC#1466)
  Update buoys file and fix boundary point jobs (NOAA-EMC#1465)
  Move Fit2Obs to stand-alone job (NOAA-EMC#1456)
  Rework arch job dependencies (NOAA-EMC#1455)
  Stage bias files for UFSDA aerosols (NOAA-EMC#1370)
  Add logging level to logger (NOAA-EMC#1442)
  Update hash for GDASApp to db2f998 (NOAA-EMC#1443)
  Remove para module paths for ncdiag on WCOSS2 (NOAA-EMC#1437)
  Use P8 settings for C384 atm by default (NOAA-EMC#1440)
  Enforce rstprod on relevant tarballs (NOAA-EMC#1436)

Externals.cfg

@@ -29,7 +29,7 @@ protocol = git
 required = True
 
 [GSI-EnKF]
-hash = 31b8b29
+hash = 113e307
 local_path = sorc/gsi_enkf.fd
 repo_url = https://github.com/NOAA-EMC/GSI.git
 protocol = git
@@ -43,14 +43,14 @@ protocol = git
 required = False
 
 [GSI-Monitor]
-hash = c64cc47
+hash = 8cf16de
 local_path = sorc/gsi_monitor.fd
 repo_url = https://github.com/NOAA-EMC/GSI-monitor.git
 protocol = git
 required = False
 
 [GDASApp]
-hash = 6aa55fa
+hash = db2f998
 local_path = sorc/gdas.cd
 repo_url = https://github.com/NOAA-EMC/GDASApp.git
 protocol = git

env/HERA.env

@@ -133,6 +133,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
     export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}"
 
 elif [[ "${step}" = "sfcanl" ]]; then
+
     nth_max=$((npe_node_max / npe_node_sfcanl))
 
     export NTHREADS_CYCLE=${nth_sfcanl:-14}
@@ -291,4 +292,14 @@ elif [[ "${step}" = "gempak" ]]; then
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
     export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
+
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/JET.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanalrun atmensanalrun"
+    echo "atmanalrun atmensanalrun aeroanlrun"
     echo "anal sfcanl fcst post vrfy metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"
@@ -59,6 +59,14 @@ elif [[ "${step}" = "atmensanalrun" ]]; then
     [[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max}
     export APRUN_ATMENSANAL="${launcher} ${npe_atmensanalrun}"
 
+elif [[ "${step}" = "aeroanlrun" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_aeroanlrun))
+
+    export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}}
+    [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
+    export APRUN_AEROANL="${launcher} ${npe_aeroanlrun}"
+
 elif [[ "${step}" = "anal" ]]; then
 
     nth_max=$((npe_node_max / npe_node_anal))
@@ -204,4 +212,12 @@ elif [[ "${step}" = "gempak" ]]; then
 
      echo "WARNING: ${step} is not enabled on ${machine}!"
 
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/ORION.env

@@ -42,7 +42,7 @@ if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
     export sys_tp="ORION"
     export launcher_PREP="srun"
 
-elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]]|| [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"
     if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
@@ -290,4 +290,13 @@ elif [[ "${step}" = "gempak" ]]; then
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
     export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/S4.env

@@ -260,4 +260,13 @@ elif [[ "${step}" = "awips" ]]; then
 elif [[ "${step}" = "gempak" ]]; then
 
      echo "WARNING: ${step} is not enabled on S4!"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/WCOSS2.env

@@ -290,6 +290,14 @@ elif [[ "${step}" = "gempak" ]]; then
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
     export APRUN_GEMPAKCFP="${launcher} -np ${npe_gempak} ${mpmd_opt}"
 
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -np ${npe_fit2obs}"
+
 elif [[ "${step}" = "waveawipsbulls" ]]; then
 
     unset PERL5LIB

jobs/JGDAS_FIT2OBS

@@ -0,0 +1,88 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "fit2obs" -c "base fit2obs"
+
+
+##############################################
+# Set variables used in the script
+##############################################
+
+export CDUMP=${RUN/enkf}
+
+# Ignore spelling warning; nothing is misspelled
+# shellcheck disable=SC2153
+CDATE=$(${NDATE} -"${VBACKUP_FITS}" "${PDY}${cyc}") # set CDATE to lookback cycle for use in fit2obs package
+export CDATE
+vday=${CDATE:0:8}
+vcyc=${CDATE:8:2}
+
+export COM_INA=${ROTDIR}/gdas.${vday}/${vcyc}/atmos
+# We want to defer variable expansion, so ignore warning about single quotes
+# shellcheck disable=SC2016
+export COM_INF='$ROTDIR/vrfyarch/gfs.$fdy/$fzz'
+export COM_PRP=${ROTDIR}/gdas.${vday}/${vcyc}/obs
+
+export PRPI=${COM_PRP}/${RUN}.t${vcyc}z.prepbufr
+export sig1=${COM_INA}/${RUN}.t${vcyc}z.atmanl.nc
+export sfc1=${COM_INA}/${RUN}.t${vcyc}z.atmanl.nc
+export CNVS=${COM_INA}/${RUN}.t${vcyc}z.cnvstat
+
+export OUTPUT_FILETYPE=${OUTPUT_FILETYPE:-netcdf}
+
+export FIT_DIR=${ARCDIR}/fits
+[[ ! -d "${FIT_DIR}" ]] && mkdir -p "${FIT_DIR}"
+export HORZ_DIR=${ARCDIR}/horiz
+[[ ! -d "${HORZ_DIR}" ]] && mkdir -p "${HORZ_DIR}"
+export COMLOX=${DATA}/fitx
+[[ ! -d "${COMLOX}" ]] && mkdir -p "${COMLOX}"
+
+echo "echo err_chk">"${DATA}"/err_chk; chmod 755 "${DATA}"/err_chk
+echo "echo postmsg">"${DATA}"/postmsg; chmod 755 "${DATA}"/postmsg
+
+##############################################
+# Check spinup and available inputs
+##############################################
+
+# Ignore spelling warning; nothing is misspelled
+# shellcheck disable=SC2153
+if [[ ${CDATE} -gt ${SDATE} ]]; then
+  for file in ${PRPI} ${sig1} ${sfc1} ${CNVS}; do
+    if [[ ! -f "${file}" ]]; then
+      echo "FATAL ERROR: FILE MISSING: ${file}"
+      exit 1
+    fi
+  done
+
+  ##############################################
+  # RUN FIT2OBS VERIFICATION
+  ##############################################
+
+  "${fitdir}/batrun/excfs_gdas_vrfyfits.sh.ecf"
+  status=$?
+  [[ ${status} -ne 0 ]] && exit "${status}"
+
+  ##############################################
+  # End JOB SPECIFIC work
+  ##############################################
+
+  ##############################################
+  # Final processing
+  ##############################################
+  if [[ -e "${pgmout}" ]] ; then
+    cat "${pgmout}"
+  fi
+
+else
+
+  echo "Too early for FIT2OBS to run. Exiting."
+
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || (echo "FATAL ERROR: ${DATAROOT} does not exist. ABORT!"; exit 1)
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/JGLOBAL_WAVE_POST_BNDPNT

@@ -34,8 +34,7 @@ export WAV_MOD_TAG=${CDUMP}wave${waveMEMB}
 
 export CFP_VERBOSE=1
 
-export FHMAX_WAV_PNT=180
-if [ ${FHMAX_WAV} -lt ${FHMAX_WAV_PNT} ] ; then export FHMAX_WAV_IBP=${FHMAX_WAV} ; fi
+export FHMAX_WAV_PNT=${FHMAX_WAV_IBP}
 export DOSPC_WAV='YES' # Spectral post
 export DOBLL_WAV='NO' # Bulletin post
 export DOBNDPNT_WAV='YES'  #not boundary points

jobs/JGLOBAL_WAVE_POST_BNDPNTBLL

@@ -37,8 +37,7 @@ export WAV_MOD_TAG=${CDUMP}wave${waveMEMB}
 
 export CFP_VERBOSE=1
 
-export FHMAX_WAV_PNT=180
-if [ ${FHMAX_WAV} -lt ${FHMAX_WAV_PNT} ] ; then export FHMAX_WAV_IBP=${FHMAX_WAV} ; fi
+export FHMAX_WAV_PNT=${FHMAX_WAV_IBP}
 export DOSPC_WAV='NO' # Spectral post
 export DOBLL_WAV='YES' # Bulletin post
 export DOBNDPNT_WAV='YES'  #boundary points

jobs/rocoto/fcst.sh

@@ -18,10 +18,29 @@ module load prod_util
 if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
   module load cray-pals
 fi
+if [[ "${MACHINE_ID}" = "hera" ]]; then
+  module use "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/modulefiles/core"
+  module load "miniconda3/4.6.14"
+  module load "gfs_workflow/1.0.0"
+# TODO: orion and wcoss2 will be uncommented when they are ready.  This comment block will be removed in the next PR
+#elif [[ "${MACHINE_ID}" = "orion" ]]; then
+#  module use "/home/rmahajan/opt/global-workflow/modulefiles/core"
+#  module load "python/3.7.5"
+#  module load "gfs_workflow/1.0.0"
+#elif [[ "${MACHINE_ID}" = "wcoss2" ]]; then
+#  module load "python/3.7.5"
+fi
 module list
 unset MACHINE_ID
 set_trace
 
+###############################################################
+# exglobal_forecast.py requires the following in PYTHONPATH
+# This will be moved to a module load when ready
+pygwPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/pygw/src:${HOMEgfs}/ush/python/pygfs"
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${pygwPATH}"
+export PYTHONPATH
+
 export job="fcst"
 export jobid="${job}.$$"
 

jobs/rocoto/fit2obs.sh

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+echo
+echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ==============="
+. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="fit2obs"
+export jobid="${job}.$$"
+
+###############################################################
+echo
+echo "=============== START TO RUN FIT2OBS ==============="
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGDAS_FIT2OBS"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+exit 0

jobs/rocoto/vrfy.sh

@@ -57,27 +57,6 @@ if [ ${RUNMOS} = "YES" -a ${CDUMP} = "gfs" ]; then
 fi
 
 
-###############################################################
-echo
-echo "=============== START TO RUN FIT2OBS VERIFICATION ==============="
-if [ ${VRFYFITS} = "YES" -a ${CDUMP} = ${CDFNL} -a ${CDATE} != ${SDATE} ]; then
-
-    export CDUMPFCST=${VDUMP}
-    export TMPDIR="${RUNDIR}/${CDATE}/${CDUMP}"
-    [[ ! -d ${TMPDIR} ]] && mkdir -p ${TMPDIR}
-
-    xdate=$(${NDATE} -${VBACKUP_FITS} ${CDATE})
-
-    export RUN_ENVIR_SAVE=${RUN_ENVIR}
-    export RUN_ENVIR="netcdf"
-
-    ${PREPQFITSH} ${PSLOT} ${xdate} ${ROTDIR} ${ARCDIR} ${TMPDIR}
-
-    export RUN_ENVIR=${RUN_ENVIR_SAVE}
-
-fi
-
-
 ###############################################################
 echo
 echo "=============== START TO RUN RADMON DATA EXTRACTION ==============="

modulefiles/module_base.wcoss2.lua

@@ -26,15 +26,10 @@ load(pathJoin("prod_util", os.getenv("prod_util_ver")))
 load(pathJoin("grib_util", os.getenv("grib_util_ver")))
 load(pathJoin("bufr_dump", os.getenv("bufr_dump_ver")))
 load(pathJoin("util_shared", os.getenv("util_shared_ver")))
-load(pathJoin("crtm", os.getenv("crtm_ver")))
 load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
-load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
-
-pushenv("HPC_OPT", "/apps/ops/para/libs")
-append_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304")
-append_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7")
-
 load(pathJoin("ncdiag", os.getenv("ncdiag_ver")))
+load(pathJoin("crtm", os.getenv("crtm_ver")))
+load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
 
 prepend_path("MODULEPATH", pathJoin("/lfs/h2/emc/global/save/emc.global/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles"))
 load(pathJoin("prepobs", os.getenv("prepobs_run_ver")))

parm/config/config.aeroanl

@@ -19,6 +19,6 @@ export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
 export JEDIVAREXE=${HOMEgfs}/exec/fv3jedi_var.x
-export CRTM_VER="2.3.0"
+export crtm_VERSION="2.3.0"
 
 echo "END: config.aeroanl"

parm/config/config.base.emc.dyn

@@ -379,6 +379,7 @@ export binary_diag=".false."
 
 # Verification options
 export DO_METP="YES"         # Run METPLUS jobs - set METPLUS settings in config.metp
+export DO_FIT2OBS="YES"      # Run fit to observations package
 
 # Archiving options
 export HPSSARCH="@HPSSARCH@"        # save data to HPSS archive

parm/config/config.defaults.s2sw

@@ -14,25 +14,6 @@ FHOUT_HF_GFS=-1
 min_seaice="1.0e-6"
 use_cice_alb=".true."
 
-# config.ufs  # TODO: This is hard-wired for P8 and needs to be refactored.  For now, use case C384
-# TODO: Q. for @jessicameixner-noaa: can we make these defaults in config.ufs for C384?
-case "${CASE}" in
-  "C384")
-    DELTIM=300
-    layout_x_gfs=8
-    layout_y_gfs=8
-    WRITE_GROUP_GFS=1
-    WRTTASK_PER_GROUP_GFS=24
-    #The settings below will result in S2SWA running 35 days under 8 hours wallclock on hera
-    #layout_x_gfs=24
-    #layout_y_gfs=16
-    #WRTTASK_PER_GROUP_GFS=86
-    ;;
-  *)  # All other ${CASE}
-    echo "config.defaults.s2sw: Using default settings for CASE=${CASE}"
-    ;;
-esac
-
 # config.wave
 
 waveGRD='mx025'

parm/config/config.fcst

@@ -71,6 +71,7 @@ fi
 #######################################################################
 
 export FORECASTSH="$HOMEgfs/scripts/exglobal_forecast.sh"
+#export FORECASTSH="$HOMEgfs/scripts/exglobal_forecast.py"  # Temp. while this is worked on
 export FCSTEXECDIR="$HOMEgfs/exec"
 export FCSTEXEC="ufs_model.x"