Move Fit2Obs to stand-alone job

env/HERA.env

@@ -133,6 +133,7 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
     export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}"
 
 elif [[ "${step}" = "sfcanl" ]]; then
+
     nth_max=$((npe_node_max / npe_node_sfcanl))
 
     export NTHREADS_CYCLE=${nth_sfcanl:-14}
@@ -291,4 +292,14 @@ elif [[ "${step}" = "gempak" ]]; then
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
     export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
+
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/JET.env

@@ -212,4 +212,12 @@ elif [[ "${step}" = "gempak" ]]; then
 
      echo "WARNING: ${step} is not enabled on ${machine}!"
 
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/ORION.env

@@ -290,4 +290,13 @@ elif [[ "${step}" = "gempak" ]]; then
     export NTHREADS_GEMPAK=${nth_gempak:-1}
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
     export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/S4.env

@@ -260,4 +260,13 @@ elif [[ "${step}" = "awips" ]]; then
 elif [[ "${step}" = "gempak" ]]; then
 
      echo "WARNING: ${step} is not enabled on S4!"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs}"
+
 fi

env/WCOSS2.env

@@ -290,6 +290,14 @@ elif [[ "${step}" = "gempak" ]]; then
     [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
     export APRUN_GEMPAKCFP="${launcher} -np ${npe_gempak} ${mpmd_opt}"
 
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -np ${npe_fit2obs}"
+
 elif [[ "${step}" = "waveawipsbulls" ]]; then
 
     unset PERL5LIB

jobs/JGDAS_FIT2OBS

@@ -0,0 +1,84 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "fit2obs" -c "base fit2obs"
+
+
+##############################################
+# Set variables used in the script
+##############################################
+
+export CDUMP=${RUN/enkf}
+
+export xdate=$(${NDATE} -"${VBACKUP_FITS}" "${CDATE}")
+export CDATE=${xdate} # set CDATE to xdate for use in fit2obs package
+export vday=$(echo "${xdate}" | cut -c1-8)
+export vcyc=$(echo "${xdate}" | cut -c9-10)
+
+export COM_INA=${ROTDIR}/gdas.${vday}/${vcyc}/atmos
+# We want to defer variable expansion, so ignore warning about single quotes
+# shellcheck disable=SC2016
+export COM_INF='$ROTDIR/vrfyarch/gfs.$fdy/$fzz'
+export COM_PRP=${ROTDIR}/gdas.${vday}/${vcyc}/obs
+
+export PRPI=${COM_PRP}/${RUN}.t${vcyc}z.prepbufr
+export sig1=${COM_INA}/${RUN}.t${vcyc}z.atmanl.nc
+export sfc1=${COM_INA}/${RUN}.t${vcyc}z.atmanl.nc
+export CNVS=${COM_INA}/${RUN}.t${vcyc}z.cnvstat
+
+export OUTPUT_FILETYPE=${OUTPUT_FILETYPE:-netcdf}
+
+export FIT_DIR=${ARCDIR}/fits
+[[ ! -d "${FIT_DIR}" ]] && mkdir -p "${FIT_DIR}"
+export HORZ_DIR=${ARCDIR}/horiz
+[[ ! -d "${HORZ_DIR}" ]] && mkdir -p "${HORZ_DIR}"
+export COMLOX=${DATA}/fitx
+[[ ! -d "${COMLOX}" ]] && mkdir -p "${COMLOX}"
+
+echo "echo err_chk">"${DATA}"/err_chk; chmod 755 "${DATA}"/err_chk
+echo "echo postmsg">"${DATA}"/postmsg; chmod 755 "${DATA}"/postmsg
+
+##############################################
+# Check spinup and available inputs
+##############################################
+
+if [[ ${xdate} -gt ${SDATE} ]]; then
+  for file in ${PRPI} ${sig1} ${sfc1} ${CNVS}; do
+    if [[ ! -f "${file}" ]]; then
+      echo "FATAL ERROR: FILE MISSING: ${file}"
+      exit 1
+    fi
+  done
+
+  ##############################################
+  # RUN FIT2OBS VERIFICATION
+  ##############################################
+
+  "${fitdir}/batrun/excfs_gdas_vrfyfits.sh.ecf"
+  status=$?
+  [[ ${status} -ne 0 ]] && exit "${status}"
+
+  ##############################################
+  # End JOB SPECIFIC work
+  ##############################################
+
+  ##############################################
+  # Final processing
+  ##############################################
+  if [[ -e "${pgmout}" ]] ; then
+    cat "${pgmout}"
+  fi
+
+else
+
+  echo "Too early for FIT2OBS to run. Exiting."
+
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || (echo "FATAL ERROR: ${DATAROOT} does not exist. ABORT!"; exit 1)
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/rocoto/fit2obs.sh

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+echo
+echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ==============="
+. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+status=$?
+[[ ${status} -ne 0 ]] && exit ${status}
+
+export job="fit2obs"
+export jobid="${job}.$$"
+
+###############################################################
+echo
+echo "=============== START TO RUN FIT2OBS ==============="
+# Execute the JJOB
+${HOMEgfs}/jobs/JGDAS_FIT2OBS
+status=$?
+[[ ${status} -ne 0 ]] && exit ${status}
+
+exit 0

jobs/rocoto/vrfy.sh

@@ -57,27 +57,6 @@ if [ ${RUNMOS} = "YES" -a ${CDUMP} = "gfs" ]; then
 fi
 
 
-###############################################################
-echo
-echo "=============== START TO RUN FIT2OBS VERIFICATION ==============="
-if [ ${VRFYFITS} = "YES" -a ${CDUMP} = ${CDFNL} -a ${CDATE} != ${SDATE} ]; then
-
-    export CDUMPFCST=${VDUMP}
-    export TMPDIR="${RUNDIR}/${CDATE}/${CDUMP}"
-    [[ ! -d ${TMPDIR} ]] && mkdir -p ${TMPDIR}
-
-    xdate=$(${NDATE} -${VBACKUP_FITS} ${CDATE})
-
-    export RUN_ENVIR_SAVE=${RUN_ENVIR}
-    export RUN_ENVIR="netcdf"
-
-    ${PREPQFITSH} ${PSLOT} ${xdate} ${ROTDIR} ${ARCDIR} ${TMPDIR}
-
-    export RUN_ENVIR=${RUN_ENVIR_SAVE}
-
-fi
-
-
 ###############################################################
 echo
 echo "=============== START TO RUN RADMON DATA EXTRACTION ==============="

parm/config/config.base.emc.dyn

@@ -379,6 +379,7 @@ export binary_diag=".false."
 
 # Verification options
 export DO_METP="YES"         # Run METPLUS jobs - set METPLUS settings in config.metp
+export DO_FIT2OBS="YES"      # Run fit to observations package
 
 # Archiving options
 export HPSSARCH="@HPSSARCH@"        # save data to HPSS archive

parm/config/config.fit2obs

@@ -0,0 +1,30 @@
+#! /usr/bin/env bash
+
+########## config.fit2obs ##########
+# Fit to Observations
+
+echo "BEGIN: config.fit2obs"
+
+# Get task specific resources
+. ${EXPDIR}/config.resources fit2obs
+
+export fit_ver="wflow.1.0"
+export fitdir="${BASE_GIT}/Fit2Obs/${fit_ver}"
+
+export HOMEcfs=${fitdir}
+export EXECcfs=${HOMEcfs}/exec
+export USHcfs=${HOMEcfs}/ush
+
+export PRVT=${HOMEgfs}/fix/gsi/prepobs_errtable.global
+export HYBLEVS=${HOMEgfs}/fix/am/global_hyblev.l${LEVS}.txt
+
+export VBACKUP_FITS=24
+export OUTPUT_FILETYPE="netcdf"
+export CONVNETC="YES"
+export ACPROFit="YES"
+
+if [[ ${netcdf_diag:-".false."} = ".true." ]]; then
+    export CONVNETC="YES"
+fi
+
+echo "END: config.fit2obs"

parm/config/config.resources

@@ -12,7 +12,7 @@ if [ $# -ne 1 ]; then
     echo "atmanalprep atmanalrun atmanalpost"
     echo "atmensanalprep atmensanalrun atmensanalpost"
     echo "aeroanlinit aeroanlrun aeroanlfinal"
-    echo "anal sfcanl analcalc analdiag gldas fcst post vrfy metp arch echgres"
+    echo "anal sfcanl analcalc analdiag gldas fcst post vrfy fit2obs metp arch echgres"
     echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
     echo "init_chem mom6ic ocnpost"
     echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt"
@@ -623,6 +623,14 @@ elif [ ${step} = "vrfy" ]; then
     fi
     export is_exclusive=True
 
+elif [ ${step} = "fit2obs" ]; then
+
+    export wtime_fit2obs="00:20:00"
+    export npe_fit2obs=3
+    export nth_fit2obs=1
+    export npe_node_fit2obs=1
+    if [[ ${machine} == "WCOSS2" ]]; then export npe_node_fit2obs=3 ; fi
+
 elif [ ${step} = "metp" ]; then
 
     export nth_metp=1

parm/config/config.vrfy

@@ -8,12 +8,8 @@ echo "BEGIN: config.vrfy"
 # Get task specific resources
 . $EXPDIR/config.resources vrfy
 
-export VDUMP="gfs"                # Verifying dump
-export CDUMPFCST="gdas"           # Fit-to-obs with GDAS/GFS prepbufr
 export CDFNL="gdas"               # Scores verification against GDAS/GFS analysis
-
 export MKPGB4PRCP="YES"           # Make 0.25-deg pgb files in ARCDIR for precip verification
-export VRFYFITS="YES"             # Fit to observations
 export VRFYRAD="YES"              # Radiance data assimilation monitoring
 export VRFYOZN="YES"              # Ozone data assimilation monitoring
 export VRFYMINMON="YES"           # GSI minimization monitoring
@@ -22,38 +18,6 @@ export VRFYGENESIS="YES"          # Cyclone genesis verification
 export VRFYFSU="NO"               # Cyclone genesis verification (FSU)
 export RUNMOS="NO"                # whether to run entire MOS package
 
-#-------------------------------------------------
-# Fit to Observations
-#-------------------------------------------------
-
-if [ $VRFYFITS = "YES" ]; then
-
-    export fit_ver="newm.1.3"
-    export fitdir="$BASE_GIT/verif/global/Fit2Obs/${fit_ver}/batrun"
-    export PRVT=$HOMEgfs/fix/gsi/prepobs_errtable.global
-    export HYBLEVS=$HOMEgfs/fix/am/global_hyblev.l${LEVS}.txt
-    export CUE2RUN=$QUEUE
-
-    export VBACKUP_FITS=24
-
-    export CONVNETC="NO"
-    if [ ${netcdf_diag:-".false."} = ".true." ]; then
-        export CONVNETC="YES"
-    fi
-
-    if [ $machine = "HERA" ]; then
-        export PREPQFITSH="$fitdir/subfits_hera_slurm"
-    elif [ $machine = "S4" ]; then
-        export PREPQFITSH="$fitdir/subfits_s4_slurm"
-    elif [ $machine = "ORION" ]; then
-        export PREPQFITSH="$fitdir/subfits_orion_netcdf"
-    else
-        echo "Fit2Obs NOT supported on this machine"
-    fi
-
-fi
-
-
 #----------------------------------------------------------
 # Minimization, Radiance and Ozone Monitoring
 #----------------------------------------------------------

workflow/applications.py

@@ -105,6 +105,7 @@ def __init__(self, conf: Configuration) -> None:
         self.do_awips = _base.get('DO_AWIPS', False)
         self.do_wafs = _base.get('WAFSF', False)
         self.do_vrfy = _base.get('DO_VRFY', True)
+        self.do_fit2obs = _base.get('DO_FIT2OBS', True)
         self.do_metp = _base.get('DO_METP', False)
         self.do_jedivar = _base.get('DO_JEDIVAR', False)
         self.do_jediens = _base.get('DO_JEDIENS', False)
@@ -186,7 +187,7 @@ def _cycled_configs(self):
         if self.do_ocean:
             configs += ['ocnpost']
 
-        configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'arch']
+        configs += ['sfcanl', 'analcalc', 'fcst', 'post', 'vrfy', 'fit2obs', 'arch']
 
         if self.do_gldas:
             configs += ['gldas']
@@ -401,6 +402,9 @@ def _get_cycled_task_names(self):
                 gdas_tasks += wave_bndpnt_tasks
             gdas_tasks += wave_post_tasks
 
+        if self.do_fit2obs:
+            gdas_tasks += ['fit2obs']
+
         gdas_tasks += gdas_gfs_common_cleanup_tasks
 
         # Collect "gfs" cycle tasks

workflow/rocoto/workflow_tasks.py

@@ -993,6 +993,20 @@ def vrfy(self):
 
         return task
 
+    def fit2obs(self):
+        deps = []
+        dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps)
+
+        cycledef = 'gdas_half,gdas' if self.cdump in ['gdas'] else self.cdump
+
+        resources = self.get_resource('fit2obs')
+        task = create_wf_task('fit2obs', resources, cdump=self.cdump, envar=self.envars, dependency=dependencies,
+                              cycledef=cycledef)
+
+        return task
+
     def metp(self):
         deps = []
         dep_dict = {'type': 'metatask', 'name': f'{self.cdump}post'}
@@ -1020,6 +1034,9 @@ def arch(self):
         if self.app_config.do_vrfy:
             dep_dict = {'type': 'task', 'name': f'{self.cdump}vrfy'}
             deps.append(rocoto.add_dependency(dep_dict))
+        if self.app_config.do_fit2obs and self.cdump in ['gdas']:
+            dep_dict = {'type': 'task', 'name': f'{self.cdump}fit2obs'}
+            deps.append(rocoto.add_dependency(dep_dict))
         if self.app_config.do_metp and self.cdump in ['gfs']:
             dep_dict = {'type': 'metatask', 'name': f'{self.cdump}metp'}
             deps.append(rocoto.add_dependency(dep_dict))