Merge branch 'NOAA-EMC:develop' into feature/fit2obs

env/JET.env

@@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
-    echo "atmanalrun atmensanalrun"
+    echo "atmanalrun atmensanalrun aeroanlrun"
     echo "anal sfcanl fcst post vrfy metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"
@@ -59,6 +59,14 @@ elif [[ "${step}" = "atmensanalrun" ]]; then
     [[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max}
     export APRUN_ATMENSANAL="${launcher} ${npe_atmensanalrun}"
 
+elif [[ "${step}" = "aeroanlrun" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_aeroanlrun))
+
+    export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}}
+    [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
+    export APRUN_AEROANL="${launcher} ${npe_aeroanlrun}"
+
 elif [[ "${step}" = "anal" ]]; then
 
     nth_max=$((npe_node_max / npe_node_anal))

parm/config/config.aeroanl

@@ -19,6 +19,6 @@ export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
 export JEDIVAREXE=${HOMEgfs}/exec/fv3jedi_var.x
-export CRTM_VER="2.3.0"
+export crtm_VERSION="2.3.0"
 
 echo "END: config.aeroanl"

parm/config/config.resources

@@ -60,11 +60,12 @@ if [ ${step} = "prep" ]; then
       export memory_prep="40G"
     fi
 
-elif [ ${step} = "aerosol_init" ]; then
+elif [[ "${step}" = "aerosol_init" ]]; then
     export wtime_aerosol_init="00:05:00"
     export npe_aerosol_init=1
     export nth_aerosol_init=1
-    export npe_node_aerosol_init=$(echo "${npe_node_max} / ${nth_aerosol_init}" | bc)
+    npe_node_aerosol_init=$(echo "${npe_node_max} / ${nth_aerosol_init}" | bc)
+    export npe_node_aerosol_init
     export NTASKS=${npe_aerosol_init}
     export memory_aerosol_init="6G"
 

parm/parm_gdas/aero_crtm_coeff.yaml

@@ -1,23 +1,13 @@
 mkdir:
-- !ENV ${DATA}/crtm/
+- $(DATA)/crtm/
 copy:
-- - !ENV ${FV3JEDI_FIX}/crtm/${CRTM_VER}/AerosolCoeff.bin
-  - !ENV ${DATA}/crtm/AerosolCoeff.bin
-- - !ENV ${FV3JEDI_FIX}/crtm/${CRTM_VER}/CloudCoeff.bin
-  - !ENV ${DATA}/crtm/CloudCoeff.bin
-- - !ENV ${FV3JEDI_FIX}/crtm/${CRTM_VER}/v.viirs-m_npp.SpcCoeff.bin
-  - !ENV ${DATA}/crtm/v.viirs-m_npp.SpcCoeff.bin
-- - !ENV ${FV3JEDI_FIX}/crtm/${CRTM_VER}/v.viirs-m_npp.TauCoeff.bin
-  - !ENV ${DATA}/crtm/v.viirs-m_npp.TauCoeff.bin
-- - !ENV ${FV3JEDI_FIX}/crtm/${CRTM_VER}/v.viirs-m_j1.SpcCoeff.bin
-  - !ENV ${DATA}/crtm/v.viirs-m_j1.SpcCoeff.bin
-- - !ENV ${FV3JEDI_FIX}/crtm/${CRTM_VER}/v.viirs-m_j1.TauCoeff.bin
-  - !ENV ${DATA}/crtm/v.viirs-m_j1.TauCoeff.bin
-- - !ENV ${FV3JEDI_FIX}/crtm/${CRTM_VER}/NPOESS.VISice.EmisCoeff.bin
-  - !ENV ${DATA}/crtm/NPOESS.VISice.EmisCoeff.bin
-- - !ENV ${FV3JEDI_FIX}/crtm/${CRTM_VER}/NPOESS.VISland.EmisCoeff.bin
-  - !ENV ${DATA}/crtm/NPOESS.VISland.EmisCoeff.bin
-- - !ENV ${FV3JEDI_FIX}/crtm/${CRTM_VER}/NPOESS.VISsnow.EmisCoeff.bin
-  - !ENV ${DATA}/crtm/NPOESS.VISsnow.EmisCoeff.bin
-- - !ENV ${FV3JEDI_FIX}/crtm/${CRTM_VER}/NPOESS.VISwater.EmisCoeff.bin
-  - !ENV ${DATA}/crtm/NPOESS.VISwater.EmisCoeff.bin
+- [$(FV3JEDI_FIX)/crtm/$(crtm_VERSION)/AerosolCoeff.bin, $(DATA)crtm/]
+- [$(FV3JEDI_FIX)/crtm/$(crtm_VERSION)/CloudCoeff.bin, $(DATA)/crtm/]
+- [$(FV3JEDI_FIX)/crtm/$(crtm_VERSION)/v.viirs-m_npp.SpcCoeff.bin, $(DATA)/crtm/]
+- [$(FV3JEDI_FIX)/crtm/$(crtm_VERSION)/v.viirs-m_npp.TauCoeff.bin, $(DATA)/crtm/]
+- [$(FV3JEDI_FIX)/crtm/$(crtm_VERSION)/v.viirs-m_j1.SpcCoeff.bin, $(DATA)/crtm/]
+- [$(FV3JEDI_FIX)/crtm/$(crtm_VERSION)/v.viirs-m_j1.TauCoeff.bin, $(DATA)/crtm/]
+- [$(FV3JEDI_FIX)/crtm/$(crtm_VERSION)/NPOESS.VISice.EmisCoeff.bin, $(DATA)/crtm/]
+- [$(FV3JEDI_FIX)/crtm/$(crtm_VERSION)/NPOESS.VISland.EmisCoeff.bin, $(DATA)/crtm/]
+- [$(FV3JEDI_FIX)/crtm/$(crtm_VERSION)/NPOESS.VISsnow.EmisCoeff.bin, $(DATA)/crtm/]
+- [$(FV3JEDI_FIX)/crtm/$(crtm_VERSION)/NPOESS.VISwater.EmisCoeff.bin, $(DATA)/crtm/]

ush/python/pygfs/task/analysis.py

@@ -32,6 +32,10 @@ def initialize(self) -> None:
         obs_dict = self.get_obs_dict()
         FileHandler(obs_dict).sync()
 
+        # some analyses need to stage bias corrections
+        bias_dict = self.get_bias_dict()
+        FileHandler(bias_dict).sync()
+
     @logit(logger)
     def get_obs_dict(self: Task) -> Dict[str, Any]:
         """Compile a dictionary of observation files to copy
@@ -64,6 +68,44 @@ def get_obs_dict(self: Task) -> Dict[str, Any]:
         }
         return obs_dict
 
+    @logit(logger)
+    def get_bias_dict(self: Task) -> Dict[str, Any]:
+        """Compile a dictionary of observation files to copy
+
+        This method uses the OBS_LIST configuration variable to generate a dictionary
+        from a list of YAML files that specify what observation bias correction files
+        are to be copied to the run directory from the observation input directory
+
+        Parameters
+        ----------
+
+        Returns
+        ----------
+        bias_dict: Dict
+            a dictionary containing the list of observation bias files to copy for FileHandler
+        """
+        logger.debug(f"OBS_LIST: {self.task_config['OBS_LIST']}")
+        obs_list_config = parse_j2yaml(self.task_config["OBS_LIST"], self.task_config)
+        logger.debug(f"obs_list_config: {obs_list_config}")
+        # get observers from master dictionary
+        observers = obs_list_config['observers']
+        copylist = []
+        for ob in observers:
+            if 'obs bias' in ob.keys():
+                obfile = ob['obs bias']['input file']
+                obdir = os.path.dirname(obfile)
+                basename = os.path.basename(obfile)
+                prefix = '.'.join(basename.split('.')[:-2])
+                for file in ['satbias.nc4', 'satbias_cov.nc4', 'tlapse.txt']:
+                    bfile = f"{prefix}.{file}"
+                    copylist.append([os.path.join(self.task_config.comin_ges_atm, bfile), os.path.join(obdir, bfile)])
+
+        bias_dict = {
+            'mkdir': [os.path.join(self.runtime_config.DATA, 'bc')],
+            'copy': copylist
+        }
+        return bias_dict
+
     @logit(logger)
     def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None:
         """Add cubed-sphere increments to cubed-sphere backgrounds