Add GDASApp mode letkf to run_jedi_exe.py (#223)

parm/atm/obs/config/lgetkf_amsua_n19.yaml

@@ -26,6 +26,36 @@ obs operator:
     CoefficientPath: ./crtm/
 obs error:
   covariance model: diagonal
+obs bias:
+  input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias.${GDATE}.nc4
+  output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias.${CDATE}.nc4
+  variational bc:
+    predictors:
+    - name: constant
+    - name: lapse_rate
+      order: 2
+      tlapse: &amsua_n19_tlapse !ENV ${DATA}/obs/${GPREFIX}amsua_n19.tlapse.${GDATE}.txt
+    - name: lapse_rate
+      tlapse: *amsua_n19_tlapse
+    - name: emissivity
+    - name: scan_angle
+      order: 4
+    - name: scan_angle
+      order: 3
+    - name: scan_angle
+      order: 2
+    - name: scan_angle
+  covariance:
+    minimal required obs number: 20
+    variance range: [1.0e-6, 10.0]
+    step size: 1.0e-4
+    largest analysis variance: 10000.0
+    prior:
+      input file: !ENV ${DATA}/obs/${GPREFIX}amsua_n19.satbias_cov.${GDATE}.nc4
+      inflation:
+        ratio: 1.1
+        ratio for small dataset: 2.0
+    output file: !ENV ${DATA}/bc/${APREFIX}amsua_n19.satbias_cov.${CDATE}.nc4
 obs filters:
 - filter: Bounds Check
   filter variables:

ush/examples/run_jedi_exe/3dvar_hera.yaml

@@ -5,7 +5,7 @@ template: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/varia
 config:
   berror_yaml: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/berror/staticb_gsibec.yaml
   obs_dir: obs
-  diag_dig: diags
+  diag_dir: diags
   crtm_coeff_dir: crtm
   bias_in_dir: obs
   bias_out_dir: bc

ush/examples/run_jedi_exe/3dvar_orion.yaml

@@ -5,7 +5,7 @@ template: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/variational/3dvar
 config:
   berror_yaml: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/berror/staticb_gsibec.yaml
   obs_dir: obs
-  diag_dig: diags
+  diag_dir: diags
   crtm_coeff_dir: crtm
   bias_in_dir: obs
   bias_out_dir: bc

ush/examples/run_jedi_exe/letkf_hera.yaml

@@ -0,0 +1,37 @@
+working directory: /scratch2/NCEPDEV/stmp1/Cory.R.Martin/gdas_single_test_letkf
+GDASApp home: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp
+GDASApp mode: letkf
+template: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/lgetkf/lgetkf.yaml
+config:
+  obs_dir: obs
+  diag_dir: diags
+  crtm_coeff_dir: crtm
+  bias_in_dir: obs
+  bias_out_dir: bc
+  obs_yaml_dir: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/config
+  executable: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/build/bin/fv3jedi_letkf.x
+  obs_list: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/lists/lgetkf_prototype.yaml
+  gdas_fix_root: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix
+  atm: true
+  layout_x: 1
+  layout_y: 1
+  atm_window_length: PT6H
+  valid_time: 2021-12-21T06:00:00Z
+  dump: gdas
+  case: C48
+  case_anl: C48
+  case_enkf: C48
+  staticb_type: gsibec
+  dohybvar: no
+  levs: 128
+  nmem: 10
+  comin_ens: /scratch2/NCEPDEV/stmp1/Russ.Treadon/comrot/prufsda/enkfgdas.20211221/00/atmos
+  interp_method: barycentric
+job options:
+  machine: hera
+  account: da-cpu
+  queue: debug
+  partition: hera
+  walltime: '30:00'
+  ntasks: 6
+  modulepath: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/modulefiles

ush/run_jedi_exe.py

@@ -39,7 +39,7 @@ def run_jedi_exe(yamlconfig):
 
     # check if the specified app mode is valid
     app_mode = all_config_dict['GDASApp mode']
-    supported_app_modes = ['hofx', 'variational']
+    supported_app_modes = ['hofx', 'variational', 'letkf']
     if app_mode not in supported_app_modes:
         raise KeyError(f"'{app_mode}' not supported. " +
                        "Current GDASApp modes supported are: " +
@@ -58,7 +58,7 @@ def run_jedi_exe(yamlconfig):
     sys.path.append(ufsda_path)
     import ufsda
     from ufsda.misc_utils import calc_fcst_steps
-    from ufsda.stage import gdas_single_cycle, gdas_fix
+    from ufsda.stage import gdas_single_cycle, gdas_fix, background_ens, atm_obs, bias_obs
 
     # compute config for YAML for executable
     executable_subconfig = all_config_dict['config']
@@ -111,12 +111,16 @@ def run_jedi_exe(yamlconfig):
         'window_begin': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}",
         'prev_valid_time': f"{prev_cycle.strftime('%Y-%m-%dT%H:%M:%SZ')}",
         'atm_window_length': executable_subconfig['atm_window_length'],
+        'ATM_WINDOW_LENGTH': f"PT{assim_freq}H",
+        'ATM_WINDOW_BEGIN': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}",
+        'COMOUT': os.path.join(workdir, 'obs'),
         'CASE': executable_subconfig['case'],
         'CASE_ANL': executable_subconfig.get('case_anl', executable_subconfig['case']),
         'CASE_ENKF': executable_subconfig.get('case_enkf', executable_subconfig['case']),
         'DOHYBVAR': executable_subconfig.get('dohybvar', False),
         'LEVS': str(executable_subconfig['levs']),
         'NMEM_ENKF': executable_subconfig.get('nmem', 0),
+        'COMIN_GES_ENS': executable_subconfig.get('comin_ens','./'),
         'forecast_steps': calc_fcst_steps(executable_subconfig.get('forecast_step', 'PT6H'),
                                           executable_subconfig['atm_window_length']),
         'BKG_TSTEP': executable_subconfig.get('forecast_step', 'PT6H'),
@@ -132,7 +136,14 @@ def run_jedi_exe(yamlconfig):
     genYAML(yamlconfig, output=output_file)
     logging.info(f'Wrote YAML file to {output_file}')
     # use R2D2 to stage backgrounds, obs, bias correction files, etc.
-    ufsda.stage.gdas_single_cycle(var_config)
+    if app_mode in ['variational', 'hofx']:
+        ufsda.stage.gdas_single_cycle(var_config)
+    # stage ensemble backgrouns for letkf
+    if app_mode in ['letkf']:
+        ufsda.stage.background_ens(var_config)
+        ufsda.stage.atm_obs(var_config)
+        ufsda.stage.bias_obs(var_config)
+
     # link additional fix files needed (CRTM, fieldmetadata, etc.)
     gdasfix = executable_subconfig['gdas_fix_root']
     ufsda.stage.gdas_fix(gdasfix, workdir, var_config)
@@ -141,7 +152,7 @@ def run_jedi_exe(yamlconfig):
     ufsda.disk_utils.symlink(executable_subconfig['executable'], os.path.join(workdir, baseexe))
     # create output directories
     ufsda.disk_utils.mkdir(os.path.join(workdir, 'diags'))
-    if app_mode in ['variational']:
+    if app_mode in ['variational', 'letkf']:
         ufsda.disk_utils.mkdir(os.path.join(workdir, 'anl'))
         ufsda.disk_utils.mkdir(os.path.join(workdir, 'bc'))
     baseexe = os.path.join(workdir, baseexe)

ush/ufsda/stage.py

@@ -32,7 +32,7 @@
 import ioda_conv_engines as iconv
 from orddicts import DefaultOrderedDict
 
-__all__ = ['atm_background', 'atm_obs', 'bias_obs', 'background', 'fv3jedi', 'obs', 'berror', 'gdas_fix', 'gdas_single_cycle']
+__all__ = ['atm_background', 'atm_obs', 'bias_obs', 'background', 'background_ens', 'fv3jedi', 'obs', 'berror', 'gdas_fix', 'gdas_single_cycle']
 
 logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
                     level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
@@ -476,3 +476,37 @@ def berror(config):
     except FileExistsError:
         os.remove(jedi_staticb_dir)
         os.symlink(config['staticb_dir'], jedi_staticb_dir)
+
+
+def background_ens(config):
+    """
+    Stage backgrounds and create links for analysis
+    This involves:
+    - cp RESTART to RESTART_GES
+    - ln RESTART_GES to analysis/bkg
+    - mkdir analysis/anl
+    """
+    for imem in range(1, config['NMEM_ENKF']+1):
+        memchar = f"mem{imem:03d}"
+        print(f"background_ens:  process {memchar}")
+        rst_dir = os.path.join(config['COMIN_GES_ENS'], memchar, 'RESTART')
+        ges_dir = os.path.join(config['COMIN_GES_ENS'], memchar, 'RESTART_GES')
+        jedi_bkg_mem = os.path.join(config['DATA'], 'bkg', memchar)
+        jedi_bkg_dir = os.path.join(config['DATA'], 'bkg', memchar, 'RESTART')
+        jedi_anl_dir = os.path.join(config['DATA'], 'anl', memchar)
+        mkdir(jedi_bkg_mem)
+
+        # copy RESTART to RESTART_GES
+        try:
+            shutil.copytree(rst_dir, ges_dir)
+        except FileExistsError:
+            shutil.rmtree(ges_dir)
+            shutil.copytree(rst_dir, ges_dir)
+        try:
+            os.symlink(ges_dir, jedi_bkg_dir)
+        except FileExistsError:
+            os.remove(jedi_bkg_dir)
+            os.symlink(ges_dir, jedi_bkg_dir)
+        mkdir(jedi_anl_dir)
+
+