clean up run_jedi_exe, extend genYAML for 4dhox, update 3d and 4dhofx…

… example yamls (#223)

parm/atm/obs/lists/gdas_prototype_3d.yaml

@@ -0,0 +1,13 @@
+observers:
+- !INC ${OBS_YAML_DIR}/amsua_n19.yaml
+- !INC ${OBS_YAML_DIR}/sondes.yaml
+##- !INC ${OBS_YAML_DIR}/atms_n20.yaml
+##- !INC ${OBS_YAML_DIR}/aircraft.yaml
+##- !INC ${OBS_YAML_DIR}/satwind.yaml
+##- !INC ${OBS_YAML_DIR}/omi_aura.yaml
+##- !INC ${OBS_YAML_DIR}/ompsnp_npp.yaml
+##- !INC ${OBS_YAML_DIR}/ompstc8_npp.yaml
+##- !INC ${OBS_YAML_DIR}/cris-fsr_n20.yaml
+##- !INC ${OBS_YAML_DIR}/cris-fsr_npp.yaml
+##- !INC ${OBS_YAML_DIR}/sfc.yaml
+#- !INC ${OBS_YAML_DIR}/sfcship.yaml

parm/atm/obs/lists/gdas_prototype_4d.yaml

@@ -0,0 +1,13 @@
+observers:
+##- !INC ${OBS_YAML_DIR}/amsua_n19.yaml
+- !INC ${OBS_YAML_DIR}/sondes.yaml
+##- !INC ${OBS_YAML_DIR}/atms_n20.yaml
+##- !INC ${OBS_YAML_DIR}/aircraft.yaml
+##- !INC ${OBS_YAML_DIR}/satwind.yaml
+##- !INC ${OBS_YAML_DIR}/omi_aura.yaml
+##- !INC ${OBS_YAML_DIR}/ompsnp_npp.yaml
+##- !INC ${OBS_YAML_DIR}/ompstc8_npp.yaml
+##- !INC ${OBS_YAML_DIR}/cris-fsr_n20.yaml
+##- !INC ${OBS_YAML_DIR}/cris-fsr_npp.yaml
+##- !INC ${OBS_YAML_DIR}/sfc.yaml
+#- !INC ${OBS_YAML_DIR}/sfcship.yaml

ush/examples/run_jedi_exe/3dhofx_hera.yaml

@@ -1,15 +1,15 @@
 working directory: /scratch2/NCEPDEV/stmp1/Cory.R.Martin/gdas_single_test_hofx3d
 GDASApp home: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp
 GDASApp mode: hofx
-executable options:
+template: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/hofx/hofx_nomodel.yaml
+config:
   obs_yaml_dir: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/config
-  yaml_template: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/hofx/hofx_nomodel.yaml
   executable: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/build/bin/fv3jedi_hofx_nomodel.x
-  obs_list: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype.yaml
+  obs_list: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype_3d.yaml
   gdas_fix_root: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix
   atm: true
-  layout_x: 4
-  layout_y: 4
+  layout_x: 1
+  layout_y: 1
   atm_window_length: PT6H
   valid_time: 2021-08-01T00:00:00Z
   dump: gdas
@@ -22,5 +22,6 @@ job options:
   queue: debug
   partition: hera
   walltime: '30:00'
-  ntasks: 96
+  ntasks: 6
+  ntasks-per-node: 2
   modulepath: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/modulefiles

ush/examples/run_jedi_exe/3dhofx_orion.yaml

@@ -1,15 +1,15 @@
 working directory: /work2/noaa/stmp/cmartin/gdas_single_test_hofx3d
 GDASApp home: /work2/noaa/da/cmartin/GDASApp/work/GDASApp
 GDASApp mode: hofx
-executable options:
+template: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/hofx/hofx_nomodel.yaml
+config:
   obs_yaml_dir: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/config
-  yaml_template: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/hofx/hofx_nomodel.yaml
   executable: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/build/bin/fv3jedi_hofx_nomodel.x
-  obs_list: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype.yaml
+  obs_list: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype_3d.yaml
   gdas_fix_root: /work2/noaa/da/cmartin/GDASApp/fix
   atm: true
-  layout_x: 4
-  layout_y: 4
+  layout_x: 1
+  layout_y: 1
   atm_window_length: PT6H
   valid_time: 2021-08-01T00:00:00Z
   dump: gdas
@@ -22,5 +22,6 @@ job options:
   queue: debug
   partition: debug
   walltime: '30:00'
-  ntasks: 96
+  ntasks: 6
+  ntasks-per-node: 2
   modulepath: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/modulefiles

ush/examples/run_jedi_exe/3dvar_hera.yaml

@@ -11,7 +11,7 @@ config:
   bias_out_dir: bc
   obs_yaml_dir: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/config
   executable: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/build/bin/fv3jedi_var.x
-  obs_list: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype.yaml
+  obs_list: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype_3d.yaml
   gdas_fix_root: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix
   atm: true
   layout_x: 1
@@ -31,6 +31,6 @@ job options:
   account: da-cpu
   queue: debug
   partition: hera
-  walltime: '10:00'
+  walltime: '30:00'
   ntasks: 6
   modulepath: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/modulefiles

ush/examples/run_jedi_exe/3dvar_orion.yaml

@@ -11,7 +11,7 @@ config:
   bias_out_dir: bc
   obs_yaml_dir: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/config
   executable: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/build/bin/fv3jedi_var.x
-  obs_list: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype.yaml
+  obs_list: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype_3d.yaml
   gdas_fix_root: /work2/noaa/da/cmartin/GDASApp/fix
   atm: true
   layout_x: 1
@@ -31,6 +31,6 @@ job options:
   account: da-cpu
   queue: debug
   partition: debug
-  walltime: '10:00'
+  walltime: '30:00'
   ntasks: 6
   modulepath: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/modulefiles

ush/examples/run_jedi_exe/4dhofx_hera.yaml

@@ -1,15 +1,15 @@
 working directory: /scratch2/NCEPDEV/stmp1/Cory.R.Martin/gdas_single_test_hofx4d
 GDASApp home: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp
 GDASApp mode: hofx
-executable options:
+template: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/hofx/hofx4d.yaml
+config:
   obs_yaml_dir: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/config
-  yaml_template: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/hofx/hofx4d.yaml
   executable: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/build/bin/fv3jedi_hofx.x
-  obs_list: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype.yaml
+  obs_list: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype_4d.yaml
   gdas_fix_root: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/fix
   atm: true
-  layout_x: 4
-  layout_y: 4
+  layout_x: 1
+  layout_y: 1
   atm_window_length: PT6H
   forecast_step: PT1H
   valid_time: 2021-08-01T00:00:00Z
@@ -23,5 +23,6 @@ job options:
   queue: debug
   partition: hera
   walltime: '30:00'
-  ntasks: 96
+  ntasks: 6
+  ntasks-per-node: 2
   modulepath: /scratch1/NCEPDEV/da/Cory.R.Martin/GDASApp/work/GDASApp/modulefiles

ush/examples/run_jedi_exe/4dhofx_orion.yaml

@@ -1,15 +1,15 @@
 working directory: /work2/noaa/stmp/cmartin/gdas_single_test_hofx4d
 GDASApp home: /work2/noaa/da/cmartin/GDASApp/work/GDASApp
 GDASApp mode: hofx
-executable options:
+template: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/hofx/hofx4d.yaml
+config:
   obs_yaml_dir: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/config
-  yaml_template: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/hofx/hofx4d.yaml
   executable: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/build/bin/fv3jedi_hofx.x
-  obs_list: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype.yaml
+  obs_list: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/parm/atm/obs/lists/gdas_prototype_4d.yaml
   gdas_fix_root: /work2/noaa/da/cmartin/GDASApp/fix
   atm: true
-  layout_x: 4
-  layout_y: 4
+  layout_x: 1
+  layout_y: 1
   atm_window_length: PT6H
   forecast_step: PT1H
   valid_time: 2021-08-01T00:00:00Z
@@ -23,5 +23,6 @@ job options:
   queue: debug
   partition: debug
   walltime: '30:00'
-  ntasks: 96
+  ntasks: 6
+  ntasks-per-node: 2
   modulepath: /work2/noaa/da/cmartin/GDASApp/work/GDASApp/modulefiles

ush/run_jedi_exe.py

@@ -39,7 +39,7 @@ def run_jedi_exe(yamlconfig):
 
     # check if the specified app mode is valid
     app_mode = all_config_dict['GDASApp mode']
-    supported_app_modes = ['hofx', 'variational', 'gw_scripts']
+    supported_app_modes = ['hofx', 'variational']
     if app_mode not in supported_app_modes:
         raise KeyError(f"'{app_mode}' not supported. " +
                        "Current GDASApp modes supported are: " +
@@ -78,136 +78,73 @@ def run_jedi_exe(yamlconfig):
     oprefix = executable_subconfig['dump'] + ".t" + str(cyc) + "z."
     gprefix = executable_subconfig['dump'] + ".t" + str(gcyc) + "z."
 
-    if app_mode in ['hofx', 'variational']:
-        single_exec = True
-        var_config = {
-            'DATA': os.path.join(workdir),
-            'APREFIX': str(oprefix),
-            'OPREFIX': str(oprefix),
-            'GPREFIX': str(gprefix),
-            'BERROR_YAML': executable_subconfig.get('berror_yaml', './'),
-            'STATICB_TYPE': executable_subconfig.get('staticb_type', 'gsibec'),
-            'OBS_YAML_DIR': executable_subconfig['obs_yaml_dir'],
-            'OBS_LIST': executable_subconfig['obs_list'],
-            'atm': executable_subconfig.get('atm', False),
-            'layout_x': str(executable_subconfig['layout_x']),
-            'layout_y': str(executable_subconfig['layout_y']),
-            'BKG_DIR': os.path.join(workdir, 'bkg'),
-            'fv3jedi_fix_dir': os.path.join(workdir, 'fv3jedi'),
-            'fv3jedi_fieldmetadata_dir': os.path.join(workdir, 'fv3jedi'),
-            'ANL_DIR': os.path.join(workdir, 'anl'),
-            'fv3jedi_staticb_dir': os.path.join(workdir, 'berror'),
-            'BIAS_IN_DIR': os.path.join(workdir, 'obs'),
-            'BIAS_OUT_DIR': os.path.join(workdir, 'bc'),
-            'CRTM_COEFF_DIR': os.path.join(workdir, 'crtm'),
-            'BIAS_PREFIX': f"{executable_subconfig['dump']}.t{gcyc}z.",
-            'BIAS_DATE': f"{gdate}",
-            'DIAG_DIR': os.path.join(workdir, 'diags'),
-            'OBS_DIR': os.path.join(workdir, 'obs'),
-            'OBS_PREFIX': f"{executable_subconfig['dump']}.t{cyc}z.",
-            'OBS_DATE': f"{cdate}",
-            'CDATE': f"{cdate}",
-            'GDATE': f"{gdate}",
-            'valid_time': f"{valid_time.strftime('%Y-%m-%dT%H:%M:%SZ')}",
-            'window_begin': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}",
-            'prev_valid_time': f"{prev_cycle.strftime('%Y-%m-%dT%H:%M:%SZ')}",
-            'atm_window_length': executable_subconfig['atm_window_length'],
-            'CASE': executable_subconfig['case'],
-            'CASE_ANL': executable_subconfig.get('case_anl', executable_subconfig['case']),
-            'CASE_ENKF': executable_subconfig.get('case_enkf', executable_subconfig['case']),
-            'DOHYBVAR': executable_subconfig.get('dohybvar', False),
-            'LEVS': str(executable_subconfig['levs']),
-            'NMEM_ENKF': executable_subconfig.get('nmem', 0),
-            'forecast_steps': calc_fcst_steps(executable_subconfig.get('forecast_step', 'PT6H'),
-                                              executable_subconfig['atm_window_length']),
-            'BKG_TSTEP': executable_subconfig.get('forecast_step', 'PT6H'),
-            'INTERP_METHOD': executable_subconfig.get('interp_method', 'barycentric'),
-        }
-        output_file = os.path.join(workdir, f"gdas_{app_mode}.yaml")
-        # set some environment variables
-        os.environ['PARMgfs'] = os.path.join(all_config_dict['GDASApp home'], 'parm')
-        for key, value in var_config.items():
-            os.environ[key] = str(value)
-        # generate YAML for executable based on input config
-        logging.info(f'Using yamlconfig {yamlconfig}')
-        genYAML(yamlconfig, output=output_file)
-        logging.info(f'Wrote YAML file to {output_file}')
-        # use R2D2 to stage backgrounds, obs, bias correction files, etc.
-        ufsda.stage.gdas_single_cycle(var_config)
-        # link additional fix files needed (CRTM, fieldmetadata, etc.)
-        gdasfix = executable_subconfig['gdas_fix_root']
-        ufsda.stage.gdas_fix(gdasfix, workdir, var_config)
-        # link executable
-        baseexe = os.path.basename(executable_subconfig['executable'])
-        ufsda.disk_utils.symlink(executable_subconfig['executable'], os.path.join(workdir, baseexe))
-        # create output directories
-        ufsda.disk_utils.mkdir(os.path.join(workdir, 'diags'))
-        if app_mode in ['variational']:
-            ufsda.disk_utils.mkdir(os.path.join(workdir, 'anl'))
-            ufsda.disk_utils.mkdir(os.path.join(workdir, 'bc'))
-        baseexe = os.path.join(workdir, baseexe)
-    else:
-        baseexe = ''
-        output_file = ''
-        single_exec = False
-        gdasapp_home = os.path.join(all_config_dict['GDASApp home'])
-        gdasapp_bin = os.path.join(gdasapp_home, 'build', 'bin')
-        homegfs = os.path.join(workdir, 'HOMEgfs')
-        aprun_socaanal = all_config_dict['job options']['mpiexec']+' '+str(all_config_dict['job options']['ntasks'])
-        comin_ges_src = os.path.join(all_config_dict['model backgrounds']['ocn'], 'RESTART')
-        ufsda.disk_utils.mkdir(os.path.join(workdir, 'RESTART'))
-        ufsda.disk_utils.copytree(comin_ges_src, os.path.join(workdir, 'RESTART'))
-
-        runtime_envar = {
-            'CDATE': cdate,
-            'GDATE': gdate,
-            'gcyc': gcyc,
-            'PDY': pdy,
-            'cyc': cyc,
-            'assim_freq': '6',
-            'COMOUT': workdir,
-            'DATA': os.path.join(workdir, 'analysis'),
-            'COMIN_OBS': all_config_dict['r2d2 options']['root'],
-            'COMIN_GES': all_config_dict['model backgrounds']['ocn'],
-            'CDUMP': 'gdas',
-            'GDUMP': 'gdas',
-            'CASE_ANL': "C48",
-            'CASE': 'C48',
-            'DOHYBVAR': 'False',
-            'CASE_ENKF': "C192",
-            'LEVS': '75',
-            'OBS_YAML_DIR': executable_subconfig['obs_yaml_dir'],
-            'OBS_LIST': executable_subconfig['obs_list'],
-            'OBS_YAML': executable_subconfig['obs_list'],
-            'JEDI_BIN': gdasapp_bin,
-            'HOMEgfs': homegfs,
-            'SOCA_INPUT_FIX_DIR': all_config_dict['jedi static']['soca']['path'],
-            'STATICB_DIR': os.path.join(workdir, 'soca_static'),
-            'R2D2_OBS_DB': 'shared',
-            'R2D2_OBS_DUMP': all_config_dict['r2d2 options']['obs_dump'],
-            'R2D2_OBS_SRC': all_config_dict['r2d2 options']['obs_src'],
-            'R2D2_OBS_WINDOW': '24',
-            'FV3JEDI_STAGE_YAML': os.path.join(gdasapp_home, 'test', 'soca', 'testinput', 'dumy.yaml'),
-            'DOMAIN_STACK_SIZE': all_config_dict['fms']['domain_stack_size'],
-            'SOCA_VARS': all_config_dict['jedi options']['soca']['variables'],
-            'SOCA_NINNER': all_config_dict['jedi options']['soca']['ninner'],
-        }
-
-        # do something to resolve gw env. variables
-        runtime_envar_yaml = os.path.join(workdir, 'runtime_envar.yaml')
-        f = open(runtime_envar_yaml, 'w')
-        yaml.dump(runtime_envar, f, sort_keys=False, default_flow_style=False)
-        bashout = os.path.join(workdir, 'load_envar.sh')
-        export_envar(runtime_envar_yaml, bashout)
-
-        # link gdas.cd
-        ufsda.mkdir(os.path.join(homegfs, 'sorc'))
-        ufsda_link = os.path.join(homegfs, 'sorc', 'gdas.cd')
-        ufsda.disk_utils.symlink(all_config_dict['GDASApp home'],
-                                 ufsda_link)
-        ush_link = os.path.join(homegfs, 'ush')
-        ufsda.disk_utils.symlink(os.path.join(all_config_dict['GDASApp home'], 'ush'),
-                                 ush_link)
+    single_exec = True
+    var_config = {
+        'DATA': os.path.join(workdir),
+        'APREFIX': str(oprefix),
+        'OPREFIX': str(oprefix),
+        'GPREFIX': str(gprefix),
+        'BERROR_YAML': executable_subconfig.get('berror_yaml', './'),
+        'STATICB_TYPE': executable_subconfig.get('staticb_type', 'gsibec'),
+        'OBS_YAML_DIR': executable_subconfig['obs_yaml_dir'],
+        'OBS_LIST': executable_subconfig['obs_list'],
+        'atm': executable_subconfig.get('atm', False),
+        'layout_x': str(executable_subconfig['layout_x']),
+        'layout_y': str(executable_subconfig['layout_y']),
+        'BKG_DIR': os.path.join(workdir, 'bkg'),
+        'fv3jedi_fix_dir': os.path.join(workdir, 'fv3jedi'),
+        'fv3jedi_fieldmetadata_dir': os.path.join(workdir, 'fv3jedi'),
+        'ANL_DIR': os.path.join(workdir, 'anl'),
+        'fv3jedi_staticb_dir': os.path.join(workdir, 'berror'),
+        'BIAS_IN_DIR': os.path.join(workdir, 'obs'),
+        'BIAS_OUT_DIR': os.path.join(workdir, 'bc'),
+        'CRTM_COEFF_DIR': os.path.join(workdir, 'crtm'),
+        'BIAS_PREFIX': f"{executable_subconfig['dump']}.t{gcyc}z.",
+        'BIAS_DATE': f"{gdate}",
+        'DIAG_DIR': os.path.join(workdir, 'diags'),
+        'OBS_DIR': os.path.join(workdir, 'obs'),
+        'OBS_PREFIX': f"{executable_subconfig['dump']}.t{cyc}z.",
+        'OBS_DATE': f"{cdate}",
+        'CDATE': f"{cdate}",
+        'GDATE': f"{gdate}",
+        'valid_time': f"{valid_time.strftime('%Y-%m-%dT%H:%M:%SZ')}",
+        'window_begin': f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}",
+        'prev_valid_time': f"{prev_cycle.strftime('%Y-%m-%dT%H:%M:%SZ')}",
+        'atm_window_length': executable_subconfig['atm_window_length'],
+        'CASE': executable_subconfig['case'],
+        'CASE_ANL': executable_subconfig.get('case_anl', executable_subconfig['case']),
+        'CASE_ENKF': executable_subconfig.get('case_enkf', executable_subconfig['case']),
+        'DOHYBVAR': executable_subconfig.get('dohybvar', False),
+        'LEVS': str(executable_subconfig['levs']),
+        'NMEM_ENKF': executable_subconfig.get('nmem', 0),
+        'forecast_steps': calc_fcst_steps(executable_subconfig.get('forecast_step', 'PT6H'),
+                                          executable_subconfig['atm_window_length']),
+        'BKG_TSTEP': executable_subconfig.get('forecast_step', 'PT6H'),
+        'INTERP_METHOD': executable_subconfig.get('interp_method', 'barycentric'),
+    }
+    output_file = os.path.join(workdir, f"gdas_{app_mode}.yaml")
+    # set some environment variables
+    os.environ['PARMgfs'] = os.path.join(all_config_dict['GDASApp home'], 'parm')
+    for key, value in var_config.items():
+        os.environ[key] = str(value)
+    # generate YAML for executable based on input config
+    logging.info(f'Using yamlconfig {yamlconfig}')
+    genYAML(yamlconfig, output=output_file)
+    logging.info(f'Wrote YAML file to {output_file}')
+    # use R2D2 to stage backgrounds, obs, bias correction files, etc.
+    ufsda.stage.gdas_single_cycle(var_config)
+    # link additional fix files needed (CRTM, fieldmetadata, etc.)
+    gdasfix = executable_subconfig['gdas_fix_root']
+    ufsda.stage.gdas_fix(gdasfix, workdir, var_config)
+    # link executable
+    baseexe = os.path.basename(executable_subconfig['executable'])
+    ufsda.disk_utils.symlink(executable_subconfig['executable'], os.path.join(workdir, baseexe))
+    # create output directories
+    ufsda.disk_utils.mkdir(os.path.join(workdir, 'diags'))
+    if app_mode in ['variational']:
+        ufsda.disk_utils.mkdir(os.path.join(workdir, 'anl'))
+        ufsda.disk_utils.mkdir(os.path.join(workdir, 'bc'))
+    baseexe = os.path.join(workdir, baseexe)
 
     # generate job submission script
     job_script = ufsda.misc_utils.create_batch_job(all_config_dict['job options'],