-
Notifications
You must be signed in to change notification settings - Fork 32
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enable run_jedi_exe.py to run 3dvar (#278)
* Restore 3dvar capability to run_jedi_exe.py (#223) * Correct python norm errors (#223) * Remove yaml_template from 3dvar_orion.yaml (#223) * add genYAML.py; clean up run_jedi_exe.py (#223) * Update genYAML to use ufsda genYAML.py (#223) * add ufsda.genYAML to __init__.py (#223) * remove genYAML from yamltools.py (#223) * remove __main__ from genYAML.py * revert changes to ush/genYAML * remove stage from init, simplify genYAML * import specific ufsda.stage functions in run_jedi_exe.py * replace exclusive with mem=0 in sbatch jobcard * add ufsda.stage import to marine_analysis_prep script * use RussTreadon-NOAA/global-workflow feature/ufsda_stage in automated tests * Revert "use RussTreadon-NOAA/global-workflow feature/ufsda_stage in automated tests" This reverts commit 3c828e2. * update keywords in hera 3dvar yaml * provide default for NMEM_ENKF in run_jedi_exe.py
- Loading branch information
1 parent
ef9106c
commit f7c23af
Showing
11 changed files
with
151 additions
and
121 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,8 @@ | ||
| from .disk_utils import mkdir, symlink | ||
| from .ufs_yaml import gen_yaml, parse_config | ||
| import ufsda.stage | ||
| import ufsda.archive | ||
| import ufsda.r2d2 | ||
| import ufsda.post | ||
| import ufsda.yamltools | ||
| import ufsda.genYAML | ||
| from .misc_utils import isTrue, create_batch_job, submit_batch_job |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| #!/usr/bin/env python3 | ||
| # genYAML | ||
| # generate YAML using ufsda python module, | ||
| # current runtime env, and optional input YAML | ||
| import argparse | ||
| import datetime as dt | ||
| import logging | ||
| import os | ||
| import re | ||
| import yaml | ||
| from pygw.template import Template, TemplateConstants | ||
| from pygw.yaml_file import YAMLFile | ||
|
|
||
|
|
||
| def genYAML(yamlconfig, output=None): | ||
| logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') | ||
| # open YAML file to get config | ||
| try: | ||
| with open(yamlconfig, 'r') as yamlconfig_opened: | ||
| all_config_dict = yaml.safe_load(yamlconfig_opened) | ||
| logging.info(f'Loading configuration from {yamlconfig}') | ||
| except Exception as e: | ||
| logging.error(f'Error occurred when attempting to load: {yamlconfig}, error: {e}') | ||
|
|
||
| if not output: | ||
| output_file = all_config_dict['output'] | ||
| else: | ||
| output_file = output | ||
|
|
||
| template = all_config_dict['template'] | ||
| config_dict = all_config_dict['config'] | ||
| # what if the config_dict has environment variables that need substituted? | ||
| pattern = re.compile(r'.*?\${(\w+)}.*?') | ||
| for key, value in config_dict.items(): | ||
| if type(value) == str: | ||
| match = pattern.findall(value) | ||
| if match: | ||
| fullvalue = value | ||
| for g in match: | ||
| config_dict[key] = fullvalue.replace( | ||
| f'${{{g}}}', os.environ.get(g, f'${{{g}}}') | ||
| ) | ||
| # NOTE the following is a hack until YAMLFile can take in an input config dict | ||
| # if something in the template is expected to be an env var | ||
| # but it is not defined in the env, problems will arise | ||
| # so we set the env var in this subprocess for the substitution to occur | ||
| for key, value in config_dict.items(): | ||
| os.environ[key] = str(value) | ||
| # next we need to compute a few things | ||
| runtime_config = get_runtime_config(dict(os.environ, **config_dict)) | ||
| # now run the global-workflow parser | ||
| outconfig = YAMLFile(path=template) | ||
| outconfig = Template.substitute_structure(outconfig, TemplateConstants.DOUBLE_CURLY_BRACES, config_dict.get) | ||
| outconfig = Template.substitute_structure(outconfig, TemplateConstants.DOLLAR_PARENTHESES, config_dict.get) | ||
| outconfig = Template.substitute_structure(outconfig, TemplateConstants.DOUBLE_CURLY_BRACES, runtime_config.get) | ||
| outconfig = Template.substitute_structure(outconfig, TemplateConstants.DOLLAR_PARENTHESES, runtime_config.get) | ||
| outconfig.save(output_file) | ||
|
|
||
|
|
||
| def get_runtime_config(config_dict): | ||
| # compute some runtime variables | ||
| # this will probably need pulled out somewhere else eventually | ||
| # a temporary hack to get UFO evaluation stuff and ATM VAR going again | ||
| valid_time = dt.datetime.strptime(config_dict['CDATE'], '%Y%m%d%H') | ||
| assim_freq = int(config_dict.get('assim_freq', 6)) | ||
| window_begin = valid_time - dt.timedelta(hours=assim_freq/2) | ||
| window_end = valid_time + dt.timedelta(hours=assim_freq/2) | ||
| component_dict = { | ||
| 'atmos': 'ATM', | ||
| 'chem': 'AERO', | ||
| 'ocean': 'SOCA', | ||
| 'land': 'land', | ||
| } | ||
| win_begin_var = component_dict[config_dict.get('COMPONENT', 'atmos')] + '_WINDOW_BEGIN' | ||
| win_end_var = component_dict[config_dict.get('COMPONENT', 'atmos')] + '_WINDOW_END' | ||
| win_len_var = component_dict[config_dict.get('COMPONENT', 'atmos')] + '_WINDOW_LENGTH' | ||
| bkg_string_var = 'BKG_YYYYmmddHHMMSS' | ||
| bkg_isotime_var = 'BKG_ISOTIME' | ||
| npx_ges_var = 'npx_ges' | ||
| npy_ges_var = 'npy_ges' | ||
| npz_ges_var = 'npz_ges' | ||
| npx_anl_var = 'npx_anl' | ||
| npy_anl_var = 'npy_anl' | ||
| npz_anl_var = 'npz_anl' | ||
|
|
||
| runtime_config = { | ||
| win_begin_var: f"{window_begin.strftime('%Y-%m-%dT%H:%M:%SZ')}", | ||
| win_end_var: f"{window_end.strftime('%Y-%m-%dT%H:%M:%SZ')}", | ||
| win_len_var: f"PT{assim_freq}H", | ||
| bkg_string_var: f"{valid_time.strftime('%Y%m%d.%H%M%S')}", | ||
| bkg_isotime_var: f"{valid_time.strftime('%Y-%m-%dT%H:%M:%SZ')}", | ||
| npx_ges_var: f"{int(os.environ['CASE'][1:]) + 1}", | ||
| npy_ges_var: f"{int(os.environ['CASE'][1:]) + 1}", | ||
| npz_ges_var: f"{int(os.environ['LEVS']) - 1}", | ||
| npx_anl_var: f"{int(os.environ['CASE_ENKF'][1:]) + 1}", | ||
| npy_anl_var: f"{int(os.environ['CASE_ENKF'][1:]) + 1}", | ||
| npz_anl_var: f"{int(os.environ['LEVS']) - 1}", | ||
| } | ||
|
|
||
| return runtime_config |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.