From 45c3d2f1c8a75315bbfa5e39621c1b603bfecf49 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 22 Jun 2022 14:11:02 -0600 Subject: [PATCH] feature 1653 develop climo day_interval NA value (#1671) * moved logic to handle climo_mean/stdev dictionaries into met_config utility * moved constant variable to constants and made name uppercase * per #1599, added logic to read VAR variables for climo_mean/stdev (i.e. GRID_STAT_CLIMO_MEAN_VAR1_NAME). Added function to easily format field info for a single field or list of fields with fewer arguments * added function to format field info that is easier to call than get_field_info and can eventually replace it * added line breaks to match pep8 standards, ci-run-all-diff * per 1599, added documentation for new config variables * replaced calls to get_field_info with format_field_info since it is much simpler * fixed bug where field list is never returned from function call * removed whitespace * modified logic to report an error if no commands were run at all * only report an error if the process list contains a wrapper that should generate at least 1 command. Example and CyclonePlotter wrappers do not run any shell commands, so they are excluded * per feedback in PR #1658 review, added glossary entry that is wrapper independent and provide reference to that entry instead of duplicating information for each wrapper * per #1653, add unit tests to check that day_interval value of climo_mean/stdev dictionary can be NA or an integer * per #1653, modified function to read climo variables to support NA instead of just integers * per #1653, updated unit tests to test both climo_mean and climo_stdev variables, added tests for all other climo variables --- .../grid_stat/test_grid_stat_wrapper.py | 3 + .../pytests/met_config/test_met_config.py | 140 +++++++++++++++--- metplus/util/met_config.py | 2 +- 3 files changed, 125 insertions(+), 20 deletions(-) diff --git a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py index 34385c23c2..e29aeaae5a 100644 --- a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py +++ b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py @@ -516,6 +516,9 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, ({'GRID_STAT_CLIMO_MEAN_DAY_INTERVAL': '30', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {day_interval = 30;}'}), + ({'GRID_STAT_CLIMO_MEAN_DAY_INTERVAL': 'NA', }, + {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {day_interval = NA;}'}), + ({'GRID_STAT_CLIMO_MEAN_HOUR_INTERVAL': '12', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {hour_interval = 12;}'}), diff --git a/internal_tests/pytests/met_config/test_met_config.py b/internal_tests/pytests/met_config/test_met_config.py index c67cf09e0b..0f990e678b 100644 --- a/internal_tests/pytests/met_config/test_met_config.py +++ b/internal_tests/pytests/met_config/test_met_config.py @@ -11,37 +11,139 @@ # 0 no relevant config set ({}, ''), # 1 _FIELD set - ({'GRID_STAT_CLIMO_MEAN_FIELD': '{name="TMP"; level="(*,*)";}'}, + ({'APP_CLIMO__FIELD': '{name="TMP"; level="(*,*)";}'}, '{name="TMP"; level="(*,*)";}'), # 2 VAR1 name/level set - ({'GRID_STAT_CLIMO_MEAN_VAR1_NAME': 'TMP', - 'GRID_STAT_CLIMO_MEAN_VAR1_LEVELS': '"(*,*)"'}, + ({'APP_CLIMO__VAR1_NAME': 'TMP', + 'APP_CLIMO__VAR1_LEVELS': '"(*,*)"'}, '{ name="TMP"; level="(*,*)"; }'), # 3 VAR1/2 name/level set - ({'GRID_STAT_CLIMO_MEAN_VAR1_NAME': 'TMP', - 'GRID_STAT_CLIMO_MEAN_VAR1_LEVELS': '"(*,*)"', - 'GRID_STAT_CLIMO_MEAN_VAR2_NAME': 'PRES', - 'GRID_STAT_CLIMO_MEAN_VAR2_LEVELS': '"(0,*,*)"'}, + ({'APP_CLIMO__VAR1_NAME': 'TMP', + 'APP_CLIMO__VAR1_LEVELS': '"(*,*)"', + 'APP_CLIMO__VAR2_NAME': 'PRES', + 'APP_CLIMO__VAR2_LEVELS': '"(0,*,*)"'}, '{ name="TMP"; level="(*,*)"; },{ name="PRES"; level="(0,*,*)"; }'), # 4 VAR1 name/level and FIELD set - prefer VAR - ({'GRID_STAT_CLIMO_MEAN_FIELD': '{name="TEMP"; level="(0,*,*)";}', - 'GRID_STAT_CLIMO_MEAN_VAR1_NAME': 'TMP', - 'GRID_STAT_CLIMO_MEAN_VAR1_LEVELS': '"(*,*)"'}, + ({'APP_CLIMO__FIELD': '{name="TEMP"; level="(0,*,*)";}', + 'APP_CLIMO__VAR1_NAME': 'TMP', + 'APP_CLIMO__VAR1_LEVELS': '"(*,*)"'}, '{ name="TMP"; level="(*,*)"; }'), ] ) def test_read_climo_field(metplus_config, config_overrides, expected_value): - app_name = 'grid_stat' - climo_type = 'MEAN' - expected_var = f'{app_name}_CLIMO_{climo_type}_FIELD'.upper() - config = metplus_config() + app_name = 'app' + for climo_type in ('MEAN', 'STDEV'): + expected_var = f'{app_name}_CLIMO_{climo_type}_FIELD'.upper() + config = metplus_config() + + # set config values + for key, value in config_overrides.items(): + key_sub = key.replace('', climo_type) + value_sub = value.replace('', climo_type.lower()) + config.set('config', key_sub, value_sub) - # set config values - for key, value in config_overrides.items(): - config.set('config', key, value) + _read_climo_field(climo_type, config, app_name) + assert config.getraw('config', expected_var) == expected_value - _read_climo_field(climo_type, config, app_name) - assert config.getraw('config', expected_var) == expected_value +@pytest.mark.parametrize( + 'config_overrides, expected_value', [ + # 0 no relevant config set + ({}, ''), + # 1 file name single + ({'APP_CLIMO__FILE_NAME': 'some/file/path'}, + 'climo_ = {file_name = ["some/file/path"];}'), + # 2 file name multiple + ({'APP_CLIMO__FILE_NAME': 'some/file/path, other/path'}, + 'climo_ = {file_name = ["some/file/path", "other/path"];}'), + # 3 field single + ({'APP_CLIMO__FIELD': '{name="TMP"; level="(*,*)";}'}, + 'climo_ = {field = [{name="TMP"; level="(*,*)";}];}'), + # 4 field multiple + ({'APP_CLIMO__FIELD': ('{name="TMP"; level="(*,*)";},' + '{name="TEMP"; level="P500";}')}, + ('climo_ = {field = [{name="TMP"; level="(*,*)";}, ' + '{name="TEMP"; level="P500";}];}')), + # 5 use fcst no other climo_ + ({'APP_CLIMO__USE_FCST': 'TRUE'}, + 'climo_ = fcst;'), + # 6 use obs no other climo_ + ({'APP_CLIMO__USE_OBS': 'TRUE'}, + 'climo_ = obs;'), + # 7 use fcst with other climo_ + ({'APP_CLIMO__REGRID_METHOD': 'NEAREST', + 'APP_CLIMO__USE_FCST': 'TRUE'}, + 'climo_ = {regrid = {method = NEAREST;}}climo_ = fcst;'), + # 8 use obs with other climo_ + ({'APP_CLIMO__REGRID_METHOD': 'NEAREST', + 'APP_CLIMO__USE_OBS': 'TRUE'}, + 'climo_ = {regrid = {method = NEAREST;}}climo_ = obs;'), + # 9 regrid method + ({'APP_CLIMO__REGRID_METHOD': 'NEAREST', }, + 'climo_ = {regrid = {method = NEAREST;}}'), + # 10 regrid width + ({'APP_CLIMO__REGRID_WIDTH': '1', }, + 'climo_ = {regrid = {width = 1;}}'), + # 11 regrid vld_thresh + ({'APP_CLIMO__REGRID_VLD_THRESH': '0.5', }, + 'climo_ = {regrid = {vld_thresh = 0.5;}}'), + # 12 regrid shape + ({'APP_CLIMO__REGRID_SHAPE': 'SQUARE', }, + 'climo_ = {regrid = {shape = SQUARE;}}'), + # 13 time_interp_method + ({'APP_CLIMO__TIME_INTERP_METHOD': 'NEAREST', }, + 'climo_ = {time_interp_method = NEAREST;}'), + # 14 match_month + ({'APP_CLIMO__MATCH_MONTH': 'True', }, + 'climo_ = {match_month = TRUE;}'), + # 15 day_interval - int + ({'APP_CLIMO__DAY_INTERVAL': '30', }, + 'climo_ = {day_interval = 30;}'), + # 16 day_interval - NA + ({'APP_CLIMO__DAY_INTERVAL': 'NA', }, + 'climo_ = {day_interval = NA;}'), + # 17 hour_interval + ({'APP_CLIMO__HOUR_INTERVAL': '12', }, + 'climo_ = {hour_interval = 12;}'), + # 18 all + ({ + 'APP_CLIMO__FILE_NAME': '/some/climo_/file.txt', + 'APP_CLIMO__FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', + 'APP_CLIMO__REGRID_METHOD': 'NEAREST', + 'APP_CLIMO__REGRID_WIDTH': '1', + 'APP_CLIMO__REGRID_VLD_THRESH': '0.5', + 'APP_CLIMO__REGRID_SHAPE': 'SQUARE', + 'APP_CLIMO__TIME_INTERP_METHOD': 'NEAREST', + 'APP_CLIMO__MATCH_MONTH': 'True', + 'APP_CLIMO__DAY_INTERVAL': '30', + 'APP_CLIMO__HOUR_INTERVAL': '12', + }, + ('climo_ = {file_name = ' + '["/some/climo_/file.txt"];' + 'field = [{name="CLM_NAME"; level="(0,0,*,*)";}];' + 'regrid = {method = NEAREST;width = 1;' + 'vld_thresh = 0.5;shape = SQUARE;}' + 'time_interp_method = NEAREST;' + 'match_month = TRUE;day_interval = 30;' + 'hour_interval = 12;}')), + ] +) +def test_handle_climo_dict(metplus_config, config_overrides, expected_value): + app_name = 'app' + for climo_type in ('MEAN', 'STDEV'): + expected_var = f'METPLUS_CLIMO_{climo_type}_DICT' + config = metplus_config() + output_dict = {} + + # set config values + for key, value in config_overrides.items(): + key_sub = key.replace('', climo_type) + value_sub = value.replace('', climo_type.lower()) + config.set('config', key_sub, value_sub) + + handle_climo_dict(config, app_name, output_dict) + print(output_dict) + expected_sub = expected_value.replace('', climo_type.lower()) + assert output_dict[expected_var] == expected_sub @pytest.mark.parametrize( 'name, data_type, mp_configs, extra_args', [ diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index b2cebe49e9..807cf10a3d 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -773,7 +773,7 @@ def handle_climo_dict(config, app_name, output_dict): }), 'time_interp_method': ('string', 'remove_quotes,uppercase'), 'match_month': ('bool', 'uppercase'), - 'day_interval': 'int', + 'day_interval': ('string', 'remove_quotes,uppercase'), 'hour_interval': 'int', 'file_type': ('string', 'remove_quotes'), }