From df4458bc040e1c1c8c2f915e206b6a167ca5d3a5 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 13 Apr 2021 09:58:45 -0600 Subject: [PATCH 01/42] modified handle_mask function to use new approach to setting env vars for MET configs, call old function read_mask_poly to handle old env var name VERIFICATION_MASK --- metplus/wrappers/command_builder.py | 80 ++++++++++++++------------- metplus/wrappers/grid_stat_wrapper.py | 6 +- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index ee5348e10d..79d3be26ad 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1952,54 +1952,56 @@ def handle_time_summary_dict(self, c_dict, remove_bracket_list=None): for list_values in remove_bracket_list: c_dict[list_values] = c_dict[list_values].strip('[]') - def handle_mask(self, single_value=False, c_dict=None): + def handle_mask(self, single_value=False, get_flags=False): """! Read mask dictionary values and set them into env_var_list @param single_value if True, only a single value for grid and poly are allowed. If False, they should be treated as as list - @param c_dict (optional) dictionary to set VERIFICATION_MASK key - with mask.grid value to support old MET config environment variable + @param get_flags if True, read grid_flag and poly_flag values """ app = self.app_name.upper() - tmp_dict = {} - extra_args = {} - if single_value: - set_met_config = self.set_met_config_string - else: - set_met_config = self.set_met_config_list - - set_met_config(tmp_dict, - [f'{app}_MASK_GRID', - f'{app}_GRID'], - 'grid', - 'MASK_GRID', - allow_empty=True) - set_met_config(tmp_dict, - [f'{app}_MASK_POLY', - f'{app}_VERIFICATION_MASK_TEMPLATE', - f'{app}_POLY'], - 'poly', - 'MASK_POLY', - allow_empty=True) - - # set VERIFICATION MASK to support old method of setting mask.poly - mask_poly_string = tmp_dict.get('MASK_POLY') - if mask_poly_string: - mask_poly_string = ( - mask_poly_string.split('=')[1].strip().strip(';').strip('[]') + + dict_name = 'mask' + dict_items = [] + + data_type = 'string' if single_value else 'list' + + dict_items.append( + self.get_met_config( + name='grid', + data_type=data_type, + metplus_configs=[f'{app}_MASK_GRID', + f'{app}_GRID'], + extra_args={'allow_empty': True} ) - else: - mask_poly_string = '' + ) - if c_dict: - c_dict['VERIFICATION_MASK'] = mask_poly_string + dict_items.append( + self.get_met_config( + name='poly', + data_type=data_type, + metplus_configs=[f'{app}_MASK_POLY', + f'{app}_VERIFICATION_MASK_TEMPLATE', + f'{app}_POLY'], + extra_args={'allow_empty': True} + ) + ) + # get grid_flag and poly_flag if requested + if get_flags: + dict_items.append( + self.get_met_config(name='grid_flag', + data_type='string', + metplus_configs=[f'{app}_MASK_GRID_FLAG'], + extra_args={'remove_quotes': True}) + ) + dict_items.append( + self.get_met_config(name='poly_flag', + data_type='string', + metplus_configs=[f'{app}_MASK_POLY_FLAG'], + extra_args={'remove_quotes': True}) + ) - # set METPLUS_MASK_DICT env var with mask items if set - mask_dict_string = self.format_met_config_dict(tmp_dict, - 'mask', - ['MASK_GRID', - 'MASK_POLY']) - self.env_var_dict['METPLUS_MASK_DICT'] = mask_dict_string + self.handle_met_config_dict(dict_name, dict_items) def get_met_config_function(self, item_type): """! Return function to use based on item type diff --git a/metplus/wrappers/grid_stat_wrapper.py b/metplus/wrappers/grid_stat_wrapper.py index ff6b64bac9..1ce1291ec1 100755 --- a/metplus/wrappers/grid_stat_wrapper.py +++ b/metplus/wrappers/grid_stat_wrapper.py @@ -173,8 +173,10 @@ def create_c_dict(self): 'GRID_STAT_NEIGHBORHOOD_SHAPE', 'SQUARE') ) - self.handle_mask(single_value=False, - c_dict=c_dict) + self.handle_mask(single_value=False) + + # handle setting VERIFICATION_MASK for old wrapped MET config files + c_dict['MASK_POLY_TEMPLATE'] = self.read_mask_poly() self.handle_climo_cdf_dict() From 7d0f1d658b2972cf0d5bff56a1585987d7b59d95 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 13 Apr 2021 10:03:45 -0600 Subject: [PATCH 02/42] try moving fcst/obs options to FCST/OBS_VAR_OPTIONS, ci-run-diff --- .../climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf index 02eaa48a38..b73052d5fd 100644 --- a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf +++ b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf @@ -43,18 +43,20 @@ MODE_MERGE_THRESH = ge10.0, ge20.0 MODE_MERGE_FLAG = THRESH -MODE_MET_CONFIG_OVERRIDES = fcst = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } obs = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } match_flag = NO_MERGE; mask = { poly_flag = BOTH; } weight = {aspect_diff = 1.0;} fcst_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } obs_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } nc_pairs_flag = {polylines = FALSE; } +#MODE_MET_CONFIG_OVERRIDES = fcst = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } obs = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } match_flag = NO_MERGE; mask = { poly_flag = BOTH; } weight = {aspect_diff = 1.0;} fcst_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } obs_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } nc_pairs_flag = {polylines = FALSE; } +MODE_MET_CONFIG_OVERRIDES = match_flag = NO_MERGE; mask = { poly_flag = BOTH; } weight = {aspect_diff = 1.0;} fcst_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } obs_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } nc_pairs_flag = {polylines = FALSE; } # Forecast Reflectivity Variable Information MODEL = CESM FCST_VAR1_NAME = PRECT FCST_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" -FCST_VAR1_OPTIONS = convert(x) = 86400000*x; +FCST_VAR1_OPTIONS = convert(x) = 86400000*x; filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; # MRMS Reflecivitiy Variable Information OBTYPE = GPCP OBS_VAR1_NAME = precip OBS_VAR1_LEVELS = "(0,*,*)" +OBS_VAR1_OPTIONS = filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped MODE_REGRID_TO_GRID = FCST From 849c41dbb7eaafe505e4c25743ac87506643d4cb Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 13 Apr 2021 11:10:53 -0600 Subject: [PATCH 03/42] started adding support for new env vars for the wrapped MET configs --- metplus/wrappers/mode_wrapper.py | 19 +++++++++++++++++-- parm/met_config/MODEConfig_wrapped | 8 ++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 30e239b450..30ebfa685a 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -36,8 +36,20 @@ class MODEWrapper(CompareGriddedWrapper): 'METPLUS_OBS_MERGE_THRESH', 'METPLUS_OBS_MERGE_FLAG', 'METPLUS_MASK_POLY', + 'METPLUS_MASK_DICT', 'METPLUS_OUTPUT_PREFIX', 'METPLUS_GRID_RES', + 'METPLUS_FCST_FILTER_ATTR_NAME', + 'METPLUS_FCST_FILTER_ATTR_THRESH', + 'METPLUS_FCST_CENSOR_THRESH', + 'METPLUS_FCST_CENSOR_VAL', + 'METPLUS_FCST_VLD_THRESH', + 'METPLUS_OBS_FILTER_ATTR_NAME', + 'METPLUS_OBS_FILTER_ATTR_THRESH', + 'METPLUS_OBS_CENSOR_THRESH', + 'METPLUS_OBS_CENSOR_VAL', + 'METPLUS_OBS_VLD_THRESH', + 'METPLUS_MATCH_FLAG', ] def __init__(self, config, instance=None, config_overrides={}): @@ -153,8 +165,11 @@ def create_c_dict(self): 'MODE_MERGE_CONFIG_FILE', '') ) - mask_poly = self.read_mask_poly() - c_dict['MASK_POLY_TEMPLATE'] = mask_poly + self.handle_mask(single_value=True, + get_flags=True) + + # handle setting VERIFICATION_MASK for old wrapped MET config files + c_dict['MASK_POLY_TEMPLATE'] = self.read_mask_poly() c_dict['MASK_POLY_IS_LIST'] = False return c_dict diff --git a/parm/met_config/MODEConfig_wrapped b/parm/met_config/MODEConfig_wrapped index f4c77f0485..cf87777688 100644 --- a/parm/met_config/MODEConfig_wrapped +++ b/parm/met_config/MODEConfig_wrapped @@ -95,12 +95,8 @@ max_centroid_dist = 800.0/grid_res; // // Verification masking regions // -mask = { - grid = ""; - grid_flag = NONE; // Apply to NONE, FCST, OBS, or BOTH - ${METPLUS_MASK_POLY} - poly_flag = NONE; // Apply to NONE, FCST, OBS, or BOTH -} +//mask = { +${METPLUS_MASK_DICT} //////////////////////////////////////////////////////////////////////////////// From 292d97432740c59bd45268b19d7901927790a978 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 13 Apr 2021 11:11:35 -0600 Subject: [PATCH 04/42] change use case conf back because the fix did not work --- .../climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf index b73052d5fd..02eaa48a38 100644 --- a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf +++ b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf @@ -43,20 +43,18 @@ MODE_MERGE_THRESH = ge10.0, ge20.0 MODE_MERGE_FLAG = THRESH -#MODE_MET_CONFIG_OVERRIDES = fcst = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } obs = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } match_flag = NO_MERGE; mask = { poly_flag = BOTH; } weight = {aspect_diff = 1.0;} fcst_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } obs_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } nc_pairs_flag = {polylines = FALSE; } -MODE_MET_CONFIG_OVERRIDES = match_flag = NO_MERGE; mask = { poly_flag = BOTH; } weight = {aspect_diff = 1.0;} fcst_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } obs_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } nc_pairs_flag = {polylines = FALSE; } +MODE_MET_CONFIG_OVERRIDES = fcst = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } obs = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } match_flag = NO_MERGE; mask = { poly_flag = BOTH; } weight = {aspect_diff = 1.0;} fcst_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } obs_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } nc_pairs_flag = {polylines = FALSE; } # Forecast Reflectivity Variable Information MODEL = CESM FCST_VAR1_NAME = PRECT FCST_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" -FCST_VAR1_OPTIONS = convert(x) = 86400000*x; filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; +FCST_VAR1_OPTIONS = convert(x) = 86400000*x; # MRMS Reflecivitiy Variable Information OBTYPE = GPCP OBS_VAR1_NAME = precip OBS_VAR1_LEVELS = "(0,*,*)" -OBS_VAR1_OPTIONS = filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped MODE_REGRID_TO_GRID = FCST From 3f027819a8e914314b190653e56bb6728f6db305 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 15 Apr 2021 11:15:36 -0600 Subject: [PATCH 05/42] added support for many MODEConfig variables --- docs/Users_Guide/glossary.rst | 175 ++++++++++++ docs/Users_Guide/wrappers.rst | 261 ++++++++++++++++++ metplus/wrappers/mode_wrapper.py | 112 ++++++++ parm/met_config/MODEConfig_wrapped | 62 ++--- .../use_cases/met_tool_wrapper/MODE/MODE.conf | 61 +++- 5 files changed, 624 insertions(+), 47 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 30ee719644..fa5e5acdbc 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -6354,3 +6354,178 @@ METplus Configuration Glossary Specify the value for 'ops_hit_window.end' in the MET configuration file for TCGen. | *Used by:* TCGen + + MODE_FCST_FILTER_ATTR_NAME + Specify the value for 'fcst.filter_attr_name' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_FCST_FILTER_ATTR_THRESH + Specify the value for 'fcst.filter_attr_thresh' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_FCST_CENSOR_THRESH + Specify the value for 'fcst.censor_thresh' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_FCST_CENSOR_VAL + Specify the value for 'fcst.censor_val' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_FCST_VLD_THRESH + Specify the value for 'fcst.vld_thresh' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_OBS_FILTER_ATTR_NAME + Specify the value for 'obs.filter_attr_name' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_OBS_FILTER_ATTR_THRESH + Specify the value for 'obs.filter_attr_thresh' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_OBS_CENSOR_THRESH + Specify the value for 'obs.censor_thresh' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_OBS_CENSOR_VAL + Specify the value for 'obs.censor_val' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_OBS_VLD_THRESH + Specify the value for 'obs.vld_thresh' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_WEIGHT_CENTROID_DIST + Specify the value for 'weight.centroid_dist' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_WEIGHT_BOUNDARY_DIST + Specify the value for 'weight.boundary_dist' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_WEIGHT_CONVEX_HULL_DIST + Specify the value for 'weight.convex_hull_dist' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_WEIGHT_ANGLE_DIFF + Specify the value for 'weight.angle_diff' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_WEIGHT_ASPECT_DIFF + Specify the value for 'weight.aspect_diff' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_WEIGHT_AREA_RATIO + Specify the value for 'weight.area_ratio' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_WEIGHT_INT_AREA_RATIO + Specify the value for 'weight.int_area_ratio' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_WEIGHT_CURVATURE_RATIO + Specify the value for 'weight.curvature_ratio' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_WEIGHT_COMPLEXITY_RATIO + Specify the value for 'weight.complexity_ratio' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_WEIGHT_INTEN_PERC_RATIO + Specify the value for 'weight.inten_perc_ratio' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_WEIGHT_INTEN_PERC_VALUE + Specify the value for 'weight.inten_perc_value' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_NC_PAIRS_FLAG_LATLON + Specify the value for 'nc_pairs_flag.latlon' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_NC_PAIRS_FLAG_RAW + Specify the value for 'nc_pairs_flag.raw' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_NC_PAIRS_FLAG_OBJECT_RAW + Specify the value for 'nc_pairs_flag.object_raw' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_NC_PAIRS_FLAG_OBJECT_ID + Specify the value for 'nc_pairs_flag.object_id' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_NC_PAIRS_FLAG_CLUSTER_ID + Specify the value for 'nc_pairs_flag.cluster_id' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_NC_PAIRS_FLAG_POLYLINES + Specify the value for 'nc_pairs_flag.polylines' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_MASK_GRID + Specify the value for 'mask.grid' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_MASK_GRID_FLAG + Specify the value for 'mask.grid_flag' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_MASK_POLY + Specify the value for 'mask.poly' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_MASK_POLY_FLAG + Specify the value for 'mask.poly_flag' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_MATCH_FLAG + Specify the value for 'match_flag' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_MAX_CENTROID_DIST + Specify the value for 'max_centroid_dist' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_TOTAL_INTEREST_THRESH + Specify the value for 'total_interest_thresh' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_INTEREST_FUNCTION_CENTROID_DIST + Specify the value for 'interest_function.centroid_dist' in the MET configuration file for MODE. + + | *Used by:* MODE diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index d1e03eb611..7ae3e3e60b 100755 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -1814,6 +1814,41 @@ Configuration | :term:`MODE_SKIP_IF_OUTPUT_EXISTS` | :term:`MODE_DESC` | :term:`MODE_MET_CONFIG_OVERRIDES` +| :term:`MODE_WEIGHT_CENTROID_DIST` +| :term:`MODE_WEIGHT_BOUNDARY_DIST` +| :term:`MODE_WEIGHT_CONVEX_HULL_DIST` +| :term:`MODE_WEIGHT_ANGLE_DIFF` +| :term:`MODE_WEIGHT_ASPECT_DIFF` +| :term:`MODE_WEIGHT_AREA_RATIO` +| :term:`MODE_WEIGHT_INT_AREA_RATIO` +| :term:`MODE_WEIGHT_CURVATURE_RATIO` +| :term:`MODE_WEIGHT_COMPLEXITY_RATIO` +| :term:`MODE_WEIGHT_INTEN_PERC_RATIO` +| :term:`MODE_WEIGHT_INTEN_PERC_VALUE` +| :term:`MODE_MASK_GRID` +| :term:`MODE_MASK_GRID_FLAG` +| :term:`MODE_MASK_POLY` +| :term:`MODE_MASK_POLY_FLAG` +| :term:`MODE_FCST_FILTER_ATTR_NAME` +| :term:`MODE_FCST_FILTER_ATTR_THRESH` +| :term:`MODE_FCST_CENSOR_THRESH` +| :term:`MODE_FCST_CENSOR_VAL` +| :term:`MODE_FCST_VLD_THRESH` +| :term:`MODE_OBS_FILTER_ATTR_NAME` +| :term:`MODE_OBS_FILTER_ATTR_THRESH` +| :term:`MODE_OBS_CENSOR_THRESH` +| :term:`MODE_OBS_CENSOR_VAL` +| :term:`MODE_OBS_VLD_THRESH` +| :term:`MODE_NC_PAIRS_FLAG_LATLON` +| :term:`MODE_NC_PAIRS_FLAG_RAW` +| :term:`MODE_NC_PAIRS_FLAG_OBJECT_RAW` +| :term:`MODE_NC_PAIRS_FLAG_OBJECT_ID` +| :term:`MODE_NC_PAIRS_FLAG_CLUSTER_ID` +| :term:`MODE_NC_PAIRS_FLAG_POLYLINES` +| :term:`MODE_MATCH_FLAG` +| :term:`MODE_MAX_CENTROID_DIST` +| :term:`MODE_TOTAL_INTEREST_THRESH` +| :term:`MODE_INTEREST_FUNCTION_CENTROID_DIST` | :term:`FCST_MODE_VAR_NAME` (optional) | :term:`FCST_MODE_VAR_LEVELS` (optional) | :term:`FCST_MODE_VAR_THRESH` (optional) @@ -2076,6 +2111,232 @@ Below the file contents are descriptions of each environment variable referenced * - :term:`MODE_MET_CONFIG_OVERRIDES` - n/a +**${METPLUS_FCST_FILTER_ATTR_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_FCST_FILTER_ATTR_NAME` + - fcst.filter_attr_name + +**${METPLUS_FCST_FILTER_ATTR_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_FCST_FILTER_ATTR_THRESH` + - fcst.filter_attr_thresh + +**${METPLUS_FCST_CENSOR_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_FCST_CENSOR_THRESH` + - fcst.censor_thresh + +**${METPLUS_FCST_CENSOR_VAL}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_FCST_CENSOR_VAL` + - fcst.censor_val + +**${METPLUS_FCST_VLD_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_FCST_VLD_THRESH` + - fcst.vld_thresh + +**${METPLUS_OBS_FILTER_ATTR_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_OBS_FILTER_ATTR_NAME` + - obs.filter_attr_name + +**${METPLUS_OBS_FILTER_ATTR_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_OBS_FILTER_ATTR_THRESH` + - obs.filter_attr_thresh + +**${METPLUS_OBS_CENSOR_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_OBS_CENSOR_THRESH` + - obs.censor_thresh + +**${METPLUS_OBS_CENSOR_VAL}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_OBS_CENSOR_VAL` + - obs.censor_val + +**${METPLUS_OBS_VLD_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_OBS_VLD_THRESH` + - obs.vld_thresh + +**${METPLUS_MASK_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_MASK_GRID` + - mask.grid + * - :term:`MODE_MASK_GRID_FLAG` + - mask.grid_flag + * - :term:`MODE_MASK_POLY` + - mask.poly + * - :term:`MODE_MASK_POLY_FLAG` + - mask.poly_flag + +**${METPLUS_MATCH_FLAG}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_MATCH_FLAG` + - match_flag + +**${METPLUS_WEIGHT_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_WEIGHT_CENTROID_DIST` + - weight.centroid_dist + * - :term:`MODE_WEIGHT_BOUNDARY_DIST` + - weight.boundary_dist + * - :term:`MODE_WEIGHT_CONVEX_HULL_DIST` + - weight.convex_hull_dist + * - :term:`MODE_WEIGHT_ANGLE_DIFF` + - weight.angle_diff + * - :term:`MODE_WEIGHT_ASPECT_DIFF` + - weight.aspect_diff + * - :term:`MODE_WEIGHT_AREA_RATIO` + - weight.area_ratio + * - :term:`MODE_WEIGHT_INT_AREA_RATIO` + - weight.int_area_ratio + * - :term:`MODE_WEIGHT_CURVATURE_RATIO` + - weight.curvature_ratio + * - :term:`MODE_WEIGHT_COMPLEXITY_RATIO` + - weight.complexity_ratio + * - :term:`MODE_WEIGHT_INTEN_PERC_RATIO` + - weight.inten_perc_ratio + * - :term:`MODE_WEIGHT_INTEN_PERC_VALUE` + - weight.inten_perc_value + +**${METPLUS_NC_PAIRS_FLAG_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_NC_PAIRS_FLAG_LATLON` + - nc_pairs_flag.latlon + * - :term:`MODE_NC_PAIRS_FLAG_RAW` + - nc_pairs_flag.raw + * - :term:`MODE_NC_PAIRS_FLAG_OBJECT_RAW` + - nc_pairs_flag.object_raw + * - :term:`MODE_NC_PAIRS_FLAG_OBJECT_ID` + - nc_pairs_flag.object_id + * - :term:`MODE_NC_PAIRS_FLAG_CLUSTER_ID` + - nc_pairs_flag.cluster_id + * - :term:`MODE_NC_PAIRS_FLAG_POLYLINES` + - nc_pairs_flag.polylines + +**${METPLUS_MAX_CENTROID_DIST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_MAX_CENTROID_DIST` + - max_centroid_dist + +**${METPLUS_INTEREST_FUNCTION_CENTROID_DIST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_INTEREST_FUNCTION_CENTROID_DIST` + - interest_function.centroid_dist + +**${METPLUS_TOTAL_INTEREST_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_TOTAL_INTEREST_THRESH` + - total_interest_thresh + + + + .. _mtd_wrapper: MTD diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 30ebfa685a..e19ca70823 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -50,6 +50,34 @@ class MODEWrapper(CompareGriddedWrapper): 'METPLUS_OBS_CENSOR_VAL', 'METPLUS_OBS_VLD_THRESH', 'METPLUS_MATCH_FLAG', + 'METPLUS_WEIGHT_DICT', + 'METPLUS_NC_PAIRS_FLAG_DICT', + 'METPLUS_MAX_CENTROID_DIST', + 'METPLUS_TOTAL_INTEREST_THRESH', + 'METPLUS_INTEREST_FUNCTION_CENTROID_DIST', + ] + + WEIGHTS = [ + ('centroid_dist', 'float'), + ('boundary_dist', 'float'), + ('convex_hull_dist', 'float'), + ('angle_diff', 'float'), + ('aspect_diff', 'float'), + ('area_ratio', 'float'), + ('int_area_ratio', 'float'), + ('curvature_ratio', 'float'), + ('complexity_ratio', 'float'), + ('inten_perc_ratio', 'float'), + ('inten_perc_value', 'int'), + ] + + NC_PAIRS_FLAGS = [ + 'latlon', + 'raw', + 'object_raw', + 'object_id', + 'cluster_id', + 'polylines', ] def __init__(self, config, instance=None, config_overrides={}): @@ -145,6 +173,46 @@ def create_c_dict(self): f'METPLUS_{data_type}_MERGE_FLAG', remove_quotes=True) + self.set_met_config_list(self.env_var_dict, + [f'{data_type}_MODE_FILTER_ATTR_NAME', + f'MODE_{data_type}_FILTER_ATTR_NAME', + 'MODE_FILTER_ATTR_NAME'], + 'filter_attr_name', + f'METPLUS_{data_type}_FILTER_ATTR_NAME') + + self.set_met_config_list(self.env_var_dict, + [f'{data_type}_MODE_FILTER_ATTR_THRESH', + f'MODE_{data_type}_FILTER_ATTR_THRESH', + 'MODE_FILTER_ATTR_THRESH'], + 'filter_attr_thresh', + f'METPLUS_{data_type}_FILTER_ATTR_THRESH', + remove_quotes=True) + + self.set_met_config_list(self.env_var_dict, + [f'{data_type}_MODE_CENSOR_THRESH', + f'MODE_{data_type}_CENSOR_THRESH', + 'MODE_CENSOR_THRESH'], + 'censor_thresh', + f'METPLUS_{data_type}_CENSOR_THRESH', + remove_quotes=True) + + self.set_met_config_list(self.env_var_dict, + [f'{data_type}_MODE_CENSOR_VAL', + f'MODE_{data_type}_CENSOR_VAL', + f'{data_type}_MODE_CENSOR_VALUE', + f'MODE_{data_type}_CENSOR_VALUE', + 'MODE_CENSOR_VAL', + 'MODE_CENSOR_VALUE'], + 'censor_val', + f'METPLUS_{data_type}_CENSOR_VAL', + remove_quotes=True) + + self.set_met_config_float(self.env_var_dict, + [f'{data_type}_MODE_VLD_THRESH', + f'MODE_{data_type}_VLD_THRESH'], + 'vld_thresh', + f'METPLUS_{data_type}_VLD_THRESH') + # set c_dict values for old method of setting env vars for name in ['CONV_RADIUS', 'CONV_THRESH', @@ -158,6 +226,30 @@ def create_c_dict(self): 'grid_res', 'METPLUS_GRID_RES') + self.set_met_config_string(self.env_var_dict, + ['MODE_MATCH_FLAG'], + 'match_flag', + 'METPLUS_MATCH_FLAG', + remove_quotes=True) + + self.handle_weight() + self.handle_flags('nc_pairs') + + self.add_met_config(name='total_interest_thresh', + data_type='float', + metplus_configs=['MODE_TOTAL_INTEREST_THRESH']) + + self.add_met_config(name='max_centroid_dist', + data_type='string', + metplus_configs=['MODE_MAX_CENTROID_DIST'], + remove_quotes=True) + + self.set_met_config_string(self.env_var_dict, + ['MODE_INTEREST_FUNCTION_CENTROID_DIST'], + 'centroid_dist', + 'METPLUS_INTEREST_FUNCTION_CENTROID_DIST', + remove_quotes=True) + c_dict['ALLOW_MULTIPLE_FILES'] = False c_dict['MERGE_CONFIG_FILE'] = ( @@ -174,6 +266,26 @@ def create_c_dict(self): return c_dict + def handle_weight(self): + """! Read weight dictionary values and set them into env_var_list + + """ + app = self.app_name.upper() + + dict_name = 'weight' + dict_items = [] + + for name, data_type in self.WEIGHTS: + dict_items.append( + self.get_met_config( + name=name, + data_type=data_type, + metplus_configs=[f'{app}_WEIGHT_{name.upper()}'], + ) + ) + + self.handle_met_config_dict(dict_name, dict_items) + def set_environment_variables(self, time_info): """!Set environment variables that will be read set when running this tool. diff --git a/parm/met_config/MODEConfig_wrapped b/parm/met_config/MODEConfig_wrapped index cf87777688..837a8cd85f 100644 --- a/parm/met_config/MODEConfig_wrapped +++ b/parm/met_config/MODEConfig_wrapped @@ -48,13 +48,13 @@ ${METPLUS_QUILT} fcst = { ${METPLUS_FCST_FIELD} - censor_thresh = []; - censor_val = []; + ${METPLUS_FCST_CENSOR_THRESH} + ${METPLUS_FCST_CENSOR_VAL} ${METPLUS_FCST_CONV_RADIUS} ${METPLUS_FCST_CONV_THRESH} - vld_thresh = 0.5; - filter_attr_name = []; - filter_attr_thresh = []; + ${METPLUS_FCST_VLD_THRESH} + ${METPLUS_FCST_FILTER_ATTR_NAME} + ${METPLUS_FCST_FILTER_ATTR_THRESH} ${METPLUS_FCST_MERGE_THRESH} ${METPLUS_FCST_MERGE_FLAG} } @@ -62,13 +62,13 @@ fcst = { obs = { ${METPLUS_OBS_FIELD} - censor_thresh = []; - censor_val = []; + ${METPLUS_OBS_CENSOR_THRESH} + ${METPLUS_OBS_CENSOR_VAL} ${METPLUS_OBS_CONV_RADIUS} ${METPLUS_OBS_CONV_THRESH} - vld_thresh = 0.5; - filter_attr_name = []; - filter_attr_thresh = []; + ${METPLUS_OBS_VLD_THRESH} + ${METPLUS_OBS_FILTER_ATTR_NAME} + ${METPLUS_OBS_FILTER_ATTR_THRESH} ${METPLUS_OBS_MERGE_THRESH} ${METPLUS_OBS_MERGE_FLAG} } @@ -83,12 +83,14 @@ mask_missing_flag = BOTH; // // Match objects between the forecast and observation fields // -match_flag = MERGE_BOTH; +//match_flag = +${METPLUS_MATCH_FLAG} // // Maximum centroid distance for objects to be compared // -max_centroid_dist = 800.0/grid_res; +//max_centroid_dist = +${METPLUS_MAX_CENTROID_DIST} //////////////////////////////////////////////////////////////////////////////// @@ -103,19 +105,8 @@ ${METPLUS_MASK_DICT} // // Fuzzy engine weights // -weight = { - centroid_dist = 2.0; - boundary_dist = 4.0; - convex_hull_dist = 0.0; - angle_diff = 1.0; - aspect_diff = 0.0; - area_ratio = 1.0; - int_area_ratio = 2.0; - curvature_ratio = 0.0; - complexity_ratio = 0.0; - inten_perc_ratio = 0.0; - inten_perc_value = 50; -} +//weight = { +${METPLUS_WEIGHT_DICT} //////////////////////////////////////////////////////////////////////////////// @@ -124,11 +115,7 @@ weight = { // interest_function = { - centroid_dist = ( - ( 0.0, 1.0 ) - ( 60.0/grid_res, 1.0 ) - ( 600.0/grid_res, 0.0 ) - ); + ${METPLUS_INTEREST_FUNCTION_CENTROID_DIST} boundary_dist = ( ( 0.0, 1.0 ) @@ -180,7 +167,8 @@ interest_function = { // // Total interest threshold for determining matches // -total_interest_thresh = 0.7; +//total_interest_thresh = +${METPLUS_TOTAL_INTEREST_THRESH} // // Interest threshold for printing output pair information @@ -228,14 +216,10 @@ plot_gcarc_flag = FALSE; // NetCDF matched pairs, PostScript, and contingency table output files // ps_plot_flag = TRUE; -nc_pairs_flag = { - latlon = TRUE; - raw = TRUE; - object_raw = TRUE; - object_id = TRUE; - cluster_id = TRUE; - polylines = TRUE; -} + +//nc_pairs_flag = { +${METPLUS_NC_PAIRS_FLAG_DICT} + ct_stats_flag = TRUE; //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf index 955ad55d99..73b5ae634a 100644 --- a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf +++ b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf @@ -48,6 +48,14 @@ LOOP_ORDER = times # Verbosity of MET output - overrides LOG_VERBOSITY for MODE only #LOG_MODE_VERBOSITY = 2 +# Name to identify model (forecast) data in output +MODEL = WRF + +MODE_DESC = NA + +# Name to identify observation data in output +OBTYPE = WRF + # Location of MET config file to pass to the MODE # References CONFIG_DIR from the [dir] section MODE_CONFIG_FILE = {CONFIG_DIR}/MODEConfig_wrapped @@ -63,18 +71,55 @@ MODE_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_NAME}_ # Not used if unset or set to an empty string MODE_MERGE_CONFIG_FILE = -# Name to identify model (forecast) data in output -MODEL = WRF - -MODE_DESC = NA - -# Name to identify observation data in output -OBTYPE = WRF - #MODE_DESC = MODE_GRID_RES = 40 +#MODE_MASK_GRID = +#MODE_MASK_GRID_FLAG = +#MODE_MASK_POLY = +#MODE_MASK_POLY_FLAG = + +#MODE_FCST_FILTER_ATTR_NAME = +#MODE_FCST_FILTER_ATTR_THRESH = +#MODE_FCST_CENSOR_THRESH = +#MODE_FCST_CENSOR_VAL = +#MODE_FCST_VLD_THRESH = + +#MODE_OBS_FILTER_ATTR_NAME = +#MODE_OBS_FILTER_ATTR_THRESH = +#MODE_OBS_CENSOR_THRESH = +#MODE_OBS_CENSOR_VAL = +#MODE_OBS_VLD_THRESH = + +#MODE_MATCH_FLAG = + +#MODE_WEIGHT_CENTROID_DIST = +#MODE_WEIGHT_BOUNDARY_DIST = +#MODE_WEIGHT_CONVEX_HULL_DIST = +#MODE_WEIGHT_ANGLE_DIFF = +#MODE_WEIGHT_ASPECT_DIFF = +#MODE_WEIGHT_AREA_RATIO = +#MODE_WEIGHT_INT_AREA_RATIO = +#MODE_WEIGHT_CURVATURE_RATIO = +#MODE_WEIGHT_COMPLEXITY_RATIO = +#MODE_WEIGHT_INTEN_PERC_RATIO = +#MODE_WEIGHT_INTEN_PERC_VALUE = + +#MODE_NC_PAIRS_FLAG_LATLON = +#MODE_NC_PAIRS_FLAG_RAW = +#MODE_NC_PAIRS_FLAG_OBJECT_RAW = +#MODE_NC_PAIRS_FLAG_OBJECT_ID = +#MODE_NC_PAIRS_FLAG_CLUSTER_ID = +#MODE_NC_PAIRS_FLAG_POLYLINES = + +#MODE_MAX_CENTROID_DIST = + +#MODE_TOTAL_INTEREST_THRESH = + +#MODE_INTEREST_FUNCTION_CENTROID_DIST = + + # turn on quilting MODE_QUILT = True From d02a487da5f9ba2c99e9f3f77d1a3d640d6bce7c Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 15 Apr 2021 12:18:30 -0600 Subject: [PATCH 06/42] removed MODE_MET_CONFIG_OVERRIDES and replaced the values with the new config variables, ci-run-diff --- ...E_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf | 38 +++++++++++++------ .../MODE_fcstFV3_obsGOES_BrightnessTemp.conf | 20 +++++++++- ...DE_fcstFV3_obsGOES_BrightnessTempObjs.conf | 25 +++++++++++- .../MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf | 26 +++++++++++-- ...GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf | 11 +++++- 5 files changed, 103 insertions(+), 17 deletions(-) diff --git a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf index 02eaa48a38..23b1e0499f 100644 --- a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf +++ b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf @@ -31,6 +31,20 @@ LEAD_SEQ = 24, 48 # specified, then repeat for the next item in the PROCESS_LIST. LOOP_ORDER = times + +# Forecast Reflectivity Variable Information +MODEL = CESM +FCST_VAR1_NAME = PRECT +FCST_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" +FCST_VAR1_OPTIONS = convert(x) = 86400000*x; + +# MRMS Reflecivitiy Variable Information +OBTYPE = GPCP +OBS_VAR1_NAME = precip +OBS_VAR1_LEVELS = "(0,*,*)" + +MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped + MODE_GRID_RES = 1 MODE_QUILT = True @@ -43,20 +57,22 @@ MODE_MERGE_THRESH = ge10.0, ge20.0 MODE_MERGE_FLAG = THRESH -MODE_MET_CONFIG_OVERRIDES = fcst = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } obs = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } match_flag = NO_MERGE; mask = { poly_flag = BOTH; } weight = {aspect_diff = 1.0;} fcst_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } obs_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } nc_pairs_flag = {polylines = FALSE; } +#MODE_MET_CONFIG_OVERRIDES = fcst = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } obs = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } match_flag = NO_MERGE; mask = { poly_flag = BOTH; } weight = {aspect_diff = 1.0;} fcst_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } obs_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } nc_pairs_flag = {polylines = FALSE; } -# Forecast Reflectivity Variable Information -MODEL = CESM -FCST_VAR1_NAME = PRECT -FCST_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" -FCST_VAR1_OPTIONS = convert(x) = 86400000*x; +MODE_FCST_FILTER_ATTR_NAME = AREA +MODE_FCST_FILTER_ATTR_THRESH = >=7 -# MRMS Reflecivitiy Variable Information -OBTYPE = GPCP -OBS_VAR1_NAME = precip -OBS_VAR1_LEVELS = "(0,*,*)" +MODE_OBS_FILTER_ATTR_NAME = AREA +MODE_OBS_FILTER_ATTR_THRESH = >=7 + +MODE_MATCH_FLAG = NO_MERGE + +MODE_NC_PAIRS_FLAG_POLYLINES = False + +MODE_MASK_POLY_FLAG = BOTH + +MODE_WEIGHT_ASPECT_DIFF = 1.0 -MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped MODE_REGRID_TO_GRID = FCST [dir] diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf index 7fce5b90f1..1dd2e9359b 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf @@ -35,7 +35,25 @@ MODE_MERGE_FLAG = NONE MODE_GRID_RES = 3 -MODE_MET_CONFIG_OVERRIDES = fcst = { censor_thresh = [<=0]; censor_val = [9999];} obs = { censor_thresh = [<=0]; censor_val = [9999];} max_centroid_dist = 600.0/grid_res; weight = {centroid_dist = 4.0;boundary_dist = 3.0;convex_hull_dist = 1.0;area_ratio = 4.0;int_area_ratio = 3.0;} interest_function = { centroid_dist = ( (0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) );} total_interest_thresh = 0.65; +#MODE_MET_CONFIG_OVERRIDES = fcst = { censor_thresh = [<=0]; censor_val = [9999];} obs = { censor_thresh = [<=0]; censor_val = [9999];} max_centroid_dist = 600.0/grid_res; weight = {centroid_dist = 4.0;boundary_dist = 3.0;convex_hull_dist = 1.0;area_ratio = 4.0;int_area_ratio = 3.0;} interest_function = { centroid_dist = ( (0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) );} total_interest_thresh = 0.65; + +MODE_FCST_CENSOR_THRESH = <=0 +MODE_FCST_CENSOR_VAL = 9999 + +MODE_OBS_CENSOR_THRESH = <=0 +MODE_OBS_CENSOR_VAL = 9999 + +MODE_MAX_CENTROID_DIST = 600.0/grid_res + +MODE_WEIGHT_CENTROID_DIST = 4.0 +MODE_WEIGHT_BOUNDARY_DIST = 3.0 +MODE_WEIGHT_CONVEX_HULL_DIST = 1.0 +MODE_WEIGHT_AREA_RATIO = 4.0 +MODE_WEIGHT_INT_AREA_RATIO = 3.0 + +MODE_INTEREST_FUNCTION_CENTROID_DIST = ( ( 0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) ) + +MODE_TOTAL_INTEREST_THRESH = 0.65 # Forecast Brightness Temperature Variable Information diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf index 0196ca8227..7fd69e65ec 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf @@ -52,7 +52,30 @@ MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped MODE_GRID_RES = 3 -MODE_MET_CONFIG_OVERRIDES = fcst = {censor_thresh = [<=0]; censor_val = [9999];} obs = {censor_thresh = [<=0]; censor_val = [9999];} max_centroid_dist = 600.0/grid_res; mask = {poly_flag = BOTH; } weight = {centroid_dist = 4.0; boundary_dist = 3.0;convex_hull_dist = 1.0;area_ratio = 4.0; int_area_ratio = 3.0;} interest_function = { centroid_dist = ( ( 0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) );} total_interest_thresh = 0.65; nc_pairs_flag = {polylines = FALSE;} +#MODE_MET_CONFIG_OVERRIDES = fcst = {censor_thresh = [<=0]; censor_val = [9999];} obs = {censor_thresh = [<=0]; censor_val = [9999];} max_centroid_dist = 600.0/grid_res; mask = {poly_flag = BOTH; } weight = {centroid_dist = 4.0; boundary_dist = 3.0;convex_hull_dist = 1.0;area_ratio = 4.0; int_area_ratio = 3.0;} interest_function = { centroid_dist = ( ( 0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) );} total_interest_thresh = 0.65; nc_pairs_flag = {polylines = FALSE;} + +MODE_FCST_CENSOR_THRESH = <=0 +MODE_FCST_CENSOR_VAL = 9999 + +MODE_OBS_CENSOR_THRESH = <=0 +MODE_OBS_CENSOR_VAL = 9999 + +MODE_MAX_CENTROID_DIST = 600.0/grid_res + +MODE_MASK_POLY_FLAG = BOTH + +MODE_WEIGHT_CENTROID_DIST = 4.0 +MODE_WEIGHT_BOUNDARY_DIST = 3.0 +MODE_WEIGHT_CONVEX_HULL_DIST = 1.0 +MODE_WEIGHT_AREA_RATIO = 4.0 +MODE_WEIGHT_INT_AREA_RATIO = 3.0 + +MODE_INTEREST_FUNCTION_CENTROID_DIST = ( ( 0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) ) + +MODE_TOTAL_INTEREST_THRESH = 0.65 + +MODE_NC_PAIRS_FLAG_POLYLINES = False + MODE_REGRID_TO_GRID = NONE diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf index 976f162e7b..a501e3b08f 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf @@ -34,19 +34,39 @@ MODE_MERGE_FLAG = NONE MODE_GRID_RES = 4 -MODE_MET_CONFIG_OVERRIDES = fcst = {censor_thresh = [>0&&<0.75]; censor_val = [-9999.0]; filter_attr_name = ["AREA"];filter_attr_thresh = [>=4];convert(x) = x / 0.0254;} obs = {censor_thresh = [>0&&<0.75];censor_val = [-9999.0];filter_attr_name = ["AREA"];filter_attr_thresh = [>=4];convert(x) = MM_to_IN(x);} match_flag =NO_MERGE; max_centroid_dist = 400.0/grid_res; mask = {poly_flag = BOTH; } weight = {inten_perc_value = 99;} total_interest_thresh = 0.5; +#MODE_MET_CONFIG_OVERRIDES = fcst = {censor_thresh = [>0&&<0.75]; censor_val = [-9999.0]; filter_attr_name = ["AREA"];filter_attr_thresh = [>=4];convert(x) = x / 0.0254;} obs = {censor_thresh = [>0&&<0.75];censor_val = [-9999.0];filter_attr_name = ["AREA"];filter_attr_thresh = [>=4];convert(x) = MM_to_IN(x);} match_flag =NO_MERGE; max_centroid_dist = 400.0/grid_res; mask = {poly_flag = BOTH; } weight = {inten_perc_value = 99;} total_interest_thresh = 0.5; + +MODE_FCST_CENSOR_THRESH = >0&&<0.75 +MODE_FCST_CENSOR_VAL = -9999.0 +MODE_FCST_FILTER_ATTR_NAME = AREA +MODE_FCST_FILTER_ATTR_THRESH = >=4 + +MODE_OBS_CENSOR_THRESH = >0&&<0.75 +MODE_OBS_CENSOR_VAL = -9999.0 +MODE_OBS_FILTER_ATTR_NAME = AREA +MODE_OBS_FILTER_ATTR_THRESH = >=4 + +MODE_MATCH_FLAG = NO_MERGE + +MODE_MAX_CENTROID_DIST = 400.0/grid_res + +MODE_MASK_POLY_FLAG = BOTH + +MODE_WEIGHT_INTEN_PERC_VALUE = 99 + +MODE_TOTAL_INTEREST_THRESH = 0.5 # Forecast Reflectivity Variable Information MODEL = HRRRv4_HAILCAST FCST_VAR1_NAME = HAIL FCST_VAR1_LEVELS = L0 -#FCST_VAR1_OPTIONS = convert(x) = x / 0.0254; +FCST_VAR1_OPTIONS = convert(x) = x / 0.0254 # MRMS Reflecivitiy Variable Information OBTYPE = MRMS OBS_VAR1_NAME = MESHMax60min OBS_VAR1_LEVELS = Z500 -#OBS_VAR1_OPTIONS = censor_thresh = eq-3; censor_val = -9999; convert(x) = MM_to_IN(x); +OBS_VAR1_OPTIONS = convert(x) = MM_to_IN(x); #CONFIG_DIR={PARM_BASE}/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2 #MODE_CONFIG_FILE = {CONFIG_DIR}/MODEConfig_hailcast diff --git a/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf b/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf index d604f85b40..139284023d 100644 --- a/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf +++ b/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf @@ -107,7 +107,16 @@ MODE_MERGE_THRESH = >=1.25 MODE_GRID_RES = 12.7 -MODE_MET_CONFIG_OVERRIDES = obs = {censor_thresh = [ >1.00 && <=1.28, >1.28 ]; censor_val = [ 1.00 , -9999 ]; } match_flag = NO_MERGE; mask = {poly_flag = BOTH; } total_interest_thresh = 0.8; +#MODE_MET_CONFIG_OVERRIDES = obs = {censor_thresh = [ >1.00 && <=1.28, >1.28 ]; censor_val = [ 1.00 , -9999 ]; } match_flag = NO_MERGE; mask = {poly_flag = BOTH; } total_interest_thresh = 0.8; + +MODE_OBS_CENSOR_THRESH = >1.00 && <=1.28, >1.28 +MODE_OBS_CENSOR_VAL = 1.00 , -9999 + +MODE_MATCH_FLAG = NO_MERGE + +MODE_MASK_POLY_FLAG = BOTH + +MODE_TOTAL_INTEREST_THRESH = 0.8 # prefix to add to MODE output filenames MODE_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_NAME} From ea7dd43ed77af45a257de1c8c5fd37c30ef8e4b9 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 15 Apr 2021 12:49:54 -0600 Subject: [PATCH 07/42] fixed typo in functiona argument, ci-run-diff --- metplus/wrappers/mode_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index e19ca70823..33942c979e 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -242,7 +242,7 @@ def create_c_dict(self): self.add_met_config(name='max_centroid_dist', data_type='string', metplus_configs=['MODE_MAX_CENTROID_DIST'], - remove_quotes=True) + extra_args={'remove_quotes': True}) self.set_met_config_string(self.env_var_dict, ['MODE_INTEREST_FUNCTION_CENTROID_DIST'], From 3b2d919d7171f36c40f8fba961e19211d6a6d668 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 15 Apr 2021 14:32:39 -0600 Subject: [PATCH 08/42] started pytests for mode to test new configs --- .../pytests/mode/test_mode_wrapper.py | 376 ++++++++++++++++++ 1 file changed, 376 insertions(+) create mode 100644 internal_tests/pytests/mode/test_mode_wrapper.py diff --git a/internal_tests/pytests/mode/test_mode_wrapper.py b/internal_tests/pytests/mode/test_mode_wrapper.py new file mode 100644 index 0000000000..d5cee884eb --- /dev/null +++ b/internal_tests/pytests/mode/test_mode_wrapper.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 + +import os +import sys +import re +import logging +from collections import namedtuple +import pytest +import datetime + +import produtil + +from metplus.wrappers.mode_wrapper import MODEWrapper +from metplus.util import met_util as util +from metplus.util import time_util + +@pytest.mark.parametrize( + 'config_overrides, env_var_values', [ + ({'MODEL': 'my_model'}, + {'METPLUS_MODEL': 'model = "my_model";'}), + + ({'MODE_DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + + ({'DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + + ({'OBTYPE': 'my_obtype'}, + {'METPLUS_OBTYPE': 'obtype = "my_obtype";'}), + + ({'MODE_REGRID_TO_GRID': 'FCST', + }, + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + + ({'MODE_REGRID_METHOD': 'NEAREST', + }, + {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), + + ({'MODE_REGRID_WIDTH': '1', + }, + {'METPLUS_REGRID_DICT': 'regrid = {width = 1;}'}), + + ({'MODE_REGRID_VLD_THRESH': '0.5', + }, + {'METPLUS_REGRID_DICT': 'regrid = {vld_thresh = 0.5;}'}), + + ({'MODE_REGRID_SHAPE': 'SQUARE', + }, + {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + + ({'MODE_REGRID_TO_GRID': 'FCST', + 'MODE_REGRID_METHOD': 'NEAREST', + 'MODE_REGRID_WIDTH': '1', + 'MODE_REGRID_VLD_THRESH': '0.5', + 'MODE_REGRID_SHAPE': 'SQUARE', + }, + {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' + )}), + + ({'MODE_CLIMO_MEAN_INPUT_TEMPLATE': + '/some/path/climo/filename.nc', + }, + {'METPLUS_CLIMO_MEAN_FILE': + 'file_name = ["/some/path/climo/filename.nc"];', + }), + ({'MODE_CLIMO_STDEV_INPUT_TEMPLATE': + '/some/path/climo/stdfile.nc', + }, + {'METPLUS_CLIMO_STDEV_FILE': + 'climo_stdev = { file_name = ["/some/path/climo/stdfile.nc"]; }', + }), + # mask grid and poly (old config var) + ({'MODE_MASK_GRID': 'FULL', + 'MODE_VERIFICATION_MASK_TEMPLATE': 'one, two', + }, + {'METPLUS_MASK_DICT': + 'mask = {grid = ["FULL"];poly = ["one", "two"];}', + }), + # mask grid and poly (new config var) + ({'MODE_MASK_GRID': 'FULL', + 'MODE_MASK_POLY': 'one, two', + }, + {'METPLUS_MASK_DICT': + 'mask = {grid = ["FULL"];poly = ["one", "two"];}', + }), + # mask grid value + ({'MODE_MASK_GRID': 'FULL', + }, + {'METPLUS_MASK_DICT': + 'mask = {grid = ["FULL"];}', + }), + # mask grid empty string (should create empty list) + ({'MODE_MASK_GRID': '', + }, + {'METPLUS_MASK_DICT': + 'mask = {grid = [];}', + }), + # mask poly (old config var) + ({'MODE_VERIFICATION_MASK_TEMPLATE': 'one, two', + }, + {'METPLUS_MASK_DICT': + 'mask = {poly = ["one", "two"];}', + }), + # mask poly (new config var) + ({'MODE_MASK_POLY': 'one, two', + }, + {'METPLUS_MASK_DICT': + 'mask = {poly = ["one", "two"];}', + }), + + ({'MODE_NEIGHBORHOOD_COV_THRESH': '>=0.5'}, + {'METPLUS_NBRHD_COV_THRESH': 'cov_thresh = [>=0.5];'}), + + ({'MODE_NEIGHBORHOOD_WIDTH': '1,2'}, + {'METPLUS_NBRHD_WIDTH': 'width = [1, 2];'}), + + ({'MODE_NEIGHBORHOOD_SHAPE': 'CIRCLE'}, + {'METPLUS_NBRHD_SHAPE': 'shape = CIRCLE;'}), + + ({'MODE_OUTPUT_PREFIX': 'my_output_prefix'}, + {'METPLUS_OUTPUT_PREFIX': 'output_prefix = "my_output_prefix";'}), + + ({'MODE_OUTPUT_FLAG_FHO': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {fho = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_CTC': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {ctc = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_CTS': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {cts = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_MCTC': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {mctc = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_MCTS': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {mcts = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_CNT': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {cnt = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_SL1L2': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {sl1l2 = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_SAL1L2': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {sal1l2 = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_VL1L2': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {vl1l2 = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_VAL1L2': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {val1l2 = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_VCNT': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {vcnt = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_PCT': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pct = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_PSTD': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pstd = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_PJC': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pjc = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_PRC': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {prc = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_ECLV': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {eclv = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_NBRCTC': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {nbrctc = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_NBRCTS': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {nbrcts = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_NBRCNT': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {nbrcnt = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_GRAD': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {grad = STAT;}'}), + + ({'MODE_OUTPUT_FLAG_DMAP': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {dmap = STAT;}'}), + + ({ + 'MODE_OUTPUT_FLAG_FHO': 'STAT', + 'MODE_OUTPUT_FLAG_CTC': 'STAT', + 'MODE_OUTPUT_FLAG_CTS': 'STAT', + 'MODE_OUTPUT_FLAG_MCTC': 'STAT', + 'MODE_OUTPUT_FLAG_MCTS': 'STAT', + 'MODE_OUTPUT_FLAG_CNT': 'STAT', + 'MODE_OUTPUT_FLAG_SL1L2': 'STAT', + 'MODE_OUTPUT_FLAG_SAL1L2': 'STAT', + 'MODE_OUTPUT_FLAG_VL1L2': 'STAT', + 'MODE_OUTPUT_FLAG_VAL1L2': 'STAT', + 'MODE_OUTPUT_FLAG_VCNT': 'STAT', + 'MODE_OUTPUT_FLAG_PCT': 'STAT', + 'MODE_OUTPUT_FLAG_PSTD': 'STAT', + 'MODE_OUTPUT_FLAG_PJC': 'STAT', + 'MODE_OUTPUT_FLAG_PRC': 'STAT', + 'MODE_OUTPUT_FLAG_ECLV': 'STAT', + 'MODE_OUTPUT_FLAG_NBRCTC': 'STAT', + 'MODE_OUTPUT_FLAG_NBRCTS': 'STAT', + 'MODE_OUTPUT_FLAG_NBRCNT': 'STAT', + 'MODE_OUTPUT_FLAG_GRAD': 'STAT', + 'MODE_OUTPUT_FLAG_DMAP': 'STAT', + }, + { + 'METPLUS_OUTPUT_FLAG_DICT': ( + 'output_flag = {fho = STAT;ctc = STAT;cts = STAT;' + 'mctc = STAT;mcts = STAT;cnt = STAT;sl1l2 = STAT;' + 'sal1l2 = STAT;vl1l2 = STAT;val1l2 = STAT;' + 'vcnt = STAT;pct = STAT;pstd = STAT;pjc = STAT;' + 'prc = STAT;eclv = STAT;nbrctc = STAT;nbrcts = STAT;' + 'nbrcnt = STAT;grad = STAT;dmap = STAT;}')}), + + ({'MODE_NC_PAIRS_FLAG_LATLON': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {latlon = TRUE;}'}), + + ({'MODE_NC_PAIRS_FLAG_RAW': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {raw = TRUE;}'}), + + ({'MODE_NC_PAIRS_FLAG_DIFF': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {diff = TRUE;}'}), + + ({'MODE_NC_PAIRS_FLAG_CLIMO': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {climo = TRUE;}'}), + + ({'MODE_NC_PAIRS_FLAG_CLIMO_CDP': 'TRUE', }, + { + 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {climo_cdp = TRUE;}'}), + + ({'MODE_NC_PAIRS_FLAG_WEIGHT': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {weight = TRUE;}'}), + + ({'MODE_NC_PAIRS_FLAG_NBRHD': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {nbrhd = TRUE;}'}), + + ({'MODE_NC_PAIRS_FLAG_FOURIER': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {fourier = TRUE;}'}), + + ({'MODE_NC_PAIRS_FLAG_GRADIENT': 'TRUE', }, + { + 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {gradient = TRUE;}'}), + + ({'MODE_NC_PAIRS_FLAG_DISTANCE_MAP': 'TRUE', }, + { + 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {distance_map = TRUE;}'}), + + ({'MODE_NC_PAIRS_FLAG_APPLY_MASK': 'TRUE', }, + { + 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {apply_mask = TRUE;}'}), + + ({ + 'MODE_NC_PAIRS_FLAG_LATLON': 'TRUE', + 'MODE_NC_PAIRS_FLAG_RAW': 'TRUE', + 'MODE_NC_PAIRS_FLAG_DIFF': 'TRUE', + 'MODE_NC_PAIRS_FLAG_CLIMO': 'TRUE', + 'MODE_NC_PAIRS_FLAG_CLIMO_CDP': 'TRUE', + 'MODE_NC_PAIRS_FLAG_WEIGHT': 'TRUE', + 'MODE_NC_PAIRS_FLAG_NBRHD': 'TRUE', + 'MODE_NC_PAIRS_FLAG_FOURIER': 'TRUE', + 'MODE_NC_PAIRS_FLAG_GRADIENT': 'TRUE', + 'MODE_NC_PAIRS_FLAG_DISTANCE_MAP': 'TRUE', + 'MODE_NC_PAIRS_FLAG_APPLY_MASK': 'TRUE', + }, + { + 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {latlon = TRUE;raw = TRUE;diff = TRUE;climo = TRUE;climo_cdp = TRUE;weight = TRUE;nbrhd = TRUE;fourier = TRUE;gradient = TRUE;distance_map = TRUE;apply_mask = TRUE;}'}), + + ({'MODE_CLIMO_CDF_CDF_BINS': '1', }, + {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {cdf_bins = 1.0;}'}), + + ({'MODE_CLIMO_CDF_CENTER_BINS': 'True', }, + {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {center_bins = TRUE;}'}), + + ({'MODE_CLIMO_CDF_WRITE_BINS': 'False', }, + {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {write_bins = FALSE;}'}), + + ({ + 'MODE_CLIMO_CDF_CDF_BINS': '1', + 'MODE_CLIMO_CDF_CENTER_BINS': 'True', + 'MODE_CLIMO_CDF_WRITE_BINS': 'False', + }, + { + 'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {cdf_bins = 1.0;center_bins = TRUE;write_bins = FALSE;}'}), + + ] +) +def test_mode_single_field(metplus_config, config_overrides, + env_var_values): + fcst_dir = '/some/path/fcst' + obs_dir = '/some/path/obs' + fcst_name = 'APCP' + fcst_level = 'A03' + obs_name = 'APCP_03' + obs_level_no_quotes = '(*,*)' + obs_level = f'"{obs_level_no_quotes}"' + fcst_fmt = f'field = [{{ name="{fcst_name}"; level="{fcst_level}"; }}];' + obs_fmt = (f'field = [{{ name="{obs_name}"; ' + f'level="{obs_level_no_quotes}"; }}];') + config = metplus_config() + + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'MODE') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') + config.set('config', 'INIT_BEG', '2005080700') + config.set('config', 'INIT_END', '2005080712') + config.set('config', 'INIT_INCREMENT', '12H') + config.set('config', 'LEAD_SEQ', '12H') + config.set('config', 'LOOP_ORDER', 'times') + config.set('config', 'MODE_CONFIG_FILE', + '{PARM_BASE}/met_config/MODEConfig_wrapped') + config.set('config', 'FCST_MODE_INPUT_DIR', fcst_dir) + config.set('config', 'OBS_MODE_INPUT_DIR', obs_dir) + config.set('config', 'FCST_MODE_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') + config.set('config', 'OBS_MODE_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d%H}/obs_file') + config.set('config', 'MODE_OUTPUT_DIR', + '{OUTPUT_BASE}/MODE/output') + config.set('config', 'MODE_OUTPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}') + + config.set('config', 'FCST_VAR1_NAME', fcst_name) + config.set('config', 'FCST_VAR1_LEVELS', fcst_level) + config.set('config', 'OBS_VAR1_NAME', obs_name) + config.set('config', 'OBS_VAR1_LEVELS', obs_level) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = MODEWrapper(config) + assert(wrapper.isOK) + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + expected_cmds = [(f"{app_path} {verbosity} " + f"{fcst_dir}/2005080700/fcst_file_F012 " + f"{obs_dir}/2005080712/obs_file " + f"{config_file} -outdir {out_dir}/2005080712"), + (f"{app_path} {verbosity} " + f"{fcst_dir}/2005080712/fcst_file_F012 " + f"{obs_dir}/2005080800/obs_file " + f"{config_file} -outdir {out_dir}/2005080800"), + ] + + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert(cmd == expected_cmd) + + # check that environment variables were set properly + for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert(match is not None) + value = match.split('=', 1)[1] + if env_var_key == 'METPLUS_FCST_FIELD': + assert(value == fcst_fmt) + elif env_var_key == 'METPLUS_OBS_FIELD': + assert (value == obs_fmt) + else: + assert(env_var_values.get(env_var_key, '') == value) From 6d84cf0eea6fefad69cb4dcdc162821a135dcd93 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 19 Apr 2021 16:53:03 -0600 Subject: [PATCH 09/42] added support for rest of MODE config vars that reference grid_res, removed setting METPLUS_GRID_RES from the wrapped config and instead set each METplus variable that references grid_res using {MODE_GRID_RES} if the grid_res value needs to be changed from the default --- metplus/wrappers/command_builder.py | 2 +- metplus/wrappers/mode_wrapper.py | 14 +++++++++ parm/met_config/MODEConfig_wrapped | 16 +++++----- .../use_cases/met_tool_wrapper/MODE/MODE.conf | 29 ++++++++++++++----- .../MODE_fcstFV3_obsGOES_BrightnessTemp.conf | 16 ++++++++-- ...DE_fcstFV3_obsGOES_BrightnessTempObjs.conf | 16 ++++++++-- .../MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf | 6 +--- 7 files changed, 71 insertions(+), 28 deletions(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 18f1e2b077..d0ad1b364b 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -421,7 +421,7 @@ def get_env_copy(self, var_list=None): # insert escape characters to allow export command to be copyable clean_env = self.env[var].replace('"', r'\"').replace(r'\\"', r'\\\"') line = 'export ' + var + '="' + clean_env + '"' - + line = line.replace('\n', '') out += line + '; ' return out diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 33942c979e..c2ea525a32 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -55,6 +55,8 @@ class MODEWrapper(CompareGriddedWrapper): 'METPLUS_MAX_CENTROID_DIST', 'METPLUS_TOTAL_INTEREST_THRESH', 'METPLUS_INTEREST_FUNCTION_CENTROID_DIST', + 'METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST', + 'METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST', ] WEIGHTS = [ @@ -250,6 +252,18 @@ def create_c_dict(self): 'METPLUS_INTEREST_FUNCTION_CENTROID_DIST', remove_quotes=True) + self.set_met_config_string(self.env_var_dict, + ['MODE_INTEREST_FUNCTION_BOUNDARY_DIST'], + 'boundary_dist', + 'METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST', + remove_quotes=True) + + self.set_met_config_string(self.env_var_dict, + ['MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST'], + 'convex_hull_dist', + 'METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST', + remove_quotes=True) + c_dict['ALLOW_MULTIPLE_FILES'] = False c_dict['MERGE_CONFIG_FILE'] = ( diff --git a/parm/met_config/MODEConfig_wrapped b/parm/met_config/MODEConfig_wrapped index 837a8cd85f..eb3338fd35 100644 --- a/parm/met_config/MODEConfig_wrapped +++ b/parm/met_config/MODEConfig_wrapped @@ -33,7 +33,11 @@ ${METPLUS_REGRID_DICT} // // Approximate grid resolution (km) // -${METPLUS_GRID_RES} +// Note: grid_res is not used in the wrapped MODE config file +// MODE_GRID_RES can be set in the METplus config files and +// and referenced in any variable overrides that reference grid_res in the +// default file: share/met/config/MODEConfig_default +// See parm/use_cases/met_tool_wrapper/MODE/MODE.conf for an example //////////////////////////////////////////////////////////////////////////////// @@ -117,15 +121,9 @@ interest_function = { ${METPLUS_INTEREST_FUNCTION_CENTROID_DIST} - boundary_dist = ( - ( 0.0, 1.0 ) - ( 400.0/grid_res, 0.0 ) - ); + ${METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST} - convex_hull_dist = ( - ( 0.0, 1.0 ) - ( 400.0/grid_res, 0.0 ) - ); + ${METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST} angle_diff = ( ( 0.0, 1.0 ) diff --git a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf index 73b5ae634a..248c856a8e 100644 --- a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf +++ b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf @@ -73,8 +73,30 @@ MODE_MERGE_CONFIG_FILE = #MODE_DESC = +# Note: this is not actually set in the MODEConfig_wrapped file +# It can be referenced by other METplus config variables to override values MODE_GRID_RES = 40 +MODE_MAX_CENTROID_DIST = 800.0/{MODE_GRID_RES} + +MODE_INTEREST_FUNCTION_CENTROID_DIST = ( + ( 0.0, 1.0 ) + ( 60.0/{MODE_GRID_RES}, 1.0 ) + ( 600.0/{MODE_GRID_RES}, 0.0 ) + ) + +MODE_INTEREST_FUNCTION_BOUNDARY_DIST = ( + ( 0.0, 1.0 ) + ( 400.0/{MODE_GRID_RES}, 0.0 ) + ) + +MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = ( + ( 0.0, 1.0 ) + ( 400.0/{MODE_GRID_RES}, 0.0 ) + ) + +#MODE_TOTAL_INTEREST_THRESH = + #MODE_MASK_GRID = #MODE_MASK_GRID_FLAG = #MODE_MASK_POLY = @@ -113,13 +135,6 @@ MODE_GRID_RES = 40 #MODE_NC_PAIRS_FLAG_CLUSTER_ID = #MODE_NC_PAIRS_FLAG_POLYLINES = -#MODE_MAX_CENTROID_DIST = - -#MODE_TOTAL_INTEREST_THRESH = - -#MODE_INTEREST_FUNCTION_CENTROID_DIST = - - # turn on quilting MODE_QUILT = True diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf index 1dd2e9359b..e65f65eff1 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf @@ -35,7 +35,19 @@ MODE_MERGE_FLAG = NONE MODE_GRID_RES = 3 -#MODE_MET_CONFIG_OVERRIDES = fcst = { censor_thresh = [<=0]; censor_val = [9999];} obs = { censor_thresh = [<=0]; censor_val = [9999];} max_centroid_dist = 600.0/grid_res; weight = {centroid_dist = 4.0;boundary_dist = 3.0;convex_hull_dist = 1.0;area_ratio = 4.0;int_area_ratio = 3.0;} interest_function = { centroid_dist = ( (0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) );} total_interest_thresh = 0.65; +MODE_MAX_CENTROID_DIST = 600.0/{MODE_GRID_RES} + +MODE_INTEREST_FUNCTION_CENTROID_DIST = ( ( 0.0, 1.0 ) ( 60.0/{MODE_GRID_RES}, 1.0 ) ( 450.0/{MODE_GRID_RES}, 0.0 ) ) + +MODE_INTEREST_FUNCTION_BOUNDARY_DIST = ( + ( 0.0, 1.0 ) + ( 400.0/{MODE_GRID_RES}, 0.0 ) + ) + +MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = ( + ( 0.0, 1.0 ) + ( 400.0/{MODE_GRID_RES}, 0.0 ) + ) MODE_FCST_CENSOR_THRESH = <=0 MODE_FCST_CENSOR_VAL = 9999 @@ -43,7 +55,6 @@ MODE_FCST_CENSOR_VAL = 9999 MODE_OBS_CENSOR_THRESH = <=0 MODE_OBS_CENSOR_VAL = 9999 -MODE_MAX_CENTROID_DIST = 600.0/grid_res MODE_WEIGHT_CENTROID_DIST = 4.0 MODE_WEIGHT_BOUNDARY_DIST = 3.0 @@ -51,7 +62,6 @@ MODE_WEIGHT_CONVEX_HULL_DIST = 1.0 MODE_WEIGHT_AREA_RATIO = 4.0 MODE_WEIGHT_INT_AREA_RATIO = 3.0 -MODE_INTEREST_FUNCTION_CENTROID_DIST = ( ( 0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) ) MODE_TOTAL_INTEREST_THRESH = 0.65 diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf index 7fd69e65ec..617e1ae205 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf @@ -52,7 +52,19 @@ MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped MODE_GRID_RES = 3 -#MODE_MET_CONFIG_OVERRIDES = fcst = {censor_thresh = [<=0]; censor_val = [9999];} obs = {censor_thresh = [<=0]; censor_val = [9999];} max_centroid_dist = 600.0/grid_res; mask = {poly_flag = BOTH; } weight = {centroid_dist = 4.0; boundary_dist = 3.0;convex_hull_dist = 1.0;area_ratio = 4.0; int_area_ratio = 3.0;} interest_function = { centroid_dist = ( ( 0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) );} total_interest_thresh = 0.65; nc_pairs_flag = {polylines = FALSE;} +MODE_MAX_CENTROID_DIST = 600.0/{MODE_GRID_RES} + +MODE_INTEREST_FUNCTION_CENTROID_DIST = ( ( 0.0, 1.0 ) ( 60.0/{MODE_GRID_RES}, 1.0 ) ( 450.0/{MODE_GRID_RES}, 0.0 ) ) + +MODE_INTEREST_FUNCTION_BOUNDARY_DIST = ( + ( 0.0, 1.0 ) + ( 400.0/{MODE_GRID_RES}, 0.0 ) + ) + +MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = ( + ( 0.0, 1.0 ) + ( 400.0/{MODE_GRID_RES}, 0.0 ) + ) MODE_FCST_CENSOR_THRESH = <=0 MODE_FCST_CENSOR_VAL = 9999 @@ -60,7 +72,6 @@ MODE_FCST_CENSOR_VAL = 9999 MODE_OBS_CENSOR_THRESH = <=0 MODE_OBS_CENSOR_VAL = 9999 -MODE_MAX_CENTROID_DIST = 600.0/grid_res MODE_MASK_POLY_FLAG = BOTH @@ -70,7 +81,6 @@ MODE_WEIGHT_CONVEX_HULL_DIST = 1.0 MODE_WEIGHT_AREA_RATIO = 4.0 MODE_WEIGHT_INT_AREA_RATIO = 3.0 -MODE_INTEREST_FUNCTION_CENTROID_DIST = ( ( 0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) ) MODE_TOTAL_INTEREST_THRESH = 0.65 diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf index a501e3b08f..54738e4558 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf @@ -32,10 +32,6 @@ MODE_MERGE_THRESH = >=0.0 MODE_MERGE_FLAG = NONE -MODE_GRID_RES = 4 - -#MODE_MET_CONFIG_OVERRIDES = fcst = {censor_thresh = [>0&&<0.75]; censor_val = [-9999.0]; filter_attr_name = ["AREA"];filter_attr_thresh = [>=4];convert(x) = x / 0.0254;} obs = {censor_thresh = [>0&&<0.75];censor_val = [-9999.0];filter_attr_name = ["AREA"];filter_attr_thresh = [>=4];convert(x) = MM_to_IN(x);} match_flag =NO_MERGE; max_centroid_dist = 400.0/grid_res; mask = {poly_flag = BOTH; } weight = {inten_perc_value = 99;} total_interest_thresh = 0.5; - MODE_FCST_CENSOR_THRESH = >0&&<0.75 MODE_FCST_CENSOR_VAL = -9999.0 MODE_FCST_FILTER_ATTR_NAME = AREA @@ -48,7 +44,7 @@ MODE_OBS_FILTER_ATTR_THRESH = >=4 MODE_MATCH_FLAG = NO_MERGE -MODE_MAX_CENTROID_DIST = 400.0/grid_res +MODE_MAX_CENTROID_DIST = 100.0 MODE_MASK_POLY_FLAG = BOTH From b2f9265fabad5cb983b56620a80a6cbbdfcbf447 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 19 Apr 2021 17:04:57 -0600 Subject: [PATCH 10/42] added variables that reference grid_res since grid_res value is changed in use case conf --- ...E_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf | 19 +++++++++++++++++++ ...GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf index 23b1e0499f..de48212d3c 100644 --- a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf +++ b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf @@ -47,6 +47,25 @@ MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped MODE_GRID_RES = 1 +MODE_MAX_CENTROID_DIST = 800.0/{MODE_GRID_RES} + +MODE_INTEREST_FUNCTION_CENTROID_DIST = ( + ( 0.0, 1.0 ) + ( 60.0/{MODE_GRID_RES}, 1.0 ) + ( 600.0/{MODE_GRID_RES}, 0.0 ) + ) + +MODE_INTEREST_FUNCTION_BOUNDARY_DIST = ( + ( 0.0, 1.0 ) + ( 400.0/{MODE_GRID_RES}, 0.0 ) + ) + +MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = ( + ( 0.0, 1.0 ) + ( 400.0/{MODE_GRID_RES}, 0.0 ) + ) + + MODE_QUILT = True MODE_CONV_RADIUS = 2 diff --git a/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf b/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf index 139284023d..3c42444f35 100644 --- a/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf +++ b/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf @@ -107,6 +107,24 @@ MODE_MERGE_THRESH = >=1.25 MODE_GRID_RES = 12.7 +MODE_MAX_CENTROID_DIST = 800.0/{MODE_GRID_RES} + +MODE_INTEREST_FUNCTION_CENTROID_DIST = ( + ( 0.0, 1.0 ) + ( 60.0/{MODE_GRID_RES}, 1.0 ) + ( 600.0/{MODE_GRID_RES}, 0.0 ) + ) + +MODE_INTEREST_FUNCTION_BOUNDARY_DIST = ( + ( 0.0, 1.0 ) + ( 400.0/{MODE_GRID_RES}, 0.0 ) + ) + +MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = ( + ( 0.0, 1.0 ) + ( 400.0/{MODE_GRID_RES}, 0.0 ) + ) + #MODE_MET_CONFIG_OVERRIDES = obs = {censor_thresh = [ >1.00 && <=1.28, >1.28 ]; censor_val = [ 1.00 , -9999 ]; } match_flag = NO_MERGE; mask = {poly_flag = BOTH; } total_interest_thresh = 0.8; MODE_OBS_CENSOR_THRESH = >1.00 && <=1.28, >1.28 From 4d36737c889a4c1025d450b32fb41f550ed4b2bc Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 19 Apr 2021 17:05:28 -0600 Subject: [PATCH 11/42] no longer set MODE_GRID_RES in wrapped MET config since it should be referenced in other METplus config variables instead, ci-run-diff --- metplus/wrappers/mode_wrapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index c2ea525a32..95a068756b 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -38,7 +38,6 @@ class MODEWrapper(CompareGriddedWrapper): 'METPLUS_MASK_POLY', 'METPLUS_MASK_DICT', 'METPLUS_OUTPUT_PREFIX', - 'METPLUS_GRID_RES', 'METPLUS_FCST_FILTER_ATTR_NAME', 'METPLUS_FCST_FILTER_ATTR_THRESH', 'METPLUS_FCST_CENSOR_THRESH', From cb5fa6134c0e7e5e11e36925285c37a5b40840f8 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 19 Apr 2021 18:31:29 -0600 Subject: [PATCH 12/42] added support to set default value for string or list MET config value to get around issues with grid_res --- metplus/wrappers/command_builder.py | 75 +++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index d0ad1b364b..7bd2ba3850 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1345,6 +1345,28 @@ def set_time_dict_for_single_runtime(self, c_dict): self.config.logger.error("Could not get [INIT/VALID] time information from configuration file") self.isOK = False + def _get_config_or_default(self, mp_config_name, get_function, + default=None): + conf_value = '' + + # if no possible METplus config variables are not set + if mp_config_name is None: + # if no default defined, return without doing anything + if not default: + return None + + # if METplus config variable is set, read the value + else: + conf_value = get_function('config', + mp_config_name, + '') + + # if variable is not set and there is a default defined, set default + if not conf_value and default: + conf_value = default + + return conf_value + def set_met_config_list(self, c_dict, mp_config, met_config_name, c_dict_key=None, **kwargs): """! Get list from METplus configuration file and format it to be passed @@ -1364,14 +1386,20 @@ def set_met_config_list(self, c_dict, mp_config, met_config_name, value @param remove_quotes if True, output value without quotes. Default value is False + @param default (Optional) if set, use this value as default + if config is not set """ mp_config_name = self.get_mp_config_name(mp_config) - if mp_config_name is None: + conf_value = self._get_config_or_default( + mp_config_name, + get_function=self.config.getraw, + default=kwargs.get('default') + ) + if conf_value is None: return - conf_value = util.getlist(self.config.getraw('config', - mp_config_name, - '')) + # convert value from config to a list + conf_value = util.getlist(conf_value) if conf_value or kwargs.get('allow_empty', False): conf_value = str(conf_value) # if not removing quotes, escape any quotes found in list items @@ -1404,27 +1432,28 @@ def set_met_config_string(self, c_dict, mp_config, met_config_name, set to None (default) then use upper-case of met_config_name @param remove_quotes if True, output value without quotes. Default value is False + @param default (Optional) if set, use this value as default + if config is not set """ mp_config_name = self.get_mp_config_name(mp_config) - if mp_config_name is None: + conf_value = self._get_config_or_default( + mp_config_name, + get_function=self.config.getraw, + default=kwargs.get('default') + ) + if not conf_value: return - conf_value = self.config.getraw('config', mp_config_name, '') - if conf_value: - if not c_dict_key: - c_key = met_config_name.upper() - else: - c_key = c_dict_key - - conf_value = util.remove_quotes(conf_value) - # add quotes back if remote quotes is False - if not kwargs.get('remove_quotes'): - conf_value = f'"{conf_value}"' + conf_value = util.remove_quotes(conf_value) + # add quotes back if remote quotes is False + if not kwargs.get('remove_quotes'): + conf_value = f'"{conf_value}"' - if kwargs.get('uppercase', False): - conf_value = conf_value.upper() + if kwargs.get('uppercase', False): + conf_value = conf_value.upper() - c_dict[c_key] = f'{met_config_name} = {conf_value};' + c_key = c_dict_key if c_dict_key else met_config_name.upper() + c_dict[c_key] = f'{met_config_name} = {conf_value};' def set_met_config_number(self, c_dict, num_type, mp_config, met_config_name, c_dict_key=None, **kwargs): @@ -1440,6 +1469,8 @@ def set_met_config_number(self, c_dict, num_type, mp_config, to determine the key in c_dict to set (upper-case) if c_dict_key is None @param c_dict_key optional argument to specify c_dict key to store result. If set to None (default) then use upper-case of met_config_name + @param default (Optional) if set, use this value as default + if config is not set """ mp_config_name = self.get_mp_config_name(mp_config) if mp_config_name is None: @@ -1465,14 +1496,16 @@ def set_met_config_int(self, c_dict, mp_config_name, met_config_name, self.set_met_config_number(c_dict, 'int', mp_config_name, met_config_name, - c_dict_key=c_dict_key) + c_dict_key=c_dict_key, + **kwargs) def set_met_config_float(self, c_dict, mp_config_name, met_config_name, c_dict_key=None, **kwargs): self.set_met_config_number(c_dict, 'float', mp_config_name, met_config_name, - c_dict_key=c_dict_key) + c_dict_key=c_dict_key, + **kwargs) def set_met_config_thresh(self, c_dict, mp_config, met_config_name, c_dict_key=None, **kwargs): From 49ae35a62c488d1c815ba57e2818c7cb3a81946d Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 19 Apr 2021 18:32:10 -0600 Subject: [PATCH 13/42] set METPLUS_GRID_RES again and set default value for variables that reference grid_res so they are set correctly --- metplus/wrappers/mode_wrapper.py | 20 +++++++++++++++----- parm/met_config/MODEConfig_wrapped | 7 ++----- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 95a068756b..02d4311d1d 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -38,6 +38,7 @@ class MODEWrapper(CompareGriddedWrapper): 'METPLUS_MASK_POLY', 'METPLUS_MASK_DICT', 'METPLUS_OUTPUT_PREFIX', + 'METPLUS_GRID_RES', 'METPLUS_FCST_FILTER_ATTR_NAME', 'METPLUS_FCST_FILTER_ATTR_THRESH', 'METPLUS_FCST_CENSOR_THRESH', @@ -151,7 +152,8 @@ def create_c_dict(self): 'MODE_CONV_RADIUS'], 'conv_radius', f'METPLUS_{data_type}_CONV_RADIUS', - remove_quotes=True) + remove_quotes=True, + default='60.0/grid_res') self.set_met_config_list(self.env_var_dict, [f'{data_type}_MODE_CONV_THRESH', @@ -243,25 +245,33 @@ def create_c_dict(self): self.add_met_config(name='max_centroid_dist', data_type='string', metplus_configs=['MODE_MAX_CENTROID_DIST'], - extra_args={'remove_quotes': True}) + extra_args={'remove_quotes': True, + 'default': '800.0/grid_res'}) self.set_met_config_string(self.env_var_dict, ['MODE_INTEREST_FUNCTION_CENTROID_DIST'], 'centroid_dist', 'METPLUS_INTEREST_FUNCTION_CENTROID_DIST', - remove_quotes=True) + remove_quotes=True, + default=('((0.0,1.0)' + '(60.0/grid_res,1.0)' + '(600.0/grid_res,0.0))')) self.set_met_config_string(self.env_var_dict, ['MODE_INTEREST_FUNCTION_BOUNDARY_DIST'], 'boundary_dist', 'METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST', - remove_quotes=True) + remove_quotes=True, + default=('((0.0,1.0)' + '(400.0/grid_res,0.0))')) self.set_met_config_string(self.env_var_dict, ['MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST'], 'convex_hull_dist', 'METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST', - remove_quotes=True) + remove_quotes=True, + default=('((0.0,1.0)' + '(400.0/grid_res,0.0))')) c_dict['ALLOW_MULTIPLE_FILES'] = False diff --git a/parm/met_config/MODEConfig_wrapped b/parm/met_config/MODEConfig_wrapped index eb3338fd35..a08a76164c 100644 --- a/parm/met_config/MODEConfig_wrapped +++ b/parm/met_config/MODEConfig_wrapped @@ -33,11 +33,8 @@ ${METPLUS_REGRID_DICT} // // Approximate grid resolution (km) // -// Note: grid_res is not used in the wrapped MODE config file -// MODE_GRID_RES can be set in the METplus config files and -// and referenced in any variable overrides that reference grid_res in the -// default file: share/met/config/MODEConfig_default -// See parm/use_cases/met_tool_wrapper/MODE/MODE.conf for an example +// grid_res = +${METPLUS_GRID_RES} //////////////////////////////////////////////////////////////////////////////// From b89889990033a013017f9eceadf52bb39af7d5d5 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 19 Apr 2021 18:32:23 -0600 Subject: [PATCH 14/42] removed unnecessary default values, ci-run-diff --- .../use_cases/met_tool_wrapper/MODE/MODE.conf | 20 ------------------- ...E_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf | 19 ------------------ .../MODE_fcstFV3_obsGOES_BrightnessTemp.conf | 14 ++----------- ...DE_fcstFV3_obsGOES_BrightnessTempObjs.conf | 14 ++----------- .../MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf | 2 +- ...GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf | 20 ------------------- 6 files changed, 5 insertions(+), 84 deletions(-) diff --git a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf index 248c856a8e..dde404ff19 100644 --- a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf +++ b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf @@ -73,28 +73,8 @@ MODE_MERGE_CONFIG_FILE = #MODE_DESC = -# Note: this is not actually set in the MODEConfig_wrapped file -# It can be referenced by other METplus config variables to override values MODE_GRID_RES = 40 -MODE_MAX_CENTROID_DIST = 800.0/{MODE_GRID_RES} - -MODE_INTEREST_FUNCTION_CENTROID_DIST = ( - ( 0.0, 1.0 ) - ( 60.0/{MODE_GRID_RES}, 1.0 ) - ( 600.0/{MODE_GRID_RES}, 0.0 ) - ) - -MODE_INTEREST_FUNCTION_BOUNDARY_DIST = ( - ( 0.0, 1.0 ) - ( 400.0/{MODE_GRID_RES}, 0.0 ) - ) - -MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = ( - ( 0.0, 1.0 ) - ( 400.0/{MODE_GRID_RES}, 0.0 ) - ) - #MODE_TOTAL_INTEREST_THRESH = #MODE_MASK_GRID = diff --git a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf index de48212d3c..23b1e0499f 100644 --- a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf +++ b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf @@ -47,25 +47,6 @@ MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped MODE_GRID_RES = 1 -MODE_MAX_CENTROID_DIST = 800.0/{MODE_GRID_RES} - -MODE_INTEREST_FUNCTION_CENTROID_DIST = ( - ( 0.0, 1.0 ) - ( 60.0/{MODE_GRID_RES}, 1.0 ) - ( 600.0/{MODE_GRID_RES}, 0.0 ) - ) - -MODE_INTEREST_FUNCTION_BOUNDARY_DIST = ( - ( 0.0, 1.0 ) - ( 400.0/{MODE_GRID_RES}, 0.0 ) - ) - -MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = ( - ( 0.0, 1.0 ) - ( 400.0/{MODE_GRID_RES}, 0.0 ) - ) - - MODE_QUILT = True MODE_CONV_RADIUS = 2 diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf index e65f65eff1..f8381030da 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf @@ -35,19 +35,9 @@ MODE_MERGE_FLAG = NONE MODE_GRID_RES = 3 -MODE_MAX_CENTROID_DIST = 600.0/{MODE_GRID_RES} +MODE_MAX_CENTROID_DIST = 600.0/grid_res -MODE_INTEREST_FUNCTION_CENTROID_DIST = ( ( 0.0, 1.0 ) ( 60.0/{MODE_GRID_RES}, 1.0 ) ( 450.0/{MODE_GRID_RES}, 0.0 ) ) - -MODE_INTEREST_FUNCTION_BOUNDARY_DIST = ( - ( 0.0, 1.0 ) - ( 400.0/{MODE_GRID_RES}, 0.0 ) - ) - -MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = ( - ( 0.0, 1.0 ) - ( 400.0/{MODE_GRID_RES}, 0.0 ) - ) +MODE_INTEREST_FUNCTION_CENTROID_DIST = ( ( 0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) ) MODE_FCST_CENSOR_THRESH = <=0 MODE_FCST_CENSOR_VAL = 9999 diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf index 617e1ae205..aea3f16adc 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf @@ -52,19 +52,9 @@ MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped MODE_GRID_RES = 3 -MODE_MAX_CENTROID_DIST = 600.0/{MODE_GRID_RES} +MODE_MAX_CENTROID_DIST = 600.0/grid_res -MODE_INTEREST_FUNCTION_CENTROID_DIST = ( ( 0.0, 1.0 ) ( 60.0/{MODE_GRID_RES}, 1.0 ) ( 450.0/{MODE_GRID_RES}, 0.0 ) ) - -MODE_INTEREST_FUNCTION_BOUNDARY_DIST = ( - ( 0.0, 1.0 ) - ( 400.0/{MODE_GRID_RES}, 0.0 ) - ) - -MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = ( - ( 0.0, 1.0 ) - ( 400.0/{MODE_GRID_RES}, 0.0 ) - ) +MODE_INTEREST_FUNCTION_CENTROID_DIST = ( ( 0.0, 1.0 ) ( 60.0/grid_res, 1.0 ) ( 450.0/grid_res, 0.0 ) ) MODE_FCST_CENSOR_THRESH = <=0 MODE_FCST_CENSOR_VAL = 9999 diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf index 54738e4558..d41834257c 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf @@ -44,7 +44,7 @@ MODE_OBS_FILTER_ATTR_THRESH = >=4 MODE_MATCH_FLAG = NO_MERGE -MODE_MAX_CENTROID_DIST = 100.0 +MODE_MAX_CENTROID_DIST = 400.0/grid_res MODE_MASK_POLY_FLAG = BOTH diff --git a/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf b/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf index 3c42444f35..418f42d396 100644 --- a/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf +++ b/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf @@ -107,26 +107,6 @@ MODE_MERGE_THRESH = >=1.25 MODE_GRID_RES = 12.7 -MODE_MAX_CENTROID_DIST = 800.0/{MODE_GRID_RES} - -MODE_INTEREST_FUNCTION_CENTROID_DIST = ( - ( 0.0, 1.0 ) - ( 60.0/{MODE_GRID_RES}, 1.0 ) - ( 600.0/{MODE_GRID_RES}, 0.0 ) - ) - -MODE_INTEREST_FUNCTION_BOUNDARY_DIST = ( - ( 0.0, 1.0 ) - ( 400.0/{MODE_GRID_RES}, 0.0 ) - ) - -MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = ( - ( 0.0, 1.0 ) - ( 400.0/{MODE_GRID_RES}, 0.0 ) - ) - -#MODE_MET_CONFIG_OVERRIDES = obs = {censor_thresh = [ >1.00 && <=1.28, >1.28 ]; censor_val = [ 1.00 , -9999 ]; } match_flag = NO_MERGE; mask = {poly_flag = BOTH; } total_interest_thresh = 0.8; - MODE_OBS_CENSOR_THRESH = >1.00 && <=1.28, >1.28 MODE_OBS_CENSOR_VAL = 1.00 , -9999 From 898fd0a13f4528703f0ea14d4029b1fb6ceff598 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 12:00:18 -0600 Subject: [PATCH 15/42] added optional argument to set env var name if it differs from the uppercase of MET config name --- metplus/util/met_dictionary_info.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/metplus/util/met_dictionary_info.py b/metplus/util/met_dictionary_info.py index e35f49d415..da65dac4d1 100644 --- a/metplus/util/met_dictionary_info.py +++ b/metplus/util/met_dictionary_info.py @@ -10,22 +10,28 @@ """ class METConfigInfo: - """! Stores information for a member of a MET dictionary, including - name of variable, list of METplus config variables that can be used - to set the value, the data type of the item, and any additional - requirements such as remove quotes or make uppercase. + """! Stores information for a member of a MET config variables that + can be used to set the value, the data type of the item, + optional name of environment variable to set (without METPLUS_ prefix) + if it differs from the name, + and any additional requirements such as remove quotes or make uppercase. """ def __init__(self, name, data_type, + env_var_name=None, metplus_configs=None, extra_args=None): self.name = name self.data_type = data_type self.metplus_configs = metplus_configs self.extra_args = extra_args + self.env_var_name = env_var_name if env_var_name else name def __repr__(self): - return (f'{self.__class__.__name__}({self.name}, {self.data_type}' - f', {self.metplus_configs}, {self.extra_args})') + return (f'{self.__class__.__name__}({self.name}, {self.data_type}, ' + f'{self.env_var_name}, ' + f'{self.metplus_configs}, ' + f'{self.extra_args}' + ')') @property def name(self): @@ -45,6 +51,16 @@ def data_type(self): def data_type(self, data_type): self._data_type = data_type + @property + def env_var_name(self): + return self._env_var_name + + @env_var_name.setter + def env_var_name(self, env_var_name): + if not isinstance(env_var_name, str): + raise TypeError("Name must be a string") + self._env_var_name = env_var_name + @property def metplus_configs(self): return self._metplus_configs From 40a0eb0768189c4a9d8b9117bce64834e7ac60f5 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 12:00:56 -0600 Subject: [PATCH 16/42] read env_var_name and prepend METPLUS_ if it is not found to use as dictionary key --- metplus/wrappers/command_builder.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 7bd2ba3850..c3e1d468c3 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -2061,10 +2061,14 @@ def handle_met_config_item(self, item, output_dict=None): if output_dict is None: output_dict = self.env_var_dict + env_var_name = item.env_var_name.upper() + if not env_var_name.startswith('METPLUS_'): + env_var_name = f'METPLUS_{env_var_name}' + set_met_config(output_dict, item.metplus_configs, item.name, - c_dict_key=f'METPLUS_{item.name.upper()}', + c_dict_key=env_var_name, **item.extra_args) return True @@ -2095,6 +2099,8 @@ def add_met_config(self, **kwargs): arguments, which includes the following: @param name MET config variable name to set @param data_type type of variable to set, i.e. string, list, bool + @param env_var_name environment variable name to set (uses + name if not set) with or without METPLUS_ prefix @param metplus_configs variables from METplus config that should be read to get the value. This can be a list of variable names in order of precedence (first variable is used if it is set, From f2d22e6a2d4c3d1b9f82c7eb128d5eec250eefee Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 12:02:35 -0600 Subject: [PATCH 17/42] clean up formatting --- metplus/wrappers/mode_wrapper.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 02d4311d1d..28f710c00a 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -101,9 +101,12 @@ def add_merge_config_file(self, time_info): def create_c_dict(self): c_dict = super().create_c_dict() - c_dict['VERBOSITY'] = self.config.getstr('config', 'LOG_MODE_VERBOSITY', + c_dict['VERBOSITY'] = self.config.getstr('config', + 'LOG_MODE_VERBOSITY', c_dict['VERBOSITY']) - c_dict['CONFIG_FILE'] = self.config.getraw('config', 'MODE_CONFIG_FILE', '') + c_dict['CONFIG_FILE'] = self.config.getraw('config', + 'MODE_CONFIG_FILE', + '') if not c_dict['CONFIG_FILE']: self.log_error('MODE_CONFIG_FILE must be set') self.isOK = False From 5e06a0323bfc572723a897c569edd544103ee503 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 12:03:04 -0600 Subject: [PATCH 18/42] added default values for variables that reference grid_res so they will properly read grid_res if it is overridden from default value --- metplus/wrappers/mode_wrapper.py | 104 ++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 28f710c00a..fc5d95d9a9 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -82,6 +82,19 @@ class MODEWrapper(CompareGriddedWrapper): 'polylines', ] + DEFAULT_VALUES = { + 'FCST_CONV_RADIUS': '60.0/grid_res', + 'OBS_CONV_RADIUS': '60.0/grid_res', + 'MAX_CENTROID_DIST': '800.0/grid_res', + 'INTEREST_FUNCTION_CENTROID_DIST': ('((0.0,1.0)' + '(60.0/grid_res,1.0)' + '(600.0/grid_res,0.0))'), + 'INTEREST_FUNCTION_BOUNDARY_DIST': ('((0.0,1.0)' + '(400.0/grid_res,0.0))'), + 'INTEREST_FUNCTION_CONVEX_HULL_DIST': ('((0.0,1.0)' + '(400.0/grid_res,0.0))'), + } + def __init__(self, config, instance=None, config_overrides={}): # only set app variables if not already set by MTD (subclass) if not hasattr(self, 'app_name'): @@ -150,13 +163,15 @@ def create_c_dict(self): 'METPLUS_QUILT') for data_type in ['FCST', 'OBS']: - self.set_met_config_list(self.env_var_dict, - [f'{data_type}_MODE_CONV_RADIUS', - 'MODE_CONV_RADIUS'], - 'conv_radius', - f'METPLUS_{data_type}_CONV_RADIUS', - remove_quotes=True, - default='60.0/grid_res') + self.set_met_config_list( + self.env_var_dict, + [f'{data_type}_MODE_CONV_RADIUS', + 'MODE_CONV_RADIUS'], + 'conv_radius', + f'METPLUS_{data_type}_CONV_RADIUS', + remove_quotes=True, + default=self.DEFAULT_VALUES.get(f'{data_type}_CONV_RADIUS'), + ) self.set_met_config_list(self.env_var_dict, [f'{data_type}_MODE_CONV_THRESH', @@ -245,36 +260,51 @@ def create_c_dict(self): data_type='float', metplus_configs=['MODE_TOTAL_INTEREST_THRESH']) - self.add_met_config(name='max_centroid_dist', - data_type='string', - metplus_configs=['MODE_MAX_CENTROID_DIST'], - extra_args={'remove_quotes': True, - 'default': '800.0/grid_res'}) - - self.set_met_config_string(self.env_var_dict, - ['MODE_INTEREST_FUNCTION_CENTROID_DIST'], - 'centroid_dist', - 'METPLUS_INTEREST_FUNCTION_CENTROID_DIST', - remove_quotes=True, - default=('((0.0,1.0)' - '(60.0/grid_res,1.0)' - '(600.0/grid_res,0.0))')) - - self.set_met_config_string(self.env_var_dict, - ['MODE_INTEREST_FUNCTION_BOUNDARY_DIST'], - 'boundary_dist', - 'METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST', - remove_quotes=True, - default=('((0.0,1.0)' - '(400.0/grid_res,0.0))')) - - self.set_met_config_string(self.env_var_dict, - ['MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST'], - 'convex_hull_dist', - 'METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST', - remove_quotes=True, - default=('((0.0,1.0)' - '(400.0/grid_res,0.0))')) + self.add_met_config( + name='max_centroid_dist', + data_type='string', + metplus_configs=['MODE_MAX_CENTROID_DIST'], + extra_args={ + 'remove_quotes': True, + 'default': self.DEFAULT_VALUES.get('MAX_CENTROID_DIST') + } + ) + self.add_met_config( + name='centroid_dist', + data_type='string', + env_var_name='INTEREST_FUNCTION_CENTROID_DIST', + metplus_configs=['MODE_INTEREST_FUNCTION_CENTROID_DIST'], + extra_args={ + 'remove_quotes': True, + 'default': self.DEFAULT_VALUES.get( + 'INTEREST_FUNCTION_CENTROID_DIST' + ) + } + ) + self.add_met_config( + name='boundary_dist', + data_type='string', + env_var_name='INTEREST_FUNCTION_BOUNDARY_DIST', + metplus_configs=['MODE_INTEREST_FUNCTION_BOUNDARY_DIST'], + extra_args={ + 'remove_quotes': True, + 'default': self.DEFAULT_VALUES.get( + 'INTEREST_FUNCTION_BOUNDARY_DIST' + ) + } + ) + self.add_met_config( + name='convex_hull_dist', + data_type='string', + env_var_name='INTEREST_FUNCTION_CONVEX_HULL_DIST', + metplus_configs=['MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST'], + extra_args={ + 'remove_quotes': True, + 'default': self.DEFAULT_VALUES.get( + 'INTEREST_FUNCTION_CONVEX_HULL_DIST' + ) + } + ) c_dict['ALLOW_MULTIPLE_FILES'] = False From 6e04137633ed5492259c8040a341bbeab1ddf9ff Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 16:29:32 -0600 Subject: [PATCH 19/42] make mask grid_flag and poly_flag uppercase --- metplus/wrappers/command_builder.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index c3e1d468c3..35648d7539 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -2018,13 +2018,15 @@ def handle_mask(self, single_value=False, get_flags=False): self.get_met_config(name='grid_flag', data_type='string', metplus_configs=[f'{app}_MASK_GRID_FLAG'], - extra_args={'remove_quotes': True}) + extra_args={'remove_quotes': True, + 'uppercase': True}) ) dict_items.append( self.get_met_config(name='poly_flag', data_type='string', metplus_configs=[f'{app}_MASK_POLY_FLAG'], - extra_args={'remove_quotes': True}) + extra_args={'remove_quotes': True, + 'uppercase': True}) ) self.handle_met_config_dict(dict_name, dict_items) From f9d2771a7970cee8d7e0f10668d4963f22ee3e6b Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 16:30:44 -0600 Subject: [PATCH 20/42] no need to set isOK to false because log_error does that automatically --- metplus/wrappers/mode_wrapper.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index fc5d95d9a9..d4dcf1a7b4 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -122,7 +122,6 @@ def create_c_dict(self): '') if not c_dict['CONFIG_FILE']: self.log_error('MODE_CONFIG_FILE must be set') - self.isOK = False c_dict['OBS_INPUT_DIR'] = \ self.config.getdir('OBS_MODE_INPUT_DIR', '') @@ -131,7 +130,6 @@ def create_c_dict(self): 'OBS_MODE_INPUT_TEMPLATE') if not c_dict['OBS_INPUT_TEMPLATE']: self.log_error('OBS_MODE_INPUT_TEMPLATE must be set') - self.isOK = False c_dict['OBS_INPUT_DATATYPE'] = \ self.config.getstr('config', 'OBS_MODE_INPUT_DATATYPE', '') @@ -142,14 +140,12 @@ def create_c_dict(self): 'FCST_MODE_INPUT_TEMPLATE') if not c_dict['FCST_INPUT_TEMPLATE']: self.log_error('OBS_MODE_INPUT_TEMPLATE must be set') - self.isOK = False c_dict['FCST_INPUT_DATATYPE'] = \ self.config.getstr('config', 'FCST_MODE_INPUT_DATATYPE', '') c_dict['OUTPUT_DIR'] = self.config.getdir('MODE_OUTPUT_DIR', '') if not c_dict['OUTPUT_DIR']: self.log_error('MODE_OUTPUT_DIR must be set') - self.isOK = False c_dict['OUTPUT_TEMPLATE'] = ( self.config.getraw('config', From c7ada93cc65302fb0c3a7332546caa97a41e1c63 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 16:31:12 -0600 Subject: [PATCH 21/42] rearrange list of env vars that are set by wrapper for organization sake --- metplus/wrappers/mode_wrapper.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index d4dcf1a7b4..2bca148f3c 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -30,25 +30,24 @@ class MODEWrapper(CompareGriddedWrapper): 'METPLUS_FCST_CONV_THRESH', 'METPLUS_FCST_MERGE_THRESH', 'METPLUS_FCST_MERGE_FLAG', - 'METPLUS_OBS_FIELD', - 'METPLUS_OBS_CONV_RADIUS', - 'METPLUS_OBS_CONV_THRESH', - 'METPLUS_OBS_MERGE_THRESH', - 'METPLUS_OBS_MERGE_FLAG', - 'METPLUS_MASK_POLY', - 'METPLUS_MASK_DICT', - 'METPLUS_OUTPUT_PREFIX', - 'METPLUS_GRID_RES', 'METPLUS_FCST_FILTER_ATTR_NAME', 'METPLUS_FCST_FILTER_ATTR_THRESH', 'METPLUS_FCST_CENSOR_THRESH', 'METPLUS_FCST_CENSOR_VAL', 'METPLUS_FCST_VLD_THRESH', + 'METPLUS_OBS_FIELD', + 'METPLUS_OBS_CONV_RADIUS', + 'METPLUS_OBS_CONV_THRESH', + 'METPLUS_OBS_MERGE_THRESH', + 'METPLUS_OBS_MERGE_FLAG', 'METPLUS_OBS_FILTER_ATTR_NAME', 'METPLUS_OBS_FILTER_ATTR_THRESH', 'METPLUS_OBS_CENSOR_THRESH', 'METPLUS_OBS_CENSOR_VAL', 'METPLUS_OBS_VLD_THRESH', + 'METPLUS_MASK_DICT', + 'METPLUS_OUTPUT_PREFIX', + 'METPLUS_GRID_RES', 'METPLUS_MATCH_FLAG', 'METPLUS_WEIGHT_DICT', 'METPLUS_NC_PAIRS_FLAG_DICT', From e36034fe132bb37657b99878c32baa05bda765c8 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 16:32:28 -0600 Subject: [PATCH 22/42] add synonyms for METplus config variables like FCST_MODE_X and MODE_FCST_X so both formats are supported --- metplus/wrappers/mode_wrapper.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 2bca148f3c..fd23938968 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -161,6 +161,7 @@ def create_c_dict(self): self.set_met_config_list( self.env_var_dict, [f'{data_type}_MODE_CONV_RADIUS', + f'MODE_{data_type}_CONV_RADIUS', 'MODE_CONV_RADIUS'], 'conv_radius', f'METPLUS_{data_type}_CONV_RADIUS', @@ -170,6 +171,7 @@ def create_c_dict(self): self.set_met_config_list(self.env_var_dict, [f'{data_type}_MODE_CONV_THRESH', + f'MODE_{data_type}_CONV_THRESH', 'MODE_CONV_THRESH'], 'conv_thresh', f'METPLUS_{data_type}_CONV_THRESH', @@ -177,6 +179,7 @@ def create_c_dict(self): self.set_met_config_list(self.env_var_dict, [f'{data_type}_MODE_MERGE_THRESH', + f'MODE_{data_type}_MERGE_THRESH', 'MODE_MERGE_THRESH'], 'merge_thresh', f'METPLUS_{data_type}_MERGE_THRESH', @@ -184,6 +187,7 @@ def create_c_dict(self): self.set_met_config_string(self.env_var_dict, [f'{data_type}_MODE_MERGE_FLAG', + f'MODE_{data_type}_MERGE_FLAG', 'MODE_MERGE_FLAG'], 'merge_flag', f'METPLUS_{data_type}_MERGE_FLAG', @@ -225,7 +229,9 @@ def create_c_dict(self): self.set_met_config_float(self.env_var_dict, [f'{data_type}_MODE_VLD_THRESH', - f'MODE_{data_type}_VLD_THRESH'], + f'{data_type}_MODE_VALID_THRESH', + f'MODE_{data_type}_VLD_THRESH', + f'MODE_{data_type}_VALID_THRESH'], 'vld_thresh', f'METPLUS_{data_type}_VLD_THRESH') From 5fb79c62ce563e711cdf01e0716a8ced0219d510 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 16:33:51 -0600 Subject: [PATCH 23/42] auto set uppercase for flag variables --- metplus/wrappers/mode_wrapper.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index fd23938968..2fff899e0f 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -191,7 +191,8 @@ def create_c_dict(self): 'MODE_MERGE_FLAG'], 'merge_flag', f'METPLUS_{data_type}_MERGE_FLAG', - remove_quotes=True) + remove_quotes=True, + uppercase=True) self.set_met_config_list(self.env_var_dict, [f'{data_type}_MODE_FILTER_ATTR_NAME', @@ -252,7 +253,8 @@ def create_c_dict(self): ['MODE_MATCH_FLAG'], 'match_flag', 'METPLUS_MATCH_FLAG', - remove_quotes=True) + remove_quotes=True, + uppercase=True) self.handle_weight() self.handle_flags('nc_pairs') From 880efe4d3876af2a5ee003de7a16194b132e2fdf Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 16:34:57 -0600 Subject: [PATCH 24/42] if MODE_GRID_RES is set, set default values for any MET config variable that references grid_res in the default config so they are re-read with the new grid_res value --- metplus/wrappers/mode_wrapper.py | 33 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 2fff899e0f..346434abf0 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -157,6 +157,18 @@ def create_c_dict(self): 'quilt', 'METPLUS_QUILT') + self.set_met_config_float(self.env_var_dict, + 'MODE_GRID_RES', + 'grid_res', + 'METPLUS_GRID_RES') + + # if MODE_GRID_RES is not set, then unset the default values + defaults = self.DEFAULT_VALUES.copy() + if not self.env_var_dict.get('METPLUS_GRID_RES'): + for default_key in self.DEFAULT_VALUES: + defaults[default_key] = None + + # read forecast and observation field variables for data_type in ['FCST', 'OBS']: self.set_met_config_list( self.env_var_dict, @@ -166,7 +178,7 @@ def create_c_dict(self): 'conv_radius', f'METPLUS_{data_type}_CONV_RADIUS', remove_quotes=True, - default=self.DEFAULT_VALUES.get(f'{data_type}_CONV_RADIUS'), + default=defaults.get(f'{data_type}_CONV_RADIUS'), ) self.set_met_config_list(self.env_var_dict, @@ -244,11 +256,6 @@ def create_c_dict(self): value = self.get_env_var_value(f'METPLUS_{data_type}_{name}') c_dict[f'{data_type}_{name}'] = value - self.set_met_config_float(self.env_var_dict, - 'MODE_GRID_RES', - 'grid_res', - 'METPLUS_GRID_RES') - self.set_met_config_string(self.env_var_dict, ['MODE_MATCH_FLAG'], 'match_flag', @@ -269,7 +276,7 @@ def create_c_dict(self): metplus_configs=['MODE_MAX_CENTROID_DIST'], extra_args={ 'remove_quotes': True, - 'default': self.DEFAULT_VALUES.get('MAX_CENTROID_DIST') + 'default': defaults.get('MAX_CENTROID_DIST') } ) self.add_met_config( @@ -279,9 +286,7 @@ def create_c_dict(self): metplus_configs=['MODE_INTEREST_FUNCTION_CENTROID_DIST'], extra_args={ 'remove_quotes': True, - 'default': self.DEFAULT_VALUES.get( - 'INTEREST_FUNCTION_CENTROID_DIST' - ) + 'default': defaults.get('INTEREST_FUNCTION_CENTROID_DIST') } ) self.add_met_config( @@ -291,9 +296,7 @@ def create_c_dict(self): metplus_configs=['MODE_INTEREST_FUNCTION_BOUNDARY_DIST'], extra_args={ 'remove_quotes': True, - 'default': self.DEFAULT_VALUES.get( - 'INTEREST_FUNCTION_BOUNDARY_DIST' - ) + 'default': defaults.get('INTEREST_FUNCTION_BOUNDARY_DIST') } ) self.add_met_config( @@ -303,9 +306,7 @@ def create_c_dict(self): metplus_configs=['MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST'], extra_args={ 'remove_quotes': True, - 'default': self.DEFAULT_VALUES.get( - 'INTEREST_FUNCTION_CONVEX_HULL_DIST' - ) + 'default': defaults.get('INTEREST_FUNCTION_CONVEX_HULL_DIST') } ) From b6571c8318336190449371a169e7deef6c218052 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 16:36:18 -0600 Subject: [PATCH 25/42] wrote unit tests for mode to check that commands are generated for 2 run times and that each METplus config variable that controls an environment variable used in the wrapped MET config is set properly. Also check that default values are used only if MODE_GRID_RES is set --- .../pytests/mode/test_mode_wrapper.py | 552 ++++++++++-------- 1 file changed, 308 insertions(+), 244 deletions(-) diff --git a/internal_tests/pytests/mode/test_mode_wrapper.py b/internal_tests/pytests/mode/test_mode_wrapper.py index d5cee884eb..787f427738 100644 --- a/internal_tests/pytests/mode/test_mode_wrapper.py +++ b/internal_tests/pytests/mode/test_mode_wrapper.py @@ -14,8 +14,52 @@ from metplus.util import met_util as util from metplus.util import time_util +fcst_dir = '/some/path/fcst' +obs_dir = '/some/path/obs' +fcst_name = 'APCP' +fcst_level = 'A03' +obs_name = 'APCP_03' +obs_level_no_quotes = '(*,*)' +obs_level = f'"{obs_level_no_quotes}"' +fcst_fmt = f'field = {{ name="{fcst_name}"; level="{fcst_level}"; }};' +obs_fmt = (f'field = {{ name="{obs_name}"; ' + f'level="{obs_level_no_quotes}"; }};') + + +def set_minimum_config_settings(config): + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'MODE') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') + config.set('config', 'INIT_BEG', '2005080700') + config.set('config', 'INIT_END', '2005080712') + config.set('config', 'INIT_INCREMENT', '12H') + config.set('config', 'LEAD_SEQ', '12H') + config.set('config', 'LOOP_ORDER', 'times') + config.set('config', 'MODE_CONFIG_FILE', + '{PARM_BASE}/met_config/MODEConfig_wrapped') + config.set('config', 'FCST_MODE_INPUT_DIR', fcst_dir) + config.set('config', 'OBS_MODE_INPUT_DIR', obs_dir) + config.set('config', 'FCST_MODE_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') + config.set('config', 'OBS_MODE_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d%H}/obs_file') + config.set('config', 'MODE_OUTPUT_DIR', + '{OUTPUT_BASE}/MODE/output') + config.set('config', 'MODE_OUTPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}') + + config.set('config', 'FCST_VAR1_NAME', fcst_name) + config.set('config', 'FCST_VAR1_LEVELS', fcst_level) + config.set('config', 'OBS_VAR1_NAME', obs_name) + config.set('config', 'OBS_VAR1_LEVELS', obs_level) + @pytest.mark.parametrize( - 'config_overrides, env_var_values', [ + 'config_overrides, expected_output', [ ({'MODEL': 'my_model'}, {'METPLUS_MODEL': 'model = "my_model";'}), @@ -58,280 +102,213 @@ 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' )}), - ({'MODE_CLIMO_MEAN_INPUT_TEMPLATE': - '/some/path/climo/filename.nc', - }, - {'METPLUS_CLIMO_MEAN_FILE': - 'file_name = ["/some/path/climo/filename.nc"];', - }), - ({'MODE_CLIMO_STDEV_INPUT_TEMPLATE': - '/some/path/climo/stdfile.nc', - }, - {'METPLUS_CLIMO_STDEV_FILE': - 'climo_stdev = { file_name = ["/some/path/climo/stdfile.nc"]; }', - }), - # mask grid and poly (old config var) + ({'MODE_QUILT': 'True'}, + {'METPLUS_QUILT': 'quilt = TRUE;'}), + + ({'MODE_FCST_CONV_RADIUS': '40.0/grid_res'}, + {'METPLUS_FCST_CONV_RADIUS': 'conv_radius = [40.0/grid_res];'}), + ({'MODE_OBS_CONV_RADIUS': '40.0/grid_res'}, + {'METPLUS_OBS_CONV_RADIUS': 'conv_radius = [40.0/grid_res];'}), + + ({'FCST_MODE_CONV_THRESH': '>=10.0'}, + {'METPLUS_FCST_CONV_THRESH': 'conv_thresh = [>=10.0];'}), + ({'OBS_MODE_CONV_THRESH': '>=10.0'}, + {'METPLUS_OBS_CONV_THRESH': 'conv_thresh = [>=10.0];'}), + + ({'FCST_MODE_MERGE_THRESH': '>=10.0'}, + {'METPLUS_FCST_MERGE_THRESH': 'merge_thresh = [>=10.0];'}), + ({'OBS_MODE_MERGE_THRESH': '>=10.0'}, + {'METPLUS_OBS_MERGE_THRESH': 'merge_thresh = [>=10.0];'}), + + ({'FCST_MODE_MERGE_FLAG': 'Thresh'}, + {'METPLUS_FCST_MERGE_FLAG': 'merge_flag = THRESH;'}), + ({'OBS_MODE_MERGE_FLAG': 'Thresh'}, + {'METPLUS_OBS_MERGE_FLAG': 'merge_flag = THRESH;'}), + + ({'MODE_FCST_FILTER_ATTR_NAME': 'ONE, TWO'}, + {'METPLUS_FCST_FILTER_ATTR_NAME': 'filter_attr_name = ["ONE", "TWO"];'}), + ({'MODE_OBS_FILTER_ATTR_NAME': 'ONE, TWO'}, + {'METPLUS_OBS_FILTER_ATTR_NAME': 'filter_attr_name = ["ONE", "TWO"];'}), + + ({'MODE_FCST_FILTER_ATTR_THRESH': '>=1.0, >=2.0'}, + {'METPLUS_FCST_FILTER_ATTR_THRESH': 'filter_attr_thresh = [>=1.0, >=2.0];'}), + ({'MODE_OBS_FILTER_ATTR_THRESH': '>=1.0, >=2.0'}, + {'METPLUS_OBS_FILTER_ATTR_THRESH': 'filter_attr_thresh = [>=1.0, >=2.0];'}), + + ({'MODE_FCST_CENSOR_THRESH': '>=1.0, >=2.0'}, + {'METPLUS_FCST_CENSOR_THRESH': 'censor_thresh = [>=1.0, >=2.0];'}), + ({'MODE_OBS_CENSOR_THRESH': '>=1.0, >=2.0'}, + {'METPLUS_OBS_CENSOR_THRESH': 'censor_thresh = [>=1.0, >=2.0];'}), + + ({'MODE_FCST_CENSOR_VAL': '1.0, 2.0'}, + {'METPLUS_FCST_CENSOR_VAL': 'censor_val = [1.0, 2.0];'}), + ({'MODE_OBS_CENSOR_VAL': '1.0, 2.0'}, + {'METPLUS_OBS_CENSOR_VAL': 'censor_val = [1.0, 2.0];'}), + + ({'MODE_FCST_VLD_THRESH': '0.6'}, + {'METPLUS_FCST_VLD_THRESH': 'vld_thresh = 0.6;'}), + ({'MODE_OBS_VLD_THRESH': '0.6'}, + {'METPLUS_OBS_VLD_THRESH': 'vld_thresh = 0.6;'}), + + # mask grid value ({'MODE_MASK_GRID': 'FULL', - 'MODE_VERIFICATION_MASK_TEMPLATE': 'one, two', }, - {'METPLUS_MASK_DICT': - 'mask = {grid = ["FULL"];poly = ["one", "two"];}', + {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";}', }), - # mask grid and poly (new config var) - ({'MODE_MASK_GRID': 'FULL', - 'MODE_MASK_POLY': 'one, two', + # mask poly (old config var) + ({'MODE_VERIFICATION_MASK_TEMPLATE': 'one', }, - {'METPLUS_MASK_DICT': - 'mask = {grid = ["FULL"];poly = ["one", "two"];}', + {'METPLUS_MASK_DICT': 'mask = {poly = "one";}', }), - # mask grid value - ({'MODE_MASK_GRID': 'FULL', + # mask poly (new config var) + ({'MODE_MASK_POLY': 'one', }, - {'METPLUS_MASK_DICT': - 'mask = {grid = ["FULL"];}', + {'METPLUS_MASK_DICT': 'mask = {poly = "one";}', }), - # mask grid empty string (should create empty list) - ({'MODE_MASK_GRID': '', + # mask poly_flag + ({'MODE_MASK_POLY_FLAG': 'Fcst', }, - {'METPLUS_MASK_DICT': - 'mask = {grid = [];}', + {'METPLUS_MASK_DICT': 'mask = {poly_flag = FCST;}', }), - # mask poly (old config var) - ({'MODE_VERIFICATION_MASK_TEMPLATE': 'one, two', + # mask grid_flag + ({'MODE_MASK_GRID_FLAG': 'obs', }, - {'METPLUS_MASK_DICT': - 'mask = {poly = ["one", "two"];}', + {'METPLUS_MASK_DICT': 'mask = {grid_flag = OBS;}', }), - # mask poly (new config var) - ({'MODE_MASK_POLY': 'one, two', + # mask all + ({'MODE_MASK_GRID': 'FULL', + 'MODE_MASK_POLY': 'one', + 'MODE_MASK_POLY_FLAG': 'Fcst', + 'MODE_MASK_GRID_FLAG': 'obs', }, - {'METPLUS_MASK_DICT': - 'mask = {poly = ["one", "two"];}', + {'METPLUS_MASK_DICT': ('mask = {grid = "FULL";poly = "one";' + 'grid_flag = OBS;poly_flag = FCST;}'), }), - - ({'MODE_NEIGHBORHOOD_COV_THRESH': '>=0.5'}, - {'METPLUS_NBRHD_COV_THRESH': 'cov_thresh = [>=0.5];'}), - - ({'MODE_NEIGHBORHOOD_WIDTH': '1,2'}, - {'METPLUS_NBRHD_WIDTH': 'width = [1, 2];'}), - - ({'MODE_NEIGHBORHOOD_SHAPE': 'CIRCLE'}, - {'METPLUS_NBRHD_SHAPE': 'shape = CIRCLE;'}), - ({'MODE_OUTPUT_PREFIX': 'my_output_prefix'}, {'METPLUS_OUTPUT_PREFIX': 'output_prefix = "my_output_prefix";'}), - ({'MODE_OUTPUT_FLAG_FHO': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {fho = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_CTC': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {ctc = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_CTS': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {cts = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_MCTC': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {mctc = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_MCTS': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {mcts = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_CNT': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {cnt = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_SL1L2': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {sl1l2 = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_SAL1L2': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {sal1l2 = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_VL1L2': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {vl1l2 = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_VAL1L2': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {val1l2 = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_VCNT': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {vcnt = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_PCT': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pct = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_PSTD': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pstd = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_PJC': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pjc = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_PRC': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {prc = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_ECLV': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {eclv = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_NBRCTC': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {nbrctc = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_NBRCTS': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {nbrcts = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_NBRCNT': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {nbrcnt = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_GRAD': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {grad = STAT;}'}), - - ({'MODE_OUTPUT_FLAG_DMAP': 'STAT', }, - {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {dmap = STAT;}'}), - - ({ - 'MODE_OUTPUT_FLAG_FHO': 'STAT', - 'MODE_OUTPUT_FLAG_CTC': 'STAT', - 'MODE_OUTPUT_FLAG_CTS': 'STAT', - 'MODE_OUTPUT_FLAG_MCTC': 'STAT', - 'MODE_OUTPUT_FLAG_MCTS': 'STAT', - 'MODE_OUTPUT_FLAG_CNT': 'STAT', - 'MODE_OUTPUT_FLAG_SL1L2': 'STAT', - 'MODE_OUTPUT_FLAG_SAL1L2': 'STAT', - 'MODE_OUTPUT_FLAG_VL1L2': 'STAT', - 'MODE_OUTPUT_FLAG_VAL1L2': 'STAT', - 'MODE_OUTPUT_FLAG_VCNT': 'STAT', - 'MODE_OUTPUT_FLAG_PCT': 'STAT', - 'MODE_OUTPUT_FLAG_PSTD': 'STAT', - 'MODE_OUTPUT_FLAG_PJC': 'STAT', - 'MODE_OUTPUT_FLAG_PRC': 'STAT', - 'MODE_OUTPUT_FLAG_ECLV': 'STAT', - 'MODE_OUTPUT_FLAG_NBRCTC': 'STAT', - 'MODE_OUTPUT_FLAG_NBRCTS': 'STAT', - 'MODE_OUTPUT_FLAG_NBRCNT': 'STAT', - 'MODE_OUTPUT_FLAG_GRAD': 'STAT', - 'MODE_OUTPUT_FLAG_DMAP': 'STAT', - }, - { - 'METPLUS_OUTPUT_FLAG_DICT': ( - 'output_flag = {fho = STAT;ctc = STAT;cts = STAT;' - 'mctc = STAT;mcts = STAT;cnt = STAT;sl1l2 = STAT;' - 'sal1l2 = STAT;vl1l2 = STAT;val1l2 = STAT;' - 'vcnt = STAT;pct = STAT;pstd = STAT;pjc = STAT;' - 'prc = STAT;eclv = STAT;nbrctc = STAT;nbrcts = STAT;' - 'nbrcnt = STAT;grad = STAT;dmap = STAT;}')}), - - ({'MODE_NC_PAIRS_FLAG_LATLON': 'TRUE', }, - {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {latlon = TRUE;}'}), - - ({'MODE_NC_PAIRS_FLAG_RAW': 'TRUE', }, - {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {raw = TRUE;}'}), - - ({'MODE_NC_PAIRS_FLAG_DIFF': 'TRUE', }, - {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {diff = TRUE;}'}), - - ({'MODE_NC_PAIRS_FLAG_CLIMO': 'TRUE', }, - {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {climo = TRUE;}'}), - - ({'MODE_NC_PAIRS_FLAG_CLIMO_CDP': 'TRUE', }, - { - 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {climo_cdp = TRUE;}'}), - - ({'MODE_NC_PAIRS_FLAG_WEIGHT': 'TRUE', }, - {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {weight = TRUE;}'}), - - ({'MODE_NC_PAIRS_FLAG_NBRHD': 'TRUE', }, - {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {nbrhd = TRUE;}'}), - - ({'MODE_NC_PAIRS_FLAG_FOURIER': 'TRUE', }, - {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {fourier = TRUE;}'}), - - ({'MODE_NC_PAIRS_FLAG_GRADIENT': 'TRUE', }, - { - 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {gradient = TRUE;}'}), + ({'MODE_GRID_RES': '40.0'}, + {'METPLUS_GRID_RES': 'grid_res = 40.0;'}), + + ({'MODE_MATCH_FLAG': 'merge_both'}, + {'METPLUS_MATCH_FLAG': 'match_flag = MERGE_BOTH;'}), + + # single weight values + ({'MODE_WEIGHT_CENTROID_DIST': '4.0', }, + {'METPLUS_WEIGHT_DICT': 'weight = {centroid_dist = 4.0;}'}), + ({'MODE_WEIGHT_BOUNDARY_DIST': '4.0', }, + {'METPLUS_WEIGHT_DICT': 'weight = {boundary_dist = 4.0;}'}), + ({'MODE_WEIGHT_CONVEX_HULL_DIST': '4.0', }, + {'METPLUS_WEIGHT_DICT': 'weight = {convex_hull_dist = 4.0;}'}), + ({'MODE_WEIGHT_ANGLE_DIFF': '4.0', }, + {'METPLUS_WEIGHT_DICT': 'weight = {angle_diff = 4.0;}'}), + ({'MODE_WEIGHT_ASPECT_DIFF': '4.0', }, + {'METPLUS_WEIGHT_DICT': 'weight = {aspect_diff = 4.0;}'}), + ({'MODE_WEIGHT_AREA_RATIO': '4.0', }, + {'METPLUS_WEIGHT_DICT': 'weight = {area_ratio = 4.0;}'}), + ({'MODE_WEIGHT_INT_AREA_RATIO': '4.0', }, + {'METPLUS_WEIGHT_DICT': 'weight = {int_area_ratio = 4.0;}'}), + ({'MODE_WEIGHT_CURVATURE_RATIO': '4.0', }, + {'METPLUS_WEIGHT_DICT': 'weight = {curvature_ratio = 4.0;}'}), + ({'MODE_WEIGHT_COMPLEXITY_RATIO': '4.0', }, + {'METPLUS_WEIGHT_DICT': 'weight = {complexity_ratio = 4.0;}'}), + ({'MODE_WEIGHT_INTEN_PERC_RATIO': '4.0', }, + {'METPLUS_WEIGHT_DICT': 'weight = {inten_perc_ratio = 4.0;}'}), + ({'MODE_WEIGHT_INTEN_PERC_VALUE': '40', }, + {'METPLUS_WEIGHT_DICT': 'weight = {inten_perc_value = 40;}'}), + + # all weight values + ({'MODE_WEIGHT_CENTROID_DIST': '4.0', + 'MODE_WEIGHT_BOUNDARY_DIST': '4.0', + 'MODE_WEIGHT_CONVEX_HULL_DIST': '4.0', + 'MODE_WEIGHT_ANGLE_DIFF': '4.0', + 'MODE_WEIGHT_ASPECT_DIFF': '4.0', + 'MODE_WEIGHT_AREA_RATIO': '4.0', + 'MODE_WEIGHT_INT_AREA_RATIO': '4.0', + 'MODE_WEIGHT_CURVATURE_RATIO': '4.0', + 'MODE_WEIGHT_COMPLEXITY_RATIO': '4.0', + 'MODE_WEIGHT_INTEN_PERC_RATIO': '4.0', + 'MODE_WEIGHT_INTEN_PERC_VALUE': '40', + }, + {'METPLUS_WEIGHT_DICT': ('weight = {centroid_dist = 4.0;' + 'boundary_dist = 4.0;convex_hull_dist = 4.0;' + 'angle_diff = 4.0;aspect_diff = 4.0;' + 'area_ratio = 4.0;int_area_ratio = 4.0;' + 'curvature_ratio = 4.0;' + 'complexity_ratio = 4.0;' + 'inten_perc_ratio = 4.0;' + 'inten_perc_value = 40;}')}), - ({'MODE_NC_PAIRS_FLAG_DISTANCE_MAP': 'TRUE', }, - { - 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {distance_map = TRUE;}'}), + ({'MODE_NC_PAIRS_FLAG_LATLON': 'FALSE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {latlon = FALSE;}'}), - ({'MODE_NC_PAIRS_FLAG_APPLY_MASK': 'TRUE', }, - { - 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {apply_mask = TRUE;}'}), + ({'MODE_NC_PAIRS_FLAG_RAW': 'FALSE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {raw = FALSE;}'}), - ({ - 'MODE_NC_PAIRS_FLAG_LATLON': 'TRUE', - 'MODE_NC_PAIRS_FLAG_RAW': 'TRUE', - 'MODE_NC_PAIRS_FLAG_DIFF': 'TRUE', - 'MODE_NC_PAIRS_FLAG_CLIMO': 'TRUE', - 'MODE_NC_PAIRS_FLAG_CLIMO_CDP': 'TRUE', - 'MODE_NC_PAIRS_FLAG_WEIGHT': 'TRUE', - 'MODE_NC_PAIRS_FLAG_NBRHD': 'TRUE', - 'MODE_NC_PAIRS_FLAG_FOURIER': 'TRUE', - 'MODE_NC_PAIRS_FLAG_GRADIENT': 'TRUE', - 'MODE_NC_PAIRS_FLAG_DISTANCE_MAP': 'TRUE', - 'MODE_NC_PAIRS_FLAG_APPLY_MASK': 'TRUE', - }, - { - 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {latlon = TRUE;raw = TRUE;diff = TRUE;climo = TRUE;climo_cdp = TRUE;weight = TRUE;nbrhd = TRUE;fourier = TRUE;gradient = TRUE;distance_map = TRUE;apply_mask = TRUE;}'}), + ({'MODE_NC_PAIRS_FLAG_OBJECT_RAW': 'FALSE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {object_raw = FALSE;}'}), - ({'MODE_CLIMO_CDF_CDF_BINS': '1', }, - {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {cdf_bins = 1.0;}'}), + ({'MODE_NC_PAIRS_FLAG_OBJECT_ID': 'FALSE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {object_id = FALSE;}'}), - ({'MODE_CLIMO_CDF_CENTER_BINS': 'True', }, - {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {center_bins = TRUE;}'}), + ({'MODE_NC_PAIRS_FLAG_CLUSTER_ID': 'FALSE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {cluster_id = FALSE;}'}), - ({'MODE_CLIMO_CDF_WRITE_BINS': 'False', }, - {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {write_bins = FALSE;}'}), + ({'MODE_NC_PAIRS_FLAG_POLYLINES': 'FALSE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {polylines = FALSE;}'}), ({ - 'MODE_CLIMO_CDF_CDF_BINS': '1', - 'MODE_CLIMO_CDF_CENTER_BINS': 'True', - 'MODE_CLIMO_CDF_WRITE_BINS': 'False', + 'MODE_NC_PAIRS_FLAG_LATLON': 'FALSE', + 'MODE_NC_PAIRS_FLAG_RAW': 'FALSE', + 'MODE_NC_PAIRS_FLAG_OBJECT_RAW': 'FALSE', + 'MODE_NC_PAIRS_FLAG_OBJECT_ID': 'FALSE', + 'MODE_NC_PAIRS_FLAG_CLUSTER_ID': 'FALSE', + 'MODE_NC_PAIRS_FLAG_POLYLINES': 'FALSE', }, { - 'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {cdf_bins = 1.0;center_bins = TRUE;write_bins = FALSE;}'}), + 'METPLUS_NC_PAIRS_FLAG_DICT': ('nc_pairs_flag = {latlon = FALSE;' + 'raw = FALSE;object_raw = FALSE;' + 'object_id = FALSE;' + 'cluster_id = FALSE;' + 'polylines = FALSE;}')}), + + ({'MODE_MAX_CENTROID_DIST': '400.0/grid_res'}, + {'METPLUS_MAX_CENTROID_DIST': 'max_centroid_dist = 400.0/grid_res;'}), + + ({'MODE_TOTAL_INTEREST_THRESH': '0.4'}, + {'METPLUS_TOTAL_INTEREST_THRESH': 'total_interest_thresh = 0.4;'}), + + ({'MODE_INTEREST_FUNCTION_CENTROID_DIST': ('((0.0, 2.0) ' + '(30.0/grid_res, 1.0)' + '300.0/grid_res, 1.0))')}, + {'METPLUS_INTEREST_FUNCTION_CENTROID_DIST': ('centroid_dist = (' + '(0.0, 2.0) ' + '(30.0/grid_res, 1.0)' + '300.0/grid_res, 1.0)' + ');')}), + + ({'MODE_INTEREST_FUNCTION_BOUNDARY_DIST': ('((0.0, 2.0) ' + '200.0/grid_res, 1.0))')}, + {'METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST': ('boundary_dist = (' + '(0.0, 2.0) ' + '200.0/grid_res, 1.0)' + ');')}), + + ({'MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST': ('((0.0, 2.0) ' + '200.0/grid_res, 1.0))')}, + {'METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST': ('convex_hull_dist = (' + '(0.0, 2.0) ' + '200.0/grid_res, 1.0)' + ');')}), ] ) def test_mode_single_field(metplus_config, config_overrides, - env_var_values): - fcst_dir = '/some/path/fcst' - obs_dir = '/some/path/obs' - fcst_name = 'APCP' - fcst_level = 'A03' - obs_name = 'APCP_03' - obs_level_no_quotes = '(*,*)' - obs_level = f'"{obs_level_no_quotes}"' - fcst_fmt = f'field = [{{ name="{fcst_name}"; level="{fcst_level}"; }}];' - obs_fmt = (f'field = [{{ name="{obs_name}"; ' - f'level="{obs_level_no_quotes}"; }}];') + expected_output): config = metplus_config() - # set config variables to prevent command from running and bypass check - # if input files actually exist - config.set('config', 'DO_NOT_RUN_EXE', True) - config.set('config', 'INPUT_MUST_EXIST', False) - - # set process and time config variables - config.set('config', 'PROCESS_LIST', 'MODE') - config.set('config', 'LOOP_BY', 'INIT') - config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') - config.set('config', 'INIT_BEG', '2005080700') - config.set('config', 'INIT_END', '2005080712') - config.set('config', 'INIT_INCREMENT', '12H') - config.set('config', 'LEAD_SEQ', '12H') - config.set('config', 'LOOP_ORDER', 'times') - config.set('config', 'MODE_CONFIG_FILE', - '{PARM_BASE}/met_config/MODEConfig_wrapped') - config.set('config', 'FCST_MODE_INPUT_DIR', fcst_dir) - config.set('config', 'OBS_MODE_INPUT_DIR', obs_dir) - config.set('config', 'FCST_MODE_INPUT_TEMPLATE', - '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') - config.set('config', 'OBS_MODE_INPUT_TEMPLATE', - '{valid?fmt=%Y%m%d%H}/obs_file') - config.set('config', 'MODE_OUTPUT_DIR', - '{OUTPUT_BASE}/MODE/output') - config.set('config', 'MODE_OUTPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}') - - config.set('config', 'FCST_VAR1_NAME', fcst_name) - config.set('config', 'FCST_VAR1_LEVELS', fcst_level) - config.set('config', 'OBS_VAR1_NAME', obs_name) - config.set('config', 'OBS_VAR1_LEVELS', obs_level) + # set config variables needed to run + set_minimum_config_settings(config) # set config variable overrides for key, value in config_overrides.items(): @@ -358,6 +335,28 @@ def test_mode_single_field(metplus_config, config_overrides, all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") + # set default values in expected output list + # only if they are not set and if MODE_GRID_RES is set + if 'MODE_GRID_RES' in config_overrides: + met_remove_prefixes = ['interest_function_', 'fcst_', 'obs_'] + met_lists = ['conv_radius'] + for name, default_val in wrapper.DEFAULT_VALUES.items(): + if f'MODE_{name}' not in config_overrides: + met_name = name.lower() + # remove prefix that corresponds to dictionary not variable + for met_remove_prefix in met_remove_prefixes: + if met_name.startswith(met_remove_prefix): + met_name = met_name.split(met_remove_prefix)[1] + break + + # convert value to list if variable expects a list + if met_name in met_lists: + default_val = f'[{default_val}]' + + expected_output[f'METPLUS_{name}'] = ( + f'{met_name} = {default_val};' + ) + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert(cmd == expected_cmd) @@ -373,4 +372,69 @@ def test_mode_single_field(metplus_config, config_overrides, elif env_var_key == 'METPLUS_OBS_FIELD': assert (value == obs_fmt) else: - assert(env_var_values.get(env_var_key, '') == value) + assert(expected_output.get(env_var_key, '') == value) + +@pytest.mark.parametrize( + 'config_name, env_var_name, met_name, var_type', [ + ('FCST_MODE_CONV_RADIUS', 'METPLUS_FCST_CONV_RADIUS', + 'conv_radius', 'list'), + ('MODE_FCST_CONV_RADIUS', 'METPLUS_FCST_CONV_RADIUS', + 'conv_radius', 'list'), + ('MODE_CONV_RADIUS', 'METPLUS_FCST_CONV_RADIUS', + 'conv_radius', 'list'), + + ('FCST_MODE_CONV_THRESH', 'METPLUS_FCST_CONV_THRESH', + 'conv_thresh', 'list'), + ('MODE_FCST_CONV_THRESH', 'METPLUS_FCST_CONV_THRESH', + 'conv_thresh', 'list'), + ('MODE_CONV_THRESH', 'METPLUS_FCST_CONV_THRESH', + 'conv_thresh', 'list'), + + ('FCST_MODE_MERGE_THRESH', 'METPLUS_FCST_MERGE_THRESH', + 'merge_thresh', 'list'), + ('MODE_FCST_MERGE_THRESH', 'METPLUS_FCST_MERGE_THRESH', + 'merge_thresh', 'list'), + ('MODE_MERGE_THRESH', 'METPLUS_FCST_MERGE_THRESH', + 'merge_thresh', 'list'), + + ('FCST_MODE_MERGE_FLAG', 'METPLUS_FCST_MERGE_FLAG', + 'merge_flag', 'upper'), + ('MODE_FCST_MERGE_FLAG', 'METPLUS_FCST_MERGE_FLAG', + 'merge_flag', 'upper'), + ('MODE_MERGE_FLAG', 'METPLUS_FCST_MERGE_FLAG', + 'merge_flag', 'upper'), + + ('FCST_MODE_VLD_THRESH', 'METPLUS_FCST_VLD_THRESH', + 'vld_thresh', 'float'), + ('FCST_MODE_VALID_THRESH', 'METPLUS_FCST_VLD_THRESH', + 'vld_thresh', 'float'), + ('MODE_FCST_VLD_THRESH', 'METPLUS_FCST_VLD_THRESH', + 'vld_thresh', 'float'), + ('MODE_FCST_VALID_THRESH', 'METPLUS_FCST_VLD_THRESH', + 'vld_thresh', 'float'), + + ] +) +def test_config_synonyms(metplus_config, config_name, env_var_name, + met_name, var_type): + """! Ensure that different METplus config variable names set the correct + value in the environment variables list + """ + in_value = 'out_value' + + if var_type == 'list': + out_value = f'[{in_value}]' + elif var_type == 'upper': + out_value = in_value.upper() + elif var_type == 'float': + in_value = out_value = 4.0 + + config = metplus_config() + set_minimum_config_settings(config) + config.set('config', config_name, in_value) + wrapper = MODEWrapper(config) + assert(wrapper.isOK) + + + expected_output = f'{met_name} = {out_value};' + assert(wrapper.env_var_dict[env_var_name] == expected_output) From 76b404119e35810061d05b23a1d852142c97a601 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 16:36:55 -0600 Subject: [PATCH 26/42] formatting and ci-run-diff --- parm/use_cases/met_tool_wrapper/MODE/MODE.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf index dde404ff19..9ae7eaf7d2 100644 --- a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf +++ b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf @@ -60,9 +60,11 @@ OBTYPE = WRF # References CONFIG_DIR from the [dir] section MODE_CONFIG_FILE = {CONFIG_DIR}/MODEConfig_wrapped -# grid to remap data. Value is set as the 'to_grid' variable in the 'regrid' dictionary -# See MET User's Guide for more information MODE_REGRID_TO_GRID = NONE +#MODE_REGRID_METHOD = +#MODE_REGRID_WIDTH = +#ODE_REGRID_VLD_THRESH = +#MODE_REGRID_SHAPE = MODE_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_NAME}_{CURRENT_OBS_LEVEL} From a5cf4820064369e3773f1b4a4addd90b620c918e Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 16:54:23 -0600 Subject: [PATCH 27/42] removed unused variable and reordered variables --- .../MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf index 23b1e0499f..9f61d6108c 100644 --- a/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf +++ b/parm/use_cases/model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf @@ -38,6 +38,13 @@ FCST_VAR1_NAME = PRECT FCST_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" FCST_VAR1_OPTIONS = convert(x) = 86400000*x; +MODE_FCST_FILTER_ATTR_NAME = AREA +MODE_FCST_FILTER_ATTR_THRESH = >=7 + +MODE_OBS_FILTER_ATTR_NAME = AREA +MODE_OBS_FILTER_ATTR_THRESH = >=7 + + # MRMS Reflecivitiy Variable Information OBTYPE = GPCP OBS_VAR1_NAME = precip @@ -57,14 +64,6 @@ MODE_MERGE_THRESH = ge10.0, ge20.0 MODE_MERGE_FLAG = THRESH -#MODE_MET_CONFIG_OVERRIDES = fcst = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } obs = { filter_attr_name = ["AREA"]; filter_attr_thresh = [>=7]; } match_flag = NO_MERGE; mask = { poly_flag = BOTH; } weight = {aspect_diff = 1.0;} fcst_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } obs_raw_plot = { color_table = "MET_BASE/colortables/met_default.ctable"; } nc_pairs_flag = {polylines = FALSE; } - -MODE_FCST_FILTER_ATTR_NAME = AREA -MODE_FCST_FILTER_ATTR_THRESH = >=7 - -MODE_OBS_FILTER_ATTR_NAME = AREA -MODE_OBS_FILTER_ATTR_THRESH = >=7 - MODE_MATCH_FLAG = NO_MERGE MODE_NC_PAIRS_FLAG_POLYLINES = False From 91c24b198fc4e3a9e552a60d4bf36a374343792e Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 17:16:36 -0600 Subject: [PATCH 28/42] added missing METplus config variables --- docs/Users_Guide/glossary.rst | 10 ++++++++++ docs/Users_Guide/wrappers.rst | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index fa5e5acdbc..6d52cd3d60 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -6529,3 +6529,13 @@ METplus Configuration Glossary Specify the value for 'interest_function.centroid_dist' in the MET configuration file for MODE. | *Used by:* MODE + + MODE_INTEREST_FUNCTION_BOUNDARY_DIST + Specify the value for 'interest_function.boundary_dist' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST + Specify the value for 'interest_function.convex_hull_dist' in the MET configuration file for MODE. + + | *Used by:* MODE diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 7ae3e3e60b..23ccaef5e7 100755 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -1849,6 +1849,8 @@ Configuration | :term:`MODE_MAX_CENTROID_DIST` | :term:`MODE_TOTAL_INTEREST_THRESH` | :term:`MODE_INTEREST_FUNCTION_CENTROID_DIST` +| :term:`MODE_INTEREST_FUNCTION_BOUNDARY_DIST` +| :term:`MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST` | :term:`FCST_MODE_VAR_NAME` (optional) | :term:`FCST_MODE_VAR_LEVELS` (optional) | :term:`FCST_MODE_VAR_THRESH` (optional) @@ -2323,6 +2325,28 @@ Below the file contents are descriptions of each environment variable referenced * - :term:`MODE_INTEREST_FUNCTION_CENTROID_DIST` - interest_function.centroid_dist +**${METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_INTEREST_FUNCTION_BOUNDARY_DIST` + - interest_function.boundary_dist + +**${METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST` + - interest_function.convex_hull_dist + **${METPLUS_TOTAL_INTEREST_THRESH}** .. list-table:: From 58a9ab1098b01a999249690c7b65d623eda2c79e Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 20 Apr 2021 17:17:32 -0600 Subject: [PATCH 29/42] rearranged config variables to group related items for clarify, removed comments that can be found in the documentation, and moved all variables into [config] section --- .../use_cases/met_tool_wrapper/MODE/MODE.conf | 206 +++++------------- 1 file changed, 59 insertions(+), 147 deletions(-) diff --git a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf index 9ae7eaf7d2..35bc4c2f6d 100644 --- a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf +++ b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf @@ -1,82 +1,95 @@ -# MODE METplus Configuration - -# section heading for [config] variables - all items below this line and -# before the next section heading correspond to the [config] section [config] -# List of applications to run - only MODE for this case +# MODE METplus Configuration + PROCESS_LIST = MODE -# time looping - options are INIT, VALID, RETRO, and REALTIME -# If set to INIT or RETRO: -# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set -# If set to VALID or REALTIME: -# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +LOOP_ORDER = times LOOP_BY = INIT -# Format of INIT_BEG and INIT_END using % items -# %Y = 4 digit year, %m = 2 digit month, %d = 2 digit day, etc. -# see www.strftime.org for more information -# %Y%m%d%H expands to YYYYMMDDHH INIT_TIME_FMT = %Y%m%d%H - -# Start time for METplus run - must match INIT_TIME_FMT INIT_BEG=2005080700 - -# End time for METplus run - must match INIT_TIME_FMT INIT_END=2005080700 - -# Increment between METplus runs (in seconds if no units are specified) -# Must be >= 60 seconds INIT_INCREMENT = 12H -# List of forecast leads to process for each run time (init or valid) -# In hours if units are not specified -# If unset, defaults to 0 (don't loop through forecast leads) LEAD_SEQ = 12 -# Order of loops to process data - Options are times, processes -# Not relevant if only one item is in the PROCESS_LIST -# times = run all wrappers in the PROCESS_LIST for a single run time, then -# increment the run time and run all wrappers again until all times have -# been evaluated. -# processes = run the first wrapper in the PROCESS_LIST for all times -# specified, then repeat for the next item in the PROCESS_LIST until all -# wrappers have been run -LOOP_ORDER = times - -# Verbosity of MET output - overrides LOG_VERBOSITY for MODE only #LOG_MODE_VERBOSITY = 2 -# Name to identify model (forecast) data in output +FCST_MODE_INPUT_DIR = {INPUT_BASE}/met_test/data/sample_fcst +FCST_MODE_INPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/wrfprs_ruc13_{lead?fmt=%HH}.tm00_G212 + +OBS_MODE_INPUT_DIR = {INPUT_BASE}/met_test/data/sample_fcst +OBS_MODE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d%H}/wrfprs_ruc13_00.tm00_G212 + +MODE_OUTPUT_DIR = {OUTPUT_BASE}/mode +MODE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d%H} + + MODEL = WRF MODE_DESC = NA -# Name to identify observation data in output OBTYPE = WRF -# Location of MET config file to pass to the MODE -# References CONFIG_DIR from the [dir] section -MODE_CONFIG_FILE = {CONFIG_DIR}/MODEConfig_wrapped +MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped + +FCST_VAR1_NAME = RH +FCST_VAR1_LEVELS = P500 + +FCST_MODE_CONV_RADIUS = 5 +FCST_MODE_CONV_THRESH = >=80.0 +FCST_MODE_MERGE_THRESH = >=75.0 +FCST_MODE_MERGE_FLAG = NONE + +#MODE_FCST_FILTER_ATTR_NAME = +#MODE_FCST_FILTER_ATTR_THRESH = +#MODE_FCST_CENSOR_THRESH = +#MODE_FCST_CENSOR_VAL = +#MODE_FCST_VLD_THRESH = + +FCST_IS_PROB = false + + +OBS_VAR1_NAME = RH +OBS_VAR1_LEVELS = P500 + +OBS_MODE_CONV_RADIUS = 5 +OBS_MODE_CONV_THRESH = >=80.0 +OBS_MODE_MERGE_THRESH = >=75.0 +OBS_MODE_MERGE_FLAG = NONE + +#MODE_OBS_FILTER_ATTR_NAME = +#MODE_OBS_FILTER_ATTR_THRESH = +#MODE_OBS_CENSOR_THRESH = +#MODE_OBS_CENSOR_VAL = +#MODE_OBS_VLD_THRESH = + +OBS_IS_PROB = false + + +FCST_MODE_FILE_WINDOW_BEGIN = 0 +FCST_MODE_FILE_WINDOW_END = 0 +OBS_MODE_FILE_WINDOW_BEGIN = 0 +OBS_MODE_FILE_WINDOW_END = 0 + MODE_REGRID_TO_GRID = NONE #MODE_REGRID_METHOD = #MODE_REGRID_WIDTH = -#ODE_REGRID_VLD_THRESH = +#MODE_REGRID_VLD_THRESH = #MODE_REGRID_SHAPE = MODE_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_NAME}_{CURRENT_OBS_LEVEL} -# Location of merge config file to pass to the MODE -# References CONFIG_DIR from the [dir] section -# Not used if unset or set to an empty string MODE_MERGE_CONFIG_FILE = -#MODE_DESC = - MODE_GRID_RES = 40 +#MODE_INTEREST_FUNCTION_CENTROID_DIST = +#MODE_INTEREST_FUNCTION_BOUNDARY_DIST = +#MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = + #MODE_TOTAL_INTEREST_THRESH = #MODE_MASK_GRID = @@ -84,18 +97,6 @@ MODE_GRID_RES = 40 #MODE_MASK_POLY = #MODE_MASK_POLY_FLAG = -#MODE_FCST_FILTER_ATTR_NAME = -#MODE_FCST_FILTER_ATTR_THRESH = -#MODE_FCST_CENSOR_THRESH = -#MODE_FCST_CENSOR_VAL = -#MODE_FCST_VLD_THRESH = - -#MODE_OBS_FILTER_ATTR_NAME = -#MODE_OBS_FILTER_ATTR_THRESH = -#MODE_OBS_CENSOR_THRESH = -#MODE_OBS_CENSOR_VAL = -#MODE_OBS_VLD_THRESH = - #MODE_MATCH_FLAG = #MODE_WEIGHT_CENTROID_DIST = @@ -117,93 +118,4 @@ MODE_GRID_RES = 40 #MODE_NC_PAIRS_FLAG_CLUSTER_ID = #MODE_NC_PAIRS_FLAG_POLYLINES = -# turn on quilting MODE_QUILT = True - -# convolution radius list -FCST_MODE_CONV_RADIUS = 5 - -# convolution radius list -OBS_MODE_CONV_RADIUS = 5 - -# convolution threshold list -FCST_MODE_CONV_THRESH = >=80.0 - -# convolution threshold list -OBS_MODE_CONV_THRESH = >=80.0 - -# merge threshold list -FCST_MODE_MERGE_THRESH = >=75.0 - -# merge threshold list -OBS_MODE_MERGE_THRESH = >=75.0 - -# merge flag: options are NONE, THRESH, ENGINE, or BOTH -FCST_MODE_MERGE_FLAG = NONE - -# merge flag: options are NONE, THRESH, ENGINE, or BOTH -OBS_MODE_MERGE_FLAG = NONE - -# List of variables to compare in MODE - FCST_VAR1 variables correspond -# to OBS_VAR1 variables - -# Name of forecast variable 1 -FCST_VAR1_NAME = RH - -# List of levels to evaluate for forecast variable 1 -# P500 = 500 mb pressure level in GRIB file -FCST_VAR1_LEVELS = P500 - -# Name of observation variable 1 -OBS_VAR1_NAME = RH - -# List of levels to evaluate for observation variable 1 -# P500 = 500 mb pressure level in GRIB file -# must be the same length as FCST_VAR1_LEVELS -OBS_VAR1_LEVELS = P500 - -# Time relative to valid time (in seconds) to allow files to be considered -# valid. Set both BEGIN and END to 0 to require the exact time in the filename -# Not used in this example. -FCST_MODE_FILE_WINDOW_BEGIN = 0 -FCST_MODE_FILE_WINDOW_END = 0 -OBS_MODE_FILE_WINDOW_BEGIN = 0 -OBS_MODE_FILE_WINDOW_END = 0 - -# Set to true if forecast data is probabilistic -FCST_IS_PROB = false - -# Set to true if observation data is probabilistic -# Only used if configuring forecast data as the 'OBS' input -OBS_IS_PROB = false - -# Used to specify a verification mask poly file for MODE -# Not used for this example. -MODE_MASK_POLY = - -# End of [config] section and start of [dir] section -[dir] - -# location of configuration files used by MET applications -CONFIG_DIR={PARM_BASE}/met_config - -# directory containing forecast input to MODE -FCST_MODE_INPUT_DIR = {INPUT_BASE}/met_test/data/sample_fcst - -# directory containing observation input to MODE -OBS_MODE_INPUT_DIR = {INPUT_BASE}/met_test/data/sample_fcst - -# directory to write output from MODE -MODE_OUTPUT_DIR = {OUTPUT_BASE}/mode - -# End of [dir] section and start of [filename_templates] section -[filename_templates] - -# Template to look for forecast input to MODE relative to FCST_MODE_INPUT_DIR -FCST_MODE_INPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/wrfprs_ruc13_{lead?fmt=%HH}.tm00_G212 - -# Template to look for observation input to MODE relative to OBS_MODE_INPUT_DIR -OBS_MODE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d%H}/wrfprs_ruc13_00.tm00_G212 - -# Optional subdirectories relative to MODE_OUTPUT_DIR to write output from MODE -MODE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d%H} From 1d785969085a7bbb3dc60b8bb6494fc3c53539a5 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 21 Apr 2021 14:10:43 -0600 Subject: [PATCH 30/42] added support for setting obs_quality in PointStat wrapped config --- docs/Users_Guide/glossary.rst | 5 +++++ docs/Users_Guide/wrappers.rst | 13 +++++++++++++ .../pytests/point_stat/test_point_stat_wrapper.py | 3 +++ parm/met_config/PointStatConfig_wrapped | 3 ++- 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 6d52cd3d60..649692db6a 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -6539,3 +6539,8 @@ METplus Configuration Glossary Specify the value for 'interest_function.convex_hull_dist' in the MET configuration file for MODE. | *Used by:* MODE + + POINT_STAT_OBS_QUALITY + Specify the value for 'obs_quality' in the MET configuration file for PointStat. + + | *Used by:* PointStat diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 23ccaef5e7..817e19cba7 100755 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -3002,6 +3002,7 @@ Configuration | :term:`POINT_STAT_CLIMO_CDF_BINS` | :term:`POINT_STAT_CLIMO_CDF_CENTER_BINS` | :term:`POINT_STAT_CLIMO_CDF_WRITE_BINS` +| :term:`POINT_STAT_OBS_QUALITY` | :term:`FCST_POINT_STAT_WINDOW_BEGIN` (optional) | :term:`FCST_POINT_STAT_WINDOW_END` (optional) | :term:`OBS_POINT_STAT_WINDOW_BEGIN` (optional) @@ -3249,6 +3250,18 @@ Below the file contents are descriptions of each environment variable referenced * - :term:`POINT_STAT_CLIMO_CDF_WRITE_BINS` - climo_cdf.write_bins +**${METPLUS_OBS_QUALITY}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT_STAT_OBS_QUALITY` + - obs_quality + + .. _py_embed_ingest_wrapper: PyEmbedIngest diff --git a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py index e3111925e5..5aa7b8e151 100755 --- a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py +++ b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py @@ -152,6 +152,9 @@ def point_stat_wrapper(metplus_config): { 'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {cdf_bins = 1.0;center_bins = TRUE;write_bins = FALSE;}'}), + ({'POINT_STAT_OBS_QUALITY': '1, 2, 3', }, + {'METPLUS_OBS_QUALITY': 'obs_quality = ["1", "2", "3"];'}), + ] ) def test_point_stat_all_fields(metplus_config, config_overrides, diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index 134723e7da..f3526859e4 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -57,7 +57,8 @@ obs = { // ${METPLUS_MESSAGE_TYPE} sid_exc = []; -obs_quality = [ "1", "2", "3" ]; +//obs_quality = +${METPLUS_OBS_QUALITY} duplicate_flag = NONE; obs_summary = NONE; obs_perc_value = 50; From 1d886b89f821c5092300136862dda383f610d3b2 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 21 Apr 2021 14:12:52 -0600 Subject: [PATCH 31/42] added new config to basic use case --- parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf index fe60e962b6..9e17dff4e6 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf @@ -50,6 +50,8 @@ LOOP_ORDER = processes # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped +#POINT_STAT_OBS_QUALITY = 1, 2, 3 + #POINT_STAT_CLIMO_CDF_BINS = 1 #POINT_STAT_CLIMO_CDF_CENTER_BINS = False #POINT_STAT_CLIMO_CDF_WRITE_BINS = True From bc01b400b699d4ce3bf0533c2cd76d22e6016bbf Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 21 Apr 2021 14:13:26 -0600 Subject: [PATCH 32/42] removed obs_quality from POINT_STAT_MET_CONFIG_OVERRIDES because it is no longer needed (default is same as value specified) ci-run-diff --- .../UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf b/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf index 2bc487424e..8f4104600e 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf @@ -76,7 +76,7 @@ BOTH_VAR1_NAME = TMP BOTH_VAR1_LEVELS = P925-950, P850-800, P700-650 POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped -POINT_STAT_MET_CONFIG_OVERRIDES = obs_quality = []; output_flag = {fho = BOTH; ctc = BOTH; cts = STAT; cnt = BOTH; eclv = BOTH; mpr = BOTH;} +POINT_STAT_MET_CONFIG_OVERRIDES = output_flag = {fho = BOTH; ctc = BOTH; cts = STAT; cnt = BOTH; eclv = BOTH; mpr = BOTH;} # Regrid to specified grid. Indicate NONE if no regridding, or the grid id From 91d163b50f0feb7d41c97340acc1016d1d494768 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 21 Apr 2021 14:30:55 -0600 Subject: [PATCH 33/42] added support for all output_flag dictionary values in PointStat, removed POINT_STAT_MET_CONFIG_OVERRIDES in favor of new variables, ci-run-diff --- docs/Users_Guide/glossary.rst | 95 +++++++++++++++++++ docs/Users_Guide/wrappers.rst | 66 +++++++++++++ .../point_stat/test_point_stat_wrapper.py | 81 ++++++++++++++++ metplus/wrappers/point_stat_wrapper.py | 29 ++++++ parm/met_config/PointStatConfig_wrapped | 22 +---- .../met_tool_wrapper/PointStat/PointStat.conf | 20 ++++ ...I2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf | 7 +- 7 files changed, 299 insertions(+), 21 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 649692db6a..222e2e0b5c 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -6544,3 +6544,98 @@ METplus Configuration Glossary Specify the value for 'obs_quality' in the MET configuration file for PointStat. | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_FHO + Specify the value for 'output_flag.fho' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_CTC + Specify the value for 'output_flag.ctc' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_CTS + Specify the value for 'output_flag.cts' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_MCTC + Specify the value for 'output_flag.mctc' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_MCTS + Specify the value for 'output_flag.mcts' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_CNT + Specify the value for 'output_flag.cnt' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_SL1L2 + Specify the value for 'output_flag.sl1l2' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_SAL1L2 + Specify the value for 'output_flag.sal1l2' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_VL1L2 + Specify the value for 'output_flag.vl1l2' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_VAL1L2 + Specify the value for 'output_flag.val1l2' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_VCNT + Specify the value for 'output_flag.vcnt' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_PCT + Specify the value for 'output_flag.pct' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_PSTD + Specify the value for 'output_flag.pstd' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_PJC + Specify the value for 'output_flag.pjc' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_PRC + Specify the value for 'output_flag.prc' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_ECNT + Specify the value for 'output_flag.ecnt' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_RPS + Specify the value for 'output_flag.rps' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_ECLV + Specify the value for 'output_flag.eclv' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OUTPUT_FLAG_MPR + Specify the value for 'output_flag.mpr' in the MET configuration file for PointStat. + + | *Used by:* PointStat diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 817e19cba7..9f1bb5a815 100755 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -3003,6 +3003,25 @@ Configuration | :term:`POINT_STAT_CLIMO_CDF_CENTER_BINS` | :term:`POINT_STAT_CLIMO_CDF_WRITE_BINS` | :term:`POINT_STAT_OBS_QUALITY` +| :term:`POINT_STAT_OUTPUT_FLAG_FHO` +| :term:`POINT_STAT_OUTPUT_FLAG_CTC` +| :term:`POINT_STAT_OUTPUT_FLAG_CTS` +| :term:`POINT_STAT_OUTPUT_FLAG_MCTC` +| :term:`POINT_STAT_OUTPUT_FLAG_MCTS` +| :term:`POINT_STAT_OUTPUT_FLAG_CNT` +| :term:`POINT_STAT_OUTPUT_FLAG_SL1L2` +| :term:`POINT_STAT_OUTPUT_FLAG_SAL1L2` +| :term:`POINT_STAT_OUTPUT_FLAG_VL1L2` +| :term:`POINT_STAT_OUTPUT_FLAG_VAL1L2` +| :term:`POINT_STAT_OUTPUT_FLAG_VCNT` +| :term:`POINT_STAT_OUTPUT_FLAG_PCT` +| :term:`POINT_STAT_OUTPUT_FLAG_PSTD` +| :term:`POINT_STAT_OUTPUT_FLAG_PJC` +| :term:`POINT_STAT_OUTPUT_FLAG_PRC` +| :term:`POINT_STAT_OUTPUT_FLAG_ECNT` +| :term:`POINT_STAT_OUTPUT_FLAG_RPS` +| :term:`POINT_STAT_OUTPUT_FLAG_ECLV` +| :term:`POINT_STAT_OUTPUT_FLAG_MPR` | :term:`FCST_POINT_STAT_WINDOW_BEGIN` (optional) | :term:`FCST_POINT_STAT_WINDOW_END` (optional) | :term:`OBS_POINT_STAT_WINDOW_BEGIN` (optional) @@ -3261,6 +3280,53 @@ Below the file contents are descriptions of each environment variable referenced * - :term:`POINT_STAT_OBS_QUALITY` - obs_quality +**${METPLUS_OUTPUT_FLAG_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT_STAT_OUTPUT_FLAG_FHO` + - output_flag.fho + * - :term:`POINT_STAT_OUTPUT_FLAG_CTC` + - output_flag.ctc + * - :term:`POINT_STAT_OUTPUT_FLAG_CTS` + - output_flag.cts + * - :term:`POINT_STAT_OUTPUT_FLAG_MCTC` + - output_flag.mctc + * - :term:`POINT_STAT_OUTPUT_FLAG_MCTS` + - output_flag.mcts + * - :term:`POINT_STAT_OUTPUT_FLAG_CNT` + - output_flag.cnt + * - :term:`POINT_STAT_OUTPUT_FLAG_SL1L2` + - output_flag.sl1l2 + * - :term:`POINT_STAT_OUTPUT_FLAG_SAL1L2` + - output_flag.sal1l2 + * - :term:`POINT_STAT_OUTPUT_FLAG_VL1L2` + - output_flag.vl1l2 + * - :term:`POINT_STAT_OUTPUT_FLAG_VAL1L2` + - output_flag.val1l2 + * - :term:`POINT_STAT_OUTPUT_FLAG_VCNT` + - output_flag.vcnt + * - :term:`POINT_STAT_OUTPUT_FLAG_PCT` + - output_flag.pct + * - :term:`POINT_STAT_OUTPUT_FLAG_PSTD` + - output_flag.pstd + * - :term:`POINT_STAT_OUTPUT_FLAG_PJC` + - output_flag.pjc + * - :term:`POINT_STAT_OUTPUT_FLAG_PRC` + - output_flag.prc + * - :term:`POINT_STAT_OUTPUT_FLAG_ECNT` + - output_flag.ecnt + * - :term:`POINT_STAT_OUTPUT_FLAG_RPS` + - output_flag.rps + * - :term:`POINT_STAT_OUTPUT_FLAG_ECLV` + - output_flag.eclv + * - :term:`POINT_STAT_OUTPUT_FLAG_MPR` + - output_flag.mpr + .. _py_embed_ingest_wrapper: diff --git a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py index 5aa7b8e151..fd6dcf52ca 100755 --- a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py +++ b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py @@ -155,6 +155,87 @@ def point_stat_wrapper(metplus_config): ({'POINT_STAT_OBS_QUALITY': '1, 2, 3', }, {'METPLUS_OBS_QUALITY': 'obs_quality = ["1", "2", "3"];'}), + ({'POINT_STAT_OUTPUT_FLAG_FHO': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {fho = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_CTC': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {ctc = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_CTS': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {cts = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_MCTC': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {mctc = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_MCTS': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {mcts = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_CNT': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {cnt = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_SL1L2': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {sl1l2 = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_SAL1L2': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {sal1l2 = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_VL1L2': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {vl1l2 = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_VAL1L2': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {val1l2 = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_VCNT': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {vcnt = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_PCT': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pct = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_PSTD': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pstd = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_PJC': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pjc = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_PRC': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {prc = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_ECNT': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {ecnt = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_RPS': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {rps = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_ECLV': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {eclv = BOTH;}'}), + + ({'POINT_STAT_OUTPUT_FLAG_MPR': 'BOTH', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {mpr = BOTH;}'}), + + ({ + 'POINT_STAT_OUTPUT_FLAG_FHO': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_CTC': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_CTS': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_MCTC': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_MCTS': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_CNT': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_SL1L2': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_SAL1L2': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_VL1L2': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_VAL1L2': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_VCNT': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_PCT': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_PSTD': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_PJC': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_PRC': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_ECNT': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_RPS': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_ECLV': 'BOTH', + 'POINT_STAT_OUTPUT_FLAG_MPR': 'BOTH', + }, + { + 'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {fho = BOTH;ctc = BOTH;cts = BOTH;mctc = BOTH;mcts = BOTH;cnt = BOTH;sl1l2 = BOTH;sal1l2 = BOTH;vl1l2 = BOTH;val1l2 = BOTH;vcnt = BOTH;pct = BOTH;pstd = BOTH;pjc = BOTH;prc = BOTH;ecnt = BOTH;rps = BOTH;eclv = BOTH;mpr = BOTH;}'}), + ] ) def test_point_stat_all_fields(metplus_config, config_overrides, diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index 5e8ab2f347..46b302149c 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -35,8 +35,31 @@ class PointStatWrapper(CompareGriddedWrapper): 'METPLUS_MASK_SID', 'METPLUS_OUTPUT_PREFIX', 'METPLUS_CLIMO_CDF_DICT', + 'METPLUS_OBS_QUALITY', + 'METPLUS_OUTPUT_FLAG_DICT', ] + OUTPUT_FLAGS = ['fho', + 'ctc', + 'cts', + 'mctc', + 'mcts', + 'cnt', + 'sl1l2', + 'sal1l2', + 'vl1l2', + 'val1l2', + 'vcnt', + 'pct', + 'pstd', + 'pjc', + 'prc', + 'ecnt', + 'rps', + 'eclv', + 'mpr', + ] + def __init__(self, config, instance=None, config_overrides={}): self.app_name = 'point_stat' self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), @@ -164,6 +187,12 @@ def create_c_dict(self): False) ) + self.add_met_config(name='obs_quality', + data_type='list', + metplus_configs=['POINT_STAT_OBS_QUALITY']) + + self.handle_flags('output') + if not c_dict['FCST_INPUT_TEMPLATE']: self.log_error('Must set FCST_POINT_STAT_INPUT_TEMPLATE ' 'in config file') diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index f3526859e4..20ffa7c434 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -190,26 +190,8 @@ hira = { // // Statistical output types // -output_flag = { - fho = NONE; - ctc = NONE; - cts = NONE; - mctc = NONE; - mcts = NONE; - cnt = NONE; - sl1l2 = STAT; - sal1l2 = NONE; - vl1l2 = STAT; - val1l2 = NONE; - vcnt = NONE; - pct = NONE; - pstd = NONE; - pjc = NONE; - prc = NONE; - ecnt = NONE; // Only for HiRA. - eclv = NONE; - mpr = NONE; -} +//output_flag = { +${METPLUS_OUTPUT_FLAG_DICT} //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf index 9e17dff4e6..3fb888158a 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf @@ -52,6 +52,26 @@ POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped #POINT_STAT_OBS_QUALITY = 1, 2, 3 +#POINT_STAT_OUTPUT_FLAG_FHO = +#POINT_STAT_OUTPUT_FLAG_CTC = +#POINT_STAT_OUTPUT_FLAG_CTS = +#POINT_STAT_OUTPUT_FLAG_MCTC = +#POINT_STAT_OUTPUT_FLAG_MCTS = +#POINT_STAT_OUTPUT_FLAG_CNT = +#POINT_STAT_OUTPUT_FLAG_SL1L2 = +#POINT_STAT_OUTPUT_FLAG_SAL1L2 = +#POINT_STAT_OUTPUT_FLAG_VL1L2 = +#POINT_STAT_OUTPUT_FLAG_VAL1L2 = +#POINT_STAT_OUTPUT_FLAG_VCNT = +#POINT_STAT_OUTPUT_FLAG_PCT = +#POINT_STAT_OUTPUT_FLAG_PSTD = +#POINT_STAT_OUTPUT_FLAG_PJC = +#POINT_STAT_OUTPUT_FLAG_PRC = +#POINT_STAT_OUTPUT_FLAG_ECNT = +#POINT_STAT_OUTPUT_FLAG_RPS = +#POINT_STAT_OUTPUT_FLAG_ECLV = +#POINT_STAT_OUTPUT_FLAG_MPR = + #POINT_STAT_CLIMO_CDF_BINS = 1 #POINT_STAT_CLIMO_CDF_CENTER_BINS = False #POINT_STAT_CLIMO_CDF_WRITE_BINS = True diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf b/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf index 8f4104600e..fd935f62de 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf @@ -76,8 +76,13 @@ BOTH_VAR1_NAME = TMP BOTH_VAR1_LEVELS = P925-950, P850-800, P700-650 POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped -POINT_STAT_MET_CONFIG_OVERRIDES = output_flag = {fho = BOTH; ctc = BOTH; cts = STAT; cnt = BOTH; eclv = BOTH; mpr = BOTH;} +POINT_STAT_OUTPUT_FLAG_FHO = BOTH +POINT_STAT_OUTPUT_FLAG_CTC = BOTH +POINT_STAT_OUTPUT_FLAG_CTS = STAT +POINT_STAT_OUTPUT_FLAG_CNT = BOTH +POINT_STAT_OUTPUT_FLAG_ECLV = BOTH +POINT_STAT_OUTPUT_FLAG_MPR = BOTH # Regrid to specified grid. Indicate NONE if no regridding, or the grid id # (e.g. G212) From f2389debea3fdf06b6a269b58ec9087fe0ccd489 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 21 Apr 2021 14:51:40 -0600 Subject: [PATCH 34/42] added name of variable above env var --- parm/met_config/PointStatConfig_wrapped | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index 20ffa7c434..4a10bc06be 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -9,12 +9,14 @@ // // Output model name to be written // +// model = ${METPLUS_MODEL} // // Output description to be written // May be set separately in each "obs.field" entry // +// desc = ${METPLUS_DESC} //////////////////////////////////////////////////////////////////////////////// @@ -22,6 +24,7 @@ ${METPLUS_DESC} // // Verification grid // +// regrid = { ${METPLUS_REGRID_DICT} //////////////////////////////////////////////////////////////////////////////// @@ -55,6 +58,7 @@ obs = { // Point observation filtering options // May be set separately in each "obs.field" entry // +// message_type = ${METPLUS_MESSAGE_TYPE} sid_exc = []; //obs_quality = @@ -110,7 +114,6 @@ climo_stdev = { } time_interp_method = DW_MEAN; - match_month = TRUE; day_interval = 31; hour_interval = 6; } @@ -126,6 +129,7 @@ ${METPLUS_CLIMO_CDF_DICT} // // Point observation time window // +// obs_window = { ${METPLUS_OBS_WINDOW_DICT} //////////////////////////////////////////////////////////////////////////////// @@ -196,8 +200,9 @@ ${METPLUS_OUTPUT_FLAG_DICT} //////////////////////////////////////////////////////////////////////////////// tmp_dir = "/tmp"; +// output_prefix = ${METPLUS_OUTPUT_PREFIX} -//version = "V9.0"; +//version = "V10.0.0"; //////////////////////////////////////////////////////////////////////////////// From 28f507444cec6a98f09826070075501eb0baf682 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 21 Apr 2021 14:56:54 -0600 Subject: [PATCH 35/42] add output flags that were set to STAT in wrapped config file previously, ci-run-diff --- parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf | 4 ++-- .../met_tool_wrapper/PointStat/PointStat_once_per_field.conf | 3 +++ .../PointStat/PointStat_python_embedding.conf | 3 +++ ...ointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf | 4 ++++ .../PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf | 4 ++++ .../UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf | 2 ++ 6 files changed, 18 insertions(+), 2 deletions(-) diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf index 3fb888158a..9905255121 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf @@ -58,9 +58,9 @@ POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped #POINT_STAT_OUTPUT_FLAG_MCTC = #POINT_STAT_OUTPUT_FLAG_MCTS = #POINT_STAT_OUTPUT_FLAG_CNT = -#POINT_STAT_OUTPUT_FLAG_SL1L2 = +POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT #POINT_STAT_OUTPUT_FLAG_SAL1L2 = -#POINT_STAT_OUTPUT_FLAG_VL1L2 = +POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT #POINT_STAT_OUTPUT_FLAG_VAL1L2 = #POINT_STAT_OUTPUT_FLAG_VCNT = #POINT_STAT_OUTPUT_FLAG_PCT = diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat_once_per_field.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat_once_per_field.conf index 5499674651..4e43fe21ba 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat_once_per_field.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat_once_per_field.conf @@ -50,6 +50,9 @@ LOOP_ORDER = processes # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped +POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT +POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT + # Time relative to each input file's valid time (in seconds if no units are specified) for data within the file to be # considered valid. Values are set in the 'obs_window' dictionary in the PointStat config file OBS_POINT_STAT_WINDOW_BEGIN = -5400 diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat_python_embedding.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat_python_embedding.conf index 64b1067359..9146e6802d 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat_python_embedding.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat_python_embedding.conf @@ -50,6 +50,9 @@ LOOP_ORDER = processes # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped +POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT +POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT + # Time relative to each input file's valid time (in seconds if no units are specified) for data within the file to be # considered valid. Values are set in the 'obs_window' dictionary in the PointStat config file OBS_POINT_STAT_WINDOW_BEGIN = -5400 diff --git a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf index 1b060df70a..53bfb64262 100644 --- a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf +++ b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf @@ -32,6 +32,10 @@ PB2NC_SKIP_IF_OUTPUT_EXISTS = True PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped +POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT +POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT + + # Either conus_sfc or upper_air PB2NC_VERTICAL_LOCATION = upper_air diff --git a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf index d06c35afaf..c6a3b12a53 100644 --- a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf +++ b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf @@ -33,6 +33,10 @@ LEAD_SEQ = 0 OBS_WINDOW_BEGIN = -2700 OBS_WINDOW_END = 2700 +POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT +POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT + + # Either conus_sfc or upper_air PB2NC_VERTICAL_LOCATION = conus_sfc diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf b/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf index fd935f62de..5ae3dd5970 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf @@ -77,6 +77,8 @@ BOTH_VAR1_LEVELS = P925-950, P850-800, P700-650 POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped +POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT +POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT POINT_STAT_OUTPUT_FLAG_FHO = BOTH POINT_STAT_OUTPUT_FLAG_CTC = BOTH POINT_STAT_OUTPUT_FLAG_CTS = STAT From 81728c23608496fa3c361cc1227ada8693d6fc8b Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 21 Apr 2021 15:11:20 -0600 Subject: [PATCH 36/42] changed values in wrapped PointStat config to match default PointStat in MET repo -- this may require updates to some use case configs if this changes the output, ci-run-diff --- parm/met_config/PointStatConfig_wrapped | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index 4a10bc06be..cc940eb962 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -96,7 +96,7 @@ climo_mean = { shape = SQUARE; } - time_interp_method = NEAREST; + time_interp_method = DW_MEAN; day_interval = 31; hour_interval = 6; } @@ -170,8 +170,8 @@ interp = { type = [ { - method = BILIN; - width = 2; + method = NEAREST; + width = 1; } ]; } From ba139b1053505abde90e1812b444eeba006592c4 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 21 Apr 2021 15:50:20 -0600 Subject: [PATCH 37/42] install cartopy 0.18 because new version won't install properly using this script --- ci/jobs/python_requirements/get_cartopy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jobs/python_requirements/get_cartopy.sh b/ci/jobs/python_requirements/get_cartopy.sh index 2c66df9fdd..e3ccc8cd18 100755 --- a/ci/jobs/python_requirements/get_cartopy.sh +++ b/ci/jobs/python_requirements/get_cartopy.sh @@ -15,7 +15,7 @@ yum -y install geos yum -y install geos-devel pip3 install --upgrade cython numpy pyshp six pip3 install shapely --no-binary shapely -pip3 install cartopy +pip3 install cartopy==0.18.0 #some cartopy functionality fails without scipy pip3 install scipy From 5b1e1e0623b890ee3079328d6da3d937152d5d06 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 21 Apr 2021 15:53:17 -0600 Subject: [PATCH 38/42] revert back changes to match default MET config to ensure no differences, ci-run-diff --- parm/met_config/PointStatConfig_wrapped | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index cc940eb962..4a10bc06be 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -96,7 +96,7 @@ climo_mean = { shape = SQUARE; } - time_interp_method = DW_MEAN; + time_interp_method = NEAREST; day_interval = 31; hour_interval = 6; } @@ -170,8 +170,8 @@ interp = { type = [ { - method = NEAREST; - width = 1; + method = BILIN; + width = 2; } ]; } From 690d254f337f4f91a3abb1ce64042d45a52a40e8 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 22 Apr 2021 13:20:09 -0600 Subject: [PATCH 39/42] changes to support handling MET config variables in a common way including dictionaries to allow easy addition of nested dictionaries --- .../command_builder/test_command_builder.py | 57 ++++++++++++++- metplus/util/met_dictionary_info.py | 24 ++++++- metplus/wrappers/command_builder.py | 47 +++++++++---- metplus/wrappers/compare_gridded_wrapper.py | 69 +++++++++++++++++++ metplus/wrappers/ensemble_stat_wrapper.py | 41 ----------- 5 files changed, 180 insertions(+), 58 deletions(-) diff --git a/internal_tests/pytests/command_builder/test_command_builder.py b/internal_tests/pytests/command_builder/test_command_builder.py index a54be0f831..ee0e25acf9 100644 --- a/internal_tests/pytests/command_builder/test_command_builder.py +++ b/internal_tests/pytests/command_builder/test_command_builder.py @@ -729,9 +729,9 @@ def test_set_met_config_list_allow_empty(metplus_config, mp_config_name, ('bad_name', None), ] ) -def test_get_met_config_function(metplus_config, data_type, expected_function): +def test_set_met_config_function(metplus_config, data_type, expected_function): cbw = CommandBuilder(metplus_config()) - function_found = cbw.get_met_config_function(data_type) + function_found = cbw.set_met_config_function(data_type) function_name = function_found.__name__ if function_found else None assert(function_name == expected_function) @@ -772,3 +772,56 @@ def test_add_met_config(metplus_config): print(f"env_var_dict: {cbw.env_var_dict}") expected_value = f'valid_freq = {value};' assert(cbw.env_var_dict['METPLUS_VALID_FREQ'] == expected_value) + +def test_handle_met_config_dict_nested(metplus_config): + dict_name = 'outer' + beg = -3 + end = 5 + sub_dict_name = 'inner' + sub_dict_value1 = 'value1' + sub_dict_value2 = 'value2' + expected_value = ( + f'{dict_name} = {{beg = {beg};end = {end};{sub_dict_name} = ' + f'{{var1 = {sub_dict_value1};var2 = {sub_dict_value2};}}}}' + ) + + config = metplus_config() + config.set('config', 'APP_OUTER_BEG', beg) + config.set('config', 'APP_OUTER_END', end) + config.set('config', 'APP_OUTER_INNER_VAR1', sub_dict_value1) + config.set('config', 'APP_OUTER_INNER_VAR2', sub_dict_value2) + cbw = CommandBuilder(config) + + dict_items = [] + item = met_config(name='beg', + data_type='int', + metplus_configs=['APP_OUTER_BEG']) + dict_items.append(item) + item = met_config(name='end', + data_type='int', + metplus_configs=['APP_OUTER_END']) + dict_items.append(item) + + sub_dict_items = [] + item = met_config(name='var1', + data_type='string', + metplus_configs=['APP_OUTER_INNER_VAR1'], + extra_args={'remove_quotes': True}) + sub_dict_items.append(item) + item = met_config(name='var2', + data_type='string', + metplus_configs=['APP_OUTER_INNER_VAR2'], + extra_args={'remove_quotes': True}) + sub_dict_items.append(item) + + dict_items.append( + cbw.get_met_config( + name=sub_dict_name, + data_type='dict', + children=sub_dict_items, + ) + ) + + cbw.handle_met_config_dict(dict_name, dict_items) + print(f"env_var_dict: {cbw.env_var_dict}") + assert(cbw.env_var_dict.get('METPLUS_OUTER_DICT') == expected_value) diff --git a/metplus/util/met_dictionary_info.py b/metplus/util/met_dictionary_info.py index da65dac4d1..f9de77a5dd 100644 --- a/metplus/util/met_dictionary_info.py +++ b/metplus/util/met_dictionary_info.py @@ -9,6 +9,7 @@ Output Files: N/A """ + class METConfigInfo: """! Stores information for a member of a MET config variables that can be used to set the value, the data type of the item, @@ -19,18 +20,21 @@ class METConfigInfo: def __init__(self, name, data_type, env_var_name=None, metplus_configs=None, - extra_args=None): + extra_args=None, + children=None): self.name = name self.data_type = data_type self.metplus_configs = metplus_configs self.extra_args = extra_args self.env_var_name = env_var_name if env_var_name else name + self.children = children def __repr__(self): return (f'{self.__class__.__name__}({self.name}, {self.data_type}, ' f'{self.env_var_name}, ' f'{self.metplus_configs}, ' f'{self.extra_args}' + f', {self.children}' ')') @property @@ -86,3 +90,21 @@ def extra_args(self, extra_args): self._extra_args = args + @property + def children(self): + return self._children + + @children.setter + def children(self, children): + if not children and self.data_type == 'dict': + raise TypeError("Must have children if data_type is dict.") + + if children: + if self.data_type != 'dict': + raise TypeError("data_type must be dict to have children. " + f"data_type is {self.data_type}") + + + + + self._children = children diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 35648d7539..6771782797 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -2031,7 +2031,7 @@ def handle_mask(self, single_value=False, get_flags=False): self.handle_met_config_dict(dict_name, dict_items) - def get_met_config_function(self, item_type): + def set_met_config_function(self, item_type): """! Return function to use based on item type @param item_type type of MET config variable to obtain @@ -2056,10 +2056,14 @@ def get_met_config_function(self, item_type): return None def handle_met_config_item(self, item, output_dict=None): - set_met_config = self.get_met_config_function(item.data_type) - if not set_met_config: - return False + """! Reads info from METConfigInfo object, gets value from + METplusConfig, and formats it based on the specifications. Sets + value in output dictionary with key starting with METPLUS_. + @param item METConfigInfo object to read and determine what to get + @param output_dict (optional) dictionary to save formatted output + If unset, use self.env_var_dict. + """ if output_dict is None: output_dict = self.env_var_dict @@ -2067,6 +2071,25 @@ def handle_met_config_item(self, item, output_dict=None): if not env_var_name.startswith('METPLUS_'): env_var_name = f'METPLUS_{env_var_name}' + # handle dictionary item + if item.data_type == 'dict': + env_var_name = f'{env_var_name}_DICT' + tmp_dict = {} + for child in item.children: + if not self.handle_met_config_item(child, tmp_dict): + return False + + dict_string = self.format_met_config_dict(tmp_dict, + item.name, + keys=None) + output_dict[env_var_name] = dict_string + return True + + # handle non-dictionary item + set_met_config = self.set_met_config_function(item.data_type) + if not set_met_config: + return False + set_met_config(output_dict, item.metplus_configs, item.name, @@ -2083,17 +2106,13 @@ def handle_met_config_dict(self, dict_name, dict_items, if output_dict is None: output_dict = self.env_var_dict - tmp_dict = {} - for item in dict_items: - self.handle_met_config_item(item, output_dict=tmp_dict) - - dict_string = self.format_met_config_dict(tmp_dict, - dict_name, - keys=None) - env_var_name = f'METPLUS_{dict_name.upper()}_DICT' - output_dict[env_var_name] = dict_string + final_met_config = self.get_met_config( + name=dict_name, + data_type='dict', + children=dict_items, + ) - return True + return self.handle_met_config_item(final_met_config, output_dict) def add_met_config(self, **kwargs): """! Create METConfigInfo object from arguments and process diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index 4785de7821..b8161ce93f 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -431,3 +431,72 @@ def handle_climo_cdf_dict(self): 'CLIMO_CDF_WRITE_BINS']) ) self.env_var_dict['METPLUS_CLIMO_CDF_DICT'] = climo_cdf + + def handle_interp_dict(self, uses_field=False): + """! Reads config variables for interp dictionary, i.e. + _INTERP_VLD_THRESH, _INTERP_SHAPE, _INTERP_METHOD, and + _INTERP_WIDTH. Also _INTERP_FIELD if specified + + @param uses_field if True, read field variable as well + (default is False) + """ + app = self.app_name.upper() + + dict_name = 'interp' + dict_items = [] + + metplus_prefix = f'{app}_{dict_name.upper()}_' + + # items to set for interp dictionary + # key is MET config name and used for METplus config and env var names + # value is a tuple of data type of item and names of any children items + interp_items = { + 'vld_thresh': ('float', None), + 'shape': ('string', None), + 'type': ('dict', [('method', 'string'), + ('width', 'int')]), + } + + if uses_field: + interp_items['field'] = ('string', None) + + for name, (data_type, kids) in interp_items.items(): + metplus_name = f'{metplus_prefix}{name.upper()}' + metplus_configs = [] + + # if dictionary, read get children from MET config + if data_type == 'dict': + children = [] + for kid, kid_type in kids: + # add APP_INTERP_TYPE_METHOD and APP_INTERP_METHOD + metplus_configs.append(f'{metplus_name}_{kid.upper()}') + metplus_configs.append(f'{metplus_prefix}{kid.upper()}') + + child_item = self.get_met_config( + name=kid, + data_type=kid_type, + metplus_configs=metplus_configs.copy(), + extra_args={'remove_quotes': True} + ) + children.append(child_item) + + # reset metplus config list for next kid + metplus_configs.clear() + + # set metplus_configs + metplus_configs = None + else: + children = None + metplus_configs.append(metplus_name) + + dict_items.append( + self.get_met_config( + name=name, + data_type=data_type, + metplus_configs=metplus_configs, + extra_args={'remove_quotes': True}, + children=children, + ) + ) + + self.handle_met_config_dict(dict_name, dict_items) diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index fd8387f121..a770be4541 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -400,47 +400,6 @@ def handle_nbrhd_prob_dict(self): ) self.env_var_dict['METPLUS_NBRHD_PROB_DICT'] = nbrhd_prob - def handle_interp_dict(self): - tmp_dict = {} - self.set_met_config_string(tmp_dict, - 'ENSEMBLE_STAT_INTERP_FIELD', - 'field', - 'INTERP_FIELD', - remove_quotes=True) - self.set_met_config_float(tmp_dict, - 'ENSEMBLE_STAT_INTERP_VLD_THRESH', - 'vld_thresh', - 'INTERP_VLD_THRESH') - self.set_met_config_string(tmp_dict, - 'ENSEMBLE_STAT_INTERP_SHAPE', - 'shape', - 'INTERP_SHAPE', - remove_quotes=True) - self.set_met_config_string(tmp_dict, - 'ENSEMBLE_STAT_INTERP_METHOD', - 'method', - 'INTERP_METHOD', - remove_quotes=True) - self.set_met_config_int(tmp_dict, - 'ENSEMBLE_STAT_INTERP_WIDTH', - 'width', - 'INTERP_WIDTH') - - tmp_dict['INTERP_TYPE'] = self.format_met_config_type(tmp_dict, - 'interp') - - - interp = ( - self.format_met_config_dict(tmp_dict, - 'interp', - ['INTERP_FIELD', - 'INTERP_VLD_THRESH', - 'INTERP_SHAPE', - 'INTERP_TYPE', - ]) - ) - self.env_var_dict['METPLUS_INTERP_DICT'] = interp - def format_met_config_type(self, c_dict, dict_name): """! Format type item for MET config From 921053b188b0495846473e5e07251d5b679d8531 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 22 Apr 2021 13:20:46 -0600 Subject: [PATCH 40/42] added support for climo_mean.time_interp_method, climo_stdev.time_interp_method, and interp dictionary --- docs/Users_Guide/glossary.rst | 30 +++++++++++++ docs/Users_Guide/wrappers.rst | 45 +++++++++++++++++++ .../point_stat/test_point_stat_wrapper.py | 32 +++++++++++++ metplus/wrappers/point_stat_wrapper.py | 24 ++++++++++ parm/met_config/PointStatConfig_wrapped | 19 +++----- .../met_tool_wrapper/PointStat/PointStat.conf | 8 ++++ 6 files changed, 145 insertions(+), 13 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 222e2e0b5c..73cf60a695 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -6639,3 +6639,33 @@ METplus Configuration Glossary Specify the value for 'output_flag.mpr' in the MET configuration file for PointStat. | *Used by:* PointStat + + POINT_STAT_INTERP_VLD_THRESH + Specify the value for 'interp.vld_thresh' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_INTERP_SHAPE + Specify the value for 'interp.shape' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_INTERP_TYPE_METHOD + Specify the value for 'interp.type.method' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_INTERP_TYPE_WIDTH + Specify the value for 'interp.type.width' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD + Specify the value for 'climo_mean.time_interp_method' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD + Specify the value for 'climo_stdev.time_interp_method' in the MET configuration file for PointStat. + + | *Used by:* PointStat diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 9f1bb5a815..7821d0c103 100755 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -3022,6 +3022,12 @@ Configuration | :term:`POINT_STAT_OUTPUT_FLAG_RPS` | :term:`POINT_STAT_OUTPUT_FLAG_ECLV` | :term:`POINT_STAT_OUTPUT_FLAG_MPR` +| :term:`POINT_STAT_INTERP_VLD_THRESH` +| :term:`POINT_STAT_INTERP_SHAPE` +| :term:`POINT_STAT_INTERP_TYPE_METHOD` +| :term:`POINT_STAT_INTERP_TYPE_WIDTH` +| :term:`POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD` +| :term:`POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD` | :term:`FCST_POINT_STAT_WINDOW_BEGIN` (optional) | :term:`FCST_POINT_STAT_WINDOW_END` (optional) | :term:`OBS_POINT_STAT_WINDOW_BEGIN` (optional) @@ -3327,6 +3333,45 @@ Below the file contents are descriptions of each environment variable referenced * - :term:`POINT_STAT_OUTPUT_FLAG_MPR` - output_flag.mpr +**${METPLUS_INTERP_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT_STAT_INTERP_VLD_THRESH` + - interp.vld_thresh + * - :term:`POINT_STAT_INTERP_SHAPE` + - interp.shape + * - :term:`POINT_STAT_INTERP_TYPE_METHOD` + - interp.type.method + * - :term:`POINT_STAT_INTERP_TYPE_WIDTH` + - interp.type.width + +**${METPLUS_CLIMO_MEAN_TIME_INTERP_METHOD}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD` + - climo_mean.time_interp_method + +**${METPLUS_CLIMO_STDEV_TIME_INTERP_METHOD}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD` + - climo_stdev.time_interp_method + .. _py_embed_ingest_wrapper: diff --git a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py index fd6dcf52ca..7961e48d59 100755 --- a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py +++ b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py @@ -236,6 +236,38 @@ def point_stat_wrapper(metplus_config): { 'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {fho = BOTH;ctc = BOTH;cts = BOTH;mctc = BOTH;mcts = BOTH;cnt = BOTH;sl1l2 = BOTH;sal1l2 = BOTH;vl1l2 = BOTH;val1l2 = BOTH;vcnt = BOTH;pct = BOTH;pstd = BOTH;pjc = BOTH;prc = BOTH;ecnt = BOTH;rps = BOTH;eclv = BOTH;mpr = BOTH;}'}), + ({'POINT_STAT_INTERP_VLD_THRESH': '0.5', }, + {'METPLUS_INTERP_DICT': 'interp = {vld_thresh = 0.5;}'}), + + ({'POINT_STAT_INTERP_SHAPE': 'SQUARE', }, + {'METPLUS_INTERP_DICT': 'interp = {shape = SQUARE;}'}), + + ({'POINT_STAT_INTERP_TYPE_METHOD': 'BILIN', }, + {'METPLUS_INTERP_DICT': 'interp = {type = {method = BILIN;}}'}), + + ({'POINT_STAT_INTERP_TYPE_WIDTH': '2', }, + {'METPLUS_INTERP_DICT': 'interp = {type = {width = 2;}}'}), + + ({ + 'POINT_STAT_INTERP_VLD_THRESH': '0.5', + 'POINT_STAT_INTERP_SHAPE': 'SQUARE', + 'POINT_STAT_INTERP_TYPE_METHOD': 'BILIN', + 'POINT_STAT_INTERP_TYPE_WIDTH': '2', + }, + { + 'METPLUS_INTERP_DICT': ('interp = {' + 'vld_thresh = 0.5;shape = SQUARE;' + 'type = {method = BILIN;width = 2;}}')}), + + ({'POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD': 'NEAREST', }, + {'METPLUS_CLIMO_MEAN_TIME_INTERP_METHOD': ( + 'time_interp_method = NEAREST;' + )}), + + ({'POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD': 'NEAREST', }, + {'METPLUS_CLIMO_STDEV_TIME_INTERP_METHOD': ( + 'time_interp_method = NEAREST;' + )}), ] ) def test_point_stat_all_fields(metplus_config, config_overrides, diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index 46b302149c..dbe0d8b68b 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -37,6 +37,9 @@ class PointStatWrapper(CompareGriddedWrapper): 'METPLUS_CLIMO_CDF_DICT', 'METPLUS_OBS_QUALITY', 'METPLUS_OUTPUT_FLAG_DICT', + 'METPLUS_INTERP_DICT', + 'METPLUS_CLIMO_MEAN_TIME_INTERP_METHOD', + 'METPLUS_CLIMO_STDEV_TIME_INTERP_METHOD', ] OUTPUT_FLAGS = ['fho', @@ -193,6 +196,27 @@ def create_c_dict(self): self.handle_flags('output') + self.handle_interp_dict() + + self.add_met_config( + name='time_interp_method', + data_type='string', + env_var_name='CLIMO_MEAN_TIME_INTERP_METHOD', + metplus_configs=['POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD'], + extra_args={'remove_quotes': True, + 'uppercase': True, + }, + ) + self.add_met_config( + name='time_interp_method', + data_type='string', + env_var_name='CLIMO_STDEV_TIME_INTERP_METHOD', + metplus_configs=['POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD'], + extra_args={'remove_quotes': True, + 'uppercase': True, + }, + ) + if not c_dict['FCST_INPUT_TEMPLATE']: self.log_error('Must set FCST_POINT_STAT_INPUT_TEMPLATE ' 'in config file') diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index 4a10bc06be..9263e3468c 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -96,7 +96,8 @@ climo_mean = { shape = SQUARE; } - time_interp_method = NEAREST; + //time_interp_method = + ${METPLUS_CLIMO_MEAN_TIME_INTERP_METHOD} day_interval = 31; hour_interval = 6; } @@ -113,7 +114,8 @@ climo_stdev = { shape = SQUARE; } - time_interp_method = DW_MEAN; + //time_interp_method = + ${METPLUS_CLIMO_STDEV_TIME_INTERP_METHOD} day_interval = 31; hour_interval = 6; } @@ -164,17 +166,8 @@ boot = { // // Interpolation methods // -interp = { - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = BILIN; - width = 2; - } - ]; -} +//interp = { +${METPLUS_INTERP_DICT} //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf index 9905255121..187fe99345 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf @@ -52,6 +52,14 @@ POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped #POINT_STAT_OBS_QUALITY = 1, 2, 3 +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST +#POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD = + +#POINT_STAT_INTERP_VLD_THRESH = +#POINT_STAT_INTERP_SHAPE = +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + #POINT_STAT_OUTPUT_FLAG_FHO = #POINT_STAT_OUTPUT_FLAG_CTC = #POINT_STAT_OUTPUT_FLAG_CTS = From a415a4e26b25c4dc90d7c4c4360f0fa5ad8b1c5e Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 22 Apr 2021 13:29:56 -0600 Subject: [PATCH 41/42] added old defaults from wrapped config, ci-run-diff --- .../met_tool_wrapper/PointStat/PointStat_once_per_field.conf | 5 +++++ .../PointStat/PointStat_python_embedding.conf | 5 +++++ ...intStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf | 4 ++++ .../PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf | 4 ++++ ...UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf | 5 +++++ 5 files changed, 23 insertions(+) diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat_once_per_field.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat_once_per_field.conf index 4e43fe21ba..fb56e2f52a 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat_once_per_field.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat_once_per_field.conf @@ -50,6 +50,11 @@ LOOP_ORDER = processes # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST + +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat_python_embedding.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat_python_embedding.conf index 9146e6802d..7e07e8624f 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat_python_embedding.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat_python_embedding.conf @@ -50,6 +50,11 @@ LOOP_ORDER = processes # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST + +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT diff --git a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf index 53bfb64262..14ef8c0ab6 100644 --- a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf +++ b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf @@ -32,6 +32,10 @@ PB2NC_SKIP_IF_OUTPUT_EXISTS = True PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT diff --git a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf index c6a3b12a53..770939bf73 100644 --- a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf +++ b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf @@ -33,6 +33,10 @@ LEAD_SEQ = 0 OBS_WINDOW_BEGIN = -2700 OBS_WINDOW_END = 2700 +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf b/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf index 5ae3dd5970..d524045343 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf @@ -77,6 +77,11 @@ BOTH_VAR1_LEVELS = P925-950, P850-800, P700-650 POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped +POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST + +POINT_STAT_INTERP_TYPE_METHOD = BILIN +POINT_STAT_INTERP_TYPE_WIDTH = 2 + POINT_STAT_OUTPUT_FLAG_SL1L2 = STAT POINT_STAT_OUTPUT_FLAG_VL1L2 = STAT POINT_STAT_OUTPUT_FLAG_FHO = BOTH From 63e4233c04b883fcbab5ea976e0a4a8904b265e6 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 22 Apr 2021 13:34:47 -0600 Subject: [PATCH 42/42] specify version of cartopy to avoid failure --- ci/jobs/python_requirements/get_cartopy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jobs/python_requirements/get_cartopy.sh b/ci/jobs/python_requirements/get_cartopy.sh index 2c66df9fdd..e3ccc8cd18 100755 --- a/ci/jobs/python_requirements/get_cartopy.sh +++ b/ci/jobs/python_requirements/get_cartopy.sh @@ -15,7 +15,7 @@ yum -y install geos yum -y install geos-devel pip3 install --upgrade cython numpy pyshp six pip3 install shapely --no-binary shapely -pip3 install cartopy +pip3 install cartopy==0.18.0 #some cartopy functionality fails without scipy pip3 install scipy