Update Develop-ref after #1930, #1927, dtcenter/MET#2335 Attempt 2 (#…

…1936)

Co-authored-by: johnhg <johnhg@ucar.edu>
Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Co-authored-by: Mrinal Biswas <biswas@ucar.edu>
Co-authored-by: Julie Prestopnik <jpresto@ucar.edu>
Co-authored-by: Christina Kalb <kalb@mohawk.rap.ucar.edu>
Co-authored-by: Hank Fisher <fisherh@seneca.rap.ucar.edu>
Co-authored-by: jprestop <jpresto@ucar.edu>
Co-authored-by: cristianastan2 <cstan@gmu.edu>
Co-authored-by: John Halley Gotway <johnhg@ucar.edu>
Co-authored-by: bikegeek <minnawin@ucar.edu>
Co-authored-by: Lisa Goodrich <lisag@ucar.edu>
Co-authored-by: Julie Prestopnik <jpresto@seneca.rap.ucar.edu>
Co-authored-by: Hank Fisher <fisherh@ucar.edu>
Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com>
Co-authored-by: j-opatz <jopatz@ucar.edu>
Co-authored-by: Christina Kalb <kalb@ucar.edu>
Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com>
Co-authored-by: Howard Soh <hsoh@seneca.rap.ucar.edu>
Co-authored-by: Molly Smith <molly.b.smith@noaa.gov>
Co-authored-by: hsoh-u <hsoh@ucar.edu>
Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com>
Co-authored-by: Will Mayfield <59745143+willmayfield@users.noreply.github.com>

internal/tests/pytests/wrappers/command_builder/test_command_builder.py

@@ -6,7 +6,7 @@
 import datetime
 
 from metplus.wrappers.command_builder import CommandBuilder
-from metplus.util import ti_calculate
+from metplus.util import ti_calculate, add_field_info_to_time_info
 
 
 def get_data_dir(config):
@@ -27,9 +27,9 @@ def test_find_data_no_dated(metplus_config, data_type):
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['fcst_level'] = "6"
-    v['obs_level'] = "6"
+    var_info = {}
+    var_info['fcst_level'] = "6"
+    var_info['obs_level'] = "6"
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000",'%Y%m%d%H%M')
     task_info['lead'] = 0
@@ -39,7 +39,8 @@ def test_find_data_no_dated(metplus_config, data_type):
     pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 3600
     pcw.c_dict[f'{data_type}INPUT_DIR'] = get_data_dir(pcw.config)
     pcw.c_dict[f'{data_type}INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}"
-    obs_file = pcw.find_data(time_info, v, data_type)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file = pcw.find_data(time_info, data_type)
     assert obs_file == pcw.c_dict[f'{data_type}INPUT_DIR']+'/20180201_0045'
 
 
@@ -67,7 +68,7 @@ def test_find_data_not_a_path(metplus_config, data_type):
     pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 0
     pcw.c_dict[f'{data_type}INPUT_DIR'] = ''
     pcw.c_dict[f'{data_type}INPUT_TEMPLATE'] = 'G003'
-    obs_file = pcw.find_data(time_info, var_info=None, data_type=data_type)
+    obs_file = pcw.find_data(time_info, data_type=data_type)
     assert obs_file == 'G003'
 
 
@@ -76,8 +77,8 @@ def test_find_obs_no_dated(metplus_config):
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['obs_level'] = "6"
+    var_info = {}
+    var_info['obs_level'] = "6"
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
     task_info['lead'] = 0
@@ -87,7 +88,8 @@ def test_find_obs_no_dated(metplus_config):
     pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
     pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config)
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}"
-    obs_file = pcw.find_obs(time_info, v)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file = pcw.find_obs(time_info)
     assert obs_file == pcw.c_dict['OBS_INPUT_DIR'] + '/20180201_0045'
 
 
@@ -96,8 +98,8 @@ def test_find_obs_dated(metplus_config):
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['obs_level'] = "6"
+    var_info = {}
+    var_info['obs_level'] = "6"
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
     task_info['lead'] = 0
@@ -107,7 +109,8 @@ def test_find_obs_dated(metplus_config):
     pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
     pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config)
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}'
-    obs_file = pcw.find_obs(time_info, v)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file = pcw.find_obs(time_info)
     assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180201/20180201_0013'
 
 
@@ -126,8 +129,8 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds)
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['obs_level'] = "6"
+    var_info = {}
+    var_info['obs_level'] = "6"
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("2020020112", '%Y%m%d%H')
     task_info['lead'] = 0
@@ -136,7 +139,8 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds)
     pcw.c_dict['OFFSETS'] = offsets
     pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config)
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}"
-    obs_file, time_info = pcw.find_obs_offset(time_info, v)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file, time_info = pcw.find_obs_offset(time_info)
 
     print(f"OBSFILE: {obs_file}")
     print(f"EXPECTED FILE: {expected_file}")
@@ -153,8 +157,8 @@ def test_find_obs_dated_previous_day(metplus_config):
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['obs_level'] = "6"
+    var_info = {}
+    var_info['obs_level'] = "6"
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
     task_info['lead'] = 0
@@ -164,7 +168,8 @@ def test_find_obs_dated_previous_day(metplus_config):
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}'
     pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600
     pcw.c_dict['OBS_FILE_WINDOW_END'] = 0
-    obs_file = pcw.find_obs(time_info, v)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file = pcw.find_obs(time_info)
     assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180131/20180131_2345'
 
 
@@ -173,8 +178,9 @@ def test_find_obs_dated_next_day(metplus_config):
     config = metplus_config
 
     pcw = CommandBuilder(config)
-    v = {}
-    v['obs_level'] = "6"
+    var_info = {
+        'obs_level': "6"
+    }
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802012345", '%Y%m%d%H%M')
     task_info['lead'] = 0
@@ -184,7 +190,8 @@ def test_find_obs_dated_next_day(metplus_config):
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}'
     pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = 0
     pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
-    obs_file = pcw.find_obs(time_info, v)
+    add_field_info_to_time_info(time_info, var_info)
+    obs_file = pcw.find_obs(time_info)
     assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180202/20180202_0013'
 
 

internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py

@@ -59,6 +59,49 @@ def set_minimum_config_settings(config, set_fields=True):
         config.set('config', 'OBS_VAR1_LEVELS', obs_level)
 
 
+@pytest.mark.parametrize(
+    'config_overrides, expected_filename', [
+        # 0 - set forecast level
+        ({'FCST_VAR1_NAME': 'fcst_file',
+          'FCST_VAR1_LEVELS': 'A06',
+          'OBS_VAR1_NAME': 'obs_file',
+          'OBS_VAR1_LEVELS': 'A06',
+          'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE': '{fcst_name}_A{level?fmt=%3H}',
+          },
+         f'{fcst_dir}/fcst_file_A006'),
+        # 1 - don't set forecast level
+        ({'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE': 'fcst_file_A{level?fmt=%3H}'},
+         f'{fcst_dir}/fcst_file_A000'),
+    ]
+)
+@pytest.mark.wrapper_c
+def test_ensemble_stat_level_in_template(metplus_config, config_overrides,
+                                         expected_filename):
+
+    config = metplus_config
+
+    set_minimum_config_settings(config, set_fields=False)
+
+    # set config variable overrides
+    for key, value in config_overrides.items():
+        config.set('config', key, value)
+
+    wrapper = EnsembleStatWrapper(config)
+    assert wrapper.isOK
+
+    file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR')
+    file_list_file = f"{file_list_dir}/20050807000000_12_ensemble_stat.txt"
+    if os.path.exists(file_list_file):
+        os.remove(file_list_file)
+
+    wrapper.run_all_times()
+    assert os.path.exists(file_list_file)
+    with open(file_list_file, 'r') as file_handle:
+        filenames = file_handle.read().splitlines()[1:]
+    assert len(filenames) == 1
+    assert filenames[0] == expected_filename
+
+
 @pytest.mark.parametrize(
     'config_overrides, env_var_values', [
         # 0 : no ens, 1 fcst, 1 obs
@@ -577,8 +620,7 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides,
 
     app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
     verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
-    file_list_dir = os.path.join(wrapper.config.getdir('STAGING_DIR'),
-                                 'file_lists')
+    file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR')
     config_file = wrapper.c_dict.get('CONFIG_FILE')
     out_dir = wrapper.c_dict.get('OUTPUT_DIR')
     expected_cmds = [(f"{app_path} {verbosity} "
@@ -655,8 +697,7 @@ def test_ensemble_stat_fill_missing(metplus_config, config_overrides,
 
     wrapper = EnsembleStatWrapper(config)
 
-    file_list_file = os.path.join(wrapper.config.getdir('STAGING_DIR'),
-                                  'file_lists',
+    file_list_file = os.path.join(wrapper.config.getdir('FILE_LISTS_DIR'),
                                   '20050807000000_12_ensemble_stat.txt')
     if os.path.exists(file_list_file):
         os.remove(file_list_file)

metplus/util/string_manip.py

@@ -239,6 +239,9 @@ def format_thresh(thresh_str):
     @returns string of comma-separated list of the threshold(s) with
      letter format, i.e. gt3,le5.5,eq7
     """
+    if isinstance(thresh_str, list):
+        return format_thresh(','.join(thresh_str))
+
     formatted_thresh_list = []
     # separate thresholds by comma and strip off whitespace around values
     thresh_list = [thresh.strip() for thresh in thresh_str.split(',')]

metplus/util/time_util.py

@@ -13,6 +13,8 @@
 from dateutil.relativedelta import relativedelta
 import re
 
+from .string_manip import split_level, format_thresh
+
 '''!@namespace TimeInfo
 @brief Utility to handle timing in METplus wrappers
 @code{.sh}
@@ -332,20 +334,10 @@ def _format_time_list(string_value, get_met_format, sort_list=True):
 
 
 def ti_calculate(input_dict_preserve):
-    out_dict = {}
+    # copy input dictionary so valid or init can be removed to recalculate it
+    # without modifying the input to the function
     input_dict = input_dict_preserve.copy()
-
-    KEYS_TO_COPY = ['custom', 'instance']
-
-    # set output dictionary to input items
-    if 'now' in input_dict.keys():
-        out_dict['now'] = input_dict['now']
-        out_dict['today'] = out_dict['now'].strftime('%Y%m%d')
-
-    # copy over values of some keys if it is set in input dictionary
-    for key in KEYS_TO_COPY:
-        if key in input_dict.keys():
-            out_dict[key] = input_dict[key]
+    out_dict = input_dict
 
     # read in input dictionary items and compute missing items
     # valid inputs: valid, init, lead, offset
@@ -381,7 +373,6 @@ def ti_calculate(input_dict_preserve):
     else:
         out_dict['lead'] = relativedelta(seconds=0)
 
-
     # set offset to 0 if not specified
     if 'offset_hours' in input_dict.keys():
         out_dict['offset'] = datetime.timedelta(hours=input_dict['offset_hours'])
@@ -390,7 +381,6 @@ def ti_calculate(input_dict_preserve):
     else:
         out_dict['offset'] = datetime.timedelta(seconds=0)
 
-
     # if init and valid are set, check which was set first via loop_by
     # remove the other to recalculate
     if 'init' in input_dict.keys() and 'valid' in input_dict.keys():
@@ -509,3 +499,25 @@ def add_to_time_input(time_input, clock_time=None, instance=None, custom=None):
     # otherwise leave it unset so it can be set within the wrapper
     if custom:
         time_input['custom'] = custom
+
+
+def add_field_info_to_time_info(time_info, var_info):
+    """!Add field information from var_info to the time_info dictionary to use
+    in string template substitution. Sets new items in time_info.
+
+    @param time_info dictionary containing time information to substitute
+    filename template tags
+    @param var_info dictionary containing information for the fields to process
+    """
+    if var_info is None:
+        return
+
+    for key, value in var_info.items():
+        # skip index and extra field info
+        if key == 'index' or key.endswith('extra'):
+            continue
+
+        if key.endswith('thresh'):
+            value = format_thresh(value)
+
+        time_info[key] = value

metplus/wrappers/ascii2nc_wrapper.py

@@ -297,7 +297,7 @@ def find_input_files(self, time_info):
             return self.infiles
 
         # get list of files even if only one is found (return_list=True)
-        obs_path = self.find_obs(time_info, var_info=None, return_list=True)
+        obs_path = self.find_obs(time_info, return_list=True)
         if obs_path is None:
             return None