From f18831d74168a339ea47e72d2c81d02206629895 Mon Sep 17 00:00:00 2001 From: CPKalb Date: Thu, 12 Apr 2018 13:23:53 -0600 Subject: [PATCH 01/14] Update grid2grid_hwt_env.conf --- parm/use_cases/hwt/grid2grid_hwt_env.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/use_cases/hwt/grid2grid_hwt_env.conf b/parm/use_cases/hwt/grid2grid_hwt_env.conf index 4e575d0111..fd7261932d 100644 --- a/parm/use_cases/hwt/grid2grid_hwt_env.conf +++ b/parm/use_cases/hwt/grid2grid_hwt_env.conf @@ -51,7 +51,7 @@ OBS_REFC_0_THRESH = 25, 30, 35, 40, 45, 50 [filename_templates] # HRRRe -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[FCST_INPUT_TEMPLATE_IN]} +FCST_GRID_STAT_INPUT_TEMPLATE = $FCST_INPUT_TEMPLATE_IN # MRMS REFC -OBS_GRID_STAT_INPUT_TEMPLATE = {ENV[OBS_INPUT_TEMPLATE_IN]} +OBS_GRID_STAT_INPUT_TEMPLATE = $OBS_INPUT_TEMPLATE_IN From 1dc0a636ece5e10c3ecd45b64ea2fb8e13546dbd Mon Sep 17 00:00:00 2001 From: CPKalb Date: Thu, 12 Apr 2018 15:03:13 -0600 Subject: [PATCH 02/14] Rename grid_stat_wrapper_hwt.py to grid_stat_wrapper_hwt_mult.py --- ush/{grid_stat_wrapper_hwt.py => grid_stat_wrapper_hwt_mult.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ush/{grid_stat_wrapper_hwt.py => grid_stat_wrapper_hwt_mult.py} (100%) diff --git a/ush/grid_stat_wrapper_hwt.py b/ush/grid_stat_wrapper_hwt_mult.py similarity index 100% rename from ush/grid_stat_wrapper_hwt.py rename to ush/grid_stat_wrapper_hwt_mult.py From 2f62f2adc7cb2ceaed9a76299638d0f754e748db Mon Sep 17 00:00:00 2001 From: CPKalb Date: Thu, 12 Apr 2018 15:11:01 -0600 Subject: [PATCH 03/14] Rename ush/grid_stat_wrapper_hwt_mult.py to parm/use_cases/hwt/grid_stat_wrapper_hwt_mult.py --- {ush => parm/use_cases/hwt}/grid_stat_wrapper_hwt_mult.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {ush => parm/use_cases/hwt}/grid_stat_wrapper_hwt_mult.py (100%) diff --git a/ush/grid_stat_wrapper_hwt_mult.py b/parm/use_cases/hwt/grid_stat_wrapper_hwt_mult.py similarity index 100% rename from ush/grid_stat_wrapper_hwt_mult.py rename to parm/use_cases/hwt/grid_stat_wrapper_hwt_mult.py From 55bde9093d16d8c02ab08e6d65e0c0d1f26a59a8 Mon Sep 17 00:00:00 2001 From: Minna Win Date: Thu, 12 Apr 2018 21:29:54 +0000 Subject: [PATCH 04/14] Regression test for produtil, using pytest-regtest. --- .../produtil/test_produtil_regression.py | 156 ++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 internal_tests/pytests/produtil/test_produtil_regression.py diff --git a/internal_tests/pytests/produtil/test_produtil_regression.py b/internal_tests/pytests/produtil/test_produtil_regression.py new file mode 100644 index 0000000000..9666c0dffc --- /dev/null +++ b/internal_tests/pytests/produtil/test_produtil_regression.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +from __future__ import print_function +import os +import subprocess +#import produtil.setup +import produtil +import sys +import logging +import pytest +import config_metplus +import config_launcher as launcher +import met_util as util + + +# +# These are tests (not necessarily unit tests) for the +# MET Point-Stat Wrapper, PointStatWrapper.py +# NOTE: This test requires pytest, which is NOT part of the standard Python +# library. +# These tests require one configuration file in addition to the three +# required METplus configuration files: point_stat_test.conf. This contains +# the information necessary for running all the tests. Each test can be +# customized to replace various settings if needed. +# + +# +# -----------Mandatory----------- +# configuration and fixture to support METplus configuration files beyond +# the metplus_data, metplus_system, and metplus_runtime conf files. +# + + +# Add a test configuration +# def pytest_addoption(parser): +# parser.addoption("-c", action="store", help=" -c ") +# +# +# # @pytest.fixture +# def cmdopt(request): +# return request.config.getoption("-c") + + +# ------------------------ + +def get_config_obj(): + """! Create the configuration object that is used by all tests""" + file_list = ["METplus/internal_tests/pytests/produtil"] + config_obj = config_metplus.setup(file_list[0]) + + return config_obj + + +def test_getstr_ok(regtest): + """! Test that the expected string is retrieved via produtil's getstr + method + """ + conf_obj = get_config_obj() + str_value = conf_obj.getstr('config', 'STRING_VALUE') + expected_str_value = "someStringValue!#@$%" + # print(str_value, file=regtest) + regtest.write("done") + + +def test_getint_ok(regtest): + """! Test that the expected int in the produtil_test.conf file has been + retrieved correctly. + """ + conf_obj = get_config_obj() + expected_int_value = int(2908887) + int_value = conf_obj.getint('config', 'INT_VALUE') + # print(int_value, file=regtest) + regtest.write("done") + + +def test_getraw_ok(regtest): + """! Test that the raw value in the produtil_test.conf file has been + retrieved correctly. + """ + conf_obj = get_config_obj() + expected_raw = 'GRIB_lvl_type = 100' + raw_value = conf_obj.getraw('config', 'RAW_VALUE') + # print(raw_value, file=regtest) + # regtest.write("done") + + +def test_getdir_ok(regtest): + """! Test that the directory in the produtil_test.conf file has been + correctly retrieved. + """ + conf_obj = get_config_obj() + expected_dir = "/tmp/some_dir" + dir_retrieved = conf_obj.getdir('DIR_VALUE') + # print(dir_retrieved, file=regtest) + + +def test_getdir_compound_ok(regtest): + """! Test that directories created from other directories, ie. + BASE_DIR = /base/dir + SPECIFIC_DIR = {BASE_DIR}/specific/dir + + correctly returns the directory path for SPECIFIC_DIR + """ + expected_specific_dir = "/tmp/specific_place" + conf_obj = get_config_obj() + specific_dir = conf_obj.getdir('SPECIFIC_DIR') + print(specific_dir, file=regtest) + + +def test_no_value_as_string(regtest): + """! Tests that a key with no value returns an empty string.""" + + conf_obj = get_config_obj() + expected_unassigned = '' + unassigned = conf_obj.getstr('config', 'UNASSIGNED_VALUE') + # print(unassigned, file=regtest) + + +def test_no_value_as_list(regtest): + """! Tests that a key with no list of strings returns an empty list.""" + + conf_obj = get_config_obj() + expected_unassigned = [] + unassigned = util.getlist(conf_obj.getstr('config', 'UNASSIGNED_VALUE')) + assert unassigned == expected_unassigned + # print(unassigned, file=regtest) + + +def test_new_lines_in_conf(regtest): + """! Test that any newlines in the configuration file are handled + properly + """ + + conf_obj = get_config_obj() + expected_string = \ + "very long line requiring newline character to be tested 12345\n67890 end of the line." + long_line = conf_obj.getstr('config', 'NEW_LINES') + assert long_line == expected_string + # print(long_line, file=regtest) + + +def test_get_exe_ok(regtest): + """! Test that executables are correctly retrieved.""" + conf_obj = get_config_obj() + expected_exe = '/usr/local/bin/wgrib2' + executable = conf_obj.getexe('WGRIB2') + assert executable == expected_exe + # print(executable, file=regtest) + + +def test_get_bool(regtest): + """! Test that boolean values are correctly retrieved.""" + conf_obj = get_config_obj() + bool_val = conf_obj.getbool('config', 'BOOL_VALUE') + assert bool_val is True + # print(bool_val, file=regtest) + From 7635579a417399e46e533214c1ab9ec38f240399 Mon Sep 17 00:00:00 2001 From: CPKalb Date: Thu, 12 Apr 2018 15:45:36 -0600 Subject: [PATCH 05/14] Create grid_stat_wrapper.py For HWT, wrapper with ability to read environmental parameters in the filename templates --- parm/use_cases/hwt/grid_stat_wrapper.py | 287 ++++++++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 parm/use_cases/hwt/grid_stat_wrapper.py diff --git a/parm/use_cases/hwt/grid_stat_wrapper.py b/parm/use_cases/hwt/grid_stat_wrapper.py new file mode 100644 index 0000000000..36c824555f --- /dev/null +++ b/parm/use_cases/hwt/grid_stat_wrapper.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python + +''' +Program Name: grid_stat_wrapper.py +Contact(s): George McCabe +Abstract: +History Log: Initial version +Usage: +Parameters: None +Input Files: +Output Files: +Condition codes: 0 for success, 1 for failure +''' + +from __future__ import (print_function, division) + +import logging +import os +import sys +import met_util as util +import re +import csv +import subprocess +from command_builder import CommandBuilder +from task_info import TaskInfo +import string_template_substitution as sts + + +class GridStatWrapper(CommandBuilder): + + def __init__(self, p, logger): + super(GridStatWrapper, self).__init__(p, logger) + met_install_dir = p.getdir('MET_INSTALL_DIR') + self.app_path = os.path.join(met_install_dir, 'bin/grid_stat') + self.app_name = os.path.basename(self.app_path) + + def set_output_dir(self, outdir): + self.outdir = "-outdir "+outdir + + def get_command(self): + if self.app_path is None: + self.logger.error(self.app_name + ": No app path specified. \ + You must use a subclass") + return None + + cmd = self.app_path + " " + for a in self.args: + cmd += a + " " + + if len(self.infiles) == 0: + self.logger.error(self.app_name+": No input filenames specified") + return None + + for f in self.infiles: + cmd += f + " " + + if self.param != "": + cmd += self.param + " " + + if self.outdir == "": + self.logger.error(self.app_name+": No output directory specified") + return None + + cmd += self.outdir + return cmd + + def find_model(self, lead, init_time, level): + model_dir = self.p.getstr('config', 'FCST_GRID_STAT_INPUT_DIR') + max_forecast = self.p.getint('config', 'FCST_MAX_FORECAST') + init_interval = self.p.getint('config', 'FCST_INIT_INTERVAL') + lead_check = lead + time_check = init_time + time_offset = 0 + found = False + while lead_check <= max_forecast: + model_template = os.path.expandvars(self.p.getraw('filename_templates', + 'FCST_GRID_STAT_INPUT_TEMPLATE')) + # split by - to handle a level that is a range, such as 0-10 + model_ss = sts.StringSub(self.logger, model_template, + init=time_check, + lead=str(lead_check).zfill(2), + level=str(level.split('-')[0]).zfill(2)) + model_file = model_ss.doStringSub() + model_path = os.path.join(model_dir, model_file) + if os.path.exists(model_path): + found = True + break + elif os.path.exists(model_path+".gz"): + with gzip.open(model_path+".gz", 'rb') as infile: + with open(model_path, 'wb') as outfile: + outfile.write(infile.read()) + infile.close() + outfile.close() + # TODO: change model_path to path without gz + # set found to true and break + time_check = util.shift_time(time_check, -init_interval) + lead_check = lead_check + init_interval + + if found: + return model_path + else: + return '' + + def run_at_time(self, init_time, valid_time): + task_info = TaskInfo() + task_info.init_time = init_time + task_info.valid_time = valid_time + var_list = util.parse_var_list(self.p) + + lead_seq = util.getlistint(self.p.getstr('config', 'LEAD_SEQ')) + for lead in lead_seq: + task_info.lead = lead + for var_info in var_list: + self.run_at_time_once(task_info, var_info) + + + def run_at_time_once(self, ti, v): + valid_time = ti.getValidTime() + init_time = ti.getInitTime() + grid_stat_base_dir = self.p.getstr('config', 'GRID_STAT_OUT_DIR') + if self.p.getbool('config', 'LOOP_BY_INIT'): + grid_stat_out_dir = os.path.join(grid_stat_base_dir, + init_time, "grid_stat") + else: + grid_stat_out_dir = os.path.join(grid_stat_base_dir, + valid_time, "grid_stat") + fcst_level = v.fcst_level + fcst_level_type = "" + if(fcst_level[0].isalpha()): + fcst_level_type = fcst_level[0] + fcst_level = fcst_level[1:] + obs_level = v.obs_level + obs_level_type = "" + if(obs_level[0].isalpha()): + obs_level_type = obs_level[0] + obs_level = obs_level[1:] + model_type = self.p.getstr('config', 'MODEL_TYPE') + obs_dir = self.p.getstr('config', 'OBS_GRID_STAT_INPUT_DIR') + obs_template = self.p.getraw('filename_templates', + 'OBS_GRID_STAT_INPUT_TEMPLATE') + model_dir = self.p.getstr('config', 'FCST_GRID_STAT_INPUT_DIR') + config_dir = self.p.getstr('config', 'CONFIG_DIR') + + ymd_v = valid_time[0:8] + if not os.path.exists(grid_stat_out_dir): + os.makedirs(grid_stat_out_dir) + + # get model to compare + model_path = self.find_model(ti.lead, init_time, fcst_level) + + if model_path == "": + print("ERROR: COULD NOT FIND FILE IN "+model_dir) + return + self.add_input_file(model_path) + # TODO: Handle range of levels + obsSts = sts.StringSub(self.logger, + obs_template, + valid=valid_time, + init=init_time, + level=str(obs_level.split('-')[0]).zfill(2)) + obs_file = obsSts.doStringSub() + + obs_path = os.path.join(obs_dir, obs_file) + self.add_input_file(obs_path) + self.set_param_file(self.p.getstr('config', 'GRID_STAT_CONFIG')) + self.set_output_dir(grid_stat_out_dir) + + # set up environment variables for each grid_stat run + # get fcst and obs thresh parameters + # verify they are the same size + + fcst_str = "FCST_"+v.fcst_name+"_"+fcst_level+"_THRESH" + obs_str = "OBS_"+v.obs_name+"_"+obs_level+"_THRESH" + fcst_cat_thresh = "" + obs_cat_thresh = "" + fcst_threshs = [] + obs_threshs = [] + + if self.p.has_option('config', fcst_str): + fcst_threshs = util.getlistfloat(self.p.getstr('config', fcst_str)) + fcst_cat_thresh = "cat_thresh=[ " + for fcst_thresh in fcst_threshs: + fcst_cat_thresh += "gt"+str(fcst_thresh)+", " + fcst_cat_thresh = fcst_cat_thresh[0:-2]+" ];" + + if self.p.has_option('config', obs_str): + obs_threshs = util.getlistfloat(self.p.getstr('config', obs_str)) + obs_cat_thresh = "cat_thresh=[ " + for obs_thresh in obs_threshs: + obs_cat_thresh += "gt"+str(obs_thresh)+", " + obs_cat_thresh = obs_cat_thresh[0:-2]+" ];" + + if len(fcst_threshs) != len(obs_threshs): + self.logger.error("run_example: Number of forecast and "\ + "observation thresholds must be the same") + exit(1) + + # TODO: Allow NetCDF level with more than 2 dimensions i.e. (1,*,*) + # TODO: Need to check data type for PROB fcst? non PROB obs? + + fcst_field = "" + obs_field = "" +# TODO: change PROB mode to put all cat thresh values in 1 item + if self.p.getbool('config', 'FCST_IS_PROB'): + for fcst_thresh in fcst_threshs: + fcst_field += "{ name=\"PROB\"; level=\""+fcst_level_type + \ + fcst_level.zfill(2) + "\"; prob={ name=\"" + \ + v.fcst_name + \ + "\"; thresh_lo="+str(fcst_thresh)+"; } }," + for obs_thresh in obs_threshs: + obs_field += "{ name=\""+v.obs_name+"_"+obs_level.zfill(2) + \ + "\"; level=\"(*,*)\"; cat_thresh=[ gt" + \ + str(obs_thresh)+" ]; }," + else: +# data_type = self.p.getstr('config', 'OBS_NATIVE_DATA_TYPE') + obs_data_type = util.get_filetype(self.p, obs_path) + model_data_type = util.get_filetype(self.p, model_path) + if obs_data_type == "NETCDF": + + obs_field += "{ name=\"" + v.obs_name+"_" + obs_level.zfill(2) + \ + "\"; level=\"(*,*)\"; " + + else: + obs_field += "{ name=\""+v.obs_name + \ + "\"; level=\"["+obs_level_type + \ + obs_level.zfill(2)+"]\"; " + + if model_data_type == "NETCDF": + fcst_field += "{ name=\""+v.fcst_name+"_"+fcst_level.zfill(2) + \ + "\"; level=\"(*,*)\"; " + else: + fcst_field += "{ name=\""+v.fcst_name + \ + "\"; level=\"["+fcst_level_type + \ + fcst_level.zfill(2)+"]\"; " + + fcst_field += fcst_cat_thresh+" }," + +# obs_field += "{ name=\"" + v.obs_name+"_" + obs_level.zfill(2) + \ +# "\"; level=\"(*,*)\"; " + obs_field += obs_cat_thresh+ " }," + + # remove last comma and } to be added back after extra options + fcst_field = fcst_field[0:-2] + obs_field = obs_field[0:-2] + + fcst_field += v.fcst_extra+"}" + obs_field += v.obs_extra+"}" + + ob_type = self.p.getstr('config', "OB_TYPE") + + self.add_env_var("MODEL", model_type) + self.add_env_var("FCST_VAR", v.fcst_name) + self.add_env_var("OBS_VAR", v.obs_name) + # TODO: Change ACCUM to LEVEL in GridStatConfig_MEAN/PROB and here + self.add_env_var("ACCUM", v.fcst_level) + self.add_env_var("OBTYPE", ob_type) + self.add_env_var("CONFIG_DIR", config_dir) + self.add_env_var("FCST_FIELD", fcst_field) + self.add_env_var("OBS_FIELD", obs_field) + self.add_env_var("MET_VALID_HHMM", valid_time[4:8]) + cmd = self.get_command() + + self.logger.debug("") + self.logger.debug("ENVIRONMENT FOR NEXT COMMAND: ") + self.print_env_item("MODEL") + self.print_env_item("FCST_VAR") + self.print_env_item("OBS_VAR") + self.print_env_item("ACCUM") + self.print_env_item("OBTYPE") + self.print_env_item("CONFIG_DIR") + self.print_env_item("FCST_FIELD") + self.print_env_item("OBS_FIELD") + self.print_env_item("MET_VALID_HHMM") + self.logger.debug("") + self.logger.debug("COPYABLE ENVIRONMENT FOR NEXT COMMAND: ") + self.print_env_copy(["MODEL", "FCST_VAR", "OBS_VAR", + "ACCUM", "OBTYPE", "CONFIG_DIR", + "FCST_FIELD", "OBS_FIELD", + "MET_VALID_HHMM"]) + self.logger.debug("") + cmd = self.get_command() + if cmd is None: + print("ERROR: grid_stat could not generate command") + return + self.logger.info("") + self.build() +self.clear() From fe729d7f05f396fe1a3e5d8cc3f55a4e9cdc6b88 Mon Sep 17 00:00:00 2001 From: CPKalb Date: Thu, 12 Apr 2018 15:53:53 -0600 Subject: [PATCH 06/14] Update run_metplus_hwt.py --- parm/use_cases/hwt/run_metplus_hwt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/use_cases/hwt/run_metplus_hwt.py b/parm/use_cases/hwt/run_metplus_hwt.py index 71ef4f9b5d..14b40d125b 100644 --- a/parm/use_cases/hwt/run_metplus_hwt.py +++ b/parm/use_cases/hwt/run_metplus_hwt.py @@ -17,3 +17,5 @@ os.environ["FCST_MAX_FCST_IN"] = "48" os.environ["OUT_DIR_IN"] = "/home/christina.kalb/metout" + +os.system('./master_metplus.py -c ../parm/use_cases/hwt/grid2grid_hwt_env.conf') From 0060a5fc5339881f26269f179dc7a1d85cf6b9ea Mon Sep 17 00:00:00 2001 From: CPKalb Date: Thu, 12 Apr 2018 15:59:39 -0600 Subject: [PATCH 07/14] Update run_metplus_hwt.py --- parm/use_cases/hwt/run_metplus_hwt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/use_cases/hwt/run_metplus_hwt.py b/parm/use_cases/hwt/run_metplus_hwt.py index 14b40d125b..5eb5bb75e9 100644 --- a/parm/use_cases/hwt/run_metplus_hwt.py +++ b/parm/use_cases/hwt/run_metplus_hwt.py @@ -6,7 +6,7 @@ os.environ["OBS_INPUT_DIR_IN"] = "/raid/efp/se2018/ftp/gsd" os.environ["FCST_INPUT_TEMPLATE_IN"] = "{init?fmt=%Y%m%d%H}/hrrre01_{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.grib2" -os.environ["OBS_INPUT_TEMPLATE_IN"] = "{valid?fmt=%m}/{valid?fmt=%d}/hrrre01_{valid?fmt=%Y%m%d%H}f000.grib2" +os.environ["OBS_INPUT_TEMPLATE_IN"] = "{valid?fmt=%Y%m%d%H}/hrrre01_{valid?fmt=%Y%m%d%H}f000.grib2" os.environ["MODEL_TYPE_IN"] = "HRRRe01" os.environ["OB_TYPE_IN"] = "MRMS" From 85d9ab407bb4d231bb8085205ae5f1763ec6f159 Mon Sep 17 00:00:00 2001 From: Minna Win Date: Fri, 13 Apr 2018 18:21:15 +0000 Subject: [PATCH 08/14] Removed two tests that weren't relevant --- .../pytests/pb2nc/test_pb2nc_wrapper.py | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py b/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py index 1f9e746ef7..8666c1287b 100644 --- a/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py +++ b/internal_tests/pytests/pb2nc/test_pb2nc_wrapper.py @@ -98,12 +98,12 @@ def test_config(key, value): assert (pb_key == value) -@pytest.mark.parametrize( - 'key, value', [ - ('NC_FILE_TMPL', 'nam.t{init?fmt=%HH}z.prepbufr.tm{lead?fmt=%HH}'), - ('NC_FILE_TMPL', 'prepbufr.gdas.{valid?fmt=%Y%m%d%HH}') - ] -) +#@pytest.mark.parametrize( +# 'key, value', [ +# ('NC_FILE_TMPL', 'nam.t{init?fmt=%HH}z.prepbufr.tm{lead?fmt=%HH}'), +# ('NC_FILE_TMPL', 'prepbufr.gdas.{valid?fmt=%Y%m%d%HH}') +# ] +#) # def test_set_attribute_after_wrapper_creation(key, value): # # Test that we can change the attribute defined in the config file # pb = pb2nc_wrapper() @@ -509,12 +509,5 @@ def test_config(key, value): # assert True # # -# def test_refactor(): -# pb = pb2nc_wrapper() -# relevant = pb.get_files_by_time() -# assert True is True # -def test_run(): - pb = pb2nc_wrapper() - pb.run_all_times() - assert True + From 9527f15430387e3c6a84705965372ab23820deeb Mon Sep 17 00:00:00 2001 From: Minna Win Date: Fri, 13 Apr 2018 18:24:00 +0000 Subject: [PATCH 09/14] Forgot the TIME_METHOD setting --- .../pytests/point_stat/point_stat_test_conus_sfc.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal_tests/pytests/point_stat/point_stat_test_conus_sfc.conf b/internal_tests/pytests/point_stat/point_stat_test_conus_sfc.conf index 6a45bec3b3..442f7f7798 100644 --- a/internal_tests/pytests/point_stat/point_stat_test_conus_sfc.conf +++ b/internal_tests/pytests/point_stat/point_stat_test_conus_sfc.conf @@ -26,6 +26,10 @@ POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/{OBS_NAME} ## PROCESS_LIST. LOOP_METHOD = processes +# Time method by which to perform validation, either by init time or by valid +# time. Indicate by either BY_VALID or BY_INIT +TIME_METHOD = BY_VALID + # MET point_stat config file #POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/Mallory_PointStatConfig_conus_sfc POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_conus_sfc From 95031766cffaaa4312b1a9648a7ca88fe3679038 Mon Sep 17 00:00:00 2001 From: Minna Win Date: Fri, 13 Apr 2018 18:24:45 +0000 Subject: [PATCH 10/14] Added the missing TIME_METHOD value --- .../pytests/point_stat/point_stat_test_upper_air.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal_tests/pytests/point_stat/point_stat_test_upper_air.conf b/internal_tests/pytests/point_stat/point_stat_test_upper_air.conf index 9d69b835ed..4b46f42df2 100644 --- a/internal_tests/pytests/point_stat/point_stat_test_upper_air.conf +++ b/internal_tests/pytests/point_stat/point_stat_test_upper_air.conf @@ -27,6 +27,10 @@ POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/{MODEL_NAME} ## PROCESS_LIST. LOOP_METHOD = processes +# Time method by which to perform validation, either by init time or by valid +# time. Indicate by either BY_VALID or BY_INIT +TIME_METHOD = BY_VALID + # MET point_stat config file POINT_STAT_CONFIG_FILE = {PARM_BASE}/met_config/PointStatConfig_upper_air From c4b73c1070d2162f33e04c9906dd5c989a613ab9 Mon Sep 17 00:00:00 2001 From: Minna Win Date: Fri, 13 Apr 2018 18:31:10 +0000 Subject: [PATCH 11/14] Renamed to correctly spell the 'substitution' portion of the directory name. --- .../test_string_template_substitution.py | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 internal_tests/pytests/StringTemplateSubstitution/test_string_template_substitution.py diff --git a/internal_tests/pytests/StringTemplateSubstitution/test_string_template_substitution.py b/internal_tests/pytests/StringTemplateSubstitution/test_string_template_substitution.py new file mode 100644 index 0000000000..1ee1149d98 --- /dev/null +++ b/internal_tests/pytests/StringTemplateSubstitution/test_string_template_substitution.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import pytest +from string_template_substitution import StringSub +import logging + + +def test_cycle_hour(): + cycle_string = "00" + valid_string = "20180103" + logger = logging.getLogger("dummy") + templ = "prefix.{valid?fmt=%Y%m%d}.tm{cycle?fmt=%H}" + ss = StringSub(logger, templ, valid=valid_string, cycle=cycle_string) + expected_hours = int(0) + assert (ss.cycle_time_hours == expected_hours) + + +def test_offset_hour(): + logger = logging.getLogger("dummy") + expected_hour = int(3) + offset = "03" + templ = "prefix.{valid?fmt=%Y%m%d}.tm{offset?fmt=%H}" + ss = StringSub(logger, templ, offset=offset) + assert (ss.offset_hour == expected_hour) + + +@pytest.mark.parametrize( + 'key, value', [ + ('00', '20180103060000'), + ('03', '20180103030000'), + ('06', '20180103000000'), + ('72', '20171231060000') + ] +) +def test_calc_valid_for_prepbufr(key, value): + # Verify that the previous day is correctly calculated when + # the negative_offset_hour > cycle_hour + cycle_hour = "00" + init_string = "2018010306" + logger = logging.getLogger("dummy") + templ = "prefix.{valid?fmt=%Y%m%d%H}.tm{cycle?fmt=%H}z.tm{" \ + "offset?fmt=%H}.nc" + + ss = StringSub(logger, templ, init=init_string, cycle=cycle_hour, + offset=key) + valid_time = ss.calc_valid_for_prepbufr() + assert (valid_time == value) + + +def test_gdas_substitution(): + # Test that the string template substitution works correctly for GDAS + # prepbufr files, which do not make use of the cycle hour or the offset + # to generate the valid time. + valid_string = "2018010411" + logger = logging.getLogger("testing") + templ = "prepbufr.gdas.{valid?fmt=%Y%m%d%H}.nc" + expected_filename = 'prepbufr.gdas.' + valid_string + '.nc' + ss = StringSub(logger, templ, valid=valid_string) + filename = ss.doStringSub() + print("expected filename ", expected_filename, " gdas filename: ", + filename) + assert(filename == expected_filename) + +@pytest.mark.parametrize( + 'key, value', [ + ('38', 'prepbufr.nam.2018010311.t38z.tm03.nc'), + ('380', 'prepbufr.nam.2018011717.t380z.tm03.nc') + + ] +) +def test_nam_substitution_HH(key, value): + # Test that the substitution works correctly when given an init time, + # cycle hour, and negative offset hour. + init_string = "20180102" + cycle_string = key + offset_string = '03' + expected_filename = value + logger = logging.getLogger("test") + templ = \ + 'prepbufr.nam.{valid?fmt=%Y%m%d%H}.t{cycle?fmt=%HH}z.tm{' \ + 'offset?fmt=%HH}.nc' + ss = StringSub(logger, templ, init=init_string, cycle=cycle_string, + offset=offset_string) + filename = ss.doStringSub() + print('nam filename: ', filename) + assert (filename == expected_filename) + + +@pytest.mark.parametrize( + 'key, value', [ + ('18', 'prepbufr.nam.2018010215.t018z.tm03.nc'), + ('03', 'prepbufr.nam.2018010200.t003z.tm03.nc'), + + ] +) +def test_nam_substitution_HHH(key, value): + # Test that the substitution works correctly when given an init time, + # cycle hour, and negative offset hour. + init_string = "20180102" + cycle_string = key + offset_string = '03' + expected_filename = value + logger = logging.getLogger("test") + templ = \ + 'prepbufr.nam.{valid?fmt=%Y%m%d%H}.t{cycle?fmt=%HHH}z.tm{' \ + 'offset?fmt=%HH}.nc' + ss = StringSub(logger, templ, init=init_string, cycle=cycle_string, + offset=offset_string) + filename = ss.doStringSub() + print('nam filename: ', filename) + assert (filename == expected_filename) + + +@pytest.mark.parametrize( + 'key, value', [ + ('38', 'prepbufr.nam.2018010311.t01_13_59_59z.tm03.nc'), + ('380', 'prepbufr.nam.2018011717.t15_20_00_00z.tm03.nc') + ] +) +def test_nam_substitution_dHMS(key, value): + # Test that the substitution works correctly when given an init time, + # cycle hour, and negative offset hour. + init_string = "20180102" + cycle_string = key + offset_string = '03' + expected_filename = value + logger = logging.getLogger("test") + templ = \ + 'prepbufr.nam.{valid?fmt=%Y%m%d%H}.t{cycle?fmt=%dd%HH%M%S}z.tm{' \ + 'offset?fmt=%HH}.nc' + ss = StringSub(logger, templ, init=init_string, cycle=cycle_string, + offset=offset_string) + filename = ss.doStringSub() + print('nam filename: ', filename) + assert (filename == expected_filename) From 704a4c0795f73bf8202a76d98331ad1a1eca2670 Mon Sep 17 00:00:00 2001 From: bikegeek <3753118+bikegeek@users.noreply.github.com> Date: Fri, 13 Apr 2018 12:33:36 -0600 Subject: [PATCH 12/14] Delete test_string_template_substitution.py in the internal_tests/pytests/StringTemplateSubstitution directory --- .../test_string_template_substitution.py | 137 ------------------ 1 file changed, 137 deletions(-) delete mode 100644 internal_tests/pytests/StringTemplateSubstition/test_string_template_substitution.py diff --git a/internal_tests/pytests/StringTemplateSubstition/test_string_template_substitution.py b/internal_tests/pytests/StringTemplateSubstition/test_string_template_substitution.py deleted file mode 100644 index 1ee1149d98..0000000000 --- a/internal_tests/pytests/StringTemplateSubstition/test_string_template_substitution.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function - -import pytest -from string_template_substitution import StringSub -import logging - - -def test_cycle_hour(): - cycle_string = "00" - valid_string = "20180103" - logger = logging.getLogger("dummy") - templ = "prefix.{valid?fmt=%Y%m%d}.tm{cycle?fmt=%H}" - ss = StringSub(logger, templ, valid=valid_string, cycle=cycle_string) - expected_hours = int(0) - assert (ss.cycle_time_hours == expected_hours) - - -def test_offset_hour(): - logger = logging.getLogger("dummy") - expected_hour = int(3) - offset = "03" - templ = "prefix.{valid?fmt=%Y%m%d}.tm{offset?fmt=%H}" - ss = StringSub(logger, templ, offset=offset) - assert (ss.offset_hour == expected_hour) - - -@pytest.mark.parametrize( - 'key, value', [ - ('00', '20180103060000'), - ('03', '20180103030000'), - ('06', '20180103000000'), - ('72', '20171231060000') - ] -) -def test_calc_valid_for_prepbufr(key, value): - # Verify that the previous day is correctly calculated when - # the negative_offset_hour > cycle_hour - cycle_hour = "00" - init_string = "2018010306" - logger = logging.getLogger("dummy") - templ = "prefix.{valid?fmt=%Y%m%d%H}.tm{cycle?fmt=%H}z.tm{" \ - "offset?fmt=%H}.nc" - - ss = StringSub(logger, templ, init=init_string, cycle=cycle_hour, - offset=key) - valid_time = ss.calc_valid_for_prepbufr() - assert (valid_time == value) - - -def test_gdas_substitution(): - # Test that the string template substitution works correctly for GDAS - # prepbufr files, which do not make use of the cycle hour or the offset - # to generate the valid time. - valid_string = "2018010411" - logger = logging.getLogger("testing") - templ = "prepbufr.gdas.{valid?fmt=%Y%m%d%H}.nc" - expected_filename = 'prepbufr.gdas.' + valid_string + '.nc' - ss = StringSub(logger, templ, valid=valid_string) - filename = ss.doStringSub() - print("expected filename ", expected_filename, " gdas filename: ", - filename) - assert(filename == expected_filename) - -@pytest.mark.parametrize( - 'key, value', [ - ('38', 'prepbufr.nam.2018010311.t38z.tm03.nc'), - ('380', 'prepbufr.nam.2018011717.t380z.tm03.nc') - - ] -) -def test_nam_substitution_HH(key, value): - # Test that the substitution works correctly when given an init time, - # cycle hour, and negative offset hour. - init_string = "20180102" - cycle_string = key - offset_string = '03' - expected_filename = value - logger = logging.getLogger("test") - templ = \ - 'prepbufr.nam.{valid?fmt=%Y%m%d%H}.t{cycle?fmt=%HH}z.tm{' \ - 'offset?fmt=%HH}.nc' - ss = StringSub(logger, templ, init=init_string, cycle=cycle_string, - offset=offset_string) - filename = ss.doStringSub() - print('nam filename: ', filename) - assert (filename == expected_filename) - - -@pytest.mark.parametrize( - 'key, value', [ - ('18', 'prepbufr.nam.2018010215.t018z.tm03.nc'), - ('03', 'prepbufr.nam.2018010200.t003z.tm03.nc'), - - ] -) -def test_nam_substitution_HHH(key, value): - # Test that the substitution works correctly when given an init time, - # cycle hour, and negative offset hour. - init_string = "20180102" - cycle_string = key - offset_string = '03' - expected_filename = value - logger = logging.getLogger("test") - templ = \ - 'prepbufr.nam.{valid?fmt=%Y%m%d%H}.t{cycle?fmt=%HHH}z.tm{' \ - 'offset?fmt=%HH}.nc' - ss = StringSub(logger, templ, init=init_string, cycle=cycle_string, - offset=offset_string) - filename = ss.doStringSub() - print('nam filename: ', filename) - assert (filename == expected_filename) - - -@pytest.mark.parametrize( - 'key, value', [ - ('38', 'prepbufr.nam.2018010311.t01_13_59_59z.tm03.nc'), - ('380', 'prepbufr.nam.2018011717.t15_20_00_00z.tm03.nc') - ] -) -def test_nam_substitution_dHMS(key, value): - # Test that the substitution works correctly when given an init time, - # cycle hour, and negative offset hour. - init_string = "20180102" - cycle_string = key - offset_string = '03' - expected_filename = value - logger = logging.getLogger("test") - templ = \ - 'prepbufr.nam.{valid?fmt=%Y%m%d%H}.t{cycle?fmt=%dd%HH%M%S}z.tm{' \ - 'offset?fmt=%HH}.nc' - ss = StringSub(logger, templ, init=init_string, cycle=cycle_string, - offset=offset_string) - filename = ss.doStringSub() - print('nam filename: ', filename) - assert (filename == expected_filename) From 83099d8ac411f4139725ac1f2cf0f48e9f26d6a7 Mon Sep 17 00:00:00 2001 From: Minna Win Date: Fri, 13 Apr 2018 21:02:45 +0000 Subject: [PATCH 13/14] Conf file for initial testing of logging prior to refactor. --- .../pytests/logging/test_logging.py | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 internal_tests/pytests/logging/test_logging.py diff --git a/internal_tests/pytests/logging/test_logging.py b/internal_tests/pytests/logging/test_logging.py new file mode 100644 index 0000000000..c1721d9046 --- /dev/null +++ b/internal_tests/pytests/logging/test_logging.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import logging +import re +import os +import pytest +import met_util as util +import config_metplus + +# +# -----------Mandatory----------- +# configuration and fixture to support METplus configuration files beyond +# the metplus_data, metplus_system, and metplus_runtime conf files. +# + + +# Add a test configuration +def pytest_addoption(parser): + parser.addoption("-c", action="store", help=" -c ") + + +# @pytest.fixture +def cmdopt(request): + return request.config.getoption("-c") + +@pytest.fixture +def get_test_config(): + config_instance = config_metplus.setup() + return config_instance + + +@pytest.fixture +def get_test_logger(): + # Create a logger object based on the logging_test.conf + config_instance = get_test_config() + fixture_logger = util.get_logger(config_instance) + return fixture_logger + +# --------------Tests go here ------------ + + +def test_log_level(): + # Verify that the log level is set to what we indicated in the config file. + fixture_logger = get_test_logger() + # Expecting log level = DEBUG as set in the test config file. + level = logging.getLevelName('DEBUG') + assert fixture_logger.isEnabledFor(level) + + +def test_log_level_key(): + # Verify that the LOG_LEVEL key is in the config file + config_instance = get_test_config() + section = 'config' + option = 'LOG_LEVEL' + assert config_instance.has_option(section, option) + + +def test_logdir_exists(): + # Verify that the expected log dir exists. + config = get_test_config() + log_dir = config.get('config', 'LOG_DIR') + # Verify that a logfile exists in the log dir, with a filename + # like {LOG_DIR}/master_metplus.YYYYMMDD.log + assert os.path.exists(log_dir) + + +def test_logfile_exists(): + # Verify that a logfile with format master_metplus.YYYYMMDD.log exists + # We are assuming that there can be numerous files in the log directory. + config = get_test_config() + log_dir = config.get('config', 'LOG_DIR') + + # Only check for the log file if the log directory is present + if os.path.exists(log_dir): + files = [f for f in os.listdir(log_dir)] + match = re.match(r'master_metplus.[0-9]{8}.log', files[0]) + if match: + # Check the first file + assert match.group(0) + else: + # This file doesn't match the expected file name format. + assert False + else: + # There is no log directory + assert False + + + + + + + + From 59da8f98b6c89c0b01913b2579e2c72279367b1f Mon Sep 17 00:00:00 2001 From: Minna Win Date: Fri, 13 Apr 2018 21:05:51 +0000 Subject: [PATCH 14/14] Config file for testing logging prior to refactor. --- .../pytests/logging/logging_test.conf | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 internal_tests/pytests/logging/logging_test.conf diff --git a/internal_tests/pytests/logging/logging_test.conf b/internal_tests/pytests/logging/logging_test.conf new file mode 100644 index 0000000000..dc3e29fce9 --- /dev/null +++ b/internal_tests/pytests/logging/logging_test.conf @@ -0,0 +1,28 @@ +## Configuration file for testing logging in MET+ +## Override any key-values from the metplus_system.conf and metplus_data.conf files. +## ***NOTE***: Values are set to what is on DTC host 'eyewall' + +[config] +# Logging levels: DEBUG, INFO, WARN, ERROR (most verbose is DEBUG) +LOG_LEVEL = DEBUG +LOG_DIR = {OUTPUT_BASE}/logs +LOG_FILENAME = {LOG_DIR}/master_metplus.log ;; NOTE: current YYYYMMDD is inserted before the rightmost . filename extension + + +[dir] +# These should be located at some test directory +PROJ_DIR = /d1/minnawin/logging_test_output +MET_INSTALL_DIR = /usr/local/met-6.1 +METPLUS_BASE = /home/minnawin/latest +OUTPUT_BASE = /d1/minnawin/logging_test_output +TMP_DIR = /d1/minnawin/logging_test_output/tmp + +[exe] +WGRIB2 = /usr/local/bin/wgrib2 +RM_EXE = /bin/rm +CUT_EXE = /usr/bin/cut +TR_EXE = /usr/bin/tr +NCAP2_EXE =/usr/local/nco/bin/ncap2 +CONVERT_EXE =/usr/bin/convert +NCDUMP_EXE = /usr/local/bin/ncdump +EGREP_EXE = /bin/egrep