From f332db6818759934fb33a27a4795ed17cd998801 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:46:25 -0600 Subject: [PATCH] Code coverage in SonarQube (#448) * write sounding_data.dat file in output directory instead of current working directory * refactored plot classes - renamed logger class variables to be consistent and call utility functions to greatly simplify the main functions of each plot that are handled in the same way * Refactored unit tests so they can be run from any directory, not just the directory containing the test. Added pytest fixtures to handle setup of environment and remove files and intermed_files directories. Set env var TEST_DIR to the current test directory and reference it in yaml files to ensure that test input/output is read/written from the correct test directory. Changed file paths to be relative to test directory instead of current working directory * added PyCharm-generated files that shouldn't change * run pytests from top directory and generate code coverage report to send to SonarQube server * clone METcalcpy into different directory to prevent pytest to running its tests * install METcalcpy into different directory * cleanup * remove unnecessary call to remove sounding_data.dat file that is removed when scrubbing the output dir * remove unused variable * removed unused variable and whitespace * clean up linter complaints * add prints to help debugging if test fails * clean up tests to run faster by running plot script once and checking for all files --- .coveragerc | 2 + .github/workflows/sonarqube.yml | 30 +++ .github/workflows/unit_tests.yaml | 46 +--- .idea/.gitignore | 9 + .idea/METplotpy.iml | 15 ++ .idea/inspectionProfiles/Project_Default.xml | 14 ++ .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 7 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .../sonarqube/sonar-project.properties | 1 + metplotpy/plots/bar/bar.py | 47 +--- metplotpy/plots/base_plot.py | 7 +- metplotpy/plots/box/box.py | 53 ++-- metplotpy/plots/contour/contour.py | 56 ++--- metplotpy/plots/eclv/eclv.py | 58 ++--- metplotpy/plots/ens_ss/ens_ss.py | 52 ++-- .../equivalence_testing_bounds.py | 55 ++--- metplotpy/plots/histogram/hist.py | 50 ++-- metplotpy/plots/histogram/prob_hist.py | 26 +- metplotpy/plots/histogram/rank_hist.py | 28 +-- metplotpy/plots/histogram/rel_hist.py | 29 +-- metplotpy/plots/histogram_2d/histogram_2d.py | 25 +- metplotpy/plots/hovmoeller/hovmoeller.py | 18 +- metplotpy/plots/line/line.py | 54 ++-- metplotpy/plots/mpr_plot/mpr_plot.py | 17 +- .../performance_diagram.py | 18 +- .../plots/reliability_diagram/reliability.py | 25 +- metplotpy/plots/revision_box/revision_box.py | 27 +- .../plots/revision_series/revision_series.py | 46 +--- metplotpy/plots/roc_diagram/roc_diagram.py | 18 +- metplotpy/plots/scatter/scatter.py | 20 +- metplotpy/plots/skew_t/skew_t.py | 107 ++++---- .../plots/taylor_diagram/taylor_diagram.py | 24 +- metplotpy/plots/tcmpr_plots/tcmpr.py | 19 +- metplotpy/plots/util.py | 39 +++ metplotpy/plots/wind_rose/wind_rose.py | 20 +- test/bar/custom_bar.yaml | 4 +- test/bar/custom_defaultpoints1_bar.yaml | 4 +- test/bar/custom_points1_bar.yaml | 6 +- test/bar/{intermed_files => }/nan.points1 | 0 test/bar/test_bar.py | 158 ++++-------- test/bar/threshold_bar.yaml | 4 +- test/box/custom_box.yaml | 4 +- test/box/custom_box_defaultpoints1.yaml | 4 +- test/box/custom_box_points1.yaml | 6 +- test/box/simple_box.yaml | 4 +- test/box/test_box.py | 120 +++------ test/conftest.py | 33 +++ test/contour/custom_contour.yaml | 4 +- test/contour/test_contour.py | 15 +- .../test_difficulty_index_plotting.py | 8 +- test/eclv/custom_eclv.yaml | 4 +- test/eclv/custom_eclv_ctc.yaml | 4 +- test/eclv/custom_eclv_pct.yaml | 4 +- test/eclv/test_eclv.py | 29 +-- test/ens_ss/custom2_ens_ss.yaml | 6 +- test/ens_ss/custom_ens_ss.yaml | 4 +- test/ens_ss/test_ens_ss.py | 27 +- .../custom_equivalence_testing_bounds.yaml | 4 +- .../custom_equivalence_testing_bounds2.yaml | 6 +- .../test_equivalence_testing_bounds.py | 78 +++--- test/histogram/prob_hist.yaml | 4 +- test/histogram/rank_hist.yaml | 4 +- test/histogram/rel_hist.yaml | 4 +- test/histogram/test_prob_hist.py | 30 ++- test/histogram/test_rank_hist.py | 32 ++- test/histogram/test_rel_hist.py | 29 +-- test/histogram_2d/custom_histogram_2d.yaml | 4 +- test/histogram_2d/minimal_histogram_2d.yaml | 4 +- test/histogram_2d/test_histogram_2d.py | 34 +-- test/line/custom_line.yaml | 4 +- test/line/custom_line2.yaml | 6 +- test/line/custom_line_from_zero.yaml | 4 +- test/line/custom_line_groups.yaml | 4 +- test/line/custom_line_groups2.yaml | 6 +- test/line/fbias_fixed_vars_vals.yaml | 6 +- test/line/mv_custom_vert_line.yaml | 6 +- test/line/test_line_groups_plot.py | 34 +-- test/line/test_line_plot.py | 122 +++++---- test/mpr_plot/mpr_plot_custom.yaml | 4 +- test/mpr_plot/test_mpr_plot.py | 41 ++-- .../custom_performance_diagram.yaml | 4 +- ...om_performance_diagram_defaultpoints1.yaml | 4 +- .../custom_performance_diagram_points1.yaml | 6 +- .../test_performance_diagram.py | 102 +++----- .../custom_reliability_diagram.yaml | 4 +- .../custom_reliability_points1.yaml | 6 +- .../custom_reliability_use_defaults.yaml | 4 +- .../test_reliability_diagram.py | 71 ++---- test/revision_box/custom2_revision_box.yaml | 6 +- test/revision_box/custom_revision_box.yaml | 4 +- test/revision_box/test_revision_box.py | 73 ++---- .../custom2_revision_series.yaml | 6 +- .../custom_revision_series.yaml | 4 +- test/revision_series/test_revision_series.py | 71 ++---- test/roc_diagram/CTC_ROC.yaml | 4 +- test/roc_diagram/CTC_ROC_ee.yaml | 4 +- test/roc_diagram/CTC_ROC_summary.yaml | 6 +- test/roc_diagram/CTC_ROC_thresh.yaml | 4 +- test/roc_diagram/CTC_ROC_thresh_dump_pts.yaml | 6 +- .../CTC_ROC_thresh_reverse_pts.yaml | 4 +- test/roc_diagram/CTC_wind_reformatted.yaml | 4 +- test/roc_diagram/PCT_ROC.yaml | 4 +- test/roc_diagram/custom_roc_diagram.yaml | 4 +- test/roc_diagram/test_roc_diagram.py | 231 +++++------------- test/roc_diagram/test_roc_diagram.yaml | 4 +- test/skew_t/test_skew_t.py | 165 +++---------- test/skew_t/test_skew_t.yaml | 6 +- .../taylor_diagram/taylor_diagram_custom.yaml | 4 +- .../taylor_diagram/test_neg_and_pos_corr.yaml | 4 +- test/taylor_diagram/test_pos_corr.yaml | 4 +- test/taylor_diagram/test_taylor_diagram.py | 74 +++--- test/tcmpr_plots/test_tcmpr_multi_plots.yaml | 6 +- test/tcmpr_plots/test_tcmpr_plots.py | 23 +- test/util/test_util.py | 54 ++-- test/wind_rose/minimal_wind_rose.yaml | 4 +- test/wind_rose/test_wind_rose.py | 97 +++----- test/wind_rose/wind_rose_custom.yaml | 4 +- test/wind_rose/wind_rose_custom2_points.yaml | 4 +- test/wind_rose/wind_rose_custom_points.yaml | 6 +- 121 files changed, 1168 insertions(+), 1931 deletions(-) create mode 100644 .coveragerc create mode 100644 .idea/.gitignore create mode 100644 .idea/METplotpy.iml create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml rename test/bar/{intermed_files => }/nan.points1 (100%) create mode 100644 test/conftest.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..4edd7b1a --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +relative_files = True diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml index d48e1237..cb809cbe 100644 --- a/.github/workflows/sonarqube.yml +++ b/.github/workflows/sonarqube.yml @@ -48,6 +48,36 @@ jobs: # Disable shallow clones for better analysis fetch-depth: 0 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Retrieve METcalcpy repository develop branch + run: | + metcalcpy_dir="$RUNNER_WORKSPACE/METcalcpy" + git clone https://github.com/dtcenter/METcalcpy ${metcalcpy_dir} + cd ${metcalcpy_dir} + git -C ${metcalcpy_dir} checkout develop + python -m pip install -e ${metcalcpy_dir} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + python -m pip install pytest-cov + + - name: Run Pytests + run: coverage run -m pytest + + - name: Output coverage report + run: coverage report -m + if: always() + + - name: Generate XML coverage report + run: coverage xml + if: always() + - name: Get branch name id: get_branch_name run: echo branch_name=${GITHUB_REF#refs/heads/} >> $GITHUB_OUTPUT diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index a3c07bc5..27d578c2 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -39,10 +39,11 @@ jobs: - name: Retrieve METcalcpy repository develop branch run: | - /usr/bin/git clone https://github.com/dtcenter/METcalcpy - cd METcalcpy - /usr/bin/git checkout develop - python -m pip install -e . + metcalcpy_dir="$RUNNER_WORKSPACE/METcalcpy" + git clone https://github.com/dtcenter/METcalcpy ${metcalcpy_dir} + cd ${metcalcpy_dir} + git -C ${metcalcpy_dir} checkout develop + python -m pip install -e ${metcalcpy_dir} - name: Install dependencies run: | @@ -55,43 +56,8 @@ jobs: # run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" # id: extract_branch - - name: Test with pytest - run: | - cd test - cd bar - pytest test_bar.py - cd ../box - pytest test_box.py - cd ../contour - pytest test_contour.py - cd ../eclv - pytest test_eclv.py - cd ../ens_ss - pytest test_ens_ss.py - cd ../equivalence_testing_bounds - pytest test_equivalence_testing_bounds.py - cd ../line - pytest test_line_groups_plot.py - pytest test_line_plot.py - cd ../mpr_plot - pytest test_mpr_plot.py - cd ../performance_diagram - pytest test_performance_diagram.py - cd ../reliability_diagram - pytest test_reliability_diagram.py - cd ../roc_diagram - pytest test_roc_diagram.py - cd ../taylor_diagram - pytest test_taylor_diagram.py - cd ../wind_rose - pytest test_wind_rose.py - cd ../histogram - pytest test_prob_hist.py - pytest test_rank_hist.py - pytest test_rel_hist.py - cd ../tcmpr_plots - pytest --capture=fd test_tcmpr_plots.py + run: pytest diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..abf271a3 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,9 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +/sonarlint* diff --git a/.idea/METplotpy.iml b/.idea/METplotpy.iml new file mode 100644 index 00000000..2d40d024 --- /dev/null +++ b/.idea/METplotpy.iml @@ -0,0 +1,15 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 00000000..1bbf15dd --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,14 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 00000000..105ce2da --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..c58198b3 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..8f21a77e --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index 894fec47..0777e588 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -5,6 +5,7 @@ sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME sonar.sources=metplotpy,test sonar.coverage.exclusions=test/** +sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 # SonarQube server diff --git a/metplotpy/plots/bar/bar.py b/metplotpy/plots/bar/bar.py index f02042c9..cb881886 100644 --- a/metplotpy/plots/bar/bar.py +++ b/metplotpy/plots/bar/bar.py @@ -19,7 +19,6 @@ import pandas as pd import plotly.graph_objects as go -import yaml from plotly.graph_objects import Figure from plotly.subplots import make_subplots @@ -53,11 +52,11 @@ def __init__(self, parameters: dict) -> None: # the # config file that represents the BasePlot object (Bar). self.config_obj = BarConfig(self.parameters) - self.bar_logger = self.config_obj.logger - self.bar_logger.info(f"Start bar plot: {datetime.now()}") + self.logger = self.config_obj.logger + self.logger.info(f"Start bar plot: {datetime.now()}") # Check that we have all the necessary settings for each series - self.bar_logger.info(f"Consistency checking of config settings for colors, " - f"legends, etc.") + self.logger.info("Consistency checking of config settings for colors, " + "legends, etc.") is_config_consistent = self.config_obj._config_consistency_check() if not is_config_consistent: value_error_msg = ("ValueError: The number of series defined by series_val_1 and " @@ -66,20 +65,19 @@ def __init__(self, parameters: dict) -> None: "check the number of your configuration file's " "plot_i, plot_disp, series_order, user_legend, show_legend and " "colors settings.") - self.bar_logger.error(value_error_msg) + self.logger.error(value_error_msg) raise ValueError(value_error_msg) - # Read in input data, location specified in config file - self.bar_logger.info(f"Begin reading input data: {datetime.now()}") + self.logger.info(f"Begin reading input data: {datetime.now()}") self.input_df = self._read_input_data() # Apply event equalization, if requested if self.config_obj.use_ee is True: - self.bar_logger.info(f"Performing event equalization: {datetime.now()}") + self.logger.info(f"Performing event equalization: {datetime.now()}") self.input_df = calc_util.perform_event_equalization(self.parameters, self.input_df) - self.bar_logger.info(f"End event equalization: {datetime.now()}") + self.logger.info(f"End event equalization: {datetime.now()}") # Create a list of series objects. # Each series object contains all the necessary information for plotting, @@ -91,9 +89,9 @@ def __init__(self, parameters: dict) -> None: # Need to have a self.figure that we can pass along to # the methods in base_plot.py (BasePlot class methods) to # create binary versions of the plot. - self.bar_logger.info(f"Begin creating the figure: {datetime.now()}") + self.logger.info(f"Begin creating the figure: {datetime.now()}") self._create_figure() - self.bar_logger.info(f"End creating the figure: {datetime.now()}") + self.logger.info(f"End creating the figure: {datetime.now()}") def __repr__(self): """ Implement repr which can be useful for debugging this @@ -114,7 +112,7 @@ def _read_input_data(self): Returns: """ - self.bar_logger.info(f"Finished reading input data: " + self.logger.info(f"Finished reading input data: " f"{datetime.now()}") return pd.read_csv(self.config_obj.parameters['stat_input'], sep='\t', header='infer', float_precision='round_trip') @@ -516,28 +514,7 @@ def main(config_filename=None): @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = Bar(docs) - plot.save_to_file() - # plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.bar_logger.info(f"Finished bar plot at {datetime.now()}") - except ValueError as val_er: - print(val_er) + util.make_plot(config_filename, Bar) if __name__ == "__main__": diff --git a/metplotpy/plots/base_plot.py b/metplotpy/plots/base_plot.py index 106bf3a1..de0458a2 100644 --- a/metplotpy/plots/base_plot.py +++ b/metplotpy/plots/base_plot.py @@ -74,7 +74,6 @@ def __init__(self, parameters, default_conf_filename): self.remove_file() self.config_obj = Config(self.parameters) - def get_image_format(self): """Reads the image format type from user provided image name. Uses file extension as a type. If the file extension is not valid - @@ -340,8 +339,6 @@ def save_to_file(self): Returns: """ - - image_name = self.get_config_value('plot_filename') # Suppress deprecation warnings from third-party packages that are not in our control. @@ -388,8 +385,8 @@ def show_in_browser(self): if self.figure: self.figure.show() else: - self.logger.error(f" Figure not created. Nothing to show in the " - f"browser. ") + self.logger.error(" Figure not created. Nothing to show in the " + "browser. ") print("Oops! The figure was not created. Can't show") def _add_lines(self, config_obj: Config, x_points_index: Union[list, None] = None) -> None: diff --git a/metplotpy/plots/box/box.py b/metplotpy/plots/box/box.py index bff95601..6c41762a 100644 --- a/metplotpy/plots/box/box.py +++ b/metplotpy/plots/box/box.py @@ -20,7 +20,6 @@ from typing import Union from operator import add from itertools import chain -import yaml import pandas as pd import plotly.graph_objects as go @@ -57,13 +56,13 @@ def __init__(self, parameters): # config file that represents the BasePlot object (Box). self.config_obj = BoxConfig(self.parameters) - self.box_logger = self.config_obj.logger + self.logger = self.config_obj.logger - self.box_logger.info(f"Start bar plot at {datetime.now()}") + self.logger.info(f"Start bar plot at {datetime.now()}") # Check that we have all the necessary settings for each series is_config_consistent = self.config_obj._config_consistency_check() - self.box_logger.info("Checking consistency of user_legends, colors, etc...") + self.logger.info("Checking consistency of user_legends, colors, etc...") if not is_config_consistent: raise ValueError("The number of series defined by series_val_1/2 and derived curves is" " inconsistent with the number of settings" @@ -77,9 +76,9 @@ def __init__(self, parameters): # Apply event equalization, if requested if self.config_obj.use_ee is True: - self.box_logger.info(f"Start event equalization: {datetime.now()}") + self.logger.info(f"Start event equalization: {datetime.now()}") self.input_df = calc_util.perform_event_equalization(self.parameters, self.input_df) - self.box_logger.info(f"Finish event equalization: {datetime.now()}") + self.logger.info(f"Finish event equalization: {datetime.now()}") # Create a list of series objects. # Each series object contains all the necessary information for plotting, @@ -129,7 +128,7 @@ def _create_series(self, input_data): """ - self.box_logger.info(f"Begin generating series objects: " + self.logger.info(f"Begin generating series objects: " f"{datetime.now()}") series_list = [] @@ -173,14 +172,14 @@ def _create_series(self, input_data): # reorder series series_list = self.config_obj.create_list_by_series_ordering(series_list) - self.box_logger.info(f"End generating series objects: " + self.logger.info(f"End generating series objects: " f"{datetime.now()}") return series_list def _create_figure(self): """ Create a box plot from default and custom parameters""" - self.box_logger.info(f"Begin creating the figure: " + self.logger.info(f"Begin creating the figure: " f"{datetime.now()}") self.figure = self._create_layout() self._add_xaxis() @@ -230,7 +229,7 @@ def _create_figure(self): self.figure.update_layout(boxmode='group') - self.box_logger.info(f"End creating the figure: " + self.logger.info(f"End creating the figure: " f"{datetime.now()}") def _draw_series(self, series: BoxSeries) -> None: @@ -240,7 +239,7 @@ def _draw_series(self, series: BoxSeries) -> None: :param series: Line series object with data and parameters """ - self.box_logger.info(f"Begin drawing the boxes on the plot for " + self.logger.info(f"Begin drawing the boxes on the plot for " f"{series.series_name}: " f"{datetime.now()}") # defaults markers and colors for the regular box plot @@ -284,7 +283,7 @@ def _draw_series(self, series: BoxSeries) -> None: secondary_y=series.y_axis != 1 ) - self.box_logger.info(f"End drawing the boxes on the plot: " + self.logger.info(f"End drawing the boxes on the plot: " f"{datetime.now()}") @staticmethod @@ -299,7 +298,7 @@ def _find_min_max(series: BoxSeries, yaxis_min: Union[float, None], :param yaxis_max: previously calculated max value :return: a tuple with calculated min/max """ - self.box_logger.info(f"Begin finding min and max CI values: " + self.logger.info(f"Begin finding min and max CI values: " f"{datetime.now()}") # calculate series upper and lower limits of CIs indexes = range(len(series.series_points['dbl_med'])) @@ -311,7 +310,7 @@ def _find_min_max(series: BoxSeries, yaxis_min: Union[float, None], if yaxis_min is None or yaxis_max is None: return min(low_range), max(upper_range) - self.box_logger.info(f"End finding min and max CI values: " + self.logger.info(f"End finding min and max CI values: " f"{datetime.now()}") return min(chain([yaxis_min], low_range)), max(chain([yaxis_max], upper_range)) @@ -577,7 +576,7 @@ def write_html(self) -> None: # save html self.figure.write_html(html_name, include_plotlyjs=False) - self.box_logger.info(f"End writing HTML file: " + self.logger.info(f"End writing HTML file: " f"{datetime.now()}") def write_output_file(self) -> None: @@ -637,29 +636,7 @@ def main(config_filename=None): Args: @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./custom_box.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = Box(docs) - plot.save_to_file() - #plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.box_logger.info(f"Finished box plot at {datetime.now()}") - except ValueError as ve: - print(ve) + util.make_plot(config_filename, Box) if __name__ == "__main__": diff --git a/metplotpy/plots/contour/contour.py b/metplotpy/plots/contour/contour.py index 08dfd191..353a1d72 100644 --- a/metplotpy/plots/contour/contour.py +++ b/metplotpy/plots/contour/contour.py @@ -18,7 +18,6 @@ import re import csv -import yaml import pandas as pd import plotly.graph_objects as go @@ -59,15 +58,14 @@ def __init__(self, parameters: dict) -> None: # config file that represents the BasePlot object (Line). self.config_obj = ContourConfig(self.parameters) - - self.contour_logger = self.config_obj.logger - self.contour_logger.info(f"Start contour plot: {datetime.now()}") + self.logger = self.config_obj.logger + self.logger.info(f"Start contour plot: {datetime.now()}") # Check that we have all the necessary settings for each series - self.contour_logger.info("Consistency checking of config settings for colors,legends, etc.") + self.logger.info("Consistency checking of config settings for colors,legends, etc.") is_config_consistent = self.config_obj._config_consistency_check() if not is_config_consistent: - self.contour_logger.error("ValueError: The number of series defined by " + self.logger.error("ValueError: The number of series defined by " "series_val_1 is inconsistent with the number of " "settings required for describing each series. " "Please check the number of your configuration" @@ -81,15 +79,15 @@ def __init__(self, parameters: dict) -> None: " colors settings.") # Read in input data, location specified in config file - self.contour_logger.info(f"Begin reading input data: {datetime.now()}") + self.logger.info(f"Begin reading input data: {datetime.now()}") self.input_df = self._read_input_data() # Apply event equalization, if requested if self.config_obj.use_ee is True: - self.contour_logger.info(f"Begin event equalization: {datetime.now()} ") + self.logger.info(f"Begin event equalization: {datetime.now()} ") self.input_df = calc_util.perform_event_equalization(self.parameters, self.input_df) - self.contour_logger.info(f"Event equalization complete: {datetime.now()}") + self.logger.info(f"Event equalization complete: {datetime.now()}") # Create a list of series objects. # Each series object contains all the necessary information for plotting, @@ -140,7 +138,7 @@ def _create_series(self, input_data): """ series_list = [] - self.contour_logger.info(f"Generating series objects: {datetime.now()}") + self.logger.info(f"Generating series objects: {datetime.now()}") # add series for y1 axis num_series_y1 = len(self.config_obj.get_series_y()) for i, name in enumerate(self.config_obj.get_series_y()): @@ -150,7 +148,7 @@ def _create_series(self, input_data): # reorder series series_list = self.config_obj.create_list_by_series_ordering(series_list) - self.contour_logger.info(f"Finished creating series objects: {datetime.now()}") + self.logger.info(f"Finished creating series objects: {datetime.now()}") return series_list def _create_figure(self): @@ -158,7 +156,7 @@ def _create_figure(self): Create a Contour plot from defaults and custom parameters """ - self.contour_logger.info(f"Creating the figure: {datetime.now()}") + self.logger.info(f"Creating the figure: {datetime.now()}") # create and draw the plot self.figure = self._create_layout() self._add_xaxis() @@ -201,7 +199,7 @@ def _create_figure(self): 'ticks': "outside" } ) - self.contour_logger.info(f"Figure creating complete: {datetime.now()}") + self.logger.info(f"Figure creating complete: {datetime.now()}") def _draw_series(self, series: Series) -> None: """ @@ -209,7 +207,7 @@ def _draw_series(self, series: Series) -> None: :param series: Contour series object with data and parameters """ - self.contour_logger.info(f"Drawing the data: {datetime.now()}") + self.logger.info(f"Drawing the data: {datetime.now()}") line_width = self.config_obj.linewidth_list[series.idx] if self.config_obj.add_contour_overlay is False: line_width = 0 @@ -249,7 +247,7 @@ def _draw_series(self, series: Series) -> None: zauto=zauto ) ) - self.contour_logger.info(f"Finished drawing data: {datetime.now()}") + self.logger.info(f"Finished drawing data: {datetime.now()}") def _create_layout(self) -> Figure: """ @@ -390,7 +388,7 @@ def write_output_file(self) -> None: """ saves series points to the files """ - self.contour_logger.info(f"Writing output file: {datetime.now()}") + self.logger.info(f"Writing output file: {datetime.now()}") # Open file, name it based on the stat_input config setting, # (the input data file) except replace the .data @@ -425,7 +423,7 @@ def write_output_file(self) -> None: file.writelines('\n') file.writelines('\n') file.close() - self.contour_logger.info(f"Finished writing output file: {datetime.now()}") + self.logger.info(f"Finished writing output file: {datetime.now()}") def main(config_filename=None): @@ -437,29 +435,7 @@ def main(config_filename=None): Args: @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./custom_line_plot.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = Contour(docs) - plot.save_to_file() - # plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.contour_logger.info(f"Finished contour plot at {datetime.now()}") - except ValueError as val_er: - print(val_er) + util.make_plot(config_filename, Contour) if __name__ == "__main__": diff --git a/metplotpy/plots/eclv/eclv.py b/metplotpy/plots/eclv/eclv.py index 6b0a7a55..5860b572 100644 --- a/metplotpy/plots/eclv/eclv.py +++ b/metplotpy/plots/eclv/eclv.py @@ -17,13 +17,13 @@ import csv from operator import add from typing import Union -import yaml import itertools import plotly.graph_objects as go from datetime import datetime from metcalcpy.event_equalize import event_equalize + from metplotpy.plots.base_plot import BasePlot from metplotpy.plots.constants import PLOTLY_AXIS_LINE_COLOR, PLOTLY_AXIS_LINE_WIDTH from metplotpy.plots.eclv.eclv_config import EclvConfig @@ -60,14 +60,14 @@ def __init__(self, parameters: dict) -> None: # config file that represents the BasePlot object (Eclv). self.config_obj = EclvConfig(self.parameters) - self.eclv_logger = self.config_obj.logger + self.logger = self.config_obj.logger - self.eclv_logger.info(f"Start eclv plot: {datetime.now()}") + self.logger.info(f"Start eclv plot: {datetime.now()}") # Check that we have all the necessary settings for each series is_config_consistent = self.config_obj._config_consistency_check() if not is_config_consistent: - self.line_logger.error("ValueError: The number of series defined by " + self.logger.error("ValueError: The number of series defined by " "series_val_1 is " "inconsistent with the number of settings " "required for" @@ -85,12 +85,12 @@ def __init__(self, parameters: dict) -> None: " colors, and series_symbols settings.") # Read in input data, location specified in config file - self.eclv_logger.info(f"Begin reading input data: {datetime.now()}") + self.logger.info(f"Begin reading input data: {datetime.now()}") self.input_df = self._read_input_data() # Apply event equalization, if requested if self.config_obj.use_ee is True: - self.line_logger.info(f"Performing event equalization: {datetime.now()}") + self.logger.info(f"Performing event equalization: {datetime.now()}") fix_vals_permuted_list = [] for key in self.config_obj.fixed_vars_vals_input: @@ -106,7 +106,7 @@ def __init__(self, parameters: dict) -> None: self.parameters['series_val_1'], fix_vals_keys, fix_vals_permuted_list, True, True) - self.eclv_logger.info(f"End even equalization: {datetime.now()}") + self.logger.info(f"End even equalization: {datetime.now()}") # Create a list of series objects. # Each series object contains all the necessary information for plotting, @@ -119,9 +119,9 @@ def __init__(self, parameters: dict) -> None: # Need to have a self.figure that we can pass along to # the methods in base_plot.py (BasePlot class methods) to # create binary versions of the plot. - self.eclv_logger.info(f"Begin creating the figure: {datetime.now()}") + self.logger.info(f"Begin creating the figure: {datetime.now()}") self._create_figure() - self.eclv_logger.info(f"End creating the figure: {datetime.now()}") + self.logger.info(f"End creating the figure: {datetime.now()}") def __repr__(self): """ Implement repr which can be useful for debugging this @@ -150,7 +150,7 @@ def _create_series(self, input_data): """ - self.eclv_logger.info(f"Begin creating series objects: {datetime.now()}") + self.logger.info(f"Begin creating series objects: {datetime.now()}") series_list = [] # add series for y1 axis @@ -163,7 +163,7 @@ def _create_series(self, input_data): # reorder series series_list = self.config_obj.create_list_by_series_ordering(series_list) - self.eclv_logger.info(f"Finished creating series objects:" + self.logger.info(f"Finished creating series objects:" f" {datetime.now()}") return series_list @@ -171,7 +171,7 @@ def _create_figure(self): """ Create a eclv plot from defaults and custom parameters """ - self.eclv_logger.info(f"Begin creating the figure: {datetime.now()}") + self.logger.info(f"Begin creating the figure: {datetime.now()}") # create and draw the plot self.figure = self._create_layout() self._add_xaxis() @@ -227,7 +227,7 @@ def _create_figure(self): # add x2 axis self._add_x2axis(n_stats) - self.eclv_logger.info(f"Finished creating the figure: {datetime.now()}") + self.logger.info(f"Finished creating the figure: {datetime.now()}") def _add_x2axis(self, n_stats) -> None: """ @@ -287,7 +287,7 @@ def _draw_series(self, series: Series, :param series: Eclv series object with data and parameters :param x_points_index_adj: values for adjusting x-values position """ - self.eclv_logger.info(f"Begin drawing the series : {datetime.now()}") + self.logger.info(f"Begin drawing the series : {datetime.now()}") # pct series can have mote than one line for ind, series_points in enumerate(series.series_points): y_points = series_points['dbl_med'] @@ -334,7 +334,7 @@ def _draw_series(self, series: Series, secondary_y=False ) - self.eclv_logger.info(f"Finished drawing the series :" + self.logger.info(f"Finished drawing the series :" f" {datetime.now()}") def write_output_file(self) -> None: @@ -342,7 +342,7 @@ def write_output_file(self) -> None: saves series points to the files """ - self.eclv_logger.info(f"Begin writing output file: {datetime.now()}") + self.logger.info(f"Begin writing output file: {datetime.now()}") # Open file, name it based on the stat_input config setting, # (the input data file) except replace the .data @@ -381,7 +381,7 @@ def write_output_file(self) -> None: file.writelines('\n') file.writelines('\n') file.close() - self.eclv_logger.info(f"Finished writing output file: {datetime.now()}") + self.logger.info(f"Finished writing output file: {datetime.now()}") def main(config_filename=None): @@ -394,29 +394,7 @@ def main(config_filename=None): @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./custom_eclv_plot.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = Eclv(docs) - plot.save_to_file() - # plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.eclv_logger.info(f"Finished ECLV plot: {datetime.now()}") - except ValueError as val_er: - print(val_er) + util.make_plot(config_filename, Eclv) if __name__ == "__main__": diff --git a/metplotpy/plots/ens_ss/ens_ss.py b/metplotpy/plots/ens_ss/ens_ss.py index 4d4e490e..d6d153ee 100644 --- a/metplotpy/plots/ens_ss/ens_ss.py +++ b/metplotpy/plots/ens_ss/ens_ss.py @@ -18,7 +18,6 @@ import re import itertools -import yaml import numpy as np import pandas as pd @@ -59,14 +58,14 @@ def __init__(self, parameters: dict) -> None: # config file that represents the BasePlot object (EnsSs). self.config_obj = EnsSsConfig(self.parameters) - self.ens_logger = self.config_obj.logger - self.ens_logger.info(f"Start Ens_ss plot: {datetime.now()}") + self.logger = self.config_obj.logger + self.logger.info(f"Start Ens_ss plot: {datetime.now()}") # Check that we have all the necessary settings for each series - self.ens_logger.info(f"Consistency checking of config settings for colors, " + self.logger.info(f"Consistency checking of config settings for colors, " f"legends, etc.{datetime.now()}") is_config_consistent = self.config_obj._config_consistency_check() - self.ens_logger.info(f"Finished consistency checking of config settings for colors, " + self.logger.info(f"Finished consistency checking of config settings for colors, " f"legends, etc.{datetime.now()}") if not is_config_consistent: value_error_msg = ("ValueError: The number of series defined by " @@ -76,18 +75,18 @@ def __init__(self, parameters: dict) -> None: "check the number of your configuration file's " "plot_i, plot_disp, series_order, user_legend, show_legend and " "colors settings.") - self.ens_logger.error(value_error_msg) + self.logger.error(value_error_msg) raise ValueError(value_error_msg) # Read in input data, location specified in config file - self.ens_logger.info(f"Begin reading input data: {datetime.now()}") + self.logger.info(f"Begin reading input data: {datetime.now()}") self.input_df = self._read_input_data() # Apply event equalization, if requested if self.config_obj.use_ee is True: - self.ens_logger.info(f"Performing event equalization: {datetime.now()}") + self.logger.info(f"Performing event equalization: {datetime.now()}") self._perform_event_equalization() - self.ens_logger.info(f"Finished event equalization: {datetime.now()}") + self.logger.info(f"Finished event equalization: {datetime.now()}") # Create a list of series objects. # Each series object contains all the necessary information for plotting, @@ -197,7 +196,7 @@ def _create_series(self, input_data): """ - self.ens_logger.info(f"Begin creating series objects: {datetime.now()}") + self.logger.info(f"Begin creating series objects: {datetime.now()}") series_list = [] @@ -211,7 +210,7 @@ def _create_series(self, input_data): # reorder series series_list = self.config_obj.create_list_by_series_ordering(series_list) - self.ens_logger.info(f"Finished creating series objects:" + self.logger.info(f"Finished creating series objects:" f" {datetime.now()}") return series_list @@ -221,7 +220,7 @@ def _create_figure(self): Create a Ensemble spread-skill plot from defaults and custom parameters """ - self.ens_logger.info(f"Begin creating the figure: {datetime.now()}") + self.logger.info(f"Begin creating the figure: {datetime.now()}") # create and draw the plot self.figure = self._create_layout() @@ -253,7 +252,7 @@ def _create_figure(self): self._yaxis_limits() self._y2axis_limits() - self.ens_logger.info(f"Finished creating the figure: {datetime.now()}") + self.logger.info(f"Finished creating the figure: {datetime.now()}") def _add_y2axis(self) -> None: """ @@ -293,7 +292,7 @@ def _draw_series(self, series: EnsSsSeries) -> None: :param series: EnsSs series object with data and parameters """ - self.ens_logger.info(f"Begin drawing the series on the plot:" + self.logger.info(f"Begin drawing the series on the plot:" f" {datetime.now()}") # add the plot @@ -335,7 +334,7 @@ def _draw_series(self, series: EnsSsSeries) -> None: secondary_y=True ) - self.ens_logger.info(f"Finished drawing the series on the plot:" + self.logger.info(f"Finished drawing the series on the plot:" f" {datetime.now()}") def _create_layout(self) -> Figure: @@ -557,28 +556,7 @@ def main(config_filename=None): Args: @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = EnsSs(docs) - plot.save_to_file() - # plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.ens_logger.info(f"Finished EnsSs plot: {datetime.now()}") - except ValueError as val_er: - print(val_er) + util.make_plot(config_filename, EnsSs) if __name__ == "__main__": diff --git a/metplotpy/plots/equivalence_testing_bounds/equivalence_testing_bounds.py b/metplotpy/plots/equivalence_testing_bounds/equivalence_testing_bounds.py index 698ab88c..5d3798c0 100644 --- a/metplotpy/plots/equivalence_testing_bounds/equivalence_testing_bounds.py +++ b/metplotpy/plots/equivalence_testing_bounds/equivalence_testing_bounds.py @@ -39,6 +39,7 @@ class EquivalenceTestingBounds(BasePlot): """ Generates a Plotly Equivalence Testing Bounds plot . """ + LONG_NAME = 'Equivalence Testing Bounds' def __init__(self, parameters: dict) -> None: """ Creates a Plotly Equivalence Testing Bounds plot, based on @@ -57,8 +58,8 @@ def __init__(self, parameters: dict) -> None: # config file that represents the BasePlot object (EquivalenceTestingBounds). self.config_obj = LineConfig(self.parameters) - self.eq_logger = self.config_obj.logger - self.eq_logger.info(f"Start equivalence testing bounds: {datetime.now()}") + self.logger = self.config_obj.logger + self.logger.info(f"Start equivalence testing bounds: {datetime.now()}") # Check that we have all the necessary settings for each series is_config_consistent = self.config_obj._config_consistency_check() @@ -69,12 +70,12 @@ def __init__(self, parameters: dict) -> None: " the number of your configuration file's plot_i," " plot_disp, series_order, user_legend," " colors, show_legend and series_symbols settings.") - self.eq_logger.error(f"ValueError: {error_msg}: {datetime.now()}") + self.logger.error(f"ValueError: {error_msg}: {datetime.now()}") raise ValueError(error_msg) # Read in input data, location specified in config file self.input_df = self._read_input_data() - self.eq_logger.info(f"Finished reading input data: {datetime.now()}") + self.logger.info(f"Finished reading input data: {datetime.now()}") # Apply event equalization, if requested if self.config_obj.use_ee is True: @@ -113,7 +114,7 @@ def _read_input_data(self): Returns: """ - self.eq_logger.info(f"Begin reading input data: {datetime.now()}") + self.logger.info(f"Begin reading input data: {datetime.now()}") return pd.read_csv(self.config_obj.parameters['stat_input'], sep='\t', header='infer', float_precision='round_trip') @@ -134,7 +135,7 @@ def _create_series(self, input_data): """ - self.eq_logger.info(f"Creating series object: {datetime.now()}") + self.logger.info(f"Creating series object: {datetime.now()}") series_list = [] # add series for y1 axis @@ -184,7 +185,7 @@ def _create_series(self, input_data): # reorder series series_list = self.config_obj.create_list_by_series_ordering(series_list) - self.eq_logger.info(f"Finished creating series object:" + self.logger.info(f"Finished creating series object:" f" {datetime.now()}") return series_list @@ -192,7 +193,7 @@ def _create_figure(self): """ Create an Equivalence Testing Bounds plot from defaults and custom parameters """ - self.eq_logger.info(f"Creating the figure: {datetime.now()}") + self.logger.info(f"Creating the figure: {datetime.now()}") # create and draw the plot self.figure = self._create_layout() @@ -211,7 +212,7 @@ def _create_figure(self): self._draw_series(series, ind) ind = ind + 1 - self.eq_logger.info(f"Finished creating the figure: {datetime.now()}") + self.logger.info(f"Finished creating the figure: {datetime.now()}") def _draw_series(self, series: LineSeries, ind: int) -> None: """ @@ -221,7 +222,7 @@ def _draw_series(self, series: LineSeries, ind: int) -> None: :param x_points_index_adj: values for adjusting x-values position """ - self.eq_logger.info(f"Start drawing the lines on the plot: {datetime.now()}") + self.logger.info(f"Start drawing the lines on the plot: {datetime.now()}") ci_tost_up = series.series_points['ci_tost'][1] ci_tost_lo = series.series_points['ci_tost'][0] dif = series.series_points['dif'] @@ -279,7 +280,7 @@ def _draw_series(self, series: LineSeries, ind: int) -> None: } ) - self.eq_logger.info(f"Finished drawing the lines on the plot: {datetime.now()}") + self.logger.info(f"Finished drawing the lines on the plot: {datetime.now()}") def _create_layout(self) -> Figure: """ @@ -500,7 +501,7 @@ def write_html(self) -> None: Plotly.js """ - self.eq_logger.info(f"Write html file: {datetime.now()}") + self.logger.info(f"Write html file: {datetime.now()}") if self.config_obj.create_html is True: # construct the fle name from plot_filename @@ -510,14 +511,14 @@ def write_html(self) -> None: # save html self.figure.write_html(html_name, include_plotlyjs=False) - self.eq_logger.info(f"Finished writing html file: {datetime.now()}") + self.logger.info(f"Finished writing html file: {datetime.now()}") def write_output_file(self) -> None: """ Formats y1 and y2 series point data to the 2-dim arrays and saves them to the files """ - self.eq_logger.info(f"Write output file: {datetime.now()}") + self.logger.info(f"Write output file: {datetime.now()}") # if points_path parameter doesn't exist, # open file, name it based on the stat_input config setting, @@ -559,7 +560,7 @@ def write_output_file(self) -> None: # save points self._save_points(ci_tost_df.values.tolist(), filename) - self.eq_logger.info(f"Finished writing the output file: {datetime.now()}") + self.logger.info(f"Finished writing the output file: {datetime.now()}") @staticmethod def _save_points(points: list, output_file: str) -> None: @@ -600,29 +601,7 @@ def main(config_filename=None): Args: @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./custom_line_plot.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = EquivalenceTestingBounds(docs) - plot.save_to_file() - # plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.eq_logger.info(f"Finished equivalence testing bounds: {datetime.now()}") - except ValueError as val_er: - print(val_er) + util.make_plot(config_filename, EquivalenceTestingBounds) if __name__ == "__main__": diff --git a/metplotpy/plots/histogram/hist.py b/metplotpy/plots/histogram/hist.py index 97eec445..a3012a5c 100644 --- a/metplotpy/plots/histogram/hist.py +++ b/metplotpy/plots/histogram/hist.py @@ -64,15 +64,15 @@ def __init__(self, parameters: dict) -> None: getattr(sys.modules['metplotpy.plots.histogram.hist_config'], self.config_obj_name)(self.parameters) - self.hist_logger = self.config_obj.logger - self.hist_logger.info(f"Begin [rank|probability|relative frequency] histogram:" + self.logger = self.config_obj.logger + self.logger.info(f"Begin [rank|probability|relative frequency] histogram:" f" {datetime.now()}") # Check that we have all the necessary settings for each ser - self.hist_logger.info(f"Performing consistency check for settings in config " + self.logger.info(f"Performing consistency check for settings in config " f"file: {datetime.now()}") is_config_consistent = self.config_obj._config_consistency_check() - self.hist_logger.info(f"Finished with consistency check: {datetime.now()}") + self.logger.info(f"Finished with consistency check: {datetime.now()}") if not is_config_consistent: error_msg = ("The number of ser defined by series_val_1 is" " inconsistent with the number of settings" @@ -80,18 +80,18 @@ def __init__(self, parameters: dict) -> None: " the number of your configuration file's " " plot_disp, series_order, user_legend, show_legend" " colors settings.") - self.hist_logger.error(f"ValueError: {error_msg}") + self.logger.error(f"ValueError: {error_msg}") raise ValueError(error_msg) # Read in input data, location specified in config file self.input_df = self._read_input_data() - self.hist_logger.info(f"Finished reading input data: {datetime.now()}") + self.logger.info(f"Finished reading input data: {datetime.now()}") # Apply event equalization, if requested if self.config_obj.use_ee is True: - self.hist_logger.info(f"Begin event equalization: {datetime.now()}") + self.logger.info(f"Begin event equalization: {datetime.now()}") self._perform_event_equalization() - self.hist_logger.info(f"Event equalization complete: {datetime.now()}") + self.logger.info(f"Event equalization complete: {datetime.now()}") # Create a list of ser objects. # Each ser object contains all the necessary information for plotting, @@ -193,7 +193,7 @@ def _read_input_data(self): """ - self.hist_logger.info(f"Reading input data: {datetime.now()}") + self.logger.info(f"Reading input data: {datetime.now()}") return pd.read_csv(self.config_obj.parameters['stat_input'], sep='\t', header='infer', float_precision='round_trip') @@ -218,7 +218,7 @@ def _create_series(self, input_data): """ - self.hist_logger.info(f"Creating the series objects: {datetime.now()}") + self.logger.info(f"Creating the series objects: {datetime.now()}") series_list = [] hist_series_type = \ getattr(sys.modules['metplotpy.plots.histogram.hist_series'], @@ -247,14 +247,14 @@ def _create_series(self, input_data): # reorder ser series_list = self.config_obj.create_list_by_series_ordering(series_list) - self.hist_logger.info(f"Finished creating the series objects: {datetime.now()}") + self.logger.info(f"Finished creating the series objects: {datetime.now()}") return series_list def _create_figure(self): """ Create a box plot from defaults and custom parameters """ - self.hist_logger.info(f"Begin creating the histogram figure: {datetime.now()}") + self.logger.info(f"Begin creating the histogram figure: {datetime.now()}") # create and draw the plot self.figure = self._create_layout() self._add_xaxis() @@ -271,7 +271,7 @@ def _create_figure(self): self.config_obj ) - self.hist_logger.info(f"Finished creating the histogram figure: " + self.logger.info(f"Finished creating the histogram figure: " f"{datetime.now()}") def _draw_series(self, series: HistSeries) -> None: @@ -299,7 +299,7 @@ def _create_layout(self) -> Figure: :return: Figure object """ - self.hist_logger.info(f"Creating the layout: {datetime.now()}") + self.logger.info(f"Creating the layout: {datetime.now()}") # create annotation annotation = [ @@ -344,14 +344,14 @@ def _create_layout(self) -> Figure: plot_bgcolor=PLOTLY_PAPER_BGCOOR ) - self.hist_logger.info(f"Finished creating the layout: {datetime.now()}") + self.logger.info(f"Finished creating the layout: {datetime.now()}") return fig def _add_xaxis(self) -> None: """ Configures and adds x-axis to the plot """ - self.hist_logger.info(f"Configuring and adding the x-axis: {datetime.now()}") + self.logger.info(f"Configuring and adding the x-axis: {datetime.now()}") self.figure.update_xaxes(title_text=self.config_obj.xaxis, linecolor=PLOTLY_AXIS_LINE_COLOR, @@ -370,7 +370,7 @@ def _add_xaxis(self) -> None: tickfont={'size': self.config_obj.x_tickfont_size}, dtick=self._get_dtick() ) - self.hist_logger.info(f"Finished configuring and adding the x-axis:" + self.logger.info(f"Finished configuring and adding the x-axis:" f" {datetime.now()}") def _add_yaxis(self) -> None: @@ -378,7 +378,7 @@ def _add_yaxis(self) -> None: Configures and adds y-axis to the plot """ - self.hist_logger.info(f"Configuring and adding the y-axis: {datetime.now()}") + self.logger.info(f"Configuring and adding the y-axis: {datetime.now()}") self.figure.update_yaxes(title_text= util.apply_weight_style(self.config_obj.yaxis_1, self.config_obj.parameters[ @@ -400,7 +400,7 @@ def _add_yaxis(self) -> None: tickangle=self.config_obj.y_tickangle, tickfont={'size': self.config_obj.y_tickfont_size} ) - self.hist_logger.info(f"Finished configuring and adding the y-axis:" + self.logger.info(f"Finished configuring and adding the y-axis:" f" {datetime.now()}") def _add_legend(self) -> None: @@ -409,7 +409,7 @@ def _add_legend(self) -> None: and attaches it to the initial Figure """ - self.hist_logger.info(f"Adding the legend: {datetime.now()}") + self.logger.info(f"Adding the legend: {datetime.now()}") self.figure.update_layout(legend={'x': self.config_obj.bbox_x, 'y': self.config_obj.bbox_y, 'xanchor': 'center', @@ -425,7 +425,7 @@ def _add_legend(self) -> None: 'color': "black" } }) - self.hist_logger.info(f"Finished adding the legend: {datetime.now()}") + self.logger.info(f"Finished adding the legend: {datetime.now()}") def write_html(self) -> None: """ @@ -433,7 +433,7 @@ def write_html(self) -> None: Plotly.js """ - self.hist_logger.info(f"Begin writing html: {datetime.now()}") + self.logger.info(f"Begin writing html: {datetime.now()}") if self.config_obj.create_html is True: # construct the file name from plot_filename @@ -444,13 +444,13 @@ def write_html(self) -> None: # save html self.figure.write_html(html_name, include_plotlyjs=False) - self.hist_logger.info(f"Finished writing html: {datetime.now()}") + self.logger.info(f"Finished writing html: {datetime.now()}") def write_output_file(self) -> None: """ saves box points to the file """ - self.hist_logger.info(f"Begin writing the output file: {datetime.now()}") + self.logger.info(f"Begin writing the output file: {datetime.now()}") # if points_path parameter doesn't exist, # open file, name it based on the stat_input config setting, @@ -481,4 +481,4 @@ def write_output_file(self) -> None: file.writelines('\n') file.close() - self.hist_logger.info(f"Finished writing the output file: {datetime.now()}") + self.logger.info(f"Finished writing the output file: {datetime.now()}") diff --git a/metplotpy/plots/histogram/prob_hist.py b/metplotpy/plots/histogram/prob_hist.py index 1f92a914..d4cd7aae 100644 --- a/metplotpy/plots/histogram/prob_hist.py +++ b/metplotpy/plots/histogram/prob_hist.py @@ -15,7 +15,6 @@ from typing import Union from datetime import datetime -import yaml from metplotpy.plots.histogram.hist import Hist from metplotpy.plots.histogram.hist_series import HistSeries @@ -26,12 +25,10 @@ class ProbHist(Hist): """ Generates a Plotly Probability Histogram or Histograms of probability integral transform plot for 1 or more traces """ - + LONG_NAME = 'probability histogram' config_obj_name = 'ProbHistogramConfig' series_obj = 'ProbHistSeries' - - def _get_x_points(self, series: HistSeries) -> list: x_points = [] for ser in self.series_list: @@ -66,30 +63,17 @@ def main(config_filename=None): Args: @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - + params = util.get_params(config_filename) try: - - plot = ProbHist(docs) + plot = ProbHist(params) plot.save_to_file() # plot.show_in_browser() plot.write_html() plot.write_output_file() log_level = plot.get_config_value('log_level') log_filename = plot.get_config_value('log_filename') - prob_logger = util.get_common_logger(log_level, log_filename) - prob_logger.info(f"Finished probability histogram: {datetime.now()}") + logger = util.get_common_logger(log_level, log_filename) + logger.info(f"Finished probability histogram: {datetime.now()}") except ValueError as val_er: print(val_er) diff --git a/metplotpy/plots/histogram/rank_hist.py b/metplotpy/plots/histogram/rank_hist.py index c0f0ce99..a9a054b9 100644 --- a/metplotpy/plots/histogram/rank_hist.py +++ b/metplotpy/plots/histogram/rank_hist.py @@ -23,13 +23,12 @@ class RankHist(Hist): """ Generates a Plotly Histograms of ensemble rank plot for 1 or more traces """ - + LONG_NAME = 'rank histogram' config_obj_name='RankHistogramConfig' series_obj='RankHistSeries' def _get_x_points(self, series: HistSeries) -> list: - self.hist_logger.info(f"Retrieving x points for rank histogram:" - f" {datetime.now()}") + self.logger.info(f"Retrieving x points for {self.LONG_NAME}: {datetime.now()}") return sorted(series.series_data['i_value'].unique()) @@ -42,28 +41,7 @@ def main(config_filename=None): Args: @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = RankHist(docs) - plot.save_to_file() - # plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.hist_logger.info(f"Finished creating rank histogram: {datetime.now()}") - except ValueError as val_er: - print(val_er) + util.make_plot(config_filename, RankHist) if __name__ == "__main__": diff --git a/metplotpy/plots/histogram/rel_hist.py b/metplotpy/plots/histogram/rel_hist.py index bebb7e84..3040451e 100644 --- a/metplotpy/plots/histogram/rel_hist.py +++ b/metplotpy/plots/histogram/rel_hist.py @@ -25,13 +25,12 @@ class RelHist(Hist): """ Generates a Plotly Relative Histogram or Histograms of relative position plot for 1 or more traces """ - + LONG_NAME = 'relative frequency histogram' config_obj_name='RelHistogramConfig' series_obj='RelHistSeries' def _get_x_points(self, series: HistSeries) -> list: - self.hist_logger.info(f"Retrieving x points for relative frequency histogram: " - f"{datetime.now()}") + self.logger.info(f"Retrieving x points for {self.LONG_NAME}: {datetime.now()}") return sorted(series.series_data['i_value'].unique()) @@ -45,29 +44,7 @@ def main(config_filename=None): Args: @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = RelHist(docs) - plot.save_to_file() - # plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.hist_logger.info(f"Finished creating the relative frequency histogram: " - f"{datetime.now()}") - except ValueError as val_er: - print(val_er) + util.make_plot(config_filename, RelHist) if __name__ == "__main__": diff --git a/metplotpy/plots/histogram_2d/histogram_2d.py b/metplotpy/plots/histogram_2d/histogram_2d.py index 648bd921..3f22aa06 100644 --- a/metplotpy/plots/histogram_2d/histogram_2d.py +++ b/metplotpy/plots/histogram_2d/histogram_2d.py @@ -169,9 +169,6 @@ def write_output_file(self): self.logger.info(f"Finished writing plot to output file: {datetime.now()}") - - - def _read_input_data(self): """ Read the input data file and store as an xarray @@ -187,32 +184,16 @@ def _read_input_data(self): try: ds = xr.open_dataset(self.input_file) except IOError: - print("Unable to open input file") + print(f"Unable to open input file: {self.input_file}") sys.exit(1) self.logger.info(f"Finished reading input data: {datetime.now()}") return ds def main(config_filename=None): - metplotpy_base = os.getenv('METPLOTPY_BASE') - if not metplotpy_base: - metplotpy_base = '' - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - + params = util.get_params(config_filename) try: - h = Histogram_2d(docs) + h = Histogram_2d(params) h.save_to_file() h.logger.info(f"Finished generating histogram 2D plot: {datetime.now()}") except ValueError as ve: diff --git a/metplotpy/plots/hovmoeller/hovmoeller.py b/metplotpy/plots/hovmoeller/hovmoeller.py index 59d5801d..b189a903 100644 --- a/metplotpy/plots/hovmoeller/hovmoeller.py +++ b/metplotpy/plots/hovmoeller/hovmoeller.py @@ -231,23 +231,7 @@ def main(config_filename=None): """ - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r'): - try: - # Use the METcalcpy parser to parse config files with environment - # variables that - # look like: - # input_file: !ENV '${ENV_NAME}/some_input_file.nc' - # This supports METplus hovmoeller use case. - config = metcalcpy.util.read_env_vars_in_config.parse_config(config_file) - except yaml.YAMLError as exc: - print(exc) - + config = util.get_params(config_filename) try: plot = Hovmoeller(config) plot.save_to_file() diff --git a/metplotpy/plots/line/line.py b/metplotpy/plots/line/line.py index 40cc05a4..21151dc4 100644 --- a/metplotpy/plots/line/line.py +++ b/metplotpy/plots/line/line.py @@ -65,9 +65,9 @@ def __init__(self, parameters: dict) -> None: # config file that represents the BasePlot object (Line). self.config_obj = LineConfig(self.parameters) - self.line_logger = self.config_obj.logger + self.logger = self.config_obj.logger - self.line_logger.info(f"Begin creating the line plot: {datetime.now()}") + self.logger.info(f"Begin creating the line plot: {datetime.now()}") # Check that we have all the necessary settings for each series is_config_consistent = self.config_obj._config_consistency_check() @@ -78,7 +78,7 @@ def __init__(self, parameters: dict) -> None: "the number of your configuration file's plot_ci, " "plot_disp, series_order, user_legend " "colors, series_symbols, show_legend settings.") - self.line_logger.error(f"ValueError: {error_msg}: {datetime.now()}") + self.logger.error(f"ValueError: {error_msg}: {datetime.now()}") raise ValueError(error_msg) # Read in input data, location specified in config file @@ -86,10 +86,10 @@ def __init__(self, parameters: dict) -> None: # Apply event equalization, if requested if self.config_obj.use_ee is True: - self.line_logger.info(f"Begin event equalization: {datetime.now()}") + self.logger.info(f"Begin event equalization: {datetime.now()}") self.input_df = calc_util.perform_event_equalization(self.parameters, self.input_df) - self.line_logger.info(f"Finished event equalization: {datetime.now()}") + self.logger.info(f"Finished event equalization: {datetime.now()}") # Create a list of series objects. # Each series object contains all the necessary information for plotting, @@ -147,7 +147,7 @@ def _create_series(self, input_data): """ - self.line_logger.info(f"Begin creating the series objects: {datetime.now()}") + self.logger.info(f"Begin creating the series objects: {datetime.now()}") series_list = [] # add series for y1 axis @@ -192,14 +192,14 @@ def _create_series(self, input_data): # reorder series series_list = self.config_obj.create_list_by_series_ordering(series_list) - self.line_logger.info(f"Finished creating the series objects: {datetime.now()}") + self.logger.info(f"Finished creating the series objects: {datetime.now()}") return series_list def _create_figure(self) -> None: """ Create a line plot from defaults and custom parameters """ - self.line_logger.info(f"Begin create the figure: {datetime.now()}") + self.logger.info(f"Begin create the figure: {datetime.now()}") # create and draw the plot self.figure = self._create_layout() @@ -270,7 +270,7 @@ def _create_figure(self) -> None: if self.config_obj.start_from_zero is True: self.figure.update_xaxes(range=[0, len(x_points_index) - 1]) - self.line_logger.info(f"Finished creating the figure: {datetime.now()}") + self.logger.info(f"Finished creating the figure: {datetime.now()}") def _draw_series(self, series: Series, x_points_index_adj: Union[list, None] = None) \ @@ -281,7 +281,7 @@ def _draw_series(self, series: Series, x_points_index_adj: Union[list, None] = :param series: Line series object with data and parameters :param x_points_index_adj: values for adjusting x-values position """ - self.line_logger.info(f"Begin drawing the lines on the plot: {datetime.now()}") + self.logger.info(f"Begin drawing the lines on the plot: {datetime.now()}") y_points = series.series_points['dbl_med'] # show or not ci @@ -354,7 +354,7 @@ def _draw_series(self, series: Series, x_points_index_adj: Union[list, None] = secondary_y=series.y_axis != 1 ) - self.line_logger.info(f"Finished drawing the lines on the plot:" + self.logger.info(f"Finished drawing the lines on the plot:" f" {datetime.now()}") def _create_layout(self) -> Figure: @@ -440,7 +440,7 @@ def _adjust_for_vertical(self, x_points_index: list) -> None: :param x_points_index: list of indexws for the original x -axis """ - self.line_logger.info(f"Begin switching x and y axis: {datetime.now()}") + self.logger.info(f"Begin switching x and y axis: {datetime.now()}") odered_indy_label = self.config_obj.create_list_by_plot_val_ordering( self.config_obj.indy_label) if self.config_obj.vert_plot is True: @@ -460,7 +460,7 @@ def _adjust_for_vertical(self, x_points_index: list) -> None: } ) - self.line_logger.info(f"Finished switching x and y axis: {datetime.now()}") + self.logger.info(f"Finished switching x and y axis: {datetime.now()}") def _add_xaxis(self) -> None: """ @@ -703,7 +703,7 @@ def write_output_file(self) -> None: files """ - self.line_logger.info(f"Begin writing to output file: {datetime.now()}") + self.logger.info(f"Begin writing to output file: {datetime.now()}") # if points_path parameter doesn't exist, # open file, name it based on the stat_input config setting, # (the input data file) except replace the .data @@ -753,7 +753,7 @@ def write_output_file(self) -> None: self._save_points(all_points_1, filename + ".points1") self._save_points(all_points_2, filename + ".points2") - self.line_logger.info(f"Finished writing to output file: {datetime.now()}") + self.logger.info(f"Finished writing to output file: {datetime.now()}") @staticmethod def _find_min_max(series: LineSeries, yaxis_min: Union[float, None], @@ -856,29 +856,7 @@ def main(config_filename=None): @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./custom_line_plot.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = Line(docs) - plot.save_to_file() - #plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.line_logger.info(f"Finished creating line plot: {datetime.now()}") - except ValueError as val_er: - print(val_er) + util.make_plot(config_filename, Line) if __name__ == "__main__": diff --git a/metplotpy/plots/mpr_plot/mpr_plot.py b/metplotpy/plots/mpr_plot/mpr_plot.py index 7f6d65a0..329d5b27 100644 --- a/metplotpy/plots/mpr_plot/mpr_plot.py +++ b/metplotpy/plots/mpr_plot/mpr_plot.py @@ -587,22 +587,9 @@ def main(config_filename=None): The location of the input data is defined in either the default or custom config file. """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./custom_line_plot.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - + params = util.get_params(config_filename) try: - plot = MprPlot(docs) + plot = MprPlot(params) plot.save_to_file() if plot.config_obj.show_in_browser: plot.show_in_browser() diff --git a/metplotpy/plots/performance_diagram/performance_diagram.py b/metplotpy/plots/performance_diagram/performance_diagram.py index 4507f6ef..e5f92ed2 100644 --- a/metplotpy/plots/performance_diagram/performance_diagram.py +++ b/metplotpy/plots/performance_diagram/performance_diagram.py @@ -21,7 +21,6 @@ from matplotlib.colors import LinearSegmentedColormap from matplotlib.font_manager import FontProperties import numpy as np -import yaml import pandas as pd from metplotpy.plots.base_plot import BasePlot import metcalcpy.util.utils as calc_util @@ -471,23 +470,10 @@ def main(config_filename=None): Returns: """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./custom_performance_diagram.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - + params = util.get_params(config_filename) try: # create a performance diagram - PerformanceDiagram(docs) + PerformanceDiagram(params) except ValueError as value_error: print(value_error) diff --git a/metplotpy/plots/reliability_diagram/reliability.py b/metplotpy/plots/reliability_diagram/reliability.py index 22764c86..b83f9572 100644 --- a/metplotpy/plots/reliability_diagram/reliability.py +++ b/metplotpy/plots/reliability_diagram/reliability.py @@ -19,7 +19,6 @@ from datetime import datetime from typing import Union -import yaml import numpy as np import pandas as pd @@ -731,29 +730,7 @@ def main(config_filename=None): Args: @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./custom_line_plot.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = Reliability(docs) - plot.save_to_file() - #plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.logger.info(f"Finished generating reliability diagram: {datetime.now()}") - except ValueError as val_er: - print(val_er) + util.make_plot(config_filename, Reliability) if __name__ == "__main__": diff --git a/metplotpy/plots/revision_box/revision_box.py b/metplotpy/plots/revision_box/revision_box.py index 3b7b5518..d9336712 100644 --- a/metplotpy/plots/revision_box/revision_box.py +++ b/metplotpy/plots/revision_box/revision_box.py @@ -13,7 +13,6 @@ import os import re from datetime import datetime -import yaml import plotly.graph_objects as go from metplotpy.plots.base_plot import BasePlot @@ -30,7 +29,7 @@ class RevisionBox(Box): """ Generates a Plotly Revision box plot for 1 or more boxes. """ - + LONG_NAME = 'revision box' defaults_name = 'revision_box_defaults.yaml' def __init__(self, parameters: dict) -> None: @@ -303,29 +302,7 @@ def main(config_filename=None): Args: @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./custom_revision_box.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = RevisionBox(docs) - plot.save_to_file() - # plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.logger.info(f"Finished revision box plot: {datetime.now()}") - except ValueError as ve: - print(ve) + util.make_plot(config_filename, RevisionBox) if __name__ == "__main__": diff --git a/metplotpy/plots/revision_series/revision_series.py b/metplotpy/plots/revision_series/revision_series.py index 8ef2fd02..acf7621e 100644 --- a/metplotpy/plots/revision_series/revision_series.py +++ b/metplotpy/plots/revision_series/revision_series.py @@ -38,7 +38,7 @@ class RevisionSeries(Line): """ Generates a Plotly Revision Series plot for 1 or more traces where each dot is represented by a text point data file. """ - + LONG_NAME = 'revision series' defaults_name = 'revision_series_defaults.yaml' def __init__(self, parameters: dict) -> None: @@ -59,7 +59,8 @@ def __init__(self, parameters: dict) -> None: # config file that represents the BasePlot object (RevisionSeries). self.config_obj = RevisionSeriesConfig(self.parameters) - self.config_obj.logger.info(f'Begin revision series plotting.') + self.logger = self.config_obj.logger + self.logger.info('Begin revision series plotting.') # Check that we have all the necessary settings for each series is_config_consistent = self.config_obj._config_consistency_check() @@ -70,7 +71,7 @@ def __init__(self, parameters: dict) -> None: " the number of your configuration file's plot_i," " plot_disp, series_order, user_legend," " colors, show_legend and series_symbols settings.") - self.config_obj.logger.error(f"ValueError: {value_error_msg}") + self.logger.error(f"ValueError: {value_error_msg}") raise ValueError(value_error_msg) # Read in input data, location specified in config file @@ -133,8 +134,7 @@ def _create_figure(self): Create a Revision Series plot from defaults and custom parameters """ - self.config_obj.logger.info(f"Begin creating the revision series figure:" - f" {datetime.now()}") + self.logger.info(f"Begin creating the {self.LONG_NAME} figure: {datetime.now()}") # create and draw the plot self.figure = self._create_layout() self._add_xaxis() @@ -179,8 +179,7 @@ def _create_figure(self): # apply y axis limits self._yaxis_limits() - self.config_obj.logger.info(f"Finish creating revision series figure: " - f"{datetime.now()}") + self.logger.info(f"Finish creating {self.LONG_NAME} figure: {datetime.now()}") def _draw_series(self, series: Series, x_points_index_adj: Union[list, None] = None) -> None: """ @@ -190,7 +189,7 @@ def _draw_series(self, series: Series, x_points_index_adj: Union[list, None] = N :param x_points_index_adj: values for adjusting x-values position """ - self.config_obj.logger.info(f"Draw the formatted series: {datetime.now()}") + self.logger.info(f"Draw the formatted series: {datetime.now()}") y_points = series.series_points['points']['stat_value'].tolist() # add the plot @@ -208,7 +207,7 @@ def _draw_series(self, series: Series, x_points_index_adj: Union[list, None] = N ), secondary_y=False ) - self.config_obj.logger.info(f"Finished drawing series: {datetime.now()}") + self.logger.info(f"Finished drawing series: {datetime.now()}") def _add_xaxis(self) -> None: """ @@ -239,7 +238,7 @@ def _calc_stag_adjustments(self) -> list: :return: the list of the adjustment values """ - self.config_obj.logger.info("Calculating the x-axis adjustment.") + self.logger.info("Calculating the x-axis adjustment.") # get the total number of series num_stag = len(self.config_obj.all_series_y1) @@ -260,7 +259,7 @@ def write_output_file(self) -> None: """ Formats y1 series point data and saves them to the files """ - self.config_obj.logger.info("Write output file") + self.logger.info("Write output file") # if points_path parameter doesn't exist, # open file, name it based on the stat_input config setting, # (the input data file) except replace the .data @@ -305,30 +304,7 @@ def main(config_filename=None): Args: @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./revision_series_defaults.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - try: - plot = RevisionSeries(docs) - plot.save_to_file() - #plot.show_in_browser() - plot.write_html() - plot.write_output_file() - plot.config_obj.logger.info(f"Finished creating revision series plot: " - f"{datetime.now()}") - except ValueError as val_er: - print(val_er) + util.make_plot(config_filename, RevisionSeries) if __name__ == "__main__": diff --git a/metplotpy/plots/roc_diagram/roc_diagram.py b/metplotpy/plots/roc_diagram/roc_diagram.py index f12c3fa5..c728e0c3 100644 --- a/metplotpy/plots/roc_diagram/roc_diagram.py +++ b/metplotpy/plots/roc_diagram/roc_diagram.py @@ -15,7 +15,6 @@ import os from datetime import datetime -import yaml import re import warnings # with warnings.catch_warnings(): @@ -564,22 +563,9 @@ def main(config_filename=None): @param config_filename: default is None, the name of the custom config file to apply Returns: """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./custom_performance_diagram.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - + params = util.get_params(config_filename) try: - r = ROCDiagram(docs) + r = ROCDiagram(params) r.save_to_file() r.write_html() diff --git a/metplotpy/plots/scatter/scatter.py b/metplotpy/plots/scatter/scatter.py index 0c8f6c9f..24203d87 100644 --- a/metplotpy/plots/scatter/scatter.py +++ b/metplotpy/plots/scatter/scatter.py @@ -18,6 +18,9 @@ import pandas as pd from plots.base_plot import BasePlot +from metplotpy.plots import util + + class Scatter(BasePlot): """ Generates a Plotly scatter plot, @@ -126,20 +129,11 @@ def main(): The location of the input data is defined in either the default or custom config file. """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - with open("./custom_scatter.yaml", 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - - + params = util.get_params("./custom_scatter.yaml") try: - s = Scatter(docs) - s.save_to_file() - s.show_in_browser() + s = Scatter(params) + s.save_to_file() + s.show_in_browser() except ValueError as ve: print(ve) diff --git a/metplotpy/plots/skew_t/skew_t.py b/metplotpy/plots/skew_t/skew_t.py index caadfb99..63a0a32e 100644 --- a/metplotpy/plots/skew_t/skew_t.py +++ b/metplotpy/plots/skew_t/skew_t.py @@ -37,7 +37,7 @@ warnings.filterwarnings(action='ignore', category=UserWarning) -def extract_sounding_data(input_file): +def extract_sounding_data(input_file, output_directory): with open(input_file) as infile: data = infile.readlines() @@ -64,12 +64,14 @@ def extract_sounding_data(input_file): # Save the sounding data into a text file, which will then be converted into a # pandas dataframe. - with open("sounding_data.dat", "w") as txt_file: + sounding_data_file = os.path.join(output_directory, 'sounding_data.dat') + os.makedirs(output_directory, exist_ok=True) + with open(sounding_data_file, "w") as txt_file: for line in sounding_data: txt_file.write("".join(line) + "\n") # Read in the current sounding data file, replacing any 9999 values with NaN. - df_raw: pandas.DataFrame = pd.read_csv("sounding_data.dat", delim_whitespace=True, + df_raw: pandas.DataFrame = pd.read_csv(sounding_data_file, delim_whitespace=True, skiprows=1, na_values=['9999']) @@ -503,8 +505,6 @@ def create_skew_t(input_file: str, config: dict) -> None: Return: None, generate plots as png files in the specified output file directory. ''' - - file_only = os.path.basename(input_file) logger.info(f" Creating skew T plots for input file {file_only} ") @@ -515,7 +515,7 @@ def create_skew_t(input_file: str, config: dict) -> None: f"GENERATED.") return - sounding_df, plevs = extract_sounding_data(input_file) + sounding_df, plevs = extract_sounding_data(input_file, config['output_directory']) # Check if sounding data consists entirely of na-values. all_na = check_for_all_na(sounding_df) @@ -690,59 +690,50 @@ def main(config_filename=None): Returns: ''' - if not config_filename: - config_file = util.read_config_from_command_line() + config = util.get_params(config_filename) + + # Set up the logging. + log_dir = config['log_directory'] + log_file = config['log_filename'] + log_full_path = os.path.join(log_dir, log_file) + try: + os.makedirs(log_dir, exist_ok=True) + except FileExistsError: + # If directory already exists, this is OK. Continue. + pass + + log_level = config['log_level'] + format_str = "'%(asctime)s||%(levelname)s||%(funcName)s||%(message)s'" + if log_level == 'DEBUG': + logging.basicConfig(filename=log_full_path, level=logging.DEBUG, + format=format_str, + filemode='w') + elif log_level == 'INFO': + logging.basicConfig(filename=log_full_path, level=logging.INFO, + format=format_str, + filemode='w') + elif log_level == 'WARNING': + logging.basicConfig(filename=log_full_path, level=logging.WARNING, + format=format_str, + filemode='w') else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - config = yaml.load(stream, Loader=yaml.FullLoader) - - # Set up the logging. - log_dir = config['log_directory'] - log_file = config['log_filename'] - log_full_path = os.path.join(log_dir, log_file) - try: - os.makedirs(log_dir, exist_ok=True) - except FileExistsError: - # If directory already exists, this is OK. Continue. - pass - - log_level = config['log_level'] - format_str = "'%(asctime)s||%(levelname)s||%(funcName)s||%(message)s'" - if log_level == 'DEBUG': - logging.basicConfig(filename=log_full_path, level=logging.DEBUG, - format=format_str, - filemode='w') - elif log_level == 'INFO': - logging.basicConfig(filename=log_full_path, level=logging.INFO, - format=format_str, - filemode='w') - elif log_level == 'WARNING': - logging.basicConfig(filename=log_full_path, level=logging.WARNING, - format=format_str, - filemode='w') - else: - # log_level == 'ERROR' - logging.basicConfig(filename=log_full_path, level=logging.ERROR, - format=format_str, - filemode='w') - - # Get the list of input files to visualize. - input_dir = config['input_directory'] - file_ext = config['input_file_extension'] - files_of_interest = [] - - for root, dir, files in os.walk(input_dir): - for item in files: - if item.endswith(file_ext): - files_of_interest.append(os.path.join(root, item)) - # Create skew T diagrams for each input file. - for file_of_interest in files_of_interest: - create_skew_t(file_of_interest, config) - - except yaml.YAMLError as exc: - logger.error(f"YAMLError: {exc}") + # log_level == 'ERROR' + logging.basicConfig(filename=log_full_path, level=logging.ERROR, + format=format_str, + filemode='w') + + # Get the list of input files to visualize. + input_dir = config['input_directory'] + file_ext = config['input_file_extension'] + files_of_interest = [] + + for root, _, files in os.walk(input_dir): + for item in files: + if item.endswith(file_ext): + files_of_interest.append(os.path.join(root, item)) + # Create skew T diagrams for each input file. + for file_of_interest in files_of_interest: + create_skew_t(file_of_interest, config) if __name__ == "__main__": diff --git a/metplotpy/plots/taylor_diagram/taylor_diagram.py b/metplotpy/plots/taylor_diagram/taylor_diagram.py index f80a112c..6dfa5d02 100644 --- a/metplotpy/plots/taylor_diagram/taylor_diagram.py +++ b/metplotpy/plots/taylor_diagram/taylor_diagram.py @@ -22,14 +22,11 @@ import warnings from datetime import datetime import matplotlib.pyplot as plt -import numpy -import pandas from matplotlib.font_manager import FontProperties from matplotlib.projections import PolarAxes import mpl_toolkits.axisartist.floating_axes as fa import mpl_toolkits.axisartist.grid_finder as gf import numpy as np -import yaml import pandas as pd from metplotpy.plots import constants from metplotpy.plots.base_plot import BasePlot @@ -192,7 +189,7 @@ def _create_figure(self) -> None: rlocs = np.concatenate((-rlocs[:0:-1], rlocs)) # Convert to polar angles - tick_locations: numpy.array = np.arccos(rlocs) + tick_locations: np.array = np.arccos(rlocs) # positions gl1 = gf.FixedLocator(tick_locations) tf1 = gf.DictFormatter(dict(zip(tick_locations, map(str, rlocs)))) @@ -361,25 +358,12 @@ def main(config_filename=None): Returns: """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./custom_taylor_diagram.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs: dict = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) - + params = util.get_params(config_filename) try: - TaylorDiagram(docs) + TaylorDiagram(params) except ValueError as value_error: - logger = util.get_common_logger(docs['log_level'], docs['log_filename']) + logger = util.get_common_logger(params['log_level'], params['log_filename']) logger.error(f"ValueError {value_error}") diff --git a/metplotpy/plots/tcmpr_plots/tcmpr.py b/metplotpy/plots/tcmpr_plots/tcmpr.py index 034fbfb5..9988e064 100755 --- a/metplotpy/plots/tcmpr_plots/tcmpr.py +++ b/metplotpy/plots/tcmpr_plots/tcmpr.py @@ -523,18 +523,7 @@ def main(config_filename=None): Args: @param config_filename: default is None, the name of the custom config file to apply """ - - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) + docs = util.get_params(config_filename) # Determine location of the default YAML config files and then # read defaults stored in YAML formatted file into the dictionary @@ -543,11 +532,7 @@ def main(config_filename=None): else: location = os.path.realpath(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'config')) - with open(os.path.join(location, "tcmpr_defaults.yaml"), 'r') as stream: - try: - defaults = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) + defaults = util.get_params(os.path.join(location, "tcmpr_defaults.yaml")) # merge user defined parameters into defaults if they exist docs = {**defaults, **docs} diff --git a/metplotpy/plots/util.py b/metplotpy/plots/util.py index ab94c2b4..dc024000 100644 --- a/metplotpy/plots/util.py +++ b/metplotpy/plots/util.py @@ -18,6 +18,7 @@ import logging import gc import re +from datetime import datetime import matplotlib import numpy as np from typing import Union @@ -26,6 +27,7 @@ from metplotpy.plots.context_filter import ContextFilter as cf import metcalcpy.util.pstd_statistics as pstats import metcalcpy.util.ctc_statistics as cstats +from metcalcpy.util.read_env_vars_in_config import parse_config COLORSCALES = { 'green_red': ['#E6FFE2', '#B3FAAD', '#74F578', '#30D244', '#00A01E', '#F6A1A2', @@ -72,6 +74,43 @@ def read_config_from_command_line(): return args.Path +def get_params(config_filename): + """!Read config_filename or get config file from command line, then parse + config file and return it as a dictionary. + + @param config_filename The full path to the config file or None + @returns dictionary containing parameters for plot + """ + config_file = config_filename if config_filename else read_config_from_command_line() + return parse_config(config_file) + + +def make_plot(config_filename, plot_class): + """!Get plot parameters and create the plot. + + @param config_filename The full path to the config or None + @param plot_class class of plot to produce, e.g. Bar or Box + @returns plot class object or None if something went wrong + """ + # Retrieve the contents of the custom config file to over-ride + # or augment settings defined by the default config file. + params = get_params(config_filename) + try: + plot = plot_class(params) + plot.save_to_file() + #if plot.config_obj.show_in_browser: + # plot.show_in_browser() + plot.write_html() + plot.write_output_file() + name = plot_class.__name__ if not hasattr(plot_class, 'LONG_NAME') else plot_class.LONG_NAME + plot.logger.info(f"Finished {name} plot at {datetime.now()}") + return plot + except ValueError as val_er: + print(val_er) + + return None + + def alpha_blending(hex_color: str, alpha: float) -> str: """ Alpha color blending as if on the white background. Useful for gridlines diff --git a/metplotpy/plots/wind_rose/wind_rose.py b/metplotpy/plots/wind_rose/wind_rose.py index 895d84a4..cc4c9f9f 100644 --- a/metplotpy/plots/wind_rose/wind_rose.py +++ b/metplotpy/plots/wind_rose/wind_rose.py @@ -20,7 +20,6 @@ from typing import Union import pandas as pd import numpy as np -import yaml import re from pathlib import Path @@ -410,25 +409,14 @@ def main(config_filename=None): The location of the input data is defined in either the default or custom config file. """ + params = util.get_params(config_filename) - # Retrieve the contents of the custom config file to over-ride - # or augment settings defined by the default config file. - # with open("./mpr_plot_custom.yaml", 'r') as stream: - if not config_filename: - config_file = util.read_config_from_command_line() - else: - config_file = config_filename - with open(config_file, 'r') as stream: - try: - docs = yaml.load(stream, Loader=yaml.FullLoader) - except yaml.YAMLError as exc: - print(exc) # point to data file in the test dir - if 'stat_input' not in docs: - docs['stat_input'] = str(Path(__file__).parent.parent.parent.parent) + '/test/wind_rose/point_stat_mpr.txt' + if 'stat_input' not in params: + params['stat_input'] = str(Path(__file__).parent.parent.parent.parent) + '/test/wind_rose/point_stat_mpr.txt' try: - plot = WindRosePlot(docs) + plot = WindRosePlot(params) plot.save_to_file() if plot.config_obj.show_in_browser: plot.show_in_browser() diff --git a/test/bar/custom_bar.yaml b/test/bar/custom_bar.yaml index 755b219f..3361c2c7 100644 --- a/test/bar/custom_bar.yaml +++ b/test/bar/custom_bar.yaml @@ -135,8 +135,8 @@ ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -stat_input: ./bar.data -plot_filename: ./bar.png +stat_input: !ENV '${TEST_DIR}/bar.data' +plot_filename: !ENV '${TEST_DIR}/bar.png' # To save your log output to a file, specify a path and filename and uncomment the line below. Make sure you have # permissions to the directory you specify. The default, as specified in the default config file is stdout. diff --git a/test/bar/custom_defaultpoints1_bar.yaml b/test/bar/custom_defaultpoints1_bar.yaml index 87991b2b..ba734352 100644 --- a/test/bar/custom_defaultpoints1_bar.yaml +++ b/test/bar/custom_defaultpoints1_bar.yaml @@ -130,8 +130,8 @@ ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -stat_input: ./bar.data -plot_filename: ./bar_defaultpoints1.png +stat_input: !ENV '${TEST_DIR}/bar.data' +plot_filename: !ENV '${TEST_DIR}/bar_defaultpoints1.png' show_legend: -True -True \ No newline at end of file diff --git a/test/bar/custom_points1_bar.yaml b/test/bar/custom_points1_bar.yaml index 7921e801..bfc23987 100644 --- a/test/bar/custom_points1_bar.yaml +++ b/test/bar/custom_points1_bar.yaml @@ -15,7 +15,7 @@ con_series: create_html: 'False' derived_series_1: [] dump_points_1: 'True' -points_path: './intermed_files' +points_path: !ENV '${TEST_DIR}/intermed_files' eqbound_high: 0.001 eqbound_low: -0.001 event_equal: 'False' @@ -129,8 +129,8 @@ ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -stat_input: ./bar.data -plot_filename: ./bar_points1.png +stat_input: !ENV '${TEST_DIR}/bar.data' +plot_filename: !ENV '${TEST_DIR}/bar_points1.png' show_legend: -True diff --git a/test/bar/intermed_files/nan.points1 b/test/bar/nan.points1 similarity index 100% rename from test/bar/intermed_files/nan.points1 rename to test/bar/nan.points1 diff --git a/test/bar/test_bar.py b/test/bar/test_bar.py index 18eb9fae..eb0c5fee 100644 --- a/test/bar/test_bar.py +++ b/test/bar/test_bar.py @@ -5,60 +5,46 @@ from metplotpy.plots.bar import bar # from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) +CLEANUP_FILES = ['bar.png', 'bar.points1'] + @pytest.fixture -def setup(): +def setup(remove_files, setup_env): # Cleanup the plotfile and point1 output file from any previous run - cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_bar.yaml" + remove_files(cwd, CLEANUP_FILES) + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_bar.yaml" # Invoke the command to generate a Bar plot based on # the custom_bar.yaml custom config file. bar.main(custom_config_filename) + @pytest.fixture -def setup_nones(): +def setup_nones(remove_files, setup_env): # Cleanup the plotfile and point1 output file from any previous run - cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "bar_with_nones.yaml" + remove_files(cwd, CLEANUP_FILES) + setup_env(cwd) + custom_config_filename = f"{cwd}/bar_with_nones.yaml" # Invoke the command to generate a Bar plot based on # the custom_bar.yaml custom config file. bar.main(custom_config_filename) -def cleanup(): - # remove the bbar.png and .points files - # from any previous runs - try: - path = os.getcwd() - plot_file = 'bar.png' - points_file_1 = 'bar.points1' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file_1)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - - -@pytest.mark.parametrize("test_input, expected", - (["./bar_expected.png", True], ["./bar.png", True], ["./bar.points1", True])) -def test_files_exist(setup, test_input, expected): +def test_files_exist(setup, remove_files): """ Checking that the plot and data files are getting created """ - assert os.path.isfile(test_input) == expected - cleanup() + check_files = ('bar.png', 'bar.points1') + for test_input in check_files: + print(f'Checking if {cwd}/{test_input} is found') + assert os.path.isfile(f"{cwd}/{test_input}") + remove_files(cwd, check_files) -def test_no_nans_in_points_file(setup): +def test_no_nans_in_points_file(setup, remove_files): """ Checking that the points1 intermediate file does not have any NaN's. This is indicative of a problem with the _create_series_points() method. @@ -68,35 +54,33 @@ def test_no_nans_in_points_file(setup): # Fail if there are any NaN's-this indicates something went wrong with the # line_series.py module's _create_series_points() method. nans_found = False - with open("./bar.points1", "r") as f: + with open(f"{cwd}/bar.points1", "r") as f: data = f.read() if "NaN" in data: nans_found = True - assert nans_found == False - cleanup() + assert not nans_found # Verify that the nan.points1 file does indeed trigger a "nans_found" - with open("./intermed_files/nan.points1", "r") as f: + with open(f"{cwd}/nan.points1", "r") as f: data = f.read() if "NaN" in data: nans_found = True - # assert - assert nans_found == True + assert nans_found + remove_files(cwd, CLEANUP_FILES) @pytest.mark.skip("fails on linux host machines") -def test_images_match(setup): +def test_images_match(setup, remove_files): """ Compare an expected plot with the newly created plot to verify that the plot hasn't changed in appearance. """ - comparison = CompareImages('./bar_expected.png', './bar.png') + comparison = CompareImages(f'{cwd}/bar_expected.png', f'{cwd}/bar.png') assert comparison.mssim == 1 - - cleanup() + remove_files(cwd, CLEANUP_FILES) @pytest.mark.skip("fails on linux host machines") @@ -106,108 +90,72 @@ def test_none_data_images_match(setup_nones): newly created plot to verify that the plot hasn't changed in appearance. """ - comparison = CompareImages('expected_with_nones.png', './bar_with_nones.png') + comparison = CompareImages(f'{cwd}/expected_with_nones.png', f'{cwd}/bar_with_nones.png') assert comparison.mssim == 1 try: - path = os.getcwd() - plot_file = 'bar_with_nones.png' - os.remove(os.path.join(path, plot_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. + plot_file = f'{cwd}/bar_with_nones.png' + os.remove(plot_file) + except OSError: pass -@pytest.mark.parametrize("test_input, expected", - (["./bar_points1.png", True], ["./intermed_files/bar.points1", True])) -def test_point_and_plot_files_exist(test_input, expected): +def test_point_and_plot_files_exist(setup_env, remove_files): """ Checking that the plot and (specified location) intermediate file are getting created """ - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_points1_bar.yaml" - intermed_dir = os.path.join(os.getcwd(), 'intermed_files') + check_files = ("bar_points1.png", "intermed_files/bar.points1") + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_points1_bar.yaml" + intermed_dir = os.path.join(cwd, 'intermed_files') try: os.mkdir(intermed_dir) - except FileExistsError as e: + except FileExistsError: pass # Invoke the command to generate a Bar plot based on # the custom_bar.yaml custom config file. bar.main(custom_config_filename) - assert os.path.isfile(test_input) == expected - # remove the .png and .points files - try: - path = os.getcwd() - plot_file = 'bar_points1.png' - points_file_1 = 'bar.points1' - subdir = os.path.join(path, 'intermed_files') - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(subdir, points_file_1)) - os.rmdir(intermed_dir) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass + for test_input in check_files: + assert os.path.isfile(f"{cwd}/{test_input}") + remove_files(cwd, check_files) -@pytest.mark.parametrize("test_input, expected", - (["./bar_defaultpoints1.png", True], ["./bar.points1", True])) -def test_point_and_plot_files_exist(test_input, expected): +def test_point_and_plot_files_exist_default(setup_env, remove_files): """ Checking that the plot and (specified location) intermediate file are getting created """ - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_defaultpoints1_bar.yaml" + check_files = ("bar_defaultpoints1.png", "bar.points1") + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_defaultpoints1_bar.yaml" # Invoke the command to generate a Bar plot based on # the custom_bar.yaml custom config file. bar.main(custom_config_filename) - assert os.path.isfile(test_input) == expected + for test_input in check_files: + assert os.path.isfile(f"{cwd}/{test_input}") # remove the .png and .points files - try: - path = os.getcwd() - plot_file = 'bar_defaultpoints1.png' - points_file_1 = 'bar.points1' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file_1)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - + remove_files(cwd, check_files) @pytest.mark.skip("fails on linux host machines") -def test_threshold_plotting(): +def test_threshold_plotting(setup_env, remove_files): """ Verify that the bar plot using data with thresholds is correct. """ - # Cleanup the plotfile and point1 output file from any previous run - cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "threshold_bar.yaml" + remove_files(cwd, CLEANUP_FILES) + setup_env(cwd) + custom_config_filename = f"{cwd}/threshold_bar.yaml" # Invoke the command to generate a Bar plot based on # the custom_bar.yaml custom config file. bar.main(custom_config_filename) - comparison = CompareImages('./expected_threshold.png', './threshold_bar.png') + comparison = CompareImages(f'{cwd}/expected_threshold.png', f'{cwd}/threshold_bar.png') assert comparison.mssim == 1 - try: - path = os.getcwd() - plot_file = 'threshold_bar.png' - os.remove(os.path.join(path, plot_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - + remove_files(cwd, ['threshold_bar.png']) diff --git a/test/bar/threshold_bar.yaml b/test/bar/threshold_bar.yaml index 1cee538e..36efd591 100644 --- a/test/bar/threshold_bar.yaml +++ b/test/bar/threshold_bar.yaml @@ -84,7 +84,7 @@ plot_ci: - none plot_disp: - 'True' -plot_filename: threshold_bar.png +plot_filename: !ENV '${TEST_DIR}/threshold_bar.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -112,7 +112,7 @@ show_nstats: 'False' show_signif: - 'False' start_from_zero: 'False' -stat_input: threshold_bar.data +stat_input: !ENV '${TEST_DIR}/threshold_bar.data' sync_yaxes: 'False' title: MET_HYSPLIT_DUST_CSI_Verification_DAY2_12Z_202202 - G246 title_align: 0.5 diff --git a/test/box/custom_box.yaml b/test/box/custom_box.yaml index 0952747f..bd2a1b61 100644 --- a/test/box/custom_box.yaml +++ b/test/box/custom_box.yaml @@ -180,8 +180,8 @@ ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -stat_input: ./box.data -plot_filename: ./box.png +stat_input: !ENV '${TEST_DIR}/box.data' +plot_filename: !ENV '${TEST_DIR}/box.png' # To save your log output to a file, specify a path and filename and uncomment the line below. Make sure you have # permissions to the directory you specify. The default, as specified in the default config file is stdout. diff --git a/test/box/custom_box_defaultpoints1.yaml b/test/box/custom_box_defaultpoints1.yaml index d746a38b..59720457 100644 --- a/test/box/custom_box_defaultpoints1.yaml +++ b/test/box/custom_box_defaultpoints1.yaml @@ -174,8 +174,8 @@ ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -stat_input: ./box.data -plot_filename: ./box_defaultpoints1.png +stat_input: !ENV '${TEST_DIR}/box.data' +plot_filename: !ENV '${TEST_DIR}/box_defaultpoints1.png' show_legend: -True -True diff --git a/test/box/custom_box_points1.yaml b/test/box/custom_box_points1.yaml index d5962f1c..c6be626e 100644 --- a/test/box/custom_box_points1.yaml +++ b/test/box/custom_box_points1.yaml @@ -101,7 +101,7 @@ plot_stat: median plot_type: png16m plot_units: in plot_width: 11.0 -points_path: './intermed_files' +points_path: !ENV '${TEST_DIR}/intermed_files' random_seed: null @@ -173,8 +173,8 @@ ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -stat_input: ./box.data -plot_filename: ./box_points1.png +stat_input: !ENV '${TEST_DIR}/box.data' +plot_filename: !ENV '${TEST_DIR}/box_points1.png' show_legend: -True -True diff --git a/test/box/simple_box.yaml b/test/box/simple_box.yaml index 7bccefaf..c1fc33aa 100644 --- a/test/box/simple_box.yaml +++ b/test/box/simple_box.yaml @@ -112,8 +112,8 @@ title: Simple Example-Box xaxis: FCST_LEAD xaxis_reverse: 'False' -stat_input: ./box.data -plot_filename: ./default_box.png +stat_input: !ENV '${TEST_DIR}/box.data' +plot_filename: !ENV '${TEST_DIR}/default_box.png' show_legend: -True diff --git a/test/box/test_box.py b/test/box/test_box.py index f6fd6fb5..25ab21c4 100644 --- a/test/box/test_box.py +++ b/test/box/test_box.py @@ -3,46 +3,30 @@ from metplotpy.plots.box import box #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) +CLEANUP_FILES = ['box.png', 'box.points1'] @pytest.fixture -def setup(): +def setup(remove_files, setup_env): # Cleanup the plotfile and point1 output file from any previous run - cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - #os.environ['METPLOTPY_BASE'] = "../../metplotpy" - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_box.yaml" + remove_files(cwd, CLEANUP_FILES) + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_box.yaml" # Invoke the command to generate a Performance Diagram based on # the test_custom_performance_diagram.yaml custom config file. box.main(custom_config_filename) -def cleanup(): - # remove the box.png and .points files - # from any previous runs - try: - path = os.getcwd() - plot_file = 'box.png' - points_file_1 = 'box.points1' - os.remove(os.path.join(path, points_file_1)) - os.remove(os.path.join(path, plot_file)) - - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - - @pytest.mark.parametrize("test_input, expected", - (["./box.png", True],["./box.points1", True])) -def test_files_exist(setup, test_input, expected): + (["box.png", True], ["box.points1", True])) +def test_files_exist(setup, test_input, expected, remove_files): """ Checking that the plot and data files are getting created """ - assert os.path.isfile(test_input) == expected - cleanup() + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, CLEANUP_FILES) + @pytest.mark.skip("fails on linux hosts") def test_images_match(setup): @@ -51,78 +35,57 @@ def test_images_match(setup): newly created plot to verify that the plot hasn't changed in appearance. """ - comparison = CompareImages('./box_expected.png', './box.png') + comparison = CompareImages(f'{cwd}/box_expected.png', f'{cwd}/box.png') assert comparison.mssim == 1 - cleanup() + remove_files(cwd, CLEANUP_FILES) @pytest.mark.parametrize("test_input, expected", - (["./box_expected.png", True],["./box_points1.png", True],["./intermed_files/box.points1", True])) -def test_points1_file_exist(test_input, expected): + (["box_expected.png", True], ["box_points1.png", True], ["intermed_files/box.points1", True])) +def test_points1_file_exist(setup_env, test_input, expected, remove_files): """ Checking that the plot is created and points1 output files is created where specified in the custom_box_points1.yaml file """ - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_box_points1.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_box_points1.yaml" try: - os.mkdir(os.path.join(os.getcwd(), './intermed_files')) - except FileExistsError as e: + os.mkdir(os.path.join(cwd, 'intermed_files')) + except FileExistsError: pass # Invoke the command to generate a box plot based on # the custom_box_points1.yaml custom config file. box.main(custom_config_filename) - assert os.path.isfile(test_input) == expected - try: - path = os.getcwd() - plot_file = 'box_points1.png' - points_file_1 = 'box.points1' - subdir = os.path.join(path, './intermed_files') - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(subdir, points_file_1)) - os.rmdir(subdir) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, ['box_points1.png']) @pytest.mark.parametrize("test_input, expected", - (["./box_defaultpoints1.png", True],["./box.points1", True])) -def test_defaultpoints1_file_exist(test_input, expected): + (["box_defaultpoints1.png", True], ["box.points1", True])) +def test_defaultpoints1_file_exist(setup_env, test_input, expected, remove_files): """ Checking that the plot is created and points1 output files is created in the default location (i.e. the current working dir, where the box.data file resides) """ - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_box_defaultpoints1.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_box_defaultpoints1.yaml" # Invoke the command to generate a box plot based on # the custom_box_defaultpoints1.yaml custom config file. box.main(custom_config_filename) - assert os.path.isfile(test_input) == expected + assert os.path.isfile(f"{cwd}/{test_input}") == expected # remove the created plot and intermediate .points1 file - try: - path = os.getcwd() - plot_file = 'box_defaultpoints1.png' - points_file_1 = 'box.points1' - os.remove(os.path.join(path, points_file_1)) - os.remove(os.path.join(path, plot_file)) - - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass + remove_files(cwd, ['box_defaultpoints1.png', 'box.points1']) -def test_no_nans_in_points_file(setup): +def test_no_nans_in_points_file(setup, remove_files): """ Checking that the points1 file does not contain NaN's """ - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_box_defaultpoints1.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + custom_config_filename = f"{cwd}/custom_box_defaultpoints1.yaml" # Invoke the command to generate a box plot based on # the custom_box_defaultpoints1.yaml custom config file. @@ -132,33 +95,22 @@ def test_no_nans_in_points_file(setup): # Fail if there are any NaN's-this indicates something went wrong with the # line_series.py module's _create_series_points() method. nans_found = False - with open("./box.points1", "r") as f: + with open(f"{cwd}/box.points1", "r") as f: data = f.read() if "NaN" in data: nans_found = True - assert nans_found == False - cleanup() + assert not nans_found + remove_files(cwd, CLEANUP_FILES) # Verify that the nan.points1 file does indeed trigger a "nans_found" - with open("./nan.points1", "r") as f: + with open(f"{cwd}/nan.points1", "r") as f: data = f.read() if "NaN" in data: nans_found = True # assert - assert nans_found == True + assert nans_found # remove the created plot and intermediate .points1 file - try: - path = os.getcwd() - plot_file = 'box_defaultpoints1.png' - points_file_1 = 'box.points1' - os.remove(os.path.join(path, points_file_1)) - os.remove(os.path.join(path, plot_file)) - - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - + remove_files(cwd, ['box_defaultpoints1.png', 'box.points1']) diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..c234f017 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,33 @@ +import pytest +import os +import shutil + + +@pytest.fixture +def setup_env(): + def set_environ(test_dir): + print("Setting up environment") + os.environ['METPLOTPY_BASE'] = f"{test_dir}/../../" + os.environ['TEST_DIR'] = test_dir + return set_environ + + +@pytest.fixture() +def remove_files(): + def remove_the_files(test_dir, file_list): + print("Removing the files") + # loop over list of files under test_dir and remove them + for file in file_list: + try: + os.remove(os.path.join(test_dir, file)) + except OSError: + pass + + # also remove intermed_files directory if it exists + print("Removing intermed_files directory if it exists") + try: + shutil.rmtree(f"{test_dir}/intermed_files") + except FileNotFoundError: + pass + + return remove_the_files diff --git a/test/contour/custom_contour.yaml b/test/contour/custom_contour.yaml index c6d08fb2..b53e5a71 100644 --- a/test/contour/custom_contour.yaml +++ b/test/contour/custom_contour.yaml @@ -150,8 +150,8 @@ ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -stat_input: ./contour.data -plot_filename: ./contour.png +stat_input: !ENV '${TEST_DIR}/contour.data' +plot_filename: !ENV '${TEST_DIR}/contour.png' # To save your log output to a file, specify a path and filename and uncomment the line below. Make sure you have # permissions to the directory you specify. The default, as specified in the default config file is stdout. diff --git a/test/contour/test_contour.py b/test/contour/test_contour.py index 6a8d6e91..7aff0c56 100644 --- a/test/contour/test_contour.py +++ b/test/contour/test_contour.py @@ -3,6 +3,7 @@ from metplotpy.plots.contour import contour #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) @pytest.fixture def setup(): @@ -10,23 +11,21 @@ def setup(): cleanup() # Set up the METPLOTPY_BASE so that met_plot.py will correctly find # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd # Invoke the command to generate a contour plot based on # the config yaml files. - contour.main("custom_contour.yaml") + contour.main(f"{cwd}/custom_contour.yaml") def cleanup(): # remove the previously created files try: - path = os.getcwd() plot_file = 'contour.png' - os.remove(os.path.join(path, plot_file)) - - + os.remove(os.path.join(cwd, plot_file)) except OSError as e: # Typically, when files have already been removed or # don't exist. Ignore. @@ -34,7 +33,7 @@ def cleanup(): @pytest.mark.parametrize("test_input, expected", - (["./contour_expected.png", True], ["./contour.png", True] + ([f"{cwd}/contour_expected.png", True], [f"{cwd}/contour.png", True] )) def test_files_exist(setup, test_input, expected): """ @@ -50,6 +49,6 @@ def test_images_match(setup): newly created plots to verify that the plot hasn't changed in appearance. """ - comparison = CompareImages('./contour_expected.png', './contour.png') + comparison = CompareImages(f'{cwd}/contour_expected.png', f'{cwd}/contour.png') assert comparison.mssim == 1 cleanup() diff --git a/test/difficulty_index/test_difficulty_index_plotting.py b/test/difficulty_index/test_difficulty_index_plotting.py index a6f9c9ae..5dfe50c8 100755 --- a/test/difficulty_index/test_difficulty_index_plotting.py +++ b/test/difficulty_index/test_difficulty_index_plotting.py @@ -1,12 +1,15 @@ +import os from . import example_difficulty_index as edi +cwd = os.path.dirname(__file__) + def test_difficulty_index_plot(): """ Compare difficulty index values to ensure correctness. """ - file1 = 'swh_North_Pacific_5dy_ensemble.npz' + file1 = f'{cwd}/swh_North_Pacific_5dy_ensemble.npz' lats, lons, fieldijn = edi.load_data(file1) muij, sigmaij = edi.compute_stats(fieldijn) @@ -16,5 +19,6 @@ def test_difficulty_index_plot(): assert 9.475065612792969 == muij[25][100] + if __name__ == "__main__": - test_difficulty_index_plot() + test_difficulty_index_plot() diff --git a/test/eclv/custom_eclv.yaml b/test/eclv/custom_eclv.yaml index aa40457a..18ddd739 100644 --- a/test/eclv/custom_eclv.yaml +++ b/test/eclv/custom_eclv.yaml @@ -112,8 +112,8 @@ ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -stat_input: ./eclv.data -plot_filename: ./eclv.png +stat_input: !ENV '${TEST_DIR}/eclv.data' +plot_filename: !ENV '${TEST_DIR}/eclv.png' # To save your log output to a file, specify a path and filename and uncomment the line below. Make sure you have # permissions to the directory you specify. The default, as specified in the default config file is stdout. diff --git a/test/eclv/custom_eclv_ctc.yaml b/test/eclv/custom_eclv_ctc.yaml index 7cadc20c..80d7c731 100644 --- a/test/eclv/custom_eclv_ctc.yaml +++ b/test/eclv/custom_eclv_ctc.yaml @@ -111,8 +111,8 @@ ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -stat_input: ./eclv_ctc.data -plot_filename: ./eclv_ctc.png +stat_input: !ENV '${TEST_DIR}/eclv_ctc.data' +plot_filename: !ENV '${TEST_DIR}/eclv_ctc.png' show_legend: -True diff --git a/test/eclv/custom_eclv_pct.yaml b/test/eclv/custom_eclv_pct.yaml index b41331c8..bc6e3a4a 100644 --- a/test/eclv/custom_eclv_pct.yaml +++ b/test/eclv/custom_eclv_pct.yaml @@ -102,7 +102,7 @@ ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -stat_input: ./eclv_pct.data -plot_filename: ./eclv_pct.png +stat_input: !ENV '${TEST_DIR}/eclv_pct.data' +plot_filename: !ENV '${TEST_DIR}/eclv_pct.png' show_legend: -True \ No newline at end of file diff --git a/test/eclv/test_eclv.py b/test/eclv/test_eclv.py index 9dd8cda9..37bf24b6 100644 --- a/test/eclv/test_eclv.py +++ b/test/eclv/test_eclv.py @@ -3,6 +3,7 @@ from metplotpy.plots.eclv import eclv #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) @pytest.fixture def setup(): @@ -10,26 +11,26 @@ def setup(): cleanup() # Set up the METPLOTPY_BASE so that met_plot.py will correctly find # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd # Invoke the command to generate a ECLV plots based on # the config yaml files. - eclv.main("custom_eclv_pct.yaml") - eclv.main("custom_eclv.yaml") - eclv.main("custom_eclv_ctc.yaml") + eclv.main(f"{cwd}/custom_eclv_pct.yaml") + eclv.main(f"{cwd}/custom_eclv.yaml") + eclv.main(f"{cwd}/custom_eclv_ctc.yaml") def cleanup(): # remove the previously created files try: - path = os.getcwd() plot_file = 'eclv_pct.png' - os.remove(os.path.join(path, plot_file)) + os.remove(os.path.join(cwd, plot_file)) plot_file = 'eclv.png' - os.remove(os.path.join(path, plot_file)) + os.remove(os.path.join(cwd, plot_file)) plot_file = 'eclv_ctc.png' - os.remove(os.path.join(path, plot_file)) + os.remove(os.path.join(cwd, plot_file)) except OSError as e: # Typically, when files have already been removed or @@ -38,9 +39,9 @@ def cleanup(): @pytest.mark.parametrize("test_input, expected", - (["./eclv_pct_expected.png", True], ["./eclv_pct.png", True], - ["./eclv_ctc_expected.png", True], ["./eclv_ctc.png", True], - ["./eclv_expected.png", True], ["./eclv.png", True])) + ([f"{cwd}/eclv_pct_expected.png", True], [f"{cwd}/eclv_pct.png", True], + [f"{cwd}/eclv_ctc_expected.png", True], [f"{cwd}/eclv_ctc.png", True], + [f"{cwd}/eclv_expected.png", True], [f"{cwd}/eclv.png", True])) def test_files_exist(setup, test_input, expected): """ Checking that the plot files are getting created @@ -55,12 +56,12 @@ def test_images_match(setup): newly created plots to verify that the plot hasn't changed in appearance. """ - comparison = CompareImages('./eclv_pct_expected.png', './eclv_pct.png') + comparison = CompareImages(f'{cwd}/eclv_pct_expected.png', './eclv_pct.png') assert comparison.mssim == 1 - comparison = CompareImages('./eclv_expected.png', './eclv.png') + comparison = CompareImages(f'{cwd}/eclv_expected.png', './eclv.png') assert comparison.mssim == 1 - comparison = CompareImages('./eclv_ctc_expected.png', './eclv_ctc.png') + comparison = CompareImages(f'{cwd}/eclv_ctc_expected.png', './eclv_ctc.png') assert comparison.mssim == 1 cleanup() diff --git a/test/ens_ss/custom2_ens_ss.yaml b/test/ens_ss/custom2_ens_ss.yaml index b9be350d..a01d1cbd 100644 --- a/test/ens_ss/custom2_ens_ss.yaml +++ b/test/ens_ss/custom2_ens_ss.yaml @@ -112,9 +112,9 @@ y2tlab_orient: 1 y2tlab_perp: 1 y2tlab_size: 1.0 -stat_input: ./ens_ss.data -plot_filename: ./intermed_files/ens_ss.png -points_path: ./intermed_files +stat_input: !ENV '${TEST_DIR}/ens_ss.data' +plot_filename: !ENV '${TEST_DIR}/intermed_files/ens_ss.png' +points_path: !ENV '${TEST_DIR}/intermed_files' show_legend: -True -True \ No newline at end of file diff --git a/test/ens_ss/custom_ens_ss.yaml b/test/ens_ss/custom_ens_ss.yaml index 231d2b79..3bfe0585 100644 --- a/test/ens_ss/custom_ens_ss.yaml +++ b/test/ens_ss/custom_ens_ss.yaml @@ -119,8 +119,8 @@ y2tlab_orient: 1 y2tlab_perp: 1 y2tlab_size: 1.0 -stat_input: ./ens_ss.data -plot_filename: ./ens_ss.png +stat_input: !ENV '${TEST_DIR}/ens_ss.data' +plot_filename: !ENV '${TEST_DIR}/ens_ss.png' # To save your log output to a file, specify a path and filename and uncomment the line below. Make sure you have # permissions to the directory you specify. The default, as specified in the default config file is stdout. diff --git a/test/ens_ss/test_ens_ss.py b/test/ens_ss/test_ens_ss.py index d4d85018..9962fc56 100644 --- a/test/ens_ss/test_ens_ss.py +++ b/test/ens_ss/test_ens_ss.py @@ -3,6 +3,7 @@ #from metcalcpy.compare_images import CompareImages from metplotpy.plots.ens_ss import ens_ss +cwd = os.path.dirname(__file__) @pytest.fixture def setup(): @@ -10,8 +11,9 @@ def setup(): cleanup() # Set up the METPLOTPY_BASE so that met_plot.py will correctly find # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_ens_ss.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd + custom_config_filename = f"{cwd}/custom_ens_ss.yaml" # Invoke the command to generate a Bar plot based on # the custom_ens_ss.yaml custom config file. @@ -22,11 +24,10 @@ def cleanup(): # remove the .png and .points files # from any previous runs try: - path = os.getcwd() plot_file = 'ens_ss.png' points_file_1 = 'ens_ss.points1' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file_1)) + os.remove(os.path.join(cwd, plot_file)) + os.remove(os.path.join(cwd, points_file_1)) except OSError as er: # Typically when files have already been removed or # don't exist. Ignore. @@ -34,7 +35,7 @@ def cleanup(): @pytest.mark.parametrize("test_input, expected", - (["./ens_ss.png", True],["./ens_ss.png", True],["./ens_ss.points1", True])) + ([f"{cwd}/ens_ss.png", True],[f"{cwd}/ens_ss.png", True],[f"{cwd}/ens_ss.points1", True])) def test_files_exist( setup, test_input, expected): """ Checking that the plot and data files are getting created @@ -49,23 +50,24 @@ def test_images_match(setup): newly created plot to verify that the plot hasn't changed in appearance. """ - comparison = CompareImages('./ens_ss_expected.png', './ens_ss.png') + comparison = CompareImages(f'{cwd}/ens_ss_expected.png', f'{cwd}/ens_ss.png') assert comparison.mssim == 1 cleanup() @pytest.mark.parametrize("test_input, expected", - (["./intermed_files/ens_ss.png", True], ["./intermed_files/ens_ss.points1", True])) + ([f"{cwd}/intermed_files/ens_ss.png", True], [f"{cwd}/intermed_files/ens_ss.points1", True])) def test_files_exist(test_input, expected): """ Checking that the plot and data files are getting created """ try: - os.mkdir(os.path.join(os.getcwd(), './intermed_files')) + os.mkdir(os.path.join(cwd, 'intermed_files')) except FileExistsError as e: pass - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom2_ens_ss.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd + custom_config_filename = f"{cwd}/custom2_ens_ss.yaml" # Invoke the command to generate a Bar plot based on # the custom_ens_ss.yaml custom config file. @@ -73,8 +75,7 @@ def test_files_exist(test_input, expected): assert os.path.isfile(test_input) == expected cleanup() try: - path = os.getcwd() - subdir = os.path.join(path, "./intermed_files") + subdir = os.path.join(cwd, "intermed_files") plot_file = 'ens_ss.png' points_file_1 = 'ens_ss.points1' os.remove(os.path.join(subdir, plot_file)) diff --git a/test/equivalence_testing_bounds/custom_equivalence_testing_bounds.yaml b/test/equivalence_testing_bounds/custom_equivalence_testing_bounds.yaml index 03cb8f2d..ffec3e5d 100644 --- a/test/equivalence_testing_bounds/custom_equivalence_testing_bounds.yaml +++ b/test/equivalence_testing_bounds/custom_equivalence_testing_bounds.yaml @@ -163,8 +163,8 @@ ytlab_horiz: 0.5 ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -plot_filename: ./equivalence_testing_bounds.png -stat_input: ./equivalence_testing_bounds.data +plot_filename: !ENV '${TEST_DIR}/equivalence_testing_bounds.png' +stat_input: !ENV '${TEST_DIR}/equivalence_testing_bounds.data' show_legend: - 'True' - 'True' diff --git a/test/equivalence_testing_bounds/custom_equivalence_testing_bounds2.yaml b/test/equivalence_testing_bounds/custom_equivalence_testing_bounds2.yaml index 42681fc2..f087c6b8 100644 --- a/test/equivalence_testing_bounds/custom_equivalence_testing_bounds2.yaml +++ b/test/equivalence_testing_bounds/custom_equivalence_testing_bounds2.yaml @@ -83,7 +83,7 @@ plot_stat: median plot_type: png16m plot_units: in plot_width: 11.0 -points_path: ./intermed_files +points_path: !ENV '${TEST_DIR}/intermed_files' random_seed: null series_line_style: - '-' @@ -164,8 +164,8 @@ ytlab_horiz: 0.5 ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -plot_filename: ./intermed_files/equivalence_testing_bounds.png -stat_input: ./equivalence_testing_bounds.data +plot_filename: !ENV '${TEST_DIR}/intermed_files/equivalence_testing_bounds.png' +stat_input: !ENV '${TEST_DIR}/equivalence_testing_bounds.data' show_legend: - 'True' - 'True' diff --git a/test/equivalence_testing_bounds/test_equivalence_testing_bounds.py b/test/equivalence_testing_bounds/test_equivalence_testing_bounds.py index 3bfcd5e3..e7e849e6 100644 --- a/test/equivalence_testing_bounds/test_equivalence_testing_bounds.py +++ b/test/equivalence_testing_bounds/test_equivalence_testing_bounds.py @@ -3,92 +3,72 @@ from metplotpy.plots.equivalence_testing_bounds import equivalence_testing_bounds as etb #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) +CLEANUP_FILES = ['equivalence_testing_bounds.png', 'equivalence_testing_bounds.points1'] @pytest.fixture -def setup(): +def setup(remove_files, setup_env): # Cleanup the plotfile and point1 output file from any previous run - cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./custom_equivalence_testing_bounds.yaml" + remove_files(cwd, CLEANUP_FILES) + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_equivalence_testing_bounds.yaml" # Invoke the command to generate an equivalence testing boundary plot based on # the custom config file. etb.main(custom_config_filename) -def cleanup(): - # remove the line.png and .points1 file - # from any previous runs - try: - path = os.getcwd() - plot_file = 'equivalence_testing_bounds.png' - points_file_1 = 'equivalence_testing_bounds.points1' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file_1)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - - @pytest.mark.parametrize("test_input,expected", - (["./equivalence_testing_bounds.png", True], - ["./equivalence_testing_bounds.points1", True])) -def test_files_exist(setup, test_input, expected): + (["equivalence_testing_bounds.png", True], + ["equivalence_testing_bounds.points1", True])) +def test_files_exist(setup, test_input, expected, remove_files): ''' Checking that the plot and data files are getting created ''' - assert os.path.isfile(test_input) == expected - cleanup() + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, CLEANUP_FILES) + @pytest.mark.skip("skimage differences causing failure") -def test_images_match(setup): +def test_images_match(setup, remove_files): ''' Compare an expected plot with the newly created plot to verify that the plot hasn't changed in appearance. ''' - path = os.getcwd() - plot_file = './equivalence_testing_bounds.png' - actual_file = os.path.join(path, plot_file) - comparison = CompareImages('./equivalence_testing_bounds_expected.png', actual_file) + plot_file = 'equivalence_testing_bounds.png' + actual_file = os.path.join(cwd, plot_file) + comparison = CompareImages(f'{cwd}/equivalence_testing_bounds_expected.png', actual_file) assert comparison.mssim == 1 - cleanup() + remove_files(cwd, CLEANUP_FILES) @pytest.mark.parametrize("test_input,expected", - (["./intermed_files/equivalence_testing_bounds.png", True], - ["./intermed_files/equivalence_testing_bounds.points1", True])) -def test_files_exist(test_input, expected): + (["intermed_files/equivalence_testing_bounds.png", True], + ["intermed_files/equivalence_testing_bounds.points1", True])) +def test_files_exist(setup_env, test_input, expected, remove_files): ''' Checking that the plot and data files are getting created ''' - intermed_dir = os.path.join(os.getcwd(), 'intermed_files') + intermed_dir = os.path.join(cwd, 'intermed_files') try: os.mkdir(intermed_dir) - except FileExistsError as e: + except FileExistsError: pass - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./custom_equivalence_testing_bounds2.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_equivalence_testing_bounds2.yaml" # Invoke the command to generate an equivalence testing boundary plot based on # the custom config file. etb.main(custom_config_filename) - assert os.path.isfile(test_input) == expected + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, ['intermed_files/equivalence_testing_bounds.png', + 'intermed_files/equivalence_testing_bounds.points1', + 'intermed_files/equivalence_testing_bounds.html']) try: - path = os.getcwd() - plot_file = 'equivalence_testing_bounds.png' - points_file_1 = 'equivalence_testing_bounds.points1' - subdir = os.path.join(path, 'intermed_files') - os.remove(os.path.join(subdir, plot_file)) - os.remove(os.path.join(subdir, points_file_1)) os.rmdir(intermed_dir) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. + except OSError: pass diff --git a/test/histogram/prob_hist.yaml b/test/histogram/prob_hist.yaml index 0600576d..1b72e5c2 100644 --- a/test/histogram/prob_hist.yaml +++ b/test/histogram/prob_hist.yaml @@ -1,5 +1,5 @@ -stat_input: ./prob_hist.data -plot_filename: ./prob_hist.png +stat_input: !ENV '${TEST_DIR}/prob_hist.data' +plot_filename: !ENV '${TEST_DIR}/prob_hist.png' caption_align: 0.0 caption_col: '#333333' diff --git a/test/histogram/rank_hist.yaml b/test/histogram/rank_hist.yaml index 40f12bdd..8cf23ec9 100644 --- a/test/histogram/rank_hist.yaml +++ b/test/histogram/rank_hist.yaml @@ -1,5 +1,5 @@ -stat_input: ./rank_hist.data -plot_filename: ./rank_hist.png +stat_input: !ENV '${TEST_DIR}/rank_hist.data' +plot_filename: !ENV '${TEST_DIR}/rank_hist.png' caption_align: 0.0 caption_col: '#333333' diff --git a/test/histogram/rel_hist.yaml b/test/histogram/rel_hist.yaml index 322e83f0..ebb76e03 100644 --- a/test/histogram/rel_hist.yaml +++ b/test/histogram/rel_hist.yaml @@ -1,5 +1,5 @@ -plot_filename: ./rel_hist.png -stat_input: ./rel_hist.data +plot_filename: !ENV '${TEST_DIR}/rel_hist.png' +stat_input: !ENV '${TEST_DIR}/rel_hist.data' caption_align: 0.0 diff --git a/test/histogram/test_prob_hist.py b/test/histogram/test_prob_hist.py index 4d41f0d7..fc9b3ab4 100644 --- a/test/histogram/test_prob_hist.py +++ b/test/histogram/test_prob_hist.py @@ -3,15 +3,15 @@ from metplotpy.plots.histogram import prob_hist #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) + @pytest.fixture -def setup(): - # Cleanup the plotfile output file from any previous run +def setup(setup_env): cleanup() + setup_env(cwd) - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "prob_hist.yaml" - + custom_config_filename = f"{cwd}/prob_hist.yaml" prob_hist.main(custom_config_filename) @@ -19,22 +19,20 @@ def cleanup(): # remove the rel_hist.png # from any previous runs try: - path = os.getcwd() - plot_file = './prob_hist.png' - os.remove(os.path.join(path, plot_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. + plot_file = 'prob_hist.png' + os.remove(os.path.join(cwd, plot_file)) + except OSError: pass + @pytest.mark.parametrize("test_input, expected", - (["./prob_hist_expected.png", True], - ["./prob_hist.png", True])) + (["prob_hist_expected.png", True], + ["prob_hist.png", True])) def test_files_exist(setup, test_input, expected): """ Checking that the plot and data files are getting created """ - assert os.path.isfile(test_input) == expected + assert os.path.isfile(f"{cwd}/{test_input}") == expected cleanup() @pytest.mark.skip("Image comparisons fail during Github Actions checks.") @@ -44,7 +42,7 @@ def test_images_match(setup): newly created plot to verify that the plot hasn't changed in appearance. """ - comparison = CompareImages("./prob_hist_expected.png", - "./prob_hist.png") + comparison = CompareImages(f"{cwd}/prob_hist_expected.png", + f"{cwd}/prob_hist.png") assert comparison.mssim == 1 cleanup() diff --git a/test/histogram/test_rank_hist.py b/test/histogram/test_rank_hist.py index e2ba10e3..4b9d5b08 100644 --- a/test/histogram/test_rank_hist.py +++ b/test/histogram/test_rank_hist.py @@ -3,16 +3,15 @@ from metplotpy.plots.histogram import rank_hist #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) + @pytest.fixture -def setup(): - # Cleanup the plotfile output file from any previous run +def setup(setup_env): cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "rank_hist.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/rank_hist.yaml" rank_hist.main(custom_config_filename) @@ -20,24 +19,23 @@ def cleanup(): # remove the rel_hist.png # from any previous runs try: - path = os.getcwd() - plot_file = './rank_hist.png' - os.remove(os.path.join(path, plot_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. + plot_file = 'rank_hist.png' + os.remove(os.path.join(cwd, plot_file)) + except OSError: pass + @pytest.mark.parametrize("test_input, expected", - (["./rank_hist_expected.png", True], - ["./rank_hist.png", True])) + (["rank_hist_expected.png", True], + ["rank_hist.png", True])) def test_files_exist(setup, test_input, expected): """ Checking that the plot and data files are getting created """ - assert os.path.isfile(test_input) == expected + assert os.path.isfile(f"{cwd}/{test_input}") == expected cleanup() + @pytest.mark.skip("Image comparisons fail during Github Actions checks.") def test_images_match(setup): """ @@ -45,7 +43,7 @@ def test_images_match(setup): newly created plot to verify that the plot hasn't changed in appearance. """ - comparison = CompareImages("./rank_hist_expected.png", - "./rank_hist.png") + comparison = CompareImages(f"{cwd}/rank_hist_expected.png", + f"{cwd}/rank_hist.png") assert comparison.mssim == 1 cleanup() diff --git a/test/histogram/test_rel_hist.py b/test/histogram/test_rel_hist.py index 2da38dec..a491283c 100644 --- a/test/histogram/test_rel_hist.py +++ b/test/histogram/test_rel_hist.py @@ -3,13 +3,15 @@ from metplotpy.plots.histogram import rel_hist #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) + @pytest.fixture -def setup(): +def setup(setup_env): # Cleanup the plotfile output file from any previous run cleanup() - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "rel_hist.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/rel_hist.yaml" # Invoke the command to generate a histogram based on # the rel_hist.yaml custom config file. @@ -20,24 +22,23 @@ def cleanup(): # remove the rel_hist.png # from any previous runs try: - path = os.getcwd() - plot_file = './rel_hist.png' - os.remove(os.path.join(path, plot_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. + plot_file = 'rel_hist.png' + os.remove(os.path.join(cwd, plot_file)) + except OSError: pass + @pytest.mark.parametrize("test_input, expected", - (["./rel_hist_expected.png", True], - ["./rel_hist.png", True])) + (["rel_hist_expected.png", True], + ["rel_hist.png", True])) def test_files_exist(setup, test_input, expected): """ Checking that the plot and data files are getting created """ - assert os.path.isfile(test_input) == expected + assert os.path.isfile(f"{cwd}/{test_input}") == expected cleanup() + @pytest.mark.skip("Image comparisons fail in Github Actions checks.") def test_images_match(setup): """ @@ -45,7 +46,7 @@ def test_images_match(setup): newly created plot to verify that the plot hasn't changed in appearance. """ - comparison = CompareImages("./rel_hist_expected.png", - "./rel_hist.png") + comparison = CompareImages(f"{cwd}/rel_hist_expected.png", + f"{cwd}/rel_hist.png") assert comparison.mssim == 1 cleanup() diff --git a/test/histogram_2d/custom_histogram_2d.yaml b/test/histogram_2d/custom_histogram_2d.yaml index 1cc13895..292349c2 100644 --- a/test/histogram_2d/custom_histogram_2d.yaml +++ b/test/histogram_2d/custom_histogram_2d.yaml @@ -1,5 +1,5 @@ # replace with the location and name of your choosing: -plot_filename: ./custom_tmp_z2_p500.png +plot_filename: !ENV '${TEST_DIR}/custom_tmp_z2_p500.png' height: 800 width: 1200 @@ -26,7 +26,7 @@ pdf_max: 4.0 # dump_points_2: 'False' # the input data file (replace ./ with the full path) -stat_input: ./grid_diag_temperature.nc +stat_input: !ENV '${TEST_DIR}/grid_diag_temperature.nc' # To save your log output to a file, specify a path and filename and uncomment the line below. Make sure you have # permissions to the directory you specify. The default, as specified in the default config file is stdout. diff --git a/test/histogram_2d/minimal_histogram_2d.yaml b/test/histogram_2d/minimal_histogram_2d.yaml index cfb8c80c..291feb3b 100644 --- a/test/histogram_2d/minimal_histogram_2d.yaml +++ b/test/histogram_2d/minimal_histogram_2d.yaml @@ -1,5 +1,5 @@ # minimal custom config file. Use all default settings in the histogram_2d_defaults.yaml # configuration file -stat_input: ./grid_diag_temperature.nc -plot_filename: ./tmp_z2_p500.png +stat_input: !ENV '${TEST_DIR}/grid_diag_temperature.nc' +plot_filename: !ENV '${TEST_DIR}/tmp_z2_p500.png' diff --git a/test/histogram_2d/test_histogram_2d.py b/test/histogram_2d/test_histogram_2d.py index 4e6e9e95..1bc96a04 100644 --- a/test/histogram_2d/test_histogram_2d.py +++ b/test/histogram_2d/test_histogram_2d.py @@ -1,20 +1,19 @@ import os -import sys import pytest from metplotpy.plots.histogram_2d import histogram_2d as h2d - # from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) + + @pytest.fixture -def setup(): +def setup(setup_env): # Cleanup the plotfile and point1 output file from any previous run # cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./minimal_histogram_2d.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/minimal_histogram_2d.yaml" # print("\n current directory: ", os.getcwd()) # print("\ncustom config file: ", custom_config_filename, '\n') @@ -29,26 +28,11 @@ def setup(): # change the stat_input and plot_filename to explicitly point to this directory before # running the test below because xarray cannot handle relative paths when reading in # filenames -def test_plot_exists(setup): +def test_plot_exists(setup, remove_files): ''' Checking that only the "defaults" plot file is getting created ''' test_input = "tmp_z2_p500.png" - from os.path import exists - file_exists = exists(test_input) - if file_exists: - assert os.path.exists(test_input) - else: - print("File doesn't exist") - assert False - - # cleanup - try: - path = os.getcwd() - plot_file = 'tmp_z2_p500.png' - # os.remove(os.path.join(path, plot_file)) - except OSError: - # Typically when files have already been removed or - # don't exist. Ignore. - pass + assert os.path.exists(f"{cwd}/{test_input}") + remove_files(cwd, ['tmp_z2_p500.png']) diff --git a/test/line/custom_line.yaml b/test/line/custom_line.yaml index b189b8ff..e38a6d85 100644 --- a/test/line/custom_line.yaml +++ b/test/line/custom_line.yaml @@ -94,7 +94,7 @@ plot_disp: - 'True' - 'True' - 'True' -plot_filename: ./line.png +plot_filename: !ENV '${TEST_DIR}/line.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -153,7 +153,7 @@ show_signif: - 'False' - 'False' - 'False' -stat_input: ./line.data +stat_input: !ENV '${TEST_DIR}/line.data' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/line/custom_line2.yaml b/test/line/custom_line2.yaml index c827af9b..cafc9708 100644 --- a/test/line/custom_line2.yaml +++ b/test/line/custom_line2.yaml @@ -95,14 +95,14 @@ plot_disp: - 'True' - 'True' - 'True' -plot_filename: ./line.png +plot_filename: !ENV '${TEST_DIR}/line.png' plot_height: 8.5 plot_res: 72 plot_stat: median plot_type: png16m plot_units: in plot_width: 11.0 -points_path: ./intermed_files +points_path: !ENV '${TEST_DIR}/intermed_files' random_seed: null series_line_style: - '-' @@ -149,7 +149,7 @@ show_signif: - 'False' - 'False' - 'False' -stat_input: ./line.data +stat_input: !ENV '${TEST_DIR}/line.data' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/line/custom_line_from_zero.yaml b/test/line/custom_line_from_zero.yaml index cda69849..9d320c8c 100644 --- a/test/line/custom_line_from_zero.yaml +++ b/test/line/custom_line_from_zero.yaml @@ -95,7 +95,7 @@ plot_disp: - 'True' - 'True' - 'True' -plot_filename: ./line_from_zero.png +plot_filename: !ENV '${TEST_DIR}/line_from_zero.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -155,7 +155,7 @@ show_signif: - 'False' - 'False' start_from_zero: 'True' -stat_input: ./line.data +stat_input: !ENV '${TEST_DIR}/line.data' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/line/custom_line_groups.yaml b/test/line/custom_line_groups.yaml index 6985141a..0eafc444 100644 --- a/test/line/custom_line_groups.yaml +++ b/test/line/custom_line_groups.yaml @@ -87,7 +87,7 @@ plot_disp: - 'True' - 'True' - 'True' -plot_filename: ./line_groups.png +plot_filename: !ENV '${TEST_DIR}/line_groups.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -125,7 +125,7 @@ show_signif: - 'False' - 'False' - 'False' -stat_input: ./line_groups.data +stat_input: !ENV '${TEST_DIR}/line_groups.data' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/line/custom_line_groups2.yaml b/test/line/custom_line_groups2.yaml index ff6e6b15..0eedd0cf 100644 --- a/test/line/custom_line_groups2.yaml +++ b/test/line/custom_line_groups2.yaml @@ -92,14 +92,14 @@ plot_disp: - 'True' - 'True' - 'True' -plot_filename: ./intermed_files/line_groups.png +plot_filename: !ENV '${TEST_DIR}/intermed_files/line_groups.png' plot_height: 8.5 plot_res: 72 plot_stat: median plot_type: png16m plot_units: in plot_width: 11.0 -points_path: ./intermed_files +points_path: !ENV '${TEST_DIR}/intermed_files' random_seed: null series_line_style: - '-' @@ -131,7 +131,7 @@ show_signif: - 'False' - 'False' - 'False' -stat_input: ./line_groups.data +stat_input: !ENV '${TEST_DIR}/line_groups.data' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/line/fbias_fixed_vars_vals.yaml b/test/line/fbias_fixed_vars_vals.yaml index 211f7389..75005942 100644 --- a/test/line/fbias_fixed_vars_vals.yaml +++ b/test/line/fbias_fixed_vars_vals.yaml @@ -166,7 +166,7 @@ plot_ci: plot_disp: - 'True' - 'True' -plot_filename: ./fbias_fixed_vars.png +plot_filename: !ENV '${TEST_DIR}/fbias_fixed_vars.png' plot_height: 8.5 plot_res: 72 plot_stat: mean @@ -180,7 +180,7 @@ start_from_zero: True # that is used by METviewer (created when dump_points_1 is set to True) # if dump_points_1 is True and this is uncommented, the points1 file # will be saved in the default location (i.e. where the input data file is stored). -points_path: ./ +points_path: !ENV '${TEST_DIR}/' random_seed: null series_line_style: @@ -207,7 +207,7 @@ show_nstats: 'False' show_signif: - 'False' - 'False' -stat_input: ./fbias_data.txt +stat_input: !ENV '${TEST_DIR}/fbias_data.txt' sync_yaxes: 'False' title: "Fixed variable fcst_thresh >0.0 for FBIAS" title_align: 0.5 diff --git a/test/line/mv_custom_vert_line.yaml b/test/line/mv_custom_vert_line.yaml index 1a9bc292..d26335f0 100644 --- a/test/line/mv_custom_vert_line.yaml +++ b/test/line/mv_custom_vert_line.yaml @@ -102,7 +102,7 @@ plot_ci: plot_disp: - 'True' - 'True' -plot_filename: ./vert_line_plot.png +plot_filename: !ENV '${TEST_DIR}/vert_line_plot.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -114,7 +114,7 @@ plot_width: 11.0 # that is used by METviewer (created when dump_points_1 is set to True) # if dump_points_1 is True and this is uncommented, the points1 file # will be saved in the default location (i.e. where the input data file is stored). -points_path: ./intermed_files +points_path: !ENV '${TEST_DIR}/intermed_files' random_seed: null series_line_style: - '-' @@ -140,7 +140,7 @@ show_nstats: 'True' show_signif: - 'True' - 'True' -stat_input: ./vert_line_plot_data.txt +stat_input: !ENV '${TEST_DIR}/vert_line_plot_data.txt' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/line/test_line_groups_plot.py b/test/line/test_line_groups_plot.py index e4133e54..a0756c03 100644 --- a/test/line/test_line_groups_plot.py +++ b/test/line/test_line_groups_plot.py @@ -3,6 +3,7 @@ from metplotpy.plots.line import line as l #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) @pytest.fixture def setup(): @@ -10,8 +11,9 @@ def setup(): cleanup() # Set up the METPLOTPY_BASE so that met_plot.py will correctly find # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./custom_line_groups.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd + custom_config_filename = f"{cwd}/custom_line_groups.yaml" # Invoke the command to generate a Performance Diagram based on # the test_custom_performance_diagram.yaml custom config file. @@ -22,15 +24,14 @@ def cleanup(): # remove the line.png and .points files # from any previous runs try: - path = os.getcwd() plot_file = 'line_groups.png' points_file_1 = 'line_groups.points1' points_file_2 = 'line_groups.points2' html_file = 'line_groups.html' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file_1)) - os.remove(os.path.join(path, points_file_2)) - os.remove(os.path.join(path, html_file)) + os.remove(os.path.join(cwd, plot_file)) + os.remove(os.path.join(cwd, points_file_1)) + os.remove(os.path.join(cwd, points_file_2)) + os.remove(os.path.join(cwd, html_file)) except OSError as e: # Typically when files have already been removed or # don't exist. Ignore. @@ -38,7 +39,7 @@ def cleanup(): @pytest.mark.parametrize("test_input,expected", - (["./line_groups.png", True], ["./line_groups.points1", True])) + ([f"{cwd}/line_groups.png", True], [f"{cwd}/line_groups.points1", True])) def test_files_exist(setup, test_input, expected): ''' Checking that the plot and data files are getting created @@ -53,27 +54,27 @@ def test_images_match(setup): newly created plot to verify that the plot hasn't changed in appearance. ''' - path = os.getcwd() plot_file = 'line_groups.png' - actual_file = os.path.join(path, plot_file) - comparison = CompareImages('./line_groups_expected.png', actual_file) + actual_file = os.path.join(cwd, plot_file) + comparison = CompareImages(f'{cwd}/line_groups_expected.png', actual_file) assert comparison.mssim == 1 cleanup() @pytest.mark.parametrize("test_input,expected", - (["./intermed_files/line_groups.png", True], ["./intermed_files/line_groups.points1", True])) + ([f"{cwd}/intermed_files/line_groups.png", True], [f"{cwd}/intermed_files/line_groups.points1", True])) def test_files_exist( test_input, expected): ''' Checking that the plot and data files are getting created ''' - intermed_dir = './intermed_files' + intermed_dir = f'{cwd}/intermed_files' try: os.mkdir(intermed_dir) except FileExistsError as e: pass - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./custom_line_groups2.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd + custom_config_filename = f"{cwd}/custom_line_groups2.yaml" # Invoke the command to generate a Performance Diagram based on # the test_custom_performance_diagram.yaml custom config file. @@ -82,8 +83,7 @@ def test_files_exist( test_input, expected): # remove the files that were created, cleanup() isn't applicable for this test. try: - path = os.getcwd() - subdir = os.path.join(path, intermed_dir) + subdir = os.path.join(cwd, intermed_dir) plot_file = 'line_groups.png' points_file_1 = 'line_groups.points1' points_file_2 = 'line_groups.points2' diff --git a/test/line/test_line_plot.py b/test/line/test_line_plot.py index 203de672..bb4972dc 100644 --- a/test/line/test_line_plot.py +++ b/test/line/test_line_plot.py @@ -3,6 +3,7 @@ import os from metplotpy.plots.line import line as l +cwd = os.path.dirname(__file__) # from metcalcpy.compare_images import CompareImages @@ -13,8 +14,9 @@ def setup(): cleanup() # Set up the METPLOTPY_BASE so that met_plot.py will correctly find # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_line.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd + custom_config_filename = f"{cwd}/custom_line.yaml" # Invoke the command to generate a line plot based on # the custom config file. @@ -25,13 +27,12 @@ def cleanup(): # remove the line.png and .points files # from any previous runs try: - path = os.getcwd() plot_file = 'line.png' points_file_1 = 'line.points1' points_file_2 = 'line.points2' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file_1)) - os.remove(os.path.join(path, points_file_2)) + os.remove(os.path.join(cwd, plot_file)) + os.remove(os.path.join(cwd, points_file_1)) + os.remove(os.path.join(cwd, points_file_2)) except OSError as e: # Typically when files have already been removed or # don't exist. Ignore. @@ -39,8 +40,8 @@ def cleanup(): @pytest.mark.parametrize("test_input,expected", - (["./line.png", True], ["./line.points1", False], - ["./line.points2", False])) + ([f"{cwd}/line.png", True], [f"{cwd}/line.points1", False], + [f"{cwd}/line.points2", False])) def test_files_exist(setup, test_input, expected): ''' Checking that the plot file is getting created but the @@ -51,8 +52,8 @@ def test_files_exist(setup, test_input, expected): @pytest.mark.parametrize("test_input,expected", - (["./line.png", True], ["./intermed_files/line.points1", True], - ["./intermed_files/line.points2", True])) + ([f"{cwd}/line.png", True], [f"{cwd}/intermed_files/line.points1", True], + [f"{cwd}/intermed_files/line.points2", True])) def test_points_files_exist(test_input, expected): ''' Checking that the plot and point data files are getting created @@ -60,12 +61,13 @@ def test_points_files_exist(test_input, expected): # create the intermediate directory to store the .points1 and .points2 files try: - os.mkdir(os.path.join(os.getcwd(), 'intermed_files')) + os.mkdir(os.path.join(cwd, 'intermed_files')) except FileExistsError as e: pass - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_line2.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd + custom_config_filename = f"{cwd}/custom_line2.yaml" l.main(custom_config_filename) # Test for expected values @@ -73,12 +75,11 @@ def test_points_files_exist(test_input, expected): # cleanup intermediate files and plot try: - path = os.getcwd() plot_file = 'line.png' points_file_1 = 'line.points1' points_file_2 = 'line.points2' - intermed_path = os.path.join(path, 'intermed_files') - os.remove(os.path.join(path, plot_file)) + intermed_path = os.path.join(cwd, 'intermed_files') + os.remove(os.path.join(cwd, plot_file)) os.remove(os.path.join(intermed_path, points_file_1)) os.remove(os.path.join(intermed_path, points_file_2)) except OSError as e: @@ -94,19 +95,20 @@ def test_no_nans_in_points_files(): # create the intermediate directory to store the .points1 and .points2 files try: - os.mkdir(os.path.join(os.getcwd(), 'intermed_files')) + os.mkdir(os.path.join(cwd, 'intermed_files')) except FileExistsError as e: pass - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_line2.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd + custom_config_filename = f"{cwd}/custom_line2.yaml" l.main(custom_config_filename) # Check for NaN's in the intermediate files, line.points1 and line.points2 # Fail if there are any NaN's-this indicates something went wrong with the # line_series.py module's _create_series_points() method. nans_found = False - with open("./intermed_files/line.points1", "r") as f: + with open(f"{cwd}/intermed_files/line.points1", "r") as f: data = f.read() if "NaN" in data: nans_found = True @@ -114,7 +116,7 @@ def test_no_nans_in_points_files(): assert nans_found == False # Now check line.points2 - with open("./intermed_files/line.points2", "r") as f: + with open(f"{cwd}/intermed_files/line.points2", "r") as f: data = f.read() if "NaN" in data: nans_found = True @@ -122,7 +124,7 @@ def test_no_nans_in_points_files(): assert nans_found == False # Verify that the nan.points1 file does indeed trigger a "nans_found" - with open("./nan.points1", "r") as f: + with open(f"{cwd}/nan.points1", "r") as f: data = f.read() if "NaN" in data: nans_found = True @@ -132,12 +134,11 @@ def test_no_nans_in_points_files(): # cleanup intermediate files and plot try: - path = os.getcwd() plot_file = 'line.png' points_file_1 = 'line.points1' points_file_2 = 'line.points2' - intermed_path = os.path.join(path, 'intermed_files') - os.remove(os.path.join(path, plot_file)) + intermed_path = os.path.join(cwd, 'intermed_files') + os.remove(os.path.join(cwd, plot_file)) os.remove(os.path.join(intermed_path, points_file_1)) os.remove(os.path.join(intermed_path, points_file_2)) except OSError as e: @@ -153,10 +154,9 @@ def test_images_match(setup): newly created plot to verify that the plot hasn't changed in appearance. ''' - path = os.getcwd() plot_file = './line.png' - actual_file = os.path.join(path, plot_file) - comparison = CompareImages('./line_expected.png', actual_file) + actual_file = os.path.join(cwd, plot_file) + comparison = CompareImages(f'{cwd}/line_expected.png', actual_file) # !!!WARNING!!! SOMETIMES FILE SIZES DIFFER IN SPITE OF THE PLOTS LOOKING THE SAME # THIS TEST IS NOT 100% RELIABLE because of differences in machines, OS, etc. @@ -174,16 +174,16 @@ def test_new_images_match(): # Set up the METPLOTPY_BASE so that met_plot.py will correctly find # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_line_from_zero.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd + custom_config_filename = f"{cwd}/custom_line_from_zero.yaml" # Invoke the command to generate a Performance Diagram based on # the test_custom_performance_diagram.yaml custom config file. l.main(custom_config_filename) - path = os.getcwd() plot_file = 'line_from_zero.png' - actual_file = os.path.join(path, plot_file) - comparison = CompareImages('./line_expected_from_zero.png', actual_file) + actual_file = os.path.join(cwd, plot_file) + comparison = CompareImages(f'{cwd}/line_expected_from_zero.png', actual_file) # !!!WARNING!!! SOMETIMES FILE SIZES DIFFER IN SPITE OF THE PLOTS LOOKING THE SAME # THIS TEST IS NOT 100% RELIABLE because of differences in machines, OS, etc. @@ -191,9 +191,8 @@ def test_new_images_match(): # cleanup plot try: - path = os.getcwd() plot_file = 'line_from_zero.png' - os.remove(os.path.join(path, plot_file)) + os.remove(os.path.join(cwd, plot_file)) except OSError as e: # Typically when files have already been removed or # don't exist. Ignore. @@ -215,21 +214,21 @@ def test_vertical_plot(): except FileExistsError as e: pass - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "mv_custom_vert_line.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIRR'] = cwd + custom_config_filename = f"{cwd}/mv_custom_vert_line.yaml" l.main(custom_config_filename) try: - path = os.getcwd() plot_file = 'vert_line_plot.png' points_file_1 = 'vert_line_plot.points1' points_file_2 = 'vert_line_plot.points2' - intermed_path = os.path.join(path, 'intermed_files') + intermed_path = os.path.join(cwd, 'intermed_files') # Retrieve the .points1 files generated by METviewer and METplotpy respectively - mv_df = pd.read_csv('./intermed_files/vert_plot_y1_from_metviewer.points1', + mv_df = pd.read_csv(f'{cwd}/intermed_files/vert_plot_y1_from_metviewer.points1', sep=" ", header=None) - mpp_df = pd.read_csv('./intermed_files/vert_line_plot.points1', sep=" ", + mpp_df = pd.read_csv(f'{cwd}/intermed_files/vert_line_plot.points1', sep=" ", header=None) # ----------------------- @@ -259,7 +258,7 @@ def test_vertical_plot(): os.remove(os.path.join(path, plot_file)) os.remove(os.path.join(intermed_path, points_file_1)) os.remove(os.path.join(intermed_path, points_file_2)) - os.remove('./intermed_files/vert_plot_y1_from_metviewer.points1') + os.remove(f'{cwd}/intermed_files/vert_plot_y1_from_metviewer.points1') except OSError as e: # Typically when files have already been removed or # don't exist. Ignore. @@ -274,25 +273,25 @@ def test_fixed_var_val(): """ # Set up the METPLOTPY_BASE so that met_plot.py will correctly find # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "fbias_fixed_vars_vals.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd + custom_config_filename = f"{cwd}/fbias_fixed_vars_vals.yaml" # Invoke the command to generate a line plot based on # the custom config file. l.main(custom_config_filename) - expected_points = "../intermed_files/mv_fixed_var_vals.points1" + expected_points = f"{cwd}/intermed_files/mv_fixed_var_vals.points1" try: - path = os.getcwd() plot_file = 'fbias_fixed_vars_reformatted_input.png' - intermed_path = os.path.join(path, 'intermed_files') + intermed_path = os.path.join(cwd, 'intermed_files') # Retrieve the .points1 files generated by METviewer and METplotpy respectively - mv_df = pd.read_csv('./intermed_files/mv_fixed_var_vals.points1', + mv_df = pd.read_csv(f'{cwd}/intermed_files/mv_fixed_var_vals.points1', sep="\t", header=None) - mpp_df = pd.read_csv('./fbias.points1', sep="\t", header=None) + mpp_df = pd.read_csv(f'{cwd}/fbias.points1', sep="\t", header=None) # Verify that the values in the generated points1 file are identical # to those in the METviewer points1 file. @@ -311,12 +310,12 @@ def test_fixed_var_val(): assert mv_df.iloc[i][0] == mpp_df.iloc[i][0] # Clean up the fbias.points1, fbias.points2, and .png files - os.remove('fbias.points1') - os.remove('fbias.points2') - os.remove('fbias_fixed_vars_reformatted_input.png') - os.remove('./intermed_files/mv_fixed_var_vals.points1') + os.remove(f'{cwd}/fbias.points1') + os.remove(f'{cwd}/fbias.points2') + os.remove(f'{cwd}/fbias_fixed_vars_reformatted_input.png') + os.remove(f'{cwd}/intermed_files/mv_fixed_var_vals.points1') os.remove(expected_points) - os.rmdir('./intermed_files') + os.rmdir(f'{cwd}/intermed_files') except OSError as e: # Typically when files have already been removed or @@ -335,20 +334,19 @@ def test_fixed_var_val_image_compare(): # Set up the METPLOTPY_BASE so that met_plot.py will correctly find # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "fbias_fixed_vars_vals.yaml" + os.environ['METPLOTPY_BASE'] = f"{cwd}/../../" + os.environ['TEST_DIR'] = cwd + custom_config_filename = f"{cwd}/fbias_fixed_vars_vals.yaml" # Invoke the command to generate a line plot based on # the custom config file. l.main(custom_config_filename) - expected_plot = "./expected_fbias_fixed_vars.png" + expected_plot = f"{cwd}/expected_fbias_fixed_vars.png" try: - path = os.getcwd() plot_file = 'fbias_fixed_vars.png' - - created_file = os.path.join(path, plot_file) + created_file = os.path.join(cwd, plot_file) # first verify that the output plot was created if os.path.exists(created_file): @@ -364,8 +362,8 @@ def test_fixed_var_val_image_compare(): assert comparison.mssim == 1 # Clean up the fbias.points1, fbias.points2 and the png files - os.remove('fbias.points1') - os.remove('fbias.points2') + os.remove(f'{cwd}/fbias.points1') + os.remove(f'{cwd}/fbias.points2') os.remove(created_file) except OSError as e: diff --git a/test/mpr_plot/mpr_plot_custom.yaml b/test/mpr_plot/mpr_plot_custom.yaml index 7c20cf56..aad4b441 100644 --- a/test/mpr_plot/mpr_plot_custom.yaml +++ b/test/mpr_plot/mpr_plot_custom.yaml @@ -1,5 +1,5 @@ wind_rose: True -plot_filename: mpr_plots.png +plot_filename: !ENV '${TEST_DIR}/mpr_plots.png' wind_rose_breaks: - 0.0 - 1.0 @@ -28,7 +28,7 @@ angularaxis_ticktext: - 'S' - 'W' mpr_file_list: - - ./point_stat_mpr.txt + - !ENV '${TEST_DIR}/point_stat_mpr.txt' width: 1200 height: 7500 marker_color: 'rgb(194,189,251)' diff --git a/test/mpr_plot/test_mpr_plot.py b/test/mpr_plot/test_mpr_plot.py index df5bf1ee..ee724505 100644 --- a/test/mpr_plot/test_mpr_plot.py +++ b/test/mpr_plot/test_mpr_plot.py @@ -3,50 +3,39 @@ from metplotpy.plots.mpr_plot import mpr_plot #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) +CLEANUP_FILES = ['mpr_plots.png'] + @pytest.fixture -def setup(): +def setup(setup_env, remove_files): # Cleanup the plotfile and point1 output file from any previous run - cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "mpr_plot_custom.yaml" + remove_files(cwd, CLEANUP_FILES) + setup_env(cwd) + custom_config_filename = f"{cwd}/mpr_plot_custom.yaml" # Invoke the command to generate a Performance Diagram based on # the custom_performance_diagram.yaml custom config file. mpr_plot.main(custom_config_filename) -def cleanup(): - # remove the line.png and .points files - # from any previous runs - try: - path = os.getcwd() - plot_file = 'mpr_plots.png' - os.remove(os.path.join(path, plot_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - - @pytest.mark.parametrize("test_input, expected", - (["./mpr_plots.png", True],["./mpr_plots_expected.png", True])) -def test_files_exist(setup, test_input, expected): + (["mpr_plots.png", True], ["mpr_plots_expected.png", True])) +def test_files_exist(setup, test_input, expected, remove_files): """ Checking that the plot and data files are getting created """ - assert os.path.isfile(test_input) == expected - cleanup() + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, CLEANUP_FILES) + @pytest.mark.skip("unreliable-sometimes fails due to differences between machines.") -def test_images_match(setup): +def test_images_match(setup, remove_files): """ Compare an expected plot with the newly created plot to verify that the plot hasn't changed in appearance. """ - comparison = CompareImages('./mpr_plots_expected.png', './mpr_plots.png') + comparison = CompareImages(f'{cwd}/mpr_plots_expected.png', f'{cwd}/mpr_plots.png') assert comparison.mssim == 1 - cleanup() + remove_files(cwd, CLEANUP_FILES) diff --git a/test/performance_diagram/custom_performance_diagram.yaml b/test/performance_diagram/custom_performance_diagram.yaml index 03bb45fd..6527cb1e 100644 --- a/test/performance_diagram/custom_performance_diagram.yaml +++ b/test/performance_diagram/custom_performance_diagram.yaml @@ -104,8 +104,8 @@ series_line_style: - "--" # dotted line - ":" -stat_input: ./plot_20200317_151252.data -plot_filename: ./performance_diagram_actual.png +stat_input: !ENV '${TEST_DIR}/plot_20200317_151252.data' +plot_filename: !ENV '${TEST_DIR}/performance_diagram_actual.png' event_equal: False # To save your log output to a file, specify a path and filename and uncomment the line below. Make sure you have diff --git a/test/performance_diagram/custom_performance_diagram_defaultpoints1.yaml b/test/performance_diagram/custom_performance_diagram_defaultpoints1.yaml index 2aaa321d..d6e5480f 100644 --- a/test/performance_diagram/custom_performance_diagram_defaultpoints1.yaml +++ b/test/performance_diagram/custom_performance_diagram_defaultpoints1.yaml @@ -102,6 +102,6 @@ series_line_style: - "--" # dotted line - ":" -stat_input: ./plot_20200317_151252.data -plot_filename: ./performance_diagram_defaultpoints1.png +stat_input: !ENV '${TEST_DIR}/plot_20200317_151252.data' +plot_filename: !ENV '${TEST_DIR}/performance_diagram_defaultpoints1.png' event_equalize: False diff --git a/test/performance_diagram/custom_performance_diagram_points1.yaml b/test/performance_diagram/custom_performance_diagram_points1.yaml index 67a62c84..aaf20796 100644 --- a/test/performance_diagram/custom_performance_diagram_points1.yaml +++ b/test/performance_diagram/custom_performance_diagram_points1.yaml @@ -5,7 +5,7 @@ title: Performance Diagram ("Custom title") xaxis: Success Ratio dump_points_1: 'True' -points_path: './intermed_files' +points_path: !ENV '${TEST_DIR}/intermed_files' # support two y-axes yaxis_1: Probability of Detection (PODY) @@ -99,6 +99,6 @@ series_line_style: - "--" # dotted line - ":" -stat_input: ./plot_20200317_151252.data -plot_filename: ./performance_diagram_actual_points1.png +stat_input: !ENV '${TEST_DIR}/plot_20200317_151252.data' +plot_filename: !ENV '${TEST_DIR}/performance_diagram_actual_points1.png' event_equalize: False diff --git a/test/performance_diagram/test_performance_diagram.py b/test/performance_diagram/test_performance_diagram.py index 47a8961b..511e8a8d 100644 --- a/test/performance_diagram/test_performance_diagram.py +++ b/test/performance_diagram/test_performance_diagram.py @@ -4,14 +4,17 @@ from metplotpy.plots.performance_diagram import performance_diagram as pd #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) + +CLEANUP_FILES = ['performance_diagram_actual.png', 'plot_20200317_151252.points1'] + + @pytest.fixture -def setup(): +def setup(setup_env, remove_files): + setup_env(cwd) # Cleanup the plotfile and point1 output file from any previous run - # cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./custom_performance_diagram.yaml" + remove_files(cwd, CLEANUP_FILES) + custom_config_filename = f"{cwd}/custom_performance_diagram.yaml" # print("\n current directory: ", os.getcwd()) # print("\ncustom config file: ", custom_config_filename, '\n') @@ -22,43 +25,27 @@ def setup(): pd.main(custom_config_filename) -def cleanup(): - # remove the performance_diagram_expected.png and plot_20200317_151252.points1 files - # from any previous runs - try: - path = os.getcwd() - plot_file = 'performance_diagram_actual.png' - points_file = 'plot_20200317_151252.points1' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - - -@pytest.mark.parametrize("test_input,expected_bool",(["./performance_diagram_actual.png", True], ["./plot_20200317_151252.points1", False])) -def test_plot_exists(setup, test_input, expected_bool): +@pytest.mark.parametrize("test_input,expected_bool",(["performance_diagram_actual.png", True], ["plot_20200317_151252.points1", False])) +def test_plot_exists(setup, test_input, expected_bool, remove_files): ''' Checking that only the plot file is getting created and the .point1 data file is not (dump_points_1 is 'False' in the test config file) ''' - assert os.path.isfile(test_input) == expected_bool - # cleanup() + assert os.path.isfile(f"{cwd}/{test_input}") == expected_bool + remove_files(cwd, CLEANUP_FILES) @pytest.mark.parametrize("test_input,expected_bool",(["./performance_diagram_actual_points1.png", True], ["./intermed_files/plot_20200317_151252.points1", True])) -def test_files_exist(test_input, expected_bool): +def test_files_exist(setup_env, test_input, expected_bool, remove_files): ''' Checking that only the plot file is getting created and the .point1 data file is not (dump_points_1 is 'False' in the test config file) ''' - - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./custom_performance_diagram_points1.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_performance_diagram_points1.yaml" try: - os.mkdir(os.path.join(os.getcwd(), 'intermed_files')) - except FileExistsError as e: + os.mkdir(os.path.join(cwd, 'intermed_files')) + except FileExistsError: pass # Invoke the command to generate a Performance Diagram based on @@ -67,32 +54,25 @@ def test_files_exist(test_input, expected_bool): # or augment settings defined by the default config file. pd.main(custom_config_filename) - assert os.path.isfile(test_input) == expected_bool + assert os.path.isfile(f"{cwd}/{test_input}") == expected_bool + remove_files(cwd, ['performance_diagram_actual_points1.png', 'intermed_files/plot_20200317_151252.points1']) try: - path = os.getcwd() - plot_file = 'performance_diagram_actual_points1.png' - points_file = './intermed_files/plot_20200317_151252.points1' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file)) - os.rmdir(os.path.join(path, './intermed_files')) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. + os.rmdir(os.path.join(cwd, 'intermed_files')) + except OSError: pass -@pytest.mark.parametrize("test_input,expected_bool",(["./performance_diagram_defaultpoints1.png", True], ["./plot_20200317_151252.points1", True])) -def test_files_exist(test_input, expected_bool): +@pytest.mark.parametrize("test_input,expected_bool", (["performance_diagram_defaultpoints1.png", True], ["plot_20200317_151252.points1", True])) +def test_files_exist(setup_env, test_input, expected_bool, remove_files): ''' Checking that only the plot file is getting created and the .point1 data file is not (dump_points_1 is 'False' in the test config file) ''' - - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./custom_performance_diagram_defaultpoints1.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_performance_diagram_defaultpoints1.yaml" try: - os.mkdir(os.path.join(os.getcwd(), 'intermed_files')) - except FileExistsError as e: + os.mkdir(os.path.join(cwd, 'intermed_files')) + except FileExistsError: pass # Invoke the command to generate a Performance Diagram based on @@ -101,34 +81,26 @@ def test_files_exist(test_input, expected_bool): # or augment settings defined by the default config file. pd.main(custom_config_filename) - assert os.path.isfile(test_input) == expected_bool + assert os.path.isfile(f"{cwd}/{test_input}") == expected_bool + remove_files(cwd, ['performance_diagram_defaultpoints1.png', 'plot_20200317_151252.points1']) try: - path = os.getcwd() - plot_file = 'performance_diagram_defaultpoints1.png' - points_file = 'plot_20200317_151252.points1' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file)) - os.rmdir(os.path.join(path,'./intermed_files')) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. + os.rmdir(os.path.join(cwd, 'intermed_files')) + except OSError: pass @pytest.mark.skip() -def test_images_match(setup): +def test_images_match(setup, remove_files): ''' Compare an expected plot with the newly created plot to verify that the plot hasn't changed in appearance. ''' - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./custom_performance_diagram.yaml" + custom_config_filename = f"{cwd}/custom_performance_diagram.yaml" pd.main(custom_config_filename) - path = os.getcwd() plot_file = 'performance_diagram_actual.png' - actual_file = os.path.join(path, plot_file) - comparison = CompareImages('./performance_diagram_expected.png',actual_file) + actual_file = os.path.join(cwd, plot_file) + comparison = CompareImages(f'{cwd}/performance_diagram_expected.png', actual_file) assert comparison.mssim == 1 - cleanup() + remove_files(cwd, CLEANUP_FILES) diff --git a/test/reliability_diagram/custom_reliability_diagram.yaml b/test/reliability_diagram/custom_reliability_diagram.yaml index be310b07..30c8e6bb 100644 --- a/test/reliability_diagram/custom_reliability_diagram.yaml +++ b/test/reliability_diagram/custom_reliability_diagram.yaml @@ -81,7 +81,7 @@ plot_ci: - none plot_disp: - 'True' -plot_filename: ./custom_reliability_diagram.png +plot_filename: !ENV '${TEST_DIR}/custom_reliability_diagram.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -111,7 +111,7 @@ series_val_2: {} show_nstats: 'False' show_signif: - 'False' -stat_input: ./plot_20210311_145053.data +stat_input: !ENV '${TEST_DIR}/plot_20210311_145053.data' summary_curves: [] sync_yaxes: 'False' title: test title diff --git a/test/reliability_diagram/custom_reliability_points1.yaml b/test/reliability_diagram/custom_reliability_points1.yaml index c2ba8608..adcbb785 100644 --- a/test/reliability_diagram/custom_reliability_points1.yaml +++ b/test/reliability_diagram/custom_reliability_points1.yaml @@ -96,7 +96,7 @@ plot_stat: median plot_type: png16m plot_units: in plot_width: 11.0 -points_path: ./intermed_files +points_path: !ENV '${TEST_DIR}/intermed_files' random_seed: null rely_event_hist: 'True' series_line_style: @@ -180,8 +180,8 @@ ytlab_horiz: 0.5 ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -plot_filename: ./intermed_files/reliability.png -stat_input: ./reliability.data +plot_filename: !ENV '${TEST_DIR}/intermed_files/reliability.png' +stat_input: !ENV '${TEST_DIR}/reliability.data' noskill_line_col: 'green' reference_line_col: 'blue' show_legend: diff --git a/test/reliability_diagram/custom_reliability_use_defaults.yaml b/test/reliability_diagram/custom_reliability_use_defaults.yaml index 344a6bb1..b2a2d711 100644 --- a/test/reliability_diagram/custom_reliability_use_defaults.yaml +++ b/test/reliability_diagram/custom_reliability_use_defaults.yaml @@ -179,8 +179,8 @@ ytlab_horiz: 0.5 ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -plot_filename: ./reliability.png -stat_input: ./reliability.data +plot_filename: !ENV '${TEST_DIR}/reliability.png' +stat_input: !ENV '${TEST_DIR}/reliability.data' show_legend: - 'True' - 'True' diff --git a/test/reliability_diagram/test_reliability_diagram.py b/test/reliability_diagram/test_reliability_diagram.py index a6413636..0ceeb222 100644 --- a/test/reliability_diagram/test_reliability_diagram.py +++ b/test/reliability_diagram/test_reliability_diagram.py @@ -4,81 +4,62 @@ from metplotpy.plots.reliability_diagram import reliability as r #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) +CLEANUP_FILES = ['reliability.png', 'reliability.points1'] + @pytest.fixture -def setup(): +def setup(setup_env, remove_files): # Cleanup the plotfile and point1 output file from any previous run - cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./custom_reliability_use_defaults.yaml" + remove_files(cwd, CLEANUP_FILES) + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_reliability_use_defaults.yaml" # Invoke the command to generate a Performance Diagram based on # the test_custom_performance_diagram.yaml custom config file. r.main(custom_config_filename) -def cleanup(): - # remove the line.png and .points files - # from any previous runs - try: - path = os.getcwd() - plot_file = 'reliability.png' - points_file_1 = 'reliability.points1' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file_1)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - - @pytest.mark.parametrize("test_input,expected", - (["./reliability.png", True], ["./reliability.points1", True])) -def test_files_exist(setup, test_input, expected): + ([CLEANUP_FILES[0], True], [CLEANUP_FILES[1], True])) +def test_files_exist(setup, test_input, expected, remove_files): ''' Checking that the plot and data files are getting created ''' - assert os.path.isfile(test_input) == expected - cleanup() + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, CLEANUP_FILES) + @pytest.mark.skip("depends on machine on which this is run") -def test_images_match(setup): +def test_images_match(setup, remove_files): ''' Compare an expected plot with the newly created plot to verify that the plot hasn't changed in appearance. ''' - comparison = CompareImages('reliability.png', 'reliability_expected.png') + comparison = CompareImages(f'{cwd}/{CLEANUP_FILES[0]}', f'{cwd}/{CLEANUP_FILES[1]}') assert comparison.mssim == 1 - cleanup() + remove_files(cwd, CLEANUP_FILES) + @pytest.mark.parametrize("test_input,expected", - (["./intermed_files/reliability.png", True], ["./intermed_files/reliability.points1", True])) -def test_files_exist(test_input, expected): + (["intermed_files/reliability.png", True], ["intermed_files/reliability.points1", True])) +def test_files_exist(setup_env, test_input, expected, remove_files): ''' Checking that the plot and data files are getting created ''' try: - os.mkdir(os.path.join(os.getcwd(), 'intermed_files')) - except FileExistsError as e: + os.mkdir(os.path.join(cwd, 'intermed_files')) + except FileExistsError: pass - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./custom_reliability_points1.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_reliability_points1.yaml" r.main(custom_config_filename) - assert os.path.isfile(test_input) == expected + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, ['intermed_files/reliability.png', 'intermed_files/reliability.points1']) try: - path = os.getcwd() - plot_file = 'reliability.png' - points_file = 'reliability.points1' - subdir = os.path.join(path, './intermed_files') - os.remove(os.path.join(subdir, plot_file)) - os.remove(os.path.join(subdir, points_file)) - os.rmdir(subdir) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. + os.rmdir(os.path.join(cwd, 'intermed_files')) + except OSError: pass diff --git a/test/revision_box/custom2_revision_box.yaml b/test/revision_box/custom2_revision_box.yaml index 600dd96b..2334f4f8 100644 --- a/test/revision_box/custom2_revision_box.yaml +++ b/test/revision_box/custom2_revision_box.yaml @@ -82,7 +82,7 @@ plot_caption: '' plot_disp: - 'True' - 'True' -plot_filename: ./intermed_files/revision_box.png +plot_filename: !ENV '${TEST_DIR}/intermed_files/revision_box.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -99,7 +99,7 @@ series_val_1: model: - AFWAOCv3.5.1_d01 - NoahMPv3.5.1_d01 -stat_input: ./revision_box.data +stat_input: !ENV '${TEST_DIR}/revision_box.data' title: test title title_align: 0.5 title_offset: -2 @@ -130,7 +130,7 @@ ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -points_path: ./intermed_files +points_path: !ENV '${TEST_DIR}/intermed_files' show_legend: - 'True' - 'True' \ No newline at end of file diff --git a/test/revision_box/custom_revision_box.yaml b/test/revision_box/custom_revision_box.yaml index 55760b48..fed2f274 100644 --- a/test/revision_box/custom_revision_box.yaml +++ b/test/revision_box/custom_revision_box.yaml @@ -82,7 +82,7 @@ plot_caption: '' plot_disp: - 'True' - 'True' -plot_filename: ./revision_box.png +plot_filename: !ENV '${TEST_DIR}/revision_box.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -99,7 +99,7 @@ series_val_1: model: - AFWAOCv3.5.1_d01 - NoahMPv3.5.1_d01 -stat_input: ./revision_box.data +stat_input: !ENV '${TEST_DIR}/revision_box.data' title: test title title_align: 0.5 title_offset: -2 diff --git a/test/revision_box/test_revision_box.py b/test/revision_box/test_revision_box.py index 44760e65..6932327f 100644 --- a/test/revision_box/test_revision_box.py +++ b/test/revision_box/test_revision_box.py @@ -3,85 +3,60 @@ from metplotpy.plots.revision_box import revision_box #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) +CLEANUP_FILES = ['revision_box.png', 'revision_box.points1'] + @pytest.fixture -def setup(): +def setup(remove_files, setup_env): # Cleanup the plotfile and point1 output file from any previous run - cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_revision_box.yaml" + remove_files(cwd, CLEANUP_FILES) + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_revision_box.yaml" # Invoke the command to generate a Revision Box plot based on # the custom_revision_box.yaml custom config file. revision_box.main(custom_config_filename) -def cleanup(): - # remove the .png and .points files - # from any previous runs - try: - path = os.getcwd() - plot_file = 'revision_box.png' - points_file_1 = 'revision_box.points1' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file_1)) - except OSError as er: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - - @pytest.mark.parametrize("test_input, expected", - (["./revision_box.png", True],["./revision_box.points1", True])) -def test_files_exist( setup, test_input, expected): + (["revision_box.png", True], ["revision_box.points1", True])) +def test_files_exist(setup, test_input, expected, remove_files): """ Checking that the plot and data files are getting created """ - assert os.path.isfile(test_input) == expected - cleanup() + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, CLEANUP_FILES) + @pytest.mark.skip("fails on linux hosts") -def test_images_match(setup): +def test_images_match(setup, remove_files): """ Compare an expected plot with the newly created plot to verify that the plot hasn't changed in appearance. """ - comparison = CompareImages('./revision_box_expected.png', './revision_box.png') + comparison = CompareImages(f'{cwd}/revision_box_expected.png', f'{cwd}/revision_box.png') assert comparison.mssim == 1 - cleanup() + remove_files(cwd, CLEANUP_FILES) @pytest.mark.parametrize("test_input, expected", - (["./intermed_files/revision_box.png", True], ["./intermed_files/revision_box.points1", True])) -def test_files_exist(test_input, expected): + (["intermed_files/revision_box.png", True], ["intermed_files/revision_box.points1", True])) +def test_files_exist(setup_env, test_input, expected, remove_files): """ Checking that the plot and data files are getting created """ + setup_env(cwd) try: - os.mkdir(os.path.join(os.getcwd(), './intermed_files')) - except FileExistsError as e: + os.mkdir(os.path.join(cwd, 'intermed_files')) + except FileExistsError: pass - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom2_revision_box.yaml" + + custom_config_filename = f"{cwd}/custom2_revision_box.yaml" # Invoke the command to generate a Bar plot based on # the custom_ens_ss.yaml custom config file. revision_box.main(custom_config_filename) - assert os.path.isfile(test_input) == expected - cleanup() - try: - path = os.getcwd() - subdir = os.path.join(path, "./intermed_files") - plot_file = 'revision_box.png' - points_file_1 = 'revision_box.points1' - os.remove(os.path.join(subdir, plot_file)) - os.remove(os.path.join(subdir, points_file_1)) - os.rmdir(subdir) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, CLEANUP_FILES) diff --git a/test/revision_series/custom2_revision_series.yaml b/test/revision_series/custom2_revision_series.yaml index 59778bdf..9c05a30b 100644 --- a/test/revision_series/custom2_revision_series.yaml +++ b/test/revision_series/custom2_revision_series.yaml @@ -69,7 +69,7 @@ plot_caption: '' plot_disp: - 'True' - 'True' -plot_filename: ./intermed_files/revision_series.png +plot_filename: !ENV '${TEST_DIR}/intermed_files/revision_series.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -95,7 +95,7 @@ series_val_1: - NoahMPv3.5.1_d01 start_from_zero: 'False' -stat_input: ./revision_series.data +stat_input: !ENV '${TEST_DIR}/revision_series.data' title: Revision series title_align: 0.5 title_offset: -2 @@ -125,7 +125,7 @@ ytlab_horiz: 0.5 ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -points_path: ./intermed_files +points_path: !ENV '${TEST_DIR}/intermed_files' show_legend: - 'True' - 'True' \ No newline at end of file diff --git a/test/revision_series/custom_revision_series.yaml b/test/revision_series/custom_revision_series.yaml index d84bbe34..30ddf814 100644 --- a/test/revision_series/custom_revision_series.yaml +++ b/test/revision_series/custom_revision_series.yaml @@ -69,7 +69,7 @@ plot_caption: '' plot_disp: - 'True' - 'True' -plot_filename: ./revision_series.png +plot_filename: !ENV '${TEST_DIR}/revision_series.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -95,7 +95,7 @@ series_val_1: - NoahMPv3.5.1_d01 start_from_zero: 'False' -stat_input: ./revision_series.data +stat_input: !ENV '${TEST_DIR}/revision_series.data' title: Revision series title_align: 0.5 title_offset: -2 diff --git a/test/revision_series/test_revision_series.py b/test/revision_series/test_revision_series.py index 5eb4e6ec..c381900a 100644 --- a/test/revision_series/test_revision_series.py +++ b/test/revision_series/test_revision_series.py @@ -3,47 +3,34 @@ from metplotpy.plots.revision_series import revision_series #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) +CLEANUP_FILES = ('revision_series.png', 'revision_series.points1') + @pytest.fixture -def setup(): +def setup(remove_files, setup_env): # Cleanup the plotfile and point1 output file from any previous run - cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom_revision_series.yaml" + remove_files(cwd, CLEANUP_FILES) + setup_env(cwd) + custom_config_filename = f"{cwd}/custom_revision_series.yaml" # Invoke the command to generate a Revision Series plot based on # the custom_revision_series.yaml custom config file. revision_series.main(custom_config_filename) -def cleanup(): - # remove the .png and .points files - # from any previous runs - try: - path = os.getcwd() - plot_file = 'revision_series.png' - points_file_1 = 'revision_series.points1' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file_1)) - except OSError as er: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - - @pytest.mark.parametrize("test_input, expected", - (["./revision_series.png", True],["./revision_series.points1", True])) -def test_files_exist( setup, test_input, expected): + (["revision_series.png", True], ["revision_series.points1", True])) +def test_files_exist(setup, test_input, expected, remove_files): """ Checking that the plot and data files are getting created """ - assert os.path.isfile(test_input) == expected - cleanup() + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, CLEANUP_FILES) + @pytest.mark.skip("fails on linux hosts") -def test_images_match(setup): +def test_images_match(setup, remove_files): """ Compare an expected plot with the newly created plot to verify that the plot hasn't @@ -51,37 +38,27 @@ def test_images_match(setup): """ comparison = CompareImages('./revision_series_expected.png', './revision_series.png') assert comparison.mssim == 1 - cleanup() + remove_files(cwd, CLEANUP_FILES) @pytest.mark.parametrize("test_input, expected", - (["./intermed_files/revision_series.png", True], ["./intermed_files/revision_series.points1", True])) -def test_files_exist(test_input, expected): + (["intermed_files/revision_series.png", True], + ["intermed_files/revision_series.points1", True])) +def test_files_exist(setup_env, test_input, expected, remove_files): """ Checking that the plot and data files are getting created """ try: - os.mkdir(os.path.join(os.getcwd(), './intermed_files')) - except FileExistsError as e: + os.mkdir(os.path.join(cwd, 'intermed_files')) + except FileExistsError: pass - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "custom2_revision_series.yaml" + + setup_env(cwd) + custom_config_filename = f"{cwd}/custom2_revision_series.yaml" # Invoke the command to generate a Bar plot based on # the custom_ens_ss.yaml custom config file. revision_series.main(custom_config_filename) - assert os.path.isfile(test_input) == expected - cleanup() - try: - path = os.getcwd() - subdir = os.path.join(path, "./intermed_files") - plot_file = 'revision_series.png' - points_file_1 = 'revision_series.points1' - os.remove(os.path.join(subdir, plot_file)) - os.remove(os.path.join(subdir, points_file_1)) - os.rmdir(subdir) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, CLEANUP_FILES) diff --git a/test/roc_diagram/CTC_ROC.yaml b/test/roc_diagram/CTC_ROC.yaml index 57eccbea..f4a72858 100644 --- a/test/roc_diagram/CTC_ROC.yaml +++ b/test/roc_diagram/CTC_ROC.yaml @@ -62,7 +62,7 @@ plot_ci: - none plot_disp: - 'True' -plot_filename: ./CTC_ROC.png +plot_filename: !ENV '${TEST_DIR}/CTC_ROC.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -88,7 +88,7 @@ series_val_2: {} show_nstats: 'False' show_signif: - 'False' -stat_input: CTC_ROC.data +stat_input: !ENV '${TEST_DIR}/CTC_ROC.data' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/roc_diagram/CTC_ROC_ee.yaml b/test/roc_diagram/CTC_ROC_ee.yaml index 94103e4d..4391d2fd 100644 --- a/test/roc_diagram/CTC_ROC_ee.yaml +++ b/test/roc_diagram/CTC_ROC_ee.yaml @@ -62,7 +62,7 @@ plot_ci: - none plot_disp: - 'True' -plot_filename: ./CTC_ROC_ee.png +plot_filename: !ENV '${TEST_DIR}/CTC_ROC_ee.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -88,7 +88,7 @@ series_val_2: {} show_nstats: 'False' show_signif: - 'False' -stat_input: CTC_ROC.data +stat_input: !ENV '${TEST_DIR}/CTC_ROC.data' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/roc_diagram/CTC_ROC_summary.yaml b/test/roc_diagram/CTC_ROC_summary.yaml index bc5f4c23..e24ca49a 100644 --- a/test/roc_diagram/CTC_ROC_summary.yaml +++ b/test/roc_diagram/CTC_ROC_summary.yaml @@ -77,7 +77,7 @@ plot_disp: - 'True' - 'True' - 'True' -plot_filename: ./CTC_ROC_summary.png +plot_filename: !ENV '${TEST_DIR}/CTC_ROC_summary.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -123,7 +123,7 @@ show_signif: - 'False' - 'False' start_from_zero: 'False' -stat_input: CTC_ROC_summary.data +stat_input: !ENV '${TEST_DIR}/CTC_ROC_summary.data' summary_curve: median sync_yaxes: 'False' title: test title @@ -174,5 +174,5 @@ ytlab_horiz: 0.5 ytlab_orient: 1 ytlab_perp: 0.5 ytlab_size: 1 -points_path: './intermed_files' +points_path: !ENV '${TEST_DIR}/intermed_files' diff --git a/test/roc_diagram/CTC_ROC_thresh.yaml b/test/roc_diagram/CTC_ROC_thresh.yaml index 61a2dc76..7bbc26ca 100644 --- a/test/roc_diagram/CTC_ROC_thresh.yaml +++ b/test/roc_diagram/CTC_ROC_thresh.yaml @@ -64,7 +64,7 @@ plot_ci: - none plot_disp: - 'True' -plot_filename: ./CTC_ROC_thresh.png +plot_filename: !ENV '${TEST_DIR}/CTC_ROC_thresh.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -90,7 +90,7 @@ series_val_2: {} show_nstats: 'False' show_signif: - 'False' -stat_input: CTC_ROC_thresh.data +stat_input: !ENV '${TEST_DIR}/CTC_ROC_thresh.data' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/roc_diagram/CTC_ROC_thresh_dump_pts.yaml b/test/roc_diagram/CTC_ROC_thresh_dump_pts.yaml index 41ae9784..3432b2ae 100644 --- a/test/roc_diagram/CTC_ROC_thresh_dump_pts.yaml +++ b/test/roc_diagram/CTC_ROC_thresh_dump_pts.yaml @@ -64,14 +64,14 @@ plot_ci: - none plot_disp: - 'True' -plot_filename: ./CTC_ROC_thresh_dump_pts.png +plot_filename: !ENV '${TEST_DIR}/CTC_ROC_thresh_dump_pts.png' plot_height: 8.5 plot_res: 72 plot_stat: median plot_type: png16m plot_units: in plot_width: 11.0 -points_path: './intermed_files' +points_path: !ENV '${TEST_DIR}/intermed_files' random_seed: null reverse_connection_order: 'True' roc_ctc: true @@ -91,7 +91,7 @@ series_val_2: {} show_nstats: 'False' show_signif: - 'False' -stat_input: CTC_ROC_thresh.data +stat_input: !ENV '${TEST_DIR}/CTC_ROC_thresh.data' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/roc_diagram/CTC_ROC_thresh_reverse_pts.yaml b/test/roc_diagram/CTC_ROC_thresh_reverse_pts.yaml index ce8717b2..d6d0e4a7 100644 --- a/test/roc_diagram/CTC_ROC_thresh_reverse_pts.yaml +++ b/test/roc_diagram/CTC_ROC_thresh_reverse_pts.yaml @@ -64,7 +64,7 @@ plot_ci: - none plot_disp: - 'True' -plot_filename: ./CTC_ROC_thresh.png +plot_filename: !ENV '${TEST_DIR}/CTC_ROC_thresh.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -90,7 +90,7 @@ series_val_2: {} show_nstats: 'False' show_signif: - 'False' -stat_input: CTC_ROC_thresh.data +stat_input: !ENV '${TEST_DIR}/CTC_ROC_thresh.data' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/roc_diagram/CTC_wind_reformatted.yaml b/test/roc_diagram/CTC_wind_reformatted.yaml index 4b3912fa..3874c2d2 100644 --- a/test/roc_diagram/CTC_wind_reformatted.yaml +++ b/test/roc_diagram/CTC_wind_reformatted.yaml @@ -64,7 +64,7 @@ plot_ci: - none plot_disp: - 'True' -plot_filename: ./CTC_wind_reformatted.png +plot_filename: !ENV '${TEST_DIR}/CTC_wind_reformatted.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -90,7 +90,7 @@ series_val_2: {} show_nstats: 'False' show_signif: - 'False' -stat_input: ctc_reformatted_wind_P500.data +stat_input: !ENV '${TEST_DIR}/ctc_reformatted_wind_P500.data' sync_yaxes: 'False' title: test title title_align: 0.5 diff --git a/test/roc_diagram/PCT_ROC.yaml b/test/roc_diagram/PCT_ROC.yaml index 4020f310..8d0b8b64 100644 --- a/test/roc_diagram/PCT_ROC.yaml +++ b/test/roc_diagram/PCT_ROC.yaml @@ -62,7 +62,7 @@ plot_ci: - none plot_disp: - 'True' -plot_filename: ./PCT_ROC.png +plot_filename: !ENV '${TEST_DIR}/PCT_ROC.png' plot_height: 8.5 plot_res: 72 plot_stat: median @@ -88,7 +88,7 @@ series_val_2: {} show_nstats: 'False' show_signif: - 'False' -stat_input: PCT_ROC.data +stat_input: !ENV '${TEST_DIR}/PCT_ROC.data' sync_yaxes: 'False' title: test title PCT ROC data diff --git a/test/roc_diagram/custom_roc_diagram.yaml b/test/roc_diagram/custom_roc_diagram.yaml index 1b5a53a3..3a1e463d 100644 --- a/test/roc_diagram/custom_roc_diagram.yaml +++ b/test/roc_diagram/custom_roc_diagram.yaml @@ -98,8 +98,8 @@ reverse_connection_order: False # Make the plot generated in METviewer interactive create_html: 'True' -stat_input: ./plot_20200507_074426.data -plot_filename: ./roc_diagram_custom.png +stat_input: !ENV '${TEST_DIR}/plot_20200507_074426.data' +plot_filename: !ENV '${TEST_DIR}/roc_diagram_custom.png' # To save your log output to a file, specify a path and filename and uncomment the line below. Make sure you have # permissions to the directory you specify. The default, as specified in the default config file is stdout. diff --git a/test/roc_diagram/test_roc_diagram.py b/test/roc_diagram/test_roc_diagram.py index 25201458..a5c22d37 100644 --- a/test/roc_diagram/test_roc_diagram.py +++ b/test/roc_diagram/test_roc_diagram.py @@ -7,17 +7,17 @@ # from metcalcpy.compare_images import CompareImages import metcalcpy.util.ctc_statistics as ctc +cwd = os.path.dirname(__file__) + # Fixture used for the image comparison # test. @pytest.fixture -def setup(): +def setup(setup_env): # Cleanup the plotfile and point1 output file from any previous run cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./CTC_ROC_thresh.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/CTC_ROC_thresh.yaml" # print("\n current directory: ", os.getcwd()) # print("\ncustom config file: ", custom_config_filename, '\n') @@ -27,14 +27,12 @@ def setup(): @pytest.fixture -def setup_rev_points(): +def setup_rev_points(setup_env): # Cleanup the plotfile and point1 output file from any previous run - path = os.getcwd() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./CTC_ROC_thresh_reverse_pts.yaml" - print("\n current directory: ", os.getcwd()) + cleanup() + setup_env(cwd) + custom_config_filename = f"{cwd}/CTC_ROC_thresh_reverse_pts.yaml" + print("\n current directory: ", cwd) print("\ncustom config file: ", custom_config_filename, '\n') # Invoke the command to generate a Performance Diagram based on @@ -43,20 +41,15 @@ def setup_rev_points(): @pytest.fixture -def setup_summary(): +def setup_summary(setup_env): # Cleanup the plotfile and point1 output file from any previous run - path = os.getcwd() - - subdir_path = os.path.join(path, 'intermed_files') - - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./CTC_ROC_summary.yaml" + cleanup() + setup_env(cwd) + custom_config_filename = f"{cwd}/CTC_ROC_summary.yaml" try: - os.mkdir(os.path.join(os.getcwd(), 'intermed_files')) - except FileExistsError as e: + os.mkdir(os.path.join(cwd, 'intermed_files')) + except FileExistsError: pass # Invoke the command to generate a Performance Diagram based on @@ -65,22 +58,19 @@ def setup_summary(): @pytest.fixture -def setup_dump_points(): +def setup_dump_points(setup_env): # Cleanup the plotfile and point1 output file from any previous run - path = os.getcwd() + cleanup() + setup_env(cwd) # put any intermediate files in the intermed_files subdirectory of this current # working directory. *NOTE* This MUST match with what you set up in CTC_ROC_thresh.yaml for the # points_path configuration setting. - subdir_path = os.path.join(path, 'intermed_files') - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./CTC_ROC_thresh_dump_pts.yaml" + custom_config_filename = f"{cwd}/CTC_ROC_thresh_dump_pts.yaml" # print("\n current directory: ", os.getcwd()) # print("\ncustom config file: ", custom_config_filename, '\n') try: - os.mkdir(os.path.join(os.getcwd(), 'intermed_files')) - except FileExistsError as e: + os.mkdir(os.path.join(cwd, 'intermed_files')) + except FileExistsError: pass # Invoke the command to generate a Performance Diagram based on @@ -93,28 +83,25 @@ def cleanup(): # from any previous runs # The subdir_path is where the .points1 file will be stored try: - path = os.getcwd() plot_file = 'CTC_ROC_thresh.png' - html_file = '.html' - os.remove(os.path.join(path, html_file)) - os.remove(os.path.join(path, plot_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. + html_file = 'CTC_ROC_thresh.html' + os.remove(os.path.join(cwd, html_file)) + os.remove(os.path.join(cwd, plot_file)) + except OSError: pass @pytest.mark.parametrize("test_input,expected_boolean", - (["./CTC_ROC_thresh_expected.png", True], ["./CTC_ROC_thresh.points1", False])) + (["CTC_ROC_thresh_expected.png", True], ["CTC_ROC_thresh.points1", False])) def test_files_exist(setup, test_input, expected_boolean): ''' Checking that the plot file is getting created but the points1 file is NOT ''' - assert os.path.isfile(test_input) == expected_boolean + assert os.path.isfile(f"{cwd}/{test_input}") == expected_boolean cleanup() -def test_expected_CTC_thresh_dump_points(setup_dump_points): +def test_expected_ctc_thresh_dump_points(setup_dump_points, remove_files): ''' For test data, verify that the points in the .points1 file are in the directory we specified and the values @@ -123,48 +110,25 @@ def test_expected_CTC_thresh_dump_points(setup_dump_points): ''' expected_pody = pd.Series([1, 0.8457663, 0.7634846, 0.5093934, 0.1228585, 0]) expected_pofd = pd.Series([1, 0.0688293, 0.049127, 0.0247044, 0.0048342, 0]) - df = pd.read_csv("./intermed_files/CTC_ROC_thresh.points1", sep='\t', header='infer') + df = pd.read_csv(f"{cwd}/intermed_files/CTC_ROC_thresh.points1", sep='\t', header='infer') pofd = df.iloc[:, 0] pody = df.iloc[:, 1] for index, expected in enumerate(expected_pody): - if ctc.round_half_up(expected) - ctc.round_half_up(pody[index]) == 0.0: - pass - else: - assert False + assert ctc.round_half_up(expected) - ctc.round_half_up(pody[index]) == 0.0 # if we get here, then all elements matched in value and position - assert True # do the same test for pofd for index, expected in enumerate(expected_pofd): - if ctc.round_half_up(expected) - ctc.round_half_up(pofd[index]) == 0.0: - pass - else: - assert False + assert ctc.round_half_up(expected) - ctc.round_half_up(pofd[index]) == 0.0 # different cleanup than what is provided by cleanup() # clean up the intermediate subdirectory and other files - try: - path = os.getcwd() - plot_file = 'CTC_ROC_thresh_dump_pts.png' - html_file = '.html' - points_file = 'CTC_ROC_thresh.points1' - intermed_path = os.path.join(path, "intermed_files") - os.remove(os.path.join(intermed_path, points_file)) - os.rmdir(intermed_path) - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, html_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - - # if we get here, then all elements matched in value and position - assert True + remove_files(cwd, ['CTC_ROC_thresh_dump_pts.png', 'CTC_ROC_thresh_dump_pts.html']) -def test_expected_CTC_summary(setup_summary): +def test_expected_ctc_summary(setup_summary, remove_files): ''' For test data, verify that the points in the .points1 file are in the directory we specified and the values @@ -174,48 +138,25 @@ def test_expected_CTC_summary(setup_summary): expected_pofd = pd.Series([1, 0.0052708, 0, 1, 0.0084788, 0, 1, 0.0068247, 0]) expected_pody = pd.Series([1, 0.0878715, 0, 1, 0.1166785, 0, 1, 0.1018776, 0]) - df = pd.read_csv("./intermed_files/CTC_ROC_summary.points1", sep='\t', header='infer') + df = pd.read_csv(f"{cwd}/intermed_files/CTC_ROC_summary.points1", sep='\t', header='infer') pofd = df.iloc[:, 0] pody = df.iloc[:, 1] for index, expected in enumerate(expected_pody): - if ctc.round_half_up(expected) - ctc.round_half_up(pody[index]) == 0.0: - pass - else: - assert False + assert ctc.round_half_up(expected) - ctc.round_half_up(pody[index]) == 0.0 # if we get here, then all elements matched in value and position - assert True # do the same test for pofd for index, expected in enumerate(expected_pofd): - if ctc.round_half_up(expected) - ctc.round_half_up(pofd[index]) == 0.0: - pass - else: - assert False + assert ctc.round_half_up(expected) - ctc.round_half_up(pofd[index]) == 0.0 # different cleanup than what is provided by cleanup() # clean up the intermediate subdirectory and other files - try: - path = os.getcwd() - plot_file = 'CTC_ROC_summary.png' - html_file = '.html' - points_file = 'CTC_ROC_summary.points1' - intermed_path = os.path.join(path, "intermed_files") - os.remove(os.path.join(intermed_path, points_file)) - os.rmdir(intermed_path) - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, html_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - - # if we get here, then all elements matched in value and position - assert True + remove_files(cwd, ['CTC_ROC_summary.png', 'CTC_ROC_summary.html']) -def test_expected_CTC_thresh_points_reversed(setup_rev_points): +def test_expected_ctc_thresh_points_reversed(setup_rev_points, remove_files): ''' For test data, verify that the points in the .points1 file match what is expected (within round-off tolerance/acceptable precision) when @@ -225,45 +166,26 @@ def test_expected_CTC_thresh_points_reversed(setup_rev_points): expected_pody = pd.Series([1, 0.8457663, 0.7634846, 0.5093934, 0.1228585, 0]) expected_pofd = pd.Series([1, 0.0688293, 0.0491275, 0.0247044, 0.0048342, 0]) - df = pd.read_csv("./CTC_ROC_thresh.points1", sep='\t', header='infer') + df = pd.read_csv(f"{cwd}/CTC_ROC_thresh.points1", sep='\t', header='infer') pofd = df.iloc[:, 0] pody = df.iloc[:, 1] for index, expected in enumerate(expected_pody): - if ctc.round_half_up(expected) - ctc.round_half_up(pody[index]) == 0.0: - pass - else: - assert False + assert ctc.round_half_up(expected) - ctc.round_half_up(pody[index]) == 0.0 - # if we get here, then all elements matched in value and position - assert True + # if we get here, then all elements matched in value and position # do the same test for pofd for index, expected in enumerate(expected_pofd): - if ctc.round_half_up(expected) - ctc.round_half_up(pofd[index]) == 0.0: - pass - else: - assert False + assert ctc.round_half_up(expected) - ctc.round_half_up(pofd[index]) == 0.0 - # if we get here, then all elements matched in value and position - assert True + # if we get here, then all elements matched in value and position # different cleanup than what is provided by cleanup() - try: - path = os.getcwd() - plot_file = 'CTC_ROC_thresh.png' - points_file = 'CTC_ROC_thresh.points1' - html_file = '.html' - os.remove(os.path.join(path, points_file)) - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, html_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass + remove_files(cwd, ['CTC_ROC_thresh.png', 'CTC_ROC_thresh.points1', 'CTC_ROC_thresh.html']) -def test_ee_returns_empty_df(capsys): +def test_ee_returns_empty_df(capsys, remove_files): ''' use CTC_ROC.data with event equalization set to True. This will result in an empty data frame returned from event equalization. Check for @@ -274,7 +196,7 @@ def test_ee_returns_empty_df(capsys): ''' - custom_config_filename = "./CTC_ROC_ee.yaml" + custom_config_filename = f"{cwd}/CTC_ROC_ee.yaml" roc.main(custom_config_filename) captured = capsys.readouterr() expected = '\nINFO: No resulting data after performing event equalization of axis 1\n' \ @@ -284,18 +206,11 @@ def test_ee_returns_empty_df(capsys): assert captured.out == expected # Clean up - try: - path = os.getcwd() - plot_file = 'CTC_ROC_ee.png' - os.remove(os.path.join(path, plot_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass + remove_files(cwd, ['CTC_ROC_ee.png', 'CTC_ROC_ee.html']) @pytest.mark.skip("skip image comparison") -def test_images_match(setup): +def test_images_match(setup, remove_files): ''' Compare an expected plot with the newly created plot to verify that the plot hasn't @@ -305,47 +220,36 @@ def test_images_match(setup): can sometimes be a different size than the expected (which was generated using the same configuration file and data!) ''' - path = os.getcwd() - plot_file = './CTC_ROC_thresh.png' - actual_file = os.path.join(path, plot_file) - comparison = CompareImages('./CTC_ROC_thresh.png', actual_file) + plot_file = 'CTC_ROC_thresh.png' + actual_file = os.path.join(cwd, plot_file) + comparison = CompareImages(f'{cwd}/CTC_ROC_thresh.png', actual_file) assert comparison.mssim == 1 # cleanup # different cleanup than what is provided by cleanup() - try: - plot_file = 'CTC_ROC_thresh.png' - html_file = '.html' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, html_file)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass + remove_files(cwd, ['CTC_ROC_thresh.png', 'CTC_ROC_thresh.html']) -def test_pct_plot_exists(): +def test_pct_plot_exists(remove_files): ''' Verify that the ROC diagram is generated ''' - custom_config_filename = "./PCT_ROC.yaml" - output_plot = "./PCT_ROC.png" + custom_config_filename = f"{cwd}/PCT_ROC.yaml" + output_plot = f"{cwd}/PCT_ROC.png" print("\n Testing for existence of PCT ROC plot...") roc.main(custom_config_filename) - assert os.path.isfile(output_plot) == True - os.remove(os.path.join(output_plot)) + assert os.path.isfile(output_plot) + remove_files(cwd, ['PCT_ROC.png', 'PCT_ROC.html']) -def test_pct_no_warnings(): +def test_pct_no_warnings(remove_files): ''' Verify that the ROC diagram is generated without FutureWarnings ''' - custom_config_filename = "./PCT_ROC.yaml" - output_plot = "./PCT_ROC.png" - + custom_config_filename = f"{cwd}/PCT_ROC.yaml" print("\n Testing for FutureWarning..") try: @@ -354,26 +258,25 @@ def test_pct_no_warnings(): print("FutureWarning generated") # FutureWarning generated, test fails assert False - else: - assert True - os.remove(os.path.join(output_plot)) + + remove_files(cwd, ['PCT_ROC.png', 'PCT_ROC.html']) -def test_ctc_reformatted(): +def test_ctc_reformatted(remove_files): ''' Verify that the ROC diagram is generated successfully from reformatted CTC data. ''' - custom_config_filename = "./CTC_wind_reformatted.yaml" - output_plot = "./CTC_wind_reformatted.png" + custom_config_filename = f"{cwd}/CTC_wind_reformatted.yaml" + output_plot = f"{cwd}/CTC_wind_reformatted.png" print("\n Testing for presence of the CTC_wind_reformatted.png plot...") roc.main(custom_config_filename) - assert os.path.isfile(output_plot) == True + assert os.path.isfile(output_plot) # Checking for plot size isn't reliable #expected_filesize = int(43239) #plot_filesize = int(os.path.getsize(output_plot)) #assert plot_filesize >= expected_filesize - os.remove(os.path.join(output_plot)) + remove_files(cwd, ['CTC_wind_reformatted.png', 'CTC_wind_reformatted.html']) diff --git a/test/roc_diagram/test_roc_diagram.yaml b/test/roc_diagram/test_roc_diagram.yaml index cf85a3e1..4ccee027 100644 --- a/test/roc_diagram/test_roc_diagram.yaml +++ b/test/roc_diagram/test_roc_diagram.yaml @@ -92,6 +92,6 @@ caption_align: .1 #0 # axis parallel location adjustment # Make the plot generated in METviewer interactive create_html: 'True' # input file is relative to the roc_diagram.py module -stat_input: ./CTC_ROC_thresh.data -plot_filename: ./roc_diagram_actual.png +stat_input: !ENV '${TEST_DIR}/CTC_ROC_thresh.data' +plot_filename: !ENV '${TEST_DIR}/roc_diagram_actual.png' diff --git a/test/skew_t/test_skew_t.py b/test/skew_t/test_skew_t.py index d870b0e7..42e003ec 100644 --- a/test/skew_t/test_skew_t.py +++ b/test/skew_t/test_skew_t.py @@ -6,17 +6,12 @@ from metplotpy.plots.skew_t import skew_t as skew_t # from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) -def test_files_exist(): - ''' - Checking that only the expected plot files are getting created and - input files with only fill/missing data are not created. - ''' - - os.environ['METPLOTPY_BASE'] = "../../" - cur_dir = os.getcwd() - custom_config_filename = os.path.join(cur_dir, "test_skew_t.yaml") +def test_skew_t(setup_env): + setup_env(cwd) + custom_config_filename = os.path.join(cwd, "test_skew_t.yaml") # Invoke the command to generate a skew-T Diagram based on # the test_skew_tm.yaml custom config file. @@ -24,7 +19,7 @@ def test_files_exist(): # Verify that files for the ssh052023 data exists for the 0,6, 12,18,24, 30, 42, # 48, 54, and 60 hour data. - output_dir = os.path.join(cur_dir, 'output' ) + output_dir = os.path.join(cwd, 'output') # Some of these data files have incomplete data so check for the expected hour # plots. @@ -32,7 +27,7 @@ def test_files_exist(): print(f"Output dir: {output_dir}") file_ext = '.png' files_of_interest = [] - for root, dir, files in os.walk(output_dir): + for root, _, files in os.walk(output_dir): for item in files: if item.endswith(file_ext): # print(f"Item of interest: {item}") @@ -40,30 +35,37 @@ def test_files_exist(): base_file = os.path.basename(full_file) files_of_interest.append(base_file) - # List of files for the sh052023 data (which is missing data for hours 66-240). - data_some_missing_data = ['ssh052023_avno_doper_2023010100_diag', - 'ssh052023_avno_doper_2023010106_diag'] + _check_files_exist(files_of_interest) + _check_files_not_created(files_of_interest) + _check_empty_input(files_of_interest) + + # Clean up all png files + shutil.rmtree(output_dir) + # If running without the ' -p no:logging' option, then uncomment to ensure that log + # files are removed. + # shutil.rmtree('./logs') + +def _check_files_exist(files_of_interest): + ''' + Checking that only the expected plot files are getting created and + input files with only fill/missing data are not created. + ''' + # List of files for the sh052023 data (which is missing data for hours 66-240). # Config file is requesting all the available sounding hours - expected_hours_for_2023_010100 = range(0,61,6) - expected_hours_for_2023_010106 = range(0,49,6) + data_some_missing_data = { + '2023010100': range(0, 61, 6), + '2023010106': range(0, 49, 6), + } # Create a list of expected base file names with their expected hours. expected_base_filenames = [] - for base in data_some_missing_data: - # Expected base for expected plot output name of format: - # ssh_052023_avno_doper_202301010[0|6]_diag_[0-9]{1,2}_hr - if base == 'ssh052023_avno_doper_2023010100_diag': - # Working with the 2023010100 date file - for cur_hr in expected_hours_for_2023_010100: - base_hr = base + '_' + str(cur_hr) + '_hr' - expected_base_filenames.append(base_hr) - - elif base == 'ssh052023_avno_doper_2023010106_diag': - # Working with the 2023010106 date - for cur_hr in expected_hours_for_2023_010106: - base_hr = base + '_' + str(cur_hr) + '_hr' - expected_base_filenames.append(base_hr) + # Expected base for expected plot output name of format: + # ssh_052023_avno_doper_202301010[0|6]_diag_[0-9]{1,2}_hr + for filetime, expected_hours in data_some_missing_data.items(): + for cur_hr in expected_hours: + base_hr = f'ssh052023_avno_doper_{filetime}_diag_{cur_hr}_hr' + expected_base_filenames.append(base_hr) # Subset only the files that correspond to the sh052023 data subset_files_of_interest = [] @@ -78,51 +80,13 @@ def test_files_exist(): if expected in subset_files_of_interest: num_found += 1 - if len(subset_files_of_interest) == num_found: - assert True - else: - assert False + assert len(subset_files_of_interest) == num_found - # Clean up all png files - temp_datafile = os.path.join(cur_dir, 'sounding_data.dat') - os.remove(temp_datafile) - shutil.rmtree(output_dir) - # If running without the ' -p no:logging' option, then uncomment to ensure that log - # files are removed. - # shutil.rmtree('./logs') - -def test_files_not_created(): +def _check_files_not_created(files_of_interest): ''' Checking that input files with only fill/missing data are not created. ''' - - os.environ['METPLOTPY_BASE'] = "../../" - cur_dir = os.getcwd() - custom_config_filename = os.path.join(cur_dir, "test_skew_t.yaml") - - # Invoke the command to generate a skew-T Diagram based on - # the test_skew_tm.yaml custom config file. - skew_t.main(custom_config_filename) - - # Verify that files for the ssh052023 data exists for the 0,6, 12,18,24, 30, 42, - # 48, 54, and 60 hour data. - output_dir = os.path.join(cur_dir, 'output' ) - - # Some of these data files have incomplete data so check for the expected hour - # plots. - - print(f"Output dir: {output_dir}") - file_ext = '.png' - files_of_interest = [] - for root, dir, files in os.walk(output_dir): - for item in files: - if item.endswith(file_ext): - # print(f"Item of interest: {item}") - full_file = os.path.join(root, item) - base_file = os.path.basename(full_file) - files_of_interest.append(base_file) - # List of files with no sounding data (9999 for all fields and times) no_sounding_data = ['ssh162023_avno_doper_2023022712_diag', 'ssh162023_avno_doper_2023022800_diag', @@ -153,52 +117,13 @@ def test_files_not_created(): if cur in subsetted_basenames: fail_counter += 1 - if fail_counter == 0: - assert True - else: - assert False - - - # Clean up all png files - temp_datafile = os.path.join(cur_dir, 'sounding_data.dat') - os.remove(temp_datafile) - shutil.rmtree(output_dir) - # If running with the ' -p no:logging' option, then uncomment to ensure that log - # files are removed. - # shutil.rmtree('./logs') + assert fail_counter == 0 -def test_empty_input(): +def _check_empty_input(files_of_interest): ''' Checking that empty input file is not creating any plots. ''' - - os.environ['METPLOTPY_BASE'] = "../../" - cur_dir = os.getcwd() - custom_config_filename = os.path.join(cur_dir, "test_skew_t.yaml") - - # Invoke the command to generate a skew-T Diagram based on - # the test_skew_tm.yaml custom config file. - skew_t.main(custom_config_filename) - - # Verify that files for the ssh052023 data exists for the 0,6, 12,18,24, 30, 42, - # 48, 54, and 60 hour data. - output_dir = os.path.join(cur_dir, 'output') - - # Some of these data files have incomplete data so check for the expected hour - # plots. - - print(f"Output dir: {output_dir}") - file_ext = '.png' - files_of_interest = [] - for root, dir, files in os.walk(output_dir): - for item in files: - if item.endswith(file_ext): - # print(f"Item of interest: {item}") - full_file = os.path.join(root, item) - base_file = os.path.basename(full_file) - files_of_interest.append(base_file) - # List of empty files no_data_empty_file = ['sal092022_avno_doper_2022092800_diag'] @@ -213,18 +138,6 @@ def test_empty_input(): subsetted_files_of_interest.append(cur) match_found = re.match(r'^sal092022_avno_doper_2022092800_diag', - no_data_empty_file[0]) - if match_found in subsetted_files_of_interest: - # The output file was created when it shouldn't have been, fail. - assert False - else: - # The output file was NOT created, as expected. Pass. - assert True - - # Clean up all png files - temp_datafile = os.path.join(cur_dir, 'sounding_data.dat') - os.remove(temp_datafile) - shutil.rmtree(output_dir) - # If running without the ' -p no:logging' option, then uncomment to ensure that log - # files are removed. - # shutil.rmtree('./logs') + no_data_empty_file[0]) + # The output file was created when it shouldn't have been, fail. + assert match_found not in subsetted_files_of_interest diff --git a/test/skew_t/test_skew_t.yaml b/test/skew_t/test_skew_t.yaml index 696d5817..858fc8e0 100644 --- a/test/skew_t/test_skew_t.yaml +++ b/test/skew_t/test_skew_t.yaml @@ -4,11 +4,11 @@ # Input and output file information -input_directory: './data/2023/' +input_directory: !ENV '${TEST_DIR}/data/2023/' input_file_extension: '.dat' -output_directory: ./output +output_directory: !ENV '${TEST_DIR}/output' log_level: "INFO" -log_directory: ./logs +log_directory: !ENV '${TEST_DIR}/logs' log_filename: 'tc_diags_skewt.log' # Sounding hours of interest. If all_sounding_hours is set to False, then the diff --git a/test/taylor_diagram/taylor_diagram_custom.yaml b/test/taylor_diagram/taylor_diagram_custom.yaml index 8d6a3523..26fce36e 100644 --- a/test/taylor_diagram/taylor_diagram_custom.yaml +++ b/test/taylor_diagram/taylor_diagram_custom.yaml @@ -1,8 +1,8 @@ --- # custom config file to override some of the settings in the # default config file taylor_diagram_defaults.yaml -stat_input: ./plot_dlwr_sample.data -plot_filename: ./taylor_diagram_custom.png +stat_input: !ENV '${TEST_DIR}/plot_dlwr_sample.data' +plot_filename: !ENV '${TEST_DIR}/taylor_diagram_custom.png' # change the caption text plot_caption: "Custom caption" diff --git a/test/taylor_diagram/test_neg_and_pos_corr.yaml b/test/taylor_diagram/test_neg_and_pos_corr.yaml index 0e166f23..454f1120 100644 --- a/test/taylor_diagram/test_neg_and_pos_corr.yaml +++ b/test/taylor_diagram/test_neg_and_pos_corr.yaml @@ -1,8 +1,8 @@ --- # custom config file to override some of the settings in the # default config file taylor_diagram_defaults.yaml -stat_input: ./plot_dlwr_sample.data -plot_filename: ./test_neg_and_pos_corr_plot.png +stat_input: !ENV '${TEST_DIR}/plot_dlwr_sample.data' +plot_filename: !ENV '${TEST_DIR}/test_neg_and_pos_corr_plot.png' taylor_show_gamma: True # Show only positive values of correlation taylor_voc: False diff --git a/test/taylor_diagram/test_pos_corr.yaml b/test/taylor_diagram/test_pos_corr.yaml index e30d4ec5..27c995fb 100644 --- a/test/taylor_diagram/test_pos_corr.yaml +++ b/test/taylor_diagram/test_pos_corr.yaml @@ -1,8 +1,8 @@ --- # custom config file to override some of the settings in the # default config file taylor_diagram_defaults.yaml -stat_input: ./plot_dlwr_sample.data -plot_filename: ./test_pos_corr_plot.png +stat_input: !ENV '${TEST_DIR}/plot_dlwr_sample.data' +plot_filename: !ENV '${TEST_DIR}/test_pos_corr_plot.png' taylor_show_gamma: True # Show only positive values of correlation taylor_voc: True \ No newline at end of file diff --git a/test/taylor_diagram/test_taylor_diagram.py b/test/taylor_diagram/test_taylor_diagram.py index 7006af0c..415bee98 100644 --- a/test/taylor_diagram/test_taylor_diagram.py +++ b/test/taylor_diagram/test_taylor_diagram.py @@ -2,39 +2,39 @@ from metplotpy.plots.taylor_diagram import taylor_diagram as td #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) -def test_pos_corr_file_exists(): - os.environ['METPLOTPY_BASE'] = "../../" - test_config_filename = "test_pos_corr.yaml" + +def test_pos_corr_file_exists(setup_env): + setup_env(cwd) + + test_config_filename = f"{cwd}/test_pos_corr.yaml" td.main(test_config_filename) # Verify that a plot was generated - plot_file = "test_pos_corr_plot.png" - expected_file = "expected_pos_corr_plot.png" - path = os.getcwd() - assert os.path.isfile(plot_file) == True - + plot_file = f"{cwd}/test_pos_corr_plot.png" + expected_file = f"{cwd}/expected_pos_corr_plot.png" + assert os.path.isfile(plot_file) # image comparison #comparison = CompareImages(plot_file,expected_file) # assert comparison.mssim >= .99 # Clean up - os.remove(os.path.join(path, plot_file)) + os.remove(plot_file) -def test_pos_corr_file_exists(): - os.environ['METPLOTPY_BASE'] = "../../" - test_config_filename = "test_pos_corr.yaml" +def test_pos_corr_file_exists(setup_env): + setup_env(cwd) + test_config_filename = f"{cwd}/test_pos_corr.yaml" td.main(test_config_filename) # Verify that a plot was generated - plot_file = "test_pos_corr_plot.png" - path = os.getcwd() - assert os.path.isfile(plot_file) == True + plot_file = f"{cwd}/test_pos_corr_plot.png" + assert os.path.isfile(plot_file) # Clean up - os.remove(os.path.join(path, plot_file)) + os.remove(plot_file) # Not reliable when the expected image is generated on a Mac and then this # test is run on non-Mac machine. @@ -55,47 +55,47 @@ def test_pos_corr_file_exists(): # # Clean up # os.remove(os.path.join(path, plot_file)) -def test_neg_and_pos_corr_file_exists(): - os.environ['METPLOTPY_BASE'] = "../../" - test_config_filename = "test_neg_and_pos_corr.yaml" + +def test_neg_and_pos_corr_file_exists(setup_env): + setup_env(cwd) + test_config_filename = f"{cwd}/test_neg_and_pos_corr.yaml" td.main(test_config_filename) # Verify that a plot was generated - plot_file = "test_neg_and_pos_corr_plot.png" - path = os.getcwd() - assert os.path.isfile(plot_file) == True + plot_file = f"{cwd}/test_neg_and_pos_corr_plot.png" + assert os.path.isfile(plot_file) # Clean up - os.remove(os.path.join(path, plot_file)) + os.remove(plot_file) + # Not reliable when the expected image is generated on a Mac and then this # test is run on non-Mac machine. -def test_neg_and_pos_corr_images_match(): - os.environ['METPLOTPY_BASE'] = "../../" - test_config_filename = "test_neg_and_pos_corr.yaml" +def test_neg_and_pos_corr_images_match(setup_env): + setup_env(cwd) + test_config_filename = f"{cwd}/test_neg_and_pos_corr.yaml" td.main(test_config_filename) # Verify that a plot was generated - plot_file = "test_neg_and_pos_corr_plot.png" - expected_file = "expected_neg_and_pos_corr_plot.png" - path = os.getcwd() + plot_file = f"{cwd}/test_neg_and_pos_corr_plot.png" + expected_file = f"{cwd}/expected_neg_and_pos_corr_plot.png" # image comparison, with allowance of .99 match instead of 100% match #comparison = CompareImages(plot_file, expected_file) #assert comparison.mssim >= .99 # Clean up - os.remove(os.path.join(path, plot_file)) + os.remove(plot_file) + -def test_custom_plot_exists(): - os.environ['METPLOTPY_BASE'] = "../../" - test_config_filename = "taylor_diagram_custom.yaml" +def test_custom_plot_exists(setup_env): + setup_env(cwd) + test_config_filename = f"{cwd}/taylor_diagram_custom.yaml" td.main(test_config_filename) # Verify that a plot was generated - plot_file = "./taylor_diagram_custom.png" - path = os.getcwd() - assert os.path.isfile(plot_file) == True + plot_file = f"{cwd}/taylor_diagram_custom.png" + assert os.path.isfile(plot_file) # Clean up - os.remove(os.path.join(path, plot_file)) + os.remove(plot_file) diff --git a/test/tcmpr_plots/test_tcmpr_multi_plots.yaml b/test/tcmpr_plots/test_tcmpr_multi_plots.yaml index 81abf145..f94694ec 100755 --- a/test/tcmpr_plots/test_tcmpr_multi_plots.yaml +++ b/test/tcmpr_plots/test_tcmpr_multi_plots.yaml @@ -102,7 +102,7 @@ series_symbols_size: - 7 show_nstats: 'True' -tcst_dir: './Data/' +tcst_dir: !ENV '${TEST_DIR}/Data/' #tcst_dir: '/path/to/tcmpr_sample_data' tcst_files: [ ] title: '' @@ -133,11 +133,11 @@ event_equal: 'True' skill_ref: - HFSA -plot_dir: './output' +plot_dir: !ENV '${TEST_DIR}/output' prefix: '' subtitle: log_level: INFO -log_filename: './output/tcmpr_log.out' +log_filename: !ENV '${TEST_DIR}/output/tcmpr_log.out' baseline_file: './metplotpy/plots/tcmpr_plots/hfip_baseline.dat' column_info_file: './metplotpy/plots/tcmpr_plots/plot_tcmpr_hdr.dat' hfip_bsln: 'no' diff --git a/test/tcmpr_plots/test_tcmpr_plots.py b/test/tcmpr_plots/test_tcmpr_plots.py index c35ed402..56f7764a 100644 --- a/test/tcmpr_plots/test_tcmpr_plots.py +++ b/test/tcmpr_plots/test_tcmpr_plots.py @@ -1,32 +1,26 @@ import pytest import os -import re + import metplotpy.plots.tcmpr_plots.tcmpr as t +cwd = os.path.dirname(__file__) -@pytest.fixture -def setup(): - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../.." - custom_config_filename = "./test_tcmpr_multi_plots.yaml" +@pytest.fixture +def setup(setup_env): + setup_env(cwd) + custom_config_filename = f"{cwd}/test_tcmpr_multi_plots.yaml" # Invoke the command to generate a line plot based on # the custom config file t.main(custom_config_filename) - def test_plots_created(setup): - - - # Check for presence of fourteen plots (seven plot types, one each for the # ABS(X-Y) and TK_ERR columns. expected_num_plots = 14 - test_dir = os.getcwd() - output_dir = os.path.join(test_dir, 'output') + output_dir = os.path.join(cwd, 'output') only_files = os.listdir(output_dir) assert len(only_files) == expected_num_plots @@ -81,7 +75,6 @@ def test_plots_created(setup): # assert cur_file_size >= expected_size # - # Clean up try: for cur_file in only_files: @@ -89,4 +82,4 @@ def test_plots_created(setup): os.rmdir(output_dir) except FileNotFoundError: # If files already cleaned up, then ignore error - pass \ No newline at end of file + pass diff --git a/test/util/test_util.py b/test/util/test_util.py index 46427056..b22bcc04 100644 --- a/test/util/test_util.py +++ b/test/util/test_util.py @@ -1,8 +1,11 @@ +import os import pandas as pd import metplotpy.plots.util as util import gc +cwd = os.path.dirname(__file__) + def test_is_threshold(): """ @@ -14,20 +17,14 @@ def test_is_threshold(): not_thresholds = [1.0, 11, 77] mixed_thresholds = ['==0.0', '> 11', 21, '>= .5', '== 3'] - if not util.is_threshold_value(not_thresholds): - assert True - - if util.is_threshold_value(mixed_thresholds): - assert True + assert not any(util.is_threshold_value(not_thresholds)) + assert any(util.is_threshold_value(mixed_thresholds)) # test series series_not_thresholds = pd.Series(not_thresholds, range(len(not_thresholds))) series_mixed_thresholds = pd.Series(mixed_thresholds, range(len(mixed_thresholds))) - if not util.is_threshold_value(series_not_thresholds): - assert True - - if util.is_threshold_value(series_mixed_thresholds): - assert True + assert not any(util.is_threshold_value(series_not_thresholds)) + assert any(util.is_threshold_value(series_mixed_thresholds)) def test_sort_threshold_values(): @@ -41,8 +38,8 @@ def test_sort_threshold_values(): sorted_list = util.sort_threshold_values(threshold_list) - for idx, sorted in enumerate(sorted_list): - if sorted != expected_list[idx]: + for idx, sorted_val in enumerate(sorted_list): + if sorted_val != expected_list[idx]: assert False @@ -57,8 +54,8 @@ def test_sort_threshold_values_whitespace(): sorted_list = util.sort_threshold_values(threshold_list) - for idx, sorted in enumerate(sorted_list): - if sorted != expected_list[idx]: + for idx, sorted_val in enumerate(sorted_list): + if sorted_val != expected_list[idx]: assert False @@ -73,10 +70,11 @@ def test_sort_threshold_values_floats(): sorted_list = util.sort_threshold_values(threshold_list) - for idx, sorted in enumerate(sorted_list): - if sorted != expected_list[idx]: + for idx, sorted_val in enumerate(sorted_list): + if sorted_val != expected_list[idx]: assert False + def test_is_thresh(): """ Verify that the regular expression used to test for column labels that are threshold-types (i.e. fcst_thresh, obs_thresh, @@ -90,21 +88,17 @@ def test_is_thresh(): counter = 0 expected_none = len(no_thresh_cols) for cur_col in no_thresh_cols: - if not util.is_thresh_column(cur_col): - counter += 1 + if not util.is_thresh_column(cur_col): + counter += 1 - if counter == expected_none: - assert True + assert counter == expected_none counter = 0 expected_all = len(thresh_cols) for cur_col in thresh_cols: if util.is_thresh_column(cur_col): counter +=1 - if counter == expected_all: - assert True - else: - assert False + assert counter == expected_all expected_found = 2 counter = 0 @@ -112,10 +106,7 @@ def test_is_thresh(): if util.is_thresh_column(cur_col): counter +=1 - if counter == expected_found: - assert True - else: - assert False + assert counter == expected_found def test_filter_by_fixed_vars(): @@ -125,7 +116,6 @@ def test_filter_by_fixed_vars(): """ - settings_dict = {'fcst_thresh': ['NA', '>0.0', '>=0.254'], 'obtype': ['CCPA'], 'vx_mask': ['CONUS'], 'fcst_init_beg': ['2023-06-24 12:00:00']} @@ -135,7 +125,7 @@ def test_filter_by_fixed_vars(): "vx_mask in ('CONUS')", "fcst_init_beg in ('2023-06-24 12:00:00')"] - input_df = pd.read_csv("./full_thresh_data.txt", sep='\t') + input_df = pd.read_csv(f"{cwd}/full_thresh_data.txt", sep='\t') filtered_df = util.filter_by_fixed_vars(input_df, settings_dict) expected_df = input_df.copy(deep=True) @@ -146,20 +136,17 @@ def test_filter_by_fixed_vars(): # Verify that the expected and filtered dataframes have the same dimensions assert filtered_df.shape == expected_df.shape - filtered_list = filtered_df['fcst_thresh'].to_list() expected_list = expected_df['fcst_thresh'].to_list() for filtered in filtered_list: assert filtered in expected_list - # Clean up previous dataframes del intermed_df del expected_df gc.collect() - # Now test when there is only one value in fcst_thresh and it is NA settings_dict2 = {'fcst_thresh': ['NA'], 'obtype': ['CCPA'], 'vx_mask': ['CONUS'], 'fcst_init_beg': ['2023-06-24 12:00:00']} @@ -178,7 +165,6 @@ def test_filter_by_fixed_vars(): assert filtered_df.shape == expected_df.shape - filtered_list = filtered_df['fcst_thresh'].to_list() expected_list = expected_df['fcst_thresh'].to_list() diff --git a/test/wind_rose/minimal_wind_rose.yaml b/test/wind_rose/minimal_wind_rose.yaml index 256d73bd..76db4667 100644 --- a/test/wind_rose/minimal_wind_rose.yaml +++ b/test/wind_rose/minimal_wind_rose.yaml @@ -1,8 +1,8 @@ # minimal config file to use all the settings in the wind_rose_default.yaml file. # **NOTE**: update the stat_input and plot_filename settings with the full path to your # input file and output file, respectively. -stat_input: ./point_stat_mpr.txt -plot_filename: ./wind_rose_default.png +stat_input: !ENV '${TEST_DIR}/point_stat_mpr.txt' +plot_filename: !ENV '${TEST_DIR}/wind_rose_default.png' # Optional, uncomment and set to directory to store the .points1 file # that is used by METviewer (created when dump_points_1 is set to True) diff --git a/test/wind_rose/test_wind_rose.py b/test/wind_rose/test_wind_rose.py index 774a5917..04ac6284 100644 --- a/test/wind_rose/test_wind_rose.py +++ b/test/wind_rose/test_wind_rose.py @@ -3,15 +3,15 @@ from metplotpy.plots.wind_rose import wind_rose #from metcalcpy.compare_images import CompareImages +cwd = os.path.dirname(__file__) +CLEANUP_FILES = ['wind_rose_custom.png', 'point_stat_mpr.points1'] @pytest.fixture -def setup(): +def setup(remove_files, setup_env): # Cleanup the plotfile and point1 output file from any previous run - cleanup() - # Set up the METPLOTPY_BASE so that met_plot.py will correctly find - # the config directory containing all the default config files. - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "./wind_rose_custom.yaml" + remove_files(cwd, CLEANUP_FILES) + setup_env(cwd) + custom_config_filename = f"{cwd}/wind_rose_custom.yaml" # Invoke the command to generate a Wind rose Diagram based on # a custom config file. @@ -22,108 +22,81 @@ def cleanup(): # remove the .png and .points files # from any previous runs try: - path = os.getcwd() plot_file = 'wind_rose_custom.png' points_file_1 = 'point_stat_mpr.points1' - os.remove(os.path.join(path, plot_file)) - os.remove(os.path.join(path, points_file_1)) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. + os.remove(os.path.join(cwd, plot_file)) + os.remove(os.path.join(cwd, points_file_1)) + except OSError: pass @pytest.mark.parametrize("test_input,expected", (["wind_rose_expected.png", True], ["wind_rose_expected.points", True])) -def test_files_exist(setup, test_input, expected): +def test_files_exist(setup, test_input, expected, remove_files): ''' Checking that the plot and data files are getting created ''' - assert os.path.isfile(test_input) == expected - cleanup() - + assert os.path.isfile(f"{cwd}/{test_input}") == expected + remove_files(cwd, CLEANUP_FILES) @pytest.mark.parametrize("test_input,expected", - (["./intermed_files/wind_rose_custom_points.png", True], ["./intermed_files/point_stat_mpr.points1", True])) -def test_points1_files_exist( test_input, expected): + (["intermed_files/wind_rose_custom_points.png", True], ["intermed_files/point_stat_mpr.points1", True])) +def test_points1_files_exist(setup_env, test_input, expected, remove_files): ''' Checking that the plot file and points1 file are getting created where expected plot and point file are being saved in the intermed_files subdir ''' - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "wind_rose_custom_points.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/wind_rose_custom_points.yaml" try: - os.mkdir(os.path.join(os.getcwd(), './intermed_files')) - except FileExistsError as e: + os.mkdir(os.path.join(cwd, 'intermed_files')) + except FileExistsError: pass # Invoke the command to generate a Wind Rose Diagram based on # a custom config file. wind_rose.main(custom_config_filename) - assert os.path.isfile(test_input) == expected + assert os.path.isfile(f"{cwd}/{test_input}") == expected # remove the plot and points1 files that were created - cleanup() - try: - path = os.getcwd() - subdir = os.path.join(path, 'intermed_files') - plot_file = 'wind_rose_custom_points.png' - points_file_1 = 'point_stat_mpr.points1' - os.remove(os.path.join(subdir, plot_file)) - os.remove(os.path.join(subdir, points_file_1)) - os.rmdir(subdir) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass + remove_files(cwd, CLEANUP_FILES) @pytest.mark.parametrize("test_input,expected", - (["./intermed_files/wind_rose_points2.png", True], - ["./intermed_files/point_stat_mpr.points1", True])) -def test_points1_files_exist(test_input, expected): + (["intermed_files/wind_rose_points2.png", True], + ["intermed_files/point_stat_mpr.points1", True])) +def test_points1_files_exist(setup_env, test_input, expected, remove_files): ''' Checking that the plot file and points1 file are getting created where expected plot and point file are being saved in the intermed_files subdir. Verify that when no stat_file is specified, the point_stat_mpr.txt in the test dir is being used. ''' - os.environ['METPLOTPY_BASE'] = "../../" - custom_config_filename = "wind_rose_custom2_points.yaml" + setup_env(cwd) + custom_config_filename = f"{cwd}/wind_rose_custom2_points.yaml" try: - os.mkdir(os.path.join(os.getcwd(), './intermed_files')) - except FileExistsError as e: + os.mkdir(os.path.join(cwd, 'intermed_files')) + except FileExistsError: pass # Invoke the command to generate a Wind Rose Diagram based on # a custom config file. wind_rose.main(custom_config_filename) - assert os.path.isfile(test_input) == expected + assert os.path.isfile(f"{cwd}/{test_input}") == expected + + # remove the plot and points1 files and intermed_files that were created + remove_files(cwd, CLEANUP_FILES) + - # remove the plot and points1 files that were created - cleanup() - try: - path = os.getcwd() - subdir = os.path.join(path, 'intermed_files') - plot_file = 'wind_rose_points2.png' - points_file_1 = 'point_stat_mpr.points1' - os.remove(os.path.join(subdir, plot_file)) - os.remove(os.path.join(subdir, points_file_1)) - os.rmdir(subdir) - except OSError as e: - # Typically when files have already been removed or - # don't exist. Ignore. - pass - @pytest.mark.skip("unreliable sometimes fails due to differences in machines.") -def test_images_match(setup): +def test_images_match(setup, remove_files): ''' Compare an expected plot with the newly created plot to verify that the plot hasn't changed in appearance. ''' - comparison = CompareImages('./wind_rose_expected.png', './wind_rose_custom.png') + comparison = CompareImages(f'{cwd}/wind_rose_expected.png', f'{cwd}/wind_rose_custom.png') assert comparison.mssim == 1 - cleanup() + remove_files(cwd, CLEANUP_FILES) diff --git a/test/wind_rose/wind_rose_custom.yaml b/test/wind_rose/wind_rose_custom.yaml index 0851ec38..605a8792 100644 --- a/test/wind_rose/wind_rose_custom.yaml +++ b/test/wind_rose/wind_rose_custom.yaml @@ -33,9 +33,9 @@ angularaxis_ticktext: # !!!!!!!! IMPORTANT !!!!!! # Uncomment the stat_input setting and specify the full path to the # point_stat_mpr.txt file -stat_input: ./point_stat_mpr.txt +stat_input: !ENV '${TEST_DIR}/point_stat_mpr.txt' -plot_filename: ./wind_rose_custom.png +plot_filename: !ENV '${TEST_DIR}/wind_rose_custom.png' dump_points: True # Optional, uncomment and set to directory to store the .points1 file diff --git a/test/wind_rose/wind_rose_custom2_points.yaml b/test/wind_rose/wind_rose_custom2_points.yaml index b9177f49..544405ff 100644 --- a/test/wind_rose/wind_rose_custom2_points.yaml +++ b/test/wind_rose/wind_rose_custom2_points.yaml @@ -29,8 +29,8 @@ angularaxis_ticktext: - 'E' - 'S' - 'W' -plot_filename: ./intermed_files/wind_rose_points2.png +plot_filename: !ENV '${TEST_DIR}/intermed_files/wind_rose_points2.png' dump_points: True -points_path: ./intermed_files +points_path: !ENV '${TEST_DIR}/intermed_files' # No stat_input, force creation of 'default' points file name show_in_browser: False \ No newline at end of file diff --git a/test/wind_rose/wind_rose_custom_points.yaml b/test/wind_rose/wind_rose_custom_points.yaml index 61842b59..56c1347c 100644 --- a/test/wind_rose/wind_rose_custom_points.yaml +++ b/test/wind_rose/wind_rose_custom_points.yaml @@ -29,8 +29,8 @@ angularaxis_ticktext: - 'E' - 'S' - 'W' -plot_filename: ./intermed_files/wind_rose_custom_points.png +plot_filename: !ENV '${TEST_DIR}/intermed_files/wind_rose_custom_points.png' dump_points: True -points_path: ./intermed_files -stat_input: ./point_stat_mpr.txt +points_path: !ENV '${TEST_DIR}/intermed_files' +stat_input: !ENV '${TEST_DIR}/point_stat_mpr.txt' show_in_browser: False \ No newline at end of file