diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index af9ed8e8..3d2eb265 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -17,8 +17,8 @@ jobs: name: Build Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.8' - name: Install dependencies @@ -28,7 +28,7 @@ jobs: python -m pip install -r docs/requirements.txt - name: Build docs run: ./.github/jobs/build_documentation.sh - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: documentation diff --git a/.github/workflows/trigger_metplus.yml b/.github/workflows/trigger_metplus.yml index f9c78188..bd70bc1a 100644 --- a/.github/workflows/trigger_metplus.yml +++ b/.github/workflows/trigger_metplus.yml @@ -20,7 +20,7 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - - uses: actions/github-script@v6 + - uses: actions/github-script@v7 with: github-token: ${{ secrets.METPLUS_BOT_TOKEN }} script: | diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 8a83305a..1b0deeb0 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -33,9 +33,9 @@ jobs: python-version: ["3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -64,11 +64,12 @@ jobs: pytest test_utils.py pytest test_validate_mv_python.py pytest test_future_warnings.py + pytest test_sl1l2.py coverage run -m pytest test_agg_eclv.py test_agg_stats_and_boot.py test_agg_stats_with_groups.py test_calc_difficulty_index.py test_convert_lon_indices.py test_event_equalize.py test_event_equalize_against_values.py test_lon_360_to_180.py test_statistics.py test_tost_paired.py test_utils.py test_future_warnings.py coverage html - name: Archive code coverage results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: code-coverage-report path: test/htmlcov/index.html diff --git a/.github/workflows/vulnerabilities.yaml b/.github/workflows/vulnerabilities.yaml index f9dcb5b2..a532781f 100644 --- a/.github/workflows/vulnerabilities.yaml +++ b/.github/workflows/vulnerabilities.yaml @@ -22,7 +22,7 @@ jobs: selftest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: install run: | python -m venv env/ diff --git a/docs/Users_Guide/difficulty_index.rst b/docs/Users_Guide/difficulty_index.rst new file mode 100644 index 00000000..c10dfb1d --- /dev/null +++ b/docs/Users_Guide/difficulty_index.rst @@ -0,0 +1,171 @@ +**************** +Difficulty Index +**************** + +Description +=========== + +This module is used to calculate the difficulty of a decision based on a set of forecasts, +such as an ensemble, for quantities such as wind speed or significant wave height as a +function of space and time. + +Example +======= + +An example Use-Case for running Difficulty Index for Wind Speed can be found in the `METplus documentation `_. + +Decision Difficulty Index Computation +===================================== + +Consider the following formulation of a forecast decision difficulty index: + + .. math :: d_{i,j} = \frac{A(\bar{x}_{i,j})}{2}(\frac{(\sigma/\bar{x})_{i,j}}{(\sigma/\bar{x})_{ref}}+[1-\frac{1}{2}|P(x_{i,j}\geq thresh)-P(x_{i,j}`_) - .. dropdown:: Enhancements - * **Upgrade to using Python 3.10.4** (`#270 `_) - * Enhance the Release Notes by adding dropdown menus(`#292 `_) - * **Updated Analysis tools to handle TC-RMW functionality** (`#308 `_) - .. dropdown:: Internal - * Add 'License.txt' to the METcalcpy repo (`#279 `_) - * Update cloud python to 3.10(`#170 `_) - * Update cloud python to 3.8(`#169 `_) - * Changed Project to Cycle assignment in github(`#301 `_) - * Partial completion of logging for STIGS(`#46 `_) + * Change second person references to third (`#315 `_) + * Enhanced documentation for Difficulty index (`#332 `_) .. dropdown:: Bugfixes - * **Remaining reference to ARIMA in utils.py compute_std_err_from_median_variance_inflation_factor()** (`#254 `_) - * **Replace frame.append method to pandas.concat in agg_stat.py** (`#296 `_) - * **Fixed Ratio Derived curve not generating** (`#302 `_) - * **Fixed PSTD_ROC_AUC calculation** (`#306 `_) + * Add missing reliability statistics (`#330 `_) + +METcalcpy Version 3.0.0-beta1 release notes (20230915) +------------------------------------------------------ + + .. dropdown:: New Functionality + + .. dropdown:: Enhancements + + .. dropdown:: Internal + + .. dropdown:: Bugfixes + + * Remove reset_index from various calculations (`#322 `_) + METcalcpy Upgrade Instructions ============================== diff --git a/docs/conf.py b/docs/conf.py index be015a95..040227cf 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,7 +41,8 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx_gallery.gen_gallery', - 'sphinx_design'] + 'sphinx_design', + 'sphinx_rtd_theme',] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/index.rst b/docs/index.rst index de2a64eb..ab455bf2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,14 +1,14 @@ -########################### +=========================== METcalcpy version |version| -########################### +=========================== Developed by the `Developmental Testbed Center `_, Boulder, CO .. image:: _static/METplus_banner_photo_web.png -**History** - +History +------- The Model Evaluation Tools (MET) were developed by the Developmental Testbed Center (DTC) and released in January 2008. The goal of the tools was to @@ -37,7 +37,8 @@ diagnostics capability for NOAA's UFS and a component of NCAR's SIMA modeling frameworks. It is being actively developed by NCAR, ESRL, EMC and is open to community contributions. -**METplus Concept** +METplus Concept +--------------- METplus is the overarching, or umbrella, repository and hence framework for the Unified Forecast System verification capability. It is intended to be @@ -76,7 +77,8 @@ was developed because CESM is comprised of a number of different components that are developed and managed independently. Each component also may have additional "external" dependencies that need to be maintained independently. -**Acronyms** +Acronyms +-------- * **MET** - Model Evaluation Tools * **DTC** - Developmental Testbed Center @@ -93,7 +95,8 @@ additional "external" dependencies that need to be maintained independently. * **GSD** - Global Systems Division -**Authors** +Authors +------- Many authors, listed below in alphabetical order, have contributed to the documentation of METplus. To cite this documentation in publications, please refer to the METcalcpy User's Guide :ref:`Citation Instructions`. @@ -121,7 +124,8 @@ To cite this documentation in publications, please refer to the METcalcpy User's -**Indices** +Indices +======= * :ref:`genindex` * :ref:`modindex` diff --git a/docs/requirements.txt b/docs/requirements.txt index 0b266552..acd19d3d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,6 @@ -sphinx-gallery -sphinxcontrib-bibtex +sphinx-gallery==0.14.0 +pillow==10.0.1 +sphinxcontrib-bibtex==2.6.1 +sphinx==5.3.0 sphinx-design==0.3.0 +sphinx_rtd_theme==1.3.0 diff --git a/docs/version b/docs/version index 8eef5147..bcaa9617 100644 --- a/docs/version +++ b/docs/version @@ -1 +1 @@ -__version__="2.1.0" +__version__="3.0.0-beta3" diff --git a/internal/scripts/installation/modulefiles/2.1.0_wcoss2 b/internal/scripts/installation/modulefiles/3.0.0_wcoss2 similarity index 83% rename from internal/scripts/installation/modulefiles/2.1.0_wcoss2 rename to internal/scripts/installation/modulefiles/3.0.0_wcoss2 index 73c909ba..1655a07b 100644 --- a/internal/scripts/installation/modulefiles/2.1.0_wcoss2 +++ b/internal/scripts/installation/modulefiles/3.0.0_wcoss2 @@ -3,7 +3,7 @@ ## METcalcpy ## proc ModulesHelp { } { - puts stderr "Sets up the paths and environment variables to use the METcalcpy-2.1.0. + puts stderr "Sets up the paths and environment variables to use the METcalcpy-3.0.0. *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" } @@ -12,4 +12,4 @@ module load intel module use /apps/dev/modulefiles/ module load ve/evs/2.0 -prepend-path PYTHONPATH /apps/sw_review/emc/METcalcpy/2.1.0 +prepend-path PYTHONPATH /apps/sw_review/emc/METcalcpy/3.0.0 diff --git a/metcalcpy/util/pstd_statistics.py b/metcalcpy/util/pstd_statistics.py index 8d9f96da..6a890d7a 100644 --- a/metcalcpy/util/pstd_statistics.py +++ b/metcalcpy/util/pstd_statistics.py @@ -41,16 +41,13 @@ def calculate_pstd_brier(input_data, columns_names): t_table = df_pct_perm['n_i'].sum() o_bar_table = df_pct_perm['oy_i'].sum() / t_table o_bar = input_data[0, get_column_index_by_name(columns_names, 'o_bar')] - - t_table.reset_index(inplace=True, drop=True) - reliability = calc_reliability(t_table, df_pct_perm) resolution = calc_resolution(t_table, df_pct_perm, o_bar) uncertainty = calc_uncertainty(o_bar_table) brier = reliability - resolution + uncertainty result = round_half_up(brier, PRECISION) - except (TypeError, ZeroDivisionError, Warning, ValueError): + except (TypeError, ZeroDivisionError, Warning, ValueError, AttributeError): result = None warnings.filterwarnings('ignore') return result @@ -75,8 +72,6 @@ def calculate_pstd_bss_smpl(input_data, columns_names): t_table = df_pct_perm['n_i'].sum() o_bar_table = df_pct_perm['oy_i'].sum() / t_table o_bar = input_data[0, get_column_index_by_name(columns_names, 'o_bar')] - - t_table.reset_index(inplace=True, drop=True) reliability = calc_reliability(t_table, df_pct_perm) resolution = calc_resolution(t_table, df_pct_perm, o_bar) uncertainty = calc_uncertainty(o_bar_table) @@ -161,7 +156,6 @@ def calculate_pstd_resolution(input_data, columns_names): df_pct_perm = _calc_common_stats(columns_names, input_data) o_bar = input_data[0, get_column_index_by_name(columns_names, 'o_bar')] t_table = df_pct_perm['n_i'].sum() - t_table.reset_index(inplace=True, drop=True) resolution = calc_resolution(t_table, df_pct_perm, o_bar) result = round_half_up(resolution, PRECISION) except (TypeError, ZeroDivisionError, Warning, ValueError): diff --git a/metcalcpy/util/sl1l2_statistics.py b/metcalcpy/util/sl1l2_statistics.py index 9324a0dd..12508b45 100644 --- a/metcalcpy/util/sl1l2_statistics.py +++ b/metcalcpy/util/sl1l2_statistics.py @@ -524,10 +524,13 @@ def calculate_bcmse(input_data, columns_names, aggregation=False): """ warnings.filterwarnings('error') try: + mse = calculate_mse(input_data, columns_names, aggregation) me = calculate_me(input_data, columns_names, aggregation) result = mse - me ** 2 result = round_half_up(result, PRECISION) + if result < 0: + return 0. except (TypeError, Warning): result = None warnings.filterwarnings('ignore') diff --git a/metcalcpy/util/utils.py b/metcalcpy/util/utils.py index c8b99616..5fb9b7c1 100644 --- a/metcalcpy/util/utils.py +++ b/metcalcpy/util/utils.py @@ -691,7 +691,14 @@ def equalize_axis_data(fix_vals_keys, fix_vals_permuted, params, input_data, axi for fcst_var, fcst_var_stats in fcst_var_val.items(): series_data_for_ee = pd.DataFrame() - for fcst_var_stat in fcst_var_stats: + fcst_var_stats_current = fcst_var_stats + # if the data comes from agg_stat workflow it doesn't contain statistics values. + # They will be calculated later. In this case The code should not loop over all + # requested statistics but instead should do it only ones to avoid data multiplication + if 'stat_name' not in input_data.keys(): + fcst_var_stats_current = [fcst_var_stats[0]] + + for fcst_var_stat in fcst_var_stats_current: # for each series for the specified axis if len(params['series_val_' + axis]) == 0: series_data_for_ee = input_data diff --git a/test/test_sl1l2.py b/test/test_sl1l2.py new file mode 100644 index 00000000..2cbce584 --- /dev/null +++ b/test/test_sl1l2.py @@ -0,0 +1,22 @@ +import numpy as np +import pytest + +from metcalcpy.util import sl1l2_statistics as sl1l2 + +def test_calculate_bcmse(): + # Test that negative BCMSE values are no longer returned. + input_data_list = [] + + # These data produce negative BCMSE values. Test that the modified code no longer returns negative values + # for the BCMSE. + input_data_list.append(np.array([[4.37978400e+01, 4.70115800e+01, 1.91825108e+03, 2.21008843e+03, 2.05900571e+03, 1.00000000e+00]])) + input_data_list.append(np.array([[8.66233900e+01, 4.83037900e+01, 7.50361146e+03, 2.33325660e+03, 4.18423840e+03, 1.00000000e+00]])) + input_data_list.append(np.array([[3.68089000e+01, 1.64253370e+02, 1.35489535e+03, 2.69791703e+04, 6.04598647e+03, 1.00000000e+00]])) + columns_names = np.array(['fbar', 'obar', 'ffbar', 'oobar', 'fobar', 'total'], dtype='= 0. + + +