From c05d62e736b6a80491836967ab3685f2f46a82b2 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 20 Oct 2022 09:56:48 -0600 Subject: [PATCH] Update Develop-ref after dtcenter/MET#2310 and dtcenter/MET#2300 (#1880) Co-authored-by: johnhg Co-authored-by: jprestop Co-authored-by: Mrinal Biswas Co-authored-by: Julie Prestopnik Co-authored-by: j-opatz Co-authored-by: Kathryn Newman Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: Lisa Goodrich Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Julie Prestopnik Co-authored-by: Christina Kalb Co-authored-by: Hank Fisher Co-authored-by: cristianastan2 Co-authored-by: John Halley Gotway Co-authored-by: bikegeek Co-authored-by: Christina Kalb Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: Howard Soh Co-authored-by: Molly Smith Co-authored-by: hsoh-u Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: Hank Fisher --- .github/jobs/get_use_cases_to_run.sh | 6 +- .github/jobs/run_difference_tests.sh | 4 +- .github/jobs/save_error_logs.sh | 4 +- .github/jobs/set_job_controls.sh | 12 +- .github/workflows/documentation.yml | 8 +- .github/workflows/testing.yml | 20 +-- docs/Release_Guide/met_development.rst | 1 + docs/Release_Guide/met_official.rst | 1 + docs/Release_Guide/metcalcpy_development.rst | 1 + docs/Release_Guide/metcalcpy_official.rst | 1 + docs/Release_Guide/metdataio_development.rst | 1 + docs/Release_Guide/metdataio_official.rst | 1 + docs/Release_Guide/metplotpy_development.rst | 1 + docs/Release_Guide/metplotpy_official.rst | 1 + docs/Release_Guide/metplus_development.rst | 1 + docs/Release_Guide/metplus_official.rst | 1 + docs/Release_Guide/metviewer_development.rst | 1 + docs/Release_Guide/metviewer_official.rst | 1 + .../update_upgrade_instructions.rst | 26 ++++ docs/Users_Guide/index.rst | 1 + docs/Users_Guide/overview.rst | 4 - docs/Users_Guide/release-notes.rst | 62 +++++---- metplus/util/diff_util.py | 126 +++++++++++++++++- 23 files changed, 215 insertions(+), 70 deletions(-) create mode 100644 docs/Release_Guide/release_steps/update_upgrade_instructions.rst diff --git a/.github/jobs/get_use_cases_to_run.sh b/.github/jobs/get_use_cases_to_run.sh index 341d1c4801..0d06a85df0 100755 --- a/.github/jobs/get_use_cases_to_run.sh +++ b/.github/jobs/get_use_cases_to_run.sh @@ -47,9 +47,9 @@ echo Array of groups to run is: $matrix # if matrix is still empty, exit 1 to fail step and skip rest of workflow if [ "$matrix" == "[]" ]; then echo No tests to run! - echo ::set-output name=run_some_tests::false + echo "run_some_tests=false" >> $GITHUB_OUTPUT exit 0 fi -echo ::set-output name=run_some_tests::true -echo ::set-output name=matrix::{\"categories\":$(echo $matrix)}\" +echo "run_some_tests=true" >> $GITHUB_OUTPUT +echo "matrix={\"categories\":$(echo $matrix)}\"" >> $GITHUB_OUTPUT diff --git a/.github/jobs/run_difference_tests.sh b/.github/jobs/run_difference_tests.sh index 609fa570e7..49e7242ec8 100755 --- a/.github/jobs/run_difference_tests.sh +++ b/.github/jobs/run_difference_tests.sh @@ -14,10 +14,10 @@ artifact_name=$2 .github/jobs/setup_and_run_diff.py ${matrix_categories} $artifact_name if [ "$( ls -A ${RUNNER_WORKSPACE}/diff)" ]; then - echo ::set-output name=upload_diff::true + echo "upload_diff=true" >> $GITHUB_OUTPUT mkdir -p artifact/diff-${artifact_name} cp -r ${RUNNER_WORKSPACE}/diff/* artifact/diff-${artifact_name} exit 2 fi -echo ::set-output name=upload_diff::false +echo "upload_diff=false" >> $GITHUB_OUTPUT diff --git a/.github/jobs/save_error_logs.sh b/.github/jobs/save_error_logs.sh index 0662056729..b2fe320b4b 100755 --- a/.github/jobs/save_error_logs.sh +++ b/.github/jobs/save_error_logs.sh @@ -8,7 +8,7 @@ .github/jobs/copy_error_logs.py ${RUNNER_WORKSPACE}/output artifact/error_logs if [ -d "artifact/error_logs" ]; then - echo ::set-output name=upload_error_logs::true + echo "upload_error_logs=true" >> $GITHUB_OUTPUT else - echo ::set-output name=upload_error_logs::false + echo "upload_error_logs=false" >> $GITHUB_OUTPUT fi diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index 432c57a89a..5e72bc273c 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -83,16 +83,16 @@ else fi fi -echo ::set-output name=run_get_image::$run_get_image -echo ::set-output name=run_get_input_data::$run_get_input_data -echo ::set-output name=run_diff::$run_diff -echo ::set-output name=run_save_truth_data::$run_save_truth_data -echo ::set-output name=external_trigger::$external_trigger +echo "run_get_image=$run_get_image" >> $GITHUB_OUTPUT +echo "run_get_input_data=$run_get_input_data" >> $GITHUB_OUTPUT +echo "run_diff=$run_diff" >> $GITHUB_OUTPUT +echo "run_save_truth_data=$run_save_truth_data" >> $GITHUB_OUTPUT +echo "external_trigger=$external_trigger" >> $GITHUB_OUTPUT # get branch name branch_name=`${GITHUB_WORKSPACE}/.github/jobs/print_branch_name.py` -echo ::set-output name=branch_name::$branch_name +echo "branch_name=$branch_name" >> $GITHUB_OUTPUT # get use cases to run .github/jobs/get_use_cases_to_run.sh $run_use_cases $run_all_use_cases $run_unit_tests diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 2f43da4edc..a14891b7fb 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -16,8 +16,8 @@ jobs: name: Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: '3.6' - name: Install dependencies @@ -26,12 +26,12 @@ jobs: python -m pip install python-dateutil requests Pillow - name: Build Documentation run: ./.github/jobs/build_documentation.sh - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: always() with: name: METplus_documentation path: artifact/documentation - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: failure() with: name: documentation_warnings.log diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 305e2e80f2..f60cb04e59 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -52,7 +52,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set job controls id: job_status run: .github/jobs/set_job_controls.sh @@ -75,8 +75,8 @@ jobs: needs: job_control if: ${{ needs.job_control.outputs.run_get_image == 'true' }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: '3.6' - name: Get METplus Image @@ -115,7 +115,7 @@ jobs: matrix: ${{fromJson(needs.job_control.outputs.matrix)}} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Free disk space run: .github/jobs/free_disk_space.sh @@ -130,7 +130,7 @@ jobs: id: get-artifact-name run: | artifact_name=`.github/jobs/get_artifact_name.sh ${{ matrix.categories }}` - echo ::set-output name=artifact_name::${artifact_name} + echo "artifact_name=${artifact_name}" >> $GITHUB_OUTPUT # run use case tests - name: Run Use Cases @@ -158,14 +158,14 @@ jobs: run: .github/jobs/copy_output_to_artifact.sh ${{ steps.get-artifact-name.outputs.artifact_name }} - name: Upload output data artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ always() && steps.run_tests.conclusion != 'skipped' && !startsWith(matrix.categories,'pytests') }} with: name: ${{ steps.get-artifact-name.outputs.artifact_name }} path: artifact/${{ steps.get-artifact-name.outputs.artifact_name }} - name: Upload error logs artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ always() && steps.save-errors.outputs.upload_error_logs }} with: name: error_logs @@ -173,7 +173,7 @@ jobs: if-no-files-found: ignore - name: Upload difference data artifact - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ always() && steps.run-diff.outputs.upload_diff == 'true' }} with: name: diff-${{ steps.get-artifact-name.outputs.artifact_name }} @@ -186,8 +186,8 @@ jobs: needs: [use_case_tests] if: ${{ needs.job_control.outputs.run_save_truth_data == 'true' }} steps: - - uses: actions/checkout@v2 - - uses: actions/download-artifact@v2 + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 - run: .github/jobs/create_output_data_volumes.sh env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} diff --git a/docs/Release_Guide/met_development.rst b/docs/Release_Guide/met_development.rst index a37f672b5d..3690a94682 100644 --- a/docs/Release_Guide/met_development.rst +++ b/docs/Release_Guide/met_development.rst @@ -11,6 +11,7 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/met/update_version_development.rst .. include:: release_steps/update_release_notes_development.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst diff --git a/docs/Release_Guide/met_official.rst b/docs/Release_Guide/met_official.rst index 98bcc841fc..81e8269abf 100644 --- a/docs/Release_Guide/met_official.rst +++ b/docs/Release_Guide/met_official.rst @@ -11,6 +11,7 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/met/update_version_official.rst .. include:: release_steps/update_release_notes_official.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/met/create_release_reference_branch.rst diff --git a/docs/Release_Guide/metcalcpy_development.rst b/docs/Release_Guide/metcalcpy_development.rst index a9a74901f2..936a9c9ce7 100644 --- a/docs/Release_Guide/metcalcpy_development.rst +++ b/docs/Release_Guide/metcalcpy_development.rst @@ -11,6 +11,7 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metcalcpy/update_version.rst .. include:: release_steps/update_release_notes_development.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst diff --git a/docs/Release_Guide/metcalcpy_official.rst b/docs/Release_Guide/metcalcpy_official.rst index 4a66380180..f5b5ef3d40 100644 --- a/docs/Release_Guide/metcalcpy_official.rst +++ b/docs/Release_Guide/metcalcpy_official.rst @@ -11,6 +11,7 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metcalcpy/update_version_official.rst .. include:: release_steps/update_release_notes_official.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_branch.rst diff --git a/docs/Release_Guide/metdataio_development.rst b/docs/Release_Guide/metdataio_development.rst index a3a66e2925..810725ccf6 100644 --- a/docs/Release_Guide/metdataio_development.rst +++ b/docs/Release_Guide/metdataio_development.rst @@ -14,6 +14,7 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metdataio/update_version.rst .. include:: release_steps/update_release_notes_development.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst diff --git a/docs/Release_Guide/metdataio_official.rst b/docs/Release_Guide/metdataio_official.rst index c20542c5c4..d0b54155f3 100644 --- a/docs/Release_Guide/metdataio_official.rst +++ b/docs/Release_Guide/metdataio_official.rst @@ -14,6 +14,7 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metdataio/update_version_official.rst .. include:: release_steps/update_release_notes_official.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_branch.rst diff --git a/docs/Release_Guide/metplotpy_development.rst b/docs/Release_Guide/metplotpy_development.rst index 7913284c0f..3e78b5c84d 100644 --- a/docs/Release_Guide/metplotpy_development.rst +++ b/docs/Release_Guide/metplotpy_development.rst @@ -12,6 +12,7 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/metplotpy/update_version.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/update_release_notes_development.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst diff --git a/docs/Release_Guide/metplotpy_official.rst b/docs/Release_Guide/metplotpy_official.rst index 20ac22ca69..2f6a68f1f3 100644 --- a/docs/Release_Guide/metplotpy_official.rst +++ b/docs/Release_Guide/metplotpy_official.rst @@ -11,6 +11,7 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metplotpy/update_version_official.rst .. include:: release_steps/update_release_notes_official.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_branch.rst diff --git a/docs/Release_Guide/metplus_development.rst b/docs/Release_Guide/metplus_development.rst index 2bd5843439..6dd1b4b1f2 100644 --- a/docs/Release_Guide/metplus_development.rst +++ b/docs/Release_Guide/metplus_development.rst @@ -11,6 +11,7 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metplus/update_version.rst .. include:: release_steps/update_release_notes_development.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/metplus/create_release_extra.rst diff --git a/docs/Release_Guide/metplus_official.rst b/docs/Release_Guide/metplus_official.rst index 47a1ba344d..f2a9acb2d3 100644 --- a/docs/Release_Guide/metplus_official.rst +++ b/docs/Release_Guide/metplus_official.rst @@ -11,6 +11,7 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metplus/update_release_date.rst .. include:: release_steps/update_release_notes_official.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/metplus/update_manage_externals.rst .. include:: release_steps/merge_release_issue.rst diff --git a/docs/Release_Guide/metviewer_development.rst b/docs/Release_Guide/metviewer_development.rst index 4a11effc2b..6b2cea9e8b 100644 --- a/docs/Release_Guide/metviewer_development.rst +++ b/docs/Release_Guide/metviewer_development.rst @@ -11,6 +11,7 @@ Create a new vX.Y.Z-betaN or vX.Y.Z-rcN development release from the develop bra .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metviewer/update_version.rst .. include:: release_steps/update_release_notes_development.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst diff --git a/docs/Release_Guide/metviewer_official.rst b/docs/Release_Guide/metviewer_official.rst index bd5f4c2727..624195d6d1 100644 --- a/docs/Release_Guide/metviewer_official.rst +++ b/docs/Release_Guide/metviewer_official.rst @@ -11,6 +11,7 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/create_release_feature_branch.rst .. include:: release_steps/metviewer/update_version_official.rst .. include:: release_steps/update_release_notes_official.rst +.. include:: release_steps/update_upgrade_instructions.rst .. include:: release_steps/rotate_authorship.rst .. include:: release_steps/merge_release_issue.rst .. include:: release_steps/create_release_branch.rst diff --git a/docs/Release_Guide/release_steps/update_upgrade_instructions.rst b/docs/Release_Guide/release_steps/update_upgrade_instructions.rst new file mode 100644 index 0000000000..bb28f258b2 --- /dev/null +++ b/docs/Release_Guide/release_steps/update_upgrade_instructions.rst @@ -0,0 +1,26 @@ +Update Upgrade Instructions +--------------------------- + +Occasionally, changes will be made to software that will require users to make +changes to their configuration files in order to use the latest release. For +example, when ensemble post-processing was added to Gen-Ens-Prod and removed +from Ensemble-Stat, users were required to make changes in their configuration +files. + +.. note:: + + This section is not always applicable. + + +To alert the users to the necessary steps involved with the upgrade: + +* Update the Upgrade Instructions section of the release-notes.rst file found + in the |projectRepo| User's Guide directory with the necessary information. + +* Add an "upgrade instructions" link next to the appropriate METplus component + (|projectRepo|) after "latest" and before "development" in + the :ref:`METplus Components Release Note Links ` + section in the release-notes.rst file in the METplus User's Guide in the + `METplus GitHub repository `__ ONLY + if there are Upgrade Instructions for this release. + diff --git a/docs/Users_Guide/index.rst b/docs/Users_Guide/index.rst index e08f2ea38d..e5afd2f3df 100644 --- a/docs/Users_Guide/index.rst +++ b/docs/Users_Guide/index.rst @@ -78,6 +78,7 @@ is sponsored by NSF. :numbered: 4 overview + release-notes getting_started installation systemconfiguration diff --git a/docs/Users_Guide/overview.rst b/docs/Users_Guide/overview.rst index 02dc581823..a6b61780e4 100644 --- a/docs/Users_Guide/overview.rst +++ b/docs/Users_Guide/overview.rst @@ -689,10 +689,6 @@ METplus Components Python Requirements - -.. _release-notes: - -.. include:: release-notes.rst - Future development plans ======================== diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index d1ee026553..d85d9fef80 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -1,46 +1,37 @@ -METplus Release Notes -===================== +*************************** +METplus Release Information +*************************** + +.. _release-notes: Users can view the :ref:`releaseCycleStages` section of the Release Guide for descriptions of the development releases (including beta releases and release candidates), official releases, and bugfix releases for the METplus Components. +.. _components-release-notes: + METplus Components Release Note Links -------------------------------------- - -Release Notes - Latest Official Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* `MET `__ -* `METviewer `__ -* `METplotpy `__ -* `METcalcpy `__ -* `METdatadb `__ -* `METexpress `__ -* `METplus Wrappers `__ - -Release Notes - Development Release -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* `MET `__ -* `METviewer `__ -* `METplotpy `__ -* `METcalcpy `__ -* `METdatadb `__ -* `METexpress `__ -* `METplus Wrappers `__ +===================================== + +* MET (`latest `__, `development `__) +* METviewer (`latest `__, `development `__) +* METplotpy (`latest `__, `development `__) +* METcalcpy (`latest `__, `development `__) +* METdataio (`latest `__, `development `__) +* METexpress (`latest `__, `development `__) +* METplus Wrappers (`latest `__, :ref:`upgrade instructions `, `development `__) + METplus Wrappers Release Notes ------------------------------- +============================== -When applicable, release notes are followed by the GitHub issue number which -describes the bugfix, enhancement, or new feature: -https://github.com/dtcenter/METplus/issues +When applicable, release notes are followed by the `GitHub issue `__ number which +describes the bugfix, enhancement, or new feature. METplus Version 5.0.0 Beta 3 Release Notes (2022-09-21) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------------------------------- .. warning:: **MAJOR CHANGES**: @@ -87,7 +78,7 @@ METplus Version 5.0.0 Beta 3 Release Notes (2022-09-21) METplus Version 5.0.0 Beta 2 Release Notes (2022-08-03) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------------------------------- * Enhancements: @@ -103,7 +94,7 @@ METplus Version 5.0.0 Beta 2 Release Notes (2022-08-03) METplus Version 5.0.0 Beta 1 Release Notes (2022-06-22) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------------------------------- * Enhancements: @@ -136,3 +127,10 @@ METplus Version 5.0.0 Beta 1 Release Notes (2022-06-22) * Document GitHub Discussions procedure for the Contributor's Guide (`#1159 `_) * Create a METplus "Release Guide" describing how to build releases for the METplus components (`#673 `_) * Update documentation about viewing RTD URLs on branches (`#1512 `_) + +.. _upgrade-instructions: + +METplus Wrappers Upgrade Instructions +===================================== + +Coming Soon! diff --git a/metplus/util/diff_util.py b/metplus/util/diff_util.py index d6625454cd..15eaeee787 100644 --- a/metplus/util/diff_util.py +++ b/metplus/util/diff_util.py @@ -1,6 +1,7 @@ import os import netCDF4 import filecmp +import csv from PIL import Image, ImageChops import numpy @@ -25,11 +26,23 @@ '.pdf', ] +CSV_EXTENSIONS = [ + '.csv', +] + UNSUPPORTED_EXTENSIONS = [ ] +# number of decision places to accept float differences +# Note: Completing METplus issue #1873 could allow this to be set to 6 +ROUNDING_PRECISION = 5 + def get_file_type(filepath): _, file_extension = os.path.splitext(filepath) + + if file_extension in CSV_EXTENSIONS: + return 'csv' + if file_extension in IMAGE_EXTENSIONS: return 'image' @@ -56,6 +69,7 @@ def get_file_type(filepath): return 'unknown' + def compare_dir(dir_a, dir_b, debug=False, save_diff=False): # if input are files and not directories, compare them if os.path.isfile(dir_a): @@ -137,6 +151,7 @@ def compare_dir(dir_a, dir_b, debug=False, save_diff=False): "**************************************************\n\n") return diff_files + def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None, save_diff=False): # dir_a and dir_b are only needed if comparing file lists that need those @@ -149,7 +164,7 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None, if not os.path.exists(filepath_b): if debug: print(f"ERROR: File does not exist: {filepath_b}") - return (filepath_a, '', 'file not found (in truth but missing now)', '') + return filepath_a, '', 'file not found (in truth but missing now)', '' file_type = get_file_type(filepath_a) if file_type.startswith('skip'): @@ -158,12 +173,21 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None, if file_type.startswith('unsupported'): print(f"Unsupported file type encountered: {file_type.split('.')[1]}") - return (filepath_a, filepath_b, file_type, '') + return filepath_a, filepath_b, file_type, '' + + if file_type == 'csv': + print('Comparing CSV') + if not compare_csv_files(filepath_a, filepath_b): + print(f'ERROR: CSV file differs: {filepath_b}') + return filepath_a, filepath_b, 'CSV diff', '' + + print("No differences in CSV files") + return True if file_type == 'netcdf': print("Comparing NetCDF") if not nc_is_equal(filepath_a, filepath_b): - return (filepath_a, filepath_b, 'NetCDF diff', '') + return filepath_a, filepath_b, 'NetCDF diff', '' print("No differences in NetCDF files") return True @@ -179,7 +203,7 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None, if diff_file is False: diff_file = '' - return (filepath_a, filepath_b, 'PDF diff', diff_file) + return filepath_a, filepath_b, 'PDF diff', diff_file if file_type == 'image': print("Comparing images") @@ -192,7 +216,7 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None, if diff_file is False: diff_file = '' - return (filepath_a, filepath_b, 'Image diff', diff_file) + return filepath_a, filepath_b, 'Image diff', diff_file # if not any of the above types, use diff to compare print("Comparing text files") @@ -200,7 +224,7 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None, # if files differ, open files and handle expected diffs if not compare_txt_files(filepath_a, filepath_b, dir_a, dir_b): print(f"ERROR: File differs: {filepath_b}") - return (filepath_a, filepath_b, 'Text diff', '') + return filepath_a, filepath_b, 'Text diff', '' print("No differences in text files") return True @@ -209,6 +233,7 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None, return True + def compare_pdf_as_images(filepath_a, filepath_b, save_diff=False): try: from pdf2image import convert_from_path @@ -234,6 +259,7 @@ def compare_pdf_as_images(filepath_a, filepath_b, save_diff=False): return True + def compare_image_files(filepath_a, filepath_b, save_diff=False): image_a = Image.open(filepath_a) image_b = Image.open(filepath_b) @@ -246,6 +272,7 @@ def compare_image_files(filepath_a, filepath_b, save_diff=False): return save_diff_file(image_diff, filepath_b) + def compare_images(image_a, image_b): """! Compare pillow image objects. Returns difference image object if there are differences or None if not. @@ -264,6 +291,7 @@ def compare_images(image_a, image_b): return image_diff return None + def save_diff_file(image_diff, filepath_b): rel_path, file_extension = os.path.splitext(filepath_b) diff_file = f'{rel_path}_diff.png' @@ -271,6 +299,86 @@ def save_diff_file(image_diff, filepath_b): image_diff.save(diff_file, "PNG") return diff_file + +def compare_csv_files(filepath_a, filepath_b): + lines_a = [] + lines_b = [] + + with open(filepath_a, 'r') as file_handle: + csv_read = csv.DictReader(file_handle, delimiter=',') + for row in csv_read: + lines_a.append(row) + + with open(filepath_b, 'r') as file_handle: + csv_read = csv.DictReader(file_handle, delimiter=',') + for row in csv_read: + lines_b.append(row) + + keys_a = lines_a[0].keys() + keys_b = lines_b[0].keys() + # compare header columns and report error if they differ + if len(keys_a) != len(keys_b): + print(f'ERROR: Different number of columns in TRUTH ({len(keys_a)}) ' + f'than in OUTPUT ({len(keys_b)})') + only_a = [item for item in keys_a if item not in keys_b] + if only_a: + print(f'Columns only in TRUTH: {",".join(only_a)}') + + only_b = [item for item in keys_b if item not in keys_a] + if only_b: + print(f'Columns only in OUTPUT: {",".join(only_b)}') + return False + + # compare number of lines and error if they differ + if len(lines_a) != len(lines_b): + print(f'ERROR: Different number of lines in TRUTH ({len(lines_a)}) ' + f'than in OUTPUT ({len(lines_b)})') + return False + + # compare each CSV column + status = True + for num, (line_a, line_b) in enumerate(zip(lines_a, lines_b), start=1): + for key in keys_a: + val_a = line_a[key] + val_b = line_b[key] + if val_a == val_b: + continue + # prevent error if values are diffs are less than + # ROUNDING_PRECISION decimal places + # METplus issue #1873 addresses the real problem + try: + if is_equal_rounded(val_a, val_b): + continue + print(f"ERROR: Line {num} - {key} differs by " + f"less than {ROUNDING_PRECISION} decimals: " + f"TRUTH = {val_a}, OUTPUT = {val_b}") + status = False + except ValueError: + # handle values that can't be cast to float + print(f"ERROR: Line {num} - {key} differs: " + f"TRUTH = {val_a}, OUTPUT = {val_b}") + status = False + + return status + + +def is_equal_rounded(value_a, value_b): + if _truncate_float(value_a) == _truncate_float(value_b): + return True + if _round_float(value_a) == _round_float(value_b): + return True + return False + + +def _truncate_float(value): + factor = 1 / (10 ** ROUNDING_PRECISION) + return float(value) // factor * factor + + +def _round_float(value): + return round(float(value), ROUNDING_PRECISION) + + def compare_txt_files(filepath_a, filepath_b, dir_a=None, dir_b=None): with open(filepath_a, 'r') as file_handle: lines_a = file_handle.read().splitlines() @@ -317,7 +425,8 @@ def compare_txt_files(filepath_a, filepath_b, dir_a=None, dir_b=None): header_a = lines_a.pop(0).split()[1:] header_b = lines_b.pop(0).split()[1:] else: - header_a = header_b = None + header_a = None + header_b = None if len(lines_a) != len(lines_b): print(f"ERROR: Different number of lines in {filepath_b}") @@ -348,6 +457,7 @@ def compare_txt_files(filepath_a, filepath_b, dir_a=None, dir_b=None): return all_good + def diff_text_lines(lines_a, lines_b, dir_a=None, dir_b=None, print_error=False, @@ -382,6 +492,7 @@ def diff_text_lines(lines_a, lines_b, return all_good + def nc_is_equal(file_a, file_b, fields=None, debug=False): """! Check if two NetCDF files have the same data @param file_a first file to compare @@ -462,6 +573,7 @@ def nc_is_equal(file_a, file_b, fields=None, debug=False): return is_equal + if __name__ == '__main__': dir_a = sys.argv[1] dir_b = sys.argv[2]