diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 94344733d4..d6f4f86ed7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,21 +89,29 @@ jobs: fail-fast: false matrix: categories: - - "met_tool_wrapper" - - "air_quality_and_comp" - - "climate" - - "convection_allowing_models:0-5" - - "convection_allowing_models:6" + - "met_tool_wrapper:0-53" + - "air_quality_and_comp:0" + - "climate:0-1" + - "convection_allowing_models:0" + - "convection_allowing_models:1" + - "convection_allowing_models:2-6" - "convection_allowing_models:7" - - "convection_allowing_models:8+" - - "cryosphere" - - "data_assimilation" - - "marine_and_coastal" - - "medium_range:0-4" - - "medium_range:5" - - "medium_range:6+" - - "precipitation" - - "s2s,space_weather,tc_and_extra_tc" + - "convection_allowing_models:8" + - "cryosphere:0" + - "data_assimilation:0" + - "marine_and_coastal:0" + - "medium_range:0" + - "medium_range:1-2" + - "medium_range:3-5" + - "medium_range:6" + - "medium_range:7-8" + - "precipitation:0" + - "precipitation:1" + - "precipitation:2" + - "precipitation:3-8" + - "s2s:0" + - "space_weather:0-1" + - "tc_and_extra_tc:0-2" steps: - uses: actions/checkout@v2 - uses: ./ci/actions/run_tests diff --git a/ci/jobs/create_output_data_volumes.sh b/ci/jobs/create_output_data_volumes.sh index 05d30e01f3..7a65e68fc8 100755 --- a/ci/jobs/create_output_data_volumes.sh +++ b/ci/jobs/create_output_data_volumes.sh @@ -31,10 +31,16 @@ for vol_name in use_cases_*; do image_name=dtcenter/metplus-data-dev:output-${branch_name}-${vol_name#use_cases_} echo Creating Docker data volume: ${image_name} + + start_time=$SECONDS echo docker build -t ${image_name} --build-arg vol_name=${vol_name} ${docker_data_output_dir} docker build -t ${image_name} --build-arg vol_name=${vol_name} ${docker_data_output_dir} + echo Build took $(( SECONDS - start_time)) + + start_time=$SECONDS echo docker push ${image_name} docker push ${image_name} + echo Push took $(( SECONDS - start_time)) # remove data after it has been added to data volume rm -rf ${docker_data_output_dir}/$vol_name diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index 65392f50eb..6f03f102b7 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -579,6 +579,8 @@ properly, then delete the feature branch from dtcenter/METplus. This will avoid confusion if this branch diverges from the branch on the forked repository that will be used in the final pull request. +.. _add_use_case_to_test_suite: + Add use case to the test suite ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -599,11 +601,17 @@ definition line to this file and add your new use case under it. Each use case in that category will be found on its own line after this line. The use cases can be defined using 3 different formats:: - - :: - :::: + :: + :::: + :::::: + +**** + +The index is the number associated with the use case so it can be referenced +easily. The first index number in a new category should be 0. +Each use case added should have an index that is one greater than the previous. -**** +**::** This format should only be used if the use case has only 1 configuration file and no additional Python package dependencies besides the ones that are @@ -612,13 +620,13 @@ used for the use case relative to METplus/parm/use_cases. The filename of the config file without the .conf extension will be used as the name of the use case. Example:: - model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf + 6::model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf The above example will be named 'PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr' and will run using the configuration file listed. -**::** +**::::** This format is required if the use case contains multiple configuration files. Instead of forcing the script to guess which conf file should be used as the @@ -626,12 +634,12 @@ name of the use case, you must explicitly define it. The name of the use case must be separated from the with '::' and each conf file path or conf variable override must be separated by a comma. Example:: - GridStat_multiple_config:: met_tool_wrapper/GridStat/GridStat.conf,met_tool_wrapper/GridStat/GridStat_forecast.conf,met_tool_wrapper/GridStat/GridStat_observation.conf + 44::GridStat_multiple_config:: met_tool_wrapper/GridStat/GridStat.conf,met_tool_wrapper/GridStat/GridStat_forecast.conf,met_tool_wrapper/GridStat/GridStat_observation.conf The above example is named 'GridStat_multiple_config' and uses 3 .conf files. Use cases with only one configuration file can also use this format is desired. -**::::** +**::::::** This format is used if there are additional Python packages required to run the use case. is a list of packages to install before running @@ -639,7 +647,7 @@ the use case separated by commas. Example:: - TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf,user_env_vars.MET_PYTHON_EXE=python3::pygrib,metpy + 8::TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf,user_env_vars.MET_PYTHON_EXE=python3::pygrib,metpy The above example is named TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics. @@ -685,86 +693,106 @@ example of a script that uses Conda to install a package:: echo Installing xesmf with conda conda install -c conda-forge dask netCDF4 xesmf +.. _add_new_category_to_test_runs: + Add new category to test runs ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you are adding a new use case category, you will need to add a new entry -to the main.yml file found in the .github/workflows directory in the METplus -repository. -Find the job in the main.yml file named "use_case_tests" and add the new -category to the matrix categories list if it is not already there:: +Add a new entry to the main.yml file found in the .github/workflows directory +in the METplus repository. +Find the job in the main.yml file named "use_case_tests" and add a new entry +in the "categories" list with the category of the new use case followed by a +colon and the index of the use case from the list described in the +:ref:`add_use_case_to_test_suite` section. For example, if the new use case +is found in the medium_range category with an index of 9, and the main.yml file +has the following entries in the categories list:: use_case_tests: name: Use Case Tests runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, 'ci-doc-only')" needs: [get_image, update_data_volumes] strategy: fail-fast: false matrix: categories: - - "met_tool_wrapper" - - "air_quality_and_comp" - - "climate" - - "convection_allowing_models:0-5" - - "convection_allowing_models:6+" - - "cryosphere" - - "data_assimilation" - - "marine_and_coastal" - - "medium_range:0-4" - - "medium_range:5" - - "medium_range:6+" - - "precipitation" - - "s2s,space_weather,tc_and_extra_tc" - -Multiple Categories in One Test -""""""""""""""""""""""""""""""" - -If the use cases run quickly and you want to run multiple categories in one -job, you can add additional categories to this argument separated by commas or -ampersands, i.e. category1,category2. Do not include any spaces around the -commas. Example:: - - "s2s,space_weather,tc_and_extra_tc" + - "met_tool_wrapper:0-53" + - "air_quality_and_comp:0" + - "climate:0-1" + - "convection_allowing_models:0" + - "convection_allowing_models:1" + - "convection_allowing_models:2-6" + - "convection_allowing_models:7" + - "convection_allowing_models:8" + - "cryosphere:0" + - "data_assimilation:0" + - "marine_and_coastal:0" + - "medium_range:0" + - "medium_range:1-2" + - "medium_range:3-5" + - "medium_range:6" + - "medium_range:7-8" + - "precipitation:0" + - "precipitation:1" + - "precipitation:2" + - "precipitation:3-8" + - "s2s:0" + - "space_weather:0-1" + - "tc_and_extra_tc:0-2" + +then add the following to the list:: + + - "medium_range:9" + +.. note:: + Make sure that the indentation matches the other lines and use spaces to + indent instead of tabs. + +New use cases are added as a separate item to make reviewing the test results +easier. A new use case will produce new output data that is not found in the +"truth" data set which is compared the output of the use case runs to check +if code changes altered the final results. Isolating the new output will make +it easier to verify that the only differences are caused by the new data. +It also makes it easier to check the size of the output data and length of time +the use case takes to run to determine if it can be added to an existing group +or if it should remain in its own group. + .. _subset_category: Subset Category into Multiple Tests """"""""""""""""""""""""""""""""""" -If all of the use cases in a given category take a long time to run, you can -separate them into multiple test jobs. Add a colon (:), then define +Use cases can be separated into multiple test jobs. +Add a colon (:), then define the cases to run for the job. Use cases are numbered starting with 0 and are in order of how they are found in the all_use_cases.txt file. The argument supports a comma-separated list of numbers. Example:: - "data_assimilation:0,2,4" - ... - "data_assimilation:1,3" + - "data_assimilation:0,2,4" + - "data_assimilation:1,3" The above example will run a job with data_assimilation use cases 0, 2, and 4, then another job with data_assimilation use cases 1 and 3. It also supports a range of numbers separated with a dash. Example:: - "data_assimilation:0-3" - ... - "data_assimilation:4+" + - "data_assimilation:0-3" + - "data_assimilation:4-5" The above example will run a job with data_assimilation 0, 1, 2, and 3, then -another job with data_assimilation 4 and higher. If you split up use cases -into a subset, we recommend that you add a plus sign (+) to the end of the last -number specified in case additional use cases are added to the category. +another job with data_assimilation 4 and 5. You can also use a combination of commas and dashes to define the list of cases to run. Example:: - "data_assimilation:0-2,4+" + "data_assimilation:0-2,4" ... "data_assimilation:3" -The above example will run data_assimilation 0, 1, 2, 4, and above in one +The above example will run data_assimilation 0, 1, 2, and 4 in one job, then data_assimilation 3 in another job. Monitoring Automated Tests @@ -1009,8 +1037,32 @@ completed successfully. If the circle on the left side is yellow, then the run has not completed yet. If everything ran smoothly, clean up the files on the web server. +Consider rearranging the use case groups +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If another group of use cases in the same category exists, consider adding the +new use case to an existing group to speed up execution. +If a new use case runs quickly (check the time next to the use case group in +the diagram found on the Summary page of each GitHub Actions run), +produces a reasonably small sized output data +artifact (found at the bottom of a completed GitHub Actions run), and the same +applies to another group of same category, it would make sense to combine them. +In the .github/workflow/main.yml file, modify the categories list under the +"use_case_tests" job (see :ref:`add_new_category_to_test_runs`). For example, +if the following is found in the list:: + + - "met_tool_wrapper:0-53" + +and the new use case is defined with:: + + - "met_tool_wrapper:54" + +then combine the two list items into a single item:: + + - "met_tool_wrapper:0-54" + Create a pull request from develop into develop-ref -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The addition of a new use case results in new output data. When this happens, the reference branch needs to be updated so that future pull requests will diff --git a/internal_tests/use_cases/all_use_cases.txt b/internal_tests/use_cases/all_use_cases.txt index 4cbac94a14..2e0958e34b 100644 --- a/internal_tests/use_cases/all_use_cases.txt +++ b/internal_tests/use_cases/all_use_cases.txt @@ -1,117 +1,127 @@ Category: met_tool_wrapper -CyclonePlotter::met_tool_wrapper/CyclonePlotter/CyclonePlotter.conf,user_env_vars.MET_PYTHON_EXE=python3::cartopy,matplotlib -PCPCombine_python_embedding:: met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf,user_env_vars.MET_PYTHON_EXE=python3::h5py -ASCII2NC:: met_tool_wrapper/ASCII2NC/ASCII2NC.conf -met_tool_wrapper/ASCII2NC/ASCII2NC_python_embedding.conf -ASCII2NC_python_embedding_user_py:: met_tool_wrapper/ASCII2NC/ASCII2NC_python_embedding_user_py.conf -PyEmbedIngest:: met_tool_wrapper/PyEmbedIngest/PyEmbedIngest.conf -EnsembleStat:: met_tool_wrapper/EnsembleStat/EnsembleStat.conf -EnsembleStat_python_embedding:: met_tool_wrapper/EnsembleStat/EnsembleStat_python_embedding.conf -Example:: met_tool_wrapper/Example/Example.conf -GempakToCF:: met_tool_wrapper/GempakToCF/GempakToCF.conf -GenVxMask:: met_tool_wrapper/GenVxMask/GenVxMask.conf -GenVxMask_multiple:: met_tool_wrapper/GenVxMask/GenVxMask_multiple.conf -GenVxMask_with_arguments:: met_tool_wrapper/GenVxMask/GenVxMask_with_arguments.conf -GridDiag:: met_tool_wrapper/GridDiag/GridDiag.conf -GridStat:: met_tool_wrapper/GridStat/GridStat.conf -MODE:: met_tool_wrapper/MODE/MODE.conf -MODE_python_emedding:: met_tool_wrapper/MODE/MODE_python_embedding.conf -MTD:: met_tool_wrapper/MTD/MTD.conf -MTD_python_embedding:: met_tool_wrapper/MTD/MTD_python_embedding.conf -PB2NC:: met_tool_wrapper/PB2NC/PB2NC.conf -PCPCombine_sum:: met_tool_wrapper/PCPCombine/PCPCombine_sum.conf -PCPCombine_add:: met_tool_wrapper/PCPCombine/PCPCombine_add.conf -PCPCombine_bucket:: met_tool_wrapper/PCPCombine/PCPCombine_bucket.conf -PCPCombine_user_defined:: met_tool_wrapper/PCPCombine/PCPCombine_user_defined.conf -PCPCombine_derive:: met_tool_wrapper/PCPCombine/PCPCombine_derive.conf -PCPCombine_loop_custom:: met_tool_wrapper/PCPCombine/PCPCombine_loop_custom.conf -PCPCombine_subtract:: met_tool_wrapper/PCPCombine/PCPCombine_subtract.conf -PointStat:: met_tool_wrapper/PointStat/PointStat.conf -PointStat_python_embedding:: met_tool_wrapper/PointStat/PointStat_python_embedding.conf -Point2Grid:: met_tool_wrapper/Point2Grid/Point2Grid.conf -PointStat_once_per_field:: met_tool_wrapper/PointStat/PointStat_once_per_field.conf -RegridDataPlane:: met_tool_wrapper/RegridDataPlane/RegridDataPlane.conf -RegridDataPlane_multi_field_multi_file:: met_tool_wrapper/RegridDataPlane/RegridDataPlane_multi_field_multi_file.conf -RegridDataPlane_multi_field_one_file:: met_tool_wrapper/RegridDataPlane/RegridDataPlane_multi_field_one_file.conf -RegridDataPlane_python_embedding:: met_tool_wrapper/RegridDataPlane/RegridDataPlane_python_embedding.conf -StatAnalysis:: met_tool_wrapper/StatAnalysis/StatAnalysis.conf -StatAnalysis_python_embedding:: met_tool_wrapper/StatAnalysis/StatAnalysis_python_embedding.conf -SeriesAnalysis:: met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf::netCDF4 -SeriesAnalysis_python_embedding:: met_tool_wrapper/SeriesAnalysis/SeriesAnalysis_python_embedding.conf::netCDF4 -TCGen:: met_tool_wrapper/TCGen/TCGen.conf -TCPairs_extra_tropical:: met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf -TCPairs_tropical:: met_tool_wrapper/TCPairs/TCPairs_tropical.conf -TCRMW:: met_tool_wrapper/TCRMW/TCRMW.conf -TCStat:: met_tool_wrapper/TCStat/TCStat.conf -GridStat_multiple_config:: met_tool_wrapper/GridStat/GridStat.conf,met_tool_wrapper/GridStat/GridStat_forecast.conf,met_tool_wrapper/GridStat/GridStat_observation.conf -met_tool_wrapper/PlotDataPlane/PlotDataPlane_grib1.conf -met_tool_wrapper/PlotDataPlane/PlotDataPlane_netcdf.conf -met_tool_wrapper/PlotDataPlane/PlotDataPlane_python_embedding.conf -met_tool_wrapper/ExtractTiles/ExtractTiles.conf -met_tool_wrapper/UserScript/UserScript_run_once.conf -met_tool_wrapper/UserScript/UserScript_run_once_for_each.conf -met_tool_wrapper/UserScript/UserScript_run_once_per_init.conf -met_tool_wrapper/UserScript/UserScript_run_once_per_lead.conf -met_tool_wrapper/UserScript/UserScript_run_once_per_valid.conf +0::CyclonePlotter::met_tool_wrapper/CyclonePlotter/CyclonePlotter.conf,user_env_vars.MET_PYTHON_EXE=python3::cartopy,matplotlib +1::PCPCombine_python_embedding:: met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf,user_env_vars.MET_PYTHON_EXE=python3::h5py +2::ASCII2NC:: met_tool_wrapper/ASCII2NC/ASCII2NC.conf +3::met_tool_wrapper/ASCII2NC/ASCII2NC_python_embedding.conf +4::ASCII2NC_python_embedding_user_py:: met_tool_wrapper/ASCII2NC/ASCII2NC_python_embedding_user_py.conf +5::PyEmbedIngest:: met_tool_wrapper/PyEmbedIngest/PyEmbedIngest.conf +6::EnsembleStat:: met_tool_wrapper/EnsembleStat/EnsembleStat.conf +7::EnsembleStat_python_embedding:: met_tool_wrapper/EnsembleStat/EnsembleStat_python_embedding.conf +8::Example:: met_tool_wrapper/Example/Example.conf +9::GempakToCF:: met_tool_wrapper/GempakToCF/GempakToCF.conf +10::GenVxMask:: met_tool_wrapper/GenVxMask/GenVxMask.conf +11::GenVxMask_multiple:: met_tool_wrapper/GenVxMask/GenVxMask_multiple.conf +12::GenVxMask_with_arguments:: met_tool_wrapper/GenVxMask/GenVxMask_with_arguments.conf +13::GridDiag:: met_tool_wrapper/GridDiag/GridDiag.conf +14::GridStat:: met_tool_wrapper/GridStat/GridStat.conf +15::MODE:: met_tool_wrapper/MODE/MODE.conf +16::MODE_python_emedding:: met_tool_wrapper/MODE/MODE_python_embedding.conf +17::MTD:: met_tool_wrapper/MTD/MTD.conf +18::MTD_python_embedding:: met_tool_wrapper/MTD/MTD_python_embedding.conf +19::PB2NC:: met_tool_wrapper/PB2NC/PB2NC.conf +20::PCPCombine_sum:: met_tool_wrapper/PCPCombine/PCPCombine_sum.conf +21::PCPCombine_add:: met_tool_wrapper/PCPCombine/PCPCombine_add.conf +22::PCPCombine_bucket:: met_tool_wrapper/PCPCombine/PCPCombine_bucket.conf +23::PCPCombine_user_defined:: met_tool_wrapper/PCPCombine/PCPCombine_user_defined.conf +24::PCPCombine_derive:: met_tool_wrapper/PCPCombine/PCPCombine_derive.conf +25::PCPCombine_loop_custom:: met_tool_wrapper/PCPCombine/PCPCombine_loop_custom.conf +26::PCPCombine_subtract:: met_tool_wrapper/PCPCombine/PCPCombine_subtract.conf +27::PointStat:: met_tool_wrapper/PointStat/PointStat.conf +28::PointStat_python_embedding:: met_tool_wrapper/PointStat/PointStat_python_embedding.conf +29::Point2Grid:: met_tool_wrapper/Point2Grid/Point2Grid.conf +30::PointStat_once_per_field:: met_tool_wrapper/PointStat/PointStat_once_per_field.conf +31::RegridDataPlane:: met_tool_wrapper/RegridDataPlane/RegridDataPlane.conf +32::RegridDataPlane_multi_field_multi_file:: met_tool_wrapper/RegridDataPlane/RegridDataPlane_multi_field_multi_file.conf +33::RegridDataPlane_multi_field_one_file:: met_tool_wrapper/RegridDataPlane/RegridDataPlane_multi_field_one_file.conf +34::RegridDataPlane_python_embedding:: met_tool_wrapper/RegridDataPlane/RegridDataPlane_python_embedding.conf +35::StatAnalysis:: met_tool_wrapper/StatAnalysis/StatAnalysis.conf +36::StatAnalysis_python_embedding:: met_tool_wrapper/StatAnalysis/StatAnalysis_python_embedding.conf +37::SeriesAnalysis:: met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf::netCDF4 +38::SeriesAnalysis_python_embedding:: met_tool_wrapper/SeriesAnalysis/SeriesAnalysis_python_embedding.conf::netCDF4 +39::TCGen:: met_tool_wrapper/TCGen/TCGen.conf +40::TCPairs_extra_tropical:: met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf +41::TCPairs_tropical:: met_tool_wrapper/TCPairs/TCPairs_tropical.conf +42::TCRMW:: met_tool_wrapper/TCRMW/TCRMW.conf +43::TCStat:: met_tool_wrapper/TCStat/TCStat.conf +44::GridStat_multiple_config:: met_tool_wrapper/GridStat/GridStat.conf,met_tool_wrapper/GridStat/GridStat_forecast.conf,met_tool_wrapper/GridStat/GridStat_observation.conf +45::met_tool_wrapper/PlotDataPlane/PlotDataPlane_grib1.conf +46::met_tool_wrapper/PlotDataPlane/PlotDataPlane_netcdf.conf +47::met_tool_wrapper/PlotDataPlane/PlotDataPlane_python_embedding.conf +48::met_tool_wrapper/ExtractTiles/ExtractTiles.conf +49::met_tool_wrapper/UserScript/UserScript_run_once.conf +50::met_tool_wrapper/UserScript/UserScript_run_once_for_each.conf +51::met_tool_wrapper/UserScript/UserScript_run_once_per_init.conf +52::met_tool_wrapper/UserScript/UserScript_run_once_per_lead.conf +53::met_tool_wrapper/UserScript/UserScript_run_once_per_valid.conf + + +Category: air_quality_and_comp +0::EnsembleStat_fcstICAP_obsMODIS_aod::model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf,user_env_vars.MET_PYTHON_EXE=python3::netCDF4 + Category: climate -GridStat_fcstCESM_obsGFS_ConusTemp::model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf -MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip::model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf +0::GridStat_fcstCESM_obsGFS_ConusTemp::model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf +1::MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip::model_applications/climate/MODE_fcstCESM_obsGPCP_AsianMonsoonPrecip.conf + Category: convection_allowing_models -EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField::model_applications/convection_allowing_models/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf -MODE_fcstHRRR_obsMRMS_Hail_GRIB2::model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf -EnsembleStat_fcstHRRR_fcstOnly_SurrogateSevere::model_applications/convection_allowing_models/EnsembleStat_fcstHRRR_fcstOnly_SurrogateSevere.conf -GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere::model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf -GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb::model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf -Point2Grid_obsLSR_ObsOnly_PracticallyPerfect::model_applications/convection_allowing_models/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf -MODE_fcstFV3_obsGOES_BrightnessTemp::model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf,model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp/ci_overrides.conf -MODE_fcstFV3_obsGOES_BrightnessTempObjs:: model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf -GridStat_fcstFV3_obsGOES_BrightnessTempDmap:: model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf +0::EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField::model_applications/convection_allowing_models/EnsembleStat_fcstHRRRE_obsHRRRE_Sfc_MultiField.conf +1::MODE_fcstHRRR_obsMRMS_Hail_GRIB2::model_applications/convection_allowing_models/MODE_fcstHRRR_obsMRMS_Hail_GRIB2.conf +2::EnsembleStat_fcstHRRR_fcstOnly_SurrogateSevere::model_applications/convection_allowing_models/EnsembleStat_fcstHRRR_fcstOnly_SurrogateSevere.conf +3::GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere::model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf +4::GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb::model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf +5::Point2Grid_obsLSR_ObsOnly_PracticallyPerfect::model_applications/convection_allowing_models/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf +6::GridStat_fcstFV3_obsGOES_BrightnessTempDmap:: model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf +7::MODE_fcstFV3_obsGOES_BrightnessTemp::model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp.conf,model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTemp/ci_overrides.conf +8::MODE_fcstFV3_obsGOES_BrightnessTempObjs:: model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf -Category:cryosphere -GridStat_MODE_fcstIMS_obsNCEP_sea_ice::model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf + +Category: cryosphere +0::GridStat_MODE_fcstIMS_obsNCEP_sea_ice::model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf + Category: data_assimilation -StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface::model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.conf,user_env_vars.MET_PYTHON_EXE=python3::netCDF4 +0::StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface::model_applications/data_assimilation/StatAnalysis_fcstHAFS_obsPrepBufr_JEDI_IODA_interface.conf,user_env_vars.MET_PYTHON_EXE=python3::netCDF4 + + +Category: marine_and_coastal +0::PlotDataPlane_obsHYCOM_coordTripolar::model_applications/marine_and_coastal/PlotDataPlane_obsHYCOM_coordTripolar.conf,user_env_vars.MET_PYTHON_EXE=python3::xesmf Category: medium_range -PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr:: model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf -TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.conf::netCDF4 -TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf::netCDF4 -GridStat_fcstGFS_obsGFS_climoNCEP_MultiField:: model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf -GridStat_fcstGFS_obsGFS_Sfc_MultiField:: model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf -model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf -TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_IVT:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_IVT.conf,user_env_vars.MET_PYTHON_EXE=python3::pygrib,metpy,netCDF4 -#commented_out_use_case.conf -TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf,user_env_vars.MET_PYTHON_EXE=python3::pygrib,metpy,netCDF4 -UserScript_fcstGEFS_Difficulty_Index::model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.conf::METcalcpy,METplotpy +0::PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr:: model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.conf +1::TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByInit.conf::netCDF4 +2::TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf::netCDF4 +3::GridStat_fcstGFS_obsGFS_climoNCEP_MultiField:: model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf +4::GridStat_fcstGFS_obsGFS_Sfc_MultiField:: model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf +5::UserScript_fcstGEFS_Difficulty_Index::model_applications/medium_range/UserScript_fcstGEFS_Difficulty_Index.conf::METcalcpy,METplotpy +6::model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf +7::TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_IVT:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_IVT.conf,user_env_vars.MET_PYTHON_EXE=python3::pygrib,metpy,netCDF4 +8::TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics:: model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf,user_env_vars.MET_PYTHON_EXE=python3::pygrib,metpy,netCDF4 + + Category: precipitation -GridStat_fcstGFS_obsCCPA_GRIB:: model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf -EnsembleStat_fcstHRRRE_FcstOnly_NetCDF:: model_applications/precipitation/EnsembleStat_fcstHRRRE_FcstOnly_NetCDF.conf -GridStat_fcstHREFmean_obsStgIV_Gempak:: model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf -GridStat_fcstHREFmean_obsStgIV_NetCDF:: model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf -GridStat_fcstHRRR-TLE_obsStgIV_GRIB:: model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf -MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB:: model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.conf -MTD_fcstHRRR-TLE_obsMRMS:: model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.conf +0::EnsembleStat_fcstHRRRE_FcstOnly_NetCDF:: model_applications/precipitation/EnsembleStat_fcstHRRRE_FcstOnly_NetCDF.conf +1::GridStat_fcstHREFmean_obsStgIV_Gempak:: model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf +2::GridStat_fcstHREFmean_obsStgIV_NetCDF:: model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf +3::GridStat_fcstGFS_obsCCPA_GRIB:: model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf +4::GridStat_fcstHRRR-TLE_obsStgIV_GRIB:: model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf +5::MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB:: model_applications/precipitation/MTD_fcstHRRR-TLE_FcstOnly_RevisionSeries_GRIB.conf +6::MTD_fcstHRRR-TLE_obsMRMS:: model_applications/precipitation/MTD_fcstHRRR-TLE_obsMRMS.conf + Category: s2s -GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast:: model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf::netCDF4 +0::GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast:: model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf::netCDF4 -Category: space_weather -GridStat_fcstGloTEC_obsGloTEC_vx7:: model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf -GenVxMask_fcstGloTEC_FcstOnly_solar_altitude:: model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.conf -Category: tc_and_extra_tc -TCRMW_fcstGFS_fcstOnly_gonzalo:: model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.conf -CyclonePlotter_fcstGFS_obsGFS_OPC:: model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf:: cartopy,matplotlib -UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF:: model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf,model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF/ci_overrides.conf +Category: space_weather +0::GridStat_fcstGloTEC_obsGloTEC_vx7:: model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf +1::GenVxMask_fcstGloTEC_FcstOnly_solar_altitude:: model_applications/space_weather/GenVxMask_fcstGloTEC_FcstOnly_solar_altitude.conf -Category: air_quality_and_comp -EnsembleStat_fcstICAP_obsMODIS_aod::model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf,user_env_vars.MET_PYTHON_EXE=python3::netCDF4 -Category: marine_and_coastal -PlotDataPlane_obsHYCOM_coordTripolar::model_applications/marine_and_coastal/PlotDataPlane_obsHYCOM_coordTripolar.conf,user_env_vars.MET_PYTHON_EXE=python3::xesmf +Category: tc_and_extra_tc +0::TCRMW_fcstGFS_fcstOnly_gonzalo:: model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.conf +1::CyclonePlotter_fcstGFS_obsGFS_OPC:: model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf:: cartopy,matplotlib +2::UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF:: model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf,model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF/ci_overrides.conf diff --git a/internal_tests/use_cases/metplus_use_case_suite.py b/internal_tests/use_cases/metplus_use_case_suite.py index 3053f19086..5f0397afb5 100644 --- a/internal_tests/use_cases/metplus_use_case_suite.py +++ b/internal_tests/use_cases/metplus_use_case_suite.py @@ -220,7 +220,17 @@ def add_use_case_groups(self, categories, case_slice=None): for category in categories: use_cases_list.extend(self.all_cases[category]) - use_cases_to_run = subset_list(use_cases_list, case_slice) + # if argument is a slice object or None, call subset_list util + if isinstance(case_slice, slice) or case_slice is None: + use_cases_to_run = subset_list(use_cases_list, case_slice) + # if not, get all use cases that have an index in the list + else: + case_indices = case_slice + if not isinstance(case_slice, list): + case_indices = [case_indices] + use_cases_to_run = [use_case for use_case in use_cases_list + if use_case['index'] in case_indices] + self.add_use_case_group('&'.join(categories), use_cases=use_cases_to_run, @@ -363,7 +373,8 @@ def parse_all_use_cases_file(): for category, use_case_list in category_dict.items(): all_cases[category] = [] for use_case in use_case_list: - name, *rest = use_case.split('::') + index, name, *rest = use_case.split('::') + index = int(index.strip()) name = name.strip() requirements = [] @@ -384,7 +395,8 @@ def parse_all_use_cases_file(): set_output = f'config.USE_CASE_NAME={name}' config_args.append(set_output) - use_case_dict = {'name': name, + use_case_dict = {'index': index, + 'name': name, 'config_args': config_args, 'requirements': requirements} all_cases[category].append(use_case_dict) @@ -408,13 +420,13 @@ def parse_all_use_cases_file(): # slice(5) == medium range 0-4 all_use_cases.add_use_case_groups('medium_range', - case_slice=slice(5)) + case_slice=range(5)) # 5 = medium range 5 all_use_cases.add_use_case_groups('medium_range', case_slice=5) - # slice(6, None) == medium range 6+ + # slice(6, None) == medium range 6-8 all_use_cases.add_use_case_groups('medium_range', - case_slice=slice(6, None)) + case_slice=range(6, 9)) all_use_cases.add_use_case_groups('precipitation')