diff --git a/.github/actions/run_tests/entrypoint.sh b/.github/actions/run_tests/entrypoint.sh index 6c5da65ea2..bd2aa579dc 100644 --- a/.github/actions/run_tests/entrypoint.sh +++ b/.github/actions/run_tests/entrypoint.sh @@ -61,7 +61,7 @@ if [[ "$INPUT_CATEGORIES" == pytests* ]]; then for x in `cat $PYTESTS_GROUPS_FILEPATH`; do marker="${x//_or_/ or }" marker="${marker//not_/not }" - command+="/usr/local/conda/envs/${METPLUS_ENV_TAG}/bin/pytest -vv --cov=../../../metplus -m \"$marker\"" + command+="/usr/local/conda/envs/${METPLUS_ENV_TAG}/bin/pytest -vv --cov=../../../metplus --cov-append -m \"$marker\"" command+=";if [ \$? != 0 ]; then status=1; fi;" done command+="if [ \$status != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi" diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 0c927c2223..d2b2475fcf 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -11,7 +11,7 @@ }, { "category": "air_quality_and_comp", - "index_list": "0", + "index_list": "0-6", "run": false }, { diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index e2e52287fd..5da4da815e 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -394,13 +394,18 @@ METplus Configuration Glossary | *Used by:* RegridDataPlane + LOG_TC_DIAG_VERBOSITY + Overrides the log verbosity for TCDiag only. If not set, the verbosity level is controlled by :term:`LOG_MET_VERBOSITY`. + + | *Used by:* TCDiag + LOG_TC_PAIRS_VERBOSITY Overrides the log verbosity for TCPairs only. If not set, the verbosity level is controlled by :term:`LOG_MET_VERBOSITY`. | *Used by:* TCPairs LOG_TC_RMW_VERBOSITY - Overrides the log verbosity for TCRMW only. If not set, the verbosity level is controlled by :term:`LOG_MET_VERBOSITY`. + Overrides the log verbosity for TCRMW only. If not set, the verbosity level is controlled by :term:`LOG_MET_VERBOSITY`. | *Used by:* TCRMW @@ -499,6 +504,12 @@ METplus Configuration Glossary | *Used by:* TCRMW + TC_DIAG_CONFIG_FILE + Path to configuration file read by tc_diag. + If unset, parm/met_config/TCDiagConfig_wrapped will be used. + + | *Used by:* TCDiag + ASCII2NC_INPUT_FORMAT Optional string to specify the format of the input data. Valid options are "met_point", "little_r", "surfrad", "wwsis", "aeronet", "aeronetv2", or "aeronetv3." @@ -4705,6 +4716,11 @@ METplus Configuration Glossary | *Used by:* TCRMW + TC_DIAG_INPUT_DATATYPE + Specify the data type of the input directory for input files used with the MET tc_diag tool. Used to set the 'file_type' value of the data dictionary in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + TC_RMW_INPUT_DIR Directory containing input data to TCRMW. This variable is optional because you can specify the full path to the input files using :term:`TC_RMW_INPUT_TEMPLATE`. @@ -10388,3 +10404,284 @@ METplus Configuration Glossary Specify the value for 'regrid.censor_val' in the MET configuration file for TCRMW. | *Used by:* TCRMW + + TC_DIAG_INPUT_DIR + Directory containing input data to TCDiag. This variable is optional because you can specify the full path to the input files using :term:`TC_DIAG_INPUT_TEMPLATE`. + + | *Used by:* TCDiag + + TC_DIAG_INPUT_TEMPLATE + Filename template of the input data used by TCDiag. See also :term:`TC_DIAG_INPUT_DIR`. + + | *Used by:* TCDiag + + TC_DIAG_DECK_INPUT_DIR + Directory containing ADECK input data to TCDiag. This variable is optional because you can specify the full path to the input files using :term:`TC_DIAG_DECK_TEMPLATE`. + + | *Used by:* TCDiag + + TC_DIAG_DECK_TEMPLATE + Filename template of the ADECK input data used by TCDiag. See also :term:`TC_DIAG_DECK_INPUT_DIR`. + + | *Used by:* TCDiag + + TC_DIAG_OUTPUT_DIR + Directory to write output data from TCDiag. This variable is optional because you can specify the full path to the output file using :term:`TC_DIAG_OUTPUT_TEMPLATE`. + + | *Used by:* TCDiag + + TC_DIAG_OUTPUT_TEMPLATE + Filename template of write the output data generated by TCDiag. See also :term:`TC_DIAG_OUTPUT_DIR`. + + | *Used by:* TCDiag + + TC_DIAG_INPUT_FILE_LIST + Specifies an explicit path to a file list file to pass into tc_diag. + If set, :term:`TC_DIAG_INPUT_TEMPLATE` and :term:`TC_DIAG_INPUT_DIR` + are ignored. + + | *Used by:* TCDiag + + TC_DIAG_MET_CONFIG_OVERRIDES + Override any variables in the MET configuration file that are not + supported by the wrapper. This should be set to the full variable name + and value that you want to override, including the equal sign and the + ending semi-colon. The value is directly appended to the end of the + wrapped MET config file. + + Example: + TC_DIAG_MET_CONFIG_OVERRIDES = desc = "override_desc"; model = "override_model"; + + See :ref:`Overriding Unsupported MET config file settings` for more information + + | *Used by:* TCDiag + + TC_DIAG_SKIP_IF_OUTPUT_EXISTS + If True, do not run app if output file already exists. Set to False to overwrite files. + + | *Used by:* TCDiag + + TC_DIAG_MODEL + Specify the value for 'model' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_STORM_ID + Specify the value for 'storm_id' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_BASIN + Specify the value for 'basin' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_CYCLONE + Specify the value for 'cyclone' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_INIT_INCLUDE + Specify the value for 'init_inc' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_VALID_BEG + Specify the value for 'valid_beg' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_VALID_END + Specify the value for 'valid_end' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_VALID_INCLUDE + Specify the value for 'valid_inc' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_VALID_EXCLUDE + Specify the value for 'valid_exc' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_VALID_HOUR + Specify the value for 'valid_hour' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_DIAG_SCRIPT + Specify the value for 'diag_script' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_DOMAIN_INFO_DOMAIN + Specify the value for the nth 'domain_info.domain' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_DOMAIN_INFO_N_RANGE + Specify the value for the nth 'domain_info.n_range' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_DOMAIN_INFO_N_AZIMUTH + Specify the value for the nth 'domain_info.n_azimuth' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_DOMAIN_INFO_DELTA_RANGE_KM + Specify the value for the nth 'domain_info.delta_range_km' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_DOMAIN_INFO_DIAG_SCRIPT + Specify the value for the nth 'domain_info.diag_script' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_CENSOR_THRESH + Specify the value for 'censor_thresh' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_CENSOR_VAL + Specify the value for 'censor_val' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_CONVERT + Specify the value for 'convert' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_DATA_DOMAIN + Specify the value for 'data.domain' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_DATA_LEVEL + Specify the value for 'data.level' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_REGRID_METHOD + Specify the value for 'regrid.method' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_REGRID_WIDTH + Specify the value for 'regrid.width' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_REGRID_VLD_THRESH + Specify the value for 'regrid.vld_thresh' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_REGRID_SHAPE + Specify the value for 'regrid.shape' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_COMPUTE_TANGENTIAL_AND_RADIAL_WINDS + Specify the value for 'compute_tangential_and_radial_winds' in the MET configuration file for TCDiag . + + | *Used by:* TCDiag + + TC_DIAG_U_WIND_FIELD_NAME + Specify the value for 'u_wind_field_name' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_V_WIND_FIELD_NAME + Specify the value for 'v_wind_field_name' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_TANGENTIAL_VELOCITY_FIELD_NAME + Specify the value for 'tangential_velocity_field_name' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_TANGENTIAL_VELOCITY_LONG_FIELD_NAME + Specify the value for 'tangential_velocity_long_field_name' in the MET configuration file for TCDiag . + + | *Used by:* TCDiag + + TC_DIAG_RADIAL_VELOCITY_FIELD_NAME + Specify the value for 'radial_velocity_field_name' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_RADIAL_VELOCITY_LONG_FIELD_NAME + Specify the value for 'radial_velocity_long_field_name' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_VORTEX_REMOVAL + Specify the value for 'vortex_removal' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_NC_RNG_AZI_FLAG + Specify the value for 'nc_rng_azi_flag' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_NC_DIAG_FLAG + Specify the value for 'nc_diag_flag' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_CIRA_DIAG_FLAG + Specify the value for 'cira_diag_flag' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + TC_DIAG_OUTPUT_PREFIX + Specify the value for 'output_prefix' in the MET configuration file for TCDiag. + + | *Used by:* TCDiag + + MODE_MULTIVAR_INTENSITY + Specify the value for 'multivar_intensity' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_FCST_MULTIVAR_NAME + Specify the value for 'fcst.multivar_name' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_FCST_MULTIVAR_UNITS + Specify the value for 'fcst.multivar_units' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_OBS_MULTIVAR_NAME + Specify the value for 'obs.multivar_name' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_OBS_MULTIVAR_UNITS + Specify the value for 'obs.multivar_units' in the MET configuration file for MODE. + + | *Used by:* MODE diff --git a/docs/Users_Guide/quicksearch.rst b/docs/Users_Guide/quicksearch.rst index 7fca01c204..58caef38f2 100644 --- a/docs/Users_Guide/quicksearch.rst +++ b/docs/Users_Guide/quicksearch.rst @@ -32,6 +32,7 @@ Use Cases by MET Tool: | `RegridDataPlane <../search.html?q=RegridDataPlaneToolUseCase&check_keywords=yes&area=default>`_ | `SeriesAnalysis <../search.html?q=SeriesAnalysisUseCase&check_keywords=yes&area=default>`_ | `StatAnalysis <../search.html?q=StatAnalysisToolUseCase&check_keywords=yes&area=default>`_ + | `TCDiag <../search.html?q=TCDiagToolUseCase&check_keywords=yes&area=default>`_ | `TCMPRPlotter <../search.html?q=TCMPRPlotterUseCase&check_keywords=yes&area=default>`_ | `TCGen <../search.html?q=TCGenToolUseCase&check_keywords=yes&area=default>`_ | `TCPairs <../search.html?q=TCPairsToolUseCase&check_keywords=yes&area=default>`_ @@ -59,6 +60,7 @@ Use Cases by MET Tool: | **RegridDataPlane**: *RegridDataPlaneToolUseCase* | **SeriesAnalysis**: *SeriesAnalysisUseCase* | **StatAnalysis**: *StatAnalysisToolUseCase* + | **TCDiag**: *TCDiagToolUseCase* | **TCMPRPlotter**: *TCMPRPlotterUseCase* | **TCGen**: *TCGenToolUseCase* | **TCPairs**: *TCPairsToolUseCase* diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index df1680d803..88c8212d3a 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -35,6 +35,9 @@ METplus Version 5.1.0-rc1 Release Notes (2023-06-07) .. dropdown:: Enhancements + * Add support for new multivariate MODE settings + (`#2197 `_) + .. dropdown:: Bugfix * Define the order of the forecast variables numerically rather than alphabetically @@ -52,6 +55,11 @@ METplus Version 5.1.0-rc1 Release Notes (2023-06-07) * StatAnalysis time shifting failure (`#2168 `_) + .. dropdown:: New Wrappers + + * TCDiag (beta) + (`#1626 `_) + .. dropdown:: New Use Cases * Read in Argo profile data netCDF files for use in METplus with python embedding @@ -73,6 +81,8 @@ METplus Version 5.1.0-rc1 Release Notes (2023-06-07) (`#2147 `_) * Update the development release guide instructions to remove references to a Coordinated release (`#2159 `_) + * Refactored code to resolve many SonarQube items + (`#1610 `_) METplus Version 5.1.0-beta2 Release Notes (2023-04-26) ------------------------------------------------------ diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index a900dcd864..daf201eb69 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -4055,16 +4055,21 @@ METplus Configuration | :term:`FCST_MODE_IS_PROB` | :term:`FCST_MODE_PROB_IN_GRIB_PDS` | :term:`MODE_MULTIVAR_LOGIC` +| :term:`MODE_MULTIVAR_INTENSITY` | :term:`FCST_MODE_VAR_NAME` | :term:`FCST_MODE_VAR_LEVELS` | :term:`FCST_MODE_VAR_THRESH` | :term:`FCST_MODE_VAR_OPTIONS` | :term:`MODE_FCST_FILE_TYPE` +| :term:`MODE_FCST_MULTIVAR_NAME` +| :term:`MODE_FCST_MULTIVAR_UNITS` | :term:`OBS_MODE_VAR_NAME` | :term:`OBS_MODE_VAR_LEVELS` | :term:`OBS_MODE_VAR_THRESH` | :term:`OBS_MODE_VAR_OPTIONS` | :term:`MODE_OBS_FILE_TYPE` +| :term:`MODE_OBS_MULTIVAR_NAME` +| :term:`MODE_OBS_MULTIVAR_UNITS` | .. warning:: **DEPRECATED:** @@ -4184,6 +4189,17 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`MODE_MULTIVAR_LOGIC` - multivar_logic +**${METPLUS_MULTIVAR_INTENSITY}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_MULTIVAR_INTENSITY` + - multivar_intensity + **${METPLUS_FCST_FIELD}** .. list-table:: @@ -4258,6 +4274,28 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`MODE_FCST_FILE_TYPE` - fcst.file_type +**${METPLUS_FCST_MULTIVAR_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_FCST_MULTIVAR_NAME` + - fcst.multivar_name + +**${METPLUS_FCST_MULTIVAR_UNITS}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_FCST_MULTIVAR_UNITS` + - fcst.multivar_units + **${METPLUS_OBS_FIELD}** @@ -4333,6 +4371,28 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`MODE_OBS_FILE_TYPE` - obs.file_type +**${METPLUS_OBS_MULTIVAR_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_OBS_MULTIVAR_NAME` + - obs.multivar_name + +**${METPLUS_OBS_MULTIVAR_UNITS}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_OBS_MULTIVAR_UNITS` + - obs.multivar_units + **${METPLUS_MASK_POLY}** .. list-table:: @@ -7624,6 +7684,494 @@ see :ref:`How METplus controls MET config file settings`. - hss_ec_value +.. _tc_diag_wrapper: + +TCDiag +====== + +Description +----------- + +Used to configure the MET tool TC-Diag. + +METplus Configuration +--------------------- + +| :term:`TC_DIAG_INPUT_DIR` +| :term:`TC_DIAG_DECK_INPUT_DIR` +| :term:`TC_DIAG_OUTPUT_DIR` +| :term:`TC_DIAG_DECK_TEMPLATE` +| :term:`TC_DIAG_INPUT_TEMPLATE` +| :term:`TC_DIAG_INPUT_FILE_LIST` +| :term:`TC_DIAG_OUTPUT_TEMPLATE` +| :term:`LOG_TC_DIAG_VERBOSITY` +| :term:`TC_DIAG_CONFIG_FILE` +| :term:`TC_DIAG_MODEL` +| :term:`TC_DIAG_STORM_ID` +| :term:`TC_DIAG_BASIN` +| :term:`TC_DIAG_CYCLONE` +| :term:`TC_DIAG_INIT_INCLUDE` +| :term:`TC_DIAG_VALID_BEG` +| :term:`TC_DIAG_VALID_END` +| :term:`TC_DIAG_VALID_INCLUDE` +| :term:`TC_DIAG_VALID_EXCLUDE` +| :term:`TC_DIAG_VALID_HOUR` +| :term:`TC_DIAG_DIAG_SCRIPT` +| :term:`TC_DIAG_DOMAIN_INFO_DOMAIN` +| :term:`TC_DIAG_DOMAIN_INFO_N_RANGE` +| :term:`TC_DIAG_DOMAIN_INFO_N_AZIMUTH` +| :term:`TC_DIAG_DOMAIN_INFO_DELTA_RANGE_KM` +| :term:`TC_DIAG_DOMAIN_INFO_DIAG_SCRIPT` +| :term:`TC_DIAG_CENSOR_THRESH` +| :term:`TC_DIAG_CENSOR_VAL` +| :term:`TC_DIAG_CONVERT` +| :term:`TC_DIAG_INPUT_DATATYPE` +| :term:`TC_DIAG_DATA_DOMAIN` +| :term:`TC_DIAG_DATA_LEVEL` +| :term:`TC_DIAG_REGRID_METHOD` +| :term:`TC_DIAG_REGRID_WIDTH` +| :term:`TC_DIAG_REGRID_VLD_THRESH` +| :term:`TC_DIAG_REGRID_SHAPE` +| :term:`TC_DIAG_REGRID_CENSOR_THRESH` +| :term:`TC_DIAG_REGRID_CENSOR_VAL` +| :term:`TC_DIAG_REGRID_CONVERT` +| :term:`TC_DIAG_COMPUTE_TANGENTIAL_AND_RADIAL_WINDS` +| :term:`TC_DIAG_U_WIND_FIELD_NAME` +| :term:`TC_DIAG_V_WIND_FIELD_NAME` +| :term:`TC_DIAG_TANGENTIAL_VELOCITY_FIELD_NAME` +| :term:`TC_DIAG_TANGENTIAL_VELOCITY_LONG_FIELD_NAME` +| :term:`TC_DIAG_RADIAL_VELOCITY_FIELD_NAME` +| :term:`TC_DIAG_RADIAL_VELOCITY_LONG_FIELD_NAME` +| :term:`TC_DIAG_VORTEX_REMOVAL` +| :term:`TC_DIAG_NC_RNG_AZI_FLAG` +| :term:`TC_DIAG_NC_DIAG_FLAG` +| :term:`TC_DIAG_CIRA_DIAG_FLAG` +| :term:`TC_DIAG_OUTPUT_PREFIX` +| :term:`TC_DIAG_SKIP_IF_OUTPUT_EXISTS` +| :term:`TC_DIAG_MET_CONFIG_OVERRIDES` +| + +.. _tc-diag-met-conf: + +MET Configuration +----------------- + +Below is the wrapped MET configuration file used for this wrapper. +Environment variables are used to control entries in this configuration file. +The default value for each environment variable is obtained from +(except where noted below): + +`MET_INSTALL_DIR/share/met/config/TCDiagConfig_default `_ + +Below the file contents are descriptions of each environment variable +referenced in this file and the corresponding METplus configuration item used +to set the value of the environment variable. For detailed examples showing +how METplus sets the values of these environment variables, +see :ref:`How METplus controls MET config file settings`. + +.. literalinclude:: ../../parm/met_config/TCDiagConfig_wrapped + +**${METPLUS_MODEL}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`MODEL` + - model + +**${METPLUS_STORM_ID}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_STORM_ID` + - storm_id + +**${METPLUS_BASIN}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_BASIN` + - basin + +**${METPLUS_CYCLONE}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_CYCLONE` + - cyclone + +**${METPLUS_INIT_INCLUDE_LIST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_INIT_INCLUDE` + - init_inc + +**${METPLUS_VALID_BEG}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_VALID_BEG` + - valid_beg + +**${METPLUS_VALID_END}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_VALID_END` + - valid_end + +**${METPLUS_VALID_INCLUDE_LIST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_VALID_INCLUDE` + - valid_inc + +**${METPLUS_VALID_EXCLUDE_LIST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_VALID_EXCLUDE` + - valid_exc + +**${METPLUS_VALID_HOUR_LIST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_VALID_HOUR` + - valid_hour + +**${METPLUS_LEAD_LIST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`LEAD_SEQ` + - lead + +**${METPLUS_DIAG_SCRIPT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_DIAG_SCRIPT` + - diag_script + +**${METPLUS_DOMAIN_INFO_LIST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_DOMAIN_INFO_DOMAIN` + - domain_info.domain + * - :term:`TC_DIAG_DOMAIN_INFO_N_RANGE` + - domain_info.n_range + * - :term:`TC_DIAG_DOMAIN_INFO_N_AZIMUTH` + - domain_info.n_azimuth + * - :term:`TC_DIAG_DOMAIN_INFO_DELTA_RANGE_KM` + - domain_info.delta_range_km + * - :term:`TC_DIAG_DOMAIN_INFO_DIAG_SCRIPT` + - domain_info.diag_script + +**${METPLUS_CENSOR_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_CENSOR_THRESH` + - censor_thresh + +**${METPLUS_CENSOR_VAL}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_CENSOR_VAL` + - censor_val + +**${METPLUS_CONVERT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_CONVERT` + - convert + +**${METPLUS_DATA_FIELD}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`BOTH_VAR_NAME` + - data.field.name + * - :term:`BOTH_VAR_LEVELS` + - data.field.level + * - :term:`BOTH_VAR_OPTIONS` + - n/a + +.. note:: For more information on controlling the field attributes in METplus, please see the :ref:`Field_Info` section of the User's Guide. + +**${METPLUS_DATA_FILE_TYPE}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_INPUT_DATATYPE` + - data.file_type + +**${METPLUS_DATA_DOMAIN}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_DATA_DOMAIN` + - data.domain + +**${METPLUS_DATA_LEVEL}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_DATA_LEVEL` + - data.level + +**${METPLUS_REGRID_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_REGRID_SHAPE` + - regrid.shape + * - :term:`TC_DIAG_REGRID_METHOD` + - regrid.method + * - :term:`TC_DIAG_REGRID_WIDTH` + - regrid.width + * - :term:`TC_DIAG_REGRID_VLD_THRESH` + - regrid.vld_thresh + * - :term:`TC_DIAG_REGRID_CONVERT` + - regrid.convert + * - :term:`TC_DIAG_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`TC_DIAG_REGRID_CENSOR_VAL` + - regrid.censor_val + +**${METPLUS_COMPUTE_TANGENTIAL_AND_RADIAL_WINDS}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_COMPUTE_TANGENTIAL_AND_RADIAL_WINDS` + - compute_tangential_and_radial_winds + +**${METPLUS_U_WIND_FIELD_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_U_WIND_FIELD_NAME` + - u_wind_field_name + +**${METPLUS_V_WIND_FIELD_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_V_WIND_FIELD_NAME` + - v_wind_field_name + +**${METPLUS_TANGENTIAL_VELOCITY_FIELD_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_TANGENTIAL_VELOCITY_FIELD_NAME` + - tangential_velocity_field_name + +**${METPLUS_TANGENTIAL_VELOCITY_LONG_FIELD_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_TANGENTIAL_VELOCITY_LONG_FIELD_NAME` + - tangential_velocity_long_field_name + +**${METPLUS_RADIAL_VELOCITY_FIELD_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_RADIAL_VELOCITY_FIELD_NAME` + - radial_velocity_field_name + +**${METPLUS_RADIAL_VELOCITY_LONG_FIELD_NAME}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_RADIAL_VELOCITY_LONG_FIELD_NAME` + - radial_velocity_long_field_name + +**${METPLUS_VORTEX_REMOVAL}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_VORTEX_REMOVAL` + - vortex_removal + +**${METPLUS_NC_RNG_AZI_FLAG}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_NC_RNG_AZI_FLAG` + - nc_rng_azi_flag + +**${METPLUS_NC_DIAG_FLAG}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_NC_DIAG_FLAG` + - nc_diag_flag + +**${METPLUS_CIRA_DIAG_FLAG}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_CIRA_DIAG_FLAG` + - cira_diag_flag + +**${METPLUS_OUTPUT_PREFIX}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_OUTPUT_PREFIX` + - output_prefix + +**${METPLUS_MET_CONFIG_OVERRIDES}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_MET_CONFIG_OVERRIDES` + - n/a + + .. _tc_gen_wrapper: TCGen diff --git a/docs/_static/met_tool_wrapper-TCDiag.png b/docs/_static/met_tool_wrapper-TCDiag.png new file mode 100644 index 0000000000..9bdf0b2398 Binary files /dev/null and b/docs/_static/met_tool_wrapper-TCDiag.png differ diff --git a/docs/use_cases/met_tool_wrapper/TCDiag/README.rst b/docs/use_cases/met_tool_wrapper/TCDiag/README.rst new file mode 100644 index 0000000000..b9803f5489 --- /dev/null +++ b/docs/use_cases/met_tool_wrapper/TCDiag/README.rst @@ -0,0 +1,2 @@ +TCDiag +------ diff --git a/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py b/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py new file mode 100644 index 0000000000..d2a8f0b75d --- /dev/null +++ b/docs/use_cases/met_tool_wrapper/TCDiag/TCDiag.py @@ -0,0 +1,116 @@ +""" +TCDiag: Basic Use Case +====================== + +met_tool_wrapper/TCDiag/TCDiag.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# TODO: Add content here + +############################################################################## +# Datasets +# -------- +# +# **Forecast:** GFS FV3 +# **Track:** A Deck +# +# **Location:** All of the input data required for this use case can be found +# in the met_test sample data tarball. Click here to the METplus releases page +# and download sample data for the appropriate release: +# https://github.com/dtcenter/METplus/releases +# +# This tarball should be unpacked into the directory that you will set the +# value of INPUT_BASE. See `Running METplus`_ section for more information. +# + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes the METplus TCDiag wrapper to search for +# the desired ADECK file and forecast files that are correspond to the track. +# It generates a command to run tc_diag if all required files are found. + +############################################################################## +# METplus Workflow +# ---------------- +# +# TCDiag is the only tool called in this example. It processes the following +# run times: +# +# **Init:** 2016-09-29- 00Z +# **Forecast lead:** 141, 143, and 147 hour +# + +############################################################################## +# METplus Configuration +# --------------------- +# +# parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf + +############################################################################## +# MET Configuration +# --------------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. note:: See the :ref:`TCDiag MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/TCDiagConfig_wrapped + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. +# + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in met_tool_wrapper/TCDiag (relative to **OUTPUT_BASE**) +# and will contain the following files: +# +# * tc_diag_aal142016.nc +# + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * TCDiagToolUseCase +# * GRIB2FileUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# +# +# sphinx_gallery_thumbnail_path = '_static/met_tool_wrapper-TCDiag.png' +# diff --git a/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.py b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.py new file mode 100644 index 0000000000..26c14bafb6 --- /dev/null +++ b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.py @@ -0,0 +1,138 @@ +""" +GridStat: Cloud Fractions with Neighborhood and Probabilities +============================================================= + +model_applications/air_quality_and_comp/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# This use case captures various statistical measures of two model comparisons +# for low and total cloud fractions with different neighborhood +# settings for internal model metrics and to aid in future model updates +# + +############################################################################## +# Datasets +# -------- +# +# | **Forecast:** Global Forecast System (GFS) +# | **Observations:** ECMWF Reanalysis, Version 5 (ERA5) +# | **Grid:** GPP 17km masking region +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword +# in the observation input template settings. The same Python script can processes both forecast and +# observation datasets, but only the observation dataset is not +# set up for native ingest by MET. Two separate forecast fields are verified against two respective observation fields, +# with the Python script being passed the input file, the model name, the variable name being analyzed, +# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. +# This process is repeated with 3 instance names to GridStat, each with a different setting for regridding, +# neighborhood evaluation, thresholding, output line types, and output prefix names. + +############################################################################## +# METplus Workflow +# ---------------- +# +# GridStat is the only MET tool called in this example. +# It processes the following run time: +# +# | **Init:** 2022-07-03 12Z +# | **Forecast lead:** 36 hour +# +# Because instance names are used, GridStat will run 3 times for this 1 initalization time. + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads the default configuration file found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line: +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.conf + +############################################################################## +# MET Configuration +# ----------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped + +############################################################################## +# Python Embedding +# ---------------- +# +# This use case utilizes 1 Python script to read and process the observation fields. +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac/read_input_data.py + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in model_applications/air_quality_and_comp/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac +# (relative to **OUTPUT_BASE**) +# and will contain the following files: +# +# * grid_stat_GFS_to_ERA5_F36_CloudFracs_360000L_20220705_000000V_pairs.nc +# * grid_stat_GFS_to_ERA5_F36_CloudFracs_360000L_20220705_000000V.stat +# * grid_stat_GFS_to_ERA5_F36_CloudFracs_NBR_360000L_20220705_000000V_pairs.nc +# * grid_stat_GFS_to_ERA5_F36_CloudFracs_NBR_360000L_20220705_000000V.stat +# * grid_stat_GFS_to_ERA5_F36_CloudFracs_PROB_360000L_20220705_000000V_pairs.nc +# * grid_stat_GFS_to_ERA5_F36_CloudFracs_PRB_360000L_20220705_000000V.stat + + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * GridStatToolUseCase +# * NetCDFFileUseCase +# * AirQualityAndCompAppUseCase +# * PythonEmbeddingFileUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# sphinx_gallery_thumbnail_path = '_static/air_quality_and_comp-GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.png' +# diff --git a/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.py b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.py new file mode 100644 index 0000000000..9ea7468677 --- /dev/null +++ b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.py @@ -0,0 +1,138 @@ +""" +GridStat: Cloud Fractions with Neighborhood and Probabilities +============================================================= + +model_applications/air_quality_and_comp/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# This use case captures various statistical measures of two model comparisons +# for low and total cloud fraction with different neighborhood and probability +# settings for internal model metrics and to aid in future model updates +# + +############################################################################## +# Datasets +# -------- +# +# | **Forecast:** Global Forecast System (GFS) +# | **Observations:** Modern-Era Retrospective analysis for Research and Applications, Version 2 (MERRA2) +# | **Grid:** GPP 17km masking region +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword +# in the observation input template settings. The same Python script processes both forecast and +# observation datasets, but only the observation dataset is not set up for native ingest by MET. +# Two separate forecast fields are verified against two respective observation fields, +# with the Python script being passed the input file, the model name, the variable name being analyzed, +# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. +# This process is repeated with 3 instance names to GridStat, each with a different setting for regridding, +# neighborhood evaluation, thresholding, output line types, and output prefix names. + +############################################################################## +# METplus Workflow +# ---------------- +# +# GridStat is the only MET tool called in this example. +# It processes the following run time: +# +# | **Init:** 2021-07-03 12Z +# | **Forecast lead:** 36 hour +# +# Because instance names are used, GridStat will run 3 times for this 1 initalization time. + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads the default configuration file found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line: +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.conf + +############################################################################## +# MET Configuration +# ----------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped + +############################################################################## +# Python Embedding +# ---------------- +# +# This use case utilizes 1 Python script to read and process both forecast and +# observation fields. +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in model_applications/air_quality_and_comp/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac +# (relative to **OUTPUT_BASE**) +# and will contain the following files: +# +# * grid_stat_GFS_TO_MERRA2_F36_CloudFracs_360000L_20210705_000000V_pairs.nc +# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_360000L_20210705_000000V.stat +# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_NBR_360000L_20210705_000000V_pairs.nc +# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_NBR_360000L_20210705_000000V.stat +# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_PROB_360000L_20210705_000000V_pairs.nc +# * grid_stat_GFS_to_MERRA2_F36_CloudFracs_PROB_360000L_20210705_000000V.stat + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * GridStatToolUseCase +# * NetCDFFileUseCase +# * AirQualityAndCompAppUseCase +# * PythonEmbeddingFileUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# sphinx_gallery_thumbnail_path = '_static/air_quality_and_comp-GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.png' +# diff --git a/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.py b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.py new file mode 100644 index 0000000000..918e4a8f98 --- /dev/null +++ b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.py @@ -0,0 +1,137 @@ +""" +GridStat: Cloud Pressure and Temperature Heights +================================================ + +model_applications/air_quality_and_comp/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# This use case captures various statistical measures of two model comparisons +# for cloud top pressures and temperatures with different neighborhood +# settings for internal model metrics and to aid in future model updates +# + +############################################################################## +# Datasets +# -------- +# +# | **Forecast:** Global Forecast System (GFS) +# | **Observations:** Satellite ClOud and Radiation Property retrieval System (SatCORPS) +# | **Grid:** GPP 17km masking region +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword +# in the observation input template settings. The same Python script can processes both forecast and +# observation datasets, but only the observation dataset is not +# set up for native ingest by MET. Two separate forecast fields are verified against two respective observation fields, +# with the Python script being passed the input file, the model name, the variable name being analyzed, +# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. +# This process is repeated with 2 instance names to GridStat, each with a different setting for regridding, +# neighborhood evaluation, thresholding, output line types, and output prefix names. + +############################################################################## +# METplus Workflow +# ---------------- +# +# GridStat is the only MET tool called in this example. +# It processes the following run time: +# +# | **Init:** 2022-07-03 12Z +# | **Forecast lead:** 36 hour +# +# Because instance names are used, GridStat will run 2 times for this 1 initalization time. + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads the default configuration file found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line: +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.conf + +############################################################################## +# MET Configuration +# ----------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped + +############################################################################## +# Python Embedding +# ---------------- +# +# This use case utilizes 1 Python script to read and process the observation fields. +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp/read_input_data.py + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in model_applications/air_quality_and_comp/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp +# (relative to **OUTPUT_BASE**) +# and will contain the following files: +# +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V_pairs.nc +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V_ctc.txt +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V_cts.txt +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_360000L_20220705_000000V.stat +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_NBR_360000L_20220705_000000V_pairs.nc +# * grid_stat_GFS_to_SATCORPS_F36_CloudHgts_NBR_360000L_20220705_000000V.stat + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * GridStatToolUseCase +# * NetCDFFileUseCase +# * AirQualityAndCompAppUseCase +# * PythonEmbeddingFileUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# sphinx_gallery_thumbnail_path = '_static/air_quality_and_comp-GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.png' +# diff --git a/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.py b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.py new file mode 100644 index 0000000000..f50a0be14a --- /dev/null +++ b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.py @@ -0,0 +1,135 @@ +""" +GridStat: Cloud Height with Neighborhood and Probabilities +========================================================== + +model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# This use case captures various statistical measures of two model comparisons +# for cloud base height with different neighborhood settings for internal +# model metrics and to aid in future model updates +# + +############################################################################## +# Datasets +# -------- +# +# | **Forecast:** Model for Prediction Across Scales (MPAS) +# | **Observations:** ECMWF Reanalysis, Version 5 (ERA5) +# | **Grid:** GPP 17km masking region +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword +# in the forecast and observation input template settings. The same Python script processes both forecast and +# observation datasets. The forecast field is verified against the respective observation field, +# with the Python script being passed the input file, the model name, the variable name being analyzed, +# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. +# This process is repeated with 2 instance names to GridStat, each with a different setting for regridding, +# neighborhood evaluation, thresholding, output line types, and output prefix names. + +############################################################################## +# METplus Workflow +# ---------------- +# +# GridStat is the only MET tool called in this example. +# It processes the following run time: +# +# | **Init:** 2020-07-23 00Z +# | **Forecast lead:** 36 hour +# +# Because instance names are used, GridStat will run 2 times for this 1 initalization time. + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads the default configuration file found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line: +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.conf + +############################################################################## +# MET Configuration +# ----------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped + +############################################################################## +# Python Embedding +# ---------------- +# +# This use case utilizes 1 Python script to read and process both forecast and +# observation fields. +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsERA5_cloudBaseHgt/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsERA5_cloudBaseHgt/read_input_data.py + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsERA5_cloudBaseHgt +# (relative to **OUTPUT_BASE**) +# and will contain the following files: +# +# * grid_stat_MPAS_to_ERA5_F36_CloudBaseHgt_360000L_20200724_120000V_pairs.nc +# * grid_stat_MPAS_to_ERA5_F36_CloudBaseHgt_360000L_20200724_120000V.stat +# * grid_stat_MPAS_to_ERA5_F36_CloudBaseHgt_NBR_360000L_20200724_120000V_pairs.nc +# * grid_stat_MPAS_to_ERA5_F36_CloudBaseHgt_NBR_360000L_20200724_120000V.stat + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * GridStatToolUseCase +# * NetCDFFileUseCase +# * AirQualityAndCompAppUseCase +# * PythonEmbeddingFileUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# sphinx_gallery_thumbnail_path = '_static/air_quality_and_comp-GridStat_fcstMPAS_obsERA5_cloudBaseHgt.png' +# diff --git a/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.py b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.py new file mode 100644 index 0000000000..a3587f7a19 --- /dev/null +++ b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.py @@ -0,0 +1,137 @@ +""" +GridStat: Cloud Fractions with Neighborhood and Probabilities +============================================================= + +model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# This use case captures various statistical measures of two model comparisons +# for low and total cloud fraction with different neighborhood and probability +# settings for internal model metrics and to aid in future model updates +# + +############################################################################## +# Datasets +# -------- +# +# | **Forecast:** Model for Prediction Across Scales (MPAS) +# | **Observations:** Modern-Era Retrospective analysis for Research and Applications, Version 2 (MERRA2) +# | **Grid:** GPP 17km masking region +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword +# in the forecast and observation input template settings. The same Python script processes both forecast and +# observation datasets. Two separate forecast fields are verified against two respective observation fields, +# with the Python script being passed the input file, the model name, the variable name being analyzed, +# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. +# This process is repeated with 3 instance names to GridStat, each with a different setting for regridding, +# neighborhood evaluation, thresholding, output line types, and output prefix names. + +############################################################################## +# METplus Workflow +# ---------------- +# +# GridStat is the only MET tool called in this example. +# It processes the following run time: +# +# | **Init:** 2020-07-23 00Z +# | **Forecast lead:** 36 hour +# +# Because instance names are used, GridStat will run 3 times for this 1 initalization time. + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads the default configuration file found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line: +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.conf + +############################################################################## +# MET Configuration +# ----------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped + +############################################################################## +# Python Embedding +# ---------------- +# +# This use case utilizes 1 Python script to read and process both forecast and +# observation fields. +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac/read_input_data.py + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac +# (relative to **OUTPUT_BASE**) +# and will contain the following files: +# +# * grid_stat_MPAS_to_MERRA2_F36_CloudFracs_360000L_20200724_120000V_pairs.nc +# * grid_stat_MPAS_to_MERRA2_F36_CloudFracs_360000L_20200724_120000V.stat +# * grid_stat_MPAS_to_MERRA2_F36_CloudFracs_NBR_360000L_20200724_120000V_pairs.nc +# * grid_stat_MPAS_to_MERRA2_F36_CloudFracs_NBR_360000L_20200724_120000V.stat +# * grid_stat_MPAS_to_MERRA2_F36_CloudFracs_PROB_360000L_20200724_120000V_pairs.nc +# * grid_stat_MPAS_to_MERRA2_F36_CloudFracs_PROB_360000L_20200724_120000V.stat + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * GridStatToolUseCase +# * NetCDFFileUseCase +# * AirQualityAndCompAppUseCase +# * PythonEmbeddingFileUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# sphinx_gallery_thumbnail_path = '_static/air_quality_and_comp-GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.png' +# diff --git a/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.py b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.py new file mode 100644 index 0000000000..67dcb7e67e --- /dev/null +++ b/docs/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.py @@ -0,0 +1,137 @@ +""" +GridStat: Cloud Fractions with Neighborhood and Probabilities +============================================================= + +model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# This use case captures various statistical measures of two model comparisons +# for low and total cloud fraction with different neighborhood and probability +# settings for internal model metrics and to aid in future model updates +# + +############################################################################## +# Datasets +# -------- +# +# | **Forecast:** Model for Prediction Across Scales (MPAS) +# | **Observations:** Satellite ClOud and Radiation Property retrieval System (SatCORPS) +# | **Grid:** GPP 17km masking region +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes Python Embedding, which is called using the PYTHON_NUMPY keyword +# in the forecast and observation input template settings. The same Python script processes both forecast and +# observation datasets. Two separate forecast fields are verified against two respective observation fields, +# with the Python script being passed the input file, the model name, the variable name being analyzed, +# the initialization and valid times, and a flag to indicate if the field passed is observation or forecast. +# This process is repeated with 3 instance names to GridStat, each with a different setting for regridding, +# neighborhood evaluation, thresholding, output line types, and output prefix names. + +############################################################################## +# METplus Workflow +# ---------------- +# +# GridStat is the only MET tool called in this example. +# It processes the following run time: +# +# | **Init:** 2020-07-23 00Z +# | **Forecast lead:** 36 hour +# +# Because instance names are used, GridStat will run 3 times for this 1 initalization time. + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads the default configuration file found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line: +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.conf + +############################################################################## +# MET Configuration +# ----------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. note:: See the :ref:`GridStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/GridStatConfig_wrapped + +############################################################################## +# Python Embedding +# ---------------- +# +# This use case utilizes 1 Python script to read and process both forecast and +# observation fields. +# parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac/read_input_data.py +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac/read_input_data.py + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac +# (relative to **OUTPUT_BASE**) +# and will contain the following files: +# +# * grid_stat_MPAS_F36_CloudFracs_360000L_20200724_120000V_pairs.nc +# * grid_stat_MPAS_F36_CloudFracs_360000L_20200724_120000V.stat +# * grid_stat_MPAS_F36_CloudFracs_NBR_360000L_20200724_120000V_pairs.nc +# * grid_stat_MPAS_F36_CloudFracs_NBR_360000L_20200724_120000V.stat +# * grid_stat_MPAS_F36_CloudFracs_PROB_360000L_20200724_120000V_pairs.nc +# * grid_stat_MPAS_F36_CloudFracs_PROB_360000L_20200724_120000V.stat + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * GridStatToolUseCase +# * NetCDFFileUseCase +# * AirQualityAndCompAppUseCase +# * PythonEmbeddingFileUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# sphinx_gallery_thumbnail_path = '_static/air_quality_and_comp-GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.png' +# diff --git a/internal/scripts/docker_env/README.md b/internal/scripts/docker_env/README.md index 53a8eedbdc..80ff1ff7fb 100644 --- a/internal/scripts/docker_env/README.md +++ b/internal/scripts/docker_env/README.md @@ -403,7 +403,7 @@ cd /home/met_test/geovista /home/met_test/.conda/envs/geovista.${METPLUS_ENV_VERSION}/bin/pip3 install --no-deps --editable . ``` -## pandac.v5.1 (from py_embed_base.v5.1) +## pandac.v5.1 (from metplotpy.v5.1) ### Docker @@ -411,7 +411,7 @@ cd /home/met_test/geovista export METPLUS_ENV_VERSION=v5.1 docker build -t dtcenter/metplus-envs:pandac.${METPLUS_ENV_VERSION} \ --build-arg METPLUS_ENV_VERSION \ - --build-arg BASE_ENV=py_embed_base \ + --build-arg BASE_ENV=metplotpy \ --build-arg ENV_NAME=pandac . docker push dtcenter/metplus-envs:pandac.${METPLUS_ENV_VERSION} ``` diff --git a/internal/scripts/docker_env/scripts/pandac_env.sh b/internal/scripts/docker_env/scripts/pandac_env.sh index 70c81b5899..a17cd2400b 100755 --- a/internal/scripts/docker_env/scripts/pandac_env.sh +++ b/internal/scripts/docker_env/scripts/pandac_env.sh @@ -19,11 +19,25 @@ METPLUS_VERSION=$1 ENV_NAME=pandac.${METPLUS_VERSION} # Conda environment to use as base for new environment -BASE_ENV=py_embed_base.${METPLUS_VERSION} +#BASE_ENV=py_embed_base.${METPLUS_VERSION} +BASE_ENV=metplotpy.${METPLUS_VERSION} conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} -conda install -y --name ${ENV_NAME} -c conda-forge scipy==1.10.1 -conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.6.3 +#conda install -y --name ${ENV_NAME} -c conda-forge scipy==1.10.1 +#if [ $? != 0 ]; then +# echo install of scipy==1.10.1 failed +# exit 1 +#fi + conda install -y --name ${ENV_NAME} -c conda-forge pygrib==2.1.4 +if [ $? != 0 ]; then + echo install of pygrib==2.1.4 failed + exit 1 +fi +#conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.6.3 +#if [ $? != 0 ]; then +# echo install of matplotlib==3.6.3 failed +# exit 1 +#fi diff --git a/internal/tests/pytests/util/config/test_config.py b/internal/tests/pytests/util/config/test_config.py index be8f925fb9..164865e553 100644 --- a/internal/tests/pytests/util/config/test_config.py +++ b/internal/tests/pytests/util/config/test_config.py @@ -72,7 +72,7 @@ def test_getstr(metplus_config, input_value, default, result): ('/some/test/dir', None, '/some/test/dir'), ('/some//test/dir', None, '/some/test/dir'), ('/path/to', None, 'ValueError'), - (None, None, 'NoOptionError'), + (None, None, ''), (None, '/default/path', '/default/path'), ] @@ -83,11 +83,9 @@ def test_getdir(metplus_config, input_value, default, result): if input_value is not None: conf.set('config', 'TEST_GETDIR', input_value) - # catch NoOptionError exception and pass test if default is None + # catch ValueError and pass test if value is invalid and no default try: assert result == conf.getdir('TEST_GETDIR', default=default) - except NoOptionError: - assert result == 'NoOptionError' except ValueError: assert result == 'ValueError' diff --git a/internal/tests/pytests/util/config_metplus/test_config_metplus.py b/internal/tests/pytests/util/config_metplus/test_config_metplus.py index 8dda66dea1..ca6daf44f6 100644 --- a/internal/tests/pytests/util/config_metplus/test_config_metplus.py +++ b/internal/tests/pytests/util/config_metplus/test_config_metplus.py @@ -162,8 +162,8 @@ def test_find_var_indices_fcst(metplus_config, ) @pytest.mark.util def test_get_field_search_prefixes(data_type, met_tool, expected_out): - assert(config_metplus.get_field_search_prefixes(data_type, - met_tool) == expected_out) + assert(config_metplus._get_field_search_prefixes(data_type, + met_tool) == expected_out) # search prefixes are valid prefixes to append to field info variables diff --git a/internal/tests/pytests/util/string_manip/test_util_string_manip.py b/internal/tests/pytests/util/string_manip/test_util_string_manip.py index d53f19b469..38eb260440 100644 --- a/internal/tests/pytests/util/string_manip/test_util_string_manip.py +++ b/internal/tests/pytests/util/string_manip/test_util_string_manip.py @@ -8,6 +8,18 @@ from metplus.util.string_manip import * from metplus.util.string_manip import _fix_list +@pytest.mark.parametrize( + 'template, expected_output', [ + ('', ''), + ('{init?fmt=%Y%m%d}_i{init?fmt=%H%M}_m0_f*', + r'{init?fmt=%Y%m%d}_i{init?fmt=%H%M}_m0_f*'), + ('pgbf{lead?fmt=%HHH}.gfs.{init?fmt=%Y%m%d%H}', + r'pgbf.*\.gfs\.{init?fmt=%Y%m%d%H}'), + ] +) +@pytest.mark.util +def test_template_to_regex(template, expected_output): + assert template_to_regex(template) == expected_output @pytest.mark.parametrize( 'subset_definition, expected_result', [ diff --git a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py index 07ff4bb110..4781dfe073 100644 --- a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py +++ b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py @@ -329,6 +329,21 @@ def set_minimum_config_settings(config): ({'MODE_MASK_MISSING_FLAG': 'BOTH', }, {'METPLUS_MASK_MISSING_FLAG': 'mask_missing_flag = BOTH;'}), + ({'MODE_MULTIVAR_INTENSITY': 'false, true,true', }, + {'METPLUS_MULTIVAR_INTENSITY': 'multivar_intensity = [FALSE, TRUE, TRUE];'}), + + ({'MODE_FCST_MULTIVAR_NAME': 'Snow', }, + {'METPLUS_FCST_MULTIVAR_NAME': 'multivar_name = "Snow";'}), + + ({'MODE_FCST_MULTIVAR_UNITS': 'L0', }, + {'METPLUS_FCST_MULTIVAR_UNITS': 'multivar_units = "L0";'}), + + ({'MODE_OBS_MULTIVAR_NAME': 'Precip', }, + {'METPLUS_OBS_MULTIVAR_NAME': 'multivar_name = "Precip";'}), + + ({'MODE_OBS_MULTIVAR_UNITS': 'Z10', }, + {'METPLUS_OBS_MULTIVAR_UNITS': 'multivar_units = "Z10";'}), + ] ) @pytest.mark.wrapper_a diff --git a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py index 1ef38667f1..de609612ab 100644 --- a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py +++ b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py @@ -198,7 +198,7 @@ def test_valid_init_env_vars(metplus_config, config_overrides, runtime_settings_dict_list = wrapper._get_all_runtime_settings(time_input) assert runtime_settings_dict_list - first_runtime_only = [runtime_settings_dict_list[0]] + first_runtime_only = runtime_settings_dict_list[0] wrapper._run_stat_analysis_job(first_runtime_only) all_cmds = wrapper.all_commands diff --git a/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py new file mode 100644 index 0000000000..faddf14e20 --- /dev/null +++ b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 + +import pytest + +import os +from datetime import datetime + +from metplus.wrappers.tc_diag_wrapper import TCDiagWrapper + +deck_template = 'aal14{date?fmt=%Y}_short.dat' +input_template = 'gfs.subset.t00z.pgrb2.0p25.f*' +output_template = 'tc_diag_aal14{date?fmt=%Y}.nc' + +time_fmt = '%Y%m%d%H' +run_times = ['2016092900'] + +data_fmt = ( + 'field = [{ name="PRMSL"; level="L0"; },' + '{ name="TMP"; level="P1000"; },' + '{ name="TMP"; level="P900"; },' + '{ name="TMP"; level="P800"; },' + '{ name="TMP"; level="P700"; },' + '{ name="TMP"; level="P500"; },' + '{ name="TMP"; level="P100"; }];' +) + + +def get_data_dir(config): + return os.path.join(config.getdir('METPLUS_BASE'), + 'internal', 'tests', 'data', 'tc_pairs') + + +def set_minimum_config_settings(config): + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'TCDiag') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', time_fmt) + config.set('config', 'INIT_BEG', run_times[0]) + config.set('config', 'INIT_END', run_times[-1]) + config.set('config', 'INIT_INCREMENT', '6H') + config.set('config', 'TC_DIAG_CONFIG_FILE', + '{PARM_BASE}/met_config/TCDiagConfig_wrapped') + config.set('config', 'TC_DIAG_DECK_TEMPLATE', deck_template) + config.set('config', 'TC_DIAG_INPUT_TEMPLATE', input_template) + config.set('config', 'TC_DIAG_OUTPUT_DIR', + '{OUTPUT_BASE}/TCDiag/output') + config.set('config', 'TC_DIAG_OUTPUT_TEMPLATE', output_template) + + config.set('config', 'BOTH_VAR1_NAME', 'PRMSL') + config.set('config', 'BOTH_VAR1_LEVELS', 'L0') + config.set('config', 'BOTH_VAR2_NAME', 'TMP') + config.set('config', 'BOTH_VAR2_LEVELS', 'P1000, P900, P800, P700, P500, P100') + + +@pytest.mark.parametrize( + 'config_overrides, env_var_values', [ + ({}, {}), + ({'MODEL': 'GFSO, OFCL'}, {'METPLUS_MODEL': 'model = ["GFSO", "OFCL"];'}), + ({'TC_DIAG_STORM_ID': 'AL092022'}, {'METPLUS_STORM_ID': 'storm_id = "AL092022";'}), + ({'TC_DIAG_BASIN': 'AL'}, {'METPLUS_BASIN': 'basin = "AL";'}), + ({'TC_DIAG_CYCLONE': '14'}, {'METPLUS_CYCLONE': 'cyclone = "14";'}), + ({'TC_DIAG_INIT_INC': '20220924_00', }, {'METPLUS_INIT_INCLUDE': 'init_inc = "20220924_00";'}), + + ({'TC_DIAG_VALID_BEG': '20220924_00', }, {'METPLUS_VALID_BEG': 'valid_beg = "20220924_00";'}), + + ({'TC_DIAG_VALID_END': '20220924_00', }, {'METPLUS_VALID_END': 'valid_end = "20220924_00";'}), + + ({'TC_DIAG_VALID_INC': '20220924_00, 20220924_18', }, + {'METPLUS_VALID_INCLUDE_LIST': 'valid_inc = ["20220924_00", "20220924_18"];'}), + + ({'TC_DIAG_VALID_EXC': '20220924_00, 20220924_18', }, + {'METPLUS_VALID_EXCLUDE_LIST': 'valid_exc = ["20220924_00", "20220924_18"];'}), + + ({'TC_DIAG_VALID_HOUR': '12, 18', }, {'METPLUS_VALID_HOUR_LIST': 'valid_hour = ["12", "18"];'}), + + ({'LEAD_SEQ': '0,6,12,18,24', }, {'METPLUS_LEAD_LIST': 'lead = ["00", "06", "12", "18", "24"];'}), + + ({'TC_DIAG_DIAG_SCRIPT': 'MET_BASE/python/tc_diag/compute_tc_diagnostics.py', }, + {'METPLUS_DIAG_SCRIPT': 'diag_script = ["MET_BASE/python/tc_diag/compute_tc_diagnostics.py"];'}), + # domain_info 1 dictionary in list + ({'TC_DIAG_DOMAIN_INFO1_DOMAIN': 'parent', }, + {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{domain = "parent";}];'}), + + ({'TC_DIAG_DOMAIN_INFO1_N_RANGE': '150', }, + {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{n_range = 150;}];'}), + + ({'TC_DIAG_DOMAIN_INFO1_N_AZIMUTH': '8', }, + {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{n_azimuth = 8;}];'}), + + ({'TC_DIAG_DOMAIN_INFO1_DELTA_RANGE_KM': '10.0', }, + {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{delta_range_km = 10.0;}];'}), + ({'TC_DIAG_DOMAIN_INFO1_DIAG_SCRIPT': 'MET_BASE/python/tc_diag/compute_all_diagnostics.py,MET_BASE/python/tc_diag/compute_custom_diagnostics.py', }, + {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{diag_script = ["MET_BASE/python/tc_diag/compute_all_diagnostics.py", "MET_BASE/python/tc_diag/compute_custom_diagnostics.py"];}];'}), + ({'TC_DIAG_DOMAIN_INFO1_DOMAIN': 'parent', + 'TC_DIAG_DOMAIN_INFO1_N_RANGE': '150', + 'TC_DIAG_DOMAIN_INFO1_N_AZIMUTH': '8', + 'TC_DIAG_DOMAIN_INFO1_DELTA_RANGE_KM': '10.0', + 'TC_DIAG_DOMAIN_INFO1_DIAG_SCRIPT': 'MET_BASE/python/tc_diag/compute_all_diagnostics.py,MET_BASE/python/tc_diag/compute_custom_diagnostics.py', + }, + {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{domain = "parent";n_range = 150;n_azimuth = 8;delta_range_km = 10.0;diag_script = ["MET_BASE/python/tc_diag/compute_all_diagnostics.py", "MET_BASE/python/tc_diag/compute_custom_diagnostics.py"];}];'}), + # domain_info 2 dictionaries in list + ({'TC_DIAG_DOMAIN_INFO1_DOMAIN': 'parent', + 'TC_DIAG_DOMAIN_INFO1_N_RANGE': '150', + 'TC_DIAG_DOMAIN_INFO1_N_AZIMUTH': '8', + 'TC_DIAG_DOMAIN_INFO1_DELTA_RANGE_KM': '10.0', + 'TC_DIAG_DOMAIN_INFO1_DIAG_SCRIPT': 'MET_BASE/python/tc_diag/compute_all_diagnostics.py,MET_BASE/python/tc_diag/compute_custom_diagnostics.py', + 'TC_DIAG_DOMAIN_INFO2_DOMAIN': 'nest', + 'TC_DIAG_DOMAIN_INFO2_N_RANGE': '100', + 'TC_DIAG_DOMAIN_INFO2_N_AZIMUTH': '7', + 'TC_DIAG_DOMAIN_INFO2_DELTA_RANGE_KM': '12.0', + 'TC_DIAG_DOMAIN_INFO2_DIAG_SCRIPT': 'MET_BASE/python/tc_diag/compute_sst_diagnostics.py', + }, + {'METPLUS_DOMAIN_INFO_LIST': 'domain_info = [{domain = "parent";n_range = 150;n_azimuth = 8;delta_range_km = 10.0;diag_script = ["MET_BASE/python/tc_diag/compute_all_diagnostics.py", "MET_BASE/python/tc_diag/compute_custom_diagnostics.py"];},{domain = "nest";n_range = 100;n_azimuth = 7;delta_range_km = 12.0;diag_script = ["MET_BASE/python/tc_diag/compute_sst_diagnostics.py"];}];'}), + + ({'TC_DIAG_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_CENSOR_THRESH': 'censor_thresh = [>12000, <5000];'}), + + ({'TC_DIAG_CENSOR_VAL': '12000,5000', }, + {'METPLUS_CENSOR_VAL': 'censor_val = [12000, 5000];'}), + + ({'TC_DIAG_CONVERT': '2*x', }, + {'METPLUS_CONVERT': 'convert(x) = 2*x;'}), + + ({'TC_DIAG_DATA_DOMAIN': 'parent,nest', }, + {'METPLUS_DATA_DOMAIN': 'domain = ["parent", "nest"];'}), + + ({'TC_DIAG_DATA_LEVEL': 'P500, P850', }, + {'METPLUS_DATA_LEVEL': 'level = ["P500", "P850"];'}), + + ({'TC_DIAG_REGRID_METHOD': 'NEAREST',}, + {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), + + ({'TC_DIAG_REGRID_WIDTH': '1',}, + {'METPLUS_REGRID_DICT': 'regrid = {width = 1;}'}), + + ({'TC_DIAG_REGRID_VLD_THRESH': '0.5',}, + {'METPLUS_REGRID_DICT': 'regrid = {vld_thresh = 0.5;}'}), + + ({'TC_DIAG_REGRID_SHAPE': 'SQUARE',}, + {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + + ({'TC_DIAG_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'TC_DIAG_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'TC_DIAG_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + + ({'TC_DIAG_REGRID_METHOD': 'NEAREST', + 'TC_DIAG_REGRID_WIDTH': '1', + 'TC_DIAG_REGRID_VLD_THRESH': '0.5', + 'TC_DIAG_REGRID_SHAPE': 'SQUARE', + 'TC_DIAG_REGRID_CONVERT': '2*x', + 'TC_DIAG_REGRID_CENSOR_THRESH': '>12000,<5000', + 'TC_DIAG_REGRID_CENSOR_VAL': '12000,5000', + }, + {'METPLUS_REGRID_DICT': ('regrid = {method = NEAREST;' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' + ) + }), + + ({'TC_DIAG_COMPUTE_TANGENTIAL_AND_RADIAL_WINDS': 'true', }, + {'METPLUS_COMPUTE_TANGENTIAL_AND_RADIAL_WINDS': 'compute_tangential_and_radial_winds = TRUE;'}), + + ({'TC_DIAG_U_WIND_FIELD_NAME': 'UGRD', }, + {'METPLUS_U_WIND_FIELD_NAME': 'u_wind_field_name = "UGRD";'}), + + ({'TC_DIAG_V_WIND_FIELD_NAME': 'VGRD', }, + {'METPLUS_V_WIND_FIELD_NAME': 'v_wind_field_name = "VGRD";'}), + + ({'TC_DIAG_TANGENTIAL_VELOCITY_FIELD_NAME': 'VT', }, + {'METPLUS_TANGENTIAL_VELOCITY_FIELD_NAME': 'tangential_velocity_field_name = "VT";'}), + + ({'TC_DIAG_TANGENTIAL_VELOCITY_LONG_FIELD_NAME': 'Tangential Velocity', }, + {'METPLUS_TANGENTIAL_VELOCITY_LONG_FIELD_NAME': 'tangential_velocity_long_field_name = "Tangential Velocity";'}), + + ({'TC_DIAG_RADIAL_VELOCITY_FIELD_NAME': 'VR', }, + {'METPLUS_RADIAL_VELOCITY_FIELD_NAME': 'radial_velocity_field_name = "VR";'}), + + ({'TC_DIAG_RADIAL_VELOCITY_LONG_FIELD_NAME': 'Radial Velocity', }, + {'METPLUS_RADIAL_VELOCITY_LONG_FIELD_NAME': 'radial_velocity_long_field_name = "Radial Velocity";'}), + + ({'TC_DIAG_VORTEX_REMOVAL': 'False', }, {'METPLUS_VORTEX_REMOVAL': 'vortex_removal = FALSE;'}), + + ({'TC_DIAG_NC_RNG_AZI_FLAG': 'true', }, {'METPLUS_NC_RNG_AZI_FLAG': 'nc_rng_azi_flag = TRUE;'}), + + ({'TC_DIAG_NC_DIAG_FLAG': 'true', }, {'METPLUS_NC_DIAG_FLAG': 'nc_diag_flag = TRUE;'}), + + ({'TC_DIAG_CIRA_DIAG_FLAG': 'True', }, {'METPLUS_CIRA_DIAG_FLAG': 'cira_diag_flag = TRUE;'}), + + ({'TC_DIAG_OUTPUT_PREFIX': 'my_prefix', }, {'METPLUS_OUTPUT_PREFIX': 'output_prefix = "my_prefix";'}), + + ] +) +@pytest.mark.wrapper +def test_tc_diag_run(metplus_config, config_overrides, + env_var_values): + config = metplus_config + + set_minimum_config_settings(config) + + test_data_dir = get_data_dir(config) + deck_dir = os.path.join(test_data_dir, 'bdeck') + + config.set('config', 'TC_DIAG_DECK_INPUT_DIR', deck_dir) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = TCDiagWrapper(config) + assert wrapper.isOK + + file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR') + file_list_file = f"{file_list_dir}/aal142016_short.dat_data_files.txt" + if os.path.exists(file_list_file): + os.remove(file_list_file) + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + + expected_cmds = [(f"{app_path} " + f"-deck {deck_dir}/aal142016_short.dat " + f"-data {file_list_file} " + f"-config {config_file} " + f"-out {out_dir}/tc_diag_aal142016.nc " + f"{verbosity}"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + assert len(all_cmds) == len(expected_cmds) + + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd + + # check that environment variables were set properly + for env_var_key in env_var_keys: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + print(f'Checking env var: {env_var_key}') + actual_value = match.split('=', 1)[1] + if env_var_key == 'METPLUS_DATA_FIELD': + assert actual_value == data_fmt + else: + assert env_var_values.get(env_var_key, '') == actual_value + + +@pytest.mark.wrapper +def test_get_config_file(metplus_config): + fake_config_name = '/my/config/file' + + config = metplus_config + config.set('config', 'INIT_TIME_FMT', time_fmt) + config.set('config', 'INIT_BEG', run_times[0]) + default_config_file = os.path.join(config.getdir('PARM_BASE'), + 'met_config', + 'TCDiagConfig_wrapped') + + wrapper = TCDiagWrapper(config) + assert wrapper.c_dict['CONFIG_FILE'] == default_config_file + + config.set('config', 'TC_DIAG_CONFIG_FILE', fake_config_name) + wrapper = TCDiagWrapper(config) + assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index 62ad8b0863..8e901c141f 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -78,6 +78,7 @@ def test_read_storm_info(metplus_config, config_overrides, isOK): wrapper = TCPairsWrapper(config) assert wrapper.isOK == isOK + @pytest.mark.parametrize( 'storm_id,basin,cyclone', [ ('AL092011', 'al', '09'), @@ -540,6 +541,23 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, 'field_source = "GFS_0p50";match_to_track = ["OFCL"]' ';diag_name = ["MY_NAME2"];}];' )}), + # -diag arguments with multiple models + ('VALID', { + 'MODEL': 'MYNN, H19C', + 'TC_PAIRS_DIAG_TEMPLATE1': '/some/path/{valid?fmt=%Y%m%d%H}_{model}.dat', + 'TC_PAIRS_DIAG_SOURCE1': 'TCDIAG', + 'TC_PAIRS_DIAG_TEMPLATE2': '/some/path/rt_{valid?fmt=%Y%m%d%H}.dat', + 'TC_PAIRS_DIAG_SOURCE2': 'LSDIAG_RT', + }, + {'DIAG_ARG': ('-diag TCDIAG /some/path/2014121318_H19C.dat /some/path/2014121318_MYNN.dat ' + '-diag LSDIAG_RT /some/path/rt_2014121318.dat'), + 'METPLUS_MODEL': 'model = ["MYNN", "H19C"];'}), + # -diag arguments with wildcard model + ('VALID', { + 'TC_PAIRS_DIAG_TEMPLATE1': 'bmlq2014123118.{model}.0104', + 'TC_PAIRS_DIAG_SOURCE1': 'TCDIAG', + }, + {'DIAG_ARG': '-diag TCDIAG /bmlq2014123118.gfso.0104',}), ] ) @pytest.mark.wrapper @@ -556,6 +574,8 @@ def test_tc_pairs_run(metplus_config, loop_by, config_overrides, config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir) + if 'DIAG_ARG' in env_var_values and 'TC_PAIRS_DIAG_DIR1' not in config_overrides: + config.set('config', 'TC_PAIRS_DIAG_DIR1', bdeck_dir) # set config variable overrides for key, value in config_overrides.items(): @@ -589,6 +609,7 @@ def test_tc_pairs_run(metplus_config, loop_by, config_overrides, diag_arg = '' if 'DIAG_ARG' in env_var_values: diag_arg = f" {env_var_values['DIAG_ARG']}" + diag_arg = diag_arg.replace('', bdeck_dir) expected_cmds = [(f"{app_path} {verbosity} " f"-bdeck {bdeck_dir}/bmlq2014123118.gfso.0104 " diff --git a/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py b/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py new file mode 100644 index 0000000000..a1bfe04064 --- /dev/null +++ b/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 + +import pytest + +import os +from datetime import datetime + +from metplus.wrappers.tcrmw_wrapper import TCRMWWrapper + +deck_template = 'aal14{date?fmt=%Y}_short.dat' +input_template = 'gfs.subset.t00z.pgrb2.0p25.f*' +output_template = 'tc_rmw_aal14{date?fmt=%Y}.nc' + +time_fmt = '%Y%m%d%H' +run_times = ['2016092900'] + +data_fmt = ( + 'field = [{ name="PRMSL"; level="L0"; },' + '{ name="TMP"; level="P1000"; },' + '{ name="TMP"; level="P900"; },' + '{ name="TMP"; level="P800"; },' + '{ name="TMP"; level="P700"; },' + '{ name="TMP"; level="P500"; },' + '{ name="TMP"; level="P100"; }];' +) + + +def get_data_dir(config): + return os.path.join(config.getdir('METPLUS_BASE'), + 'internal', 'tests', 'data', 'tc_pairs') + + +def set_minimum_config_settings(config): + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'TCRMW') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', time_fmt) + config.set('config', 'INIT_BEG', run_times[0]) + config.set('config', 'INIT_END', run_times[-1]) + config.set('config', 'INIT_INCREMENT', '6H') + config.set('config', 'TC_RMW_CONFIG_FILE', + '{PARM_BASE}/met_config/TCRMWConfig_wrapped') + config.set('config', 'TC_RMW_DECK_TEMPLATE', deck_template) + config.set('config', 'TC_RMW_INPUT_TEMPLATE', input_template) + config.set('config', 'TC_RMW_OUTPUT_DIR', + '{OUTPUT_BASE}/TCRMW/output') + config.set('config', 'TC_RMW_OUTPUT_TEMPLATE', output_template) + + config.set('config', 'BOTH_VAR1_NAME', 'PRMSL') + config.set('config', 'BOTH_VAR1_LEVELS', 'L0') + config.set('config', 'BOTH_VAR2_NAME', 'TMP') + config.set('config', 'BOTH_VAR2_LEVELS', 'P1000, P900, P800, P700, P500, P100') + + +@pytest.mark.parametrize( + 'config_overrides, env_var_values', [ + ({}, {}), + ({'MODEL': 'GFSO'}, {'METPLUS_MODEL': 'model = "GFSO";'}), + ({'TC_RMW_STORM_ID': 'AL092022'}, {'METPLUS_STORM_ID': 'storm_id = "AL092022";'}), + ({'TC_RMW_BASIN': 'AL'}, {'METPLUS_BASIN': 'basin = "AL";'}), + ({'TC_RMW_CYCLONE': '14'}, {'METPLUS_CYCLONE': 'cyclone = "14";'}), + ({'TC_RMW_INIT_INC': '20220924_00', }, {'METPLUS_INIT_INCLUDE': 'init_inc = "20220924_00";'}), + + ({'TC_RMW_VALID_BEG': '20220924_00', }, {'METPLUS_VALID_BEG': 'valid_beg = "20220924_00";'}), + + ({'TC_RMW_VALID_END': '20220924_00', }, {'METPLUS_VALID_END': 'valid_end = "20220924_00";'}), + + ({'TC_RMW_VALID_INC': '20220924_00, 20220924_18', }, + {'METPLUS_VALID_INCLUDE_LIST': 'valid_inc = ["20220924_00", "20220924_18"];'}), + + ({'TC_RMW_VALID_EXC': '20220924_00, 20220924_18', }, + {'METPLUS_VALID_EXCLUDE_LIST': 'valid_exc = ["20220924_00", "20220924_18"];'}), + + ({'TC_RMW_VALID_HOUR': '12, 18', }, {'METPLUS_VALID_HOUR_LIST': 'valid_hour = ["12", "18"];'}), + + ({'LEAD_SEQ': '0,6,12,18,24', }, {'METPLUS_LEAD_LIST': 'lead = ["00", "06", "12", "18", "24"];'}), + + ({'TC_RMW_REGRID_METHOD': 'NEAREST',}, + {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), + + ({'TC_RMW_REGRID_WIDTH': '1',}, + {'METPLUS_REGRID_DICT': 'regrid = {width = 1;}'}), + + ({'TC_RMW_REGRID_VLD_THRESH': '0.5',}, + {'METPLUS_REGRID_DICT': 'regrid = {vld_thresh = 0.5;}'}), + + ({'TC_RMW_REGRID_SHAPE': 'SQUARE',}, + {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + + ({'TC_RMW_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'TC_RMW_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'TC_RMW_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + + ({'TC_RMW_REGRID_METHOD': 'NEAREST', + 'TC_RMW_REGRID_WIDTH': '1', + 'TC_RMW_REGRID_VLD_THRESH': '0.5', + 'TC_RMW_REGRID_SHAPE': 'SQUARE', + 'TC_RMW_REGRID_CONVERT': '2*x', + 'TC_RMW_REGRID_CENSOR_THRESH': '>12000,<5000', + 'TC_RMW_REGRID_CENSOR_VAL': '12000,5000', + }, + {'METPLUS_REGRID_DICT': ('regrid = {method = NEAREST;' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' + ) + }), + ({'TC_RMW_N_RANGE': '10', }, + {'METPLUS_N_RANGE': 'n_range = 10;'}), + + ({'TC_RMW_N_AZIMUTH': '12', }, + {'METPLUS_N_AZIMUTH': 'n_azimuth = 12;'}), + + ({'TC_RMW_MAX_RANGE_KM': '13', }, + {'METPLUS_MAX_RANGE_KM': 'max_range_km = 13.0;'}), + + ({'TC_RMW_DELTA_RANGE_KM': '14', }, + {'METPLUS_DELTA_RANGE_KM': 'delta_range_km = 14.0;'}), + + ({'TC_RMW_RMW_SCALE': '15', }, + {'METPLUS_RMW_SCALE': 'rmw_scale = 15.0;'}), + + ] +) +@pytest.mark.wrapper +def test_tc_rmw_run(metplus_config, config_overrides, + env_var_values): + config = metplus_config + + set_minimum_config_settings(config) + + test_data_dir = get_data_dir(config) + deck_dir = os.path.join(test_data_dir, 'bdeck') + + config.set('config', 'TC_RMW_DECK_INPUT_DIR', deck_dir) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = TCRMWWrapper(config) + assert wrapper.isOK + + file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR') + file_list_file = f"{file_list_dir}/aal142016_short.dat_data_files.txt" + if os.path.exists(file_list_file): + os.remove(file_list_file) + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + + expected_cmds = [(f"{app_path} " + f"-adeck {deck_dir}/aal142016_short.dat " + f"-data {file_list_file} " + f"-config {config_file} " + f"-out {out_dir}/tc_rmw_aal142016.nc " + f"{verbosity}"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + assert len(all_cmds) == len(expected_cmds) + + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd + + # check that environment variables were set properly + for env_var_key in env_var_keys: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + print(f'Checking env var: {env_var_key}') + actual_value = match.split('=', 1)[1] + if env_var_key == 'METPLUS_DATA_FIELD': + assert actual_value == data_fmt + else: + assert env_var_values.get(env_var_key, '') == actual_value + + +@pytest.mark.wrapper +def test_get_config_file(metplus_config): + fake_config_name = '/my/config/file' + + config = metplus_config + config.set('config', 'INIT_TIME_FMT', time_fmt) + config.set('config', 'INIT_BEG', run_times[0]) + default_config_file = os.path.join(config.getdir('PARM_BASE'), + 'met_config', + 'TCRMWConfig_wrapped') + + wrapper = TCRMWWrapper(config) + assert wrapper.c_dict['CONFIG_FILE'] == default_config_file + + config.set('config', 'TC_RMW_CONFIG_FILE', fake_config_name) + wrapper = TCRMWWrapper(config) + assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name diff --git a/internal/tests/use_cases/all_use_cases.txt b/internal/tests/use_cases/all_use_cases.txt index 82d761688a..16325b6bf1 100644 --- a/internal/tests/use_cases/all_use_cases.txt +++ b/internal/tests/use_cases/all_use_cases.txt @@ -61,9 +61,16 @@ Category: met_tool_wrapper 59::IODA2NC::met_tool_wrapper/IODA2NC/IODA2NC.conf 60::PointStat_python_embedding_obs:: met_tool_wrapper/PointStat/PointStat_python_embedding_obs.conf 61::PlotPointObs:: met_tool_wrapper/PlotPointObs/PlotPointObs.conf +#62::TCDiag:: met_tool_wrapper/TCDiag/TCDiag.conf Category: air_quality_and_comp 0::EnsembleStat_fcstICAP_obsMODIS_aod::model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf +1::GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac::model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsSATCORPS_lowAndTotalCloudFrac.conf:: pandac_env, py_embed +2::GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp::model_applications/air_quality_and_comp/GridStat_fcstGFS_obsSATCORPS_cloudTopPressAndTemp.conf:: pandac_env, py_embed +3::GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac::model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsMERRA2_lowAndTotalCloudFrac.conf:: pandac_env, py_embed +4::GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac::model_applications/air_quality_and_comp/GridStat_fcstGFS_obsMERRA2_lowAndTotalCloudFrac.conf:: pandac_env, py_embed +5::GridStat_fcstMPAS_obsERA5_cloudBaseHgt::model_applications/air_quality_and_comp/GridStat_fcstMPAS_obsERA5_cloudBaseHgt.conf:: pandac_env, py_embed +6::GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac::model_applications/air_quality_and_comp/GridStat_fcstGFS_obsERA5_lowAndTotalCloudFrac.conf:: pandac_env, py_embed Category: climate diff --git a/metplus/VERSION b/metplus/VERSION index bfea8475c1..f8ee15fd4c 100644 --- a/metplus/VERSION +++ b/metplus/VERSION @@ -1 +1 @@ -5.1.0-rc1-dev +5.1.0-dev diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index 61d0cad416..f90aeed939 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -160,7 +160,7 @@ def _parse_launch_args(args): bad = False for arg in args: m = re.match(r'''(?x) - (?P
[a-zA-Z][a-zA-Z0-9_]*) + (?P
[a-zA-Z]\w*) \.(?P