diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4e6e3872b7..b0f3ef9c7e 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -301,7 +301,7 @@ jobs: matrix: include: - jobid: 'job1' - tests: 'point_stat stat_analysis_ps' + tests: 'point_stat stat_analysis_ps pair_stat' - jobid: 'job2' tests: 'grid_stat stat_analysis_gs' - jobid: 'job3' diff --git a/Make-include b/Make-include index 6b88767ffb..3fb84f12e5 100644 --- a/Make-include +++ b/Make-include @@ -22,6 +22,7 @@ MET_CPPFLAGS = -I${top_builddir}/src/basic/vx_cal \ -I${top_builddir}/src/libcode/vx_gnomon \ -I${top_builddir}/src/libcode/vx_grid \ -I${top_builddir}/src/libcode/vx_gsl_prob \ + -I${top_builddir}/src/libcode/vx_ioda \ -I${top_builddir}/src/libcode/vx_nav \ -I${top_builddir}/src/libcode/vx_nc_obs \ -I${top_builddir}/src/libcode/vx_nc_util \ @@ -69,6 +70,7 @@ MET_LDFLAGS = -L${top_builddir}/src/basic/vx_cal \ -L${top_builddir}/src/libcode/vx_gnomon \ -L${top_builddir}/src/libcode/vx_grid \ -L${top_builddir}/src/libcode/vx_gsl_prob \ + -L${top_builddir}/src/libcode/vx_ioda \ -L${top_builddir}/src/libcode/vx_nav \ -L${top_builddir}/src/libcode/vx_nc_obs \ -L${top_builddir}/src/libcode/vx_nc_util \ diff --git a/config.h.in b/config.h.in index 931b16f6f2..e66bc1d535 100644 --- a/config.h.in +++ b/config.h.in @@ -48,6 +48,9 @@ /* "build modis" */ #undef ENABLE_MODIS +/* "build pair_stat" */ +#undef ENABLE_PAIR_STAT + /* "build pb2nc" */ #undef ENABLE_PB2NC diff --git a/configure b/configure index 758f3502f6..19da79f5f4 100755 --- a/configure +++ b/configure @@ -711,6 +711,8 @@ ENABLE_WAVELET_STAT_FALSE ENABLE_WAVELET_STAT_TRUE ENABLE_STAT_ANALYSIS_FALSE ENABLE_STAT_ANALYSIS_TRUE +ENABLE_PAIR_STAT_FALSE +ENABLE_PAIR_STAT_TRUE ENABLE_POINT_STAT_FALSE ENABLE_POINT_STAT_TRUE ENABLE_PLOT_POINT_OBS_FALSE @@ -913,6 +915,7 @@ enable_point2grid enable_shift_data_plane enable_plot_point_obs enable_point_stat +enable_pair_stat enable_stat_analysis enable_wavelet_stat enable_series_analysis @@ -1645,6 +1648,7 @@ Optional Features: --disable-plot_point_obs Disable compilation of plot_point_obs --disable-point_stat Disable compilation of point_stat + --disable-pair_stat Disable compilation of pair_stat --disable-stat_analysis Disable compilation of stat_analysis --disable-wavelet_stat Disable compilation of wavelet_stat --disable-series_analysis @@ -6538,6 +6542,41 @@ else printf "%s\n" "$as_me: point_stat will not be compiled" >&6;} fi +# pair_stat + +# Check whether --enable-pair_stat was given. +if test ${enable_pair_stat+y} +then : + enableval=$enable_pair_stat; case "${enableval}" in + yes | no ) ENABLE_PAIR_STAT="${enableval}" ;; + *) as_fn_error $? "bad value ${enableval} for --disable-pair_stat" "$LINENO" 5 ;; + esac +else $as_nop + ENABLE_PAIR_STAT="yes" + +fi + + + if test "x$ENABLE_PAIR_STAT" = "xyes"; then + ENABLE_PAIR_STAT_TRUE= + ENABLE_PAIR_STAT_FALSE='#' +else + ENABLE_PAIR_STAT_TRUE='#' + ENABLE_PAIR_STAT_FALSE= +fi + + +if test "x$ENABLE_PAIR_STAT" = "xyes"; then + +printf "%s\n" "#define ENABLE_PAIR_STAT /**/" >>confdefs.h + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: pair_stat will be compiled" >&5 +printf "%s\n" "$as_me: pair_stat will be compiled" >&6;} +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: pair_stat will not be compiled" >&5 +printf "%s\n" "$as_me: pair_stat will not be compiled" >&6;} +fi + # stat_analysis # Check whether --enable-stat_analysis was given. @@ -10158,7 +10197,7 @@ fi # Create configured files -ac_config_files="$ac_config_files Makefile scripts/Rscripts/Makefile scripts/Rscripts/include/Makefile scripts/python/Makefile scripts/python/examples/Makefile scripts/python/met/Makefile scripts/python/pyembed/Makefile scripts/python/utility/Makefile scripts/python/tc_diag/Makefile scripts/python/tc_diag/atcf_tools/Makefile scripts/python/tc_diag/config/Makefile scripts/python/tc_diag/diag_lib/Makefile scripts/python/tc_diag/tc_diag_driver/Makefile data/Makefile data/climo/Makefile data/climo/seeps/Makefile data/colortables/Makefile data/colortables/NCL_colortables/Makefile data/config/Makefile data/map/Makefile data/map/admin_by_country/Makefile data/poly/Makefile data/poly/HMT_masks/Makefile data/poly/NCEP_masks/Makefile data/ps/Makefile data/table_files/Makefile data/tc_data/Makefile src/Makefile src/basic/Makefile src/basic/enum_to_string/Makefile src/basic/vx_cal/Makefile src/basic/vx_config/Makefile src/basic/vx_log/Makefile src/basic/vx_math/Makefile src/basic/vx_util/Makefile src/basic/vx_util_math/Makefile src/libcode/Makefile src/libcode/vx_afm/Makefile src/libcode/vx_analysis_util/Makefile src/libcode/vx_color/Makefile src/libcode/vx_data2d/Makefile src/libcode/vx_data2d_factory/Makefile src/libcode/vx_data2d_grib/Makefile src/libcode/vx_data2d_grib2/Makefile src/libcode/vx_data2d_nc_met/Makefile src/libcode/vx_data2d_nc_wrf/Makefile src/libcode/vx_data2d_nc_cf/Makefile src/libcode/vx_data2d_ugrid/Makefile src/libcode/vx_geodesy/Makefile src/libcode/vx_gis/Makefile src/libcode/vx_gnomon/Makefile src/libcode/vx_grid/Makefile src/libcode/vx_gsl_prob/Makefile src/libcode/vx_nav/Makefile src/libcode/vx_solar/Makefile src/libcode/vx_nc_obs/Makefile src/libcode/vx_nc_util/Makefile src/libcode/vx_pb_util/Makefile src/libcode/vx_plot_util/Makefile src/libcode/vx_ps/Makefile src/libcode/vx_pxm/Makefile src/libcode/vx_render/Makefile src/libcode/vx_shapedata/Makefile src/libcode/vx_stat_out/Makefile src/libcode/vx_statistics/Makefile src/libcode/vx_time_series/Makefile src/libcode/vx_physics/Makefile src/libcode/vx_series_data/Makefile src/libcode/vx_regrid/Makefile src/libcode/vx_tc_util/Makefile src/libcode/vx_summary/Makefile src/libcode/vx_python3_utils/Makefile src/libcode/vx_data2d_python/Makefile src/libcode/vx_bool_calc/Makefile src/libcode/vx_pointdata_python/Makefile src/libcode/vx_seeps/Makefile src/tools/Makefile src/tools/core/Makefile src/tools/core/ensemble_stat/Makefile src/tools/core/grid_stat/Makefile src/tools/core/mode/Makefile src/tools/core/mode_analysis/Makefile src/tools/core/pcp_combine/Makefile src/tools/core/point_stat/Makefile src/tools/core/series_analysis/Makefile src/tools/core/stat_analysis/Makefile src/tools/core/wavelet_stat/Makefile src/tools/other/Makefile src/tools/other/ascii2nc/Makefile src/tools/other/lidar2nc/Makefile src/tools/other/gen_ens_prod/Makefile src/tools/other/gen_vx_mask/Makefile src/tools/other/gis_utils/Makefile src/tools/other/ioda2nc/Makefile src/tools/other/madis2nc/Makefile src/tools/other/mode_graphics/Makefile src/tools/other/modis_regrid/Makefile src/tools/other/pb2nc/Makefile src/tools/other/plot_data_plane/Makefile src/tools/other/plot_point_obs/Makefile src/tools/other/wwmca_tool/Makefile src/tools/other/gsi_tools/Makefile src/tools/other/regrid_data_plane/Makefile src/tools/other/point2grid/Makefile src/tools/other/shift_data_plane/Makefile src/tools/other/mode_time_domain/Makefile src/tools/other/grid_diag/Makefile src/tools/tc_utils/Makefile src/tools/tc_utils/tc_dland/Makefile src/tools/tc_utils/tc_pairs/Makefile src/tools/tc_utils/tc_stat/Makefile src/tools/tc_utils/tc_gen/Makefile src/tools/tc_utils/rmw_analysis/Makefile src/tools/tc_utils/tc_rmw/Makefile src/tools/tc_utils/tc_diag/Makefile" +ac_config_files="$ac_config_files Makefile scripts/Rscripts/Makefile scripts/Rscripts/include/Makefile scripts/python/Makefile scripts/python/examples/Makefile scripts/python/met/Makefile scripts/python/pyembed/Makefile scripts/python/utility/Makefile scripts/python/tc_diag/Makefile scripts/python/tc_diag/atcf_tools/Makefile scripts/python/tc_diag/config/Makefile scripts/python/tc_diag/diag_lib/Makefile scripts/python/tc_diag/tc_diag_driver/Makefile data/Makefile data/climo/Makefile data/climo/seeps/Makefile data/colortables/Makefile data/colortables/NCL_colortables/Makefile data/config/Makefile data/map/Makefile data/map/admin_by_country/Makefile data/poly/Makefile data/poly/HMT_masks/Makefile data/poly/NCEP_masks/Makefile data/ps/Makefile data/table_files/Makefile data/tc_data/Makefile src/Makefile src/basic/Makefile src/basic/enum_to_string/Makefile src/basic/vx_cal/Makefile src/basic/vx_config/Makefile src/basic/vx_log/Makefile src/basic/vx_math/Makefile src/basic/vx_util/Makefile src/basic/vx_util_math/Makefile src/libcode/Makefile src/libcode/vx_afm/Makefile src/libcode/vx_analysis_util/Makefile src/libcode/vx_color/Makefile src/libcode/vx_data2d/Makefile src/libcode/vx_data2d_factory/Makefile src/libcode/vx_data2d_grib/Makefile src/libcode/vx_data2d_grib2/Makefile src/libcode/vx_data2d_nc_met/Makefile src/libcode/vx_data2d_nc_wrf/Makefile src/libcode/vx_data2d_nc_cf/Makefile src/libcode/vx_data2d_ugrid/Makefile src/libcode/vx_geodesy/Makefile src/libcode/vx_gis/Makefile src/libcode/vx_gnomon/Makefile src/libcode/vx_grid/Makefile src/libcode/vx_gsl_prob/Makefile src/libcode/vx_nav/Makefile src/libcode/vx_solar/Makefile src/libcode/vx_nc_obs/Makefile src/libcode/vx_nc_util/Makefile src/libcode/vx_pb_util/Makefile src/libcode/vx_plot_util/Makefile src/libcode/vx_ps/Makefile src/libcode/vx_pxm/Makefile src/libcode/vx_render/Makefile src/libcode/vx_shapedata/Makefile src/libcode/vx_stat_out/Makefile src/libcode/vx_statistics/Makefile src/libcode/vx_time_series/Makefile src/libcode/vx_physics/Makefile src/libcode/vx_series_data/Makefile src/libcode/vx_regrid/Makefile src/libcode/vx_tc_util/Makefile src/libcode/vx_summary/Makefile src/libcode/vx_python3_utils/Makefile src/libcode/vx_data2d_python/Makefile src/libcode/vx_bool_calc/Makefile src/libcode/vx_pointdata_python/Makefile src/libcode/vx_seeps/Makefile src/libcode/vx_ioda/Makefile src/tools/Makefile src/tools/core/Makefile src/tools/core/ensemble_stat/Makefile src/tools/core/grid_stat/Makefile src/tools/core/mode/Makefile src/tools/core/mode_analysis/Makefile src/tools/core/pcp_combine/Makefile src/tools/core/point_stat/Makefile src/tools/core/pair_stat/Makefile src/tools/core/series_analysis/Makefile src/tools/core/stat_analysis/Makefile src/tools/core/wavelet_stat/Makefile src/tools/other/Makefile src/tools/other/ascii2nc/Makefile src/tools/other/lidar2nc/Makefile src/tools/other/gen_ens_prod/Makefile src/tools/other/gen_vx_mask/Makefile src/tools/other/gis_utils/Makefile src/tools/other/ioda2nc/Makefile src/tools/other/madis2nc/Makefile src/tools/other/mode_graphics/Makefile src/tools/other/modis_regrid/Makefile src/tools/other/pb2nc/Makefile src/tools/other/plot_data_plane/Makefile src/tools/other/plot_point_obs/Makefile src/tools/other/wwmca_tool/Makefile src/tools/other/gsi_tools/Makefile src/tools/other/regrid_data_plane/Makefile src/tools/other/point2grid/Makefile src/tools/other/shift_data_plane/Makefile src/tools/other/mode_time_domain/Makefile src/tools/other/grid_diag/Makefile src/tools/tc_utils/Makefile src/tools/tc_utils/tc_dland/Makefile src/tools/tc_utils/tc_pairs/Makefile src/tools/tc_utils/tc_stat/Makefile src/tools/tc_utils/tc_gen/Makefile src/tools/tc_utils/rmw_analysis/Makefile src/tools/tc_utils/tc_rmw/Makefile src/tools/tc_utils/tc_diag/Makefile" if test -n "$MET_DEVELOPMENT"; then @@ -10391,6 +10430,10 @@ if test -z "${ENABLE_POINT_STAT_TRUE}" && test -z "${ENABLE_POINT_STAT_FALSE}"; as_fn_error $? "conditional \"ENABLE_POINT_STAT\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${ENABLE_PAIR_STAT_TRUE}" && test -z "${ENABLE_PAIR_STAT_FALSE}"; then + as_fn_error $? "conditional \"ENABLE_PAIR_STAT\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi if test -z "${ENABLE_STAT_ANALYSIS_TRUE}" && test -z "${ENABLE_STAT_ANALYSIS_FALSE}"; then as_fn_error $? "conditional \"ENABLE_STAT_ANALYSIS\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 @@ -11121,6 +11164,7 @@ do "src/libcode/vx_bool_calc/Makefile") CONFIG_FILES="$CONFIG_FILES src/libcode/vx_bool_calc/Makefile" ;; "src/libcode/vx_pointdata_python/Makefile") CONFIG_FILES="$CONFIG_FILES src/libcode/vx_pointdata_python/Makefile" ;; "src/libcode/vx_seeps/Makefile") CONFIG_FILES="$CONFIG_FILES src/libcode/vx_seeps/Makefile" ;; + "src/libcode/vx_ioda/Makefile") CONFIG_FILES="$CONFIG_FILES src/libcode/vx_ioda/Makefile" ;; "src/tools/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/Makefile" ;; "src/tools/core/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/core/Makefile" ;; "src/tools/core/ensemble_stat/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/core/ensemble_stat/Makefile" ;; @@ -11129,6 +11173,7 @@ do "src/tools/core/mode_analysis/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/core/mode_analysis/Makefile" ;; "src/tools/core/pcp_combine/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/core/pcp_combine/Makefile" ;; "src/tools/core/point_stat/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/core/point_stat/Makefile" ;; + "src/tools/core/pair_stat/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/core/pair_stat/Makefile" ;; "src/tools/core/series_analysis/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/core/series_analysis/Makefile" ;; "src/tools/core/stat_analysis/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/core/stat_analysis/Makefile" ;; "src/tools/core/wavelet_stat/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/core/wavelet_stat/Makefile" ;; diff --git a/configure.ac b/configure.ac index 03436847af..5f189e0ec7 100644 --- a/configure.ac +++ b/configure.ac @@ -898,6 +898,26 @@ else AC_MSG_NOTICE([point_stat will not be compiled]) fi +# pair_stat + +AC_ARG_ENABLE(pair_stat, + [AS_HELP_STRING([--disable-pair_stat], [Disable compilation of pair_stat])], + [case "${enableval}" in + yes | no ) ENABLE_PAIR_STAT="${enableval}" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --disable-pair_stat) ;; + esac], + [ENABLE_PAIR_STAT="yes"] +) + +AM_CONDITIONAL([ENABLE_PAIR_STAT], [test "x$ENABLE_PAIR_STAT" = "xyes"]) + +if test "x$ENABLE_PAIR_STAT" = "xyes"; then + AC_DEFINE([ENABLE_PAIR_STAT], [], ["build pair_stat"]) + AC_MSG_NOTICE([pair_stat will be compiled]) +else + AC_MSG_NOTICE([pair_stat will not be compiled]) +fi + # stat_analysis AC_ARG_ENABLE(stat_analysis, @@ -1370,6 +1390,7 @@ AC_CONFIG_FILES([Makefile src/libcode/vx_bool_calc/Makefile src/libcode/vx_pointdata_python/Makefile src/libcode/vx_seeps/Makefile + src/libcode/vx_ioda/Makefile src/tools/Makefile src/tools/core/Makefile src/tools/core/ensemble_stat/Makefile @@ -1378,6 +1399,7 @@ AC_CONFIG_FILES([Makefile src/tools/core/mode_analysis/Makefile src/tools/core/pcp_combine/Makefile src/tools/core/point_stat/Makefile + src/tools/core/pair_stat/Makefile src/tools/core/series_analysis/Makefile src/tools/core/stat_analysis/Makefile src/tools/core/wavelet_stat/Makefile diff --git a/data/config/IODADataConfig_default b/data/config/IODADataConfig_default new file mode 100644 index 0000000000..ec1bb571f7 --- /dev/null +++ b/data/config/IODADataConfig_default @@ -0,0 +1,38 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// IODA Data configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Mapping of input IODA variable names to output variables names. +// The default IODA map, obs_var_map, is appended to this map. +// +obs_name_map = []; + +// +// Default mapping for Metadata. +// +metadata_map = [ + { key = "message_type"; val = "msg_type,station_ob"; }, + { key = "station_id"; val = "station_id,report_identifier"; }, + { key = "pressure"; val = "air_pressure,pressure"; }, + { key = "height"; val = "height,height_above_mean_sea_level"; }, + { key = "datetime"; val = "datetime,dateTime"; }, + { key = "elevation"; val = "elevation,station_elevation"; }, + { key = "nlocs"; val = "Location"; } +]; + +// +// Default mapping for obs to qc. +// +obs_to_qc_map = [ + { key = "wind_from_direction"; val = "eastward_wind,northward_wind"; }, + { key = "wind_speed"; val = "eastward_wind,northward_wind"; } +]; + +missing_thresh = [ <=-1e9, >=1e9, ==-9999 ]; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/data/config/Makefile.am b/data/config/Makefile.am index f6ed69c168..02678017c7 100644 --- a/data/config/Makefile.am +++ b/data/config/Makefile.am @@ -27,6 +27,7 @@ config_DATA = \ GridStatConfig_default \ GridDiagConfig_default \ IODA2NCConfig_default \ + IODADataConfig_default \ Madis2NcConfig_default \ MODEAnalysisConfig_default \ MODEConfig_default \ @@ -34,6 +35,7 @@ config_DATA = \ MTDConfig_default \ PB2NCConfig_default \ PointStatConfig_default \ + PairStatConfig_default \ SeriesAnalysisConfig_default \ STATAnalysisConfig_default \ STATAnalysisConfig_GO_Index \ diff --git a/data/config/Makefile.in b/data/config/Makefile.in index 6ff5c30139..5268ac67b5 100644 --- a/data/config/Makefile.in +++ b/data/config/Makefile.in @@ -318,6 +318,7 @@ config_DATA = \ GridStatConfig_default \ GridDiagConfig_default \ IODA2NCConfig_default \ + IODADataConfig_default \ Madis2NcConfig_default \ MODEAnalysisConfig_default \ MODEConfig_default \ @@ -325,6 +326,7 @@ config_DATA = \ MTDConfig_default \ PB2NCConfig_default \ PointStatConfig_default \ + PairStatConfig_default \ SeriesAnalysisConfig_default \ STATAnalysisConfig_default \ STATAnalysisConfig_GO_Index \ diff --git a/data/config/PairStatConfig_default b/data/config/PairStatConfig_default new file mode 100644 index 0000000000..e00d933d18 --- /dev/null +++ b/data/config/PairStatConfig_default @@ -0,0 +1,191 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Pair-Stat configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// Non-empty string overrides input data values +// +model = ""; + +// +// Output description to be written +// Non-empty string overrides input data values +// May be set separately in each "obs.pairs" entry +// +desc = ""; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Forecast and observation pairs to be verified +// +fcst = { + pairs = [ + { + name = "TMP"; + level = [ "Z2" ]; + } + ]; +} + +obs = { + pairs = [ + { + name = "TMP"; + level = [ "Z2" ]; + } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Forecast and observation data censoring, thresholding, and filtering options +// May be set separately in each "fcst.pairs" or "obs.pairs" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = [ NA ]; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Matched pairs filtering and control options +// May be set separately in each "obs.pairs" entry +// +mpr_column = []; +mpr_thresh = []; +mpr_str_inc = []; +mpr_str_exc = []; +eclv_points = 0.05; +hss_ec_value = NA; +rank_corr_flag = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology mean data +// May be set separately in the "fcst" and "obs" dictionaries +// +climo_mean = { + + file_name = []; + field = []; + + time_interp_method = DW_MEAN; + day_interval = 31; + hour_interval = 6; +} + +// +// Climatology standard deviation data +// May be set separately in the "fcst" and "obs" dictionaries +// +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +// +// Climatology distribution settings +// May be set separately in each "obs.pairs" entry +// +climo_cdf = { + cdf_bins = 1; + center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation time window +// May be set separately in each "obs.pairs" entry +// +obs_window = { + beg = -5400; + end = 5400; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// May be set separately in each "obs.pairs" entry +// +mask = { + grid = [ "FULL" ]; + poly = []; + sid = []; + llpnt = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// May be set separately in each "obs.pairs" entry +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Threshold for SEEPS p1 (Probability of being dry) +// +seeps_p1_thresh = >=0.1&&<=0.85; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// May be set separately in each "obs.pairs" entry +// +output_flag = { + fho = NONE; + ctc = NONE; + cts = NONE; + mctc = NONE; + mcts = NONE; + cnt = NONE; + sl1l2 = NONE; + sal1l2 = NONE; + vl1l2 = NONE; + val1l2 = NONE; + vcnt = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; + mpr = NONE; + seeps = NONE; + seeps_mpr = NONE; +} + +//////////////////////////////////////////////////////////////////////////////// + +point_weight_flag = NONE; + +tmp_dir = "/tmp"; +version = "V12.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/docs/Users_Guide/grid-stat.rst b/docs/Users_Guide/grid-stat.rst index f69abaadae..bfc7c90aaa 100644 --- a/docs/Users_Guide/grid-stat.rst +++ b/docs/Users_Guide/grid-stat.rst @@ -9,7 +9,7 @@ Introduction The Grid-Stat tool functions in much the same way as the Point-Stat tool, except that the verification statistics it calculates are for a matched forecast-observation grid (as opposed to a set of observation points). Neither the forecast nor the observation grid needs to be identical to the final matched grid. If the forecast grid is different from the final matched grid, then forecast values are regridded (interpolated) to the final matched grid. The same procedure is followed for observations. No regridding is necessary if the forecast and observation grids are identical but remains optional. A smoothing operation may be performed on the forecast and observation fields prior to verification. All the matched forecast-observation grid points are used to compute the verification statistics. In addition to traditional verification approaches, the Grid-Stat tool includes Fourier decompositions, gradient statistics, distance metrics, and neighborhood methods, designed to examine forecast performance as a function of spatial scale. -Scientific and statistical aspects of the Grid-Stat tool are briefly described in this section, followed by practical details regarding usage and output from the tool. +Scientific and statistical aspects of the Grid-Stat tool are briefly described in this section. Practical aspects of the Grid-Stat tool are described in :numref:`grid-stat_practical_info`. Scientific and Statistical Aspects ================================== @@ -163,6 +163,8 @@ In some cases, a user may be interested in a much higher threshold than :math:`2 Since :math:`G_\beta` is so sensitive to the choice of :math:`\beta`, which is defined relative to the number of points in the verification domain, :math:`G_\beta` is only computed for the full verification domain. :math:`G_\beta` is reported as a bad data value for any masking region subsets of the full verification domain. +.. _grid-stat_practical_info: + Practical Information ===================== diff --git a/docs/Users_Guide/index.rst b/docs/Users_Guide/index.rst index 1df5c65e53..5b72cd0733 100644 --- a/docs/Users_Guide/index.rst +++ b/docs/Users_Guide/index.rst @@ -54,6 +54,7 @@ The National Center for Atmospheric Research (NCAR) is sponsored by NSF. The DTC gen-ens-prod masking point-stat + pair-stat grid-stat ensemble-stat wavelet-stat diff --git a/docs/Users_Guide/pair-stat.rst b/docs/Users_Guide/pair-stat.rst new file mode 100644 index 0000000000..3577bcfdfb --- /dev/null +++ b/docs/Users_Guide/pair-stat.rst @@ -0,0 +1,98 @@ +.. _pair-stat: + +*************** +Pair-Stat Tool +*************** + +Introduction +============ + +The Pair-Stat tool provides verification statistics for forecast and observation data that has already been paired. While no smoothing, regridding, or interpolation methods apply, the Pair-Stat tool filters and groups the pairs temporally and spatially. It then computes continuous, categorical, and probabilistic verification statistics. The categorical and probabilistic statistics generally are derived by applying a threshold to the forecast and observation values. Confidence intervals - representing the uncertainty in the verification measures - are computed for the verification statistics. + +Scientific and statistical aspects of the Pair-Stat tool are discussed in the following section. Practical aspects of the Pair-Stat tool are described in :numref:`pair-stat_practical_info`. + +Scientific and Statistical Aspects +================================== + +The statistical methods and measures computed by the Pair-Stat tool are described briefly in this section. In addition, :numref:`matching-methods` discusses the various interpolation options available for matching the forecast grid point values to the observation points. The statistical measures computed by the Pair-Stat tool are described briefly in :numref:`PS_Statistical-measures` and in more detail in :numref:`Appendix C, Section %s `. :numref:`PS_Statistical-confidence-intervals` describes the methods for computing confidence intervals that are applied to some of the measures computed by the Pair-Stat tool; more detail on confidence intervals is provided in :numref:`Appendix D, Section %s `. + +.. _pair-stat_practical_info: + +Practical Information +===================== + +.. note:: + + Only **-format mpr** is supported at this time. Support for **-format python** and **-format ioda** will be added in future development cycles. + + +The Pair-Stat tool is used to perform verification of a gridded model field using point observations. The gridded model field to be verified must be in one of the supported file formats. The point observations must be formatted as the NetCDF output of the point reformatting tools described in :numref:`reformat_point`. The Pair-Stat tool provides the capability of interpolating the gridded forecast data to the observation points using a variety of methods as described in :numref:`matching-methods`. The Pair-Stat tool computes a number of continuous statistics on the matched pair data as well as discrete statistics once the matched pair data have been thresholded. + +If no matched pairs are found for a particular verification task, a report listing counts for reasons why the observations were not used is written to the log output at the default verbosity level of 2. If matched pairs are found, this report is written at verbosity level 3. Inspecting these rejection reason counts is the first step in determining why Pair-Stat found no matched pairs. The order of the log messages matches the order in which the processing logic is applied. Start from the last log message and work your way up, considering each of the non-zero rejection reason counts. Verbosity level 9 prints a very detailed explanation about why each observation is used or skipped for each verification task. + +pair_stat Usage +--------------- + +The usage statement for the Pair-Stat tool is shown below: + +.. code-block:: none + + Usage: pair_stat + -pairs file_1 ... file_n | file_list + -format type + -config config_file + [-outdir path] + [-log file] + [-v level] + +pair_stat has three required arguments and accepts optional ones. + +Required Arguments for pair_stat +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. The **-pairs** argument defines one or more input files containing forecast/observation pairs. + May be set as a list of file names (**file_1 ... file_n**) or as an ASCII file containing + a list file names (**file_list**). May be used multiple times (required). + +2. The **format type** argument defines the input pairs file format and may be set to + "mpr" or "ioda" (required). + +3. The **-config config_file** argument is a PairStatConfig file containing the desired + configuration settings (required). + +Optional Arguments for pair_stat +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +4. The **-outdir path** option overrides the output directory which defaults to the current + working directory (optional). + +5. The **-log file** option directs output and errors to the specified log file. + All messages will be written to that file as well as standard out and error. + Thus, users can save the messages without having to redirect the output on the command line. + The default behavior is no log file. + +6. The **-v level** option indicates the desired level of verbosity. + The value of "level" will override the default setting of 2. + Setting the verbosity to 0 will make the tool run with no log messages, + while increasing the verbosity will increase the amount of logging. + +An example of the pair_stat calling sequence is shown below: + +.. code-block:: none + + pair_stat \ + -pairs point_stat_mpr.txt \ + -format mpr \ + -config PairStatConfig + +In this example, the Pair-Stat tool reads matched pair data (**-format mpr**) from **point_stat_mpr.txt** +and applies the configuration options specified in the **PairStatConfig** file. + +pair_stat Configuration File +----------------------------- + +The default configuration file for the Pair-Stat tool named **PairStatConfig_default** can be found in the installed *share/met/config* directory. Users are encouraged to make a copy prior to modifying its contents. The configuration file options are described in the subsections below. + +Note that environment variables may be used when editing configuration files, as described in the :numref:`config_env_vars`. + +TODO: Complete documentation diff --git a/docs/Users_Guide/point-stat.rst b/docs/Users_Guide/point-stat.rst index 50fd7c3015..fc764c1962 100644 --- a/docs/Users_Guide/point-stat.rst +++ b/docs/Users_Guide/point-stat.rst @@ -9,7 +9,7 @@ Introduction The Point-Stat tool provides verification statistics for forecasts at observation points (as opposed to over gridded analyses). The Point-Stat tool matches gridded forecasts to point observation locations and supports several different interpolation options. The tool then computes continuous, categorical, spatial, and probabilistic verification statistics. The categorical and probabilistic statistics generally are derived by applying a threshold to the forecast and observation values. Confidence intervals - representing the uncertainty in the verification measures - are computed for the verification statistics. -Scientific and statistical aspects of the Point-Stat tool are discussed in the following section. Practical aspects of the Point-Stat tool are described in :numref:`tc-stat_practical-information`. +Scientific and statistical aspects of the Point-Stat tool are discussed in the following section. Practical aspects of the Point-Stat tool are described in :numref:`point-stat_practical_info`. Scientific and Statistical Aspects ================================== @@ -257,7 +257,7 @@ MET provides non-parametric bootstrap confidence intervals for many categorical For more information on confidence intervals pertaining to verification measures, see :ref:`Wilks (2011) `, :ref:`Jolliffe and Stephenson (2012) `, and Bradley (2008). -.. _tc-stat_practical-information: +.. _point-stat_practical_info: Practical Information ===================== diff --git a/internal/test_unit/bin/unit_test.sh b/internal/test_unit/bin/unit_test.sh index bd16f1d44b..e3b0927628 100755 --- a/internal/test_unit/bin/unit_test.sh +++ b/internal/test_unit/bin/unit_test.sh @@ -32,6 +32,75 @@ done UNIT=${MET_TEST_BASE}/python/unit.py # Unit test XML +UNIT_XML_ORG="unit_ascii2nc.xml \ + unit_ascii2nc_indy.xml \ + unit_madis2nc.xml \ + unit_trmm2nc.xml \ + unit_pb2nc.xml \ + unit_pb2nc_indy.xml \ + unit_gen_vx_mask.xml \ + unit_gen_ens_prod.xml \ + unit_pcp_combine.xml \ + unit_wwmca_regrid.xml \ + unit_point_stat.xml \ + unit_stat_analysis_ps.xml \ + unit_pair_stat.xml \ + unit_duplicate_flag.xml \ + unit_obs_summary.xml \ + unit_grid_stat.xml \ + unit_stat_analysis_gs.xml \ + unit_wavelet_stat.xml \ + unit_stat_analysis_ws.xml \ + unit_ensemble_stat.xml \ + unit_stat_analysis_es.xml \ + unit_mode.xml \ + unit_mode_multivar.xml \ + unit_mode_analysis.xml \ + unit_plot_point_obs.xml \ + unit_plot_data_plane.xml \ + unit_wwmca_plot.xml \ + unit_series_analysis.xml \ + unit_tc_dland.xml \ + unit_tc_pairs.xml \ + unit_tc_stat.xml \ + unit_plot_tc.xml \ + unit_tc_rmw.xml \ + unit_rmw_analysis.xml \ + unit_tc_diag.xml \ + unit_tc_gen.xml \ + unit_met_test_scripts.xml \ + unit_modis.xml \ + unit_ref_config_lead_00.xml \ + unit_ref_config_lead_12.xml \ + unit_ref_config_lead_24.xml \ + unit_ref_config_lead_36.xml \ + unit_ref_config_lead_48.xml \ + unit_ref_config.xml \ + unit_mode_graphics.xml \ + unit_regrid.xml \ + unit_gsi_tools.xml \ + unit_aeronet.xml \ + unit_shift_data_plane.xml \ + unit_mtd.xml \ + unit_climatology_1.0deg.xml \ + unit_climatology_1.5deg.xml \ + unit_climatology_2.5deg.xml \ + unit_climatology_mixed.xml \ + unit_grib_tables.xml \ + unit_grid_weight.xml \ + unit_netcdf.xml \ + unit_hira.xml \ + unit_interp_shape.xml \ + unit_lidar2nc.xml \ + unit_ioda2nc.xml \ + unit_airnow.xml \ + unit_python.xml \ + unit_point2grid.xml \ + unit_perc_thresh.xml \ + unit_gaussian.xml \ + unit_grid_diag.xml \ + unit_quality_filter.xml" + UNIT_XML="unit_ascii2nc.xml \ unit_ascii2nc_indy.xml \ unit_madis2nc.xml \ diff --git a/internal/test_unit/config/PairStatConfig_mpr b/internal/test_unit/config/PairStatConfig_mpr new file mode 100644 index 0000000000..89386d25a6 --- /dev/null +++ b/internal/test_unit/config/PairStatConfig_mpr @@ -0,0 +1,193 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Pair-Stat configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// Non-empty string overrides input data values +// +model = ""; + +// +// Output description to be written +// Non-empty string overrides input data values +// May be set separately in each "obs.pairs" entry +// +desc = ""; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Forecast and observation pairs to be verified +// +fcst = { + pairs = [ + { + name = "TMP"; + level = "Z2"; + cnt_thresh = [ NA, >273 ]; + mpr_str_inc = [ { key = "VX_MASK"; val = "DTC165,DTC166"; }, + { key = "OBTYPE"; val = "ADPSFC"; } ]; + }, + { + name = "TMP"; + level = "P850-700"; + mpr_str_inc = [ { key = "VX_MASK"; val = "DTC165,DTC166"; }, + { key = "OBTYPE"; val = "ADPUPA"; } ]; + } + ]; +} + +obs = fcst; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Forecast and observation data censoring, thresholding, and filtering options +// May be set separately in each "fcst.pairs" or "obs.pairs" entry +// +censor_thresh = []; +censor_val = []; +cat_thresh = [ NA ]; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Matched pairs filtering and control options +// May be set separately in each "obs.pairs" entry +// +mpr_column = []; +mpr_thresh = []; +mpr_str_inc = [ { key = "VX_MASK"; val = "DTC165,DTC166"; } ]; +mpr_str_exc = []; +eclv_points = 0.05; +hss_ec_value = NA; +rank_corr_flag = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Climatology mean data +// May be set separately in the "fcst" and "obs" dictionaries +// +climo_mean = { + + file_name = []; + field = []; + + time_interp_method = DW_MEAN; + day_interval = 31; + hour_interval = 6; +} + +// +// Climatology standard deviation data +// May be set separately in the "fcst" and "obs" dictionaries +// +climo_stdev = climo_mean; +climo_stdev = { + file_name = []; +} + +// +// Climatology distribution settings +// May be set separately in each "obs.pairs" entry +// +climo_cdf = { + cdf_bins = 1; + center_bins = FALSE; + write_bins = TRUE; + direct_prob = FALSE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Point observation time window +// May be set separately in each "obs.pairs" entry +// +obs_window = { + beg = -5400; + end = 5400; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// May be set separately in each "obs.pairs" entry +// +mask = { + grid = [ "FULL", "DTC165", "DTC166" ]; + poly = []; + sid = []; + llpnt = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// May be set separately in each "obs.pairs" entry +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = ""; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Threshold for SEEPS p1 (Probability of being dry) +// +seeps_p1_thresh = >=0.1&&<=0.85; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// May be set separately in each "obs.pairs" entry +// +output_flag = { + fho = NONE; + ctc = NONE; + cts = NONE; + mctc = NONE; + mcts = NONE; + cnt = BOTH; + sl1l2 = BOTH; + sal1l2 = NONE; + vl1l2 = NONE; + val1l2 = NONE; + vcnt = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; + mpr = NONE; + seeps = NONE; + seeps_mpr = NONE; +} + +//////////////////////////////////////////////////////////////////////////////// + +point_weight_flag = NONE; + +tmp_dir = "/tmp"; +version = "V12.0.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/internal/test_unit/xml/unit_pair_stat.xml b/internal/test_unit/xml/unit_pair_stat.xml new file mode 100644 index 0000000000..5d87ca940f --- /dev/null +++ b/internal/test_unit/xml/unit_pair_stat.xml @@ -0,0 +1,65 @@ + + + + + + + + + + +]> + + + + + + + + + &TEST_DIR; + true + + + &MET_BIN;/pair_stat + \ + -pairs &OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_120000L_20120409_120000V.stat \ + -format mpr \ + -config &CONFIG_DIR;/PairStatConfig_mpr \ + -out &OUTPUT_DIR;/pair_stat/pair_stat_format_mpr -v 1 + + + &OUTPUT_DIR;/pair_stat/pair_stat_format_mpr.stat + &OUTPUT_DIR;/pair_stat/pair_stat_format_mpr_cnt.txt + &OUTPUT_DIR;/pair_stat/pair_stat_format_mpr_sl1l2.txt + + + + + + &MET_BIN;/pair_stat + 1 + \ + -pairs python_input \ + -format python \ + -config &CONFIG_DIR;/PairStatConfig_mpr + + + + + + + + &MET_BIN;/pair_stat + 1 + \ + -pairs ioda_input \ + -format ioda \ + -config &CONFIG_DIR;/PairStatConfig_mpr + + + + + + diff --git a/scripts/Makefile b/scripts/Makefile index 6660bff06f..f927d9327b 100755 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -33,8 +33,7 @@ export TEST_OUT_DIR ## were built ## - -TESTS := $(shell grep "define ENABLE_" ../config.h | grep -v "ENABLE_PYTHON" | grep -v "ENABLE_UGRID" | cut -d" " -f2 | cut -b8- | tr A-Z a-z) +TESTS := $(shell grep "define ENABLE_" ../config.h | grep -v "ENABLE_PAIR_STAT" | grep -v "ENABLE_PYTHON" | grep -v "ENABLE_UGRID" | cut -d" " -f2 | cut -b8- | tr A-Z a-z) ######################################################################## diff --git a/src/basic/vx_cal/is_leap_year.cc b/src/basic/vx_cal/is_leap_year.cc index a383041475..256cc36fb9 100644 --- a/src/basic/vx_cal/is_leap_year.cc +++ b/src/basic/vx_cal/is_leap_year.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include "vx_cal.h" #include "vx_log.h" @@ -34,7 +35,15 @@ static constexpr double DAY_EPSILON = 0.00002; //////////////////////////////////////////////////////////////////////// +const array monthly_days = { + 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 +}; + +//////////////////////////////////////////////////////////////////////// +bool check_time_units_and_offset(int sec_per_unit, double time_value); + +//////////////////////////////////////////////////////////////////////// int is_leap_year(int year) { @@ -54,12 +63,6 @@ return 1; } -//////////////////////////////////////////////////////////////////////// - -const int monthly_days[NO_OF_MONTHS] = { - 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 -}; - //////////////////////////////////////////////////////////////////////// int get_days_from_mmdd(int month, int day, bool no_leap) { @@ -98,11 +101,11 @@ void increase_one_month(int &year, int &month) { //////////////////////////////////////////////////////////////////////// -void adjuste_day_for_month_year_units(int &day, int &month, int &year, double month_fraction) { +void adjust_day_for_month_year_units(int &day, int &month, int &year, double month_fraction) { // Compute remaining days from the month fraction bool day_adjusted = false; - const int day_offset = (int)(month_fraction * DAYS_PER_MONTH + 0.5); - const char *method_name = "adjuste_day_for_month_year_units() -> "; + const auto day_offset = (int)(month_fraction * DAYS_PER_MONTH + 0.5); + const char *method_name = "adjust_day_for_month_year_units() -> "; day += day_offset; if (day == 1 && abs(month_fraction-0.5) < DAY_EPSILON) { @@ -130,8 +133,8 @@ void adjuste_day_for_month_year_units(int &day, int &month, int &year, double mo //////////////////////////////////////////////////////////////////////// -void adjuste_day_for_day_units(int &day, int &month, int &year, double time_value) { - int day_offset = day + (int)time_value; +void adjust_day_for_day_units(int &day, int &month, int &year, int time_value) { + int day_offset = day + time_value; if (day_offset < 0) { while (day_offset < 0) { decrease_one_month(year, month); @@ -146,7 +149,6 @@ void adjuste_day_for_day_units(int &day, int &month, int &year, double time_valu } day = day_offset; if (day == 0) day = 1; - } //////////////////////////////////////////////////////////////////////// @@ -162,25 +164,27 @@ unixtime add_to_unixtime(unixtime base_unixtime, int sec_per_unit, unixtime ut; auto time_value_ut = (unixtime)time_value; double time_fraction = time_value - (double)time_value_ut; - const char *method_name = "add_to_unixtime() -> "; + const char *method_name = "add_to_unixtime(double) -> "; + + if (!check_time_units_and_offset(sec_per_unit, time_value)) { + mlog << Error << "\n" << method_name + << " the negative offset (" << time_value + << ") is not supported for unit months and years\n\n"; + exit(-1); + } if (sec_per_unit == SEC_MONTH || sec_per_unit == SEC_YEAR) { - if (time_value < 0) { - mlog << Error << "\n" << method_name - << " the negative offset (" << time_value - << ") is not supported for unit months and years\n\n"; - exit(-1); - } unix_to_mdyhms(base_unixtime, month, day, year, hour, minute, second); // Update year and compute remaining months int month_offset; + double month_fraction = time_fraction; if (sec_per_unit == SEC_YEAR) { year += time_value_ut; - time_fraction *= NO_OF_MONTHS; // 12 months/year - month_offset = (int)time_fraction; - time_fraction -= month_offset; + month_fraction *= NO_OF_MONTHS; // 12 months/year + month_offset = (int)month_fraction; + month_fraction -= month_offset; } else month_offset = (int)time_value_ut; @@ -189,44 +193,8 @@ unixtime add_to_unixtime(unixtime base_unixtime, int sec_per_unit, increase_one_month(year, month); } // Compute remaining days - //adjuste_day_for_month_year_units(day, month, year, time_fraction); - //ut = mdyhms_to_unix(month, day, year, hour, minute, second); - double day_offset; - if (day == 1) { - if (abs(time_fraction-0.5) < DAY_EPSILON) day = 15; - else { - day_offset = time_fraction * DAYS_PER_MONTH; - day += (int)day_offset; - if (day_offset - (int)day_offset > 0.5) day++; - } - } - else { - day_offset = time_fraction * DAYS_PER_MONTH; - time_value_ut = (int)day_offset; - day += time_value_ut; - if (day_offset - time_value_ut > 0.5) day++; - if (day > DAYS_PER_MONTH) { - day -= DAYS_PER_MONTH; - increase_one_month(year, month); - } - } - - int day_org = day; - bool day_adjusted = false; - int max_day = monthly_days[month-1]; - if (day > max_day) { - day = max_day; - day_adjusted = true; - if (month == 2 && is_leap_year(year)) { - if (day_org == 29) day_adjusted = false; - day = 29; - } - } + adjust_day_for_month_year_units(day, month, year, month_fraction); ut = mdyhms_to_unix(month, day, year, hour, minute, second); - if (day_adjusted) { - mlog << Debug(2) << method_name << "adjusted day " << day_org - << " to " << day << " for " << year << "-" << month << "\n"; - } } else if (!no_leap || sec_per_unit != 86400) { // seconds, minute, hours, and day unit with leap year @@ -241,7 +209,7 @@ unixtime add_to_unixtime(unixtime base_unixtime, int sec_per_unit, } else { // no_leap year && unit = day unix_to_mdyhms(base_unixtime, month, day, year, hour, minute, second); - adjuste_day_for_day_units(day, month, year, time_value); + adjust_day_for_day_units(day, month, year, (int)time_value); ut = mdyhms_to_unix(month, day, year, hour, minute, second); if (time_fraction > (1-TIME_EPSILON) ) ut += (unixtime)sec_per_unit; else if (time_fraction > TIME_EPSILON) ut += (unixtime)(time_fraction * sec_per_unit); @@ -250,9 +218,67 @@ unixtime add_to_unixtime(unixtime base_unixtime, int sec_per_unit, << unix_to_yyyymmdd_hhmmss(base_unixtime) << " plus " << time_value << " times " << sec_per_unit << " seconds = " << unix_to_yyyymmdd_hhmmss(ut) << "\n"; + + return ut; +} + +//////////////////////////////////////////////////////////////////////// + +unixtime add_to_unixtime(unixtime base_unixtime, int sec_per_unit, + unixtime time_value, bool no_leap) { + int month; + int day; + int year; + int hour; + int minute; + int second; + unixtime ut; + const char *method_name = "add_to_unixtime(unixtime) -> "; + + if (!check_time_units_and_offset(sec_per_unit, (double)time_value)) { + mlog << Error << "\n" << method_name + << " the negative offset (" << time_value + << ") is not supported for unit months and years\n\n"; + exit(-1); + } + + if (sec_per_unit == SEC_YEAR) { + unix_to_mdyhms(base_unixtime, month, day, year, hour, minute, second); + year += time_value; + ut = mdyhms_to_unix(month, day, year, hour, minute, second); + } + else if (sec_per_unit == SEC_MONTH) { + unix_to_mdyhms(base_unixtime, month, day, year, hour, minute, second); + for (int idx=0; idx= 0); + } + return valid; +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/basic/vx_cal/vx_cal.h b/src/basic/vx_cal/vx_cal.h index 5cccd82ab7..9b6f32513b 100644 --- a/src/basic/vx_cal/vx_cal.h +++ b/src/basic/vx_cal/vx_cal.h @@ -42,6 +42,8 @@ static const int mjd_ut0 = 40587; // mjd of Jan 1, 1970 //////////////////////////////////////////////////////////////////////// +extern unixtime add_to_unixtime (unixtime u, int sec_per_unit, unixtime time_value, bool no_leap); + extern unixtime add_to_unixtime (unixtime u, int sec_per_unit, double time_value, bool no_leap); extern unixtime doyhms_to_unix (int doy, int year, int hour, int minute, int second); diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index 9f1366a681..686e3dd52b 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -609,6 +609,8 @@ static const char conf_key_censor_thresh[] = "censor_thresh"; static const char conf_key_censor_val[] = "censor_val"; static const char conf_key_mpr_column[] = "mpr_column"; static const char conf_key_mpr_thresh[] = "mpr_thresh"; +static const char conf_key_mpr_str_inc[] = "mpr_str_inc"; +static const char conf_key_mpr_str_exc[] = "mpr_str_exc"; static const char conf_key_cnt_thresh[] = "cnt_thresh"; static const char conf_key_cnt_logic[] = "cnt_logic"; static const char conf_key_cat_thresh[] = "cat_thresh"; @@ -717,6 +719,7 @@ static const char conf_key_ugrid_metadata_map[] = "ugrid_metadata_map"; // // Entries to override file metadata // + static const char conf_key_set_attr_name[] = "set_attr_name"; static const char conf_key_set_attr_units[] = "set_attr_units"; static const char conf_key_set_attr_level[] = "set_attr_level"; @@ -738,6 +741,7 @@ static const char conf_key_is_prob[] = "is_prob"; // // Climatology data parameter key names // + static const char conf_key_climo_mean[] = "climo_mean"; static const char conf_key_climo_mean_field[] = "climo_mean.field"; static const char conf_key_climo_stdev[] = "climo_stdev"; @@ -746,6 +750,7 @@ static const char conf_key_climo_stdev_field[] = "climo_stdev.field"; // // Climatology distribution parameter key names // + static const char conf_key_climo_cdf[] = "climo_cdf"; static const char conf_key_cdf_bins[] = "cdf_bins"; static const char conf_key_center_bins[] = "center_bins"; @@ -758,6 +763,7 @@ static const char conf_key_hour_interval[] = "hour_interval"; // // Point-Stat specific parameter key names // + static const char conf_key_hira[] = "hira"; static const char conf_key_land_mask[] = "land_mask"; static const char conf_key_land_mask_flag[] = "land_mask.flag"; @@ -767,9 +773,17 @@ static const char conf_key_use_obs_thresh[] = "use_obs_thresh"; static const char conf_key_interp_fcst_thresh[] = "interp_fcst_thresh"; static const char conf_key_point_weight_flag[] = "point_weight_flag"; +// +// Pair-Stat specific parameter key names +// + +static const char conf_key_fcst_pairs[] = "fcst.pairs"; +static const char conf_key_obs_pairs[] = "obs.pairs"; + // // Grid-Stat specific parameter key names // + static const char conf_key_nc_pairs_var_name[] = "nc_pairs_var_name"; static const char conf_key_nc_pairs_var_suffix[] = "nc_pairs_var_suffix"; // nc_pairs_var_str is deprecated and replaced by nc_pairs_var_suffix diff --git a/src/basic/vx_config/data_file_type.h b/src/basic/vx_config/data_file_type.h index 618e1cccb5..e40a9bcdb7 100644 --- a/src/basic/vx_config/data_file_type.h +++ b/src/basic/vx_config/data_file_type.h @@ -8,20 +8,16 @@ /////////////////////////////////////////////////////////////////////////////// - #ifndef __DATA_FILE_TYPE_H__ #define __DATA_FILE_TYPE_H__ - /////////////////////////////////////////////////////////////////////////////// +// +// Enumeration of supported file types +// - // - // Enumeration of supported gridded file types - // - -enum GrdFileType -{ +enum GrdFileType { FileType_None, // Default @@ -37,36 +33,27 @@ enum GrdFileType FileType_Python_Numpy, // Python script using numpy array and attributes dictionary FileType_Python_Xarray, // Python script using xarray dataplane FileType_UGrid, // Unstructured grid + FileType_Pairs, // Paired data }; - /////////////////////////////////////////////////////////////////////////////// - -inline bool is_netcdf_grdfiletype(const GrdFileType _t) - -{ - -return ( (_t == FileType_NcMet) || (_t == FileType_General_Netcdf) || - (_t == FileType_NcWrf) || (_t == FileType_NcPinterp) || - (_t == FileType_NcCF) ); - +inline bool is_netcdf_grdfiletype(const GrdFileType _t) { + return(_t == FileType_NcMet || + _t == FileType_General_Netcdf || + _t == FileType_NcWrf || + _t == FileType_NcPinterp || + _t == FileType_NcCF); } - /////////////////////////////////////////////////////////////////////////////// - -inline bool is_python_grdfiletype(const GrdFileType _t) - -{ - -return ( (_t == FileType_Python_Xarray) || (_t == FileType_Python_Numpy) ); - +inline bool is_python_grdfiletype(const GrdFileType _t) { + return(_t == FileType_Python_Xarray || + _t == FileType_Python_Numpy); } - /////////////////////////////////////////////////////////////////////////////// #endif /* __DATA_FILE_TYPE_H__ */ diff --git a/src/basic/vx_config/grdfiletype_to_string.cc b/src/basic/vx_config/grdfiletype_to_string.cc index c9e89451c3..39ac64b246 100644 --- a/src/basic/vx_config/grdfiletype_to_string.cc +++ b/src/basic/vx_config/grdfiletype_to_string.cc @@ -57,6 +57,7 @@ switch ( t ) { case FileType_Python_Numpy: s = "FileType_Python_Numpy"; break; case FileType_Python_Xarray: s = "FileType_Python_Xarray"; break; case FileType_UGrid: s = "FileType_UGrid"; break; + case FileType_Pairs: s = "FileType_Pairs"; break; default: s = "(bad value)"; @@ -92,6 +93,7 @@ else if ( strcmp(text, "FileType_Bufr" ) == 0 ) { t = FileType_Bufr; else if ( strcmp(text, "FileType_Python_Numpy" ) == 0 ) { t = FileType_Python_Numpy; return true; } else if ( strcmp(text, "FileType_Python_Xarray" ) == 0 ) { t = FileType_Python_Xarray; return true; } else if ( strcmp(text, "FileType_UGrid" ) == 0 ) { t = FileType_UGrid; return true; } +else if ( strcmp(text, "FileType_Pairs" ) == 0 ) { t = FileType_Pairs; return true; } // // nope // diff --git a/src/libcode/Makefile.am b/src/libcode/Makefile.am index ff98f4df20..8c3fde5e8e 100644 --- a/src/libcode/Makefile.am +++ b/src/libcode/Makefile.am @@ -64,6 +64,7 @@ SUBDIRS += vx_data2d_factory \ vx_nc_obs \ vx_solar \ vx_bool_calc \ - vx_seeps + vx_seeps \ + vx_ioda MAINTAINERCLEANFILES = Makefile.in diff --git a/src/libcode/Makefile.in b/src/libcode/Makefile.in index f7629e1481..2a480310da 100644 --- a/src/libcode/Makefile.in +++ b/src/libcode/Makefile.in @@ -161,7 +161,7 @@ DIST_SUBDIRS = vx_grid vx_data2d vx_nc_util vx_data2d_grib \ vx_pxm vx_render vx_ps vx_plot_util vx_shapedata vx_statistics \ vx_stat_out vx_analysis_util vx_tc_util vx_summary \ vx_time_series vx_physics vx_series_data vx_regrid vx_nc_obs \ - vx_solar vx_bool_calc vx_seeps + vx_solar vx_bool_calc vx_seeps vx_ioda am__DIST_COMMON = $(srcdir)/Makefile.in DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) am__relativize = \ @@ -355,7 +355,7 @@ SUBDIRS = vx_grid vx_data2d vx_nc_util vx_data2d_grib vx_data2d_nc_met \ vx_pxm vx_render vx_ps vx_plot_util vx_shapedata vx_statistics \ vx_stat_out vx_analysis_util vx_tc_util vx_summary \ vx_time_series vx_physics vx_series_data vx_regrid vx_nc_obs \ - vx_solar vx_bool_calc vx_seeps + vx_solar vx_bool_calc vx_seeps vx_ioda MAINTAINERCLEANFILES = Makefile.in all: all-recursive diff --git a/src/libcode/vx_analysis_util/mode_job.cc b/src/libcode/vx_analysis_util/mode_job.cc index e97c8c3994..067c70771f 100644 --- a/src/libcode/vx_analysis_util/mode_job.cc +++ b/src/libcode/vx_analysis_util/mode_job.cc @@ -323,7 +323,7 @@ if ( n_dump == 0 ) { dump_at.set_precision(get_precision()); dump_at.set_bad_data_value(bad_data_double); dump_at.set_bad_data_str(na_str); - dump_at.set_delete_trailing_blank_rows(1); + dump_at.set_delete_trailing_blank_rows(true); // Write out the MODE header columns for (j=0; j +#include +#include +#include + +#include "var_info.h" +#include "var_info_pairs.h" + +#include "util_constants.h" + +#include "vx_math.h" +#include "vx_util.h" +#include "vx_log.h" +#include "vx_data2d.h" + +using namespace std; + +/////////////////////////////////////////////////////////////////////////////// +// +// Code for class VarInfoPairs +// +/////////////////////////////////////////////////////////////////////////////// + +VarInfoPairs::VarInfoPairs() { + + init_from_scratch(); +} + +/////////////////////////////////////////////////////////////////////////////// + +VarInfoPairs::~VarInfoPairs() { + + clear(); +} + +/////////////////////////////////////////////////////////////////////////////// + +VarInfoPairs::VarInfoPairs(const VarInfoPairs &f) { + + init_from_scratch(); + + assign(f); +} + +/////////////////////////////////////////////////////////////////////////////// + +VarInfoPairs & VarInfoPairs::operator=(const VarInfoPairs &f) { + + if ( this == &f ) return *this; + + assign(f); + + return *this; +} + +/////////////////////////////////////////////////////////////////////////////// + +VarInfo *VarInfoPairs::clone() const { + + VarInfoPairs *ret = new VarInfoPairs(*this); + + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + +void VarInfoPairs::init_from_scratch() { + + // First call the parent's initialization + VarInfo::init_from_scratch(); + + clear(); + + return; +} + +/////////////////////////////////////////////////////////////////////////////// + +void VarInfoPairs::assign(const VarInfoPairs &v) { + + // First call the parent's assign + VarInfo::assign(v); + + return; +} + +/////////////////////////////////////////////////////////////////////////////// + +void VarInfoPairs::clear() { + + // First call the parent's clear + VarInfo::clear(); + + return; +} + +/////////////////////////////////////////////////////////////////////////////// + +void VarInfoPairs::dump(ostream &out) const { + + // Dump out the contents + out << "VarInfoPairs::dump():\n"; + + VarInfo::dump(out); + + return; +} + +/////////////////////////////////////////////////////////////////////////////// + +void VarInfoPairs::set_dict(Dictionary & dict) { + + VarInfo::set_dict(dict); + + // + // Parse the required name and optional level strings + // + ConcatString nstr = dict.lookup_string(conf_key_name); + ReqName = nstr; + Name = nstr; + + ConcatString lstr = dict.lookup_string(conf_key_level, false); + Level.set_req_name(lstr.c_str()); + Level.set_name(lstr.c_str()); + + VarInfo::set_magic(nstr, lstr); + + return; +} + +/////////////////////////////////////////////////////////////////////////////// + +bool VarInfoPairs::is_precipitation() const { + bool status = false; + + // + // Check set_attrs entry + // + if(!is_bad_data(SetAttrIsPrecipitation)) { + return(SetAttrIsPrecipitation != 0); + } + + return status; +} + +/////////////////////////////////////////////////////////////////////////////// + +bool VarInfoPairs::is_specific_humidity() const { + bool status = false; + + // + // Check set_attrs entry + // + if(!is_bad_data(SetAttrIsSpecificHumidity)) { + return(SetAttrIsSpecificHumidity != 0); + } + + return status; +} + +/////////////////////////////////////////////////////////////////////////////// + +bool VarInfoPairs::is_u_wind() const { + bool status = false; + + // + // Check set_attrs entry + // + if(!is_bad_data(SetAttrIsUWind)) { + return(SetAttrIsUWind != 0); + } + + return status; +} + +/////////////////////////////////////////////////////////////////////////////// + +bool VarInfoPairs::is_v_wind() const { + bool status = false; + + // + // Check set_attrs entry + // + if(!is_bad_data(SetAttrIsVWind)) { + return(SetAttrIsVWind != 0); + } + + return status; +} + +/////////////////////////////////////////////////////////////////////////////// + +bool VarInfoPairs::is_wind_speed() const { + bool status = false; + + // + // Check set_attrs entry + // + if(!is_bad_data(SetAttrIsWindSpeed)) { + return(SetAttrIsWindSpeed != 0); + } + + return status; +} + +/////////////////////////////////////////////////////////////////////////////// + +bool VarInfoPairs::is_wind_direction() const { + bool status = false; + + // + // Check set_attrs entry + // + if(!is_bad_data(SetAttrIsWindDirection)) { + return(SetAttrIsWindDirection != 0); + } + + return status; +} + +/////////////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d/var_info_pairs.h b/src/libcode/vx_data2d/var_info_pairs.h new file mode 100644 index 0000000000..9d5644a6fc --- /dev/null +++ b/src/libcode/vx_data2d/var_info_pairs.h @@ -0,0 +1,73 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2024 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __VAR_INFO_PAIRS_H__ +#define __VAR_INFO_PAIRS_H__ + +/////////////////////////////////////////////////////////////////////////////// + +#include "var_info.h" +#include "vx_config.h" + +#include "data_file_type.h" + +/////////////////////////////////////////////////////////////////////////////// + +class VarInfoPairs : public VarInfo { + + private: + + void init_from_scratch(); + void assign(const VarInfoPairs &); + + public: + VarInfoPairs(); + ~VarInfoPairs(); + VarInfoPairs(const VarInfoPairs &); + VarInfoPairs & operator=(const VarInfoPairs &); + VarInfo *clone() const; + + void dump(std::ostream &) const; + void clear(); + + // + // get stuff + // + + GrdFileType file_type() const; + + // + // set stuff + // + + void set_dict(Dictionary &); + + // + // do stuff + // + + bool is_precipitation() const; + bool is_specific_humidity() const; + bool is_u_wind() const; + bool is_v_wind() const; + bool is_wind_speed() const; + bool is_wind_direction() const; +}; + +/////////////////////////////////////////////////////////////////////////////// + +inline GrdFileType VarInfoPairs::file_type() const { return FileType_Pairs; } + +/////////////////////////////////////////////////////////////////////////////// + +#endif // __VAR_INFO_PAIRS_H__ + +/////////////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_factory/var_info_factory.cc b/src/libcode/vx_data2d_factory/var_info_factory.cc index 32a398fb39..968d68dfd4 100644 --- a/src/libcode/vx_data2d_factory/var_info_factory.cc +++ b/src/libcode/vx_data2d_factory/var_info_factory.cc @@ -27,6 +27,7 @@ #include "var_info_nc_cf.h" #include "var_info_nc_met.h" #include "var_info_nc_wrf.h" +#include "var_info_pairs.h" #include "var_info_ugrid.h" #ifdef WITH_PYTHON @@ -115,6 +116,10 @@ VarInfo * VarInfoFactory::new_var_info(GrdFileType type) ugrid_compile_error(method_name); #endif + case FileType_Pairs: + vi = new VarInfoPairs; + break; + case FileType_HdfEos: mlog << Error << "\n" << method_name << "Support for GrdFileType = " << grdfiletype_to_string(type) diff --git a/src/libcode/vx_data2d_grib/data2d_grib.h b/src/libcode/vx_data2d_grib/data2d_grib.h index bcfbc45ab9..1834d18bf5 100644 --- a/src/libcode/vx_data2d_grib/data2d_grib.h +++ b/src/libcode/vx_data2d_grib/data2d_grib.h @@ -102,7 +102,7 @@ class MetGrib1DataFile : public Met2dDataFile { //////////////////////////////////////////////////////////////////////// -inline double MetGrib1DataFile::operator () (int x, int y) const { return get(x, y) ; } +inline double MetGrib1DataFile::operator () (int x, int y) const { return get(x, y); } inline GrdFileType MetGrib1DataFile::file_type () const { return FileType_Gb1; } diff --git a/src/libcode/vx_data2d_grib2/var_info_grib2.h b/src/libcode/vx_data2d_grib2/var_info_grib2.h index 2ce946d0db..a7044fced7 100644 --- a/src/libcode/vx_data2d_grib2/var_info_grib2.h +++ b/src/libcode/vx_data2d_grib2/var_info_grib2.h @@ -161,21 +161,14 @@ inline int VarInfoGrib2::der_type() const { return DerType; } inline int VarInfoGrib2::stat_type() const { return StatType; } inline int VarInfoGrib2::perc_val() const { return PercVal; } -inline int VarInfoGrib2::aerosol_type() const { return AerosolType; } -inline int VarInfoGrib2::aerosol_interval_type() const { return AerosolIntervalType; } -inline double VarInfoGrib2::aerosol_size_lower() const { return AerosolSizeLower; } -inline double VarInfoGrib2::aerosol_size_upper() const { return AerosolSizeUpper; } - -inline int VarInfoGrib2::n_ipdtmpl() const { - return IPDTmplIndex.n(); } -inline int VarInfoGrib2::ipdtmpl_index(int i) const { - return IPDTmplIndex[i]; } -inline int VarInfoGrib2::ipdtmpl_val(int i) const { - return IPDTmplVal[i]; } - -/////////////////////////////////////////////////////////////////////////////// - - +inline int VarInfoGrib2::aerosol_type() const { return AerosolType; } +inline int VarInfoGrib2::aerosol_interval_type() const { return AerosolIntervalType; } +inline double VarInfoGrib2::aerosol_size_lower() const { return AerosolSizeLower; } +inline double VarInfoGrib2::aerosol_size_upper() const { return AerosolSizeUpper; } + +inline int VarInfoGrib2::n_ipdtmpl() const { return IPDTmplIndex.n(); } +inline int VarInfoGrib2::ipdtmpl_index(int i) const { return IPDTmplIndex[i]; } +inline int VarInfoGrib2::ipdtmpl_val(int i) const { return IPDTmplVal[i]; } /////////////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.h b/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.h index 8fee4ead2d..12c3faab09 100644 --- a/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.h +++ b/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.h @@ -86,14 +86,14 @@ class VarInfoNcCF : public VarInfo /////////////////////////////////////////////////////////////////////////////// -inline GrdFileType VarInfoNcCF::file_type() const { return FileType_NcCF; } -inline const LongArray & VarInfoNcCF::dimension() const { return Dimension; } -inline int VarInfoNcCF::dimension(int i) const { return (int) Dimension[i]; } -inline int VarInfoNcCF::n_dimension() const { return Dimension.n_elements();} -inline const NumArray & VarInfoNcCF::dim_value() const { return Dim_value; } -inline double VarInfoNcCF::dim_value(int i) const { return Dim_value[i]; } -inline const BoolArray & VarInfoNcCF::is_offset() const { return Is_offset; } -inline bool VarInfoNcCF::is_offset(int i) const { return Is_offset[i]; } +inline GrdFileType VarInfoNcCF::file_type() const { return FileType_NcCF; } +inline const LongArray & VarInfoNcCF::dimension() const { return Dimension; } +inline int VarInfoNcCF::dimension(int i) const { return (int)Dimension[i]; } +inline int VarInfoNcCF::n_dimension() const { return Dimension.n_elements(); } +inline const NumArray & VarInfoNcCF::dim_value() const { return Dim_value; } +inline double VarInfoNcCF::dim_value(int i) const { return Dim_value[i]; } +inline const BoolArray & VarInfoNcCF::is_offset() const { return Is_offset; } +inline bool VarInfoNcCF::is_offset(int i) const { return Is_offset[i]; } /////////////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.h b/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.h index 44f4f48cfd..f76554c6d1 100644 --- a/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.h +++ b/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.h @@ -213,7 +213,7 @@ class VarInfoNcWrf : public VarInfo // get stuff // - GrdFileType file_type() const; + GrdFileType file_type() const; const LongArray & dimension() const; int dimension(int i) const; const NumArray & dim_value() const; diff --git a/src/libcode/vx_data2d_python/data2d_python.h b/src/libcode/vx_data2d_python/data2d_python.h index ff205885bd..5376fc20d4 100644 --- a/src/libcode/vx_data2d_python/data2d_python.h +++ b/src/libcode/vx_data2d_python/data2d_python.h @@ -101,7 +101,7 @@ class MetPythonDataFile : public Met2dDataFile { inline double MetPythonDataFile::operator () (int x, int y) const { return get(x, y); } -inline GrdFileType MetPythonDataFile::file_type () const { return Type; } +inline GrdFileType MetPythonDataFile::file_type () const { return Type; } //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d_ugrid/data2d_ugrid.h b/src/libcode/vx_data2d_ugrid/data2d_ugrid.h index de549b0640..0e1399118e 100644 --- a/src/libcode/vx_data2d_ugrid/data2d_ugrid.h +++ b/src/libcode/vx_data2d_ugrid/data2d_ugrid.h @@ -123,7 +123,7 @@ class MetUGridDataFile : public Met2dDataFile { //////////////////////////////////////////////////////////////////////// -inline GrdFileType MetUGridDataFile::file_type () const { return FileType_UGrid; } +inline GrdFileType MetUGridDataFile::file_type() const { return FileType_UGrid; } inline ConcatString MetUGridDataFile::coordinate_file() const { return _file->coordinate_nc(); } diff --git a/src/libcode/vx_data2d_ugrid/var_info_ugrid.h b/src/libcode/vx_data2d_ugrid/var_info_ugrid.h index a3e05c3d88..6289fd62dc 100644 --- a/src/libcode/vx_data2d_ugrid/var_info_ugrid.h +++ b/src/libcode/vx_data2d_ugrid/var_info_ugrid.h @@ -86,14 +86,14 @@ class VarInfoUGrid : public VarInfo /////////////////////////////////////////////////////////////////////////////// -inline GrdFileType VarInfoUGrid::file_type() const { return FileType_UGrid; } -inline const LongArray & VarInfoUGrid::dimension() const { return Dimension; } -inline int VarInfoUGrid::dimension(int i) const { return Dimension[i]; } -inline int VarInfoUGrid::n_dimension() const { return Dimension.n_elements();} -inline const NumArray & VarInfoUGrid::dim_value() const { return Dim_value; } -inline double VarInfoUGrid::dim_value(int i) const { return Dim_value[i]; } -inline const BoolArray & VarInfoUGrid::is_offset() const { return Is_offset; } -inline bool VarInfoUGrid::is_offset(int i) const { return Is_offset[i]; } +inline GrdFileType VarInfoUGrid::file_type() const { return FileType_UGrid; } +inline const LongArray & VarInfoUGrid::dimension() const { return Dimension; } +inline int VarInfoUGrid::dimension(int i) const { return Dimension[i]; } +inline int VarInfoUGrid::n_dimension() const { return Dimension.n_elements(); } +inline const NumArray & VarInfoUGrid::dim_value() const { return Dim_value; } +inline double VarInfoUGrid::dim_value(int i) const { return Dim_value[i]; } +inline const BoolArray & VarInfoUGrid::is_offset() const { return Is_offset; } +inline bool VarInfoUGrid::is_offset(int i) const { return Is_offset[i]; } /////////////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_ioda/Makefile.am b/src/libcode/vx_ioda/Makefile.am new file mode 100644 index 0000000000..cf4a03d43f --- /dev/null +++ b/src/libcode/vx_ioda/Makefile.am @@ -0,0 +1,17 @@ +## @start 1 +## Makefile.am -- Process this file with automake to produce Makefile.in +## @end 1 + +MAINTAINERCLEANFILES = Makefile.in + +# Include the project definitions + +include ${top_srcdir}/Make-include + +# The library + +noinst_LIBRARIES = libvx_ioda.a +libvx_ioda_a_SOURCES = \ + ioda.cc ioda.h \ + ioda_data_conf_info.cc ioda_data_conf_info.h +libvx_ioda_a_CPPFLAGS = ${MET_CPPFLAGS} diff --git a/src/libcode/vx_ioda/Makefile.in b/src/libcode/vx_ioda/Makefile.in new file mode 100644 index 0000000000..1c3ec0c9bd --- /dev/null +++ b/src/libcode/vx_ioda/Makefile.in @@ -0,0 +1,667 @@ +# Makefile.in generated by automake 1.16.5 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2021 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +subdir = src/libcode/vx_ioda +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +LIBRARIES = $(noinst_LIBRARIES) +AR = ar +ARFLAGS = cru +AM_V_AR = $(am__v_AR_@AM_V@) +am__v_AR_ = $(am__v_AR_@AM_DEFAULT_V@) +am__v_AR_0 = @echo " AR " $@; +am__v_AR_1 = +libvx_ioda_a_AR = $(AR) $(ARFLAGS) +libvx_ioda_a_LIBADD = +am_libvx_ioda_a_OBJECTS = libvx_ioda_a-ioda.$(OBJEXT) \ + libvx_ioda_a-ioda_data_conf_info.$(OBJEXT) +libvx_ioda_a_OBJECTS = $(am_libvx_ioda_a_OBJECTS) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/libvx_ioda_a-ioda.Po \ + ./$(DEPDIR)/libvx_ioda_a-ioda_data_conf_info.Po +am__mv = mv -f +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +AM_V_CXX = $(am__v_CXX_@AM_V@) +am__v_CXX_ = $(am__v_CXX_@AM_DEFAULT_V@) +am__v_CXX_0 = @echo " CXX " $@; +am__v_CXX_1 = +CXXLD = $(CXX) +CXXLINK = $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \ + -o $@ +AM_V_CXXLD = $(am__v_CXXLD_@AM_V@) +am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@) +am__v_CXXLD_0 = @echo " CXXLD " $@; +am__v_CXXLD_1 = +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libvx_ioda_a_SOURCES) +DIST_SOURCES = $(libvx_ioda_a_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BUFRLIB_NAME = @BUFRLIB_NAME@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPPFLAGS = @CPPFLAGS@ +CSCOPE = @CSCOPE@ +CTAGS = @CTAGS@ +CXX = @CXX@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +ETAGS = @ETAGS@ +EXEEXT = @EXEEXT@ +F77 = @F77@ +FC_LIBS = @FC_LIBS@ +FFLAGS = @FFLAGS@ +FLIBS = @FLIBS@ +GRIB2CLIB_NAME = @GRIB2CLIB_NAME@ +GRIB2_DEP_LIBS = @GRIB2_DEP_LIBS@ +GRIB2_MET_LIBS = @GRIB2_MET_LIBS@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +LDFLAGS = @LDFLAGS@ +LEX = @LEX@ +LEXLIB = @LEXLIB@ +LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +MET_ATLAS = @MET_ATLAS@ +MET_ATLASINC = @MET_ATLASINC@ +MET_ATLASLIB = @MET_ATLASLIB@ +MET_BUFR = @MET_BUFR@ +MET_BUFRLIB = @MET_BUFRLIB@ +MET_CAIRO = @MET_CAIRO@ +MET_CAIROINC = @MET_CAIROINC@ +MET_CAIROLIB = @MET_CAIROLIB@ +MET_CXX_STANDARD = @MET_CXX_STANDARD@ +MET_ECKIT = @MET_ECKIT@ +MET_ECKITINC = @MET_ECKITINC@ +MET_ECKITLIB = @MET_ECKITLIB@ +MET_FREETYPE = @MET_FREETYPE@ +MET_FREETYPEINC = @MET_FREETYPEINC@ +MET_FREETYPELIB = @MET_FREETYPELIB@ +MET_GRIB2C = @MET_GRIB2C@ +MET_GRIB2CINC = @MET_GRIB2CINC@ +MET_GRIB2CLIB = @MET_GRIB2CLIB@ +MET_GSL = @MET_GSL@ +MET_GSLINC = @MET_GSLINC@ +MET_GSLLIB = @MET_GSLLIB@ +MET_HDF = @MET_HDF@ +MET_HDF5 = @MET_HDF5@ +MET_HDF5INC = @MET_HDF5INC@ +MET_HDF5LIB = @MET_HDF5LIB@ +MET_HDFEOS = @MET_HDFEOS@ +MET_HDFEOSINC = @MET_HDFEOSINC@ +MET_HDFEOSLIB = @MET_HDFEOSLIB@ +MET_HDFINC = @MET_HDFINC@ +MET_HDFLIB = @MET_HDFLIB@ +MET_NETCDF = @MET_NETCDF@ +MET_NETCDFINC = @MET_NETCDFINC@ +MET_NETCDFLIB = @MET_NETCDFLIB@ +MET_PROJ = @MET_PROJ@ +MET_PROJINC = @MET_PROJINC@ +MET_PROJLIB = @MET_PROJLIB@ +MET_PYTHON_BIN_EXE = @MET_PYTHON_BIN_EXE@ +MET_PYTHON_CC = @MET_PYTHON_CC@ +MET_PYTHON_LD = @MET_PYTHON_LD@ +MKDIR_P = @MKDIR_P@ +OBJEXT = @OBJEXT@ +OPENMP_CFLAGS = @OPENMP_CFLAGS@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PYTHON_DEP_LIBS = @PYTHON_DEP_LIBS@ +PYTHON_MET_LIBS = @PYTHON_MET_LIBS@ +RANLIB = @RANLIB@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +UGRID_DEP_LIBS = @UGRID_DEP_LIBS@ +UGRID_MET_LIBS = @UGRID_MET_LIBS@ +VERSION = @VERSION@ +YACC = @YACC@ +YFLAGS = @YFLAGS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_F77 = @ac_ct_F77@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +sysconfdir = @sysconfdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +MAINTAINERCLEANFILES = Makefile.in + +# The library +noinst_LIBRARIES = libvx_ioda.a +libvx_ioda_a_SOURCES = \ + ioda.cc ioda.h \ + ioda_data_conf_info.cc ioda_data_conf_info.h + +libvx_ioda_a_CPPFLAGS = ${MET_CPPFLAGS} +all: all-am + +.SUFFIXES: +.SUFFIXES: .cc .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/libcode/vx_ioda/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/libcode/vx_ioda/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstLIBRARIES: + -test -z "$(noinst_LIBRARIES)" || rm -f $(noinst_LIBRARIES) + +libvx_ioda.a: $(libvx_ioda_a_OBJECTS) $(libvx_ioda_a_DEPENDENCIES) $(EXTRA_libvx_ioda_a_DEPENDENCIES) + $(AM_V_at)-rm -f libvx_ioda.a + $(AM_V_AR)$(libvx_ioda_a_AR) libvx_ioda.a $(libvx_ioda_a_OBJECTS) $(libvx_ioda_a_LIBADD) + $(AM_V_at)$(RANLIB) libvx_ioda.a + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_ioda_a-ioda.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_ioda_a-ioda_data_conf_info.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.cc.o: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $< + +.cc.obj: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +libvx_ioda_a-ioda.o: ioda.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_ioda_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_ioda_a-ioda.o -MD -MP -MF $(DEPDIR)/libvx_ioda_a-ioda.Tpo -c -o libvx_ioda_a-ioda.o `test -f 'ioda.cc' || echo '$(srcdir)/'`ioda.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_ioda_a-ioda.Tpo $(DEPDIR)/libvx_ioda_a-ioda.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ioda.cc' object='libvx_ioda_a-ioda.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_ioda_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_ioda_a-ioda.o `test -f 'ioda.cc' || echo '$(srcdir)/'`ioda.cc + +libvx_ioda_a-ioda.obj: ioda.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_ioda_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_ioda_a-ioda.obj -MD -MP -MF $(DEPDIR)/libvx_ioda_a-ioda.Tpo -c -o libvx_ioda_a-ioda.obj `if test -f 'ioda.cc'; then $(CYGPATH_W) 'ioda.cc'; else $(CYGPATH_W) '$(srcdir)/ioda.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_ioda_a-ioda.Tpo $(DEPDIR)/libvx_ioda_a-ioda.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ioda.cc' object='libvx_ioda_a-ioda.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_ioda_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_ioda_a-ioda.obj `if test -f 'ioda.cc'; then $(CYGPATH_W) 'ioda.cc'; else $(CYGPATH_W) '$(srcdir)/ioda.cc'; fi` + +libvx_ioda_a-ioda_data_conf_info.o: ioda_data_conf_info.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_ioda_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_ioda_a-ioda_data_conf_info.o -MD -MP -MF $(DEPDIR)/libvx_ioda_a-ioda_data_conf_info.Tpo -c -o libvx_ioda_a-ioda_data_conf_info.o `test -f 'ioda_data_conf_info.cc' || echo '$(srcdir)/'`ioda_data_conf_info.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_ioda_a-ioda_data_conf_info.Tpo $(DEPDIR)/libvx_ioda_a-ioda_data_conf_info.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ioda_data_conf_info.cc' object='libvx_ioda_a-ioda_data_conf_info.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_ioda_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_ioda_a-ioda_data_conf_info.o `test -f 'ioda_data_conf_info.cc' || echo '$(srcdir)/'`ioda_data_conf_info.cc + +libvx_ioda_a-ioda_data_conf_info.obj: ioda_data_conf_info.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_ioda_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_ioda_a-ioda_data_conf_info.obj -MD -MP -MF $(DEPDIR)/libvx_ioda_a-ioda_data_conf_info.Tpo -c -o libvx_ioda_a-ioda_data_conf_info.obj `if test -f 'ioda_data_conf_info.cc'; then $(CYGPATH_W) 'ioda_data_conf_info.cc'; else $(CYGPATH_W) '$(srcdir)/ioda_data_conf_info.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_ioda_a-ioda_data_conf_info.Tpo $(DEPDIR)/libvx_ioda_a-ioda_data_conf_info.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ioda_data_conf_info.cc' object='libvx_ioda_a-ioda_data_conf_info.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_ioda_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_ioda_a-ioda_data_conf_info.obj `if test -f 'ioda_data_conf_info.cc'; then $(CYGPATH_W) 'ioda_data_conf_info.cc'; else $(CYGPATH_W) '$(srcdir)/ioda_data_conf_info.cc'; fi` + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(LIBRARIES) +installdirs: +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." + -test -z "$(MAINTAINERCLEANFILES)" || rm -f $(MAINTAINERCLEANFILES) +clean: clean-am + +clean-am: clean-generic clean-noinstLIBRARIES mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/libvx_ioda_a-ioda.Po + -rm -f ./$(DEPDIR)/libvx_ioda_a-ioda_data_conf_info.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/libvx_ioda_a-ioda.Po + -rm -f ./$(DEPDIR)/libvx_ioda_a-ioda_data_conf_info.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-generic clean-noinstLIBRARIES cscopelist-am ctags \ + ctags-am distclean distclean-compile distclean-generic \ + distclean-tags distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic pdf pdf-am ps ps-am \ + tags tags-am uninstall uninstall-am + +.PRECIOUS: Makefile + + +# Include the project definitions + +include ${top_srcdir}/Make-include + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/libcode/vx_ioda/ioda.cc b/src/libcode/vx_ioda/ioda.cc new file mode 100644 index 0000000000..f85046e062 --- /dev/null +++ b/src/libcode/vx_ioda/ioda.cc @@ -0,0 +1,637 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2024 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "file_exists.h" + +#include "string_fxns.h" +#include "vx_log.h" +#include "nc_utils.h" +#include "ioda.h" + +using namespace std; +using namespace netCDF; + + +//////////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////////// + +static bool has_postfix(const std::string &, std::string const &); + +//////////////////////////////////////////////////////////////////////// + +bool iodaReader::check_missing_thresh(double value) const { + bool check = false; + for(int idx=0; idx " + << "trouble getting " << GET_NC_NAME(var) << "\n\n"; + exit(1); + } + } + return status; +} + +//////////////////////////////////////////////////////////////////////// + +bool iodaReader::get_meta_data_double(const char *metadata_key, double *metadata_buf) { + bool status = false; + + ConcatString metadata_name = find_meta_name(metadata_key, metadata_vars); + if(metadata_name.nonempty()) { + NcVar meta_var = get_var(f_in, metadata_name.c_str(), metadata_group_name); + if(IS_VALID_NC(meta_var)) { + static const char *method_name = "iodaReader::get_meta_data_double() -> "; + status = get_nc_data(&meta_var, metadata_buf, nlocs); + if (!status) mlog << Debug(3) << method_name + << "trouble getting " << metadata_name << "\n"; + } + } + if(status) { + for(int idx=0; idx= 8) { + for(int idx=0; idx *iodaReader::get_point_pairs( + const char *var_name_f, const char *var_name_o, + const char *group_name_f, const char *group_name_o, const int channel) { + ConcatString log_var_name_f = var_name_f; + ConcatString log_var_name_o = var_name_o; + vector obs_val_f(nlocs, bad_data_double); + vector obs_val_o(nlocs, bad_data_double); + + if (!read_obs_data(obs_val_f.data(), var_name_f, group_name_f, channel)) { + clear(); + exit(-1); + } + if (!read_obs_data(obs_val_o.data(), var_name_o, group_name_o, channel)) { + clear(); + exit(-1); + } + + point_pairs.clear(); + point_pairs.resize(nlocs); + for (int i=0; i 0) { + for (int idx=0; idx hdr_vld_block(nlocs*(ndatetime+1), 0); + vector hdr_vld_block2(nlocs, nullptr); + for (int i=0; i hdr_vld_block(nlocs*(ndatetime+1), 0); + status = get_nc_data(&in_hdr_vld_var, hdr_vld_block.data()); + + for (int i=0; i hdr_ut_arr(nlocs); + status = get_nc_data(hdr_vld_var, hdr_ut_arr.data()); + if (status) { + for (int i=0; i hdr_ut_arr(nlocs); + status = get_nc_data(hdr_vld_var, hdr_ut_arr.data()); + if (status) { + for (int i=0; i< nlocs; i++) { + vld_arr[i] = add_to_unixtime(base_ut, sec_per_unit, + hdr_ut_arr[i], no_leap_year); + } + } + } + return status; +} + +//////////////////////////////////////////////////////////////////////// + +void iodaReader::read_ioda(netCDF::NcFile *_f_in) { + static const char *method_name = "iodaReader::read_ioda() -> "; + bool error_out = true; + + f_in = _f_in; + + clear(); + + get_dim_names(f_in, &dim_names); + nlocs_name = find_meta_name("nlocs", dim_names); + if(nlocs_name.empty()) { + mlog << Error << "\n" << method_name + << "trouble getting the dimension name for nlocs\n\n"; + exit(1); + } + else { + nlocs = get_dim_value(f_in, nlocs_name.c_str(), error_out); // number of locations + if(nlocs == 0) { + mlog << Error << "\n" << method_name + << "No ioda record (the dimension size of " + << nlocs_name << " is " << nlocs << ")\n\n"; + exit(1); + } + } + + if ( find_ioda_format(_f_in) == e_ioda_format::v1 ) { + get_obs_metadata_names_v1(); + } + else { + get_obs_metadata_names_v2(); + } + if(mlog.verbosity_level() >= 6) { + for(int idx=0; idx= var_dim_names.n()) { + mlog << Error << "\n" << method_name + << "Channel (" << channel << ") is out of bound (" << var_dim_names.n() + << ") for " << log_var_name << "\n\n"; + return status; + } + if (1 == var_dim_names.n()) { + mlog << Debug(1) << method_name + << "Channel (" << channel << ") is configured to the variable " + << log_var_name << " which does not have the chenell dimension\n"; + status = get_nc_data(&obs_var, data_buf, nlocs); + } + else { + LongArray lengths; + LongArray offsets; + if (nlocs_name == var_dim_names[0]) { + lengths.add(nlocs); + lengths.add(1); + offsets.add(0); + offsets.add(channel); + } + else { + lengths.add(1); + lengths.add(nlocs); + offsets.add(0); + offsets.add(channel); + } + status = get_nc_data(&obs_var, data_buf, lengths, offsets); + } + } + + if (!status) { + mlog << Error << "\n" << method_name + << "trouble getting " << log_var_name<< "\n\n"; + } + + return status; + +} + +//////////////////////////////////////////////////////////////////////// + +bool iodaReader::read_string_data(const char *var_name, vector &hdr_data, int str_length) { + bool status = false; + char hdr_val[512]; + static const char *method_name = "iodaReader::read_string_data -> "; + static const char *method_name_s = "iodaReader::read_string_data() "; + + hdr_data.clear(); + NcVar hdr_var = get_var(f_in, var_name, metadata_group_name); + if (IS_INVALID_NC(hdr_var)) return status; + hdr_data.resize(nlocs, ""); + if (NC_STRING == GET_NC_TYPE_ID(hdr_var)) { + vector hdr_data2(nlocs, nullptr); + for (int i=0; i hdr_data2(nlocs*(str_length+1),0); + if ((status = get_meta_data_chars(hdr_var, hdr_data2.data()))) { + for (int i=0; i= postfix_len) { + return (0 == str_buf.compare(buf_len - postfix_len, postfix_len, postfix)); + } else { + return false; + } +} + +//////////////////////////////////////////////////////////////////////// + diff --git a/src/libcode/vx_ioda/ioda.h b/src/libcode/vx_ioda/ioda.h new file mode 100644 index 0000000000..f0bc3a3124 --- /dev/null +++ b/src/libcode/vx_ioda/ioda.h @@ -0,0 +1,118 @@ + + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __IODA_H__ +#define __IODA_H__ + + +//////////////////////////////////////////////////////////////////////// + +#include + +#include "pair_base.h" +#include "ioda_data_conf_info.h" + +//////////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////////// + +enum class e_ioda_format { v1, v2 }; + +//////////////////////////////////////////////////////////////////////// + +struct iodaHeaders { + void clear(); +}; + +//////////////////////////////////////////////////////////////////////// + +struct iodaMetadata { + + void clear(); +}; + +//////////////////////////////////////////////////////////////////////// + +class iodaReader { + public: + + int nlocs; + int nrecs; + int nvars; + int ndatetime; + int nstring; + + StringArray dim_names; + StringArray metadata_vars; + StringArray obs_value_vars; + + ConcatString nlocs_name; + ConcatString datetime_name; + ConcatString lon_name; + ConcatString lat_name; + ConcatString msg_type_name; + ConcatString station_id_name; + + std::vector lat_arr; + std::vector lon_arr; + std::vector elv_arr; + std::vector vld_arr; // valid times` + + std::vector msg_types; + std::vector station_ids; + + std::vector obs_pres_arr; + std::vector obs_hght_arr; + + private: + netCDF::NcFile *f_in; + IODADataConfInfo conf_info; + + e_ioda_format ioda_format; + std::vector point_pairs; + + public: + bool check_missing_thresh(double value) const; + void clear(); + e_ioda_format get_format_ver() const; + std::vector *get_point_pairs(const char *var_name_f, const char *var_name_o, + const char *group_name_f=nullptr, + const char *group_name_o=nullptr, + const int channel=bad_data_int); + bool is_in_metadata_map(const std::string &metadata_key, const StringArray &available_list); + void read_ioda(netCDF::NcFile *f_in); + void set_data_config(const char *, const char *); + + private: + e_ioda_format find_ioda_format(netCDF::NcFile *_f_in); + ConcatString find_meta_name(const std::string &meta_key, const StringArray &available_names); + bool get_meta_data_chars(netCDF::NcVar &var, char *metadata_buf) const; + bool get_meta_data_double(const char *metadata_key, double *metadata_buf); + void get_obs_metadata_names_v1(); + void get_obs_metadata_names_v2(); + void read_header(); + void read_metadata_names(); + bool read_obs_data(double *data_buf, const char *var_name, + const char *group_name, const int channel); + bool read_string_data(const char *var_name, + std::vector &hdr_data, int str_length); + bool read_time(); + bool read_time_as_number(netCDF::NcVar *hdr_vld_var); + +}; + + +//////////////////////////////////////////////////////////////////////// + +inline e_ioda_format iodaReader::get_format_ver() const { return ioda_format; }; + +//////////////////////////////////////////////////////////////////////// + +#endif /* __IODA_H__ */ + + +//////////////////////////////////////////////////////////////////////// + diff --git a/src/libcode/vx_ioda/ioda_data_conf_info.cc b/src/libcode/vx_ioda/ioda_data_conf_info.cc new file mode 100644 index 0000000000..0aed5ffe33 --- /dev/null +++ b/src/libcode/vx_ioda/ioda_data_conf_info.cc @@ -0,0 +1,104 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2024 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include +#include + +#include "ioda_data_conf_info.h" + +#include "vx_log.h" + +using namespace std; + + +//////////////////////////////////////////////////////////////////////// +// +// Code for class IODADataConfInfo +// +//////////////////////////////////////////////////////////////////////// + +IODADataConfInfo::IODADataConfInfo() { + init_from_scratch(); +} + +//////////////////////////////////////////////////////////////////////// + +IODADataConfInfo::~IODADataConfInfo() { + clear(); +} + +//////////////////////////////////////////////////////////////////////// + +void IODADataConfInfo::init_from_scratch() { + clear(); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void IODADataConfInfo::clear() { + + // Initialize values + metadata_map.clear(); + missing_thresh.clear(); + obs_name_map.clear(); + obs_to_qc_map.clear(); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void IODADataConfInfo::read_data_config(const char *default_file_name, + const char *user_file_name) { + + // Read the config file constants + conf.read(replace_path(config_const_filename).c_str()); + + conf.read(replace_path(DEF_DATA_CONFIG_NAME).c_str()); + + // Read the default config file + conf.read(default_file_name); + + // Read the user-specified config file + if (user_file_name && strcmp(user_file_name, "")) conf.read(user_file_name); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void IODADataConfInfo::process_data_config() { + static const char *method_name = "IODADataConfInfo::process_data_config() -> "; + + // Initialize + clear(); + + // Conf: missing_thresh + missing_thresh = conf.lookup_thresh_array(conf_key_missing_thresh, false); + + // Conf: obs_name_map + obs_name_map = parse_conf_obs_name_map(&conf); + metadata_map = parse_conf_metadata_map(&conf); + obs_to_qc_map = parse_conf_obs_to_qc_map(&conf); + + mlog << Debug(5) << method_name + << "obs_name_map: " << obs_name_map.size() + << ", metadata_map: " << metadata_map.size() + << ", obs_to_qc_map: " << obs_to_qc_map.size() << "\n"; + return; +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_ioda/ioda_data_conf_info.h b/src/libcode/vx_ioda/ioda_data_conf_info.h new file mode 100644 index 0000000000..df8cb3a5cb --- /dev/null +++ b/src/libcode/vx_ioda/ioda_data_conf_info.h @@ -0,0 +1,72 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2024 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#ifndef __IODADATA_CONF_INFO_H__ +#define __IODADATA_CONF_INFO_H__ + +//////////////////////////////////////////////////////////////////////// + +#include + +#include "vx_config.h" +#include "vx_analysis_util.h" +#include "vx_grid.h" +#include "vx_util.h" +#include "vx_cal.h" +#include "vx_math.h" + +//////////////////////////////////////////////////////////////////////// + +static constexpr int string_data_len = 512; + +static constexpr char metadata_group_name[] = "MetaData"; +static constexpr char qc_group_name[] = "QCFlags"; +static constexpr char qc_postfix[] = "PreQC"; +static constexpr char obs_group_name[] = "ObsValue"; +static constexpr char derived_obs_group_name[] = "DerivedObsValue"; + +static constexpr char DEF_DATA_CONFIG_NAME[] = "MET_BASE/config/IODADataConfig_default"; + +//////////////////////////////////////////////////////////////////////// + + +class IODADataConfInfo { + + private: + + void init_from_scratch(); + + public: + + // IODAData configuration object + MetConfig conf; + + ThreshArray missing_thresh; // Fill value thresh array + + std::map obs_name_map; + std::map metadata_map; + std::map obs_to_qc_map; + + IODADataConfInfo(); + ~IODADataConfInfo(); + + void clear(); + + std::map getObsVarMap() const { return obs_name_map; } + + void read_data_config(const char *, const char *); + void process_data_config(); +}; + +//////////////////////////////////////////////////////////////////////// + +#endif /* __IODADATA_CONF_INFO_H__ */ + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index 177febf1df..8a0475f8d2 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -1315,6 +1315,14 @@ bool get_nc_data(NcVar *var, time_t *data) { //////////////////////////////////////////////////////////////////////// +bool get_nc_data(NcVar *var, unixtime *data) { + bool return_status = get_nc_data_(var, data, (unixtime)bad_data_int); + + return return_status; +} + +//////////////////////////////////////////////////////////////////////// + bool get_nc_data(NcVar *var, int *data) { bool return_status = get_nc_data_(var, data, bad_data_int); return return_status; diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index 671b3bf609..e2bad06942 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -225,6 +225,7 @@ extern bool get_nc_data(netCDF::NcVar *, uchar *data, bool allow_conversion=fal extern bool get_nc_data(netCDF::NcVar *, float *data); extern bool get_nc_data(netCDF::NcVar *, double *data); extern bool get_nc_data(netCDF::NcVar *, time_t *data); +extern bool get_nc_data(netCDF::NcVar *, unixtime *data); extern bool get_nc_data(netCDF::NcVar *, unsigned short *data); extern bool get_nc_data(netCDF::NcVar *, int *data, const LongArray &curs); diff --git a/src/libcode/vx_shapedata/engine.cc b/src/libcode/vx_shapedata/engine.cc index 0809afd999..33095e6305 100644 --- a/src/libcode/vx_shapedata/engine.cc +++ b/src/libcode/vx_shapedata/engine.cc @@ -3029,7 +3029,7 @@ void write_engine_stats(ModeFuzzyEngine & eng, const Grid & grid, AsciiTable & a eng.conf_info.conf.output_precision()); at.set_bad_data_value(bad_data_double); // Set the bad data value at.set_bad_data_str(na_str); // Set the bad data string - at.set_delete_trailing_blank_rows(1); // No trailing blank rows + at.set_delete_trailing_blank_rows(true); // No trailing blank rows // // calculate n_valid diff --git a/src/libcode/vx_stat_out/Makefile.am b/src/libcode/vx_stat_out/Makefile.am index 219ba60ddf..83473eaa97 100644 --- a/src/libcode/vx_stat_out/Makefile.am +++ b/src/libcode/vx_stat_out/Makefile.am @@ -14,5 +14,6 @@ noinst_LIBRARIES = libvx_stat_out.a libvx_stat_out_a_SOURCES = \ stat_columns.cc stat_columns.h \ stat_hdr_columns.cc stat_hdr_columns.h \ + stat_hdr_info.cc stat_hdr_info.h \ vx_stat_out.h libvx_stat_out_a_CPPFLAGS = ${MET_CPPFLAGS} diff --git a/src/libcode/vx_stat_out/Makefile.in b/src/libcode/vx_stat_out/Makefile.in index 77d31a4ea1..117c6ff9bc 100644 --- a/src/libcode/vx_stat_out/Makefile.in +++ b/src/libcode/vx_stat_out/Makefile.in @@ -108,7 +108,8 @@ am__v_AR_1 = libvx_stat_out_a_AR = $(AR) $(ARFLAGS) libvx_stat_out_a_LIBADD = am_libvx_stat_out_a_OBJECTS = libvx_stat_out_a-stat_columns.$(OBJEXT) \ - libvx_stat_out_a-stat_hdr_columns.$(OBJEXT) + libvx_stat_out_a-stat_hdr_columns.$(OBJEXT) \ + libvx_stat_out_a-stat_hdr_info.$(OBJEXT) libvx_stat_out_a_OBJECTS = $(am_libvx_stat_out_a_OBJECTS) AM_V_P = $(am__v_P_@AM_V@) am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) @@ -126,7 +127,8 @@ DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) depcomp = $(SHELL) $(top_srcdir)/depcomp am__maybe_remake_depfiles = depfiles am__depfiles_remade = ./$(DEPDIR)/libvx_stat_out_a-stat_columns.Po \ - ./$(DEPDIR)/libvx_stat_out_a-stat_hdr_columns.Po + ./$(DEPDIR)/libvx_stat_out_a-stat_hdr_columns.Po \ + ./$(DEPDIR)/libvx_stat_out_a-stat_hdr_info.Po am__mv = mv -f AM_V_lt = $(am__v_lt_@AM_V@) am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) @@ -349,6 +351,7 @@ noinst_LIBRARIES = libvx_stat_out.a libvx_stat_out_a_SOURCES = \ stat_columns.cc stat_columns.h \ stat_hdr_columns.cc stat_hdr_columns.h \ + stat_hdr_info.cc stat_hdr_info.h \ vx_stat_out.h libvx_stat_out_a_CPPFLAGS = ${MET_CPPFLAGS} @@ -402,6 +405,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_stat_out_a-stat_columns.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_stat_out_a-stat_hdr_columns.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libvx_stat_out_a-stat_hdr_info.Po@am__quote@ # am--include-marker $(am__depfiles_remade): @$(MKDIR_P) $(@D) @@ -451,6 +455,20 @@ libvx_stat_out_a-stat_hdr_columns.obj: stat_hdr_columns.cc @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_stat_out_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_stat_out_a-stat_hdr_columns.obj `if test -f 'stat_hdr_columns.cc'; then $(CYGPATH_W) 'stat_hdr_columns.cc'; else $(CYGPATH_W) '$(srcdir)/stat_hdr_columns.cc'; fi` +libvx_stat_out_a-stat_hdr_info.o: stat_hdr_info.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_stat_out_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_stat_out_a-stat_hdr_info.o -MD -MP -MF $(DEPDIR)/libvx_stat_out_a-stat_hdr_info.Tpo -c -o libvx_stat_out_a-stat_hdr_info.o `test -f 'stat_hdr_info.cc' || echo '$(srcdir)/'`stat_hdr_info.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_stat_out_a-stat_hdr_info.Tpo $(DEPDIR)/libvx_stat_out_a-stat_hdr_info.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='stat_hdr_info.cc' object='libvx_stat_out_a-stat_hdr_info.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_stat_out_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_stat_out_a-stat_hdr_info.o `test -f 'stat_hdr_info.cc' || echo '$(srcdir)/'`stat_hdr_info.cc + +libvx_stat_out_a-stat_hdr_info.obj: stat_hdr_info.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_stat_out_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT libvx_stat_out_a-stat_hdr_info.obj -MD -MP -MF $(DEPDIR)/libvx_stat_out_a-stat_hdr_info.Tpo -c -o libvx_stat_out_a-stat_hdr_info.obj `if test -f 'stat_hdr_info.cc'; then $(CYGPATH_W) 'stat_hdr_info.cc'; else $(CYGPATH_W) '$(srcdir)/stat_hdr_info.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/libvx_stat_out_a-stat_hdr_info.Tpo $(DEPDIR)/libvx_stat_out_a-stat_hdr_info.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='stat_hdr_info.cc' object='libvx_stat_out_a-stat_hdr_info.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libvx_stat_out_a_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o libvx_stat_out_a-stat_hdr_info.obj `if test -f 'stat_hdr_info.cc'; then $(CYGPATH_W) 'stat_hdr_info.cc'; else $(CYGPATH_W) '$(srcdir)/stat_hdr_info.cc'; fi` + ID: $(am__tagged_files) $(am__define_uniq_tagged_files); mkid -fID $$unique tags: tags-am @@ -577,6 +595,7 @@ clean-am: clean-generic clean-noinstLIBRARIES mostlyclean-am distclean: distclean-am -rm -f ./$(DEPDIR)/libvx_stat_out_a-stat_columns.Po -rm -f ./$(DEPDIR)/libvx_stat_out_a-stat_hdr_columns.Po + -rm -f ./$(DEPDIR)/libvx_stat_out_a-stat_hdr_info.Po -rm -f Makefile distclean-am: clean-am distclean-compile distclean-generic \ distclean-tags @@ -624,6 +643,7 @@ installcheck-am: maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/libvx_stat_out_a-stat_columns.Po -rm -f ./$(DEPDIR)/libvx_stat_out_a-stat_hdr_columns.Po + -rm -f ./$(DEPDIR)/libvx_stat_out_a-stat_hdr_info.Po -rm -f Makefile maintainer-clean-am: distclean-am maintainer-clean-generic diff --git a/src/libcode/vx_stat_out/stat_hdr_info.cc b/src/libcode/vx_stat_out/stat_hdr_info.cc new file mode 100644 index 0000000000..19187b5534 --- /dev/null +++ b/src/libcode/vx_stat_out/stat_hdr_info.cc @@ -0,0 +1,453 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2024 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include +#include +#include + +#include "vx_log.h" + +#include "stat_hdr_info.h" + +using namespace std; + +//////////////////////////////////////////////////////////////////////// + +static const string case_str = "CASE"; + +//////////////////////////////////////////////////////////////////////// +// +// Code for StatHdrInfo structure. +// +//////////////////////////////////////////////////////////////////////// + +StatHdrInfo::StatHdrInfo() { + clear(); +} + +//////////////////////////////////////////////////////////////////////// + +void StatHdrInfo::clear() { + model.clear(); + desc.clear(); + fcst_lead.clear(); + fcst_valid_beg = fcst_valid_end = (unixtime) 0; + obs_lead.clear(); + obs_valid_beg = obs_valid_end = (unixtime) 0; + fcst_var.clear(); + fcst_units.clear(); + fcst_lev.clear(); + obs_var.clear(); + obs_units.clear(); + obs_lev.clear(); + obtype.clear(); + vx_mask.clear(); + interp_mthd.clear(); + interp_pnts.clear(); + fcst_thresh.clear(); + obs_thresh.clear(); + cov_thresh.clear(); + alpha.clear(); + + return; +} + +//////////////////////////////////////////////////////////////////////// +// +// Keep track the unique STAT header entries for each line. +// +//////////////////////////////////////////////////////////////////////// + +void StatHdrInfo::add(const STATLine &line) { + ConcatString cs; + if(!model.has(line.model())) + model.add(line.model()); + if(!desc.has(line.desc())) + desc.add(line.desc()); + if(!fcst_lead.has(line.fcst_lead())) + fcst_lead.add(line.fcst_lead()); + if(fcst_valid_beg == (unixtime) 0 || line.fcst_valid_beg() < fcst_valid_beg) + fcst_valid_beg = line.fcst_valid_beg(); + if(fcst_valid_end == (unixtime) 0 || line.fcst_valid_end() > fcst_valid_end) + fcst_valid_end = line.fcst_valid_end(); + if(!obs_lead.has(line.obs_lead())) + obs_lead.add(line.obs_lead()); + if(obs_valid_beg == (unixtime) 0 || line.obs_valid_beg() < obs_valid_beg) + obs_valid_beg = line.obs_valid_beg(); + if(obs_valid_end == (unixtime) 0 || line.obs_valid_end() > obs_valid_end) + obs_valid_end = line.obs_valid_end(); + if(!fcst_var.has(line.fcst_var())) + fcst_var.add(line.fcst_var()); + if(!fcst_units.has(line.fcst_units())) + fcst_units.add(line.fcst_units()); + if(!fcst_lev.has(line.fcst_lev())) + fcst_lev.add(line.fcst_lev()); + if(!obs_var.has(line.obs_var())) + obs_var.add(line.obs_var()); + if(!obs_units.has(line.obs_units())) + obs_units.add(line.obs_units()); + if(!obs_lev.has(line.obs_lev())) + obs_lev.add(line.obs_lev()); + if(!obtype.has(line.obtype())) + obtype.add(line.obtype()); + if(!vx_mask.has(line.vx_mask())) + vx_mask.add(line.vx_mask()); + if(!interp_mthd.has(line.interp_mthd())) + interp_mthd.add(line.interp_mthd()); + if(!interp_pnts.has(line.interp_pnts())) + interp_pnts.add(line.interp_pnts()); + cs = line.get_item("FCST_THRESH", false); + cs.strip_paren(); + if(!fcst_thresh.has(cs)) fcst_thresh.add(cs); + cs = line.get_item("OBS_THRESH", false); + cs.strip_paren(); + if(!obs_thresh.has(cs)) obs_thresh.add(cs); + cs = line.get_item("COV_THRESH", false); + cs.strip_paren(); + if(!cov_thresh.has(cs)) cov_thresh.add(cs); + if(!alpha.has(line.alpha())) + alpha.add(line.alpha()); + + return; +} + +//////////////////////////////////////////////////////////////////////// +// +// Check for and print debug and warning messages about multiple header +// column values found. +// +//////////////////////////////////////////////////////////////////////// + +void StatHdrInfo::check_shc(const ConcatString &cur_case) { + + // MODEL + if(model.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << model.n() + << " unique MODEL values: " + << write_css(model) << "\n"; + } + + // DESC + if(desc.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << desc.n() + << " unique DESC values: " + << write_css(desc) << "\n"; + } + + // FCST_LEAD + if(fcst_lead.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << fcst_lead.n() + << " unique FCST_LEAD values: " + << write_css_hhmmss(fcst_lead) << "\n"; + } + + // OBS_LEAD + if(obs_lead.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << obs_lead.n() + << " unique OBS_LEAD values: " + << write_css_hhmmss(obs_lead) << "\n"; + } + + // FCST_VAR + if(fcst_var.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << fcst_var.n() + << " unique FCST_VAR values: " + << write_css(fcst_var) << "\n"; + } + + // FCST_UNITS + if(fcst_units.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << fcst_units.n() + << " unique FCST_UNITS values: " + << write_css(fcst_units) << "\n"; + } + + // FCST_LEV + if(fcst_lev.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << fcst_lev.n() + << " unique FCST_LEV values: " + << write_css(fcst_lev) << "\n"; + } + + // OBS_VAR + if(obs_var.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << obs_var.n() + << " unique OBS_VAR values: " + << write_css(obs_var) << "\n"; + } + + // OBS_UNITS + if(obs_units.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << obs_units.n() + << " unique OBS_UNITS values: " + << write_css(obs_units) << "\n"; + } + + // OBS_LEV + if(obs_lev.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << obs_lev.n() + << " unique OBS_LEV values: " + << write_css(obs_lev) << "\n"; + } + + // OBTYPE + if(obtype.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << obtype.n() + << " unique OBTYPE values: " + << write_css(obtype) << "\n"; + } + + // VX_MASK + if(vx_mask.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << vx_mask.n() + << " unique VX_MASK values: " + << write_css(vx_mask) << "\n"; + } + + // INTERP_MTHD + if(interp_mthd.n() > 1) { + mlog << Warning + << "For case \"" << cur_case << "\", found " + << interp_mthd.n() + << " unique INTERP_MTHD values: " + << write_css(interp_mthd) << ".\n"; + } + + // INTERP_PNTS + if(interp_pnts.n() > 1) { + mlog << Warning + << "For case \"" << cur_case << "\", found " + << interp_pnts.n() + << " unique INTERP_PNTS values: " + << write_css(interp_pnts) << ".\n"; + } + + // FCST_THRESH + if(fcst_thresh.n() > 1) { + mlog << Warning + << "For case \"" << cur_case << "\", found " + << fcst_thresh.n() + << " unique FCST_THRESH values: " + << write_css(fcst_thresh) << "\n"; + } + + // OBS_THRESH + if(obs_thresh.n() > 1) { + mlog << Warning + << "For case \"" << cur_case << "\", found " + << obs_thresh.n() + << " unique OBS_THRESH values: " + << write_css(obs_thresh) << "\n"; + } + + // COV_THRESH + if(cov_thresh.n() > 1) { + mlog << Warning + << "For case \"" << cur_case << "\", found " + << cov_thresh.n() + << " unique COV_THRESH values: " + << write_css(cov_thresh) << ".\n"; + } + + // ALPHA + if(alpha.n() > 1) { + mlog << Warning + << "For case \"" << cur_case << "\", found " + << alpha.n() + << " unique ALPHA values: " + << write_css(alpha) << ".\n"; + } + + return; +} + +//////////////////////////////////////////////////////////////////////// +// +// Use the StatHdrInfo struct to populate a StatHdrColumns object +// +//////////////////////////////////////////////////////////////////////// + +StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, + const StringArray &case_cols, + const StringArray &hdr_cols, + const StringArray &hdr_vals, + const STATLineType lt) { + ThreshArray ta; + ConcatString css; + double out_alpha; + int wdth; + StatHdrColumns shc; + + // MODEL + shc.set_model(get_col_css(cur_case, "MODEL", model, false).c_str()); + + // DESC + shc.set_desc(get_col_css(cur_case, "DESC", desc, false).c_str()); + + // FCST_LEAD + css = write_css_hhmmss(fcst_lead); + if(fcst_lead.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << fcst_lead.n() + << " unique FCST_LEAD values: " << css << "\n"; + } + shc.set_fcst_lead_sec(fcst_lead.max()); + + // FCST_VALID_BEG, FCST_VALID_END + shc.set_fcst_valid_beg(fcst_valid_beg); + shc.set_fcst_valid_end(fcst_valid_end); + + // OBS_LEAD + css = write_css_hhmmss(obs_lead); + if(obs_lead.n() > 1) { + mlog << Debug(2) + << "For case \"" << cur_case << "\", found " + << obs_lead.n() + << " unique OBS_LEAD values: " << css << "\n"; + } + shc.set_obs_lead_sec(obs_lead.max()); + + // OBS_VALID_BEG, OBS_VALID_END + shc.set_obs_valid_beg(obs_valid_beg); + shc.set_obs_valid_end(obs_valid_end); + + // FCST_VAR + shc.set_fcst_var(get_col_css(cur_case, "FCST_VAR", fcst_var, false)); + + // FCST_UNITS + shc.set_fcst_units(get_col_css(cur_case, "FCST_UNITS", fcst_units, false)); + + // FCST_LEV + shc.set_fcst_lev(get_col_css(cur_case, "FCST_LEV", fcst_lev, false).c_str()); + + // OBS_VAR + shc.set_obs_var(get_col_css(cur_case, "OBS_VAR", obs_var, false)); + + // OBS_UNITS + shc.set_obs_units(get_col_css(cur_case, "OBS_UNITS", obs_units, false)); + + // OBS_LEV + shc.set_obs_lev(get_col_css(cur_case, "OBS_LEV", obs_lev, false).c_str()); + + // OBTYPE + shc.set_obtype(get_col_css(cur_case, "OBTYPE", obtype, false).c_str()); + + // VX_MASK + shc.set_mask(get_col_css(cur_case, "VX_MASK", vx_mask, false).c_str()); + + // INTERP_MTHD + shc.set_interp_mthd(get_col_css(cur_case, "INTERP_MTHD", interp_mthd, true)); + + // INTERP_PNTS + css = write_css(interp_pnts); + if(interp_pnts.n() == 0 || interp_pnts.n() > 1) { + mlog << Warning + << "For case \"" << cur_case << "\", found " + << interp_pnts.n() + << " unique INTERP_PNTS values: " << css << ".\n"; + wdth = bad_data_int; + } + else { + wdth = nint(sqrt(interp_pnts[0])); + } + shc.set_interp_wdth(wdth); + + // FCST_THRESH + ta.clear(); + ta.add_css(get_col_css(cur_case, "FCST_THRESH", fcst_thresh, true).c_str()); + shc.set_fcst_thresh(ta); + + // OBS_THRESH + ta.clear(); + ta.add_css(get_col_css(cur_case, "OBS_THRESH", obs_thresh, true).c_str()); + shc.set_obs_thresh(ta); + + // COV_THRESH + ta.clear(); + ta.add_css(get_col_css(cur_case, "COV_THRESH", cov_thresh, true).c_str()); + shc.set_cov_thresh(ta); + + // ALPHA + css = write_css(alpha); + if(alpha.n() == 0 || alpha.n() > 1) { + mlog << Warning + << "For case \"" << cur_case << "\", found " + << alpha.n() + << " unique ALPHA values: " << css << ".\n"; + out_alpha = bad_data_double; + } + else { + out_alpha = alpha[0]; + } + shc.set_alpha(out_alpha); + + // LINE_TYPE + shc.set_line_type(statlinetype_to_string(lt)); + + // Apply the -set_hdr options + StringArray case_vals = cur_case.split(":"); + shc.apply_set_hdr_opts(hdr_cols, hdr_vals, case_cols, case_vals); + + return shc; +} + +//////////////////////////////////////////////////////////////////////// + +ConcatString StatHdrInfo::get_col_css(const ConcatString &cur_case, + const char *col_name, + const StringArray &col_vals, + bool warning) const { + + // Build comma-separated list of column values + ConcatString css(write_css(col_vals)); + + // Check for multiple entries + if(col_vals.n() > 1) { + ConcatString msg; + msg << "For case \"" << cur_case << "\", found " + << col_vals.n() << " unique " << col_name + << " values: " << css << "\n"; + if(warning) mlog << Warning << msg; + else mlog << Debug(2) << msg; + } + + return css; +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_stat_out/stat_hdr_info.h b/src/libcode/vx_stat_out/stat_hdr_info.h new file mode 100644 index 0000000000..f2a2e0ffe6 --- /dev/null +++ b/src/libcode/vx_stat_out/stat_hdr_info.h @@ -0,0 +1,54 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2024 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#ifndef __STAT_HDR_INFO_H__ +#define __STAT_HDR_INFO_H__ + +//////////////////////////////////////////////////////////////////////// + +#include "vx_config.h" +#include "vx_util.h" +#include "vx_analysis_util.h" + +#include "stat_hdr_columns.h" + +//////////////////////////////////////////////////////////////////////// + +struct StatHdrInfo { + StringArray model, desc; + StringArray fcst_var, fcst_units, fcst_lev; + StringArray obs_var, obs_units, obs_lev; + StringArray obtype, vx_mask, interp_mthd; + StringArray fcst_thresh, obs_thresh, cov_thresh; + NumArray fcst_lead, obs_lead, interp_pnts, alpha; + unixtime fcst_valid_beg, fcst_valid_end; + unixtime obs_valid_beg, obs_valid_end; + + StatHdrInfo(); + + void clear(); + void add(const STATLine &line); + void check_shc(const ConcatString &cur_case); + StatHdrColumns get_shc(const ConcatString &cur_case, + const StringArray &case_cols, + const StringArray &hdr_cols, + const StringArray &hdr_vals, + const STATLineType lt); + ConcatString get_col_css(const ConcatString &cur_case, + const char *col_name, + const StringArray &col_vals, + bool warning) const; +}; + +//////////////////////////////////////////////////////////////////////// + +#endif // __STAT_HDR_INFO_H__ + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_stat_out/vx_stat_out.h b/src/libcode/vx_stat_out/vx_stat_out.h index 3c2f669cc4..9746c4d1e4 100644 --- a/src/libcode/vx_stat_out/vx_stat_out.h +++ b/src/libcode/vx_stat_out/vx_stat_out.h @@ -20,6 +20,7 @@ #include "stat_columns.h" #include "stat_hdr_columns.h" +#include "stat_hdr_info.h" //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_statistics/pair_base.cc b/src/libcode/vx_statistics/pair_base.cc index d6d44f101b..a4c74c535e 100644 --- a/src/libcode/vx_statistics/pair_base.cc +++ b/src/libcode/vx_statistics/pair_base.cc @@ -35,15 +35,38 @@ static bool sort_obs(ob_val_t a, ob_val_t b) { return a.val &m) { - // Check for constant length - if(sa.n() != ta.n()) { - mlog << Error << "\nVxPairBase::set_mpr_thresh() -> " - << "the \"" << conf_key_mpr_column << "\" (" - << write_css(sa) << ") and \"" << conf_key_mpr_thresh - << "\" (" << write_css(ta) - << ") config file entries must have the same length!\n\n"; - exit(1); - } + mpr_thr_inc_map = m; + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void VxPairBase::set_mpr_str_inc_map(const map &m) { + + mpr_str_inc_map = m; + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void VxPairBase::set_mpr_str_exc_map(const map &m) { - mpr_column = sa; - mpr_thresh = ta; + mpr_str_exc_map = m; return; } diff --git a/src/libcode/vx_statistics/pair_base.h b/src/libcode/vx_statistics/pair_base.h index a734d1af31..62e5ec5c03 100644 --- a/src/libcode/vx_statistics/pair_base.h +++ b/src/libcode/vx_statistics/pair_base.h @@ -32,20 +32,34 @@ struct ob_val_t { std::string qc; }; -struct station_values_t { +struct station_value_base_t { - void clear(); + void clear_base(); std::string typ; std::string sid; double lat; double lon; - double x; - double y; - double wgt; unixtime ut; double lvl; double elv; +}; + +struct point_pair_t : station_value_base_t { + + void clear(); + + double fval; + double oval; +}; + +struct station_values_t : station_value_base_t { + + void clear(); + + double x; + double y; + double wgt; double fcmn; double fcsd; double ocmn; @@ -272,8 +286,15 @@ class VxPairBase { ////////////////////////////////////////////////////////////////// - StringArray mpr_column; // Names of MPR columns or diffs of columns - ThreshArray mpr_thresh; // Filtering thresholds for the MPR columns + // Mapping of numeric MPR columns or diffs of colums to + // inclusion thresholds + std::map mpr_thr_inc_map; + + // Mapping of string MPR columns to list of inclusion + // and exclusion strings + std::map mpr_str_inc_map; + std::map mpr_str_exc_map; + ////////////////////////////////////////////////////////////////// @@ -364,7 +385,9 @@ class VxPairBase { void set_interp(int i_interp, InterpMthd mthd, int width, GridTemplateFactory::GridTemplates shape); - void set_mpr_thresh(const StringArray &, const ThreshArray &); + void set_mpr_thr_inc_map(const std::map &); + void set_mpr_str_inc_map(const std::map &); + void set_mpr_str_exc_map(const std::map &); void set_climo_cdf_info_ptr(const ClimoCDFInfo *); diff --git a/src/libcode/vx_statistics/pair_data_point.cc b/src/libcode/vx_statistics/pair_data_point.cc index f913767739..20dc2d38e2 100644 --- a/src/libcode/vx_statistics/pair_data_point.cc +++ b/src/libcode/vx_statistics/pair_data_point.cc @@ -606,7 +606,7 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, // Check matched pair filtering options ConcatString reason_cs; if(!check_mpr_thresh(fcst_v, obs_v, cpi, - mpr_column, mpr_thresh, &reason_cs)) { + mpr_thr_inc_map, &reason_cs)) { if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { mlog << Debug(REJECT_DEBUG_LEVEL) @@ -742,13 +742,13 @@ bool check_fo_thresh(double f, double o, const ClimoPntInfo &cpi, //////////////////////////////////////////////////////////////////////// bool check_mpr_thresh(double f, double o, const ClimoPntInfo &cpi, - const StringArray &col_sa, const ThreshArray &col_ta, + const map &m, ConcatString *reason_ptr) { // Initialize if(reason_ptr) reason_ptr->erase(); // Check arrays - if(col_sa.n() == 0 || col_ta.n() == 0) return true; + if(m.size() == 0) return true; bool keep = true; bool absv = false; @@ -756,18 +756,18 @@ bool check_mpr_thresh(double f, double o, const ClimoPntInfo &cpi, ConcatString cs; double v, v_cur; - // Loop over all the column filter names - for(int i=0; i &m) { // Check for no work to be done - if(col_sa.n() == 0 && col_ta.n() == 0) return; - - // Check for constant length - if(col_sa.n() != col_ta.n()) { - mlog << Error << "\napply_mpr_thresh_mask() -> " - << "the \"" << conf_key_mpr_column << "\" (" - << write_css(col_sa) << ") and \"" << conf_key_mpr_thresh - << "\" (" << write_css(col_ta) - << ") config file entries must have the same length!\n\n"; - exit(1); - } + if(m.size() == 0) return; int nxy = fcst_dp.nx() * fcst_dp.ny(); int n_skip = 0; @@ -884,8 +877,7 @@ void apply_mpr_thresh_mask(DataPlane &fcst_dp, DataPlane &obs_dp, (ocsd_flag && is_bad_data(cpi.ocsd))) continue; // Discard pairs which do not meet the threshold criteria - if(!check_mpr_thresh(fcst_dp.buf()[i], obs_dp.buf()[i], cpi, - col_sa, col_ta)) { + if(!check_mpr_thresh(fcst_dp.buf()[i], obs_dp.buf()[i], cpi, m)) { // Increment skip counter n_skip++; @@ -902,9 +894,7 @@ void apply_mpr_thresh_mask(DataPlane &fcst_dp, DataPlane &obs_dp, mlog << Debug(3) << "Discarded " << n_skip << " of " << nxy - << " pairs for matched pair filtering columns (" - << write_css(col_sa) << ") and thresholds (" - << col_ta.get_str() << ").\n"; + << " pairs when applying matched pair filtering thresholds.\n"; return; } diff --git a/src/libcode/vx_statistics/pair_data_point.h b/src/libcode/vx_statistics/pair_data_point.h index 5f43a5e185..aa0e135204 100644 --- a/src/libcode/vx_statistics/pair_data_point.h +++ b/src/libcode/vx_statistics/pair_data_point.h @@ -141,7 +141,7 @@ extern bool check_fo_thresh(double, double, const ClimoPntInfo &, const SetLogic); extern bool check_mpr_thresh(double, double, const ClimoPntInfo &, - const StringArray &, const ThreshArray &, + const std::map &, ConcatString * = 0); extern double get_mpr_column_value(double, double, const ClimoPntInfo &, @@ -150,7 +150,7 @@ extern double get_mpr_column_value(double, double, const ClimoPntInfo &, extern void apply_mpr_thresh_mask(DataPlane &, DataPlane &, DataPlane &, DataPlane &, DataPlane &, DataPlane &, - const StringArray &, const ThreshArray &); + const std::map &); extern bool check_seeps_thresh(double, double, const StringArray &, const ThreshArray &, diff --git a/src/tools/core/Makefile.am b/src/tools/core/Makefile.am index 91fd210cf0..5372ec7474 100644 --- a/src/tools/core/Makefile.am +++ b/src/tools/core/Makefile.am @@ -42,6 +42,10 @@ if ENABLE_POINT_STAT SUBDIRS += point_stat endif +if ENABLE_PAIR_STAT + SUBDIRS += pair_stat +endif + if ENABLE_STAT_ANALYSIS SUBDIRS += stat_analysis endif diff --git a/src/tools/core/Makefile.in b/src/tools/core/Makefile.in index cacc9eabec..cd0038cec6 100644 --- a/src/tools/core/Makefile.in +++ b/src/tools/core/Makefile.in @@ -93,9 +93,10 @@ host_triplet = @host@ @ENABLE_MODE_ANALYSIS_TRUE@am__append_4 = mode_analysis @ENABLE_PCP_COMBINE_TRUE@am__append_5 = pcp_combine @ENABLE_POINT_STAT_TRUE@am__append_6 = point_stat -@ENABLE_STAT_ANALYSIS_TRUE@am__append_7 = stat_analysis -@ENABLE_WAVELET_STAT_TRUE@am__append_8 = wavelet_stat -@ENABLE_SERIES_ANALYSIS_TRUE@am__append_9 = series_analysis +@ENABLE_PAIR_STAT_TRUE@am__append_7 = pair_stat +@ENABLE_STAT_ANALYSIS_TRUE@am__append_8 = stat_analysis +@ENABLE_WAVELET_STAT_TRUE@am__append_9 = wavelet_stat +@ENABLE_SERIES_ANALYSIS_TRUE@am__append_10 = series_analysis subdir = src/tools/core ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/configure.ac @@ -159,7 +160,8 @@ am__define_uniq_tagged_files = \ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ done | $(am__uniquify_input)` DIST_SUBDIRS = ensemble_stat grid_stat mode mode_analysis pcp_combine \ - point_stat stat_analysis wavelet_stat series_analysis + point_stat pair_stat stat_analysis wavelet_stat \ + series_analysis am__DIST_COMMON = $(srcdir)/Makefile.in DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) am__relativize = \ @@ -348,7 +350,8 @@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ SUBDIRS = $(am__append_1) $(am__append_2) $(am__append_3) \ $(am__append_4) $(am__append_5) $(am__append_6) \ - $(am__append_7) $(am__append_8) $(am__append_9) + $(am__append_7) $(am__append_8) $(am__append_9) \ + $(am__append_10) MAINTAINERCLEANFILES = Makefile.in all: all-recursive diff --git a/src/tools/core/ensemble_stat/ensemble_stat.cc b/src/tools/core/ensemble_stat/ensemble_stat.cc index 68cfe50cc1..7b327b443f 100644 --- a/src/tools/core/ensemble_stat/ensemble_stat.cc +++ b/src/tools/core/ensemble_stat/ensemble_stat.cc @@ -2077,7 +2077,7 @@ void setup_table(AsciiTable &at) { at.set_bad_data_str(na_str); // Don't write out trailing blank rows - at.set_delete_trailing_blank_rows(1); + at.set_delete_trailing_blank_rows(true); return; } diff --git a/src/tools/core/grid_stat/grid_stat.cc b/src/tools/core/grid_stat/grid_stat.cc index a1c43f0824..5e565ebcbd 100644 --- a/src/tools/core/grid_stat/grid_stat.cc +++ b/src/tools/core/grid_stat/grid_stat.cc @@ -587,7 +587,7 @@ void setup_table(AsciiTable &at) { at.set_bad_data_str(na_str); // Don't write out trailing blank rows - at.set_delete_trailing_blank_rows(1); + at.set_delete_trailing_blank_rows(true); return; } @@ -823,12 +823,11 @@ void process_scores() { << (ocsd_dp.is_empty() ? 0 : 1) << " standard deviation field(s).\n"; // Apply MPR threshold filters - if(conf_info.vx_opt[i].mpr_sa.n() > 0) { + if(conf_info.vx_opt[i].mpr_thr_inc_map.size() > 0) { apply_mpr_thresh_mask(fcst_dp, obs_dp, fcmn_dp, fcsd_dp, ocmn_dp, ocsd_dp, - conf_info.vx_opt[i].mpr_sa, - conf_info.vx_opt[i].mpr_ta); + conf_info.vx_opt[i].mpr_thr_inc_map); } // Setup the first pass through the data diff --git a/src/tools/core/grid_stat/grid_stat_conf_info.cc b/src/tools/core/grid_stat/grid_stat_conf_info.cc index f9b704e455..c29579e710 100644 --- a/src/tools/core/grid_stat/grid_stat_conf_info.cc +++ b/src/tools/core/grid_stat/grid_stat_conf_info.cc @@ -599,8 +599,7 @@ void GridStatVxOpt::clear() { var_name.clear(); var_suffix.clear(); - mpr_sa.clear(); - mpr_ta.clear(); + mpr_thr_inc_map.clear(); fcat_ta.clear(); ocat_ta.clear(); @@ -734,8 +733,26 @@ void GridStatVxOpt::process_config( for(i=0; i " + << "The length of \"" << conf_key_mpr_column << "\" and \"" + << conf_key_mpr_thresh << "\" must match (" << mpr_sa.n() + << " != " << mpr_ta.n() << ")!\n\n"; + exit(1); + } + + // Store in map + for(int i=0; i mpr_thr_inc_map; ThreshArray fcat_ta; // fcst categorical thresholds ThreshArray ocat_ta; // obs categorical thresholds diff --git a/src/tools/core/mode/mode_exec.cc b/src/tools/core/mode/mode_exec.cc index 081266cae6..bc72929284 100644 --- a/src/tools/core/mode/mode_exec.cc +++ b/src/tools/core/mode/mode_exec.cc @@ -2510,7 +2510,7 @@ void ModeExecutive::write_ct_stats() engine.conf_info.conf.output_precision()); cts_at.set_bad_data_value(bad_data_double); // Set the bad data value cts_at.set_bad_data_str(na_str); // Set the bad data string - cts_at.set_delete_trailing_blank_rows(1); // No trailing blank rows + cts_at.set_delete_trailing_blank_rows(true); // No trailing blank rows // // Write out the MODE header columns diff --git a/src/tools/core/pair_stat/.gitignore b/src/tools/core/pair_stat/.gitignore new file mode 100644 index 0000000000..547b0cdd41 --- /dev/null +++ b/src/tools/core/pair_stat/.gitignore @@ -0,0 +1,6 @@ +pair_stat +*.o +*.a +.deps +Makefile +*.dSYM diff --git a/src/tools/core/pair_stat/Makefile.am b/src/tools/core/pair_stat/Makefile.am new file mode 100644 index 0000000000..f8bbc63022 --- /dev/null +++ b/src/tools/core/pair_stat/Makefile.am @@ -0,0 +1,51 @@ +## @start 1 +## Makefile.am -- Process this file with automake to produce Makefile.in +## @end 1 + +MAINTAINERCLEANFILES = Makefile.in + +# Include the project definitions + +include ${top_srcdir}/Make-include + +# The program + +bin_PROGRAMS = pair_stat +pair_stat_SOURCES = pair_stat.cc \ + pair_stat_conf_info.cc +pair_stat_CPPFLAGS = ${MET_CPPFLAGS} +pair_stat_LDFLAGS = ${MET_LDFLAGS} +pair_stat_LDADD = -lvx_stat_out \ + -lvx_statistics \ + -lvx_shapedata \ + -lvx_analysis_util \ + -lvx_data2d_factory \ + -lvx_data2d_nc_met \ + -lvx_data2d_grib \ + $(GRIB2_MET_LIBS) \ + -lvx_data2d_nc_wrf \ + -lvx_data2d_nc_cf \ + $(UGRID_MET_LIBS) \ + $(PYTHON_MET_LIBS) \ + -lvx_data2d \ + -lvx_nc_obs \ + -lvx_seeps \ + -lvx_nc_util \ + -lvx_regrid \ + -lvx_grid \ + -lvx_geodesy \ + -lvx_seeps \ + -lvx_config \ + -lvx_gsl_prob \ + -lvx_color \ + -lvx_util_math \ + -lvx_util \ + $(PYTHON_MET_LIBS) \ + -lvx_math \ + -lvx_cal \ + -lvx_log \ + $(GRIB2_DEP_LIBS) $(UGRID_DEP_LIBS) $(PYTHON_DEP_LIBS) \ + -lm -lproj -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas + +EXTRA_DIST = pair_stat.h \ + pair_stat_conf_info.h diff --git a/src/tools/core/pair_stat/Makefile.in b/src/tools/core/pair_stat/Makefile.in new file mode 100644 index 0000000000..48f6f33d88 --- /dev/null +++ b/src/tools/core/pair_stat/Makefile.in @@ -0,0 +1,730 @@ +# Makefile.in generated by automake 1.16.5 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2021 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +bin_PROGRAMS = pair_stat$(EXEEXT) +subdir = src/tools/core/pair_stat +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__installdirs = "$(DESTDIR)$(bindir)" +PROGRAMS = $(bin_PROGRAMS) +am_pair_stat_OBJECTS = pair_stat-pair_stat.$(OBJEXT) \ + pair_stat-pair_stat_conf_info.$(OBJEXT) +pair_stat_OBJECTS = $(am_pair_stat_OBJECTS) +am__DEPENDENCIES_1 = +pair_stat_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) +pair_stat_LINK = $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) \ + $(pair_stat_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/pair_stat-pair_stat.Po \ + ./$(DEPDIR)/pair_stat-pair_stat_conf_info.Po +am__mv = mv -f +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ + $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) +AM_V_CXX = $(am__v_CXX_@AM_V@) +am__v_CXX_ = $(am__v_CXX_@AM_DEFAULT_V@) +am__v_CXX_0 = @echo " CXX " $@; +am__v_CXX_1 = +CXXLD = $(CXX) +CXXLINK = $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) \ + -o $@ +AM_V_CXXLD = $(am__v_CXXLD_@AM_V@) +am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@) +am__v_CXXLD_0 = @echo " CXXLD " $@; +am__v_CXXLD_1 = +SOURCES = $(pair_stat_SOURCES) +DIST_SOURCES = $(pair_stat_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BUFRLIB_NAME = @BUFRLIB_NAME@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CFLAGS = @CFLAGS@ +CPPFLAGS = @CPPFLAGS@ +CSCOPE = @CSCOPE@ +CTAGS = @CTAGS@ +CXX = @CXX@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +ETAGS = @ETAGS@ +EXEEXT = @EXEEXT@ +F77 = @F77@ +FC_LIBS = @FC_LIBS@ +FFLAGS = @FFLAGS@ +FLIBS = @FLIBS@ +GRIB2CLIB_NAME = @GRIB2CLIB_NAME@ +GRIB2_DEP_LIBS = @GRIB2_DEP_LIBS@ +GRIB2_MET_LIBS = @GRIB2_MET_LIBS@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +LDFLAGS = @LDFLAGS@ +LEX = @LEX@ +LEXLIB = @LEXLIB@ +LEX_OUTPUT_ROOT = @LEX_OUTPUT_ROOT@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LTLIBOBJS = @LTLIBOBJS@ +MAKEINFO = @MAKEINFO@ +MET_ATLAS = @MET_ATLAS@ +MET_ATLASINC = @MET_ATLASINC@ +MET_ATLASLIB = @MET_ATLASLIB@ +MET_BUFR = @MET_BUFR@ +MET_BUFRLIB = @MET_BUFRLIB@ +MET_CAIRO = @MET_CAIRO@ +MET_CAIROINC = @MET_CAIROINC@ +MET_CAIROLIB = @MET_CAIROLIB@ +MET_CXX_STANDARD = @MET_CXX_STANDARD@ +MET_ECKIT = @MET_ECKIT@ +MET_ECKITINC = @MET_ECKITINC@ +MET_ECKITLIB = @MET_ECKITLIB@ +MET_FREETYPE = @MET_FREETYPE@ +MET_FREETYPEINC = @MET_FREETYPEINC@ +MET_FREETYPELIB = @MET_FREETYPELIB@ +MET_GRIB2C = @MET_GRIB2C@ +MET_GRIB2CINC = @MET_GRIB2CINC@ +MET_GRIB2CLIB = @MET_GRIB2CLIB@ +MET_GSL = @MET_GSL@ +MET_GSLINC = @MET_GSLINC@ +MET_GSLLIB = @MET_GSLLIB@ +MET_HDF = @MET_HDF@ +MET_HDF5 = @MET_HDF5@ +MET_HDF5INC = @MET_HDF5INC@ +MET_HDF5LIB = @MET_HDF5LIB@ +MET_HDFEOS = @MET_HDFEOS@ +MET_HDFEOSINC = @MET_HDFEOSINC@ +MET_HDFEOSLIB = @MET_HDFEOSLIB@ +MET_HDFINC = @MET_HDFINC@ +MET_HDFLIB = @MET_HDFLIB@ +MET_NETCDF = @MET_NETCDF@ +MET_NETCDFINC = @MET_NETCDFINC@ +MET_NETCDFLIB = @MET_NETCDFLIB@ +MET_PROJ = @MET_PROJ@ +MET_PROJINC = @MET_PROJINC@ +MET_PROJLIB = @MET_PROJLIB@ +MET_PYTHON_BIN_EXE = @MET_PYTHON_BIN_EXE@ +MET_PYTHON_CC = @MET_PYTHON_CC@ +MET_PYTHON_LD = @MET_PYTHON_LD@ +MKDIR_P = @MKDIR_P@ +OBJEXT = @OBJEXT@ +OPENMP_CFLAGS = @OPENMP_CFLAGS@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PYTHON_DEP_LIBS = @PYTHON_DEP_LIBS@ +PYTHON_MET_LIBS = @PYTHON_MET_LIBS@ +RANLIB = @RANLIB@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +STRIP = @STRIP@ +UGRID_DEP_LIBS = @UGRID_DEP_LIBS@ +UGRID_MET_LIBS = @UGRID_MET_LIBS@ +VERSION = @VERSION@ +YACC = @YACC@ +YFLAGS = @YFLAGS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_F77 = @ac_ct_F77@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +srcdir = @srcdir@ +sysconfdir = @sysconfdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +MAINTAINERCLEANFILES = Makefile.in +pair_stat_SOURCES = pair_stat.cc \ + pair_stat_conf_info.cc + +pair_stat_CPPFLAGS = ${MET_CPPFLAGS} +pair_stat_LDFLAGS = ${MET_LDFLAGS} +pair_stat_LDADD = -lvx_stat_out \ + -lvx_statistics \ + -lvx_shapedata \ + -lvx_analysis_util \ + -lvx_data2d_factory \ + -lvx_data2d_nc_met \ + -lvx_data2d_grib \ + $(GRIB2_MET_LIBS) \ + -lvx_data2d_nc_wrf \ + -lvx_data2d_nc_cf \ + $(UGRID_MET_LIBS) \ + $(PYTHON_MET_LIBS) \ + -lvx_data2d \ + -lvx_nc_obs \ + -lvx_seeps \ + -lvx_nc_util \ + -lvx_regrid \ + -lvx_grid \ + -lvx_geodesy \ + -lvx_seeps \ + -lvx_config \ + -lvx_gsl_prob \ + -lvx_color \ + -lvx_util_math \ + -lvx_util \ + $(PYTHON_MET_LIBS) \ + -lvx_math \ + -lvx_cal \ + -lvx_log \ + $(GRIB2_DEP_LIBS) $(UGRID_DEP_LIBS) $(PYTHON_DEP_LIBS) \ + -lm -lproj -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas + +EXTRA_DIST = pair_stat.h \ + pair_stat_conf_info.h + +all: all-am + +.SUFFIXES: +.SUFFIXES: .cc .o .obj +$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/tools/core/pair_stat/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/tools/core/pair_stat/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): +install-binPROGRAMS: $(bin_PROGRAMS) + @$(NORMAL_INSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(bindir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(bindir)" || exit 1; \ + fi; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p \ + ; then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' \ + -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-binPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' \ + `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(bindir)" && rm -f $$files + +clean-binPROGRAMS: + -test -z "$(bin_PROGRAMS)" || rm -f $(bin_PROGRAMS) + +pair_stat$(EXEEXT): $(pair_stat_OBJECTS) $(pair_stat_DEPENDENCIES) $(EXTRA_pair_stat_DEPENDENCIES) + @rm -f pair_stat$(EXEEXT) + $(AM_V_CXXLD)$(pair_stat_LINK) $(pair_stat_OBJECTS) $(pair_stat_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pair_stat-pair_stat.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/pair_stat-pair_stat_conf_info.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.cc.o: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $< + +.cc.obj: +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +pair_stat-pair_stat.o: pair_stat.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(pair_stat_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT pair_stat-pair_stat.o -MD -MP -MF $(DEPDIR)/pair_stat-pair_stat.Tpo -c -o pair_stat-pair_stat.o `test -f 'pair_stat.cc' || echo '$(srcdir)/'`pair_stat.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/pair_stat-pair_stat.Tpo $(DEPDIR)/pair_stat-pair_stat.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='pair_stat.cc' object='pair_stat-pair_stat.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(pair_stat_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o pair_stat-pair_stat.o `test -f 'pair_stat.cc' || echo '$(srcdir)/'`pair_stat.cc + +pair_stat-pair_stat.obj: pair_stat.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(pair_stat_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT pair_stat-pair_stat.obj -MD -MP -MF $(DEPDIR)/pair_stat-pair_stat.Tpo -c -o pair_stat-pair_stat.obj `if test -f 'pair_stat.cc'; then $(CYGPATH_W) 'pair_stat.cc'; else $(CYGPATH_W) '$(srcdir)/pair_stat.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/pair_stat-pair_stat.Tpo $(DEPDIR)/pair_stat-pair_stat.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='pair_stat.cc' object='pair_stat-pair_stat.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(pair_stat_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o pair_stat-pair_stat.obj `if test -f 'pair_stat.cc'; then $(CYGPATH_W) 'pair_stat.cc'; else $(CYGPATH_W) '$(srcdir)/pair_stat.cc'; fi` + +pair_stat-pair_stat_conf_info.o: pair_stat_conf_info.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(pair_stat_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT pair_stat-pair_stat_conf_info.o -MD -MP -MF $(DEPDIR)/pair_stat-pair_stat_conf_info.Tpo -c -o pair_stat-pair_stat_conf_info.o `test -f 'pair_stat_conf_info.cc' || echo '$(srcdir)/'`pair_stat_conf_info.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/pair_stat-pair_stat_conf_info.Tpo $(DEPDIR)/pair_stat-pair_stat_conf_info.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='pair_stat_conf_info.cc' object='pair_stat-pair_stat_conf_info.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(pair_stat_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o pair_stat-pair_stat_conf_info.o `test -f 'pair_stat_conf_info.cc' || echo '$(srcdir)/'`pair_stat_conf_info.cc + +pair_stat-pair_stat_conf_info.obj: pair_stat_conf_info.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(pair_stat_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT pair_stat-pair_stat_conf_info.obj -MD -MP -MF $(DEPDIR)/pair_stat-pair_stat_conf_info.Tpo -c -o pair_stat-pair_stat_conf_info.obj `if test -f 'pair_stat_conf_info.cc'; then $(CYGPATH_W) 'pair_stat_conf_info.cc'; else $(CYGPATH_W) '$(srcdir)/pair_stat_conf_info.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/pair_stat-pair_stat_conf_info.Tpo $(DEPDIR)/pair_stat-pair_stat_conf_info.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='pair_stat_conf_info.cc' object='pair_stat-pair_stat_conf_info.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(pair_stat_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o pair_stat-pair_stat_conf_info.obj `if test -f 'pair_stat_conf_info.cc'; then $(CYGPATH_W) 'pair_stat_conf_info.cc'; else $(CYGPATH_W) '$(srcdir)/pair_stat_conf_info.cc'; fi` + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am +check: check-am +all-am: Makefile $(PROGRAMS) +installdirs: + for dir in "$(DESTDIR)$(bindir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." + -test -z "$(MAINTAINERCLEANFILES)" || rm -f $(MAINTAINERCLEANFILES) +clean: clean-am + +clean-am: clean-binPROGRAMS clean-generic mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/pair_stat-pair_stat.Po + -rm -f ./$(DEPDIR)/pair_stat-pair_stat_conf_info.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: install-binPROGRAMS + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/pair_stat-pair_stat.Po + -rm -f ./$(DEPDIR)/pair_stat-pair_stat_conf_info.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-binPROGRAMS + +.MAKE: install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ + clean-binPROGRAMS clean-generic cscopelist-am ctags ctags-am \ + distclean distclean-compile distclean-generic distclean-tags \ + distdir dvi dvi-am html html-am info info-am install \ + install-am install-binPROGRAMS install-data install-data-am \ + install-dvi install-dvi-am install-exec install-exec-am \ + install-html install-html-am install-info install-info-am \ + install-man install-pdf install-pdf-am install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic pdf pdf-am \ + ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-binPROGRAMS + +.PRECIOUS: Makefile + + +# Include the project definitions + +include ${top_srcdir}/Make-include + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/tools/core/pair_stat/pair_stat.cc b/src/tools/core/pair_stat/pair_stat.cc new file mode 100644 index 0000000000..279d6a56db --- /dev/null +++ b/src/tools/core/pair_stat/pair_stat.cc @@ -0,0 +1,1313 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2024 +// ** University Corporation for Atmospheric Research led(UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// +// +// Filename: pair_stat.cc +// +// Description: +// Based on user-specified parameters, this tool reads already +// computed forecast and observation pair data, subsets them by +// attribute, time, and space, and computes many verification +// scores and statistics, including confidence intervals, to +// summarize the comparison. +// +// Mod# Date Name Description +// ---- ---- ---- ----------- +// 000 11/07/24 Halley Gotway MET #3006 New +// +//////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "main.h" +#include "pair_stat.h" + +#include "vx_statistics.h" +#include "vx_nc_util.h" +#include "vx_regrid.h" +#include "vx_log.h" +#include "seeps.h" + +#include "nc_obs_util.h" +#include "nc_point_obs_in.h" + +#ifdef WITH_PYTHON +#include "data2d_nc_met.h" +#include "pointdata_python.h" +#endif + +using namespace std; +using namespace netCDF; + +//////////////////////////////////////////////////////////////////////// + +static void process_command_line(int, char **); +static void setup_pairs(); + +static void setup_txt_files(); +static void setup_table (AsciiTable &); + +static ConcatString build_outfile_name(const char *); + +static void process_mpr_pairs(const ConcatString &, PairsFormat); +static void process_ioda_pairs(const ConcatString &); +static void process_scores(); + +static void do_cts (vector &, int, const PairDataPoint *); +static void do_mcts (MCTSInfo &, int, const PairDataPoint *); +static void do_cnt_sl1l2 (const PairStatVxOpt &, const PairDataPoint *); +static void do_vl1l2 (vector &, int, const PairDataPoint *, const PairDataPoint *); +static void do_pct (const PairStatVxOpt &, const PairDataPoint *); + +static void finish_txt_files(); + +static void clean_up(); + +static void usage(); + +static void set_pairs(const StringArray &); +static void set_format(const StringArray &); +static void set_config(const StringArray &); +static void set_out(const StringArray &); + +//////////////////////////////////////////////////////////////////////// + +int met_main(int argc, char *argv[]) { + + // Process the command line arguments + process_command_line(argc, argv); + + // Process each pairs file + for(int i=0; i 0) pairs_files.add(sa); + else pairs_files.add(tmp_src[i]); + } + + // Check for required argruments + if(pairs_files.n() == 0) { + mlog << Error << "\n" << method_name + << "The \"-pairs\" command line option is required!\n\n"; + usage(); + } + if(pairs_format == PairsFormat::None) { + mlog << Error << "\n" << method_name + << "The \"-format\" command line option is required!\n\n"; + usage(); + } + if(config_file.empty()) { + mlog << Error << "\n" << method_name + << "The \"-config\" command line option is required!\n\n"; + usage(); + } + + // List of config files to be read + StringArray config_files; + + // Add ConfigConstants + config_files.add(replace_path(config_const_filename)); + + // Add the IODA Data config file + if(pairs_format == PairsFormat::IODA) { + config_files.add(replace_path(ioda_data_config_filename)); + } + + // Add the default config file + config_files.add(replace_path(default_config_filename)); + + // Add the user config file + config_files.add(config_file); + + // Read the config files + conf_info.read_config(config_files); + + // Process the configuration + conf_info.process_config(pairs_format); + + // Process the masks + conf_info.process_masks(); + + // Setup the VxPairDataPoint objects + conf_info.set_vx_pd(); + + // List the input pair files + mlog << Debug(1) + << "Reading " << pairs_files.n() << " \"" + << pairsformat_to_string(pairs_format) << "\" format pairs file(s): " + << write_css(pairs_files) << "\n"; + + // Use the first verification task to set the random number generator + // and seed value for bootstrap confidence intervals + rng_set(rng_ptr, + conf_info.vx_opt[0].boot_info.rng.c_str(), + conf_info.vx_opt[0].boot_info.seed.c_str()); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +static void setup_txt_files() { + + // Create output file names for the stat file and optional text files + ConcatString base_name(build_outfile_name("")); + + ///////////////////////////////////////////////////////////////////// + // + // Setup the output STAT file + // + ///////////////////////////////////////////////////////////////////// + + // Get the maximum number of data columns + int n_prob = conf_info.get_max_n_fprob_thresh(); + int n_cat = conf_info.get_max_n_cat_thresh() + 1; + int n_eclv = conf_info.get_max_n_eclv_points(); + + int max_prob_col = get_n_pjc_columns(n_prob); + int max_mctc_col = get_n_mctc_columns(n_cat); + + // Determine the maximum number of data columns + int max_col = (max_prob_col > max_stat_col ? max_prob_col : max_stat_col); + if (max_mctc_col > max_col) max_col = max_mctc_col; + + // Add the header columns + max_col += n_header_columns + 1; + + // Initialize file stream + stat_out = (ofstream *) nullptr; + + // Build the file name + stat_file << base_name << stat_file_ext; + + // Create the output STAT file + open_txt_file(stat_out, stat_file.c_str()); + + // Setup the STAT AsciiTable + stat_at.set_size(conf_info.n_stat_row() + 1, max_col); + setup_table(stat_at); + + // Write the text header row + write_header_row(nullptr, 0, 1, stat_at, 0, 0); + + // Initialize the row index to 1 to account for the header + i_stat_row = 1; + + ///////////////////////////////////////////////////////////////////// + // + // Setup each of the optional output text files + // + ///////////////////////////////////////////////////////////////////// + + // Loop through output file type + for(int i=0; i " + << "the \"-format python\" option is not supported yet!\n\n"; + exit(1); + } + + // + // Open the input file + // + if(!f.open(file_name.c_str())) { + mlog << Warning << "\n" << method_name + << "can't open matched pair file \"" << file_name + << "\" for reading!\n\n"; + return; + } + + // + // Count the number read and kept + // + int n_read = 0; + int n_keep = 0; + + // + // Process the STAT lines + // + STATLine line; + while(f >> line) { + + // Skip header and non-MPR lines + if(line.is_header() || line.type() != STATLineType::mpr) continue; + + n_read++; + + if(conf_info.add_mpr_line(line)) n_keep++; + } + + mlog << Debug(3) << "Keeping " << n_keep << " of " << n_read + << " MPR lines from \"" << file_name << "\".\n"; + + return; +} + +//////////////////////////////////////////////////////////////////////// + +static void process_ioda_pairs(const ConcatString &) { + + // Add support for -format ioda + mlog << Error << "\nprocess_ioda_pairs() -> " + << "the \"-format ioda\" option is not supported yet!\n\n"; + exit(1); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +static void process_scores() { + ConcatString cs; + + // Initialize pointers + vector cts_info; + vector vl1l2_info; + MCTSInfo mcts_info; + + mlog << Debug(2) + << "\n" << sep_str << "\n\n"; + + // Setup the output text files as requested in the config file + setup_txt_files(); + + // Store the maximum number of each threshold type + int n_cat = conf_info.get_max_n_cat_thresh(); + int n_wind = conf_info.get_max_n_wind_thresh(); + + // Allocate space for output statistics types + cts_info.resize(n_cat); + vl1l2_info.resize(n_wind); + + // Compute scores for each PairData object and write output + int i_vx = -1; + for(auto &vx : conf_info.vx_opt) { + + i_vx++; + + // Store masking region as the only "case" information + StringArray empty_sa; + StringArray case_cols; + case_cols.add("VX_MASK"); + + // Loop through the verification masking regions + for(int i_mask=0; i_maskmagic_str() + << " versus " << vx.vx_pd.obs_info->magic_str() + << ", over region " << pd_ptr->mask_name + << ", using " << pd_ptr->n_obs << " matched pairs.\n"; + + // Process percentile thresholds + vx.set_perc_thresh(pd_ptr); + + // Write out the MPR lines + if(vx.output_flag[i_mpr] != STATOutputType::None) { + write_mpr_row(shc, pd_ptr, + vx.output_flag[i_mpr], + stat_at, i_stat_row, + txt_at[i_mpr], i_txt_row[i_mpr], false); + shc = in_shc; + } + + // Write out the SEEPS MPR lines + if(vx.output_flag[i_seeps_mpr] != STATOutputType::None) { + write_seeps_mpr_row(shc, pd_ptr, + vx.output_flag[i_seeps_mpr], + stat_at, i_stat_row, + txt_at[i_seeps_mpr], i_txt_row[i_seeps_mpr], false); + shc = in_shc; + } + + // Write out the SEEPS lines + if(vx.output_flag[i_seeps] != STATOutputType::None) { + compute_aggregated_seeps(pd_ptr, &pd_ptr->seeps_agg); + write_seeps_row(shc, &pd_ptr->seeps_agg, + vx.output_flag[i_seeps], + stat_at, i_stat_row, + txt_at[i_seeps], i_txt_row[i_seeps]); + shc = in_shc; + } + + // Compute CTS scores + if(!vx.vx_pd.fcst_info->is_prob() && + vx.fcat_ta.n() > 0 && + (vx.output_flag[i_fho] != STATOutputType::None || + vx.output_flag[i_ctc] != STATOutputType::None || + vx.output_flag[i_cts] != STATOutputType::None || + vx.output_flag[i_eclv] != STATOutputType::None)) { + + // Initialize + for(int i_cat=0; i_catis_prob() && + vx.fcat_ta.n() > 1 && + (vx.output_flag[i_mctc] != STATOutputType::None || + vx.output_flag[i_mcts] != STATOutputType::None)) { + + // Initialize + mcts_info.clear(); + + // Compute MCTS Info + do_mcts(mcts_info, i_vx, pd_ptr); + + if(mcts_info.cts.n_pairs() == 0) continue; + + // Write out MCTC + if(vx.output_flag[i_mctc] != STATOutputType::None) { + write_mctc_row(shc, mcts_info, + vx.output_flag[i_mctc], + stat_at, i_stat_row, + txt_at[i_mctc], i_txt_row[i_mctc]); + shc = in_shc; + } + + // Write out MCTS + if(vx.output_flag[i_mcts] != STATOutputType::None) { + write_mcts_row(shc, mcts_info, + vx.output_flag[i_mcts], + stat_at, i_stat_row, + txt_at[i_mcts], i_txt_row[i_mcts]); + shc = in_shc; + } + } // end Compute MCTS scores + + // Compute CNT, SL1L2, and SAL1L2 scores + if(!vx.vx_pd.fcst_info->is_prob() && + (vx.output_flag[i_cnt] != STATOutputType::None || + vx.output_flag[i_sl1l2] != STATOutputType::None || + vx.output_flag[i_sal1l2] != STATOutputType::None)) { + do_cnt_sl1l2(vx, pd_ptr); + } + + // Compute VL1L2 and VAL1L2 partial sums for UGRD and VGRD + if(!vx.vx_pd.fcst_info->is_prob() && + vx.vx_pd.fcst_info->is_v_wind() && + vx.vx_pd.fcst_info->uv_index() >= 0 && + (vx.output_flag[i_vl1l2] != STATOutputType::None || + vx.output_flag[i_val1l2] != STATOutputType::None || + vx.output_flag[i_vcnt] != STATOutputType::None)) { + + // Store the forecast variable name + shc.set_fcst_var(ugrd_vgrd_abbr_str); + + // Store the observation variable name + shc.set_obs_var(ugrd_vgrd_abbr_str); + + // Initialize + for(int i_wind=0; i_winduv_index(); + + // Check to make sure the masking regions match + if(conf_info.vx_opt[i_vx].get_n_mask() != + conf_info.vx_opt[u_vx].get_n_mask()) { + mlog << Warning << "\nprocess_scores() -> " + << "when computing VL1L2 and/or VAL1L2 vector " + << "partial sums, the U and V components must " + << "be processed using the same set of mask regions. " + << "Failing to do so will cause unexpected results!\n\n"; + } + + // Compute VL1L2 and VAL1L2 + do_vl1l2(vl1l2_info, i_vx, + &conf_info.vx_opt[u_vx].vx_pd.pd[i_mask], + &conf_info.vx_opt[i_vx].vx_pd.pd[i_mask]); + + // Loop through all of the wind speed thresholds + for(int i_wind=0; i_wind 0) { + write_vl1l2_row(shc, vl1l2_info[i_wind], + vx.output_flag[i_vl1l2], + stat_at, i_stat_row, + txt_at[i_vl1l2], i_txt_row[i_vl1l2]); + } + + // Write out VAL1L2 + if(vx.output_flag[i_val1l2] != STATOutputType::None && + vl1l2_info[i_wind].vacount > 0) { + write_val1l2_row(shc, vl1l2_info[i_wind], + vx.output_flag[i_val1l2], + stat_at, i_stat_row, + txt_at[i_val1l2], i_txt_row[i_val1l2]); + } + + // Write out VCNT + if(vx.output_flag[i_vcnt] != STATOutputType::None && + vl1l2_info[i_wind].vcount > 0) { + write_vcnt_row(shc, vl1l2_info[i_wind], + vx.output_flag[i_vcnt], + stat_at, i_stat_row, + txt_at[i_vcnt], i_txt_row[i_vcnt]); + } + } // end for i_wind + + shc = in_shc; + } // end Compute VL1L2 and VAL1L2 + + // Compute PCT counts and scores + if(vx.vx_pd.fcst_info->is_prob() && + (vx.output_flag[i_pct] != STATOutputType::None || + vx.output_flag[i_pstd] != STATOutputType::None || + vx.output_flag[i_pjc] != STATOutputType::None || + vx.output_flag[i_prc] != STATOutputType::None || + vx.output_flag[i_eclv] != STATOutputType::None)) { + do_pct(conf_info.vx_opt[i_vx], pd_ptr); + } + } // end for i_mask + + mlog << Debug(2) << "\n" << sep_str << "\n\n"; + + } // end for i_vx + + return; +} + +//////////////////////////////////////////////////////////////////////// + +static void do_cts(vector &cts_info, int i_vx, const PairDataPoint *pd_ptr) { + + mlog << Debug(2) + << "Computing Categorical Statistics.\n"; + + // + // Set up the CTSInfo thresholds and alpha values + // + int n_cat = conf_info.vx_opt[i_vx].fcat_ta.n(); + for(int i=0; if_na.n() == 0 || pd_ptr->o_na.n() == 0) return; + + // + // Compute the stats, normal confidence intervals, and bootstrap + // bootstrap confidence intervals + // + CTSInfo *cts_info_ptr = cts_info.data(); + if(conf_info.vx_opt[i_vx].boot_info.interval == BootIntervalType::BCA) { + compute_cts_stats_ci_bca(rng_ptr, *pd_ptr, + conf_info.vx_opt[i_vx].boot_info.n_rep, + cts_info_ptr, n_cat, + conf_info.vx_opt[i_vx].output_flag[i_cts] != STATOutputType::None, + conf_info.vx_opt[i_vx].rank_corr_flag, + conf_info.tmp_dir.c_str()); + } + else { + compute_cts_stats_ci_perc(rng_ptr, *pd_ptr, + conf_info.vx_opt[i_vx].boot_info.n_rep, + conf_info.vx_opt[i_vx].boot_info.rep_prop, + cts_info_ptr, n_cat, + conf_info.vx_opt[i_vx].output_flag[i_cts] != STATOutputType::None, + conf_info.vx_opt[i_vx].rank_corr_flag, + conf_info.tmp_dir.c_str()); + } + + return; +} + +//////////////////////////////////////////////////////////////////////// + +static void do_mcts(MCTSInfo &mcts_info, int i_vx, const PairDataPoint *pd_ptr) { + + mlog << Debug(2) + << "Computing Multi-Category Statistics.\n"; + + // + // Set up the MCTSInfo size, thresholds, and alpha values + // + mcts_info.cts.set_size(conf_info.vx_opt[i_vx].fcat_ta.n() + 1); + mcts_info.cts.set_ec_value(conf_info.vx_opt[i_vx].hss_ec_value); + mcts_info.set_fthresh(conf_info.vx_opt[i_vx].fcat_ta); + mcts_info.set_othresh(conf_info.vx_opt[i_vx].ocat_ta); + mcts_info.allocate_n_alpha(conf_info.vx_opt[i_vx].get_n_ci_alpha()); + + for(int i=0; if_na.n() == 0 || pd_ptr->o_na.n() == 0) return; + + // + // Compute the stats, normal confidence intervals, and bootstrap + // bootstrap confidence intervals + // + if(conf_info.vx_opt[i_vx].boot_info.interval == BootIntervalType::BCA) { + compute_mcts_stats_ci_bca(rng_ptr, *pd_ptr, + conf_info.vx_opt[i_vx].boot_info.n_rep, + mcts_info, + conf_info.vx_opt[i_vx].output_flag[i_mcts] != STATOutputType::None, + conf_info.vx_opt[i_vx].rank_corr_flag, + conf_info.tmp_dir.c_str()); + } + else { + compute_mcts_stats_ci_perc(rng_ptr, *pd_ptr, + conf_info.vx_opt[i_vx].boot_info.n_rep, + conf_info.vx_opt[i_vx].boot_info.rep_prop, + mcts_info, + conf_info.vx_opt[i_vx].output_flag[i_mcts] != STATOutputType::None, + conf_info.vx_opt[i_vx].rank_corr_flag, + conf_info.tmp_dir.c_str()); + } + + return; +} + +//////////////////////////////////////////////////////////////////////// + +static void do_cnt_sl1l2(const PairStatVxOpt &vx_opt, const PairDataPoint *pd_ptr) { + + mlog << Debug(2) + << "Computing Scalar Partial Sums and Continuous Statistics.\n"; + + // Determine the number of observation climo CDF bins + int n_bin = (pd_ptr->ocmn_na.n_valid() > 0 && + pd_ptr->ocsd_na.n_valid() > 0 ? + vx_opt.get_n_cdf_bin() : 1); + + if(n_bin > 1) { + mlog << Debug(2) + << "Applying " << n_bin << " climatology bins.\n"; + } + + // Set flags + bool do_sl1l2 = (vx_opt.output_flag[i_sl1l2] != STATOutputType::None || + vx_opt.output_flag[i_sal1l2] != STATOutputType::None); + bool do_cnt = (vx_opt.output_flag[i_cnt] != STATOutputType::None); + bool precip_flag = (vx_opt.vx_pd.fcst_info->is_precipitation() && + vx_opt.vx_pd.obs_info->is_precipitation()); + + // Create statistics objects + vector cnt_info(n_bin); + vector sl1l2_info(n_bin); + + // Process each continuous filtering threshold + for(int i=0; isubset_pairs_cnt_thresh( + vx_opt.fcnt_ta[i], + vx_opt.ocnt_ta[i], + vx_opt.cnt_logic); + + // Check for no matched pairs to process + if(pd_thr.n_obs == 0) continue; + + // Process the climo CDF bins + for(int j=0; j 1) pd = subset_climo_cdf_bin(pd_thr, + vx_opt.cdf_info.cdf_ta, j); + else pd = pd_thr; + + // Check for no matched pairs to process + if(pd.n_obs == 0) continue; + + // Compute and write SL1L2 and SAL1L2 output + if(do_sl1l2) { + + // Store thresholds + sl1l2_info[j].fthresh = vx_opt.fcnt_ta[i]; + sl1l2_info[j].othresh = vx_opt.ocnt_ta[i]; + sl1l2_info[j].logic = vx_opt.cnt_logic; + + // Compute partial sums + sl1l2_info[j].set(pd); + + // Write out SL1L2 + if((n_bin == 1 || vx_opt.cdf_info.write_bins) && + vx_opt.output_flag[i_sl1l2] != STATOutputType::None && + sl1l2_info[j].scount > 0) { + + write_sl1l2_row(shc, sl1l2_info[j], + vx_opt.output_flag[i_sl1l2], + j, n_bin, stat_at, i_stat_row, + txt_at[i_sl1l2], i_txt_row[i_sl1l2]); + } + + // Write out SAL1L2 + if((n_bin == 1 || vx_opt.cdf_info.write_bins) && + vx_opt.output_flag[i_sal1l2] != STATOutputType::None && + sl1l2_info[j].sacount > 0) { + + write_sal1l2_row(shc, sl1l2_info[j], + vx_opt.output_flag[i_sal1l2], + j, n_bin, stat_at, i_stat_row, + txt_at[i_sal1l2], i_txt_row[i_sal1l2]); + } + } // end do_sl1l2 + + // Compute and write CNT output + if(do_cnt) { + + // Store thresholds + cnt_info[j].fthresh = vx_opt.fcnt_ta[i]; + cnt_info[j].othresh = vx_opt.ocnt_ta[i]; + cnt_info[j].logic = vx_opt.cnt_logic; + + // Setup the CNTInfo alpha values + cnt_info[j].allocate_n_alpha(vx_opt.get_n_ci_alpha()); + for(int k=0; k 0) { + + write_cnt_row(shc, cnt_info[j], + vx_opt.output_flag[i_cnt], j, n_bin, + stat_at, i_stat_row, txt_at[i_cnt], i_txt_row[i_cnt]); + } + } // end if do_cnt + } // end for j (n_bin) + + // Write the mean of the climo CDF bins + if(n_bin > 1) { + + // Compute SL1L2 climo CDF bin means + if(vx_opt.output_flag[i_sl1l2] != STATOutputType::None || + vx_opt.output_flag[i_sal1l2] != STATOutputType::None) { + + SL1L2Info sl1l2_mean; + compute_sl1l2_mean((SL1L2Info *) sl1l2_info.data(), n_bin, sl1l2_mean); + + // Write out SL1L2 + if(vx_opt.output_flag[i_sl1l2] != STATOutputType::None && + sl1l2_mean.scount > 0) { + + write_sl1l2_row(shc, sl1l2_mean, + vx_opt.output_flag[i_sl1l2], + -1, n_bin, stat_at, i_stat_row, + txt_at[i_sl1l2], i_txt_row[i_sl1l2]); + } + + // Write out SAL1L2 + if(vx_opt.output_flag[i_sal1l2] != STATOutputType::None && + sl1l2_mean.sacount > 0) { + + write_sal1l2_row(shc, sl1l2_mean, + vx_opt.output_flag[i_sal1l2], + -1, n_bin, stat_at, i_stat_row, + txt_at[i_sal1l2], i_txt_row[i_sal1l2]); + } + } + + // Compute CNT climo CDF bin means + if(vx_opt.output_flag[i_cnt] != STATOutputType::None) { + + CNTInfo cnt_mean; + compute_cnt_mean((CNTInfo *) cnt_info.data(), n_bin, cnt_mean); + + if(cnt_mean.n > 0) { + + write_cnt_row(shc, cnt_mean, + vx_opt.output_flag[i_cnt], + -1, n_bin, stat_at, i_stat_row, + txt_at[i_cnt], i_txt_row[i_cnt]); + } + } + } // end if n_bin > 1 + + } // end for i (fcnt_ta) + + return; +} + +//////////////////////////////////////////////////////////////////////// + +static void do_vl1l2(vector &v_info, int i_vx, + const PairDataPoint *pd_u_ptr, + const PairDataPoint *pd_v_ptr) { + + mlog << Debug(2) + << "Computing Vector Partial Sums and Continuous Vector Statistics.\n"; + + // + // Check that the number of pairs are the same + // + if(pd_u_ptr->n_obs != pd_v_ptr->n_obs) { + mlog << Error << "\ndo_vl1l2() -> " + << "unequal number of UGRD and VGRD pairs (" + << pd_u_ptr->n_obs << " != " << pd_v_ptr->n_obs + << ")\n\n"; + exit(1); + } + + // + // Set all of the VL1L2Info objects + // + for(int i=0; iocmn_na.n_valid() > 0 && + pd_ptr->ocsd_na.n_valid() > 0 ? + vx_opt.get_n_cdf_bin() : 1); + + if(n_bin > 1) { + mlog << Debug(2) + << "Applying " << n_bin << " climatology bins.\n"; + } + + // Create statistics objects + vector pct_info(n_bin); + + // Process each probabilistic observation threshold + for(int i=0; i 1) pd = subset_climo_cdf_bin(*pd_ptr, + vx_opt.cdf_info.cdf_ta, j); + else pd = *pd_ptr; + + // Store thresholds + pct_info[j].fthresh = vx_opt.fcat_ta; + pct_info[j].othresh = vx_opt.ocat_ta[i]; + pct_info[j].allocate_n_alpha(vx_opt.get_n_ci_alpha()); + + for(int k=0; k 1) { + + PCTInfo pct_mean; + compute_pct_mean((PCTInfo *) pct_info.data(), n_bin, pct_mean); + + // Write out PSTD + if(vx_opt.output_flag[i_pstd] != STATOutputType::None) { + write_pstd_row(shc, pct_mean, + vx_opt.output_flag[i_pstd], + -1, n_bin, stat_at, i_stat_row, + txt_at[i_pstd], i_txt_row[i_pstd]); + } + } // end if n_bin > 1 + + } // end for i (ocnt_ta) + + return; +} + +//////////////////////////////////////////////////////////////////////// + +static void finish_txt_files() { + + // Write out the contents of the STAT AsciiTable and + // close the STAT output files + if(stat_out) { + *stat_out << stat_at; + close_txt_file(stat_out, stat_file.c_str()); + } + + // Finish up each of the optional text files + for(int i=0; i +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pair_stat_conf_info.h" + +#include "vx_data2d_factory.h" +#include "vx_grid.h" +#include "vx_util.h" +#include "vx_stat_out.h" +#include "vx_gsl_prob.h" + +//////////////////////////////////////////////////////////////////////// +// +// Constants +// +//////////////////////////////////////////////////////////////////////// + +static const char * program_name = "pair_stat"; + +// Default configuration file name +static const char * default_config_filename = + "MET_BASE/config/PairStatConfig_default"; +static const char * ioda_data_config_filename = + "MET_BASE/config/IODADataConfig_default"; + +// Header columns +static const char * const * txt_columns[n_txt] = { + fho_columns, ctc_columns, cts_columns, + mctc_columns, mcts_columns, + cnt_columns, sl1l2_columns, sal1l2_columns, + vcnt_columns, vl1l2_columns, val1l2_columns, + pct_columns, pstd_columns, pjc_columns, + prc_columns, eclv_columns, + mpr_columns, seeps_mpr_columns, seeps_columns +}; + +// Length of header columns +static const int n_txt_columns[n_txt] = { + n_fho_columns, n_ctc_columns, n_cts_columns, + n_mctc_columns, n_mcts_columns, + n_cnt_columns, n_sl1l2_columns, n_sal1l2_columns, + n_vcnt_columns, n_vl1l2_columns, n_val1l2_columns, + n_pct_columns, n_pstd_columns, n_pjc_columns, + n_prc_columns, n_eclv_columns, + n_mpr_columns, n_seeps_mpr_columns, n_seeps_columns +}; + +// Text file abbreviations +static const char * const txt_file_abbr[n_txt] = { + "fho", "ctc", "cts", + "mctc", "mcts", + "cnt", "sl1l2", "sal1l2", + "vcnt", "vl1l2", "val1l2", + "pct", "pstd", "pjc", + "prc", "eclv", + "mpr", "seeps_mpr", "seeps" +}; + +/////////////////////////////////////////////////////////////////////////////// +// +// Variables for Command Line Arguments +// +/////////////////////////////////////////////////////////////////////////////// + +// Input files +static StringArray pairs_files; +static PairsFormat pairs_format; + +// Input Config file +static ConcatString config_file; +static PairStatConfInfo conf_info; + +// Optional arguments +static ConcatString out_base; + +/////////////////////////////////////////////////////////////////////////////// +// +// Variables for Output Files +// +/////////////////////////////////////////////////////////////////////////////// + +// Timing information +static unixtime fcst_valid_ut = (unixtime) 0; +static int fcst_lead_sec = bad_data_int; + +// Output STAT file +static ConcatString stat_file; +static std::ofstream *stat_out = (std::ofstream *) nullptr; +static AsciiTable stat_at; +static int i_stat_row; + +// Optional ASCII output files +static ConcatString txt_file[n_txt]; +static std::ofstream *txt_out[n_txt]; +static AsciiTable txt_at[n_txt]; +static int i_txt_row[n_txt]; + +/////////////////////////////////////////////////////////////////////////////// +// +// Miscellaneous Variables +// +/////////////////////////////////////////////////////////////////////////////// + +// Data file factory and input files +static Met2dDataFileFactory mtddf_factory; +static Met2dDataFile *fcst_mtddf = (Met2dDataFile *) nullptr; + +// Pointer to the random number generator to be used +static gsl_rng *rng_ptr = (gsl_rng *) nullptr; + +// Strings to be output in the STAT and optional text files +static StatHdrColumns shc; + +/////////////////////////////////////////////////////////////////////////////// + +#endif // __PAIR_STAT_H__ + +/////////////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/pair_stat/pair_stat_conf_info.cc b/src/tools/core/pair_stat/pair_stat_conf_info.cc new file mode 100644 index 0000000000..b2de29fdc2 --- /dev/null +++ b/src/tools/core/pair_stat/pair_stat_conf_info.cc @@ -0,0 +1,1254 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2024 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include +#include +#include + +#include "pair_stat_conf_info.h" +#include "nc_obs_util.h" +#include "vx_data2d_factory.h" +#include "vx_data2d.h" +#include "vx_log.h" + +using namespace std; + +//////////////////////////////////////////////////////////////////////// +// +// Code for class PairsFormat enumeration +// +//////////////////////////////////////////////////////////////////////// + +PairsFormat string_to_pairsformat(const string &s) { + PairsFormat t; + ConcatString cs(s); + cs.set_upper(); + + if(cs == "MPR") t = PairsFormat::MPR; + else if(cs == "PYTHON") t = PairsFormat::Python; + else if(cs == "IODA") t = PairsFormat::IODA; + else t = PairsFormat::None; + + return t; +} + +//////////////////////////////////////////////////////////////////////// + +ConcatString pairsformat_to_string(const PairsFormat t) { + ConcatString s; + + switch(t) { + case PairsFormat::MPR: s = "mpr"; break; + case PairsFormat::Python: s = "python"; break; + case PairsFormat::IODA: s = "ioda"; break; + default: s = na_str; break; + } + + return s; +} + +//////////////////////////////////////////////////////////////////////// +// +// Code for class PairStatConfInfo +// +//////////////////////////////////////////////////////////////////////// + +PairStatConfInfo::PairStatConfInfo() { + + init_from_scratch(); +} + +//////////////////////////////////////////////////////////////////////// + +void PairStatConfInfo::init_from_scratch() { + + clear(); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void PairStatConfInfo::clear() { + + // Initialize values + model.clear(); + vx_opt.clear(); + mask_area_map.clear(); + mask_sid_map.clear(); + point_weight_flag = PointWeightType::None; + tmp_dir.clear(); + version.clear(); + seeps_climo_name.clear(); + seeps_p1_thresh.clear(); + + // Set count to zero + n_vx = 0; + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void PairStatConfInfo::read_config(const StringArray &sa) { + + // Read each specified config file in order + for(int i=0; i= 5) conf.dump(cout); + + // Initialize + clear(); + + // Conf: version + version = parse_conf_version(&conf); + + // Conf: model + model = parse_conf_string(&conf, conf_key_model, false); + + // Conf: point_weight_flag + point_weight_flag = parse_conf_point_weight_flag(&conf); + + // Conf: tmp_dir + tmp_dir = parse_conf_tmp_dir(&conf); + + // Conf: fcst.pairs and obs.pairs + fdict = conf.lookup_array(conf_key_fcst_pairs); + odict = conf.lookup_array(conf_key_obs_pairs); + + // Determine the number of fields (name/level) to be verified + n_fvx = parse_conf_n_vx(fdict); + n_ovx = parse_conf_n_vx(odict); + + // Check for a valid number of verification tasks + if(n_fvx == 0 || n_fvx != n_ovx) { + mlog << Error << "\nPairStatConfInfo::process_config() -> " + << "The number of verification tasks in \"" + << conf_key_obs_pairs << "\" (" << n_ovx + << ") must be non-zero and match the number in \"" + << conf_key_fcst_pairs << "\" (" << n_fvx << ").\n\n"; + exit(1); + } + + // Allocate memory for the verification task options + n_vx = n_fvx; + vx_opt.resize(n_vx); + + // Check for consistent number of climatology fields + check_climo_n_vx(fdict, n_vx); + check_climo_n_vx(odict, n_vx); + + // Conf: threshold for SEEPS p1 + seeps_p1_thresh = conf.lookup_thresh(conf_key_seeps_p1_thresh); + + // Conf: SEEPS climo filename + seeps_climo_name = conf.lookup_string(conf_key_seeps_point_climo_name, false); + + // Parse settings for each verification task + for(i=0; iis_u_wind() && + vx_opt[i].vx_pd.obs_info->is_u_wind()) { + + // Search for corresponding v-wind + for(j=0; jis_v_wind() && + vx_opt[j].vx_pd.obs_info->is_v_wind() && + vx_opt[i].is_uv_match(vx_opt[j])) { + + mlog << Debug(3) << "U-wind field array entry " << i+1 + << " matches V-wind field array entry " << j+1 << ".\n"; + + // Print warning about multiple matches + if(vx_opt[i].vx_pd.fcst_info->uv_index() >= 0 || + vx_opt[i].vx_pd.obs_info->uv_index() >= 0) { + mlog << Warning << "\nPairStatConfInfo::process_config() -> " + << "For U-wind, found multiple matching V-wind field array entries! " + << "Using the first match found. Set the \"level\" strings to " + << "differentiate between them.\n\n"; + } + // Use the first match + else { + vx_opt[i].vx_pd.fcst_info->set_uv_index(j); + vx_opt[i].vx_pd.obs_info->set_uv_index(j); + } + } + } + + // No match found + if(vx_opt[i].vx_pd.fcst_info->uv_index() < 0 || + vx_opt[i].vx_pd.obs_info->uv_index() < 0) { + mlog << Debug(3) << "U-wind field array entry " << i+1 + << " has no matching V-wind field array entry.\n"; + } + + } + // Process v-wind + else if(vx_opt[i].vx_pd.fcst_info->is_v_wind() && + vx_opt[i].vx_pd.obs_info->is_v_wind()) { + + // Search for corresponding u-wind + for(j=0; jis_u_wind() && + vx_opt[j].vx_pd.obs_info->is_u_wind() && + vx_opt[i].is_uv_match(vx_opt[j])) { + + mlog << Debug(3) << "V-wind field array entry " << i+1 + << " matches U-wind field array entry " << j+1 << ".\n"; + + // Print warning about multiple matches + if(vx_opt[i].vx_pd.fcst_info->uv_index() >= 0 || + vx_opt[i].vx_pd.obs_info->uv_index() >= 0) { + mlog << Warning << "\nPairStatConfInfo::process_config() -> " + << "For U-wind, found multiple matching V-wind field array entries! " + << "Using the first match found. Set the \"level\" strings to " + << "differentiate between them.\n\n"; + } + // Use the first match + else { + vx_opt[i].vx_pd.fcst_info->set_uv_index(j); + vx_opt[i].vx_pd.obs_info->set_uv_index(j); + } + } + } + + // No match found + if(vx_opt[i].vx_pd.fcst_info->uv_index() < 0 || + vx_opt[i].vx_pd.obs_info->uv_index() < 0) { + mlog << Debug(3) << "V-wind field array entry " << i+1 + << " has no matching U-wind field array entry.\n"; + } + + } + } // end for i + } // end if + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void PairStatConfInfo::process_flags() { + int i, j; + bool output_ascii_flag = false; + + // Initialize + for(i=0; i " + << "At least one output STAT type must be requested in \"" + << conf_key_output_flag << "\".\n\n"; + exit(1); + } + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void PairStatConfInfo::process_masks() { + MaskPlane mp; + ConcatString name; + + mlog << Debug(2) + << "Processing masking regions.\n"; + + // Mapping of grid definition strings to mask names + map grid_map; + map poly_map; + map sid_map; + map point_map; + + // Initiailize + mask_area_map.clear(); + mask_sid_map.clear(); + + // Process the masks for each vx task + for(auto &vx : vx_opt) { + + // Initialize + vx.mask_name.clear(); + + // Parse the masking grids + for(int i=0; iis_u_wind() && + vx.vx_pd.obs_info->is_u_wind()) || + (vx.vx_pd.fcst_info->is_v_wind() && + vx.vx_pd.obs_info->is_v_wind())) { + vflag = true; + break; + } + } + + return vflag; +} + +//////////////////////////////////////////////////////////////////////// + +bool PairStatConfInfo::add_mpr_line(const STATLine &l) { + bool keep = false; + + // Attempt to add line to each verification task + for(auto &vx : vx_opt) { + if(vx.add_mpr_line(l)) keep = true; + } + + return keep; +} + +//////////////////////////////////////////////////////////////////////// +// +// Code for class PairStatVxOpt +// +//////////////////////////////////////////////////////////////////////// + +PairStatVxOpt::PairStatVxOpt() { + + init_from_scratch(); +} + +//////////////////////////////////////////////////////////////////////// + +void PairStatVxOpt::init_from_scratch() { + + clear(); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void PairStatVxOpt::clear() { + + // Initialize values + vx_pd.clear(); + vx_hdr.clear(); + + beg_ds = end_ds = bad_data_int; + + fcat_ta.clear(); + ocat_ta.clear(); + + fcnt_ta.clear(); + ocnt_ta.clear(); + cnt_logic = SetLogic::None; + + fwind_ta.clear(); + owind_ta.clear(); + wind_logic = SetLogic::None; + + mask_grid.clear(); + mask_poly.clear(); + mask_sid.clear(); + mask_llpnt.clear(); + + mpr_thr_inc_map.clear(); + mpr_str_inc_map.clear(); + mpr_str_exc_map.clear(); + + mask_name.clear(); + + eclv_points.clear(); + + cdf_info.clear(); + + ci_alpha.clear(); + + boot_info.clear(); + + hss_ec_value = bad_data_double; + rank_corr_flag = false; + + for(int i=0; ireq_level_name(), + v.vx_pd.fcst_info->req_level_name()) || + !is_req_level_match( vx_pd.obs_info->req_level_name(), + v.vx_pd.obs_info->req_level_name()) || + !(beg_ds == v.beg_ds ) || + !(end_ds == v.end_ds ) || + !(mask_grid == v.mask_grid ) || + !(mask_poly == v.mask_poly ) || + !(mask_sid == v.mask_sid ) || + !(mask_llpnt == v.mask_llpnt ) || + !(mask_name == v.mask_name )) match = false; + + return match; +} + +//////////////////////////////////////////////////////////////////////// + +void PairStatVxOpt::process_config(PairsFormat ftype, + Dictionary &fdict, Dictionary &odict) { + int n; + VarInfoFactory info_factory; + mapoutput_map; + Dictionary *dict; + const char *method_name = "PairStatVxOpt::process_config() -> "; + + // Initialize + clear(); + + // Allocate new VarInfo objects + vx_pd.set_fcst_info(info_factory.new_var_info(FileType_Pairs)); + vx_pd.set_obs_info(info_factory.new_var_info(FileType_Pairs)); + + // Set the VarInfo objects + vx_pd.fcst_info->set_dict(fdict); + vx_pd.obs_info->set_dict(odict); + + // Dump the contents of the current VarInfo + if(mlog.verbosity_level() >= 5) { + mlog << Debug(5) + << "Parsed forecast field:\n"; + vx_pd.fcst_info->dump(cout); + mlog << Debug(5) + << "Parsed observation field:\n"; + vx_pd.obs_info->dump(cout); + } + + // No support for wind direction + if(vx_pd.fcst_info->is_wind_direction() || + vx_pd.obs_info->is_wind_direction()) { + mlog << Error << "\n" << method_name + << "wind direction may not be verified using pair_stat.\n\n"; + exit(1); + } + + // Check that the observation field does not contain probabilities + if(vx_pd.obs_info->is_prob()) { + mlog << Error << "\n" << method_name + << "the observation field cannot contain probabilities.\n\n"; + exit(1); + } + + // Conf: output_flag + output_map = parse_conf_output_flag(&odict, txt_file_type, n_txt); + + // Populate the output_flag array with map values + for(int i=0; i= 5) { + mlog << Debug(5) + << "Parsed thresholds:\n" + << "Matched pair filter columns: " << write_css(mpr_sa) << "\n" + << "Matched pair filter thresholds: " << mpr_ta.get_str() << "\n" + << "Forecast categorical thresholds: " << fcat_ta.get_str() << "\n" + << "Observed categorical thresholds: " << ocat_ta.get_str() << "\n" + << "Forecast continuous thresholds: " << fcnt_ta.get_str() << "\n" + << "Observed continuous thresholds: " << ocnt_ta.get_str() << "\n" + << "Continuous threshold logic: " << setlogic_to_string(cnt_logic) << "\n" + << "Forecast wind speed thresholds: " << fwind_ta.get_str() << "\n" + << "Observed wind speed thresholds: " << owind_ta.get_str() << "\n" + << "Wind speed threshold logic: " << setlogic_to_string(wind_logic) << "\n"; + } + + // Verifying a probability field + if(vx_pd.fcst_info->is_prob()) { + fcat_ta = string_to_prob_thresh(fcat_ta.get_str().c_str()); + } + + // Check for equal threshold length for non-probability fields + if(!vx_pd.fcst_info->is_prob() && + fcat_ta.n() != ocat_ta.n()) { + + mlog << Error << "\n" << method_name + << "The number of thresholds for each field in \"fcst." + << conf_key_cat_thresh + << "\" must match the number of thresholds for each " + << "field in \"obs." << conf_key_cat_thresh << "\".\n\n"; + exit(1); + } + + // Add default continuous thresholds until the counts match + n = max(fcnt_ta.n(), ocnt_ta.n()); + while(fcnt_ta.n() < n) fcnt_ta.add(na_str); + while(ocnt_ta.n() < n) ocnt_ta.add(na_str); + + // Add default wind speed thresholds until the counts match + n = max(fwind_ta.n(), owind_ta.n()); + while(fwind_ta.n() < n) fwind_ta.add(na_str); + while(owind_ta.n() < n) owind_ta.add(na_str); + + // Verifying with multi-category contingency tables + if(!vx_pd.fcst_info->is_prob() && + (output_flag[i_mctc] != STATOutputType::None || + output_flag[i_mcts] != STATOutputType::None)) { + check_mctc_thresh(fcat_ta); + check_mctc_thresh(ocat_ta); + } + + // Conf: mask_grid + mask_grid = odict.lookup_string_array(conf_key_mask_grid); + + // Conf: mask_poly + mask_poly = odict.lookup_string_array(conf_key_mask_poly); + + // Conf: mask_sid + mask_sid = odict.lookup_string_array(conf_key_mask_sid); + + // Conf: mask_llpnt + mask_llpnt = parse_conf_llpnt_mask(&odict); + + // Conf: eclv_points + eclv_points = parse_conf_eclv_points(&odict); + + // Conf: climo_cdf + cdf_info = parse_conf_climo_cdf(&odict); + + // Conf: ci_alpha + ci_alpha = parse_conf_ci_alpha(&odict); + + // Conf: boot + boot_info = parse_conf_boot(&odict); + + // Conf: hss_ec_value + hss_ec_value = odict.lookup_double(conf_key_hss_ec_value); + + // Conf: rank_corr_flag + rank_corr_flag = odict.lookup_bool(conf_key_rank_corr_flag); + + // Conf: desc + vx_pd.set_desc(parse_conf_string(&odict, conf_key_desc, false).c_str()); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void PairStatVxOpt::set_vx_pd(PairStatConfInfo *conf_info) { + ConcatString cs; + StringArray sa; + + // Setup the VxPairDataPoint object for each mask + + // Check for at least one masking region + if(get_n_mask() == 0) { + mlog << Error << "\nPairStatVxOpt::set_vx_pd() -> " + << "At least one output masking region must be requested in \"" + << conf_key_mask_grid << "\", \"" + << conf_key_mask_poly << "\", \"" + << conf_key_mask_sid << "\", or \"" + << conf_key_mask_llpnt << "\".\n\n"; + exit(1); + } + + // Define the dimensions with n_msg_typ = n_interp = 1 + vx_pd.set_size(1, get_n_mask(), 1); + + // Size the header objects + vx_hdr.resize(get_n_mask()); + + // Store the MPR filtering maps + vx_pd.set_mpr_thr_inc_map(mpr_thr_inc_map); + vx_pd.set_mpr_str_inc_map(mpr_str_inc_map); + vx_pd.set_mpr_str_exc_map(mpr_str_exc_map); + + // Store the climo CDF info + vx_pd.set_climo_cdf_info_ptr(&cdf_info); + + // Define the masking information: grid, poly, sid, point + int n; + + // Define the grid masks + for(int i=0; imask_area_map[mask_name[n]])); + } + + // Define the poly masks + for(int i=0; imask_area_map[mask_name[n]])); + } + + // Define the station ID masks + for(int i=0; imask_sid_map[mask_name[n]])); + } + + // Define the Lat/Lon point masks + for(int i=0; i<(int) mask_llpnt.size(); i++) { + n = i + mask_grid.n() + mask_poly.n() + mask_sid.n(); + vx_pd.set_mask_llpnt(n, mask_name[n].c_str(), &mask_llpnt[i]); + } + + // After sizing VxPairDataPoint, add settings for each array element + if(output_flag[i_seeps_mpr] != STATOutputType::None || + output_flag[i_seeps] != STATOutputType::None) { + vx_pd.load_seeps_climo(conf_info->seeps_climo_name); + vx_pd.set_seeps_thresh(conf_info->seeps_p1_thresh); + } + + return; +} + +//////////////////////////////////////////////////////////////////////// + +void PairStatVxOpt::set_perc_thresh(const PairDataPoint *pd_ptr) { + + // + // Compute percentiles for forecast and observation thresholds, + // but not for wind speed or climatology CDF thresholds. + // + if(!fcat_ta.need_perc() && !ocat_ta.need_perc() && + !fcnt_ta.need_perc() && !ocnt_ta.need_perc()) return; + + // + // Sort the input arrays + // + NumArray f_sort = pd_ptr->f_na; + NumArray o_sort = pd_ptr->o_na; + NumArray fcmn_sort = pd_ptr->fcmn_na; + NumArray ocmn_sort = pd_ptr->ocmn_na; + f_sort.sort_array(); + o_sort.sort_array(); + fcmn_sort.sort_array(); + ocmn_sort.sort_array(); + + // + // Compute percentiles + // + fcat_ta.set_perc(&f_sort, &o_sort, &fcmn_sort, &ocmn_sort, &fcat_ta, &ocat_ta); + ocat_ta.set_perc(&f_sort, &o_sort, &fcmn_sort, &ocmn_sort, &fcat_ta, &ocat_ta); + fcnt_ta.set_perc(&f_sort, &o_sort, &fcmn_sort, &ocmn_sort, &fcnt_ta, &ocnt_ta); + ocnt_ta.set_perc(&f_sort, &o_sort, &fcmn_sort, &ocmn_sort, &fcnt_ta, &ocnt_ta); + + return; +} + +//////////////////////////////////////////////////////////////////////// + +int PairStatVxOpt::n_txt_row(int i_txt_row) const { + int n = 0; + int n_bin; + int n_pd = get_n_mask(); + const char *method_name = "PairStatVxOpt::n_txt_row(int) -> "; + + // Range check + if(i_txt_row < 0 || i_txt_row >= n_txt) { + mlog << Error << "\n" << method_name + << "range check error for " << i_txt_row << "\n\n"; + exit(1); + } + + // Check if this output line type is requested + if(output_flag[i_txt_row] == STATOutputType::None) return 0; + + bool prob_flag = vx_pd.fcst_info->is_prob(); + bool vect_flag = vx_pd.fcst_info->is_v_wind() && + vx_pd.fcst_info->uv_index() >= 0; + + // Determine row multiplier for climatology bins + if(cdf_info.write_bins) { + n_bin = get_n_cdf_bin(); + if(n_bin > 1) n_bin++; + } + else { + n_bin = 1; + } + + // Switch on the index of the line type + switch(i_txt_row) { + + case i_fho: + case i_ctc: + // Number of FHO or CTC lines = + // Message Types * Masks * Interpolations * Thresholds + n = (prob_flag ? 0 : n_pd * get_n_cat_thresh()); + break; + + case i_cts: + // Number of CTS lines = + // Message Types * Masks * Interpolations * Thresholds * + // Alphas + n = (prob_flag ? 0 : n_pd * get_n_cat_thresh() * + get_n_ci_alpha()); + break; + + case i_mctc: + // Number of MCTC lines = + // Message Types * Masks * Interpolations + n = (prob_flag ? 0 : n_pd); + break; + + case i_mcts: + // Number of MCTS lines = + // Message Types * Masks * Interpolations * Alphas + n = (prob_flag ? 0 : n_pd * get_n_ci_alpha()); + break; + + case i_cnt: + // Number of CNT lines = + // Message Types * Masks * Interpolations * Thresholds * + // Climo Bins * Alphas + n = (prob_flag ? 0 : n_pd * get_n_cnt_thresh() * n_bin * + get_n_ci_alpha()); + break; + + case i_sl1l2: + case i_sal1l2: + // Number of SL1L2 and SAL1L2 lines = + // Message Types * Masks * Interpolations * Thresholds * + // Climo Bins + n = (prob_flag ? 0 : n_pd * get_n_cnt_thresh() * n_bin); + break; + + case i_vcnt: + // Number of VCNT lines = + // Message Types * Masks * Interpolations * Thresholds * + // Alphas + n = (!vect_flag ? 0 : n_pd * + get_n_wind_thresh() * get_n_ci_alpha()); + break; + + case i_vl1l2: + case i_val1l2: + // Number of VL1L2 or VAL1L2 lines = + // Message Types * Masks * Interpolations * Thresholds + n = (!vect_flag ? 0 : n_pd * + get_n_wind_thresh()); + break; + + case i_pct: + case i_pjc: + case i_prc: + // Number of PCT, PJC, or PRC lines possible = + // Message Types * Masks * Interpolations * Thresholds * + // Climo Bins + n = (!prob_flag ? 0 : n_pd * get_n_oprob_thresh() * n_bin); + break; + + case i_pstd: + // Number of PSTD lines = + // Message Types * Masks * Interpolations * Thresholds * + // Alphas * Climo Bins + n = (!prob_flag ? 0 : n_pd * + get_n_oprob_thresh() * get_n_ci_alpha() * n_bin); + break; + + case i_eclv: + // Number of CTC -> ECLV lines = + // Message Types * Masks * Interpolations * Thresholds * + // Climo Bins + n = (prob_flag ? 0 : n_pd * get_n_cat_thresh() * n_bin); + + // Number of PCT -> ECLV lines = + // Message Types * Masks * Interpolations * + // Observation Probability Thresholds * + // Forecast Probability Thresholds * Climo Bins + n += (!prob_flag ? 0 : n_pd * + get_n_oprob_thresh() * get_n_fprob_thresh() * n_bin); + break; + + case i_mpr: + // Compute the number of matched pairs to be written + n = vx_pd.get_n_pair(); + break; + + case i_seeps_mpr: + // Compute the number of matched pairs to be written + n = vx_pd.get_n_pair(); + break; + + case i_seeps: + // Compute the number of matched pairs to be written + n = vx_pd.get_n_pair(); + break; + + default: + mlog << Error << "\n" << method_name + << "unexpected output type index value: " << i_txt_row + << "\n\n"; + exit(1); + } + return n; +} + +//////////////////////////////////////////////////////////////////////// + +int PairStatVxOpt::get_n_cnt_thresh() const { + return (!vx_pd.fcst_info || vx_pd.fcst_info->is_prob()) ? + 0 : fcnt_ta.n(); +} + +//////////////////////////////////////////////////////////////////////// + +int PairStatVxOpt::get_n_cat_thresh() const { + return (!vx_pd.fcst_info || vx_pd.fcst_info->is_prob()) ? + 0 : fcat_ta.n(); +} + +//////////////////////////////////////////////////////////////////////// + +int PairStatVxOpt::get_n_wind_thresh() const { + return (!vx_pd.fcst_info || vx_pd.fcst_info->is_prob()) ? + 0 : fwind_ta.n(); +} + +//////////////////////////////////////////////////////////////////////// + +int PairStatVxOpt::get_n_fprob_thresh() const { + return (!vx_pd.fcst_info || !vx_pd.fcst_info->is_prob()) ? + 0 : fcat_ta.n(); +} + +//////////////////////////////////////////////////////////////////////// + +int PairStatVxOpt::get_n_oprob_thresh() const { + return (!vx_pd.fcst_info || !vx_pd.fcst_info->is_prob()) ? + 0 : ocat_ta.n(); +} + +//////////////////////////////////////////////////////////////////////// + +bool PairStatVxOpt::is_keeper_mpr(const STATLine &l) const { + + // Check name and level strings + if(l.fcst_var() != vx_pd.fcst_info->name_attr() || + l.fcst_lev() != vx_pd.fcst_info->level_attr() || + l.obs_var() != vx_pd.obs_info->name_attr() || + l.obs_lev() != vx_pd.obs_info->level_attr()) return false; + + // Check MPR thresholds + for(auto &m : mpr_thr_inc_map) { + if(!m.second.check_dbl(atof(l.get_item(m.first.c_str())))) return false; + } + + // Check MPR string inclusions + for(auto &m : mpr_str_inc_map) { + if(!m.second.has(l.get_item(m.first.c_str()))) return false; + } + + // Check MPR string exclusions + for(auto &m : mpr_str_exc_map) { + if(m.second.has(l.get_item(m.first.c_str()))) return false; + } + + return true; +} + +//////////////////////////////////////////////////////////////////////// + +bool PairStatVxOpt::add_mpr_line(const STATLine &l) { + bool keep = false; + + // Check filtering options + if(is_keeper_mpr(l)) { + + // Parse climo data from the line + ClimoPntInfo cpi; + + // In met-6.1 and later: + // - CLIMO was replaced by CLIMO_MEAN + if(l.has("CLIMO")) { + double cmn = atof(l.get_item("CLIMO")); + double csd = bad_data_double; + cpi.set(cmn, csd, cmn, csd); + } + // In met-12.0.0 and later: + // - CLIMO_MEAN was replaced by OBS_CLIMO_MEAN + // - CLIMO_STDEV was replaced by OBS_CLIMO_STDEV + // - CLIMO_CDF was replaced by OBS_CLIMO_CDF + else if(l.has("CLIMO_MEAN")) { + double cmn = atof(l.get_item("CLIMO_MEAN")); + double csd = atof(l.get_item("CLIMO_STDEV")); + cpi.set(cmn, csd, cmn, csd); + } + else { + cpi.set(atof(l.get_item("FCST_CLIMO_MEAN")), + atof(l.get_item("FCST_CLIMO_STDEV")), + atof(l.get_item("OBS_CLIMO_MEAN")), + atof(l.get_item("OBS_CLIMO_STDEV"))); + } + + // Attempt to add to each masking region + for(int i_mask=0; i_mask + +#include "vx_config.h" +#include "vx_data2d.h" +#include "vx_grid.h" +#include "vx_util.h" +#include "vx_cal.h" +#include "vx_math.h" +#include "vx_gsl_prob.h" +#include "vx_analysis_util.h" +#include "vx_statistics.h" +#include "vx_stat_out.h" + +//////////////////////////////////////////////////////////////////////// + +// Reference global 1/10-th degree grid +static const LatLonData GlobalTenthData = + { "GlobalTenthDegree", -90.0, -0.0, 0.1, 0.1, 1801, 3601 }; +static Grid grid(GlobalTenthData); + +// Indices for the output flag types in the configuration file +static const int i_fho = 0; +static const int i_ctc = 1; +static const int i_cts = 2; +static const int i_mctc = 3; +static const int i_mcts = 4; + +static const int i_cnt = 5; +static const int i_sl1l2 = 6; +static const int i_sal1l2 = 7; + +static const int i_vcnt = 8; +static const int i_vl1l2 = 9; +static const int i_val1l2 = 10; + +static const int i_pct = 11; +static const int i_pstd = 12; +static const int i_pjc = 13; +static const int i_prc = 14; +static const int i_eclv = 15; + +static const int i_mpr = 16; +static const int i_seeps_mpr = 17; +static const int i_seeps = 18; + +static const int n_txt = 19; + +// Text file type +static const STATLineType txt_file_type[n_txt] = { + + STATLineType::fho, // 0 + STATLineType::ctc, // 1 + STATLineType::cts, // 2 + STATLineType::mctc, // 3 + STATLineType::mcts, // 4 + + STATLineType::cnt, // 5 + STATLineType::sl1l2, // 6 + STATLineType::sal1l2, // 7 + + STATLineType::vcnt, // 8 + STATLineType::vl1l2, // 9 + STATLineType::val1l2, // 10 + + STATLineType::pct, // 11 + STATLineType::pstd, // 12 + STATLineType::pjc, // 13 + STATLineType::prc, // 14 + STATLineType::eclv, // 15 + + STATLineType::mpr, // 16 + STATLineType::seeps_mpr, // 17 + STATLineType::seeps // 18 +}; + +/////////////////////////////////////////////////////////////////////////////// + +// +// Supported input pairs formats +// + +enum class PairsFormat { + None, // Default + MPR, // ASCII files containing MET MPR lines + Python, // Stat MPR data via Python embedding + IODA, // IODA pairs file +}; + +/////////////////////////////////////////////////////////////////////////////// + +extern ConcatString pairsformat_to_string(const PairsFormat); +extern PairsFormat string_to_pairsformat(const std::string &); + +/////////////////////////////////////////////////////////////////////////////// + +class PairStatConfInfo; // forward reference + +/////////////////////////////////////////////////////////////////////////////// + +class PairStatVxOpt { + + private: + + void init_from_scratch(); + + public: + + PairStatVxOpt(); + + ////////////////////////////////////////////////////////////////// + + VxPairDataPoint vx_pd; // Matched pair data [n_mask] + std::vector vx_hdr; // Track header inputs [n_mask] + + int beg_ds; // Begin observation time window offset + int end_ds; // End observation time window offset + + ThreshArray fcat_ta; // Array for fcst categorical thresholds + ThreshArray ocat_ta; // Array for obs categorical thresholds + + ThreshArray fcnt_ta; // Array for fcst continuous thresholds + ThreshArray ocnt_ta; // Array for obs continuous thresholds + SetLogic cnt_logic; // Array of continuous threshold field logic + + ThreshArray fwind_ta; // Array for fcst wind speed thresholds + ThreshArray owind_ta; // Array for obs wind speed thresholds + SetLogic wind_logic; // Array of wind speed field logic + + StringArray mask_grid; // Masking grid strings + StringArray mask_poly; // Masking polyline strings + StringArray mask_sid; // Masking station ID's + + // Matched pair inclusion thresholds + std::map mpr_thr_inc_map; + + // Matched pair inclusion and exclusion strings + std::map mpr_str_inc_map; + std::map mpr_str_exc_map; + + // Vector of MaskLatLon objects defining Lat/Lon Point masks + std::vector mask_llpnt; + + StringArray mask_name; // Masking names + + NumArray eclv_points; // ECLV points + + ClimoCDFInfo cdf_info; // Climo CDF info + + NumArray ci_alpha; // Alpha value for confidence intervals + + BootInfo boot_info; // Bootstrapping information + + double hss_ec_value; // HSS expected correct value + bool rank_corr_flag; // Flag for computing rank correlations + + // Output file options + STATOutputType output_flag[n_txt]; // Flag for each output line type + + ////////////////////////////////////////////////////////////////// + + void clear(); + + void process_config(PairsFormat, Dictionary &, Dictionary &); + void set_vx_pd(PairStatConfInfo *); + bool is_uv_match(const PairStatVxOpt &) const; + + void set_perc_thresh(const PairDataPoint *); + + // Compute the number of output lines for this task + int n_txt_row(int i) const; + + int get_n_mask() const; + + int get_n_cnt_thresh() const; + int get_n_cat_thresh() const; + int get_n_wind_thresh() const; + + int get_n_fprob_thresh() const; + int get_n_oprob_thresh() const; + + int get_n_eclv_points() const; + int get_n_cdf_bin() const; + int get_n_ci_alpha() const; + + // Check for matches + bool is_keeper_mpr(const STATLine &) const; + + // Add paired data + bool add_mpr_line(const STATLine &); +}; + +//////////////////////////////////////////////////////////////////////// + +inline int PairStatVxOpt::get_n_mask() const { return mask_name.n(); } +inline int PairStatVxOpt::get_n_eclv_points() const { return eclv_points.n(); } +inline int PairStatVxOpt::get_n_cdf_bin() const { return cdf_info.n_bin; } +inline int PairStatVxOpt::get_n_ci_alpha() const { return ci_alpha.n(); } + +//////////////////////////////////////////////////////////////////////// + +class PairStatConfInfo { + + private: + + void init_from_scratch(); + + // Number of verification tasks + int n_vx; + + public: + + PairStatConfInfo(); + + ////////////////////////////////////////////////////////////////// + + // Pair-Stat configuration object + MetConfig conf; + + // Model name from the Pair-Stat config file + ConcatString model; // Model name + + std::vector vx_opt; // Vector of vx options [n_vx] + + // Mapping of mask names to DataPlanes + std::map mask_area_map; + + // Mapping of mask names to Station ID lists + std::map mask_sid_map; + + PointWeightType point_weight_flag; // Point weighting flag + + ConcatString tmp_dir; // Directory for temporary files + ConcatString version; // Config file version + + ConcatString seeps_climo_name; // SEESP climo filename + SingleThresh seeps_p1_thresh; // SEESP p1 threshold + + // Summary of output file options across all verification tasks + STATOutputType output_flag[n_txt]; // Flag for each output line type + + ////////////////////////////////////////////////////////////////// + + void clear(); + + void read_config(const StringArray &); + + void process_config(PairsFormat); + void process_flags(); + void process_masks(); + void set_vx_pd(); + + // Dump out the counts + int get_n_vx() const; + + // Compute the maximum number of output lines possible based + // on the contents of the configuration file + int n_txt_row(int i) const; + int n_stat_row() const; + + // Maximum across all verification tasks + int get_max_n_cat_thresh() const; + int get_max_n_cnt_thresh() const; + int get_max_n_wind_thresh() const; + int get_max_n_fprob_thresh() const; + int get_max_n_oprob_thresh() const; + int get_max_n_eclv_points() const; + + // Check for any verification of vectors + bool get_vflag() const; + + // Add paired data + bool add_mpr_line(const STATLine &); +}; + +//////////////////////////////////////////////////////////////////////// + +inline int PairStatConfInfo::get_n_vx() const { return(n_vx); } + +//////////////////////////////////////////////////////////////////////// + +#endif /* __PAIR_STAT_CONF_INFO_H__ */ + +//////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/point_stat/point_stat.cc b/src/tools/core/point_stat/point_stat.cc index 735f85107e..de924f6009 100644 --- a/src/tools/core/point_stat/point_stat.cc +++ b/src/tools/core/point_stat/point_stat.cc @@ -568,7 +568,7 @@ void setup_table(AsciiTable &at) { at.set_bad_data_str(na_str); // Don't write out trailing blank rows - at.set_delete_trailing_blank_rows(1); + at.set_delete_trailing_blank_rows(true); return; } diff --git a/src/tools/core/point_stat/point_stat_conf_info.cc b/src/tools/core/point_stat/point_stat_conf_info.cc index cfd5e10432..f5b89dad4f 100644 --- a/src/tools/core/point_stat/point_stat_conf_info.cc +++ b/src/tools/core/point_stat/point_stat_conf_info.cc @@ -762,8 +762,7 @@ void PointStatVxOpt::clear() { mask_sid.clear(); mask_llpnt.clear(); - mpr_sa.clear(); - mpr_ta.clear(); + mpr_thr_inc_map.clear(); mask_name.clear(); @@ -919,8 +918,26 @@ void PointStatVxOpt::process_config(GrdFileType ftype, int_to_setlogic(odict.lookup_int(conf_key_wind_logic))); // Conf: mpr_column and mpr_thresh - mpr_sa = odict.lookup_string_array(conf_key_mpr_column); - mpr_ta = odict.lookup_thresh_array(conf_key_mpr_thresh); + StringArray mpr_sa(odict.lookup_string_array(conf_key_mpr_column)); + ThreshArray mpr_ta(odict.lookup_thresh_array(conf_key_mpr_thresh)); + + // Check for the same length + if(mpr_sa.n() != mpr_ta.n()) { + mlog << Error << "\nPointStatVxOpt::process_config() -> " + << "The length of \"" << conf_key_mpr_column << "\" and \"" + << conf_key_mpr_thresh << "\" must match (" << mpr_sa.n() + << " != " << mpr_ta.n() << ")!\n\n"; + exit(1); + } + + // Store in map + for(int i=0; i= 5) { @@ -1089,7 +1106,7 @@ void PointStatVxOpt::set_vx_pd(PointStatConfInfo *conf_info) { vx_pd.set_size(n_msg_typ, n_mask, n_interp); // Store the MPR filter threshold - vx_pd.set_mpr_thresh(mpr_sa, mpr_ta); + vx_pd.set_mpr_thr_inc_map(mpr_thr_inc_map); // Store the climo CDF info vx_pd.set_climo_cdf_info_ptr(&cdf_info); diff --git a/src/tools/core/point_stat/point_stat_conf_info.h b/src/tools/core/point_stat/point_stat_conf_info.h index 9db3081dd7..732a3fd4ec 100644 --- a/src/tools/core/point_stat/point_stat_conf_info.h +++ b/src/tools/core/point_stat/point_stat_conf_info.h @@ -129,8 +129,8 @@ class PointStatVxOpt { StringArray mask_poly; // Masking polyline strings StringArray mask_sid; // Masking station ID's - StringArray mpr_sa; // MPR column names - ThreshArray mpr_ta; // MPR column thresholds + // Matched pair inclusion thresholds + std::map mpr_thr_inc_map; // Vector of MaskLatLon objects defining Lat/Lon Point masks std::vector mask_llpnt; diff --git a/src/tools/core/stat_analysis/aggr_stat_line.cc b/src/tools/core/stat_analysis/aggr_stat_line.cc index f7b9ccaf72..de93c55669 100644 --- a/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -66,433 +66,8 @@ using namespace std; //////////////////////////////////////////////////////////////////////// static bool is_precip_var_name(const ConcatString &s); -static const string case_str = "CASE"; static bool is_vector_dir_stat(const STATLineType &t, const ConcatString &s); -//////////////////////////////////////////////////////////////////////// -// -// Code for StatHdrInfo structure. -// -//////////////////////////////////////////////////////////////////////// - -StatHdrInfo::StatHdrInfo() { - clear(); -} - -//////////////////////////////////////////////////////////////////////// - -void StatHdrInfo::clear() { - model.clear(); - desc.clear(); - fcst_lead.clear(); - fcst_valid_beg = fcst_valid_end = (unixtime) 0; - obs_lead.clear(); - obs_valid_beg = obs_valid_end = (unixtime) 0; - fcst_var.clear(); - fcst_units.clear(); - fcst_lev.clear(); - obs_var.clear(); - obs_units.clear(); - obs_lev.clear(); - obtype.clear(); - vx_mask.clear(); - interp_mthd.clear(); - interp_pnts.clear(); - fcst_thresh.clear(); - obs_thresh.clear(); - cov_thresh.clear(); - alpha.clear(); - - return; -} - -//////////////////////////////////////////////////////////////////////// -// -// Keep track the unique STAT header entries for each line. -// -//////////////////////////////////////////////////////////////////////// - -void StatHdrInfo::add(const STATLine &line) { - ConcatString cs; - if(!model.has(line.model())) - model.add(line.model()); - if(!desc.has(line.desc())) - desc.add(line.desc()); - if(!fcst_lead.has(line.fcst_lead())) - fcst_lead.add(line.fcst_lead()); - if(fcst_valid_beg == (unixtime) 0 || line.fcst_valid_beg() < fcst_valid_beg) - fcst_valid_beg = line.fcst_valid_beg(); - if(fcst_valid_end == (unixtime) 0 || line.fcst_valid_end() > fcst_valid_end) - fcst_valid_end = line.fcst_valid_end(); - if(!obs_lead.has(line.obs_lead())) - obs_lead.add(line.obs_lead()); - if(obs_valid_beg == (unixtime) 0 || line.obs_valid_beg() < obs_valid_beg) - obs_valid_beg = line.obs_valid_beg(); - if(obs_valid_end == (unixtime) 0 || line.obs_valid_end() > obs_valid_end) - obs_valid_end = line.obs_valid_end(); - if(!fcst_var.has(line.fcst_var())) - fcst_var.add(line.fcst_var()); - if(!fcst_units.has(line.fcst_units())) - fcst_units.add(line.fcst_units()); - if(!fcst_lev.has(line.fcst_lev())) - fcst_lev.add(line.fcst_lev()); - if(!obs_var.has(line.obs_var())) - obs_var.add(line.obs_var()); - if(!obs_units.has(line.obs_units())) - obs_units.add(line.obs_units()); - if(!obs_lev.has(line.obs_lev())) - obs_lev.add(line.obs_lev()); - if(!obtype.has(line.obtype())) - obtype.add(line.obtype()); - if(!vx_mask.has(line.vx_mask())) - vx_mask.add(line.vx_mask()); - if(!interp_mthd.has(line.interp_mthd())) - interp_mthd.add(line.interp_mthd()); - if(!interp_pnts.has(line.interp_pnts())) - interp_pnts.add(line.interp_pnts()); - cs = line.get_item("FCST_THRESH", false); - cs.strip_paren(); - if(!fcst_thresh.has(cs)) fcst_thresh.add(cs); - cs = line.get_item("OBS_THRESH", false); - cs.strip_paren(); - if(!obs_thresh.has(cs)) obs_thresh.add(cs); - cs = line.get_item("COV_THRESH", false); - cs.strip_paren(); - if(!cov_thresh.has(cs)) cov_thresh.add(cs); - if(!alpha.has(line.alpha())) - alpha.add(line.alpha()); - - return; -} - -//////////////////////////////////////////////////////////////////////// -// -// Check for and print debug and warning messages about multiple header -// column values found. -// -//////////////////////////////////////////////////////////////////////// - -void StatHdrInfo::check_shc(const ConcatString &cur_case) { - - // MODEL - if(model.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << model.n() - << " unique MODEL values: " - << write_css(model) << "\n"; - } - - // DESC - if(desc.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << desc.n() - << " unique DESC values: " - << write_css(desc) << "\n"; - } - - // FCST_LEAD - if(fcst_lead.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << fcst_lead.n() - << " unique FCST_LEAD values: " - << write_css_hhmmss(fcst_lead) << "\n"; - } - - // OBS_LEAD - if(obs_lead.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << obs_lead.n() - << " unique OBS_LEAD values: " - << write_css_hhmmss(obs_lead) << "\n"; - } - - // FCST_VAR - if(fcst_var.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << fcst_var.n() - << " unique FCST_VAR values: " - << write_css(fcst_var) << "\n"; - } - - // FCST_UNITS - if(fcst_units.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << fcst_units.n() - << " unique FCST_UNITS values: " - << write_css(fcst_units) << "\n"; - } - - // FCST_LEV - if(fcst_lev.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << fcst_lev.n() - << " unique FCST_LEV values: " - << write_css(fcst_lev) << "\n"; - } - - // OBS_VAR - if(obs_var.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << obs_var.n() - << " unique OBS_VAR values: " - << write_css(obs_var) << "\n"; - } - - // OBS_UNITS - if(obs_units.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << obs_units.n() - << " unique OBS_UNITS values: " - << write_css(obs_units) << "\n"; - } - - // OBS_LEV - if(obs_lev.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << obs_lev.n() - << " unique OBS_LEV values: " - << write_css(obs_lev) << "\n"; - } - - // OBTYPE - if(obtype.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << obtype.n() - << " unique OBTYPE values: " - << write_css(obtype) << "\n"; - } - - // VX_MASK - if(vx_mask.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << vx_mask.n() - << " unique VX_MASK values: " - << write_css(vx_mask) << "\n"; - } - - // INTERP_MTHD - if(interp_mthd.n() > 1) { - mlog << Warning - << "For case \"" << cur_case << "\", found " - << interp_mthd.n() - << " unique INTERP_MTHD values: " - << write_css(interp_mthd) << ".\n"; - } - - // INTERP_PNTS - if(interp_pnts.n() > 1) { - mlog << Warning - << "For case \"" << cur_case << "\", found " - << interp_pnts.n() - << " unique INTERP_PNTS values: " - << write_css(interp_pnts) << ".\n"; - } - - // FCST_THRESH - if(fcst_thresh.n() > 1) { - mlog << Warning - << "For case \"" << cur_case << "\", found " - << fcst_thresh.n() - << " unique FCST_THRESH values: " - << write_css(fcst_thresh) << "\n"; - } - - // OBS_THRESH - if(obs_thresh.n() > 1) { - mlog << Warning - << "For case \"" << cur_case << "\", found " - << obs_thresh.n() - << " unique OBS_THRESH values: " - << write_css(obs_thresh) << "\n"; - } - - // COV_THRESH - if(cov_thresh.n() > 1) { - mlog << Warning - << "For case \"" << cur_case << "\", found " - << cov_thresh.n() - << " unique COV_THRESH values: " - << write_css(cov_thresh) << ".\n"; - } - - // ALPHA - if(alpha.n() > 1) { - mlog << Warning - << "For case \"" << cur_case << "\", found " - << alpha.n() - << " unique ALPHA values: " - << write_css(alpha) << ".\n"; - } - - return; -} - -//////////////////////////////////////////////////////////////////////// -// -// Use the StatHdrInfo struct to populate a StatHdrColumns object -// -//////////////////////////////////////////////////////////////////////// - -StatHdrColumns StatHdrInfo::get_shc(const ConcatString &cur_case, - const StringArray &case_cols, - const StringArray &hdr_cols, - const StringArray &hdr_vals, - const STATLineType lt) { - ThreshArray ta; - ConcatString css; - double out_alpha; - int wdth; - StatHdrColumns shc; - - // MODEL - shc.set_model(get_col_css(cur_case, "MODEL", model, false).c_str()); - - // DESC - shc.set_desc(get_col_css(cur_case, "DESC", desc, false).c_str()); - - // FCST_LEAD - css = write_css_hhmmss(fcst_lead); - if(fcst_lead.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << fcst_lead.n() - << " unique FCST_LEAD values: " << css << "\n"; - } - shc.set_fcst_lead_sec(fcst_lead.max()); - - // FCST_VALID_BEG, FCST_VALID_END - shc.set_fcst_valid_beg(fcst_valid_beg); - shc.set_fcst_valid_end(fcst_valid_end); - - // OBS_LEAD - css = write_css_hhmmss(obs_lead); - if(obs_lead.n() > 1) { - mlog << Debug(2) - << "For case \"" << cur_case << "\", found " - << obs_lead.n() - << " unique OBS_LEAD values: " << css << "\n"; - } - shc.set_obs_lead_sec(obs_lead.max()); - - // OBS_VALID_BEG, OBS_VALID_END - shc.set_obs_valid_beg(obs_valid_beg); - shc.set_obs_valid_end(obs_valid_end); - - // FCST_VAR - shc.set_fcst_var(get_col_css(cur_case, "FCST_VAR", fcst_var, false)); - - // FCST_UNITS - shc.set_fcst_units(get_col_css(cur_case, "FCST_UNITS", fcst_units, false)); - - // FCST_LEV - shc.set_fcst_lev(get_col_css(cur_case, "FCST_LEV", fcst_lev, false).c_str()); - - // OBS_VAR - shc.set_obs_var(get_col_css(cur_case, "OBS_VAR", obs_var, false)); - - // OBS_UNITS - shc.set_obs_units(get_col_css(cur_case, "OBS_UNITS", obs_units, false)); - - // OBS_LEV - shc.set_obs_lev(get_col_css(cur_case, "OBS_LEV", obs_lev, false).c_str()); - - // OBTYPE - shc.set_obtype(get_col_css(cur_case, "OBTYPE", obtype, false).c_str()); - - // VX_MASK - shc.set_mask(get_col_css(cur_case, "VX_MASK", vx_mask, false).c_str()); - - // INTERP_MTHD - shc.set_interp_mthd(get_col_css(cur_case, "INTERP_MTHD", interp_mthd, true)); - - // INTERP_PNTS - css = write_css(interp_pnts); - if(interp_pnts.n() == 0 || interp_pnts.n() > 1) { - mlog << Warning - << "For case \"" << cur_case << "\", found " - << interp_pnts.n() - << " unique INTERP_PNTS values: " << css << ".\n"; - wdth = bad_data_int; - } - else { - wdth = nint(sqrt(interp_pnts[0])); - } - shc.set_interp_wdth(wdth); - - // FCST_THRESH - ta.clear(); - ta.add_css(get_col_css(cur_case, "FCST_THRESH", fcst_thresh, true).c_str()); - shc.set_fcst_thresh(ta); - - // OBS_THRESH - ta.clear(); - ta.add_css(get_col_css(cur_case, "OBS_THRESH", obs_thresh, true).c_str()); - shc.set_obs_thresh(ta); - - // COV_THRESH - ta.clear(); - ta.add_css(get_col_css(cur_case, "COV_THRESH", cov_thresh, true).c_str()); - shc.set_cov_thresh(ta); - - // ALPHA - css = write_css(alpha); - if(alpha.n() == 0 || alpha.n() > 1) { - mlog << Warning - << "For case \"" << cur_case << "\", found " - << alpha.n() - << " unique ALPHA values: " << css << ".\n"; - out_alpha = bad_data_double; - } - else { - out_alpha = alpha[0]; - } - shc.set_alpha(out_alpha); - - // LINE_TYPE - shc.set_line_type(statlinetype_to_string(lt)); - - // Apply the -set_hdr options - StringArray case_vals = cur_case.split(":"); - shc.apply_set_hdr_opts(hdr_cols, hdr_vals, case_cols, case_vals); - - return shc; -} - -//////////////////////////////////////////////////////////////////////// - -ConcatString StatHdrInfo::get_col_css(const ConcatString &cur_case, - const char *col_name, - const StringArray &col_vals, - bool warning) const { - - // Build comma-separated list of column values - ConcatString css(write_css(col_vals)); - - // Check for multiple entries - if(col_vals.n() > 1) { - ConcatString msg; - msg << "For case \"" << cur_case << "\", found " - << col_vals.n() << " unique " << col_name - << " values: " << css << "\n"; - if(warning) mlog << Warning << msg; - else mlog << Debug(2) << msg; - } - - return css; -} - //////////////////////////////////////////////////////////////////////// // // Code for AggrENSInfo structure @@ -2173,7 +1748,7 @@ void aggr_mpr_lines(LineDataFile &f, STATAnalysisJob &job, // if(m[key].fcst_var != cur.fcst_var || m[key].obs_var != cur.obs_var) { - mlog << Error << "\nread_mpr_lines() -> " + mlog << Error << "\naggr_mpr_lines() -> " << "both the forecast and observation variable types must " << "remain constant. Try setting \"-fcst_var\" and/or " << "\"-obs_var\".\n" diff --git a/src/tools/core/stat_analysis/aggr_stat_line.h b/src/tools/core/stat_analysis/aggr_stat_line.h index d0ec0d9c7b..ab82d17e69 100644 --- a/src/tools/core/stat_analysis/aggr_stat_line.h +++ b/src/tools/core/stat_analysis/aggr_stat_line.h @@ -60,32 +60,6 @@ static const int min_time_series = 10; //////////////////////////////////////////////////////////////////////// -struct StatHdrInfo { - StringArray model, desc; - StringArray fcst_var, fcst_units, fcst_lev; - StringArray obs_var, obs_units, obs_lev; - StringArray obtype, vx_mask, interp_mthd; - StringArray fcst_thresh, obs_thresh, cov_thresh; - NumArray fcst_lead, obs_lead, interp_pnts, alpha; - unixtime fcst_valid_beg, fcst_valid_end; - unixtime obs_valid_beg, obs_valid_end; - - StatHdrInfo(); - - void clear(); - void add(const STATLine &line); - void check_shc(const ConcatString &cur_case); - StatHdrColumns get_shc(const ConcatString &cur_case, - const StringArray &case_cols, - const StringArray &hdr_cols, - const StringArray &hdr_vals, - const STATLineType lt); - ConcatString get_col_css(const ConcatString &cur_case, - const char *col_name, - const StringArray &col_vals, - bool warning) const; -}; - struct AggrSummaryInfo { StatHdrInfo hdr; std::map val; diff --git a/src/tools/core/stat_analysis/stat_analysis_job.cc b/src/tools/core/stat_analysis/stat_analysis_job.cc index c4a786b11f..2b722f0f02 100644 --- a/src/tools/core/stat_analysis/stat_analysis_job.cc +++ b/src/tools/core/stat_analysis/stat_analysis_job.cc @@ -4407,7 +4407,7 @@ void setup_table(AsciiTable &at, int n_hdr_cols, int prec) { at.set_bad_data_str(na_str); // Don't write out trailing blank rows - at.set_delete_trailing_blank_rows(1); + at.set_delete_trailing_blank_rows(true); return; } diff --git a/src/tools/core/wavelet_stat/wavelet_stat.cc b/src/tools/core/wavelet_stat/wavelet_stat.cc index 89d05ce6ae..7b7ac5c634 100644 --- a/src/tools/core/wavelet_stat/wavelet_stat.cc +++ b/src/tools/core/wavelet_stat/wavelet_stat.cc @@ -647,7 +647,7 @@ void setup_table(AsciiTable &at) { at.set_bad_data_str(na_str); // Don't write out trailing blank rows - at.set_delete_trailing_blank_rows(1); + at.set_delete_trailing_blank_rows(true); return; } diff --git a/src/tools/other/gsi_tools/gsi_util.cc b/src/tools/other/gsi_tools/gsi_util.cc index 82b4000ae9..3c717b146b 100644 --- a/src/tools/other/gsi_tools/gsi_util.cc +++ b/src/tools/other/gsi_tools/gsi_util.cc @@ -351,7 +351,7 @@ void setup_table(AsciiTable &at) { at.set_bad_data_str(na_str); // Don't write out trailing blank rows - at.set_delete_trailing_blank_rows(1); + at.set_delete_trailing_blank_rows(true); return; } diff --git a/src/tools/other/ioda2nc/Makefile.am b/src/tools/other/ioda2nc/Makefile.am index 9694454dac..3f7ae84c90 100644 --- a/src/tools/other/ioda2nc/Makefile.am +++ b/src/tools/other/ioda2nc/Makefile.am @@ -27,6 +27,7 @@ ioda2nc_LDADD = -lvx_stat_out \ -lvx_data2d_nc_cf \ $(UGRID_MET_LIBS) \ $(PYTHON_MET_LIBS) \ + -lvx_ioda \ -lvx_data2d \ -lvx_nc_obs \ -lvx_nc_util \ @@ -43,6 +44,7 @@ ioda2nc_LDADD = -lvx_stat_out \ -lvx_math \ -lvx_cal \ -lvx_log \ + -lvx_statistics \ $(GRIB2_DEP_LIBS) $(UGRID_DEP_LIBS) $(PYTHON_DEP_LIBS) \ -lm -lproj -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas $(BLIB_NAME) \ $(FLIBS) diff --git a/src/tools/other/ioda2nc/Makefile.in b/src/tools/other/ioda2nc/Makefile.in index cff7fbea89..9824efbd2e 100644 --- a/src/tools/other/ioda2nc/Makefile.in +++ b/src/tools/other/ioda2nc/Makefile.in @@ -361,6 +361,7 @@ ioda2nc_LDADD = -lvx_stat_out \ -lvx_data2d_nc_cf \ $(UGRID_MET_LIBS) \ $(PYTHON_MET_LIBS) \ + -lvx_ioda \ -lvx_data2d \ -lvx_nc_obs \ -lvx_nc_util \ @@ -377,6 +378,7 @@ ioda2nc_LDADD = -lvx_stat_out \ -lvx_math \ -lvx_cal \ -lvx_log \ + -lvx_statistics \ $(GRIB2_DEP_LIBS) $(UGRID_DEP_LIBS) $(PYTHON_DEP_LIBS) \ -lm -lproj -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas $(BLIB_NAME) \ $(FLIBS) diff --git a/src/tools/other/ioda2nc/ioda2nc.cc b/src/tools/other/ioda2nc/ioda2nc.cc index 4e19fd2794..c40ed32147 100644 --- a/src/tools/other/ioda2nc/ioda2nc.cc +++ b/src/tools/other/ioda2nc/ioda2nc.cc @@ -37,6 +37,7 @@ #include "main.h" #include "apply_mask.h" +#include "ioda.h" #include "ioda2nc_conf_info.h" #include "vx_log.h" #include "vx_nc_util.h" @@ -60,20 +61,9 @@ using namespace netCDF; // Constants // -static const char *DEF_CONFIG_NAME = "MET_BASE/config/IODA2NCConfig_default"; +static constexpr char DEF_CONFIG_NAME[] = "MET_BASE/config/IODA2NCConfig_default"; -static const char *program_name = "ioda2nc"; - -static const int REJECT_DEBUG_LEVEL = 9; -static const int string_data_len = 512; - -static const char *metadata_group_name = "MetaData"; -static const char *qc_group_name = "QCFlags"; -static const char *qc_postfix = "PreQC"; -static const char *obs_group_name = "ObsValue"; -static const char *derived_obs_group_name = "DerivedObsValue"; - -enum class e_ioda_format { v1, v2 }; +static constexpr char program_name[] = "ioda2nc"; //////////////////////////////////////////////////////////////////////// @@ -81,6 +71,8 @@ enum class e_ioda_format { v1, v2 }; // Variables for command line arguments // +static iodaReader ioda_reader; + // StringArray to store IODA file name static StringArray ioda_files; @@ -101,7 +93,8 @@ static MaskPoly mask_poly; static StringArray mask_sid; // Beginning and ending retention times -static unixtime valid_beg_ut, valid_end_ut; +static unixtime valid_beg_ut; +static unixtime valid_end_ut; // Number of IODA messages to process from the command line static int nmsg = -1; @@ -138,15 +131,15 @@ static NcFile *f_out = (NcFile *) nullptr; static void initialize(); static void process_command_line(int, char **); -static void open_netcdf(); +static void open_netcdf_output(); static void process_ioda_file(int); static void write_netcdf_hdr_data(); static void clean_up(); -static void addObservation(const float *obs_arr, const ConcatString &hdr_typ, +static void addObservation(const double *obs_arr, const ConcatString &hdr_typ, const ConcatString &hdr_sid, const time_t hdr_vld, - const float hdr_lat, const float hdr_lon, const float hdr_elv, - const float quality_mark, const int buf_size); + const double hdr_lat, const double hdr_lon, const double hdr_elv, + const double quality_mark); static bool keep_message_type(const char *); static bool keep_station_id(const char *); @@ -167,17 +160,15 @@ static void set_valid_end_time(const StringArray &); static void set_verbosity(const StringArray &); static bool check_core_data(const bool, const bool, - StringArray &, StringArray &, e_ioda_format); -static bool check_missing_thresh(float value); + const StringArray &, const StringArray &, e_ioda_format); static ConcatString find_meta_name(string meta_key, StringArray available_names); -static bool get_meta_data_float(NcFile *, StringArray &, const char *, float *, - const int); +static bool get_meta_data_double(NcFile *, StringArray &, const char *, double *, + const int); static bool get_meta_data_strings(NcVar &, char *); static bool get_meta_data_strings(NcVar &, char **); -static bool get_obs_data_float(NcFile *, const ConcatString, NcVar *, - float *, int *, const int, const e_ioda_format); +static bool get_obs_data_double(NcFile *, const ConcatString &, NcVar *, + double *, int *, const int, const e_ioda_format); static bool has_postfix(const std::string &, std::string const &); -static bool is_in_metadata_map(string metadata_key, StringArray &available_list); //////////////////////////////////////////////////////////////////////// @@ -191,11 +182,10 @@ int met_main(int argc, char *argv[]) { process_command_line(argc, argv); // Open the NetCDF file - open_netcdf(); + open_netcdf_output(); // Process each IODA file for(int i=0; i= valid_beg_ut if(valid_beg_ut != (unixtime) 0 && valid_end_ut != (unixtime) 0 && valid_beg_ut > valid_end_ut) { - mlog << Error << "\nprocess_command_line() -> " + mlog << Error << "\n" << method_name << "the ending time (" << unix_to_yyyymmdd_hhmmss(valid_end_ut) << ") must be greater than the beginning time (" << unix_to_yyyymmdd_hhmmss(valid_beg_ut) << ").\n\n"; @@ -329,7 +319,7 @@ void process_command_line(int argc, char **argv) { //////////////////////////////////////////////////////////////////////// -void open_netcdf() { +static void open_netcdf_output() { // Create the output netCDF file for writing mlog << Debug(1) << "Creating NetCDF File:\t\t" << ncfile << "\n"; @@ -337,7 +327,7 @@ void open_netcdf() { // Check for a valid file if(IS_INVALID_NC_P(f_out)) { - mlog << Error << "\nopen_netcdf() -> " + mlog << Error << "\nopen_netcdf_output() -> " << "trouble opening output file: " << ncfile << "\n\n"; delete f_out; @@ -360,36 +350,52 @@ void open_netcdf() { //////////////////////////////////////////////////////////////////////// -void process_ioda_file(int i_pb) { - int npbmsg, npbmsg_total; - int idx, i_msg, i_read, n_file_obs, n_hdr_obs; - int rej_typ, rej_sid, rej_vld, rej_grid, rej_poly; - int rej_elv, rej_nobs; - double x, y; +static void process_ioda_file(int i_pb) { + int npbmsg; + int npbmsg_total; + int idx; + int i_msg; + int n_file_obs; + int n_hdr_obs; + int rej_typ; + int rej_sid; + int rej_vld; + int rej_grid; + int rej_poly; + int rej_elv; + int rej_nobs; + double x; + double y; - bool status; - bool is_time_offset = false; - bool is_time_string = false; unixtime file_ut; unixtime adjusted_file_ut; - unixtime msg_ut, beg_ut, end_ut; - unixtime min_msg_ut, max_msg_ut; + unixtime msg_ut; + unixtime beg_ut; + unixtime end_ut; + unixtime min_msg_ut; + unixtime max_msg_ut; - ConcatString file_name, blk_prefix, blk_file, log_message; + ConcatString log_message; ConcatString prefix; - ConcatString start_time_str, end_time_str; - char min_time_str[max_str_len], max_time_str[max_str_len]; + ConcatString start_time_str; + ConcatString end_time_str; + char max_time_str[max_str_len]; + char min_time_str[max_str_len]; char hdr_typ[max_str_len]; ConcatString hdr_sid; char modified_hdr_typ[max_str_len]; - double hdr_lat, hdr_lon, hdr_elv; + double hdr_lat; + double hdr_lon; + double hdr_elv; unixtime hdr_vld_ut; - float obs_arr[OBS_ARRAY_LEN]; + double obs_arr[OBS_ARRAY_LEN]; const int debug_level_for_performance = 3; - int start_t, end_t, method_start, method_end; - start_t = end_t = method_start = method_end = clock(); + clock_t start_t; + clock_t end_t; + clock_t method_start; + clock_t method_end; IntArray diff_file_times; int diff_file_time_count; @@ -403,6 +409,8 @@ void process_ioda_file(int i_pb) { conf_info.area_mask.ny() > 0); bool apply_poly_mask = (conf_info.poly_mask.n_points() > 0); + start_t = end_t = method_start = method_end = clock(); + // List the IODA file being processed mlog << Debug(1) << "Processing IODA File:\t" << ioda_files[i_pb]<< "\n"; @@ -415,153 +423,41 @@ void process_ioda_file(int i_pb) { << "\" for reading.\n\n"; delete f_in; f_in = (NcFile *) nullptr; + clean_up(); exit(1); } - // Initialize - hdr_typ[0] = 0; - file_ut = beg_ut = end_ut = hdr_vld_ut = (unixtime) 0; - filtered_times.clear(); - min_msg_ut = max_msg_ut = (unixtime) 0; - min_time_str[0] = 0; - max_time_str[0] = 0; - modified_hdr_typ[0] = 0; - - // Set the file name for the IODA file - file_name << ioda_files[i_pb]; - - int nrecs = 0; - int nlocs = 0; - int nstring = 0; - int nvars = 0; - StringArray dim_names; - StringArray metadata_vars; - StringArray obs_value_vars; - bool error_out = true; - e_ioda_format ioda_format = e_ioda_format::v2; - - get_dim_names(f_in, &dim_names); - ConcatString nlocs_name = find_meta_name("nlocs", dim_names); - if(0 < nlocs_name.length()) nlocs = get_dim_value(f_in, nlocs_name.c_str(), error_out); // number of locations - - nvars = bad_data_int ; - nstring = string_data_len; - if (! has_nc_group(f_in, obs_group_name)) ioda_format = e_ioda_format::v1; - - if ( ioda_format == e_ioda_format::v1 ) { - StringArray var_names; - get_var_names(f_in, &var_names); - for(idx=0; idx= 8) { - for(idx=0; idx= 6) { - for(idx=0; idx hdr_lat_arr (nlocs); - vector hdr_lon_arr (nlocs); - vector hdr_elv_arr (nlocs); - vector obs_pres_arr (nlocs); - vector obs_hght_arr (nlocs); - vector hdr_time_arr (nlocs); - char *hdr_vld_block = new char[nlocs*ndatetime]; - char *hdr_msg_types = nullptr; - char *hdr_station_ids = nullptr; - char **hdr_vld_block2 = nullptr; - char **hdr_msg_types2 = nullptr; - char **hdr_station_ids2 = nullptr; vector v_qc_data; - vector v_obs_data; - - if (is_time_string) { - hdr_vld_block2 = (char**) calloc(nlocs, sizeof(char*)); - for (int i=0; i v_obs_data; StringArray raw_var_names; - if(do_all_vars || obs_var_names.n() == 0) raw_var_names = obs_value_vars; + if(do_all_vars || obs_var_names.n() == 0) raw_var_names = ioda_reader.obs_value_vars; else raw_var_names = obs_var_names; - NcVar obs_var, qc_var; + NcVar obs_var; + NcVar qc_var; ConcatString unit_attr; ConcatString desc_attr; for(idx=0; idx message_type_map = conf_info.getMessageTypeMap(); // Initialize @@ -720,41 +540,21 @@ void process_ioda_file(int i_pb) { for(idx=0; idx 0) { - if(bin_count > 0 && (i_read+1)%bin_count == 0) { - cout << nint((double) (i_read+1)/npbmsg*100.0) << "% " << flush; - showed_progress = true; - if(mlog.verbosity_level() >= debug_level_for_performance) { - end_t = clock(); - cout << (end_t-start_t)/double(CLOCKS_PER_SEC) - << " seconds\n"; - start_t = clock(); - } + for(int i_read=0; i_read 0 && bin_count > 0 + && (i_read+1)%bin_count == 0) { + cout << nint((double) (i_read+1)/npbmsg*100.0) << "% " << flush; + showed_progress = true; + if(mlog.verbosity_level() >= debug_level_for_performance) { + end_t = clock(); + cout << (end_t-start_t)/double(CLOCKS_PER_SEC) + << " seconds\n"; + start_t = clock(); } } - if (is_time_offset) { - msg_ut = add_to_unixtime(base_ut, sec_per_unit, - hdr_time_arr[i_read], no_leap_year); - } - else if (is_time_string) { - char valid_time[nstring+1]; - - m_strncpy(valid_time, (const char *)hdr_vld_block2[i_read], - nstring, method_name_s, "valid_time", true); - valid_time[nstring] = 0; - msg_ut = yyyymmddThhmmss_to_unix(valid_time); - } - else { - char valid_time[ndatetime+1]; - m_strncpy(valid_time, (const char *)(hdr_vld_block + (i_read * ndatetime)), - ndatetime, method_name_s, "valid_time", true); - valid_time[ndatetime] = 0; - msg_ut = yyyymmddThhmmss_to_unix(valid_time); - } + msg_ut = ioda_reader.vld_arr[i_read]; // Check to make sure that the message time hasn't changed // from one IODA message to the next @@ -800,14 +600,8 @@ void process_ioda_file(int i_pb) { } if(has_msg_type) { - if (nullptr != hdr_msg_types2) { - m_strncpy(hdr_typ, hdr_msg_types2[i_read], nstring, method_name_s, "hdr_typ2"); - } - else { - m_strncpy(hdr_typ, hdr_msg_types+(i_read*nstring), nstring, method_name_s, "hdr_typ"); - - } - m_rstrip(hdr_typ, nstring); + m_strncpy(hdr_typ, ioda_reader.msg_types[i_read].c_str(), + nstring, method_name_s, "hdr_typ"); // If the message type is not listed in the configuration // file and it is not the case that all message types should be @@ -832,13 +626,8 @@ void process_ioda_file(int i_pb) { if(has_station_id) { char tmp_sid[nstring+1]; - if (nullptr != hdr_station_ids2) { - m_strncpy(tmp_sid, hdr_station_ids2[i_read], nstring, method_name_s, "tmp_sid2"); - } - else { - m_strncpy(tmp_sid, hdr_station_ids+(i_read*nstring), nstring, method_name_s, "tmp_sid"); - } - m_rstrip(tmp_sid, nstring, false); + m_strncpy(tmp_sid, ioda_reader.station_ids[i_read].c_str(), + nstring, method_name_s, "tmp_sid"); m_replace_char(tmp_sid, ' ', '_'); hdr_sid = tmp_sid; } @@ -856,13 +645,13 @@ void process_ioda_file(int i_pb) { // LON LAT DHR ELV TYP T29 ITP // Longitude - hdr_lon = hdr_lon_arr[i_read]; + hdr_lon = ioda_reader.lon_arr[i_read]; // Latitude - hdr_lat = hdr_lat_arr[i_read]; + hdr_lat = ioda_reader.lat_arr[i_read]; // Elevation - hdr_elv = hdr_elv_arr[i_read]; + hdr_elv = ioda_reader.elv_arr[i_read]; // Compute the valid time and check if it is within the // specified valid range @@ -912,15 +701,14 @@ void process_ioda_file(int i_pb) { // Check if the message elevation is within the specified range. // Missing data values for elevation are retained. - if(!check_missing_thresh(hdr_elv) && + if(!ioda_reader.check_missing_thresh(hdr_elv) && (hdr_elv < conf_info.beg_elev || hdr_elv > conf_info.end_elev) ) { rej_elv++; continue; } // Store the index to the header data - obs_arr[0] = (float)nc_point_obs.get_hdr_index(); - + obs_arr[0] = (double)nc_point_obs.get_hdr_index(); n_hdr_obs = 0; for(idx=0; idx " + static const string method_name = "write_netcdf_hdr_data()"; + mlog << Error << "\n" << method_name << " -> " << "No IODA records retained. Nothing to write.\n\n"; // Delete the NetCDF file remove_temp_file(ncfile); exit(1); } + nc_point_obs.init_netcdf(obs_cnt, hdr_cnt, program_name); + // Make sure all obs data is processed before handling header StringArray nc_var_name_arr; StringArray nc_var_unit_arr; @@ -1105,30 +879,30 @@ void write_netcdf_hdr_data() { //////////////////////////////////////////////////////////////////////// -void addObservation(const float *obs_arr, const ConcatString &hdr_typ, +static void addObservation(const double *obs_arr, const ConcatString &hdr_typ, const ConcatString &hdr_sid, const time_t hdr_vld, - const float hdr_lat, const float hdr_lon, const float hdr_elv, - const float quality_mark, const int buf_size) + const double hdr_lat, const double hdr_lon, const double hdr_elv, + const double quality_mark) { // Write the quality flag to the netCDF file ConcatString obs_qty; - if(check_missing_thresh(quality_mark)) + if(ioda_reader.check_missing_thresh(quality_mark)) obs_qty.add("NA"); else obs_qty.format("%d", nint(quality_mark)); - int var_index = obs_arr[1]; + auto var_index = (int)obs_arr[1]; map name_map = conf_info.getObsVarMap(); string var_name = obs_var_names[var_index]; string out_name = name_map[var_name]; - Observation obs = Observation(hdr_typ.text(), - hdr_sid.text(), - hdr_vld, - hdr_lat, hdr_lon, hdr_elv, - obs_qty.text(), - var_index, - obs_arr[2], obs_arr[3], obs_arr[4], - (0addObservationObj(obs); return; @@ -1136,7 +910,7 @@ void addObservation(const float *obs_arr, const ConcatString &hdr_typ, //////////////////////////////////////////////////////////////////////// -void clean_up() { +static void clean_up() { nc_point_obs.close(); @@ -1150,7 +924,7 @@ void clean_up() { //////////////////////////////////////////////////////////////////////// -bool keep_message_type(const char *mt_str) { +static bool keep_message_type(const char *mt_str) { bool keep = conf_info.message_type.n_elements() == 0 || conf_info.message_type.has(mt_str, false); @@ -1163,7 +937,7 @@ bool keep_message_type(const char *mt_str) { //////////////////////////////////////////////////////////////////////// -bool keep_station_id(const char *sid_str) { +static bool keep_station_id(const char *sid_str) { bool keep = (conf_info.station_id.n_elements() == 0 || conf_info.station_id.has(sid_str, false)); @@ -1177,8 +951,8 @@ bool keep_station_id(const char *sid_str) { //////////////////////////////////////////////////////////////////////// -bool keep_valid_time(const unixtime ut, - const unixtime min_ut, const unixtime max_ut) { +static bool keep_valid_time(const unixtime ut, + const unixtime min_ut, const unixtime max_ut) { bool keep = true; // If min_ut and max_ut both set, check the range @@ -1204,34 +978,32 @@ bool keep_valid_time(const unixtime ut, //////////////////////////////////////////////////////////////////////// -bool check_core_data(const bool has_msg_type, const bool has_station_id, - StringArray &dim_names, StringArray &metadata_vars, - e_ioda_format ioda_format) { +static bool check_core_data(const bool has_msg_type, const bool has_station_id, + const StringArray &dim_names, const StringArray &metadata_vars, + e_ioda_format ioda_format_ver) { bool is_netcdf_ready = true; static const char *method_name = "check_core_data() -> "; - StringArray &t_core_dims = (ioda_format == e_ioda_format::v2) + StringArray &t_core_dims = (ioda_format_ver == e_ioda_format::v2) ? core_dims : core_dims_v1; for(int idx=0; idx " << "core dimension \"" << t_core_dims[idx] << "\" is missing.\n\n"; is_netcdf_ready = false; } } - if (ioda_format == e_ioda_format::v1) { - if(has_msg_type || has_station_id) { - if (!is_in_metadata_map("nstring", dim_names)) { - mlog << Error << "\n" << method_name << "-> " - << "core dimension \"nstring\" is missing.\n\n"; - is_netcdf_ready = false; - } + if ((ioda_format_ver == e_ioda_format::v1) && (has_msg_type || has_station_id)) { + if (!ioda_reader.is_in_metadata_map("nstring", dim_names)) { + mlog << Error << "\n" << method_name << "-> " + << "core dimension \"nstring\" is missing.\n\n"; + is_netcdf_ready = false; } } for(int idx=0; idx " << "core variable \"" << core_meta_vars[idx] << "\" is missing.\n\n"; is_netcdf_ready = false; @@ -1242,68 +1014,7 @@ bool check_core_data(const bool has_msg_type, const bool has_station_id, //////////////////////////////////////////////////////////////////////// -bool check_missing_thresh(float value) { - bool check = false; - for(int idx=0; idx 0) { - NcVar meta_var = get_var(f_in, metadata_name.c_str(), metadata_group_name); - if(IS_VALID_NC(meta_var)) { - status = get_nc_data(&meta_var, metadata_buf, nlocs); - if(!status) mlog << Debug(3) << method_name - << "trouble getting " << metadata_name << "\n"; - } - } - else mlog << Debug(4) << method_name - << "Metadata for " << metadata_key << " does not exist!\n"; - if(status) { - for(int idx=0; idx 0) { + if(var_name.nonempty()) { ConcatString qc_name = var_name; ConcatString qc_group = qc_postfix; - if (ioda_format == e_ioda_format::v2) { + if (ioda_format_ver == e_ioda_format::v2) { NcGroup nc_grp = get_nc_group(f_in, qc_postfix); if (IS_INVALID_NC(nc_grp)) qc_group = qc_group_name; StringArray qc_names = conf_info.obs_to_qc_map[var_name]; @@ -1370,7 +1081,7 @@ bool get_obs_data_float(NcFile *f_in, const ConcatString var_name, } if(status) { for(int idx=0; idx= postfix_len) { - return (0 == str_buf.compare(buf_len - postfix_len, postfix_len, postfix)); - } else { - return false; - } -} - -//////////////////////////////////////////////////////////////////////// - -bool is_in_metadata_map(std::string metadata_key, StringArray &available_list) { - bool found = available_list.has(metadata_key); - - if (!found) { - StringArray alt_names = conf_info.metadata_map[metadata_key]; - if (alt_names.n() > 0) { - for (int idx=0; idx group_map; - group_map = parse_conf_message_type_group_map(&conf); + map group_map = parse_conf_message_type_group_map(&conf); // Expand the values for any message type group names for(i=0; i 15) { mlog << Warning << "\nIODA2NCConfInfo::process_config() -> " @@ -165,11 +169,8 @@ void IODA2NCConfInfo::process_config() { << ") should be set between 0 and 15.\n\n"; } - // Conf: obs_name_map - obs_name_map = parse_conf_obs_name_map(&conf); + // Conf: message_type_map message_type_map = parse_conf_message_type_map(&conf); - metadata_map = parse_conf_metadata_map(&conf); - obs_to_qc_map = parse_conf_obs_to_qc_map(&conf); return; } diff --git a/src/tools/other/ioda2nc/ioda2nc_conf_info.h b/src/tools/other/ioda2nc/ioda2nc_conf_info.h index 93c4e51d2c..c026afd107 100644 --- a/src/tools/other/ioda2nc/ioda2nc_conf_info.h +++ b/src/tools/other/ioda2nc/ioda2nc_conf_info.h @@ -21,10 +21,11 @@ #include "vx_util.h" #include "vx_cal.h" #include "vx_math.h" +#include "ioda_data_conf_info.h" //////////////////////////////////////////////////////////////////////// -class IODA2NCConfInfo { +class IODA2NCConfInfo : public IODADataConfInfo { private: @@ -33,7 +34,6 @@ class IODA2NCConfInfo { public: // IODA2NC configuration object - MetConfig conf; // Store data parsed from the IODA2NC configuration object StringArray message_type; // Obseration message type @@ -49,15 +49,11 @@ class IODA2NCConfInfo { double end_elev; double beg_level; // Range of level values to be retained double end_level; - StringArray obs_var; // IODA variiable names + StringArray obs_var; // IODA variable names int quality_mark_thresh; // Quality marks to be retained - ThreshArray missing_thresh; // Fill value thresh array ConcatString version; // Config file version - std::map obs_name_map; std::map message_type_map; - std::map metadata_map; - std::map obs_to_qc_map; StringArray surface_message_types; TimeSummaryInfo timeSummaryInfo; @@ -66,7 +62,6 @@ class IODA2NCConfInfo { void clear(); - std::map getObsVarMap() const { return obs_name_map; } std::map getMessageTypeMap() const { return message_type_map; } TimeSummaryInfo getSummaryInfo() const { return timeSummaryInfo; }; diff --git a/src/tools/tc_utils/tc_gen/tc_gen.cc b/src/tools/tc_utils/tc_gen/tc_gen.cc index c02f65e9b4..63ed8da6ed 100644 --- a/src/tools/tc_utils/tc_gen/tc_gen.cc +++ b/src/tools/tc_utils/tc_gen/tc_gen.cc @@ -1909,7 +1909,7 @@ void setup_table(AsciiTable &at) { at.set_bad_data_str(na_str); // Don't write out trailing blank rows - at.set_delete_trailing_blank_rows(1); + at.set_delete_trailing_blank_rows(true); return; } diff --git a/src/tools/tc_utils/tc_pairs/tc_pairs.cc b/src/tools/tc_utils/tc_pairs/tc_pairs.cc index 1516d8745d..0a32b5d5ff 100644 --- a/src/tools/tc_utils/tc_pairs/tc_pairs.cc +++ b/src/tools/tc_utils/tc_pairs/tc_pairs.cc @@ -2264,7 +2264,7 @@ void setup_table(AsciiTable &at) { at.set_bad_data_str(na_str); // Don't write out trailing blank rows - at.set_delete_trailing_blank_rows(1); + at.set_delete_trailing_blank_rows(true); return; } diff --git a/src/tools/tc_utils/tc_stat/tc_stat_job.cc b/src/tools/tc_utils/tc_stat/tc_stat_job.cc index eb908b928f..66940a1bf7 100644 --- a/src/tools/tc_utils/tc_stat/tc_stat_job.cc +++ b/src/tools/tc_utils/tc_stat/tc_stat_job.cc @@ -1264,7 +1264,7 @@ void TCStatJob::dump_pair(const TrackPairInfo &pair, ofstream *out, out_at.set_precision(get_precision()); out_at.set_bad_data_value(bad_data_double); out_at.set_bad_data_str(na_str); - out_at.set_delete_trailing_blank_rows(1); + out_at.set_delete_trailing_blank_rows(true); // Setup header columns tchc.clear(); @@ -2015,7 +2015,7 @@ void TCStatJobSummary::clear() { SummaryMap.clear(); // Set to default value - ColumnUnion = default_column_union; + ColumnUnion = default_tc_column_union; OutAlpha = default_tc_alpha; FSPThresh = default_fsp_thresh; @@ -2125,7 +2125,7 @@ ConcatString TCStatJobSummary::serialize() const { // Add summary job-specific options for(i=0; i