diff --git a/met/data/config/ConfigConstants b/met/data/config/ConfigConstants index ee0d4855c0..33e5060e8b 100644 --- a/met/data/config/ConfigConstants +++ b/met/data/config/ConfigConstants @@ -88,6 +88,7 @@ AW_MEAN = 20; GAUSSIAN = 21; MAXGAUSS = 22; GEOG_MATCH = 23; +HIRA = 24; // Interpolation types NONE = 1; @@ -169,3 +170,9 @@ BETA = 8; TOP = TRUE; BOTTOM = FALSE; +// Normalization types +NONE = 1; +CLIMO_ANOM = 2; +CLIMO_STD_ANOM = 3; +FCST_ANOM = 4; +FCST_STD_ANOM = 5; diff --git a/met/data/config/GenEnsProdConfig_default b/met/data/config/GenEnsProdConfig_default index 09746b626e..0b053ba65c 100644 --- a/met/data/config/GenEnsProdConfig_default +++ b/met/data/config/GenEnsProdConfig_default @@ -36,10 +36,11 @@ regrid = { // // May be set separately in each "field" entry // -censor_thresh = []; -censor_val = []; -cat_thresh = []; -nc_var_str = ""; +censor_thresh = []; +censor_val = []; +cat_thresh = []; +nc_var_str = ""; +normalize_flag = NONE; // // Ensemble fields to be processed diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h index f3c5550f02..9c27e436d3 100644 --- a/met/src/basic/vx_config/config_constants.h +++ b/met/src/basic/vx_config/config_constants.h @@ -471,6 +471,20 @@ enum MatchType { MatchType_NoMerge // Match with no additional merging }; +//////////////////////////////////////////////////////////////////////// + +// +// Enumeration for normalization options +// + +enum NormalizeType { + NormalizeType_None, // No normalization + NormalizeType_ClimoAnom, // Subtract climo mean + NormalizeType_ClimoStdAnom, // Subtract climo mean and divide by standard deviation + NormalizeType_FcstAnom, // Subtract ensemble mean + NormalizeType_FcstStdAnom // Subtract ensemble mean and divide by standard deviation +}; + //////////////////////////////////////////////////////////////////////// // // Constants used in configuartion files @@ -752,6 +766,12 @@ static const char conf_key_dist_parm[] = "dist_parm"; static const char conf_key_inst_bias_scale[] = "inst_bias_scale"; static const char conf_key_inst_bias_offset[] = "inst_bias_offset"; +// +// Gen-Ens-Prod specific parameter key names +// + +static const char conf_key_normalize_flag[] = "normalize_flag"; + // Distribution options static const char conf_val_normal[] = "NORMAL"; static const char conf_val_exponential[] = "EXPONENTIAL"; @@ -760,6 +780,12 @@ static const char conf_val_gamma[] = "GAMMA"; static const char conf_val_uniform[] = "UNIFORM"; static const char conf_val_beta[] = "BETA"; +// Normalization options +static const char conf_val_climo_anom[] = "CLIMO_ANOM"; +static const char conf_val_climo_std_anom[] = "CLIMO_STD_ANOM"; +static const char conf_val_fcst_anom[] = "FCST_ANOM"; +static const char conf_val_fcst_std_anom[] = "FCST_STD_ANOM"; + // // STAT-Analysis specific parameter key names // diff --git a/met/src/basic/vx_config/config_util.cc b/met/src/basic/vx_config/config_util.cc index c466a9f2de..71d23a0598 100644 --- a/met/src/basic/vx_config/config_util.cc +++ b/met/src/basic/vx_config/config_util.cc @@ -141,9 +141,10 @@ RegridInfo::RegridInfo() { void RegridInfo::validate() { // Check for unsupported regridding options - if(method == InterpMthd_Best || + if(method == InterpMthd_Best || method == InterpMthd_Geog_Match || - method == InterpMthd_Gaussian) { + method == InterpMthd_Gaussian || + method == InterpMthd_HiRA) { mlog << Error << "\nRegridInfo::validate() -> " << "\"" << interpmthd_to_string(method) << "\" not valid for regridding, only interpolating.\n\n"; @@ -2272,6 +2273,7 @@ InterpMthd int_to_interpmthd(int i) { else if(i == conf_const.lookup_int(interpmthd_gaussian_str)) m = InterpMthd_Gaussian; else if(i == conf_const.lookup_int(interpmthd_maxgauss_str)) m = InterpMthd_MaxGauss; else if(i == conf_const.lookup_int(interpmthd_geog_match_str)) m = InterpMthd_Geog_Match; + else if(i == conf_const.lookup_int(interpmthd_hira_str)) m = InterpMthd_HiRA; else { mlog << Error << "\nconf_int_to_interpmthd() -> " << "Unexpected value of " << i @@ -2833,6 +2835,47 @@ ConcatString matchtype_to_string(MatchType type) { /////////////////////////////////////////////////////////////////////////////// +NormalizeType int_to_normalizetype(int v) { + NormalizeType t = NormalizeType_None; + + // Convert integer to enumerated NormalizeType + if(v == conf_const.lookup_int(conf_val_none)) t = NormalizeType_None; + else if(v == conf_const.lookup_int(conf_val_climo_anom)) t = NormalizeType_ClimoAnom; + else if(v == conf_const.lookup_int(conf_val_climo_std_anom)) t = NormalizeType_ClimoStdAnom; + else if(v == conf_const.lookup_int(conf_val_fcst_anom)) t = NormalizeType_FcstAnom; + else if(v == conf_const.lookup_int(conf_val_fcst_std_anom)) t = NormalizeType_FcstStdAnom; + else { + mlog << Error << "\nint_to_normalizetype() -> " + << "Unexpected value of " << v << ".\n\n"; + exit(1); + } + + return(t); +} + +/////////////////////////////////////////////////////////////////////////////// + +ConcatString normalizetype_to_string(NormalizeType type) { + ConcatString s; + + // Convert enumerated NormalizeType to string + switch(type) { + case(NormalizeType_None): s = conf_val_none; break; + case(NormalizeType_ClimoAnom): s = conf_val_climo_anom; break; + case(NormalizeType_ClimoStdAnom): s = conf_val_climo_std_anom; break; + case(NormalizeType_FcstAnom): s = conf_val_fcst_anom; break; + case(NormalizeType_FcstStdAnom): s = conf_val_fcst_std_anom; break; + default: + mlog << Error << "\nnormalizetype_to_string() -> " + << "Unexpected NormalizeType value of " << type << ".\n\n"; + exit(1); + } + + return(s); +} + +/////////////////////////////////////////////////////////////////////////////// + DistType int_to_disttype(int v) { DistType t = DistType_None; @@ -2979,3 +3022,34 @@ StringArray parse_conf_ens_member_ids(Dictionary *dict) { } /////////////////////////////////////////////////////////////////////////////// + +NormalizeType parse_conf_normalize_flag(Dictionary *dict) { + NormalizeType t = NormalizeType_None; + int v; + + if(!dict) { + mlog << Error << "\nparse_conf_normalize_type() -> " + << "empty dictionary!\n\n"; + exit(1); + } + + // Get the integer flag value for the current entry + v = dict->lookup_int(conf_key_normalize_flag); + + // Convert integer to enumerated WaveletType + if(v == conf_const.lookup_int(conf_val_none)) t = NormalizeType_None; + else if(v == conf_const.lookup_int(conf_val_climo_anom)) t = NormalizeType_ClimoAnom; + else if(v == conf_const.lookup_int(conf_val_climo_std_anom)) t = NormalizeType_ClimoStdAnom; + else if(v == conf_const.lookup_int(conf_val_fcst_anom)) t = NormalizeType_FcstAnom; + else if(v == conf_const.lookup_int(conf_val_fcst_std_anom)) t = NormalizeType_FcstStdAnom; + else { + mlog << Error << "\nparse_conf_normalize_flag() -> " + << "Unexpected config file value of " << v << " for \"" + << conf_key_normalize_flag << "\".\n\n"; + exit(1); + } + + return(t); +} + +/////////////////////////////////////////////////////////////////////////////// diff --git a/met/src/basic/vx_config/config_util.h b/met/src/basic/vx_config/config_util.h index 5afcc974a1..8c32c2c5ce 100644 --- a/met/src/basic/vx_config/config_util.h +++ b/met/src/basic/vx_config/config_util.h @@ -78,6 +78,7 @@ extern map extern void parse_conf_range_int(Dictionary *dict, int &beg, int &end); extern void parse_conf_range_double(Dictionary *dict, double &beg, double &end); extern StringArray parse_conf_ens_member_ids(Dictionary *dict); +extern NormalizeType parse_conf_normalize_flag(Dictionary *dict); extern void check_mask_names(const StringArray &); @@ -119,6 +120,9 @@ extern ConcatString obssummary_to_string(ObsSummary, int); extern MatchType int_to_matchtype(int); extern ConcatString matchtype_to_string(MatchType); +extern NormalizeType int_to_normalizetype(int); +extern ConcatString normalizetype_to_string(NormalizeType); + extern DistType int_to_disttype(int); extern DistType string_to_disttype(const char *); extern ConcatString disttype_to_string(DistType); diff --git a/met/src/tools/other/gen_ens_prod/gen_ens_prod.cc b/met/src/tools/other/gen_ens_prod/gen_ens_prod.cc index 32f77cbf5c..be8503f244 100644 --- a/met/src/tools/other/gen_ens_prod/gen_ens_prod.cc +++ b/met/src/tools/other/gen_ens_prod/gen_ens_prod.cc @@ -17,6 +17,7 @@ // 000 09/10/21 Halley Gotway MET #1904 Initial version. // 001 11/15/21 Halley Gotway MET #1968 Ensemble -ctrl error check. // 002 01/14/21 McCabe MET #1695 All members in one file. +// 002 02/17/22 Halley Gotway MET #1918 Add normalize_flag. // //////////////////////////////////////////////////////////////////////// @@ -53,7 +54,12 @@ using namespace std; static void process_command_line(int, char **); static void process_grid(const Grid &); static void process_ensemble(); + +static void get_ens_mean_stdev(GenEnsProdVarInfo *, DataPlane &, DataPlane &); static bool get_data_plane(const char *, GrdFileType, VarInfo *, DataPlane &); +static void normalize_data(DataPlane &, NormalizeType, + const DataPlane &, const DataPlane &, + const DataPlane &, const DataPlane &); static void clear_counts(); static void track_counts(GenEnsProdVarInfo *, const DataPlane &, bool, @@ -267,64 +273,14 @@ void process_grid(const Grid &fcst_grid) { //////////////////////////////////////////////////////////////////////// -bool get_data_plane(const char *infile, GrdFileType ftype, - VarInfo *info, DataPlane &dp) { - bool found; - Met2dDataFile *mtddf = (Met2dDataFile *) 0; - - // Read the current ensemble file - if(!(mtddf = mtddf_factory.new_met_2d_data_file(infile, ftype))) { - mlog << Error << "\nget_data_plane() -> " - << "trouble reading file \"" << infile << "\"\n\n"; - exit(1); - } - - // Read the gridded data field - if((found = mtddf->data_plane(*info, dp))) { - - // Setup the verification grid, if necessary - if(nxy == 0) process_grid(mtddf->grid()); - - // Create the output file, if necessary - if(nc_out == (NcFile *) 0) setup_nc_file(); - - // Regrid, if requested and necessary - if(!(mtddf->grid() == grid)) { - mlog << Debug(1) - << "Regridding field \"" << info->magic_str() - << "\" to the verification grid.\n"; - dp = met_regrid(dp, mtddf->grid(), grid, info->regrid()); - } - - // Store the valid time, if not already set - if(ens_valid_ut == (unixtime) 0) { - ens_valid_ut = dp.valid(); - } - // Check to make sure that the valid time doesn't change - else if(ens_valid_ut != dp.valid()) { - mlog << Warning << "\nget_data_plane() -> " - << "The valid time has changed, " - << unix_to_yyyymmdd_hhmmss(ens_valid_ut) - << " != " << unix_to_yyyymmdd_hhmmss(dp.valid()) - << " in \"" << infile << "\"\n\n"; - } - - } // end if found - - // Deallocate the data file pointer, if necessary - if(mtddf) { delete mtddf; mtddf = (Met2dDataFile *) 0; } - - return(found); -} - -//////////////////////////////////////////////////////////////////////// - void process_ensemble() { int i_var, i_ens, n_ens_vld, n_ens_inputs; bool need_reset; - DataPlane ens_dp, ctrl_dp, cmn_dp, csd_dp; + DataPlane ens_dp, ctrl_dp; + DataPlane cmn_dp, csd_dp; + DataPlane emn_dp, esd_dp; unixtime max_init_ut = bad_data_ll; - VarInfo * var_info; + VarInfo *var_info; ConcatString ens_file; // Loop through each of the ensemble fields to be processed @@ -342,7 +298,7 @@ void process_ensemble() { n_ens_inputs = (*var_it)->inputs_n(); for(i_ens=n_ens_vld=0; i_ens < n_ens_inputs; i_ens++) { - // get file and VarInfo to process + // Get file and VarInfo to process ens_file = (*var_it)->get_file(i_ens); var_info = (*var_it)->get_var_info(i_ens); @@ -382,9 +338,19 @@ void process_ensemble() { conf_info.conf.lookup_array(conf_key_climo_stdev_field, false), i_var, ens_valid_ut, grid); + // Compute the ensemble summary data, if needed + if((*var_it)->normalize_flag == NormalizeType_FcstAnom || + (*var_it)->normalize_flag == NormalizeType_FcstStdAnom ) { + get_ens_mean_stdev((*var_it), emn_dp, esd_dp); + } + else { + emn_dp.erase(); + esd_dp.erase(); + } + // Read ensemble control member data, if provided if(ctrl_file.nonempty()) { - VarInfo * ctrl_info = (*var_it)->get_ctrl(i_ens); + VarInfo *ctrl_info = (*var_it)->get_ctrl(i_ens); mlog << Debug(3) << "\n" << "Reading control field: " @@ -400,6 +366,12 @@ void process_ensemble() { exit(1); } + // Normalize, if requested + if((*var_it)->normalize_flag != NormalizeType_None) { + normalize_data(ctrl_dp, (*var_it)->normalize_flag, + cmn_dp, csd_dp, emn_dp, esd_dp); + } + // Apply current data to the running sums and counts track_counts(*var_it, ctrl_dp, true, cmn_dp, csd_dp); @@ -414,6 +386,12 @@ void process_ensemble() { } // end if need_reset + // Normalize, if requested + if((*var_it)->normalize_flag != NormalizeType_None) { + normalize_data(ens_dp, (*var_it)->normalize_flag, + cmn_dp, csd_dp, emn_dp, esd_dp); + } + // Apply current data to the running sums and counts track_counts(*var_it, ens_dp, false, cmn_dp, csd_dp); @@ -447,6 +425,247 @@ void process_ensemble() { //////////////////////////////////////////////////////////////////////// +void get_ens_mean_stdev(GenEnsProdVarInfo *ens_info, + DataPlane &emn_dp, DataPlane &esd_dp) { + int i_ens, nxy, j; + double ens; + NumArray emn_cnt_na, emn_sum_na; + NumArray esd_cnt_na, esd_sum_na, esd_ssq_na; + VarInfo *var_info; + ConcatString ens_file; + DataPlane ens_dp; + + // Check for null pointer + if(!ens_info) { + mlog << Error << "\nget_ens_mean_stdev() -> " + << "null pointer!\n\n"; + exit(1); + } + + mlog << Debug(3) + << "Computing the ensemble mean and standard deviation for " + << ens_info->raw_magic_str << "\n"; + + // Loop over the ensemble inputs + for(i_ens=0; i_ens < ens_info->inputs_n(); i_ens++) { + + // Get file and VarInfo to process + ens_file = ens_info->get_file(i_ens); + var_info = ens_info->get_var_info(i_ens); + + // Skip bad data files + if(!ens_file_vld[ens_info->get_file_index(i_ens)]) continue; + + // Read data and track the valid data count + if(!get_data_plane(ens_file.c_str(), etype, + var_info, ens_dp)) { + mlog << Warning << "\nget_ens_mean_stdev() -> " + << "ensemble field \"" << var_info->magic_str() + << "\" not found in file \"" << ens_file << "\"\n\n"; + continue; + } + + // Initialize sums, if needed + if(emn_cnt_na.n() == 0) { + nxy = ens_dp.nx()*ens_dp.ny(); + emn_cnt_na.set_const(0.0, nxy); + emn_sum_na = emn_cnt_na; + esd_cnt_na = emn_cnt_na; + esd_sum_na = emn_cnt_na; + esd_ssq_na = emn_cnt_na; + } + + // Update the counts and sums + for(j=0; jget_ctrl(i_ens); + + // Error out if missing + if(!get_data_plane(ctrl_file.c_str(), etype, + var_info, ens_dp)) { + mlog << Error << "\nget_ens_mean_stdev() -> " + << "control member ensemble field \"" + << var_info->magic_str() + << "\" not found in file \"" << ctrl_file << "\"\n\n"; + exit(1); + } + + // Update counts and sums + for(j=0; j " + << "trouble reading file \"" << infile << "\"\n\n"; + exit(1); + } + + // Read the gridded data field + if((found = mtddf->data_plane(*info, dp))) { + + // Setup the verification grid, if necessary + if(nxy == 0) process_grid(mtddf->grid()); + + // Create the output file, if necessary + if(nc_out == (NcFile *) 0) setup_nc_file(); + + // Regrid, if requested and necessary + if(!(mtddf->grid() == grid)) { + mlog << Debug(1) + << "Regridding field \"" << info->magic_str() + << "\" to the verification grid.\n"; + dp = met_regrid(dp, mtddf->grid(), grid, info->regrid()); + } + + // Store the valid time, if not already set + if(ens_valid_ut == (unixtime) 0) { + ens_valid_ut = dp.valid(); + } + // Check to make sure that the valid time doesn't change + else if(ens_valid_ut != dp.valid()) { + mlog << Warning << "\nget_data_plane() -> " + << "The valid time has changed, " + << unix_to_yyyymmdd_hhmmss(ens_valid_ut) + << " != " << unix_to_yyyymmdd_hhmmss(dp.valid()) + << " in \"" << infile << "\"\n\n"; + } + + } // end if found + + // Deallocate the data file pointer, if necessary + if(mtddf) { delete mtddf; mtddf = (Met2dDataFile *) 0; } + + return(found); +} + +//////////////////////////////////////////////////////////////////////// + +static void normalize_data(DataPlane &dp, NormalizeType t, + const DataPlane &cmn_dp, const DataPlane &csd_dp, + const DataPlane &emn_dp, const DataPlane &esd_dp) { + + mlog << Debug(3) << "Normalizing ensemble data using " + << normalizetype_to_string(t) << ".\n"; + + // Check for climo mean + if((t == NormalizeType_ClimoAnom || t == NormalizeType_ClimoStdAnom) && + dp.nxy() != cmn_dp.nxy()) { + mlog << Error << "\nnormalize_data()-> " + << "the climatology mean field is required for " + << normalizetype_to_string(t) << "!\n\n"; + exit(1); + } + + // Check for climo standard deviation + if(t == NormalizeType_ClimoStdAnom && + dp.nxy() != csd_dp.nxy()) { + mlog << Error << "\nnormalize_data()-> " + << "the climatology standard deviation field is required for " + << normalizetype_to_string(t) << "!\n\n"; + exit(1); + } + + // Supported types + switch(t) { + + case NormalizeType_None: + break; + + case NormalizeType_ClimoAnom: + dp.anomaly(cmn_dp); + break; + + case NormalizeType_ClimoStdAnom: + dp.standard_anomaly(cmn_dp, csd_dp); + break; + + case NormalizeType_FcstAnom: + dp.anomaly(emn_dp); + break; + + case NormalizeType_FcstStdAnom: + dp.standard_anomaly(emn_dp, esd_dp); + break; + + default: + mlog << Error << "\nnormalize_data()-> " + << "unexpected NormalizeType value (" + << t << ")\n\n"; + exit(1); + } // end switch + + return; +} + +//////////////////////////////////////////////////////////////////////// + void clear_counts() { int i, j; @@ -471,7 +690,7 @@ void clear_counts() { //////////////////////////////////////////////////////////////////////// -void track_counts(GenEnsProdVarInfo * ens_info, const DataPlane &ens_dp, bool is_ctrl, +void track_counts(GenEnsProdVarInfo *ens_info, const DataPlane &ens_dp, bool is_ctrl, const DataPlane &cmn_dp, const DataPlane &csd_dp) { int i, j, k; double ens, cmn, csd; @@ -582,7 +801,7 @@ void setup_nc_file() { //////////////////////////////////////////////////////////////////////// -void write_ens_nc(GenEnsProdVarInfo * ens_info, int n_ens_vld, +void write_ens_nc(GenEnsProdVarInfo *ens_info, int n_ens_vld, const DataPlane &ens_dp, const DataPlane &cmn_dp, const DataPlane &csd_dp) { @@ -839,7 +1058,7 @@ void write_ens_nc(GenEnsProdVarInfo * ens_info, int n_ens_vld, //////////////////////////////////////////////////////////////////////// -void write_ens_var_float(GenEnsProdVarInfo * ens_info, float *ens_data, const DataPlane &dp, +void write_ens_var_float(GenEnsProdVarInfo *ens_info, float *ens_data, const DataPlane &dp, const char *type_str, const char *long_name_str) { NcVar ens_var; @@ -895,7 +1114,7 @@ void write_ens_var_float(GenEnsProdVarInfo * ens_info, float *ens_data, const Da //////////////////////////////////////////////////////////////////////// -void write_ens_var_int(GenEnsProdVarInfo * ens_info, int *ens_data, const DataPlane &dp, +void write_ens_var_int(GenEnsProdVarInfo *ens_info, int *ens_data, const DataPlane &dp, const char *type_str, const char *long_name_str) { NcVar ens_var; @@ -942,7 +1161,7 @@ void write_ens_var_int(GenEnsProdVarInfo * ens_info, int *ens_data, const DataPl //////////////////////////////////////////////////////////////////////// -void write_ens_data_plane(GenEnsProdVarInfo * ens_info, const DataPlane &ens_dp, const DataPlane &dp, +void write_ens_data_plane(GenEnsProdVarInfo *ens_info, const DataPlane &ens_dp, const DataPlane &dp, const char *type_str, const char *long_name_str) { // Allocate memory for this data diff --git a/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc b/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc index 917e66c76a..e22b8b964d 100644 --- a/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc +++ b/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc @@ -62,10 +62,9 @@ void GenEnsProdConfInfo::clear() { desc.clear(); for(; var_it != ens_input.end(); var_it++) { - if(*var_it) { delete *var_it; } - } + ens_input.clear(); cdf_info.clear(); nbrhd_prob.clear(); @@ -258,6 +257,9 @@ void GenEnsProdConfInfo::process_config(GrdFileType etype, StringArray * ens_fil // Keep track of the maximum number of thresholds if(ens_info->cat_ta.n() > max_n_cat) max_n_cat = ens_info->cat_ta.n(); + // Conf: normalize_flag + ens_info->normalize_flag = parse_conf_normalize_flag(&i_edict); + // Conf: ensemble_flag ens_info->nc_info = parse_nc_info(&i_edict); diff --git a/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.h b/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.h index 657a908026..e6ea171ee5 100644 --- a/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.h +++ b/met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.h @@ -122,7 +122,8 @@ inline int GenEnsProdConfInfo::get_compression_level() { return(conf.nc_compress class GenEnsProdVarInfo: public EnsVarInfo { public: - GenEnsProdNcOutInfo nc_info; // Ensemble product outputs + NormalizeType normalize_flag; // Ensemble normalization logic + GenEnsProdNcOutInfo nc_info; // Ensemble product outputs }; #endif /* __GEN_ENS_PROD_CONF_INFO_H__ */