Skip to content

Commit

Permalink
Feature 1918 climo_ens_member_id (#2075)
Browse files Browse the repository at this point in the history
* Per #1918, store the ensemble_member_id string in the EnsVarInfo class so that we can use it later, if needed, when reading climatological data which may also make use of that string.

* Per #1918, update gen_ens_prod to set the MET_ENS_MEMBER_ID environment variable when reading climatology data if the ens_member_ids config option has been set and the normalizing relative to climatology has been requested.

* Per #1918, add log messages to read_climo.cc and gen_ens_prod.cc to clarify what data is being read from which climo data files.

* Added documentation on MET_ENS_MEMBER_ID usage in climo file name

* updated usage langauge

* Per #1918, adding gen_ens_prod unit test to demonstrate using ENS_MEMBER_ID to read climo data separately for each member.

* Per #1918, adding gen_ens_prod unit test to demonstrate using ENS_MEMBER_ID to read climo data separately for each member.

Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com>
  • Loading branch information
JohnHalleyGotway and j-opatz authored Mar 2, 2022
1 parent 7d63e0f commit 107ac32
Show file tree
Hide file tree
Showing 10 changed files with 277 additions and 18 deletions.
12 changes: 12 additions & 0 deletions met/docs/Users_Guide/gen-ens-prod.rst
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,18 @@ Each value in the array will replace the text **MET_ENS_MEMBER_ID**.
];
}
This replacement behavior can also be applied to climatology file name entry, in the
climo_mean and climo_stdev dictionaries.

.. code-block:: none
climo_mean = {
file_name = ["/path/to/file/memberMET_ENS_MEMBER_ID-mean.nc"];
}
This substitution method can only be used if **ens_member_ids** has at least one entry
and the **normalize** option is set to **CLIMO_ANOM** or **CLIMO_STD_ANOM**.

**control_id** is a string that is substituted in the same way as the **ens_member_ids** values
to read a control member. This value is only used when the **-ctrl** command line argument is
used. The value should not be found in the **ens_member_ids** array.
Expand Down
6 changes: 6 additions & 0 deletions met/src/libcode/vx_data2d/var_info.cc
Original file line number Diff line number Diff line change
Expand Up @@ -892,6 +892,12 @@ int EnsVarInfo::get_file_index(int index) {

////////////////////////////////////////////////////////////////////////

ConcatString EnsVarInfo::get_ens_member_id(int index) {
return inputs[index].ens_member_id;
}

////////////////////////////////////////////////////////////////////////

ConcatString raw_magic_str(Dictionary i_edict, GrdFileType file_type) {
ConcatString magic_str;

Expand Down
8 changes: 5 additions & 3 deletions met/src/libcode/vx_data2d/var_info.h
Original file line number Diff line number Diff line change
Expand Up @@ -258,9 +258,10 @@ inline int VarInfo::accum_attr() const { return(SetAttrAccum); }
//

struct InputInfo {
VarInfo * var_info; // Variable information to read
int file_index; // Index in file_list of file to read
StringArray * file_list; // Array of files (unallocated)
VarInfo * var_info; // Variable information to read
int file_index; // Index in file_list of file to read
StringArray * file_list; // Array of files (unallocated)
ConcatString ens_member_id; // MET_ENS_MEMBER_ID string
};

////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -292,6 +293,7 @@ class EnsVarInfo {
VarInfo * get_var_info(int index=0);
ConcatString get_file(int index=0);
int get_file_index(int index=0);
ConcatString get_ens_member_id(int index=0);

ConcatString nc_var_str; // Ensemble variable name strings
ThreshArray cat_ta; // Ensemble categorical thresholds
Expand Down
6 changes: 6 additions & 0 deletions met/src/libcode/vx_statistics/read_climo.cc
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,12 @@ void read_climo_file(const char *climo_file, GrdFileType ctype,
}
}

// Print log message for matching record
mlog << Debug(4)
<< "Found matching " << cur_ut_cs << " \""
<< info->magic_str() << "\" climatology field in file \""
<< climo_file << "\".\n";

// Regrid, if needed
if(!(mtddf->grid() == vx_grid)) {
mlog << Debug(2)
Expand Down
80 changes: 67 additions & 13 deletions met/src/tools/other/gen_ens_prod/gen_ens_prod.cc
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ static void process_command_line(int, char **);
static void process_grid(const Grid &);
static void process_ensemble();

static void get_climo_mean_stdev(GenEnsProdVarInfo *, int,
bool, int, DataPlane &, DataPlane &);
static void get_ens_mean_stdev(GenEnsProdVarInfo *, DataPlane &, DataPlane &);
static bool get_data_plane(const char *, GrdFileType, VarInfo *, DataPlane &);

Expand Down Expand Up @@ -272,7 +274,7 @@ void process_grid(const Grid &fcst_grid) {

void process_ensemble() {
int i_var, i_ens, n_ens_vld, n_ens_inputs;
bool need_reset;
bool need_reset, set_climo_ens_mem_id;
DataPlane ens_dp, ctrl_dp;
DataPlane cmn_dp, csd_dp;
DataPlane emn_dp, esd_dp;
Expand All @@ -287,6 +289,13 @@ void process_ensemble() {
// Need to reinitialize counts and sums for each ensemble field
need_reset = true;

// When normalizing relative to climatology with MET_ENS_MEMBER_ID set,
// read climatology separately for each member
set_climo_ens_mem_id =
(conf_info.ens_member_ids.n() > 1) &&
((*var_it)->normalize == NormalizeType_ClimoAnom ||
(*var_it)->normalize == NormalizeType_ClimoStdAnom);

// Print out the normalization flag
cs << cs_erase;
if((*var_it)->normalize != NormalizeType_None) {
Expand All @@ -310,9 +319,9 @@ void process_ensemble() {
// Skip bad data files
if(!ens_file_vld[(*var_it)->get_file_index(i_ens)]) continue;

mlog << Debug(3) << "\n"
<< "Reading field: "
<< var_info->magic_str() << "\n";
mlog << Debug(3)
<< "\nReading ensemble field \""
<< var_info->magic_str() << "\".\n";

// Read data and track the valid data count
if(!get_data_plane(ens_file.c_str(), etype,
Expand All @@ -335,17 +344,13 @@ void process_ensemble() {
clear_counts();

// Read climatology data for this field
cmn_dp = read_climo_data_plane(
conf_info.conf.lookup_array(conf_key_climo_mean_field, false),
i_var, ens_valid_ut, grid);

csd_dp = read_climo_data_plane(
conf_info.conf.lookup_array(conf_key_climo_stdev_field, false),
i_var, ens_valid_ut, grid);
get_climo_mean_stdev((*var_it), i_var,
set_climo_ens_mem_id,
i_ens, cmn_dp, csd_dp);

// Compute the ensemble summary data, if needed
if((*var_it)->normalize == NormalizeType_FcstAnom ||
(*var_it)->normalize == NormalizeType_FcstStdAnom ) {
(*var_it)->normalize == NormalizeType_FcstStdAnom) {
get_ens_mean_stdev((*var_it), emn_dp, esd_dp);
}
else {
Expand All @@ -371,6 +376,13 @@ void process_ensemble() {
exit(1);
}

// Read climo data with MET_ENS_MEMBER_ID set
if(set_climo_ens_mem_id) {
get_climo_mean_stdev((*var_it), i_var,
set_climo_ens_mem_id, i_ens,
cmn_dp, csd_dp);
}

// Normalize, if requested
if((*var_it)->normalize != NormalizeType_None) {
normalize_data(ctrl_dp, (*var_it)->normalize,
Expand All @@ -391,6 +403,13 @@ void process_ensemble() {

} // end if need_reset

// Read climo data with MET_ENS_MEMBER_ID set
if(set_climo_ens_mem_id) {
get_climo_mean_stdev((*var_it), i_var,
set_climo_ens_mem_id, i_ens,
cmn_dp, csd_dp);
}

// Normalize, if requested
if((*var_it)->normalize != NormalizeType_None) {
normalize_data(ens_dp, (*var_it)->normalize,
Expand All @@ -411,7 +430,7 @@ void process_ensemble() {
if(((double) n_ens_vld/n_ens_inputs) < conf_info.vld_ens_thresh) {
mlog << Error << "\nprocess_ensemble() -> "
<< n_ens_vld << " of " << n_ens_inputs
<< " (" << (double)n_ens_vld/n_ens_inputs << ")"
<< " (" << (double)n_ens_vld/n_ens_inputs << ")"
<< " fields found for \"" << (*var_it)->get_var_info()->magic_str()
<< "\" does not meet the threshold specified by \""
<< conf_key_ens_ens_thresh << "\" (" << conf_info.vld_ens_thresh
Expand All @@ -430,6 +449,41 @@ void process_ensemble() {

////////////////////////////////////////////////////////////////////////

void get_climo_mean_stdev(GenEnsProdVarInfo *ens_info, int i_var,
bool set_ens_mem_id, int i_ens,
DataPlane &cmn_dp, DataPlane &csd_dp) {

// Set the MET_ENS_MEMBER_ID environment variable
if(set_ens_mem_id) {
setenv(met_ens_member_id, ens_info->get_ens_member_id(i_ens).c_str(), 1);
}

mlog << Debug(4)
<< "Reading climatology mean data for ensemble field \""
<< ens_info->get_var_info(i_ens)->magic_str() << "\".\n";

cmn_dp = read_climo_data_plane(
conf_info.conf.lookup_array(conf_key_climo_mean_field, false),
i_var, ens_valid_ut, grid);

mlog << Debug(4)
<< "Reading climatology standard deviation data for ensemble field \""
<< ens_info->get_var_info(i_ens)->magic_str() << "\".\n";

csd_dp = read_climo_data_plane(
conf_info.conf.lookup_array(conf_key_climo_stdev_field, false),
i_var, ens_valid_ut, grid);

// Unset the MET_ENS_MEMBER_ID environment variable
if(set_ens_mem_id) {
unsetenv(met_ens_member_id);
}

return;
}

////////////////////////////////////////////////////////////////////////

void get_ens_mean_stdev(GenEnsProdVarInfo *ens_info,
DataPlane &emn_dp, DataPlane &esd_dp) {
int i_ens, nxy, j;
Expand Down
2 changes: 1 addition & 1 deletion met/src/tools/other/gen_ens_prod/gen_ens_prod.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
//
// Mod# Date Name Description
// ---- ---- ---- -----------
// 000 09/10/21 Halley Gotway Initial version (MET #1904).
// 000 09/10/21 Halley Gotway MET #1904 Initial version.
//
////////////////////////////////////////////////////////////////////////

Expand Down
2 changes: 2 additions & 0 deletions met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ void GenEnsProdConfInfo::process_config(GrdFileType etype, StringArray * ens_fil
input_info.var_info = next_var;
input_info.file_index = 0;
input_info.file_list = ens_files;
input_info.ens_member_id = ens_member_ids[j];
ens_info->add_input(input_info);

// Add InputInfo to ens info list for each ensemble file provided
Expand All @@ -221,6 +222,7 @@ void GenEnsProdConfInfo::process_config(GrdFileType etype, StringArray * ens_fil
input_info.var_info = NULL;
input_info.file_index = k;
input_info.file_list = ens_files;
input_info.ens_member_id = ens_member_ids[j];
ens_info->add_input(input_info);
} // end for k

Expand Down
161 changes: 161 additions & 0 deletions test/config/GenEnsProdConfig_climo_anom_ens_member_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
////////////////////////////////////////////////////////////////////////////////
//
// Gen-Ens-Prod configuration file.
//
// For additional information, please see the MET User's Guide.
//
////////////////////////////////////////////////////////////////////////////////

//
// Output model name to be written
//
model = "CFSv2";

//
// Output description to be written
// May be set separately in each "obs.field" entry
//
desc = "NA";

////////////////////////////////////////////////////////////////////////////////

//
// Verification grid
// May be set separately in each "field" entry
//
regrid = {
to_grid = NONE;
method = NEAREST;
width = 1;
vld_thresh = 0.5;
shape = SQUARE;
}

////////////////////////////////////////////////////////////////////////////////

//
// May be set separately in each "field" entry
//
censor_thresh = [];
censor_val = [];
normalize = CLIMO_STD_ANOM;
cat_thresh = [];
nc_var_str = "";

//
// Ensemble fields to be processed
//
ens = {
file_type = NETCDF_NCCF;
ens_thresh = 0.3;
vld_thresh = 0.3;

field = [
{
name = "fcst";
level = "(MET_ENS_MEMBER_ID,0,*,*)";
cat_thresh = [ <-0.43, >-0.43&&<=0.43, >0.43 ];
}
];
}

//
// IDs for ensemble members
// Only set if processing a single file with all ensembles
//
ens_member_ids = ["0","1","2"];
control_id = "";

////////////////////////////////////////////////////////////////////////////////

//
// Neighborhood ensemble probabilities
//
nbrhd_prob = {
width = [ 5 ];
shape = CIRCLE;
vld_thresh = 0.0;
}

//
// NMEP smoothing methods
//
nmep_smooth = {
vld_thresh = 0.0;
shape = CIRCLE;
gaussian_dx = 81.27;
gaussian_radius = 120;
type = [
{
method = GAUSSIAN;
width = 1;
}
];
}

////////////////////////////////////////////////////////////////////////////////

//
// Climatology data
//
climo_mean = {

file_name = [ "${CLIMO_MEAN_FILE}" ];
field = [
{
name = "series_cnt_FBAR";
level = "(*,*)";
}
];

regrid = {
method = NEAREST;
width = 1;
vld_thresh = 0.5;
shape = SQUARE;
}

time_interp_method = DW_MEAN;
day_interval = NA;
hour_interval = NA;
}

climo_stdev = {

file_name = [ "${CLIMO_STDEV_FILE}" ];
field = [
{
name = "series_cnt_FSTDEV";
level = "(*,*)";
}
];
}

////////////////////////////////////////////////////////////////////////////////

//
// Ensemble product output types
// May be set separately in each "ens.field" entry
//
ensemble_flag = {
latlon = TRUE;
mean = TRUE;
stdev = TRUE;
minus = FALSE;
plus = FALSE;
min = FALSE;
max = FALSE;
range = FALSE;
vld_count = FALSE;
frequency = TRUE;
nep = FALSE;
nmep = FALSE;
climo = FALSE;
climo_cdp = FALSE;
}

////////////////////////////////////////////////////////////////////////////////

version = "V10.1.0";

////////////////////////////////////////////////////////////////////////////////
Loading

0 comments on commit 107ac32

Please sign in to comment.