From 4c612e9a0d8e4e92f3489234008b7d6c33de71f8 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 30 Aug 2021 13:31:14 -0600 Subject: [PATCH] Per #1891, tighten up the gen_vx_mask logic a bit. Use a global config for the default NC compression level. Then read the input and mask fields using local variables instead of the global one to ensure they don't affect eachother. Also took the opportunity to refine the logic so that we only create a mtddf object in one spot, in the get_data_plane() function. --- .../tools/other/gen_vx_mask/gen_vx_mask.cc | 186 +++++++++--------- met/src/tools/other/gen_vx_mask/gen_vx_mask.h | 17 +- 2 files changed, 98 insertions(+), 105 deletions(-) diff --git a/met/src/tools/other/gen_vx_mask/gen_vx_mask.cc b/met/src/tools/other/gen_vx_mask/gen_vx_mask.cc index 3b20c7ad60..3b220f8e75 100644 --- a/met/src/tools/other/gen_vx_mask/gen_vx_mask.cc +++ b/met/src/tools/other/gen_vx_mask/gen_vx_mask.cc @@ -24,7 +24,8 @@ // 006 07/09/18 Bullock Add shapefile masking type. // 007 04/08/19 Halley Gotway Add percentile thresholds. // 008 04/06/20 Halley Gotway Generalize input_grid option. -// 009 06/01/21 Seth Linden Change -type from optional to required +// 009 06/01/21 Seth Linden Change -type from optional to required. +// 010 08/30/21 Halley Gotway MET #1891 fix input and mask fields. // //////////////////////////////////////////////////////////////////////// @@ -102,7 +103,7 @@ void process_command_line(int argc, char **argv) { if(argc == 1) usage(); // Initialize the configuration object - config.read(replace_path(config_const_filename).c_str()); + global_config.read(replace_path(config_const_filename).c_str()); // Parse the command line into tokens cline.set(argc, argv); @@ -161,15 +162,6 @@ void process_command_line(int argc, char **argv) { //////////////////////////////////////////////////////////////////////// void process_input_grid(DataPlane &dp) { - Met2dDataFileFactory mtddf_factory; - Met2dDataFile *mtddf_ptr = (Met2dDataFile *) 0; - GrdFileType ftype = FileType_None; - - // Get the gridded file type from the data config string, if present - if(input_field_str.length() > 0) { - config.read_string(input_field_str.c_str()); - ftype = parse_conf_file_type(&config); - } // Parse info.name as a white-space separated string StringArray sa; @@ -193,28 +185,12 @@ void process_input_grid(DataPlane &dp) { << "Use the grid defined by file \"" << input_gridname << "\".\n"; - // Attempt to open the data file - mtddf_ptr = mtddf_factory.new_met_2d_data_file(input_gridname.c_str(), ftype); - if(!mtddf_ptr) { - mlog << Error << "\nprocess_input_grid() -> " - << "can't open input file \"" << input_gridname << "\"\n\n"; - exit(1); - } - - // Read the input data plane, if requested - if(input_field_str.length() > 0) { - get_data_plane(mtddf_ptr, input_field_str.c_str(), dp); - } - // Check for the output of a previous call to this tool - else if(get_gen_vx_mask_data(mtddf_ptr, dp)) { - } - - // Extract the grid - grid = mtddf_ptr->grid(); + // Read the input grid and data plane, if requested + get_data_plane(input_gridname, input_field_str, true, dp, grid); } // If not yet set, fill the input data plane with zeros - if(dp.is_empty()) { + if(dp.is_empty()) { dp.set_size(grid.nx(), grid.ny()); dp.set_constant(0.0); } @@ -223,18 +199,12 @@ void process_input_grid(DataPlane &dp) { << "Parsed Input Grid:\t" << grid.name() << " (" << grid.nx() << " x " << grid.ny() << ")\n"; - // Clean up - if(mtddf_ptr) { delete mtddf_ptr; mtddf_ptr = (Met2dDataFile *) 0; } - return; } //////////////////////////////////////////////////////////////////////// void process_mask_file(DataPlane &dp) { - Met2dDataFileFactory mtddf_factory; - Met2dDataFile *mtddf_ptr = (Met2dDataFile *) 0; - GrdFileType ftype = FileType_None; // Initialize solar_ut = (unixtime) 0; @@ -282,26 +252,10 @@ void process_mask_file(DataPlane &dp) { // Otherwise, process the mask file as a gridded data file else { - // Get the gridded file type from the mask config string, if present - if(mask_field_str.length() > 0) { - config.read_string(mask_field_str.c_str()); - ftype = parse_conf_file_type(&config); - } - - mtddf_ptr = mtddf_factory.new_met_2d_data_file(mask_filename.c_str(), ftype); - if(!mtddf_ptr) { - mlog << Error << "\nprocess_mask_file() -> " - << "can't open gridded mask data file \"" << mask_filename << "\"\n\n"; - exit(1); - } - - // Read mask_field, if specified - if(mask_field_str.length() > 0) { - get_data_plane(mtddf_ptr, mask_field_str.c_str(), dp); - } - - // Extract the grid - grid_mask = mtddf_ptr->grid(); + // Read the mask grid and data plane, if requested + get_data_plane(mask_filename, mask_field_str, + mask_type == MaskType_Data, + dp, grid_mask); mlog << Debug(2) << "Parsed Mask Grid:\t" << grid_mask.name() @@ -393,70 +347,105 @@ void process_mask_file(DataPlane &dp) { exit(1); } - // Clean up - if(mtddf_ptr) { delete mtddf_ptr; mtddf_ptr = (Met2dDataFile *) 0; } - return; } //////////////////////////////////////////////////////////////////////// -void get_data_plane(Met2dDataFile *mtddf_ptr, - const char *config_str, DataPlane &dp) { - VarInfoFactory vi_factory; - VarInfo *vi_ptr = (VarInfo *) 0; - double dmin, dmax; +void get_data_plane(const ConcatString &file_name, + const ConcatString &config_str, + bool read_gen_vx_mask_output, + DataPlane &dp, Grid &dp_grid) { + ConcatString local_cs = config_str; + GrdFileType ftype = FileType_None; - // Parse the config string - config.read_string(config_str); + // Initialize to the global configuration + MetConfig local_config = global_config; - // Allocate new VarInfo object - vi_ptr = vi_factory.new_var_info(mtddf_ptr->file_type()); - if(!vi_ptr) { - mlog << Error << "\nget_data_plane() -> " - << "can't allocate new VarInfo pointer.\n\n"; - exit(1); + // Parse non-empty config strings + if(local_cs.length() > 0) { + local_config.read_string(local_cs.c_str()); + ftype = parse_conf_file_type(&local_config); } - // Read config into the VarInfo object - vi_ptr->set_dict(config); - - // Get data plane from the file for this VarInfo object - if(!mtddf_ptr->data_plane(*vi_ptr, dp)) { + // Attempt to open the data file + Met2dDataFileFactory mtddf_factory; + Met2dDataFile *mtddf_ptr = (Met2dDataFile *) 0; + mtddf_ptr = mtddf_factory.new_met_2d_data_file(file_name.c_str(), ftype); + if(!mtddf_ptr) { mlog << Error << "\nget_data_plane() -> " - << "trouble reading field \"" << config_str - << "\" from file \"" << mtddf_ptr->filename() << "\"\n\n"; + << "can't open input file \"" << file_name << "\"\n\n"; exit(1); } - // Dump the range of data values read - dp.data_range(dmin, dmax); - mlog << Debug(3) - << "Read field \"" << vi_ptr->magic_str() << "\" from \"" - << mtddf_ptr->filename() << "\" with data ranging from " - << dmin << " to " << dmax << ".\n"; + // Read gen_vx_mask output from a previous run + if(read_gen_vx_mask_output && + local_cs.length() == 0 && + mtddf_ptr->file_type() == FileType_NcMet) { + if(get_gen_vx_mask_config_str((MetNcMetDataFile *) mtddf_ptr, local_cs)) { + local_config.read_string(local_cs.c_str()); + } + } + + // Read data plane, if requested + if(local_cs.length() > 0) { + + // Allocate new VarInfo object + VarInfoFactory vi_factory; + VarInfo *vi_ptr = (VarInfo *) 0; + vi_ptr = vi_factory.new_var_info(mtddf_ptr->file_type()); + if(!vi_ptr) { + mlog << Error << "\nget_data_plane() -> " + << "can't allocate new VarInfo pointer.\n\n"; + exit(1); + } + + // Read config into the VarInfo object + vi_ptr->set_dict(local_config); + + // Get data plane from the file for this VarInfo object + if(!mtddf_ptr->data_plane(*vi_ptr, dp)) { + mlog << Error << "\nget_data_plane() -> " + << "trouble reading field \"" << local_cs + << "\" from file \"" << mtddf_ptr->filename() << "\"\n\n"; + exit(1); + } + + // Dump the range of data values read + double dmin, dmax; + dp.data_range(dmin, dmax); + mlog << Debug(3) + << "Read field \"" << vi_ptr->magic_str() << "\" from \"" + << mtddf_ptr->filename() << "\" with data ranging from " + << dmin << " to " << dmax << ".\n"; + + // Clean up + if(vi_ptr) { delete vi_ptr; vi_ptr = (VarInfo *) 0; } + + } // end if + + // Extract the grid + dp_grid = mtddf_ptr->grid(); // Clean up - if(vi_ptr) { delete vi_ptr; vi_ptr = (VarInfo *) 0; } + if(mtddf_ptr) { delete mtddf_ptr; mtddf_ptr = (Met2dDataFile *) 0; } return; } //////////////////////////////////////////////////////////////////////// -bool get_gen_vx_mask_data(Met2dDataFile *mtddf_ptr, DataPlane &dp) { +bool get_gen_vx_mask_config_str(MetNcMetDataFile *mnmdf_ptr, + ConcatString &config_str) { bool status = false; - ConcatString tool, config_str; + ConcatString tool; int i; - // Must be MET NetCDF format - if(mtddf_ptr->file_type() != FileType_NcMet) return(status); - - // Cast pointer of correct type - MetNcMetDataFile *mnmdf_ptr = (MetNcMetDataFile *) mtddf_ptr; + // Check for null pointer + if(!mnmdf_ptr) return(status); // Check for the MET_tool global attribute - if(!get_global_att(mnmdf_ptr->MetNc->Nc, (string)"MET_tool", tool)) return(status); + if(!get_global_att(mnmdf_ptr->MetNc->Nc, (string) "MET_tool", tool)) return(status); // Check for gen_vx_mask output if(tool != program_name) return(status); @@ -469,8 +458,9 @@ bool get_gen_vx_mask_data(Met2dDataFile *mtddf_ptr, DataPlane &dp) { mnmdf_ptr->MetNc->Var[i].name == "lon") continue; // Read the first non-lat/lon variable - config_str << "'name=\"" << mnmdf_ptr->MetNc->Var[i].name << "\"; level=\"(*,*)\";'"; - get_data_plane(mtddf_ptr, config_str.c_str(), dp); + config_str << cs_erase + << "'name=\"" << mnmdf_ptr->MetNc->Var[i].name + << "\"; level=\"(*,*)\";'"; status = true; break; } @@ -1231,7 +1221,7 @@ void write_netcdf(const DataPlane &dp) { } int deflate_level = compress_level; - if (deflate_level < 0) deflate_level = config.nc_compression(); + if (deflate_level < 0) deflate_level = global_config.nc_compression(); // Define Variables mask_var = add_var(f_out, string(mask_name), ncFloat, lat_dim, lon_dim, deflate_level); @@ -1444,7 +1434,7 @@ void usage() { << mlog.verbosity_level() << ") (optional).\n" << "\t\t\"-compress level\" overrides the compression level of " - << "NetCDF variable (" << config.nc_compression() + << "NetCDF variable (" << global_config.nc_compression() << ") (optional).\n\n" << flush; diff --git a/met/src/tools/other/gen_vx_mask/gen_vx_mask.h b/met/src/tools/other/gen_vx_mask/gen_vx_mask.h index 70181b8b4e..fe6f6f901c 100644 --- a/met/src/tools/other/gen_vx_mask/gen_vx_mask.h +++ b/met/src/tools/other/gen_vx_mask/gen_vx_mask.h @@ -17,7 +17,8 @@ // 000 12/09/14 Halley Gotway New // 001 06/02/16 Halley Gotway Add box masking type. // 002 11/15/16 Halley Gotway Add solar masking types. -// 003 06/03/21 Seth Linden Changed default mask type to MaskType_None +// 003 06/03/21 Seth Linden Changed default mask type to MaskType_None. +// 004 08/30/21 Halley Gotway MET #1891 fix input and mask fields. // //////////////////////////////////////////////////////////////////////// @@ -32,6 +33,7 @@ using namespace std; #include "vx_cal.h" #include "mask_poly.h" #include "vx_grid.h" +#include "data2d_nc_met.h" #include "data_plane.h" #include "vx_data2d.h" #include "vx_data2d_factory.h" @@ -112,7 +114,7 @@ static MaskPoly poly_mask; static Grid grid, grid_mask; // Configuration object for reading config strings -static MetConfig config; +static MetConfig global_config; static int compress_level = -1; //////////////////////////////////////////////////////////////////////// @@ -120,10 +122,11 @@ static int compress_level = -1; static void process_command_line(int, char **); static void process_input_grid(DataPlane &dp); static void process_mask_file(DataPlane &dp); -static void get_data_plane(Met2dDataFile *mtddf_ptr, - const char *config_str, DataPlane &dp); -static bool get_gen_vx_mask_data(Met2dDataFile *mtddf_ptr, - DataPlane &dp); +static void get_data_plane(const ConcatString &file_name, + const ConcatString &config_str, bool, + DataPlane &dp, Grid &dp_grid); +static bool get_gen_vx_mask_config_str(MetNcMetDataFile *, + ConcatString &); static void get_shapefile_outline(ShpPolyRecord &shape); static void apply_poly_mask(DataPlane &dp); static void apply_shape_mask(DataPlane &dp); @@ -135,7 +138,7 @@ static void apply_data_mask(DataPlane &dp); static void apply_solar_mask(DataPlane &dp); static void apply_lat_lon_mask(DataPlane &dp); static DataPlane combine(const DataPlane &dp_data, - const DataPlane &dp_mask, SetLogic); + const DataPlane &dp_mask, SetLogic); static void write_netcdf(const DataPlane &dp); static void usage(); static void set_type(const StringArray &);