Per #2609, update logic in vx_series_data library to accept optional …

…args to control searching all files lists, erroring out, and printing warnings. The default settings maintain existing functionality. Update tc_diag code to suppress errors and warnings due to missing data. Still do need to suppress additional warnings from the vx_data libraries.

src/libcode/vx_series_data/series_data.cc

@@ -20,9 +20,11 @@
 
 ////////////////////////////////////////////////////////////////////////
 
-void get_series_entry(int i_series, VarInfo* data_info,
+bool get_series_entry(int i_series, VarInfo* data_info,
         const StringArray& search_files, const GrdFileType type,
-        DataPlane& dp, Grid& grid) {
+        DataPlane& dp, Grid& grid,
+        bool error_out, bool print_warning,
+        bool search_all_files) {
    int i;
    bool found;
 
@@ -39,42 +41,47 @@ void get_series_entry(int i_series, VarInfo* data_info,
       int i_cur = (i_series + i) % search_files.n();
 
       found = read_single_entry(data_info, search_files[i_cur],
-                                type, dp, grid);
+                                type, dp, grid, print_warning);
 
-      if(found) break;
+      // Break out of the loop if data was found or not searching all files
+      if(found || !search_all_files) break;
 
    } // end for i
 
-   // Error out if not found
-   if(!found) {
+   // Error out if not found and specified
+   if(!found && error_out) {
       mlog << Error << "\nget_series_entry() -> "
            << "Could not find data for " << data_info->magic_time_str()
-           << " in file list:\n:" << write_css(search_files) << "\n\n";
+           << " in file list:\n" << write_css(search_files) << "\n\n";
       exit(1);
    }
 
-   return;
+   return(found);
 }
 
 ////////////////////////////////////////////////////////////////////////
 
 bool read_single_entry(VarInfo* info, const ConcatString& filename,
-        const GrdFileType type, DataPlane& dp, Grid& grid) {
+        const GrdFileType type, DataPlane& dp, Grid& grid,
+        bool print_warning) {
 
    Met2dDataFileFactory mtddf_factory;
    Met2dDataFile* mtddf = (Met2dDataFile*) 0;
 
    // Check that file exists
    if(!file_exists(filename.c_str())) {
-       mlog << Warning << "\nread_single_entry() -> "
-            << "File does not exist: " << filename << "\n\n";
-       return(false);
+      if(print_warning) {
+         mlog << Warning << "\nread_single_entry() -> "
+              << "File does not exist: " << filename << "\n\n";
+      }
+      return(false);
    }
 
    // Open data file
    mtddf = mtddf_factory.new_met_2d_data_file(filename.c_str(), type);
 
    // Attempt to read gridded data
+   // TODO: Should we pass print_warning as an option to data_plane?
    bool found = mtddf->data_plane(*info, dp);
 
    // Store grid

src/libcode/vx_series_data/series_data.h

@@ -21,11 +21,15 @@
 
 ////////////////////////////////////////////////////////////////////////
 
-void get_series_entry(int, VarInfo*, const StringArray&,
-                      const GrdFileType, DataPlane&, Grid&);
+bool get_series_entry(int, VarInfo*, const StringArray&,
+                      const GrdFileType, DataPlane&, Grid&,
+                      bool error_out=true,
+                      bool print_warning=true,
+                      bool search_all_files=true);
 
 bool read_single_entry(VarInfo*, const ConcatString&, const GrdFileType,
-                       DataPlane&, Grid&);
+                       DataPlane&, Grid&,
+                       bool print_warning=true);
 
 ////////////////////////////////////////////////////////////////////////
 

src/tools/tc_utils/tc_diag/tc_diag.cc

@@ -343,6 +343,10 @@ void process_tracks(TrackInfoArray& tracks) {
            << unix_to_yyyymmddhh(init_ta.max()) << ".\n\n";
       exit(1);
    }
+   // Store the initialization time
+   else {
+      init_ut = init_ta[0];
+   }
 
    return;
 }
@@ -612,6 +616,14 @@ void set_data(const StringArray& a) {
    for(int i=2; i<a.n(); i++) sa.add(a[i]);
    info.data_files = parse_file_list(sa);
 
+   // Sanity check that input files exist
+   for(int i=0; i<info.data_files.n(); i++) {
+      if(!file_exists(info.data_files[i].c_str())) {
+         mlog << Warning << "\nset_data() -> "
+              << "File does not exist: " << info.data_files[i] << "\n\n";
+      }
+   }
+
    // Store the data in the map
    if(data_opt_map.count(domain) == 0) data_opt_map[domain]  = info;
    else                                data_opt_map[domain] += info;
@@ -966,7 +978,7 @@ void process_fields(const TrackInfoArray &tracks,
    Grid grid_dp;
    VarInfoFactory vi_factory;
    VarInfo *vi = (VarInfo *) 0;
-   StringArray tmp_key_sa;
+   StringArray tmp_key_sa, fields_missing;
 
    // TODO: Consider adding vortex removal logic here
    // Read in the full set of fields required for vortex removal
@@ -980,9 +992,18 @@ void process_fields(const TrackInfoArray &tracks,
       vi->set_valid(vld_ut);
 
       // Find data for this track point
-      get_series_entry(i_vld, vi,
+      bool status = get_series_entry(i_vld, vi,
                        di.data_files, file_type,
-                       data_dp, grid_dp);
+                       data_dp, grid_dp,
+                       false, false, false);
+
+      // Keep track of missing fields
+      if(!status) {
+         fields_missing.add(vi->magic_str());
+
+         // Store the requested valid time
+         data_dp.set_valid(vld_ut);
+      }
 
       // Do coordinate transformation for each track point
       for(j=0; j<tracks.n(); j++) {
@@ -1014,6 +1035,17 @@ void process_fields(const TrackInfoArray &tracks,
 
    } // end for i
 
+   // Log missing fields
+   if(fields_missing.n() > 0) {
+      mlog << Debug(2) << "For the "
+           << domain << " domain, "
+           << sec_to_hhmmss(vld_ut - init_ut) << " lead time, and "
+           << unix_to_yyyymmdd_hhmmss(vld_ut) << " valid time, "
+           << fields_missing.n() << " of " << di.var_info_ptr.size()
+           << " requested fields missing:\n"
+           << write_css(fields_missing) << "\n";
+   }
+
    // Loop over the current set of temp files
    for(i=0; i<tmp_key_sa.n(); i++) {
 
@@ -1643,7 +1675,15 @@ void TmpFileInfo::write_nc_data(const VarInfo *vi, const DataPlane &dp_in,
    ri.shape      = GridTemplateFactory::GridTemplate_Square;
 
    // Do the cylindrical coordinate transformation
-   dp_out = met_regrid(dp_in, grid_in, grid_out, ri);
+   if(dp_in.nxy() > 0) {
+      dp_out = met_regrid(dp_in, grid_in, grid_out, ri);
+   }
+   // Handle empty input fields
+   else {
+      dp_out.set_valid(dp_in.valid());
+      dp_out.set_size(ra_grid.range_n(), ra_grid.azimuth_n(),
+                      bad_data_double);
+   }
 
    // Logic for pressure level data
    bool is_prs = (vi->level().type() == LevelType_Pres);

src/tools/tc_utils/tc_diag/tc_diag.h

@@ -91,6 +91,8 @@ static ConcatString out_prefix;
 //
 ////////////////////////////////////////////////////////////////////////
 
+static unixtime init_ut = (unixtime) 0;
+
 class OutFileInfo {
 
    private: