Merging Howard's ioda development branch into the pair_stat tool

development branch.

Merge remote-tracking branch 'origin/feature_3007_vx_ioda' into feature_3006_pair_stat

Make-include

@@ -22,6 +22,7 @@ MET_CPPFLAGS = -I${top_builddir}/src/basic/vx_cal \
 	     -I${top_builddir}/src/libcode/vx_gnomon \
 	     -I${top_builddir}/src/libcode/vx_grid \
 	     -I${top_builddir}/src/libcode/vx_gsl_prob \
+	     -I${top_builddir}/src/libcode/vx_ioda \
 	     -I${top_builddir}/src/libcode/vx_nav \
 	     -I${top_builddir}/src/libcode/vx_nc_obs \
 	     -I${top_builddir}/src/libcode/vx_nc_util \
@@ -69,6 +70,7 @@ MET_LDFLAGS = -L${top_builddir}/src/basic/vx_cal \
 	     -L${top_builddir}/src/libcode/vx_gnomon \
 	     -L${top_builddir}/src/libcode/vx_grid \
 	     -L${top_builddir}/src/libcode/vx_gsl_prob \
+	     -L${top_builddir}/src/libcode/vx_ioda \
 	     -L${top_builddir}/src/libcode/vx_nav \
 	     -L${top_builddir}/src/libcode/vx_nc_obs \
 	     -L${top_builddir}/src/libcode/vx_nc_util \

configure

@@ -10197,7 +10197,7 @@ fi
 
 # Create configured files
 
-ac_config_files="$ac_config_files Makefile scripts/Rscripts/Makefile scripts/Rscripts/include/Makefile scripts/python/Makefile scripts/python/examples/Makefile scripts/python/met/Makefile scripts/python/pyembed/Makefile scripts/python/utility/Makefile scripts/python/tc_diag/Makefile scripts/python/tc_diag/atcf_tools/Makefile scripts/python/tc_diag/config/Makefile scripts/python/tc_diag/diag_lib/Makefile scripts/python/tc_diag/tc_diag_driver/Makefile data/Makefile data/climo/Makefile data/climo/seeps/Makefile data/colortables/Makefile data/colortables/NCL_colortables/Makefile data/config/Makefile data/map/Makefile data/map/admin_by_country/Makefile data/poly/Makefile data/poly/HMT_masks/Makefile data/poly/NCEP_masks/Makefile data/ps/Makefile data/table_files/Makefile data/tc_data/Makefile src/Makefile src/basic/Makefile src/basic/enum_to_string/Makefile src/basic/vx_cal/Makefile src/basic/vx_config/Makefile src/basic/vx_log/Makefile src/basic/vx_math/Makefile src/basic/vx_util/Makefile src/basic/vx_util_math/Makefile src/libcode/Makefile src/libcode/vx_afm/Makefile src/libcode/vx_analysis_util/Makefile src/libcode/vx_color/Makefile src/libcode/vx_data2d/Makefile src/libcode/vx_data2d_factory/Makefile src/libcode/vx_data2d_grib/Makefile src/libcode/vx_data2d_grib2/Makefile src/libcode/vx_data2d_nc_met/Makefile src/libcode/vx_data2d_nc_wrf/Makefile src/libcode/vx_data2d_nc_cf/Makefile src/libcode/vx_data2d_ugrid/Makefile src/libcode/vx_geodesy/Makefile src/libcode/vx_gis/Makefile src/libcode/vx_gnomon/Makefile src/libcode/vx_grid/Makefile src/libcode/vx_gsl_prob/Makefile src/libcode/vx_nav/Makefile src/libcode/vx_solar/Makefile src/libcode/vx_nc_obs/Makefile src/libcode/vx_nc_util/Makefile src/libcode/vx_pb_util/Makefile src/libcode/vx_plot_util/Makefile src/libcode/vx_ps/Makefile src/libcode/vx_pxm/Makefile src/libcode/vx_render/Makefile src/libcode/vx_shapedata/Makefile src/libcode/vx_stat_out/Makefile src/libcode/vx_statistics/Makefile src/libcode/vx_time_series/Makefile src/libcode/vx_physics/Makefile src/libcode/vx_series_data/Makefile src/libcode/vx_regrid/Makefile src/libcode/vx_tc_util/Makefile src/libcode/vx_summary/Makefile src/libcode/vx_python3_utils/Makefile src/libcode/vx_data2d_python/Makefile src/libcode/vx_bool_calc/Makefile src/libcode/vx_pointdata_python/Makefile src/libcode/vx_seeps/Makefile src/tools/Makefile src/tools/core/Makefile src/tools/core/ensemble_stat/Makefile src/tools/core/grid_stat/Makefile src/tools/core/mode/Makefile src/tools/core/mode_analysis/Makefile src/tools/core/pcp_combine/Makefile src/tools/core/point_stat/Makefile src/tools/core/pair_stat/Makefile src/tools/core/series_analysis/Makefile src/tools/core/stat_analysis/Makefile src/tools/core/wavelet_stat/Makefile src/tools/other/Makefile src/tools/other/ascii2nc/Makefile src/tools/other/lidar2nc/Makefile src/tools/other/gen_ens_prod/Makefile src/tools/other/gen_vx_mask/Makefile src/tools/other/gis_utils/Makefile src/tools/other/ioda2nc/Makefile src/tools/other/madis2nc/Makefile src/tools/other/mode_graphics/Makefile src/tools/other/modis_regrid/Makefile src/tools/other/pb2nc/Makefile src/tools/other/plot_data_plane/Makefile src/tools/other/plot_point_obs/Makefile src/tools/other/wwmca_tool/Makefile src/tools/other/gsi_tools/Makefile src/tools/other/regrid_data_plane/Makefile src/tools/other/point2grid/Makefile src/tools/other/shift_data_plane/Makefile src/tools/other/mode_time_domain/Makefile src/tools/other/grid_diag/Makefile src/tools/tc_utils/Makefile src/tools/tc_utils/tc_dland/Makefile src/tools/tc_utils/tc_pairs/Makefile src/tools/tc_utils/tc_stat/Makefile src/tools/tc_utils/tc_gen/Makefile src/tools/tc_utils/rmw_analysis/Makefile src/tools/tc_utils/tc_rmw/Makefile src/tools/tc_utils/tc_diag/Makefile"
+ac_config_files="$ac_config_files Makefile scripts/Rscripts/Makefile scripts/Rscripts/include/Makefile scripts/python/Makefile scripts/python/examples/Makefile scripts/python/met/Makefile scripts/python/pyembed/Makefile scripts/python/utility/Makefile scripts/python/tc_diag/Makefile scripts/python/tc_diag/atcf_tools/Makefile scripts/python/tc_diag/config/Makefile scripts/python/tc_diag/diag_lib/Makefile scripts/python/tc_diag/tc_diag_driver/Makefile data/Makefile data/climo/Makefile data/climo/seeps/Makefile data/colortables/Makefile data/colortables/NCL_colortables/Makefile data/config/Makefile data/map/Makefile data/map/admin_by_country/Makefile data/poly/Makefile data/poly/HMT_masks/Makefile data/poly/NCEP_masks/Makefile data/ps/Makefile data/table_files/Makefile data/tc_data/Makefile src/Makefile src/basic/Makefile src/basic/enum_to_string/Makefile src/basic/vx_cal/Makefile src/basic/vx_config/Makefile src/basic/vx_log/Makefile src/basic/vx_math/Makefile src/basic/vx_util/Makefile src/basic/vx_util_math/Makefile src/libcode/Makefile src/libcode/vx_afm/Makefile src/libcode/vx_analysis_util/Makefile src/libcode/vx_color/Makefile src/libcode/vx_data2d/Makefile src/libcode/vx_data2d_factory/Makefile src/libcode/vx_data2d_grib/Makefile src/libcode/vx_data2d_grib2/Makefile src/libcode/vx_data2d_nc_met/Makefile src/libcode/vx_data2d_nc_wrf/Makefile src/libcode/vx_data2d_nc_cf/Makefile src/libcode/vx_data2d_ugrid/Makefile src/libcode/vx_geodesy/Makefile src/libcode/vx_gis/Makefile src/libcode/vx_gnomon/Makefile src/libcode/vx_grid/Makefile src/libcode/vx_gsl_prob/Makefile src/libcode/vx_ioda/Makefile src/libcode/vx_nav/Makefile src/libcode/vx_solar/Makefile src/libcode/vx_nc_obs/Makefile src/libcode/vx_nc_util/Makefile src/libcode/vx_pb_util/Makefile src/libcode/vx_plot_util/Makefile src/libcode/vx_ps/Makefile src/libcode/vx_pxm/Makefile src/libcode/vx_render/Makefile src/libcode/vx_shapedata/Makefile src/libcode/vx_stat_out/Makefile src/libcode/vx_statistics/Makefile src/libcode/vx_time_series/Makefile src/libcode/vx_physics/Makefile src/libcode/vx_series_data/Makefile src/libcode/vx_regrid/Makefile src/libcode/vx_tc_util/Makefile src/libcode/vx_summary/Makefile src/libcode/vx_python3_utils/Makefile src/libcode/vx_data2d_python/Makefile src/libcode/vx_bool_calc/Makefile src/libcode/vx_pointdata_python/Makefile src/libcode/vx_seeps/Makefile src/tools/Makefile src/tools/core/Makefile src/tools/core/ensemble_stat/Makefile src/tools/core/grid_stat/Makefile src/tools/core/mode/Makefile src/tools/core/mode_analysis/Makefile src/tools/core/pcp_combine/Makefile src/tools/core/point_stat/Makefile src/tools/core/pair_stat/Makefile src/tools/core/series_analysis/Makefile src/tools/core/stat_analysis/Makefile src/tools/core/wavelet_stat/Makefile src/tools/other/Makefile src/tools/other/ascii2nc/Makefile src/tools/other/lidar2nc/Makefile src/tools/other/gen_ens_prod/Makefile src/tools/other/gen_vx_mask/Makefile src/tools/other/gis_utils/Makefile src/tools/other/ioda2nc/Makefile src/tools/other/madis2nc/Makefile src/tools/other/mode_graphics/Makefile src/tools/other/modis_regrid/Makefile src/tools/other/pb2nc/Makefile src/tools/other/plot_data_plane/Makefile src/tools/other/plot_point_obs/Makefile src/tools/other/wwmca_tool/Makefile src/tools/other/gsi_tools/Makefile src/tools/other/regrid_data_plane/Makefile src/tools/other/point2grid/Makefile src/tools/other/shift_data_plane/Makefile src/tools/other/mode_time_domain/Makefile src/tools/other/grid_diag/Makefile src/tools/tc_utils/Makefile src/tools/tc_utils/tc_dland/Makefile src/tools/tc_utils/tc_pairs/Makefile src/tools/tc_utils/tc_stat/Makefile src/tools/tc_utils/tc_gen/Makefile src/tools/tc_utils/rmw_analysis/Makefile src/tools/tc_utils/tc_rmw/Makefile src/tools/tc_utils/tc_diag/Makefile"
 
 
 if test -n "$MET_DEVELOPMENT"; then
@@ -11164,6 +11164,7 @@ do
     "src/libcode/vx_bool_calc/Makefile") CONFIG_FILES="$CONFIG_FILES src/libcode/vx_bool_calc/Makefile" ;;
     "src/libcode/vx_pointdata_python/Makefile") CONFIG_FILES="$CONFIG_FILES src/libcode/vx_pointdata_python/Makefile" ;;
     "src/libcode/vx_seeps/Makefile") CONFIG_FILES="$CONFIG_FILES src/libcode/vx_seeps/Makefile" ;;
+    "src/libcode/vx_ioda/Makefile") CONFIG_FILES="$CONFIG_FILES src/libcode/vx_ioda/Makefile" ;;
     "src/tools/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/Makefile" ;;
     "src/tools/core/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/core/Makefile" ;;
     "src/tools/core/ensemble_stat/Makefile") CONFIG_FILES="$CONFIG_FILES src/tools/core/ensemble_stat/Makefile" ;;

configure.ac

@@ -1390,6 +1390,7 @@ AC_CONFIG_FILES([Makefile
                  src/libcode/vx_bool_calc/Makefile
                  src/libcode/vx_pointdata_python/Makefile
                  src/libcode/vx_seeps/Makefile
+                 src/libcode/vx_ioda/Makefile
                  src/tools/Makefile
                  src/tools/core/Makefile
                  src/tools/core/ensemble_stat/Makefile

src/basic/vx_cal/is_leap_year.cc

@@ -16,6 +16,7 @@
 #include <string.h>
 #include <stdlib.h>
 #include <cmath>
+#include <array>
 
 #include "vx_cal.h"
 #include "vx_log.h"
@@ -34,7 +35,15 @@ static constexpr double DAY_EPSILON = 0.00002;
 
 ////////////////////////////////////////////////////////////////////////
 
+const array<int, NO_OF_MONTHS> monthly_days = {
+  31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31
+};
+
+////////////////////////////////////////////////////////////////////////
 
+bool check_time_units_and_offset(int sec_per_unit, double time_value);
+
+////////////////////////////////////////////////////////////////////////
 int is_leap_year(int year)
 
 {
@@ -54,12 +63,6 @@ return 1;
 }
 
 
-////////////////////////////////////////////////////////////////////////
-
-const int monthly_days[NO_OF_MONTHS] = {
-  31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31
-};
-
 ////////////////////////////////////////////////////////////////////////
 
 int get_days_from_mmdd(int month, int day, bool no_leap) {
@@ -98,11 +101,11 @@ void increase_one_month(int &year, int &month) {
 
 ////////////////////////////////////////////////////////////////////////
 
-void adjuste_day_for_month_year_units(int &day, int &month, int &year, double month_fraction) {
+void adjust_day_for_month_year_units(int &day, int &month, int &year, double month_fraction) {
   // Compute remaining days from the month fraction
   bool day_adjusted = false;
-  const int day_offset = (int)(month_fraction * DAYS_PER_MONTH + 0.5);
-  const char *method_name = "adjuste_day_for_month_year_units() -> ";
+  const auto day_offset = (int)(month_fraction * DAYS_PER_MONTH + 0.5);
+  const char *method_name = "adjust_day_for_month_year_units() -> ";
 
   day += day_offset;
   if (day == 1 && abs(month_fraction-0.5) < DAY_EPSILON) {
@@ -130,8 +133,8 @@ void adjuste_day_for_month_year_units(int &day, int &month, int &year, double mo
 
 ////////////////////////////////////////////////////////////////////////
 
-void adjuste_day_for_day_units(int &day, int &month, int &year, double time_value) {
-  int day_offset = day + (int)time_value;
+void adjust_day_for_day_units(int &day, int &month, int &year, int time_value) {
+  int day_offset = day + time_value;
   if (day_offset < 0) {
     while (day_offset < 0) {
       decrease_one_month(year, month);
@@ -146,7 +149,6 @@ void adjuste_day_for_day_units(int &day, int &month, int &year, double time_valu
   }
   day = day_offset;
   if (day == 0) day = 1;
-
 }
 
 ////////////////////////////////////////////////////////////////////////
@@ -162,25 +164,27 @@ unixtime add_to_unixtime(unixtime base_unixtime, int sec_per_unit,
   unixtime ut;
   auto time_value_ut = (unixtime)time_value;
   double time_fraction = time_value - (double)time_value_ut;
-  const char *method_name = "add_to_unixtime() -> ";
+  const char *method_name = "add_to_unixtime(double) -> ";
+
+  if (!check_time_units_and_offset(sec_per_unit, time_value)) {
+    mlog << Error << "\n" << method_name
+         << " the negative offset (" << time_value
+         << ") is not supported for unit months and years\n\n";
+    exit(-1);
+  }
 
   if (sec_per_unit == SEC_MONTH || sec_per_unit == SEC_YEAR) {
-    if (time_value < 0) {
-      mlog << Error << "\n" << method_name
-           << " the negative offset (" << time_value
-           << ") is not supported for unit months and years\n\n";
-      exit(-1);
-    }
 
     unix_to_mdyhms(base_unixtime, month, day, year, hour, minute, second);
 
     // Update year and compute remaining months
     int month_offset;
+    double month_fraction = time_fraction;
     if (sec_per_unit == SEC_YEAR) {
       year += time_value_ut;
-      time_fraction *= NO_OF_MONTHS;    // 12 months/year
-      month_offset = (int)time_fraction;
-      time_fraction -= month_offset;
+      month_fraction *= NO_OF_MONTHS;    // 12 months/year
+      month_offset = (int)month_fraction;
+      month_fraction -= month_offset;
     }
     else month_offset = (int)time_value_ut;
 
@@ -189,44 +193,8 @@ unixtime add_to_unixtime(unixtime base_unixtime, int sec_per_unit,
       increase_one_month(year, month);
     }
     // Compute remaining days
-    //adjuste_day_for_month_year_units(day, month, year, time_fraction);
-    //ut = mdyhms_to_unix(month, day, year, hour, minute, second);
-    double day_offset;
-    if (day == 1) {
-      if (abs(time_fraction-0.5) < DAY_EPSILON) day = 15;
-      else {
-        day_offset = time_fraction * DAYS_PER_MONTH;
-        day += (int)day_offset;
-        if (day_offset - (int)day_offset > 0.5) day++;
-      }
-    }
-    else {
-      day_offset = time_fraction * DAYS_PER_MONTH;
-      time_value_ut = (int)day_offset;
-      day += time_value_ut;
-      if (day_offset - time_value_ut > 0.5) day++;
-      if (day > DAYS_PER_MONTH) {
-         day -= DAYS_PER_MONTH;
-         increase_one_month(year, month);
-      }
-    }
-
-    int day_org = day;
-    bool day_adjusted = false;
-    int max_day = monthly_days[month-1];
-    if (day > max_day) {
-      day = max_day;
-      day_adjusted = true;
-      if (month == 2 && is_leap_year(year)) {
-        if (day_org == 29) day_adjusted = false;
-        day = 29;
-      }
-    }
+    adjust_day_for_month_year_units(day, month, year, month_fraction);
     ut = mdyhms_to_unix(month, day, year, hour, minute, second);
-    if (day_adjusted) {
-      mlog << Debug(2) << method_name << "adjusted day " << day_org
-           << " to " << day << " for " << year << "-" << month << "\n";
-    }
   }
   else if (!no_leap || sec_per_unit != 86400) {
     // seconds, minute, hours, and day unit with leap year
@@ -241,7 +209,7 @@ unixtime add_to_unixtime(unixtime base_unixtime, int sec_per_unit,
   }
   else {    //  no_leap year && unit = day
     unix_to_mdyhms(base_unixtime, month, day, year, hour, minute, second);
-    adjuste_day_for_day_units(day, month, year, time_value);
+    adjust_day_for_day_units(day, month, year, (int)time_value);
     ut = mdyhms_to_unix(month, day, year, hour, minute, second);
     if (time_fraction > (1-TIME_EPSILON) ) ut += (unixtime)sec_per_unit;
     else if (time_fraction > TIME_EPSILON) ut += (unixtime)(time_fraction * sec_per_unit);
@@ -250,9 +218,67 @@ unixtime add_to_unixtime(unixtime base_unixtime, int sec_per_unit,
        << unix_to_yyyymmdd_hhmmss(base_unixtime)
        << " plus " << time_value << " times " << sec_per_unit
        << " seconds = " << unix_to_yyyymmdd_hhmmss(ut) << "\n";
+
+  return ut;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+unixtime add_to_unixtime(unixtime base_unixtime, int sec_per_unit,
+                         unixtime time_value, bool no_leap) {
+  int month;
+  int day;
+  int year;
+  int hour;
+  int minute;
+  int second;
+  unixtime ut;
+  const char *method_name = "add_to_unixtime(unixtime) -> ";
+
+  if (!check_time_units_and_offset(sec_per_unit, (double)time_value)) {
+    mlog << Error << "\n" << method_name
+         << " the negative offset (" << time_value
+         << ") is not supported for unit months and years\n\n";
+    exit(-1);
+  }
+
+  if (sec_per_unit == SEC_YEAR) {
+    unix_to_mdyhms(base_unixtime, month, day, year, hour, minute, second);
+    year += time_value;
+    ut = mdyhms_to_unix(month, day, year, hour, minute, second);
+  }
+  else if (sec_per_unit == SEC_MONTH) {
+    unix_to_mdyhms(base_unixtime, month, day, year, hour, minute, second);
+    for (int idx=0; idx<time_value; idx++) {
+      increase_one_month(year, month);
+    }
+    ut = mdyhms_to_unix(month, day, year, hour, minute, second);
+  }
+  else if (!no_leap || sec_per_unit != 86400) {
+    // seconds, minute, hours, and day unit with leap year
+    ut = base_unixtime + sec_per_unit * time_value;
+  }
+  else {    //  no_leap year && unit = day
+    unix_to_mdyhms(base_unixtime, month, day, year, hour, minute, second);
+    adjust_day_for_day_units(day, month, year, (int)time_value);
+    ut = mdyhms_to_unix(month, day, year, hour, minute, second);
+  }
+  mlog << Debug(5) << method_name
+       << unix_to_yyyymmdd_hhmmss(base_unixtime)
+       << " plus " << time_value << " times " << sec_per_unit
+       << " seconds = " << unix_to_yyyymmdd_hhmmss(ut) << "\n";
 
   return ut;
 }
 
 ////////////////////////////////////////////////////////////////////////
 
+bool check_time_units_and_offset(int sec_per_unit, double time_value) {
+  bool valid = true;
+  if (sec_per_unit == SEC_MONTH || sec_per_unit == SEC_YEAR) {
+    valid = (time_value >= 0);
+  }
+  return valid;
+}
+
+////////////////////////////////////////////////////////////////////////

src/basic/vx_cal/vx_cal.h

@@ -42,6 +42,8 @@ static const int mjd_ut0 = 40587;   //  mjd of Jan 1, 1970
 
 ////////////////////////////////////////////////////////////////////////
 
+extern  unixtime  add_to_unixtime (unixtime u, int sec_per_unit, unixtime time_value, bool no_leap);
+
 extern  unixtime  add_to_unixtime (unixtime u, int sec_per_unit, double time_value, bool no_leap);
 
 extern  unixtime  doyhms_to_unix  (int doy, int year, int hour, int minute, int second);

src/libcode/Makefile.am

@@ -64,6 +64,7 @@ SUBDIRS += vx_data2d_factory \
 	vx_nc_obs \
 	vx_solar \
 	vx_bool_calc \
-	vx_seeps
+	vx_seeps \
+	vx_ioda
 
 MAINTAINERCLEANFILES 	= Makefile.in

src/libcode/Makefile.in

@@ -161,7 +161,7 @@ DIST_SUBDIRS = vx_grid vx_data2d vx_nc_util vx_data2d_grib \
 	vx_pxm vx_render vx_ps vx_plot_util vx_shapedata vx_statistics \
 	vx_stat_out vx_analysis_util vx_tc_util vx_summary \
 	vx_time_series vx_physics vx_series_data vx_regrid vx_nc_obs \
-	vx_solar vx_bool_calc vx_seeps
+	vx_solar vx_bool_calc vx_seeps vx_ioda
 am__DIST_COMMON = $(srcdir)/Makefile.in
 DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
 am__relativize = \
@@ -355,7 +355,7 @@ SUBDIRS = vx_grid vx_data2d vx_nc_util vx_data2d_grib vx_data2d_nc_met \
 	vx_pxm vx_render vx_ps vx_plot_util vx_shapedata vx_statistics \
 	vx_stat_out vx_analysis_util vx_tc_util vx_summary \
 	vx_time_series vx_physics vx_series_data vx_regrid vx_nc_obs \
-	vx_solar vx_bool_calc vx_seeps
+	vx_solar vx_bool_calc vx_seeps vx_ioda
 MAINTAINERCLEANFILES = Makefile.in
 all: all-recursive
 

src/libcode/vx_ioda/Makefile.am

@@ -0,0 +1,17 @@
+## @start 1
+## Makefile.am -- Process this file with automake to produce Makefile.in
+## @end 1
+
+MAINTAINERCLEANFILES	= Makefile.in
+
+# Include the project definitions
+
+include ${top_srcdir}/Make-include
+
+# The library
+
+noinst_LIBRARIES = libvx_ioda.a
+libvx_ioda_a_SOURCES = \
+              ioda.cc ioda.h \
+              ioda_data_conf_info.cc ioda_data_conf_info.h
+libvx_ioda_a_CPPFLAGS = ${MET_CPPFLAGS}