Feature 1471 python_grid (#1704)

* Per #1471, defined a parse_grid_string() function in the vx_statistics library and then updated vx_data2d_python to call that function. However, this creates a circular dependency because vx_data2d_python now depends on vx_statistics.

* Per #1471, because of the change in dependencies, I had to modify many, many Makefile.am files to link to the -lvx_statistics after -lvx_data2d_python. This is not great, but I didn't find a better solution.

* Per #1471, add a sanity check to make sure the grid and data dimensions actually match.

* Per #1471, add 3 new unit tests to demonstrate setting the python grid as a named grid, grid specification string, or a gridded data file.

* Per #1471, document python grid changes in appendix F.

* Per #1471, just spacing.

* Per #1471, lots of Makefile.am changes to get this code to compile on kiowa. Worringly, it compiled and linked fine on my Mac laptop but not on kiowa. Must be some large differences in the linker logic.

Co-authored-by: John Halley Gotway <johnhg@kiowa.rap.ucar.edu>

met/docs/Users_Guide/appendixF.rst

@@ -65,7 +65,9 @@ The data must be loaded into a 2D NumPy array named **met_data**. In addition th
      'long_name': 'FooBar',
      'level':     'Surface',
      'units':     'None',
-  
+ 
+     # Define 'grid' as a string or a dictionary
+ 
      'grid': {
         'type': 'Lambert Conformal',
         'hemisphere': 'N',
@@ -83,12 +85,32 @@ The data must be loaded into a 2D NumPy array named **met_data**. In addition th
         'ny': 129,
       }
   
-     }
+  }
+
+
+In the dictionary, valid time, initialization time, lead time and accumulation time (if any) must be indicated by strings. Valid and initialization times must be given in YYYYMMDD[_HH[MMSS]] format, and lead and accumulation times must be given in HH[MMSS] format, where the square brackets indicate optional elements. The dictionary must also include strings for the name, long_name, level, and units to describe the data. The rest of the **attrs** dictionary gives the grid size and projection information in the same format that is used in the netCDF files written out by the MET tools. Those entries are also listed below. Note that the **grid** entry in the **attrs** dictionary can either be defined as a string or as a dictionary itself.
+
+If specified as a string, **grid** can be defined as follows:
+
+• As a named grid:
+
+.. code-block:: none
 
+  'grid': 'G212'
+
+• As a grid specification string, as described in :ref:`appendixB`:
+
+.. code-block:: none
+
+  'grid': 'lambert 185 129 12.19 -133.459 -95 40.635 6371.2 25 25 N'
+
+• As the path to an existing gridded data file:
+
+.. code-block:: none
 
-In the dictionary, valid time, initialization time, lead time and accumulation time (if any) must be indicated by strings. Valid and initialization times must be given in YYYYMMDD[_HH[MMSS]] format, and lead and accumulation times must be given in HH[MMSS] format, where the square brackets indicate optional elements. The dictionary must also include strings for the name, long_name, level, and units to describe the data. The rest of the **attrs** dictionary gives the grid size and projection information in the same format that is used in the netCDF files written out by the MET tools. Those entries are also listed below. Note that the **grid** entry in the **attrs** dictionary is itself a dictionary.
+  'grid': '/path/to/sample_data.grib'
 
-The supported grid **type** strings are described below:
+When specified as a dictionary, the contents of the **grid** dictionary vary based on the grid **type** string. The entries for the supported grid types are described below:
 
 • **Lambert Conformal** grid dictionary entries:
 

met/internal_tests/basic/vx_util/Makefile.am

@@ -82,18 +82,30 @@ endif
 test_ascii_header_SOURCES = test_ascii_header.cc
 test_ascii_header_CPPFLAGS = ${MET_CPPFLAGS}
 test_ascii_header_LDFLAGS = -L. ${MET_LDFLAGS}
-test_ascii_header_LDADD = -lvx_util \
+test_ascii_header_LDADD = -lvx_stat_out \
+	-lvx_statistics \
+	-lvx_shapedata \
+	-lvx_gsl_prob \
+	-lvx_analysis_util \
+	-lvx_shapedata \
+	-lvx_util \
+	$(PYTHON_LIBS) \
+	-lvx_statistics \
+	-lvx_data2d_factory \
+	-lvx_data2d_nc_met \
+	-lvx_data2d_nc_pinterp \
+	$(PYTHON_LIBS) \
+	-lvx_data2d_nccf \
+	-lvx_data2d_grib $(GRIB2_LIBS) \
+	-lvx_data2d \
+	-lvx_nc_util \
+	-lvx_regrid \
+	-lvx_grid \
 	-lvx_config \
 	-lvx_gsl_prob \
 	-lvx_cal \
 	-lvx_util \
 	-lvx_math \
+	-lvx_color \
 	-lvx_log \
-	-lgsl -lgslcblas
-
-if ENABLE_PYTHON
-test_ascii_header_LDADD += $(MET_PYTHON_LD) -lvx_data2d_python -lvx_python3_utils -lvx_data2d_python -lvx_python3_utils -lvx_math 
-test_ascii_header_LDADD += -lvx_data2d_python -lvx_python3_utils -lvx_data2d_python -lvx_python3_utils
-test_ascii_header_LDADD += -lvx_math -lvx_grid -lvx_util -lvx_data2d -lvx_config -lvx_gsl_prob -lvx_cal -lvx_math -lvx_util
-endif
-
+	-lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas -lvx_util

met/internal_tests/libcode/vx_data2d_factory/Makefile.am

@@ -25,20 +25,27 @@ test_is_grib_LDADD = -lvx_data2d_factory \
 test_factory_SOURCES = test_factory.cc
 test_factory_CPPFLAGS = ${MET_CPPFLAGS}
 test_factory_LDFLAGS = -L. ${MET_LDFLAGS}
-test_factory_LDADD = -lvx_data2d_factory \
+test_factory_LDADD = -lvx_stat_out \
+	-lvx_statistics \
+	-lvx_shapedata \
+	-lvx_gsl_prob \
+	-lvx_analysis_util \
+	-lvx_data2d_factory \
 	-lvx_data2d_nc_met \
 	-lvx_data2d_grib $(GRIB2_LIBS) \
 	-lvx_data2d_nc_pinterp \
 	$(PYTHON_LIBS) \
 	-lvx_data2d_nccf \
+	-lvx_statistics \
 	-lvx_data2d \
 	-lvx_nc_util \
+	-lvx_regrid \
 	-lvx_grid \
 	-lvx_config \
 	-lvx_gsl_prob \
+	-lvx_cal \
 	-lvx_util \
 	-lvx_math \
 	-lvx_color \
-	-lvx_cal \
 	-lvx_log \
 	-lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas

met/internal_tests/libcode/vx_nc_util/Makefile.am

@@ -19,21 +19,32 @@ noinst_PROGRAMS = test_pressure_levels
 test_pressure_levels_SOURCES = test_pressure_levels.cc
 test_pressure_levels_CPPFLAGS = ${MET_CPPFLAGS}
 test_pressure_levels_LDFLAGS = -L. ${MET_LDFLAGS}
-test_pressure_levels_LDADD = -lvx_tc_util \
+test_pressure_levels_LDADD = -lvx_stat_out \
+	-lvx_statistics \
+	-lvx_shapedata \
+	-lvx_gsl_prob \
+	-lvx_analysis_util \
+	-lvx_tc_util \
+	-lvx_shapedata \
+	-lvx_util \
+	$(PYTHON_LIBS) \
+	-lvx_statistics \
+	-lvx_data2d_factory \
+	-lvx_data2d_nc_met \
+	-lvx_data2d_nc_pinterp \
+	$(PYTHON_LIBS) \
+	-lvx_data2d_nccf \
+	-lvx_data2d_grib $(GRIB2_LIBS) \
 	-lvx_data2d \
 	-lvx_nc_util \
+	-lvx_regrid \
+	-lvx_grid \
 	-lvx_config \
 	-lvx_gsl_prob \
 	-lvx_cal \
 	-lvx_util \
 	-lvx_math \
+	-lvx_color \
 	-lvx_log \
-	-lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas
+	-lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas -lvx_util
 
-if ENABLE_PYTHON
-test_pressure_levels_LDADD += $(MET_PYTHON_LD)
-test_pressure_levels_LDADD += -lvx_data2d_python -lvx_python3_utils
-test_pressure_levels_LDADD += -lvx_data2d_python -lvx_python3_utils
-test_pressure_levels_LDADD += -lvx_grid -lvx_util -lvx_config
-test_pressure_levels_LDADD += -lvx_data2d -lvx_gsl_prob -lvx_util -lvx_math -lvx_cal -lvx_config
-endif

met/internal_tests/libcode/vx_tc_util/Makefile.am

@@ -20,39 +20,64 @@ noinst_PROGRAMS = test_read \
 test_read_SOURCES = test_read.cc
 test_read_CPPFLAGS = ${MET_CPPFLAGS}
 test_read_LDFLAGS = -L. ${MET_LDFLAGS}
-test_read_LDADD = -lvx_tc_util \
+test_read_LDADD = -lvx_stat_out \
+	-lvx_statistics \
+	-lvx_shapedata \
+	-lvx_gsl_prob \
+	-lvx_analysis_util \
+	-lvx_tc_util \
+	-lvx_shapedata \
+	-lvx_util \
+	$(PYTHON_LIBS) \
+	-lvx_statistics \
+	-lvx_data2d_factory \
+	-lvx_data2d_nc_met \
+	-lvx_data2d_nc_pinterp \
+	$(PYTHON_LIBS) \
+	-lvx_data2d_nccf \
+	-lvx_data2d_grib $(GRIB2_LIBS) \
+	-lvx_data2d \
+	-lvx_nc_util \
+	-lvx_regrid \
+	-lvx_grid \
 	-lvx_config \
 	-lvx_gsl_prob \
 	-lvx_cal \
 	-lvx_util \
 	-lvx_math \
+	-lvx_color \
 	-lvx_log \
-	-lgsl -lgslcblas
-
-if ENABLE_PYTHON
-test_read_LDADD += $(MET_PYTHON_LD)
-test_read_LDADD += -lvx_data2d_python -lvx_python3_utils
-test_read_LDADD += -lvx_data2d_python -lvx_python3_utils
-test_read_LDADD += -lvx_grid -lvx_util -lvx_config
-test_read_LDADD += -lvx_data2d -lvx_gsl_prob -lvx_util -lvx_math -lvx_cal -lvx_config
-endif
+	-lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas -lvx_util
 
 test_read_prob_SOURCES = test_read_prob.cc
 test_read_prob_CPPFLAGS = ${MET_CPPFLAGS}
 test_read_prob_LDFLAGS = -L. ${MET_LDFLAGS}
-test_read_prob_LDADD = -lvx_tc_util \
+test_read_prob_LDADD = -lvx_stat_out \
+	-lvx_statistics \
+	-lvx_shapedata \
+	-lvx_gsl_prob \
+	-lvx_analysis_util \
+	-lvx_tc_util \
+	-lvx_shapedata \
+	-lvx_util \
+	$(PYTHON_LIBS) \
+	-lvx_statistics \
+	-lvx_data2d_factory \
+	-lvx_data2d_nc_met \
+	-lvx_data2d_nc_pinterp \
+	$(PYTHON_LIBS) \
+	-lvx_data2d_nccf \
+	-lvx_data2d_grib $(GRIB2_LIBS) \
+	-lvx_data2d \
+	-lvx_nc_util \
+	-lvx_regrid \
+	-lvx_grid \
 	-lvx_config \
 	-lvx_gsl_prob \
 	-lvx_cal \
 	-lvx_util \
 	-lvx_math \
+	-lvx_color \
 	-lvx_log \
-	-lgsl -lgslcblas
+	-lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas -lvx_util
 
-if ENABLE_PYTHON
-test_read_prob_LDADD += $(MET_PYTHON_LD)
-test_read_prob_LDADD += -lvx_data2d_python -lvx_python3_utils
-test_read_prob_LDADD += -lvx_data2d_python -lvx_python3_utils
-test_read_prob_LDADD += -lvx_grid -lvx_util -lvx_config
-test_read_prob_LDADD += -lvx_data2d -lvx_gsl_prob -lvx_util -lvx_math -lvx_cal -lvx_config
-endif

met/internal_tests/tools/other/mode_time_domain/Makefile.am

@@ -56,6 +56,7 @@ test_velocity_LDADD = \
 	-lvx_data2d_grib $(GRIB2_LIBS) \
 	-lvx_data2d_nc_pinterp \
 	$(PYTHON_LIBS) \
+	-lvx_statistics \
 	-lvx_data2d_nccf \
 	-lvx_data2d \
 	-lvx_nc_util \
@@ -69,3 +70,4 @@ test_velocity_LDADD = \
 	-lvx_color \
 	-lvx_log \
 	-lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas -lz
+

met/scripts/python/Makefile.am

@@ -27,6 +27,7 @@ pythonscriptsdir = $(pkgdatadir)/python
 
 pythonscripts_DATA = \
 	read_ascii_numpy.py \
+	read_ascii_numpy_grid.py \
 	read_ascii_xarray.py \
 	read_ascii_point.py \
 	read_ascii_mpr.py

met/scripts/python/read_ascii_numpy_grid.py

@@ -0,0 +1,53 @@
+from __future__ import print_function
+
+import numpy as np
+import os
+import sys
+
+###########################################
+
+print('Python Script:\t', sys.argv[0])
+
+   ##
+   ##  input file specified on the command line
+   ##  load the data into the numpy array
+   ##
+
+if len(sys.argv) == 3:
+    # Read the input file as the first argument
+    input_file = os.path.expandvars(sys.argv[1])
+    data_name  = sys.argv[2]
+    try:
+        # Print some output to verify that this script ran
+        print("Input File:\t" + repr(input_file))
+        print("Data Name:\t" + repr(data_name))
+        met_data = np.loadtxt(input_file)
+        print("Data Shape:\t" + repr(met_data.shape))
+        print("Data Type:\t" + repr(met_data.dtype))
+    except NameError:
+        print("Can't find the input file")
+else:
+    print("ERROR: read_ascii_numpy.py -> Must specify exactly one input file and a name for the data.")
+    sys.exit(1)
+
+###########################################
+
+   ##
+   ##  create the metadata dictionary
+   ##
+
+attrs = {
+
+   'valid': '20050807_120000',
+   'init':  '20050807_000000',
+   'lead':  '120000',
+   'accum': '120000',
+
+   'name':      data_name,
+   'long_name': data_name + '_word',
+   'level':     'Surface',
+   'units':     'None',
+   'grid':      os.path.expandvars(os.getenv('PYTHON_GRID'))
+}
+
+print("Attributes:\t" + repr(attrs))

met/src/libcode/vx_data2d_python/dataplane_from_numpy_array.cc

@@ -12,6 +12,7 @@
 #include "string.h"
 
 #include "vx_python3_utils.h"
+#include "vx_statistics.h"
 #include "check_endian.h"
 
 #include "data_plane.h"
@@ -219,9 +220,43 @@ dp_out.set_accum(t);
 
      ////////////////////
 
-PyObject * py_grid = attrs.lookup_dict("grid");
+   //
+   // attempt to parse "grid" as a string
+   //
+
+s = attrs.lookup_string("grid", false);
+
+if ( s.nonempty() ) {
+
+   grid_out = parse_grid_string(s.c_str());
+
+}
+else {
+
+   //
+   // otherwise, parse "grid" as a dictionary
+   //
+
+   PyObject * py_grid = attrs.lookup_dict("grid");
 
-grid_from_python_dict(Python3_Dict(py_grid), grid_out);
+   grid_from_python_dict(Python3_Dict(py_grid), grid_out);
+
+}
+
+   //
+   // make sure the grid and data dimensions match
+   //
+
+if ( grid_out.nx() != Nx || grid_out.ny() != Ny ) {
+
+   mlog << Error << "\ndataplane_from_numpy_array() -> "
+        << "the grid dimensions (" << grid_out.nx() << ", "
+        << grid_out.ny() << ") and data dimensions (" << Nx
+        << ", " << Ny << ") do not match!\n\n";
+
+   exit ( 1 );
+
+}
 
      ////////////////////