Feature 1319 no pickle

met/data/wrappers/Makefile.am

@@ -23,10 +23,12 @@ wrappersdir = $(pkgdatadir)/wrappers
 wrappers_DATA = \
      generic_python.py \
      generic_pickle.py \
-     read_pickle_dataplane.py \
-     write_pickle_dataplane.py \
+     read_tmp_dataplane.py \
+     write_tmp_dataplane.py \
      write_pickle_mpr.py \
-     write_pickle_point.py
+     read_tmp_ascii.py \
+     write_pickle_point.py \
+     write_tmp_point.py
 
 EXTRA_DIST = ${wrappers_DATA}
 

met/data/wrappers/read_tmp_ascii.py

@@ -0,0 +1,49 @@
+"""
+Module Name: read_tmp_ascii.py
+
+Read MET Point Observations from a text file created by write_tmp_point.py script
+  or MET Matched Pairs from a text file created by write_tmp_mpr.py script
+
+Point observation format:
+    Message_Type, Station_ID, Valid_Time, Lat, Lon, Elevation,
+    GRIB_Code or Variable_Name, Level, Height, QC_String, Observation_Value
+
+Version  Date
+1.0.0    2021/02/18  David Fillmore  Initial version
+"""
+
+__author__ = 'David Fillmore'
+__version__ = '1.0.0'
+__email__ = 'met_help@ucar.edu'
+
+import argparse
+
+point_data = None
+
+def read_tmp_ascii(filename):
+    """
+    Arguments:
+        filename (string): temporary file created by write_tmp_point.py
+
+    Returns:
+        (list of lists): point data
+    """
+    print('read_tmp_ascii:' + filename)
+    f = open(filename, 'r')
+    lines = f.readlines()
+    f.close()
+
+    global point_data
+    point_data = [eval(line.strip('\n')) for line in lines]
+
+    return point_data
+
+if __name__ == '__main__':
+    """
+    Parse command line arguments
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--filename', type=str)
+    args = parser.parse_args()
+
+    data = read_tmp_ascii(args.filename)

met/data/wrappers/read_tmp_dataplane.py

@@ -0,0 +1,36 @@
+########################################################################
+#
+#    Reads temporary file into memory.
+#
+#    usage:  /path/to/python read_tmp_dataplane.py dataplane.tmp
+#
+########################################################################
+
+import sys
+import numpy as np
+import netCDF4 as nc
+
+print('Python Script:\t', sys.argv[0])
+met_info = {}
+
+netcdf_filename = sys.argv[1]
+print('Read NetCDF:\t',  netcdf_filename)
+
+# read NetCDF file
+ds = nc.Dataset(netcdf_filename, 'r')
+met_data = ds['met_data'][:]
+met_attrs = {}
+grid = {}
+for attr, attr_val in ds.__dict__.items():
+    if 'grid' in attr:
+        grid_attr = attr.split('.')[1]
+        grid[grid_attr] = attr_val
+    else:
+        met_attrs[attr] = attr_val
+grid['nx'], grid['ny'] = int(grid['nx']), int(grid['ny'])
+met_attrs['grid'] = grid
+met_attrs['name'] = met_attrs['_name']
+del met_attrs['_name']
+met_info['met_data'] = met_data
+met_info['attrs'] = met_attrs
+print(met_info)

met/data/wrappers/write_pickle_mpr.py

@@ -18,6 +18,7 @@
 print('Write Pickle:\t',  sys.argv[1])
 
 pickle_filename = sys.argv[1]
+tmp_filename = pickle_filename + '.txt'
 
 pyembed_module_name = sys.argv[2]
 sys.argv = sys.argv[2:]
@@ -28,6 +29,8 @@
 met_in = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(met_in)
 
-print(met_in)
+f = open(tmp_filename, 'w')
+for line in met_in.mpr_data:
+    f.write(str(line) + '\n')
 
 pickle.dump( met_in.mpr_data, open( pickle_filename, "wb" ) )

met/data/wrappers/write_pickle_point.py

@@ -18,6 +18,7 @@
 print('Write Pickle:\t',  sys.argv[1])
 
 pickle_filename = sys.argv[1]
+tmp_filename = pickle_filename + '.txt'
 
 pyembed_module_name = sys.argv[2]
 sys.argv = sys.argv[2:]
@@ -28,4 +29,8 @@
 met_in = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(met_in)
 
+f = open(tmp_filename, 'w')
+for line in met_in.point_data:
+    f.write(str(line) + '\n')
+
 pickle.dump( met_in.point_data, open( pickle_filename, "wb" ) )

met/data/wrappers/write_tmp_dataplane.py

@@ -0,0 +1,63 @@
+########################################################################
+#
+#    Adapted from a script provided by George McCabe
+#    Adapted by Randy Bullock
+#
+#    usage:  /path/to/python write_tmp_dataplane.py \
+#            tmp_output_filename <user_python_script>.py <args>
+#
+########################################################################
+
+import os
+import sys
+import importlib.util
+import netCDF4 as nc
+
+print('Python Script:\t', sys.argv[0])
+print('User Command:\t',  sys.argv[2:])
+
+netcdf_filename = sys.argv[1]
+
+print('Write NetCDF:\t',  netcdf_filename)
+
+pyembed_module_name = sys.argv[2]
+sys.argv = sys.argv[2:]
+
+if not pyembed_module_name.endswith('.py'):
+    pyembed_module_name += '.py'
+
+user_base = os.path.basename(pyembed_module_name).replace('.py','')
+
+spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name)
+met_in = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(met_in)
+
+met_info = {'met_data': met_in.met_data}
+if hasattr(met_in.met_data, 'attrs') and met_in.met_data.attrs:
+    attrs = met_in.met_data.attrs
+else:
+    attrs = met_in.attrs
+met_info['attrs'] = attrs
+
+print('write_tmp_dataplane')
+print(met_info)
+
+# write NetCDF file
+ds = nc.Dataset(netcdf_filename, 'w')
+
+nx, ny = met_in.met_data.shape
+ds.createDimension('x', nx)
+ds.createDimension('y', ny)
+dp = ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y'))
+dp[:] = met_in.met_data
+
+for attr, attr_val in met_info['attrs'].items():
+    print(attr, attr_val, type(attr_val))
+    if attr == 'name':
+        setattr(ds, '_name', attr_val)
+    if type(attr_val) == str:
+        setattr(ds, attr, attr_val)
+    if type(attr_val) == dict:
+        for key in attr_val:
+            setattr(ds, attr + '.' + key, attr_val[key])
+ds.close()

met/data/wrappers/write_pickle_dataplane.py → met/data/wrappers/write_tmp_point.py

@@ -3,40 +3,30 @@
 #    Adapted from a script provided by George McCabe
 #    Adapted by Randy Bullock
 #
-#    usage:  /path/to/python write_pickle_dataplane.py \
-#            pickle_output_filename <user_python_script>.py <args>
+#    usage:  /path/to/python write_tmp_point.py \
+#            tmp_ascii_output_filename <user_python_script>.py <args>
 #
 ########################################################################
 
 import os
 import sys
-import pickle
 import importlib.util
-import xarray as xr
 
 print('Python Script:\t', sys.argv[0])
 print('User Command:\t',  sys.argv[2:])
-print('Write Pickle:\t',  sys.argv[1])
+print('Write Temporary Ascii:\t',  sys.argv[1])
 
-pickle_filename = sys.argv[1]
+tmp_filename = sys.argv[1]
 
 pyembed_module_name = sys.argv[2]
 sys.argv = sys.argv[2:]
 
-if not pyembed_module_name.endswith('.py'):
-    pyembed_module_name += '.py'
-
 user_base = os.path.basename(pyembed_module_name).replace('.py','')
 
 spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name)
 met_in = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(met_in)
 
-if isinstance(met_in.met_data, xr.DataArray):
-    met_info = { 'attrs': met_in.met_data.attrs, 'met_data': met_in.met_data }
-else:
-    met_info = { 'attrs': met_in.attrs, 'met_data': met_in.met_data }
-
-print(met_info)
-
-pickle.dump( met_info, open( pickle_filename, "wb" ) )
+f = open(tmp_filename, 'w')
+for line in met_in.point_data:
+    f.write(str(line) + '\n')