Skip to content

Commit

Permalink
#2202 Added APIs to check the point data (missing data and data types)
Browse files Browse the repository at this point in the history
  • Loading branch information
Howard Soh committed Jul 22, 2022
1 parent 3d13abc commit ec9a1ce
Showing 1 changed file with 122 additions and 33 deletions.
155 changes: 122 additions & 33 deletions scripts/python/met_point_obs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
- This is the base class and the customized script should extend the met_point_obs.
- The customized script (for example "custom_reader") must implement "def read_data(self, args)"
which fills the array (list) variables at __init__().
which fills the array (python list or numpy array) variables at __init__().
- The args can be 1) single string argument, 2) the list of arguments, and 3) the dictionary of arguments.
- The variable "met_point_data" must be set for MET tools
- The customized script is expected to include following codes:
Expand All @@ -28,6 +28,8 @@ class met_point_obs(ABC):
'''
classdocs
'''
ERROR_P = " ==ERROR_PYTHON=="
INFO_P = " ==INFO_PYTHON=="

python_prefix = 'PYTHON_POINT_RAW'

Expand All @@ -41,39 +43,97 @@ def __init__(self, use_var_id=True):
self.nhdr = 0
self.npbhdr = 0
self.nhdr_typ = 0 # type table
self.nhdr_sid = 0 #station_uid table
self.nhdr_sid = 0 # station_id table
self.nhdr_vld = 0 # valid time strings
self.hdr_typ = [] # (nhdr)
self.hdr_sid = [] # (nhdr)
self.hdr_vld = [] # (nhdr)
self.hdr_lat = [] # (nhdr)
self.hdr_lon = [] # (nhdr)
self.hdr_elv = [] # (nhdr)
self.hdr_typ_table = [] # (nhdr_typ, mxstr2)
self.hdr_sid_table = [] # (nhdr_sid, mxstr2)
self.hdr_vld_table = [] # (nhdr_vld, mxstr)
self.hdr_prpt_typ = [] # optional
self.hdr_irpt_typ = [] # optional
self.hdr_inst_typ = [] # optional
self.hdr_typ = [] # (nhdr) integer
self.hdr_sid = [] # (nhdr) integer
self.hdr_vld = [] # (nhdr) integer
self.hdr_lat = [] # (nhdr) float
self.hdr_lon = [] # (nhdr) float
self.hdr_elv = [] # (nhdr) float
self.hdr_typ_table = [] # (nhdr_typ, mxstr2) string
self.hdr_sid_table = [] # (nhdr_sid, mxstr2) string
self.hdr_vld_table = [] # (nhdr_vld, mxstr) string

#Observation data
self.nobs = 0
self.nobs_qty = 0
self.nobs_var = 0
self.obs_qty = [] # (nobs_qty )
self.obs_hid = [] # (nobs)
self.obs_vid = [] # (nobs) veriable index or GRIB code
self.obs_lvl = [] # (nobs)
self.obs_hgt = [] # (nobs)
self.obs_val = [] # (nobs)
self.obs_qty_table = [] # (nobs_qty, mxstr)
self.obs_var_table = [] # (nobs_var, mxstr2) optional if GRIB code at self.obs_vid
self.obs_qty = [] # (nobs_qty) integer, index of self.obs_qty_table
self.obs_hid = [] # (nobs) integer
self.obs_vid = [] # (nobs) integer, veriable index from self.obs_var_table or GRIB code
self.obs_lvl = [] # (nobs) float
self.obs_hgt = [] # (nobs) float
self.obs_val = [] # (nobs) float
self.obs_qty_table = [] # (nobs_qty, mxstr) string
self.obs_var_table = [] # (nobs_var, mxstr2) string, required if self.use_var_id is True

# Optional variables for PREPBUFR, not supported yet
self.hdr_prpt_typ = [] # optional
self.hdr_irpt_typ = [] # optional
self.hdr_inst_typ = [] # optional

@abstractmethod
def read_data(self, args):
# args should be list or dictionary
# The variables at __init__ should be filled as python list or numpy array
pass

def check_data_member_float(self, local_var, var_name):
if 0 == len(local_var):
self.log_error("{v} is empty".format(v=var_name))
elif isinstance(local_var, list):
if 0 <= str(type(local_var[0])).find('numpy'):
self.log_info("Recommend using numpy instead of python list for {v} ({t}) to avoid side effect".format(
v=var_name, t=type(local_var[0])))
elif not isinstance(local_var[0], (int, float)):
self.log_error("Not supported data type ({t}) for {v}[0] (int or float only)".format(
v=var_name, t=local_var[0].type))
elif not isinstance(local_var, np.ndarray):
self.log_error("Not supported data type ({t}) for {v} (list and numpy.ndarray)".format(
v=var_name, t=type(local_var)))

def check_data_member_int(self, local_var, var_name):
if 0 == len(local_var):
self.log_error("{v} is empty".format(v=var_name))
elif isinstance(local_var, list):
if 0 <= str(type(local_var[0])).find('numpy'):
self.log_info("Recommend using numpy instead of python list for {v} ({t}) to avoid side effect".format(
v=var_name, t=type(local_var[0])))
elif not isinstance(local_var[0], int):
self.log_error("Not supported data type ({t}) for {v}[0] (int only)".format(
v=var_name, t=type(local_var[0])))
elif not isinstance(local_var, np.ndarray):
self.log_error("Not supported data type ({t}) for {v} (list and numpy.ndarray)".format(
v=var_name, t=type(local_var)))

def check_data_member_string(self, local_var, var_name):
if 0 == len(local_var):
self.log_error("{v} is empty".format(v=var_name))
elif not isinstance(local_var, (list)):
self.log_error("Not supported data type ({t}) for {v} (list)".format(
v=var_name, t=type(local_var)))

def check_point_data(self):
self.check_data_member_int(self.hdr_typ,'hdr_typ')
self.check_data_member_int(self.hdr_sid,'hdr_sid')
self.check_data_member_int(self.hdr_vld,'hdr_vld')
self.check_data_member_float(self.hdr_lat,'hdr_lat')
self.check_data_member_float(self.hdr_lon,'hdr_lon')
self.check_data_member_float(self.hdr_elv,'hdr_elv')
self.check_data_member_string(self.hdr_typ_table,'hdr_typ_table')
self.check_data_member_string(self.hdr_sid_table,'hdr_sid_table')
self.check_data_member_string(self.hdr_vld_table,'hdr_vld_table')

self.check_data_member_int(self.obs_qty,'obs_qty')
self.check_data_member_int(self.obs_hid,'obs_hid')
self.check_data_member_int(self.obs_vid,'obs_vid')
self.check_data_member_float(self.obs_lvl,'obs_lvl')
self.check_data_member_float(self.obs_hgt,'obs_hgt')
self.check_data_member_float(self.obs_val,'obs_val'0)
self.check_data_member_string(self.obs_qty_table,'bs_qty_table')
self.check_data_member_string(self.obs_var_table,'bs_var_table')

def get_point_data(self):
if self.nhdr <= 0:
self.nhdr = len(self.hdr_lat)
Expand All @@ -91,8 +151,18 @@ def get_point_data(self):
self.nobs_qty = len(self.obs_qty_table)
if self.nobs_var <= 0:
self.nobs_var = len(self.obs_var_table)
self.check_point_data()
return self.__dict__

def log_error(self, err_msgs):
print(self.ERROR_P)
for err_line in err_msgs.split('\n'):
print('{p} {m}'.format(p=self.ERROR_P, m=err_line))
print(self.ERROR_P)

def log_info(self, info_msg):
print('{p} {m}'.format(p=self.INFO_P, m=info_msg))

@staticmethod
def is_python_script(arg_value):
return arg_value.startswith(met_point_obs.python_prefix)
Expand Down Expand Up @@ -120,31 +190,50 @@ def print_data(key, data_array, show_count=COUNT_SHOW):
def print_point_data(met_point_data, print_subset=True):
print(' === MET point data by python embedding ===')
if print_subset:
met_point_obs.print_data('nhdr', met_point_data['nhdr'])
met_point_obs.print_data('nobs' , met_point_data['nobs'])
met_point_obs.print_data('nhdr',met_point_data['nhdr'])
met_point_obs.print_data('nobs',met_point_data['nobs'])
met_point_obs.print_data('use_var_id',met_point_data['use_var_id'])
met_point_obs.print_data('hdr_typ',met_point_data['hdr_typ'])
met_point_obs.print_data('hdr_typ_table',met_point_data['hdr_typ_table'])
met_point_obs.print_data('hdr_sid',met_point_data['hdr_sid'])
met_point_obs.print_data('hdr_sid_table',met_point_data['hdr_sid_table'])
met_point_obs.print_data('hdr_vld',met_point_data['hdr_vld'])
met_point_obs.print_data('hdr_vld_table',met_point_data['hdr_vld_table'])
met_point_obs.print_data('hdr_lat',met_point_data['hdr_lat'])
met_point_obs.print_data('hdr_lon',met_point_data['hdr_lon'])
met_point_obs.print_data('hdr_elv',met_point_data['hdr_elv'])
met_point_obs.print_data('obs_hid',met_point_data['obs_hid'])
met_point_obs.print_data('obs_vid',met_point_data['obs_vid'])
met_point_obs.print_data('obs_val',met_point_data['obs_val'])
met_point_obs.print_data('obs_var_table',met_point_data['obs_var_table'])
met_point_obs.print_data('obs_qty',met_point_data['obs_qty'])
met_point_obs.print_data('obs_qty_table',met_point_data['obs_qty_table'])
met_point_obs.print_data('hdr_typ_table',met_point_data['hdr_typ_table'])
met_point_obs.print_data('hdr_sid_table',met_point_data['hdr_sid_table'])
met_point_obs.print_data('hdr_vld_table',met_point_data['hdr_vld_table'])
met_point_obs.print_data('obs_lvl',met_point_data['obs_lvl'])
met_point_obs.print_data('obs_hgt',met_point_data['obs_hgt'])
met_point_obs.print_data('obs_val',met_point_data['obs_val'])

else:
print('All',met_point_data)
print(" nhdr: ",met_point_data['nhdr'])
print(" nobs: ",met_point_data['nobs'])
print('use_var_id: ',met_point_data['use_var_id'])
print('hdr_typ: ',met_point_data['hdr_typ'])
print('obs_vid: ',met_point_data['obs_vid'])
print('obs_val: ',met_point_data['obs_val'])
print('obs_var_table: ',met_point_data['obs_var_table'])
print('obs_qty_table: ',met_point_data['obs_qty_table'])
print('hdr_typ_table: ',met_point_data['hdr_typ_table'])
print('hdr_sid: ',met_point_data['hdr_sid'])
print('hdr_sid_table: ',met_point_data['hdr_sid_table'])
print('hdr_vld: ',met_point_data['hdr_vld'])
print('hdr_vld_table: ',met_point_data['hdr_vld_table'])
print('hdr_lat: ',met_point_data['hdr_lat'])
print('hdr_lon: ',met_point_data['hdr_lon'])
print('hdr_elv: ',met_point_data['hdr_elv'])
print('obs_hid: ',met_point_data['obs_hid'])
print('obs_vid: ',met_point_data['obs_vid'])
print('obs_var_table: ',met_point_data['obs_var_table'])
print('obs_qty: ',met_point_data['obs_qty'])
print('obs_qty_table: ',met_point_data['obs_qty_table'])
print('obs_lvl: ',met_point_data['obs_lvl'])
print('obs_hgt: ',met_point_data['obs_hgt'])
print('obs_val: ',met_point_data['obs_val'])

print(' === MET point data by python embedding ===')


Expand All @@ -160,9 +249,9 @@ def read_data(self, arg_map={}):
self.hdr_lon = np.array([ -89., -89., -89., -89., -89., -89., -89., -89., -89., -92., -92., -92., -92., -89., -89., -89., -89., -89., -89., -89., -89., -89., -92., -92., -92., -92. ])
self.hdr_elv = np.array([ 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220. ])

self.obs_qid = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])
self.obs_hid = np.array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 25 ])
self.obs_vid = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ])
self.obs_qty = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])
self.obs_lvl = np.array([ 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000. ])
self.obs_hgt = np.array([ 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2. ])
self.obs_val = np.array([ 292., 292.5, 293., 293.5, 294., 294.5, 295., 295.5, 296., 292., 293.4, 293., 296., 294., 92., 92.5, 93., 93.5, 94., 94.5, 95., 95.5, 96., 92., 93.4, 93., 96., 94. ])
Expand Down

0 comments on commit ec9a1ce

Please sign in to comment.