From dca0b258c021ba4958e0c65a36290a30a966a6bc Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Tue, 14 Sep 2021 11:28:24 -0600 Subject: [PATCH 1/3] #1913 Changed MAX_PBL to 10000. Filter out if the PBL input data is invalid --- met/src/tools/other/pb2nc/pb2nc.cc | 72 ++++++++++++++++-------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/met/src/tools/other/pb2nc/pb2nc.cc b/met/src/tools/other/pb2nc/pb2nc.cc index 588e3a62e7..e14df8235a 100644 --- a/met/src/tools/other/pb2nc/pb2nc.cc +++ b/met/src/tools/other/pb2nc/pb2nc.cc @@ -167,7 +167,7 @@ static float static_dummy_200[MAX_CAPE_LEVEL]; static float static_dummy_201[MAX_CAPE_LEVEL+1]; #define ROG 287.04 -#define MAX_PBL 5000 +#define MAX_PBL 10000 #define MAX_PBL_LEVEL 256 #define PBL_DEBUG_LEVEL 8 static bool IGNORE_Q_PBL = true; @@ -375,6 +375,7 @@ static void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_co const ConcatString &hdr_typ, const ConcatString &hdr_sid); static int interpolate_by_pressure(int length, float *pres_data, float *var_data); static void interpolate_pqtzuv(float*, float*, float*); +static bool is_valid_pb_data(float pb_value); static void log_merged_tqz_uv(map pqtzuv_map_tq, map pqtzuv_map_uv, map &pqtzuv_map_merged, @@ -1556,35 +1557,36 @@ void process_pbfile(int i_pb) { if (cal_cape) { if (cape_member_cnt >= 3) cape_level++; } - if (do_pbl && !is_eq(pqtzuv[0], bad_data_float)) { - // Allocated memory is deleted after all observations are processed - float *tmp_pqtzuv = new float [mxr8vt]; - - for(kk=0; kk 0) { + bool has_uv = is_valid_pb_data(pqtzuv[4]) && is_valid_pb_data(pqtzuv[5]); + bool has_tq = is_valid_pb_data(pqtzuv[2]) && + (IGNORE_Q_PBL || is_valid_pb_data(pqtzuv[1])) && + (IGNORE_Z_PBL || is_valid_pb_data(pqtzuv[3])); + if (has_tq || has_uv) { + // Allocated memory is deleted after all observations are processed + float *tmp_pqtzuv = new float [mxr8vt]; + + for(kk=0; kk MAX_PBL) { - mlog << Warning << "\nComputed PBL (" << obs_arr[4] << " from " - << pbl_value << ") is too high, Reset to " << MAX_PBL - << " " << hdr_info<< "\n\n"; - obs_arr[4] = MAX_PBL; + mlog << Warning << "\nNot saved the computed PBL (" << obs_arr[4] << " from " + << pbl_value << ") because of the MAX PBL " << MAX_PBL + << " (" << hdr_info<< ")\n\n"; } - - addObservation(obs_arr, (string)hdr_typ, (string)hdr_sid, hdr_vld_ut, - hdr_lat, hdr_lon, hdr_elv, pbl_qm, OBS_BUFFER_SIZE); + else addObservation(obs_arr, (string)hdr_typ, (string)hdr_sid, hdr_vld_ut, + hdr_lat, hdr_lon, hdr_elv, pbl_qm, OBS_BUFFER_SIZE); } } @@ -3211,6 +3211,12 @@ void interpolate_pqtzuv(float *prev_pqtzuv, float *cur_pqtzuv, float *next_pqtzu //////////////////////////////////////////////////////////////////////// +static bool is_valid_pb_data(float pb_value) { + return (!is_eq(pb_value,bad_data_float) && pb_value < r8bfms); +} + +//////////////////////////////////////////////////////////////////////// + void merge_records(float *first_pqtzuv, map pqtzuv_map_pivot, map pqtzuv_map_aux, map &pqtzuv_map_merged) { From c0f7b49e00a10c23b8a3c1105a164bf9677e29db Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Wed, 15 Sep 2021 14:00:29 -0600 Subject: [PATCH 2/3] #1912 Replaced is_eq to is_bad_data --- met/src/tools/other/pb2nc/pb2nc.cc | 26 ++++++++++---------- met/src/tools/other/point2grid/point2grid.cc | 22 ++++++++--------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/met/src/tools/other/pb2nc/pb2nc.cc b/met/src/tools/other/pb2nc/pb2nc.cc index e14df8235a..c2365bda4a 100644 --- a/met/src/tools/other/pb2nc/pb2nc.cc +++ b/met/src/tools/other/pb2nc/pb2nc.cc @@ -2466,7 +2466,7 @@ void addObservation(const float *obs_arr, const ConcatString &hdr_typ, // Write the quality flag to the netCDF file ConcatString obs_qty; int quality_code = nint(quality_mark); - if (quality_code == bad_data_int || quality_mark > r8bfms) { + if (is_bad_data(quality_code) || quality_mark > r8bfms) { obs_qty.add("NA"); } else { @@ -2822,7 +2822,7 @@ void display_bufr_variables(const StringArray &all_vars, const StringArray &all_ void copy_pqtzuv(float *to_pqtzuv, float *from_pqtzuv, bool copy_all) { int start_idx = (copy_all ? 0 : 1); for (int index = start_idx; index < mxr8vt; index++) { - if (copy_all || !is_eq(from_pqtzuv[index], bad_data_float)) + if (copy_all || !is_bad_data(from_pqtzuv[index])) to_pqtzuv[index] = from_pqtzuv[index]; } } @@ -2996,8 +2996,8 @@ float compute_pbl(map pqtzuv_map_tq, pbl_data_hgt[index] = pqtzuv[3]; pbl_data_ugrd[index] = pqtzuv[4]; pbl_data_vgrd[index] = pqtzuv[5]; - if (!is_eq(pbl_data_spfh[index], bad_data_float)) spfh_cnt++; - if (!is_eq(pbl_data_hgt[index], bad_data_float)) hgt_cnt++; + if (is_valid_pb_data(pbl_data_spfh[index])) spfh_cnt++; + if (is_valid_pb_data(pbl_data_hgt[index])) hgt_cnt++; selected_levels.add(nint(it->first)); } @@ -3017,7 +3017,7 @@ float compute_pbl(map pqtzuv_map_tq, break; } } - if (!is_eq(highest_pressure, bad_data_float)) { + if (!is_bad_data(highest_pressure)) { index = MAX_PBL_LEVEL - 1; for (; it!=pqtzuv_map_tq.end(); ++it) { int pres_level = nint(it->first); @@ -3069,7 +3069,7 @@ float compute_pbl(map pqtzuv_map_tq, //SUBROUTINE CALPBL(T,Q,P,Z,U,V,MZBL,HPBL,jpbl) calpbl_(pbl_data_temp, pbl_data_spfh, pbl_data_pres, pbl_data_hgt, pbl_data_ugrd, pbl_data_vgrd, &mzbl, &hpbl, &jpbl); - if (is_eq(hpbl, bad_data_float)) + if (!is_valid_pb_data(hpbl)) mlog << Debug(5) << method_name << " fail to compute PBL. TQ records: " << tq_count << " UV records: " << uv_count << " merged records: " << pqtzuv_map_merged.size() << "\n"; @@ -3089,7 +3089,7 @@ void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code, ConcatString hdr_info; hdr_info << unix_to_yyyymmdd_hhmmss(hdr_vld_ut) << " " << hdr_typ << " " << hdr_sid; - if (is_eq(pbl_value, bad_data_float)) { + if (is_bad_data(pbl_value)) { mlog << Warning << "\nFailed to compute PBL (" << hdr_info << ")\n\n"; } else if (pbl_value < hdr_elv) { @@ -3129,7 +3129,7 @@ int interpolate_by_pressure(int length, float *pres_data, float *var_data) { skip_missing = false; count_interpolated = 0; for (idx=0; idx= 4) { if (from_min_value > data_value) from_min_value = data_value; @@ -1134,8 +1134,8 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, unixtime ref_ut = (unixtime) 0; unixtime tmp_time; if( conf_info.valid_time > 0 ) { - if (!is_eq(bad_data_int, conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds; - if (!is_eq(bad_data_int, conf_info.end_ds)) valid_end_ut += conf_info.end_ds; + if (!is_bad_data(conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds; + if (!is_bad_data(conf_info.end_ds)) valid_end_ut += conf_info.end_ds; ref_ut = get_reference_unixtime(&time_var, sec_per_unit, no_leap_year); } for (int i=0; i= to_size ) { mlog << Error << "\n" << method_name << "the mapped cell is out of range: " @@ -1978,7 +1978,7 @@ static unixtime find_valid_time(NcVar time_var) { } } - if (valid_time == bad_data_int) { + if (is_bad_data(valid_time)) { mlog << Error << "\n" << method_name << "trouble finding time variable from \"" << InputFilename << "\"\n\n"; @@ -2401,7 +2401,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, for (int dIdx=0; dIdx Date: Wed, 15 Sep 2021 15:05:26 -0600 Subject: [PATCH 3/3] #1913 Changed is_eq to is_bad_data --- met/src/tools/other/pb2nc/pb2nc.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/met/src/tools/other/pb2nc/pb2nc.cc b/met/src/tools/other/pb2nc/pb2nc.cc index c2365bda4a..f16ed62b97 100644 --- a/met/src/tools/other/pb2nc/pb2nc.cc +++ b/met/src/tools/other/pb2nc/pb2nc.cc @@ -3090,7 +3090,7 @@ void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code, hdr_info << unix_to_yyyymmdd_hhmmss(hdr_vld_ut) << " " << hdr_typ << " " << hdr_sid; if (is_bad_data(pbl_value)) { - mlog << Warning << "\nFailed to compute PBL (" << hdr_info << ")\n\n"; + mlog << Warning << "\nFailed to compute PBL " << pbl_value << " (" << hdr_info << ")\n\n"; } else if (pbl_value < hdr_elv) { mlog << Warning << "\nNot saved because the computed PBL (" << pbl_value @@ -3129,7 +3129,7 @@ int interpolate_by_pressure(int length, float *pres_data, float *var_data) { skip_missing = false; count_interpolated = 0; for (idx=0; idx