Skip to content

Commit

Permalink
Feature 1913 MAX_PBL (#1916)
Browse files Browse the repository at this point in the history 
* #1913 Changed MAX_PBL to 10000. Filter out if the PBL input data is invalid

* #1912 Replaced is_eq to is_bad_data

* #1913 Changed is_eq to is_bad_data

Co-authored-by: Howard Soh <hsoh@kiowa.rap.ucar.edu>
  • Loading branch information
@hsoh-u
hsoh-u and Howard Soh authored Sep 16, 2021
1 parent 054fe2c commit c104425
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 56 deletions.
96 changes: 51 additions & 45 deletions met/src/tools/other/pb2nc/pb2nc.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ static float static_dummy_200[MAX_CAPE_LEVEL];
static float static_dummy_201[MAX_CAPE_LEVEL+1];

#define ROG 287.04
#define MAX_PBL 5000
#define MAX_PBL 10000
#define MAX_PBL_LEVEL 256
#define PBL_DEBUG_LEVEL 8
static bool IGNORE_Q_PBL = true;
Expand Down Expand Up @@ -374,6 +374,7 @@ static void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_co
const ConcatString &hdr_typ, const ConcatString &hdr_sid);
static int interpolate_by_pressure(int length, float *pres_data, float *var_data);
static void interpolate_pqtzuv(float*, float*, float*);
static bool is_valid_pb_data(float pb_value);
static void log_merged_tqz_uv(map<float, float*> pqtzuv_map_tq,
map<float, float*> pqtzuv_map_uv,
map<float, float*> &pqtzuv_map_merged,
Expand Down Expand Up @@ -1562,35 +1563,36 @@ void process_pbfile(int i_pb) {
if (cal_cape) {
if (cape_member_cnt >= 3) cape_level++;
}
if (do_pbl && !is_eq(pqtzuv[0], bad_data_float)) {

// Allocated memory is deleted after all observations are processed
float *tmp_pqtzuv = new float [mxr8vt];

for(kk=0; kk<mxr8vt; kk++) tmp_pqtzuv[kk] = pqtzuv[kk];

bool has_tq = false;
bool has_uv = false;
if (!is_eq(tmp_pqtzuv[4],bad_data_float) &&
!is_eq(tmp_pqtzuv[5],bad_data_float)) {
has_uv = true;
pqtzuv_map_uv[pqtzuv[0]] = tmp_pqtzuv;
}
if (!is_eq(tmp_pqtzuv[2],bad_data_float) &&
(!is_eq(tmp_pqtzuv[1],bad_data_float) || IGNORE_Q_PBL) &&
(!is_eq(tmp_pqtzuv[3],bad_data_float) || IGNORE_Z_PBL)) {
has_tq = true;
pqtzuv_map_tq[pqtzuv[0]] = tmp_pqtzuv;
if (do_pbl) {
bool excluded = true;
// Excludes a record if the pressure value is not valid
if (is_valid_pb_data(pqtzuv[0]) && pqtzuv[0] > 0) {
bool has_uv = is_valid_pb_data(pqtzuv[4]) && is_valid_pb_data(pqtzuv[5]);
bool has_tq = is_valid_pb_data(pqtzuv[2]) &&
(IGNORE_Q_PBL || is_valid_pb_data(pqtzuv[1])) &&
(IGNORE_Z_PBL || is_valid_pb_data(pqtzuv[3]));
if (has_tq || has_uv) {
// Allocated memory is deleted after all observations are processed
float *tmp_pqtzuv = new float [mxr8vt];

for(kk=0; kk<mxr8vt; kk++) tmp_pqtzuv[kk] = pqtzuv[kk];

if (has_uv) pqtzuv_map_uv[pqtzuv[0]] = tmp_pqtzuv;
if (has_tq) pqtzuv_map_tq[pqtzuv[0]] = tmp_pqtzuv;
pqtzuv_list.push_back(tmp_pqtzuv);
excluded = false;
}
}
if (!(has_tq || has_uv)) {
mlog << Debug(5) << method_name << " PBL: excluded " << lv
<< "-th level record:" << " T=" << tmp_pqtzuv[2]
<< ", U=" << tmp_pqtzuv[4] << ", V=" << tmp_pqtzuv[5]
<< ", Q=" << tmp_pqtzuv[0] << ", Z=" << tmp_pqtzuv[3]
if (excluded) {
mlog << Debug(5) << method_name
<< " PBL: excluded " << lv << "-th level record:"
<< " P=" << pqtzuv[0] << " T=" << pqtzuv[2]
<< ", U=" << pqtzuv[4] << ", V=" << pqtzuv[5]
<< ", Q=" << pqtzuv[1] << ", Z=" << pqtzuv[3]
<< " valid_time: " << unix_to_yyyymmdd_hhmmss(hdr_vld_ut)
<< " " << hdr_typ << " " << hdr_sid << "\n";
}
pqtzuv_list.push_back(tmp_pqtzuv);
}
} // end for lv

Expand Down Expand Up @@ -2477,7 +2479,7 @@ void addObservation(const float *obs_arr, const ConcatString &hdr_typ,
// Write the quality flag to the netCDF file
ConcatString obs_qty;
int quality_code = nint(quality_mark);
if (quality_code == bad_data_int || quality_mark > r8bfms) {
if (is_bad_data(quality_code) || quality_mark > r8bfms) {
obs_qty.add("NA");
}
else {
Expand Down Expand Up @@ -2833,7 +2835,7 @@ void display_bufr_variables(const StringArray &all_vars, const StringArray &all_
void copy_pqtzuv(float *to_pqtzuv, float *from_pqtzuv, bool copy_all) {
int start_idx = (copy_all ? 0 : 1);
for (int index = start_idx; index < mxr8vt; index++) {
if (copy_all || !is_eq(from_pqtzuv[index], bad_data_float))
if (copy_all || !is_bad_data(from_pqtzuv[index]))
to_pqtzuv[index] = from_pqtzuv[index];
}
}
Expand Down Expand Up @@ -3007,8 +3009,8 @@ float compute_pbl(map<float, float*> pqtzuv_map_tq,
pbl_data_hgt[index] = pqtzuv[3];
pbl_data_ugrd[index] = pqtzuv[4];
pbl_data_vgrd[index] = pqtzuv[5];
if (!is_eq(pbl_data_spfh[index], bad_data_float)) spfh_cnt++;
if (!is_eq(pbl_data_hgt[index], bad_data_float)) hgt_cnt++;
if (is_valid_pb_data(pbl_data_spfh[index])) spfh_cnt++;
if (is_valid_pb_data(pbl_data_hgt[index])) hgt_cnt++;
selected_levels.add(nint(it->first));
}

Expand All @@ -3028,7 +3030,7 @@ float compute_pbl(map<float, float*> pqtzuv_map_tq,
break;
}
}
if (!is_eq(highest_pressure, bad_data_float)) {
if (!is_bad_data(highest_pressure)) {
index = MAX_PBL_LEVEL - 1;
for (; it!=pqtzuv_map_tq.end(); ++it) {
int pres_level = nint(it->first);
Expand Down Expand Up @@ -3080,7 +3082,7 @@ float compute_pbl(map<float, float*> pqtzuv_map_tq,
//SUBROUTINE CALPBL(T,Q,P,Z,U,V,MZBL,HPBL,jpbl)
calpbl_(pbl_data_temp, pbl_data_spfh, pbl_data_pres, pbl_data_hgt,
pbl_data_ugrd, pbl_data_vgrd, &mzbl, &hpbl, &jpbl);
if (is_eq(hpbl, bad_data_float))
if (!is_valid_pb_data(hpbl))
mlog << Debug(5) << method_name << " fail to compute PBL. TQ records: "
<< tq_count << " UV records: " << uv_count << " merged records: "
<< pqtzuv_map_merged.size() << "\n";
Expand All @@ -3100,8 +3102,8 @@ void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code,
ConcatString hdr_info;
hdr_info << unix_to_yyyymmdd_hhmmss(hdr_vld_ut)
<< " " << hdr_typ << " " << hdr_sid;
if (is_eq(pbl_value, bad_data_float)) {
mlog << Warning << "\nFailed to compute PBL " << hdr_info << "\n\n";
if (is_bad_data(pbl_value)) {
mlog << Warning << "\nFailed to compute PBL " << pbl_value << " (" << hdr_info << ")\n\n";
}
else if (pbl_value < hdr_elv) {
mlog << Warning << "\nNot saved because the computed PBL (" << pbl_value
Expand All @@ -3118,14 +3120,12 @@ void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code,
<< " lat: " << hdr_lat << ", lon: " << hdr_lon
<< ", elv: " << hdr_elv << " " << hdr_info << "\n\n";
if (obs_arr[4] > MAX_PBL) {
mlog << Warning << "\nComputed PBL (" << obs_arr[4] << " from "
<< pbl_value << ") is too high, Reset to " << MAX_PBL
<< " " << hdr_info<< "\n\n";
obs_arr[4] = MAX_PBL;
mlog << Warning << "\nNot saved the computed PBL (" << obs_arr[4] << " from "
<< pbl_value << ") because of the MAX PBL " << MAX_PBL
<< " (" << hdr_info<< ")\n\n";
}

addObservation(obs_arr, (string)hdr_typ, (string)hdr_sid, hdr_vld_ut,
hdr_lat, hdr_lon, hdr_elv, pbl_qm, OBS_BUFFER_SIZE);
else addObservation(obs_arr, (string)hdr_typ, (string)hdr_sid, hdr_vld_ut,
hdr_lat, hdr_lon, hdr_elv, pbl_qm, OBS_BUFFER_SIZE);
}
}

Expand All @@ -3142,7 +3142,7 @@ int interpolate_by_pressure(int length, float *pres_data, float *var_data) {
skip_missing = false;
count_interpolated = 0;
for (idx=0; idx<length; idx++) {
if (is_eq(var_data[idx], bad_data_float)) {
if (is_bad_data(var_data[idx])) {
skip_missing = true;
}
else {
Expand Down Expand Up @@ -3200,15 +3200,15 @@ void interpolate_pqtzuv(float *prev_pqtzuv, float *cur_pqtzuv, float *next_pqtzu
/ (log(next_pqtzuv[0]) - log(prev_pqtzuv[0]));
float ratio_pres = USE_LOG_INTERPOLATION ? p_ratio_log : p_ratio;
for (int index=1; index < mxr8vt; index++) {
if (is_eq(cur_pqtzuv[index], bad_data_float)) {
if (is_bad_data(cur_pqtzuv[index])) {
float prev_value = prev_pqtzuv[index];
float next_value = next_pqtzuv[index];
if (is_eq(prev_value, bad_data_float) || is_eq(next_value, bad_data_float)) {
if (is_bad_data(prev_value) || is_bad_data(next_value)) {
if ((!IGNORE_Q_PBL && index==1) || (!IGNORE_Z_PBL && index==3)) {
mlog << Warning << method_name << " Missing value for "
<< (is_eq(prev_value, bad_data_float) ? "previous" : "next")
<< (is_bad_data(prev_value) ? "previous" : "next")
<< " data for index " << index << " at pressure "
<< (is_eq(prev_value, bad_data_float) ? prev_pqtzuv[0] : next_pqtzuv[0])
<< (is_bad_data(prev_value) ? prev_pqtzuv[0] : next_pqtzuv[0])
<< "\n";
}
}
Expand All @@ -3224,6 +3224,12 @@ void interpolate_pqtzuv(float *prev_pqtzuv, float *cur_pqtzuv, float *next_pqtzu

////////////////////////////////////////////////////////////////////////

static bool is_valid_pb_data(float pb_value) {
return (!is_bad_data(pb_value) && pb_value < r8bfms);
}

////////////////////////////////////////////////////////////////////////

void merge_records(float *first_pqtzuv, map<float, float*> pqtzuv_map_pivot,
map<float, float*> pqtzuv_map_aux, map<float, float*> &pqtzuv_map_merged) {

Expand Down
22 changes: 11 additions & 11 deletions met/src/tools/other/point2grid/point2grid.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -787,8 +787,8 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo,
requested_valid_time = valid_time;
if (0 < valid_time) {
valid_beg_ut = valid_end_ut = valid_time;
if (!is_eq(bad_data_int, conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds;
if (!is_eq(bad_data_int, conf_info.end_ds)) valid_end_ut += conf_info.end_ds;
if (!is_bad_data(conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds;
if (!is_bad_data(conf_info.end_ds)) valid_end_ut += conf_info.end_ds;
for(idx=0; idx<hdr_valid_times.n(); idx++) {
obs_time = timestring_to_unix(hdr_valid_times[idx].c_str());
if (valid_beg_ut <= obs_time && obs_time <= valid_end_ut) {
Expand Down Expand Up @@ -906,7 +906,7 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo,
for (int dIdx=0; dIdx<cellArray.n(); dIdx++) {
from_index = cellArray[dIdx];
data_value = obs_data.get_obs_val(from_index);
if (is_eq(data_value, bad_data_float)) continue;
if (is_bad_data(data_value)) continue;

if(mlog.verbosity_level() >= 4) {
if (from_min_value > data_value) from_min_value = data_value;
Expand Down Expand Up @@ -1134,8 +1134,8 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config,
unixtime ref_ut = (unixtime) 0;
unixtime tmp_time;
if( conf_info.valid_time > 0 ) {
if (!is_eq(bad_data_int, conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds;
if (!is_eq(bad_data_int, conf_info.end_ds)) valid_end_ut += conf_info.end_ds;
if (!is_bad_data(conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds;
if (!is_bad_data(conf_info.end_ds)) valid_end_ut += conf_info.end_ds;
ref_ut = get_reference_unixtime(&time_var, sec_per_unit, no_leap_year);
}
for (int i=0; i<from_size; i++) {
Expand Down Expand Up @@ -1212,7 +1212,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config,
for (int x=0; x<nx; x++) {
for (int y=0; y<ny; y++) {
float value = to_dp.get(x, y);
if (!is_eq(value, bad_data_float) &&
if (!is_bad_data(value) &&
((has_prob_thresh && prob_cat_thresh.check(value))
|| (do_gaussian_filter && !has_prob_thresh))) {
prob_dp.set(1, x, y);
Expand Down Expand Up @@ -1319,7 +1319,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf,
for (int dIdx=0; dIdx<cellArray.n(); dIdx++) {
from_index = cellArray[dIdx];
data_value = from_data[from_index];
if (is_eq(data_value, bad_data_float)) {
if (is_bad_data(data_value)) {
missing_cnt++;
continue;
}
Expand Down Expand Up @@ -1631,7 +1631,7 @@ void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo,
for (int x=0; x<nx; x++) {
for (int y=0; y<ny; y++) {
float value = to_dp.get(x, y);
if (!is_eq(value, bad_data_float) &&
if (!is_bad_data(value) &&
((has_prob_thresh && prob_cat_thresh.check(value))
|| (do_gaussian_filter && !has_prob_thresh))) {
prob_dp.set(1, x, y);
Expand Down Expand Up @@ -1877,7 +1877,7 @@ static void get_grid_mapping_latlon(
// Build cell mapping
for (int xIdx=0; xIdx<data_size; xIdx++) {
to_offset = mapping_indices[xIdx];
if( to_offset == bad_data_int ) continue;
if( is_bad_data(to_offset) ) continue;
if( to_offset < 0 || to_offset >= to_size ) {
mlog << Error << "\n" << method_name
<< "the mapped cell is out of range: "
Expand Down Expand Up @@ -1978,7 +1978,7 @@ static unixtime find_valid_time(NcVar time_var) {
}
}

if (valid_time == bad_data_int) {
if (is_bad_data(valid_time)) {
mlog << Error << "\n" << method_name
<< "trouble finding time variable from \""
<< InputFilename << "\"\n\n";
Expand Down Expand Up @@ -2401,7 +2401,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo,
for (int dIdx=0; dIdx<cellArray.n(); dIdx++) {
from_index = cellArray[dIdx];
data_value = from_data[from_index];
if (is_eq(data_value, bad_data_float)) {
if (is_bad_data(data_value)) {
missing_count++;
continue;
}
Expand Down

0 comments on commit c104425

Please sign in to comment.