Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature 1913 MAX_PBL #1916

Merged
merged 3 commits into from
Sep 16, 2021
Merged

Feature 1913 MAX_PBL #1916

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dca0b25
#1913 Changed MAX_PBL to 10000. Filter out if the PBL input data is i…
Sep 14, 2021
c0f7b49
#1912 Replaced is_eq to is_bad_data
Sep 15, 2021
f5cf2e7
#1913 Changed is_eq to is_bad_data
Sep 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 51 additions & 45 deletions met/src/tools/other/pb2nc/pb2nc.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ static float static_dummy_200[MAX_CAPE_LEVEL];
static float static_dummy_201[MAX_CAPE_LEVEL+1];

#define ROG 287.04
#define MAX_PBL 5000
#define MAX_PBL 10000
#define MAX_PBL_LEVEL 256
#define PBL_DEBUG_LEVEL 8
static bool IGNORE_Q_PBL = true;
Expand Down Expand Up @@ -375,6 +375,7 @@ static void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_co
const ConcatString &hdr_typ, const ConcatString &hdr_sid);
static int interpolate_by_pressure(int length, float *pres_data, float *var_data);
static void interpolate_pqtzuv(float*, float*, float*);
static bool is_valid_pb_data(float pb_value);
static void log_merged_tqz_uv(map<float, float*> pqtzuv_map_tq,
map<float, float*> pqtzuv_map_uv,
map<float, float*> &pqtzuv_map_merged,
Expand Down Expand Up @@ -1556,35 +1557,36 @@ void process_pbfile(int i_pb) {
if (cal_cape) {
if (cape_member_cnt >= 3) cape_level++;
}
if (do_pbl && !is_eq(pqtzuv[0], bad_data_float)) {

// Allocated memory is deleted after all observations are processed
float *tmp_pqtzuv = new float [mxr8vt];

for(kk=0; kk<mxr8vt; kk++) tmp_pqtzuv[kk] = pqtzuv[kk];

bool has_tq = false;
bool has_uv = false;
if (!is_eq(tmp_pqtzuv[4],bad_data_float) &&
!is_eq(tmp_pqtzuv[5],bad_data_float)) {
has_uv = true;
pqtzuv_map_uv[pqtzuv[0]] = tmp_pqtzuv;
}
if (!is_eq(tmp_pqtzuv[2],bad_data_float) &&
(!is_eq(tmp_pqtzuv[1],bad_data_float) || IGNORE_Q_PBL) &&
(!is_eq(tmp_pqtzuv[3],bad_data_float) || IGNORE_Z_PBL)) {
has_tq = true;
pqtzuv_map_tq[pqtzuv[0]] = tmp_pqtzuv;
if (do_pbl) {
bool excluded = true;
// Excludes a record if the pressure value is not valid
if (is_valid_pb_data(pqtzuv[0]) && pqtzuv[0] > 0) {
bool has_uv = is_valid_pb_data(pqtzuv[4]) && is_valid_pb_data(pqtzuv[5]);
bool has_tq = is_valid_pb_data(pqtzuv[2]) &&
(IGNORE_Q_PBL || is_valid_pb_data(pqtzuv[1])) &&
(IGNORE_Z_PBL || is_valid_pb_data(pqtzuv[3]));
if (has_tq || has_uv) {
// Allocated memory is deleted after all observations are processed
float *tmp_pqtzuv = new float [mxr8vt];

for(kk=0; kk<mxr8vt; kk++) tmp_pqtzuv[kk] = pqtzuv[kk];

if (has_uv) pqtzuv_map_uv[pqtzuv[0]] = tmp_pqtzuv;
if (has_tq) pqtzuv_map_tq[pqtzuv[0]] = tmp_pqtzuv;
pqtzuv_list.push_back(tmp_pqtzuv);
excluded = false;
}
}
if (!(has_tq || has_uv)) {
mlog << Debug(5) << method_name << " PBL: excluded " << lv
<< "-th level record:" << " T=" << tmp_pqtzuv[2]
<< ", U=" << tmp_pqtzuv[4] << ", V=" << tmp_pqtzuv[5]
<< ", Q=" << tmp_pqtzuv[0] << ", Z=" << tmp_pqtzuv[3]
if (excluded) {
mlog << Debug(5) << method_name
<< " PBL: excluded " << lv << "-th level record:"
<< " P=" << pqtzuv[0] << " T=" << pqtzuv[2]
<< ", U=" << pqtzuv[4] << ", V=" << pqtzuv[5]
<< ", Q=" << pqtzuv[1] << ", Z=" << pqtzuv[3]
<< " valid_time: " << unix_to_yyyymmdd_hhmmss(hdr_vld_ut)
<< " " << hdr_typ << " " << hdr_sid << "\n";
}
pqtzuv_list.push_back(tmp_pqtzuv);
}
} // end for lv

Expand Down Expand Up @@ -2464,7 +2466,7 @@ void addObservation(const float *obs_arr, const ConcatString &hdr_typ,
// Write the quality flag to the netCDF file
ConcatString obs_qty;
int quality_code = nint(quality_mark);
if (quality_code == bad_data_int || quality_mark > r8bfms) {
if (is_bad_data(quality_code) || quality_mark > r8bfms) {
obs_qty.add("NA");
}
else {
Expand Down Expand Up @@ -2820,7 +2822,7 @@ void display_bufr_variables(const StringArray &all_vars, const StringArray &all_
void copy_pqtzuv(float *to_pqtzuv, float *from_pqtzuv, bool copy_all) {
int start_idx = (copy_all ? 0 : 1);
for (int index = start_idx; index < mxr8vt; index++) {
if (copy_all || !is_eq(from_pqtzuv[index], bad_data_float))
if (copy_all || !is_bad_data(from_pqtzuv[index]))
to_pqtzuv[index] = from_pqtzuv[index];
}
}
Expand Down Expand Up @@ -2994,8 +2996,8 @@ float compute_pbl(map<float, float*> pqtzuv_map_tq,
pbl_data_hgt[index] = pqtzuv[3];
pbl_data_ugrd[index] = pqtzuv[4];
pbl_data_vgrd[index] = pqtzuv[5];
if (!is_eq(pbl_data_spfh[index], bad_data_float)) spfh_cnt++;
if (!is_eq(pbl_data_hgt[index], bad_data_float)) hgt_cnt++;
if (is_valid_pb_data(pbl_data_spfh[index])) spfh_cnt++;
if (is_valid_pb_data(pbl_data_hgt[index])) hgt_cnt++;
selected_levels.add(nint(it->first));
}

Expand All @@ -3015,7 +3017,7 @@ float compute_pbl(map<float, float*> pqtzuv_map_tq,
break;
}
}
if (!is_eq(highest_pressure, bad_data_float)) {
if (!is_bad_data(highest_pressure)) {
index = MAX_PBL_LEVEL - 1;
for (; it!=pqtzuv_map_tq.end(); ++it) {
int pres_level = nint(it->first);
Expand Down Expand Up @@ -3067,7 +3069,7 @@ float compute_pbl(map<float, float*> pqtzuv_map_tq,
//SUBROUTINE CALPBL(T,Q,P,Z,U,V,MZBL,HPBL,jpbl)
calpbl_(pbl_data_temp, pbl_data_spfh, pbl_data_pres, pbl_data_hgt,
pbl_data_ugrd, pbl_data_vgrd, &mzbl, &hpbl, &jpbl);
if (is_eq(hpbl, bad_data_float))
if (!is_valid_pb_data(hpbl))
mlog << Debug(5) << method_name << " fail to compute PBL. TQ records: "
<< tq_count << " UV records: " << uv_count << " merged records: "
<< pqtzuv_map_merged.size() << "\n";
Expand All @@ -3087,8 +3089,8 @@ void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code,
ConcatString hdr_info;
hdr_info << unix_to_yyyymmdd_hhmmss(hdr_vld_ut)
<< " " << hdr_typ << " " << hdr_sid;
if (is_eq(pbl_value, bad_data_float)) {
mlog << Warning << "\nFailed to compute PBL " << hdr_info << "\n\n";
if (is_bad_data(pbl_value)) {
mlog << Warning << "\nFailed to compute PBL " << pbl_value << " (" << hdr_info << ")\n\n";
}
else if (pbl_value < hdr_elv) {
mlog << Warning << "\nNot saved because the computed PBL (" << pbl_value
Expand All @@ -3105,14 +3107,12 @@ void insert_pbl(float *obs_arr, const float pbl_value, const int pbl_code,
<< " lat: " << hdr_lat << ", lon: " << hdr_lon
<< ", elv: " << hdr_elv << " " << hdr_info << "\n\n";
if (obs_arr[4] > MAX_PBL) {
mlog << Warning << "\nComputed PBL (" << obs_arr[4] << " from "
<< pbl_value << ") is too high, Reset to " << MAX_PBL
<< " " << hdr_info<< "\n\n";
obs_arr[4] = MAX_PBL;
mlog << Warning << "\nNot saved the computed PBL (" << obs_arr[4] << " from "
<< pbl_value << ") because of the MAX PBL " << MAX_PBL
<< " (" << hdr_info<< ")\n\n";
}

addObservation(obs_arr, (string)hdr_typ, (string)hdr_sid, hdr_vld_ut,
hdr_lat, hdr_lon, hdr_elv, pbl_qm, OBS_BUFFER_SIZE);
else addObservation(obs_arr, (string)hdr_typ, (string)hdr_sid, hdr_vld_ut,
hdr_lat, hdr_lon, hdr_elv, pbl_qm, OBS_BUFFER_SIZE);
}
}

Expand All @@ -3129,7 +3129,7 @@ int interpolate_by_pressure(int length, float *pres_data, float *var_data) {
skip_missing = false;
count_interpolated = 0;
for (idx=0; idx<length; idx++) {
if (is_eq(var_data[idx], bad_data_float)) {
if (is_bad_data(var_data[idx])) {
skip_missing = true;
}
else {
Expand Down Expand Up @@ -3187,15 +3187,15 @@ void interpolate_pqtzuv(float *prev_pqtzuv, float *cur_pqtzuv, float *next_pqtzu
/ (log(next_pqtzuv[0]) - log(prev_pqtzuv[0]));
float ratio_pres = USE_LOG_INTERPOLATION ? p_ratio_log : p_ratio;
for (int index=1; index < mxr8vt; index++) {
if (is_eq(cur_pqtzuv[index], bad_data_float)) {
if (is_bad_data(cur_pqtzuv[index])) {
float prev_value = prev_pqtzuv[index];
float next_value = next_pqtzuv[index];
if (is_eq(prev_value, bad_data_float) || is_eq(next_value, bad_data_float)) {
if (is_bad_data(prev_value) || is_bad_data(next_value)) {
if ((!IGNORE_Q_PBL && index==1) || (!IGNORE_Z_PBL && index==3)) {
mlog << Warning << method_name << " Missing value for "
<< (is_eq(prev_value, bad_data_float) ? "previous" : "next")
<< (is_bad_data(prev_value) ? "previous" : "next")
<< " data for index " << index << " at pressure "
<< (is_eq(prev_value, bad_data_float) ? prev_pqtzuv[0] : next_pqtzuv[0])
<< (is_bad_data(prev_value) ? prev_pqtzuv[0] : next_pqtzuv[0])
<< "\n";
}
}
Expand All @@ -3211,6 +3211,12 @@ void interpolate_pqtzuv(float *prev_pqtzuv, float *cur_pqtzuv, float *next_pqtzu

////////////////////////////////////////////////////////////////////////

static bool is_valid_pb_data(float pb_value) {
return (!is_bad_data(pb_value) && pb_value < r8bfms);
}

////////////////////////////////////////////////////////////////////////

void merge_records(float *first_pqtzuv, map<float, float*> pqtzuv_map_pivot,
map<float, float*> pqtzuv_map_aux, map<float, float*> &pqtzuv_map_merged) {

Expand Down
22 changes: 11 additions & 11 deletions met/src/tools/other/point2grid/point2grid.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -787,8 +787,8 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo,
requested_valid_time = valid_time;
if (0 < valid_time) {
valid_beg_ut = valid_end_ut = valid_time;
if (!is_eq(bad_data_int, conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds;
if (!is_eq(bad_data_int, conf_info.end_ds)) valid_end_ut += conf_info.end_ds;
if (!is_bad_data(conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds;
if (!is_bad_data(conf_info.end_ds)) valid_end_ut += conf_info.end_ds;
for(idx=0; idx<hdr_valid_times.n(); idx++) {
obs_time = timestring_to_unix(hdr_valid_times[idx].c_str());
if (valid_beg_ut <= obs_time && obs_time <= valid_end_ut) {
Expand Down Expand Up @@ -906,7 +906,7 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo,
for (int dIdx=0; dIdx<cellArray.n(); dIdx++) {
from_index = cellArray[dIdx];
data_value = obs_data.get_obs_val(from_index);
if (is_eq(data_value, bad_data_float)) continue;
if (is_bad_data(data_value)) continue;

if(mlog.verbosity_level() >= 4) {
if (from_min_value > data_value) from_min_value = data_value;
Expand Down Expand Up @@ -1134,8 +1134,8 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config,
unixtime ref_ut = (unixtime) 0;
unixtime tmp_time;
if( conf_info.valid_time > 0 ) {
if (!is_eq(bad_data_int, conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds;
if (!is_eq(bad_data_int, conf_info.end_ds)) valid_end_ut += conf_info.end_ds;
if (!is_bad_data(conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds;
if (!is_bad_data(conf_info.end_ds)) valid_end_ut += conf_info.end_ds;
ref_ut = get_reference_unixtime(&time_var, sec_per_unit, no_leap_year);
}
for (int i=0; i<from_size; i++) {
Expand Down Expand Up @@ -1212,7 +1212,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config,
for (int x=0; x<nx; x++) {
for (int y=0; y<ny; y++) {
float value = to_dp.get(x, y);
if (!is_eq(value, bad_data_float) &&
if (!is_bad_data(value) &&
((has_prob_thresh && prob_cat_thresh.check(value))
|| (do_gaussian_filter && !has_prob_thresh))) {
prob_dp.set(1, x, y);
Expand Down Expand Up @@ -1319,7 +1319,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf,
for (int dIdx=0; dIdx<cellArray.n(); dIdx++) {
from_index = cellArray[dIdx];
data_value = from_data[from_index];
if (is_eq(data_value, bad_data_float)) {
if (is_bad_data(data_value)) {
missing_cnt++;
continue;
}
Expand Down Expand Up @@ -1631,7 +1631,7 @@ void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo,
for (int x=0; x<nx; x++) {
for (int y=0; y<ny; y++) {
float value = to_dp.get(x, y);
if (!is_eq(value, bad_data_float) &&
if (!is_bad_data(value) &&
((has_prob_thresh && prob_cat_thresh.check(value))
|| (do_gaussian_filter && !has_prob_thresh))) {
prob_dp.set(1, x, y);
Expand Down Expand Up @@ -1877,7 +1877,7 @@ static void get_grid_mapping_latlon(
// Build cell mapping
for (int xIdx=0; xIdx<data_size; xIdx++) {
to_offset = mapping_indices[xIdx];
if( to_offset == bad_data_int ) continue;
if( is_bad_data(to_offset) ) continue;
if( to_offset < 0 || to_offset >= to_size ) {
mlog << Error << "\n" << method_name
<< "the mapped cell is out of range: "
Expand Down Expand Up @@ -1978,7 +1978,7 @@ static unixtime find_valid_time(NcVar time_var) {
}
}

if (valid_time == bad_data_int) {
if (is_bad_data(valid_time)) {
mlog << Error << "\n" << method_name
<< "trouble finding time variable from \""
<< InputFilename << "\"\n\n";
Expand Down Expand Up @@ -2401,7 +2401,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo,
for (int dIdx=0; dIdx<cellArray.n(); dIdx++) {
from_index = cellArray[dIdx];
data_value = from_data[from_index];
if (is_eq(data_value, bad_data_float)) {
if (is_bad_data(data_value)) {
missing_count++;
continue;
}
Expand Down