Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature 2697 pb2nc temp file #2792

Merged
merged 21 commits into from
Jan 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 2 additions & 8 deletions docs/Contributors_Guide/dev_details/tmp_file_use.rst
Original file line number Diff line number Diff line change
Expand Up @@ -39,23 +39,17 @@ those observations, and writes the result to a NetCDF output file.

PB2NC creates the following temporary files when running:

* :code:`tmp_pb2nc_blk_{PID}`, :code:`tmp_pb2nc_meta_blk_{PID}`,
:code:`tmp_pb2nc_tbl_blk_{PID}`
* :code:`tmp_pb2nc_blk_{PID}`

PB2NC assumes that each input binary file requires Fortran
blocking prior to being read by the BUFRLIB library. It applies
Fortran blocking, writes the result to this temporary file, and
uses BUFRLIB to read its contents.

* :code:`tmp_pb2nc_bufr_{PID}_tbl`: PB2NC extracts Bufr table data
* :code:`tmp_pb2nc_blk_{PID}_tbl`: PB2NC extracts Bufr table data
that is embedded in input files and writes it to this temporary
file for later use.

.. note::
The first 3 files listed above are identical. They are all
Fortran-blocked versions of the same input file. Recommend
modifying the logic to only apply Fortran blocking once.

.. _tmp_files_point2grid:

Point2Grid Tool
Expand Down
2 changes: 2 additions & 0 deletions src/basic/vx_cal/time_array.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

#include <iostream>

typedef long long unixtime;


////////////////////////////////////////////////////////////////////////

Expand Down
3 changes: 0 additions & 3 deletions src/basic/vx_cal/vx_cal.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@


#include "concat_string.h"

typedef long long unixtime;

#include "time_array.h"


Expand Down
2 changes: 1 addition & 1 deletion src/libcode/vx_data2d_nc_cf/data2d_nc_cf.cc
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ MetNcCFDataFile & MetNcCFDataFile::operator=(const MetNcCFDataFile &) {

void MetNcCFDataFile::nccf_init_from_scratch() {

_file = (NcCfFile *) 0;
_file = (NcCfFile *) nullptr;
_cur_time_index = -1;

close();
Expand Down
41 changes: 16 additions & 25 deletions src/libcode/vx_data2d_nc_cf/nc_cf_file.cc
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
////////////////////////////////////////////////////////////////////////


using namespace std;

#include <iostream>
#include <unistd.h>
#include <stdlib.h>
Expand All @@ -23,14 +21,16 @@ using namespace std;
#include <time.h>

#include <netcdf>
using namespace netCDF;

#include "vx_math.h"
#include "vx_cal.h"
#include "vx_log.h"

#include "nc_cf_file.h"

using namespace std;
using namespace netCDF;


////////////////////////////////////////////////////////////////////////

Expand Down Expand Up @@ -221,7 +221,6 @@ bool NcCfFile::open(const char * filepath)
Nvars = get_var_names(_ncFile, &varNames);
Var = new NcVarInfo [Nvars];

NcDim dim;
for (int j=0; j<Nvars; ++j) {
NcVar v = get_var(_ncFile, varNames[j].c_str());

Expand Down Expand Up @@ -360,7 +359,6 @@ bool NcCfFile::open(const char * filepath)
else {
if (use_bounds_var) {
double bounds_diff;
double time_fraction;
for(int i=0; i<n_times; i++) {
ValidTime.add(add_to_unixtime(ut, sec_per_unit, time_values[i*2+1], no_leap_year));
raw_times.add(time_values[i*2+1]);
Expand Down Expand Up @@ -909,7 +907,7 @@ double NcCfFile::getData(NcVar * var, const LongArray & a) const
double d = bad_data_double;

double fill_value;
double missing_value = get_var_missing_value(var);
//double missing_value = get_var_missing_value(var);
get_var_fill_value(var, fill_value);

status = get_nc_data(var, &d, a);
Expand Down Expand Up @@ -1052,9 +1050,6 @@ bool NcCfFile::getData(NcVar * v, const LongArray & a, DataPlane & plane) const
}

// get the data
int *i;
short *s;
float *f;
const int plane_size = nx * ny;
double *d = new double[plane_size];

Expand Down Expand Up @@ -1328,22 +1323,20 @@ NcVarInfo* NcCfFile::find_var_by_dim_name(const char *dim_name) const
NcVar *NcCfFile::find_var_by_standard_name(const char *standard_name) const
{
NcVar *var = nullptr;
if (!var) {
ConcatString att_value;
for (int i=0; i<Nvars; i++) {
if (get_var_standard_name(Var[i].var, att_value)) {
if (att_value == standard_name) {
var = Var[i].var;
break;
}
ConcatString att_value;
for (int i=0; i<Nvars; i++) {
if (get_var_standard_name(Var[i].var, att_value)) {
if (att_value == standard_name) {
var = Var[i].var;
break;
}
//if (get_var_long_name(Var[i].var, att_value) {
// if (att_value == standard_name) {
// var = &Var[i];
// break;
// }
//}
}
//if (get_var_long_name(Var[i].var, att_value) {
// if (att_value == standard_name) {
// var = &Var[i];
// break;
// }
//}
}

return var;
Expand Down Expand Up @@ -2294,7 +2287,6 @@ void NcCfFile::get_grid_mapping_polar_stereographic(const NcVar *grid_mapping_va
// units conversions.

ConcatString x_coord_units_name;
const NcVarAtt *x_coord_units_att = get_nc_att(_xCoordVar, units_att_name);
if (!get_var_units(_xCoordVar, x_coord_units_name)) {
mlog << Warning << "\n" << method_name
<< "Units not given for X coordinate variable -- assuming meters.\n\n";
Expand All @@ -2320,7 +2312,6 @@ void NcCfFile::get_grid_mapping_polar_stereographic(const NcVar *grid_mapping_va
}

ConcatString y_coord_units_name;
const NcVarAtt *y_coord_units_att = get_nc_att(_yCoordVar, units_att_name);
if (!get_var_units(_yCoordVar, y_coord_units_name)) {
mlog << Warning << "\n" << method_name
<< "Units not given for Y coordinate variable -- assuming meters.\n\n";
Expand Down
8 changes: 4 additions & 4 deletions src/libcode/vx_data2d_nc_cf/var_info_nc_cf.cc
Original file line number Diff line number Diff line change
Expand Up @@ -153,10 +153,6 @@ void VarInfoNcCF::add_dimension(int dim, bool as_offset, double dim_value) {

void VarInfoNcCF::set_magic(const ConcatString &nstr, const ConcatString &lstr) {
ConcatString tmp_str;
char *ptr = 0;
char *ptr2 = 0;
char *ptr3 = 0;
char *save_ptr = 0;
const char *method_name = "VarInfoNcCF::set_magic() -> ";

// Store the magic string
Expand All @@ -175,6 +171,10 @@ void VarInfoNcCF::set_magic(const ConcatString &nstr, const ConcatString &lstr)
add_dimension(vx_data2d_star);
}
else {
char *ptr = nullptr;
char *ptr2 = nullptr;
char *ptr3 = nullptr;
char *save_ptr = nullptr;

// Initialize the temp string
tmp_str = lstr;
Expand Down
11 changes: 3 additions & 8 deletions src/libcode/vx_data2d_ugrid/data2d_ugrid.cc
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,6 @@ bool MetUGridDataFile::data_plane(VarInfo &vinfo, DataPlane &plane)
{
bool status = false;
NcVarInfo *data_var = (NcVarInfo *)nullptr;
const long time_cnt = (long)_file->ValidTime.n();
static const string method_name
= "MetUGridDataFile::data_plane(VarInfo &, DataPlane &) -> ";

Expand Down Expand Up @@ -229,15 +228,13 @@ bool MetUGridDataFile::data_plane(VarInfo &vinfo, DataPlane &plane, NcVarInfo *d

plane.clear();

int zdim_slot = bad_data_int;
int time_dim_slot = bad_data_int;
NumArray dim_value = vinfo_nc->dim_value();
LongArray dimension(vinfo_nc->dimension());
BoolArray is_offset(vinfo_nc->is_offset());

if (nullptr != data_var) {
zdim_slot = data_var->z_slot;
time_dim_slot = data_var->t_slot;
int zdim_slot = data_var->z_slot;
int time_dim_slot = data_var->t_slot;

// Adjust dimension and is_offset
int dim_size = dimension.n_elements();
Expand Down Expand Up @@ -310,7 +307,6 @@ int MetUGridDataFile::data_plane_array(VarInfo &vinfo,
DataPlaneArray &plane_array) {
int n_rec = 0;
DataPlane plane;
bool status = false;
static const string method_name
= "MetUGridDataFile::data_plane_array(VarInfo &, DataPlaneArray &) -> ";

Expand All @@ -320,14 +316,14 @@ int MetUGridDataFile::data_plane_array(VarInfo &vinfo,
LevelInfo level = vinfo.level();
long lvl_lower = level.lower();
long lvl_upper = level.upper();
const int debug_level = 7;
ConcatString req_name = vinfo.req_name();
NcVarInfo *data_vinfo = _file->find_by_name(req_name.c_str());
if (level.type() == LevelType_Time) {
mlog << Error << "\n" << method_name
<< "LevelType_Time for unstructured grid is not enabled\n\n";
exit(1);
/* Not enabled
const int debug_level = 7;
LongArray time_offsets = collect_time_offsets(vinfo);
if (0 < time_offsets.n_elements()) {
for (int idx=0; idx<time_offsets.n_elements(); idx++) {
Expand Down Expand Up @@ -382,7 +378,6 @@ int MetUGridDataFile::data_plane_array(VarInfo &vinfo,
if (lvl_lower != lvl_upper && nullptr != data_vinfo) {
int tmp_lower = lvl_lower;
int tmp_upper = lvl_upper;
int z_offset = bad_data_int;
if (data_vinfo->z_slot >= 0) {
int zdim_size = get_dim_size(data_vinfo->var, data_vinfo->z_slot);
if (tmp_lower >= zdim_size) tmp_lower = zdim_size - 1;
Expand Down
19 changes: 7 additions & 12 deletions src/libcode/vx_data2d_ugrid/ugrid_file.cc
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,6 @@ bool UGridFile::open(const char * filepath)
bool UGridFile::open_metadata(const char * filepath)
{
unixtime ut = 0;
int sec_per_unit;
const char *method_name = "UGridFile::open_metadata() -> ";

// Open the file
Expand Down Expand Up @@ -319,6 +318,8 @@ bool UGridFile::open_metadata(const char * filepath)
ValidTime.add(ut);
}
else {
int sec_per_unit;

// Store the dimension for the time variable as the time dimension
ConcatString units;
bool use_bounds_var = false;
Expand Down Expand Up @@ -374,7 +375,6 @@ bool UGridFile::open_metadata(const char * filepath)
else {
if (use_bounds_var) {
double bounds_diff;
double time_fraction;
for(int i=0; i<n_times; i++) {
ValidTime.add(add_to_unixtime(ut, sec_per_unit, time_values[i*2+1], no_leap_year));
raw_times.add(time_values[i*2+1]);
Expand Down Expand Up @@ -419,9 +419,9 @@ bool UGridFile::open_metadata(const char * filepath)
get_dim_names(v, &dimNames);

for (int k=0; k<dim_count; ++k) {
NcDim *dim = Var[j].Dims[k];
NcDim *dim_p = Var[j].Dims[k];
const ConcatString dim_name = dimNames[k];
if ((dim && dim == _tDim) || dim_name == time_dim_name) {
if ((nullptr != dim_p && dim_p == _tDim) || dim_name == time_dim_name) {
Var[j].t_slot = k;
}
else if (dim_name == vert_dim_name) {
Expand Down Expand Up @@ -620,7 +620,6 @@ double UGridFile::getData(NcVar * var, const LongArray & a) const
double d = bad_data_double;

double fill_value;
double missing_value = get_var_missing_value(var);
get_var_fill_value(var, fill_value);

//status = get_nc_data(var, &d, a);
Expand Down Expand Up @@ -729,11 +728,8 @@ bool UGridFile::getData(NcVar * v, const LongArray & a, DataPlane & plane) const

get_nc_data(v, d, lengths, offsets);

int offset = 0;
double min_value, max_value;

min_value = 10e10;
max_value = -min_value;
double min_value = 10e10;
double max_value = -min_value;
for (int x = 0; x< nx; ++x) {
double value = d[x];
if( is_eq(value, missing_value) || is_eq(value, fill_value) ) {
Expand All @@ -748,7 +744,7 @@ bool UGridFile::getData(NcVar * v, const LongArray & a, DataPlane & plane) const

} // for x

delete [] d;
if (nullptr != d) delete [] d;

// done
ConcatString log_message;
Expand Down Expand Up @@ -824,7 +820,6 @@ bool UGridFile::get_var_info() {
}

NcDim dim;
int meta_count = 0;
int max_dim_count = 0;
ConcatString att_value;
StringArray var_names;
Expand Down
8 changes: 4 additions & 4 deletions src/libcode/vx_data2d_ugrid/var_info_ugrid.cc
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,6 @@ void VarInfoUGrid::add_dimension(int dim, bool as_offset, double dim_value) {

void VarInfoUGrid::set_magic(const ConcatString &nstr, const ConcatString &lstr) {
ConcatString tmp_str;
char *ptr = 0;
char *ptr2 = 0;
char *ptr3 = 0;
char *save_ptr = 0;
const char *method_name = "VarInfoUGrid::set_magic() -> ";

// Store the magic string
Expand All @@ -176,6 +172,10 @@ void VarInfoUGrid::set_magic(const ConcatString &nstr, const ConcatString &lstr)
add_dimension(vx_data2d_star);
}
else {
char *ptr = nullptr;
char *ptr2 = nullptr;
char *ptr3 = nullptr;
char *save_ptr = nullptr;

// Initialize the temp string
tmp_str = lstr;
Expand Down
2 changes: 0 additions & 2 deletions src/libcode/vx_grid/unstructured_grid.cc
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,6 @@ return;
ConcatString UnstructuredGrid::serialize(const char *sep) const {

ConcatString a;
char junk[256];


a << "Projection: UnstructuredGrid" << sep;

Expand Down
1 change: 0 additions & 1 deletion src/libcode/vx_nc_obs/nc_obs_util.cc
Original file line number Diff line number Diff line change
Expand Up @@ -545,7 +545,6 @@ void NetcdfObsVars::read_dims_vars(NcFile *f_in) {
////////////////////////////////////////////////////////////////////////

void NetcdfObsVars::read_header_data(MetPointHeader &hdr_data) {
bool is_valid_obs_nc = true;
long nhdr_count = get_dim_size(&hdr_dim);
int strl_len = get_dim_size(&strl_dim);
int strl2_len = strl_len;
Expand Down
4 changes: 1 addition & 3 deletions src/libcode/vx_nc_obs/nc_point_obs_out.cc
Original file line number Diff line number Diff line change
Expand Up @@ -491,7 +491,7 @@ void MetNcPointObsOut::write_obs_data()
<< ", save_raw_data: " << (do_save_raw_data ? "true" : "false")
<< "\n";

if (!do_summary || (do_summary && do_save_raw_data)) {
if (!do_summary || do_save_raw_data) {
mlog << Debug(5) << method_name << "writing "
<< (int)out_data.observations.size() << " raw data...\n";
write_obs_data(out_data.observations, do_header);
Expand All @@ -517,8 +517,6 @@ int MetNcPointObsOut::write_obs_data(const vector< Observation > observations,
const bool do_header)
{
int prev_hdr_idx = -1;
string prev_header_type = "";
string prev_station_id = "";
ConcatString obs_qty;
int headerOffset = data_buffer.cur_hdr_idx;
const string method_name = " write_obs_data()";
Expand Down
Loading