diff --git a/met/src/libcode/vx_nc_obs/nc_obs_util.cc b/met/src/libcode/vx_nc_obs/nc_obs_util.cc index 239d1da84b..0f0b991fdf 100644 --- a/met/src/libcode/vx_nc_obs/nc_obs_util.cc +++ b/met/src/libcode/vx_nc_obs/nc_obs_util.cc @@ -27,8 +27,9 @@ using namespace std; /////////////////////////////////////////////////////////////////////////////// //NcDataBuffer nc_data_buffer; -NcDataBuffer data_buffer; -struct NcHeaderData hdr_data; +//NcDataBuffer data_buffer; +//struct NcHeaderData hdr_data; + float hdr_arr_block[NC_BUFFER_SIZE_32K][HDR_ARRAY_LEN]; /////////////////////////////////////////////////////////////////////////////// @@ -272,99 +273,41 @@ NetcdfObsVars::NetcdfObsVars() /////////////////////////////////////////////////////////////////////////////// -bool NetcdfObsVars::add_header(const char *hdr_typ, const char *hdr_sid, - const time_t hdr_vld, const float hdr_lat, - const float hdr_lon, const float hdr_elv) -{ - bool added = false; - bool new_vld = false; - - // Can't filter duplicated one because header index was - // assigned before checking - int hdr_index; - add_header_strings(hdr_typ, hdr_sid); - - if (hdr_data.min_vld_time == -1 || hdr_data.min_vld_time > hdr_vld) { - if (hdr_data.min_vld_time == -1) hdr_data.max_vld_time = hdr_vld; - hdr_data.min_vld_time = hdr_vld; - new_vld = true; - } - else if (hdr_data.max_vld_time < hdr_vld) { - hdr_data.max_vld_time = hdr_vld; - new_vld = true; - } - if (new_vld || !hdr_data.vld_num_array.has(hdr_vld, hdr_index, false)) { - hdr_index = hdr_data.vld_array.n_elements(); - hdr_data.vld_array.add(unix_to_yyyymmdd_hhmmss(hdr_vld)); // Valid time - hdr_data.vld_num_array.add(hdr_vld); // Valid time - } - hdr_data.vld_idx_array.add(hdr_index); // Index of Valid time - - hdr_data.lat_array.add(hdr_lat); // Latitude - hdr_data.lon_array.add(hdr_lon); // Longitude - hdr_data.elv_array.add(hdr_elv); // Elevation - data_buffer.cur_hdr_idx++; - added = true; - return added; -} - -/////////////////////////////////////////////////////////////////////////////// - -bool NetcdfObsVars::add_header_prepbufr (const int pb_report_type, - const int in_report_type, - const int instrument_type) -{ - bool added = true; - // Can't filter duplicated one because header index was - // assigned before checking - hdr_data.prpt_typ_array.add(pb_report_type); - hdr_data.irpt_typ_array.add(in_report_type); - hdr_data.inst_typ_array.add(instrument_type); - return added; -} - -/////////////////////////////////////////////////////////////////////////////// - -bool NetcdfObsVars::add_header_strings(const char *hdr_typ, const char *hdr_sid) -{ - bool added = false; - - // Can't filter duplicated one because header index was - // assigned before checking - int hdr_index; - if (!hdr_data.typ_array.has(hdr_typ, hdr_index, false)) { - hdr_index = hdr_data.typ_array.n_elements(); - hdr_data.typ_array.add(hdr_typ); // Message type - added = true; - } - hdr_data.typ_idx_array.add(hdr_index); // Index of Message type - - if (!hdr_data.sid_array.has(hdr_sid, hdr_index, false)) { - hdr_index = hdr_data.sid_array.n_elements(); - hdr_data.sid_array.add(hdr_sid); // Station ID - added = true; - } - hdr_data.sid_idx_array.add(hdr_index); // Index of Station ID - - return added; -} - -/////////////////////////////////////////////////////////////////////////////// - -bool NetcdfObsVars::add_header_vld(const char *hdr_vld) -{ - bool added = false; - - // Can't filter duplicated one because header index was - // assigned before checking - int hdr_index; - if (!hdr_data.vld_array.has(hdr_vld, hdr_index, false)) { - hdr_index = hdr_data.typ_array.n_elements(); - hdr_data.vld_array.add(hdr_vld); // Valid time - added = true; - } - return added; -} +//bool NetcdfObsVars::add_header(const char *hdr_typ, const char *hdr_sid, +// const time_t hdr_vld, const float hdr_lat, +// const float hdr_lon, const float hdr_elv) +//{ +// bool added = false; +//// bool new_vld = false; +// +// // Can't filter duplicated one because header index was +// // assigned before checking +// int hdr_index; +// add_header_strings(hdr_typ, hdr_sid); +// +// if (hdr_data.min_vld_time == -1 || hdr_data.min_vld_time > hdr_vld) { +// if (hdr_data.min_vld_time == -1) hdr_data.max_vld_time = hdr_vld; +// hdr_data.min_vld_time = hdr_vld; +// new_vld = true; +// } +// else if (hdr_data.max_vld_time < hdr_vld) { +// hdr_data.max_vld_time = hdr_vld; +// new_vld = true; +// } +// if (new_vld || !hdr_data.vld_num_array.has(hdr_vld, hdr_index, false)) { +// hdr_index = hdr_data.vld_array.n_elements(); +// hdr_data.vld_array.add(unix_to_yyyymmdd_hhmmss(hdr_vld)); // Valid time +// hdr_data.vld_num_array.add(hdr_vld); // Valid time +// } +// hdr_data.vld_idx_array.add(hdr_index); // Index of Valid time +// +// hdr_data.lat_array.add(hdr_lat); // Latitude +// hdr_data.lon_array.add(hdr_lon); // Longitude +// hdr_data.elv_array.add(hdr_elv); // Elevation +// data_buffer.cur_hdr_idx++; +// added = true; +// return added; +//} /////////////////////////////////////////////////////////////////////////////// @@ -519,7 +462,8 @@ void NetcdfObsVars::create_pb_hdrs (NcFile *f_out, const int hdr_count) { /////////////////////////////////////////////////////////////////////////////// -void NetcdfObsVars::create_table_vars (NcFile *f_out) { +void NetcdfObsVars::create_table_vars (NcFile *f_out, NcHeaderData &hdr_data, + NcDataBuffer &data_buffer) { const string method_name = " create_table_vars()"; // Define netCDF dimensions @@ -556,43 +500,12 @@ NcDim NetcdfObsVars::create_var_obs_var (NcFile *f_out, int var_count) { //////////////////////////////////////////////////////////////////////// -int NetcdfObsVars::get_hdr_index() { - //return (nc_data_buffer.cur_hdr_idx < 0) ? 0 : nc_data_buffer.cur_hdr_idx; - return data_buffer.cur_hdr_idx; -} - -//////////////////////////////////////////////////////////////////////// - -int NetcdfObsVars::get_obs_index() { - return data_buffer.obs_data_idx; -} - -//////////////////////////////////////////////////////////////////////// - //NcHeaderData *NetcdfObsVars::get_header_data() { // return &hdr_data; //} /////////////////////////////////////////////////////////////////////////////// -void NetcdfObsVars::init_data_buffer() { - data_buffer.obs_data_idx = 0; - data_buffer.obs_data_offset = 0; - data_buffer.hdr_data_idx = 0; - data_buffer.hdr_data_offset = 0; - data_buffer.pb_hdr_data_offset = 0; - - strcpy(data_buffer.prev_hdr_typ_buf, "NotDefined"); - strcpy(data_buffer.prev_hdr_sid_buf, "NotDefined"); - strcpy(data_buffer.prev_hdr_vld_buf, "NotDefined"); - for (int index=0; index 0) obs_cnt = 0; // obs. array length (fixed dimension if obs_cnt > 0) //hdr_str_len = 0; // string length for header (message) type header - hdr_data.clear(); -} - -/////////////////////////////////////////////////////////////////////////////// - -void NetcdfObsVars::reset_header_buffer(int buf_size, bool reset_all) { - for (int i=0; i buf_size) pb_hdr_len = buf_size; - - lengths[0] = pb_hdr_len; - if(!put_nc_data((NcVar *)&hdr_prpt_typ_var, data_buf.hdr_prpt_typ_buf, lengths, offsets)) { - mlog << Error << "error writing the pb message type to the netCDF file\n\n"; - exit(1); + if (is_prepbufr) { + if (0 == raw_hdr_cnt) { + mlog << Debug(6) << method_name + << "No header for PREPBUFR report/instrument\n"; } - if(!put_nc_data((NcVar *)&hdr_irpt_typ_var, data_buf.hdr_irpt_typ_buf, lengths, offsets)) { - mlog << Error << "error writing the in message type to the netCDF file\n\n"; - exit(1); + else if (data_buf.hdr_data_offset == data_buf.pb_hdr_data_offset) { + int save_len = lengths[0]; + int pb_hdr_len = raw_hdr_cnt - offsets[0]; + if (pb_hdr_len > buf_size) pb_hdr_len = buf_size; + + lengths[0] = pb_hdr_len; + if(IS_VALID_NC(hdr_prpt_typ_var) && !put_nc_data((NcVar *)&hdr_prpt_typ_var, + data_buf.hdr_prpt_typ_buf, lengths, offsets)) { + mlog << Error << "error writing the pb message type to the netCDF file\n\n"; + exit(1); + } + if(IS_VALID_NC(hdr_irpt_typ_var) && !put_nc_data((NcVar *)&hdr_irpt_typ_var, + data_buf.hdr_irpt_typ_buf, lengths, offsets)) { + mlog << Error << "error writing the in message type to the netCDF file\n\n"; + exit(1); + } + if(IS_VALID_NC(hdr_inst_typ_var) && !put_nc_data((NcVar *)&hdr_inst_typ_var, + data_buf.hdr_inst_typ_buf, lengths, offsets)) { + mlog << Error << "error writing the instrument type to the netCDF file\n\n"; + exit(1); + } + lengths[0] = save_len; + data_buf.pb_hdr_data_offset += pb_hdr_len; } - if(!put_nc_data((NcVar *)&hdr_inst_typ_var, data_buf.hdr_inst_typ_buf, lengths, offsets)) { - mlog << Error << "error writing the instrument type to the netCDF file\n\n"; - exit(1); + else { + mlog << Debug(6) << method_name + << "Does not match header offsets for report/instrument: " << data_buf.hdr_data_offset + << " : " << data_buf.pb_hdr_data_offset << "\n"; } - lengths[0] = save_len; - data_buf.pb_hdr_data_offset += pb_hdr_len; } data_buf.hdr_data_offset += buf_size; @@ -1291,174 +1208,9 @@ void NetcdfObsVars::write_header_to_nc(NcDataBuffer &data_buf, const int buf_siz /////////////////////////////////////////////////////////////////////////////// -// Saves the headers at NcHeaderData hdr_data -// -void NetcdfObsVars::write_arr_headers() { - int cur_hdr_idx = data_buffer.cur_hdr_idx; - int buf_size = (cur_hdr_idx > OBS_BUFFER_SIZE) ? OBS_BUFFER_SIZE : cur_hdr_idx; - const string method_name = " write_arr_headers()"; - - mlog << Debug(5) << method_name << " hdr_count: " << cur_hdr_idx - << ", typ_idx_array: " << hdr_data.typ_idx_array.n_elements() - << ", sid_idx_array: " << hdr_data.sid_idx_array.n_elements() - << ", vld_idx_array: " << hdr_data.vld_idx_array.n_elements() - << ", lat_array: " << hdr_data.lat_array.n_elements() - << ", lon_array: " << hdr_data.lon_array.n_elements() - << ", elv_array: " << hdr_data.elv_array.n_elements() - << "\n"; - - int hdr_data_idx = 0; - bool is_pb_hdr = (0 < hdr_data.prpt_typ_array.n_elements()) - && !IS_INVALID_NC(hdr_prpt_typ_var); - // FIXME nc_data_buffer.obs_vars = obs_vars; - data_buffer.hdr_buf_size = buf_size; - data_buffer.hdr_data_idx = hdr_data_idx; - for (int index=0; index= buf_size) { - write_header_to_nc(data_buffer, hdr_data_idx, is_pb_hdr); - hdr_data_idx = data_buffer.hdr_data_idx; - } - } - - write_buf_headers(); -} - -/////////////////////////////////////////////////////////////////////////////// -//FIXME to remove - -void NetcdfObsVars::write_buf_headers () { - if (0 < data_buffer.hdr_data_idx) { - write_header_to_nc(data_buffer, data_buffer.hdr_data_idx); - } - write_table_vars(); -} - -/////////////////////////////////////////////////////////////////////////////// - -void NetcdfObsVars::write_header (const char *hdr_typ, const char *hdr_sid, const time_t hdr_vld, - const float hdr_lat, const float hdr_lon, const float hdr_elv) -{ - // Can't filter duplicated one because header index was - // assigned before checking - int hdr_index; - bool new_vld = false; - int hdr_data_idx = data_buffer.hdr_data_idx; - - // Message type - if (!hdr_data.typ_array.has(hdr_typ, hdr_index, false)) { - hdr_index = hdr_data.typ_array.n_elements(); - hdr_data.typ_array.add(hdr_typ); - } - data_buffer.hdr_typ_buf[hdr_data_idx] = hdr_index; - - // Station ID - if (!hdr_data.sid_array.has(hdr_sid, hdr_index, false)) { - hdr_index = hdr_data.sid_array.n_elements(); - hdr_data.sid_array.add(hdr_sid); - } - data_buffer.hdr_sid_buf[hdr_data_idx] = hdr_index; - - // Valid Time - if (hdr_data.min_vld_time == -1 || hdr_data.min_vld_time > hdr_vld) { - if (hdr_data.min_vld_time == -1) hdr_data.max_vld_time = hdr_vld; - hdr_data.min_vld_time = hdr_vld; - new_vld = true; - } - else if (hdr_data.max_vld_time < hdr_vld) { - hdr_data.max_vld_time = hdr_vld; - new_vld = true; - } - - if (new_vld || !hdr_data.vld_num_array.has(hdr_vld, hdr_index, false)) { - hdr_index = hdr_data.vld_array.n_elements(); - hdr_data.vld_array.add(unix_to_yyyymmdd_hhmmss(hdr_vld)); - hdr_data.vld_num_array.add(hdr_vld); - } - data_buffer.hdr_vld_buf[hdr_data_idx] = hdr_index; - - // Write the header array which consists of the following: - // LAT LON ELV - data_buffer.hdr_lat_buf[hdr_data_idx] = (float) hdr_lat; - data_buffer.hdr_lon_buf[hdr_data_idx] = (float) hdr_lon; - data_buffer.hdr_elv_buf[hdr_data_idx] = (float) hdr_elv; - - hdr_data_idx++; - data_buffer.hdr_data_idx = hdr_data_idx; - data_buffer.cur_hdr_idx++; - - if (hdr_data_idx >= OBS_BUFFER_SIZE) { - //FIXME - write_header_to_nc(data_buffer, OBS_BUFFER_SIZE); - } -} - -/////////////////////////////////////////////////////////////////////////////// - -void NetcdfObsVars::write_observation() -{ - if (0 < data_buffer.obs_data_idx){ - //CHECKME data_buffer.obs_vars = obs_vars; - write_obs_buffer(data_buffer.obs_data_idx); - } -} - -/////////////////////////////////////////////////////////////////////////////// - -void NetcdfObsVars::write_observation(const float obs_arr[OBS_ARRAY_LEN], const char *obs_qty) -{ - int qty_index; - int obs_data_idx = data_buffer.obs_data_idx; - if (!data_buffer.qty_data_array.has(obs_qty, qty_index, false)) { - qty_index = data_buffer.qty_data_array.n_elements(); - data_buffer.qty_data_array.add(obs_qty); - } - data_buffer.qty_idx_buf[obs_data_idx] = qty_index; - - //for (int idx=0; idx= OBS_BUFFER_SIZE) { - write_obs_buffer(OBS_BUFFER_SIZE); - } -} - -/////////////////////////////////////////////////////////////////////////////// - -void NetcdfObsVars::write_table_vars () +void NetcdfObsVars::write_table_vars (NcHeaderData &hdr_data, NcDataBuffer &data_buffer) { - mlog << Debug(5) << " write_table_vars() is called. valid hdr_typ_tbl_var: " + mlog << Debug(5) << "write_table_vars() is called. valid hdr_typ_tbl_var: " << !IS_INVALID_NC(hdr_typ_tbl_var) << "\n"; if (IS_INVALID_NC(hdr_typ_tbl_var)) mlog << Warning << "\nwrite_table_vars() is called without creating variables\n\n"; @@ -1474,91 +1226,7 @@ void NetcdfObsVars::write_table_vars () /////////////////////////////////////////////////////////////////////////////// -int NetcdfObsVars::write_observations(const vector< Observation > observations, - const bool do_header) -{ - int prev_hdr_idx = -1; - string prev_header_type = ""; - string prev_station_id = ""; - ConcatString obs_qty; - int headerOffset = data_buffer.cur_hdr_idx; - //float obs_arr[obs_arr_len]; - const string method_name = " write_observations()"; - - int obs_buf_size = observations.size(); - mlog << Debug(7) << method_name << " obs_count: " << obs_buf_size << "\n"; - if (obs_buf_size > OBS_BUFFER_SIZE) obs_buf_size = OBS_BUFFER_SIZE; - - //if (reset) { - // data_buffer.obs_vars = obs_vars; - // data_buffer.obs_buf_size = obs_buf_size; - // data_buffer.obs_data_idx = 0; - // data_buffer.obs_data_offset = 0; - // data_buffer.hdr_data_idx = 0; - // data_buffer.hdr_data_offset = 0; - // - // data_buffer.processed_count =0; - //} - float obs_arr[OBS_ARRAY_LEN]; - bool header_to_vector = IS_INVALID_NC(hdr_arr_var) - || IS_INVALID_NC(hdr_lat_var); - mlog << Debug(5) << method_name << " do_header: " << do_header - << " header_to_vector: " << header_to_vector << "\n"; - for (vector< Observation >::const_iterator obs = observations.begin(); - obs != observations.end(); ++obs) - { - data_buffer.processed_count++; - - if (do_header) { - if (obs->getHeaderIndex() != prev_hdr_idx) { - mlog << Debug(9) << method_name << " obs->getHeaderIndex(): " - << obs->getHeaderIndex() << " at obs " << data_buffer.processed_count << "\n"; - prev_hdr_idx = obs->getHeaderIndex(); - if (header_to_vector) { - add_header( - obs->getHeaderType().c_str(), - obs->getStationId().c_str(), - obs->getValidTime(), - obs->getLatitude(), - obs->getLongitude(), - obs->getElevation()); - } - else { - write_header( - obs->getHeaderType().c_str(), - obs->getStationId().c_str(), - obs->getValidTime(), - obs->getLatitude(), - obs->getLongitude(), - obs->getElevation()); - } - } - } - - obs_arr[0] = obs->getHeaderIndex(); - obs_arr[1] = (use_var_id ? obs->getVarCode() : obs->getGribCode()); - obs_arr[2] = obs->getPressureLevel(); - obs_arr[3] = obs->getHeight(); - obs_arr[4] = obs->getValue(); - obs_qty = (obs->getQualityFlag().length() == 0 ? na_str : obs->getQualityFlag().c_str()); - if (do_header) obs_arr[0] += headerOffset; - - write_observation(obs_arr, obs_qty.text()); - - } /* endfor - obs */ - - if (data_buffer.obs_data_idx > 0) { - write_obs_buffer(data_buffer.obs_data_idx); - } - - //Caller handles writing headers - - return data_buffer.processed_count; -} - -/////////////////////////////////////////////////////////////////////////////// - -void NetcdfObsVars::write_obs_buffer(const int buf_size) +void NetcdfObsVars::write_obs_buffer(NcDataBuffer &data_buffer, const int buf_size) { //CHECKME const NetcdfObsVars &obs_vars = data_buffer.obs_vars; long offsets[2] = { data_buffer.obs_data_offset, 0 }; @@ -1617,6 +1285,92 @@ void NetcdfObsVars::write_obs_buffer(const int buf_size) data_buffer.obs_data_idx = 0; } +/////////////////////////////////////////////////////////////////////////////// + +//int NetcdfObsVars::write_observations(const vector< Observation > observations, +// NcDataBuffer &data_buffer, const bool do_header) +//{ +// int prev_hdr_idx = -1; +// string prev_header_type = ""; +// string prev_station_id = ""; +// ConcatString obs_qty; +// int headerOffset = data_buffer.cur_hdr_idx; +// //float obs_arr[obs_arr_len]; +// const string method_name = " write_observations()"; +// +// int obs_buf_size = observations.size(); +// mlog << Debug(7) << method_name << " obs_count: " << obs_buf_size << "\n"; +// if (obs_buf_size > OBS_BUFFER_SIZE) obs_buf_size = OBS_BUFFER_SIZE; +// +// //if (reset) { +// // data_buffer.obs_vars = obs_vars; +// // data_buffer.obs_buf_size = obs_buf_size; +// // data_buffer.obs_data_idx = 0; +// // data_buffer.obs_data_offset = 0; +// // data_buffer.hdr_data_idx = 0; +// // data_buffer.hdr_data_offset = 0; +// // +// // data_buffer.processed_count =0; +// //} +// float obs_arr[OBS_ARRAY_LEN]; +// bool header_to_vector = IS_INVALID_NC(hdr_arr_var) +// || IS_INVALID_NC(hdr_lat_var); +// mlog << Debug(5) << method_name << " do_header: " << do_header +// << " header_to_vector: " << header_to_vector << "\n"; +// for (vector< Observation >::const_iterator obs = observations.begin(); +// obs != observations.end(); ++obs) +// { +// data_buffer.processed_count++; +// +// if (do_header) { +// if (obs->getHeaderIndex() != prev_hdr_idx) { +// mlog << Debug(9) << method_name << " obs->getHeaderIndex(): " +// << obs->getHeaderIndex() << " at obs " << data_buffer.processed_count << "\n"; +// prev_hdr_idx = obs->getHeaderIndex(); +///* +// if (header_to_vector) { +// add_header( +// obs->getHeaderType().c_str(), +// obs->getStationId().c_str(), +// obs->getValidTime(), +// obs->getLatitude(), +// obs->getLongitude(), +// obs->getElevation()); +// } +// else { +// write_header( +// obs->getHeaderType().c_str(), +// obs->getStationId().c_str(), +// obs->getValidTime(), +// obs->getLatitude(), +// obs->getLongitude(), +// obs->getElevation()); +// } +// } +//*/ +// } +// +// obs_arr[0] = obs->getHeaderIndex(); +// obs_arr[1] = (use_var_id ? obs->getVarCode() : obs->getGribCode()); +// obs_arr[2] = obs->getPressureLevel(); +// obs_arr[3] = obs->getHeight(); +// obs_arr[4] = obs->getValue(); +// obs_qty = (obs->getQualityFlag().length() == 0 ? na_str : obs->getQualityFlag().c_str()); +// if (do_header) obs_arr[0] += headerOffset; +// +// write_observation(obs_arr, obs_qty.text()); +// +// } /* endfor - obs */ +// +// if (data_buffer.obs_data_idx > 0) { +// write_obs_buffer(data_buffer, data_buffer.obs_data_idx); +// } +// +// //Caller handles writing headers +// +// return data_buffer.processed_count; +//} + //////////////////////////////////////////////////////////////////////// void NetcdfObsVars::write_obs_var_names(StringArray &obs_names) { diff --git a/met/src/libcode/vx_nc_obs/nc_obs_util.h b/met/src/libcode/vx_nc_obs/nc_obs_util.h index 5087afe35b..b5a0de3e01 100644 --- a/met/src/libcode/vx_nc_obs/nc_obs_util.h +++ b/met/src/libcode/vx_nc_obs/nc_obs_util.h @@ -128,20 +128,12 @@ struct NetcdfObsVars { NetcdfObsVars(); bool is_valid(bool exit_on_error=false); void reset(bool _use_var_id = true); - void reset_header_buffer(int buf_size, bool reset_all); - - bool add_header(const char *hdr_typ, const char *hdr_sid, const time_t hdr_vld, - const float hdr_lat, const float hdr_lon, const float hdr_elv); - bool add_header_prepbufr (const int pb_report_type, const int in_report_type, - const int instrument_type); - bool add_header_strings(const char *hdr_typ, const char *hdr_sid); - bool add_header_vld(const char *hdr_vld); void create_dimensions(NcFile *f_out); void create_hdr_vars (NcFile *f_out, const int hdr_count); void create_obs_vars (NcFile *f_out); void create_obs_name_vars (NcFile *f_out, const int var_count, const int unit_count); - void create_table_vars (NcFile *f_out); + void create_table_vars (NcFile *f_out, NcHeaderData &hdr_data, NcDataBuffer &data_buffer); void create_pb_hdrs (NcFile *f_out, const int hdr_count); NcDim create_var_obs_var (NcFile *f_out, int var_count); @@ -150,7 +142,7 @@ struct NetcdfObsVars { //NcHeaderData *get_header_data(); //void set_header_data(); - void init_data_buffer(); +// void init_data_buffer(); void read_dims_vars(NcFile *f_in); NcHeaderData read_header_data(); @@ -158,20 +150,18 @@ struct NetcdfObsVars { int *qty_idx_arr, char *obs_qty_buf); void read_pb_hdr_data(NcHeaderData *hdr_data_P); - void write_arr_headers(); - void write_buf_headers (); - void write_header (const char *hdr_typ, const char *hdr_sid, const time_t hdr_vld, - const float hdr_lat, const float hdr_lon, const float hdr_elv); void write_header_to_nc(NcDataBuffer &data_buf, const int buf_size, const bool is_pb = false); - void write_obs_buffer(const int buf_size); + void write_obs_buffer(NcDataBuffer &data_buffer, const int buf_size); void write_obs_var_names(StringArray &obs_names); void write_obs_var_units(StringArray &units); void write_obs_var_descriptions(StringArray &descriptions); - void write_observation(); - void write_observation(const float obs_arr[OBS_ARRAY_LEN], const char *obs_qty); - int write_observations(const vector< Observation > observations, - const bool do_header = true); - void write_table_vars(); +// void write_observation(); +// void write_observation(const float obs_arr[OBS_ARRAY_LEN], const char *obs_qty); +// int write_observations(const vector< Observation > observations, +// NcDataBuffer &data_buffer, const bool do_header = true); +// int write_observations(const vector< Observation > observations, +// NcDataBuffer &data_buffer, const bool do_header = true); + void write_table_vars(NcHeaderData &hdr_data, NcDataBuffer &data_buffer); }; // NetcdfObsVars diff --git a/met/src/libcode/vx_nc_obs/nc_summary.cc b/met/src/libcode/vx_nc_obs/nc_summary.cc index 7e0efcb379..50004af935 100644 --- a/met/src/libcode/vx_nc_obs/nc_summary.cc +++ b/met/src/libcode/vx_nc_obs/nc_summary.cc @@ -19,8 +19,6 @@ using namespace std; //#include #include -//#include "vx_math.h" -//#include "vx_nc_util.h" #include "write_netcdf.h" #include "nc_obs_util.h" @@ -28,8 +26,6 @@ using namespace std; //////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////// - string _secsToTimeString(const int secs) { // Get the different fields from the number of seconds @@ -53,97 +49,190 @@ string _secsToTimeString(const int secs) //////////////////////////////////////////////////////////////////////// // If raw_hdr_cnt is greater than 0, skip updating header index for obs. -void init_netcdf_output(NcFile *nc_file, NetcdfObsVars &obs_vars, - NcObsOutputData &nc_out_data, string program_name) +//void init_netcdf_output(NcFile *nc_file, NetcdfObsVars &obs_vars, +// NcObsOutputData &nc_out_data, string program_name) +//{ +// string method_name = "init_netcdf_output() "; +// SummaryObs *summary_obs = nc_out_data.summary_obs; +// bool do_summary = nc_out_data.summary_info.flag; +// +// // +// // Initialize the header and observation record counters +// // +// int obs_count = nc_out_data.observations.size(); +// int hdr_count = (nc_out_data.processed_hdr_cnt > 0) +// ? nc_out_data.processed_hdr_cnt +// : summary_obs->countHeaders(nc_out_data.observations); // count and reset header index +// if (do_summary) { +// int summary_count = summary_obs->getSummaries().size(); +// int summary_hdr_count = summary_obs->countSummaryHeaders(); +// if (nc_out_data.summary_info.raw_data) { +// obs_count += summary_count; +// hdr_count += summary_hdr_count; +// } +// else { +// if (nc_out_data.processed_hdr_cnt > 0) { +// bool hdr_cnt = obs_vars.get_hdr_index(); +// bool reset_array = true; +// obs_vars.reset_header_buffer(hdr_cnt, reset_array); +// mlog << Debug(5) << method_name << "reset headers (" << hdr_cnt << ") raw data.\n"; +// } +// obs_count = summary_count; +// hdr_count = summary_hdr_count; +// } +// } +// mlog << Debug(7) << method_name << "obs_count: " +// << obs_count << " header count: " << hdr_count << "\n"; +// obs_vars.obs_cnt = obs_count; +// obs_vars.deflate_level = nc_out_data.deflate_level; +// +// obs_vars.create_hdr_vars(nc_file, hdr_count); +// obs_vars.create_obs_vars(nc_file); +// +// // +// // Add global attributes +// // +// write_netcdf_global(nc_file, nc_file->getName().c_str(), program_name.c_str()); +// +// if (do_summary) write_summary_attributes(nc_file, nc_out_data.summary_info); +// +//} + +//////////////////////////////////////////////////////////////////////// +// If raw_hdr_cnt is greater than 0, skip updating header index for obs. + +bool get_hdr_obs_count(NcFile *nc_file, NcObsOutputData *nc_out_data, + int *obs_cnt, int *hdr_cnt) { - string method_name = "init_netcdf_output() "; - SummaryObs *summary_obs = nc_out_data.summary_obs; - bool do_summary = nc_out_data.summary_info.flag; + string method_name = "get_hdr_obs_count() "; + bool reset_header_buffer = false; + SummaryObs *summary_obs = nc_out_data->summary_obs; + bool do_summary = nc_out_data->summary_info.flag; // // Initialize the header and observation record counters // - int obs_count = nc_out_data.observations.size(); - int hdr_count = (nc_out_data.processed_hdr_cnt > 0) - ? nc_out_data.processed_hdr_cnt - : summary_obs->countHeaders(nc_out_data.observations); // count and reset header index + int obs_count = nc_out_data->observations.size(); + int hdr_count = (nc_out_data->processed_hdr_cnt > 0) + ? nc_out_data->processed_hdr_cnt + : summary_obs->countHeaders(nc_out_data->observations); // count and reset header index if (do_summary) { int summary_count = summary_obs->getSummaries().size(); int summary_hdr_count = summary_obs->countSummaryHeaders(); - if (nc_out_data.summary_info.raw_data) { + if (nc_out_data->summary_info.raw_data) { obs_count += summary_count; hdr_count += summary_hdr_count; } else { - if (nc_out_data.processed_hdr_cnt > 0) { - bool hdr_cnt = obs_vars.get_hdr_index(); - bool reset_array = true; - obs_vars.reset_header_buffer(hdr_cnt, reset_array); - mlog << Debug(5) << method_name << "reset headers (" << hdr_cnt << ") raw data.\n"; + if (nc_out_data->processed_hdr_cnt > 0) { + reset_header_buffer = true; } obs_count = summary_count; hdr_count = summary_hdr_count; } } + *obs_cnt = obs_count; + *hdr_cnt = hdr_count; mlog << Debug(7) << method_name << "obs_count: " << obs_count << " header count: " << hdr_count << "\n"; - obs_vars.obs_cnt = obs_count; - obs_vars.deflate_level = nc_out_data.deflate_level; - - obs_vars.create_hdr_vars(nc_file, hdr_count); - obs_vars.create_obs_vars(nc_file); // // Add global attributes // - write_netcdf_global(nc_file, nc_file->getName().c_str(), program_name.c_str()); - if (do_summary) write_summary_attributes(nc_file, nc_out_data.summary_info); - + if (do_summary) write_summary_attributes(nc_file, nc_out_data->summary_info); + + return reset_header_buffer; } //////////////////////////////////////////////////////////////////////// -bool write_observations(NcFile *nc_file, NetcdfObsVars &obs_vars, - NcObsOutputData &nc_out_data) -{ - string method_name = "write_observations() "; - bool do_summary = nc_out_data.summary_info.flag; - bool do_save_raw_data = nc_out_data.summary_info.raw_data; - bool do_header = (nc_out_data.processed_hdr_cnt == 0); - - mlog << Debug(5) << method_name << "do_header: " << (do_header ? "true" : "false") - << ", do_summary: " << (do_summary ? "true" : "false") - << ", save_raw_data: " << (do_save_raw_data ? "true" : "false") - << "\n"; - - if (!do_summary || (do_summary && do_save_raw_data)) { - mlog << Debug(5) << method_name << "writing " - << (int)nc_out_data.observations.size() << " raw data...\n"; - obs_vars.write_observations(nc_out_data.observations, do_header); - } - if (do_summary) { - mlog << Debug(5) << method_name << "writing summary" - << (do_save_raw_data ? " " : " (summary only)") << "...\n"; - bool tmp_do_header = true; - obs_vars.write_observations(nc_out_data.summary_obs->getSummaries(), - tmp_do_header); - } - - int obs_buf_index = obs_vars.get_obs_index(); - if (obs_buf_index > 0) { - obs_vars.write_obs_buffer(obs_buf_index); - } - - obs_vars.create_table_vars (nc_file); - obs_vars.write_arr_headers(); - //obs_vars.write_table_vars(); - - return true; +void set_nc_out_data(NcObsOutputData *out_data, vector observations, + SummaryObs *summary_obs, TimeSummaryInfo summary_info) { + out_data->processed_hdr_cnt = 0; + out_data->observations = observations; + out_data->summary_obs = summary_obs; + out_data->summary_info = summary_info; } //////////////////////////////////////////////////////////////////////// +//bool write_observations(NetcdfObsVars *obs_vars, +// NcObsOutputData &nc_out_data) +//{ +// string method_name = "write_observations() "; +// bool do_summary = nc_out_data->summary_info.flag; +// bool do_save_raw_data = nc_out_data->summary_info.raw_data; +// bool do_header = (nc_out_data->processed_hdr_cnt == 0); +// +// mlog << Debug(5) << method_name << "do_header: " << (do_header ? "true" : "false") +// << ", do_summary: " << (do_summary ? "true" : "false") +// << ", save_raw_data: " << (do_save_raw_data ? "true" : "false") +// << "\n"; +// +// if (!do_summary || (do_summary && do_save_raw_data)) { +// mlog << Debug(5) << method_name << "writing " +// << (int)nc_out_data->observations.size() << " raw data...\n"; +// obs_vars->write_observations(nc_out_data->observations, do_header); +// } +// if (do_summary) { +// mlog << Debug(5) << method_name << "writing summary" +// << (do_save_raw_data ? " " : " (summary only)") << "...\n"; +// bool tmp_do_header = true; +// obs_vars->write_observations(nc_out_data->summary_obs->getSummaries(), +// tmp_do_header); +// } +// +// int obs_buf_index = obs_vars->get_obs_index(); +// if (obs_buf_index > 0) { +// obs_vars->write_obs_buffer(obs_buf_index); +// } +// +// return true; +//} +// +//////////////////////////////////////////////////////////////////////// + +//bool write_observations(NcFile *nc_file, NetcdfObsVars &obs_vars, +// NcObsOutputData &nc_out_data) +//{ +// string method_name = "write_observations() "; +// bool do_summary = nc_out_data.summary_info.flag; +// bool do_save_raw_data = nc_out_data.summary_info.raw_data; +// bool do_header = (nc_out_data.processed_hdr_cnt == 0); +// +// mlog << Debug(5) << method_name << "do_header: " << (do_header ? "true" : "false") +// << ", do_summary: " << (do_summary ? "true" : "false") +// << ", save_raw_data: " << (do_save_raw_data ? "true" : "false") +// << "\n"; +// +// if (!do_summary || (do_summary && do_save_raw_data)) { +// mlog << Debug(5) << method_name << "writing " +// << (int)nc_out_data.observations.size() << " raw data...\n"; +// obs_vars.write_observations(nc_out_data.observations, do_header); +// } +// if (do_summary) { +// mlog << Debug(5) << method_name << "writing summary" +// << (do_save_raw_data ? " " : " (summary only)") << "...\n"; +// bool tmp_do_header = true; +// obs_vars.write_observations(nc_out_data.summary_obs->getSummaries(), +// tmp_do_header); +// } +// +// int obs_buf_index = obs_vars.get_obs_index(); +// if (obs_buf_index > 0) { +// obs_vars.write_obs_buffer(obs_buf_index); +// } +// +// obs_vars.create_table_vars (nc_file); +// obs_vars.write_arr_headers(); +// //obs_vars.write_table_vars(); +// +// return true; +//} + +//////////////////////////////////////////////////////////////////////// + void write_summary_attributes(NcFile *nc_file, TimeSummaryInfo summary_info) { add_att(nc_file, "time_summary_beg", _secsToTimeString(summary_info.beg)); @@ -184,4 +273,3 @@ void write_summary_attributes(NcFile *nc_file, TimeSummaryInfo summary_info) { } //////////////////////////////////////////////////////////////////////// - diff --git a/met/src/libcode/vx_nc_obs/nc_summary.h b/met/src/libcode/vx_nc_obs/nc_summary.h index 6a99181041..936bcf3278 100644 --- a/met/src/libcode/vx_nc_obs/nc_summary.h +++ b/met/src/libcode/vx_nc_obs/nc_summary.h @@ -16,6 +16,7 @@ #include using namespace netCDF; #include "nc_utils.h" +#include "nc_obs_util.h" #include "vx_summary.h" //////////////////////////////////////////////////////////////////////// @@ -33,10 +34,17 @@ struct NcObsOutputData { extern string _secsToTimeString(const int secs); -extern void init_netcdf_output(NcFile *, NetcdfObsVars &obs_vars, - NcObsOutputData &nc_out_data, string program_name); +// Not moved to nc_obs_util to reduce the dependency (library) -extern bool write_observations(NcFile *, NetcdfObsVars &, NcObsOutputData &nc_out_data); +//extern bool init_netcdf_output(NcFile *, NetcdfObsVars *obs_vars, +// NcObsOutputData *nc_out_data, string program_name); +extern bool get_hdr_obs_count(NcFile *nc_file, NcObsOutputData *nc_out_data, + int *obs_cnt, int *hdr_cnt); + +extern void set_nc_out_data(NcObsOutputData *out_data, vector observations, + SummaryObs *summary_obs, TimeSummaryInfo summary_info); + +//extern bool write_observations(NetcdfObsVars *, NcObsOutputData *nc_out_data); extern void write_summary_attributes(NcFile *, TimeSummaryInfo);