Skip to content

Commit

Permalink
#1581 API ica changed from obs_vars to nc_point_obs
Browse files Browse the repository at this point in the history
  • Loading branch information
Howard Soh committed Apr 9, 2021
1 parent 96c7d32 commit 8b99d3a
Show file tree
Hide file tree
Showing 8 changed files with 173 additions and 286 deletions.
78 changes: 13 additions & 65 deletions met/src/tools/other/ascii2nc/file_handler.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ FileHandler::~FileHandler()

bool FileHandler::readAsciiFiles(const vector< ConcatString > &ascii_filename_list)
{
obs_vars.init_data_buffer();
nc_point_obs.init_buffer();

// Loop through the ASCII files, reading in the observations. At the end of
// this loop, all of the observations will be in the _observations vector.
Expand Down Expand Up @@ -156,15 +156,12 @@ bool FileHandler::writeNetcdfFile(const string &nc_filename)
if (!_writeObservations())
return false;

// Add variable names
if (use_var_id) obs_vars.write_obs_var_names(obs_names);

// Close the netCDF file.

_closeNetcdf();

mlog << Debug(2) << "Finished processing " << obs_vars.obs_cnt
<< " observations for " << obs_vars.hdr_cnt << " headers.\n";
mlog << Debug(2) << "Finished processing " << nc_point_obs.get_obs_cnt()
<< " observations for " << nc_point_obs.get_hdr_cnt() << " headers.\n";

return true;
}
Expand Down Expand Up @@ -249,17 +246,15 @@ bool FileHandler::_openNetcdf(const string &nc_filename)
//
// Define the NetCDF dimensions and variables
//
obs_vars.reset(use_var_id);
obs_vars.attr_agl = true;
obs_vars.deflate_level = deflate_level;
nc_point_obs.set_netcdf(_ncFile, true);
// Note: use_var_id will be set by the handler
nc_point_obs.init_obs_vars(use_var_id, deflate_level, true);
set_nc_out_data(nc_point_obs.get_output_data(), _observations, &summary_obs, _summaryInfo);

nc_out_data.processed_hdr_cnt = 0;
nc_out_data.deflate_level = deflate_level;
nc_out_data.observations = _observations;
nc_out_data.summary_obs = &summary_obs;
nc_out_data.summary_info = _summaryInfo;
int obs_cnt, hdr_cnt;
nc_point_obs.get_dim_counts(&obs_cnt, &hdr_cnt);
nc_point_obs.init_netcdf(obs_cnt, hdr_cnt, _programName);

init_netcdf_output(_ncFile, obs_vars, nc_out_data, _programName);

//
// Initialize the header and observation record counters
Expand All @@ -271,48 +266,6 @@ bool FileHandler::_openNetcdf(const string &nc_filename)
}


////////////////////////////////////////////////////////////////////////

//bool FileHandler::_writeHdrInfo(const ConcatString &hdr_typ,
// const ConcatString &hdr_sid,
// const time_t hdr_vld,
// double lat, double lon, double elv) {
// //
// // Increment header count before writing
// //
// _hdrNum++;
// write_nc_header(obs_vars, hdr_typ, hdr_sid, hdr_vld, lat, lon, elv);
//
// return true;
//}

////////////////////////////////////////////////////////////////////////

//bool FileHandler::_writeObsInfo(int gc, float prs, float hgt, float obs,
// const ConcatString &qty) {
// float obs_arr[OBS_ARRAY_LEN];
// ConcatString obs_qty;
//
// //
// // Increment observation count before writing
// //
// _obsNum++;
//
// //
// // Build the observation array
// //
// obs_arr[0] = _hdrNum; // Index of header
// obs_arr[1] = gc; // GRIB code corresponding to the observation type
// obs_arr[2] = prs; // Pressure level (hPa) or accumulation interval (sec)
// obs_arr[3] = hgt; // Height in meters above sea level or ground level (msl or agl)
// obs_arr[4] = obs; // Observation value
//
// obs_qty = (qty.length() == 0 ? na_str : qty.text());
// write_nc_observation(obs_vars, nc_data_buffer, obs_arr, obs_qty.text());
//
// return true;
//}

////////////////////////////////////////////////////////////////////////

bool FileHandler::_addObservations(const Observation &obs)
Expand Down Expand Up @@ -384,14 +337,9 @@ bool FileHandler::_addObservations(const Observation &obs)

bool FileHandler::_writeObservations()
{
write_observations(_ncFile, obs_vars, nc_out_data);

int var_count = (use_var_id ? obs_names.n_elements() : 0);
if (var_count > 0) {
int unit_count = 0;
obs_vars.create_obs_name_vars (_ncFile, var_count, unit_count);
obs_vars.write_obs_var_names(obs_names);
}
if (use_var_id) nc_point_obs.set_using_var_id(use_var_id);
StringArray descs, units;
nc_point_obs.write_to_netcdf(obs_names, units, descs);

return true;
}
Expand Down
4 changes: 2 additions & 2 deletions met/src/tools/other/ascii2nc/file_handler.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ using namespace netCDF;
#include "vx_summary.h"

#include "nc_obs_util.h"
#include "nc_point_obs.h"
#include "nc_summary.h"

////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -85,8 +86,7 @@ class FileHandler
// Variables for writing output NetCDF file

NcFile *_ncFile;
NetcdfObsVars obs_vars;
NcObsOutputData nc_out_data;
MetNcPointObs2Write nc_point_obs;

long _nhdr;

Expand Down
56 changes: 24 additions & 32 deletions met/src/tools/other/ioda2nc/ioda2nc.cc
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*
// ** Copyright UCAR (c) 1992 - 2020
// ** Copyright UCAR (c) 2020 - 2020
// ** University Corporation for Atmospheric Research (UCAR)
// ** National Center for Atmospheric Research (NCAR)
// ** Research Applications Lab (RAL)
Expand Down Expand Up @@ -46,6 +46,7 @@ using namespace std;

#include "vx_summary.h"
#include "nc_obs_util.h"
#include "nc_point_obs.h"
#include "nc_summary.h"

////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -76,7 +77,6 @@ static ConcatString ncfile;
// Input configuration file
static ConcatString config_file;
static IODA2NCConfInfo conf_info;
static NcObsOutputData nc_out_data;

static Grid mask_grid;
static MaskPlane mask_area;
Expand Down Expand Up @@ -105,7 +105,7 @@ static IntArray filtered_times;
static bool do_summary;
static bool save_summary_only = false;
static SummaryObs *summary_obs;
static NetcdfObsVars obs_vars;
static MetNcPointObs2Write nc_point_obs;


////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -205,7 +205,7 @@ void initialize() {

n_total_obs = 0;

obs_vars.init_data_buffer();
nc_point_obs.init_buffer();

core_dims.clear();
core_dims.add("nvars");
Expand Down Expand Up @@ -323,7 +323,10 @@ void open_netcdf() {
}

// Define netCDF variables
obs_vars.reset();
int deflate_level = compress_level;
if(deflate_level < 0) deflate_level = conf_info.conf.nc_compression();
nc_point_obs.set_netcdf(f_out, true);
nc_point_obs.init_obs_vars(true, deflate_level);

// Add global attributes
write_netcdf_global(f_out, ncfile.text(), program_name);
Expand All @@ -334,7 +337,7 @@ void open_netcdf() {
////////////////////////////////////////////////////////////////////////

void process_ioda_file(int i_pb) {
int npbmsg, npbmsg_total;
int npbmsg, npbmsg_total, hdr_cnt;
int idx, i_msg, i_read, n_file_obs, i_ret, n_hdr_obs;
int rej_typ, rej_sid, rej_vld, rej_grid, rej_poly;
int rej_elv, rej_nobs;
Expand Down Expand Up @@ -525,7 +528,7 @@ void process_ioda_file(int i_pb) {
<< "trouble getting datetime\n\n";
exit(1);
}

StringArray raw_var_names;
if(do_all_vars || obs_var_names.n() == 0) raw_var_names = obs_value_vars;
else raw_var_names = obs_var_names;
Expand All @@ -535,6 +538,7 @@ void process_ioda_file(int i_pb) {
ConcatString unit_attr;
ConcatString desc_attr;
map<ConcatString,ConcatString> name_map = conf_info.getObsVarMap();
hdr_cnt = 0;
for(idx=0; idx<raw_var_names.n(); idx++ ) {
int *qc_data = new int[nlocs];
float *obs_data = new float[nlocs];
Expand Down Expand Up @@ -768,7 +772,7 @@ void process_ioda_file(int i_pb) {
}

// Store the index to the header data
obs_arr[0] = (float) obs_vars.get_hdr_index();
obs_arr[0] = (float) nc_point_obs.get_hdr_index();

n_hdr_obs = 0;
for(idx=0; idx<v_obs_data.size(); idx++ ) {
Expand All @@ -792,8 +796,8 @@ void process_ioda_file(int i_pb) {
// store the header data and increment the IODA record
// counter
if(n_hdr_obs > 0) {
obs_vars.add_header(modified_hdr_typ, hdr_sid.c_str(), hdr_vld_ut,
hdr_lat, hdr_lon, hdr_elv);
//nc_point_obs.add_header(modified_hdr_typ, hdr_sid.c_str(), hdr_vld_ut,
// hdr_lat, hdr_lon, hdr_elv);
i_msg++;
}
else {
Expand All @@ -810,8 +814,7 @@ void process_ioda_file(int i_pb) {
cout << log_message << "\n";
}

int obs_buf_index = obs_vars.get_obs_index();
if(obs_buf_index > 0) obs_vars.write_obs_buffer(obs_buf_index);
nc_point_obs.write_observation();

if(mlog.verbosity_level() > 0) cout << "\n" << flush;

Expand All @@ -831,7 +834,7 @@ void process_ioda_file(int i_pb) {
<< rej_elv << "\n"
<< "Rejected based on zero observations\t= "
<< rej_nobs << "\n"
<< "Total Records retained\t\t= "
<< "Total Records retained\t\t\t= "
<< i_msg << "\n"
<< "Total observations retained or derived\t= "
<< n_file_obs << "\n";
Expand Down Expand Up @@ -898,22 +901,17 @@ void process_ioda_file(int i_pb) {
////////////////////////////////////////////////////////////////////////

void write_netcdf_hdr_data() {
int obs_cnt, hdr_cnt;
const long hdr_count = (long) nc_point_obs.get_hdr_index();
static const string method_name = "\nwrite_netcdf_hdr_data()";

const long hdr_count = (long) obs_vars.get_hdr_index();
int deflate_level = compress_level;
if(deflate_level < 0) deflate_level = conf_info.conf.nc_compression();

nc_out_data.processed_hdr_cnt = hdr_count;
nc_out_data.deflate_level = deflate_level;
nc_out_data.observations = observations;
nc_out_data.summary_obs = summary_obs;
nc_out_data.summary_info = conf_info.getSummaryInfo();

init_netcdf_output(f_out, obs_vars, nc_out_data, program_name);
NcObsOutputData *nc_out_data = nc_point_obs.get_output_data();
set_nc_out_data(nc_out_data, observations, summary_obs, conf_info.getSummaryInfo());
nc_point_obs.get_dim_counts(&obs_cnt, &hdr_cnt);
nc_point_obs.init_netcdf(obs_cnt, hdr_cnt, program_name);

// Check for no messages retained
if(obs_vars.hdr_cnt <= 0) {
if(hdr_cnt <= 0) {
mlog << Error << method_name << " -> "
<< "No IODA reocrds retained. Nothing to write.\n\n";
// Delete the NetCDF file
Expand All @@ -922,8 +920,6 @@ void write_netcdf_hdr_data() {
}

// Make sure all obs data is processed before handling header
write_observations(f_out, obs_vars, nc_out_data);

StringArray nc_var_name_arr;
StringArray nc_var_unit_arr;
StringArray nc_var_desc_arr;
Expand All @@ -940,11 +936,7 @@ void write_netcdf_hdr_data() {
nc_var_desc_arr.add(obs_var_descs[i]);
}

obs_vars.deflate_level = deflate_level;
obs_vars.create_obs_name_vars(f_out, var_count, units_count);
obs_vars.write_obs_var_names(nc_var_name_arr);
obs_vars.write_obs_var_units(nc_var_unit_arr);
obs_vars.write_obs_var_descriptions(nc_var_desc_arr);
nc_point_obs.write_to_netcdf(nc_var_name_arr, nc_var_unit_arr, nc_var_desc_arr);

return;
}
Expand Down
2 changes: 2 additions & 0 deletions met/src/tools/other/lidar2nc/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,5 +48,7 @@ lidar2nc_LDADD = -lvx_shapedata \
-lvx_math \
-lvx_cal \
-lvx_log \
$(PYTHON_LIBS) \
-lvx_summary \
-lmfhdf -ldf -ljpeg \
-lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas -lz
Loading

0 comments on commit 8b99d3a

Please sign in to comment.