Skip to content

Commit

Permalink
#1965 Excludes the duplicated variable names from multiple input files
Browse files Browse the repository at this point in the history
  • Loading branch information
Howard Soh committed Jan 7, 2022
1 parent 1c981f4 commit 0273cf1
Showing 1 changed file with 41 additions and 31 deletions.
72 changes: 41 additions & 31 deletions met/src/tools/other/ioda2nc/ioda2nc.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,8 @@ void process_ioda_file(int i_pb) {
IntArray diff_file_times;
int diff_file_time_count;
StringArray variables_big_nlevels;
static const char *method_name = "process_ioda_file() ";
static const char *method_name = "process_ioda_file() ->";
static const char *method_name_s = "process_ioda_file() ";

bool apply_grid_mask = (conf_info.grid_mask.nx() > 0 &&
conf_info.grid_mask.ny() > 0);
Expand All @@ -388,7 +389,7 @@ void process_ioda_file(int i_pb) {

// Check for a valid file
if(IS_INVALID_NC_P(f_in)) {
mlog << Error << "\n" << method_name << "-> "
mlog << Error << "\n" << method_name
<< "can't open input NetCDF file \"" << ioda_files[i_pb]
<< "\" for reading.\n\n";
delete f_in;
Expand Down Expand Up @@ -444,7 +445,7 @@ void process_ioda_file(int i_pb) {
dim_names, metadata_vars);

if(!is_netcdf_ready) {
mlog << Error << "\n" << method_name << "-> "
mlog << Error << "\n" << method_name
<< "Please check the IODA file (required dimensions or meta variables are missing).\n\n";
delete f_in;
f_in = (NcFile *) 0;
Expand All @@ -460,7 +461,7 @@ void process_ioda_file(int i_pb) {
if(dim_names.has("nrecs")) nrecs = get_dim_value(f_in, "nrecs", false);
else {
nrecs = nvars * nlocs;
mlog << Debug(3) << "\n" << method_name << "-> "
mlog << Debug(3) << "\n" << method_name
<< "nrecs dimension does not exist, so computed\n";
}
NcVar in_hdr_vld_var = get_var(f_in, "datetime@MetaData");
Expand All @@ -472,10 +473,10 @@ void process_ioda_file(int i_pb) {
else {
NcDim datetime_dim = get_nc_dim(&in_hdr_vld_var, 1);
ndatetime = IS_VALID_NC(datetime_dim) ? get_dim_size(&datetime_dim) : nstring;
mlog << Debug(3) << "\n" << method_name << "-> "
mlog << Debug(3) << "\n" << method_name
<< "ndatetime dimension does not exist!\n";
}
mlog << Debug(5) << method_name << "-> dimensions: nvars=" << nvars << ", nlocs=" << nlocs
mlog << Debug(5) << method_name << "dimensions: nvars=" << nvars << ", nlocs=" << nlocs
<< ", nrecs=" << nrecs << ", nstring=" << nstring << ", ndatetime=" << ndatetime << "\n";

npbmsg_total = npbmsg = nlocs;
Expand Down Expand Up @@ -515,25 +516,24 @@ void process_ioda_file(int i_pb) {
}

if(!get_nc_data(&in_hdr_lat_var, hdr_lat_arr, nlocs)) {
mlog << Error << "\n" << method_name << " -> "
mlog << Error << "\n" << method_name
<< "trouble getting latitude\n\n";
exit(1);
}
if(!get_nc_data(&in_hdr_lon_var, hdr_lon_arr, nlocs)) {
mlog << Error << "\n" << method_name << " -> "
mlog << Error << "\n" << method_name
<< "trouble getting longitude\n\n";
exit(1);
}
if(!get_nc_data(&in_hdr_vld_var, hdr_vld_block, lengths, offsets)) {
mlog << Error << "\n" << method_name << " -> "
mlog << Error << "\n" << method_name
<< "trouble getting datetime\n\n";
exit(1);
}

StringArray raw_var_names;
if(do_all_vars || obs_var_names.n() == 0) raw_var_names = obs_value_vars;
else raw_var_names = obs_var_names;
if(obs_var_names.n() > 0) obs_var_names.clear();

NcVar obs_var, qc_var;
ConcatString unit_attr;
Expand All @@ -543,8 +543,8 @@ void process_ioda_file(int i_pb) {
int *qc_data = new int[nlocs];
float *obs_data = new float[nlocs];
ConcatString obs_var_name = raw_var_names[idx] + "@ObsValue";
mlog << Debug(7) << method_name << " -> "

mlog << Debug(7) << method_name
<< "processing \"" << obs_var_name << "\" variable!\n";
obs_var = get_var(f_in, obs_var_name.c_str());
v_qc_data.push_back(qc_data);
Expand All @@ -556,13 +556,16 @@ void process_ioda_file(int i_pb) {
get_var_units(&obs_var, unit_attr);
get_att_value_string(&obs_var, "long_name", desc_attr);
}
obs_var_units.add(unit_attr);
obs_var_descs.add(desc_attr);


// Replace the input variable name to the output variable name
ConcatString new_name = name_map[raw_var_names[idx]];
if (0 < new_name.length()) obs_var_names.add(new_name);
else obs_var_names.add(raw_var_names[idx]);
if (0 >= new_name.length()) new_name = raw_var_names[idx];
// Filter out the same variable names from multiple input files
if (!obs_var_names.has(new_name)) {
obs_var_names.add(new_name);
obs_var_units.add(unit_attr);
obs_var_descs.add(desc_attr);
}
}

// Initialize counts
Expand All @@ -573,7 +576,7 @@ void process_ioda_file(int i_pb) {
bool showed_progress = false;
if(mlog.verbosity_level() >= debug_level_for_performance) {
end_t = clock();
mlog << Debug(debug_level_for_performance) << " PERF: " << method_name << " "
mlog << Debug(debug_level_for_performance) << " PERF: " << method_name_s << " "
<< (end_t-start_t)/double(CLOCKS_PER_SEC)
<< " seconds for preparing\n";
start_t = clock();
Expand Down Expand Up @@ -612,7 +615,7 @@ void process_ioda_file(int i_pb) {

char valid_time[ndatetime+1];
m_strncpy(valid_time, (const char *)(hdr_vld_block + (i_read * ndatetime)),
ndatetime, method_name, "valid_time", true);
ndatetime, method_name_s, "valid_time", true);
valid_time[ndatetime] = 0;
msg_ut = yyyymmddThhmmss_to_unix(valid_time);

Expand Down Expand Up @@ -662,7 +665,7 @@ void process_ioda_file(int i_pb) {

if(has_msg_type) {
int buf_len = sizeof(modified_hdr_typ);
m_strncpy(hdr_typ, hdr_msg_types+(i_read*nstring), nstring, method_name, "hdr_typ");
m_strncpy(hdr_typ, hdr_msg_types+(i_read*nstring), nstring, method_name_s, "hdr_typ");
m_rstrip(hdr_typ, nstring);

// If the message type is not listed in the configuration
Expand All @@ -675,22 +678,22 @@ void process_ioda_file(int i_pb) {

if(0 < message_type_map.count((string)hdr_typ)) {
ConcatString mappedMessageType = message_type_map[(string)hdr_typ];
mlog << Debug(6) << "\n" << method_name << " -> "
mlog << Debug(6) << "\n" << method_name
<< "Switching report type \"" << hdr_typ
<< "\" to message type \"" << mappedMessageType << "\".\n";
if(mappedMessageType.length() < HEADER_STR_LEN) buf_len = HEADER_STR_LEN;
m_strncpy(modified_hdr_typ, mappedMessageType.c_str(), buf_len,
method_name, "modified_hdr_typ");
method_name_s, "modified_hdr_typ");
}
else {
m_strncpy(modified_hdr_typ, hdr_typ, buf_len, method_name, "modified_hdr_typ2");
m_strncpy(modified_hdr_typ, hdr_typ, buf_len, method_name_s, "modified_hdr_typ2");
}
modified_hdr_typ[buf_len-1] = 0;
}

if(has_station_id) {
char tmp_sid[nstring+1];
m_strncpy(tmp_sid, hdr_station_ids+(i_read*nstring), nstring, method_name, "tmp_sid");
m_strncpy(tmp_sid, hdr_station_ids+(i_read*nstring), nstring, method_name_s, "tmp_sid");
m_rstrip(tmp_sid, nstring);
hdr_sid = tmp_sid;
}
Expand Down Expand Up @@ -775,7 +778,16 @@ void process_ioda_file(int i_pb) {

n_hdr_obs = 0;
for(idx=0; idx<v_obs_data.size(); idx++ ) {
obs_arr[1] = idx;
int var_idx;
ConcatString out_name = name_map[raw_var_names[idx]];
if (0 >= out_name.length()) out_name = raw_var_names[idx];
if (!obs_var_names.has(out_name, var_idx)) {
mlog << Warning << "\n" << method_name
<< "Skip the variable " << out_name << " (" << raw_var_names[idx]
<< ") at " << ioda_files[i_pb] << "\n\n";
continue;
}
obs_arr[1] = var_idx;
obs_arr[2] = obs_pres_arr[i_read];
obs_arr[3] = obs_hght_arr[i_read];
obs_arr[4] = v_obs_data[idx][i_read];
Expand All @@ -795,8 +807,6 @@ void process_ioda_file(int i_pb) {
// store the header data and increment the IODA record
// counter
if(n_hdr_obs > 0) {
//nc_point_obs.add_header(modified_hdr_typ, hdr_sid.c_str(), hdr_vld_ut,
// hdr_lat, hdr_lon, hdr_elv);
i_msg++;
}
else {
Expand Down Expand Up @@ -839,12 +849,12 @@ void process_ioda_file(int i_pb) {
<< n_file_obs << "\n";

if(npbmsg == rej_vld && 0 < rej_vld) {
mlog << Warning << "\n" << method_name << " -> "
mlog << Warning << "\n" << method_name
<< "All records were filtered out by valid time.\n"
<< "\tPlease adjust time range with \"-valid_beg\" and \"-valid_end\".\n"
<< "\tmin/max obs time from IODA file: " << min_time_str
<< " and " << max_time_str << ".\n"
<< "\ttime range: " << start_time_str << " and " << end_time_str << ".\n";
<< "\ttime range: " << start_time_str << " and " << end_time_str << ".\n\n";
}
else {
mlog << Debug(1) << "Obs time between " << unix_to_yyyymmdd_hhmmss(min_msg_ut)
Expand Down Expand Up @@ -882,13 +892,13 @@ void process_ioda_file(int i_pb) {

if(mlog.verbosity_level() >= debug_level_for_performance) {
method_end = clock();
cout << " PERF: " << method_name << " "
cout << " PERF: " << method_name_s
<< (method_end-method_start)/double(CLOCKS_PER_SEC)
<< " seconds\n";
}

if(i_msg <= 0) {
mlog << Warning << "\n" << method_name << " -> "
mlog << Warning << "\n" << method_name
<< "No IODA records retained from file: "
<< ioda_files[i_pb] << "\n\n";
}
Expand Down

0 comments on commit 0273cf1

Please sign in to comment.