Skip to content

Commit

Permalink
Feature 2232 refine grid diag (#2316)
Browse files Browse the repository at this point in the history 
Co-authored-by: Seth Linden <linden@seneca.rap.ucar.edu>
Co-authored-by: John Halley Gotway <johnhg@ucar.edu>
Co-authored-by: MET Tools Test Account <met_test@seneca.rap.ucar.edu>
  • Loading branch information
@sethlinden @JohnHalleyGotway
4 people authored Oct 28, 2022
1 parent b71b1ce commit 2add447
Show file tree
Hide file tree
Showing 9 changed files with 137 additions and 56 deletions.
4 changes: 2 additions & 2 deletions internal/test_unit/config/GridDiagConfig_APCP_06_FCST_OBS
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ data = {
range = [0,25];

field = [
{ set_attr_name = "FCST_APCP"; },
{ set_attr_name = "OBS_APCP"; }
{ ${FIELD1} },
{ ${FIELD2} }
];
}

Expand Down
40 changes: 40 additions & 0 deletions internal/test_unit/xml/unit_grid_diag.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@
&DATA_DIR_OBS;/stage4_hmt/stage4_2012041018_06h.grib" \
> &OUTPUT_DIR;/grid_diag/obs_file_list; \
&MET_BIN;/grid_diag</exec>
<env>
<pair><name>FIELD1</name> <value>set_attr_name = "FCST_APCP";</value></pair>
<pair><name>FIELD2</name> <value>set_attr_name = "OBS_APCP";</value></pair>
</env>
<param> \
-data &OUTPUT_DIR;/grid_diag/fcst_file_list \
-data &OUTPUT_DIR;/grid_diag/obs_file_list \
Expand All @@ -84,4 +88,40 @@
</output>
</test>

<!-- Test the logic for avoiding name conflicts -->

<test name="grid_diag_APCP_06_VARN">
<exec>echo "&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F006.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F012.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F018.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F024.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F030.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F036.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F042.grib" \
> &OUTPUT_DIR;/grid_diag/fcst_file_list; \
echo "&DATA_DIR_OBS;/stage4_hmt/stage4_2012040906_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012040912_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012040918_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012041000_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012041006_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012041012_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012041018_06h.grib" \
> &OUTPUT_DIR;/grid_diag/obs_file_list; \
&MET_BIN;/grid_diag</exec>
<env>
<pair><name>FIELD1</name> <value>name = "APCP";</value></pair>
<pair><name>FIELD2</name> <value>name = "APCP";</value></pair>
</env>
<param> \
-data &OUTPUT_DIR;/grid_diag/fcst_file_list \
-data &OUTPUT_DIR;/grid_diag/obs_file_list \
-config &CONFIG_DIR;/GridDiagConfig_APCP_06_FCST_OBS \
-out &OUTPUT_DIR;/grid_diag/grid_diag_APCP_06_VARN.nc \
-v 3
</param>
<output>
<grid_nc>&OUTPUT_DIR;/grid_diag/grid_diag_APCP_06_VARN.nc</grid_nc>
</output>
</test>

</met_test>
2 changes: 1 addition & 1 deletion src/basic/vx_config/config.tab.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1389,7 +1389,7 @@ yyparse (void)

YYDPRINTF ((stderr, "Starting parse\n"));

yystate = 8;
yystate = 0;
yyerrstatus = 0;
yynerrs = 0;
yychar = YYEMPTY; /* Cause a token to be read. */
Expand Down
133 changes: 84 additions & 49 deletions src/tools/other/grid_diag/grid_diag.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
// 003 08/20/21 Halley Gotway Bugfix #1886 for integer overflow.
// 004 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main
// 005 10/03/22 Prestopnik MET #2227 Remove using namespace std and netCDF from header files
// 006 10/26/22 Linden MET #2232 Refine the Grid-Diag output variable names when specifying two input data sources
//
////////////////////////////////////////////////////////////////////////

Expand Down Expand Up @@ -264,7 +265,7 @@ void process_series(void) {
// Process the 1d histograms
for(int i_var=0; i_var<conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

Expand Down Expand Up @@ -346,13 +347,13 @@ void process_series(void) {
// Process the 2d joint histograms
for(int i_var=0; i_var<conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

for(int j_var=i_var+1; j_var<conf_info.get_n_data(); j_var++) {

j_var_str << cs_erase << "VAR" << j_var;
j_var_str << cs_erase << "VAR" << j_var+1;

VarInfo *joint_info = conf_info.data_info[j_var];

Expand Down Expand Up @@ -405,7 +406,7 @@ void setup_histograms(void) {

for(int i_var=0; i_var<conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

Expand Down Expand Up @@ -443,8 +444,14 @@ void setup_histograms(void) {
<< "Initializing " << data_info->magic_str_attr()
<< " histogram with " << n_bins << " bins from "
<< min << " to " << max << ".\n";

histograms[i_var_str] = vector<long long>();
init_pdf(n_bins, histograms[i_var_str]);

// Keep track of unique output variable names
if(nc_var_sa.has( data_info->magic_str_attr() )) unique_variable_names = false;
nc_var_sa.add(data_info->magic_str_attr());

} // for i_var
}

Expand All @@ -455,14 +462,14 @@ void setup_joint_histograms(void) {

for(int i_var=0; i_var<conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];
int n_bins = data_info->n_bins();

for(int j_var=i_var+1; j_var<conf_info.get_n_data(); j_var++) {

j_var_str << cs_erase << "VAR" << j_var;
j_var_str << cs_erase << "VAR" << j_var+1;

VarInfo *joint_info = conf_info.data_info[j_var];
int n_joint_bins = joint_info->n_bins();
Expand All @@ -484,48 +491,48 @@ void setup_joint_histograms(void) {
////////////////////////////////////////////////////////////////////////

void setup_nc_file(void) {
ConcatString cs, i_var_str;
ConcatString cs, i_var_str, j_var_str;

// Create NetCDF file
nc_out = open_ncfile(out_file.c_str(), true);
// Create NetCDF file
nc_out = open_ncfile(out_file.c_str(), true);

if(IS_INVALID_NC_P(nc_out)) {
if(IS_INVALID_NC_P(nc_out)) {
mlog << Error << "\nsetup_nc_file() -> "
<< "trouble opening output NetCDF file "
<< out_file << "\n\n";
exit(1);
}

// Add global attributes
write_netcdf_global(nc_out, out_file.c_str(), program_name,
NULL, NULL, conf_info.desc.c_str());
add_att(nc_out, "mask_grid", (conf_info.mask_grid_name.nonempty() ?
(string)conf_info.mask_grid_name :
na_str));
add_att(nc_out, "mask_poly", (conf_info.mask_poly_name.nonempty() ?
(string)conf_info.mask_poly_name :
na_str));

// Add time range information to the global attributes
add_att(nc_out, "init_beg", (string)unix_to_yyyymmdd_hhmmss(init_beg));
add_att(nc_out, "init_end", (string)unix_to_yyyymmdd_hhmmss(init_end));
add_att(nc_out, "valid_beg", (string)unix_to_yyyymmdd_hhmmss(valid_beg));
add_att(nc_out, "valid_end", (string)unix_to_yyyymmdd_hhmmss(valid_end));
add_att(nc_out, "lead_beg", (string)sec_to_hhmmss(lead_beg));
add_att(nc_out, "lead_end", (string)sec_to_hhmmss(lead_end));

// Write the grid size, mask size, and series length
write_nc_var_int("grid_size", "number of grid points", grid.nxy());
write_nc_var_int("mask_size", "number of mask points", conf_info.mask_area.count());
write_nc_var_int("n_series", "length of series", n_series);

// Compression level
int deflate_level = compress_level;
if(deflate_level < 0) deflate_level = conf_info.conf.nc_compression();

for(int i_var=0; i_var < conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
<< "trouble opening output NetCDF file "
<< out_file << "\n\n";
exit(1);
}

// Add global attributes
write_netcdf_global(nc_out, out_file.c_str(), program_name,
NULL, NULL, conf_info.desc.c_str());
add_att(nc_out, "mask_grid", (conf_info.mask_grid_name.nonempty() ?
(string)conf_info.mask_grid_name :
na_str));
add_att(nc_out, "mask_poly", (conf_info.mask_poly_name.nonempty() ?
(string)conf_info.mask_poly_name :
na_str));

// Add time range information to the global attributes
add_att(nc_out, "init_beg", (string)unix_to_yyyymmdd_hhmmss(init_beg));
add_att(nc_out, "init_end", (string)unix_to_yyyymmdd_hhmmss(init_end));
add_att(nc_out, "valid_beg", (string)unix_to_yyyymmdd_hhmmss(valid_beg));
add_att(nc_out, "valid_end", (string)unix_to_yyyymmdd_hhmmss(valid_end));
add_att(nc_out, "lead_beg", (string)sec_to_hhmmss(lead_beg));
add_att(nc_out, "lead_end", (string)sec_to_hhmmss(lead_end));

// Write the grid size, mask size, and series length
write_nc_var_int("grid_size", "number of grid points", grid.nxy());
write_nc_var_int("mask_size", "number of mask points", conf_info.mask_area.count());
write_nc_var_int("n_series", "length of series", n_series);

// Compression level
int deflate_level = compress_level;
if(deflate_level < 0) deflate_level = conf_info.conf.nc_compression();

for(int i_var=0; i_var < conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

Expand All @@ -534,11 +541,16 @@ void setup_nc_file(void) {
var_name.add("_");
var_name.add(data_info->level_attr());

if(multiple_data_sources && !unique_variable_names) {
var_name.add("_");
var_name.add(i_var_str);
}

// Define histogram dimensions
NcDim var_dim = add_dim(nc_out, var_name,
(long) data_info->n_bins());
data_var_dims.push_back(var_dim);

// Define histogram bins
ConcatString var_min_name = var_name;
ConcatString var_max_name = var_name;
Expand Down Expand Up @@ -575,13 +587,20 @@ void setup_nc_file(void) {
// Define histograms
for(int i_var=0; i_var < conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

// Set variable NetCDF name
ConcatString var_name = data_info->name_attr();
var_name.add("_");
var_name.add(data_info->level_attr());

if(multiple_data_sources && !unique_variable_names) {
var_name.add("_");
var_name.add(i_var_str);
}

ConcatString hist_name("hist_");
hist_name.add(var_name);
NcDim var_dim = data_var_dims[i_var];
Expand All @@ -597,21 +616,36 @@ void setup_nc_file(void) {
// Define joint histograms
for(int i_var=0; i_var < conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

for(int j_var=i_var+1; j_var<conf_info.get_n_data(); j_var++) {

j_var_str << cs_erase << "VAR" << j_var+1;

VarInfo *joint_info = conf_info.data_info[j_var];

ConcatString hist_name("hist_");
hist_name.add(data_info->name_attr());
hist_name.add("_");
hist_name.add(data_info->level_attr());

if(multiple_data_sources && !unique_variable_names) {
hist_name.add("_");
hist_name.add(i_var_str);
}

hist_name.add("_");
hist_name.add(joint_info->name_attr());
hist_name.add("_");
hist_name.add(joint_info->level_attr());

if(multiple_data_sources && !unique_variable_names) {
hist_name.add("_");
hist_name.add(j_var_str);
}

NcDim var_dim = data_var_dims[i_var];
NcDim joint_dim = data_var_dims[j_var];
vector<NcDim> dims;
Expand Down Expand Up @@ -662,7 +696,7 @@ void write_histograms(void) {

for(int i_var=0; i_var < conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];
NcVar hist_var = hist_vars[i_var];
Expand Down Expand Up @@ -690,8 +724,8 @@ void write_joint_histograms(void) {
VarInfo *joint_info = conf_info.data_info[j_var];

ij_var_str << cs_erase
<< "VAR" << i_var << "_"
<< "VAR" << j_var;
<< "VAR" << i_var+1 << "_"
<< "VAR" << j_var+1;

long long *hist = joint_histograms[ij_var_str].data();

Expand Down Expand Up @@ -816,6 +850,7 @@ void usage() {

void set_data_files(const StringArray & a) {
data_files.push_back(a);
if(data_files.size() > 0) multiple_data_sources = true;
}

////////////////////////////////////////////////////////////////////////
Expand Down
6 changes: 6 additions & 0 deletions src/tools/other/grid_diag/grid_diag.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,12 @@ vector<netCDF::NcDim> data_var_dims;
vector<netCDF::NcVar> hist_vars;
vector<netCDF::NcVar> joint_hist_vars;

static bool multiple_data_sources = false;
static bool unique_variable_names = true;

// List of output NetCDF variable names
static StringArray nc_var_sa;

////////////////////////////////////////////////////////////////////////
//
// Miscellaneous Variables
Expand Down
2 changes: 1 addition & 1 deletion src/tools/tc_utils/tc_gen/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ tc_gen_LDADD = -lvx_stat_out \
-lvx_statistics \
-lvx_gis \
-lvx_data2d \
-lvx_seeps \
-lvx_seeps \
-lvx_nc_util \
-lvx_regrid \
-lvx_grid \
Expand Down
2 changes: 1 addition & 1 deletion src/tools/tc_utils/tc_gen/Makefile.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,10 +337,10 @@ tc_gen_LDADD = -lvx_stat_out \
-lvx_statistics \
-lvx_gis \
-lvx_data2d \
-lvx_seeps \
-lvx_nc_util \
-lvx_regrid \
-lvx_grid \
-lvx_seeps \
-lvx_config \
-lvx_pb_util \
-lvx_cal \
Expand Down
2 changes: 1 addition & 1 deletion src/tools/tc_utils/tc_stat/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ tc_stat_LDADD = -lvx_stat_out \
-lvx_data2d_nccf \
-lvx_statistics \
-lvx_data2d \
-lvx_seeps \
-lvx_seeps \
-lvx_nc_util \
-lvx_regrid \
-lvx_grid \
Expand Down
2 changes: 1 addition & 1 deletion src/tools/tc_utils/tc_stat/Makefile.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,10 +340,10 @@ tc_stat_LDADD = -lvx_stat_out \
-lvx_data2d_nccf \
-lvx_statistics \
-lvx_data2d \
-lvx_seeps \
-lvx_nc_util \
-lvx_regrid \
-lvx_grid \
-lvx_seeps \
-lvx_config \
-lvx_gsl_prob \
-lvx_pb_util \
Expand Down

0 comments on commit 2add447

Please sign in to comment.