Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature 2232 refine grid diag #2316

Merged
merged 5 commits into from
Oct 28, 2022
Merged

Feature 2232 refine grid diag #2316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5fcae49
Per issue #2232, added some new global variables to keep track of uni…
Oct 25, 2022
7d22fac
Per issue #2232, modified setup_nc_file to use new flags: multiple_da…
Oct 26, 2022
5e128a2
Per #2232, add a new Grid-Diag unit test to exercise the newly added …
JohnHalleyGotway Oct 28, 2022
4dbeb11
Unrelated to #2232, these 2 Makefile.in files needed to be recreated …
Oct 28, 2022
3da9e9a
Per #2232, switch from making the VAR# strings 0-based to 1-based. So…
Oct 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions internal/test_unit/config/GridDiagConfig_APCP_06_FCST_OBS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ data = {
range = [0,25];

field = [
{ set_attr_name = "FCST_APCP"; },
{ set_attr_name = "OBS_APCP"; }
{ ${FIELD1} },
{ ${FIELD2} }
];
}

Expand Down
40 changes: 40 additions & 0 deletions internal/test_unit/xml/unit_grid_diag.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@
&DATA_DIR_OBS;/stage4_hmt/stage4_2012041018_06h.grib" \
> &OUTPUT_DIR;/grid_diag/obs_file_list; \
&MET_BIN;/grid_diag</exec>
<env>
<pair><name>FIELD1</name> <value>set_attr_name = "FCST_APCP";</value></pair>
<pair><name>FIELD2</name> <value>set_attr_name = "OBS_APCP";</value></pair>
</env>
<param> \
-data &OUTPUT_DIR;/grid_diag/fcst_file_list \
-data &OUTPUT_DIR;/grid_diag/obs_file_list \
Expand All @@ -84,4 +88,40 @@
</output>
</test>

<!-- Test the logic for avoiding name conflicts -->

<test name="grid_diag_APCP_06_VARN">
<exec>echo "&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F006.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F012.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F018.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F024.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F030.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F036.grib \
&DATA_DIR_MODEL;/grib1/gfs_hmt/gfs_2012040900_F042.grib" \
> &OUTPUT_DIR;/grid_diag/fcst_file_list; \
echo "&DATA_DIR_OBS;/stage4_hmt/stage4_2012040906_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012040912_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012040918_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012041000_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012041006_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012041012_06h.grib \
&DATA_DIR_OBS;/stage4_hmt/stage4_2012041018_06h.grib" \
> &OUTPUT_DIR;/grid_diag/obs_file_list; \
&MET_BIN;/grid_diag</exec>
<env>
<pair><name>FIELD1</name> <value>name = "APCP";</value></pair>
<pair><name>FIELD2</name> <value>name = "APCP";</value></pair>
</env>
<param> \
-data &OUTPUT_DIR;/grid_diag/fcst_file_list \
-data &OUTPUT_DIR;/grid_diag/obs_file_list \
-config &CONFIG_DIR;/GridDiagConfig_APCP_06_FCST_OBS \
-out &OUTPUT_DIR;/grid_diag/grid_diag_APCP_06_VARN.nc \
-v 3
</param>
<output>
<grid_nc>&OUTPUT_DIR;/grid_diag/grid_diag_APCP_06_VARN.nc</grid_nc>
</output>
</test>

</met_test>
2 changes: 1 addition & 1 deletion src/basic/vx_config/config.tab.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1389,7 +1389,7 @@ yyparse (void)

YYDPRINTF ((stderr, "Starting parse\n"));

yystate = 8;
yystate = 0;
yyerrstatus = 0;
yynerrs = 0;
yychar = YYEMPTY; /* Cause a token to be read. */
Expand Down
133 changes: 84 additions & 49 deletions src/tools/other/grid_diag/grid_diag.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
// 003 08/20/21 Halley Gotway Bugfix #1886 for integer overflow.
// 004 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main
// 005 10/03/22 Prestopnik MET #2227 Remove using namespace std and netCDF from header files
// 006 10/26/22 Linden MET #2232 Refine the Grid-Diag output variable names when specifying two input data sources
//
////////////////////////////////////////////////////////////////////////

Expand Down Expand Up @@ -264,7 +265,7 @@ void process_series(void) {
// Process the 1d histograms
for(int i_var=0; i_var<conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

Expand Down Expand Up @@ -346,13 +347,13 @@ void process_series(void) {
// Process the 2d joint histograms
for(int i_var=0; i_var<conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

for(int j_var=i_var+1; j_var<conf_info.get_n_data(); j_var++) {

j_var_str << cs_erase << "VAR" << j_var;
j_var_str << cs_erase << "VAR" << j_var+1;

VarInfo *joint_info = conf_info.data_info[j_var];

Expand Down Expand Up @@ -405,7 +406,7 @@ void setup_histograms(void) {

for(int i_var=0; i_var<conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

Expand Down Expand Up @@ -443,8 +444,14 @@ void setup_histograms(void) {
<< "Initializing " << data_info->magic_str_attr()
<< " histogram with " << n_bins << " bins from "
<< min << " to " << max << ".\n";

histograms[i_var_str] = vector<long long>();
init_pdf(n_bins, histograms[i_var_str]);

// Keep track of unique output variable names
if(nc_var_sa.has( data_info->magic_str_attr() )) unique_variable_names = false;
nc_var_sa.add(data_info->magic_str_attr());

} // for i_var
}

Expand All @@ -455,14 +462,14 @@ void setup_joint_histograms(void) {

for(int i_var=0; i_var<conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];
int n_bins = data_info->n_bins();

for(int j_var=i_var+1; j_var<conf_info.get_n_data(); j_var++) {

j_var_str << cs_erase << "VAR" << j_var;
j_var_str << cs_erase << "VAR" << j_var+1;

VarInfo *joint_info = conf_info.data_info[j_var];
int n_joint_bins = joint_info->n_bins();
Expand All @@ -484,48 +491,48 @@ void setup_joint_histograms(void) {
////////////////////////////////////////////////////////////////////////

void setup_nc_file(void) {
ConcatString cs, i_var_str;
ConcatString cs, i_var_str, j_var_str;

// Create NetCDF file
nc_out = open_ncfile(out_file.c_str(), true);
// Create NetCDF file
nc_out = open_ncfile(out_file.c_str(), true);

if(IS_INVALID_NC_P(nc_out)) {
if(IS_INVALID_NC_P(nc_out)) {
mlog << Error << "\nsetup_nc_file() -> "
<< "trouble opening output NetCDF file "
<< out_file << "\n\n";
exit(1);
}

// Add global attributes
write_netcdf_global(nc_out, out_file.c_str(), program_name,
NULL, NULL, conf_info.desc.c_str());
add_att(nc_out, "mask_grid", (conf_info.mask_grid_name.nonempty() ?
(string)conf_info.mask_grid_name :
na_str));
add_att(nc_out, "mask_poly", (conf_info.mask_poly_name.nonempty() ?
(string)conf_info.mask_poly_name :
na_str));

// Add time range information to the global attributes
add_att(nc_out, "init_beg", (string)unix_to_yyyymmdd_hhmmss(init_beg));
add_att(nc_out, "init_end", (string)unix_to_yyyymmdd_hhmmss(init_end));
add_att(nc_out, "valid_beg", (string)unix_to_yyyymmdd_hhmmss(valid_beg));
add_att(nc_out, "valid_end", (string)unix_to_yyyymmdd_hhmmss(valid_end));
add_att(nc_out, "lead_beg", (string)sec_to_hhmmss(lead_beg));
add_att(nc_out, "lead_end", (string)sec_to_hhmmss(lead_end));

// Write the grid size, mask size, and series length
write_nc_var_int("grid_size", "number of grid points", grid.nxy());
write_nc_var_int("mask_size", "number of mask points", conf_info.mask_area.count());
write_nc_var_int("n_series", "length of series", n_series);

// Compression level
int deflate_level = compress_level;
if(deflate_level < 0) deflate_level = conf_info.conf.nc_compression();

for(int i_var=0; i_var < conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
<< "trouble opening output NetCDF file "
<< out_file << "\n\n";
exit(1);
}

// Add global attributes
write_netcdf_global(nc_out, out_file.c_str(), program_name,
NULL, NULL, conf_info.desc.c_str());
add_att(nc_out, "mask_grid", (conf_info.mask_grid_name.nonempty() ?
(string)conf_info.mask_grid_name :
na_str));
add_att(nc_out, "mask_poly", (conf_info.mask_poly_name.nonempty() ?
(string)conf_info.mask_poly_name :
na_str));

// Add time range information to the global attributes
add_att(nc_out, "init_beg", (string)unix_to_yyyymmdd_hhmmss(init_beg));
add_att(nc_out, "init_end", (string)unix_to_yyyymmdd_hhmmss(init_end));
add_att(nc_out, "valid_beg", (string)unix_to_yyyymmdd_hhmmss(valid_beg));
add_att(nc_out, "valid_end", (string)unix_to_yyyymmdd_hhmmss(valid_end));
add_att(nc_out, "lead_beg", (string)sec_to_hhmmss(lead_beg));
add_att(nc_out, "lead_end", (string)sec_to_hhmmss(lead_end));

// Write the grid size, mask size, and series length
write_nc_var_int("grid_size", "number of grid points", grid.nxy());
write_nc_var_int("mask_size", "number of mask points", conf_info.mask_area.count());
write_nc_var_int("n_series", "length of series", n_series);

// Compression level
int deflate_level = compress_level;
if(deflate_level < 0) deflate_level = conf_info.conf.nc_compression();

for(int i_var=0; i_var < conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

Expand All @@ -534,11 +541,16 @@ void setup_nc_file(void) {
var_name.add("_");
var_name.add(data_info->level_attr());

if(multiple_data_sources && !unique_variable_names) {
var_name.add("_");
var_name.add(i_var_str);
}

// Define histogram dimensions
NcDim var_dim = add_dim(nc_out, var_name,
(long) data_info->n_bins());
data_var_dims.push_back(var_dim);

// Define histogram bins
ConcatString var_min_name = var_name;
ConcatString var_max_name = var_name;
Expand Down Expand Up @@ -575,13 +587,20 @@ void setup_nc_file(void) {
// Define histograms
for(int i_var=0; i_var < conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

// Set variable NetCDF name
ConcatString var_name = data_info->name_attr();
var_name.add("_");
var_name.add(data_info->level_attr());

if(multiple_data_sources && !unique_variable_names) {
var_name.add("_");
var_name.add(i_var_str);
}

ConcatString hist_name("hist_");
hist_name.add(var_name);
NcDim var_dim = data_var_dims[i_var];
Expand All @@ -597,21 +616,36 @@ void setup_nc_file(void) {
// Define joint histograms
for(int i_var=0; i_var < conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];

for(int j_var=i_var+1; j_var<conf_info.get_n_data(); j_var++) {

j_var_str << cs_erase << "VAR" << j_var+1;

VarInfo *joint_info = conf_info.data_info[j_var];

ConcatString hist_name("hist_");
hist_name.add(data_info->name_attr());
hist_name.add("_");
hist_name.add(data_info->level_attr());

if(multiple_data_sources && !unique_variable_names) {
hist_name.add("_");
hist_name.add(i_var_str);
}

hist_name.add("_");
hist_name.add(joint_info->name_attr());
hist_name.add("_");
hist_name.add(joint_info->level_attr());

if(multiple_data_sources && !unique_variable_names) {
hist_name.add("_");
hist_name.add(j_var_str);
}

NcDim var_dim = data_var_dims[i_var];
NcDim joint_dim = data_var_dims[j_var];
vector<NcDim> dims;
Expand Down Expand Up @@ -662,7 +696,7 @@ void write_histograms(void) {

for(int i_var=0; i_var < conf_info.get_n_data(); i_var++) {

i_var_str << cs_erase << "VAR" << i_var;
i_var_str << cs_erase << "VAR" << i_var+1;

VarInfo *data_info = conf_info.data_info[i_var];
NcVar hist_var = hist_vars[i_var];
Expand Down Expand Up @@ -690,8 +724,8 @@ void write_joint_histograms(void) {
VarInfo *joint_info = conf_info.data_info[j_var];

ij_var_str << cs_erase
<< "VAR" << i_var << "_"
<< "VAR" << j_var;
<< "VAR" << i_var+1 << "_"
<< "VAR" << j_var+1;

long long *hist = joint_histograms[ij_var_str].data();

Expand Down Expand Up @@ -816,6 +850,7 @@ void usage() {

void set_data_files(const StringArray & a) {
data_files.push_back(a);
if(data_files.size() > 0) multiple_data_sources = true;
}

////////////////////////////////////////////////////////////////////////
Expand Down
6 changes: 6 additions & 0 deletions src/tools/other/grid_diag/grid_diag.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,12 @@ vector<netCDF::NcDim> data_var_dims;
vector<netCDF::NcVar> hist_vars;
vector<netCDF::NcVar> joint_hist_vars;

static bool multiple_data_sources = false;
static bool unique_variable_names = true;

// List of output NetCDF variable names
static StringArray nc_var_sa;

////////////////////////////////////////////////////////////////////////
//
// Miscellaneous Variables
Expand Down
2 changes: 1 addition & 1 deletion src/tools/tc_utils/tc_gen/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ tc_gen_LDADD = -lvx_stat_out \
-lvx_statistics \
-lvx_gis \
-lvx_data2d \
-lvx_seeps \
-lvx_seeps \
-lvx_nc_util \
-lvx_regrid \
-lvx_grid \
Expand Down
2 changes: 1 addition & 1 deletion src/tools/tc_utils/tc_gen/Makefile.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,10 +337,10 @@ tc_gen_LDADD = -lvx_stat_out \
-lvx_statistics \
-lvx_gis \
-lvx_data2d \
-lvx_seeps \
-lvx_nc_util \
-lvx_regrid \
-lvx_grid \
-lvx_seeps \
-lvx_config \
-lvx_pb_util \
-lvx_cal \
Expand Down
2 changes: 1 addition & 1 deletion src/tools/tc_utils/tc_stat/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ tc_stat_LDADD = -lvx_stat_out \
-lvx_data2d_nccf \
-lvx_statistics \
-lvx_data2d \
-lvx_seeps \
-lvx_seeps \
-lvx_nc_util \
-lvx_regrid \
-lvx_grid \
Expand Down
2 changes: 1 addition & 1 deletion src/tools/tc_utils/tc_stat/Makefile.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,10 +340,10 @@ tc_stat_LDADD = -lvx_stat_out \
-lvx_data2d_nccf \
-lvx_statistics \
-lvx_data2d \
-lvx_seeps \
-lvx_nc_util \
-lvx_regrid \
-lvx_grid \
-lvx_seeps \
-lvx_config \
-lvx_gsl_prob \
-lvx_pb_util \
Expand Down