From d612a3b513eaf36b81a12858fa543ce97bba5795 Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Mon, 9 Oct 2023 17:15:56 -0600 Subject: [PATCH 1/7] mods to use the multivar mode default config (when in multivar mode), and to check for problematic merge config settings --- src/libcode/vx_shapedata/mode_field_info.cc | 62 ++++++++++++++++----- src/tools/core/mode/mode.cc | 7 +++ src/tools/core/mode/mode_exec.cc | 11 +++- src/tools/core/mode/mode_exec.h | 2 +- src/tools/core/mode/mode_frontend.cc | 2 +- src/tools/core/mode/multivar_frontend.cc | 3 +- 6 files changed, 68 insertions(+), 19 deletions(-) diff --git a/src/libcode/vx_shapedata/mode_field_info.cc b/src/libcode/vx_shapedata/mode_field_info.cc index 6cecb55d07..02df5c1288 100644 --- a/src/libcode/vx_shapedata/mode_field_info.cc +++ b/src/libcode/vx_shapedata/mode_field_info.cc @@ -219,32 +219,68 @@ if ( dict->lookup(conf_key_conv_thresh) ) { } -if ( dict->lookup(conf_key_merge_thresh) ) { - - merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh); - -} - if ( dict->lookup(conf_key_vld_thresh) ) { vld_thresh = dict->lookup_double(conf_key_vld_thresh); } -// for the multivar case, go without parent, and error out if merge flag is -// not set for the individual field +// For the multivar case, more complex logic regarding merge_flag and merge_thresh +// If individual entry has a merge_flag, it must have a merge_thresh (unless merge_flag=NONE) +// If individual entry has no merge_flag, check the parent and go with that flag and thresh +// If individual entry has no merge_flag, but individual entry has a merge_thresh, it's an error + if ( _multivar ) { + // set defaults to no merging + merge_thresh.clear(); + merge_flag = MergeType_None; + if ( dict->lookup(conf_key_merge_flag, false)) { - merge_flag = int_to_mergetype(dict->lookup_int(conf_key_merge_flag)); + // this individual entry has merge_flag + merge_flag = int_to_mergetype(dict->lookup_int(conf_key_merge_flag, + default_dictionary_error_out, + default_dictionary_print_warning, + false)); + if (dict->lookup(conf_key_merge_thresh, false)) { + // the individual entry also has a merge_thresh, this is good + merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh, + default_dictionary_error_out, + default_dictionary_print_warning, + false); + } else { + if (merge_flag != MergeType_None) { + mlog << Error << "\nMode_Field_Info::set() -> " + << "When 'merge_flag' is explicitly set, 'merge_thresh' must be explicitly set for multivariate mode\n\n"; + exit ( 1 ); + } + } } else { - mlog << Error << "\nMode_Field_Info::set() -> " - << "'merge_flag' must be explicitly set for all fields with multivariate mode\n\n"; - exit ( 1 ); + // individual entry does not have a merge_flag, try parent + if ( dict->lookup(conf_key_merge_flag, true)) { + // the parent does have a merge flag + merge_flag = int_to_mergetype(dict->lookup_int(conf_key_merge_flag)); + if (dict->lookup(conf_key_merge_thresh, false)) { + // individual entry has a merge_thresh but no merge_flag, this is not good + mlog << Error << "\nMode_Field_Info::set() -> " + << "When 'merge_flag' is not explicitly set, 'merge_thresh' cannot explicitly set for multivariate mode\n\n"; + exit ( 1 ); + } else { + // individual entry does not have a merge thresh, what about parent? + merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh); + if (merge_thresh_array.n() == 0 && merge_flag != MergeType_None) { + // parent has a merge_flag but no merge_thresh + mlog << Error << "\nMode_Field_Info::set() -> " + << "'merge_flag' Not equal to NONE with no 'merge_thresh', not allowed in multivariate mode\n\n"; + exit ( 1 ); + } + } + } } -} else { + } else { if ( dict->lookup(conf_key_merge_flag, true)) { merge_flag = int_to_mergetype(dict->lookup_int(conf_key_merge_flag)); } + merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh); } filter_attr_map = parse_conf_filter_attr_map(dict); diff --git a/src/tools/core/mode/mode.cc b/src/tools/core/mode/mode.cc index 079a5db734..88e18d0b40 100644 --- a/src/tools/core/mode/mode.cc +++ b/src/tools/core/mode/mode.cc @@ -115,6 +115,7 @@ const char * const program_name = "mode"; static const char default_config_filename [] = "MET_BASE/config/MODEConfig_default"; +static const char default_multivar_config_filename [] = "MET_BASE/config/MODEMultivarConfig_default"; /////////////////////////////////////////////////////////////////////// @@ -169,6 +170,12 @@ int met_main(int argc, char * argv []) if ( config.is_multivar() ) { + + // read again, this time with the mvmode defaults, which are slightly different + + config.clear(); + config.read_config(default_multivar_config_filename, user_config_filename); + // exit inside this method if something is not implemented config.check_multivar_not_implemented(); diff --git a/src/tools/core/mode/mode_exec.cc b/src/tools/core/mode/mode_exec.cc index b3f130c4ec..f927b637be 100644 --- a/src/tools/core/mode/mode_exec.cc +++ b/src/tools/core/mode/mode_exec.cc @@ -35,6 +35,7 @@ static const char * cts_str[n_cts] = {"RAW", "OBJECT"}; static const char program_name [] = "mode"; static const char * default_config_filename = "MET_BASE/config/MODEConfig_default"; +static const char * default_multivar_config_filename = "MET_BASE/config/MODEMultivarConfig_default"; // took this out of the do_conv_thresh() method static int local_r_index = -1; @@ -143,7 +144,7 @@ void ModeExecutive::clear() /////////////////////////////////////////////////////////////////////// -void ModeExecutive::init(int n_files) +void ModeExecutive::init(int n_files, bool isMultivar) { @@ -152,7 +153,11 @@ void ModeExecutive::init(int n_files) R_index = T_index = 0; // Create the default config file name - default_config_file = replace_path(default_config_filename); + if (isMultivar) { + default_config_file = replace_path(default_multivar_config_filename); + } else { + default_config_file = replace_path(default_config_filename); + } // If the merge config file was not set, use the match config file if(merge_config_file.length() == 0) @@ -226,7 +231,7 @@ void ModeExecutive::init_multivar(GrdFileType ftype, GrdFileType otype) R_index = T_index = 0; // Create the default config file name - default_config_file = replace_path(default_config_filename); + default_config_file = replace_path(default_multivar_config_filename); // If the merge config file was not set, use the match config file if(merge_config_file.length() == 0) diff --git a/src/tools/core/mode/mode_exec.h b/src/tools/core/mode/mode_exec.h index 89d0102d2a..547792c1c6 100644 --- a/src/tools/core/mode/mode_exec.h +++ b/src/tools/core/mode/mode_exec.h @@ -76,7 +76,7 @@ class ModeExecutive { void clear(); - void init(int n_files); + void init(int n_files, bool isMultivar); void init_multivar(GrdFileType ftype, GrdFileType otype); diff --git a/src/tools/core/mode/mode_frontend.cc b/src/tools/core/mode/mode_frontend.cc index 329bcaa0e7..5f8aa71182 100644 --- a/src/tools/core/mode/mode_frontend.cc +++ b/src/tools/core/mode/mode_frontend.cc @@ -78,7 +78,7 @@ int ModeFrontEnd::run(const StringArray & Argv, Processing_t ptype, int field_in process_command_line(Argv, false); - mode_exec->init(n_files); + mode_exec->init(n_files, ptype != SINGLE_VAR); ModeConfInfo & conf = mode_exec->engine.conf_info; if ( field_index >= 0 ) conf.set_field_index(field_index); diff --git a/src/tools/core/mode/multivar_frontend.cc b/src/tools/core/mode/multivar_frontend.cc index 087311dd67..6a56e48bbc 100644 --- a/src/tools/core/mode/multivar_frontend.cc +++ b/src/tools/core/mode/multivar_frontend.cc @@ -10,7 +10,8 @@ //////////////////////////////////////////////////////////////////////// -static const char mode_default_config [] = "MET_BASE/config/MODEConfig_default"; +// for multivar mode, this is the default file +static const char mode_default_config [] = "MET_BASE/config/MODEMultivarConfig_default"; static const int dir_creation_mode = 0755; From 9bb3cc092a55ca5e7fe74fbecdb53c85858f6818 Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Tue, 10 Oct 2023 11:42:06 -0600 Subject: [PATCH 2/7] added more meaningful error and debug messages --- src/libcode/vx_shapedata/mode_field_info.cc | 22 ++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/libcode/vx_shapedata/mode_field_info.cc b/src/libcode/vx_shapedata/mode_field_info.cc index 02df5c1288..6b1c42a932 100644 --- a/src/libcode/vx_shapedata/mode_field_info.cc +++ b/src/libcode/vx_shapedata/mode_field_info.cc @@ -9,6 +9,7 @@ //////////////////////////////////////////////////////////////////////// +#include #include "vx_config.h" #include "vx_data2d.h" @@ -20,6 +21,7 @@ #include "mode_field_info.h" +using std::string; //////////////////////////////////////////////////////////////////////// @@ -235,12 +237,16 @@ if ( _multivar ) { merge_thresh.clear(); merge_flag = MergeType_None; + // pull out the name + string name = var_info->name(); + if ( dict->lookup(conf_key_merge_flag, false)) { // this individual entry has merge_flag merge_flag = int_to_mergetype(dict->lookup_int(conf_key_merge_flag, default_dictionary_error_out, default_dictionary_print_warning, false)); + string merge_name = mergetype_to_string(merge_flag); if (dict->lookup(conf_key_merge_thresh, false)) { // the individual entry also has a merge_thresh, this is good merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh, @@ -250,7 +256,8 @@ if ( _multivar ) { } else { if (merge_flag != MergeType_None) { mlog << Error << "\nMode_Field_Info::set() -> " - << "When 'merge_flag' is explicitly set, 'merge_thresh' must be explicitly set for multivariate mode\n\n"; + << "Field:" << name << ". " + << " When 'merge_flag' is explicitly set, 'merge_thresh' must be explicitly set for multivariate mode\n\n"; exit ( 1 ); } } @@ -259,19 +266,28 @@ if ( _multivar ) { if ( dict->lookup(conf_key_merge_flag, true)) { // the parent does have a merge flag merge_flag = int_to_mergetype(dict->lookup_int(conf_key_merge_flag)); + string merge_name = mergetype_to_string(merge_flag); if (dict->lookup(conf_key_merge_thresh, false)) { // individual entry has a merge_thresh but no merge_flag, this is not good mlog << Error << "\nMode_Field_Info::set() -> " + << "Field:" << name << ". " << "When 'merge_flag' is not explicitly set, 'merge_thresh' cannot explicitly set for multivariate mode\n\n"; exit ( 1 ); } else { - // individual entry does not have a merge thresh, what about parent? + // individual entry doesn't have a merge_thresh, parent has a merge_flag + // expect parent to have a merge_thresh merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh); if (merge_thresh_array.n() == 0 && merge_flag != MergeType_None) { // parent has a merge_flag but no merge_thresh mlog << Error << "\nMode_Field_Info::set() -> " - << "'merge_flag' Not equal to NONE with no 'merge_thresh', not allowed in multivariate mode\n\n"; + << "Field:" << name << ". using parent merge_flag: " << merge_name + << " Parent has no 'merge_thresh', not allowed in multivariate mode\n\n"; exit ( 1 ); + } else { + string thresh_str = merge_thresh_array.thresh()->get_str(); + mlog << Debug(2) << "Field:" << name << ". Using parent merge_flag: " + << merge_name << " and parent merge_thresh:" << thresh_str + << "\n"; } } } From 46d9b5a5a8c3ca88cb2c06e29698fbc64c3a4876 Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Fri, 13 Oct 2023 10:16:04 -0600 Subject: [PATCH 3/7] Modified documentation to better describe multivariate mode including new default values --- data/config/MODEMultivarConfig_default | 7 +++++++ docs/Users_Guide/mode.rst | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/data/config/MODEMultivarConfig_default b/data/config/MODEMultivarConfig_default index 66ab5dfdf3..8d4d7f8406 100644 --- a/data/config/MODEMultivarConfig_default +++ b/data/config/MODEMultivarConfig_default @@ -53,6 +53,11 @@ quilt = FALSE; // multivar_logic = "#1 && #2 && #3"; +// +// MODE Multivar intensity computation flag +// +multivar_intensity_flag = [FALSE, TRUE, TRUE]; + // // Forecast and observation fields to be verified // @@ -84,6 +89,8 @@ fcst = { filter_attr_thresh = []; merge_thresh = >=3.5; merge_flag = NONE; + multivar_name = "ALPHA_BETA_GAMMA"; + multivar_level = "LO"; } obs = fcst; diff --git a/docs/Users_Guide/mode.rst b/docs/Users_Guide/mode.rst index 98ab658ea4..2d304b21a1 100644 --- a/docs/Users_Guide/mode.rst +++ b/docs/Users_Guide/mode.rst @@ -197,6 +197,8 @@ mode configuration file The default configuration file for the MODE tool, **MODEConfig_default**, can be found in the installed *share/met/config* directory. Another version of the configuration file is provided in *scripts/config*. We encourage users to make a copy of the configuration files prior to modifying their contents. Descriptions of **MODEConfig_default** and the required variables for any MODE configuration file are also provided below. While the configuration file contains many entries, most users will only need to change a few for their use. Specific options are described in the following subsections. +A second default configuration file for the multivar MODE option, **MODEMultivarConfig_default**, is also found in the installed *share/met/config* directory. We encourage users to make a copy of this default configuration file when setting up a multivar configuration prior to modifying content. The two default config files **MODEConfig_default** and **MODEMultivarConfig_default** are similar, with **MODEMultivarConfig_default** having example multivar specific content. + Note that environment variables may be used when editing configuration files, as described in the :numref:`config_env_vars`. _____________________ @@ -324,7 +326,7 @@ The **merge_flag** entry controls what type of merging techniques will be applie • **BOTH** indicates that both techniques should be used. -By default, the double thresholding merging technique is applied. +By default, the double thresholding **THRESH** merging technique is applied in single variable mode. The merging defaults to **NONE** with multivariate mode. _____________________ From db9ca5bfbe6aa3c2268effd1db953ba321f4590c Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Fri, 13 Oct 2023 14:51:14 -0600 Subject: [PATCH 4/7] Bug fix for unit test --- src/libcode/vx_shapedata/mode_field_info.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libcode/vx_shapedata/mode_field_info.cc b/src/libcode/vx_shapedata/mode_field_info.cc index 6b1c42a932..326470facd 100644 --- a/src/libcode/vx_shapedata/mode_field_info.cc +++ b/src/libcode/vx_shapedata/mode_field_info.cc @@ -254,12 +254,17 @@ if ( _multivar ) { default_dictionary_print_warning, false); } else { + // get the parent's merge_thresh, just to have something. Error out if the merge_flag is not none + // becAuse that is inconsistent + merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh); + if (merge_flag != MergeType_None) { mlog << Error << "\nMode_Field_Info::set() -> " << "Field:" << name << ". " << " When 'merge_flag' is explicitly set, 'merge_thresh' must be explicitly set for multivariate mode\n\n"; exit ( 1 ); } + } } else { // individual entry does not have a merge_flag, try parent From aafb2f00b904a357a987e5950cf68fe0b3af40eb Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Wed, 25 Oct 2023 13:57:08 -0600 Subject: [PATCH 5/7] cleanup and fix a bug in which the default output path was not set in multivar mode --- src/basic/vx_util/file_exists.cc | 2 -- src/libcode/vx_shapedata/mode_field_info.cc | 4 ++-- src/tools/core/mode/mode_frontend.cc | 1 - src/tools/core/mode/multivar_frontend.cc | 11 ++++++++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/basic/vx_util/file_exists.cc b/src/basic/vx_util/file_exists.cc index 6d007303a4..a51db6f164 100644 --- a/src/basic/vx_util/file_exists.cc +++ b/src/basic/vx_util/file_exists.cc @@ -1,5 +1,3 @@ - - // *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* // ** Copyright UCAR (c) 1992 - 2023 // ** University Corporation for Atmospheric Research (UCAR) diff --git a/src/libcode/vx_shapedata/mode_field_info.cc b/src/libcode/vx_shapedata/mode_field_info.cc index 326470facd..06311f5d47 100644 --- a/src/libcode/vx_shapedata/mode_field_info.cc +++ b/src/libcode/vx_shapedata/mode_field_info.cc @@ -255,7 +255,7 @@ if ( _multivar ) { false); } else { // get the parent's merge_thresh, just to have something. Error out if the merge_flag is not none - // becAuse that is inconsistent + // because that is inconsistent merge_thresh_array = dict->lookup_thresh_array(conf_key_merge_thresh); if (merge_flag != MergeType_None) { @@ -297,7 +297,7 @@ if ( _multivar ) { } } } - } else { +} else { if ( dict->lookup(conf_key_merge_flag, true)) { merge_flag = int_to_mergetype(dict->lookup_int(conf_key_merge_flag)); } diff --git a/src/tools/core/mode/mode_frontend.cc b/src/tools/core/mode/mode_frontend.cc index 5f8aa71182..a37bd89b0c 100644 --- a/src/tools/core/mode/mode_frontend.cc +++ b/src/tools/core/mode/mode_frontend.cc @@ -37,7 +37,6 @@ extern const char * const program_name; static ModeExecutive *mode_exec = 0; static int compress_level = -1; - /////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/mode/multivar_frontend.cc b/src/tools/core/mode/multivar_frontend.cc index 6a56e48bbc..c19a3ad0b0 100644 --- a/src/tools/core/mode/multivar_frontend.cc +++ b/src/tools/core/mode/multivar_frontend.cc @@ -69,6 +69,8 @@ static const char tab [] = " "; static const bool do_clusters = false; +static string default_out_dir = "."; + static ModeConfInfo config; static string mode_path; @@ -395,6 +397,12 @@ void process_command_line(const StringArray & argv) CommandLine cline; + // + // Set the default output directory + // + + outdir = replace_path(default_out_dir); + mode_path = argv[0]; cline.set(argv); @@ -672,9 +680,6 @@ void process_superobjects(ShapeData &f_result, ShapeData &o_result, mode_argv.add(junk); mode_argv.add("-outdir"); mode_argv.add(dir); - // mode_argv.add("-field_index"); - // snprintf(junk, sizeof(junk), "%d", j); - // mode_argv.add(junk); mlog << Debug(1) << "Running superobject mode \n\n"; From e16fabc963f6748b887d7cf8a0292ab71aca6556 Mon Sep 17 00:00:00 2001 From: David Albo Date: Thu, 26 Oct 2023 14:27:14 -0600 Subject: [PATCH 6/7] Changed multivar_name value to 'Super' in MODEMultivarConfig_default --- data/config/MODEMultivarConfig_default | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/config/MODEMultivarConfig_default b/data/config/MODEMultivarConfig_default index 8d4d7f8406..76207fd018 100644 --- a/data/config/MODEMultivarConfig_default +++ b/data/config/MODEMultivarConfig_default @@ -89,7 +89,7 @@ fcst = { filter_attr_thresh = []; merge_thresh = >=3.5; merge_flag = NONE; - multivar_name = "ALPHA_BETA_GAMMA"; + multivar_name = "Super"; multivar_level = "LO"; } obs = fcst; From 1ce3fd85f0794daf360f77453564e6830f09d561 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 6 Nov 2023 13:24:16 -0700 Subject: [PATCH 7/7] Update multivar_name value and description in mode.rst --- docs/Users_Guide/mode.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/Users_Guide/mode.rst b/docs/Users_Guide/mode.rst index 2d304b21a1..cd90de6b82 100644 --- a/docs/Users_Guide/mode.rst +++ b/docs/Users_Guide/mode.rst @@ -253,15 +253,13 @@ _____________________ The **multivar_intensity_flag** entry appears only in the **MODEMultivarConfig_default** file. This option is paired with the **multivar_logic** entry, and can take on a value of TRUE or FALSE for each **field**. In the multivar case, super objects are created using the **multivar_logic** settings. For each input for which **multivar_intensity_flag** is TRUE, the input is masked to be non-missing only within the super objects, and traditional mode is run on that masked input. For each input for which **multivar_intensity_flag** is FALSE, the input is skipped over. If all the multivar_intensity_flag values are FALSE, traditional mode output is created for the super objects, but with no intensity information. - _____________________ .. code-block:: none - multivar_name = "Precip"; - -The **multivar_name** entry appears only in the **MODEMultivarConfig_default** file. This option is used only when the multivar option is enabled, and only when all **multivar_intensity_flag** values are FALSE. It can be thought of as an identifier for the multivariate super object. It shows up in output files names and content. If not set the default value is "Super". It can be set separately for forecasts and observations, or as a common value for both. + multivar_name = "Super"; +The **multivar_name** entry appears only in the **MODEMultivarConfig_default** file. This option is used only when the multivar option is enabled, and only when all **multivar_intensity_flag** values are FALSE. It can be thought of as an identifier for the multivariate super object. It shows up in output files names and content. It can be set separately for forecasts and observations or as a common value for both. _____________________