Merge branch 'develop' of github.com:dtcenter/MET into develop

met/data/config/EnsembleStatConfig_default

@@ -114,7 +114,8 @@ message_type   = [ "ADPUPA" ];
 sid_inc        = [];
 sid_exc        = [];
 obs_thresh     = [ NA ];
-obs_quality    = [];
+obs_quality_inc  = [];
+obs_quality_exc  = [];
 duplicate_flag = NONE;
 obs_summary    = NONE;
 obs_perc_value = 50;

met/data/config/PointStatConfig_default

@@ -97,7 +97,8 @@ obs = fcst;
 message_type   = [ "ADPUPA" ];
 sid_inc        = [];
 sid_exc        = [];
-obs_quality    = [];
+obs_quality_inc  = [];
+obs_quality_exc  = [];
 duplicate_flag = NONE;
 obs_summary    = NONE;
 obs_perc_value = 50;

met/docs/Users_Guide/config_options.rst

@@ -1997,20 +1997,37 @@ Percentile value to use when obs_summary = PERC
   obs_perc_value = 50;
 
   
-.. _obs_quality:
+.. _obs_quality_inc:
 
-:ref:`obs_quality <obs_quality>`
+:ref:`obs_quality_inc <obs_quality_inc>`
 
-The "obs_quality" entry specifies the quality flag values that are to be
+The "obs_quality_inc" entry specifies the quality flag values that are to be
 retained and used for verification. An empty list signifies that all
 point observations should be used, regardless of their quality flag value.
 The quality flag values will vary depending on the original source of the
 observations. The quality flag values to retain should be specified as
 an array of strings, even if the values themselves are numeric.
+Note "obs_quality_inc" replaces the older option "obs_quality".
 
 .. code-block:: none
 		
-  obs_quality = [ "1", "2", "3", "9" ];
+  obs_quality_inc = [ "1", "2", "3", "9" ];
+
+  
+.. _obs_quality_exc:
+
+:ref:`obs_quality_exc <obs_quality_exc>`
+
+The "obs_quality_exc" entry specifies the quality flag values that are to be
+ignored and not used for verification. An empty list signifies that all
+point observations should be used, regardless of their quality flag value.
+The quality flag values will vary depending on the original source of the
+observations. The quality flag values to ignore should be specified as
+an array of strings, even if the values themselves are numeric.
+
+.. code-block:: none
+		
+  obs_quality_exc = [ "1", "2", "3", "9" ];
 
   
 .. _met_data_dir:

met/docs/Users_Guide/ensemble-stat.rst

@@ -149,7 +149,8 @@ ____________________
   sid_inc        = [];
   sid_exc        = [];
   duplicate_flag = NONE;
-  obs_quality    = [];
+  obs_quality_inc  = [];
+  obs_quality_exc  = [];
   obs_summary    = NONE;
   obs_perc_value = 50;
   message_type_group_map = [...];

met/docs/Users_Guide/point-stat.rst

@@ -342,7 +342,8 @@ ________________________
   sid_inc        = [];
   sid_exc        = [];
   duplicate_flag = NONE;
-  obs_quality    = [];
+  obs_quality_inc  = [];
+  obs_quality_exc  = [];
   obs_summary    = NONE;
   obs_perc_value = 50;
   message_type_group_map = [...];

met/docs/Users_Guide/release-notes.rst

@@ -7,36 +7,47 @@ describes the bugfix, enhancement, or new feature:
 
 MET Version 10.1.0-beta2 release notes (|release_date|)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-* Fix the findings from SonarQube (`#1855 <https://github.com/dtcenter/MET/issues/1855>`_).
-* Fix gen_vx_mask so that the -input_field and -mask_field options are processed independently (`#1891 <https://github.com/dtcenter/MET/issues/1891>`_).
-* Enhance Stat-Analysis to write the GO Index and CBS Index into a new STAT line type (`#1788 <https://github.com/dtcenter/MET/issues/1788>`_).
-* Enhance MET to compute the CBS Index (`#1031 <https://github.com/dtcenter/MET/issues/1031>`_).
-* Fix integer overflow in Grid-Diag (`#1886 <https://github.com/dtcenter/MET/issues/1886>`_).
-* Fix failure of Ensemble-Stat when verifying against gridded ECMWF GRIB1 files (`#1879 <https://github.com/dtcenter/MET/issues/1879>`_).
-* Fix consumption of too much memory by Stat-Analysis (`#1875 <https://github.com/dtcenter/MET/issues/1875>`_).
-* Fix failure in plot_tcmpr.R script when a directory is passed in with -lookin (`#1872 <https://github.com/dtcenter/MET/issues/1872>`_).
-* Enable TC-Pairs to only write output for a configurable list of valid times (`#1870 <https://github.com/dtcenter/MET/issues/1870>`_).
-* Add check for the start offset and data count are valid before calling NetCDF API (`#1852 <https://github.com/dtcenter/MET/issues/1852>`_).
-* Add new column, Scatter Index, for CNT line type (`#1843 <https://github.com/dtcenter/MET/issues/1843>`_).
-* Enable point2grid to support double type latitude/longitude variables (`#1838 <https://github.com/dtcenter/MET/issues/1838>`_).
-* Treat warnings from the documentation as errors to facilitate continuous integration with GHA (`#1819 <https://github.com/dtcenter/MET/issues/1819>`_).
-* Fix the output of point2grid which is flipped and rotated with lat/lon to lat/lon conversion (`#1817 <https://github.com/dtcenter/MET/issues/1817>`_).
-* Add ORANK line type to the HiRA output from Point-Stat (`#1764 <https://github.com/dtcenter/MET/issues/1764>`_).
-* Make the specification of a binary threshold in Wavelet-Stat optional (`#1746 <https://github.com/dtcenter/MET/issues/1746>`_).
-* Document the statistics from the RPS line type in Appendix C (`#1853 <https://github.com/dtcenter/MET/issues/1853>`_).
-* Fix broken URLs in default MET config files (`#1864 <https://github.com/dtcenter/MET/issues/1864>`_).
-* Add G and G-Beta to the DMAP line type from Grid-Stat (`#1673 <https://github.com/dtcenter/MET/issues/1673>`_).
-* Create and publish a PDF of the MET User's Guide (`#1453 <https://github.com/dtcenter/MET/issues/1453>`_).
-* Update documentation to reference GitHub Discussions instead of MET Help (`#1833 <https://github.com/dtcenter/MET/issues/1833>`_).
-* Update FAQ in User's Guide with info from webpage FAQ (`#1834 <https://github.com/dtcenter/MET/issues/1834>`_).  
-
+
+* New output:
+
+   * **Enhance MET to compute the CBS Index** (`#1031 <https://github.com/dtcenter/MET/issues/1031>`_).
+   * **Enhance Stat-Analysis to write the GO Index and CBS Index into a new SSIDX STAT line type** (`#1788 <https://github.com/dtcenter/MET/issues/1788>`_).
+   * **Add Scatter Index to the CNT line type** (`#1843 <https://github.com/dtcenter/MET/issues/1843>`_).
+   * **Add ORANK line type to the HiRA output from Point-Stat** (`#1764 <https://github.com/dtcenter/MET/issues/1764>`_).
+   * **Add G and G-Beta to the DMAP line type from Grid-Stat** (`#1673 <https://github.com/dtcenter/MET/issues/1673>`_).
+
+* Configuration:
+
+   * **Make the specification of a binary threshold in Wavelet-Stat optional** (`#1746 <https://github.com/dtcenter/MET/issues/1746>`_).
+   * **Enable TC-Pairs to only write output for a configurable list of valid times** (`#1870 <https://github.com/dtcenter/MET/issues/1870>`_).
+   * Fix broken URLs in default MET config files (`#1864 <https://github.com/dtcenter/MET/issues/1864>`_).
+
+* Bugfixes:
+
+   * **Fix the findings from SonarQube** (`#1855 <https://github.com/dtcenter/MET/issues/1855>`_).
+   * Fix integer overflow in Grid-Diag (`#1886 <https://github.com/dtcenter/MET/issues/1886>`_).
+   * Fix Gen-Vx-Mask so that the -input_field and -mask_field options are processed independently (`#1891 <https://github.com/dtcenter/MET/issues/1891>`_).
+   * Enable Point2Grid to support double type latitude/longitude variables (`#1838 <https://github.com/dtcenter/MET/issues/1838>`_).
+   * Fix the output of Point2Grid which is flipped and rotated with lat/lon to lat/lon conversion (`#1817 <https://github.com/dtcenter/MET/issues/1817>`_).
+   * Fix failure of Ensemble-Stat when verifying against gridded ECMWF GRIB1 files (`#1879 <https://github.com/dtcenter/MET/issues/1879>`_).
+   * Fix consumption of too much memory by Stat-Analysis (`#1875 <https://github.com/dtcenter/MET/issues/1875>`_).
+   * Add check for the start offset and data count are valid before calling NetCDF API (`#1852 <https://github.com/dtcenter/MET/issues/1852>`_).
+   * Fix failure in plot_tcmpr.R script when a directory is passed in with -lookin (`#1872 <https://github.com/dtcenter/MET/issues/1872>`_).
+
+* Documentation:
+
+   * **Create and publish a PDF of the MET User's Guide** (`#1453 <https://github.com/dtcenter/MET/issues/1453>`_).
+   * Document the statistics from the RPS line type in Appendix C (`#1853 <https://github.com/dtcenter/MET/issues/1853>`_).
+   * Treat warnings from the documentation as errors to facilitate continuous integration with GHA (`#1819 <https://github.com/dtcenter/MET/issues/1819>`_).
+   * Update documentation to reference GitHub Discussions instead of MET Help (`#1833 <https://github.com/dtcenter/MET/issues/1833>`_).
+   * Update FAQ in User's Guide with info from webpage FAQ (`#1834 <https://github.com/dtcenter/MET/issues/1834>`_).
 
 MET Version 10.1.0-beta1 release notes (20210613)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 * Add the HSS_EC statistic to the MCTS line type and a configurable option for its computation (`#1749 <http://github.com/dtcenter/MET/issues/1749>`_).
 * Implement a common API for reading and writing the common NetCDF point observation file format (`#1402 <http://github.com/dtcenter/MET/issues/1402>`_ and `#1581 <http://github.com/dtcenter/MET/issues/1581>`_).
-* Change -type for gen_vx_mask from an optional argument to a required one (`#1792 <http://github.com/dtcenter/MET/issues/1792>`_).
+* Change -type for Gen-Vx-Mask from an optional argument to a required one (`#1792 <http://github.com/dtcenter/MET/issues/1792>`_).
 * Fix python embedding when using a named grid with MET_PYTHON_EXE set (`#1798 <http://github.com/dtcenter/MET/issues/1798>`_).
 * Fix Fortify High finding for src/libcode/vx_data2d_nccf/nccf_file.cc (`#1795 <http://github.com/dtcenter/MET/issues/1795>`_).
 * Modify plot_tcmpr.R script to support plotting of extra-tropical cyclone tracks not verified against BEST tracks (`#1801 <http://github.com/dtcenter/MET/issues/1801>`_).
@@ -148,7 +159,7 @@ MET Version 10.0.0 release notes (20210510)
       * **Support additional NetCDF point observation data sources** (`#1345 <http://github.com/dtcenter/MET/issues/1345>`_, `#1509 <http://github.com/dtcenter/MET/issues/1509>`_, and `#1511 <http://github.com/dtcenter/MET/issues/1511>`_).
       * Support the 2-dimensional time variable in Himawari data files (`#1580 <http://github.com/dtcenter/MET/issues/1580>`_).
       * Create empty output files for zero input observations instead of erroring out (`#1630 <http://github.com/dtcenter/MET/issues/1630>`_).
-      * Improve the point2grid runtime performance (`#1421 <http://github.com/dtcenter/MET/issues/1421>`_).
+      * Improve the Point2Grid runtime performance (`#1421 <http://github.com/dtcenter/MET/issues/1421>`_).
       * Process point observations by variable name instead of GRIB code (`#1408 <http://github.com/dtcenter/MET/issues/1408>`_).
 
    * GIS Tools:

met/scripts/config/EnsembleStatConfig

@@ -136,7 +136,8 @@ obs = fcst;
 sid_inc        = [];
 sid_exc        = [];
 obs_thresh     = [ NA ];
-obs_quality    = [];
+obs_quality_inc  = [];
+obs_quality_exc  = [];
 duplicate_flag = NONE;
 obs_summary    = NONE;
 obs_perc_value = 50;

met/scripts/config/PointStatConfig

@@ -77,7 +77,8 @@ obs = fcst;
 //
 sid_inc        = [];
 sid_exc        = [];
-obs_quality    = [];
+obs_quality_inc  = [];
+obs_quality_exc  = [];
 duplicate_flag = NONE;
 obs_summary    = NONE;
 obs_percentile = 50;

met/src/basic/vx_config/config_constants.h

@@ -535,6 +535,8 @@ static const char conf_key_message_type[]      = "message_type";
 static const char conf_key_sid_inc[]           = "sid_inc";
 static const char conf_key_sid_exc[]           = "sid_exc";
 static const char conf_key_obs_qty[]           = "obs_quality";
+static const char conf_key_obs_qty_inc[]       = "obs_quality_inc";
+static const char conf_key_obs_qty_exc[]       = "obs_quality_exc";
 static const char conf_key_convert[]           = "convert";
 static const char conf_key_censor_thresh[]     = "censor_thresh";
 static const char conf_key_censor_val[]        = "censor_val";

met/src/basic/vx_config/config_util.cc

@@ -766,11 +766,36 @@ vector<MaskLatLon> parse_conf_llpnt_mask(Dictionary *dict) {
 
 ///////////////////////////////////////////////////////////////////////////////
 
-StringArray parse_conf_obs_qty(Dictionary *dict) {
-   const char *method_name = "parse_conf_obs_qty() -> ";
+StringArray parse_conf_obs_qty_inc(Dictionary *dict) {
+   StringArray sa;
+   const char *method_name = "parse_conf_obs_qty_inc() -> ";
+
+   // Check for old "obs_quality" entry
+   sa = dict->lookup_string_array(conf_key_obs_qty, false);
 
-   StringArray sa = parse_conf_string_array(dict, conf_key_obs_qty, method_name);
+   // Print a warning if the deprecated option was used
+   if(dict->last_lookup_status()) {
+      mlog << Warning << "\nparse_conf_obs_qty_inc() -> "
+           << "Set the \"" << conf_key_obs_qty_inc << "\" value ("
+           << write_css(sa) << ") from the deprecated \""
+           << conf_key_obs_qty << "\" configuration entry.\n"
+           << "Replace \"" << conf_key_obs_qty << "\" with \""
+           << conf_key_obs_qty_inc << "\"!\n\n";
+   }
+   else {
+      sa = parse_conf_string_array(dict, conf_key_obs_qty_inc, method_name);
+   }
+
+   return(sa);
+}
+
+///////////////////////////////////////////////////////////////////////////////
 
+StringArray parse_conf_obs_qty_exc(Dictionary *dict) {
+   const char *method_name = "parse_conf_obs_qty_exc() -> ";
+
+   StringArray sa = parse_conf_string_array(dict, conf_key_obs_qty_exc, method_name);
+
    return(sa);
 }
 

met/src/basic/vx_config/config_util.h

@@ -40,7 +40,8 @@ extern StringArray     parse_conf_sid_list(Dictionary *dict, const char *);
 extern void            parse_sid_mask(const ConcatString &, StringArray &, ConcatString &);
 extern vector<MaskLatLon>
                        parse_conf_llpnt_mask(Dictionary *dict);
-extern StringArray     parse_conf_obs_qty(Dictionary *dict);
+extern StringArray     parse_conf_obs_qty_inc(Dictionary *dict);
+extern StringArray     parse_conf_obs_qty_exc(Dictionary *dict);
 extern NumArray        parse_conf_ci_alpha(Dictionary *dict);
 extern NumArray        parse_conf_eclv_points(Dictionary *dict);
 extern ClimoCDFInfo    parse_conf_climo_cdf(Dictionary *dict);

met/src/libcode/vx_statistics/pair_data_ensemble.cc

@@ -913,8 +913,9 @@ void VxPairDataEnsemble::clear() {
 
    sid_inc_filt.clear();
    sid_exc_filt.clear();
-   obs_qty_filt.clear();
-
+   obs_qty_inc_filt.clear();
+   obs_qty_exc_filt.clear();
+
    obs_error_info = (ObsErrorInfo *) 0;
 
    fcst_ut = (unixtime) 0;
@@ -954,7 +955,8 @@ void VxPairDataEnsemble::assign(const VxPairDataEnsemble &vx_pd) {
    end_ut         = vx_pd.end_ut;
    sid_inc_filt   = vx_pd.sid_inc_filt;
    sid_exc_filt   = vx_pd.sid_exc_filt;
-   obs_qty_filt   = vx_pd.obs_qty_filt;
+   obs_qty_inc_filt = vx_pd.obs_qty_inc_filt;
+   obs_qty_exc_filt = vx_pd.obs_qty_exc_filt;
    obs_error_info = vx_pd.obs_error_info;
 
    interp_thresh  = vx_pd.interp_thresh;
@@ -1132,9 +1134,18 @@ void VxPairDataEnsemble::set_sid_exc_filt(const StringArray sa) {
 
 ////////////////////////////////////////////////////////////////////////
 
-void VxPairDataEnsemble::set_obs_qty_filt(const StringArray q) {
+void VxPairDataEnsemble::set_obs_qty_inc_filt(const StringArray q) {
 
-   obs_qty_filt = q;
+   obs_qty_inc_filt = q;
+
+   return;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+void VxPairDataEnsemble::set_obs_qty_exc_filt(const StringArray q) {
+
+   obs_qty_exc_filt = q;
 
    return;
 }
@@ -1368,16 +1379,13 @@ void VxPairDataEnsemble::add_point_obs(float *hdr_arr, int *hdr_typ_arr,
    else if(obs_info_grib->code() != nint(obs_arr[1])) {
       return;
    }
-
-   // Check if the observation quality flag is included in the list
-   if(obs_qty_filt.n() && strcmp(obs_qty, "")) {
-      bool qty_match = false;
-      for(i=0; i<obs_qty_filt.n() && !qty_match; i++)
-         if( obs_qty == obs_qty_filt[i]) qty_match = true;
-
-      if(!qty_match) return;
+
+   // Check the observation quality include and exclude options
+   if((obs_qty_inc_filt.n() > 0 && !obs_qty_inc_filt.has(obs_qty)) ||
+      (obs_qty_exc_filt.n() > 0 &&  obs_qty_exc_filt.has(obs_qty))) {
+      return;
    }
-
+   
    // Check whether the observation time falls within the valid time
    // window
    if(hdr_ut < beg_ut || hdr_ut > end_ut) return;

met/src/libcode/vx_statistics/pair_data_ensemble.h

@@ -205,8 +205,9 @@ class VxPairDataEnsemble {
 
       StringArray sid_inc_filt;  // Station ID inclusion list
       StringArray sid_exc_filt;  // Station ID exclusion list
-      StringArray obs_qty_filt;  // Observation quality markers
-
+      StringArray obs_qty_inc_filt;  // Observation quality include markers
+      StringArray obs_qty_exc_filt;  // Observation quality exclude markers
+
       //////////////////////////////////////////////////////////////////
 
       ObsErrorInfo *obs_error_info; // Pointer for observation error
@@ -249,8 +250,9 @@ class VxPairDataEnsemble {
 
       void set_sid_inc_filt(const StringArray);
       void set_sid_exc_filt(const StringArray);
-      void set_obs_qty_filt(const StringArray);
-
+      void set_obs_qty_inc_filt(const StringArray);
+      void set_obs_qty_exc_filt(const StringArray);
+
       // Call set_pd_size before set_msg_typ, set_mask_area, and set_interp
       void set_pd_size(int, int, int);