From 389e921d924678418c022f3fb8aa8eb2e3b5ad20 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 12 Nov 2021 15:05:00 -0700 Subject: [PATCH] Per #1213, add support for setting obs_quality_inc and obs_quality_exc in PointStat and EnsembleStat. POINT_STAT_OBS_QUALITY will now set obs_quality_inc if POINT_STAT_OBS_QUALITY_[INC/INCLUDE] are not set. Added documentation and unit tests for each new setting. Added commented out example in each basic use case config file. --- docs/Users_Guide/glossary.rst | 22 +++++++++- docs/Users_Guide/wrappers.rst | 44 +++++++++++++++++-- .../test_ensemble_stat_wrapper.py | 4 ++ .../point_stat/test_point_stat_wrapper.py | 6 ++- metplus/wrappers/ensemble_stat_wrapper.py | 15 +++++++ metplus/wrappers/point_stat_wrapper.py | 13 ++++-- parm/met_config/EnsembleStatConfig_wrapped | 8 +++- parm/met_config/PointStatConfig_wrapped | 9 +++- .../EnsembleStat/EnsembleStat.conf | 3 ++ .../met_tool_wrapper/PointStat/PointStat.conf | 4 +- 10 files changed, 114 insertions(+), 14 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index c924aded30..3484ae5beb 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -6661,11 +6661,19 @@ METplus Configuration Glossary | *Used by:* MODE - POINT_STAT_OBS_QUALITY - Specify the value for 'obs_quality' in the MET configuration file for PointStat. + POINT_STAT_OBS_QUALITY_INC + Specify the value for 'obs_quality_inc' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_OBS_QUALITY_EXC + Specify the value for 'obs_quality_exc' in the MET configuration file for PointStat. | *Used by:* PointStat + POINT_STAT_OBS_QUALITY + .. warning:: **DEPRECATED:** Please use :term:`POINT_STAT_OBS_QUALITY_INC` instead. + POINT_STAT_OUTPUT_FLAG_FHO Specify the value for 'output_flag.fho' in the MET configuration file for PointStat. @@ -8257,3 +8265,13 @@ METplus Configuration Glossary Specify the value for 'ens.file_type' in the MET configuration file for GenEnsProd. | *Used by:* GenEnsProd + + ENSEMBLE_STAT_OBS_QUALITY_INC + Specify the value for 'obs_quality_inc' in the MET configuration file for EnsembleStat. + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_OBS_QUALITY_EXC + Specify the value for 'obs_quality_exc' in the MET configuration file for EnsembleStat. + + | *Used by:* EnsembleStat diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 8b07448f8a..18fc97d57e 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -274,6 +274,8 @@ METplus Configuration | :term:`ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP` | :term:`ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK` | :term:`ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT` +| :term:`ENSEMBLE_STAT_OBS_QUALITY_INC` +| :term:`ENSEMBLE_STAT_OBS_QUALITY_EXC` | :term:`ENSEMBLE_STAT_MET_CONFIG_OVERRIDES` | :term:`ENSEMBLE_STAT_VERIFICATION_MASK_TEMPLATE` (optional) | :term:`ENS_VAR_NAME` (optional) @@ -829,6 +831,28 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`ENSEMBLE_STAT_OUTPUT_PREFIX` - output_prefix +**${METPLUS_OBS_QUALITY_INC}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`ENSEMBLE_STAT_OBS_QUALITY_INC` + - obs_quality_inc + +**${METPLUS_OBS_QUALITY_EXC}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`ENSEMBLE_STAT_OBS_QUALITY_EXC` + - obs_quality_exc + **${METPLUS_MET_CONFIG_OVERRIDES}** .. list-table:: @@ -4854,7 +4878,8 @@ Configuration | :term:`POINT_STAT_CLIMO_CDF_BINS` | :term:`POINT_STAT_CLIMO_CDF_CENTER_BINS` | :term:`POINT_STAT_CLIMO_CDF_WRITE_BINS` -| :term:`POINT_STAT_OBS_QUALITY` +| :term:`POINT_STAT_OBS_QUALITY_INC` +| :term:`POINT_STAT_OBS_QUALITY_EXC` | :term:`POINT_STAT_OUTPUT_FLAG_FHO` | :term:`POINT_STAT_OUTPUT_FLAG_CTC` | :term:`POINT_STAT_OUTPUT_FLAG_CTS` @@ -5196,7 +5221,18 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`POINT_STAT_CLIMO_CDF_WRITE_BINS` - climo_cdf.write_bins -**${METPLUS_OBS_QUALITY}** +**${METPLUS_OBS_QUALITY_INC}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT_STAT_OBS_QUALITY_INC` + - obs_quality_inc + +**${METPLUS_OBS_QUALITY_EXC}** .. list-table:: :widths: 5 5 @@ -5204,8 +5240,8 @@ see :ref:`How METplus controls MET config file settings`. * - METplus Config(s) - MET Config File - * - :term:`POINT_STAT_OBS_QUALITY` - - obs_quality + * - :term:`POINT_STAT_OBS_QUALITY_EXC` + - obs_quality_exc **${METPLUS_OUTPUT_FLAG_DICT}** diff --git a/internal_tests/pytests/ensemble_stat/test_ensemble_stat_wrapper.py b/internal_tests/pytests/ensemble_stat/test_ensemble_stat_wrapper.py index d0f525654f..750fe994af 100644 --- a/internal_tests/pytests/ensemble_stat/test_ensemble_stat_wrapper.py +++ b/internal_tests/pytests/ensemble_stat/test_ensemble_stat_wrapper.py @@ -540,6 +540,10 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, 'type = [{method = GAUSSIAN;width = 1;}];}' ) }), + ({'ENSEMBLE_STAT_OBS_QUALITY_INC': '2,3,4', }, + {'METPLUS_OBS_QUALITY_INC': 'obs_quality_inc = ["2", "3", "4"];'}), + ({'ENSEMBLE_STAT_OBS_QUALITY_EXC': '5,6,7', }, + {'METPLUS_OBS_QUALITY_EXC': 'obs_quality_exc = ["5", "6", "7"];'}), ] ) diff --git a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py index 7155d1d1b8..e8c2302f35 100755 --- a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py +++ b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py @@ -178,8 +178,12 @@ def test_met_dictionary_in_var_options(metplus_config): { 'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {cdf_bins = 1.0;center_bins = TRUE;write_bins = FALSE;}'}), + ({'POINT_STAT_OBS_QUALITY_INC': '2,3,4', }, + {'METPLUS_OBS_QUALITY_INC': 'obs_quality_inc = ["2", "3", "4"];'}), + ({'POINT_STAT_OBS_QUALITY_EXC': '5,6,7', }, + {'METPLUS_OBS_QUALITY_EXC': 'obs_quality_exc = ["5", "6", "7"];'}), ({'POINT_STAT_OBS_QUALITY': '1, 2, 3', }, - {'METPLUS_OBS_QUALITY': 'obs_quality = ["1", "2", "3"];'}), + {'METPLUS_OBS_QUALITY_INC': 'obs_quality_inc = ["1", "2", "3"];'}), ({'POINT_STAT_OUTPUT_FLAG_FHO': 'BOTH', }, {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {fho = BOTH;}'}), diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index e171987be8..652170ffbe 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -61,6 +61,8 @@ class EnsembleStatWrapper(CompareGriddedWrapper): 'METPLUS_OUTPUT_FLAG_DICT', 'METPLUS_ENSEMBLE_FLAG_DICT', 'METPLUS_OUTPUT_PREFIX', + 'METPLUS_OBS_QUALITY_INC', + 'METPLUS_OBS_QUALITY_EXC', ] # handle deprecated env vars used pre v4.0.0 @@ -299,6 +301,19 @@ def create_c_dict(self): c_dict['MASK_POLY_TEMPLATE'] = self.read_mask_poly() + self.add_met_config( + name='obs_quality_inc', + data_type='list', + metplus_configs=['ENSEMBLE_STAT_OBS_QUALITY_INC', + 'ENSEMBLE_STAT_OBS_QUALITY_INCLUDE'] + ) + self.add_met_config( + name='obs_quality_exc', + data_type='list', + metplus_configs=['ENSEMBLE_STAT_OBS_QUALITY_EXC', + 'ENSEMBLE_STAT_OBS_QUALITY_EXCLUDE'] + ) + # old method of setting MET config values c_dict['ENS_THRESH'] = ( self.config.getstr('config', 'ENSEMBLE_STAT_ENS_THRESH', '1.0') diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index fb1887a356..a21df7caad 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -33,7 +33,8 @@ class PointStatWrapper(CompareGriddedWrapper): 'METPLUS_MASK_SID', 'METPLUS_OUTPUT_PREFIX', 'METPLUS_CLIMO_CDF_DICT', - 'METPLUS_OBS_QUALITY', + 'METPLUS_OBS_QUALITY_INC', + 'METPLUS_OBS_QUALITY_EXC', 'METPLUS_OUTPUT_FLAG_DICT', 'METPLUS_INTERP_DICT', 'METPLUS_CLIMO_MEAN_DICT', @@ -194,9 +195,15 @@ def create_c_dict(self): False) ) - self.add_met_config(name='obs_quality', + self.add_met_config(name='obs_quality_inc', data_type='list', - metplus_configs=['POINT_STAT_OBS_QUALITY']) + metplus_configs=['POINT_STAT_OBS_QUALITY_INC', + 'POINT_STAT_OBS_QUALITY_INCLUDE', + 'POINT_STAT_OBS_QUALITY']) + self.add_met_config(name='obs_quality_exc', + data_type='list', + metplus_configs=['POINT_STAT_OBS_QUALITY_EXC', + 'POINT_STAT_OBS_QUALITY_EXCLUDE']) self.handle_flags('output') diff --git a/parm/met_config/EnsembleStatConfig_wrapped b/parm/met_config/EnsembleStatConfig_wrapped index 5331c5583c..e9ef2d5667 100644 --- a/parm/met_config/EnsembleStatConfig_wrapped +++ b/parm/met_config/EnsembleStatConfig_wrapped @@ -95,7 +95,13 @@ obs = { ${METPLUS_MESSAGE_TYPE} sid_exc = []; obs_thresh = [ NA ]; -obs_quality = []; + +//obs_quality_inc = +${METPLUS_OBS_QUALITY_INC} + +//obs_quality_exc = +${METPLUS_OBS_QUALITY_EXC} + ${METPLUS_DUPLICATE_FLAG} obs_summary = NONE; obs_perc_value = 50; diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index 2a0d8d1907..2a654d6d23 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -63,8 +63,13 @@ obs = { // message_type = ${METPLUS_MESSAGE_TYPE} sid_exc = []; -//obs_quality = -${METPLUS_OBS_QUALITY} + +//obs_quality_inc = +${METPLUS_OBS_QUALITY_INC} + +//obs_quality_exc = +${METPLUS_OBS_QUALITY_EXC} + duplicate_flag = NONE; obs_summary = NONE; obs_perc_value = 50; diff --git a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf index 261bff3325..c4c61ce8e7 100644 --- a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf +++ b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf @@ -169,6 +169,9 @@ ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP = FALSE ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK = TRUE ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT = FALSE +#ENSEMBLE_STAT_OBS_QUALITY_INC = +#ENSEMBLE_STAT_OBS_QUALITY_EXC = + # Ensemble Variables and levels as specified in the ens field dictionary # of the MET configuration file. Specify as ENS_VARn_NAME, ENS_VARn_LEVELS, # (optional) ENS_VARn_OPTION diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf index a290427772..36875e0a79 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf @@ -50,7 +50,9 @@ LOOP_ORDER = processes # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped -#POINT_STAT_OBS_QUALITY = 1, 2, 3 + +#POINT_STAT_OBS_QUALITY_INC = 1, 2, 3 +#POINT_STAT_OBS_QUALITY_EXC = POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST #POINT_STAT_CLIMO_STDEV_TIME_INTERP_METHOD =