feature 1599 set climo_mean/stdev using VAR<n> variable syntax

docs/Users_Guide/wrappers.rst

@@ -231,6 +231,9 @@ METplus Configuration
 | :term:`ENSEMBLE_STAT_SKIP_CONST`
 | :term:`ENSEMBLE_STAT_OBS_ERROR_FLAG`
 | :term:`ENSEMBLE_STAT_CLIMO_MEAN_FILE_NAME`
+| :term:`ENSEMBLE_STAT_CLIMO_MEAN_VAR<n>_NAME`
+| :term:`ENSEMBLE_STAT_CLIMO_MEAN_VAR<n>_LEVELS`
+| :term:`ENSEMBLE_STAT_CLIMO_MEAN_VAR<n>_OPTIONS`
 | :term:`ENSEMBLE_STAT_CLIMO_MEAN_FIELD`
 | :term:`ENSEMBLE_STAT_CLIMO_MEAN_REGRID_METHOD`
 | :term:`ENSEMBLE_STAT_CLIMO_MEAN_REGRID_WIDTH`
@@ -243,6 +246,9 @@ METplus Configuration
 | :term:`ENSEMBLE_STAT_CLIMO_MEAN_USE_FCST`
 | :term:`ENSEMBLE_STAT_CLIMO_MEAN_USE_OBS`
 | :term:`ENSEMBLE_STAT_CLIMO_STDEV_FILE_NAME`
+| :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR<n>_NAME`
+| :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR<n>_LEVELS`
+| :term:`ENSEMBLE_STAT_CLIMO_STDEV_VAR<n>_OPTIONS`
 | :term:`ENSEMBLE_STAT_CLIMO_STDEV_FIELD`
 | :term:`ENSEMBLE_STAT_CLIMO_STDEV_REGRID_METHOD`
 | :term:`ENSEMBLE_STAT_CLIMO_STDEV_REGRID_WIDTH`
@@ -1118,6 +1124,9 @@ METplus Configuration
 | :term:`GEN_ENS_PROD_NMEP_SMOOTH_METHOD`
 | :term:`GEN_ENS_PROD_NMEP_SMOOTH_WIDTH`
 | :term:`GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME`
+| :term:`GEN_ENS_PROD_CLIMO_MEAN_VAR<n>_NAME`
+| :term:`GEN_ENS_PROD_CLIMO_MEAN_VAR<n>_LEVELS`
+| :term:`GEN_ENS_PROD_CLIMO_MEAN_VAR<n>_OPTIONS`
 | :term:`GEN_ENS_PROD_CLIMO_MEAN_FIELD`
 | :term:`GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD`
 | :term:`GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH`
@@ -1130,6 +1139,9 @@ METplus Configuration
 | :term:`GEN_ENS_PROD_CLIMO_MEAN_USE_FCST`
 | :term:`GEN_ENS_PROD_CLIMO_MEAN_USE_OBS`
 | :term:`GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME`
+| :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR<n>_NAME`
+| :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR<n>_LEVELS`
+| :term:`GEN_ENS_PROD_CLIMO_STDEV_VAR<n>_OPTIONS`
 | :term:`GEN_ENS_PROD_CLIMO_STDEV_FIELD`
 | :term:`GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD`
 | :term:`GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH`
@@ -2917,6 +2929,9 @@ METplus Configuration
 | :term:`FCST_GRID_STAT_FILE_TYPE`
 | :term:`OBS_GRID_STAT_FILE_TYPE`
 | :term:`GRID_STAT_CLIMO_MEAN_FILE_NAME`
+| :term:`GRID_STAT_CLIMO_MEAN_VAR<n>_NAME`
+| :term:`GRID_STAT_CLIMO_MEAN_VAR<n>_LEVELS`
+| :term:`GRID_STAT_CLIMO_MEAN_VAR<n>_OPTIONS`
 | :term:`GRID_STAT_CLIMO_MEAN_FIELD`
 | :term:`GRID_STAT_CLIMO_MEAN_REGRID_METHOD`
 | :term:`GRID_STAT_CLIMO_MEAN_REGRID_WIDTH`
@@ -2929,6 +2944,9 @@ METplus Configuration
 | :term:`GRID_STAT_CLIMO_MEAN_USE_FCST`
 | :term:`GRID_STAT_CLIMO_MEAN_USE_OBS`
 | :term:`GRID_STAT_CLIMO_STDEV_FILE_NAME`
+| :term:`GRID_STAT_CLIMO_STDEV_VAR<n>_NAME`
+| :term:`GRID_STAT_CLIMO_STDEV_VAR<n>_LEVELS`
+| :term:`GRID_STAT_CLIMO_STDEV_VAR<n>_OPTIONS`
 | :term:`GRID_STAT_CLIMO_STDEV_FIELD`
 | :term:`GRID_STAT_CLIMO_STDEV_REGRID_METHOD`
 | :term:`GRID_STAT_CLIMO_STDEV_REGRID_WIDTH`
@@ -5456,6 +5474,9 @@ Configuration
 | :term:`POINT_STAT_INTERP_TYPE_METHOD`
 | :term:`POINT_STAT_INTERP_TYPE_WIDTH`
 | :term:`POINT_STAT_CLIMO_MEAN_FILE_NAME`
+| :term:`POINT_STAT_CLIMO_MEAN_VAR<n>_NAME`
+| :term:`POINT_STAT_CLIMO_MEAN_VAR<n>_LEVELS`
+| :term:`POINT_STAT_CLIMO_MEAN_VAR<n>_OPTIONS`
 | :term:`POINT_STAT_CLIMO_MEAN_FIELD`
 | :term:`POINT_STAT_CLIMO_MEAN_REGRID_METHOD`
 | :term:`POINT_STAT_CLIMO_MEAN_REGRID_WIDTH`
@@ -5468,6 +5489,9 @@ Configuration
 | :term:`POINT_STAT_CLIMO_MEAN_USE_FCST`
 | :term:`POINT_STAT_CLIMO_MEAN_USE_OBS`
 | :term:`POINT_STAT_CLIMO_STDEV_FILE_NAME`
+| :term:`POINT_STAT_CLIMO_STDEV_VAR<n>_NAME`
+| :term:`POINT_STAT_CLIMO_STDEV_VAR<n>_LEVELS`
+| :term:`POINT_STAT_CLIMO_STDEV_VAR<n>_OPTIONS`
 | :term:`POINT_STAT_CLIMO_STDEV_FIELD`
 | :term:`POINT_STAT_CLIMO_STDEV_REGRID_METHOD`
 | :term:`POINT_STAT_CLIMO_STDEV_REGRID_WIDTH`
@@ -6093,6 +6117,9 @@ METplus Configuration
 | :term:`SERIES_ANALYSIS_TC_STAT_INPUT_TEMPLATE`
 | :term:`SERIES_ANALYSIS_OUTPUT_TEMPLATE`
 | :term:`SERIES_ANALYSIS_CLIMO_MEAN_FILE_NAME`
+| :term:`SERIES_ANALYSIS_CLIMO_MEAN_VAR<n>_NAME`
+| :term:`SERIES_ANALYSIS_CLIMO_MEAN_VAR<n>_LEVELS`
+| :term:`SERIES_ANALYSIS_CLIMO_MEAN_VAR<n>_OPTIONS`
 | :term:`SERIES_ANALYSIS_CLIMO_MEAN_FIELD`
 | :term:`SERIES_ANALYSIS_CLIMO_MEAN_REGRID_METHOD`
 | :term:`SERIES_ANALYSIS_CLIMO_MEAN_REGRID_WIDTH`
@@ -6106,6 +6133,9 @@ METplus Configuration
 | :term:`SERIES_ANALYSIS_CLIMO_MEAN_USE_FCST`
 | :term:`SERIES_ANALYSIS_CLIMO_MEAN_USE_OBS`
 | :term:`SERIES_ANALYSIS_CLIMO_STDEV_FILE_NAME`
+| :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR<n>_NAME`
+| :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR<n>_LEVELS`
+| :term:`SERIES_ANALYSIS_CLIMO_STDEV_VAR<n>_OPTIONS`
 | :term:`SERIES_ANALYSIS_CLIMO_STDEV_FIELD`
 | :term:`SERIES_ANALYSIS_CLIMO_STDEV_REGRID_METHOD`
 | :term:`SERIES_ANALYSIS_CLIMO_STDEV_REGRID_WIDTH`

internal_tests/pytests/command_builder/test_command_builder.py

@@ -1,76 +1,12 @@
 #!/usr/bin/env python3
 
-import os
-import sys
-import re
-import logging
-from collections import namedtuple
-import produtil
 import pytest
-import datetime
-from metplus.wrappers.command_builder import CommandBuilder
-from metplus.util import time_util
-from metplus.util import METConfig
 
+import os
+import datetime
 
-@pytest.mark.parametrize(
-    'config_overrides, expected_value', [
-        # 0 no climo variables set
-        ({}, ''),
-        # 1 file name set only
-        ({'FILE_NAME': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'},
-         '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'),
-        # 2 input template set
-        ({'INPUT_TEMPLATE': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'},
-         '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'),
-        # 3 input template and dir set
-        ({'INPUT_DIR': '/mean/dir',
-          'INPUT_TEMPLATE': 'gs_climo_{init?fmt=%Y%m%d%H}.tmpl'},
-         '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'),
-        # 4 input template and dir set multiple templates
-        ({'INPUT_DIR': '/mean/dir',
-          'INPUT_TEMPLATE': 'gs_climo_1.tmpl, gs_climo_2.tmpl'},
-         '/mean/dir/gs_climo_1.tmpl,/mean/dir/gs_climo_2.tmpl'),
-        # 5file name, input template and dir all set
-        ({'FILE_NAME': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl',
-          'INPUT_DIR': '/mean/dir',
-          'INPUT_TEMPLATE': 'gs_climo_1.tmpl, gs_climo_2.tmpl'},
-         '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'),
-        # 6 input template is python embedding keyword and dir is set
-        ({'INPUT_DIR': '/mean/dir',
-          'INPUT_TEMPLATE': 'PYTHON_NUMPY'},
-         'PYTHON_NUMPY'),
-        # 7 input template is python embedding keyword and dir is set
-        ({'INPUT_DIR': '/mean/dir',
-          'INPUT_TEMPLATE': 'PYTHON_XARRAY'},
-         'PYTHON_XARRAY'),
-    ]
-)
-def test_read_climo_file_name(metplus_config, config_overrides,
-                                expected_value):
-    # name of app used for testing to read/set config variables
-    app_name = 'grid_stat'
-
-    # check mean and stdev climo variables
-    for climo_type in CommandBuilder.climo_types:
-        prefix = f'{app_name.upper()}_CLIMO_{climo_type.upper()}_'
-
-        config = metplus_config()
-
-        # set config values
-        for key, value in config_overrides.items():
-            config.set('config', f'{prefix}{key}', value)
-
-        cbw = CommandBuilder(config)
-
-        # set app_name to grid_stat for testing
-        cbw.app_name = app_name
-
-        cbw.read_climo_file_name(climo_type)
-        actual_value = cbw.config.getraw('config',
-                                         f'{prefix}FILE_NAME',
-                                         '')
-        assert actual_value == expected_value
+from metplus.wrappers.command_builder import CommandBuilder
+from metplus.util import ti_calculate
 
 # ------------------------
 #  test_find_data_no_dated
@@ -93,7 +29,7 @@ def test_find_data_no_dated(metplus_config, data_type):
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000",'%Y%m%d%H%M')
     task_info['lead'] = 0
-    time_info = time_util.ti_calculate(task_info)
+    time_info = ti_calculate(task_info)
 
     pcw.c_dict[f'{data_type}FILE_WINDOW_BEGIN'] = -3600
     pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 3600
@@ -120,7 +56,7 @@ def test_find_data_not_a_path(metplus_config, data_type):
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000",'%Y%m%d%H%M')
     task_info['lead'] = 0
-    time_info = time_util.ti_calculate(task_info)
+    time_info = ti_calculate(task_info)
 
     pcw.c_dict[f'{data_type}FILE_WINDOW_BEGIN'] = 0
     pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 0
@@ -138,7 +74,7 @@ def test_find_obs_no_dated(metplus_config):
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
     task_info['lead'] = 0
-    time_info = time_util.ti_calculate(task_info)
+    time_info = ti_calculate(task_info)
 
     pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600
     pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
@@ -156,7 +92,7 @@ def test_find_obs_dated(metplus_config):
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
     task_info['lead'] = 0
-    time_info = time_util.ti_calculate(task_info)
+    time_info = ti_calculate(task_info)
 
     pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600
     pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
@@ -184,7 +120,7 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds)
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("2020020112", '%Y%m%d%H')
     task_info['lead'] = 0
-    time_info = time_util.ti_calculate(task_info)
+    time_info = ti_calculate(task_info)
 
     pcw.c_dict['OFFSETS'] = offsets
     pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE') + "/internal_tests/data/obs"
@@ -209,7 +145,7 @@ def test_find_obs_dated_previous_day(metplus_config):
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
     task_info['lead'] = 0
-    time_info = time_util.ti_calculate(task_info)
+    time_info = ti_calculate(task_info)
 
     pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs"
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}'
@@ -227,7 +163,7 @@ def test_find_obs_dated_next_day(metplus_config):
     task_info = {}
     task_info['valid'] = datetime.datetime.strptime("201802012345", '%Y%m%d%H%M')
     task_info['lead'] = 0
-    time_info = time_util.ti_calculate(task_info)
+    time_info = ti_calculate(task_info)
 
     pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs"
     pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}'

internal_tests/pytests/met_config/test_met_config.py

@@ -3,6 +3,45 @@
 import pytest
 
 from metplus.util.met_config import *
+from metplus.util.met_config import _read_climo_file_name, _read_climo_field
+from metplus.util import CLIMO_TYPES
+
+@pytest.mark.parametrize(
+    'config_overrides, expected_value', [
+        # 0 no relevant config set
+        ({}, ''),
+        # 1 _FIELD set
+        ({'GRID_STAT_CLIMO_MEAN_FIELD': '{name="TMP"; level="(*,*)";}'},
+         '{name="TMP"; level="(*,*)";}'),
+        # 2 VAR1 name/level set
+        ({'GRID_STAT_CLIMO_MEAN_VAR1_NAME': 'TMP',
+          'GRID_STAT_CLIMO_MEAN_VAR1_LEVELS': '"(*,*)"'},
+         '{ name="TMP"; level="(*,*)"; }'),
+        # 3 VAR1/2 name/level set
+        ({'GRID_STAT_CLIMO_MEAN_VAR1_NAME': 'TMP',
+          'GRID_STAT_CLIMO_MEAN_VAR1_LEVELS': '"(*,*)"',
+          'GRID_STAT_CLIMO_MEAN_VAR2_NAME': 'PRES',
+          'GRID_STAT_CLIMO_MEAN_VAR2_LEVELS': '"(0,*,*)"'},
+         '{ name="TMP"; level="(*,*)"; },{ name="PRES"; level="(0,*,*)"; }'),
+        # 4 VAR1 name/level and FIELD set - prefer VAR<n>
+        ({'GRID_STAT_CLIMO_MEAN_FIELD': '{name="TEMP"; level="(0,*,*)";}',
+          'GRID_STAT_CLIMO_MEAN_VAR1_NAME': 'TMP',
+          'GRID_STAT_CLIMO_MEAN_VAR1_LEVELS': '"(*,*)"'},
+         '{ name="TMP"; level="(*,*)"; }'),
+    ]
+)
+def test_read_climo_field(metplus_config, config_overrides, expected_value):
+    app_name = 'grid_stat'
+    climo_type = 'MEAN'
+    expected_var = f'{app_name}_CLIMO_{climo_type}_FIELD'.upper()
+    config = metplus_config()
+
+    # set config values
+    for key, value in config_overrides.items():
+        config.set('config', key, value)
+
+    _read_climo_field(climo_type, config, app_name)
+    assert config.getraw('config', expected_var) == expected_value
 
 @pytest.mark.parametrize(
     'name, data_type, mp_configs, extra_args', [
@@ -57,3 +96,57 @@ def test_set_met_config_function(data_type, expected_function):
 )
 def test_format_regrid_to_grid(input, output):
     assert format_regrid_to_grid(input) == output
+
+@pytest.mark.parametrize(
+    'config_overrides, expected_value', [
+        # 0 no climo variables set
+        ({}, ''),
+        # 1 file name set only
+        ({'FILE_NAME': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'},
+         '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'),
+        # 2 input template set
+        ({'INPUT_TEMPLATE': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'},
+         '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'),
+        # 3 input template and dir set
+        ({'INPUT_DIR': '/mean/dir',
+          'INPUT_TEMPLATE': 'gs_climo_{init?fmt=%Y%m%d%H}.tmpl'},
+         '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'),
+        # 4 input template and dir set multiple templates
+        ({'INPUT_DIR': '/mean/dir',
+          'INPUT_TEMPLATE': 'gs_climo_1.tmpl, gs_climo_2.tmpl'},
+         '/mean/dir/gs_climo_1.tmpl,/mean/dir/gs_climo_2.tmpl'),
+        # 5file name, input template and dir all set
+        ({'FILE_NAME': '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl',
+          'INPUT_DIR': '/mean/dir',
+          'INPUT_TEMPLATE': 'gs_climo_1.tmpl, gs_climo_2.tmpl'},
+         '/mean/dir/gs_climo_{init?fmt=%Y%m%d%H}.tmpl'),
+        # 6 input template is python embedding keyword and dir is set
+        ({'INPUT_DIR': '/mean/dir',
+          'INPUT_TEMPLATE': 'PYTHON_NUMPY'},
+         'PYTHON_NUMPY'),
+        # 7 input template is python embedding keyword and dir is set
+        ({'INPUT_DIR': '/mean/dir',
+          'INPUT_TEMPLATE': 'PYTHON_XARRAY'},
+         'PYTHON_XARRAY'),
+    ]
+)
+def test_read_climo_file_name(metplus_config, config_overrides,
+                              expected_value):
+    # name of app used for testing to read/set config variables
+    app_name = 'grid_stat'
+
+    # check mean and stdev climo variables
+    for climo_type in CLIMO_TYPES:
+        prefix = f'{app_name.upper()}_CLIMO_{climo_type.upper()}_'
+
+        config = metplus_config()
+
+        # set config values
+        for key, value in config_overrides.items():
+            config.set('config', f'{prefix}{key}', value)
+
+        _read_climo_file_name(climo_type=climo_type,
+                              config=config,
+                              app_name=app_name)
+        actual_value = config.getraw('config', f'{prefix}FILE_NAME', '')
+        assert actual_value == expected_value

metplus/util/constants.py

@@ -1,2 +1,37 @@
-COMPRESSION_EXTENSIONS = ['.gz', '.bz2', '.zip']
+# supported file extensions that will automatically be uncompressed
+COMPRESSION_EXTENSIONS = [
+    '.gz',
+    '.bz2',
+    '.zip',
+]
 
+# Keywords of valid types of Python Embedding in MET tools
+PYTHON_EMBEDDING_TYPES = [
+    'PYTHON_NUMPY',
+    'PYTHON_XARRAY',
+    'PYTHON_PANDAS',
+]
+
+# types of climatology values that should be checked and set
+CLIMO_TYPES = [
+    'MEAN',
+    'STDEV',
+]
+
+# comparison operators that the MET tools support
+# key is an operator and value is the alphabetic equivalent
+VALID_COMPARISONS = {
+    '>=': 'ge',
+    '>': 'gt',
+    '==': 'eq',
+    '!=': 'ne',
+    '<=': 'le',
+    '<': 'lt',
+}
+
+# wrappers that do not run shell commands
+# used to check that at least one command was generated if it should
+NO_COMMAND_WRAPPERS = (
+    'Example',
+    'CyclonePlotter',
+)