Feature #344 rearrange utils

.github/jobs/get_use_case_commands.py

@@ -14,7 +14,7 @@
 sys.path.insert(0, METPLUS_TOP_DIR)
 
 from internal.tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite
-from metplus.util.met_util import expand_int_string_to_list
+from metplus.util.string_manip import expand_int_string_to_list
 from docker_utils import VERSION_EXT
 
 

docs/Contributors_Guide/deprecation.rst

@@ -26,7 +26,7 @@ wrong variable and it is using WGRIB2 = wgrib2.
 
 check_for_deprecated_config()
 -----------------------------
-In **met_util.py** there is a function called
+In **metplus/util/config_metplus.py** there is a function called
 check_for_deprecated_config. It contains a dictionary of dictionaries
 called deprecated_dict that specifies the old config name, the section
 it was found in, and a suggested alternative (None if no alternative

docs/Users_Guide/getting_started.rst

@@ -327,7 +327,7 @@ user configuration file and
 
 The last line of the screen output should match this format::
 
-    05/04 09:42:52.277 metplus (met_util.py:212) INFO: METplus has successfully finished running.
+    05/04 09:42:52.277 metplus INFO: METplus has successfully finished running.
 
 If this log message is not shown, there is likely an issue with one or more
 of the default configuration variable overrides in the
@@ -339,7 +339,7 @@ how the :ref:`common_config_variables` control a use case run.
 If the run was successful, the line above the success message should contain
 the path to the METplus log file that was generated::
 
-    05/04 09:44:21.534 metplus (met_util.py:211) INFO: Check the log file for more information: /path/to/output/logs/metplus.log.20210504094421
+    05/04 09:44:21.534 metplus INFO: Check the log file for more information: /path/to/output/logs/metplus.log.20210504094421
 
 * Review the log file and compare it to the Example.conf use case
   configuration file to see how the settings correspond to the result.

docs/Users_Guide/systemconfiguration.rst

@@ -426,7 +426,7 @@ This defines the format of the ERROR log messages. Setting the value to::
 
 Produces a log file with ERROR lines that match this format::
 
-    04/29 16:03:34.858 metplus (met_util.py:218) ERROR: METplus has finished running but had 1 error.
+    04/29 16:03:34.858 metplus (run_util.py:192) ERROR: METplus has finished running but had 1 error.
 
 The format of the timestamp is set by
 :ref:`LOG_LINE_DATE_FORMAT<log_line_date_format>`.
@@ -442,7 +442,7 @@ This defines the format of the DEBUG log messages. Setting the value to::
 
 Produces a log file with DEBUG lines that match this format::
 
-    04/29 15:54:22.851 metplus (met_util.py:207) DEBUG: METplus took 0:00:00.850983 to run.
+    04/29 15:54:22.851 metplus (run_util.py:177) DEBUG: METplus took 0:00:00.850983 to run.
 
 The format of the timestamp is set by
 :ref:`LOG_LINE_DATE_FORMAT<log_line_date_format>`.
@@ -2648,9 +2648,9 @@ In most cases, there is a simple one-to-one relationship between a deprecated co
 
 Example::
 
-    (met_util.py) ERROR: DEPRECATED CONFIG ITEMS WERE FOUND. PLEASE REMOVE/REPLACE THEM FROM CONFIG FILES
-    (met_util.py) ERROR: [dir] MODEL_DATA_DIR should be replaced with EXTRACT_TILES_GRID_INPUT_DIR
-    (met_util.py) ERROR: [config] STAT_LIST should be replaced with SERIES_ANALYSIS_STAT_LIST
+    ERROR: DEPRECATED CONFIG ITEMS WERE FOUND. PLEASE REMOVE/REPLACE THEM FROM CONFIG FILES
+    ERROR: [dir] MODEL_DATA_DIR should be replaced with EXTRACT_TILES_GRID_INPUT_DIR
+    ERROR: [config] STAT_LIST should be replaced with SERIES_ANALYSIS_STAT_LIST
 
 These cases can be handled automatically by using the :ref:`validate_config`.
 
@@ -2666,10 +2666,10 @@ Starting in METplus 3.0, users are required to either explicitly set both FCST_*
 
 Example::
 
-    (met_util.py) ERROR: If FCST_VAR1_NAME is set, the user must either set OBS_VAR1_NAME or change FCST_VAR1_NAME to BOTH_VAR1_NAME
-    (met_util.py) ERROR: If FCST_VAR2_NAME is set, the user must either set OBS_VAR2_NAME or change FCST_VAR2_NAME to BOTH_VAR2_NAME
-    (met_util.py) ERROR: If FCST_VAR1_LEVELS is set, the user must either set OBS_VAR1_LEVELS or change FCST_VAR1_LEVELS to BOTH_VAR1_LEVELS
-    (met_util.py) ERROR: If FCST_VAR2_LEVELS is set, the user must either set OBS_VAR2_LEVELS or change FCST_VAR2_LEVELS to BOTH_VAR2_LEVELS
+    ERROR: If FCST_VAR1_NAME is set, the user must either set OBS_VAR1_NAME or change FCST_VAR1_NAME to BOTH_VAR1_NAME
+    ERROR: If FCST_VAR2_NAME is set, the user must either set OBS_VAR2_NAME or change FCST_VAR2_NAME to BOTH_VAR2_NAME
+    ERROR: If FCST_VAR1_LEVELS is set, the user must either set OBS_VAR1_LEVELS or change FCST_VAR1_LEVELS to BOTH_VAR1_LEVELS
+    ERROR: If FCST_VAR2_LEVELS is set, the user must either set OBS_VAR2_LEVELS or change FCST_VAR2_LEVELS to BOTH_VAR2_LEVELS
 
 These cases can be handled automatically by using the :ref:`validate_config`, but users should review the suggested changes, as they may want to update differently.
 
@@ -2682,7 +2682,7 @@ Instead of only being able to specify FCST_PCP_COMBINE_INPUT_LEVEL, users can no
 
 Example::
 
-    (met_util.py) ERROR: [config] OBS_PCP_COMBINE_INPUT_LEVEL should be replaced with OBS_PCP_COMBINE_INPUT_ACCUMS
+    ERROR: [config] OBS_PCP_COMBINE_INPUT_LEVEL should be replaced with OBS_PCP_COMBINE_INPUT_ACCUMS
 
 These cases can be handled automatically by using the :ref:`validate_config`, but users should review the suggested changes, as they may want to include other available input accumulations.
 
@@ -2719,17 +2719,17 @@ Due to these changes, MET configuration files that refer to any of these depreca
 
 Example log output::
 
-    (met_util.py) DEBUG: Checking for deprecated environment variables in: DeprecatedConfig
-    (met_util.py) ERROR: Please remove deprecated environment variable ${GRID_VX} found in MET config file: DeprecatedConfig
-    (met_util.py) ERROR: MET to_grid variable should reference ${REGRID_TO_GRID} environment variable
-    (met_util.py) INFO: Be sure to set GRID_STAT_REGRID_TO_GRID to the correct value.
+    DEBUG: Checking for deprecated environment variables in: DeprecatedConfig
+    ERROR: Please remove deprecated environment variable ${GRID_VX} found in MET config file: DeprecatedConfig
+    ERROR: MET to_grid variable should reference ${REGRID_TO_GRID} environment variable
+    INFO: Be sure to set GRID_STAT_REGRID_TO_GRID to the correct value.
 
-    (met_util.py) ERROR: Please remove deprecated environment variable ${MET_VALID_HHMM} found in MET config file: DeprecatedConfig
-    (met_util.py) ERROR: Set GRID_STAT_CLIMO_MEAN_INPUT_[DIR/TEMPLATE] in a METplus config file to set CLIMO_MEAN_FILE in a MET config
+    ERROR: Please remove deprecated environment variable ${MET_VALID_HHMM} found in MET config file: DeprecatedConfig
+    ERROR: Set GRID_STAT_CLIMO_MEAN_INPUT_[DIR/TEMPLATE] in a METplus config file to set CLIMO_MEAN_FILE in a MET config
 
-    (met_util.py) ERROR: output_prefix variable should reference ${OUTPUT_PREFIX} environment variable
-    (met_util.py) INFO: GRID_STAT_OUTPUT_PREFIX will need to be added to the METplus config file that sets GRID_STAT_CONFIG_FILE. Set it to:
-    (met_util.py) INFO: GRID_STAT_OUTPUT_PREFIX = {CURRENT_FCST_NAME}_vs_{CURRENT_OBS_NAME}
+    ERROR: output_prefix variable should reference ${OUTPUT_PREFIX} environment variable
+    INFO: GRID_STAT_OUTPUT_PREFIX will need to be added to the METplus config file that sets GRID_STAT_CONFIG_FILE. Set it to:
+    INFO: GRID_STAT_OUTPUT_PREFIX = {CURRENT_FCST_NAME}_vs_{CURRENT_OBS_NAME}
 
 These cases can be handled automatically by using the :ref:`validate_config`, but users should review the suggested changes and make sure they add the appropriate recommended METplus configuration variables to their files to achieve the same behavior.
 

docs/use_cases/met_tool_wrapper/Example/Example.py

@@ -174,30 +174,30 @@
 #
 # You should also see a series of log output listing init/valid times, forecast lead times, and filenames derived from the filename templates. Here is an excerpt::
 #
-#   12/30 19:44:02.901 metplus (met_util.py:425) INFO: ****************************************
-#   12/30 19:44:02.901 metplus (met_util.py:426) INFO: * Running METplus
-#   12/30 19:44:02.902 metplus (met_util.py:432) INFO: *  at valid time: 201702010000
-#   12/30 19:44:02.902 metplus (met_util.py:435) INFO: ****************************************
-#   12/30 19:44:02.902 metplus.Example (example_wrapper.py:58) INFO: Running ExampleWrapper at valid time 20170201000000
-#   12/30 19:44:02.902 metplus.Example (example_wrapper.py:63) INFO: Input directory is /dir/containing/example/data
-#   12/30 19:44:02.902 metplus.Example (example_wrapper.py:64) INFO: Input template is {init?fmt=%Y%m%d}/file_{init?fmt=%Y%m%d}_{init?fmt=%2H}_F{lead?fmt=%3H}.ext
-#   12/30 19:44:02.902 metplus.Example (example_wrapper.py:79) INFO: Processing forecast lead 3 hours initialized at 2017-01-31 21Z and valid at 2017-02-01 00Z
-#   12/30 19:44:02.903 metplus.Example (example_wrapper.py:88) INFO: Looking in input directory for file: 20170131/file_20170131_21_F003.ext
-#   12/30 19:44:02.903 metplus.Example (example_wrapper.py:79) INFO: Processing forecast lead 6 hours initialized at 2017-01-31 18Z and valid at 2017-02-01 00Z
-#   12/30 19:44:02.903 metplus.Example (example_wrapper.py:88) INFO: Looking in input directory for file: 20170131/file_20170131_18_F006.ext
-#   12/30 19:44:02.904 metplus.Example (example_wrapper.py:79) INFO: Processing forecast lead 9 hours initialized at 2017-01-31 15Z and valid at 2017-02-01 00Z
-#   12/30 19:44:02.904 metplus.Example (example_wrapper.py:88) INFO: Looking in input directory for file: 20170131/file_20170131_15_F009.ext
-#   12/30 19:44:02.904 metplus.Example (example_wrapper.py:79) INFO: Processing forecast lead 12 hours initialized at 2017-01-31 12Z and valid at 2017-02-01 00Z
-#   12/30 19:44:02.904 metplus.Example (example_wrapper.py:88) INFO: Looking in input directory for file: 20170131/file_20170131_12_F012.ext
-#   12/30 19:44:02.904 metplus (met_util.py:425) INFO: ****************************************
-#   12/30 19:44:02.904 metplus (met_util.py:426) INFO: * Running METplus
-#   12/30 19:44:02.905 metplus (met_util.py:432) INFO: *  at valid time: 201702010600
-#   12/30 19:44:02.905 metplus (met_util.py:435) INFO: ****************************************
-#   12/30 19:44:02.905 metplus.Example (example_wrapper.py:58) INFO: Running ExampleWrapper at valid time 20170201060000
-#   12/30 19:44:02.905 metplus.Example (example_wrapper.py:63) INFO: Input directory is /dir/containing/example/data
-#   12/30 19:44:02.905 metplus.Example (example_wrapper.py:64) INFO: Input template is {init?fmt=%Y%m%d}/file_{init?fmt=%Y%m%d}_{init?fmt=%2H}_F{lead?fmt=%3H}.ext
-#   12/30 19:44:02.905 metplus.Example (example_wrapper.py:79) INFO: Processing forecast lead 3 hours initialized at 2017-02-01 03Z and valid at 2017-02-01 06Z
-#   12/30 19:44:02.906 metplus.Example (example_wrapper.py:88) INFO: Looking in input directory for file: 20170201/file_20170201_03_F003.ext
+#   12/30 19:44:02.901 metplus INFO: ****************************************
+#   12/30 19:44:02.901 metplus INFO: * Running METplus
+#   12/30 19:44:02.902 metplus INFO: *  at valid time: 201702010000
+#   12/30 19:44:02.902 metplus INFO: ****************************************
+#   12/30 19:44:02.902 metplus INFO: Running ExampleWrapper at valid time 20170201000000
+#   12/30 19:44:02.902 metplus INFO: Input directory is /dir/containing/example/data
+#   12/30 19:44:02.902 metplus INFO: Input template is {init?fmt=%Y%m%d}/file_{init?fmt=%Y%m%d}_{init?fmt=%2H}_F{lead?fmt=%3H}.ext
+#   12/30 19:44:02.902 metplus INFO: Processing forecast lead 3 hours initialized at 2017-01-31 21Z and valid at 2017-02-01 00Z
+#   12/30 19:44:02.903 metplus INFO: Looking in input directory for file: 20170131/file_20170131_21_F003.ext
+#   12/30 19:44:02.903 metplus INFO: Processing forecast lead 6 hours initialized at 2017-01-31 18Z and valid at 2017-02-01 00Z
+#   12/30 19:44:02.903 metplus INFO: Looking in input directory for file: 20170131/file_20170131_18_F006.ext
+#   12/30 19:44:02.904 metplus INFO: Processing forecast lead 9 hours initialized at 2017-01-31 15Z and valid at 2017-02-01 00Z
+#   12/30 19:44:02.904 metplus INFO: Looking in input directory for file: 20170131/file_20170131_15_F009.ext
+#   12/30 19:44:02.904 metplus INFO: Processing forecast lead 12 hours initialized at 2017-01-31 12Z and valid at 2017-02-01 00Z
+#   12/30 19:44:02.904 metplus INFO: Looking in input directory for file: 20170131/file_20170131_12_F012.ext
+#   12/30 19:44:02.904 metplus INFO: ****************************************
+#   12/30 19:44:02.904 metplus INFO: * Running METplus
+#   12/30 19:44:02.905 metplus INFO: *  at valid time: 201702010600
+#   12/30 19:44:02.905 metplus INFO: ****************************************
+#   12/30 19:44:02.905 metplus INFO: Running ExampleWrapper at valid time 20170201060000
+#   12/30 19:44:02.905 metplus INFO: Input directory is /dir/containing/example/data
+#   12/30 19:44:02.905 metplus INFO: Input template is {init?fmt=%Y%m%d}/file_{init?fmt=%Y%m%d}_{init?fmt=%2H}_F{lead?fmt=%3H}.ext
+#   12/30 19:44:02.905 metplus INFO: Processing forecast lead 3 hours initialized at 2017-02-01 03Z and valid at 2017-02-01 06Z
+#   12/30 19:44:02.906 metplus INFO: Looking in input directory for file: 20170201/file_20170201_03_F003.ext
 #
 
 ##############################################################################

internal/tests/pytests/util/config/test_config.py

@@ -6,8 +6,7 @@
 from configparser import NoOptionError
 from shutil import which, rmtree
 
-from metplus.util import met_util as util
-
+from metplus.util.constants import MISSING_DATA_VALUE
 
 @pytest.mark.parametrize(
     'input_value, result', [
@@ -178,14 +177,14 @@ def test_getexe(metplus_config, input_value, result):
     'input_value, default, result', [
         ('1.1', None, 1.1),
         ('1.1', 2.2, 1.1),
-        (None, None, util.MISSING_DATA_VALUE),
+        (None, None, MISSING_DATA_VALUE),
         (None, 1.1, 1.1),
         ('integer', None, None),
         ('integer', 1.1, None),
         ('0', None, 0.0),
         ('0', 2.2, 0.0),
-        ('', None, util.MISSING_DATA_VALUE),
-        ('', 2.2, util.MISSING_DATA_VALUE),
+        ('', None, MISSING_DATA_VALUE),
+        ('', 2.2, MISSING_DATA_VALUE),
     ]
 )
 def test_getfloat(metplus_config, input_value, default, result):
@@ -205,7 +204,7 @@ def test_getfloat(metplus_config, input_value, default, result):
     'input_value, default, result', [
         ('1', None, 1),
         ('1', 2, 1),
-        (None, None, util.MISSING_DATA_VALUE),
+        (None, None, MISSING_DATA_VALUE),
         (None, 1, 1),
         ('integer', None, None),
         ('integer', 1, None),
@@ -214,8 +213,8 @@ def test_getfloat(metplus_config, input_value, default, result):
         ('1.7', 2, None),
         ('1.0', None, None),
         ('1.0', 2, None),
-        ('', None, util.MISSING_DATA_VALUE),
-        ('', 2.2, util.MISSING_DATA_VALUE),
+        ('', None, MISSING_DATA_VALUE),
+        ('', 2.2, MISSING_DATA_VALUE),
     ]
 )
 @pytest.mark.util

internal/tests/pytests/util/config_metplus/test_config_metplus.py

@@ -7,7 +7,7 @@
 from datetime import datetime
 
 from metplus.util import config_metplus
-
+from metplus.util.time_util import ti_calculate
 
 @pytest.mark.util
 def test_get_default_config_list():
@@ -150,9 +150,9 @@ def test_find_var_indices_fcst(metplus_config,
     data_types = ['FCST']
     config.set('config', config_var_name, "NAME1")
     met_tool = 'grid_stat' if set_met_tool else None
-    var_name_indices = config_metplus.find_var_name_indices(config,
-                                                            data_types=data_types,
-                                                            met_tool=met_tool)
+    var_name_indices = config_metplus._find_var_name_indices(config,
+                                                             data_types=data_types,
+                                                             met_tool=met_tool)
 
     assert len(var_name_indices) == len(expected_indices)
     for actual_index in var_name_indices:
@@ -648,7 +648,7 @@ def test_find_var_indices_wrapper_specific(metplus_config, met_tool, indices):
     conf.set('config', f'{data_type}_VAR1_NAME', "NAME1")
     conf.set('config', f'{data_type}_GRID_STAT_VAR2_NAME', "GSNAME2")
 
-    var_name_indices = config_metplus.find_var_name_indices(conf, data_types=[data_type],
+    var_name_indices = config_metplus._find_var_name_indices(conf,data_types=[data_type],
                                                   met_tool=met_tool)
 
     assert var_name_indices == indices
@@ -1073,3 +1073,95 @@ def test_getraw_instance_with_unset_var(metplus_config):
     )
     new_config.set('config', 'CURRENT_FCST_NAME', 'NAME')
     assert new_config.getraw('config', 'OUTPUT_PREFIX') == 'FCST_NAME'
+
+
+@pytest.mark.parametrize(
+    'config_value, expected_result', [
+        # 2 items semi-colon at end
+        ('GRIB_lvl_typ = 234;  desc = "HI_CLOUD";',
+         'GRIB_lvl_typ = 234; desc = "HI_CLOUD";'),
+        # 2 items no semi-colon at end
+        ('GRIB_lvl_typ = 234;  desc = "HI_CLOUD"',
+         'GRIB_lvl_typ = 234; desc = "HI_CLOUD";'),
+        # 1 item semi-colon at end
+        ('GRIB_lvl_typ = 234;',
+         'GRIB_lvl_typ = 234;'),
+        # 1 item no semi-colon at end
+        ('GRIB_lvl_typ = 234',
+         'GRIB_lvl_typ = 234;'),
+    ]
+)
+@pytest.mark.util
+def test_format_var_items_options_semicolon(config_value,
+                                            expected_result):
+    time_info = {}
+
+    field_configs = {'name': 'FNAME',
+                     'levels': 'FLEVEL',
+                     'options': config_value}
+
+    var_items = config_metplus._format_var_items(field_configs, time_info)
+    result = var_items.get('extra')
+    assert result == expected_result
+
+
+@pytest.mark.parametrize(
+    'input_dict, expected_list', [
+        ({'init': datetime(2019, 2, 1, 6),
+          'lead': 7200, },
+         [
+             {'index': '1',
+              'fcst_name': 'FNAME_2019',
+              'fcst_level': 'Z06',
+              'obs_name': 'ONAME_2019',
+              'obs_level': 'L06',
+             },
+             {'index': '1',
+              'fcst_name': 'FNAME_2019',
+              'fcst_level': 'Z08',
+              'obs_name': 'ONAME_2019',
+              'obs_level': 'L08',
+             },
+         ]),
+        ({'init': datetime(2021, 4, 13, 9),
+          'lead': 10800, },
+         [
+             {'index': '1',
+              'fcst_name': 'FNAME_2021',
+              'fcst_level': 'Z09',
+              'obs_name': 'ONAME_2021',
+              'obs_level': 'L09',
+              },
+             {'index': '1',
+              'fcst_name': 'FNAME_2021',
+              'fcst_level': 'Z12',
+              'obs_name': 'ONAME_2021',
+              'obs_level': 'L12',
+              },
+         ]),
+    ]
+)
+@pytest.mark.util
+def test_sub_var_list(metplus_config, input_dict, expected_list):
+    config = metplus_config
+    config.set('config', 'FCST_VAR1_NAME', 'FNAME_{init?fmt=%Y}')
+    config.set('config', 'FCST_VAR1_LEVELS', 'Z{init?fmt=%H}, Z{valid?fmt=%H}')
+    config.set('config', 'OBS_VAR1_NAME', 'ONAME_{init?fmt=%Y}')
+    config.set('config', 'OBS_VAR1_LEVELS', 'L{init?fmt=%H}, L{valid?fmt=%H}')
+
+    time_info = ti_calculate(input_dict)
+
+    actual_temp = config_metplus.parse_var_list(config)
+
+    pp = pprint.PrettyPrinter()
+    print(f'Actual var list (before sub):')
+    pp.pprint(actual_temp)
+
+    actual_list = config_metplus.sub_var_list(actual_temp, time_info)
+    print(f'Actual var list (after sub):')
+    pp.pprint(actual_list)
+
+    assert len(actual_list) == len(expected_list)
+    for actual, expected in zip(actual_list, expected_list):
+        for key, value in expected.items():
+            assert actual.get(key) == value