moved more functions out of met_util

metplus/util/met_util.py

@@ -324,41 +324,6 @@ def get_files(filedir, filename_regex, logger=None):
                 continue
     return sorted(file_paths)
 
-def prune_empty(output_dir, logger):
-    """! Start from the output_dir, and recursively check
-        all directories and files.  If there are any empty
-        files or directories, delete/remove them so they
-        don't cause performance degradation or errors
-        when performing subsequent tasks.
-        Input:
-            @param output_dir:  The directory from which searching
-                                should begin.
-            @param logger: The logger to which all logging is
-                           directed.
-    """
-
-    # Check for empty files.
-    for root, dirs, files in os.walk(output_dir):
-        # Create a full file path by joining the path
-        # and filename.
-        for a_file in files:
-            a_file = os.path.join(root, a_file)
-            if os.stat(a_file).st_size == 0:
-                logger.debug("Empty file: " + a_file +
-                             "...removing")
-                os.remove(a_file)
-
-    # Now check for any empty directories, some
-    # may have been created when removing
-    # empty files.
-    for root, dirs, files in os.walk(output_dir):
-        for direc in dirs:
-            full_dir = os.path.join(root, direc)
-            if not os.listdir(full_dir):
-                logger.debug("Empty directory: " + full_dir +
-                             "...removing")
-                os.rmdir(full_dir)
-
 
 def shift_time_seconds(time_str, shift):
     """ Adjust time by shift seconds. Format is %Y%m%d%H%M%S
@@ -668,12 +633,6 @@ def preprocess_file(filename, data_type, config, allow_dir=False):
 
     return None
 
-def template_to_regex(template, time_info):
-    in_template = re.sub(r'\.', '\\.', template)
-    in_template = re.sub(r'{lead.*?}', '.*', in_template)
-    return do_string_sub(in_template,
-                         **time_info)
-
 
 def expand_int_string_to_list(int_string):
     """! Expand string into a list of integer values. Items are separated by
@@ -807,12 +766,6 @@ def netcdf_has_var(file_path, name, level):
     except (AttributeError, OSError, ImportError):
         return False
 
-def generate_tmp_filename():
-    import random
-    import string
-    random_string = ''.join(random.choice(string.ascii_letters)
-                            for i in range(10))
-    return f"metplus_tmp_{random_string}"
 
 def format_level(level):
     """! Format level string to prevent NetCDF level values from creating

metplus/util/string_manip.py

@@ -6,6 +6,8 @@
 
 import re
 from csv import reader
+import random
+import string
 
 from .constants import VALID_COMPARISONS
 
@@ -365,3 +367,14 @@ def round_0p5(val):
     @returns n.0, n.5, or (n+1).0 value as a result of rounding
     """
     return round(val * 2) / 2
+
+
+def generate_tmp_filename():
+    random_string = ''.join(random.choice(string.ascii_letters)
+                            for i in range(10))
+    return f"metplus_tmp_{random_string}"
+
+
+def template_to_regex(template):
+    in_template = re.sub(r'\.', '\\.', template)
+    return re.sub(r'{lead.*?}', '.*', in_template)

metplus/util/system_util.py

@@ -100,3 +100,37 @@ def get_storms(filter_filename, id_only=False, sort_column='STORM_ID'):
         storm_dict[storm] = [line for line in lines if storm in line]
 
     return storm_dict
+
+
+def prune_empty(output_dir, logger):
+    """! Start from the output_dir, and recursively check
+        all directories and files.  If there are any empty
+        files or directories, delete/remove them so they
+        don't cause performance degradation or errors
+        when performing subsequent tasks.
+
+        @param output_dir The directory from which searching should begin.
+        @param logger The logger to which all logging is directed.
+    """
+
+    # Check for empty files.
+    for root, dirs, files in os.walk(output_dir):
+        # Create a full file path by joining the path
+        # and filename.
+        for a_file in files:
+            a_file = os.path.join(root, a_file)
+            if os.stat(a_file).st_size == 0:
+                logger.debug("Empty file: " + a_file +
+                             "...removing")
+                os.remove(a_file)
+
+    # Now check for any empty directories, some
+    # may have been created when removing
+    # empty files.
+    for root, dirs, files in os.walk(output_dir):
+        for direc in dirs:
+            full_dir = os.path.join(root, direc)
+            if not os.listdir(full_dir):
+                logger.debug("Empty directory: " + full_dir +
+                             "...removing")
+                os.rmdir(full_dir)

metplus/wrappers/extract_tiles_wrapper.py

@@ -15,7 +15,7 @@
 
 from ..util import met_util as util
 from ..util import do_string_sub, ti_calculate, skip_time
-from ..util import parse_var_list, round_0p5, get_storms
+from ..util import parse_var_list, round_0p5, get_storms, prune_empty
 from .regrid_data_plane_wrapper import RegridDataPlaneWrapper
 from . import CommandBuilder
 
@@ -267,7 +267,7 @@ def run_at_time_loop_string(self, time_info):
         else:
             self.use_tc_stat_input(storm_dict, idx_dict)
 
-        util.prune_empty(self.c_dict['OUTPUT_DIR'], self.logger)
+        prune_empty(self.c_dict['OUTPUT_DIR'], self.logger)
 
     def use_tc_stat_input(self, storm_dict, idx_dict):
         """! Find storms in TCStat input file and create tiles using the storm.

metplus/wrappers/met_db_load_wrapper.py

@@ -13,10 +13,9 @@
 import os
 from datetime import datetime
 
-from ..util import met_util as util
-from ..util import time_util
+from ..util import ti_calculate
 from . import RuntimeFreqWrapper
-from ..util import do_string_sub, getlist
+from ..util import do_string_sub, getlist, generate_tmp_filename
 
 '''!@namespace METDbLoadWrapper
 @brief Parent class for wrappers that run over a grouping of times
@@ -118,7 +117,7 @@ def run_at_time_once(self, time_info):
         if time_info.get('lead') != '*':
             if (time_info.get('init') != '*'
                     or time_info.get('valid') != '*'):
-                time_info = time_util.ti_calculate(time_info)
+                time_info = ti_calculate(time_info)
 
         self.set_environment_variables(time_info)
 
@@ -234,7 +233,7 @@ def replace_values_in_xml(self, time_info):
             output_lines.append(output_line)
 
         # write tmp file with XML content with substituted values
-        out_filename = util.generate_tmp_filename()
+        out_filename = generate_tmp_filename()
         out_path = os.path.join(self.config.getdir('TMP_DIR'),
                                 out_filename)
         with open(out_path, 'w') as file_handle:

metplus/wrappers/pcp_combine_wrapper.py

@@ -12,7 +12,7 @@
 from ..util import get_seconds_from_string, ti_get_lead_string, ti_calculate
 from ..util import get_relativedelta, ti_get_seconds_from_relativedelta
 from ..util import time_string_to_met_time, seconds_to_met_time
-from ..util import parse_var_list
+from ..util import parse_var_list, template_to_regex
 from . import ReformatGriddedWrapper
 
 '''!@namespace PCPCombineWrapper
@@ -445,10 +445,10 @@ def setup_sum_method(self, time_info, lookback, data_src):
         out_accum = time_string_to_met_time(lookback, 'S')
 
         time_info['level'] = in_accum
-        pcp_regex = util.template_to_regex(
-            self.c_dict[f'{data_src}_INPUT_TEMPLATE'],
-            time_info
+        pcp_regex = template_to_regex(
+            self.c_dict[f'{data_src}_INPUT_TEMPLATE']
         )
+        pcp_regex = do_string_sub(pcp_regex, **time_info)
         pcp_regex_split = pcp_regex.split('/')
         pcp_dir = os.path.join(self.c_dict[f'{data_src}_INPUT_DIR'],
                                *pcp_regex_split[0:-1])
@@ -817,8 +817,7 @@ def get_field_string(self, time_info=None, search_accum=0, name=None,
 
         # string sub values into full field info string using search time info
         if time_info:
-            field_info = do_string_sub(field_info,
-                                       **time_info)
+            field_info = do_string_sub(field_info, **time_info)
         return field_info
 
     def find_input_file(self, init_time, valid_time, search_accum, data_src):