Bugfix #2096/#2098 develop - fix skip if output exists and do not err…

…or if no commands were run (#2099)

internal/tests/pytests/wrappers/command_builder/test_command_builder.py

@@ -855,3 +855,60 @@ def test_get_field_info_extra(metplus_config, extra, expected_value):
         add_curly_braces=False
     )[0]
     assert actual_value == expected_value
+
+
+@pytest.mark.parametrize(
+    'exists, skip, is_dir, use_prefix, run', [
+            (True, True, False, True, False),
+            (True, False, False, True, True),
+            (False, True, False, True, True),
+            (False, False, False, True, True),
+            (True, True, True, True, False),
+            (True, False, True, True, True),
+            (False, True, True, True, True),
+            (False, False, True, True, True),
+            (True, True, False, False, False),
+            (True, False, False, False, True),
+            (False, True, False, False, True),
+            (False, False, False, False, True),
+            (True, True, True, False, False),
+            (True, False, True, False, True),
+            (False, True, True, False, True),
+            (False, False, True, False, True),
+        ]
+)
+@pytest.mark.wrapper
+def test_find_and_check_output_file_skip(metplus_config, exists, skip, is_dir,
+                                         use_prefix, run):
+    app_name = 'command_builder'
+    config = metplus_config
+    if use_prefix:
+        config.set('config', f'{app_name.upper()}_OUTPUT_PREFIX', 'prefix')
+    wrapper = CommandBuilder(config)
+    wrapper.app_name = app_name
+    prefix = f'{app_name}_prefix' if use_prefix else app_name
+    exist_file = f'{prefix}_120000L_20190201_000000V.stat'
+    non_exist_file = f'{prefix}_240000L_20190201_000000V.stat'
+
+    # create fake file to test
+    create_fullpath = os.path.join(config.getdir('OUTPUT_BASE'), exist_file)
+    open(create_fullpath, 'a').close()
+
+    # set time_info, output template/dir, skip if output exists flag
+    task_info = {'valid': datetime.datetime(2019, 2, 1, 0)}
+    if is_dir:
+        task_info['lead_hours'] = 12 if exists else 24
+
+    time_info = ti_calculate(task_info)
+    wrapper.c_dict['OUTPUT_DIR'] = wrapper.config.getdir('OUTPUT_BASE')
+
+    wrapper.c_dict['SKIP_IF_OUTPUT_EXISTS'] = skip
+    if is_dir:
+        wrapper.c_dict['OUTPUT_TEMPLATE'] = ''
+    else:
+        wrapper.c_dict['OUTPUT_TEMPLATE'] = exist_file if exists else non_exist_file
+
+    result = wrapper.find_and_check_output_file(time_info, is_directory=is_dir)
+
+    # cast result to bool because None isn't equal to False
+    assert bool(result) == run

internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py

@@ -23,41 +23,6 @@ def pb2nc_wrapper(metplus_config):
     return PB2NCWrapper(config)
 
 
-@pytest.mark.parametrize(
-    'exists, skip, run', [
-            (True, True, False),
-            (True, False, True),
-            (False, True, True),
-            (False, False, True),
-        ]
-)
-@pytest.mark.wrapper
-def test_find_and_check_output_file_skip(metplus_config, exists, skip, run):
-    pb = pb2nc_wrapper(metplus_config)
-    exist_file = 'wackyfilenametocreate'
-    non_exist_file = 'wackyfilethatdoesntexist'
-
-    # create fake file to test
-    create_fullpath = os.path.join(pb.config.getdir('OUTPUT_BASE'), exist_file)
-    open(create_fullpath, 'a').close()
-
-    # set time_info, output template/dir, skip if output exists flag
-    time_info = { 'valid' : datetime.datetime(2019, 2, 1, 0) }
-    pb.c_dict['OUTPUT_DIR'] = pb.config.getdir('OUTPUT_BASE')
-
-    pb.c_dict['SKIP_IF_OUTPUT_EXISTS'] = skip
-
-    if exists:
-        pb.c_dict['OUTPUT_TEMPLATE'] = exist_file
-    else:
-        pb.c_dict['OUTPUT_TEMPLATE'] = non_exist_file
-
-    result = pb.find_and_check_output_file(time_info)
-
-    # cast result to bool because None isn't equal to False
-    assert bool(result) == run
-
-
 # ---------------------
 # test_get_command
 # test that command is generated correctly

metplus/util/config_util.py

@@ -163,8 +163,8 @@ def write_all_commands(all_commands, config):
     @returns False if no commands were provided, True otherwise
     """
     if not all_commands:
-        config.logger.error("No commands were run. "
-                            "Skip writing all_commands file")
+        config.logger.info("No commands were run. "
+                           "Skip writing all_commands file")
         return False
 
     log_timestamp = config.getstr('config', 'LOG_TIMESTAMP')

metplus/util/run_util.py

@@ -147,9 +147,7 @@ def run_metplus(config):
         # if process list contains any wrapper that should run commands
         if any([item[0] not in NO_COMMAND_WRAPPERS for item in process_list]):
             # write out all commands and environment variables to file
-            if not write_all_commands(all_commands, config):
-                # report an error if no commands were generated
-                total_errors += 1
+            write_all_commands(all_commands, config)
 
         # compute total number of errors that occurred and output results
         for process in processes:

metplus/wrappers/command_builder.py

@@ -21,7 +21,7 @@
 from ..util.constants import PYTHON_EMBEDDING_TYPES, COMPRESSION_EXTENSIONS
 from ..util import getlist, preprocess_file, loop_over_times_and_call
 from ..util import do_string_sub, ti_calculate, get_seconds_from_string
-from ..util import get_time_from_file, shift_time_seconds
+from ..util import get_time_from_file, shift_time_seconds, seconds_to_met_time
 from ..util import replace_config_from_section
 from ..util import METConfig
 from ..util import MISSING_DATA_VALUE
@@ -917,7 +917,7 @@ def find_and_check_output_file(self, time_info=None,
              template
             @param is_directory If True, check in output directory for
              any files that match the pattern
-             {app_name}_{output_prefix}*YYYYMMDD_HHMMSSV*
+             {app_name}_{output_prefix}_HHMMSSL_YYYYMMDD_HHMMSSV*
             @param output_path_template optional filename template to use
              If None, build output path template from c_dict's OUTPUT_DIR
               and OUTPUT_TEMPLATE. Default is None
@@ -955,13 +955,18 @@ def find_and_check_output_file(self, time_info=None,
         # get directory that the output file will exist
         if is_directory:
             parent_dir = output_path
-            if time_info and time_info['valid'] != '*':
-                valid_format = time_info['valid'].strftime('%Y%m%d_%H%M%S')
-            else:
-                valid_format = ''
+            valid = '*'
+            lead = '*'
+            if time_info:
+                if time_info['valid'] != '*':
+                    valid = time_info['valid'].strftime('%Y%m%d_%H%M%S')
+                if time_info['lead'] != '*':
+                    lead = seconds_to_met_time(time_info['lead_seconds'],
+                                               force_hms=True)
 
             prefix = self.get_output_prefix(time_info, set_env_vars=False)
-            search_string = f"{self.app_name}_{prefix}*{valid_format}V*"
+            prefix = f'{self.app_name}_{prefix}' if prefix else self.app_name
+            search_string = f'{prefix}_{lead}L_{valid}V*'
             search_path = os.path.join(output_path,
                                        search_string)
             if skip_if_output_exists: