Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bugfix #2096/#2098 develop - fix skip if output exists and do not error if no commands were run #2099

Merged
merged 3 commits into from
Apr 4, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -855,3 +855,60 @@ def test_get_field_info_extra(metplus_config, extra, expected_value):
add_curly_braces=False
)[0]
assert actual_value == expected_value


@pytest.mark.parametrize(
'exists, skip, is_dir, use_prefix, run', [
(True, True, False, True, False),
(True, False, False, True, True),
(False, True, False, True, True),
(False, False, False, True, True),
(True, True, True, True, False),
(True, False, True, True, True),
(False, True, True, True, True),
(False, False, True, True, True),
(True, True, False, False, False),
(True, False, False, False, True),
(False, True, False, False, True),
(False, False, False, False, True),
(True, True, True, False, False),
(True, False, True, False, True),
(False, True, True, False, True),
(False, False, True, False, True),
]
)
@pytest.mark.wrapper
def test_find_and_check_output_file_skip(metplus_config, exists, skip, is_dir,
use_prefix, run):
app_name = 'command_builder'
config = metplus_config
if use_prefix:
config.set('config', f'{app_name.upper()}_OUTPUT_PREFIX', 'prefix')
wrapper = CommandBuilder(config)
wrapper.app_name = app_name
prefix = f'{app_name}_prefix' if use_prefix else app_name
exist_file = f'{prefix}_120000L_20190201_000000V.stat'
non_exist_file = f'{prefix}_240000L_20190201_000000V.stat'

# create fake file to test
create_fullpath = os.path.join(config.getdir('OUTPUT_BASE'), exist_file)
open(create_fullpath, 'a').close()

# set time_info, output template/dir, skip if output exists flag
task_info = {'valid': datetime.datetime(2019, 2, 1, 0)}
if is_dir:
task_info['lead_hours'] = 12 if exists else 24

time_info = ti_calculate(task_info)
wrapper.c_dict['OUTPUT_DIR'] = wrapper.config.getdir('OUTPUT_BASE')

wrapper.c_dict['SKIP_IF_OUTPUT_EXISTS'] = skip
if is_dir:
wrapper.c_dict['OUTPUT_TEMPLATE'] = ''
else:
wrapper.c_dict['OUTPUT_TEMPLATE'] = exist_file if exists else non_exist_file

result = wrapper.find_and_check_output_file(time_info, is_directory=is_dir)

# cast result to bool because None isn't equal to False
assert bool(result) == run
35 changes: 0 additions & 35 deletions internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,41 +23,6 @@ def pb2nc_wrapper(metplus_config):
return PB2NCWrapper(config)


@pytest.mark.parametrize(
'exists, skip, run', [
(True, True, False),
(True, False, True),
(False, True, True),
(False, False, True),
]
)
@pytest.mark.wrapper
def test_find_and_check_output_file_skip(metplus_config, exists, skip, run):
pb = pb2nc_wrapper(metplus_config)
exist_file = 'wackyfilenametocreate'
non_exist_file = 'wackyfilethatdoesntexist'

# create fake file to test
create_fullpath = os.path.join(pb.config.getdir('OUTPUT_BASE'), exist_file)
open(create_fullpath, 'a').close()

# set time_info, output template/dir, skip if output exists flag
time_info = { 'valid' : datetime.datetime(2019, 2, 1, 0) }
pb.c_dict['OUTPUT_DIR'] = pb.config.getdir('OUTPUT_BASE')

pb.c_dict['SKIP_IF_OUTPUT_EXISTS'] = skip

if exists:
pb.c_dict['OUTPUT_TEMPLATE'] = exist_file
else:
pb.c_dict['OUTPUT_TEMPLATE'] = non_exist_file

result = pb.find_and_check_output_file(time_info)

# cast result to bool because None isn't equal to False
assert bool(result) == run


# ---------------------
# test_get_command
# test that command is generated correctly
Expand Down
4 changes: 2 additions & 2 deletions metplus/util/config_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,8 @@ def write_all_commands(all_commands, config):
@returns False if no commands were provided, True otherwise
"""
if not all_commands:
config.logger.error("No commands were run. "
"Skip writing all_commands file")
config.logger.info("No commands were run. "
"Skip writing all_commands file")
return False

log_timestamp = config.getstr('config', 'LOG_TIMESTAMP')
Expand Down
4 changes: 1 addition & 3 deletions metplus/util/run_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,9 +147,7 @@ def run_metplus(config):
# if process list contains any wrapper that should run commands
if any([item[0] not in NO_COMMAND_WRAPPERS for item in process_list]):
# write out all commands and environment variables to file
if not write_all_commands(all_commands, config):
# report an error if no commands were generated
total_errors += 1
write_all_commands(all_commands, config)

# compute total number of errors that occurred and output results
for process in processes:
Expand Down
19 changes: 12 additions & 7 deletions metplus/wrappers/command_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from ..util.constants import PYTHON_EMBEDDING_TYPES, COMPRESSION_EXTENSIONS
from ..util import getlist, preprocess_file, loop_over_times_and_call
from ..util import do_string_sub, ti_calculate, get_seconds_from_string
from ..util import get_time_from_file, shift_time_seconds
from ..util import get_time_from_file, shift_time_seconds, seconds_to_met_time
from ..util import replace_config_from_section
from ..util import METConfig
from ..util import MISSING_DATA_VALUE
Expand Down Expand Up @@ -917,7 +917,7 @@ def find_and_check_output_file(self, time_info=None,
template
@param is_directory If True, check in output directory for
any files that match the pattern
{app_name}_{output_prefix}*YYYYMMDD_HHMMSSV*
{app_name}_{output_prefix}_HHMMSSL_YYYYMMDD_HHMMSSV*
@param output_path_template optional filename template to use
If None, build output path template from c_dict's OUTPUT_DIR
and OUTPUT_TEMPLATE. Default is None
Expand Down Expand Up @@ -955,13 +955,18 @@ def find_and_check_output_file(self, time_info=None,
# get directory that the output file will exist
if is_directory:
parent_dir = output_path
if time_info and time_info['valid'] != '*':
valid_format = time_info['valid'].strftime('%Y%m%d_%H%M%S')
else:
valid_format = ''
valid = '*'
lead = '*'
if time_info:
if time_info['valid'] != '*':
valid = time_info['valid'].strftime('%Y%m%d_%H%M%S')
if time_info['lead'] != '*':
lead = seconds_to_met_time(time_info['lead_seconds'],
force_hms=True)

prefix = self.get_output_prefix(time_info, set_env_vars=False)
search_string = f"{self.app_name}_{prefix}*{valid_format}V*"
prefix = f'{self.app_name}_{prefix}' if prefix else self.app_name
search_string = f'{prefix}_{lead}L_{valid}V*'
search_path = os.path.join(output_path,
search_string)
if skip_if_output_exists:
Expand Down