Skip to content

Commit

Permalink
Bugfix #2241 Create directory containing -out_stat file (#2242)
Browse files Browse the repository at this point in the history
  • Loading branch information
@georgemccabe
georgemccabe committed Jul 11, 2023
1 parent 2c977a3 commit 09dd7f4
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 48 deletions.
2 changes: 2 additions & 0 deletions docs/Contributors_Guide/testing.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@ information about the local environment so that the tests can run.
All unit tests must include one of the custom markers listed in the
internal/tests/pytests/pytest.ini file. Some examples include:

* run_metplus
* util
* wrapper_a
* wrapper_b
* wrapper_c
* wrapper_d
* wrapper
* long
* plotting
Expand Down
120 changes: 82 additions & 38 deletions internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
config_init_beg = '20170705'
config_init_end = '20170901'


def get_config(metplus_config):
# extra_configs = []
# extra_configs.append(os.path.join(os.path.dirname(__file__),
Expand Down Expand Up @@ -44,16 +45,56 @@ def get_config(metplus_config):
return config


def tc_stat_wrapper(metplus_config):
"""! Returns a default TCStatWrapper with /path/to entries in the
metplus_system.conf and metplus_runtime.conf configuration
files. Subsequent tests can customize the final METplus configuration
to over-ride these /path/to values."""

# Default, empty TcStatWrapper with some configuration values set
# to /path/to:
@pytest.mark.parametrize(
'config_overrides, expected_dirs, expected_job_string', [
# 0: dump_row and out_stat files
({'TC_STAT_JOB_ARGS': ("-job summary -line_type TCMPR -column "
"'ABS(AMAX_WIND-BMAX_WIND)' "
"-dump_row dump_row/summary.tcst, "
"-job summary -line_type TCMPR -column "
"'ABS(AMAX_WIND-BMAX_WIND)' "
"-out_stat out_stat/stat_summary.tcst")},
['dump_row', 'out_stat'],
('jobs = ["-job summary -line_type TCMPR -column \'ABS(AMAX_WIND-BMAX_WIND)\' -dump_row dump_row/summary.tcst",'
'"-job summary -line_type TCMPR -column \'ABS(AMAX_WIND-BMAX_WIND)\' -out_stat out_stat/stat_summary.tcst"];')),
# 1: dump_row file
({'TC_STAT_JOB_ARGS': ("-job summary -line_type TCMPR -column "
"'ABS(AMAX_WIND-BMAX_WIND)' "
"-dump_row dump_row/summary.tcst")}, ['dump_row'], 'jobs = ["-job summary -line_type TCMPR -column \'ABS(AMAX_WIND-BMAX_WIND)\' -dump_row dump_row/summary.tcst"];'),
# 2: out_stat file
({'TC_STAT_JOB_ARGS': ("-job summary -line_type TCMPR -column "
"'ABS(AMAX_WIND-BMAX_WIND)' "
"-out_stat out_stat/stat_summary.tcst")},
['out_stat'],
'jobs = ["-job summary -line_type TCMPR -column \'ABS(AMAX_WIND-BMAX_WIND)\' -out_stat out_stat/stat_summary.tcst"];'),
]
)
@pytest.mark.wrapper
def test_tc_stat_handle_jobs(metplus_config, config_overrides, expected_dirs,
expected_job_string):
config = get_config(metplus_config)
return TCStatWrapper(config)

# turn off "do not run" setting so directories are created
config.set('config', 'DO_NOT_RUN_EXE', False)

# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)

# initialize wrapper and ensure it was initialized properly
wrapper = TCStatWrapper(config)
assert wrapper.isOK

# ensure job string is formatted as expected
actual_job_string = wrapper.handle_jobs(time_info={})
assert actual_job_string == expected_job_string
assert wrapper.env_var_dict['METPLUS_JOBS'] == expected_job_string

# ensure output directories are created properly
for expected in expected_dirs:
expected_dir = os.path.join(wrapper.c_dict['OUTPUT_DIR'], expected)
assert os.path.exists(expected_dir) and os.path.isdir(expected_dir)

@pytest.mark.parametrize(
'config_overrides, env_var_values', [
Expand Down Expand Up @@ -197,6 +238,9 @@ def tc_stat_wrapper(metplus_config):
# 46 out_valid_mask
({'TC_STAT_OUT_VALID_MASK': 'MET_BASE/poly/EAST.poly', },
{'METPLUS_OUT_VALID_MASK': 'out_valid_mask = "MET_BASE/poly/EAST.poly";'}),
# 47 output template
({'TC_STAT_OUTPUT_TEMPLATE': 'tc_stat.out.nc', },
{}),
]
)
Expand Down Expand Up @@ -232,7 +276,7 @@ def test_tc_stat_run(metplus_config, config_overrides, env_var_values):
verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
config_file = wrapper.c_dict.get('CONFIG_FILE')
lookin_dir = wrapper.c_dict.get('LOOKIN_DIR')
out_temp = wrapper.c_dict.get('OUTPUT_TEMPLATE')
out_temp = wrapper.c_dict.get('JOB_OUTPUT_TEMPLATE')
out_dir = wrapper.c_dict.get('OUTPUT_DIR')
out_arg = f' -out {out_dir}/{out_temp}' if out_temp else ''

Expand Down Expand Up @@ -360,24 +404,23 @@ def test_override_config_in_c_dict(metplus_config, overrides, c_dict):
@pytest.mark.parametrize(
'jobs, init_dt, expected_output', [
# single fake job
(['job1'],
None,
'jobs = ["job1"];'
),
(['job1'],
None,
'jobs = ["job1"];'
),
# 2 jobs, no time info
(['-job filter -dump_row <output_dir>/filt.tcst',
'-job rirw -line_type TCMPR '],
None,
'jobs = ["-job filter -dump_row <output_dir>/filt.tcst",'
'"-job rirw -line_type TCMPR"];'
),
(['-job filter -dump_row <output_dir>/filt.tcst',
'-job rirw -line_type TCMPR '],
None,
'jobs = ["-job filter -dump_row <output_dir>/filt.tcst",'
'"-job rirw -line_type TCMPR"];'
),
# 2 jobs, time info sub
(['-job filter -dump_row <output_dir>/{init?fmt=%Y%m%d%H}.tcst',
'-job rirw -line_type TCMPR '],
datetime.datetime(2019, 10, 31, 12),
'jobs = ["-job filter -dump_row <output_dir>/2019103112.tcst",'
'"-job rirw -line_type TCMPR"];'
datetime.datetime(2019, 10, 31, 12),
'jobs = ["-job filter -dump_row <output_dir>/2019103112.tcst",'
'"-job rirw -line_type TCMPR"];'
),
]
)
Expand All @@ -388,7 +431,8 @@ def test_handle_jobs(metplus_config, jobs, init_dt, expected_output):
else:
time_info = None

wrapper = tc_stat_wrapper(metplus_config)
config = get_config(metplus_config)
wrapper = TCStatWrapper(config)
output_base = wrapper.config.getdir('OUTPUT_BASE')
output_dir = os.path.join(output_base, 'test_handle_jobs')

Expand All @@ -411,19 +455,19 @@ def cleanup_test_dirs(parent_dirs, output_dir):
@pytest.mark.parametrize(
'jobs, init_dt, expected_output, parent_dirs', [
# single fake job, no parent dir
(['job1'],
None,
'jobs = ["job1"];',
None
),
(['job1'],
None,
'jobs = ["job1"];',
None
),
# 2 jobs, no time info, 1 parent dir
(['-job filter -dump_row <output_dir>/filt.tcst',
'-job rirw -line_type TCMPR '],
None,
'jobs = ["-job filter -dump_row <output_dir>/filt.tcst",'
'"-job rirw -line_type TCMPR"];',
['<output_dir>'],
),
(['-job filter -dump_row <output_dir>/filt.tcst',
'-job rirw -line_type TCMPR '],
None,
'jobs = ["-job filter -dump_row <output_dir>/filt.tcst",'
'"-job rirw -line_type TCMPR"];',
['<output_dir>'],
),
# 2 jobs, time info sub, 1 parent dir
(['-job filter -dump_row <output_dir>/{init?fmt=%Y%m%d%H}.tcst',
Expand Down Expand Up @@ -452,7 +496,7 @@ def cleanup_test_dirs(parent_dirs, output_dir):
'jobs = ["-job filter -dump_row <output_dir>/sub1/2019103112.tcst",'
'"-job filter -dump_row <output_dir>/sub2/20191031.tcst"];',
['<output_dir>/sub1',
'<output_dir>/sub2',],
'<output_dir>/sub2', ],
),
]
)
Expand Down
38 changes: 28 additions & 10 deletions metplus/wrappers/tc_stat_wrapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -365,16 +365,9 @@ def handle_jobs(self, time_info):
subbed_job = do_string_sub(job, **time_info) if time_info else job
formatted_jobs.append(subbed_job.strip())

# check if -dump_row is used
# if it is, create parent directory of output file
split_job = subbed_job.split(' ')
if '-dump_row' in split_job:
index = split_job.index('-dump_row') + 1
filepath = split_job[index]
self.c_dict['OUTPUT_TEMPLATE'] = filepath

if not self.find_and_check_output_file(time_info):
return None
# create parent directory of output file
if not self._create_job_out_dirs(subbed_job, time_info):
return None

job_list_string = '","'.join(formatted_jobs)
job_list_string = f'jobs = ["{job_list_string}"];'
Expand All @@ -384,6 +377,31 @@ def handle_jobs(self, time_info):

return job_list_string

def _create_job_out_dirs(self, job_args, time_info):
"""!Create output directories for output files specified by job args
like -dump_row and -out_stat to prevent the command from failing.
@param job_args list of job arguments to parse
@param time_info time dictionary used to fill in filename
template tags if used
@returns False if something went wrong trying to create directories or
True if everything went smoothly.
"""
split_job = job_args.split(' ')
for out_type in ('-dump_row', '-out_stat'):
# continue if job arg that writes a file is not found in job args
if out_type not in split_job:
continue

# if job arg is found, create parent directory of output file
index = split_job.index(out_type) + 1
filepath = split_job[index]
self.c_dict['OUTPUT_TEMPLATE'] = filepath
if not self.find_and_check_output_file(time_info):
return False

return True

def handle_out_file(self, time_info):
"""! If output template is set,
"""
Expand Down

0 comments on commit 09dd7f4

Please sign in to comment.