Bugfix #2241 Create directory containing -out_stat file

docs/Contributors_Guide/testing.rst

@@ -23,10 +23,12 @@ information about the local environment so that the tests can run.
 All unit tests must include one of the custom markers listed in the
 internal/tests/pytests/pytest.ini file. Some examples include:
 
+    * run_metplus
     * util
     * wrapper_a
     * wrapper_b
     * wrapper_c
+    * wrapper_d
     * wrapper
     * long
     * plotting

internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py

@@ -14,6 +14,7 @@
 config_init_beg = '20170705'
 config_init_end = '20170901'
 
+
 def get_config(metplus_config):
     # extra_configs = []
     # extra_configs.append(os.path.join(os.path.dirname(__file__),
@@ -44,16 +45,56 @@ def get_config(metplus_config):
     return config
 
 
-def tc_stat_wrapper(metplus_config):
-    """! Returns a default TCStatWrapper with /path/to entries in the
-         metplus_system.conf and metplus_runtime.conf configuration
-         files.  Subsequent tests can customize the final METplus configuration
-         to over-ride these /path/to values."""
-
-    # Default, empty TcStatWrapper with some configuration values set
-    # to /path/to:
+@pytest.mark.parametrize(
+    'config_overrides, expected_dirs, expected_job_string', [
+        # 0: dump_row and out_stat files
+        ({'TC_STAT_JOB_ARGS': ("-job summary -line_type TCMPR -column "
+                               "'ABS(AMAX_WIND-BMAX_WIND)' "
+                               "-dump_row dump_row/summary.tcst, "
+                               "-job summary -line_type TCMPR -column "
+                               "'ABS(AMAX_WIND-BMAX_WIND)' "
+                               "-out_stat out_stat/stat_summary.tcst")},
+         ['dump_row', 'out_stat'],
+         ('jobs = ["-job summary -line_type TCMPR -column \'ABS(AMAX_WIND-BMAX_WIND)\' -dump_row dump_row/summary.tcst",'
+          '"-job summary -line_type TCMPR -column \'ABS(AMAX_WIND-BMAX_WIND)\' -out_stat out_stat/stat_summary.tcst"];')),
+
+        # 1: dump_row file
+        ({'TC_STAT_JOB_ARGS': ("-job summary -line_type TCMPR -column "
+                               "'ABS(AMAX_WIND-BMAX_WIND)' "
+                               "-dump_row dump_row/summary.tcst")}, ['dump_row'], 'jobs = ["-job summary -line_type TCMPR -column \'ABS(AMAX_WIND-BMAX_WIND)\' -dump_row dump_row/summary.tcst"];'),
+        # 2: out_stat file
+        ({'TC_STAT_JOB_ARGS': ("-job summary -line_type TCMPR -column "
+                               "'ABS(AMAX_WIND-BMAX_WIND)' "
+                               "-out_stat out_stat/stat_summary.tcst")},
+         ['out_stat'],
+         'jobs = ["-job summary -line_type TCMPR -column \'ABS(AMAX_WIND-BMAX_WIND)\' -out_stat out_stat/stat_summary.tcst"];'),
+    ]
+)
+@pytest.mark.wrapper
+def test_tc_stat_handle_jobs(metplus_config, config_overrides, expected_dirs,
+                             expected_job_string):
     config = get_config(metplus_config)
-    return TCStatWrapper(config)
+
+    # turn off "do not run" setting so directories are created
+    config.set('config', 'DO_NOT_RUN_EXE', False)
+
+    # set config variable overrides
+    for key, value in config_overrides.items():
+        config.set('config', key, value)
+
+    # initialize wrapper and ensure it was initialized properly
+    wrapper = TCStatWrapper(config)
+    assert wrapper.isOK
+
+    # ensure job string is formatted as expected
+    actual_job_string = wrapper.handle_jobs(time_info={})
+    assert actual_job_string == expected_job_string
+    assert wrapper.env_var_dict['METPLUS_JOBS'] == expected_job_string
+
+    # ensure output directories are created properly
+    for expected in expected_dirs:
+        expected_dir = os.path.join(wrapper.c_dict['OUTPUT_DIR'], expected)
+        assert os.path.exists(expected_dir) and os.path.isdir(expected_dir)
 
 @pytest.mark.parametrize(
     'config_overrides, env_var_values', [
@@ -197,6 +238,9 @@ def tc_stat_wrapper(metplus_config):
         # 46 out_valid_mask
         ({'TC_STAT_OUT_VALID_MASK': 'MET_BASE/poly/EAST.poly', },
          {'METPLUS_OUT_VALID_MASK': 'out_valid_mask = "MET_BASE/poly/EAST.poly";'}),
+        # 47 output template
+        ({'TC_STAT_OUTPUT_TEMPLATE': 'tc_stat.out.nc', },
+         {}),
 
     ]
 )
@@ -232,7 +276,7 @@ def test_tc_stat_run(metplus_config, config_overrides, env_var_values):
     verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
     config_file = wrapper.c_dict.get('CONFIG_FILE')
     lookin_dir = wrapper.c_dict.get('LOOKIN_DIR')
-    out_temp = wrapper.c_dict.get('OUTPUT_TEMPLATE')
+    out_temp = wrapper.c_dict.get('JOB_OUTPUT_TEMPLATE')
     out_dir = wrapper.c_dict.get('OUTPUT_DIR')
     out_arg = f' -out {out_dir}/{out_temp}' if out_temp else ''
 
@@ -360,24 +404,23 @@ def test_override_config_in_c_dict(metplus_config, overrides, c_dict):
 @pytest.mark.parametrize(
     'jobs, init_dt, expected_output', [
         # single fake job
-            (['job1'],
-             None,
-             'jobs = ["job1"];'
-            ),
+        (['job1'],
+         None,
+         'jobs = ["job1"];'
+         ),
         # 2 jobs, no time info
-            (['-job filter -dump_row <output_dir>/filt.tcst',
-              '-job rirw -line_type TCMPR '],
-             None,
-             'jobs = ["-job filter -dump_row <output_dir>/filt.tcst",'
-             '"-job rirw -line_type TCMPR"];'
-            ),
-
+        (['-job filter -dump_row <output_dir>/filt.tcst',
+          '-job rirw -line_type TCMPR '],
+         None,
+         'jobs = ["-job filter -dump_row <output_dir>/filt.tcst",'
+         '"-job rirw -line_type TCMPR"];'
+         ),
         # 2 jobs, time info sub
         (['-job filter -dump_row <output_dir>/{init?fmt=%Y%m%d%H}.tcst',
           '-job rirw -line_type TCMPR '],
-         datetime.datetime(2019, 10, 31, 12),
-         'jobs = ["-job filter -dump_row <output_dir>/2019103112.tcst",'
-         '"-job rirw -line_type TCMPR"];'
+          datetime.datetime(2019, 10, 31, 12),
+          'jobs = ["-job filter -dump_row <output_dir>/2019103112.tcst",'
+          '"-job rirw -line_type TCMPR"];'
          ),
     ]
 )
@@ -388,7 +431,8 @@ def test_handle_jobs(metplus_config, jobs, init_dt, expected_output):
     else:
         time_info = None
 
-    wrapper = tc_stat_wrapper(metplus_config)
+    config = get_config(metplus_config)
+    wrapper = TCStatWrapper(config)
     output_base = wrapper.config.getdir('OUTPUT_BASE')
     output_dir = os.path.join(output_base, 'test_handle_jobs')
 
@@ -411,19 +455,19 @@ def cleanup_test_dirs(parent_dirs, output_dir):
 @pytest.mark.parametrize(
     'jobs, init_dt, expected_output, parent_dirs', [
         # single fake job, no parent dir
-            (['job1'],
-             None,
-             'jobs = ["job1"];',
-             None
-            ),
+        (['job1'],
+         None,
+         'jobs = ["job1"];',
+         None
+         ),
         # 2 jobs, no time info, 1 parent dir
-            (['-job filter -dump_row <output_dir>/filt.tcst',
-              '-job rirw -line_type TCMPR '],
-             None,
-             'jobs = ["-job filter -dump_row <output_dir>/filt.tcst",'
-             '"-job rirw -line_type TCMPR"];',
-             ['<output_dir>'],
-            ),
+        (['-job filter -dump_row <output_dir>/filt.tcst',
+          '-job rirw -line_type TCMPR '],
+         None,
+         'jobs = ["-job filter -dump_row <output_dir>/filt.tcst",'
+         '"-job rirw -line_type TCMPR"];',
+         ['<output_dir>'],
+         ),
 
         # 2 jobs, time info sub, 1 parent dir
         (['-job filter -dump_row <output_dir>/{init?fmt=%Y%m%d%H}.tcst',
@@ -452,7 +496,7 @@ def cleanup_test_dirs(parent_dirs, output_dir):
          'jobs = ["-job filter -dump_row <output_dir>/sub1/2019103112.tcst",'
          '"-job filter -dump_row <output_dir>/sub2/20191031.tcst"];',
          ['<output_dir>/sub1',
-          '<output_dir>/sub2',],
+          '<output_dir>/sub2', ],
          ),
     ]
 )

metplus/wrappers/tc_stat_wrapper.py

@@ -365,16 +365,9 @@ def handle_jobs(self, time_info):
             subbed_job = do_string_sub(job, **time_info) if time_info else job
             formatted_jobs.append(subbed_job.strip())
 
-            # check if -dump_row is used
-            # if it is, create parent directory of output file
-            split_job = subbed_job.split(' ')
-            if '-dump_row' in split_job:
-                index = split_job.index('-dump_row') + 1
-                filepath = split_job[index]
-                self.c_dict['OUTPUT_TEMPLATE'] = filepath
-
-                if not self.find_and_check_output_file(time_info):
-                    return None
+            # create parent directory of output file
+            if not self._create_job_out_dirs(subbed_job, time_info):
+                return None
 
         job_list_string = '","'.join(formatted_jobs)
         job_list_string = f'jobs = ["{job_list_string}"];'
@@ -384,6 +377,31 @@ def handle_jobs(self, time_info):
 
         return job_list_string
 
+    def _create_job_out_dirs(self, job_args, time_info):
+        """!Create output directories for output files specified by job args
+        like -dump_row and -out_stat to prevent the command from failing.
+
+        @param job_args list of job arguments to parse
+        @param time_info time dictionary used to fill in filename
+          template tags if used
+        @returns False if something went wrong trying to create directories or
+        True if everything went smoothly.
+        """
+        split_job = job_args.split(' ')
+        for out_type in ('-dump_row', '-out_stat'):
+            # continue if job arg that writes a file is not found in job args
+            if out_type not in split_job:
+                continue
+
+            # if job arg is found, create parent directory of output file
+            index = split_job.index(out_type) + 1
+            filepath = split_job[index]
+            self.c_dict['OUTPUT_TEMPLATE'] = filepath
+            if not self.find_and_check_output_file(time_info):
+                return False
+
+        return True
+
     def handle_out_file(self, time_info):
         """! If output template is set,
         """