From e3881092c182feda3165786f4ba4486dbdb6dded Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 10:26:26 -0600 Subject: [PATCH 01/10] removed unused imports --- metplus/wrappers/cyclone_plotter_wrapper.py | 2 -- metplus/wrappers/tc_stat_wrapper.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index 740141ff43..a0e0c9db5a 100644 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -37,8 +37,6 @@ WRAPPER_CANNOT_RUN = True EXCEPTION_ERR = err_msg -import produtil.setup - from ..util import met_util as util from ..util import do_string_sub from ..util import time_generator, add_to_time_input diff --git a/metplus/wrappers/tc_stat_wrapper.py b/metplus/wrappers/tc_stat_wrapper.py index 19d6ddeb26..9782f84c8d 100755 --- a/metplus/wrappers/tc_stat_wrapper.py +++ b/metplus/wrappers/tc_stat_wrapper.py @@ -16,8 +16,6 @@ import sys from datetime import datetime -from produtil.run import ExitStatusException - from ..util import getlist, mkdir_p, do_string_sub, ti_calculate from . import CommandBuilder From e7bc9e22af056df2b16fe223ab0aa2c1a2c44374 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 10:26:52 -0600 Subject: [PATCH 02/10] prevent error message if LOOP_BY is unset when it doesn't need to be set --- metplus/util/time_looping.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metplus/util/time_looping.py b/metplus/util/time_looping.py index f2e771430d..ce0b036acf 100644 --- a/metplus/util/time_looping.py +++ b/metplus/util/time_looping.py @@ -159,6 +159,9 @@ def _get_time_prefix(config): valid time, or None if not enough information was found in the config """ loop_by = config.getstr('config', 'LOOP_BY', '').upper() + if not loop_by: + return None + if loop_by in ['INIT', 'RETRO']: return 'INIT' From 4b1337da7f4158334d421b88abac81190ff38153 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 10:27:33 -0600 Subject: [PATCH 03/10] prevent error message if only running wrappers that don't run any shell commands --- metplus/util/met_util.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index 9b093be6a5..9445fc045b 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -148,15 +148,14 @@ def run_metplus(config, process_list): "Options are processes, times") return 1 - # write out all commands and environment variables to file - if not write_all_commands(all_commands, config): - # if process list contains any wrapper that should run commands, - # report an error if no commands were generated - if any([item[0] not in NO_COMMAND_WRAPPERS - for item in process_list]): + # if process list contains any wrapper that should run commands + if any([item[0] not in NO_COMMAND_WRAPPERS for item in process_list]): + # write out all commands and environment variables to file + if not write_all_commands(all_commands, config): + # report an error if no commands were generated total_errors += 1 - # compute total number of errors that occurred and output results + # compute total number of errors that occurred and output results for process in processes: if process.errors != 0: process_name = process.__class__.__name__.replace('Wrapper', '') From 361dce8077be8604722817216c31d930f8828b9d Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 12:21:41 -0600 Subject: [PATCH 04/10] per #1657, use met_util.mkdir_p instead of os.makedirs to create directories to prevent issues with parallel runs of METplus creating directories --- metplus/util/config_metplus.py | 9 +++------ metplus/util/met_util.py | 12 +++--------- metplus/wrappers/command_builder.py | 8 +++----- metplus/wrappers/series_analysis_wrapper.py | 5 ++--- metplus/wrappers/tc_pairs_wrapper.py | 3 +-- metplus/wrappers/tcmpr_plotter_wrapper.py | 2 +- 6 files changed, 13 insertions(+), 26 deletions(-) diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index e407774400..5ada71c0f5 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -251,8 +251,7 @@ def launch(config_list): # create final conf directory if it doesn't already exist final_conf_dir = os.path.dirname(final_conf) - if not os.path.exists(final_conf_dir): - os.makedirs(final_conf_dir) + util.mkdir_p(final_conf_dir) # set METPLUS_BASE/PARM_BASE conf so they can be referenced in other confs config.set('config', 'METPLUS_BASE', METPLUS_BASE) @@ -372,10 +371,8 @@ def get_logger(config, sublog=None): log_dir = config.getdir('LOG_DIR') log_level = config.getstr('config', 'LOG_LEVEL') - # Check if the directory path for the log file exists, if - # not create it. - if not os.path.exists(log_dir): - util.mkdir_p(log_dir) + # Create the log directory if it does not exist + util.mkdir_p(log_dir) if sublog is not None: logger = config.log(sublog) diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index 9445fc045b..7b458cec93 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -251,9 +251,7 @@ def handle_tmp_dir(config): # create temp dir if it doesn't exist already # this will fail if TMP_DIR is not set correctly and # env MET_TMP_DIR was not set - tmp_dir = config.getdir('TMP_DIR') - if not os.path.exists(tmp_dir): - os.makedirs(tmp_dir) + mkdir_p(config.getdir('TMP_DIR')) def handle_env_var_config(config, env_var_name, config_name): """! If environment variable is set, use that value @@ -1229,9 +1227,7 @@ def preprocess_file(filename, data_type, config, allow_dir=False): return stagefile # if it does not exist, run GempakToCF and return staged nc file # Create staging area if it does not exist - outdir = os.path.dirname(stagefile) - if not os.path.exists(outdir): - os.makedirs(outdir, mode=0o0775) + mkdir_p(os.path.dirname(stagefile)) # only import GempakToCF if needed from ..wrappers import GempakToCFWrapper @@ -1262,9 +1258,7 @@ def preprocess_file(filename, data_type, config, allow_dir=False): # Create staging area directory only if file has compression extension if any([os.path.isfile(f'{filename}{ext}') for ext in COMPRESSION_EXTENSIONS]): - outdir = os.path.dirname(outpath) - if not os.path.exists(outdir): - os.makedirs(outdir, mode=0o0775) + mkdir_p(os.path.dirname(outpath)) # uncompress gz, bz2, or zip file if os.path.isfile(filename+".gz"): diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 4148725b14..940f86c820 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -912,8 +912,7 @@ def write_list_file(self, filename, file_list, output_dir=None): list_path = os.path.join(list_dir, filename) - if not os.path.exists(list_dir): - os.makedirs(list_dir, mode=0o0775) + util.mkdir_p(list_dir) self.logger.debug("Writing list of filenames...") with open(list_path, 'w') as file_handle: @@ -1008,7 +1007,7 @@ def find_and_check_output_file(self, time_info=None, if (not os.path.exists(parent_dir) and not self.c_dict.get('DO_NOT_RUN_EXE', False)): self.logger.debug(f"Creating output directory: {parent_dir}") - os.makedirs(parent_dir) + util.mkdir_p(parent_dir) if not output_exists or not skip_if_output_exists: return True @@ -1222,8 +1221,7 @@ def get_command(self): self.log_error('Must specify path to output file') return None - if not os.path.exists(parent_dir): - os.makedirs(parent_dir) + util.mkdir_p(parent_dir) cmd += " " + out_path diff --git a/metplus/wrappers/series_analysis_wrapper.py b/metplus/wrappers/series_analysis_wrapper.py index 76f684a469..cc45658217 100755 --- a/metplus/wrappers/series_analysis_wrapper.py +++ b/metplus/wrappers/series_analysis_wrapper.py @@ -22,7 +22,7 @@ WRAPPER_CANNOT_RUN = True EXCEPTION_ERR = err_msg -from ..util import getlist, get_storms +from ..util import getlist, get_storms, mkdir_p from ..util import do_string_sub, parse_template, get_tags from ..util import get_lead_sequence, get_lead_sequence_groups from ..util import ti_get_hours_from_lead, ti_get_seconds_from_lead @@ -1044,8 +1044,7 @@ def generate_animations(self): animate_dir = os.path.join(self.c_dict['OUTPUT_DIR'], 'series_animate') - if not os.path.exists(animate_dir): - os.makedirs(animate_dir) + mkdir_p(animate_dir) for group, files in self.c_dict['PNG_FILES'].items(): # write list of files to a text file diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index 0c6b42d828..fe2d59da29 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -863,8 +863,7 @@ def read_modify_write_file(self, in_csvfile, storm_month, missing_values, @param logger the log where logging is directed """ # create output directory if it does not exist - if not os.path.exists(os.path.dirname(out_csvfile)): - os.makedirs(os.path.dirname(out_csvfile)) + mkdir_p(os.path.dirname(out_csvfile)) # Open the output csv file out_file = open(out_csvfile, "w", newline='') diff --git a/metplus/wrappers/tcmpr_plotter_wrapper.py b/metplus/wrappers/tcmpr_plotter_wrapper.py index b60b697706..0a552d8d1e 100755 --- a/metplus/wrappers/tcmpr_plotter_wrapper.py +++ b/metplus/wrappers/tcmpr_plotter_wrapper.py @@ -226,7 +226,7 @@ def run_all_times(self): if not os.path.exists(self.c_dict['OUTPUT_DIR']): self.logger.debug("Creating directory: " f"{self.c_dict['OUTPUT_DIR']}") - os.makedirs(self.c_dict['OUTPUT_DIR']) + mkdir_p(self.c_dict['OUTPUT_DIR']) self.set_environment_variables() From ce602f72ea4917257db90574ebe19d3d8c956486 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 12:21:54 -0600 Subject: [PATCH 05/10] fix indent --- metplus/wrappers/command_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 940f86c820..221f164330 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1194,7 +1194,7 @@ def get_command(self): """ if self.app_path is None: self.log_error('No app path specified. ' - 'You must use a subclass') + 'You must use a subclass') return None cmd = '{} -v {}'.format(self.app_path, self.c_dict['VERBOSITY']) From 25e92148d166040e9b0866bf530af48b5e30c8cf Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 12:23:34 -0600 Subject: [PATCH 06/10] remove redundant checks to create output directory since this is handled in the find_and_check_output_file function --- metplus/wrappers/ascii2nc_wrapper.py | 9 --------- metplus/wrappers/gen_vx_mask_wrapper.py | 9 --------- metplus/wrappers/gfdl_tracker_wrapper.py | 5 ----- metplus/wrappers/point2grid_wrapper.py | 9 --------- metplus/wrappers/tc_gen_wrapper.py | 9 --------- metplus/wrappers/tc_pairs_wrapper.py | 4 ---- metplus/wrappers/tcrmw_wrapper.py | 9 --------- 7 files changed, 54 deletions(-) diff --git a/metplus/wrappers/ascii2nc_wrapper.py b/metplus/wrappers/ascii2nc_wrapper.py index 3e110e01a6..641b336f9a 100755 --- a/metplus/wrappers/ascii2nc_wrapper.py +++ b/metplus/wrappers/ascii2nc_wrapper.py @@ -228,15 +228,6 @@ def get_command(self): out_path = self.get_output_path() cmd += ' ' + out_path - parent_dir = os.path.dirname(out_path) - if parent_dir == '': - self.log_error('Must specify path to output file') - return None - - # create full output dir if it doesn't already exist - if not os.path.exists(parent_dir): - os.makedirs(parent_dir) - # add arguments cmd += ''.join(self.args) diff --git a/metplus/wrappers/gen_vx_mask_wrapper.py b/metplus/wrappers/gen_vx_mask_wrapper.py index b3ad0eeb2c..c145f4fef0 100755 --- a/metplus/wrappers/gen_vx_mask_wrapper.py +++ b/metplus/wrappers/gen_vx_mask_wrapper.py @@ -117,15 +117,6 @@ def get_command(self): out_path = self.get_output_path() cmd += ' ' + out_path - parent_dir = os.path.dirname(out_path) - if not parent_dir: - self.log_error('Must specify path to output file') - return None - - # create full output dir if it doesn't already exist - if not os.path.exists(parent_dir): - os.makedirs(parent_dir) - # add arguments cmd += ' ' + self.args diff --git a/metplus/wrappers/gfdl_tracker_wrapper.py b/metplus/wrappers/gfdl_tracker_wrapper.py index ff192bb179..29a7097afa 100755 --- a/metplus/wrappers/gfdl_tracker_wrapper.py +++ b/metplus/wrappers/gfdl_tracker_wrapper.py @@ -656,11 +656,6 @@ def rename_fort_to_output_path(self, time_info): self.c_dict.get('OUTPUT_TEMPLATE')) output_path = do_string_sub(output_path, **time_info) - # create parent directory of output path if it does not exist - parent_dir = os.path.dirname(output_path) - if not os.path.exists(parent_dir): - self.logger.debug(f"Creating output directory: {parent_dir}") - # copy fort.64/66 file to new file name self.logger.debug(f"Copying {fort_file} file to: {output_path}") try: diff --git a/metplus/wrappers/point2grid_wrapper.py b/metplus/wrappers/point2grid_wrapper.py index e21031f5f5..a8e1be8209 100755 --- a/metplus/wrappers/point2grid_wrapper.py +++ b/metplus/wrappers/point2grid_wrapper.py @@ -138,15 +138,6 @@ def get_command(self): out_path = self.get_output_path() cmd += ' ' + out_path - parent_dir = os.path.dirname(out_path) - if parent_dir == '': - self.log_error('Must specify path to output file') - return None - - # create full output dir if it doesn't already exist - if not os.path.exists(parent_dir): - os.makedirs(parent_dir) - # add arguments cmd += ' ' + ' '.join(self.args) diff --git a/metplus/wrappers/tc_gen_wrapper.py b/metplus/wrappers/tc_gen_wrapper.py index 11b8256751..ab8873f96a 100755 --- a/metplus/wrappers/tc_gen_wrapper.py +++ b/metplus/wrappers/tc_gen_wrapper.py @@ -329,15 +329,6 @@ def get_command(self): out_path = self.get_output_path() cmd += ' -out ' + out_path - parent_dir = os.path.dirname(out_path) - if not parent_dir: - self.log_error('Must specify path to output file') - return None - - # create full output dir if it doesn't already exist - if not os.path.exists(parent_dir): - os.makedirs(parent_dir) - return cmd def run_all_times(self): diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index fe2d59da29..fff8a691ff 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -834,10 +834,6 @@ def get_command(self): self.log_error('Output path not set') return None - # create directory containing output file if it doesn't exist - if not os.path.exists(os.path.dirname(output_path)): - os.makedirs(os.path.dirname(output_path)) - cmd = '{} -v {}'.format(self.app_path, self.c_dict['VERBOSITY']) cmd += ' -bdeck {}'.format(' '.join(self.bdeck)) diff --git a/metplus/wrappers/tcrmw_wrapper.py b/metplus/wrappers/tcrmw_wrapper.py index f116a6faf4..5881f9eb32 100755 --- a/metplus/wrappers/tcrmw_wrapper.py +++ b/metplus/wrappers/tcrmw_wrapper.py @@ -193,15 +193,6 @@ def get_command(self): out_path = self.get_output_path() cmd += ' -out ' + out_path - parent_dir = os.path.dirname(out_path) - if not parent_dir: - self.log_error('Must specify path to output file') - return None - - # create full output dir if it doesn't already exist - if not os.path.exists(parent_dir): - os.makedirs(parent_dir) - # add verbosity cmd += ' -v ' + self.c_dict['VERBOSITY'] return cmd From 9effe6ca45c7482e939e824988c8f74977112dd4 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 12:24:35 -0600 Subject: [PATCH 07/10] removed function that is not used anymore --- metplus/wrappers/compare_gridded_wrapper.py | 27 --------------------- 1 file changed, 27 deletions(-) diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index dca662e89d..32fef40bac 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -312,33 +312,6 @@ def process_fields(self, time_info): # run the MET command self.build() - def create_and_set_output_dir(self, time_info): - """! Builds the full output dir path with valid or init time - Creates output directory if it doesn't already exist - Args: - @param time_info dictionary with time information - """ - out_dir = self.c_dict['OUTPUT_DIR'] - - # use output template if it is set - # if not set, do not add any extra directories to path - out_template_name = '{}_OUTPUT_TEMPLATE'.format(self.app_name.upper()) - if self.config.has_option('config', - out_template_name): - template = self.config.getraw('config', - out_template_name) - # perform string substitution to get full path - extra_path = do_string_sub(template, - **time_info) - out_dir = os.path.join(out_dir, extra_path) - - # create full output dir if it doesn't already exist - if not os.path.exists(out_dir): - os.makedirs(out_dir) - - # set output dir for wrapper - self.outdir = out_dir - def get_command(self): """! Builds the command to run the MET application @rtype string From c10ffc11ae42741f74483c63c4f04bc1b56c5171 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 12:25:56 -0600 Subject: [PATCH 08/10] added missing import of mkdir_p and changed imports to get functions that are used instead of all of met_util --- metplus/wrappers/tc_pairs_wrapper.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index fff8a691ff..e4870b7e06 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -20,9 +20,8 @@ import datetime import glob -from ..util import getlist -from ..util import time_util -from ..util import met_util as util +from ..util import getlist, get_lead_sequence, skip_time, mkdir_p +from ..util import ti_calculate from ..util import do_string_sub from ..util import get_tags from ..util.met_config import add_met_config_dict_list @@ -385,13 +384,13 @@ def run_at_time(self, input_dict): if self.c_dict['SKIP_LEAD_SEQ']: lead_seq = [0] else: - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) for lead in lead_seq: input_dict['lead'] = lead - time_info = time_util.ti_calculate(input_dict) + time_info = ti_calculate(input_dict) - if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): self.logger.debug('Skipping run time') return @@ -930,7 +929,7 @@ def _read_all_files(self, input_dict): self.edeck = [edeck_dir] # get output filename from template - time_info = time_util.ti_calculate(input_dict) + time_info = ti_calculate(input_dict) time_storm_info = self._add_storm_info_to_dict(time_info) if not self.find_and_check_output_file(time_info=time_storm_info, check_extension='.tcst'): From 42d1039fe8505400ad9f1450e15dbabfc3d8061b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 12:26:46 -0600 Subject: [PATCH 09/10] added missing import of mkdir_p and changed imports to get functions that are used instead of all of met_util --- metplus/wrappers/gen_vx_mask_wrapper.py | 10 ++++------ metplus/wrappers/point2grid_wrapper.py | 8 ++++---- metplus/wrappers/tcmpr_plotter_wrapper.py | 5 +++-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/metplus/wrappers/gen_vx_mask_wrapper.py b/metplus/wrappers/gen_vx_mask_wrapper.py index c145f4fef0..ee6b6e434f 100755 --- a/metplus/wrappers/gen_vx_mask_wrapper.py +++ b/metplus/wrappers/gen_vx_mask_wrapper.py @@ -12,9 +12,7 @@ import os -from ..util import getlist -from ..util import met_util as util -from ..util import time_util +from ..util import getlist, get_lead_sequence, skip_time, ti_calculate, mkdir_p from . import CommandBuilder from ..util import do_string_sub @@ -132,14 +130,14 @@ def run_at_time(self, input_dict): @param input_dict dictionary containing timing information @returns None """ - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) for lead in lead_seq: self.clear() input_dict['lead'] = lead - time_info = time_util.ti_calculate(input_dict) + time_info = ti_calculate(input_dict) - if util.skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): self.logger.debug('Skipping run time') continue diff --git a/metplus/wrappers/point2grid_wrapper.py b/metplus/wrappers/point2grid_wrapper.py index a8e1be8209..a2607d6858 100755 --- a/metplus/wrappers/point2grid_wrapper.py +++ b/metplus/wrappers/point2grid_wrapper.py @@ -12,8 +12,8 @@ import os -from ..util import met_util as util -from ..util import time_util +from ..util import get_lead_sequence +from ..util import ti_calculate from ..util import do_string_sub from ..util import remove_quotes from . import CommandBuilder @@ -152,12 +152,12 @@ def run_at_time(self, input_dict): Args: @param input_dict dictionary containing timing information """ - lead_seq = util.get_lead_sequence(self.config, input_dict) + lead_seq = get_lead_sequence(self.config, input_dict) for lead in lead_seq: self.clear() input_dict['lead'] = lead - time_info = time_util.ti_calculate(input_dict) + time_info = ti_calculate(input_dict) for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: if custom_string: self.logger.info(f"Processing custom string: {custom_string}") diff --git a/metplus/wrappers/tcmpr_plotter_wrapper.py b/metplus/wrappers/tcmpr_plotter_wrapper.py index 0a552d8d1e..e8408586dc 100755 --- a/metplus/wrappers/tcmpr_plotter_wrapper.py +++ b/metplus/wrappers/tcmpr_plotter_wrapper.py @@ -7,11 +7,12 @@ import shutil from ..util import getlist -from ..util import met_util as util from ..util import time_util from ..util import do_string_sub from ..util import time_generator from ..util import remove_quotes +from ..util import get_files +from ..util import mkdir_p from . import CommandBuilder class TCMPRPlotterWrapper(CommandBuilder): @@ -276,7 +277,7 @@ def get_input_files(self): self.logger.debug('Passing in directory to R script') return [input_data] - input_files = util.get_files(input_data, ".*.tcst") + input_files = get_files(input_data, ".*.tcst") self.logger.debug(f"Number of files: {len(input_files)}") return input_files From cecdb77b7703e0f3954d2540924779432458eeeb Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Aug 2022 12:27:15 -0600 Subject: [PATCH 10/10] call find_and_check_output_file to handle output path to be consistent with other wrappers --- metplus/wrappers/gen_vx_mask_wrapper.py | 3 ++- metplus/wrappers/gfdl_tracker_wrapper.py | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/metplus/wrappers/gen_vx_mask_wrapper.py b/metplus/wrappers/gen_vx_mask_wrapper.py index ee6b6e434f..b6aa36abeb 100755 --- a/metplus/wrappers/gen_vx_mask_wrapper.py +++ b/metplus/wrappers/gen_vx_mask_wrapper.py @@ -182,7 +182,8 @@ def run_at_time_all(self, time_info): temp_file = os.path.join(self.config.getdir('STAGING_DIR'), 'gen_vx_mask', f'temp_{index}.nc') - self.set_output_path(temp_file) + self.find_and_check_output_file(time_info, + output_path_template=temp_file) # run GenVxMask self.build() diff --git a/metplus/wrappers/gfdl_tracker_wrapper.py b/metplus/wrappers/gfdl_tracker_wrapper.py index 29a7097afa..bd492d04bc 100755 --- a/metplus/wrappers/gfdl_tracker_wrapper.py +++ b/metplus/wrappers/gfdl_tracker_wrapper.py @@ -272,11 +272,9 @@ def run_at_time_once(self, input_dict): self.log_error("TCVitals file not found") return False - # create output directory if it doesn't exist - output_dir = self.c_dict.get('OUTPUT_DIR') - if not os.path.exists(output_dir): - self.logger.debug(f"Creating output directory: {output_dir}") - os.makedirs(output_dir) + # get output path + if not self.find_and_check_output_file(input_dict): + return False # create sym link to output directory for all files (including tcvit) all_output_files, tc_vitals_out = (