From de7708238489d5f22e791c2d5770c05767b5cf8d Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 11:33:41 -0700 Subject: [PATCH 01/59] clean up to reduce number of indents to make code more readable --- metplus/util/config_metplus.py | 7 +------ metplus/wrappers/command_builder.py | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index b1f8431c28..962c3413d7 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -383,12 +383,8 @@ def get_logger(config, sublog=None): logger.setLevel(logging.getLevelName(log_level)) metpluslog = config.getstr('config', 'LOG_METPLUS', '') - if metpluslog: - # It is possible that more path, other than just LOG_DIR, was added - # to the metpluslog, by either a user defining more path in - # LOG_METPLUS or LOG_FILENAME_TEMPLATE definitions in their conf file. - # So lets check and make more directory if needed. + # create log directory if it does not already exist dir_name = os.path.dirname(metpluslog) if not os.path.exists(dir_name): mkdir_p(dir_name) @@ -409,7 +405,6 @@ def get_logger(config, sublog=None): stream_handler.setFormatter(formatter) logger.addHandler(stream_handler) - # set add the logger to the config config.logger = logger return logger diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index b25c0e2f77..8f68205409 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1228,6 +1228,7 @@ def run_command(self, cmd, cmd_name=None): list of all commands run. @param cmd command to run + @param cmd_name optional command name to use in the log filename @returns True on success, False otherwise """ # add command to list of all commands run @@ -1243,20 +1244,19 @@ def run_command(self, cmd, cmd_name=None): env=self.env, log_name=log_name, copyable_env=self.get_env_copy()) - if ret: - logfile_path = self.config.getstr('config', 'LOG_METPLUS') - # if MET output is written to its own logfile, get that filename - if not self.config.getbool('config', 'LOG_MET_OUTPUT_TO_METPLUS'): - logfile_path = logfile_path.replace('run_metplus', - log_name) - - self.log_error("MET command returned a non-zero return code:" - f"{cmd}") - self.logger.info("Check the logfile for more information on why " - f"it failed: {logfile_path}") - return False + if not ret: + return True - return True + logfile_path = self.config.getstr('config', 'LOG_METPLUS') + # if MET output is written to its own logfile, get that filename + if not self.config.getbool('config', 'LOG_MET_OUTPUT_TO_METPLUS'): + logfile_path = logfile_path.replace('run_metplus', log_name) + + self.log_error("MET command returned a non-zero return code:" + f"{cmd}") + self.logger.info("Check the logfile for more information on why " + f"it failed: {logfile_path}") + return False def run_all_times(self, custom=None): """! Loop over time range specified in conf file and From a945632e668d44f45ceac5fab43107baa7eaaa9c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 11:35:28 -0700 Subject: [PATCH 02/59] simplify logic for finding log filenames, remove unused function --- metplus/wrappers/command_runner.py | 117 +++++++---------------------- 1 file changed, 26 insertions(+), 91 deletions(-) diff --git a/metplus/wrappers/command_runner.py b/metplus/wrappers/command_runner.py index e6e783a975..a32964ef1b 100755 --- a/metplus/wrappers/command_runner.py +++ b/metplus/wrappers/command_runner.py @@ -49,7 +49,8 @@ def __init__(self, config, logger=None, verbose=2, skip_run=False): self.config = config self.verbose = verbose self.skip_run = skip_run - self.log_command_to_met_log = False + self.log_met_to_metplus = config.getbool('config', + 'LOG_MET_OUTPUT_TO_METPLUS') def run_cmd(self, cmd, env=None, log_name=None, copyable_env=None, **kwargs): @@ -69,7 +70,6 @@ def run_cmd(self, cmd, env=None, log_name=None, being run. @param kwargs Other options sent to the produtil Run constructor """ - if cmd is None: return cmd @@ -98,12 +98,12 @@ def run_cmd(self, cmd, env=None, log_name=None, ' contructor: %s, ' % repr(self)) # Determine where to send the output from the MET command. - log_dest = self.cmdlog_destination(cmdlog=log_name+'.log') + log_dest = self.get_log_path(log_filename=log_name+'.log') # determine if command must be run in a shell run_inshell = False if '*' in cmd or ';' in cmd or '<' in cmd or '>' in cmd: - run_inshell=True + run_inshell = True # KEEP This comment as a reference note. # Run the executable in a new process instead of through a shell. @@ -119,8 +119,7 @@ def run_cmd(self, cmd, env=None, log_name=None, the_exe = shlex.split(cmd)[0] the_args = shlex.split(cmd)[1:] if log_dest: - self.logger.debug("log_name is: %s, output sent to: %s" % (log_name, log_dest)) - + self.logger.debug("Logging command output to: %s" % log_dest) self.log_header_info(log_dest, copyable_env, cmd) if run_inshell: @@ -152,9 +151,11 @@ def run_cmd(self, cmd, env=None, log_name=None, def log_header_info(self, log_dest, copyable_env, cmd): with open(log_dest, 'a+') as log_file_handle: - # if logging MET command to its own log file, add command that was run to that log - if self.log_command_to_met_log: - # if environment variables were set and available, write them to MET tool log + # if logging MET command to its own log file, + # add command that was run to that log + if not self.log_met_to_metplus: + # if environment variables were set and available, + # write them to MET tool log if copyable_env: log_file_handle.write( "\nCOPYABLE ENVIRONMENT FOR NEXT COMMAND:\n") @@ -167,93 +168,27 @@ def log_header_info(self, log_dest, copyable_env, cmd): # write line to designate where MET tool output starts log_file_handle.write("OUTPUT:\n") - # if cmdlog=None. The returned value is either the METplus log - # or None - def cmdlog_destination(self, cmdlog=None): + def get_log_path(self, log_filename): """!Returns the location of where the command output will be sent. The METplus log, the MET log, or tty. - Args: - @param cmdlog: The cmdlog is a filename, any path info is removed. - It is joined with LOG_DIR. If cmdlog is None, - output is sent to either the METplus log or TTY. - @returns log_dest: The destination of where to send the command output. - """ - - # Check the cmdlog argument. - # ie. if cmdlog = '', or '/', or trailing slash /path/blah.log/ etc..., - # os.path.basename returns '', and we can't write to '', - # so set cmdlog to None. - if cmdlog: - cmdlog = os.path.basename(cmdlog) - if not cmdlog: cmdlog = None - # Set the default destination to None, which will be TTY - cmdlog_dest = None - - # metpluslog is the setting used to determine if output is sent to either - # a log file or tty. - # metpluslog includes /path/filename. - metpluslog = self.config.getstr('config', 'LOG_METPLUS', '') - - self.log_command_to_met_log = False - - # This block determines where to send the command output, cmdlog_dest. - # To the METplus log, a MET log, or tty. - # If no metpluslog, cmlog_dest is None, which should be interpreted as tty. - if metpluslog: - log_met_output_to_metplus = self.config.getbool('config', - 'LOG_MET_OUTPUT_TO_METPLUS') - # If cmdlog is None send output to metpluslog. - if log_met_output_to_metplus or not cmdlog: - cmdlog_dest = metpluslog - else: - self.log_command_to_met_log = True - log_timestamp = self.config.getstr('config', 'LOG_TIMESTAMP', '') - if log_timestamp: - cmdlog_dest = os.path.join(self.config.getdir('LOG_DIR'), - cmdlog + '.' + log_timestamp) - else: - cmdlog_dest = os.path.join(self.config.getdir('LOG_DIR'),cmdlog) - - - # If cmdlog_dest None we will not redirect output to a log file - # when building the Runner object, so it will end up going to tty. - return cmdlog_dest - - # This method SHOULD ONLY BE USED by wrappers that build their cmd - # outside of the command_builder.py get_command() method - # ie. such as tc_pairs wrapper. Objects that fully use the CommandBuilder - # already have the metverbosity set in the command. - def insert_metverbosity_opt(self,cmd=None): - """!Returns the cmd with the verbosity option inserted - and set after the first space found in the cmd string or - after the cmd string if there are no spaces. - - There is NO CHECKING to see if the verbosity is already - inserted in the command. If cmd is None, None is returned. - - Args: - @param cmd: One string, The cmd string to insert the -v option. - @returns cmd: The cmd string w/ -v inserted - after the first white space or end if no - spaces. If cmd is None, None is returned. + @param log_filename file name to use if logging to a separate file + @returns Log file path or None if logging to terminal """ + # if LOG_METPLUS is unset or empty, log to terminal + metplus_log = self.config.getstr('config', 'LOG_METPLUS', '') + if not metplus_log: + return None - if cmd: + # return METplus log file if logging all output there + if self.log_met_to_metplus: + return metplus_log - verbose_opt = " -v "+str(self.verbose) + " " - # None splits on whitespace space, tab, newline, return, formfeed - cmd_split = cmd.split(None, 1) + log_path = os.path.join(self.config.getdir('LOG_DIR'), log_filename) - # Handle two cases of splitting. - # /path/to/cmd - # /path/to/cmd blah blah blah .... - if len(cmd_split) == 1: - cmd = cmd_split[0] + verbose_opt - elif len(cmd_split) == 2: - cmd = cmd_split[0] + verbose_opt + cmd_split[1] - else: - self.logger.debug('Can not Insert MET verbosity option, ' - 'command unchanged, using: %s .' % repr(cmd)) + # add log timestamp to log filename if set + log_timestamp = self.config.getstr('config', 'LOG_TIMESTAMP', '') + if log_timestamp: + log_path = f'{log_path}.{log_timestamp}' - return cmd + return log_path From 11fa333720ff4f6b11678b6904356b1557f956b6 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 12:12:23 -0700 Subject: [PATCH 03/59] simplified logic to getting metplus log file and output appropriate log if logging to terminal only --- metplus/util/config_metplus.py | 45 +++++++++++----------------------- metplus/util/run_util.py | 7 +++++- 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index 962c3413d7..de0d92c5f9 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -304,39 +304,22 @@ def _set_logvars(config, logger=None): logger.info('Adding LOG_TIMESTAMP=%s' % repr(log_filenametimestamp)) config.set('config', 'LOG_TIMESTAMP', log_filenametimestamp) - log_dir = config.getdir('LOG_DIR') + metplus_log = config.strinterp( + 'config', + '{LOG_METPLUS}', + LOG_TIMESTAMP_TEMPLATE=log_filenametimestamp + ) - # NOTE: LOG_METPLUS or metpluslog is meant to include the absolute path - # and the metpluslog_filename, - # so metpluslog = /path/to/metpluslog_filename - - # if LOG_METPLUS = unset in the conf file, means NO logging. - # Also, assUmes the user has included the intended path in LOG_METPLUS. - user_defined_log_file = None - metpluslog = '' - if config.has_option('config', 'LOG_METPLUS'): - user_defined_log_file = True - # strinterp will set metpluslog to '' if LOG_METPLUS = is unset. - metpluslog = config.strinterp( - 'config', - '{LOG_METPLUS}', - LOG_TIMESTAMP_TEMPLATE=log_filenametimestamp - ) - - # test if there is any path information, if there is, - # assume it is as intended, if there is not, than add log_dir. - if metpluslog: - if os.path.basename(metpluslog) == metpluslog: - metpluslog = os.path.join(log_dir, metpluslog) - - # Setting LOG_METPLUS in the configuration object - # At this point LOG_METPLUS will have a value or '' the empty string. - if user_defined_log_file: - logger.info('Replace LOG_METPLUS with %s' % repr(metpluslog)) + # add log directory to log file path if only filename was provided + if metplus_log: + if os.path.basename(metplus_log) == metplus_log: + metplus_log = os.path.join(config.getdir('LOG_DIR'), metplus_log) + logger.info('Logging to %s' % metplus_log) else: - logger.info('Adding LOG_METPLUS=%s' % repr(metpluslog)) - # expand LOG_METPLUS to ensure it is available - config.set('config', 'LOG_METPLUS', metpluslog) + logger.info('Logging to terminal only') + + # set LOG_METPLUS with timestamp substituted + config.set('config', 'LOG_METPLUS', metplus_log) def get_logger(config, sublog=None): diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index 3298e5fdac..89e15fcef0 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -31,7 +31,12 @@ def pre_run_setup(config_inputs): logger.info('Running METplus v%s%swith command: %s', version_number, user_string, ' '.join(sys.argv)) - logger.info(f"Log file: {config.getstr('config', 'LOG_METPLUS')}") + log_file = config.getstr('config', 'LOG_METPLUS', '') + if log_file: + logger.info(f"Log file: {log_file}") + else: + logger.info('LOG_METPLUS is unset. Logging to terminal only.') + logger.info(f"METplus Base: {config.getdir('METPLUS_BASE')}") logger.info(f"Final Conf: {config.getstr('config', 'METPLUS_CONF')}") config_list = config.getstr('config', 'CONFIG_INPUT').split(',') From b8fb37f87f34acc407ad2c2379df566388b75ef0 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 12:13:01 -0700 Subject: [PATCH 04/59] remove print Running METplus version because the same info is logged in config setup --- ush/run_metplus.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ush/run_metplus.py b/ush/run_metplus.py index 1971c2f0e9..66dbb038a2 100755 --- a/ush/run_metplus.py +++ b/ush/run_metplus.py @@ -81,10 +81,6 @@ def get_config_inputs_from_command_line(): invalid flag was provided, i.e. -a. @returns list of config inputs """ - - # output version that is run to screen - print('Running METplus %s' % metplus_version) - # if not arguments were provided, print usage and exit if len(sys.argv) < 2: usage() From d6f2177f30559bba3b267c1e6f8dbb678384c9f3 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:48:48 -0700 Subject: [PATCH 05/59] change logs that are output to screen before log file is initialized to print statements so it is clear that they are not included in the logs. Removed extra info about config setup that is not relevant to user --- metplus/util/config_metplus.py | 55 +++++++++++++--------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index de0d92c5f9..eae567bd19 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -85,30 +85,25 @@ logging.Formatter.converter = time.gmtime -def setup(args, logger=None, base_confs=None): - """!The METplus setup function. - @param args list of configuration files or configuration - variable overrides. Reads all configuration inputs and returns - a configuration object. +def setup(args, base_confs=None): + """!Setup the METplusConfig by reading in default configurations and any + arguments from the command line. + + @param args list of configuration files or configuration + variable overrides. Reads all configuration inputs and returns + a configuration object + @param base_confs optional config files to read first + @returns METplusConfig object """ if base_confs is None: base_confs = _get_default_config_list() - # Setup Task logger, Until a Conf object is created, Task logger is - # only logging to tty, not a file. - if logger is None: - logger = logging.getLogger('metplus') - - logger.info('Starting METplus configuration setup.') - - override_list = _parse_launch_args(args, logger) + override_list = _parse_launch_args(args) # add default config files to override list override_list = base_confs + override_list config = launch(override_list) - logger.debug('Completed METplus configuration setup.') - return config @@ -142,7 +137,7 @@ def _get_default_config_list(parm_base=None): return default_config_list -def _parse_launch_args(args, logger): +def _parse_launch_args(args): """! Parsed arguments to scripts that launch the METplus wrappers. Options: @@ -150,7 +145,6 @@ def _parse_launch_args(args, logger): * /path/to/file.conf --- read this conf file after the default conf files @param args the script arguments, after script-specific ones are removed - @param logger a logging.Logger for log messages @returns tuple containing path to parm directory, list of config files and collections.defaultdict of explicit config overrides """ @@ -180,7 +174,7 @@ def _parse_launch_args(args, logger): filepath = arg # check if argument is a path to a file that exists if not os.path.exists(filepath): - logger.error(f'Invalid argument: {filepath}') + print(f'ERROR: Invalid argument: {filepath}') bad = True continue @@ -189,13 +183,13 @@ def _parse_launch_args(args, logger): # path exists but is not a file if not os.path.isfile(filepath): - logger.error(f'Conf is not a file: {filepath}') + print(f'ERROR: Conf is not a file: {filepath}') bad = True continue # warn and skip if file is empty if os.stat(filepath).st_size == 0: - logger.warning(f'Conf file is empty: {filepath}. Skipping') + print(f'WARNING: Conf file is empty: {filepath}. Skipping') continue # add file path to override list @@ -217,7 +211,6 @@ def launch(config_list): @param config_list list of configuration files to process """ config = METplusConfig() - logger = config.log() # set config variable for current time config.set('config', 'CLOCK_TIME', @@ -227,7 +220,7 @@ def launch(config_list): # Read in and parse all the conf files and overrides for config_item in config_list: if isinstance(config_item, str): - logger.info(f"Parsing config file: {config_item}") + print(f"Parsing config file: {config_item}") config.read(config_item) config_format_list.append(config_item) else: @@ -236,7 +229,7 @@ def launch(config_list): if not config.has_section(section): config.add_section(section) - logger.info(f"Parsing override: [{section}] {key} = {value}") + print(f"Parsing override: [{section}] {key} = {value}") config.set(section, key, value) config_format_list.append(f'{section}.{key}={value}') @@ -274,20 +267,15 @@ def launch(config_list): return config -def _set_logvars(config, logger=None): +def _set_logvars(config): """!Sets and adds the LOG_METPLUS and LOG_TIMESTAMP to the config object. If LOG_METPLUS was already defined by the user in their conf file. It expands and rewrites it in the conf object and the final file. conf file. - Args: + @param config: the config instance - @param logger: the logger, optional """ - - if logger is None: - logger = config.log() - log_timestamp_template = config.getstr('config', 'LOG_TIMESTAMP_TEMPLATE', '') if config.getbool('config', 'LOG_TIMESTAMP_USE_DATATIME', False): @@ -300,8 +288,7 @@ def _set_logvars(config, logger=None): log_filenametimestamp = date_t.strftime(log_timestamp_template) - # Adding LOG_TIMESTAMP to the final configuration file. - logger.info('Adding LOG_TIMESTAMP=%s' % repr(log_filenametimestamp)) + # add LOG_TIMESTAMP to the final configuration file config.set('config', 'LOG_TIMESTAMP', log_filenametimestamp) metplus_log = config.strinterp( @@ -314,9 +301,9 @@ def _set_logvars(config, logger=None): if metplus_log: if os.path.basename(metplus_log) == metplus_log: metplus_log = os.path.join(config.getdir('LOG_DIR'), metplus_log) - logger.info('Logging to %s' % metplus_log) + print('Logging to %s' % metplus_log) else: - logger.info('Logging to terminal only') + print('Logging to terminal only') # set LOG_METPLUS with timestamp substituted config.set('config', 'LOG_METPLUS', metplus_log) From 4fb2fa2f2e7875e7d4ff510ba8bb8a8f680a595b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:49:48 -0700 Subject: [PATCH 06/59] added function to get useful message if log file is not set and call function when logging log file so it is clear that a log file is not used --- metplus/util/run_util.py | 15 +++++++-------- metplus/util/string_manip.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index 89e15fcef0..824cc8fe62 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -5,6 +5,7 @@ from importlib import import_module from .constants import NO_COMMAND_WRAPPERS +from .string_manip import get_logfile_info from .system_util import get_user_info, write_list_to_file from .config_util import get_process_list, handle_env_var_config from .config_util import handle_tmp_dir, write_final_conf, write_all_commands @@ -31,12 +32,10 @@ def pre_run_setup(config_inputs): logger.info('Running METplus v%s%swith command: %s', version_number, user_string, ' '.join(sys.argv)) - log_file = config.getstr('config', 'LOG_METPLUS', '') - if log_file: - logger.info(f"Log file: {log_file}") - else: - logger.info('LOG_METPLUS is unset. Logging to terminal only.') + # if log file is not set, log message instructing user how to set it + log_file = get_logfile_info(config) + logger.info(f"Log file: {log_file}") logger.info(f"METplus Base: {config.getdir('METPLUS_BASE')}") logger.info(f"Final Conf: {config.getstr('config', 'METPLUS_CONF')}") config_list = config.getstr('config', 'CONFIG_INPUT').split(',') @@ -60,7 +59,7 @@ def pre_run_setup(config_inputs): logger.error("Correct configuration variables and rerun. Exiting.") logger.info("Check the log file for more information: " - f"{config.getstr('config', 'LOG_METPLUS')}") + f"{get_logfile_info(config)}") sys.exit(1) if not config.getdir('MET_INSTALL_DIR', must_exist=True): @@ -162,7 +161,7 @@ def run_metplus(config): except: logger.exception("Fatal error occurred") logger.info("Check the log file for more information: " - f"{config.getstr('config', 'LOG_METPLUS')}") + f"{get_logfile_info(config)}") return 1 @@ -179,7 +178,7 @@ def post_run_cleanup(config, app_name, total_errors): # save log file path and clock time before writing final conf file log_message = (f"Check the log file for more information: " - f"{config.getstr('config', 'LOG_METPLUS')}") + f"{get_logfile_info(config)}") start_clock_time = datetime.strptime(config.getstr('config', 'CLOCK_TIME'), '%Y%m%d%H%M%S') diff --git a/metplus/util/string_manip.py b/metplus/util/string_manip.py index 07f4610c81..2e277bda50 100644 --- a/metplus/util/string_manip.py +++ b/metplus/util/string_manip.py @@ -560,3 +560,14 @@ def find_indices_in_config_section(regex, config, sec='config', indices[index].append(identifier) return indices + + +def get_logfile_info(config): + """!Get path to log file from LOG_METPLUS config variable or return a + useful message if it is not set to instruct users how to set it. + + @param config METplusConfig object to read LOG_METPLUS from + @returns path to log file or message if unset + """ + log_file = config.getstr('config', 'LOG_METPLUS', '') + return log_file if log_file else 'Set LOG_METPLUS to write logs to a file' From cd430ffbf57aa8fc5d5870530840c76d80e27cd5 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:59:58 -0700 Subject: [PATCH 07/59] skip log message with logfile path if no log file is set --- metplus/wrappers/command_builder.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 8f68205409..1839d34416 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1247,13 +1247,16 @@ def run_command(self, cmd, cmd_name=None): if not ret: return True + self.log_error(f"Command returned a non-zero return code: {cmd}") + logfile_path = self.config.getstr('config', 'LOG_METPLUS') + if not logfile_path: + return False + # if MET output is written to its own logfile, get that filename if not self.config.getbool('config', 'LOG_MET_OUTPUT_TO_METPLUS'): logfile_path = logfile_path.replace('run_metplus', log_name) - self.log_error("MET command returned a non-zero return code:" - f"{cmd}") self.logger.info("Check the logfile for more information on why " f"it failed: {logfile_path}") return False From 79c2f1d1b4b6cfcda7a12d7cd5e2945bdea12bbb Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 14:07:27 -0700 Subject: [PATCH 08/59] log all environment variable values at DEBUG level --- metplus/wrappers/command_builder.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 1839d34416..4a9ba4131d 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -241,15 +241,8 @@ def set_environment_variables(self, time_info=None): # set user defined environment variables self.set_user_environment(time_info) - # send environment variables to logger - for msg in self.print_all_envs(print_each_item=True, - print_copyable=False): - self.logger.info(msg) - - # log environment variables that can be copied into terminal - # to rerun application if debug logging is turned on - for msg in self.print_all_envs(print_each_item=False, - print_copyable=True): + # send environment variables and copyable commands to logger + for msg in self.print_all_envs(): self.logger.debug(msg) def log_error(self, error_string): From 26d8c3698015ea0813f415ce2b3e010621224674 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 14:45:50 -0700 Subject: [PATCH 09/59] added support for setting LOG_LEVEL_TERMINAL to set log level for screen output to a different value than the log file level --- metplus/util/config_metplus.py | 53 +++++++++++++++---------------- parm/metplus_config/defaults.conf | 2 ++ 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index eae567bd19..acfe0d5913 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -309,51 +309,48 @@ def _set_logvars(config): config.set('config', 'LOG_METPLUS', metplus_log) -def get_logger(config, sublog=None): +def get_logger(config): """!This function will return a logger with a formatted file handler for writing to the LOG_METPLUS and it sets the LOG_LEVEL. If LOG_METPLUS is not defined, a logger is still returned without adding a file handler, but still setting the LOG_LEVEL. - Args: - @param config: the config instance - @param sublog the logging subdomain, or None - Returns: - logger: the logger + + @param config: the config instance + @returns logger """ _set_logvars(config) # Retrieve all logging related parameters from the param file log_dir = config.getdir('LOG_DIR') log_level = config.getstr('config', 'LOG_LEVEL') + log_level_terminal = config.getstr('config', 'LOG_LEVEL_TERMINAL') # Create the log directory if it does not exist mkdir_p(log_dir) - if sublog is not None: - logger = config.log(sublog) - else: - logger = config.log() - - # Setting of the logger level from the config instance. - # Check for log_level by Integer or LevelName. - # Try to convert the string log_level to an integer and use that, if - # it can't convert then we assume it is a valid LevelName, which - # is what is should be anyway, ie. DEBUG. - # Note: - # Earlier versions of python2 require setLevel(), argument - # to be an int. Passing in the LevelName, 'DEBUG' will disable - # logging output. Later versions of python2 will accept 'DEBUG', - # not sure which version that changed with, but the logic below - # should work for all version. I know python 2.6.6 must be an int, - # and python 2.7.5 accepts the LevelName. + logger = config.log() + try: - int_log_level = int(log_level) - logger.setLevel(int_log_level) + log_level_val = logging.getLevelName(log_level) except ValueError: - logger.setLevel(logging.getLevelName(log_level)) + print(f'ERROR: Invalid value set for LOG_LEVEL: {log_level}') + sys.exit(1) + + try: + log_level_terminal_val = logging.getLevelName(log_level_terminal) + except ValueError: + print('ERROR: Invalid value set for LOG_LEVEL_TERMINAL:' + f' {log_level_terminal}') + sys.exit(1) metpluslog = config.getstr('config', 'LOG_METPLUS', '') - if metpluslog: + if not metpluslog: + logger.setLevel(log_level_terminal_val) + else: + # set logger level to the minimum of the two log levels because + # setting level for each handler will not work otherwise + logger.setLevel(min(log_level_val, log_level_terminal_val)) + # create log directory if it does not already exist dir_name = os.path.dirname(metpluslog) if not os.path.exists(dir_name): @@ -368,11 +365,13 @@ def get_logger(config, sublog=None): # set up the file logging file_handler = logging.FileHandler(metpluslog, mode='a') file_handler.setFormatter(formatter) + file_handler.setLevel(log_level_val) logger.addHandler(file_handler) # set up console logging stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) + stream_handler.setLevel(log_level_terminal_val) logger.addHandler(stream_handler) # set add the logger to the config diff --git a/parm/metplus_config/defaults.conf b/parm/metplus_config/defaults.conf index c1e340545a..850eb68434 100644 --- a/parm/metplus_config/defaults.conf +++ b/parm/metplus_config/defaults.conf @@ -97,6 +97,8 @@ LOG_MET_OUTPUT_TO_METPLUS = yes LOG_LEVEL = INFO +LOG_LEVEL_TERMINAL = INFO + LOG_MET_VERBOSITY = 2 From ea403789ff0732255a249879d3aa8436dfbca85b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 15:53:42 -0700 Subject: [PATCH 10/59] change terminal log level to warning to limit screen output when a pytest fails --- internal/tests/pytests/minimum_pytest.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/tests/pytests/minimum_pytest.conf b/internal/tests/pytests/minimum_pytest.conf index 9982acc1a4..ae6183fe5e 100644 --- a/internal/tests/pytests/minimum_pytest.conf +++ b/internal/tests/pytests/minimum_pytest.conf @@ -5,6 +5,7 @@ MET_INSTALL_DIR = {ENV[METPLUS_TEST_MET_INSTALL_DIR]} TMP_DIR = {ENV[METPLUS_TEST_TMP_DIR]} LOG_LEVEL = DEBUG +LOG_LEVEL_TERMINAL = WARNING LOG_MET_OUTPUT_TO_METPLUS = no LOG_LINE_FORMAT = (%(filename)s) %(levelname)s: %(message)s LOG_ERR_LINE_FORMAT = {LOG_LINE_FORMAT} From 9394f94b3fae1b9a95876e4ddb5097d1c9882a2e Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 15:56:38 -0700 Subject: [PATCH 11/59] remove unnecessary extra variable, ci-run-all-cases --- metplus/util/config_metplus.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index acfe0d5913..e3a2dd7ead 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -321,12 +321,11 @@ def get_logger(config): _set_logvars(config) # Retrieve all logging related parameters from the param file - log_dir = config.getdir('LOG_DIR') log_level = config.getstr('config', 'LOG_LEVEL') log_level_terminal = config.getstr('config', 'LOG_LEVEL_TERMINAL') # Create the log directory if it does not exist - mkdir_p(log_dir) + mkdir_p(config.getdir('LOG_DIR')) logger = config.log() From 20e0728d9b5886a71ea60ff424862858e48c7a18 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 14 Dec 2022 15:57:47 -0700 Subject: [PATCH 12/59] change log level for automated tests from DEBUG to INFO to see how much faster the tests run, ci-run-all-cases --- .github/parm/test_settings.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/test_settings.conf b/.github/parm/test_settings.conf index 9927f8f7e1..05ebfe25a2 100644 --- a/.github/parm/test_settings.conf +++ b/.github/parm/test_settings.conf @@ -1,5 +1,5 @@ [config] -LOG_LEVEL = DEBUG +LOG_LEVEL = INFO LOG_MET_OUTPUT_TO_METPLUS = no LOG_LINE_FORMAT = (%(filename)s) %(levelname)s: %(message)s LOG_ERR_LINE_FORMAT = {LOG_LINE_FORMAT} From b31f69d9c3bb6e0a3563cea72a07a4cd2b3c8231 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 09:21:36 -0700 Subject: [PATCH 13/59] run GHA tests with log file and terminal level DEBUG, ci-run-all-cases --- .github/parm/test_settings.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/parm/test_settings.conf b/.github/parm/test_settings.conf index 05ebfe25a2..f75c4a251e 100644 --- a/.github/parm/test_settings.conf +++ b/.github/parm/test_settings.conf @@ -1,5 +1,6 @@ [config] -LOG_LEVEL = INFO +LOG_LEVEL = DEBUG +LOG_LEVEL_TERMINAL = DEBUG LOG_MET_OUTPUT_TO_METPLUS = no LOG_LINE_FORMAT = (%(filename)s) %(levelname)s: %(message)s LOG_ERR_LINE_FORMAT = {LOG_LINE_FORMAT} From 4ddf6ae8059db01cb4a736b95b8024101fa997ff Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 09:22:04 -0700 Subject: [PATCH 14/59] run GHA tests with log file and terminal level INFO, ci-run-all-cases --- .github/parm/test_settings.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/parm/test_settings.conf b/.github/parm/test_settings.conf index f75c4a251e..7f54618714 100644 --- a/.github/parm/test_settings.conf +++ b/.github/parm/test_settings.conf @@ -1,6 +1,6 @@ [config] -LOG_LEVEL = DEBUG -LOG_LEVEL_TERMINAL = DEBUG +LOG_LEVEL = INFO +LOG_LEVEL_TERMINAL = INFO LOG_MET_OUTPUT_TO_METPLUS = no LOG_LINE_FORMAT = (%(filename)s) %(levelname)s: %(message)s LOG_ERR_LINE_FORMAT = {LOG_LINE_FORMAT} From e7d558708ea1bf464dec36f99af76396564e0627 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 10:43:39 -0700 Subject: [PATCH 15/59] changed logs that list the length of time to run to INFO level, added 'took' to command timing log because that word can be searched to find all timing log info --- metplus/util/run_util.py | 2 +- metplus/wrappers/command_runner.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index 824cc8fe62..880652528f 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -189,7 +189,7 @@ def post_run_cleanup(config, app_name, total_errors): # compute time it took to run end_clock_time = datetime.now() total_run_time = end_clock_time - start_clock_time - logger.debug(f"{app_name} took {total_run_time} to run.") + logger.info(f"{app_name} took {total_run_time} to run.") user_info = get_user_info() user_string = f' as user {user_info}' if user_info else '' diff --git a/metplus/wrappers/command_runner.py b/metplus/wrappers/command_runner.py index a32964ef1b..1332e911a4 100755 --- a/metplus/wrappers/command_runner.py +++ b/metplus/wrappers/command_runner.py @@ -144,8 +144,8 @@ def run_cmd(self, cmd, env=None, log_name=None, # calculate time to run end_cmd_time = datetime.now() total_cmd_time = end_cmd_time - start_cmd_time - self.logger.debug(f'Finished running {the_exe} ' - f'in {total_cmd_time}') + self.logger.info(f'Finished running {the_exe} ' + f'- took {total_cmd_time}') return ret, cmd From b219ac53c1d240f6c4400a461e95bd73cace2064 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 12:30:13 -0700 Subject: [PATCH 16/59] turn on use case to test --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 056013dd1b..1ee662b076 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -17,7 +17,7 @@ { "category": "climate", "index_list": "0-1", - "run": false + "run": true }, { "category": "short_range", From 83a5ddb7fd2027b9b19baed2bab5ed739a76c4c7 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 13:42:23 -0700 Subject: [PATCH 17/59] separate out setup commands from use case commands, only add status variable checks if not running in Docker because Docker logic will be updated to run each use case in separate docker exec commands --- .github/jobs/get_use_case_commands.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 1e28e2e8f6..617b484e42 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -151,10 +151,13 @@ def main(categories, subset_list, work_dir=None, for use_case_by_requirement in use_cases_by_req: reqs = use_case_by_requirement.requirements - setup_env, py_embed_arg = handle_automation_env(host_name, reqs, work_dir) + setup_env, py_embed_arg = handle_automation_env(host_name, reqs, + work_dir) # use status variable to track if any use cases failed - use_case_cmds = ['status=0'] + use_case_cmds = [] + if host_name != 'docker': + use_case_cmds.append('status=0') for use_case in use_case_by_requirement.use_cases: # add parm/use_cases path to config args if they are conf files config_args = [] @@ -174,13 +177,14 @@ def main(categories, subset_list, work_dir=None, use_case_cmds.append(use_case_cmd) # check exit code from use case command and # set status to non-zero value on error - use_case_cmds.append("if [ $? != 0 ]; then status=1; fi") + if host_name != 'docker': + use_case_cmds.append("if [ $? != 0 ]; then status=1; fi") # if any use cases failed, force non-zero exit code with false - use_case_cmds.append("if [ $status != 0 ]; then false; fi") + if host_name != 'docker': + use_case_cmds.append("if [ $status != 0 ]; then false; fi") # add commands to set up environment before use case commands - group_commands = f"{setup_env}{';'.join(use_case_cmds)}" - all_commands.append((group_commands, reqs)) + all_commands.append((setup_env, use_case_cmds, reqs)) return all_commands @@ -203,7 +207,6 @@ def handle_command_line_args(): else: subset_list = None - # check if comparison flag should be set if len(sys.argv) > 3: do_comparison = True @@ -216,7 +219,10 @@ def handle_command_line_args(): if __name__ == '__main__': categories, subset_list, _ = handle_command_line_args() all_commands = main(categories, subset_list) - for command, requirements in all_commands: + for setup_commands, use_case_commands, requirements in all_commands: print(f"REQUIREMENTS: {','.join(requirements)}") - command_format = ';\\\n'.join(command.split(';')) - print(f"COMMAND:\n{command_format}\n") + if setup_commands: + command_format = ';\\\n'.join(setup_commands.split(';')) + print(f"SETUP COMMANDS:\n{command_format}\n") + command_format = ';\\\n'.join(use_case_commands) + print(f"USE CASE COMMANDS:\n{command_format}\n") From 901e332530b5ffaf18e4910f72c466164524fdc8 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 13:43:29 -0700 Subject: [PATCH 18/59] create Docker container and run setup commands, then docker exec each use case command separately --- .github/jobs/setup_and_run_use_cases.py | 41 ++++++++++++++++++------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 00aa682092..7fd77fefac 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -28,6 +28,7 @@ docker_output_dir = os.path.join(docker_data_dir, 'output') gha_output_dir = os.path.join(runner_workspace, 'output') + def main(): categories, subset_list, _ = ( get_use_case_commands.handle_command_line_args() @@ -61,7 +62,7 @@ def main(): ] isOK = True - for cmd, requirements in all_commands: + for setup_commands, use_case_commands, requirements in all_commands: # get environment image tag use_env = [item for item in requirements if item.endswith('_env')] @@ -110,16 +111,35 @@ def main(): **cmd_args).stdout.strip() print(f"docker ps -a\n{output}") - full_cmd = ( + all_commands = [] + all_commands.append( f"docker run -e GITHUB_WORKSPACE " f"{os.environ.get('NETWORK_ARG', '')} " f"{' '.join(volume_mounts)} " f"{volumes_from} --workdir {github_workspace} " - f'{run_tag} bash -c "{cmd}"') - print(f"RUNNING: {full_cmd}") + f'{run_tag} bash -c "{setup_commands}"' + ) + for use_case_command in use_case_commands: + all_commands.append( + 'docker exec -e GITHUB_WORKSPACE -d ' + f'{run_tag} bash -c "{use_case_command}"' + ) + all_commands.append(f'docker stop {run_tag}') + if not run_docker_commands(all_commands): + isOK = False + + if not isOK: + print("ERROR: Some commands failed.") + sys.exit(1) + + +def run_docker_commands(docker_commands): + is_ok = True + for docker_command in docker_commands: + print(f"RUNNING: {docker_command}") start_time = time.time() try: - process = subprocess.Popen(shlex.split(full_cmd), + process = subprocess.Popen(shlex.split(docker_command), shell=False, encoding='utf-8', stdout=subprocess.PIPE, @@ -133,20 +153,19 @@ def main(): print(output.strip()) rc = process.poll() if rc: - raise subprocess.CalledProcessError(rc, full_cmd) + raise subprocess.CalledProcessError(rc, docker_command) except subprocess.CalledProcessError as err: print(f"ERROR: Command failed -- {err}") - isOK = False + is_ok = False end_time = time.time() print("TIMING: Command took " f"{time.strftime('%M:%S', time.gmtime(end_time - start_time))}" - f" (MM:SS): '{full_cmd}')") + f" (MM:SS): '{docker_command}')") + + return is_ok - if not isOK: - print("ERROR: Some commands failed.") - sys.exit(1) if __name__ == '__main__': main() From e28761400f8d0ba0224f6958c485820ffc6f59ad Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 13:54:01 -0700 Subject: [PATCH 19/59] add name identifier for container so exec and stop commands can find it --- .github/jobs/setup_and_run_use_cases.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 7fd77fefac..ca92088a1e 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -114,6 +114,7 @@ def main(): all_commands = [] all_commands.append( f"docker run -e GITHUB_WORKSPACE " + f"--name {run_tag} " f"{os.environ.get('NETWORK_ARG', '')} " f"{' '.join(volume_mounts)} " f"{volumes_from} --workdir {github_workspace} " From 84d9d7de22f4f58bea28def08aefeffe61b0b508 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 14:17:54 -0700 Subject: [PATCH 20/59] docker run detached and run docker ps and images commands through run_docker_commands --- .github/jobs/setup_and_run_use_cases.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index ca92088a1e..21da65174c 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -103,23 +103,28 @@ def main(): f"{time.strftime('%M:%S', time.gmtime(end_time - start_time))}" f" (MM:SS): '{docker_build_cmd}')") - cmd_args = {'check': True, - 'encoding': 'utf-8', - 'capture_output': True, - } - output = subprocess.run(shlex.split('docker ps -a'), - **cmd_args).stdout.strip() - print(f"docker ps -a\n{output}") + # cmd_args = {'check': True, + # 'encoding': 'utf-8', + # 'capture_output': True, + # } + # output = subprocess.run(shlex.split('docker ps -a'), + # **cmd_args).stdout.strip() + # print(f"docker ps -a\n{output}") + # output = subprocess.run(shlex.split('docker images'), + # **cmd_args).stdout.strip() + # print(f"docker images\n{output}") all_commands = [] + all_commands.append('docker images') all_commands.append( - f"docker run -e GITHUB_WORKSPACE " + f"docker run -d -e GITHUB_WORKSPACE " f"--name {run_tag} " f"{os.environ.get('NETWORK_ARG', '')} " f"{' '.join(volume_mounts)} " f"{volumes_from} --workdir {github_workspace} " f'{run_tag} bash -c "{setup_commands}"' ) + all_commands.append('docker ps -a') for use_case_command in use_case_commands: all_commands.append( 'docker exec -e GITHUB_WORKSPACE -d ' From e390f1639f2028af9fa5788d1452874131dc3d27 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 14:49:23 -0700 Subject: [PATCH 21/59] start container in interactive mode detached, then exec commands to it, ci-skip-unit-tests --- .github/jobs/setup_and_run_use_cases.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 21da65174c..9a80b9a745 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -117,15 +117,15 @@ def main(): all_commands = [] all_commands.append('docker images') all_commands.append( - f"docker run -d -e GITHUB_WORKSPACE " + f"docker run -d --rm -it -e GITHUB_WORKSPACE " f"--name {run_tag} " f"{os.environ.get('NETWORK_ARG', '')} " f"{' '.join(volume_mounts)} " f"{volumes_from} --workdir {github_workspace} " - f'{run_tag} bash -c "{setup_commands}"' + f'{run_tag} bash' ) all_commands.append('docker ps -a') - for use_case_command in use_case_commands: + for use_case_command in [setup_commands] + use_case_commands: all_commands.append( 'docker exec -e GITHUB_WORKSPACE -d ' f'{run_tag} bash -c "{use_case_command}"' From e9b7377c77e0f3e8e936ca78c6d55c183517b09b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 14:56:07 -0700 Subject: [PATCH 22/59] do not exec detached because it will always return 0 error code, remove container at end of tests instead of stop, ci-skip-unit-tests --- .github/jobs/setup_and_run_use_cases.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 9a80b9a745..e87e5fd5d6 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -127,10 +127,10 @@ def main(): all_commands.append('docker ps -a') for use_case_command in [setup_commands] + use_case_commands: all_commands.append( - 'docker exec -e GITHUB_WORKSPACE -d ' - f'{run_tag} bash -c "{use_case_command}"' + f'docker exec -e GITHUB_WORKSPACE {run_tag} ' + f'bash -c "{use_case_command}"' ) - all_commands.append(f'docker stop {run_tag}') + all_commands.append(f'docker rm {run_tag}') if not run_docker_commands(all_commands): isOK = False From 3815b0e793ba064f690f6ddf7d0cd2da18b22ae5 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 15:21:31 -0700 Subject: [PATCH 23/59] run docker exec commands with bash login shell (-l), try adding export statement to bashrc so it will be read on each login shell, renamed constant variable to uppercase, ci-skip-unit-tests --- .github/jobs/get_use_case_commands.py | 2 +- .github/jobs/setup_and_run_use_cases.py | 25 +++++++------------------ 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 617b484e42..9a7c25cc21 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -66,7 +66,7 @@ def handle_automation_env(host_name, reqs, work_dir): python_dir = os.path.join('/usr', 'local', 'envs', conda_env_w_ext, 'bin') python_path = os.path.join(python_dir, 'python3') - setup_env += f' export PATH={python_dir}:$PATH;' + setup_env += f' echo "export PATH={python_dir}:$PATH;" >> /etc/bashrc' # if py_embed listed in requirements and using a Python # environment that differs from the MET env, set MET_PYTHON_EXE diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index e87e5fd5d6..634ba67415 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -28,6 +28,7 @@ docker_output_dir = os.path.join(docker_data_dir, 'output') gha_output_dir = os.path.join(runner_workspace, 'output') +RUN_TAG = 'metplus-run-env' def main(): categories, subset_list, _ = ( @@ -49,7 +50,6 @@ def main(): if os.environ.get('GITHUB_EVENT_NAME') == 'pull_request': branch_name = f"{branch_name}-pull_request" - run_tag = 'metplus-run-env' dockerfile_dir = os.path.join('.github', 'actions', 'run_tests') # use BuildKit to build image @@ -83,7 +83,7 @@ def main(): dockerfile_name = f'{dockerfile_name}_cartopy' docker_build_cmd = ( - f"docker build -t {run_tag} " + f"docker build -t {RUN_TAG} " f"--build-arg METPLUS_IMG_TAG={branch_name} " f"--build-arg METPLUS_ENV_TAG={env_tag} " f"-f {dockerfile_dir}/{dockerfile_name} ." @@ -103,34 +103,23 @@ def main(): f"{time.strftime('%M:%S', time.gmtime(end_time - start_time))}" f" (MM:SS): '{docker_build_cmd}')") - # cmd_args = {'check': True, - # 'encoding': 'utf-8', - # 'capture_output': True, - # } - # output = subprocess.run(shlex.split('docker ps -a'), - # **cmd_args).stdout.strip() - # print(f"docker ps -a\n{output}") - # output = subprocess.run(shlex.split('docker images'), - # **cmd_args).stdout.strip() - # print(f"docker images\n{output}") - all_commands = [] all_commands.append('docker images') all_commands.append( f"docker run -d --rm -it -e GITHUB_WORKSPACE " - f"--name {run_tag} " + f"--name {RUN_TAG} " f"{os.environ.get('NETWORK_ARG', '')} " f"{' '.join(volume_mounts)} " f"{volumes_from} --workdir {github_workspace} " - f'{run_tag} bash' + f'{RUN_TAG} bash' ) all_commands.append('docker ps -a') for use_case_command in [setup_commands] + use_case_commands: all_commands.append( - f'docker exec -e GITHUB_WORKSPACE {run_tag} ' - f'bash -c "{use_case_command}"' + f'docker exec -e GITHUB_WORKSPACE {RUN_TAG} ' + f'bash -cl "{use_case_command}"' ) - all_commands.append(f'docker rm {run_tag}') + all_commands.append(f'docker rm {RUN_TAG}') if not run_docker_commands(all_commands): isOK = False From 9b1b4afd471da339a0b738f5560b6accfcd2c5b9 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 15:25:07 -0700 Subject: [PATCH 24/59] run docker build command through docker command function --- .github/jobs/setup_and_run_use_cases.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 634ba67415..8ce2a7fbd6 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -88,21 +88,12 @@ def main(): f"--build-arg METPLUS_ENV_TAG={env_tag} " f"-f {dockerfile_dir}/{dockerfile_name} ." ) - print(f"Building Docker environment/branch image...\n" - f"Running: {docker_build_cmd}") - start_time = time.time() - try: - subprocess.run(shlex.split(docker_build_cmd), check=True) - except subprocess.CalledProcessError as err: - print(f"ERROR: Docker Build failed: {docker_build_cmd} -- {err}") + + print(f'Building Docker environment/branch image...') + if not run_docker_commands([docker_build_cmd]): isOK = False continue - end_time = time.time() - print("TIMING: Command took " - f"{time.strftime('%M:%S', time.gmtime(end_time - start_time))}" - f" (MM:SS): '{docker_build_cmd}')") - all_commands = [] all_commands.append('docker images') all_commands.append( From d72f7925ed02310ba08b4a67b78cbcfdb3b232c4 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 15:29:20 -0700 Subject: [PATCH 25/59] added missing semicolon --- .github/jobs/get_use_case_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 9a7c25cc21..a9589aa2af 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -66,7 +66,7 @@ def handle_automation_env(host_name, reqs, work_dir): python_dir = os.path.join('/usr', 'local', 'envs', conda_env_w_ext, 'bin') python_path = os.path.join(python_dir, 'python3') - setup_env += f' echo "export PATH={python_dir}:$PATH;" >> /etc/bashrc' + setup_env += f' echo "export PATH={python_dir}:$PATH;" >> /etc/bashrc;' # if py_embed listed in requirements and using a Python # environment that differs from the MET env, set MET_PYTHON_EXE From bd6cc1e9631b38e482dc9cf54ceef30f3a1420c3 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 15:29:40 -0700 Subject: [PATCH 26/59] force remove container at end of tests, ci-skip-unit-tests --- .github/jobs/setup_and_run_use_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 8ce2a7fbd6..b6c78242b4 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -110,7 +110,7 @@ def main(): f'docker exec -e GITHUB_WORKSPACE {RUN_TAG} ' f'bash -cl "{use_case_command}"' ) - all_commands.append(f'docker rm {RUN_TAG}') + all_commands.append(f'docker rm -f {RUN_TAG}') if not run_docker_commands(all_commands): isOK = False From 9a453b6b7b1ba078e36dd229ed95543c03abdeb1 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 15:48:43 -0700 Subject: [PATCH 27/59] fix adding export to bashrc, ci-skip-unit-tests --- .github/jobs/get_use_case_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index a9589aa2af..b74d2bf3ae 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -66,7 +66,7 @@ def handle_automation_env(host_name, reqs, work_dir): python_dir = os.path.join('/usr', 'local', 'envs', conda_env_w_ext, 'bin') python_path = os.path.join(python_dir, 'python3') - setup_env += f' echo "export PATH={python_dir}:$PATH;" >> /etc/bashrc;' + setup_env += f" echo 'export PATH={python_dir}:\\$PATH;' >> /etc/bashrc;" # if py_embed listed in requirements and using a Python # environment that differs from the MET env, set MET_PYTHON_EXE From 9817a1eba5257b5b1d94b91a080821908e0cb575 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 16:04:18 -0700 Subject: [PATCH 28/59] another test, ci-skip-unit-tests --- .github/jobs/get_use_case_commands.py | 5 ++++- .github/jobs/setup_and_run_use_cases.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index b74d2bf3ae..01bdaf091d 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -66,7 +66,7 @@ def handle_automation_env(host_name, reqs, work_dir): python_dir = os.path.join('/usr', 'local', 'envs', conda_env_w_ext, 'bin') python_path = os.path.join(python_dir, 'python3') - setup_env += f" echo 'export PATH={python_dir}:\\$PATH;' >> /etc/bashrc;" + setup_env += f" echo 'export PATH={python_dir}:\$PATH;' >> /etc/bashrc;" # if py_embed listed in requirements and using a Python # environment that differs from the MET env, set MET_PYTHON_EXE @@ -127,6 +127,9 @@ def handle_automation_env(host_name, reqs, work_dir): return setup_env, py_embed_arg +def _add_to_bashrc(command): + return f"echo '{command.replace('$', '\\$')};' >> /etc/bashrc" + def main(categories, subset_list, work_dir=None, host_name=os.environ.get('HOST_NAME')): all_commands = [] diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index b6c78242b4..52e14f9f0d 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -110,6 +110,7 @@ def main(): f'docker exec -e GITHUB_WORKSPACE {RUN_TAG} ' f'bash -cl "{use_case_command}"' ) + all_commands.append('cat /etc/bashrc') all_commands.append(f'docker rm -f {RUN_TAG}') if not run_docker_commands(all_commands): isOK = False From 8af5c65557db3afba3853931bf1eed6ccc9264af Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 16:14:01 -0700 Subject: [PATCH 29/59] comment out broken function --- .github/jobs/get_use_case_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 01bdaf091d..27a3c39ca0 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -127,8 +127,8 @@ def handle_automation_env(host_name, reqs, work_dir): return setup_env, py_embed_arg -def _add_to_bashrc(command): - return f"echo '{command.replace('$', '\\$')};' >> /etc/bashrc" +#def _add_to_bashrc(command): +# return f"echo '{command.replace('$', '\\$')};' >> /etc/bashrc" def main(categories, subset_list, work_dir=None, host_name=os.environ.get('HOST_NAME')): From 56c499907cad4daa9d89f9e5940dc44ab9f0b677 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 16:14:16 -0700 Subject: [PATCH 30/59] try grouping GHA log output, ci-skip-unit-tests --- .github/actions/run_tests/entrypoint.sh | 2 ++ .github/jobs/setup_and_run_use_cases.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.github/actions/run_tests/entrypoint.sh b/.github/actions/run_tests/entrypoint.sh index 3bfad88343..d819abf50d 100644 --- a/.github/actions/run_tests/entrypoint.sh +++ b/.github/actions/run_tests/entrypoint.sh @@ -21,7 +21,9 @@ fi # try to pull image from DockerHub DOCKERHUBTAG=dtcenter/metplus-dev:${branch_name} +echo "::group::Docker pull" time_command docker pull $DOCKERHUBTAG +echo "::endgroup::" # if unsuccessful (i.e. pull request from a fork) # then build image locally diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 52e14f9f0d..6903b313c4 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -30,6 +30,7 @@ RUN_TAG = 'metplus-run-env' + def main(): categories, subset_list, _ = ( get_use_case_commands.handle_command_line_args() @@ -90,9 +91,12 @@ def main(): ) print(f'Building Docker environment/branch image...') + print("::group::Docker build test environment") if not run_docker_commands([docker_build_cmd]): + print("::endgroup::") isOK = False continue + print("::endgroup::") all_commands = [] all_commands.append('docker images') From 7674b274fb5ca55dbbf54355692095335ebe191a Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 16:24:41 -0700 Subject: [PATCH 31/59] add GHA log group for each docker command --- .github/jobs/setup_and_run_use_cases.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 6903b313c4..5ce584236d 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -128,6 +128,7 @@ def run_docker_commands(docker_commands): is_ok = True for docker_command in docker_commands: print(f"RUNNING: {docker_command}") + print(f"::group::{docker_command}") start_time = time.time() try: process = subprocess.Popen(shlex.split(docker_command), @@ -150,6 +151,8 @@ def run_docker_commands(docker_commands): print(f"ERROR: Command failed -- {err}") is_ok = False + print("::endgroup::") + end_time = time.time() print("TIMING: Command took " f"{time.strftime('%M:%S', time.gmtime(end_time - start_time))}" From 3505446951fa1b038790e5e8224fcb81bd4f194c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 16:24:58 -0700 Subject: [PATCH 32/59] another attempt at setting path, ci-skip-unit-tests --- .github/jobs/get_use_case_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 27a3c39ca0..51480e276c 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -66,7 +66,7 @@ def handle_automation_env(host_name, reqs, work_dir): python_dir = os.path.join('/usr', 'local', 'envs', conda_env_w_ext, 'bin') python_path = os.path.join(python_dir, 'python3') - setup_env += f" echo 'export PATH={python_dir}:\$PATH;' >> /etc/bashrc;" + setup_env += f" echo 'export PATH={python_dir}:$PATH;' >> /etc/bashrc;" # if py_embed listed in requirements and using a Python # environment that differs from the MET env, set MET_PYTHON_EXE From ca5deff9704250220ac249eea1d9c19cc2e259cb Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 16:36:33 -0700 Subject: [PATCH 33/59] end group before error message so it is visible at top level, fix cat command to run inside docker, ci-skip-unit-tests --- .github/jobs/setup_and_run_use_cases.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 5ce584236d..5cd87310dc 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -109,12 +109,11 @@ def main(): f'{RUN_TAG} bash' ) all_commands.append('docker ps -a') - for use_case_command in [setup_commands] + use_case_commands: + for use_case_command in [setup_commands] + use_case_commands + ['cat /etc/bashrc']: all_commands.append( f'docker exec -e GITHUB_WORKSPACE {RUN_TAG} ' f'bash -cl "{use_case_command}"' ) - all_commands.append('cat /etc/bashrc') all_commands.append(f'docker rm -f {RUN_TAG}') if not run_docker_commands(all_commands): isOK = False @@ -148,10 +147,11 @@ def run_docker_commands(docker_commands): raise subprocess.CalledProcessError(rc, docker_command) except subprocess.CalledProcessError as err: + print("::endgroup::") print(f"ERROR: Command failed -- {err}") is_ok = False - - print("::endgroup::") + else: + print("::endgroup::") end_time = time.time() print("TIMING: Command took " From 7e96fb353fee8b67c2da3b0916cb3bd64a3241bb Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 16:48:27 -0700 Subject: [PATCH 34/59] add any env var assignment to bashrc, changed setup_env items into a list that is joined by semicolon at the end, ci-skip-unit-tests --- .github/jobs/get_use_case_commands.py | 60 +++++++++++++++------------ 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 51480e276c..033da84e89 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -60,13 +60,14 @@ def handle_automation_env(host_name, reqs, work_dir): conda_env_w_ext = f'{conda_env}{VERSION_EXT}' # start building commands to run before run_metplus.py in Docker - setup_env = 'source /etc/bashrc;' + setup_env = [] + setup_env.append('source /etc/bashrc') # add conda bin to beginning of PATH python_dir = os.path.join('/usr', 'local', 'envs', conda_env_w_ext, 'bin') python_path = os.path.join(python_dir, 'python3') - setup_env += f" echo 'export PATH={python_dir}:$PATH;' >> /etc/bashrc;" + setup_env.append(_add_to_bashrc(f'export PATH={python_dir}:$PATH')) # if py_embed listed in requirements and using a Python # environment that differs from the MET env, set MET_PYTHON_EXE @@ -86,49 +87,54 @@ def handle_automation_env(host_name, reqs, work_dir): if any([item for item in PLOTCALC_KEYWORDS if item in str(reqs).lower()]): ce_file = os.path.join(work_dir, '.github', 'parm', f'Externals_metplotcalcpy{externals_ext}') - setup_env += ( - f'cd {METPLUS_DOCKER_LOC};' - f'{work_dir}/manage_externals/checkout_externals -e {ce_file};' - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METplotpy;' - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METcalcpy;' - 'cd -;' - ) + setup_env.extend(( + f'cd {METPLUS_DOCKER_LOC}', + f'{work_dir}/manage_externals/checkout_externals -e {ce_file}', + f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METplotpy', + f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METcalcpy', + 'cd -', + )) # if metdataio is in requirements list, add command to obtain METdataio if 'metdataio' in str(reqs).lower(): ce_file = os.path.join(work_dir, '.github', 'parm', f'Externals_metdataio{externals_ext}') - setup_env += ( - f'cd {METPLUS_DOCKER_LOC};' - f'{work_dir}/manage_externals/checkout_externals -e {ce_file};' - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METdataio;' - 'cd -;' - ) + setup_env.extend(( + f'cd {METPLUS_DOCKER_LOC}', + f'{work_dir}/manage_externals/checkout_externals -e {ce_file}', + f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METdataio', + 'cd -', + )) # if gempak is in requirements list, add JRE bin to path for java if 'gempak' in str(reqs).lower(): - setup_env += 'export PATH=$PATH:/usr/lib/jvm/jre/bin;' + setup_env.append(_add_to_bashrc( + 'export PATH=$PATH:/usr/lib/jvm/jre/bin' + )) # if metplus is in requirements list, # add top of METplus repo to PYTHONPATH so metplus can be imported if 'metplus' in str(reqs).lower(): - setup_env += f'export PYTHONPATH={METPLUS_DOCKER_LOC}:$PYTHONPATH;' + setup_env.append(_add_to_bashrc( + f'export PYTHONPATH={METPLUS_DOCKER_LOC}:$PYTHONPATH' + )) # list packages in python environment that will be used if conda_env not in NOT_PYTHON_ENVS: - setup_env += ( - f'echo Using environment: dtcenter/metplus-envs:{conda_env_w_ext};' - f'echo cat /usr/local/envs/{conda_env_w_ext}/environments.yml;' - f'echo ----------------------------------------;' - f'cat /usr/local/envs/{conda_env_w_ext}/environments.yml;' - 'echo ----------------------------------------;' - ) + setup_env.extend(( + f'echo Using environment: dtcenter/metplus-envs:{conda_env_w_ext}', + f'echo cat /usr/local/envs/{conda_env_w_ext}/environments.yml', + f'echo ----------------------------------------', + f'cat /usr/local/envs/{conda_env_w_ext}/environments.yml', + 'echo ----------------------------------------', + )) - return setup_env, py_embed_arg + return ';'.join(setup_env), py_embed_arg -#def _add_to_bashrc(command): -# return f"echo '{command.replace('$', '\\$')};' >> /etc/bashrc" +def _add_to_bashrc(command): + return f"echo '{command};' >> /etc/bashrc" + def main(categories, subset_list, work_dir=None, host_name=os.environ.get('HOST_NAME')): From 3a1ab23d25e70b48acb602d8defa9e7ef8b54fd8 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 16:58:02 -0700 Subject: [PATCH 35/59] remove source of bashrc file because running with login shell, added comment to note which values were added by test script --- .github/jobs/get_use_case_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 033da84e89..f1cb2046fb 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -61,7 +61,7 @@ def handle_automation_env(host_name, reqs, work_dir): # start building commands to run before run_metplus.py in Docker setup_env = [] - setup_env.append('source /etc/bashrc') + setup_env.append(_add_to_bashrc('# BELOW WAS ADDED BY TEST SCRIPT')) # add conda bin to beginning of PATH python_dir = os.path.join('/usr', 'local', 'envs', From 806f7689422bf4c487d2f074c57c20be95ca2f79 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 17:10:44 -0700 Subject: [PATCH 36/59] added GHA log groups for all commands run through time_command, moved TIMING info after ERROR info to match other command order --- .github/jobs/bash_functions.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/jobs/bash_functions.sh b/.github/jobs/bash_functions.sh index c018add863..1e68c80614 100755 --- a/.github/jobs/bash_functions.sh +++ b/.github/jobs/bash_functions.sh @@ -1,16 +1,20 @@ #! /bin/bash # utility function to run command get log the time it took to run +# ::group:: and ::endgroup:: create collapsible log groups in GitHub Actions function time_command { local start_seconds=$SECONDS - echo "RUNNING: $*" + echo "::group::RUNNING: $*" "$@" local error=$? + echo "::endgroup::" - local duration=$(( SECONDS - start_seconds )) - echo "TIMING: Command took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS): '$*'" if [ ${error} -ne 0 ]; then echo "ERROR: '$*' exited with status = ${error}" fi + + local duration=$(( SECONDS - start_seconds )) + echo "TIMING: Command took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS): '$*'" + return $error } From 34e1abdcff43aeeff08b265334a51e79a64dba57 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 17:11:09 -0700 Subject: [PATCH 37/59] fix spacing between functions --- .github/jobs/docker_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/jobs/docker_utils.py b/.github/jobs/docker_utils.py index 4df2384b44..724da6213d 100644 --- a/.github/jobs/docker_utils.py +++ b/.github/jobs/docker_utils.py @@ -15,6 +15,7 @@ # extension to add to conda environments VERSION_EXT = '.v5' + def get_data_repo(branch_name): """! Branch names that start with main_v or contain only digits and dots with out without a prefix 'v' will return @@ -26,10 +27,12 @@ def get_data_repo(branch_name): return DOCKERHUB_METPLUS_DATA return DOCKERHUB_METPLUS_DATA_DEV + def get_dockerhub_url(branch_name): data_repo = get_data_repo(branch_name) return f'https://hub.docker.com/v2/repositories/{data_repo}/tags' + def docker_get_volumes_last_updated(current_branch): import requests dockerhub_url = get_dockerhub_url(current_branch) @@ -60,6 +63,7 @@ def docker_get_volumes_last_updated(current_branch): return volumes_last_updated + def get_branch_name(): # get branch name from env var BRANCH_NAME branch_name = os.environ.get('BRANCH_NAME') From 5fa6c63118d882c807afb0244a360312f8ab0445 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 17:11:39 -0700 Subject: [PATCH 38/59] clean up GHA log grouping, ci-skip-unit-tests --- .github/actions/run_tests/entrypoint.sh | 2 -- .github/jobs/get_data_volumes.py | 3 +++ .github/jobs/setup_and_run_use_cases.py | 6 +----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/actions/run_tests/entrypoint.sh b/.github/actions/run_tests/entrypoint.sh index d819abf50d..3bfad88343 100644 --- a/.github/actions/run_tests/entrypoint.sh +++ b/.github/actions/run_tests/entrypoint.sh @@ -21,9 +21,7 @@ fi # try to pull image from DockerHub DOCKERHUBTAG=dtcenter/metplus-dev:${branch_name} -echo "::group::Docker pull" time_command docker pull $DOCKERHUBTAG -echo "::endgroup::" # if unsuccessful (i.e. pull request from a fork) # then build image locally diff --git a/.github/jobs/get_data_volumes.py b/.github/jobs/get_data_volumes.py index b544609e52..01c706cb6c 100755 --- a/.github/jobs/get_data_volumes.py +++ b/.github/jobs/get_data_volumes.py @@ -13,6 +13,7 @@ from docker_utils import docker_get_volumes_last_updated, get_branch_name from docker_utils import get_data_repo, DOCKERHUB_METPLUS_DATA_DEV + def main(args): # get METplus version version_file = os.path.abspath(os.path.join(os.path.dirname(__file__), @@ -94,7 +95,9 @@ def main(args): print(f"CREATING DATA VOLUME FROM: {full_volume_name}") cmd = (f'docker create --name {model_app_name} ' f'{full_volume_name}') + print(f"::group::{cmd}") ret = subprocess.run(shlex.split(cmd)) + print('::endgroup::') if ret.returncode: continue diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 5cd87310dc..54d0514932 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -91,12 +91,9 @@ def main(): ) print(f'Building Docker environment/branch image...') - print("::group::Docker build test environment") if not run_docker_commands([docker_build_cmd]): - print("::endgroup::") isOK = False continue - print("::endgroup::") all_commands = [] all_commands.append('docker images') @@ -126,8 +123,7 @@ def main(): def run_docker_commands(docker_commands): is_ok = True for docker_command in docker_commands: - print(f"RUNNING: {docker_command}") - print(f"::group::{docker_command}") + print(f"::group::RUNNING {docker_command}") start_time = time.time() try: process = subprocess.Popen(shlex.split(docker_command), From 39983fb32ad39f92d6b938bcbfcb059731d1c04e Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 17:23:36 -0700 Subject: [PATCH 39/59] put TIMING inside GHA log group and output error message outside group, ci-skip-unit-tests --- .github/jobs/bash_functions.sh | 6 +++--- .github/jobs/setup_and_run_use_cases.py | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/jobs/bash_functions.sh b/.github/jobs/bash_functions.sh index 1e68c80614..e909ea4930 100755 --- a/.github/jobs/bash_functions.sh +++ b/.github/jobs/bash_functions.sh @@ -7,14 +7,14 @@ function time_command { echo "::group::RUNNING: $*" "$@" local error=$? + + local duration=$(( SECONDS - start_seconds )) + echo "TIMING: Command took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS): '$*'" echo "::endgroup::" if [ ${error} -ne 0 ]; then echo "ERROR: '$*' exited with status = ${error}" fi - local duration=$(( SECONDS - start_seconds )) - echo "TIMING: Command took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS): '$*'" - return $error } diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 54d0514932..319f40d954 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -123,6 +123,7 @@ def main(): def run_docker_commands(docker_commands): is_ok = True for docker_command in docker_commands: + error_message = None print(f"::group::RUNNING {docker_command}") start_time = time.time() try: @@ -143,17 +144,19 @@ def run_docker_commands(docker_commands): raise subprocess.CalledProcessError(rc, docker_command) except subprocess.CalledProcessError as err: - print("::endgroup::") - print(f"ERROR: Command failed -- {err}") + error_message = f"ERROR: Command failed -- {err}" is_ok = False - else: - print("::endgroup::") end_time = time.time() print("TIMING: Command took " f"{time.strftime('%M:%S', time.gmtime(end_time - start_time))}" f" (MM:SS): '{docker_command}')") + print("::endgroup::") + + if error_message: + print(error_message) + return is_ok From 5e6c4c501e7fc4723aa55084a102961f2b8695c1 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 15 Dec 2022 17:23:56 -0700 Subject: [PATCH 40/59] turn off use case group after tests, ci-run-all-diff --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 1ee662b076..056013dd1b 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -17,7 +17,7 @@ { "category": "climate", "index_list": "0-1", - "run": true + "run": false }, { "category": "short_range", From f774bff7b2384f60acce200788bb2c6ff295ab5f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 10:06:36 -0700 Subject: [PATCH 41/59] add GitHub Actions log group for full diff results so it is easier to read --- metplus/util/diff_util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metplus/util/diff_util.py b/metplus/util/diff_util.py index 15eaeee787..611732ab9b 100644 --- a/metplus/util/diff_util.py +++ b/metplus/util/diff_util.py @@ -71,6 +71,7 @@ def get_file_type(filepath): def compare_dir(dir_a, dir_b, debug=False, save_diff=False): + print('::group::Full diff results:') # if input are files and not directories, compare them if os.path.isfile(dir_a): result = compare_files(dir_a, dir_b, debug=debug, save_diff=save_diff) @@ -136,6 +137,7 @@ def compare_dir(dir_a, dir_b, debug=False, save_diff=False): print(f"ERROR: File does not exist: {filepath_a}") diff_files.append(('', filepath_b, 'file not found (new output)', '')) + print('::endgroup::') print("\n\n**************************************************\nSummary:\n") if diff_files: print("\nERROR: Some differences were found") From 39b8b8fce15bc89ec6f6e745cd75e5ba4dd07047 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 10:07:34 -0700 Subject: [PATCH 42/59] created python function to run commands with timing info, error reporting, and GitHub Actions log grouping --- .github/jobs/docker_utils.py | 56 ++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.github/jobs/docker_utils.py b/.github/jobs/docker_utils.py index 724da6213d..0e5c365f3f 100644 --- a/.github/jobs/docker_utils.py +++ b/.github/jobs/docker_utils.py @@ -1,5 +1,8 @@ import os import re +import subprocess +import shlex +import time # Utilities used by various CI jobs. Functionality includes: # - Check if Docker data volumes need to be updated. @@ -86,3 +89,56 @@ def get_branch_name(): return None return github_ref.replace('refs/heads/', '').replace('/', '_') + + +def run_commands(commands): + """!Run a list of commands via subprocess. Print the command and the length + of time it took to run. Includes ::group:: and ::endgroup:: syntax which + creates log groups in GitHub Actions log output. + + @param commands list of commands to run or a single command string + @returns True if all commands ran successfully, False if any commands fail + """ + # handle a single command string or list of command strings + if isinstance(commands, str): + command_list = [commands] + else: + command_list = commands + + is_ok = True + for command in command_list: + error_message = None + print(f"::group::RUNNING {command}") + start_time = time.time() + try: + process = subprocess.Popen(shlex.split(command), + shell=False, + encoding='utf-8', + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + # Poll process.stdout to show stdout live + while True: + output = process.stdout.readline() + if process.poll() is not None: + break + if output: + print(output.strip()) + rc = process.poll() + if rc: + raise subprocess.CalledProcessError(rc, command) + + except subprocess.CalledProcessError as err: + error_message = f"ERROR: Command failed -- {err}" + is_ok = False + + end_time = time.time() + print("TIMING: Command took " + f"{time.strftime('%M:%S', time.gmtime(end_time - start_time))}" + f" (MM:SS): '{command}')") + + print("::endgroup::") + + if error_message: + print(error_message) + + return is_ok From 69da3293934838096ec53ede340d56217b63c7dc Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 10:08:53 -0700 Subject: [PATCH 43/59] call run_commands function to consistently run processes and group logs, cleaned up logic to capitalize constants and created helper functions for better readability --- .github/jobs/get_data_volumes.py | 7 +- .github/jobs/setup_and_run_use_cases.py | 158 +++++++++++------------- 2 files changed, 74 insertions(+), 91 deletions(-) diff --git a/.github/jobs/get_data_volumes.py b/.github/jobs/get_data_volumes.py index 01c706cb6c..5c568577a1 100755 --- a/.github/jobs/get_data_volumes.py +++ b/.github/jobs/get_data_volumes.py @@ -12,6 +12,7 @@ from docker_utils import docker_get_volumes_last_updated, get_branch_name from docker_utils import get_data_repo, DOCKERHUB_METPLUS_DATA_DEV +from docker_utils import run_commands def main(args): @@ -95,11 +96,7 @@ def main(args): print(f"CREATING DATA VOLUME FROM: {full_volume_name}") cmd = (f'docker create --name {model_app_name} ' f'{full_volume_name}') - print(f"::group::{cmd}") - ret = subprocess.run(shlex.split(cmd)) - print('::endgroup::') - - if ret.returncode: + if not run_commands(cmd): continue # add name to volumes from list to pass to docker build diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 319f40d954..a08d20fb0e 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -10,26 +10,31 @@ import os import sys -import subprocess -import shlex -import time import get_use_case_commands import get_data_volumes -from docker_utils import get_branch_name, VERSION_EXT +from docker_utils import get_branch_name, VERSION_EXT, run_commands -runner_workspace = os.environ.get('RUNNER_WORKSPACE') -github_workspace = os.environ.get('GITHUB_WORKSPACE') +RUNNER_WORKSPACE = os.environ.get('RUNNER_WORKSPACE') +GITHUB_WORKSPACE = os.environ.get('GITHUB_WORKSPACE') -repo_name =os.path.basename(runner_workspace) -ws_path = os.path.join(runner_workspace, repo_name) +REPO_NAME = os.path.basename(RUNNER_WORKSPACE) +WS_PATH = os.path.join(RUNNER_WORKSPACE, REPO_NAME) -docker_data_dir = '/data' -docker_output_dir = os.path.join(docker_data_dir, 'output') -gha_output_dir = os.path.join(runner_workspace, 'output') +DOCKER_DATA_DIR = '/data' +DOCKER_OUTPUT_DIR = os.path.join(DOCKER_DATA_DIR, 'output') +GHA_OUTPUT_DIR = os.path.join(RUNNER_WORKSPACE, 'output') RUN_TAG = 'metplus-run-env' +VOLUME_MOUNTS = [ + f"-v {RUNNER_WORKSPACE}/output/mysql:/var/lib/mysql", + f"-v {GHA_OUTPUT_DIR}:{DOCKER_OUTPUT_DIR}", + f"-v {WS_PATH}:{GITHUB_WORKSPACE}", +] + +DOCKERFILE_DIR = os.path.join('.github', 'actions', 'run_tests') + def main(): categories, subset_list, _ = ( @@ -51,68 +56,53 @@ def main(): if os.environ.get('GITHUB_EVENT_NAME') == 'pull_request': branch_name = f"{branch_name}-pull_request" - dockerfile_dir = os.path.join('.github', 'actions', 'run_tests') - # use BuildKit to build image os.environ['DOCKER_BUILDKIT'] = '1' - volume_mounts = [ - f"-v {runner_workspace}/output/mysql:/var/lib/mysql", - f"-v {gha_output_dir}:{docker_output_dir}", - f"-v {ws_path}:{github_workspace}", - ] - isOK = True for setup_commands, use_case_commands, requirements in all_commands: - # get environment image tag - use_env = [item for item in requirements if item.endswith('_env')] - if use_env: - env_tag = use_env[0].replace('_env', '') - else: - env_tag = 'metplus_base' - - env_tag = f'{env_tag}{VERSION_EXT}' + env_tag = _get_metplus_env_tag(requirements) # get Dockerfile to use - dockerfile_name = 'Dockerfile.run' - if 'gempak' in str(requirements).lower(): - dockerfile_name = f'{dockerfile_name}_gempak' - elif 'gfdl' in str(requirements).lower(): - dockerfile_name = f'{dockerfile_name}_gfdl' - elif 'cartopy' in str(requirements).lower(): - dockerfile_name = f'{dockerfile_name}_cartopy' + dockerfile_name = _get_dockerfile_name(requirements) docker_build_cmd = ( f"docker build -t {RUN_TAG} " f"--build-arg METPLUS_IMG_TAG={branch_name} " f"--build-arg METPLUS_ENV_TAG={env_tag} " - f"-f {dockerfile_dir}/{dockerfile_name} ." + f"-f {DOCKERFILE_DIR}/{dockerfile_name} ." ) print(f'Building Docker environment/branch image...') - if not run_docker_commands([docker_build_cmd]): + if not run_commands([docker_build_cmd]): isOK = False continue - all_commands = [] - all_commands.append('docker images') - all_commands.append( + commands = [] + commands.append('docker images') + # start interactive container in the background + commands.append( f"docker run -d --rm -it -e GITHUB_WORKSPACE " f"--name {RUN_TAG} " f"{os.environ.get('NETWORK_ARG', '')} " - f"{' '.join(volume_mounts)} " - f"{volumes_from} --workdir {github_workspace} " + f"{' '.join(VOLUME_MOUNTS)} " + f"{volumes_from} --workdir {GITHUB_WORKSPACE} " f'{RUN_TAG} bash' ) - all_commands.append('docker ps -a') - for use_case_command in [setup_commands] + use_case_commands + ['cat /etc/bashrc']: - all_commands.append( + # list running containers + commands.append('docker ps -a') + # execute commands in running docker container + docker_commands = [setup_commands] + use_case_commands + docker_commands.append('cat /etc/bashrc') + for docker_command in docker_commands: + commands.append( f'docker exec -e GITHUB_WORKSPACE {RUN_TAG} ' - f'bash -cl "{use_case_command}"' + f'bash -cl "{docker_command}"' ) - all_commands.append(f'docker rm -f {RUN_TAG}') - if not run_docker_commands(all_commands): + # force remove container to stop and remove it + commands.append(f'docker rm -f {RUN_TAG}') + if not run_commands(commands): isOK = False if not isOK: @@ -120,45 +110,41 @@ def main(): sys.exit(1) -def run_docker_commands(docker_commands): - is_ok = True - for docker_command in docker_commands: - error_message = None - print(f"::group::RUNNING {docker_command}") - start_time = time.time() - try: - process = subprocess.Popen(shlex.split(docker_command), - shell=False, - encoding='utf-8', - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - # Poll process.stdout to show stdout live - while True: - output = process.stdout.readline() - if process.poll() is not None: - break - if output: - print(output.strip()) - rc = process.poll() - if rc: - raise subprocess.CalledProcessError(rc, docker_command) - - except subprocess.CalledProcessError as err: - error_message = f"ERROR: Command failed -- {err}" - is_ok = False - - end_time = time.time() - print("TIMING: Command took " - f"{time.strftime('%M:%S', time.gmtime(end_time - start_time))}" - f" (MM:SS): '{docker_command}')") - - print("::endgroup::") - - if error_message: - print(error_message) - - return is_ok - +def _get_metplus_env_tag(requirements): + """!Parse use case requirements to get Docker tag to obtain conda + environment to use in tests. Append version extension e.g. .v5 + + @param requirements list of use case requirements + @returns string of Docker tag + """ + use_env = [item for item in requirements if item.endswith('_env')] + if use_env: + env_tag = use_env[0].replace('_env', '') + else: + env_tag = 'metplus_base' + + return f'{env_tag}{VERSION_EXT}' + + +def _get_dockerfile_name(requirements): + """!Parse use case requirements to get name of Dockerfile to use to build + environment to use in tests. Dockerfile.run copies conda directories into + test image. Other Dockerfiles copy additional files needed to run certain + use cases. For example, cartopy uses shape files that occasionally cannot + be downloaded on the fly, so they are downloaded in advance and copied + into the test image. GEMPAK requires JavaRE. GFDL Tracker requires + NetCDF libraries and tracker executable. + + @param requirements list of use case requirements + @returns string of Dockerfile to use to create test environment + """ + if 'gempak' in str(requirements).lower(): + return f'{dockerfile_name}_gempak' + if 'gfdl' in str(requirements).lower(): + return f'{dockerfile_name}_gfdl' + if 'cartopy' in str(requirements).lower(): + return f'{dockerfile_name}_cartopy' + return 'Dockerfile.run' if __name__ == '__main__': main() From 1baa8c0fe167dd814ef5842aa5ba084346c1a6ed Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 10:10:00 -0700 Subject: [PATCH 44/59] changed calling of diff logic to start detached interactive container, run diff logic, and remove container to better organize log output in GitHub Actions --- .github/jobs/setup_and_run_diff.py | 31 ++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/.github/jobs/setup_and_run_diff.py b/.github/jobs/setup_and_run_diff.py index 9b3725c901..bb35087c24 100755 --- a/.github/jobs/setup_and_run_diff.py +++ b/.github/jobs/setup_and_run_diff.py @@ -11,6 +11,7 @@ sys.path.insert(0, ci_dir) from jobs import get_data_volumes +from jobs.docker_util import run_commands CI_JOBS_DIR = '.github/jobs' @@ -39,22 +40,32 @@ print(f"Output Volumes: {VOLUMES_FROM}") -volume_mounts = [ +VOLUME_MOUNTS = [ f'-v {WS_PATH}:{GITHUB_WORKSPACE}', f'-v {RUNNER_WORKSPACE}/output:/data/output', f'-v {RUNNER_WORKSPACE}/diff:/data/diff', ] -mount_args = ' '.join(volume_mounts) +MOUNT_ARGS = ' '.join(VOLUME_MOUNTS) # command to run inside Docker -cmd = (f'/usr/local/envs/diff{VERSION_EXT}/bin/python3 ' - f'{GITHUB_WORKSPACE}/{CI_JOBS_DIR}/run_diff_docker.py') +diff_command = (f'/usr/local/envs/diff{VERSION_EXT}/bin/python3 ' + f'{GITHUB_WORKSPACE}/{CI_JOBS_DIR}/run_diff_docker.py') + +# start detached interactive diff env container +# mount METplus code and output dir, volumes from output volumes +docker_cmd = ( + f'docker run -d --rm -it --name diff -e GITHUB_WORKSPACE {VOLUMES_FROM}' + f' {MOUNT_ARGS} dtcenter/metplus-envs:diff{VERSION_EXT} bash' +) +if not run_commands(docker_cmd): + sys.exit(1) -# run inside diff env: mount METplus code and output dir, volumes from output volumes -docker_cmd = (f'docker run -e GITHUB_WORKSPACE {VOLUMES_FROM} ' - f'{mount_args} dtcenter/metplus-envs:diff{VERSION_EXT} ' - f'bash -c "{cmd}"') +# execute command to run difference tests in Docker container +# do not run using run_commands function so GitHub Actions log grouping +# can be used to put full diff output into a group so it is easier to +# view the diff summary +docker_cmd = f'docker exec -e GITHUB_WORKSPACE diff bash -cl "{diff_command}"' print(f'RUNNING: {docker_cmd}') try: process = subprocess.Popen(shlex.split(docker_cmd), @@ -76,3 +87,7 @@ except subprocess.CalledProcessError as err: print(f"ERROR: Command failed -- {err}") sys.exit(1) + +# force remove container to stop and remove it +if not run_commands('docker rm -f diff'): + sys.exit(1) From 741bf67bde561d377e0c303812bb897b2a5ffb68 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 10:11:13 -0700 Subject: [PATCH 45/59] turn on use case to test, ci-skip-unit-tests, ci-run-diff --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 056013dd1b..8484370e51 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -57,7 +57,7 @@ { "category": "data_assimilation", "index_list": "0-1", - "run": false + "run": true }, { "category": "marine_and_cryosphere", From 3dd5b6ad47e248208b6bc3cc50e8bdb60edb3923 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 10:38:02 -0700 Subject: [PATCH 46/59] fixed typo in import, ci-skip-unit-tests, ci-run-diff --- .github/jobs/setup_and_run_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/jobs/setup_and_run_diff.py b/.github/jobs/setup_and_run_diff.py index bb35087c24..b1318bcd5f 100755 --- a/.github/jobs/setup_and_run_diff.py +++ b/.github/jobs/setup_and_run_diff.py @@ -11,7 +11,7 @@ sys.path.insert(0, ci_dir) from jobs import get_data_volumes -from jobs.docker_util import run_commands +from jobs.docker_utils import run_commands CI_JOBS_DIR = '.github/jobs' From c9a686c4acef02d3bf4ab1bb8d7f5ffb5345e33b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 10:48:22 -0700 Subject: [PATCH 47/59] turn off use case after test --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 8484370e51..056013dd1b 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -57,7 +57,7 @@ { "category": "data_assimilation", "index_list": "0-1", - "run": true + "run": false }, { "category": "marine_and_cryosphere", From 22cb0ce4390b9f703c433c2b5542a4b3803eca6c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 10:51:19 -0700 Subject: [PATCH 48/59] turn on use case group and add typo to conf to create example of how GHA logs look when a use case fails, ci-skip-unit-tests, ci-run-diff --- .github/parm/use_case_groups.json | 2 +- .../climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 056013dd1b..1ee662b076 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -17,7 +17,7 @@ { "category": "climate", "index_list": "0-1", - "run": false + "run": true }, { "category": "short_range", diff --git a/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf b/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf index f476382081..8e743c251f 100644 --- a/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf +++ b/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf @@ -42,7 +42,7 @@ LEAD_SEQ = 6, 12 FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE}/model_applications/climate/CESM FCST_GRID_STAT_INPUT_TEMPLATE = MetPlus.globe.{init?fmt=%Y-%m-%d}-00000.cam.h0.{init?fmt=%Y-%m-%d}-10800.nc -OBS_GRID_STAT_INPUT_DIR = {INPUT_BASE}/model_applications/climate/gfs_analysis +OBS_GRID_STAT_INPUT_DIR = {INPUT_BASE}/model_applications/climate/gfs_analysisX OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/gfsanl_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}_000.grb2 GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/climate/CESM_GridStat From f7bebacc3f5544c4ba529a76e4cf3a18a386f1dd Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 10:58:14 -0700 Subject: [PATCH 49/59] Revert "turn on use case group and add typo to conf to create example of how GHA logs look when a use case fails, ci-skip-unit-tests, ci-run-diff" This reverts commit 22cb0ce4390b9f703c433c2b5542a4b3803eca6c. --- .github/parm/use_case_groups.json | 2 +- .../climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 1ee662b076..056013dd1b 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -17,7 +17,7 @@ { "category": "climate", "index_list": "0-1", - "run": true + "run": false }, { "category": "short_range", diff --git a/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf b/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf index 8e743c251f..f476382081 100644 --- a/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf +++ b/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf @@ -42,7 +42,7 @@ LEAD_SEQ = 6, 12 FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE}/model_applications/climate/CESM FCST_GRID_STAT_INPUT_TEMPLATE = MetPlus.globe.{init?fmt=%Y-%m-%d}-00000.cam.h0.{init?fmt=%Y-%m-%d}-10800.nc -OBS_GRID_STAT_INPUT_DIR = {INPUT_BASE}/model_applications/climate/gfs_analysisX +OBS_GRID_STAT_INPUT_DIR = {INPUT_BASE}/model_applications/climate/gfs_analysis OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/gfsanl_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}_000.grb2 GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/climate/CESM_GridStat From d80e8f47e20b19b4754d803f5fa3503451c318ef Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 11:02:40 -0700 Subject: [PATCH 50/59] clean up function call because run_commands now accepts a string instead of requiring a list --- .github/jobs/setup_and_run_use_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index a08d20fb0e..941f86123a 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -75,7 +75,7 @@ def main(): ) print(f'Building Docker environment/branch image...') - if not run_commands([docker_build_cmd]): + if not run_commands(docker_build_cmd): isOK = False continue From e1fe6f6f2cf403e020f56961af379915c5175aa3 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 11:46:29 -0700 Subject: [PATCH 51/59] added documentation for LOG_LEVEL_TERMINAL and cleaned up info about LOG_LEVEL --- docs/Users_Guide/configuration.rst | 17 ++++++++++++++--- docs/Users_Guide/glossary.rst | 23 ++++++++++++++--------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/docs/Users_Guide/configuration.rst b/docs/Users_Guide/configuration.rst index 507f67b7b4..071292ad7e 100644 --- a/docs/Users_Guide/configuration.rst +++ b/docs/Users_Guide/configuration.rst @@ -266,6 +266,8 @@ Log File Information Where to write logs files +.. _log_metplus: + LOG_METPLUS """"""""""" @@ -348,9 +350,10 @@ How much information to log LOG_LEVEL """"""""" -This controls the level of logging output from the METplus wrappers. It does -not control the logging level of the actual MET applications. The possible -values to: +This controls the level of logging output from the METplus wrappers that is +written to the log file defined by :ref:`log_metplus`. +It does not control the logging level of the actual MET applications. +The possible values to: * CRITICAL * ERROR @@ -371,6 +374,14 @@ If a use case is producing errors, then setting:: will produce additional logging output that is helpful to discover the cause of the error. +.. _log_level_terminal: + +LOG_LEVEL_TERMINAL +"""""""""""""""""" + +This controls the level of logging that is output to the screen. +The valid values are the same as :ref:`log_level`. + LOG_MET_VERBOSITY """"""""""""""""" diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 58a81365c1..d4ebf18ae6 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -2361,15 +2361,20 @@ METplus Configuration Glossary | *Used by:* All LOG_LEVEL - Specify the level of logging. Everything above this level is sent to standard output. To quiet the output to a comfortable level, set this to "ERROR" - - Options (ordered MOST verbose to LEAST verbose): - | NOTSET - | DEBUG - | INFO - | WARNING - | ERROR - | CRITICAL + Specify the level of logging for the METplus wrapper output to the + :term:`LOG_METPLUS` log file. Log level of the applications that are + called by the wrappers are controlled with :term:`LOG_MET_VERBOSITY`. + Default log level is INFO. Set to DEBUG to see additional log output. + Log level for screen output can be set with :term:`LOG_LEVEL_TERMINAL`. + + Options (ordered LEAST verbose to MOST verbose): + CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET + + | *Used by:* All + + LOG_LEVEL_TERMINAL + Specify the level of logging for terminal screen output. + See :term:`LOG_LEVEL` for more information. | *Used by:* All From 84767d009526fc5f741f614e3e0b984ad9dfdbb4 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 11:47:26 -0700 Subject: [PATCH 52/59] if LOG_LEVEL_TERMINAL is set so that INFO log message are not output to the screen, then print important log info to the screen like starting and stopping info --- metplus/util/run_util.py | 21 ++++++++++++++++----- metplus/util/string_manip.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index 880652528f..e201392f42 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -1,11 +1,12 @@ import sys import os import shutil +import logging from datetime import datetime from importlib import import_module from .constants import NO_COMMAND_WRAPPERS -from .string_manip import get_logfile_info +from .string_manip import get_logfile_info, log_terminal_includes_info from .system_util import get_user_info, write_list_to_file from .config_util import get_process_list, handle_env_var_config from .config_util import handle_tmp_dir, write_final_conf, write_all_commands @@ -29,8 +30,13 @@ def pre_run_setup(config_inputs): user_string = f' as user {user_info} ' if user_info else ' ' config.set('config', 'METPLUS_VERSION', version_number) - logger.info('Running METplus v%s%swith command: %s', - version_number, user_string, ' '.join(sys.argv)) + running_log = (f"Running METplus v{version_number}{user_string}with " + f"command: {' '.join(sys.argv)}") + logger.info(running_log) + + # print running message if terminal log does not include INFO + if not log_terminal_includes_info(config): + print(running_log) # if log file is not set, log message instructing user how to set it log_file = get_logfile_info(config) @@ -195,8 +201,13 @@ def post_run_cleanup(config, app_name, total_errors): user_string = f' as user {user_info}' if user_info else '' if not total_errors: logger.info(log_message) - logger.info('%s has successfully finished running%s.', - app_name, user_string) + success_log = (f'{app_name} has successfully ' + f'finished running{user_string}.') + logger.info(success_log) + + # print success log message if terminal does not include INFO + if not log_terminal_includes_info(config): + print(success_log) return error_msg = (f'{app_name} has finished running{user_string} ' diff --git a/metplus/util/string_manip.py b/metplus/util/string_manip.py index 2e277bda50..3ddc53e02d 100644 --- a/metplus/util/string_manip.py +++ b/metplus/util/string_manip.py @@ -10,6 +10,7 @@ from csv import reader import random import string +import logging try: from .constants import VALID_COMPARISONS, LOWER_TO_WRAPPER_NAME @@ -571,3 +572,18 @@ def get_logfile_info(config): """ log_file = config.getstr('config', 'LOG_METPLUS', '') return log_file if log_file else 'Set LOG_METPLUS to write logs to a file' + + +def log_terminal_includes_info(config): + """!Check LOG_LEVEL_TERMINAL to see if it is set to a logging level that + includes INFO output. Check [runtime] section if not found in [config] + because the variable is moved at the end of the run. + + @param config METplusConfig object to query + @returns True if log level is set to include INFO messages. False if not. + """ + log_terminal_level = logging.getLevelName( + config.getstr('config', 'LOG_LEVEL_TERMINAL', + config.getstr('runtime', 'LOG_LEVEL_TERMINAL')) + ) + return log_terminal_level <= logging.INFO From 3bad7eff7a1167c98cb9c1387c44c386047660b0 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 12:02:55 -0700 Subject: [PATCH 53/59] set file log level to DEBUG for automated tests so that the screen output is clean with INFO level and the log files that can be downloaded contain additional DEBUG logs --- .github/parm/test_settings.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/test_settings.conf b/.github/parm/test_settings.conf index 7f54618714..5e211e806f 100644 --- a/.github/parm/test_settings.conf +++ b/.github/parm/test_settings.conf @@ -1,5 +1,5 @@ [config] -LOG_LEVEL = INFO +LOG_LEVEL = DEBUG LOG_LEVEL_TERMINAL = INFO LOG_MET_OUTPUT_TO_METPLUS = no LOG_LINE_FORMAT = (%(filename)s) %(levelname)s: %(message)s From 64e82adf4e9f1ab4a30a5360b2e35c22eb370716 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 16 Dec 2022 12:21:25 -0700 Subject: [PATCH 54/59] fixed bug in automated logic for getting Dockerfile to run tests --- .github/jobs/setup_and_run_use_cases.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 941f86123a..7453042102 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -138,13 +138,15 @@ def _get_dockerfile_name(requirements): @param requirements list of use case requirements @returns string of Dockerfile to use to create test environment """ + dockerfile_name = 'Dockerfile.run' if 'gempak' in str(requirements).lower(): return f'{dockerfile_name}_gempak' if 'gfdl' in str(requirements).lower(): return f'{dockerfile_name}_gfdl' if 'cartopy' in str(requirements).lower(): return f'{dockerfile_name}_cartopy' - return 'Dockerfile.run' + return dockerfile_name + if __name__ == '__main__': main() From 205d4608aada836a958723cd0b16b3c58d08e2fc Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 19 Dec 2022 11:31:25 -0700 Subject: [PATCH 55/59] gather list of use cases that failed or could not be run to print summary at end of all use case runs --- .github/jobs/setup_and_run_use_cases.py | 56 ++++++++++++++++++++----- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 7453042102..4330a6a653 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -60,6 +60,7 @@ def main(): os.environ['DOCKER_BUILDKIT'] = '1' isOK = True + failed_use_cases = [] for setup_commands, use_case_commands, requirements in all_commands: # get environment image tag env_tag = _get_metplus_env_tag(requirements) @@ -80,7 +81,10 @@ def main(): continue commands = [] + + # print list of existing docker images commands.append('docker images') + # start interactive container in the background commands.append( f"docker run -d --rm -it -e GITHUB_WORKSPACE " @@ -90,26 +94,58 @@ def main(): f"{volumes_from} --workdir {GITHUB_WORKSPACE} " f'{RUN_TAG} bash' ) + # list running containers commands.append('docker ps -a') - # execute commands in running docker container - docker_commands = [setup_commands] + use_case_commands - docker_commands.append('cat /etc/bashrc') - for docker_command in docker_commands: - commands.append( - f'docker exec -e GITHUB_WORKSPACE {RUN_TAG} ' - f'bash -cl "{docker_command}"' - ) - # force remove container to stop and remove it - commands.append(f'docker rm -f {RUN_TAG}') + + # execute setup commands in running docker container + commands.append(_format_docker_exec_command(setup_commands)) + + # run docker commands and skip running cases if something went wrong if not run_commands(commands): isOK = False + # force remove container if setup step fails + run_commands(f'docker rm -f {RUN_TAG}') + + # add all use cases that couldn't run to list of failed cases + failed_use_cases.extend(use_case_commands) + + continue + + # execute use cases in running docker container + # save list of use cases that failed + for use_case_command in use_case_commands: + if not run_commands(_format_docker_exec_command(use_case_command)): + failed_use_cases.append(use_case_command) + isOK = False + + # print bashrc file to see what was added by setup commands + # then force remove container to stop and remove it + if not run_commands([ + _format_docker_exec_command('cat /etc/bashrc'), + f'docker rm -f {RUN_TAG}', + ]): + isOK = False + + # print summary of use cases that failed + for failed_use_case in failed_use_cases: + print(f'ERROR: Use case failed: {failed_use_case}') + if not isOK: print("ERROR: Some commands failed.") sys.exit(1) +def _format_docker_exec_command(command): + """! Get docker exec command to call given command in a bash login shell + + @param command string of command to run in docker + @returns string of docker exec command to run command + """ + return f'docker exec -e GITHUB_WORKSPACE {RUN_TAG} bash -cl "{command}"' + + def _get_metplus_env_tag(requirements): """!Parse use case requirements to get Docker tag to obtain conda environment to use in tests. Append version extension e.g. .v5 From c679f3ea9fcc668a71aa6c6f959ea76cb0abb22f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 19 Dec 2022 11:31:44 -0700 Subject: [PATCH 56/59] turn on use case group to test, ci-skip-unit-tests --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 056013dd1b..f4cf0ce79c 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -2,7 +2,7 @@ { "category": "met_tool_wrapper", "index_list": "0-29,59-61", - "run": false + "run": true }, { "category": "met_tool_wrapper", From b8722a0449fe0fb8df511f516627f953416bc1b8 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 19 Dec 2022 11:53:02 -0700 Subject: [PATCH 57/59] purposely cause a use case to fail, ci-skip-unit-tests --- parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf index 529e6999ca..2a609bf372 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf @@ -41,7 +41,7 @@ LEAD_SEQ = 15M FCST_PCP_COMBINE_RUN = True -FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE}/met_test/new +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE}/met_test/newX FCST_PCP_COMBINE_INPUT_TEMPLATE = NEWSe_{init?fmt=%Y%m%d}_i{init?fmt=%H%M}_m0_f{valid?fmt=%H%M}.nc FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/PCPCombine/PCPCombine_add From c682a784347af5c8fc8285122d54dcc478c29ff0 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 19 Dec 2022 12:10:56 -0700 Subject: [PATCH 58/59] Revert "purposely cause a use case to fail, ci-skip-unit-tests" This reverts commit b8722a0449fe0fb8df511f516627f953416bc1b8. --- parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf index 2a609bf372..529e6999ca 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf @@ -41,7 +41,7 @@ LEAD_SEQ = 15M FCST_PCP_COMBINE_RUN = True -FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE}/met_test/newX +FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE}/met_test/new FCST_PCP_COMBINE_INPUT_TEMPLATE = NEWSe_{init?fmt=%Y%m%d}_i{init?fmt=%H%M}_m0_f{valid?fmt=%H%M}.nc FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/PCPCombine/PCPCombine_add From e39cd0d338a1842b3d42f171f3f731b8342fe076 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 19 Dec 2022 12:11:01 -0700 Subject: [PATCH 59/59] Revert "turn on use case group to test, ci-skip-unit-tests" This reverts commit c679f3ea9fcc668a71aa6c6f959ea76cb0abb22f. --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index f4cf0ce79c..056013dd1b 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -2,7 +2,7 @@ { "category": "met_tool_wrapper", "index_list": "0-29,59-61", - "run": true + "run": false }, { "category": "met_tool_wrapper",