diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index 4b537af6b4..3b384eee74 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -23,6 +23,7 @@ from produtil.config import ProdConfig from .constants import RUNTIME_CONFS, MISSING_DATA_VALUE, DEPRECATED_DICT +from .constants import UPGRADE_INSTRUCTIONS_URL, DEPRECATED_MET_LIST from .string_template_substitution import get_tags, do_string_sub from .string_manip import getlist, remove_quotes, is_python_script from .string_manip import validate_thresholds @@ -993,14 +994,20 @@ def check_for_deprecated_config(config): w_list = [] all_sed_cmds = [] + # keep track of upgrade instructions to output after all variables are checked + upgrade_notes = set() + for old, depr_info in DEPRECATED_DICT.items(): if not isinstance(depr_info, dict): continue # check if is found in old item, use regex to find vars if found if '' not in old: - handle_deprecated(old, depr_info.get('alt', ''), depr_info, - config, all_sed_cmds, w_list, e_list) + upgrade_note = handle_deprecated(old, depr_info.get('alt', ''), + depr_info, config, all_sed_cmds, + w_list, e_list) + if upgrade_note: + upgrade_notes.add(upgrade_note) continue old_regex = old.replace('', r'(\d+)') @@ -1011,18 +1018,32 @@ def check_for_deprecated_config(config): old_with_index = old.replace('', index) alt_with_index = depr_info.get('alt', '').replace('', index) - handle_deprecated(old_with_index, alt_with_index, depr_info, - config, all_sed_cmds, w_list, e_list) + upgrade_note = handle_deprecated(old_with_index, alt_with_index, + depr_info, config, all_sed_cmds, + w_list, e_list) + if upgrade_note: + upgrade_notes.add(upgrade_note) # if any warning exist, report them if w_list: for warning_msg in w_list: logger.warning(warning_msg) + if 'ensemble' in upgrade_notes: + short_msg = ('Please navigate to the upgrade instructions: ' + f'{UPGRADE_INSTRUCTIONS_URL}') + msg = ('EnsembleStat functionality has been moved to GenEnsProd. ' + 'The required changes to the config files depend on ' + 'the type of evaluation that is being performed. ' + f'{short_msg}') + + e_list.insert(0, msg) + e_list.append(short_msg) + # if any errors exist, report them and exit if e_list: - logger.error('DEPRECATED CONFIG ITEMS WERE FOUND. ' - 'PLEASE REMOVE/REPLACE THEM FROM CONFIG FILES') + logger.error('DEPRECATED CONFIG ITEMS WERE FOUND. PLEASE FOLLOW THE ' + 'INSTRUCTIONS TO UPDATE THE CONFIG FILES') for error_msg in e_list: logger.error(error_msg) return False, all_sed_cmds @@ -1065,15 +1086,17 @@ def check_for_deprecated_met_config_file(config, met_config, sed_cmds, met_tool) config.logger.error(f"Config file does not exist: {met_config}") return False - deprecated_met_list = ['MET_VALID_HHMM', 'GRID_VX', 'CONFIG_DIR'] - deprecated_output_prefix_list = ['FCST_VAR', 'OBS_VAR'] + # skip check if no deprecated variables are set + if not DEPRECATED_MET_LIST: + return all_good + config.logger.debug(f"Checking for deprecated environment variables in: {met_config}") with open(met_config, 'r') as file_handle: lines = file_handle.read().splitlines() for line in lines: - for deprecated_item in deprecated_met_list: + for deprecated_item in DEPRECATED_MET_LIST: if '${' + deprecated_item + '}' in line: all_good = False config.logger.error("Please remove deprecated environment variable " @@ -1107,28 +1130,6 @@ def check_for_deprecated_met_config_file(config, met_config, sed_cmds, met_tool) sed_cmds.append(f"#Add {add_line}") break - - for deprecated_item in deprecated_output_prefix_list: - # if deprecated item found in output prefix or to_grid line, replace line to use - # env var OUTPUT_PREFIX or REGRID_TO_GRID - if '${' + deprecated_item + '}' in line and 'output_prefix' in line: - config.logger.error("output_prefix variable should reference " - "${OUTPUT_PREFIX} environment variable") - new_line = "output_prefix = \"${OUTPUT_PREFIX}\";" - - # escape [ and ] because they are special characters in sed commands - old_line = line.rstrip().replace('[', r'\[').replace(']', r'\]') - - sed_cmds.append(f"sed -i 's|^{old_line}|{new_line}|g' {met_config}") - config.logger.info(f"You will need to add {met_tool}_OUTPUT_PREFIX to the METplus config file" - f" that sets {met_tool}_CONFIG_FILE. Set it to:") - output_prefix = _replace_output_prefix(line) - add_line = f"{met_tool}_OUTPUT_PREFIX = {output_prefix}" - config.logger.info(add_line) - sed_cmds.append(f"#Add {add_line}") - all_good = False - break - return all_good @@ -1244,14 +1245,19 @@ def find_indices_in_config_section(regex, config, sec='config', def handle_deprecated(old, alt, depr_info, config, all_sed_cmds, w_list, e_list): sec = 'config' config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') + + upgrade_note = None + # if deprecated config item is found if not config.has_option(sec, old): - return + return upgrade_note + + upgrade_note = depr_info.get('upgrade') # if it is required to remove, add to error list if not alt: e_list.append("{} should be removed".format(old)) - return + return upgrade_note e_list.append("{} should be replaced with {}".format(old, alt)) @@ -1260,6 +1266,8 @@ def handle_deprecated(old, alt, depr_info, config, all_sed_cmds, w_list, e_list) all_sed_cmds.append(f"sed -i 's|^{old}|{alt}|g' {config_file}") all_sed_cmds.append(f"sed -i 's|{{{old}}}|{{{alt}}}|g' {config_file}") + return upgrade_note + def get_custom_string_list(config, met_tool): var_name = 'CUSTOM_LOOP_LIST' diff --git a/metplus/util/constants.py b/metplus/util/constants.py index 9f3c1c1e89..fabb275975 100644 --- a/metplus/util/constants.py +++ b/metplus/util/constants.py @@ -1,5 +1,10 @@ # Constant variables used throughout the METplus wrappers source code +UPGRADE_INSTRUCTIONS_URL = ( + 'https://metplus.readthedocs.io/en/latest/Users_Guide/' + 'release-notes.html#metplus-wrappers-upgrade-instructions' +) + # dictionary used by get_wrapper_name function to easily convert wrapper # name in many formats to the correct name of the wrapper class LOWER_TO_WRAPPER_NAME = { @@ -114,35 +119,47 @@ # have the same result in a test. 0 may be a valid integer value MISSING_DATA_VALUE = -9999 -# key is the name of the depreacted variable that is no longer allowed in any config files -# value is a dictionary containing information about what to do with the deprecated config -# 'alt' is the alternative name for the deprecated config. this can be a single variable name or -# text to describe multiple variables or how to handle it. Set to None to tell the user to -# just remove the variable -# 'copy' is an optional item (defaults to True). set this to False if one cannot simply replace -# the deprecated config variable name with the value in 'alt' -# template '' : {'alt' : '', 'copy': True}, +# Dictionary used to alert users that they are using deprecated config +# variables and need to update the configs to run METplus +# key is the name of the depreacted variable that is no longer allowed in any +# config files +# value is a dictionary containing information about what to do with the +# deprecated config +# 'alt' is the alternative name for the deprecated config. this can be a +# single variable name or text to describe multiple variables or how to +# handle it. Set to None to tell the user to just remove the variable. +# 'copy' is an optional item (defaults to True). set this to False if one +# cannot simply replace the deprecated variable name with the value in 'alt' +# 'upgrade' is an optional item where the value is a keyword that will output +# additional instructions for the user. +# Valid Values: 'ensemble' DEPRECATED_DICT = { - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON': {}, - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN': {}, - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV': {}, - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS': {}, - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS': {}, - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN': {}, - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX': {}, - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE': {}, - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT': {}, - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY': {}, - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP': {}, - 'ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP': {}, - 'ENSEMBLE_STAT_NBRHD_PROB_WIDTH': {}, - 'ENSEMBLE_STAT_NBRHD_PROB_SHAPE': {}, - 'ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH': {}, - 'ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH': {}, - 'ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE': {}, - 'ENSEMBLE_STAT_NMEP_SMOOTH_METHOD': {}, - 'ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH': {}, - 'ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX': {}, - 'ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS': {}, - + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NBRHD_PROB_WIDTH': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NBRHD_PROB_SHAPE': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_METHOD': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS': {'upgrade': 'ensemble'}, } + +# List of variables in wrapped MET config files that are no longer set +# All explicitly set wrapped MET config files found in a METplus config, +# e.g. GRID_STAT_CONFIG_FILE, will be checked for these variables +# If any of these items are found, then an error will be reported +DEPRECATED_MET_LIST = [ +] diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index fb7b743b35..0981aa59f6 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -48,6 +48,7 @@ def pre_run_setup(config_inputs): config.logger.error(f"Find/Replace commands have been generated in {sed_file}") logger.error("Correct configuration variables and rerun. Exiting.") + logger.info(f"Check the log file for more information: {config.getstr('config', 'LOG_METPLUS')}") sys.exit(1) if not config.getdir('MET_INSTALL_DIR', must_exist=True): @@ -147,7 +148,8 @@ def run_metplus(config): return total_errors except: logger.exception("Fatal error occurred") - logger.info(f"Check the log file for more information: {config.getstr('config', 'LOG_METPLUS')}") + logger.info("Check the log file for more information: " + f"{config.getstr('config', 'LOG_METPLUS')}") return 1 def post_run_cleanup(config, app_name, total_errors):