From 7b365dfd87238b555d2b844814c77c184197ba8b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 14 Nov 2022 10:11:52 -0700 Subject: [PATCH] Update Develop-ref after #1930, #1927, dtcenter/MET#2335 Attempt 2 (#1936) Co-authored-by: johnhg Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Mrinal Biswas Co-authored-by: Julie Prestopnik Co-authored-by: Christina Kalb Co-authored-by: Hank Fisher Co-authored-by: jprestop Co-authored-by: cristianastan2 Co-authored-by: John Halley Gotway Co-authored-by: bikegeek Co-authored-by: Lisa Goodrich Co-authored-by: Julie Prestopnik Co-authored-by: Hank Fisher Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: j-opatz Co-authored-by: Christina Kalb Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: Howard Soh Co-authored-by: Molly Smith Co-authored-by: hsoh-u Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: Will Mayfield <59745143+willmayfield@users.noreply.github.com> --- .github/jobs/docker_setup.sh | 0 .../command_builder/test_command_builder.py | 49 +++++++++-------- .../test_ensemble_stat_wrapper.py | 49 +++++++++++++++-- metplus/util/string_manip.py | 3 ++ metplus/util/time_util.py | 42 +++++++++------ metplus/wrappers/ascii2nc_wrapper.py | 2 +- metplus/wrappers/command_builder.py | 54 +++++-------------- metplus/wrappers/compare_gridded_wrapper.py | 8 ++- metplus/wrappers/ensemble_stat_wrapper.py | 19 ++----- metplus/wrappers/ioda2nc_wrapper.py | 2 +- metplus/wrappers/mode_wrapper.py | 4 +- metplus/wrappers/mtd_wrapper.py | 15 +++--- metplus/wrappers/pb2nc_wrapper.py | 1 - metplus/wrappers/pcp_combine_wrapper.py | 8 +-- metplus/wrappers/plot_data_plane_wrapper.py | 4 +- metplus/wrappers/point2grid_wrapper.py | 3 +- metplus/wrappers/regrid_data_plane_wrapper.py | 38 ++++++------- metplus/wrappers/series_analysis_wrapper.py | 6 +-- 18 files changed, 163 insertions(+), 144 deletions(-) mode change 100644 => 100755 .github/jobs/docker_setup.sh diff --git a/.github/jobs/docker_setup.sh b/.github/jobs/docker_setup.sh old mode 100644 new mode 100755 diff --git a/internal/tests/pytests/wrappers/command_builder/test_command_builder.py b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py index c62609b983..31a2652fd1 100644 --- a/internal/tests/pytests/wrappers/command_builder/test_command_builder.py +++ b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py @@ -6,7 +6,7 @@ import datetime from metplus.wrappers.command_builder import CommandBuilder -from metplus.util import ti_calculate +from metplus.util import ti_calculate, add_field_info_to_time_info def get_data_dir(config): @@ -27,9 +27,9 @@ def test_find_data_no_dated(metplus_config, data_type): config = metplus_config pcw = CommandBuilder(config) - v = {} - v['fcst_level'] = "6" - v['obs_level'] = "6" + var_info = {} + var_info['fcst_level'] = "6" + var_info['obs_level'] = "6" task_info = {} task_info['valid'] = datetime.datetime.strptime("201802010000",'%Y%m%d%H%M') task_info['lead'] = 0 @@ -39,7 +39,8 @@ def test_find_data_no_dated(metplus_config, data_type): pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 3600 pcw.c_dict[f'{data_type}INPUT_DIR'] = get_data_dir(pcw.config) pcw.c_dict[f'{data_type}INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}" - obs_file = pcw.find_data(time_info, v, data_type) + add_field_info_to_time_info(time_info, var_info) + obs_file = pcw.find_data(time_info, data_type) assert obs_file == pcw.c_dict[f'{data_type}INPUT_DIR']+'/20180201_0045' @@ -67,7 +68,7 @@ def test_find_data_not_a_path(metplus_config, data_type): pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 0 pcw.c_dict[f'{data_type}INPUT_DIR'] = '' pcw.c_dict[f'{data_type}INPUT_TEMPLATE'] = 'G003' - obs_file = pcw.find_data(time_info, var_info=None, data_type=data_type) + obs_file = pcw.find_data(time_info, data_type=data_type) assert obs_file == 'G003' @@ -76,8 +77,8 @@ def test_find_obs_no_dated(metplus_config): config = metplus_config pcw = CommandBuilder(config) - v = {} - v['obs_level'] = "6" + var_info = {} + var_info['obs_level'] = "6" task_info = {} task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M') task_info['lead'] = 0 @@ -87,7 +88,8 @@ def test_find_obs_no_dated(metplus_config): pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600 pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config) pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}" - obs_file = pcw.find_obs(time_info, v) + add_field_info_to_time_info(time_info, var_info) + obs_file = pcw.find_obs(time_info) assert obs_file == pcw.c_dict['OBS_INPUT_DIR'] + '/20180201_0045' @@ -96,8 +98,8 @@ def test_find_obs_dated(metplus_config): config = metplus_config pcw = CommandBuilder(config) - v = {} - v['obs_level'] = "6" + var_info = {} + var_info['obs_level'] = "6" task_info = {} task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M') task_info['lead'] = 0 @@ -107,7 +109,8 @@ def test_find_obs_dated(metplus_config): pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600 pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config) pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}' - obs_file = pcw.find_obs(time_info, v) + add_field_info_to_time_info(time_info, var_info) + obs_file = pcw.find_obs(time_info) assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180201/20180201_0013' @@ -126,8 +129,8 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds) config = metplus_config pcw = CommandBuilder(config) - v = {} - v['obs_level'] = "6" + var_info = {} + var_info['obs_level'] = "6" task_info = {} task_info['valid'] = datetime.datetime.strptime("2020020112", '%Y%m%d%H') task_info['lead'] = 0 @@ -136,7 +139,8 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds) pcw.c_dict['OFFSETS'] = offsets pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config) pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}" - obs_file, time_info = pcw.find_obs_offset(time_info, v) + add_field_info_to_time_info(time_info, var_info) + obs_file, time_info = pcw.find_obs_offset(time_info) print(f"OBSFILE: {obs_file}") print(f"EXPECTED FILE: {expected_file}") @@ -153,8 +157,8 @@ def test_find_obs_dated_previous_day(metplus_config): config = metplus_config pcw = CommandBuilder(config) - v = {} - v['obs_level'] = "6" + var_info = {} + var_info['obs_level'] = "6" task_info = {} task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M') task_info['lead'] = 0 @@ -164,7 +168,8 @@ def test_find_obs_dated_previous_day(metplus_config): pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}' pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600 pcw.c_dict['OBS_FILE_WINDOW_END'] = 0 - obs_file = pcw.find_obs(time_info, v) + add_field_info_to_time_info(time_info, var_info) + obs_file = pcw.find_obs(time_info) assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180131/20180131_2345' @@ -173,8 +178,9 @@ def test_find_obs_dated_next_day(metplus_config): config = metplus_config pcw = CommandBuilder(config) - v = {} - v['obs_level'] = "6" + var_info = { + 'obs_level': "6" + } task_info = {} task_info['valid'] = datetime.datetime.strptime("201802012345", '%Y%m%d%H%M') task_info['lead'] = 0 @@ -184,7 +190,8 @@ def test_find_obs_dated_next_day(metplus_config): pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}' pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = 0 pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600 - obs_file = pcw.find_obs(time_info, v) + add_field_info_to_time_info(time_info, var_info) + obs_file = pcw.find_obs(time_info) assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180202/20180202_0013' diff --git a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py index 9328c3112a..2045453a5c 100644 --- a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py @@ -59,6 +59,49 @@ def set_minimum_config_settings(config, set_fields=True): config.set('config', 'OBS_VAR1_LEVELS', obs_level) +@pytest.mark.parametrize( + 'config_overrides, expected_filename', [ + # 0 - set forecast level + ({'FCST_VAR1_NAME': 'fcst_file', + 'FCST_VAR1_LEVELS': 'A06', + 'OBS_VAR1_NAME': 'obs_file', + 'OBS_VAR1_LEVELS': 'A06', + 'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE': '{fcst_name}_A{level?fmt=%3H}', + }, + f'{fcst_dir}/fcst_file_A006'), + # 1 - don't set forecast level + ({'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE': 'fcst_file_A{level?fmt=%3H}'}, + f'{fcst_dir}/fcst_file_A000'), + ] +) +@pytest.mark.wrapper_c +def test_ensemble_stat_level_in_template(metplus_config, config_overrides, + expected_filename): + + config = metplus_config + + set_minimum_config_settings(config, set_fields=False) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = EnsembleStatWrapper(config) + assert wrapper.isOK + + file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR') + file_list_file = f"{file_list_dir}/20050807000000_12_ensemble_stat.txt" + if os.path.exists(file_list_file): + os.remove(file_list_file) + + wrapper.run_all_times() + assert os.path.exists(file_list_file) + with open(file_list_file, 'r') as file_handle: + filenames = file_handle.read().splitlines()[1:] + assert len(filenames) == 1 + assert filenames[0] == expected_filename + + @pytest.mark.parametrize( 'config_overrides, env_var_values', [ # 0 : no ens, 1 fcst, 1 obs @@ -577,8 +620,7 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides, app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" - file_list_dir = os.path.join(wrapper.config.getdir('STAGING_DIR'), - 'file_lists') + file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR') config_file = wrapper.c_dict.get('CONFIG_FILE') out_dir = wrapper.c_dict.get('OUTPUT_DIR') expected_cmds = [(f"{app_path} {verbosity} " @@ -655,8 +697,7 @@ def test_ensemble_stat_fill_missing(metplus_config, config_overrides, wrapper = EnsembleStatWrapper(config) - file_list_file = os.path.join(wrapper.config.getdir('STAGING_DIR'), - 'file_lists', + file_list_file = os.path.join(wrapper.config.getdir('FILE_LISTS_DIR'), '20050807000000_12_ensemble_stat.txt') if os.path.exists(file_list_file): os.remove(file_list_file) diff --git a/metplus/util/string_manip.py b/metplus/util/string_manip.py index 5ddb62e867..c297dde55e 100644 --- a/metplus/util/string_manip.py +++ b/metplus/util/string_manip.py @@ -239,6 +239,9 @@ def format_thresh(thresh_str): @returns string of comma-separated list of the threshold(s) with letter format, i.e. gt3,le5.5,eq7 """ + if isinstance(thresh_str, list): + return format_thresh(','.join(thresh_str)) + formatted_thresh_list = [] # separate thresholds by comma and strip off whitespace around values thresh_list = [thresh.strip() for thresh in thresh_str.split(',')] diff --git a/metplus/util/time_util.py b/metplus/util/time_util.py index 6e6c5cfc03..cd53af2511 100755 --- a/metplus/util/time_util.py +++ b/metplus/util/time_util.py @@ -13,6 +13,8 @@ from dateutil.relativedelta import relativedelta import re +from .string_manip import split_level, format_thresh + '''!@namespace TimeInfo @brief Utility to handle timing in METplus wrappers @code{.sh} @@ -332,20 +334,10 @@ def _format_time_list(string_value, get_met_format, sort_list=True): def ti_calculate(input_dict_preserve): - out_dict = {} + # copy input dictionary so valid or init can be removed to recalculate it + # without modifying the input to the function input_dict = input_dict_preserve.copy() - - KEYS_TO_COPY = ['custom', 'instance'] - - # set output dictionary to input items - if 'now' in input_dict.keys(): - out_dict['now'] = input_dict['now'] - out_dict['today'] = out_dict['now'].strftime('%Y%m%d') - - # copy over values of some keys if it is set in input dictionary - for key in KEYS_TO_COPY: - if key in input_dict.keys(): - out_dict[key] = input_dict[key] + out_dict = input_dict # read in input dictionary items and compute missing items # valid inputs: valid, init, lead, offset @@ -381,7 +373,6 @@ def ti_calculate(input_dict_preserve): else: out_dict['lead'] = relativedelta(seconds=0) - # set offset to 0 if not specified if 'offset_hours' in input_dict.keys(): out_dict['offset'] = datetime.timedelta(hours=input_dict['offset_hours']) @@ -390,7 +381,6 @@ def ti_calculate(input_dict_preserve): else: out_dict['offset'] = datetime.timedelta(seconds=0) - # if init and valid are set, check which was set first via loop_by # remove the other to recalculate if 'init' in input_dict.keys() and 'valid' in input_dict.keys(): @@ -509,3 +499,25 @@ def add_to_time_input(time_input, clock_time=None, instance=None, custom=None): # otherwise leave it unset so it can be set within the wrapper if custom: time_input['custom'] = custom + + +def add_field_info_to_time_info(time_info, var_info): + """!Add field information from var_info to the time_info dictionary to use + in string template substitution. Sets new items in time_info. + + @param time_info dictionary containing time information to substitute + filename template tags + @param var_info dictionary containing information for the fields to process + """ + if var_info is None: + return + + for key, value in var_info.items(): + # skip index and extra field info + if key == 'index' or key.endswith('extra'): + continue + + if key.endswith('thresh'): + value = format_thresh(value) + + time_info[key] = value diff --git a/metplus/wrappers/ascii2nc_wrapper.py b/metplus/wrappers/ascii2nc_wrapper.py index 02a06fd65e..482ce6b9bf 100755 --- a/metplus/wrappers/ascii2nc_wrapper.py +++ b/metplus/wrappers/ascii2nc_wrapper.py @@ -297,7 +297,7 @@ def find_input_files(self, time_info): return self.infiles # get list of files even if only one is found (return_list=True) - obs_path = self.find_obs(time_info, var_info=None, return_list=True) + obs_path = self.find_obs(time_info, return_list=True) if obs_path is None: return None diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 43e424cea7..6af2d49a66 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -425,47 +425,39 @@ def print_env_item(self, item): """ return f"{item}={self.env[item]}" - def find_model(self, time_info, var_info=None, mandatory=True, - return_list=False): + def find_model(self, time_info, mandatory=True, return_list=False): """! Finds the model file to compare - Args: + @param time_info dictionary containing timing information - @param var_info object containing variable information @param mandatory if True, report error if not found, warning if not, default is True @rtype string @return Returns the path to an model file """ return self.find_data(time_info, - var_info=var_info, data_type="FCST", mandatory=mandatory, return_list=return_list) - def find_obs(self, time_info, var_info=None, mandatory=True, - return_list=False): + def find_obs(self, time_info, mandatory=True, return_list=False): """! Finds the observation file to compare - Args: + @param time_info dictionary containing timing information - @param var_info object containing variable information @param mandatory if True, report error if not found, warning if not, default is True @rtype string @return Returns the path to an observation file """ return self.find_data(time_info, - var_info=var_info, data_type="OBS", mandatory=mandatory, return_list=return_list) - def find_obs_offset(self, time_info, var_info=None, mandatory=True, - return_list=False): + def find_obs_offset(self, time_info, mandatory=True, return_list=False): """! Finds the observation file to compare, looping through offset list until a file is found @param time_info dictionary containing timing information - @param var_info object containing variable information @param mandatory if True, report error if not found, warning if not, default is True @rtype string @@ -483,7 +475,6 @@ def find_obs_offset(self, time_info, var_info=None, mandatory=True, time_info['offset_hours'] = offset time_info = ti_calculate(time_info) obs_path = self.find_obs(time_info, - var_info=var_info, mandatory=is_mandatory, return_list=return_list) @@ -505,12 +496,10 @@ def find_obs_offset(self, time_info, var_info=None, mandatory=True, return None, time_info - def find_data(self, time_info, var_info=None, data_type='', mandatory=True, + def find_data(self, time_info, data_type='', mandatory=True, return_list=False, allow_dir=False): """! Finds the data file to compare - Args: @param time_info dictionary containing timing information - @param var_info object containing variable information @param data_type type of data to find (i.e. FCST_ or OBS_) @param mandatory if True, report error if not found, warning if not. default is True @@ -523,21 +512,14 @@ def find_data(self, time_info, var_info=None, data_type='', mandatory=True, if data_type and not data_type.endswith('_'): data_type_fmt += '_' - if var_info is not None: - # set level based on input data type - if data_type_fmt.startswith("OBS"): - v_level = var_info['obs_level'] - else: - v_level = var_info['fcst_level'] + # set generic 'level' to level that corresponds to data_type if set + level = time_info.get(f'{data_type_fmt.lower()}level', '0') - # separate character from beginning of numeric - # level value if applicable - level = split_level(v_level)[1] + # strip off prefix letter if it exists + level = split_level(level)[1] - # set level to 0 character if it is not a number - if not level.isdigit(): - level = '0' - else: + # set level to 0 character if it is not a number, e.g. NetCDF level + if not level.isdigit(): level = '0' # if level is a range, use the first value, i.e. if 250-500 use 250 @@ -627,7 +609,7 @@ def find_exact_file(self, level, data_type, time_info, mandatory=True, check_file_list.append(full_path) # if it was set, add level back to time_info - if saved_level: + if saved_level is not None: time_info['level'] = saved_level # if multiple files are not supported by the wrapper and multiple @@ -1061,16 +1043,6 @@ def check_gempaktocf(self, gempaktocf_jar): "on how to obtain the tool: parm/use_cases/met_tool_wrapper/GempakToCF/GempakToCF.py") self.isOK = False - def add_field_info_to_time_info(self, time_info, field_info): - """!Add name and level values from field info to time info dict to be used in string substitution - Args: - @param time_info time dictionary to add items to - @param field_info field dictionary to get values from - """ - field_items = ['fcst_name', 'fcst_level', 'obs_name', 'obs_level'] - for field_item in field_items: - time_info[field_item] = field_info[field_item] if field_item in field_info else '' - def set_current_field_config(self, field_info=None): """! Sets config variables for current fcst/obs name/level that can be referenced by other config variables such as OUTPUT_PREFIX. diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index 32fef40bac..69f6bc5cb0 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -15,7 +15,7 @@ from ..util import do_string_sub, ti_calculate from ..util import parse_var_list from ..util import get_lead_sequence, skip_time, sub_var_list -from ..util import field_read_prob_info +from ..util import field_read_prob_info, add_field_info_to_time_info from . import CommandBuilder '''!@namespace CompareGriddedWrapper @@ -160,12 +160,14 @@ def run_at_time_once(self, time_info): for var_info in var_list: self.clear() self.c_dict['CURRENT_VAR_INFO'] = var_info + add_field_info_to_time_info(time_info, var_info) self.run_at_time_one_field(time_info, var_info) else: # loop over all variables and all them to the field list, # then call the app once if var_list: self.c_dict['CURRENT_VAR_INFO'] = var_list[0] + add_field_info_to_time_info(time_info, var_list[0]) self.clear() self.run_at_time_all_fields(time_info) @@ -180,7 +182,6 @@ def run_at_time_one_field(self, time_info, var_info): # get model to compare, return None if not found model_path = self.find_model(time_info, - var_info, mandatory=True, return_list=True) if model_path is None: @@ -189,7 +190,6 @@ def run_at_time_one_field(self, time_info, var_info): self.infiles.extend(model_path) # get observation to compare, return None if not found obs_path, time_info = self.find_obs_offset(time_info, - var_info, mandatory=True, return_list=True) if obs_path is None: @@ -225,7 +225,6 @@ def run_at_time_all_fields(self, time_info): # get model from first var to compare model_path = self.find_model(time_info, - var_list[0], mandatory=True, return_list=True) if not model_path: @@ -244,7 +243,6 @@ def run_at_time_all_fields(self, time_info): # get observation to from first var compare obs_path, time_info = self.find_obs_offset(time_info, - var_list[0], mandatory=True, return_list=True) if obs_path is None: diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index aa392e9b58..2c74d6294e 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -422,20 +422,9 @@ def run_at_time_all_fields(self, time_info): fill_missing=fill_missing): return - # parse optional var list for FCST and/or OBS fields - var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) - - # if empty var list for FCST/OBS, use None as first var, - # else use first var in list - if not var_list: - first_var_info = None - else: - first_var_info = var_list[0] - # get point observation file if requested if self.c_dict['OBS_POINT_INPUT_TEMPLATE']: - point_obs_path = self.find_data(time_info, first_var_info, - 'OBS_POINT') + point_obs_path = self.find_data(time_info, data_type='OBS_POINT') if point_obs_path is None: return @@ -443,13 +432,15 @@ def run_at_time_all_fields(self, time_info): # get grid observation file if requested if self.c_dict['OBS_GRID_INPUT_TEMPLATE']: - grid_obs_path = self.find_data(time_info, first_var_info, - 'OBS_GRID') + grid_obs_path = self.find_data(time_info, data_type='OBS_GRID') if grid_obs_path is None: return self.grid_obs_files.append(grid_obs_path) + # parse optional var list for FCST and/or OBS fields + var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) + # set field info fcst_field = self.get_all_field_info(var_list, 'FCST') obs_field = self.get_all_field_info(var_list, 'OBS') diff --git a/metplus/wrappers/ioda2nc_wrapper.py b/metplus/wrappers/ioda2nc_wrapper.py index 4e267e729a..dfc75b4c7a 100755 --- a/metplus/wrappers/ioda2nc_wrapper.py +++ b/metplus/wrappers/ioda2nc_wrapper.py @@ -140,7 +140,7 @@ def find_input_files(self, time_info): @returns List of files that were found or None if no files were found """ # get list of files even if only one is found (return_list=True) - obs_path = self.find_obs(time_info, var_info=None, return_list=True) + obs_path = self.find_obs(time_info, return_list=True) if obs_path is None: return None diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 1a539ea021..7d9a044b5e 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -466,12 +466,12 @@ def run_at_time_one_field(self, time_info, var_info): @param var_info object containing variable information """ # get model to compare - model_path = self.find_model(time_info, var_info) + model_path = self.find_model(time_info) if model_path is None: return # get observation to compare - obs_path = self.find_obs(time_info, var_info) + obs_path = self.find_obs(time_info) if obs_path is None: return diff --git a/metplus/wrappers/mtd_wrapper.py b/metplus/wrappers/mtd_wrapper.py index 217427badc..b74663ddab 100755 --- a/metplus/wrappers/mtd_wrapper.py +++ b/metplus/wrappers/mtd_wrapper.py @@ -15,7 +15,7 @@ from ..util import get_lead_sequence, sub_var_list from ..util import ti_calculate from ..util import do_string_sub, skip_time -from ..util import parse_var_list +from ..util import parse_var_list, add_field_info_to_time_info from . import CompareGriddedWrapper class MTDWrapper(CompareGriddedWrapper): @@ -219,6 +219,7 @@ def run_at_time_loop_string(self, input_dict): if self.c_dict.get('EXPLICIT_FILE_LIST', False): time_info = ti_calculate(input_dict) + add_field_info_to_time_info(time_info, var_info) model_list_path = do_string_sub(self.c_dict['FCST_FILE_LIST'], **time_info) self.logger.debug(f"Explicit FCST file: {model_list_path}") @@ -252,14 +253,13 @@ def run_at_time_loop_string(self, input_dict): input_dict['lead'] = lead time_info = ti_calculate(input_dict) + add_field_info_to_time_info(time_info, var_info) tasks.append(time_info) for current_task in tasks: # call find_model/obs as needed - model_file = self.find_model(current_task, var_info, - mandatory=False) - obs_file = self.find_obs(current_task, var_info, - mandatory=False) + model_file = self.find_model(current_task, mandatory=False) + obs_file = self.find_obs(current_task, mandatory=False) if model_file is None and obs_file is None: continue @@ -313,6 +313,7 @@ def run_single_mode(self, input_dict, var_info): if self.c_dict.get('EXPLICIT_FILE_LIST', False): time_info = ti_calculate(input_dict) + add_field_info_to_time_info(time_info, var_info) single_list_path = do_string_sub( self.c_dict[f'{data_src}_FILE_LIST'], **time_info @@ -334,7 +335,7 @@ def run_single_mode(self, input_dict, var_info): input_dict['lead'] = lead current_task = ti_calculate(input_dict) - single_file = find_method(current_task, var_info) + single_file = find_method(current_task) if single_file is None: continue @@ -408,7 +409,6 @@ def process_fields_one_thresh(self, time_info, var_info, model_path, fcst_field_list.extend(fcst_field) - if obs_path: obs_thresh_list = var_info['obs_thresh'] @@ -497,7 +497,6 @@ def set_environment_variables(self, time_info): self.add_env_var("MIN_VOLUME", self.c_dict["MIN_VOLUME"]) - self.add_env_var("FCST_FILE_TYPE", self.c_dict.get('FCST_FILE_TYPE', '')) self.add_env_var("OBS_FILE_TYPE", diff --git a/metplus/wrappers/pb2nc_wrapper.py b/metplus/wrappers/pb2nc_wrapper.py index fff7783f79..6d3848ca46 100755 --- a/metplus/wrappers/pb2nc_wrapper.py +++ b/metplus/wrappers/pb2nc_wrapper.py @@ -226,7 +226,6 @@ def find_input_files(self, input_dict): """ infiles, time_info = self.find_obs_offset(input_dict, - None, mandatory=True, return_list=True) diff --git a/metplus/wrappers/pcp_combine_wrapper.py b/metplus/wrappers/pcp_combine_wrapper.py index f87b07fadf..5ede0fdbcd 100755 --- a/metplus/wrappers/pcp_combine_wrapper.py +++ b/metplus/wrappers/pcp_combine_wrapper.py @@ -12,6 +12,7 @@ from ..util import get_relativedelta, ti_get_seconds_from_relativedelta from ..util import time_string_to_met_time, seconds_to_met_time from ..util import parse_var_list, template_to_regex, split_level +from ..util import add_field_info_to_time_info from . import ReformatGriddedWrapper '''!@namespace PCPCombineWrapper @@ -252,6 +253,7 @@ def run_at_time_one_field(self, time_info, var_info, data_src): return False time_info['level'] = lookback_seconds + add_field_info_to_time_info(time_info, var_info) # if method is not USER_DEFINED or DERIVE, # check that field information is set @@ -259,7 +261,7 @@ def run_at_time_one_field(self, time_info, var_info, data_src): can_run = self.setup_user_method(time_info, data_src) elif method == "DERIVE": can_run = self.setup_derive_method(time_info, lookback_seconds, - var_info, data_src) + data_src) elif method == "ADD": can_run = self.setup_add_method(time_info, lookback_seconds, data_src) @@ -494,12 +496,11 @@ def setup_add_method(self, time_info, lookback, data_src): return files_found - def setup_derive_method(self, time_info, lookback, var_info, data_src): + def setup_derive_method(self, time_info, lookback, data_src): """! Setup pcp_combine to derive stats @param time_info dictionary containing timing information @param lookback accumulation amount to compute in seconds - @param var_info object containing variable information @param data_src data type (FCST or OBS) @rtype string @return path to output file @@ -524,7 +525,6 @@ def setup_derive_method(self, time_info, lookback, var_info, data_src): level=accum_dict['level'], extra=accum_dict['extra']) input_files = self.find_data(time_info, - var_info, data_type=data_src, return_list=True) if not input_files: diff --git a/metplus/wrappers/plot_data_plane_wrapper.py b/metplus/wrappers/plot_data_plane_wrapper.py index 4f874c42f1..26f844772b 100755 --- a/metplus/wrappers/plot_data_plane_wrapper.py +++ b/metplus/wrappers/plot_data_plane_wrapper.py @@ -170,9 +170,7 @@ def find_input_files(self, time_info): self.infiles.append(self.c_dict['INPUT_TEMPLATE']) return self.infiles - file_path = self.find_data(time_info, - var_info=None, - return_list=False) + file_path = self.find_data(time_info, return_list=False) if not file_path: return None diff --git a/metplus/wrappers/point2grid_wrapper.py b/metplus/wrappers/point2grid_wrapper.py index a2607d6858..0af3fe3600 100755 --- a/metplus/wrappers/point2grid_wrapper.py +++ b/metplus/wrappers/point2grid_wrapper.py @@ -201,8 +201,7 @@ def find_input_files(self, time_info): """ # get input file # calling find_obs because we set OBS_ variables in c_dict for the input data - input_path = self.find_obs(time_info, - var_info=None) + input_path = self.find_obs(time_info) if input_path is None: return None diff --git a/metplus/wrappers/regrid_data_plane_wrapper.py b/metplus/wrappers/regrid_data_plane_wrapper.py index c58ebfc7c2..0211af4275 100755 --- a/metplus/wrappers/regrid_data_plane_wrapper.py +++ b/metplus/wrappers/regrid_data_plane_wrapper.py @@ -12,10 +12,9 @@ import os -from ..util import time_util -from ..util import do_string_sub -from ..util import parse_var_list -from ..util import get_process_list +from ..util import get_seconds_from_string, do_string_sub +from ..util import parse_var_list, get_process_list +from ..util import add_field_info_to_time_info from ..util import remove_quotes, split_level, format_level from . import ReformatGriddedWrapper @@ -173,7 +172,7 @@ def handle_output_file(self, time_info, field_info, data_type): not be run """ _, level = split_level(field_info[f'{data_type.lower()}_level']) - time_info['level'] = time_util.get_seconds_from_string(level, 'H') + time_info['level'] = get_seconds_from_string(level, 'H') return self.find_and_check_output_file(time_info) def run_once_per_field(self, time_info, var_list, data_type): @@ -189,8 +188,7 @@ def run_once_per_field(self, time_info, var_list, data_type): self.set_command_line_arguments() - self.add_field_info_to_time_info(time_info, - field_info) + add_field_info_to_time_info(time_info, field_info) input_name = field_info[f'{data_type.lower()}_name'] input_level = field_info[f'{data_type.lower()}_level'] @@ -271,8 +269,7 @@ def run_once_for_all_fields(self, time_info, var_list, data_type): self.set_command_line_arguments() for field_info in var_list: - self.add_field_info_to_time_info(time_info, - field_info) + add_field_info_to_time_info(time_info, field_info) input_name = field_info[f'{data_type.lower()}_name'] input_level = field_info[f'{data_type.lower()}_level'] @@ -289,9 +286,7 @@ def run_once_for_all_fields(self, time_info, var_list, data_type): # add list of output names self.args.append("-name " + ','.join(output_names)) - if not self.handle_output_file(time_info, - var_list[0], - data_type): + if not self.handle_output_file(time_info, var_list[0], data_type): return False # build and run commands @@ -320,7 +315,8 @@ def run_at_time_once(self, time_info, var_list, data_type): f'NAME or {data_type}_VAR_NAME.') return False - if not self.find_input_files(time_info, data_type, var_list): + add_field_info_to_time_info(time_info, var_list[0]) + if not self.find_input_files(time_info, data_type): return False # set environment variables @@ -334,12 +330,16 @@ def run_at_time_once(self, time_info, var_list, data_type): # if not running once per field, process all fields and run once return self.run_once_for_all_fields(time_info, var_list, data_type) - def find_input_files(self, time_info, data_type, var_list): - """!Get input file and verification grid to process. Use the first field in the - list to substitute level if that is provided in the filename template""" - input_path = self.find_data(time_info, - var_info=var_list[0], - data_type=data_type) + def find_input_files(self, time_info, data_type): + """!Get input file and verification grid to process. Use the first + field in the list to substitute level if that is provided in the + filename template + + @param time_info time dictionary used for string substitution + @param data_type type of data to process, i.e. FCST or OBS + @returns list of input files if files were found, None if not + """ + input_path = self.find_data(time_info, data_type=data_type) if not input_path: return None diff --git a/metplus/wrappers/series_analysis_wrapper.py b/metplus/wrappers/series_analysis_wrapper.py index 8d24ea0c30..ca3897cdf8 100755 --- a/metplus/wrappers/series_analysis_wrapper.py +++ b/metplus/wrappers/series_analysis_wrapper.py @@ -30,7 +30,7 @@ from ..util import ti_get_seconds_from_relativedelta from ..util import parse_var_list from ..util import add_to_time_input -from ..util import field_read_prob_info +from ..util import field_read_prob_info, add_field_info_to_time_info from .plot_data_plane_wrapper import PlotDataPlaneWrapper from . import RuntimeFreqWrapper @@ -803,7 +803,7 @@ def build_and_run_series_request(self, time_info, fcst_path, obs_path): self.c_dict['FCST_LIST_PATH'] = fcst_path self.c_dict['OBS_LIST_PATH'] = obs_path - self.add_field_info_to_time_info(time_info, var_info) + add_field_info_to_time_info(time_info, var_info) # get formatted field dictionary to pass into the MET config file fcst_field, obs_field = self.get_formatted_fields(var_info, @@ -916,7 +916,7 @@ def _generate_plots(self, fcst_path, time_info, storm_id): for var_info in self.c_dict['VAR_LIST']: name = var_info['fcst_name'] level = var_info['fcst_level'] - self.add_field_info_to_time_info(time_info, var_info) + add_field_info_to_time_info(time_info, var_info) # change wildcard storm ID to all_storms if storm_id == '*':