From 8bc58959c9b6d4f426a87d15c90c2a5ba48966e1 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:18:58 -0700 Subject: [PATCH] clean up formatting --- metplus/wrappers/command_builder.py | 2 +- metplus/wrappers/compare_gridded_wrapper.py | 107 +++++++----------- metplus/wrappers/ensemble_stat_wrapper.py | 6 +- metplus/wrappers/extract_tiles_wrapper.py | 12 +- metplus/wrappers/gempak_to_cf_wrapper.py | 17 +-- metplus/wrappers/py_embed_ingest_wrapper.py | 6 +- metplus/wrappers/regrid_data_plane_wrapper.py | 9 +- metplus/wrappers/runtime_freq_wrapper.py | 2 - metplus/wrappers/stat_analysis_wrapper.py | 6 +- 9 files changed, 61 insertions(+), 106 deletions(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index d971217ccc..995d62259b 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1425,7 +1425,7 @@ def get_wrapper_or_generic_config(self, generic_name, var_type='str'): Default is str. @returns value if set or empty string if not """ - name = self.get_mp_config_name( + name = self.config.get_mp_config_name( [f'{self.app_name}_{generic_name}'.upper(), generic_name.upper()] ) if not name: diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index 648166aec1..0132de56cf 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -94,11 +94,10 @@ def run_at_time_once(self, time_info): @param time_info dictionary containing timing information """ var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) - if not var_list and not self.c_dict.get('VAR_LIST_OPTIONAL', False): - self.log_error('No input fields were specified. You must set ' - f'[FCST/OBS]_VAR_[NAME/LEVELS].') - return None + self.log_error('No input fields were specified.' + ' [FCST/OBS]_VAR_NAME must be set.') + return if self.c_dict.get('ONCE_PER_FIELD', False): # loop over all fields and levels (and probability thresholds) and @@ -169,85 +168,60 @@ def run_at_time_all_fields(self, time_info): """ var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) - # get model from first var to compare - model_path = self.find_model(time_info, - mandatory=True, - return_list=True) - if not model_path: - return + # set field info + fcst_field = self.get_all_field_info(var_list, 'FCST') + obs_field = self.get_all_field_info(var_list, 'OBS') - # if there is more than 1 file, create file list file - if len(model_path) > 1: - list_filename = (f"{time_info['init_fmt']}_" - f"{time_info['lead_hours']}_" - f"{self.app_name}_fcst.txt") - model_path = self.write_list_file(list_filename, model_path) - else: - model_path = model_path[0] - - self.infiles.append(model_path) - - # get observation to from first var compare - obs_path, time_info = self.find_obs_offset(time_info, - mandatory=True, - return_list=True) - if obs_path is None: + if not fcst_field or not obs_field: + self.log_error("Could not build field info for fcst or obs") return - # if there is more than 1 file, create file list file - if len(obs_path) > 1: - list_filename = (f"{time_info['init_fmt']}_" - f"{time_info['lead_hours']}_" - f"{self.app_name}_obs.txt") - obs_path = self.write_list_file(list_filename, obs_path) - else: - obs_path = obs_path[0] - - self.infiles.append(obs_path) - - fcst_field_list = [] - obs_field_list = [] - for var_info in var_list: - next_fcst = self.get_field_info(v_level=var_info['fcst_level'], - v_thresh=var_info['fcst_thresh'], - v_name=var_info['fcst_name'], - v_extra=var_info['fcst_extra'], - d_type='FCST') - - next_obs = self.get_field_info(v_level=var_info['obs_level'], - v_thresh=var_info['obs_thresh'], - v_name=var_info['obs_name'], - v_extra=var_info['obs_extra'], - d_type='OBS') - - if next_fcst is None or next_obs is None: - return - - fcst_field_list.extend(next_fcst) - obs_field_list.extend(next_obs) - - fcst_field = ','.join(fcst_field_list) - obs_field = ','.join(obs_field_list) - self.format_field('FCST', fcst_field) self.format_field('OBS', obs_field) self.process_fields(time_info) + def get_all_field_info(self, var_list, data_type): + """!Get field info based on data type""" + + field_list = [] + for var_info in var_list: + type_lower = data_type.lower() + level = var_info[f'{type_lower}_level'] + thresh = var_info[f'{type_lower}_thresh'] + name = var_info[f'{type_lower}_name'] + extra = var_info[f'{type_lower}_extra'] + + # check if python embedding is used and set up correctly + # set env var for file type if it is used + py_embed_ok = self.check_for_python_embedding(data_type, var_info) + if not py_embed_ok: + return '' + + next_field = self.get_field_info(v_level=level, + v_thresh=thresh, + v_name=name, + v_extra=extra, + d_type=data_type) + if next_field is None: + return '' + + field_list.extend(next_field) + + return ','.join(field_list) + def process_fields(self, time_info): """! Set and print environment variables, then build/run MET command @param time_info dictionary with time information """ # set config file since command is reset after each run - self.param = do_string_sub(self.c_dict['CONFIG_FILE'], - **time_info) + self.param = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) self.set_current_field_config() # set up output dir with time info - if not self.find_and_check_output_file(time_info, - is_directory=True): + if not self.find_and_check_output_file(time_info, is_directory=True): return # set command line arguments @@ -274,8 +248,7 @@ def get_command(self): @return Returns a MET command with arguments that you can run """ if self.app_path is None: - self.log_error('No app path specified. ' - 'You must use a subclass') + self.log_error('No app path specified. You must use a subclass') return None cmd = '{} -v {} '.format(self.app_path, self.c_dict['VERBOSITY']) diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index 1de97bf155..93ddb566b0 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -506,12 +506,10 @@ def process_fields(self, time_info): @param obs_field field information formatted for MET config file """ # set config file since command is reset after each run - self.param = do_string_sub(self.c_dict['CONFIG_FILE'], - **time_info) + self.param = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) # set up output dir with time info - if not self.find_and_check_output_file(time_info, - is_directory=True): + if not self.find_and_check_output_file(time_info, is_directory=True): return # set environment variables that are passed to the MET config diff --git a/metplus/wrappers/extract_tiles_wrapper.py b/metplus/wrappers/extract_tiles_wrapper.py index 246e5af688..141cb225f7 100755 --- a/metplus/wrappers/extract_tiles_wrapper.py +++ b/metplus/wrappers/extract_tiles_wrapper.py @@ -324,8 +324,7 @@ def get_location_input_file(self, time_info, input_type): """ input_path = os.path.join(self.c_dict[f'{input_type}_INPUT_DIR'], self.c_dict[f'{input_type}_INPUT_TEMPLATE']) - input_path = do_string_sub(input_path, - **time_info) + input_path = do_string_sub(input_path, **time_info) self.logger.debug(f"Looking for {input_type} file: {input_path}") if not os.path.exists(input_path): @@ -356,8 +355,7 @@ def call_regrid_data_plane(self, time_info, track_data, input_type): self.regrid_data_plane.c_dict['VAR_LIST'] = var_list for data_type in ['FCST', 'OBS']: - grid = self.get_grid(data_type, track_data[data_type], - input_type) + grid = self.get_grid(data_type, track_data[data_type], input_type) self.regrid_data_plane.c_dict['VERIFICATION_GRID'] = grid @@ -413,14 +411,12 @@ def set_time_info_from_track_data(storm_data, storm_id=None): input_dict = {} # read forecast lead from LEAD (TC_STAT) or FCST_LEAD (MTD) - lead = storm_data.get('LEAD', - storm_data.get('FCST_LEAD')) + lead = storm_data.get('LEAD', storm_data.get('FCST_LEAD')) if lead: input_dict['lead_hours'] = lead[:-4] # read valid time from VALID (TC_STAT) or FCST_VALID (MTD) - valid = storm_data.get('VALID', - storm_data.get('FCST_VALID')) + valid = storm_data.get('VALID', storm_data.get('FCST_VALID')) if valid: valid_dt = datetime.strptime(valid, '%Y%m%d_%H%M%S') input_dict['valid'] = valid_dt diff --git a/metplus/wrappers/gempak_to_cf_wrapper.py b/metplus/wrappers/gempak_to_cf_wrapper.py index f0ddd57a6b..95f874053e 100755 --- a/metplus/wrappers/gempak_to_cf_wrapper.py +++ b/metplus/wrappers/gempak_to_cf_wrapper.py @@ -43,13 +43,11 @@ def create_c_dict(self): c_dict['INPUT_DATATYPE'] = 'GEMPAK' c_dict['INPUT_DIR'] = self.config.getdir('GEMPAKTOCF_INPUT_DIR', '') c_dict['INPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'GEMPAKTOCF_INPUT_TEMPLATE') + self.config.getraw('config', 'GEMPAKTOCF_INPUT_TEMPLATE') ) c_dict['OUTPUT_DIR'] = self.config.getdir('GEMPAKTOCF_OUTPUT_DIR', '') c_dict['OUTPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'GEMPAKTOCF_OUTPUT_TEMPLATE') + self.config.getraw('config', 'GEMPAKTOCF_OUTPUT_TEMPLATE') ) return c_dict @@ -72,16 +70,13 @@ def get_command(self): def run_at_time_once(self, time_info): """! Runs the MET application for a given time and forecast lead combination - Args: - @param time_info dictionary containing timing information + + @param time_info dictionary containing timing information """ - infile = do_string_sub(self.c_dict['INPUT_TEMPLATE'], - **time_info) - infile = os.path.join(self.c_dict.get('INPUT_DIR', ''), - infile) + infile = do_string_sub(self.c_dict['INPUT_TEMPLATE'], **time_info) + infile = os.path.join(self.c_dict.get('INPUT_DIR', ''), infile) self.infiles.append(infile) - # set environment variables self.set_environment_variables(time_info) if not self.find_and_check_output_file(time_info): diff --git a/metplus/wrappers/py_embed_ingest_wrapper.py b/metplus/wrappers/py_embed_ingest_wrapper.py index 0f6d069175..9bacd6a8be 100755 --- a/metplus/wrappers/py_embed_ingest_wrapper.py +++ b/metplus/wrappers/py_embed_ingest_wrapper.py @@ -135,8 +135,7 @@ def run_at_time_once(self, time_info): index = ingester['index'] # get grid information to project output data - output_grid = do_string_sub(ingester['output_grid'], - **time_info) + output_grid = do_string_sub(ingester['output_grid'], **time_info) rdp.clear() # get output file path @@ -148,8 +147,7 @@ def run_at_time_once(self, time_info): rdp.infiles.append(f"PYTHON_{ingester['input_type']}") for script_raw in ingester['scripts']: - script = do_string_sub(script_raw, - **time_info) + script = do_string_sub(script_raw, **time_info) rdp.infiles.append(f'-field \'name="{script}\";\'') diff --git a/metplus/wrappers/regrid_data_plane_wrapper.py b/metplus/wrappers/regrid_data_plane_wrapper.py index 7761d600e1..fdbb60bd91 100755 --- a/metplus/wrappers/regrid_data_plane_wrapper.py +++ b/metplus/wrappers/regrid_data_plane_wrapper.py @@ -344,17 +344,16 @@ def find_input_files(self, time_info, data_type): """ input_path = self.find_data(time_info, data_type=data_type) if not input_path: - return None + return False self.infiles.append(input_path) - verif_grid = do_string_sub(self.c_dict['VERIFICATION_GRID'], - **time_info) + grid = do_string_sub(self.c_dict['VERIFICATION_GRID'], **time_info) # put quotes around verification grid in case it is a grid description - self.infiles.append(f'"{verif_grid}"') + self.infiles.append(f'"{grid}"') - return self.infiles + return True def set_command_line_arguments(self): """!Returns False if command should not be run""" diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index 207ad98f1b..d4673c3712 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -301,8 +301,6 @@ def run_at_time(self, input_dict): self.logger.debug('Skipping run time') continue - # since run_all_times was not called (LOOP_BY=times) then - # get files for current run time all_files = [] self._update_list_with_new_files(time_info, all_files) self.c_dict['ALL_FILES'] = all_files diff --git a/metplus/wrappers/stat_analysis_wrapper.py b/metplus/wrappers/stat_analysis_wrapper.py index 7011ef77ef..8be92c7965 100755 --- a/metplus/wrappers/stat_analysis_wrapper.py +++ b/metplus/wrappers/stat_analysis_wrapper.py @@ -962,8 +962,7 @@ def _get_lookin_dir(self, dir_path, config_dict): @returns string of the filled directory from dir_path """ stringsub_dict = self._build_stringsub_dict(config_dict) - dir_path_filled = do_string_sub(dir_path, - **stringsub_dict) + dir_path_filled = do_string_sub(dir_path, **stringsub_dict) all_paths = [] for one_path in dir_path_filled.split(','): @@ -1124,8 +1123,7 @@ def _process_job_args(self, job_type, job, model_info, output_template, stringsub_dict) ) - output_file = os.path.join(self.c_dict['OUTPUT_DIR'], - output_filename) + output_file = os.path.join(self.c_dict['OUTPUT_DIR'], output_filename) # substitute output filename in JOBS line job = job.replace(f'[{job_type}_file]', output_file)