From 798d119445ad1354361933b4eb471a7f84730dab Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 17 Nov 2022 09:31:03 -0700 Subject: [PATCH] Feature #1569 EnsembleStat -ens_mean argument (#1952) --- docs/Users_Guide/glossary.rst | 10 +++ docs/Users_Guide/wrappers.rst | 2 + .../test_ensemble_stat_wrapper.py | 50 ++++++++++--- metplus/wrappers/ensemble_stat_wrapper.py | 72 ++++++++----------- .../EnsembleStat/EnsembleStat.conf | 3 +- 5 files changed, 83 insertions(+), 54 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 7a270558c4..9b9d02271b 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -10094,6 +10094,16 @@ METplus Configuration Glossary | *Used by:* StatAnalysis + ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR + Input directory for the optional -ens_mean file to use with the MET tool ensemble_stat. + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE + Template used to specify the optional -ens_mean file for the MET tool ensemble_stat. + + | *Used by:* EnsembleStat + TC_PAIRS_DIAG_INFO_MAP_DIAG_SOURCE Specify the value for the nth 'diag_info_map.diag_source' in the MET configuration file for TCPairs. diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 6f86e71335..41263e111b 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -193,6 +193,8 @@ METplus Configuration | :term:`ENSEMBLE_STAT_OUTPUT_TEMPLATE` | :term:`ENSEMBLE_STAT_CTRL_INPUT_DIR` | :term:`ENSEMBLE_STAT_CTRL_INPUT_TEMPLATE` +| :term:`ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE` +| :term:`ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR` | :term:`LOG_ENSEMBLE_STAT_VERBOSITY` | :term:`FCST_ENSEMBLE_STAT_INPUT_DATATYPE` | :term:`OBS_ENSEMBLE_STAT_INPUT_POINT_DATATYPE` diff --git a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py index 313beb3a86..aad6656e24 100644 --- a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py @@ -11,6 +11,9 @@ fcst_dir = '/some/path/fcst' obs_dir = '/some/path/obs' +ens_mean_dir = '/some/path/ens_mean' +ens_mean_template = 'the_ens_mean_file.nc' +obs_point_template = 'point_obs.nc' fcst_name = 'APCP' fcst_level = 'A03' obs_name = 'APCP_03' @@ -43,10 +46,10 @@ def set_minimum_config_settings(config, set_fields=True): config.set('config', 'ENSEMBLE_STAT_CONFIG_FILE', '{PARM_BASE}/met_config/EnsembleStatConfig_wrapped') config.set('config', 'FCST_ENSEMBLE_STAT_INPUT_DIR', fcst_dir) - config.set('config', 'OBS_ENSEMBLE_STAT_INPUT_DIR', obs_dir) + config.set('config', 'OBS_ENSEMBLE_STAT_GRID_INPUT_DIR', obs_dir) config.set('config', 'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE', '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') - config.set('config', 'OBS_ENSEMBLE_STAT_INPUT_TEMPLATE', + config.set('config', 'OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}/obs_file') config.set('config', 'ENSEMBLE_STAT_OUTPUT_DIR', '{OUTPUT_BASE}/EnsembleStat/output') @@ -199,11 +202,11 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, for old_env in old_env_vars: match = next((item for item in actual_env_vars if item.startswith(old_env)), None) - assert(match is not None) + assert match is not None actual_value = match.split('=', 1)[1] expected_value = env_var_values.get(old_env, '') expected_value = expected_value.replace('YMDH', ymdh) - assert(expected_value == actual_value) + assert expected_value == actual_value @pytest.mark.parametrize( @@ -601,6 +604,21 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, ({'ENSEMBLE_STAT_OBS_THRESH': 'NA, 0.5', }, {'METPLUS_OBS_THRESH': 'obs_thresh = [NA, 0.5];'}), + + ({'ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR': ens_mean_dir, + 'ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE': ens_mean_template}, + {}), + + ({'OBS_ENSEMBLE_STAT_POINT_INPUT_DIR': obs_dir, + 'OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE': obs_point_template}, + {}), + + ({'ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR': ens_mean_dir, + 'ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE': ens_mean_template, + 'OBS_ENSEMBLE_STAT_POINT_INPUT_DIR': obs_dir, + 'OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE': obs_point_template}, + {}), + ] ) @pytest.mark.wrapper_c @@ -623,12 +641,24 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides, file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR') config_file = wrapper.c_dict.get('CONFIG_FILE') out_dir = wrapper.c_dict.get('OUTPUT_DIR') + + point_obs = ' ' + ens_mean = ' ' + if 'OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE' in config_overrides: + point_obs = f' -point_obs "{obs_dir}/{obs_point_template}" ' + if 'ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE' in config_overrides: + ens_mean = f' -ens_mean {ens_mean_dir}/{ens_mean_template} ' + expected_cmds = [(f"{app_path} {verbosity} " f"{file_list_dir}/20050807000000_12_ensemble_stat.txt " - f"{config_file} -outdir {out_dir}/2005080712"), + f"{config_file}{point_obs}" + f'-grid_obs "{obs_dir}/2005080712/obs_file"{ens_mean}' + f"-outdir {out_dir}/2005080712"), (f"{app_path} {verbosity} " f"{file_list_dir}/20050807120000_12_ensemble_stat.txt " - f"{config_file} -outdir {out_dir}/2005080800"), + f"{config_file}{point_obs}" + f'-grid_obs "{obs_dir}/2005080800/obs_file"{ens_mean}' + f"-outdir {out_dir}/2005080800"), ] all_cmds = wrapper.run_all_times() @@ -641,7 +671,7 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides, for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected - assert(cmd == expected_cmd) + assert cmd == expected_cmd # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) @@ -651,11 +681,11 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides, assert(match is not None) actual_value = match.split('=', 1)[1] if env_var_key == 'METPLUS_FCST_FIELD': - assert(actual_value == fcst_fmt) + assert actual_value == fcst_fmt elif env_var_key == 'METPLUS_OBS_FIELD': - assert (actual_value == obs_fmt) + assert actual_value == obs_fmt else: - assert(env_var_values.get(env_var_key, '') == actual_value) + assert env_var_values.get(env_var_key, '') == actual_value @pytest.mark.wrapper_c diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index 4b17ce537b..adc764504a 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -196,6 +196,14 @@ def create_c_dict(self): self.log_error("Must set FCST_ENSEMBLE_STAT_INPUT_TEMPLATE or " "FCST_ENSEMBLE_STAT_INPUT_FILE_LIST") + # optional -ens_mean argument path + c_dict['ENS_MEAN_INPUT_DIR'] = ( + self.config.getdir('ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR', '')) + + c_dict['ENS_MEAN_INPUT_TEMPLATE'] = ( + self.config.getraw('config', + 'ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE')) + c_dict['OUTPUT_DIR'] = ( self.config.getdir('ENSEMBLE_STAT_OUTPUT_DIR', '') ) @@ -380,35 +388,9 @@ def get_command(self): @rtype string @return Returns a MET command with arguments that you can run """ - cmd = '{} -v {} '.format(self.app_path, self.c_dict['VERBOSITY']) - - for args in self.args: - cmd += args + " " - - if not self.infiles: - self.log_error(self.app_name+": No input filenames specified") - return None - - for infile in self.infiles: - cmd += infile + " " - - if self.param != "": - cmd += self.param + " " - - for obs_file in self.point_obs_files: - if obs_file.startswith('PYTHON'): - obs_file = f"'{obs_file}'" - cmd += "-point_obs " + obs_file + " " - - for obs_file in self.grid_obs_files: - cmd += "-grid_obs " + obs_file + " " - - if not self.outdir: - self.log_error(self.app_name+": No output directory specified") - return None - - cmd += '-outdir {}'.format(self.outdir) - return cmd + return (f"{self.app_path} -v {self.c_dict['VERBOSITY']}" + f" {' '.join(self.infiles)} {self.param}" + f" {' '.join(self.args)} -outdir {self.outdir}") def run_at_time_all_fields(self, time_info): """! Runs the MET application for a given time and forecast lead combination @@ -424,21 +406,32 @@ def run_at_time_all_fields(self, time_info): # get point observation file if requested if self.c_dict['OBS_POINT_INPUT_TEMPLATE']: - point_obs_path = self.find_data(time_info, data_type='OBS_POINT', - return_list=True) - if point_obs_path is None: + point_obs_files = self.find_data(time_info, data_type='OBS_POINT', + return_list=True) + if point_obs_files is None: return - self.point_obs_files.extend(point_obs_path) + for point_obs_path in point_obs_files: + self.args.append(f'-point_obs "{point_obs_path}"') # get grid observation file if requested if self.c_dict['OBS_GRID_INPUT_TEMPLATE']: - grid_obs_path = self.find_data(time_info, data_type='OBS_GRID', + grid_obs_files = self.find_data(time_info, data_type='OBS_GRID', + return_list=True) + if grid_obs_files is None: + return + + for grid_obs_path in grid_obs_files: + self.args.append(f'-grid_obs "{grid_obs_path}"') + + # get ens_mean file if requested + if self.c_dict['ENS_MEAN_INPUT_TEMPLATE']: + ens_mean_path = self.find_data(time_info, data_type='ENS_MEAN', return_list=True) - if grid_obs_path is None: + if ens_mean_path is None: return - self.grid_obs_files.extend(grid_obs_path) + self.args.append(f'-ens_mean {ens_mean_path[0]}') # parse optional var list for FCST and/or OBS fields var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) @@ -511,10 +504,3 @@ def process_fields(self, time_info): # run the MET command self.build() - - def clear(self): - """!Unset class variables to prepare for next run time - """ - super().clear() - self.point_obs_files = [] - self.grid_obs_files = [] diff --git a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf index 8d4a0dcbdd..03aaee0657 100644 --- a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf +++ b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf @@ -48,10 +48,11 @@ FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/arw-???-gep?/d01_{init?f OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = {INPUT_BASE}/met_test/out/ascii2nc OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = precip24_{valid?fmt=%Y%m%d%H}.nc - OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {INPUT_BASE}/met_test/data/sample_obs/ST4 OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = ST4.{valid?fmt=%Y%m%d%H}.24h +#ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR = +#ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE = ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR = ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE =