Feature #1569 EnsembleStat -ens_mean argument (#1952)

docs/Users_Guide/glossary.rst

@@ -10094,6 +10094,16 @@ METplus Configuration Glossary
 
      | *Used by:* StatAnalysis
 
+   ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR
+     Input directory for the optional -ens_mean file to use with the MET tool ensemble_stat.
+
+     | *Used by:*  EnsembleStat
+
+   ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE
+     Template used to specify the optional -ens_mean file for the MET tool ensemble_stat.
+
+     | *Used by:*  EnsembleStat
+
    TC_PAIRS_DIAG_INFO_MAP<n>_DIAG_SOURCE
      Specify the value for the nth 'diag_info_map.diag_source' in the MET configuration file for TCPairs.
 

docs/Users_Guide/wrappers.rst

@@ -193,6 +193,8 @@ METplus Configuration
 | :term:`ENSEMBLE_STAT_OUTPUT_TEMPLATE`
 | :term:`ENSEMBLE_STAT_CTRL_INPUT_DIR`
 | :term:`ENSEMBLE_STAT_CTRL_INPUT_TEMPLATE`
+| :term:`ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE`
+| :term:`ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR`
 | :term:`LOG_ENSEMBLE_STAT_VERBOSITY`
 | :term:`FCST_ENSEMBLE_STAT_INPUT_DATATYPE`
 | :term:`OBS_ENSEMBLE_STAT_INPUT_POINT_DATATYPE`

internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py

@@ -11,6 +11,9 @@
 
 fcst_dir = '/some/path/fcst'
 obs_dir = '/some/path/obs'
+ens_mean_dir = '/some/path/ens_mean'
+ens_mean_template = 'the_ens_mean_file.nc'
+obs_point_template = 'point_obs.nc'
 fcst_name = 'APCP'
 fcst_level = 'A03'
 obs_name = 'APCP_03'
@@ -43,10 +46,10 @@ def set_minimum_config_settings(config, set_fields=True):
     config.set('config', 'ENSEMBLE_STAT_CONFIG_FILE',
                '{PARM_BASE}/met_config/EnsembleStatConfig_wrapped')
     config.set('config', 'FCST_ENSEMBLE_STAT_INPUT_DIR', fcst_dir)
-    config.set('config', 'OBS_ENSEMBLE_STAT_INPUT_DIR', obs_dir)
+    config.set('config', 'OBS_ENSEMBLE_STAT_GRID_INPUT_DIR', obs_dir)
     config.set('config', 'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE',
                '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}')
-    config.set('config', 'OBS_ENSEMBLE_STAT_INPUT_TEMPLATE',
+    config.set('config', 'OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE',
                '{valid?fmt=%Y%m%d%H}/obs_file')
     config.set('config', 'ENSEMBLE_STAT_OUTPUT_DIR',
                '{OUTPUT_BASE}/EnsembleStat/output')
@@ -199,11 +202,11 @@ def test_handle_climo_file_variables(metplus_config, config_overrides,
         for old_env in old_env_vars:
             match = next((item for item in actual_env_vars if
                           item.startswith(old_env)), None)
-            assert(match is not None)
+            assert match is not None
             actual_value = match.split('=', 1)[1]
             expected_value = env_var_values.get(old_env, '')
             expected_value = expected_value.replace('YMDH', ymdh)
-            assert(expected_value == actual_value)
+            assert expected_value == actual_value
 
 
 @pytest.mark.parametrize(
@@ -601,6 +604,21 @@ def test_handle_climo_file_variables(metplus_config, config_overrides,
 
         ({'ENSEMBLE_STAT_OBS_THRESH': 'NA, 0.5', },
          {'METPLUS_OBS_THRESH': 'obs_thresh = [NA, 0.5];'}),
+
+        ({'ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR': ens_mean_dir,
+          'ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE': ens_mean_template},
+         {}),
+
+        ({'OBS_ENSEMBLE_STAT_POINT_INPUT_DIR': obs_dir,
+          'OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE': obs_point_template},
+         {}),
+
+        ({'ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR': ens_mean_dir,
+          'ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE': ens_mean_template,
+          'OBS_ENSEMBLE_STAT_POINT_INPUT_DIR': obs_dir,
+          'OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE': obs_point_template},
+         {}),
+
     ]
 )
 @pytest.mark.wrapper_c
@@ -623,12 +641,24 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides,
     file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR')
     config_file = wrapper.c_dict.get('CONFIG_FILE')
     out_dir = wrapper.c_dict.get('OUTPUT_DIR')
+
+    point_obs = ' '
+    ens_mean = ' '
+    if 'OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE' in config_overrides:
+        point_obs = f' -point_obs "{obs_dir}/{obs_point_template}" '
+    if 'ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE' in config_overrides:
+        ens_mean = f' -ens_mean {ens_mean_dir}/{ens_mean_template} '
+
     expected_cmds = [(f"{app_path} {verbosity} "
                       f"{file_list_dir}/20050807000000_12_ensemble_stat.txt "
-                      f"{config_file} -outdir {out_dir}/2005080712"),
+                      f"{config_file}{point_obs}"
+                      f'-grid_obs "{obs_dir}/2005080712/obs_file"{ens_mean}'
+                      f"-outdir {out_dir}/2005080712"),
                      (f"{app_path} {verbosity} "
                       f"{file_list_dir}/20050807120000_12_ensemble_stat.txt "
-                      f"{config_file} -outdir {out_dir}/2005080800"),
+                      f"{config_file}{point_obs}"
+                      f'-grid_obs "{obs_dir}/2005080800/obs_file"{ens_mean}'
+                      f"-outdir {out_dir}/2005080800"),
                      ]
 
     all_cmds = wrapper.run_all_times()
@@ -641,7 +671,7 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides,
 
     for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds):
         # ensure commands are generated as expected
-        assert(cmd == expected_cmd)
+        assert cmd == expected_cmd
 
         # check that environment variables were set properly
         # including deprecated env vars (not in wrapper env var keys)
@@ -651,11 +681,11 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides,
             assert(match is not None)
             actual_value = match.split('=', 1)[1]
             if env_var_key == 'METPLUS_FCST_FIELD':
-                assert(actual_value == fcst_fmt)
+                assert actual_value == fcst_fmt
             elif env_var_key == 'METPLUS_OBS_FIELD':
-                assert (actual_value == obs_fmt)
+                assert actual_value == obs_fmt
             else:
-                assert(env_var_values.get(env_var_key, '') == actual_value)
+                assert env_var_values.get(env_var_key, '') == actual_value
 
 
 @pytest.mark.wrapper_c

metplus/wrappers/ensemble_stat_wrapper.py

@@ -196,6 +196,14 @@ def create_c_dict(self):
             self.log_error("Must set FCST_ENSEMBLE_STAT_INPUT_TEMPLATE or "
                            "FCST_ENSEMBLE_STAT_INPUT_FILE_LIST")
 
+        # optional -ens_mean argument path
+        c_dict['ENS_MEAN_INPUT_DIR'] = (
+          self.config.getdir('ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR', ''))
+
+        c_dict['ENS_MEAN_INPUT_TEMPLATE'] = (
+            self.config.getraw('config',
+                               'ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE'))
+
         c_dict['OUTPUT_DIR'] = (
             self.config.getdir('ENSEMBLE_STAT_OUTPUT_DIR', '')
         )
@@ -380,35 +388,9 @@ def get_command(self):
            @rtype string
            @return Returns a MET command with arguments that you can run
         """
-        cmd = '{} -v {} '.format(self.app_path, self.c_dict['VERBOSITY'])
-
-        for args in self.args:
-            cmd += args + " "
-
-        if not self.infiles:
-            self.log_error(self.app_name+": No input filenames specified")
-            return None
-
-        for infile in self.infiles:
-            cmd += infile + " "
-
-        if self.param != "":
-            cmd += self.param + " "
-
-        for obs_file in self.point_obs_files:
-            if obs_file.startswith('PYTHON'):
-                obs_file = f"'{obs_file}'"
-            cmd += "-point_obs " + obs_file + " "
-
-        for obs_file in self.grid_obs_files:
-            cmd += "-grid_obs " + obs_file + " "
-
-        if not self.outdir:
-            self.log_error(self.app_name+": No output directory specified")
-            return None
-
-        cmd += '-outdir {}'.format(self.outdir)
-        return cmd
+        return (f"{self.app_path} -v {self.c_dict['VERBOSITY']}"
+                f" {' '.join(self.infiles)} {self.param}"
+                f" {' '.join(self.args)} -outdir {self.outdir}")
 
     def run_at_time_all_fields(self, time_info):
         """! Runs the MET application for a given time and forecast lead combination
@@ -424,21 +406,32 @@ def run_at_time_all_fields(self, time_info):
 
         # get point observation file if requested
         if self.c_dict['OBS_POINT_INPUT_TEMPLATE']:
-            point_obs_path = self.find_data(time_info, data_type='OBS_POINT',
-                                            return_list=True)
-            if point_obs_path is None:
+            point_obs_files = self.find_data(time_info, data_type='OBS_POINT',
+                                             return_list=True)
+            if point_obs_files is None:
                 return
 
-            self.point_obs_files.extend(point_obs_path)
+            for point_obs_path in point_obs_files:
+                self.args.append(f'-point_obs "{point_obs_path}"')
 
         # get grid observation file if requested
         if self.c_dict['OBS_GRID_INPUT_TEMPLATE']:
-            grid_obs_path = self.find_data(time_info, data_type='OBS_GRID',
+            grid_obs_files = self.find_data(time_info, data_type='OBS_GRID',
+                                            return_list=True)
+            if grid_obs_files is None:
+                return
+
+            for grid_obs_path in grid_obs_files:
+                self.args.append(f'-grid_obs "{grid_obs_path}"')
+
+        # get ens_mean file if requested
+        if self.c_dict['ENS_MEAN_INPUT_TEMPLATE']:
+            ens_mean_path = self.find_data(time_info, data_type='ENS_MEAN',
                                            return_list=True)
-            if grid_obs_path is None:
+            if ens_mean_path is None:
                 return
 
-            self.grid_obs_files.extend(grid_obs_path)
+            self.args.append(f'-ens_mean {ens_mean_path[0]}')
 
         # parse optional var list for FCST and/or OBS fields
         var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info)
@@ -511,10 +504,3 @@ def process_fields(self, time_info):
 
         # run the MET command
         self.build()
-
-    def clear(self):
-        """!Unset class variables to prepare for next run time
-        """
-        super().clear()
-        self.point_obs_files = []
-        self.grid_obs_files = []

parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf

@@ -48,10 +48,11 @@ FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/arw-???-gep?/d01_{init?f
 OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = {INPUT_BASE}/met_test/out/ascii2nc
 OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE = precip24_{valid?fmt=%Y%m%d%H}.nc
 
-
 OBS_ENSEMBLE_STAT_GRID_INPUT_DIR = {INPUT_BASE}/met_test/data/sample_obs/ST4
 OBS_ENSEMBLE_STAT_GRID_INPUT_TEMPLATE = ST4.{valid?fmt=%Y%m%d%H}.24h
 
+#ENSEMBLE_STAT_ENS_MEAN_INPUT_DIR =
+#ENSEMBLE_STAT_ENS_MEAN_INPUT_TEMPLATE =
 
 ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR =
 ENSEMBLE_STAT_CLIMO_MEAN_INPUT_TEMPLATE =