From 7818ad70ca546cb722652d617109ba1ac1983dfc Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 15 Aug 2022 16:37:32 -0600 Subject: [PATCH 01/25] per #1489, started implementation of PlotPointObs wrapper --- docs/Users_Guide/glossary.rst | 100 ++++++ docs/Users_Guide/wrappers.rst | 288 +++++++++++++++++- metplus/util/doc_util.py | 1 + metplus/wrappers/plot_point_obs_wrapper.py | 198 ++++++++++++ parm/met_config/PlotPointObsConfig_wrapped | 119 ++++++++ .../PlotPointObs/PlotPointObs.conf | 24 ++ 6 files changed, 729 insertions(+), 1 deletion(-) create mode 100755 metplus/wrappers/plot_point_obs_wrapper.py create mode 100644 parm/met_config/PlotPointObsConfig_wrapped create mode 100644 parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index cf8ff44469..bcbe7e6975 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -9653,3 +9653,103 @@ METplus Configuration Glossary Specify the value for 'match_points' in the MET configuration file for TCPairs. | *Used by:* TCPairs + + PLOT_POINT_OBS_MSG_TYP + Specify the value for 'msg_typ' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_SID_INC + Specify the value for 'sid_inc' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_SID_EXC + Specify the value for 'sid_exc' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_OBS_VAR + Specify the value for 'obs_var' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_OBS_GC + Specify the value for 'obs_gc' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_OBS_QUALITY + Specify the value for 'obs_quality' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_VALID_BEG + Specify the value for 'valid_beg' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_VALID_END + Specify the value for 'valid_end' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_LAT_THRESH + Specify the value for 'lat_thresh' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_LON_THRESH + Specify the value for 'lon_thresh' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_ELV_THRESH + Specify the value for 'elv_thresh' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_HGT_THRESH + Specify the value for 'hgt_thresh' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_PRS_THRESH + Specify the value for 'prs_thresh' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_OBS_THRESH + Specify the value for 'obs_thresh' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_CENSOR_THRESH + Specify the value for 'censor_thresh' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_CENSOR_VAL + Specify the value for 'censor_val' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_DOTSIZE + Specify the value for 'dotsize' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_LINE_COLOR + Specify the value for 'line_color' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_LINE_WIDTH + Specify the value for 'line_width' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_FILL_COLOR + Specify the value for 'fill_color' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index ad359da405..5f0d098135 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -5368,7 +5368,293 @@ Configuration | :term:`PLOT_DATA_PLANE_RANGE_MIN_MAX` | :term:`PLOT_DATA_PLANE_CONVERT_TO_IMAGE` | :term:`PLOT_DATA_PLANE_SKIP_IF_OUTPUT_EXISTS` -| + +.. _plot_point_obs_wrapper: + +PlotPointObs +============ + +Description +----------- + +The PlotPointObs wrapper is a Python script that encapsulates the MET +plot_point_obs tool. It provides the infrastructure to read in any input that +MET can read and plot them. + +Configuration +------------- + +| :term:`PLOT_POINT_OBS_INPUT_TEMPLATE` +| :term:`PLOT_POINT_OBS_INPUT_DIR` +| :term:`PLOT_POINT_OBS_GRID_INPUT_TEMPLATE` +| :term:`PLOT_POINT_OBS_GRID_INPUT_DIR` +| :term:`PLOT_POINT_OBS_OUTPUT_TEMPLATE` +| :term:`PLOT_POINT_OBS_OUTPUT_DIR` +| :term:`PLOT_POINT_OBS_TITLE` +| :term:`LOG_PLOT_POINT_OBS_VERBOSITY` + +| :term:`PLOT_POINT_OBS_MSG_TYP` +| :term:`PLOT_POINT_OBS_SID_INC` +| :term:`PLOT_POINT_OBS_SID_EXC` +| :term:`PLOT_POINT_OBS_OBS_VAR` +| :term:`PLOT_POINT_OBS_OBS_GC` +| :term:`PLOT_POINT_OBS_OBS_QUALITY` +| :term:`PLOT_POINT_OBS_VALID_BEG` +| :term:`PLOT_POINT_OBS_VALID_END` +| :term:`PLOT_POINT_OBS_LAT_THRESH` +| :term:`PLOT_POINT_OBS_LON_THRESH` +| :term:`PLOT_POINT_OBS_ELV_THRESH` +| :term:`PLOT_POINT_OBS_HGT_THRESH` +| :term:`PLOT_POINT_OBS_PRS_THRESH` +| :term:`PLOT_POINT_OBS_OBS_THRESH` +| :term:`PLOT_POINT_OBS_CENSOR_THRESH` +| :term:`PLOT_POINT_OBS_CENSOR_VAL` +| :term:`PLOT_POINT_OBS_DOTSIZE` +| :term:`PLOT_POINT_OBS_LINE_COLOR` +| :term:`PLOT_POINT_OBS_LINE_WIDTH` +| :term:`PLOT_POINT_OBS_FILL_COLOR` + + +.. _plot-point-obs-met-conf: + +MET Configuration +----------------- + +Below is the wrapped MET configuration file used for this wrapper. +Environment variables are used to control entries in this configuration file. +The default value for each environment variable is obtained from +(except where noted below): + +`MET_INSTALL_DIR/share/met/config/PlotPointObsConfig_default `_ + +Below the file contents are descriptions of each environment variable +referenced in this file and the corresponding METplus configuration item used +to set the value of the environment variable. For detailed examples showing +how METplus sets the values of these environment variables, +see :ref:`How METplus controls MET config file settings`. + +.. literalinclude:: ../../parm/met_config/PlotPointObsConfig_wrapped + +**${METPLUS_MSG_TYP}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_MSG_TYP` + - msg_typ + +**${METPLUS_SID_INC}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_SID_INC` + - sid_inc + +**${METPLUS_SID_EXC}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_SID_EXC` + - sid_exc + +**${METPLUS_OBS_VAR}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_OBS_VAR` + - obs_var + +**${METPLUS_OBS_GC}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_OBS_GC` + - obs_gc + +**${METPLUS_OBS_QUALITY}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_OBS_QUALITY` + - obs_quality + +**${METPLUS_VALID_BEG}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_VALID_BEG` + - valid_beg + +**${METPLUS_VALID_END}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_VALID_END` + - valid_end + +**${METPLUS_LAT_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_LAT_THRESH` + - lat_thresh + +**${METPLUS_LON_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_LON_THRESH` + - lon_thresh + +**${METPLUS_ELV_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_ELV_THRESH` + - elv_thresh + +**${METPLUS_HGT_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_HGT_THRESH` + - hgt_thresh + +**${METPLUS_PRS_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_PRS_THRESH` + - prs_thresh + +**${METPLUS_OBS_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_OBS_THRESH` + - obs_thresh + +**${METPLUS_CENSOR_THRESH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_CENSOR_THRESH` + - censor_thresh + +**${METPLUS_CENSOR_VAL}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_CENSOR_VAL` + - censor_val + +**${METPLUS_DOTSIZE}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_DOTSIZE` + - dotsize + +**${METPLUS_LINE_COLOR}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_LINE_COLOR` + - line_color + +**${METPLUS_LINE_WIDTH}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_LINE_WIDTH` + - line_width + +**${METPLUS_FILL_COLOR}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_FILL_COLOR` + - fill_color + .. _point2grid_wrapper: diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index 9338a69369..5430084d6c 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -26,6 +26,7 @@ 'pb2nc': 'PB2NC', 'pcpcombine': 'PCPCombine', 'plotdataplane': 'PlotDataPlane', + 'plotpointobs': 'PlotPointObs', 'point2grid': 'Point2Grid', 'pointtogrid': 'Point2Grid', 'pointstat': 'PointStat', diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py new file mode 100755 index 0000000000..169320a685 --- /dev/null +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -0,0 +1,198 @@ +""" +Program Name: plot_point_obs_wrapper.py +Contact(s): George McCabe +Abstract: Wrapper for plot_point_obs MET tool +History Log: Initial version +Usage: Not meant to be run +Parameters: None +Input Files: None +Output Files: None +Condition codes: 0 for success, 1 for failure +""" + +import os + +from ..util import do_string_sub, ti_calculate, get_lead_sequence +from ..util import skip_time +from . import RuntimeFreqWrapper + + +class PlotPointObsWrapper(RuntimeFreqWrapper): + """! Wrapper used to build commands to call plot_point_obs """ + + WRAPPER_ENV_VAR_KEYS = [ + 'METPLUS_GRID_DATA_DICT', + 'METPLUS_MSG_TYP', + 'METPLUS_SID_INC', + 'METPLUS_SID_EXC', + 'METPLUS_OBS_VAR', + 'METPLUS_OBS_GC', + 'METPLUS_OBS_QUALITY', + 'METPLUS_VALID_BEG', + 'METPLUS_VALID_END', + 'METPLUS_LAT_THRESH', + 'METPLUS_LON_THRESH', + 'METPLUS_ELV_THRESH', + 'METPLUS_HGT_THRESH', + 'METPLUS_PRS_THRESH', + 'METPLUS_OBS_THRESH', + 'METPLUS_CENSOR_THRESH', + 'METPLUS_CENSOR_VAL', + 'METPLUS_DOTSIZE', + 'METPLUS_LINE_COLOR', + 'METPLUS_LINE_WIDTH', + 'METPLUS_FILL_COLOR', + ] + + def __init__(self, config, instance=None): + self.app_name = 'plot_point_obs' + self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), + self.app_name) + super().__init__(config, instance=instance) + + def create_c_dict(self): + c_dict = super().create_c_dict() + app = self.app_name.upper() + + # get point obs input files + c_dict['INPUT_TEMPLATE'] = self.config.getraw('config', + f'{app}_INPUT_TEMPLATE') + c_dict['INPUT_DIR'] = self.config.getdir(f'{app}_INPUT_DIR', '') + + if not c_dict['INPUT_TEMPLATE']: + self.logger.warning(f'{app}_INPUT_TEMPLATE is required ' + 'to run PlotPointObs wrapper.') + + # get optional grid input files + c_dict['GRID_INPUT_TEMPLATE'] = self.config.getraw( + 'config', + f'{app}_GRID_INPUT_TEMPLATE' + ) + c_dict['GRID_INPUT_DIR'] = self.config.getdir(f'{app}_GRID_INPUT_DIR', + '') + + # get optional title command line argument + c_dict['TITLE'] = self.config.getraw('config', f'{app}_TITLE') + + # read config file settings + + # handle grid_data dictionary + items = { + 'field': ('list', 'remove_quotes'), + 'regrid': ('dict', '', { + 'to_grid': ('string', 'uppercase,to_grid'), + 'method': ('string', 'uppercase,remove_quotes'), + 'width': 'int', + 'vld_thresh': 'float', + 'shape': ('string', 'uppercase,remove_quotes'), + }), + 'grid_plot_info': ('dict', '', { + 'color_table': 'string', + 'plot_min': 'float', + 'plot_max': 'float', + 'colorbar_flag': 'bool', + }), + } + self.add_met_config_dict('grid_data', items) + + config_lists = [ + 'msg_typ', + 'sid_inc', + 'sid_exc', + 'obs_var', + 'obs_gc', + 'obs_quality', + 'censor_thresh', + 'censor_val', + 'line_color', + 'fill_color', + ] + for config_list in config_lists: + self.add_met_config(name=config_list, data_type='list') + + config_strings = [ + 'valid_beg', + 'valid_end', + ] + for config_string in config_strings: + self.add_met_config(name=config_string, data_type='string') + + config_threshs = [ + 'lat_thresh', + 'lon_thresh', + 'elv_thresh', + 'hgt_thresh', + 'prs_thresh', + 'obs_thresh', + ] + for config_thresh in config_threshs: + self.add_met_config(name=config_thresh, data_type='string', + extra_args={'remove_quotes': True}) + + self.add_met_config(name='dotsize(x)', data_type='string', + extra_args={'remove_quotes': True}, + metplus_configs=[f'{app}_DOTSIZE']) + self.add_met_config(name='line_width', data_type='int') + + c_dict['ALLOW_MULTIPLE_FILES'] = True + return c_dict + + def run_at_time(self, input_dict): + """! Do some processing for the current run time (init or valid) + + @param input_dict dictionary with time information of current run + """ + # fill in time info dictionary + time_info = ti_calculate(input_dict) + + # check if looping by valid or init and log time for run + loop_by = time_info['loop_by'] + current_time = time_info[loop_by + '_fmt'] + self.logger.info('Running PlotPointObsWrapper at ' + f'{loop_by} time {current_time}') + + # read input directory and template from config dictionary + input_dir = self.c_dict['INPUT_DIR'] + input_template = self.c_dict['INPUT_TEMPLATE'] + self.logger.info(f'Input directory is {input_dir}') + self.logger.info(f'Input template is {input_template}') + + # get forecast leads to loop over + lead_seq = get_lead_sequence(self.config, input_dict) + for lead in lead_seq: + + # set forecast lead time in hours + time_info['lead'] = lead + + # recalculate time info items + time_info = ti_calculate(time_info) + + if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): + self.logger.debug('Skipping run time') + continue + + for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: + if custom_string: + self.logger.info( + f"Processing custom string: {custom_string}" + ) + + time_info['custom'] = custom_string + + # log init/valid/forecast lead times for current loop iteration + self.logger.info( + 'Processing forecast lead ' + f'{time_info["lead_string"]} initialized at ' + f'{time_info["init"].strftime("%Y-%m-%d %HZ")} ' + 'and valid at ' + f'{time_info["valid"].strftime("%Y-%m-%d %HZ")}' + ) + + # perform string substitution to find filename based on + # template and current run time + filename = do_string_sub(input_template, + **time_info) + self.logger.info('Looking in input directory ' + f'for file: {filename}') + + return True diff --git a/parm/met_config/PlotPointObsConfig_wrapped b/parm/met_config/PlotPointObsConfig_wrapped new file mode 100644 index 0000000000..1e7b189263 --- /dev/null +++ b/parm/met_config/PlotPointObsConfig_wrapped @@ -0,0 +1,119 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Plot-Point-Obs configuration file. +// +// For additional information, please see the MET Users Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// Gridded data plotting options + +grid_data = { + + field = []; + + regrid = { + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; + } + + grid_plot_info = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; + colorbar_flag = TRUE; + } + +} + +//////////////////////////////////////////////////////////////////////////////// + +// Point data filtering options +// May be set separately in each "point_data" entry + +//msg_typ = +${METPLUS_MSG_TYP} + +//sid_inc = +${METPLUS_SID_INC} + +//sid_exc = +${METPLUS_SID_EXC} + +//obs_var = +${METPLUS_OBS_VAR} + +//obs_gc = +${METPLUS_OBS_GC} + +//obs_quality = +${METPLUS_OBS_QUALITY} + +//valid_beg = +${METPLUS_VALID_BEG} + +//valid_end = +${METPLUS_VALID_END} + +//lat_thresh = +${METPLUS_LAT_THRESH} + +//lon_thresh = +${METPLUS_LON_THRESH} + +//elv_thresh = +${METPLUS_ELV_THRESH} + +//hgt_thresh = +${METPLUS_HGT_THRESH} + +//prs_thresh = +${METPLUS_PRS_THRESH} + +//obs_thresh = +${METPLUS_OBS_THRESH} + +// Point data pre-processing options +// May be set separately in each "point_data" entry + +convert(x) = x; +//censor_thresh = +${METPLUS_CENSOR_THRESH} + +//censor_val = +${METPLUS_CENSOR_VAL} + + +// Point data plotting options +// May be set separately in each "point_data" entry + +//dotsize = +${METPLUS_DOTSIZE} + +//line_color = +${METPLUS_LINE_COLOR} + +//line_width = +${METPLUS_LINE_WIDTH} + +//fill_color = +${METPLUS_FILL_COLOR} + + +fill_plot_info = { // Overrides fill_color + flag = FALSE; + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; + colorbar_flag = TRUE; +} + +// Array of point data filtering, pre-processing, and plotting options +point_data = [ + { fill_color = [ 255, 0, 0 ]; } +]; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf new file mode 100644 index 0000000000..879441fd8d --- /dev/null +++ b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf @@ -0,0 +1,24 @@ +[config] + + + +#PLOT_POINT_OBS_MSG_TYP = +#PLOT_POINT_OBS_SID_INC = +#PLOT_POINT_OBS_SID_EXC = +#PLOT_POINT_OBS_OBS_VAR = +#PLOT_POINT_OBS_OBS_GC = +#PLOT_POINT_OBS_OBS_QUALITY = +#PLOT_POINT_OBS_VALID_BEG = +#PLOT_POINT_OBS_VALID_END = +#PLOT_POINT_OBS_LAT_THRESH = +#PLOT_POINT_OBS_LON_THRESH = +#PLOT_POINT_OBS_ELV_THRESH = +#PLOT_POINT_OBS_HGT_THRESH = +#PLOT_POINT_OBS_PRS_THRESH = +#PLOT_POINT_OBS_OBS_THRESH = +#PLOT_POINT_OBS_CENSOR_THRESH = +#PLOT_POINT_OBS_CENSOR_VAL = +#PLOT_POINT_OBS_DOTSIZE = +#PLOT_POINT_OBS_LINE_COLOR = +#PLOT_POINT_OBS_LINE_WIDTH = +#PLOT_POINT_OBS_FILL_COLOR = From 6ce1e9a8de2542c3453f0f55e9a398a513fd5012 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 16 Aug 2022 17:04:53 -0600 Subject: [PATCH 02/25] per #1489, continued development towards creating PlotPointObs wrapper --- docs/Users_Guide/glossary.rst | 130 ++++++++++++++++++ docs/Users_Guide/wrappers.rst | 76 +++++++++- metplus/wrappers/plot_point_obs_wrapper.py | 28 +++- parm/met_config/PlotPointObsConfig_wrapped | 37 +---- .../PlotPointObs/PlotPointObs.conf | 19 ++- 5 files changed, 254 insertions(+), 36 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index bcbe7e6975..747a20ad1e 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -9753,3 +9753,133 @@ METplus Configuration Glossary Specify the value for 'fill_color' in the MET configuration file for PlotPointObs. | *Used by:* PlotPointObs + + PLOT_POINT_OBS_INPUT_DIR + Directory containing input point observation data for PlotPointObs. + This variable is optional because you can specify the full path to the + input file(s) using :term:`PLOT_POINT_OBS_INPUT_TEMPLATE`. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_INPUT_TEMPLATE + Filename template of the input point observation file(s) for PlotPointObs. + See also :term:`PLOT_POINT_OBS_INPUT_DIR`. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_INPUT_DIR + Directory containing input grid data for PlotPointObs. + This variable is optional because you can specify the full path to the + input file using :term:`PLOT_POINT_OBS_GRID_INPUT_TEMPLATE`. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_INPUT_TEMPLATE + Filename template of the input grid file for PlotPointObs. + See also :term:`PLOT_POINT_OBS_GRID_INPUT_DIR`. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_OUTPUT_DIR + Directory containing output generated by PlotPointObs. + This variable is optional because you can specify the full path to the + output file using :term:`PLOT_POINT_OBS_OUTPUT_TEMPLATE`. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_OUTPUT_TEMPLATE + Filename template of the output generated by PlotPointObs. + See also :term:`PLOT_POINT_OBS_OUTPUT_DIR`. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_TITLE + Sets the title for the output plot generated by PlotPointObs. + + | *Used by:* PlotPointObs + + LOG_PLOT_POINT_OBS_VERBOSITY + Overrides the log verbosity for plot_point_obs only. If not set, + the verbosity level is controlled by :term:`LOG_MET_VERBOSITY`. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_FILL_PLOT_INFO_FLAG + Specify the value for 'fill_plot_info.flag' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_FILL_PLOT_INFO_COLOR_TABLE + Specify the value for 'fill_plot_info.color_table' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MIN + Specify the value for 'fill_plot_info.plot_min' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MAX + Specify the value for 'fill_plot_info.plot_max' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_FILL_PLOT_INFO_COLORBAR_FLAG + Specify the value for 'fill_plot_info.colorbar_flag' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_DATA_FIELD + Specify the value for 'grid_data.field' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_DATA_REGRID_TO_GRID + Specify the value for 'grid_data.regrid.to_grid' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_DATA_REGRID_METHOD + Specify the value for 'grid_data.regrid.method' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_DATA_REGRID_WIDTH + Specify the value for 'grid_data.regrid.width' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_DATA_REGRID_VLD_THRESH + Specify the value for 'grid_data.regrid.vld_thresh' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_DATA_REGRID_SHAPE + Specify the value for 'grid_data.regrid.shape' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLOR_TABLE + Specify the value for 'grid_data.grid_plot_info.color_table' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MIN + Specify the value for 'grid_data.grid_plot_info.plot_min' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MAX + Specify the value for 'grid_data.grid_plot_info.plot_max' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLORBAR_FLAG + Specify the value for 'grid_data.grid_plot_info.colorbar_flag' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + PLOT_POINT_OBS_POINT_DATA + Specify the value for 'point_data' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 5f0d098135..b7d44140dc 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -5392,7 +5392,16 @@ Configuration | :term:`PLOT_POINT_OBS_OUTPUT_DIR` | :term:`PLOT_POINT_OBS_TITLE` | :term:`LOG_PLOT_POINT_OBS_VERBOSITY` - +| :term:`PLOT_POINT_OBS_GRID_DATA_FIELD` +| :term:`PLOT_POINT_OBS_GRID_DATA_REGRID_TO_GRID` +| :term:`PLOT_POINT_OBS_GRID_DATA_REGRID_METHOD` +| :term:`PLOT_POINT_OBS_GRID_DATA_REGRID_WIDTH` +| :term:`PLOT_POINT_OBS_GRID_DATA_REGRID_VLD_THRESH` +| :term:`PLOT_POINT_OBS_GRID_DATA_REGRID_SHAPE` +| :term:`PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLOR_TABLE` +| :term:`PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MIN` +| :term:`PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MAX` +| :term:`PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLORBAR_FLAG` | :term:`PLOT_POINT_OBS_MSG_TYP` | :term:`PLOT_POINT_OBS_SID_INC` | :term:`PLOT_POINT_OBS_SID_EXC` @@ -5413,6 +5422,12 @@ Configuration | :term:`PLOT_POINT_OBS_LINE_COLOR` | :term:`PLOT_POINT_OBS_LINE_WIDTH` | :term:`PLOT_POINT_OBS_FILL_COLOR` +| :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_FLAG` +| :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_COLOR_TABLE` +| :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MIN` +| :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MAX` +| :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_COLORBAR_FLAG` +| :term:`PLOT_POINT_OBS_POINT_DATA` .. _plot-point-obs-met-conf: @@ -5435,6 +5450,35 @@ see :ref:`How METplus controls MET config file settings`. .. literalinclude:: ../../parm/met_config/PlotPointObsConfig_wrapped +**${METPLUS_GRID_DATA_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_GRID_DATA_FIELD` + - grid_data.field + * - :term:`PLOT_POINT_OBS_GRID_DATA_REGRID_TO_GRID` + - grid_data.regrid.to_grid + * - :term:`PLOT_POINT_OBS_GRID_DATA_REGRID_METHOD` + - grid_data.regrid.method + * - :term:`PLOT_POINT_OBS_GRID_DATA_REGRID_WIDTH` + - grid_data.regrid.width + * - :term:`PLOT_POINT_OBS_GRID_DATA_REGRID_VLD_THRESH` + - grid_data.regrid.vld_thresh + * - :term:`PLOT_POINT_OBS_GRID_DATA_REGRID_SHAPE` + - grid_data.regrid.shape + * - :term:`PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLOR_TABLE` + - grid_data.grid_plot_info.color_table + * - :term:`PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MIN` + - grid_data.grid_plot_info.plot_min + * - :term:`PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MAX` + - grid_data.grid_plot_info.plot_max + * - :term:`PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLORBAR_FLAG` + - grid_data.grid_plot_info.colorbar_flag + **${METPLUS_MSG_TYP}** .. list-table:: @@ -5655,6 +5699,36 @@ see :ref:`How METplus controls MET config file settings`. * - :term:`PLOT_POINT_OBS_FILL_COLOR` - fill_color +**${METPLUS_FILL_PLOT_INFO_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_FLAG` + - fill_plot_info.flag + * - :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_COLOR_TABLE` + - fill_plot_info.color_table + * - :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MIN` + - fill_plot_info.plot_min + * - :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MAX` + - fill_plot_info.plot_max + * - :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_COLORBAR_FLAG` + - fill_plot_info.colorbar_flag + +**${METPLUS_POINT_DATA}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_POINT_DATA` + - point_data + .. _point2grid_wrapper: diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index 169320a685..899b682ff9 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -42,6 +42,8 @@ class PlotPointObsWrapper(RuntimeFreqWrapper): 'METPLUS_LINE_COLOR', 'METPLUS_LINE_WIDTH', 'METPLUS_FILL_COLOR', + 'METPLUS_FILL_PLOT_INFO_DICT', + 'METPLUS_POINT_DATA', ] def __init__(self, config, instance=None): @@ -54,6 +56,10 @@ def create_c_dict(self): c_dict = super().create_c_dict() app = self.app_name.upper() + c_dict['VERBOSITY'] = self.config.getstr('config', + f'LOG_{app}_VERBOSITY', + c_dict['VERBOSITY']) + # get point obs input files c_dict['INPUT_TEMPLATE'] = self.config.getraw('config', f'{app}_INPUT_TEMPLATE') @@ -76,8 +82,13 @@ def create_c_dict(self): # read config file settings + # get the MET config file path or use default + c_dict['CONFIG_FILE'] = ( + self.get_config_file('PlotPointObsConfig_wrapped') + ) + # handle grid_data dictionary - items = { + self.add_met_config_dict('grid_data', { 'field': ('list', 'remove_quotes'), 'regrid': ('dict', '', { 'to_grid': ('string', 'uppercase,to_grid'), @@ -92,8 +103,7 @@ def create_c_dict(self): 'plot_max': 'float', 'colorbar_flag': 'bool', }), - } - self.add_met_config_dict('grid_data', items) + }) config_lists = [ 'msg_typ', @@ -134,6 +144,18 @@ def create_c_dict(self): metplus_configs=[f'{app}_DOTSIZE']) self.add_met_config(name='line_width', data_type='int') + self.add_met_config_dict('fill_plot_info', { + 'flag': 'bool', + 'color_table': 'string', + 'plot_min': 'float', + 'plot_max': 'float', + 'colorbar_flag': 'bool', + }) + + self.add_met_config(name='point_data', + data_type='list', + extra_args={'remove_quotes': True}) + c_dict['ALLOW_MULTIPLE_FILES'] = True return c_dict diff --git a/parm/met_config/PlotPointObsConfig_wrapped b/parm/met_config/PlotPointObsConfig_wrapped index 1e7b189263..9454e95308 100644 --- a/parm/met_config/PlotPointObsConfig_wrapped +++ b/parm/met_config/PlotPointObsConfig_wrapped @@ -8,26 +8,8 @@ // Gridded data plotting options -grid_data = { - - field = []; - - regrid = { - to_grid = NONE; - method = NEAREST; - width = 1; - vld_thresh = 0.5; - shape = SQUARE; - } - - grid_plot_info = { - color_table = "MET_BASE/colortables/met_default.ctable"; - plot_min = 0.0; - plot_max = 0.0; - colorbar_flag = TRUE; - } - -} +//grid_data = { +${METPLUS_GRID_DATA_DICT} //////////////////////////////////////////////////////////////////////////////// @@ -102,18 +84,11 @@ ${METPLUS_LINE_WIDTH} //fill_color = ${METPLUS_FILL_COLOR} - -fill_plot_info = { // Overrides fill_color - flag = FALSE; - color_table = "MET_BASE/colortables/met_default.ctable"; - plot_min = 0.0; - plot_max = 0.0; - colorbar_flag = TRUE; -} +//fill_plot_info = { +${METPLUS_FILL_PLOT_INFO_DICT} // Array of point data filtering, pre-processing, and plotting options -point_data = [ - { fill_color = [ 255, 0, 0 ]; } -]; +//point_data = +${METPLUS_POINT_DATA} //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf index 879441fd8d..9d13fcc2a7 100644 --- a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf +++ b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf @@ -1,6 +1,15 @@ [config] - +#PLOT_POINT_OBS_GRID_DATA_FIELD = +#PLOT_POINT_OBS_GRID_DATA_REGRID_TO_GRID = +#PLOT_POINT_OBS_GRID_DATA_REGRID_METHOD = +#PLOT_POINT_OBS_GRID_DATA_REGRID_WIDTH = +#PLOT_POINT_OBS_GRID_DATA_REGRID_VLD_THRESH = +#PLOT_POINT_OBS_GRID_DATA_REGRID_SHAPE = +#PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLOR_TABLE = +#PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MIN = +#PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MAX = +#PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLORBAR_FLAG = #PLOT_POINT_OBS_MSG_TYP = #PLOT_POINT_OBS_SID_INC = @@ -22,3 +31,11 @@ #PLOT_POINT_OBS_LINE_COLOR = #PLOT_POINT_OBS_LINE_WIDTH = #PLOT_POINT_OBS_FILL_COLOR = + +#PLOT_POINT_OBS_FILL_PLOT_INFO_FLAG = +#PLOT_POINT_OBS_FILL_PLOT_INFO_COLOR_TABLE = +#PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MIN = +#PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MAX = +#PLOT_POINT_OBS_FILL_PLOT_INFO_COLORBAR_FLAG = + +#PLOT_POINT_OBS_POINT_DATA = From 6d034a4766f38b6fb052fa90eb6bc60fc11b3c87 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 17 Aug 2022 16:45:25 -0600 Subject: [PATCH 03/25] added missing config variable to doc --- docs/Users_Guide/glossary.rst | 6 ++++++ docs/Users_Guide/wrappers.rst | 1 + 2 files changed, 7 insertions(+) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 747a20ad1e..53fa252be3 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -9883,3 +9883,9 @@ METplus Configuration Glossary Specify the value for 'point_data' in the MET configuration file for PlotPointObs. | *Used by:* PlotPointObs + + PLOT_POINT_OBS_SKIP_IF_OUTPUT_EXISTS + If True, do not run plot_point_obs if output file already exists. + Set to False to overwrite files. + + | *Used by:* PlotPointObs diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index b7d44140dc..638abb8039 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -5390,6 +5390,7 @@ Configuration | :term:`PLOT_POINT_OBS_GRID_INPUT_DIR` | :term:`PLOT_POINT_OBS_OUTPUT_TEMPLATE` | :term:`PLOT_POINT_OBS_OUTPUT_DIR` +| :term:`PLOT_POINT_OBS_SKIP_IF_OUTPUT_EXISTS` | :term:`PLOT_POINT_OBS_TITLE` | :term:`LOG_PLOT_POINT_OBS_VERBOSITY` | :term:`PLOT_POINT_OBS_GRID_DATA_FIELD` From 6633b1019a7e04751c4f2660203b85fbc7dcc66e Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 17 Aug 2022 16:53:10 -0600 Subject: [PATCH 04/25] per #1489, added logic to build commands --- metplus/wrappers/plot_point_obs_wrapper.py | 136 ++++++++++++--------- 1 file changed, 78 insertions(+), 58 deletions(-) diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index 899b682ff9..29ec03593e 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -76,7 +76,6 @@ def create_c_dict(self): ) c_dict['GRID_INPUT_DIR'] = self.config.getdir(f'{app}_GRID_INPUT_DIR', '') - # get optional title command line argument c_dict['TITLE'] = self.config.getraw('config', f'{app}_TITLE') @@ -159,62 +158,83 @@ def create_c_dict(self): c_dict['ALLOW_MULTIPLE_FILES'] = True return c_dict - def run_at_time(self, input_dict): - """! Do some processing for the current run time (init or valid) + def get_command(self): + return (f"{self.app_path} -v {self.c_dict['VERBOSITY']}" + f" {self.infiles[0]} {self.get_output_path()}" + f" {' '.join(self.args)}") + + def run_at_time_once(self, time_info): + """! Process runtime and try to build command to run plot_point_obs. + + @param time_info dictionary containing timing information + @returns True if command was built/run successfully or + False if something went wrong + """ + # get input files + if not self.find_input_files(time_info): + return False + + # get output path + if not self.find_and_check_output_file(time_info): + return False + + # get other configurations for command + self.set_command_line_arguments(time_info) + + # set environment variables if using config file + self.set_environment_variables(time_info) + + # build command and run + return self.build() + + def find_input_files(self, time_info): + """! Get all input files for plot_point_obs. Sets self.infiles list. + + @param time_info dictionary containing timing information + @returns List of files that were found or None if no files were found + """ + input_files = self.find_data(time_info, + return_list=True, + mandatory=mandatory) + if input_files is None: + return None + + self.infiles.extend(input_files) + + # get optional grid file path + if self.c_dict['GRID_INPUT_TEMPLATE']: + grid_file = self.find_data(time_info, + data_type='GRID', + return_list=True) + if not grid_file: + return None + + if len(grid_file) > 1: + self.log_error('More than one file found from ' + 'PLOT_POINT_OBS_GRID_INPUT_TEMPLATE: ' + f'{grid_file.split(",")}') + return None + + self.c_dict['GRID_INPUT_PATH'] = grid_file[0] - @param input_dict dictionary with time information of current run + return self.infiles + + def set_command_line_arguments(self, time_info): + """! Set all arguments for plot_point_obs command. + + @param time_info dictionary containing timing information """ - # fill in time info dictionary - time_info = ti_calculate(input_dict) - - # check if looping by valid or init and log time for run - loop_by = time_info['loop_by'] - current_time = time_info[loop_by + '_fmt'] - self.logger.info('Running PlotPointObsWrapper at ' - f'{loop_by} time {current_time}') - - # read input directory and template from config dictionary - input_dir = self.c_dict['INPUT_DIR'] - input_template = self.c_dict['INPUT_TEMPLATE'] - self.logger.info(f'Input directory is {input_dir}') - self.logger.info(f'Input template is {input_template}') - - # get forecast leads to loop over - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - - # set forecast lead time in hours - time_info['lead'] = lead - - # recalculate time info items - time_info = ti_calculate(time_info) - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue - - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info( - f"Processing custom string: {custom_string}" - ) - - time_info['custom'] = custom_string - - # log init/valid/forecast lead times for current loop iteration - self.logger.info( - 'Processing forecast lead ' - f'{time_info["lead_string"]} initialized at ' - f'{time_info["init"].strftime("%Y-%m-%d %HZ")} ' - 'and valid at ' - f'{time_info["valid"].strftime("%Y-%m-%d %HZ")}' - ) - - # perform string substitution to find filename based on - # template and current run time - filename = do_string_sub(input_template, - **time_info) - self.logger.info('Looking in input directory ' - f'for file: {filename}') - - return True + config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) + self.args.append(f'-config {config_file}') + + # if more than 1 input file was found, add them with -point_obs + for infile in self.infiles[1:]: + self.args.append(f'-point_obs {infile}') + + if self.c_dict['TITLE']: + self.args.append(f"-title {self.c_dict['TITLE']}") + + if self.c_dict['GRID_INPUT_PATH']: + grid_file = do_string_sub(self.c_dict['GRID_INPUT_PATH'], + **time_info) + self.args.append(f'-plot_grid {grid_file}') From 50d4b868b43bf4286897ffa31a70841d16c11332 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 14:14:01 -0600 Subject: [PATCH 05/25] per #1489, document runtime frequency variable and set default frequency if unset --- docs/Users_Guide/glossary.rst | 5 +++++ docs/Users_Guide/wrappers.rst | 1 + metplus/wrappers/plot_point_obs_wrapper.py | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 53fa252be3..144efc9e9b 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -9889,3 +9889,8 @@ METplus Configuration Glossary Set to False to overwrite files. | *Used by:* PlotPointObs + + PLOT_POINT_OBS_RUNTIME_FREQ + Frequency to run PlotPointObs. See :ref:`Runtime_Freq` for more information. + + | *Used by:* PlotPointObs diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 638abb8039..a668c9d1d4 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -5384,6 +5384,7 @@ MET can read and plot them. Configuration ------------- +| :term:`PLOT_POINT_OBS_RUNTIME_FREQ` | :term:`PLOT_POINT_OBS_INPUT_TEMPLATE` | :term:`PLOT_POINT_OBS_INPUT_DIR` | :term:`PLOT_POINT_OBS_GRID_INPUT_TEMPLATE` diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index 29ec03593e..7bf4d9569c 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -56,6 +56,10 @@ def create_c_dict(self): c_dict = super().create_c_dict() app = self.app_name.upper() + # set default runtime frequency if unset explicitly + if not c_dict['RUNTIME_FREQ']: + c_dict['RUNTIME_FREQ'] = 'RUN_ONCE_FOR_EACH' + c_dict['VERBOSITY'] = self.config.getstr('config', f'LOG_{app}_VERBOSITY', c_dict['VERBOSITY']) From 5a5a42f254972ff8353f05105e77e8cbda2d74e3 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 14:14:27 -0600 Subject: [PATCH 06/25] per #1489, read output path variables --- metplus/wrappers/plot_point_obs_wrapper.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index 7bf4d9569c..f179786dd7 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -80,6 +80,14 @@ def create_c_dict(self): ) c_dict['GRID_INPUT_DIR'] = self.config.getdir(f'{app}_GRID_INPUT_DIR', '') + + # get output path + c_dict['OUTPUT_TEMPLATE'] = self.config.getraw( + 'config', + f'{app}_OUTPUT_TEMPLATE' + ) + c_dict['OUTPUT_DIR'] = self.config.getdir(f'{app}_OUTPUT_DIR', '') + # get optional title command line argument c_dict['TITLE'] = self.config.getraw('config', f'{app}_TITLE') From 8c5affaba415374d1162d981940e921b8d5fa293 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 14:14:44 -0600 Subject: [PATCH 07/25] fix typo in function argument --- metplus/wrappers/plot_point_obs_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index f179786dd7..b04d3adef3 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -207,7 +207,7 @@ def find_input_files(self, time_info): """ input_files = self.find_data(time_info, return_list=True, - mandatory=mandatory) + mandatory=True) if input_files is None: return None From 502bd6b3fe29ff7fbfba8cef63a09f51825438bd Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 14:15:08 -0600 Subject: [PATCH 08/25] per #1489, rearrange order of command line args so all input file arguments come first --- metplus/wrappers/plot_point_obs_wrapper.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index b04d3adef3..9a532ec218 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -236,17 +236,18 @@ def set_command_line_arguments(self, time_info): @param time_info dictionary containing timing information """ - config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) - self.args.append(f'-config {config_file}') - # if more than 1 input file was found, add them with -point_obs for infile in self.infiles[1:]: self.args.append(f'-point_obs {infile}') - if self.c_dict['TITLE']: - self.args.append(f"-title {self.c_dict['TITLE']}") - if self.c_dict['GRID_INPUT_PATH']: grid_file = do_string_sub(self.c_dict['GRID_INPUT_PATH'], **time_info) self.args.append(f'-plot_grid {grid_file}') + + config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) + self.args.append(f'-config {config_file}') + + if self.c_dict['TITLE']: + title = do_string_sub(self.c_dict['TITLE'], **time_info) + self.args.append(f'-title "{title}"') From 3de8a7566ec6494cc968cf5bc7aa8954d94eeb46 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 14:15:28 -0600 Subject: [PATCH 09/25] added other config variables needed to run example based on MET unit test --- .../PlotPointObs/PlotPointObs.conf | 102 +++++++++++++++++- 1 file changed, 97 insertions(+), 5 deletions(-) diff --git a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf index 9d13fcc2a7..cb4af29ad6 100644 --- a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf +++ b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf @@ -1,14 +1,81 @@ [config] -#PLOT_POINT_OBS_GRID_DATA_FIELD = -#PLOT_POINT_OBS_GRID_DATA_REGRID_TO_GRID = + +# Documentation for this use case can be found at +# https://metplus.readthedocs.io/en/latest/generated/met_tool_wrapper/PlotPointObs/PlotPointObs.html + +# For additional information, please see the METplus Users Guide. +# https://metplus.readthedocs.io/en/latest/Users_Guide + +### +# Processes to run +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list +### + +PROCESS_LIST = PlotPointObs + + +### +# Time Info +# LOOP_BY options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +# LEAD_SEQ is the list of forecast leads to process +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#timing-control +### + +PLOT_POINT_OBS_RUNTIME_FREQ = RUN_ONCE_FOR_EACH + +LOOP_BY = VALID +VALID_TIME_FMT = %Y%m%d%H +VALID_BEG = 2012040912 +VALID_END = 2012040912 +VALID_INCREMENT = 1M + +LEAD_SEQ = 12H + + +### +# File I/O +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info +### + +PLOT_POINT_OBS_INPUT_DIR = {INPUT_BASE}/met_test/test_out +PLOT_POINT_OBS_INPUT_TEMPLATE = + pb2nc/ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr.tm00.nc, + ascii2nc/trmm_{valid?fmt=%Y%m%d%H}_3hr.nc + +PLOT_POINT_OBS_GRID_INPUT_DIR = {INPUT_BASE}/met_test/new/model_data/grib2/nam +PLOT_POINT_OBS_GRID_INPUT_TEMPLATE = nam_{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}.grib2 + +PLOT_POINT_OBS_OUTPUT_DIR = {OUTPUT_BASE}/plot_point_obs +PLOT_POINT_OBS_OUTPUT_TEMPLATE = nam_and_ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr_CONFIG.ps + +#PLOT_POINT_OBS_SKIP_IF_OUTPUT_EXISTS = False + + +### +# PlotPointObs Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#plotpointobs +### + +PLOT_POINT_OBS_TITLE = NAM {init?fmt=%Y%m%d%H} F{lead?fmt=%2H} vs NDAS 500mb RH and TRMM 3h > 0 + +#PLOT_POINT_OBS_CONFIG_FILE = {PARM_BASE}/met_config/PlotPointObsConfig_wrapped + +#LOG_PLOT_POINT_OBS_VERBOSITY = 2 + +PLOT_POINT_OBS_GRID_DATA_FIELD = { name = "RH"; level = "P500"; } +PLOT_POINT_OBS_GRID_DATA_REGRID_TO_GRID = NONE #PLOT_POINT_OBS_GRID_DATA_REGRID_METHOD = #PLOT_POINT_OBS_GRID_DATA_REGRID_WIDTH = #PLOT_POINT_OBS_GRID_DATA_REGRID_VLD_THRESH = #PLOT_POINT_OBS_GRID_DATA_REGRID_SHAPE = -#PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLOR_TABLE = +PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLOR_TABLE = MET_BASE/colortables/NCL_colortables/BlueGreen14.ctable #PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MIN = -#PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MAX = +PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MAX = 100.0 #PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLORBAR_FLAG = #PLOT_POINT_OBS_MSG_TYP = @@ -38,4 +105,29 @@ #PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MAX = #PLOT_POINT_OBS_FILL_PLOT_INFO_COLORBAR_FLAG = -#PLOT_POINT_OBS_POINT_DATA = +PLOT_POINT_OBS_POINT_DATA = + { + msg_typ = "ADPSFC"; + obs_gc = 61; + obs_thresh = > 0.0; + fill_color = [0, 0, 255]; + }, + { + msg_typ = "ADPSFC"; + obs_var = "RH"; + fill_color = [100, 100, 100]; + }, + { + msg_typ = "ADPUPA"; + obs_var = "RH"; + prs_thresh = == 500; + dotsize(x) = 7.5; + line_color = [0, 0, 0]; + fill_plot_info = { + flag = TRUE; + color_table = "MET_BASE/colortables/NCL_colortables/BlueGreen14.ctable"; + plot_min = 0.0; + plot_max = 100.0; + colorbar_flag = FALSE; + } + } From 4071159c66890139ece75ed7edfa5b1f064c1cfa Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 14:45:05 -0600 Subject: [PATCH 10/25] per #1489, added documentation for new use case and added quick search keyword for new wrapper --- docs/Users_Guide/quicksearch.rst | 2 + .../PlotPointObs/PlotPointObs.py | 113 ++++++++++++++++++ .../met_tool_wrapper/PlotPointObs/README.rst | 2 + 3 files changed, 117 insertions(+) create mode 100644 docs/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.py create mode 100644 docs/use_cases/met_tool_wrapper/PlotPointObs/README.rst diff --git a/docs/Users_Guide/quicksearch.rst b/docs/Users_Guide/quicksearch.rst index 1c4070a668..64de40cb9b 100644 --- a/docs/Users_Guide/quicksearch.rst +++ b/docs/Users_Guide/quicksearch.rst @@ -27,6 +27,7 @@ Use Cases by MET Tool: | `PCPCombine <../search.html?q=PCPCombineToolUseCase&check_keywords=yes&area=default>`_ | `Point2Grid <../search.html?q=Point2GridToolUseCase&check_keywords=yes&area=default>`_ | `PlotDataPlane <../search.html?q=PlotDataPlaneToolUseCase&check_keywords=yes&area=default>`_ + | `PlotPointObs <../search.html?q=PlotPointObsToolUseCase&check_keywords=yes&area=default>`_ | `PointStat <../search.html?q=PointStatToolUseCase&check_keywords=yes&area=default>`_ | `RegridDataPlane <../search.html?q=RegridDataPlaneToolUseCase&check_keywords=yes&area=default>`_ | `SeriesAnalysis <../search.html?q=SeriesAnalysisUseCase&check_keywords=yes&area=default>`_ @@ -53,6 +54,7 @@ Use Cases by MET Tool: | **PCPCombine**: *PCPCombineToolUseCase* | **Point2Grid**: *Point2GridToolUseCase* | **PlotDataPlane**: *PlotDataPlaneToolUseCase* + | **PlotPointObs**: *PlotPointObsToolUseCase* | **PointStat**: *PointStatToolUseCase* | **RegridDataPlane**: *RegridDataPlaneToolUseCase* | **SeriesAnalysis**: *SeriesAnalysisUseCase* diff --git a/docs/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.py b/docs/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.py new file mode 100644 index 0000000000..322933601f --- /dev/null +++ b/docs/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.py @@ -0,0 +1,113 @@ +""" +PlotPointObs: Basic Use Case +========================== + +met_tool_wrapper/PlotPointObs/PlotPointObs.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# Generate a postscript image to plot point observations using a gridded +# file to define the domain to plot the points. + +############################################################################## +# Datasets +# -------- +# +# | **Point Observations:** NetCDF generated by PB2NC and ASCII2NC +# | **Grid:** GRIB2 NAM +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See 'Running METplus' section for more information. +# +# | **Data Source:** NAM and NDAS +# | + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes the METplus PlotPointObs wrapper to generate a +# command to run the MET tool plot_point_obs if all required files are found. + +############################################################################## +# METplus Workflow +# ---------------- +# +# PlotPointObs is the only tool called in this example. +# It processes the following run time: +# +# | **Valid:** 2012-04-09 12Z +# | **Forecast lead:** 12 hour +# | + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads the default configuration file found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line: +# parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf + +############################################################################## +# MET Configuration +# ----------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. note:: See the :ref:`PlotPointObs MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/PlotPointObsConfig_wrapped + + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in plot_point_obs +# (relative to **OUTPUT_BASE**) +# and will contain the following file: +# +# * nam_and_ndas.20120409.t12z.prepbufr_CONFIG.ps + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * PlotPointObsToolUseCase +# * GRIBFileUseCase +# * NetCDFFileUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# sphinx_gallery_thumbnail_path = '_static/met_tool_wrapper-PlotPointObs.png' diff --git a/docs/use_cases/met_tool_wrapper/PlotPointObs/README.rst b/docs/use_cases/met_tool_wrapper/PlotPointObs/README.rst new file mode 100644 index 0000000000..672cc5d895 --- /dev/null +++ b/docs/use_cases/met_tool_wrapper/PlotPointObs/README.rst @@ -0,0 +1,2 @@ +PlotPointObs +------------ From afd236fccac3d5fcaa7499e591abd0878d71f781 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 14:45:22 -0600 Subject: [PATCH 11/25] add new use case to automated tests --- .github/parm/use_case_groups.json | 5 +++++ internal_tests/use_cases/all_use_cases.txt | 1 + 2 files changed, 6 insertions(+) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 2a4da1ef67..bfa10cfae3 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -14,6 +14,11 @@ "index_list": "59-60", "run": false }, + { + "category": "met_tool_wrapper", + "index_list": "61", + "run": true + }, { "category": "air_quality_and_comp", "index_list": "0", diff --git a/internal_tests/use_cases/all_use_cases.txt b/internal_tests/use_cases/all_use_cases.txt index 0d20c7ce2b..bac039a77a 100644 --- a/internal_tests/use_cases/all_use_cases.txt +++ b/internal_tests/use_cases/all_use_cases.txt @@ -60,6 +60,7 @@ Category: met_tool_wrapper 58::GenEnsProd::met_tool_wrapper/GenEnsProd/GenEnsProd.conf 59::IODA2NC::met_tool_wrapper/IODA2NC/IODA2NC.conf 60::PointStat_python_embedding_obs:: met_tool_wrapper/PointStat/PointStat_python_embedding_obs.conf +61::PlotPointObs:: met_tool_wrapper/PlotPointObs/PlotPointObs.conf Category: air_quality_and_comp 0::EnsembleStat_fcstICAP_obsMODIS_aod::model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf From 425171bc68c2a1ad5e39db2eb9c39d111320d2d3 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 14:55:58 -0600 Subject: [PATCH 12/25] updated instructions for adding new use case to make it more clear when a step can be skipped --- docs/Contributors_Guide/add_use_case.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index cc884963eb..ce02dec09a 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -638,7 +638,8 @@ created and tested. Trigger Input Data Ingest ------------------------- -If working in the *dtcenter/METplus repository*, please skip this step. +**IF WORKING IN THE *dtcenter/METplus REPOSITORY*, PLEASE SKIP THIS STEP.** + If working in a forked METplus repository, the newly added input data will not become available for the tests unless it is triggered from the dtcenter repository. A METplus developer will need to run the following steps. Please From 2ce8f6735a45dd211743bb0818850bceef987e44 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 14:56:25 -0600 Subject: [PATCH 13/25] updated path to input file to under met_test/new --- parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf index cb4af29ad6..304e1c5253 100644 --- a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf +++ b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf @@ -42,7 +42,7 @@ LEAD_SEQ = 12H # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info ### -PLOT_POINT_OBS_INPUT_DIR = {INPUT_BASE}/met_test/test_out +PLOT_POINT_OBS_INPUT_DIR = {INPUT_BASE}/met_test/new/test_out PLOT_POINT_OBS_INPUT_TEMPLATE = pb2nc/ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr.tm00.nc, ascii2nc/trmm_{valid?fmt=%Y%m%d%H}_3hr.nc From 97c5e8a667ab4febed55345bc9d95e8f471a4865 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:06:51 -0600 Subject: [PATCH 14/25] per #1489, added image for new use case --- docs/_static/met_tool_wrapper-PlotPointObs.png | Bin 0 -> 4668 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/_static/met_tool_wrapper-PlotPointObs.png diff --git a/docs/_static/met_tool_wrapper-PlotPointObs.png b/docs/_static/met_tool_wrapper-PlotPointObs.png new file mode 100644 index 0000000000000000000000000000000000000000..72dc459b9117a9c70d5bd208d3dc27888e52b035 GIT binary patch literal 4668 zcmaJ_WmME%zx~fJLrV`L-5@Pe5;AnBfOK~^s5C=~G}00xh{Om3Qu5#oFqD*Zmmtz5 z4RU$zde?iu-E-DHd!O~&`>b{1(@D_PR=Gz=O9%kKJvCKD{X34lgDO7m-S|T#i|>v= ze)=l%0CI?a^KOE0FjjNa(gJwyG(G?YQ2;pqTJAakq6MJ;s{ueAME75<590oh2Mhpl zE&${|9@9Jiw`JYoF8v?IDF*%DipAjnx+9Bm{;O60joH)DcHI$yx2lOB0Nkhe7a*si z(Yq~>>!~Tq83coNErPrYu(0qT9wRR#$^rH>>fUrWXE|H0C1tFbOPBiZhj5CVK6ZuF zJ|;EhTpM9EG@Jsfo!|8!ihx+O5g$SJrZ%ar&L?e+XAWhrX7taMz<$%&ZqTc<(4aGm zv_1B4uaMcB#`cBKtJB@NP&Zb*0!Htb5zdFt&WwusZ5!?^E-x9M_<3e}v&tdxpg6P&b=8+17#yu#ui3BG} zZW6Cq42uJFxPa;+RN>5w{q8gVOJn>GyT2QqxM)Fa(F>+B!&pgAtxCLEqqtah*PmWY zls&P;NwnnI$pvgR@$ezvqT%N~k3*o69`y26?63V<;xj2C>HTkM_g?walLR+#GAu(1 z4Jeh=b}rIX6=}Andf7)a zDbXJ>aIhjXJkgV8Xs;HJUu}Fla%d^}XBAImevAo`!P}|zfD*;+^ztEWC|~Tfdsf{*`$8cbKeeUXKReWV^~@yIf`EXSQr9U8A;hD2vg^Mu31d-(? zXO;rEUte8;W>40>vl7IYRkmmKXAO#>y0&I+r-C+}$Hb0ai1J8>-i4QG^>o_Kv8J-4 zLE&vFfj6tLa3fE@iQW5ST|~dI&oX-{ky^jN7QFfUFUm3Wj8D55p7u-P?kw$|?yXNV zgW+2%$c)T1S&U>Uu>%{^Heej&nRP%p7T*-!yPtBrpC;avK3sbE3*o%1 zn~gDC#R@Wth#;9D{1J0+_7$1U^9LL*87)v-$a^^zza+L(bK>Dvw^L^Kv7vh>q_woN zCzUZGCzVPfhaD|z{r<;-3W|1Fjo(nY(wcy@x-FUv^O2uG!APxLU%43FeE3Ygew*LK zKq*7usResc@7|imuem|jTmkcZKFVXrdV*^{nQ$kSEn%`UP)KoTDrHu|rE{3Ze6~InNm~?PM5PaOnEN_8rBipz1m0eXO4hZ!)xP>=h(M8k zYt7A0x%qI9SnqU=RZhs6j@dN-NGIRbTQZI1;Nxq8l7uJn9t~_BW{PB_Ac!}?`TPJB z#RVHpeVWQQAv|}AOJRJDei-<)Jvla5XkrL*STH}Kef9TTjQ{##H?t3?ZGU)wq^^9o zTpyRje{_h;@`H)hw^6E(xQZQu)Y7_+`^Zjtc5OQD{bk zT*`8)?Zs*>hHYWI6Y}WhmxIV|zmndZ|0SEao+Vb1$_e^55T9ksH9adq~4!qi|ff!wyw#q%w|gc zqnSy!LW<9_I!cyC32z>mU#t~|BVH>}LFxvVO!%*V+wTw>^x6g{M9SIKEOQUjJ~zU> z7%{m-+MeF4q zqecYR1`8CAnObJP;Vetkjt<@KQoJ~F(kiP?mtN^%8j!?ync-8&L34&3F56V{YQiyX zzIi@Boe+B;ez^tA>)xhD9K^l4Y0b)+r7CA6L7r>}3=nisL>87z7GT5T^PysW?U?jB z&!1l--YO9>eogXat#uj~9{=HHkY$*uG2;O37N2(78$g@lgO5?xFrsO!*M|cjvB*Cy z91qm?IPoahWTdGdzjoS#MZ(gW#2+u~Y~-3eE4oVuSQzLpsl#c`jjDEkz)-m)JrZIpvIS$q}NJce2K z(EZ-P9##K*VWxuf^;8Njb4bJg_2 z>>{OwxlVE~4LFhax*6Yr-Kw^Cf= zPnD}mlSkJtsEMCJQkkXvyb{PGA%_k_v8$3PXXbmRB{xDK}|D)&ZSabna=Y^ z6k*AEI-b@im7RJxi0TF^1fhaLWmMpFmb)* zoh@7G*0%8>;IiR9@0rrLqUqKY;TM4XKCxo|!%Q|Q5`6(m@=R5uh_Apo2=)3HK`2>} zia5e02}*DPx}Y2sN5CpZ$A0{F8mrJq5M4z=)rjIcqbI*%6Nyn@S&ov39eOBeQ;?WR zGxhIIb4WikW$8h*m_ZhBNXLr^NiaWSx+nDGo*?l4}F4~&5* zUc!$Z-IsexOfRSn;2(2iEpXml6n?b&81_^L4SNie4(TDy*87@wra$yXIPa#WDh8UV zIfiMt@LqciX`<;7{!+?14&606<(qkV z765t2-cM|7N`hjk5dz!Tk<&CSGlH~n6xI`xZ^L+qZM^FVRMt(f7w*Rr?>Gy^xqdT# z;FHpt=L}FOUbTNnQ%R8S5LBF%Tfsyvg=ZDK)Eu8XY$Scm*p5L>*T|l0peG|+lm^(x;c$Hz%IgJd%JS}Lx={)Lh&iR~m=>*&_ok-S z-?%++?00y;K8Tj!Lv`@(_%c&QzeFpq1xYsN)EsdN;CfQbJ&BDzqRXOjligu_8Avb8 z5d%vFEUCTs0hM6@K^ky6`n1s(m)^W74L2o_2n_$n*i)FuuWJDy1FZ4PG+vzNh0@9? zf`LN4yGs~NB9rc)!=tI)N%ad_=41KVfTl4(V1&7;5H`eq1ie&#Os$2E^Dhg1h^D>9 zYR%>iy9U{9@H?Ssi$80Grswr@E*+T>4wMj0mj^IY9m4if3cwqs1W~OZlc#HcMqoQa z1@V3-L*HxS(G}OSm-V;?@k`pf#(KS$S-`~9&fW%;`=l4cYzyW>Czl4Cn2GFeDM92! zK0Bg>gbjDu-pDI2*(CxIM(2z5a9RZVb*yQ!E5(lxh7s18c)U-6M9lbAGyG}r0(BAV zvl5`b@nN-)e#uUON%tPOo-T?F8LS3yp+m7GS^TC`*ZG&T$qDPF*4iza>FQHrAfPWN z9(YiTwU)2AKc^yw1D|$QU~~g>SVu@ME%D20c|JcbWM+UfT3L#M1fjM3x^nZ!8IpKe z9M;mWw|$S>&z^v%&^Jmi?#+LL1W$W6W{Am-IOhykttVwM9p$$@vTqv7NdTbHWL*ks z0+&4Djhknl<9FVKcN2vQ>4cwF>BXFV|6MD4I}Cj&F8QJkx+Cb z8<%sBj%9I(tYK;h4+pY}~kz*kCWN=#uduvkN^5;*SrTe1DNvp#x zbGA``Q|A316U~7c)Va~|;qA_!Ikw*|C5+v3c)1r(23oH+yULBCE!Sfme=!%jx;yu zRV+!j3ZMhcne9{IypCkM&gNnp{)V|i1~cE52UCzL zGIlcN{Q0g_jC%BgM^VThIS-s6(rR}ChdiCI>o(nMVwA9YC@*jcYQqt-QBMn21aHQa zp{6vN)re*Bp%6S|iKSb%Lkc31<4M}bZFX$}AFv7je6*lw6aA%=9~)g%!x5LZZRLfy zZR+4Y1?=6UXT%Yy?!mb7Ko>l|sNE~BRU*1YVenlxE$LI>(_fq(D3@-GGFftV3!IxT zEyxaH49|w`As>HCusfBK|{B#+BX-SXeqfndnH+;7(Dr} zZWC!MH}YIhW`lFm1yxCEPm%9L<1Man$!OU;NMblBdBa~%GgI@djZn((&^~>6PHOA@ z#U>@VcNKj+A}`Eho?e{0aNGx)y=^+1{u`NO6M*^QC0eF;*rDF>RWtar z4?FOfsWOE3CY<#o37@q_S{jPXPwL#|5 zHuQOe!>QAGAo$CYVR(N6TnQHv=|xncuO*gs^H*-;R!VPdKBUOBa-mCUt81zq1;ak23U~aOUpYY(AV&q-*e@)mFy@0j?#%#AxYPe(x7G)B@L)ebg-> x{G)KE>U(0ds}W;3+N2UP%YXiVLH=s_mZa-^$Ipnt_3p<7P*c)YM9N!7{SV?gf0qCN literal 0 HcmV?d00001 From 13a5906f9213883644ad5158670ac66d8b2ee798 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:35:50 -0600 Subject: [PATCH 15/25] fixed underline that was too short --- docs/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.py b/docs/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.py index 322933601f..ce93b0f5dd 100644 --- a/docs/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.py +++ b/docs/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.py @@ -1,6 +1,6 @@ """ PlotPointObs: Basic Use Case -========================== +============================ met_tool_wrapper/PlotPointObs/PlotPointObs.conf From 4d0ba09f2e9445417e00889c85b2dd03943447bb Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:40:13 -0600 Subject: [PATCH 16/25] turn off use case since test succeeded --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index bfa10cfae3..959babcf92 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -17,7 +17,7 @@ { "category": "met_tool_wrapper", "index_list": "61", - "run": true + "run": false }, { "category": "air_quality_and_comp", From aba0824612189dc46b6958b1a54a0076cab1c6d2 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 15:42:43 -0600 Subject: [PATCH 17/25] unset convert(x) from wrapped config in case default value in the default config changes. users should override the convert function in the point_data list --- parm/met_config/PlotPointObsConfig_wrapped | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/met_config/PlotPointObsConfig_wrapped b/parm/met_config/PlotPointObsConfig_wrapped index 9454e95308..549aaffabb 100644 --- a/parm/met_config/PlotPointObsConfig_wrapped +++ b/parm/met_config/PlotPointObsConfig_wrapped @@ -61,7 +61,8 @@ ${METPLUS_OBS_THRESH} // Point data pre-processing options // May be set separately in each "point_data" entry -convert(x) = x; +//convert(x) = x; + //censor_thresh = ${METPLUS_CENSOR_THRESH} From 8ec110da6633d701ad1503a150e2b05a2e85f200 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 18 Aug 2022 17:03:16 -0600 Subject: [PATCH 18/25] per #1489, started writing pytests for new wrapper and fixed bug uncovered by test --- .../test_plot_point_obs_wrapper.py | 133 ++++++++++++++++++ metplus/wrappers/plot_point_obs_wrapper.py | 4 +- 2 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py diff --git a/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py b/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py new file mode 100644 index 0000000000..cc3efc71d9 --- /dev/null +++ b/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 + +import pytest + +import os + +from datetime import datetime + + +from metplus.wrappers.plot_point_obs_wrapper import PlotPointObsWrapper + +obs_dir = '/some/path/obs' +grid_dir = '/some/path/grid' + + +time_fmt = '%Y%m%d%H' +run_times = ['2012040912', '2012041000'] + + +def set_minimum_config_settings(config, set_fields=True): + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'PlotPointObs') + config.set('config', 'LOOP_BY', 'VALID') + config.set('config', 'VALID_TIME_FMT', time_fmt) + config.set('config', 'VALID_BEG', run_times[0]) + config.set('config', 'VALID_END', run_times[-1]) + config.set('config', 'VALID_INCREMENT', '12H') + config.set('config', 'LEAD_SEQ', '12H') + config.set('config', 'PLOT_POINT_OBS_INPUT_DIR', obs_dir) + config.set('config', 'PLOT_POINT_OBS_INPUT_TEMPLATE', + ('pb2nc/ndas.{valid?fmt=%Y%m%d}.' + 't{valid?fmt=%H}z.prepbufr.tm00.nc')) + #config.set('config', 'PLOT_POINT_OBS_GRID_INPUT_DIR', grid_dir) + #config.set('config', 'PLOT_POINT_OBS_GRID_INPUT_TEMPLATE', + # 'nam_{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}.grib2') + config.set('config', 'PLOT_POINT_OBS_OUTPUT_DIR', + '{OUTPUT_BASE}/plot_point_obs') + config.set('config', 'PLOT_POINT_OBS_OUTPUT_TEMPLATE', + 'nam_and_ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr_CONFIG.ps') + + +@pytest.mark.parametrize( + 'config_overrides, env_var_values', [ + ({'PLOT_POINT_OBS_INPUT_TEMPLATE': ('pb2nc/ndas.{valid?fmt=%Y%m%d}.' + 't{valid?fmt=%H}z.prepbufr.tm00.nc' + ',ascii2nc/trmm_' + '{valid?fmt=%Y%m%d%H}_3hr.nc'), }, + {}), + + ] +) +@pytest.mark.wrapper_c +def test_ensemble_stat_single_field(metplus_config, config_overrides, + env_var_values): + + config = metplus_config() + + set_minimum_config_settings(config) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = PlotPointObsWrapper(config) + assert wrapper.isOK + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + input_dir = wrapper.c_dict.get('INPUT_DIR') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + expected_cmds = [ + (f"{app_path} {verbosity} " + f"{input_dir}/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc " + f"{out_dir}/nam_and_ndas.20120409.t12z.prepbufr_CONFIG.ps"), + (f"{app_path} {verbosity} " + f"{input_dir}/pb2nc/ndas.20120410.t00z.prepbufr.tm00.nc " + f"{out_dir}/nam_and_ndas.20120410.t00z.prepbufr_CONFIG.ps"), + ] + + # add -point_obs argument if template has 2 items + if ('PLOT_POINT_OBS_INPUT_TEMPLATE' in config_overrides and + len(config_overrides['PLOT_POINT_OBS_INPUT_TEMPLATE'].split(',')) > 1): + common_str = f' -point_obs {input_dir}/ascii2nc/trmm_' + expected_cmds[0] += f'{common_str}2012040912_3hr.nc' + expected_cmds[1] += f'{common_str}2012041000_3hr.nc' + + # add -config argument + expected_cmds = [f'{item} -config {config_file}' for item in expected_cmds] + + # add -title if set + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + assert len(all_cmds) == len(expected_cmds) + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert(cmd == expected_cmd) + + # check that environment variables were set properly + # including deprecated env vars (not in wrapper env var keys) + env_var_keys = (wrapper.WRAPPER_ENV_VAR_KEYS + + [name for name in env_var_values + if name not in wrapper.WRAPPER_ENV_VAR_KEYS]) + for env_var_key in env_var_keys: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert(match is not None) + actual_value = match.split('=', 1)[1] + assert(env_var_values.get(env_var_key, '') == actual_value) + + +@pytest.mark.wrapper_c +def test_get_config_file(metplus_config): + fake_config_name = '/my/config/file' + + config = metplus_config() + default_config_file = os.path.join(config.getdir('PARM_BASE'), + 'met_config', + 'PlotPointObsConfig_wrapped') + + wrapper = PlotPointObsWrapper(config) + assert wrapper.c_dict['CONFIG_FILE'] == default_config_file + + config.set('config', 'PLOT_POINT_OBS_CONFIG_FILE', fake_config_name) + wrapper = PlotPointObsWrapper(config) + assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index 9a532ec218..b785660aac 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -240,7 +240,7 @@ def set_command_line_arguments(self, time_info): for infile in self.infiles[1:]: self.args.append(f'-point_obs {infile}') - if self.c_dict['GRID_INPUT_PATH']: + if self.c_dict.get('GRID_INPUT_PATH'): grid_file = do_string_sub(self.c_dict['GRID_INPUT_PATH'], **time_info) self.args.append(f'-plot_grid {grid_file}') @@ -248,6 +248,6 @@ def set_command_line_arguments(self, time_info): config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) self.args.append(f'-config {config_file}') - if self.c_dict['TITLE']: + if self.c_dict.get('TITLE'): title = do_string_sub(self.c_dict['TITLE'], **time_info) self.args.append(f'-title "{title}"') From 87065f60ee21d058cf940b99f4bbb2e6176f0b67 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 19 Aug 2022 12:25:44 -0600 Subject: [PATCH 19/25] per #1489, added unit tests for setting env vars read by the PlotPointObs wrapped config file and made adjustments to wrapper to produce the correct results --- .../test_plot_point_obs_wrapper.py | 199 ++++++++++++++++-- metplus/wrappers/plot_point_obs_wrapper.py | 15 +- 2 files changed, 195 insertions(+), 19 deletions(-) diff --git a/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py b/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py index cc3efc71d9..a7ef9fd541 100644 --- a/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py +++ b/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py @@ -10,8 +10,27 @@ from metplus.wrappers.plot_point_obs_wrapper import PlotPointObsWrapper obs_dir = '/some/path/obs' +input_template_one = ( + 'pb2nc/ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr.tm00.nc' +) +input_template_two = ( + 'pb2nc/ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr.tm00.nc,' + 'ascii2nc/trmm_{valid?fmt=%Y%m%d%H}_3hr.nc' +) grid_dir = '/some/path/grid' +grid_template = 'nam_{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}.grib2' + +output_dir = '{OUTPUT_BASE}/plot_point_obs' +output_template = 'nam_and_ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr_CONFIG.ps' +title = 'NAM 2012040900 F12 vs NDAS 500mb RH and TRMM 3h > 0' + +point_data = ['{msg_typ = "ADPSFC";obs_gc = 61;obs_thresh = > 0.0;' + 'fill_color = [0,0,255];}', + '{msg_typ = "ADPSFC";obs_var = "RH";' + 'fill_color = [100,100,100];}'] +point_data_input = ', '.join(point_data) +point_data_format = f"[{point_data_input}];" time_fmt = '%Y%m%d%H' run_times = ['2012040912', '2012041000'] @@ -32,31 +51,171 @@ def set_minimum_config_settings(config, set_fields=True): config.set('config', 'VALID_INCREMENT', '12H') config.set('config', 'LEAD_SEQ', '12H') config.set('config', 'PLOT_POINT_OBS_INPUT_DIR', obs_dir) - config.set('config', 'PLOT_POINT_OBS_INPUT_TEMPLATE', - ('pb2nc/ndas.{valid?fmt=%Y%m%d}.' - 't{valid?fmt=%H}z.prepbufr.tm00.nc')) - #config.set('config', 'PLOT_POINT_OBS_GRID_INPUT_DIR', grid_dir) - #config.set('config', 'PLOT_POINT_OBS_GRID_INPUT_TEMPLATE', - # 'nam_{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}.grib2') - config.set('config', 'PLOT_POINT_OBS_OUTPUT_DIR', - '{OUTPUT_BASE}/plot_point_obs') - config.set('config', 'PLOT_POINT_OBS_OUTPUT_TEMPLATE', - 'nam_and_ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr_CONFIG.ps') + config.set('config', 'PLOT_POINT_OBS_INPUT_TEMPLATE', input_template_one) + config.set('config', 'PLOT_POINT_OBS_OUTPUT_DIR', output_dir) + config.set('config', 'PLOT_POINT_OBS_OUTPUT_TEMPLATE', output_template) @pytest.mark.parametrize( 'config_overrides, env_var_values', [ - ({'PLOT_POINT_OBS_INPUT_TEMPLATE': ('pb2nc/ndas.{valid?fmt=%Y%m%d}.' - 't{valid?fmt=%H}z.prepbufr.tm00.nc' - ',ascii2nc/trmm_' - '{valid?fmt=%Y%m%d%H}_3hr.nc'), }, + # 0: no additional settings + ({},{}), + # 1: input template with 2 paths + ({'PLOT_POINT_OBS_INPUT_TEMPLATE': input_template_two}, + {}), + # 2: grid input + ({'PLOT_POINT_OBS_GRID_INPUT_TEMPLATE': grid_template, + 'PLOT_POINT_OBS_GRID_INPUT_DIR': grid_dir}, {}), + # 3: input template with 2 paths and grid input + ({'PLOT_POINT_OBS_INPUT_TEMPLATE': input_template_two, + 'PLOT_POINT_OBS_GRID_INPUT_TEMPLATE': grid_template, + 'PLOT_POINT_OBS_GRID_INPUT_DIR': grid_dir}, + {}), + # 4: title + ({'PLOT_POINT_OBS_TITLE': title}, + {}), + # 5: input template with 2 paths and grid input and title + ({'PLOT_POINT_OBS_INPUT_TEMPLATE': input_template_two, + 'PLOT_POINT_OBS_GRID_INPUT_TEMPLATE': grid_template, + 'PLOT_POINT_OBS_GRID_INPUT_DIR': grid_dir, + 'PLOT_POINT_OBS_TITLE': title}, + {}), + # 6: grid_data.field + ({'PLOT_POINT_OBS_GRID_DATA_FIELD': '{ name = "RH"; level = "P500"; }'}, + {'METPLUS_GRID_DATA_DICT': 'grid_data = {field = [{ name = "RH"; level = "P500"; }];}'}), + # 7: grid_data.regrid.to_grid + ({'PLOT_POINT_OBS_GRID_DATA_REGRID_TO_GRID': 'FCST', }, + {'METPLUS_GRID_DATA_DICT': 'grid_data = {regrid = {to_grid = FCST;}}'}), + # 8: grid_data.regrid.method + ({'PLOT_POINT_OBS_GRID_DATA_REGRID_METHOD': 'NEAREST', }, + {'METPLUS_GRID_DATA_DICT': 'grid_data = {regrid = {method = NEAREST;}}'}), + # 9: grid_data.regrid.width + ({'PLOT_POINT_OBS_GRID_DATA_REGRID_WIDTH': '2', }, + {'METPLUS_GRID_DATA_DICT': 'grid_data = {regrid = {width = 2;}}'}), + # 10: grid_data.regrid.vld_thresh + ({'PLOT_POINT_OBS_GRID_DATA_REGRID_VLD_THRESH': '0.4', }, + {'METPLUS_GRID_DATA_DICT': 'grid_data = {regrid = {vld_thresh = 0.4;}}'}), + # 11: grid_data.regrid.shape + ({'PLOT_POINT_OBS_GRID_DATA_REGRID_SHAPE': 'SQUARE', }, + {'METPLUS_GRID_DATA_DICT': 'grid_data = {regrid = {shape = SQUARE;}}'}), + # 12: grid_data.plot_info.color_table + ({'PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLOR_TABLE': 'MET_BASE/colortables/met_default.ctable', }, + {'METPLUS_GRID_DATA_DICT': 'grid_data = {grid_plot_info = {color_table = "MET_BASE/colortables/met_default.ctable";}}'}), + # 13: grid_data.plot_info.plot_min + ({'PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MIN': '0.1', }, + {'METPLUS_GRID_DATA_DICT': 'grid_data = {grid_plot_info = {plot_min = 0.1;}}'}), + # 14: grid_data.plot_info.plot_max + ({'PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MAX': '100.0', }, + {'METPLUS_GRID_DATA_DICT': 'grid_data = {grid_plot_info = {plot_max = 100.0;}}'}), + # 15: grid_data.plot_info.colorbar_flag + ({'PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLORBAR_FLAG': 'false', }, + {'METPLUS_GRID_DATA_DICT': 'grid_data = {grid_plot_info = {colorbar_flag = FALSE;}}'}), + # 16: grid_data all + ({'PLOT_POINT_OBS_GRID_DATA_FIELD': '{ name = "RH"; level = "P500"; }', + 'PLOT_POINT_OBS_GRID_DATA_REGRID_TO_GRID': 'FCST', + 'PLOT_POINT_OBS_GRID_DATA_REGRID_METHOD': 'NEAREST', + 'PLOT_POINT_OBS_GRID_DATA_REGRID_WIDTH': '2', + 'PLOT_POINT_OBS_GRID_DATA_REGRID_VLD_THRESH': '0.4', + 'PLOT_POINT_OBS_GRID_DATA_REGRID_SHAPE': 'SQUARE', + 'PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLOR_TABLE': 'MET_BASE/colortables/met_default.ctable', + 'PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MIN': '0.1', + 'PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MAX': '100.0', + 'PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_COLORBAR_FLAG': 'False', + }, + {'METPLUS_GRID_DATA_DICT': 'grid_data = {field = [{ name = "RH"; level = "P500"; }];regrid = {to_grid = FCST;method = NEAREST;width = 2;vld_thresh = 0.4;shape = SQUARE;}grid_plot_info = {color_table = "MET_BASE/colortables/met_default.ctable";plot_min = 0.1;plot_max = 100.0;colorbar_flag = FALSE;}}'}), + # 17: msg_type + ({'PLOT_POINT_OBS_MSG_TYP': 'ADPUPA, AIRCFT', }, + {'METPLUS_MSG_TYP': 'msg_typ = ["ADPUPA", "AIRCFT"];'}), + # 18: sid_inc + ({'PLOT_POINT_OBS_SID_INC': '72364,72265', }, + {'METPLUS_SID_INC': 'sid_inc = ["72364", "72265"];'}), + # 19: sid_exc + ({'PLOT_POINT_OBS_SID_EXC': '72274,72426', }, + {'METPLUS_SID_EXC': 'sid_exc = ["72274", "72426"];'}), + # 20: obs_var + ({'PLOT_POINT_OBS_OBS_VAR': 'SPFH,TMP', }, + {'METPLUS_OBS_VAR': 'obs_var = ["SPFH", "TMP"];'}), + # 21: obs_gc + ({'PLOT_POINT_OBS_OBS_GC': '2,1', }, + {'METPLUS_OBS_GC': 'obs_gc = [2, 1];'}), + # 22: obs_quality + ({'PLOT_POINT_OBS_OBS_QUALITY': '2,1', }, + {'METPLUS_OBS_QUALITY': 'obs_quality = ["2", "1"];'}), + # 23: valid_beg + ({'PLOT_POINT_OBS_VALID_BEG': '20101231_00', }, + {'METPLUS_VALID_BEG': 'valid_beg = "20101231_00";'}), + # 24: valid_end + ({'PLOT_POINT_OBS_VALID_END': '20101231_12', }, + {'METPLUS_VALID_END': 'valid_end = "20101231_12";'}), + # 25: lat_thresh + ({'PLOT_POINT_OBS_LAT_THRESH': '>1.0', }, + {'METPLUS_LAT_THRESH': 'lat_thresh = >1.0;'}), + # 26: lon_thresh + ({'PLOT_POINT_OBS_LON_THRESH': '>1.0', }, + {'METPLUS_LON_THRESH': 'lon_thresh = >1.0;'}), + # 27: elv_thresh + ({'PLOT_POINT_OBS_ELV_THRESH': '>1.0', }, + {'METPLUS_ELV_THRESH': 'elv_thresh = >1.0;'}), + # 28: hgt_thresh + ({'PLOT_POINT_OBS_HGT_THRESH': '>1.0', }, + {'METPLUS_HGT_THRESH': 'hgt_thresh = >1.0;'}), + # 29: prs_thresh + ({'PLOT_POINT_OBS_PRS_THRESH': '>1.0', }, + {'METPLUS_PRS_THRESH': 'prs_thresh = >1.0;'}), + # 30: obs_thresh + ({'PLOT_POINT_OBS_OBS_THRESH': '>1.0', }, + {'METPLUS_OBS_THRESH': 'obs_thresh = >1.0;'}), + # 31: censor_thresh + ({'PLOT_POINT_OBS_CENSOR_THRESH': '>12000', }, + {'METPLUS_CENSOR_THRESH': 'censor_thresh = [>12000];'}), + # 32: censor_val + ({'PLOT_POINT_OBS_CENSOR_VAL': '>12000', }, + {'METPLUS_CENSOR_VAL': 'censor_val = [>12000];'}), + # 33: dotsize + ({'PLOT_POINT_OBS_DOTSIZE': '10.0', }, + {'METPLUS_DOTSIZE': 'dotsize(x) = 10.0;'}), + # 34: line_color + ({'PLOT_POINT_OBS_LINE_COLOR': '100,105,110', }, + {'METPLUS_LINE_COLOR': 'line_color = [100, 105, 110];'}), + # 35: line_width + ({'PLOT_POINT_OBS_LINE_WIDTH': '4', }, + {'METPLUS_LINE_WIDTH': 'line_width = 4;'}), + # 36: fill_color + ({'PLOT_POINT_OBS_FILL_COLOR': '0,10,15', }, + {'METPLUS_FILL_COLOR': 'fill_color = [0, 10, 15];'}), + # 37: fill_plot_info.flag + ({'PLOT_POINT_OBS_FILL_PLOT_INFO_FLAG': 'true', }, + {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {flag = TRUE;}'}), + # 38: fill_plot_info.color_table + ({'PLOT_POINT_OBS_FILL_PLOT_INFO_COLOR_TABLE': 'MET_BASE/colortables/met_default.ctable', }, + {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {color_table = "MET_BASE/colortables/met_default.ctable";}'}), + # 39: fill_plot_info.plot_min + ({'PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MIN': '0.1', }, + {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {plot_min = 0.1;}'}), + # 40: fill_plot_info.plot_max + ({'PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MAX': '100.0', }, + {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {plot_max = 100.0;}'}), + # 41: fill_plot_info.colorbar_flag + ({'PLOT_POINT_OBS_FILL_PLOT_INFO_COLORBAR_FLAG': 'False', }, + {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {colorbar_flag = FALSE;}'}), + # 42: fill_plot_info all + ({ + 'PLOT_POINT_OBS_FILL_PLOT_INFO_FLAG': 'true', + 'PLOT_POINT_OBS_FILL_PLOT_INFO_COLOR_TABLE': 'MET_BASE/colortables/met_default.ctable', + 'PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MIN': '0.1', + 'PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MAX': '100.0', + 'PLOT_POINT_OBS_FILL_PLOT_INFO_COLORBAR_FLAG': 'false', + }, + {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {flag = TRUE;color_table = "MET_BASE/colortables/met_default.ctable";plot_min = 0.1;plot_max = 100.0;colorbar_flag = FALSE;}'}), + # 43: point_data + ({'PLOT_POINT_OBS_POINT_DATA': point_data_input, }, + {'METPLUS_POINT_DATA': f'point_data = {point_data_format}'}), ] ) @pytest.mark.wrapper_c -def test_ensemble_stat_single_field(metplus_config, config_overrides, - env_var_values): +def test_plot_point_obs(metplus_config, config_overrides, env_var_values): config = metplus_config() @@ -90,10 +249,18 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides, expected_cmds[0] += f'{common_str}2012040912_3hr.nc' expected_cmds[1] += f'{common_str}2012041000_3hr.nc' + # add -plot_grid argument if provided + if 'PLOT_POINT_OBS_GRID_INPUT_TEMPLATE' in config_overrides: + common_str = f' -plot_grid {grid_dir}/nam_' + expected_cmds[0] += f'{common_str}2012040900_F012.grib2' + expected_cmds[1] += f'{common_str}2012040912_F012.grib2' + # add -config argument expected_cmds = [f'{item} -config {config_file}' for item in expected_cmds] # add -title if set + if 'PLOT_POINT_OBS_TITLE' in config_overrides: + expected_cmds = [f'{item} -title "{title}"' for item in expected_cmds] all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index b785660aac..3648003c87 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -116,20 +116,28 @@ def create_c_dict(self): }), }) + # string lists config_lists = [ 'msg_typ', 'sid_inc', 'sid_exc', 'obs_var', - 'obs_gc', 'obs_quality', + ] + for config_list in config_lists: + self.add_met_config(name=config_list, data_type='list') + + # integer and thresh lists + config_lists = [ + 'obs_gc', 'censor_thresh', 'censor_val', 'line_color', 'fill_color', ] for config_list in config_lists: - self.add_met_config(name=config_list, data_type='list') + self.add_met_config(name=config_list, data_type='list', + extra_args={'remove_quotes': True}) config_strings = [ 'valid_beg', @@ -152,7 +160,8 @@ def create_c_dict(self): self.add_met_config(name='dotsize(x)', data_type='string', extra_args={'remove_quotes': True}, - metplus_configs=[f'{app}_DOTSIZE']) + metplus_configs=[f'{app}_DOTSIZE'], + env_var_name='METPLUS_DOTSIZE') self.add_met_config(name='line_width', data_type='int') self.add_met_config_dict('fill_plot_info', { From 835321775347c76d6438b2660056ff56bc47b34e Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:02:27 -0600 Subject: [PATCH 20/25] added support for MET config overrides even though all of the plot_point_obs settings are supported via METplus in case additional settings are added to the config and aren't immediately added to the wrapper --- docs/Users_Guide/glossary.rst | 14 ++++++++++++++ docs/Users_Guide/wrappers.rst | 1 + parm/met_config/PlotPointObsConfig_wrapped | 2 ++ 3 files changed, 17 insertions(+) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 144efc9e9b..052c4f30d9 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -9894,3 +9894,17 @@ METplus Configuration Glossary Frequency to run PlotPointObs. See :ref:`Runtime_Freq` for more information. | *Used by:* PlotPointObs + + PLOT_POINT_OBS_MET_CONFIG_OVERRIDES + Override any variables in the MET configuration file that are not + supported by the wrapper. This should be set to the full variable name + and value that you want to override, including the equal sign and the + ending semi-colon. The value is directly appended to the end of the + wrapped MET config file. + + Example: + PLOT_POINT_OBS_MET_CONFIG_OVERRIDES = desc = "override_desc"; model = "override_model"; + + See :ref:`Overriding Unsupported MET config file settings` for more information + + | *Used by:* PlotPointObs diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index a668c9d1d4..3a1b7c0af7 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -5430,6 +5430,7 @@ Configuration | :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MAX` | :term:`PLOT_POINT_OBS_FILL_PLOT_INFO_COLORBAR_FLAG` | :term:`PLOT_POINT_OBS_POINT_DATA` +| :term:`PLOT_POINT_OBS_MET_CONFIG_OVERRIDES` .. _plot-point-obs-met-conf: diff --git a/parm/met_config/PlotPointObsConfig_wrapped b/parm/met_config/PlotPointObsConfig_wrapped index 549aaffabb..fdab1fcf4f 100644 --- a/parm/met_config/PlotPointObsConfig_wrapped +++ b/parm/met_config/PlotPointObsConfig_wrapped @@ -93,3 +93,5 @@ ${METPLUS_FILL_PLOT_INFO_DICT} ${METPLUS_POINT_DATA} //////////////////////////////////////////////////////////////////////////////// + +${METPLUS_MET_CONFIG_OVERRIDES} From e26591ce3dc6a0c4b298125debecda730696200f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:03:35 -0600 Subject: [PATCH 21/25] added convert(x) = x back to wrapped config file because MET throws an error when a config file passed to it does not contain anything. This can be removed if MET is updated to handle an empty config --- parm/met_config/PlotPointObsConfig_wrapped | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/met_config/PlotPointObsConfig_wrapped b/parm/met_config/PlotPointObsConfig_wrapped index fdab1fcf4f..9fb8fa7038 100644 --- a/parm/met_config/PlotPointObsConfig_wrapped +++ b/parm/met_config/PlotPointObsConfig_wrapped @@ -61,7 +61,7 @@ ${METPLUS_OBS_THRESH} // Point data pre-processing options // May be set separately in each "point_data" entry -//convert(x) = x; +convert(x) = x; //censor_thresh = ${METPLUS_CENSOR_THRESH} From 4c00d454a9f7ea55f1409388aff7574d5e7bef6c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:05:01 -0600 Subject: [PATCH 22/25] changed test to include both censor_thresh and censor_val since they must be set together (have the same length). Updated example value for censor_val to be a number instead of a threshold, added a check that the wrapper did not report any errors after running --- .../test_plot_point_obs_wrapper.py | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py b/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py index a7ef9fd541..3e779d81a3 100644 --- a/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py +++ b/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py @@ -17,6 +17,7 @@ 'pb2nc/ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr.tm00.nc,' 'ascii2nc/trmm_{valid?fmt=%Y%m%d%H}_3hr.nc' ) + grid_dir = '/some/path/grid' grid_template = 'nam_{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}.grib2' @@ -36,7 +37,7 @@ run_times = ['2012040912', '2012041000'] -def set_minimum_config_settings(config, set_fields=True): +def set_minimum_config_settings(config): # set config variables to prevent command from running and bypass check # if input files actually exist config.set('config', 'DO_NOT_RUN_EXE', True) @@ -166,40 +167,39 @@ def set_minimum_config_settings(config, set_fields=True): # 30: obs_thresh ({'PLOT_POINT_OBS_OBS_THRESH': '>1.0', }, {'METPLUS_OBS_THRESH': 'obs_thresh = >1.0;'}), - # 31: censor_thresh - ({'PLOT_POINT_OBS_CENSOR_THRESH': '>12000', }, - {'METPLUS_CENSOR_THRESH': 'censor_thresh = [>12000];'}), - # 32: censor_val - ({'PLOT_POINT_OBS_CENSOR_VAL': '>12000', }, - {'METPLUS_CENSOR_VAL': 'censor_val = [>12000];'}), - # 33: dotsize + # 31: censor_thresh and censor_val + ({'PLOT_POINT_OBS_CENSOR_THRESH': '>12000', + 'PLOT_POINT_OBS_CENSOR_VAL': '12000'}, + {'METPLUS_CENSOR_THRESH': 'censor_thresh = [>12000];', + 'METPLUS_CENSOR_VAL': 'censor_val = [12000];'}), + # 32: dotsize ({'PLOT_POINT_OBS_DOTSIZE': '10.0', }, {'METPLUS_DOTSIZE': 'dotsize(x) = 10.0;'}), - # 34: line_color + # 33: line_color ({'PLOT_POINT_OBS_LINE_COLOR': '100,105,110', }, {'METPLUS_LINE_COLOR': 'line_color = [100, 105, 110];'}), - # 35: line_width + # 34: line_width ({'PLOT_POINT_OBS_LINE_WIDTH': '4', }, {'METPLUS_LINE_WIDTH': 'line_width = 4;'}), - # 36: fill_color + # 35: fill_color ({'PLOT_POINT_OBS_FILL_COLOR': '0,10,15', }, {'METPLUS_FILL_COLOR': 'fill_color = [0, 10, 15];'}), - # 37: fill_plot_info.flag + # 36: fill_plot_info.flag ({'PLOT_POINT_OBS_FILL_PLOT_INFO_FLAG': 'true', }, {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {flag = TRUE;}'}), - # 38: fill_plot_info.color_table + # 37: fill_plot_info.color_table ({'PLOT_POINT_OBS_FILL_PLOT_INFO_COLOR_TABLE': 'MET_BASE/colortables/met_default.ctable', }, {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {color_table = "MET_BASE/colortables/met_default.ctable";}'}), - # 39: fill_plot_info.plot_min + # 38: fill_plot_info.plot_min ({'PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MIN': '0.1', }, {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {plot_min = 0.1;}'}), - # 40: fill_plot_info.plot_max + # 39: fill_plot_info.plot_max ({'PLOT_POINT_OBS_FILL_PLOT_INFO_PLOT_MAX': '100.0', }, {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {plot_max = 100.0;}'}), - # 41: fill_plot_info.colorbar_flag + # 40: fill_plot_info.colorbar_flag ({'PLOT_POINT_OBS_FILL_PLOT_INFO_COLORBAR_FLAG': 'False', }, {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {colorbar_flag = FALSE;}'}), - # 42: fill_plot_info all + # 41: fill_plot_info all ({ 'PLOT_POINT_OBS_FILL_PLOT_INFO_FLAG': 'true', 'PLOT_POINT_OBS_FILL_PLOT_INFO_COLOR_TABLE': 'MET_BASE/colortables/met_default.ctable', @@ -208,7 +208,7 @@ def set_minimum_config_settings(config, set_fields=True): 'PLOT_POINT_OBS_FILL_PLOT_INFO_COLORBAR_FLAG': 'false', }, {'METPLUS_FILL_PLOT_INFO_DICT': 'fill_plot_info = {flag = TRUE;color_table = "MET_BASE/colortables/met_default.ctable";plot_min = 0.1;plot_max = 100.0;colorbar_flag = FALSE;}'}), - # 43: point_data + # 42: point_data ({'PLOT_POINT_OBS_POINT_DATA': point_data_input, }, {'METPLUS_POINT_DATA': f'point_data = {point_data_format}'}), @@ -265,10 +265,11 @@ def test_plot_point_obs(metplus_config, config_overrides, env_var_values): all_cmds = wrapper.run_all_times() print(f"ALL COMMANDS: {all_cmds}") assert len(all_cmds) == len(expected_cmds) + assert not wrapper.errors for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected - assert(cmd == expected_cmd) + assert cmd == expected_cmd # check that environment variables were set properly # including deprecated env vars (not in wrapper env var keys) @@ -278,9 +279,9 @@ def test_plot_point_obs(metplus_config, config_overrides, env_var_values): for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) - assert(match is not None) + assert match is not None actual_value = match.split('=', 1)[1] - assert(env_var_values.get(env_var_key, '') == actual_value) + assert env_var_values.get(env_var_key, '') == actual_value @pytest.mark.wrapper_c From 701400bc2674589e86598eb1a0ef71e6c28a1f8f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:17:37 -0600 Subject: [PATCH 23/25] removed convert and added tmp_dir to be able to override tmp dir location and prevent empty config file from crashing MET --- parm/met_config/PlotPointObsConfig_wrapped | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parm/met_config/PlotPointObsConfig_wrapped b/parm/met_config/PlotPointObsConfig_wrapped index 9fb8fa7038..2194f01fee 100644 --- a/parm/met_config/PlotPointObsConfig_wrapped +++ b/parm/met_config/PlotPointObsConfig_wrapped @@ -61,7 +61,7 @@ ${METPLUS_OBS_THRESH} // Point data pre-processing options // May be set separately in each "point_data" entry -convert(x) = x; +//convert(x) = x; //censor_thresh = ${METPLUS_CENSOR_THRESH} @@ -92,6 +92,8 @@ ${METPLUS_FILL_PLOT_INFO_DICT} //point_data = ${METPLUS_POINT_DATA} +tmp_dir = "${MET_TMP_DIR}"; + //////////////////////////////////////////////////////////////////////////////// ${METPLUS_MET_CONFIG_OVERRIDES} From a850cd0aad3cdfbb9bb3a7d1812a3a2bd1174487 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 22 Aug 2022 15:37:10 -0600 Subject: [PATCH 24/25] fixed command in contrib guide instructions that did not work as expected --- docs/Contributors_Guide/add_use_case.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index ce02dec09a..1651d3aa40 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -953,7 +953,7 @@ Copy data from the feature directory into the next version directory **CONDITION 2:** MET Tool Wrapper Use Cases:: - from_directory=${METPLUS_DATA_TARFILE_DIR}/${METPLUS_FEATURE_BRANCH}/met_test + from_directory=${METPLUS_DATA_TARFILE_DIR}/${METPLUS_FEATURE_BRANCH}/met_test/new echo $from_directory ls $from_directory From c7d6b8e1832a4b99c26c2ba0bdc66ca346c6e30f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 22 Aug 2022 15:38:02 -0600 Subject: [PATCH 25/25] rearrange use case groups to reduce the number of met_tool_wrapper use case groups --- .github/parm/use_case_groups.json | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 959babcf92..5da16d779e 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -1,7 +1,7 @@ [ { "category": "met_tool_wrapper", - "index_list": "0-29", + "index_list": "0-29,59-61", "run": false }, { @@ -9,16 +9,6 @@ "index_list": "30-58", "run": false }, - { - "category": "met_tool_wrapper", - "index_list": "59-60", - "run": false - }, - { - "category": "met_tool_wrapper", - "index_list": "61", - "run": false - }, { "category": "air_quality_and_comp", "index_list": "0",