From bf5d9d05b7e299f77cbe8c36b591ad21ccf8a3d4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 17 Nov 2023 17:40:55 -0700 Subject: [PATCH] Update develop-ref after #2427 (#2434) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Dan Adriaansen Co-authored-by: John Halley Gotway Co-authored-by: jprestop Co-authored-by: Tracy Hertneky Co-authored-by: Giovanni Rosa Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: Mrinal Biswas Co-authored-by: j-opatz Co-authored-by: Daniel Adriaansen Co-authored-by: Jonathan Vigh Co-authored-by: root Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: Julie Prestopnik Co-authored-by: Tracy Co-authored-by: johnhg Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: metplus-bot <97135045+metplus-bot@users.noreply.github.com> Co-authored-by: Lisa Goodrich Co-authored-by: Tracy Hertneky <39317287+hertneky@users.noreply.github.com> Co-authored-by: Giovanni Rosa Co-authored-by: mrinalbiswas Co-authored-by: Christina Kalb Co-authored-by: jason-english <73247785+jason-english@users.noreply.github.com> Co-authored-by: John Sharples <41682323+John-Sharples@users.noreply.github.com> Co-authored-by: Hank Fisher Co-authored-by: reza-armuei <144857501+reza-armuei@users.noreply.github.com> Closes https://github.com/dtcenter/METplus/issues/1986 fix develop Fix broken documentation links (#2004) fix #2026 develop StatAnalysis looping (#2028) fix priority of obs_window config variables so that wrapper-specific version is preferred over generic OBS_WINDOW_BEGIN/END (#2062) fix #2070 var list numeric order (#2072) fix #2087 develop docs_pdf (#2091) fix #2096/#2098 develop - fix skip if output exists and do not error if no commands were run (#2099) Fix for Dockerfile smell DL4000 (#2112) fix #2082 develop regrid.convert/censor_thresh/censor_val (#2140) fix #2082 main_v5.0 regrid.convert/censor_thresh/censor_val (#2101) fix #2137 develop PointStat -obs_valid_beg/end (#2141) fix failured introduced by urllib3 (see https://github.com/urllib3/urllib3/issues/2168) fix #2161 develop PCPCombine additional field arguments in -subtract mode (#2162) fix #2168 develop - StatAnalysis time shift (#2169) fix releases. (#2183) fix #2189 develop - spaces in complex thresholds (#2191) fix #2179 develop TCPairs fix -diag argument (#2187) fixes (#2200) fix diff tests (#2217) fix automated tests (#2237) fix #2235 rename multivar_itensity to multivar_intensity_flag (#2236) fix #2241 Create directory containing -out_stat file (#2242) fix #2245 use unique run ID to name logger instance (#2247) fix #2244 develop fix diff tests (#2254) fixture to set pytest tmpdir (#2261) fix #1853 develop - PointStat don't require mask variables to be set (#2262) fix #2279 develop - buoy station file from 2022 (#2280) fix (#2313) fix ReadTheDocs requirements to include pillow which is a dependency of sphinx-gallery: see https://blog.readthedocs.com/defaulting-latest-build-tools/ for more info on why this was necessary fix bug described in https://github.com/pangeo-data/xESMF/issues/246 --- .github/parm/use_case_groups.json | 2 +- docs/Users_Guide/glossary.rst | 391 +++++++++++++++++- docs/Users_Guide/release-notes.rst | 51 +++ docs/Users_Guide/wrappers.rst | 365 +++++++++++++++- docs/_static/met_tool_wrapper-WaveletStat.png | Bin 0 -> 7246 bytes .../met_tool_wrapper/WaveletStat/README.rst | 2 + .../WaveletStat/WaveletStat.py | 115 ++++++ .../pytests/util/run_util/test_run_util.py | 43 +- .../util/time_looping/test_time_looping.py | 5 + .../gen_ens_prod/test_gen_ens_prod_wrapper.py | 33 +- .../wavelet_stat/test_wavelet_stat.py | 337 +++++++++++++++ internal/tests/use_cases/all_use_cases.txt | 1 + metplus/VERSION | 2 +- metplus/util/constants.py | 1 + metplus/util/met_config.py | 114 +++-- metplus/util/run_util.py | 8 +- metplus/util/time_looping.py | 9 +- metplus/wrappers/command_builder.py | 5 +- metplus/wrappers/ensemble_stat_wrapper.py | 1 + metplus/wrappers/grid_stat_wrapper.py | 1 + metplus/wrappers/mode_wrapper.py | 1 + metplus/wrappers/mtd_wrapper.py | 1 + metplus/wrappers/point_stat_wrapper.py | 1 + metplus/wrappers/tc_diag_wrapper.py | 7 +- metplus/wrappers/tc_pairs_wrapper.py | 3 +- metplus/wrappers/wavelet_stat_wrapper.py | 180 ++++++++ parm/met_config/WaveletStatConfig_wrapped | 128 ++++++ .../GenEnsProd/GenEnsProd.conf | 4 +- .../WaveletStat/WaveletStat.conf | 133 ++++++ ush/run_metplus.py | 10 +- 30 files changed, 1857 insertions(+), 97 deletions(-) create mode 100644 docs/_static/met_tool_wrapper-WaveletStat.png create mode 100644 docs/use_cases/met_tool_wrapper/WaveletStat/README.rst create mode 100644 docs/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.py create mode 100644 internal/tests/pytests/wrappers/wavelet_stat/test_wavelet_stat.py create mode 100755 metplus/wrappers/wavelet_stat_wrapper.py create mode 100644 parm/met_config/WaveletStatConfig_wrapped create mode 100644 parm/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.conf diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 1752f76df0..2ba2e011c4 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -1,7 +1,7 @@ [ { "category": "met_tool_wrapper", - "index_list": "0-29,59-62", + "index_list": "0-29,59-61,63", "run": false }, { diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index ef14ea48bc..1ab1e903bf 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -2057,7 +2057,7 @@ METplus Configuration Glossary | *Used by:* GridStat GRID_STAT_OUTPUT_TEMPLATE - Sets the subdirectories below :term:`GRID_STAT_OUTPUT_DIR` using a template to allow run time information. If LOOP_BY = VALID, default value is valid time YYYYMMDDHHMM/grid_stat. If LOOP_BY = INIT, default value is init time YYYYMMDDHHMM/grid_stat. + Sets the subdirectories below :term:`GRID_STAT_OUTPUT_DIR` using a template to allow run time information. | *Used by:* GridStat @@ -5449,10 +5449,10 @@ METplus Configuration Glossary .. warning:: **DEPRECATED:** Please use :term:`GEN_ENS_PROD_NMEP_SMOOTH_SHAPE` in :ref:`gen_ens_prod_wrapper` instead. ENSEMBLE_STAT_NMEP_SMOOTH_METHOD - .. warning:: **DEPRECATED:** Please use :term:`GEN_ENS_PROD_NMEP_SMOOTH_METHOD` in :ref:`gen_ens_prod_wrapper` instead. + .. warning:: **DEPRECATED:** Please use :term:`GEN_ENS_PROD_NMEP_SMOOTH_TYPE_METHOD` in :ref:`gen_ens_prod_wrapper` instead. ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH - .. warning:: **DEPRECATED:** Please use :term:`GEN_ENS_PROD_NMEP_SMOOTH_WIDTH` in :ref:`gen_ens_prod_wrapper` instead. + .. warning:: **DEPRECATED:** Please use :term:`GEN_ENS_PROD_NMEP_SMOOTH_TYPE_WIDTH` in :ref:`gen_ens_prod_wrapper` instead. ENSEMBLE_STAT_CENSOR_THRESH Specify the value for 'censor_thresh' in the MET configuration file for EnsembleStat. @@ -8271,12 +8271,12 @@ METplus Configuration Glossary | *Used by:* GenEnsProd - GEN_ENS_PROD_NMEP_SMOOTH_METHOD + GEN_ENS_PROD_NMEP_SMOOTH_TYPE_METHOD Specify the value for 'nmep_smooth.type.method' in the MET configuration file for GenEnsProd. | *Used by:* GenEnsProd - GEN_ENS_PROD_NMEP_SMOOTH_WIDTH + GEN_ENS_PROD_NMEP_SMOOTH_TYPE_WIDTH Specify the value for 'nmep_smooth.type.width' in the MET configuration file for GenEnsProd. | *Used by:* GenEnsProd @@ -11128,3 +11128,384 @@ METplus Configuration Glossary Specify the value for 'cat_thresh' in the MET configuration file for GridStat. | *Used by:* GridStat + + WAVELET_STAT_DESC + Specify the value for 'desc' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_REGRID_TO_GRID + Specify the value for 'regrid.to_grid' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_REGRID_METHOD + Specify the value for 'regrid.method' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_REGRID_WIDTH + Specify the value for 'regrid.width' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_REGRID_VLD_THRESH + Specify the value for 'regrid.vld_thresh' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_REGRID_SHAPE + Specify the value for 'regrid.shape' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_CENSOR_THRESH + Specify the value for 'censor_thresh' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_CENSOR_VAL + Specify the value for 'censor_val' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_MASK_MISSING_FLAG + Specify the value for 'mask_missing_flag' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_GRID_DECOMP_FLAG + Specify the value for 'grid_decomp_flag' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_TILE_WIDTH + Specify the value for 'tile.width' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_TILE_LOCATION_X_LL + Specify the value for the nth 'tile.location.x_ll' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_TILE_LOCATION_Y_LL + Specify the value for the nth 'tile.location.y_ll' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_WAVELET_TYPE + Specify the value for 'wavelet.type' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_WAVELET_MEMBER + Specify the value for 'wavelet.member' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_OUTPUT_FLAG_ISC + Specify the value for 'output_flag.isc' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_NC_PAIRS_FLAG_RAW + Specify the value for 'nc_pairs_flag.raw' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_NC_PAIRS_FLAG_DIFF + Specify the value for 'nc_pairs_flag.diff' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_PS_PLOT_FLAG + Specify the value for 'ps_plot_flag' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_FCST_RAW_PLOT_COLOR_TABLE + Specify the value for 'fcst_raw_plot.color_table' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_FCST_RAW_PLOT_PLOT_MIN + Specify the value for 'fcst_raw_plot.plot_min' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_FCST_RAW_PLOT_PLOT_MAX + Specify the value for 'fcst_raw_plot.plot_max' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_OBS_RAW_PLOT_COLOR_TABLE + Specify the value for 'obs_raw_plot.color_table' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_OBS_RAW_PLOT_PLOT_MIN + Specify the value for 'obs_raw_plot.plot_min' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_OBS_RAW_PLOT_PLOT_MAX + Specify the value for 'obs_raw_plot.plot_max' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_WVLT_PLOT_COLOR_TABLE + Specify the value for 'wvlt_plot.color_table' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_WVLT_PLOT_PLOT_MIN + Specify the value for 'wvlt_plot.plot_min' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_WVLT_PLOT_PLOT_MAX + Specify the value for 'wvlt_plot.plot_max' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_OUTPUT_PREFIX + Specify the value for 'output_prefix' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_FILE_WINDOW_BEGIN + See :term:`OBS_WAVELET_STAT_FILE_WINDOW_BEGIN` + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_FILE_WINDOW_END + See :term:`OBS_WAVELET_STAT_FILE_WINDOW_END` + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_INPUT_DATATYPE + Specify the data type of the input directory for forecast files used with the MET wavelet_stat tool. Currently valid options are NETCDF, GRIB, and GEMPAK. If set to GEMPAK, data will automatically be converted to NetCDF via GempakToCF. A corresponding variable exists for observation data called :term:`OBS_WAVELET_STAT_INPUT_DATATYPE`. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_INPUT_DIR + Input directory for forecast files to use with the MET tool wavelet_stat. A corresponding variable exists for observation data called :term:`OBS_WAVELET_STAT_INPUT_DIR`. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_INPUT_TEMPLATE + Template used to specify forecast input filenames for the MET tool wavelet_stat. A corresponding variable exists for observation data called :term:`OBS_WAVELET_STAT_INPUT_TEMPLATE`. To utilize Python Embedding as input to the MET tools, set this value to PYTHON_NUMPY or PYTHON_XARRAY. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_PROB_THRESH + Threshold values to be used for probabilistic data in wavelet_stat. The value can be a single item or a comma separated list of items that must start with a comparison operator (>,>=,==,!=,<,<=,gt,ge,eq,ne,lt,le). A corresponding variable exists for observation data called :term:`OBS_WAVELET_STAT_PROB_THRESH`. + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_FILE_WINDOW_BEGIN + Used to control the lower bound of the window around the valid time to determine if a file should be used for processing by WaveletStat. See :ref:`Directory_and_Filename_Template_Info` subsection called 'Using Windows to Find Valid Files.' Units are seconds. If :term:`OBS_WAVELET_STAT_FILE_WINDOW_BEGIN` is not set in the config file, the value of :term:`OBS_FILE_WINDOW_BEGIN` will be used instead. If both file window begin and window end values are set to 0, then METplus will require an input file with an exact time match to process. + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_FILE_WINDOW_END + Used to control the upper bound of the window around the valid time to determine if a file should be used for processing by WaveletStat. See :ref:`Directory_and_Filename_Template_Info` subsection called 'Using Windows to Find Valid Files.' Units are seconds. If :term:`OBS_WAVELET_STAT_FILE_WINDOW_END` is not set in the config file, the value of :term:`OBS_FILE_WINDOW_END` will be used instead. If both file window begin and window end values are set to 0, then METplus will require an input file with an exact time match to process. + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_INPUT_DATATYPE + See :term:`FCST_WAVELET_STAT_INPUT_DATATYPE` + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_INPUT_DIR + See :term:`FCST_WAVELET_STAT_INPUT_DIR` + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_INPUT_TEMPLATE + See :term:`FCST_WAVELET_STAT_INPUT_TEMPLATE` + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_PROB_THRESH + See :term:`FCST_WAVELET_STAT_PROB_THRESH` + + | *Used by:* WaveletStat + + WAVELET_STAT_OUTPUT_DIR + Specify the output directory where files from the MET wavelet_stat tool are written. + + | *Used by:* WaveletStat + + WAVELET_STAT_OUTPUT_TEMPLATE + Sets the subdirectories below :term:`WAVELET_STAT_OUTPUT_DIR` using a template to allow run time information. + + | *Used by:* WaveletStat + + LOG_WAVELET_STAT_VERBOSITY + Overrides the log verbosity for WaveletStat only. + If not set, the verbosity level is controlled by :term:`LOG_MET_VERBOSITY`. + + | *Used by:* WaveletStat + + WAVELET_STAT_CONFIG_FILE + Path to configuration file read by wavelet_stat. + If unset, parm/met_config/WaveletStatConfig_wrapped will be used. + + | *Used by:* WaveletStat + + WAVELET_STAT_ONCE_PER_FIELD + True/False. If True, wavelet_stat will run once to process all name/level/threshold combinations specified. + If False, it will run once for each name/level. Some cases require this to be set to False, + for example processing probablistic forecasts or precipitation accumulations. + + | *Used by:* WaveletStat + + WAVELET_STAT_CUSTOM_LOOP_LIST + Sets custom string loop list for a specific wrapper. See :term:`CUSTOM_LOOP_LIST`. + + | *Used by:* WaveletStat + + WAVELET_STAT_SKIP_IF_OUTPUT_EXISTS + If True, do not run app if output file already exists. Set to False to overwrite files. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_FILE_TYPE + Specify the value for 'fcst.file_type' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_FILE_TYPE + Specify the value for 'obs.file_type' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_IS_PROB + Wrapper-specific version of :term:`FCST_IS_PROB`. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_PROB_IN_GRIB_PDS + Wrapper-specific version of :term:`FCST_PROB_IN_GRIB_PDS`. + + | *Used by:* WaveletStat + + WAVELET_STAT_MET_CONFIG_OVERRIDES + Override any variables in the MET configuration file that are not + supported by the wrapper. This should be set to the full variable name + and value that you want to override, including the equal sign and the + ending semi-colon. The value is directly appended to the end of the + wrapped MET config file. + + Example: + WAVELET_STAT_MET_CONFIG_OVERRIDES = desc = "override_desc"; model = "override_model"; + + See :ref:`Overriding Unsupported MET config file settings` for more information + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_WINDOW_BEGIN + Passed to the WaveletStat MET config file to determine the range of data within a file that should be used for processing. Units are seconds. If the variable is not set, WaveletStat will use :term:`FCST_WINDOW_BEGIN`. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_WINDOW_END + Passed to the WaveletStat MET config file to determine the range of data within a file that should be used for processing. Units are seconds. If the variable is not set, WaveletStat will use :term:`FCST_WINDOW_END`. + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_WINDOW_BEGIN + Passed to the WaveletStat MET config file to determine the range of data within a file that should be used for processing. Units are seconds. If the variable is not set, WaveletStat will use :term:`OBS_WINDOW_BEGIN`. + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_WINDOW_END + Passed to the WaveletStat MET config file to determine the range of data within a file that should be used for processing. Units are seconds. If the variable is not set, WaveletStat will use :term:`OBS_WINDOW_END`. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_VAR_NAME + Wrapper specific field info variable. See :term:`FCST_VAR_NAME`. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_VAR_LEVELS + Wrapper specific field info variable. See :term:`FCST_VAR_LEVELS`. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_VAR_THRESH + Wrapper specific field info variable. See :term:`FCST_VAR_THRESH`. + + | *Used by:* WaveletStat + + FCST_WAVELET_STAT_VAR_OPTIONS + Wrapper specific field info variable. See :term:`FCST_VAR_OPTIONS`. + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_VAR_NAME + Wrapper specific field info variable. See :term:`OBS_VAR_NAME`. + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_VAR_LEVELS + Wrapper specific field info variable. See :term:`OBS_VAR_LEVELS`. + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_VAR_THRESH + Wrapper specific field info variable. See :term:`OBS_VAR_THRESH`. + + | *Used by:* WaveletStat + + OBS_WAVELET_STAT_VAR_OPTIONS + Wrapper specific field info variable. See :term:`OBS_VAR_OPTIONS`. + + | *Used by:* WaveletStat + + WAVELET_STAT_SKIP_VALID_TIMES + List of valid times to skip for WaveletStat only. + If set, values set in :term:`SKIP_VALID_TIMES` are ignored for WaveletStat. + See :term:`SKIP_VALID_TIMES` for formatting information. + + | *Used by:* WaveletStat + + WAVELET_STAT_INC_VALID_TIMES + List of valid times to include for WaveletStat only. + If set, values set in :term:`INC_VALID_TIMES` are ignored for WaveletStat. + See :term:`SKIP_VALID_TIMES` for formatting information. + + | *Used by:* WaveletStat + + WAVELET_STAT_SKIP_INIT_TIMES + List of initialization times to skip for WaveletStat only. + If set, values set in :term:`SKIP_INIT_TIMES` are ignored for WaveletStat. + See :term:`SKIP_VALID_TIMES` for formatting information. + + | *Used by:* WaveletStat + + WAVELET_STAT_INC_INIT_TIMES + List of initialization times to include for WaveletStat only. + If set, values set in :term:`INC_INIT_TIMES` are ignored for WaveletStat. + See :term:`SKIP_VALID_TIMES` for formatting information. + + | *Used by:* WaveletStat + + WAVELET_STAT_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat + + WAVELET_STAT_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index c00627a35e..28054ff5c2 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -30,6 +30,57 @@ When applicable, release notes are followed by the `GitHub issue `__ number which describes the bugfix, enhancement, or new feature. +METplus Version 6.0.0 Beta 2 Release Notes (2023-11-14) +------------------------------------------------------- + + .. dropdown:: Enhancements + + * Improve SeriesAnalysis ingest of multiple input files + (`#2219 `_) + * Update the TC-Diag wrapper to support updates for MET version 12.0.0 + (`#2340 `_) + * Add config option to write MET log output to terminal + (`#2377 `_) + * GenVxMask - support specification strings to define output grid + (`#2412 `_) + * Follow symbolic links when searching for files within a time window + (`#2423 `_) + + .. dropdown:: Bugfix + + * Prevent crash when empty string set for INIT_INCREMENT or VALID_INCREMENT + (`#2420 `_) + + .. dropdown:: New Wrappers + + * WaveletStat + (`#2252 `_) + + + .. dropdown:: New Use Cases + + NONE + + .. dropdown:: Documentation + + * **Add upgrade instructions for removing user wrapped MET config files** + (`#2349 `_) + * Reorder Python Wrappers - MET Configuration tables to match order in wrapped MET config file + (`#2405 `_) + * Enhancement to Difficulty Index use-case documentation + (`#2123 `_) + * Modify the Documentation Overview section in the Contributor's Guide to add Conventions + (`#1667 `_) + * Specify available tags on DockerHub + (`#2329 `_) + + .. dropdown:: Internal + + * Improve METplus test coverage + (`#2253 `_) + * Documentation: Make Headers Consistent in METplus components User's Guides + (`#898 `_) + METplus Version 6.0.0 Beta 1 Release Notes (2023-09-15) ------------------------------------------------------- diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index b29a86b7b9..1078f6466c 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -1112,8 +1112,8 @@ METplus Configuration | :term:`GEN_ENS_PROD_NMEP_SMOOTH_SHAPE` | :term:`GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX` | :term:`GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS` -| :term:`GEN_ENS_PROD_NMEP_SMOOTH_METHOD` -| :term:`GEN_ENS_PROD_NMEP_SMOOTH_WIDTH` +| :term:`GEN_ENS_PROD_NMEP_SMOOTH_TYPE_METHOD` +| :term:`GEN_ENS_PROD_NMEP_SMOOTH_TYPE_WIDTH` | :term:`GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME` | :term:`GEN_ENS_PROD_CLIMO_MEAN_VAR_NAME` | :term:`GEN_ENS_PROD_CLIMO_MEAN_VAR_LEVELS` @@ -1414,9 +1414,9 @@ ${METPLUS_NMEP_SMOOTH_DICT} - nmep_smooth.gaussian_dx * - :term:`GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS` - nmep_smooth.gaussian_radius - * - :term:`GEN_ENS_PROD_NMEP_SMOOTH_METHOD` + * - :term:`GEN_ENS_PROD_NMEP_SMOOTH_TYPE_METHOD` - nmep_smooth.type.method - * - :term:`GEN_ENS_PROD_NMEP_SMOOTH_WIDTH` + * - :term:`GEN_ENS_PROD_NMEP_SMOOTH_TYPE_WIDTH` - nmep_smooth.type.width ${METPLUS_CLIMO_MEAN_DICT} @@ -10914,3 +10914,360 @@ METplus Configuration | :term:`USER_SCRIPT_SKIP_INIT_TIMES` | :term:`USER_SCRIPT_INC_INIT_TIMES` | + +.. _wavelet_stat_wrapper: + +WaveletStat +=========== + +Description +----------- + +Used to configure the MET tool wavelet_stat. + +METplus Configuration +--------------------- + +| :term:`FCST_WAVELET_STAT_INPUT_DIR` +| :term:`OBS_WAVELET_STAT_INPUT_DIR` +| :term:`WAVELET_STAT_OUTPUT_DIR` +| :term:`FCST_WAVELET_STAT_INPUT_TEMPLATE` +| :term:`OBS_WAVELET_STAT_INPUT_TEMPLATE` +| :term:`WAVELET_STAT_OUTPUT_TEMPLATE` +| :term:`LOG_WAVELET_STAT_VERBOSITY` +| :term:`WAVELET_STAT_CONFIG_FILE` +| :term:`FCST_WAVELET_STAT_INPUT_DATATYPE` +| :term:`OBS_WAVELET_STAT_INPUT_DATATYPE` +| :term:`WAVELET_STAT_ONCE_PER_FIELD` +| :term:`WAVELET_STAT_CUSTOM_LOOP_LIST` +| :term:`WAVELET_STAT_SKIP_IF_OUTPUT_EXISTS` +| :term:`FCST_WAVELET_STAT_FILE_TYPE` +| :term:`OBS_WAVELET_STAT_FILE_TYPE` +| :term:`FCST_WAVELET_STAT_IS_PROB` +| :term:`FCST_WAVELET_STAT_PROB_IN_GRIB_PDS` +| :term:`WAVELET_STAT_MET_CONFIG_OVERRIDES` +| :term:`FCST_WAVELET_STAT_PROB_THRESH` +| :term:`OBS_WAVELET_STAT_PROB_THRESH` +| :term:`FCST_WAVELET_STAT_WINDOW_BEGIN` +| :term:`FCST_WAVELET_STAT_WINDOW_END` +| :term:`OBS_WAVELET_STAT_WINDOW_BEGIN` +| :term:`OBS_WAVELET_STAT_WINDOW_END` +| :term:`FCST_WAVELET_STAT_FILE_WINDOW_BEGIN` +| :term:`FCST_WAVELET_STAT_FILE_WINDOW_END` +| :term:`OBS_WAVELET_STAT_FILE_WINDOW_BEGIN` +| :term:`OBS_WAVELET_STAT_FILE_WINDOW_END` +| :term:`FCST_WAVELET_STAT_VAR_NAME` +| :term:`FCST_WAVELET_STAT_VAR_LEVELS` +| :term:`FCST_WAVELET_STAT_VAR_THRESH` +| :term:`FCST_WAVELET_STAT_VAR_OPTIONS` +| :term:`OBS_WAVELET_STAT_VAR_NAME` +| :term:`OBS_WAVELET_STAT_VAR_LEVELS` +| :term:`OBS_WAVELET_STAT_VAR_THRESH` +| :term:`OBS_WAVELET_STAT_VAR_OPTIONS` +| :term:`WAVELET_STAT_SKIP_VALID_TIMES` +| :term:`WAVELET_STAT_INC_VALID_TIMES` +| :term:`WAVELET_STAT_SKIP_INIT_TIMES` +| :term:`WAVELET_STAT_INC_INIT_TIMES` +| :term:`MODEL` +| :term:`WAVELET_STAT_DESC` +| :term:`OBTYPE` +| :term:`WAVELET_STAT_REGRID_TO_GRID` +| :term:`WAVELET_STAT_REGRID_METHOD` +| :term:`WAVELET_STAT_REGRID_WIDTH` +| :term:`WAVELET_STAT_REGRID_VLD_THRESH` +| :term:`WAVELET_STAT_REGRID_SHAPE` +| :term:`WAVELET_STAT_REGRID_CONVERT` +| :term:`WAVELET_STAT_REGRID_CENSOR_THRESH` +| :term:`WAVELET_STAT_REGRID_CENSOR_VAL` +| :term:`WAVELET_STAT_CENSOR_THRESH` +| :term:`WAVELET_STAT_CENSOR_VAL` +| :term:`WAVELET_STAT_MASK_MISSING_FLAG` +| :term:`WAVELET_STAT_GRID_DECOMP_FLAG` +| :term:`WAVELET_STAT_TILE_WIDTH` +| :term:`WAVELET_STAT_TILE_LOCATION_X_LL` +| :term:`WAVELET_STAT_TILE_LOCATION_Y_LL` +| :term:`WAVELET_STAT_WAVELET_TYPE` +| :term:`WAVELET_STAT_WAVELET_MEMBER` +| :term:`WAVELET_STAT_OUTPUT_FLAG_ISC` +| :term:`WAVELET_STAT_NC_PAIRS_FLAG_RAW` +| :term:`WAVELET_STAT_NC_PAIRS_FLAG_DIFF` +| :term:`WAVELET_STAT_PS_PLOT_FLAG` +| :term:`WAVELET_STAT_FCST_RAW_PLOT_COLOR_TABLE` +| :term:`WAVELET_STAT_FCST_RAW_PLOT_PLOT_MIN` +| :term:`WAVELET_STAT_FCST_RAW_PLOT_PLOT_MAX` +| :term:`WAVELET_STAT_OBS_RAW_PLOT_COLOR_TABLE` +| :term:`WAVELET_STAT_OBS_RAW_PLOT_PLOT_MIN` +| :term:`WAVELET_STAT_OBS_RAW_PLOT_PLOT_MAX` +| :term:`WAVELET_STAT_WVLT_PLOT_COLOR_TABLE` +| :term:`WAVELET_STAT_WVLT_PLOT_PLOT_MIN` +| :term:`WAVELET_STAT_WVLT_PLOT_PLOT_MAX` +| :term:`WAVELET_STAT_OUTPUT_PREFIX` + +.. _wavelet-stat-met-conf: + +MET Configuration +----------------- + +Below is the wrapped MET configuration file used for this wrapper. +Environment variables are used to control entries in this configuration file. +The default value for each environment variable is obtained from +(except where noted below): + +`MET_INSTALL_DIR/share/met/config/WaveletStatConfig_default `_ + +Below the file contents are descriptions of each environment variable +referenced in this file and the corresponding METplus configuration item used +to set the value of the environment variable. For detailed examples showing +how METplus sets the values of these environment variables, +see :ref:`How METplus controls MET config file settings`. + +.. dropdown:: Click to view parm/met_config/WaveletStatConfig_wrapped + + .. literalinclude:: ../../parm/met_config/WaveletStatConfig_wrapped + +Environment variables in wrapped MET config +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +${METPLUS_MODEL} +^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`MODEL` + - model + +${METPLUS_DESC} +^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_DESC` + - desc + +${METPLUS_OBTYPE} +^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`OBTYPE` + - obtype + +${METPLUS_REGRID_DICT} +^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_REGRID_TO_GRID` + - regrid.to_grid + * - :term:`WAVELET_STAT_REGRID_METHOD` + - regrid.method + * - :term:`WAVELET_STAT_REGRID_WIDTH` + - regrid.width + * - :term:`WAVELET_STAT_REGRID_VLD_THRESH` + - regrid.vld_thresh + * - :term:`WAVELET_STAT_REGRID_SHAPE` + - regrid.shape + +${METPLUS_CENSOR_THRESH} +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_CENSOR_THRESH` + - censor_thresh + +${METPLUS_CENSOR_VAL} +^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_CENSOR_VAL` + - censor_val + +${METPLUS_MASK_MISSING_FLAG} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_MASK_MISSING_FLAG` + - mask_missing_flag + +${METPLUS_GRID_DECOMP_FLAG} +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_GRID_DECOMP_FLAG` + - grid_decomp_flag + +${METPLUS_TILE_DICT} +^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_TILE_WIDTH` + - tile.width + * - :term:`WAVELET_STAT_TILE_LOCATION_X_LL` + - tile.location.x_ll + * - :term:`WAVELET_STAT_TILE_LOCATION_Y_LL` + - tile.location.y_ll + +${METPLUS_WAVELET_DICT} +^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_WAVELET_TYPE` + - wavelet.type + * - :term:`WAVELET_STAT_WAVELET_MEMBER` + - wavelet.member + +${METPLUS_OUTPUT_FLAG_DICT} +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_OUTPUT_FLAG_ISC` + - output_flag.isc + +${METPLUS_NC_PAIRS_FLAG_DICT} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_NC_PAIRS_FLAG_RAW` + - nc_pairs_flag.raw + * - :term:`WAVELET_STAT_NC_PAIRS_FLAG_DIFF` + - nc_pairs_flag.diff + +${METPLUS_PS_PLOT_FLAG} +^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_PS_PLOT_FLAG` + - ps_plot_flag + +${METPLUS_FCST_RAW_PLOT_DICT} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_FCST_RAW_PLOT_COLOR_TABLE` + - fcst_raw_plot.color_table + * - :term:`WAVELET_STAT_FCST_RAW_PLOT_PLOT_MIN` + - fcst_raw_plot.plot_min + * - :term:`WAVELET_STAT_FCST_RAW_PLOT_PLOT_MAX` + - fcst_raw_plot.plot_max + +${METPLUS_OBS_RAW_PLOT_DICT} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_OBS_RAW_PLOT_COLOR_TABLE` + - obs_raw_plot.color_table + * - :term:`WAVELET_STAT_OBS_RAW_PLOT_PLOT_MIN` + - obs_raw_plot.plot_min + * - :term:`WAVELET_STAT_OBS_RAW_PLOT_PLOT_MAX` + - obs_raw_plot.plot_max + +${METPLUS_WVLT_PLOT_DICT} +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_WVLT_PLOT_COLOR_TABLE` + - wvlt_plot.color_table + * - :term:`WAVELET_STAT_WVLT_PLOT_PLOT_MIN` + - wvlt_plot.plot_min + * - :term:`WAVELET_STAT_WVLT_PLOT_PLOT_MAX` + - wvlt_plot.plot_max + +${METPLUS_OUTPUT_PREFIX} +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_OUTPUT_PREFIX` + - output_prefix + +${METPLUS_MET_CONFIG_OVERRIDES} +""""""""""""""""""""""""""""""" + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`WAVELET_STAT_MET_CONFIG_OVERRIDES` + - n/a diff --git a/docs/_static/met_tool_wrapper-WaveletStat.png b/docs/_static/met_tool_wrapper-WaveletStat.png new file mode 100644 index 0000000000000000000000000000000000000000..202de6b72187e2da7fe184978ff5acc1e2e0413a GIT binary patch literal 7246 zcmV-U9I@kxP)004R> z004l5008;`004mK004C`008P>0026e000+ooVrmw0000^WmrjOO-%qQ0000800aRV z00jU50096106qa500jU50096107d~Q00RI3009630006L00RI3009620000000000 z05Sjo0096105Sjo00CS8!FB)u8;?muK~#7F)m#UBR8_v7x%W;l>75ipLVy^mNJm6a z#8s+FSy92~^6dItT?>i}mbERqEbh82imZ(y3bG=g^b$G*l2Ae_>B%IMskh9$?|<(k z#*mN%!f)N^ZC%Wx-qrYkH_P2IVsue>+$ip-GU(S zJTKzg>&5u=dc8(ViQ_n(ULP73X0ck0CQ2HOrmbetyD~eqno+B17;V+OnsRlQBPdB2 zxrE`eEPLsotE#F>ii%20N{R~#@F~2&S2+1PNphMS5|T)O^1NRXeO|ZEVHfSSqOD4- zt(25Me*|RyIpcoK1nguy`36x!cATNjj4~!O=s!CN^QE@se z>*&FQNA_RH5i~W%P+eTICaN137R8!_7`;JjFmhTQ=*zEGL5j%yE&(Dx*bcvB-lqpG zlFWu`nm1^ZVOa1cJ~E5q9Z?j#9?|3Wxt*fj1~i3=5^v$zZlW_I!ZB#ru>OMv_37Ks zVlpbNqEp_>vl@;IELDrnI{Nd<5u_3DSWLeTv?HhD!lJ#qckSG|^>B%%GB{e_BV8NY zoePOzj3$HG$`Ea;WdagGMBq$9t`R1g(we>+>8|xz0AzX)_#@XLaRDTODgi>oMI;L$ z>U2nsdZD_UFUj-fo$g*$HEML!nDOKLT|XouEWqhrj|Z=5G+gVwwO4wD2zs%4ECWVB z`*xRi^ZK=$)~`QaZM7%#GWH&%jZI`tR-MVrX|>>PQIH`cxh^XfFeX7nl+@nvI`fh8lo?X52@7p$>EAKzhJm^MUTzAG~(VNU*EKw3go>vGh0G5JUXE3>{x{ZKRBQly; zZy*Zt=Ur~eUh65yb?n=cR+o9l^y$;?o7XKW3>9GjkSOj9Uv~mQP!sqLgFsLFb{_r9 zy9@V}7z|_YH1-|{&7d=yFi_xzj|?1*n?ApW)PyEUO@-3|rB84+@D;_*6Z_a5TkaSa z|L}7QdL+hRyr2?Hvx9o3LkL1z0X}7Ac5cgD^5%<~4Y5HJ?>43l;4H!Dj7$Xru2J^? zN`Py1@frsy>mn8u2g_5et#TaS;P2*kAntHSr+0NkZ$HbMyy8BT+FdaKKwS()~4KD%$ib$?j= zVn~n$FG2Mw)4#LVOpTNUL{-h3ZUsRFJ|{0ebocbV z+JV6n?zSYPc-$_(C^mXp&0A>9gAp`ZP`gj?QW2yK2w$Uv;}xJ;R3MLy^YNa>ce>)Q z-_K}uSS)zW?LY%Nk=0;4WH!ncX`9{4YtLx)dY#6WeZ2PbWsi)He__!g)Q1zH)&3%u z(Go$g$E(xp%4-{DjJx^b$ob)8?^GMDg3Eas(xY+6=m_=IfKMA5P4Ho) zf!y>m8m;KCOU`<2L>yJ#CFU`nd8>6Kx_!AcqNv zNM-h?Ho<1(@HOI#xgwwi_ytNtmmRZ2v#j2Xzzl1&usYe>_hTir_zCu>@c~Czzlhv$TFT=oG^Ig)#b81d zM+F2~4utPH@uTy=w^%|H5*fms)0^a$&0t%30Z4!}oIC11zRhtV3+qSHuxXvBvo;c{ zF>1l(V9mjiQ|{kq9r@7AnWzdK3u{q%CxDDEb6JhOdp* z8;da-xW1ezDCoKoD03EONMmWltmRq@*qGRXOEx|c}AlLl{wBIvu7W`lK3S7B@h%?JhefN5fCUo zMh>{T3Uln)nGI)-NG>~7Q?rt*L8z-j?`X|ieUXF=J{?K`kwwTyZL~yAp0m}TzTlZ> z@E-gN3M8pigdj8|Vm!#a@Wd0RlP8CbnkZn&5kymH0S_*p)C>RupBiG5QL*m$PCxH) zUp$9AXE5X2o_i26QP3drmK*PZpmdjB$bEa>=|=|VJ~b-$snJ>U(yMl?!PD|BpXNL@ zsv##6twC-O-#tFOYS%hYjjQ-f_Tt&uj}6Uz`bNBX;iKohRVA{flKk#EybGEF#R>M> z{EuHbHGe?v)1z~q7=H4ZTN=(E0|#9EW?98&i!>>NE5CT{`0ON4MIi~6Ch0-%Up6re zKxE9pA-B(3ky-!smtUeb=p6z>L68V?%cf1MjyQwHPEu+0lGoc&_>_Jjs?=(IWCDJ< zkN&{hDjesJBOiV1oS^adJNB%_Z29U^lmust#Gv7Hmg<#7|RFy3k-d`~IWg*7!%4_F0-4ecvB#-+xr{?J~*hf+$2C zN}ws9uXMxTYrc6u;_m1A{AGXQ)BhBUPv<-_g1422kGRt~=r-Z}A=A)FJziO-4UPbT z6EAl>5j#80u?b=K{O*P4SCy7kL8!oCV_i=Y2T$h5iE*aX4j>M(^j_ODiB z&LA%Hh^Z(7qt%_*g^Z=wU~BIis3a%TS(t;2F|ijcp3oa6MBlqGYTPVcSgaaCrN=-# z5*>9x*Ads(ezy$j6=Vx~VXRT+)B%vKe9m4rwr6m^(T2E`kp4GOwGB)8>Wf$pXoUn(lVF$XaT{gSn*9KmkC(l5*xCbB}(G zyX1!GL z`C>?YMZLt$d0Ar)@m3N2q6;W@WR#>!AI0blqN^eB8zD6(bQD@kKeynHK%xAHH3j*+ zypJbC72i;0SnrOA_~wmJp5&G2m9(b5L)I?aJn!hyegg+`pk(YDto(SdY4&5#bl!&g zD{7j8444lfKu6x3z4!+8K>%q%Wh#~D)J}+vpmB3F!4aUf+&3RUoyZw4M2@)&$mD;z z*jbVdbqUb{@&tmF+gF1I(TJtzKs78_d6_lgaY?`bCwjq{J>>;p6isj>1Rts?sgWN2 z$uEAPk4*vMVuMYvl^eUIKwv5E0D9q!R*83$1>X6jW*N!rvBW00$4*+lbj4p*3?u^F zw`p)h^T_Fr{olEdZ3&37 zD%w8~B%eO@y3JWN*heD-f7rOuc-;+JOOQ|GJEuSbjnW^Pk$GK2H_Smif0zwVgj(=< zTr}rCL;dw-sAEp)4`Wvo5*2#KBVztB=-C6~nDehp(f1{}=FC40LWYqTJ^r^Ui9ElM zfj7gY!JG|osmU*r`aAvb^@nD5KQ=SI^2;}(?s^i`1nq%xB|*9+Ke!nAyjQ0loM=2U zHN4bk=dNn#{-ihkH8$yTNX6ZRdcII?&8>+Ui=B!~*h#(nY z4brGP`je~VEN2c08#)msmAlr3-EgZhu@9LDyr**eSFkq1M^1*%3u6;z3XDBxQEo%- zA@Hv@B+8mT3=#u-%2=eTJ?mlgg%7_2%MTIctt^J(LgS3qU`ZVSZ3Y9Iyc8(+W7}aW zVvr&I#~73P$cDP?fVMHAHj8~e>;hHq+BCWTCv{DA^{8pT*Uo)O9~CRO;TTa6|7sHP z3l^3kp)0oo_X8XR^(6b?6hR^nfgX`Nh({ztX)*wbd^orwEcV^#4J@x5DFFqcK_w0< z*97L&J}CrZPm(?4jo^bU2kq=vh>Of}as)*x$JF4o(0Y*5i4dnUswOLM@E=nm&& zI8t9=f+qSxKZLCO-EN;9ui0)>ov8q^>=*ekjfRPcjEshaa5ezJ))u`pt#-UFrVW7v z+ZdhFQ3fBh1dC__%H;%a7rI#16738MN(n(zxn{?^zuKB0cx;1;w(8_GJ$zy=Jjs3^x?UW$ItL7bbUKd%B-ale08>CUdfcF`yRxKyLsevm z(B4q1FLqra$d9!h-gW zF;j;|tjReVl#q-gSu_ZFsXN?eCo4jN$N;HB;t&han1l5Zu}O1Y%G&XnDWNy$3xOQl z_SI@@^u=%g#@AIvjhzX+Y7T6wJ-pRfn9J%7ruekTn`fDldROiGx?=77=)Qd0XK;0+ zC)|gKj%*=bGl7Hv3P)RJj?5qDK0@I#iVxz?yoVnax2>-)%+nc-*H)TfYLcW4jx>*$ z=C@tUTk=Tm+Ygp}{a4;rnXw`_YW#1(o``SzUA1g@toQUTS5Yp6X3fFPd9O~hZTg!z zsSgmZUcV%7+0)+YQji`3)NEp3&#|r8004ho+qEbAKy5G@>}O8&x_EZd^yz?(UAS-| z{@`r4WXs!qB+rzZq2jc%@V-XN2$|*sO$A*y3_D(~6WhE6r|bv+UH!u+j`AX7VsBkU z9MpoRzSebUQ_?dZcbonsShZ%~M#qUA8Ou+{+&agUoB`59@Huqs?C>EIy*R{v;M>Hf z{*gHQ1#%W!81tB0*Cp_W)@XK?7S*i!;LnR6NJ>to_%y^0PORZ*-4lO!e(R`_1xI^F zjJU-OKaAG%+Ncx?XbY^?kaYhmag+YnnSToL)as2NIWsrp9RI2R%3MTh;rwIb*l#16 z^7&&X&PA-O;q2jy-~QFM^A~F=wY2n^A*1M{|1RrbA>#Gm#(5-rR#fD8dngp5y<)PVt z9Hbm}uLV9eB{4ZRa?PSQxSr{j*m%UryVyb`%J$(L2&l_>YERA9PZ95ho-O(CNsJ+O z?i(QkZUV^It8!)UlNgOXdp~S!h#S7}G#AyKuP!_P{-e^xTvRsp8e&cvfSQhzI}qfm z&)SWqLDUo$>N;MAW(U(4^xEpo9qMJTuG{x@QcO6gO(&ns!Z661Q4%`Ey&)VVo_5zQ zFkI(9^{nryg`pXPJ=miWyu>nHlPHt|BPMS0BM8=4@Bgl9^;>8dsu{$EXsgVO>WI-( z>rd=#*!2~*kwV|t96$ZZLa(cQ;I3?wu(dg1s-Pe+h{ADTZHjtg z+@#!yj1X!jxrSRh@H!ylZwhgQ9jIMom`3gP9$jnPdLi5xLf_Mz+j2l7#QS$p=efHVq+e<^H&kgQ7L}WM| zAfp*Js-Wg=bvh4~@$-anHnRE2OoB83rCcr(myE33H68;xc^EBgc6u1iUyzrOgLmtw zw;kHy`h58tuiW$7xpPqk@~y>%H|4$75mcUuqKL>1@_Tn={{G?D$_9eeN+0%aDli;RaHh*eT$22tiaubE4$a$TnnQY1Ohf z7k#t(j5KwMso!u-cnowF#u2#CL|P1Q$U#u0lV8aeU3M>Q4=h^{E914b!wKlqwZ_z>2;C>fa#BHya>TA6hFSt%0iY`2N z)3AuCci(-(@Zrk5JZ=xC1?6?_%)yl)*n}f4(=}P=B}GNMckbN&!w-i_v~}@mxLizE zvV%gkW~)}dv`EG>CV?$>&?Ze$nIsx(gq5-#U3ub1N=h?O6>9-QIBO?RoGo-Z%<=4V@MWxW~Sz+rJiM~)mY zXmC&?a2Nm_f6yIXZW@1QGgphC(jROc3+bkSPhJggE`13%GYpS0lY1 zoIk>bEv~b}3FFHO{bhx5oIllUOz$7sfAHY+zJ253=;UWpf_aQyFxsY8n;vyG(=`Y- z^$Mzjh?L2qr8KFls{?tS&B-~FosBClB}IIxpSP)5kJiXqgIP;3d|rapWC3AeJCQb+ zfGNVxfJ}Dy0`e508jzUFA$goWd!5u!XBIu-YJW7RN$Acdr6i@KrQu(v=$4RRG#Z-* zh76hk(Sg>9y*sQZnzztuzKaoT>H|P36m%JL6ka+`j#XDy;`CE(O$|QC*V$}12#TY+ zI7I#Q*>R07*qoM6N<$g3htg0ssI2 literal 0 HcmV?d00001 diff --git a/docs/use_cases/met_tool_wrapper/WaveletStat/README.rst b/docs/use_cases/met_tool_wrapper/WaveletStat/README.rst new file mode 100644 index 0000000000..f6849160e8 --- /dev/null +++ b/docs/use_cases/met_tool_wrapper/WaveletStat/README.rst @@ -0,0 +1,2 @@ +WaveletStat +----------- diff --git a/docs/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.py b/docs/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.py new file mode 100644 index 0000000000..e7d746b6f3 --- /dev/null +++ b/docs/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.py @@ -0,0 +1,115 @@ +""" +WaveletStat: Basic Use Case +=========================== + +met_tool_wrapper/WaveletStat/WaveletStat.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# Compare 3 hour forecast precipitation accumulations to observations +# of 3 hour precipitation accumulation. Generate statistics of the results. + +############################################################################## +# Datasets +# -------- +# +# | **Forecast:** WRF 3 hour precipitation accumulation +# | **Observation:** NCEP Stage 2 National Precipitation Analysis - NPA (multi-sensor) 3 hour precipitation accumulation +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here for the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See the `Running METplus`_ section for more information. +# | + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes the METplus WaveletStat wrapper to search for +# files that are valid at a given run time and generate a command to run +# the MET tool wavelet_stat if all required files are found. + +############################################################################## +# METplus Workflow +# ---------------- +# +# WaveletStat is the only tool called in this example. It processes the following +# run times: +# +# | **Init:** 2005-08-07_0Z +# | **Forecast lead:** 12 hour +# | + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line: +# parm/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.conf + +############################################################################## +# MET Configuration +# ----------------- +# +# METplus sets environment variables based on user settings in the METplus configuration file. +# See :ref:`How METplus controls MET config file settings` for more details. +# +# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** +# +# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: +# :ref:`Overriding Unsupported MET config file settings` +# +# .. note:: See the :ref:`WaveletStat MET Configuration` section of the User's Guide for more information on the environment variables used in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/WaveletStatConfig_wrapped + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. +# + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for this use case will be found in wavelet_stat/2005080700 (relative to **OUTPUT_BASE**) +# and will contain the following files: +# +# * wavelet_stat_120000L_20050807_120000V_isc.txt +# * wavelet_stat_120000L_20050807_120000V.nc +# * wavelet_stat_120000L_20050807_120000V.ps +# * wavelet_stat_120000L_20050807_120000V.stat + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * WaveletStatToolUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# +# +# sphinx_gallery_thumbnail_path = '_static/met_tool_wrapper-WaveletStat.png' +# diff --git a/internal/tests/pytests/util/run_util/test_run_util.py b/internal/tests/pytests/util/run_util/test_run_util.py index 45d27fa46a..661d106c56 100644 --- a/internal/tests/pytests/util/run_util/test_run_util.py +++ b/internal/tests/pytests/util/run_util/test_run_util.py @@ -2,6 +2,7 @@ from unittest import mock import os +import re import produtil import metplus.util.run_util as ru @@ -141,23 +142,27 @@ def test_pre_run_setup_env_vars(): @pytest.mark.util def test_pre_run_setup_sed_file(capfd): - with mock.patch.object(ru.sys, 'exit') as mock_sys: - with mock.patch.object( - ru, - 'validate_config_variables', - return_value=(False, ['sed command 1', 'sed command 2']), - ): - actual = get_config_from_file('sed_run_util.conf') - mock_sys.assert_called_with(1) + with mock.patch.object( + ru, + 'validate_config_variables', + return_value=(False, ['sed command 1', 'sed command 2']), + ): + actual = get_config_from_file('sed_run_util.conf') + assert actual is None # check sed file is written correctly - sed_file = os.path.join(actual.getdir('OUTPUT_BASE'), 'sed_commands.txt') + out, err = capfd.readouterr() + sed_err_regex = r'.*Find/Replace commands have been generated in (.*)\n' + sed_file = None + match = re.match(sed_err_regex, err) + if match: + sed_file = match.group(1) + assert os.path.exists(sed_file) with open(sed_file, 'r') as f: assert f.read() == 'sed command 1\nsed command 2\n' # check correct errors logged - out, err = capfd.readouterr() expected_error_msgs = [ f'Find/Replace commands have been generated in {sed_file}', 'ERROR: Correct configuration variables and rerun. Exiting.', @@ -168,9 +173,8 @@ def test_pre_run_setup_sed_file(capfd): @pytest.mark.util def test_pre_run_setup_deprecated(capfd): - with mock.patch.object(ru.sys, 'exit') as mock_sys: - actual = get_config_from_file('sed_run_util.conf') - mock_sys.assert_called_with(1) + actual = get_config_from_file('sed_run_util.conf') + assert actual is None out, err = capfd.readouterr() @@ -185,9 +189,8 @@ def test_pre_run_setup_deprecated(capfd): @pytest.mark.util def test_pre_run_setup_no_install(capfd): - with mock.patch.object(ru.sys, 'exit') as mock_sys: - actual = get_config_from_file('no_install_run_util.conf') - mock_sys.assert_called_with(1) + actual = get_config_from_file('no_install_run_util.conf') + assert actual is None out, err = capfd.readouterr() assert 'MET_INSTALL_DIR must be set correctly to run METplus' in err @@ -433,7 +436,7 @@ def test_post_run_cleanup_no_errors(post_run_config): with mock.patch.object(ru, 'get_logfile_info', return_value='/log/file.log'): actual = ru.post_run_cleanup(post_run_config, 'fake_app', 0) - assert actual == None + assert actual is True for msg in expected_msgs: _check_log_info(post_run_config, [msg]) @@ -451,11 +454,9 @@ def test_post_run_cleanup_errors(post_run_config): ru, 'get_user_info', return_value='Allan H. Murphy' ) as mock_user: with mock.patch.object(ru, 'get_logfile_info', return_value='/log/file.log'): - with mock.patch.object(ru.sys, 'exit') as mock_sys: - actual = ru.post_run_cleanup(post_run_config, 'fake_app', 5) + actual = ru.post_run_cleanup(post_run_config, 'fake_app', 5) - mock_sys.assert_called_once_with(1) - assert actual == None + assert actual is False _check_log_info( post_run_config, ['Check the log file for more information: /log/file.log'] ) diff --git a/internal/tests/pytests/util/time_looping/test_time_looping.py b/internal/tests/pytests/util/time_looping/test_time_looping.py index de4c835779..53689acce0 100644 --- a/internal/tests/pytests/util/time_looping/test_time_looping.py +++ b/internal/tests/pytests/util/time_looping/test_time_looping.py @@ -366,6 +366,11 @@ def test_time_generator_error_check_beg_end(metplus_config, prefix): # _INCREMENT is less than 60 seconds config.set('config', f'{prefix}_INCREMENT', '10S') assert next(tl.time_generator(config)) is None + + # _INCREMENT is empty string + config.set('config', f'{prefix}_INCREMENT', '') + assert next(tl.time_generator(config)) is not None + config.set('config', f'{prefix}_INCREMENT', '1d') # _END time comes before _BEG time diff --git a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py index 572130433f..678753d59c 100644 --- a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -340,13 +340,21 @@ def handle_input_dir(config): ({'GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS': '120', }, { 'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {gaussian_radius = 120;}'}), - # 59 - ({'GEN_ENS_PROD_NMEP_SMOOTH_TYPE_METHOD': 'GAUSSIAN', }, - {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {type = [{method = GAUSSIAN;}];}'}), # 60 ({'GEN_ENS_PROD_NMEP_SMOOTH_TYPE_WIDTH': '1', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {type = [{width = 1;}];}'}), # 61 + ({'GEN_ENS_PROD_NMEP_SMOOTH_TYPE_METHOD': 'GAUSSIAN', }, + {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {type = [{method = GAUSSIAN;}];}'}), + # 62 old name without sub directory name + ({'GEN_ENS_PROD_NMEP_SMOOTH_METHOD': 'GAUSSIAN', }, + {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {type = [{method = GAUSSIAN;}];}'}), + # 63 both old and new name - should use value from new name + ({'GEN_ENS_PROD_NMEP_SMOOTH_METHOD': 'GAUSSIAN', + 'GEN_ENS_PROD_NMEP_SMOOTH_TYPE_METHOD': 'NEAREST' + }, + {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {type = [{method = NEAREST;}];}'}), + # 64 ({ 'GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH': '0.0', 'GEN_ENS_PROD_NMEP_SMOOTH_SHAPE': 'circle', @@ -362,13 +370,26 @@ def handle_input_dir(config): 'type = [{method = GAUSSIAN;width = 1;}];}' ) }), - # 62 + # 65 test from WOFS use case + ({ + 'GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH': '1.0', + 'GEN_ENS_PROD_NMEP_SMOOTH_SHAPE': 'SQUARE', + 'GEN_ENS_PROD_NMEP_SMOOTH_METHOD': 'NEAREST', + 'GEN_ENS_PROD_NMEP_SMOOTH_WIDTH': '1', + }, + { + 'METPLUS_NMEP_SMOOTH_DICT': ( + 'nmep_smooth = {vld_thresh = 1.0;shape = SQUARE;' + 'type = [{method = NEAREST;width = 1;}];}' + ) + }), + # 66 ({'GEN_ENS_PROD_ENS_MEMBER_IDS': '1,2,3,4', }, {'METPLUS_ENS_MEMBER_IDS': 'ens_member_ids = ["1", "2", "3", "4"];'}), - # 63 + # 67 ({'GEN_ENS_PROD_CONTROL_ID': '0', }, {'METPLUS_CONTROL_ID': 'control_id = "0";'}), - # 64 + # 68 ({'GEN_ENS_PROD_NORMALIZE': 'CLIMO_STD_ANOM', }, {'METPLUS_NORMALIZE': 'normalize = CLIMO_STD_ANOM;'}), diff --git a/internal/tests/pytests/wrappers/wavelet_stat/test_wavelet_stat.py b/internal/tests/pytests/wrappers/wavelet_stat/test_wavelet_stat.py new file mode 100644 index 0000000000..d944846bc6 --- /dev/null +++ b/internal/tests/pytests/wrappers/wavelet_stat/test_wavelet_stat.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 + +import pytest + +import os + +from metplus.wrappers.wavelet_stat_wrapper import WaveletStatWrapper + +fcst_dir = '/some/path/fcst' +obs_dir = '/some/path/obs' +fcst_name = 'APCP' +fcst_level = 'A03' +obs_name = 'APCP_03' +obs_level_no_quotes = '(*,*)' +obs_level = f'"{obs_level_no_quotes}"' +both_thresh = ' lt-0.5,gt-0.5 && lt0.5,gt0.5 ' +fcst_fmt = f'field = [{{ name="{fcst_name}"; level="{fcst_level}"; cat_thresh=[{both_thresh}]; }}];' +obs_fmt = (f'field = [{{ name="{obs_name}"; ' + f'level="{obs_level_no_quotes}"; cat_thresh=[{both_thresh}]; }}];') +time_fmt = '%Y%m%d%H' +run_times = ['2005080700', '2005080712'] + + +def set_minimum_config_settings(config): + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'WaveletStat') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', time_fmt) + config.set('config', 'INIT_BEG', run_times[0]) + config.set('config', 'INIT_END', run_times[-1]) + config.set('config', 'INIT_INCREMENT', '12H') + config.set('config', 'LEAD_SEQ', '12H') + config.set('config', 'LOOP_ORDER', 'times') + config.set('config', 'WAVELET_STAT_CONFIG_FILE', + '{PARM_BASE}/met_config/WaveletStatConfig_wrapped') + config.set('config', 'FCST_WAVELET_STAT_INPUT_DIR', fcst_dir) + config.set('config', 'OBS_WAVELET_STAT_INPUT_DIR', obs_dir) + config.set('config', 'FCST_WAVELET_STAT_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') + config.set('config', 'OBS_WAVELET_STAT_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d%H}/obs_file') + config.set('config', 'WAVELET_STAT_OUTPUT_DIR', + '{OUTPUT_BASE}/WaveletStat/output') + config.set('config', 'WAVELET_STAT_OUTPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}') + + config.set('config', 'FCST_VAR1_NAME', fcst_name) + config.set('config', 'FCST_VAR1_LEVELS', fcst_level) + config.set('config', 'OBS_VAR1_NAME', obs_name) + config.set('config', 'OBS_VAR1_LEVELS', obs_level) + config.set('config', 'BOTH_VAR1_THRESH', both_thresh) + + +@pytest.mark.parametrize( + 'config_overrides, expected_values', [ + # 0 generic FCST is prob + ({'FCST_IS_PROB': True}, + {'FCST_IS_PROB': True, 'OBS_IS_PROB': False}), + # 1 generic OBS is prob + ({'OBS_IS_PROB': True}, + {'FCST_IS_PROB': False, 'OBS_IS_PROB': True}), + # 2 generic FCST and OBS is prob + ({'FCST_IS_PROB': True, 'OBS_IS_PROB': True}, + {'FCST_IS_PROB': True, 'OBS_IS_PROB': True}), + # 3 generic FCST true, wrapper FCST false + ({'FCST_IS_PROB': True, 'FCST_WAVELET_STAT_IS_PROB': False}, + {'FCST_IS_PROB': False, 'OBS_IS_PROB': False}), + # 4 generic OBS true, wrapper OBS false + ({'OBS_IS_PROB': True, 'OBS_WAVELET_STAT_IS_PROB': False}, + {'FCST_IS_PROB': False, 'OBS_IS_PROB': False}), + # 5 generic FCST unset, wrapper FCST true + ({'FCST_WAVELET_STAT_IS_PROB': True}, + {'FCST_IS_PROB': True, 'OBS_IS_PROB': False}), + # 6 generic OBS unset, wrapper OBS true + ({'OBS_WAVELET_STAT_IS_PROB': True}, + {'FCST_IS_PROB': False, 'OBS_IS_PROB': True}), + # 7 generic FCST false, wrapper FCST true + ({'FCST_IS_PROB': False, 'FCST_WAVELET_STAT_IS_PROB': True}, + {'FCST_IS_PROB': True, 'OBS_IS_PROB': False}), + # 8 generic FCST true, wrapper FCST false + ({'FCST_IS_PROB': True, 'FCST_WAVELET_STAT_IS_PROB': False}, + {'FCST_IS_PROB': False, 'OBS_IS_PROB': False}), + ] +) +@pytest.mark.wrapper_b +def test_wavelet_stat_is_prob(metplus_config, config_overrides, expected_values): + config = metplus_config + + set_minimum_config_settings(config) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = WaveletStatWrapper(config) + assert wrapper.isOK + for key, expected_value in expected_values.items(): + assert expected_value == wrapper.c_dict[key] + + +@pytest.mark.parametrize( + 'config_overrides, env_var_values', [ + ({'MODEL': 'my_model'}, + {'METPLUS_MODEL': 'model = "my_model";'}), + + ({'WAVELET_STAT_DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + + ({'WAVELET_STAT_DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + + ({'OBTYPE': 'my_obtype'}, + {'METPLUS_OBTYPE': 'obtype = "my_obtype";'}), + + ({'WAVELET_STAT_REGRID_TO_GRID': 'FCST',}, + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + + ({'WAVELET_STAT_REGRID_METHOD': 'NEAREST',}, + {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), + + ({'WAVELET_STAT_REGRID_WIDTH': '1',}, + {'METPLUS_REGRID_DICT': 'regrid = {width = 1;}'}), + + ({'WAVELET_STAT_REGRID_VLD_THRESH': '0.5',}, + {'METPLUS_REGRID_DICT': 'regrid = {vld_thresh = 0.5;}'}), + + ({'WAVELET_STAT_REGRID_SHAPE': 'SQUARE',}, + {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + + ({'WAVELET_STAT_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'WAVELET_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'WAVELET_STAT_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + + ({'WAVELET_STAT_REGRID_TO_GRID': 'FCST', + 'WAVELET_STAT_REGRID_METHOD': 'NEAREST', + 'WAVELET_STAT_REGRID_WIDTH': '1', + 'WAVELET_STAT_REGRID_VLD_THRESH': '0.5', + 'WAVELET_STAT_REGRID_SHAPE': 'SQUARE', + 'WAVELET_STAT_REGRID_CONVERT': '2*x', + 'WAVELET_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', + 'WAVELET_STAT_REGRID_CENSOR_VAL': '12000,5000', + }, + {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' + )}), + + ({'WAVELET_STAT_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_CENSOR_THRESH': 'censor_thresh = [>12000, <5000];'}), + ({'WAVELET_STAT_CENSOR_VAL': '12000, 5000', }, + {'METPLUS_CENSOR_VAL': 'censor_val = [12000, 5000];'}), + + ({'WAVELET_STAT_MASK_MISSING_FLAG': 'NONE', }, + {'METPLUS_MASK_MISSING_FLAG': 'mask_missing_flag = NONE;'}), + + ({'WAVELET_STAT_GRID_DECOMP_FLAG': 'AUTO', }, + {'METPLUS_GRID_DECOMP_FLAG': 'grid_decomp_flag = AUTO;'}), + + ({'WAVELET_STAT_TILE_WIDTH': '0', }, + {'METPLUS_TILE_DICT': 'tile = {width = 0;}'}), + + ({'WAVELET_STAT_TILE_LOCATION_X_LL': '1', }, + {'METPLUS_TILE_DICT': 'tile = {location = [{x_ll = 1;}];}'}), + + ({'WAVELET_STAT_TILE_LOCATION_Y_LL': '1', }, + {'METPLUS_TILE_DICT': 'tile = {location = [{y_ll = 1;}];}'}), + + ({ + 'WAVELET_STAT_TILE_WIDTH': '1', + 'WAVELET_STAT_TILE_LOCATION1_X_LL': '1', + 'WAVELET_STAT_TILE_LOCATION1_Y_LL': '2', + 'WAVELET_STAT_TILE_LOCATION2_X_LL': '3', + 'WAVELET_STAT_TILE_LOCATION2_Y_LL': '4', + }, + {'METPLUS_TILE_DICT': 'tile = {width = 1;location = [{x_ll = 1;y_ll = 2;},{x_ll = 3;y_ll = 4;}];}'}), + + ({'WAVELET_STAT_WAVELET_TYPE': 'HAAR', }, + {'METPLUS_WAVELET_DICT': 'wavelet = {type = HAAR;}'}), + + ({'WAVELET_STAT_WAVELET_MEMBER': '2', }, + {'METPLUS_WAVELET_DICT': 'wavelet = {member = 2;}'}), + + ({ + 'WAVELET_STAT_WAVELET_TYPE': 'HAAR', + 'WAVELET_STAT_WAVELET_MEMBER': '2', + }, + {'METPLUS_WAVELET_DICT': 'wavelet = {type = HAAR;member = 2;}'}), + + ({'WAVELET_STAT_OUTPUT_FLAG_ISC': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {isc = STAT;}'}), + + ({'WAVELET_STAT_NC_PAIRS_FLAG_RAW': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {raw = TRUE;}'}), + + ({'WAVELET_STAT_NC_PAIRS_FLAG_DIFF': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {diff = TRUE;}'}), + + ({'WAVELET_STAT_NC_PAIRS_FLAG_RAW': 'TRUE', + 'WAVELET_STAT_NC_PAIRS_FLAG_DIFF': 'TRUE', + }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {raw = TRUE;diff = TRUE;}' + }), + + ({'WAVELET_STAT_FCST_RAW_PLOT_COLOR_TABLE': 'MET_BASE/colortables/met_default.ctable', }, + {'METPLUS_FCST_RAW_PLOT_DICT': 'fcst_raw_plot = {color_table = \"MET_BASE/colortables/met_default.ctable\";}'}), + + ({'WAVELET_STAT_FCST_RAW_PLOT_PLOT_MIN': '0.0', }, + {'METPLUS_FCST_RAW_PLOT_DICT': 'fcst_raw_plot = {plot_min = 0.0;}'}), + + ({'WAVELET_STAT_FCST_RAW_PLOT_PLOT_MAX': '1.0', }, + {'METPLUS_FCST_RAW_PLOT_DICT': 'fcst_raw_plot = {plot_max = 1.0;}'}), + + ({'WAVELET_STAT_FCST_RAW_PLOT_COLOR_TABLE': 'MET_BASE/colortables/met_default.ctable', + 'WAVELET_STAT_FCST_RAW_PLOT_PLOT_MIN': '0.0', + 'WAVELET_STAT_FCST_RAW_PLOT_PLOT_MAX': '1.0', + }, + {'METPLUS_FCST_RAW_PLOT_DICT': 'fcst_raw_plot = {color_table = \"MET_BASE/colortables/met_default.ctable\";plot_min = 0.0;plot_max = 1.0;}'}), + + ({'WAVELET_STAT_OBS_RAW_PLOT_PLOT_MIN': '0.0', }, + {'METPLUS_OBS_RAW_PLOT_DICT': 'obs_raw_plot = {plot_min = 0.0;}'}), + + ({'WAVELET_STAT_OBS_RAW_PLOT_PLOT_MAX': '1.0', }, + {'METPLUS_OBS_RAW_PLOT_DICT': 'obs_raw_plot = {plot_max = 1.0;}'}), + + ({'WAVELET_STAT_OBS_RAW_PLOT_COLOR_TABLE': 'MET_BASE/colortables/met_default.ctable', + 'WAVELET_STAT_OBS_RAW_PLOT_PLOT_MIN': '0.0', + 'WAVELET_STAT_OBS_RAW_PLOT_PLOT_MAX': '1.0', + }, + {'METPLUS_OBS_RAW_PLOT_DICT': 'obs_raw_plot = {color_table = \"MET_BASE/colortables/met_default.ctable\";plot_min = 0.0;plot_max = 1.0;}'}), + + ({'WAVELET_STAT_WVLT_PLOT_COLOR_TABLE': 'MET_BASE/colortables/met_default.ctable', }, + { + 'METPLUS_WVLT_PLOT_DICT': 'wvlt_plot = {color_table = \"MET_BASE/colortables/met_default.ctable\";}'}), + + ({'WAVELET_STAT_WVLT_PLOT_PLOT_MIN': '0.0', }, + {'METPLUS_WVLT_PLOT_DICT': 'wvlt_plot = {plot_min = 0.0;}'}), + + ({'WAVELET_STAT_WVLT_PLOT_PLOT_MAX': '1.0', }, + {'METPLUS_WVLT_PLOT_DICT': 'wvlt_plot = {plot_max = 1.0;}'}), + + ({'WAVELET_STAT_WVLT_PLOT_COLOR_TABLE': 'MET_BASE/colortables/NCL_colortables/BlWhRe.ctable', + 'WAVELET_STAT_WVLT_PLOT_PLOT_MIN': '0.0', + 'WAVELET_STAT_WVLT_PLOT_PLOT_MAX': '1.0', + }, + {'METPLUS_WVLT_PLOT_DICT': 'wvlt_plot = {color_table = \"MET_BASE/colortables/NCL_colortables/BlWhRe.ctable\";plot_min = 0.0;plot_max = 1.0;}'}), + + ({'WAVELET_STAT_OUTPUT_PREFIX': 'my_output_prefix'}, + {'METPLUS_OUTPUT_PREFIX': 'output_prefix = "my_output_prefix";'}), + + ({'FCST_WAVELET_STAT_FILE_TYPE': 'NETCDF_NCCF', }, + {'METPLUS_FCST_FILE_TYPE': 'file_type = NETCDF_NCCF;'}), + ({'OBS_WAVELET_STAT_FILE_TYPE': 'NETCDF_NCCF', }, + {'METPLUS_OBS_FILE_TYPE': 'file_type = NETCDF_NCCF;'}), + + ] +) +@pytest.mark.wrapper_b +def test_wavelet_stat_single_field(metplus_config, config_overrides, env_var_values): + + config = metplus_config + set_minimum_config_settings(config) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = WaveletStatWrapper(config) + assert wrapper.isOK + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + expected_cmds = [(f"{app_path} {verbosity} " + f"{fcst_dir}/2005080700/fcst_file_F012 " + f"{obs_dir}/2005080712/obs_file " + f"{config_file} -outdir {out_dir}/2005080712"), + (f"{app_path} {verbosity} " + f"{fcst_dir}/2005080712/fcst_file_F012 " + f"{obs_dir}/2005080800/obs_file " + f"{config_file} -outdir {out_dir}/2005080800"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + + assert len(all_cmds) == len(expected_cmds) + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd + + # check that environment variables were set properly + # including deprecated env vars (not in wrapper env var keys) + for env_var_key in env_var_keys: + print(f"ENV VAR: {env_var_key}") + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + actual_value = match.split('=', 1)[1] + if env_var_key == 'METPLUS_FCST_FIELD': + assert actual_value == fcst_fmt + elif env_var_key == 'METPLUS_OBS_FIELD': + assert actual_value == obs_fmt + else: + assert env_var_values.get(env_var_key, '') == actual_value + + +@pytest.mark.wrapper_b +def test_get_config_file(metplus_config): + fake_config_name = '/my/config/file' + + config = metplus_config + default_config_file = os.path.join(config.getdir('PARM_BASE'), + 'met_config', + 'WaveletStatConfig_wrapped') + + wrapper = WaveletStatWrapper(config) + assert wrapper.c_dict['CONFIG_FILE'] == default_config_file + + config.set('config', 'WAVELET_STAT_CONFIG_FILE', fake_config_name) + wrapper = WaveletStatWrapper(config) + assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name diff --git a/internal/tests/use_cases/all_use_cases.txt b/internal/tests/use_cases/all_use_cases.txt index 875caec30d..235c910c71 100644 --- a/internal/tests/use_cases/all_use_cases.txt +++ b/internal/tests/use_cases/all_use_cases.txt @@ -62,6 +62,7 @@ Category: met_tool_wrapper 60::PointStat_python_embedding_obs:: met_tool_wrapper/PointStat/PointStat_python_embedding_obs.conf 61::PlotPointObs:: met_tool_wrapper/PlotPointObs/PlotPointObs.conf 62::TCDiag:: met_tool_wrapper/TCDiag/TCDiag.conf +63::WaveletStat:: met_tool_wrapper/WaveletStat/WaveletStat.conf Category: air_quality_and_comp 0::EnsembleStat_fcstICAP_obsMODIS_aod::model_applications/air_quality_and_comp/EnsembleStat_fcstICAP_obsMODIS_aod.conf diff --git a/metplus/VERSION b/metplus/VERSION index 9d9136f3d9..23df389ae6 100644 --- a/metplus/VERSION +++ b/metplus/VERSION @@ -1 +1 @@ -6.0.0-beta2-dev +6.0.0-beta3-dev diff --git a/metplus/util/constants.py b/metplus/util/constants.py index 9a330a42e7..38f133a0e0 100644 --- a/metplus/util/constants.py +++ b/metplus/util/constants.py @@ -43,6 +43,7 @@ 'tcstat': 'TCStat', 'usage': 'Usage', 'userscript': 'UserScript', + 'waveletstat': 'WaveletStat', } # supported file extensions that will automatically be uncompressed diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index 7cf33564e1..da8268dbe5 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -13,6 +13,7 @@ from .config_metplus import parse_var_list from .field_util import format_all_field_info + class METConfig: """! Stores information for a member of a MET config variables that can be used to set the value, the data type of the item, @@ -195,6 +196,11 @@ def add_met_config_dict(config, app_name, output_dict, dict_name, items): for nickname in nicknames: metplus_configs.append(nickname) + # if list of dictionaries (dictlist) + elif 'list' in data_type: + children = get_met_config_dict_list(config, app_name, name, kids, parent=dict_name) + if not children: + continue # if dictionary, read get children from MET config else: children = [] @@ -263,15 +269,22 @@ def add_met_config_item(config, item, output_dict, depth=0): # handle dictionary or dictionary list item if 'dict' in item.data_type: tmp_dict = {} - for child in item.children: - if not add_met_config_item(config, child, tmp_dict, - depth=depth+1): + # handle list of dictionaries (dictlist) + if isinstance(item.children, dict): + dict_string = format_met_config_dict_list(config, item.name, item.children) + if dict_string is None: return False + # handle dictionary with list of children + else: + for child in item.children: + if not add_met_config_item(config, child, tmp_dict, + depth=depth+1): + return False - dict_string = format_met_config(item.data_type, - tmp_dict, - item.name, - keys=None) + dict_string = format_met_config(item.data_type, + tmp_dict, + item.name, + keys=None) # if handling dict MET config that is not nested inside another if not depth and item.data_type == 'dict': @@ -293,8 +306,7 @@ def add_met_config_item(config, item, output_dict, depth=0): **item.extra_args) -def add_met_config_dict_list(config, app_name, output_dict, dict_name, - dict_items): +def get_met_config_dict_list(config, app_name, dict_name, dict_items, parent=None): """! Read METplusConfig and format MET config variables that are a list of dictionaries. Sets value in output dict with key starting with METPLUS_. @@ -305,29 +317,34 @@ def add_met_config_dict_list(config, app_name, output_dict, dict_name, @param dict_items dictionary where the key is name of variable inside MET dictionary and the value is info about the item (see parse_item_info function for more information) + @param parent optional name of dictionary that contains the item (nested + dictionaries) """ - search_string = f'{app_name}_{dict_name}'.upper() - regex = r'^' + search_string + r'(\d+)_(\w+)$' - indices = find_indices_in_config_section(regex, config, - index_index=1, - id_index=2) + regex_end = r'(\d*)_(\w+)$' + if not parent: + search_string = f'{app_name}_{dict_name}'.upper() + regex = r'^' + search_string + regex_end + indices = find_indices_in_config_section(regex, config, + index_index=1, + id_index=2) + else: + search_string = f'{app_name}_{parent}_{dict_name}'.upper() + regex = r'^' + search_string + regex_end + indices = find_indices_in_config_section(regex, config, + index_index=1, + id_index=2) + # if no indices were found, try again excluding sub dict name + if not indices: + search_string = f'{app_name}_{parent}'.upper() + regex = r'^' + search_string + regex_end + indices = find_indices_in_config_section(regex, config, + index_index=1, + id_index=2) all_met_config_items = {} - is_ok = True - for index, items in indices.items(): + for index, _ in indices.items(): # read all variables for each index - met_config_items = {} - - # check if any variable found doesn't match valid variables - not_in_dict = [item for item in items - if item.lower() not in dict_items] - if any(not_in_dict): - for item in not_in_dict: - config.logger.error("Invalid variable: " - f"{search_string}{index}_{item}") - is_ok = False - continue - + met_config_items = [] for name, item_info in dict_items.items(): data_type, extra, kids, nicknames = _parse_item_info(item_info) metplus_configs = [f'{search_string}{index}_{name.upper()}'] @@ -338,22 +355,43 @@ def add_met_config_dict_list(config, app_name, output_dict, dict_name, extra_args=extra_args, ) - if not add_met_config_item(config, item, met_config_items): - is_ok = False + met_config_items.append(item) - dict_string = format_met_config('dict', - met_config_items, - name='') - all_met_config_items[index] = dict_string + all_met_config_items[index] = met_config_items + + return all_met_config_items + + +def add_met_config_dict_list(config, app_name, output_dict, dict_name, + dict_items): + is_ok = True + all_met_configs = get_met_config_dict_list(config, app_name, dict_name, dict_items) + if all_met_configs is None: + return False + output_string = format_met_config_dict_list(config, dict_name, all_met_configs) + if output_string is None: + is_ok = False - # format list of dictionaries - output_string = format_met_config('list', - all_met_config_items, - dict_name) output_dict[f'METPLUS_{dict_name.upper()}_LIST'] = output_string return is_ok +def format_met_config_dict_list(config, dict_name, all_met_configs): + all_met_config_items = {} + for index, met_configs in all_met_configs.items(): + met_config_items = {} + for met_config in met_configs: + if not add_met_config_item(config, met_config, met_config_items): + return None + + dict_string = format_met_config('dict', met_config_items, name='') + all_met_config_items[index] = dict_string + + # format list of dictionaries + output_string = format_met_config('list', all_met_config_items, dict_name) + return output_string + + def format_met_config(data_type, c_dict, name, keys=None): """! Return formatted variable named with any if they are set to a value. If none of the items are set, return empty string diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index 994c68d7b5..be0d5fd867 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -163,11 +163,11 @@ def pre_run_setup(config_inputs): logger.error("Correct configuration variables and rerun. Exiting.") logger.info(f"Check the log file for more information: {log_file}") - sys.exit(1) + return None if not config.getdir('MET_INSTALL_DIR', must_exist=True): logger.error('MET_INSTALL_DIR must be set correctly to run METplus') - sys.exit(1) + return None # handle dir to write temporary files handle_tmp_dir(config) @@ -335,7 +335,7 @@ def post_run_cleanup(config, app_name, total_errors): # print success log message if terminal does not include INFO if not log_terminal_includes_info(config): print(success_log) - return + return True error_msg = (f'{app_name} has finished running{user_string} ' f'but had {total_errors} error') @@ -344,4 +344,4 @@ def post_run_cleanup(config, app_name, total_errors): error_msg += '.' logger.error(error_msg) logger.info(log_message) - sys.exit(1) + return False diff --git a/metplus/util/time_looping.py b/metplus/util/time_looping.py index de7def9dc3..045c6e6ced 100644 --- a/metplus/util/time_looping.py +++ b/metplus/util/time_looping.py @@ -58,9 +58,12 @@ def time_generator(config): # if list is not provided, use _BEG, _END, and _INCREMENT start_string = config.getraw('config', f'{prefix}_BEG') end_string = config.getraw('config', f'{prefix}_END', start_string) - time_interval = get_relativedelta( - config.getstr('config', f'{prefix}_INCREMENT', '60') - ) + + time_interval = config.getstr('config', f'{prefix}_INCREMENT', '60') + # if [INIT/VALID]_INCREMENT is an empty string, set it to prevent crash + if not time_interval: + time_interval = '60' + time_interval = get_relativedelta(time_interval) start_dt = _get_current_dt(start_string, time_format, diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 13ee4b5a38..eaeb81205c 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -757,7 +757,7 @@ def _get_closest_files(self, data_dir, template, valid_time, "%Y%m%d%H%M%S").strftime("%s")) # step through all files under input directory in sorted order - for dirpath, _, all_files in os.walk(data_dir): + for dirpath, _, all_files in os.walk(data_dir, followlinks=True): for filename in sorted(all_files): fullpath = os.path.join(dirpath, filename) @@ -1399,9 +1399,6 @@ def get_output_prefix(self, time_info=None, set_env_vars=True): output_prefix_fmt = f'output_prefix = "{output_prefix}";' self.env_var_dict['METPLUS_OUTPUT_PREFIX'] = output_prefix_fmt - # set old method of setting OUTPUT_PREFIX - self.add_env_var('OUTPUT_PREFIX', output_prefix) - return output_prefix def handle_climo_dict(self): diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index 661f32cc3e..1de97bf155 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -78,6 +78,7 @@ class EnsembleStatWrapper(CompareGriddedWrapper): 'MODEL', 'OBTYPE', 'REGRID_TO_GRID', + 'OUTPUT_PREFIX', ] OUTPUT_FLAGS = [ diff --git a/metplus/wrappers/grid_stat_wrapper.py b/metplus/wrappers/grid_stat_wrapper.py index 95245dd1f8..bc60e2b7e4 100755 --- a/metplus/wrappers/grid_stat_wrapper.py +++ b/metplus/wrappers/grid_stat_wrapper.py @@ -73,6 +73,7 @@ class GridStatWrapper(CompareGriddedWrapper): 'NEIGHBORHOOD_WIDTH', 'NEIGHBORHOOD_SHAPE', 'NEIGHBORHOOD_COV_THRESH', + 'OUTPUT_PREFIX', ] OUTPUT_FLAGS = [ diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 49431c4712..ab856a1432 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -89,6 +89,7 @@ class MODEWrapper(CompareGriddedWrapper): 'OBS_MERGE_THRESH', 'FCST_MERGE_FLAG', 'OBS_MERGE_FLAG', + 'OUTPUT_PREFIX', ] WEIGHTS = { diff --git a/metplus/wrappers/mtd_wrapper.py b/metplus/wrappers/mtd_wrapper.py index 34bcc647e0..aa3a46dab5 100755 --- a/metplus/wrappers/mtd_wrapper.py +++ b/metplus/wrappers/mtd_wrapper.py @@ -55,6 +55,7 @@ class MTDWrapper(CompareGriddedWrapper): 'MIN_VOLUME', 'FCST_FILE_TYPE', 'OBS_FILE_TYPE', + 'OUTPUT_PREFIX', ] def __init__(self, config, instance=None): diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index 673fbb6d9c..e150ab4a25 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -63,6 +63,7 @@ class PointStatWrapper(CompareGriddedWrapper): 'POINT_STAT_GRID', 'POINT_STAT_STATION_ID', 'POINT_STAT_MESSAGE_TYPE', + 'OUTPUT_PREFIX', 'METPLUS_MASK_GRID', # deprecated in v5.1.0 'METPLUS_MASK_POLY', # deprecated in v5.1.0 'METPLUS_MASK_SID', # deprecated in v5.1.0 diff --git a/metplus/wrappers/tc_diag_wrapper.py b/metplus/wrappers/tc_diag_wrapper.py index 52797d9824..e72f0efffa 100755 --- a/metplus/wrappers/tc_diag_wrapper.py +++ b/metplus/wrappers/tc_diag_wrapper.py @@ -14,7 +14,7 @@ from ..util import time_util from . import RuntimeFreqWrapper -from ..util import do_string_sub, skip_time, get_lead_sequence +from ..util import do_string_sub, get_lead_sequence from ..util import parse_var_list, sub_var_list, getlist from ..util import find_indices_in_config_section from ..util.met_config import add_met_config_dict_list @@ -67,6 +67,11 @@ class TCDiagWrapper(RuntimeFreqWrapper): 'METPLUS_ONE_TIME_PER_FILE_FLAG', ] + # deprecated env vars that are no longer supported in the wrapped MET conf + DEPRECATED_WRAPPER_ENV_VAR_KEYS = [ + 'OUTPUT_PREFIX', + ] + def __init__(self, config, instance=None): self.app_name = "tc_diag" self.app_path = os.path.join(config.getdir('MET_BIN_DIR'), diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index c7b5a7cf9c..6df53af8f0 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -20,12 +20,11 @@ import datetime import glob -from ..util import getlist, get_lead_sequence, skip_time, mkdir_p +from ..util import getlist, mkdir_p from ..util import ti_calculate from ..util import do_string_sub from ..util import get_tags, find_indices_in_config_section from ..util.met_config import add_met_config_dict_list -from ..util import time_generator, log_runtime_banner, add_to_time_input from . import RuntimeFreqWrapper '''!@namespace TCPairsWrapper diff --git a/metplus/wrappers/wavelet_stat_wrapper.py b/metplus/wrappers/wavelet_stat_wrapper.py new file mode 100755 index 0000000000..2e824b4de9 --- /dev/null +++ b/metplus/wrappers/wavelet_stat_wrapper.py @@ -0,0 +1,180 @@ +''' +Program Name: wavelet_stat_wrapper.py +Contact(s): George McCabe +Abstract: +History Log: Initial version +Usage: +Parameters: None +Input Files: +Output Files: +Condition codes: 0 for success, 1 for failure +''' + +import os + +from . import CompareGriddedWrapper + +# pylint:disable=pointless-string-statement +"""!@namespace WaveletStatWrapper +@brief Wraps the MET tool wavelet_stat to compare gridded datasets +@endcode +""" + + +class WaveletStatWrapper(CompareGriddedWrapper): + """!Wraps the MET tool wavelet_stat to compare gridded datasets""" + + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + + WRAPPER_ENV_VAR_KEYS = [ + 'METPLUS_MODEL', + 'METPLUS_DESC', + 'METPLUS_OBTYPE', + 'METPLUS_REGRID_DICT', + 'METPLUS_FCST_FILE_TYPE', + 'METPLUS_FCST_FIELD', + 'METPLUS_OBS_FILE_TYPE', + 'METPLUS_OBS_FIELD', + 'METPLUS_CENSOR_THRESH', + 'METPLUS_CENSOR_VAL', + 'METPLUS_MASK_MISSING_FLAG', + 'METPLUS_GRID_DECOMP_FLAG', + 'METPLUS_TILE_DICT', + 'METPLUS_WAVELET_DICT', + 'METPLUS_OUTPUT_FLAG_DICT', + 'METPLUS_NC_PAIRS_FLAG_DICT', + 'METPLUS_PS_PLOT_FLAG', + 'METPLUS_FCST_RAW_PLOT_DICT', + 'METPLUS_OBS_RAW_PLOT_DICT', + 'METPLUS_WVLT_PLOT_DICT', + 'METPLUS_OUTPUT_PREFIX', + ] + + OUTPUT_FLAGS = [ + 'isc', + ] + + NC_PAIRS_FLAGS = [ + 'raw', + 'diff', + ] + + def __init__(self, config, instance=None): + self.app_name = 'wavelet_stat' + self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), + self.app_name) + super().__init__(config, instance=instance) + + def create_c_dict(self): + c_dict = super().create_c_dict() + app = self.app_name.upper() + c_dict['VERBOSITY'] = self.config.getstr('config', + f'LOG_{app}_VERBOSITY', + c_dict['VERBOSITY']) + + # get the MET config file path or use default + c_dict['CONFIG_FILE'] = self.get_config_file('WaveletStatConfig_wrapped') + + c_dict['OBS_INPUT_DIR'] = self.config.getdir(f'OBS_{app}_INPUT_DIR', '') + c_dict['OBS_INPUT_TEMPLATE'] = ( + self.config.getraw('config', f'OBS_{app}_INPUT_TEMPLATE') + ) + if not c_dict['OBS_INPUT_TEMPLATE']: + self.log_error(f"OBS_{app}_INPUT_TEMPLATE required to run") + + c_dict['OBS_INPUT_DATATYPE'] = ( + self.config.getstr('config', f'OBS_{app}_INPUT_DATATYPE', '') + ) + + c_dict['FCST_INPUT_DIR'] = self.config.getdir(f'FCST_{app}_INPUT_DIR', '') + c_dict['FCST_INPUT_TEMPLATE'] = ( + self.config.getraw('config', f'FCST_{app}_INPUT_TEMPLATE') + ) + if not c_dict['FCST_INPUT_TEMPLATE']: + self.log_error(f"FCST_{app}_INPUT_TEMPLATE required to run") + + c_dict['FCST_INPUT_DATATYPE'] = ( + self.config.getstr('config', f'FCST_{app}_INPUT_DATATYPE', '') + ) + + c_dict['OUTPUT_DIR'] = self.config.getdir(f'{app}_OUTPUT_DIR', '') + if not c_dict['OUTPUT_DIR']: + self.log_error(f"Must set {app}_OUTPUT_DIR") + + c_dict['OUTPUT_TEMPLATE'] = ( + self.config.getraw('config', f'{app}_OUTPUT_TEMPLATE') + ) + c_dict['ONCE_PER_FIELD'] = ( + self.config.getbool('config', + f'{app}_ONCE_PER_FIELD', + False) + ) + + c_dict['FCST_PROB_THRESH'] = ( + self.config.getstr('config', + f'FCST_{app}_PROB_THRESH', '==0.1') + ) + c_dict['OBS_PROB_THRESH'] = ( + self.config.getstr('config', + f'OBS_{app}_PROB_THRESH', '==0.1') + ) + + c_dict['ALLOW_MULTIPLE_FILES'] = False + + # MET config variables + self.add_met_config(name='censor_thresh', data_type='list', + extra_args={'remove_quotes': True}) + + self.add_met_config(name='censor_val', data_type='list', + extra_args={'remove_quotes': True}) + + self.add_met_config(name='mask_missing_flag', data_type='string', + extra_args={'remove_quotes': True, + 'uppercase': True}) + + self.add_met_config(name='grid_decomp_flag', data_type='string', + extra_args={'remove_quotes': True, + 'uppercase': True}) + + self.handle_flags('output') + self.handle_flags('nc_pairs') + + self.add_met_config(name='file_type', data_type='string', + env_var_name='FCST_FILE_TYPE', + metplus_configs=[f'{app}_FCST_FILE_TYPE', + f'FCST_{app}_FILE_TYPE', + f'{app}_FILE_TYPE'], + extra_args={'remove_quotes': True, + 'uppercase': True}) + + self.add_met_config(name='file_type', data_type='string', + env_var_name='OBS_FILE_TYPE', + metplus_configs=[f'{app}_OBS_FILE_TYPE', + f'OBS_{app}_FILE_TYPE', + f'{app}_FILE_TYPE'], + extra_args={'remove_quotes': True, + 'uppercase': True}) + + self.add_met_config_dict('tile', { + 'width': 'int', + 'location': ('dictlist', '', {'x_ll': 'int', 'y_ll': 'int'}) + }) + + self.add_met_config_dict('wavelet', { + 'type': ('string', 'remove_quotes,uppercase'), + 'member': 'int' + }) + + self.add_met_config(name='ps_plot_flag', data_type='bool') + + for config_name in ('fcst_raw_plot', 'obs_raw_plot', 'wvlt_plot'): + self.add_met_config_dict(config_name, { + 'color_table': 'string', + 'plot_min': 'float', + 'plot_max': 'float', + }) + + self.add_met_config(name='output_prefix', data_type='string') + + return c_dict diff --git a/parm/met_config/WaveletStatConfig_wrapped b/parm/met_config/WaveletStatConfig_wrapped new file mode 100644 index 0000000000..9f3f0e17f8 --- /dev/null +++ b/parm/met_config/WaveletStatConfig_wrapped @@ -0,0 +1,128 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Wavelet-Stat configuration file. +// +// For additional information, please see the MET Users Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +//model = +${METPLUS_MODEL} + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +//desc = +${METPLUS_DESC} + +// +// Output observation type to be written +// +//obtype = +${METPLUS_OBTYPE} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// May be set separately in each "field" entry +// +//regrid = { +${METPLUS_REGRID_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// May be set separately in each "field" entry +// +//censor_thresh = +${METPLUS_CENSOR_THRESH} +//censor_val = +${METPLUS_CENSOR_VAL} + +// +// Forecast and observation fields to be verified +// +fcst = { + ${METPLUS_FCST_FILE_TYPE} + ${METPLUS_FCST_FIELD} +} +obs = { + ${METPLUS_OBS_FILE_TYPE} + ${METPLUS_OBS_FIELD} +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Handle missing data +// +//mask_missing_flag = +${METPLUS_MASK_MISSING_FLAG} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Decompose the field into dyadic tiles +// +//grid_decomp_flag = +${METPLUS_GRID_DECOMP_FLAG} + +//tile = { +${METPLUS_TILE_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Wavelet to be used for the decomposition +// +//wavelet = { +${METPLUS_WAVELET_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +//output_flag = { +${METPLUS_OUTPUT_FLAG_DICT} + +// +// NetCDF matched pairs and PostScript output files +// +//nc_pairs_flag = { +${METPLUS_NC_PAIRS_FLAG_DICT} + +//ps_plot_flag = +${METPLUS_PS_PLOT_FLAG} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Plotting information +// +met_data_dir = "MET_BASE"; + +//fcst_raw_plot = { +${METPLUS_FCST_RAW_PLOT_DICT} + +//obs_raw_plot = { +${METPLUS_OBS_RAW_PLOT_DICT} + +//wvlt_plot = { +${METPLUS_WVLT_PLOT_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +//output_prefix = +${METPLUS_OUTPUT_PREFIX} + +//version = "V11.1.0"; + +//////////////////////////////////////////////////////////////////////////////// + +${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf index 3c4a300a40..55ff515571 100644 --- a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf +++ b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf @@ -123,8 +123,8 @@ GEN_ENS_PROD_ENS_THRESH = 0.8 #GEN_ENS_PROD_NMEP_SMOOTH_SHAPE = CIRCLE #GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX = 81.27 #GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS = 120 -#GEN_ENS_PROD_NMEP_SMOOTH_METHOD = GAUSSIAN -#GEN_ENS_PROD_NMEP_SMOOTH_WIDTH = 1 +#GEN_ENS_PROD_NMEP_SMOOTH_TYPE_METHOD = GAUSSIAN +#GEN_ENS_PROD_NMEP_SMOOTH_TYPE_WIDTH = 1 #GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME = #GEN_ENS_PROD_CLIMO_MEAN_FIELD = diff --git a/parm/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.conf b/parm/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.conf new file mode 100644 index 0000000000..40bbd5cac0 --- /dev/null +++ b/parm/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.conf @@ -0,0 +1,133 @@ +[config] + +# Documentation for this use case can be found at +# https://metplus.readthedocs.io/en/latest/generated/met_tool_wrapper/WaveletStat/WaveletStat.html + +# For additional information, please see the METplus Users Guide. +# https://metplus.readthedocs.io/en/latest/Users_Guide + +### +# Processes to run +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list +### + +PROCESS_LIST = WaveletStat + + +### +# Time Info +# LOOP_BY options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +# LEAD_SEQ is the list of forecast leads to process +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#timing-control +### + +LOOP_BY = INIT +INIT_TIME_FMT = %Y%m%d%H +INIT_BEG=2005080700 +INIT_END=2005080700 +INIT_INCREMENT = 12H + +LEAD_SEQ = 12 + + +### +# File I/O +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info +### + +FCST_WAVELET_STAT_INPUT_DIR = {INPUT_BASE}/met_test/data/sample_fcst +FCST_WAVELET_STAT_INPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/wrfprs_ruc13_{lead?fmt=%HH}.tm00_G212 + +OBS_WAVELET_STAT_INPUT_DIR = {INPUT_BASE}/met_test/new +OBS_WAVELET_STAT_INPUT_TEMPLATE = ST2ml{valid?fmt=%Y%m%d%H}_A03h.nc + +WAVELET_STAT_OUTPUT_DIR = {OUTPUT_BASE}/wavelet_stat +WAVELET_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H} + + +### +# Field Info +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#field-info +### + +MODEL = WRF +OBTYPE = MC_PCP + +WAVELET_STAT_ONCE_PER_FIELD = False + +#FCST_IS_PROB = false + +FCST_VAR1_NAME = APCP +FCST_VAR1_LEVELS = A03 +FCST_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + +OBS_VAR1_NAME = APCP_03 +OBS_VAR1_LEVELS = "(*,*)" +OBS_VAR1_THRESH = gt12.7, gt25.4, gt50.8, gt76.2 + + +### +# GridStat Settings (optional) +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#gridstat +### + +#LOG_WAVELET_STAT_VERBOSITY = 2 + +WAVELET_STAT_CONFIG_FILE = {PARM_BASE}/met_config/WaveletStatConfig_wrapped + +#FCST_WAVELET_STAT_FILE_TYPE = +#OBS_WAVELET_STAT_FILE_TYPE = + +#FCST_WAVELET_STAT_FILE_WINDOW_BEGIN = 0 +#FCST_WAVELET_STAT_FILE_WINDOW_END = 0 +#OBS_WAVELET_STAT_FILE_WINDOW_BEGIN = 0 +#OBS_WAVELET_STAT_FILE_WINDOW_END = 0 + +#WAVELET_STAT_MODEL = +#WAVELET_STAT_DESC = +#WAVELET_STAT_OBTYPE = + +#WAVELET_STAT_REGRID_TO_GRID = +#WAVELET_STAT_REGRID_METHOD = +#WAVELET_STAT_REGRID_WIDTH = +#WAVELET_STAT_REGRID_VLD_THRESH = +#WAVELET_STAT_REGRID_SHAPE = + +#WAVELET_STAT_CENSOR_THRESH = +#WAVELET_STAT_CENSOR_VAL = + +#WAVELET_STAT_MASK_MISSING_FLAG = + +#WAVELET_STAT_GRID_DECOMP_FLAG = + +#WAVELET_STAT_TILE_WIDTH = +#WAVELET_STAT_TILE_LOCATION1_X_LL = +#WAVELET_STAT_TILE_LOCATION1_Y_LL = + +#WAVELET_STAT_WAVELET_TYPE = +#WAVELET_STAT_WAVELET_MEMBER = + +WAVELET_STAT_OUTPUT_FLAG_ISC = STAT + +#WAVELET_STAT_NC_PAIRS_FLAG_RAW = +#WAVELET_STAT_NC_PAIRS_FLAG_DIFF = + +#WAVELET_STAT_PS_PLOT_FLAG = + +#WAVELET_STAT_FCST_RAW_PLOT_COLOR_TABLE = +#WAVELET_STAT_FCST_RAW_PLOT_PLOT_MIN = +#WAVELET_STAT_FCST_RAW_PLOT_PLOT_MAX = + +#WAVELET_STAT_OBS_RAW_PLOT_COLOR_TABLE = +#WAVELET_STAT_OBS_RAW_PLOT_PLOT_MIN = +#WAVELET_STAT_OBS_RAW_PLOT_PLOT_MAX = + +#WAVELET_STAT_WVLT_PLOT_COLOR_TABLE = +#WAVELET_STAT_WVLT_PLOT_PLOT_MIN = +#WAVELET_STAT_WVLT_PLOT_PLOT_MAX = + +#WAVELET_STAT_OUTPUT_PREFIX = diff --git a/ush/run_metplus.py b/ush/run_metplus.py index 66dbb038a2..482c67e81f 100755 --- a/ush/run_metplus.py +++ b/ush/run_metplus.py @@ -26,9 +26,7 @@ import produtil.setup -from metplus.util import metplus_check from metplus.util import pre_run_setup, run_metplus, post_run_cleanup -from metplus import __version__ as metplus_version '''!@namespace run_metplus Main script the processes all the tasks in the PROCESS_LIST @@ -42,6 +40,8 @@ def main(): config_inputs = get_config_inputs_from_command_line() config = pre_run_setup(config_inputs) + if not config: + return False # warn if calling master_metplus.py script_name = os.path.basename(__file__) @@ -52,7 +52,7 @@ def main(): total_errors = run_metplus(config) - post_run_cleanup(config, 'METplus', total_errors) + return post_run_cleanup(config, 'METplus', total_errors) def usage(): @@ -90,7 +90,6 @@ def get_config_inputs_from_command_line(): for help_arg in help_args: if help_arg in sys.argv: usage() - sys.exit(0) # pull out command line arguments config_inputs = [] @@ -121,7 +120,8 @@ def get_config_inputs_from_command_line(): if __name__ == "__main__": try: produtil.setup.setup(send_dbn=False, jobname='run-METplus') - main() + if not main(): + sys.exit(1) except Exception as exc: print(traceback.format_exc()) print('ERROR: run_metplus failed: %s' % exc)