diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index d3d398b86b..ea21353e1f 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -189,4 +189,252 @@ METplus Version 5.0.0 Beta 1 Release Notes (2022-06-22) METplus Wrappers Upgrade Instructions ===================================== -Coming Soon! +EnsembleStat/GenEnsProd +----------------------- + +Note: If :ref:`ensemble_stat_wrapper` is not found in the :term:`PROCESS_LIST` +for any use cases, then this section is not relevant. + +The METplus v5.0.0 coordinated release includes changes that remove ensemble +product generation from EnsembleStat. GenEnsProd is now required to generate +ensemble products. There are 3 situations listed below that require slightly +different modifications. + +Case 1: EnsembleStat only generating ensemble products +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If the use case had been calling EnsembleStat **WITHOUT** the -grid_obs or +-point_obs command line options, we can assume it was only doing ensemble +post-processing. +That call to EnsembleStat should be replaced with a call to GenEnsProd instead. + +Rename the following variables: +""""""""""""""""""""""""""""""" + +FCST_ENSEMBLE_STAT_INPUT_DIR => GEN_ENS_PROD_INPUT_DIR + +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE => GEN_ENS_PROD_INPUT_TEMPLATE + +ENSEMBLE_STAT_OUTPUT_DIR => GEN_ENS_PROD_OUTPUT_DIR + +ENSEMBLE_STAT_OUTPUT_TEMPLATE => GEN_ENS_PROD_OUTPUT_TEMPLATE +**and add full filename template for NetCDF output file to end of value**, i.e. +/gen_ens_prod_{valid?fmt=%Y%m%d_%H%M%S}V_ens.nc + +ENSEMBLE_STAT_N_MEMBERS => GEN_ENS_PROD_N_MEMBERS + +ENSEMBLE_STAT_ENS_THRESH => GEN_ENS_PROD_ENS_THRESH + +ENSEMBLE_STAT_ENS_VLD_THRESH => GEN_ENS_PROD_VLD_THRESH + +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON => GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN => GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV => GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV +ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS => GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS +ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS => GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS +ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN => GEN_ENS_PROD_ENSEMBLE_FLAG_MIN +ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX => GEN_ENS_PROD_ENSEMBLE_FLAG_MAX +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE => GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT => GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY => GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY +ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP => GEN_ENS_PROD_ENSEMBLE_FLAG_NEP +ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP => GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP + +ENSEMBLE_STAT_REGRID_TO_GRID => GEN_ENS_PROD_REGRID_TO_GRID +ENSEMBLE_STAT_REGRID_METHOD => GEN_ENS_PROD_REGRID_METHOD +ENSEMBLE_STAT_REGRID_WIDTH => GEN_ENS_PROD_REGRID_WIDTH +ENSEMBLE_STAT_REGRID_VLD_THRESH => GEN_ENS_PROD_REGRID_VLD_THRESH +ENSEMBLE_STAT_REGRID_SHAPE => GEN_ENS_PROD_REGRID_SHAPE +ENSEMBLE_STAT_NBRHD_PROB_WIDTH => GEN_ENS_PROD_NBRHD_PROB_WIDTH +ENSEMBLE_STAT_NBRHD_PROB_SHAPE => GEN_ENS_PROD_NBRHD_PROB_SHAPE +ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH => GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH +ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH => GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH +ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE => GEN_ENS_PROD_NMEP_SMOOTH_SHAPE +ENSEMBLE_STAT_NMEP_SMOOTH_METHOD => GEN_ENS_PROD_NMEP_SMOOTH_METHOD +ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH => GEN_ENS_PROD_NMEP_SMOOTH_WIDTH +ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX => GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX +ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS => GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS + +If ENS_VAR_ variables are not set: +""""""""""""""""""""""""""""""""""""" + +If no FCST/OBS verification is being performed in the use case using another +wrapper, then rename the FCST_VAR variables to ENS_VAR. + +e.g. + +FCST_VAR1_NAME => ENS_VAR1_NAME +FCST_VAR1_LEVELS => ENS_VAR1_LEVELS +FCST_VAR2_NAME => ENS_VAR2_NAME +FCST_VAR2_LEVELS => ENS_VAR2_LEVELS +... etc + +If FCST/OBS verification is being performed by another tool, then add +ENS_VAR variables using the corresponding FCST_VAR values. + +e.g. + +ENS_VAR1_NAME = {FCST_VAR1_NAME} +ENS_VAR1_LEVELS = {FCST_VAR1_LEVELS} +ENS_VAR2_NAME = {FCST_VAR2_NAME} +ENS_VAR2_LEVELS = {FCST_VAR2_LEVELS} +... etc + +Remove the following variables: +""""""""""""""""""""""""""""""" + +Remove any remaining ENSEMBLE_STAT_* variables that are no longer used. +Some examples: +Remove ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK +Remove ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT +Remove ENSEMBLE_STAT_MESSAGE_TYPE +Remove ENSEMBLE_STAT_OUTPUT_FLAG_ECNT +Remove ENSEMBLE_STAT_OUTPUT_FLAG_RPS +Remove ENSEMBLE_STAT_OUTPUT_FLAG_RHIST +Remove ENSEMBLE_STAT_OUTPUT_FLAG_PHIST +Remove ENSEMBLE_STAT_OUTPUT_FLAG_ORANK +Remove ENSEMBLE_STAT_OUTPUT_FLAG_SSVAR +Remove ENSEMBLE_STAT_OUTPUT_FLAG_RELP +Remove ENSEMBLE_STAT_OUTPUT_FLAG_PCT +Remove ENSEMBLE_STAT_OUTPUT_FLAG_PSTD +Remove ENSEMBLE_STAT_OUTPUT_FLAG_PJC +Remove ENSEMBLE_STAT_OUTPUT_FLAG_PRC +Remove ENSEMBLE_STAT_OUTPUT_FLAG_ECLV +Remove ENSEMBLE_STAT_DUPLICATE_FLAG +Remove ENSEMBLE_STAT_SKIP_CONST +Remove ENSEMBLE_STAT_OBS_ERROR_FLAG +Remove ENSEMBLE_STAT_ENS_SSVAR_BIN_SIZE +Remove ENSEMBLE_STAT_ENS_PHIST_BIN_SIZE +Remove ENSEMBLE_STAT_CI_ALPHA +Remove ENSEMBLE_STAT_MASK_GRID +Remove ENSEMBLE_STAT_MASK_POLY +Remove ENSEMBLE_STAT_INTERP_FIELD +Remove ENSEMBLE_STAT_INTERP_VLD_THRESH +Remove ENSEMBLE_STAT_INTERP_SHAPE +Remove ENSEMBLE_STAT_INTERP_METHOD +Remove ENSEMBLE_STAT_INTERP_WIDTH +Remove ENSEMBLE_STAT_OBS_QUALITY_INC/EXC +Remove ENSEMBLE_STAT_GRID_WEIGHT_FLAG + + + +Case 2: EnsembleStat performing ensemble verification but not generating ensemble products +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +No changes should be required for this case to continue to work as expected +except for removing configuration variables that are no longer used. +The use case will no longer generate a _ens.nc file and may create other files +(_orank.nc and txt) that contain requested output. + +Rename the following variables: +""""""""""""""""""""""""""""""" + +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN => ENSEMBLE_STAT_NC_ORANK_FLAG_MEAN +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANK => ENSEMBLE_STAT_NC_ORANK_FLAG_RANK +ENSEMBLE_STAT_ENSEMBLE_FLAG_WEIGHT => ENSEMBLE_STAT_NC_ORANK_FLAG_WEIGHT +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT => ENSEMBLE_STAT_NC_ORANK_FLAG_VLD_COUNT + + +Remove the following variables: +""""""""""""""""""""""""""""""" + +Remove any ENS_VAR_* variables +Remove ENSEMBLE_STAT_ENSEMBLE_FLAG_* +ENSEMBLE_STAT_NBRHD_PROB_WIDTH +ENSEMBLE_STAT_NBRHD_PROB_SHAPE +ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH +ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH +ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE +ENSEMBLE_STAT_NMEP_SMOOTH_METHOD +ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH +ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX +ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS + + +Case 3: EnsembleStat generating ensemble products and performing ensemble verification +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +GenEnsProd will need to be added to the PROCESS_LIST in addition to EnsembleStat to generate the ensemble verification output. + +PROCESS_LIST = ..., EnsembleStat, GenEnsProd, ... + +Set the input dir and template variables for GenEnsProd to match the values set for FCST input to EnsembleStat. Also set the output dir to match EnsembleStat output dir. + +GEN_ENS_PROD_INPUT_DIR = {FCST_ENSEMBLE_STAT_INPUT_DIR} +GEN_ENS_PROD_INPUT_TEMPLATE = {FCST_ENSEMBLE_STAT_INPUT_TEMPLATE} +GEN_ENS_PROD_OUTPUT_DIR = {ENSEMBLE_STAT_OUTPUT_DIR} + +If the EnsembleStat output template is set, then copy the value and add a template for the NetCDF output filename at the end following a forward slash ‘/’ character. + +If ENSEMBLE_STAT_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d%H} +then set +GEN_ENS_PROD_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d%H}/gen_ens_prod_{valid?fmt=%Y%m%d_%H%M%S}V_ens.nc +or something similar. + +If the EnsembleStat output template is not set, then set GenEnsProd’s template to the desired NetCDF output filename. Here is an example: + +GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{valid?fmt=%Y%m%d_%H%M%S}V_ens.nc + +Ensure that any downstream wrappers in the PROCESS_LIST are configured to read the correct GenEnsProd output file instead of the _ens.nc file that was previously generated by EnsembleStat. + +If ENS_VAR_ variables are not set, add ENS_VAR variables using the corresponding FCST_ENSEMBLE_STAT_VAR or FCST_VAR values. +If FCST_ENSEMBLE_VAR_* variables are set, then use only those values, otherwise use FCST_VAR_* + +e.g. + +ENS_VAR1_NAME = {FCST_VAR1_NAME} +ENS_VAR1_LEVELS = {FCST_VAR1_LEVELS} +ENS_VAR2_NAME = {FCST_VAR2_NAME} +ENS_VAR2_LEVELS = {FCST_VAR2_LEVELS} + + +If any of the following ENSEMBLE_STAT_* variables are set in the configuration file, then rename them to the corresponding GEN_ENS_PROD_* variable: + +ENSEMBLE_STAT_NBRHD_PROB_WIDTH => GEN_ENS_PROD_NBRHD_PROB_WIDTH +ENSEMBLE_STAT_NBRHD_PROB_SHAPE => GEN_ENS_PROD_NBRHD_PROB_SHAPE +ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH => GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH +ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH => GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH +ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE => GEN_ENS_PROD_NMEP_SMOOTH_SHAPE +ENSEMBLE_STAT_NMEP_SMOOTH_METHOD => GEN_ENS_PROD_NMEP_SMOOTH_METHOD +ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH => GEN_ENS_PROD_NMEP_SMOOTH_WIDTH +ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX => GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX +ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS => GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS +FCST_ENSEMBLE_STAT_INPUT_GRID_DATATYPE => GEN_ENS_PROD_INPUT_DATATYPE + + +If any of the following ENSEMBLE_STAT_* variables are set in the configuration file, then set the corresponding GEN_ENS_PROD_* variables to the same value or reference the ENSEMBLE_STAT_* version. + +GEN_ENS_PROD_N_MEMBERS = {ENSEMBLE_STAT_N_MEMBERS} +GEN_ENS_PROD_ENS_THRESH = {ENSEMBLE_STAT_ENS_THRESH} +GEN_ENS_PROD_REGRID_TO_GRID = {ENSEMBLE_STAT_REGRID_TO_GRID} +GEN_ENS_PROD_REGRID_METHOD = {ENSEMBLE_STAT_REGRID_METHOD} +GEN_ENS_PROD_REGRID_WIDTH = {ENSEMBLE_STAT_REGRID_WIDTH} +GEN_ENS_PROD_VLD_THRESH = {ENSEMBLE_STAT_VLD_THRESH} +GEN_ENS_PROD_SHAPE = {ENSEMBLE_STAT_SHAPE} + + +If any of the following ENSEMBLE_STAT_ENSEMBLE_FLAG_* variables are set in the configuration file, then set the corresponding GEN_ENS_PROD_ENSEMBLE_FLAG_* variables to the same value. + +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN +ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV +ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS +ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS +ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN +ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX +ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT +ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY +ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP +ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP + +e.g. + +If ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = TRUE +Add GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON = TRUE + +If any of the following ENSEMBLE_STAT_ENSEMBLE_FLAG_* variables are set in the configuration file, then rename them to the corresponding ENSEMBLE_STAT_NC_ORANK_FLAG_* variables. + +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON => ENSEMBLE_STAT_NC_ORANK_FLAG_LATLON +ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN => ENSEMBLE_STAT_NC_ORANK_FLAG_MEAN +ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT => ENSEMBLE_STAT_NC_ORANK_FLAG_VLD_COUNT diff --git a/internal/tests/pytests/util/config_metplus/test_config_metplus.py b/internal/tests/pytests/util/config_metplus/test_config_metplus.py index 8974d69b9a..71fe75138b 100644 --- a/internal/tests/pytests/util/config_metplus/test_config_metplus.py +++ b/internal/tests/pytests/util/config_metplus/test_config_metplus.py @@ -8,6 +8,7 @@ from metplus.util import config_metplus from metplus.util.time_util import ti_calculate +from metplus.util.config_validate import validate_config_variables @pytest.mark.util def test_get_default_config_list(): @@ -100,31 +101,6 @@ def test_find_indices_in_config_section(metplus_config, regex, index, assert indices == expected_result -@pytest.mark.parametrize( - 'conf_items, met_tool, expected_result', [ - ({'CUSTOM_LOOP_LIST': "one, two, three"}, '', ['one', 'two', 'three']), - ({'CUSTOM_LOOP_LIST': "one, two, three", - 'GRID_STAT_CUSTOM_LOOP_LIST': "four, five",}, 'grid_stat', ['four', 'five']), - ({'CUSTOM_LOOP_LIST': "one, two, three", - 'GRID_STAT_CUSTOM_LOOP_LIST': "four, five",}, 'point_stat', ['one', 'two', 'three']), - ({'CUSTOM_LOOP_LIST': "one, two, three", - 'ASCII2NC_CUSTOM_LOOP_LIST': "four, five",}, 'ascii2nc', ['four', 'five']), - # fails to read custom loop list for point2grid because there are underscores in name - ({'CUSTOM_LOOP_LIST': "one, two, three", - 'POINT_2_GRID_CUSTOM_LOOP_LIST': "four, five",}, 'point2grid', ['one', 'two', 'three']), - ({'CUSTOM_LOOP_LIST': "one, two, three", - 'POINT2GRID_CUSTOM_LOOP_LIST': "four, five",}, 'point2grid', ['four', 'five']), - ] -) -@pytest.mark.util -def test_get_custom_string_list(metplus_config, conf_items, met_tool, expected_result): - config = metplus_config - for conf_key, conf_value in conf_items.items(): - config.set('config', conf_key, conf_value) - - assert config_metplus.get_custom_string_list(config, met_tool) == expected_result - - @pytest.mark.parametrize( 'config_var_name, expected_indices, set_met_tool', [ ('FCST_GRID_STAT_VAR1_NAME', ['1'], True), @@ -188,97 +164,6 @@ def test_get_field_search_prefixes(data_type, met_tool, expected_out): met_tool) == expected_out) -@pytest.mark.parametrize( - 'item_list, extension, is_valid', [ - (['FCST'], 'NAME', False), - (['OBS'], 'NAME', False), - (['FCST', 'OBS'], 'NAME', True), - (['BOTH'], 'NAME', True), - (['FCST', 'OBS', 'BOTH'], 'NAME', False), - (['FCST', 'ENS'], 'NAME', False), - (['OBS', 'ENS'], 'NAME', False), - (['FCST', 'OBS', 'ENS'], 'NAME', True), - (['BOTH', 'ENS'], 'NAME', True), - (['FCST', 'OBS', 'BOTH', 'ENS'], 'NAME', False), - - (['FCST', 'OBS'], 'THRESH', True), - (['BOTH'], 'THRESH', True), - (['FCST', 'OBS', 'BOTH'], 'THRESH', False), - (['FCST', 'OBS', 'ENS'], 'THRESH', True), - (['BOTH', 'ENS'], 'THRESH', True), - (['FCST', 'OBS', 'BOTH', 'ENS'], 'THRESH', False), - - (['FCST'], 'OPTIONS', True), - (['OBS'], 'OPTIONS', True), - (['FCST', 'OBS'], 'OPTIONS', True), - (['BOTH'], 'OPTIONS', True), - (['FCST', 'OBS', 'BOTH'], 'OPTIONS', False), - (['FCST', 'ENS'], 'OPTIONS', True), - (['OBS', 'ENS'], 'OPTIONS', True), - (['FCST', 'OBS', 'ENS'], 'OPTIONS', True), - (['BOTH', 'ENS'], 'OPTIONS', True), - (['FCST', 'OBS', 'BOTH', 'ENS'], 'OPTIONS', False), - - (['FCST', 'OBS', 'BOTH'], 'LEVELS', False), - (['FCST', 'OBS'], 'LEVELS', True), - (['BOTH'], 'LEVELS', True), - (['FCST', 'OBS', 'ENS'], 'LEVELS', True), - (['BOTH', 'ENS'], 'LEVELS', True), - - ] -) -@pytest.mark.util -def test_is_var_item_valid(metplus_config, item_list, extension, is_valid): - conf = metplus_config - assert config_metplus.is_var_item_valid(item_list, '1', extension, conf)[0] == is_valid - - -@pytest.mark.parametrize( - 'item_list, configs_to_set, is_valid', [ - - (['FCST'], {'FCST_VAR1_LEVELS': 'A06', - 'OBS_VAR1_NAME': 'script_name.py something else'}, True), - (['FCST'], {'FCST_VAR1_LEVELS': 'A06', - 'OBS_VAR1_NAME': 'APCP'}, False), - (['OBS'], {'OBS_VAR1_LEVELS': '"(*,*)"', - 'FCST_VAR1_NAME': 'script_name.py something else'}, True), - (['OBS'], {'OBS_VAR1_LEVELS': '"(*,*)"', - 'FCST_VAR1_NAME': 'APCP'}, False), - - (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06', - 'OBS_VAR1_NAME': 'script_name.py something else'}, True), - (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06', - 'OBS_VAR1_NAME': 'APCP'}, False), - (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(*,*)"', - 'FCST_VAR1_NAME': 'script_name.py something else'}, True), - (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(*,*)"', - 'FCST_VAR1_NAME': 'APCP'}, False), - - (['FCST'], {'FCST_VAR1_LEVELS': 'A06, A12', - 'OBS_VAR1_NAME': 'script_name.py something else'}, False), - (['FCST'], {'FCST_VAR1_LEVELS': 'A06, A12', - 'OBS_VAR1_NAME': 'APCP'}, False), - (['OBS'], {'OBS_VAR1_LEVELS': '"(0,*,*)", "(1,*,*)"', - 'FCST_VAR1_NAME': 'script_name.py something else'}, False), - - (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06, A12', - 'OBS_VAR1_NAME': 'script_name.py something else'}, False), - (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06, A12', - 'OBS_VAR1_NAME': 'APCP'}, False), - (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(0,*,*)", "(1,*,*)"', - 'FCST_VAR1_NAME': 'script_name.py something else'}, False), - - ] -) -@pytest.mark.util -def test_is_var_item_valid_levels(metplus_config, item_list, configs_to_set, is_valid): - conf = metplus_config - for key, value in configs_to_set.items(): - conf.set('config', key, value) - - assert config_metplus.is_var_item_valid(item_list, '1', 'LEVELS', conf)[0] == is_valid - - # search prefixes are valid prefixes to append to field info variables # config_overrides are a dict of config vars and their values # search_key is the key of the field config item to check @@ -418,21 +303,20 @@ def test_parse_var_list_fcst_only(metplus_config, data_type, list_created): conf.set('config', 'FCST_VAR2_LEVELS', "LEVELS21, LEVELS22") # this should not occur because OBS variables are missing - if config_metplus.validate_configuration_variables(conf, force_check=True)[1]: - assert False + assert not validate_config_variables(conf)[0] var_list = config_metplus.parse_var_list(conf, time_info=None, data_type=data_type) # list will be created if requesting just OBS, but it should not be created if # nothing was requested because FCST values are missing if list_created: - assert(var_list[0]['fcst_name'] == "NAME1" and \ - var_list[1]['fcst_name'] == "NAME1" and \ - var_list[2]['fcst_name'] == "NAME2" and \ - var_list[3]['fcst_name'] == "NAME2" and \ - var_list[0]['fcst_level'] == "LEVELS11" and \ - var_list[1]['fcst_level'] == "LEVELS12" and \ - var_list[2]['fcst_level'] == "LEVELS21" and \ + assert(var_list[0]['fcst_name'] == "NAME1" and + var_list[1]['fcst_name'] == "NAME1" and + var_list[2]['fcst_name'] == "NAME2" and + var_list[3]['fcst_name'] == "NAME2" and + var_list[0]['fcst_level'] == "LEVELS11" and + var_list[1]['fcst_level'] == "LEVELS12" and + var_list[2]['fcst_level'] == "LEVELS21" and var_list[3]['fcst_level'] == "LEVELS22") else: assert not var_list @@ -455,7 +339,7 @@ def test_parse_var_list_obs(metplus_config, data_type, list_created): conf.set('config', 'OBS_VAR2_LEVELS', "LEVELS21, LEVELS22") # this should not occur because FCST variables are missing - if config_metplus.validate_configuration_variables(conf, force_check=True)[1]: + if validate_config_variables(conf)[0]: assert False var_list = config_metplus.parse_var_list(conf, time_info=None, data_type=data_type) @@ -492,7 +376,7 @@ def test_parse_var_list_both(metplus_config, data_type, list_created): conf.set('config', 'BOTH_VAR2_LEVELS', "LEVELS21, LEVELS22") # this should not occur because BOTH variables are used - if not config_metplus.validate_configuration_variables(conf, force_check=True)[1]: + if not validate_config_variables(conf)[0]: assert False var_list = config_metplus.parse_var_list(conf, time_info=None, data_type=data_type) @@ -523,7 +407,7 @@ def test_parse_var_list_fcst_and_obs(metplus_config): conf.set('config', 'OBS_VAR2_LEVELS', "OLEVELS21, OLEVELS22") # this should not occur because FCST and OBS variables are found - if not config_metplus.validate_configuration_variables(conf, force_check=True)[1]: + if not validate_config_variables(conf)[0]: assert False var_list = config_metplus.parse_var_list(conf) @@ -556,7 +440,7 @@ def test_parse_var_list_fcst_and_obs_alternate(metplus_config): conf.set('config', 'OBS_VAR2_LEVELS', "OLEVELS21, OLEVELS22") # configuration is invalid and parse var list should not give any results - assert(not config_metplus.validate_configuration_variables(conf, force_check=True)[1] and not config_metplus.parse_var_list(conf)) + assert(not validate_config_variables(conf)[0] and not config_metplus.parse_var_list(conf)) # VAR1 defined by OBS, VAR2 by FCST, VAR3 by both FCST AND OBS @@ -580,7 +464,7 @@ def test_parse_var_list_fcst_and_obs_and_both(metplus_config, data_type, list_le conf.set('config', 'OBS_VAR3_LEVELS', "OLEVELS31, OLEVELS32") # configuration is invalid and parse var list should not give any results - if config_metplus.validate_configuration_variables(conf, force_check=True)[1]: + if validate_config_variables(conf)[0]: assert False var_list = config_metplus.parse_var_list(conf, time_info=None, data_type=data_type) @@ -626,7 +510,7 @@ def test_parse_var_list_fcst_only_options(metplus_config, data_type, list_len): conf.set('config', 'OBS_VAR1_OPTIONS', "OOPTIONS11") # this should not occur because OBS variables are missing - if config_metplus.validate_configuration_variables(conf, force_check=True)[1]: + if validate_config_variables(conf)[0]: assert False var_list = config_metplus.parse_var_list(conf, time_info=None, data_type=data_type) @@ -963,80 +847,6 @@ def test_parse_var_list_py_embed_multi_levels(metplus_config, config_overrides, assert var_item['fcst_name'] == expected_result -@pytest.mark.parametrize( - 'input_list, expected_list', [ - ('Point2Grid', ['Point2Grid']), - # MET documentation syntax (with dashes) - ('Pcp-Combine, Grid-Stat, Ensemble-Stat', ['PCPCombine', - 'GridStat', - 'EnsembleStat']), - ('Point-Stat', ['PointStat']), - ('Mode, MODE Time Domain', ['MODE', - 'MTD']), - # actual tool name (lower case underscore) - ('point_stat, grid_stat, ensemble_stat', ['PointStat', - 'GridStat', - 'EnsembleStat']), - ('mode, mtd', ['MODE', - 'MTD']), - ('ascii2nc, pb2nc, regrid_data_plane', ['ASCII2NC', - 'PB2NC', - 'RegridDataPlane']), - ('pcp_combine, tc_pairs, tc_stat', ['PCPCombine', - 'TCPairs', - 'TCStat']), - ('gen_vx_mask, stat_analysis, series_analysis', ['GenVxMask', - 'StatAnalysis', - 'SeriesAnalysis']), - # old capitalization format - ('PcpCombine, Ascii2Nc, TcStat, TcPairs', ['PCPCombine', - 'ASCII2NC', - 'TCStat', - 'TCPairs']), - ] -) -@pytest.mark.util -def test_get_process_list(metplus_config, input_list, expected_list): - conf = metplus_config - conf.set('config', 'PROCESS_LIST', input_list) - process_list = config_metplus.get_process_list(conf) - output_list = [item[0] for item in process_list] - assert output_list == expected_list - - -@pytest.mark.parametrize( - 'input_list, expected_list', [ - # no instances - ('Point2Grid', [('Point2Grid', None)]), - # one with instance one without - ('PcpCombine, GridStat(my_instance)', [('PCPCombine', None), - ('GridStat', 'my_instance')]), - # duplicate process, one with instance one without - ('TCStat, ExtractTiles, TCStat(for_series), SeriesAnalysis', ( - [('TCStat',None), - ('ExtractTiles',None), - ('TCStat', 'for_series'), - ('SeriesAnalysis',None),])), - # two processes, both with instances - ('mode(uno), mtd(dos)', [('MODE', 'uno'), - ('MTD', 'dos')]), - # lower-case names, first with instance, second without - ('ascii2nc(some_name), pb2nc', [('ASCII2NC', 'some_name'), - ('PB2NC', None)]), - # duplicate process, both with different instances - ('tc_stat(one), tc_pairs, tc_stat(two)', [('TCStat', 'one'), - ('TCPairs', None), - ('TCStat', 'two')]), - ] -) -@pytest.mark.util -def test_get_process_list_instances(metplus_config, input_list, expected_list): - conf = metplus_config - conf.set('config', 'PROCESS_LIST', input_list) - output_list = config_metplus.get_process_list(conf) - assert output_list == expected_list - - @pytest.mark.util def test_getraw_sub_and_nosub(metplus_config): raw_string = '{MODEL}_{CURRENT_FCST_NAME}' @@ -1103,65 +913,3 @@ def test_format_var_items_options_semicolon(config_value, var_items = config_metplus._format_var_items(field_configs, time_info) result = var_items.get('extra') assert result == expected_result - - -@pytest.mark.parametrize( - 'input_dict, expected_list', [ - ({'init': datetime(2019, 2, 1, 6), - 'lead': 7200, }, - [ - {'index': '1', - 'fcst_name': 'FNAME_2019', - 'fcst_level': 'Z06', - 'obs_name': 'ONAME_2019', - 'obs_level': 'L06', - }, - {'index': '1', - 'fcst_name': 'FNAME_2019', - 'fcst_level': 'Z08', - 'obs_name': 'ONAME_2019', - 'obs_level': 'L08', - }, - ]), - ({'init': datetime(2021, 4, 13, 9), - 'lead': 10800, }, - [ - {'index': '1', - 'fcst_name': 'FNAME_2021', - 'fcst_level': 'Z09', - 'obs_name': 'ONAME_2021', - 'obs_level': 'L09', - }, - {'index': '1', - 'fcst_name': 'FNAME_2021', - 'fcst_level': 'Z12', - 'obs_name': 'ONAME_2021', - 'obs_level': 'L12', - }, - ]), - ] -) -@pytest.mark.util -def test_sub_var_list(metplus_config, input_dict, expected_list): - config = metplus_config - config.set('config', 'FCST_VAR1_NAME', 'FNAME_{init?fmt=%Y}') - config.set('config', 'FCST_VAR1_LEVELS', 'Z{init?fmt=%H}, Z{valid?fmt=%H}') - config.set('config', 'OBS_VAR1_NAME', 'ONAME_{init?fmt=%Y}') - config.set('config', 'OBS_VAR1_LEVELS', 'L{init?fmt=%H}, L{valid?fmt=%H}') - - time_info = ti_calculate(input_dict) - - actual_temp = config_metplus.parse_var_list(config) - - pp = pprint.PrettyPrinter() - print(f'Actual var list (before sub):') - pp.pprint(actual_temp) - - actual_list = config_metplus.sub_var_list(actual_temp, time_info) - print(f'Actual var list (after sub):') - pp.pprint(actual_list) - - assert len(actual_list) == len(expected_list) - for actual, expected in zip(actual_list, expected_list): - for key, value in expected.items(): - assert actual.get(key) == value diff --git a/internal/tests/pytests/util/config_util/test_config_util.py b/internal/tests/pytests/util/config_util/test_config_util.py new file mode 100644 index 0000000000..aee78545f2 --- /dev/null +++ b/internal/tests/pytests/util/config_util/test_config_util.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 + +import pytest + +import pprint +import os +from datetime import datetime + +from metplus.util.config_util import * +from metplus.util.config_metplus import parse_var_list +from metplus.util.time_util import ti_calculate + + +@pytest.mark.parametrize( + 'conf_items, met_tool, expected_result', [ + ({'CUSTOM_LOOP_LIST': "one, two, three"}, '', ['one', 'two', 'three']), + ({'CUSTOM_LOOP_LIST': "one, two, three", + 'GRID_STAT_CUSTOM_LOOP_LIST': "four, five",}, 'grid_stat', ['four', 'five']), + ({'CUSTOM_LOOP_LIST': "one, two, three", + 'GRID_STAT_CUSTOM_LOOP_LIST': "four, five",}, 'point_stat', ['one', 'two', 'three']), + ({'CUSTOM_LOOP_LIST': "one, two, three", + 'ASCII2NC_CUSTOM_LOOP_LIST': "four, five",}, 'ascii2nc', ['four', 'five']), + # fails to read custom loop list for point2grid because there are underscores in name + ({'CUSTOM_LOOP_LIST': "one, two, three", + 'POINT_2_GRID_CUSTOM_LOOP_LIST': "four, five",}, 'point2grid', ['one', 'two', 'three']), + ({'CUSTOM_LOOP_LIST': "one, two, three", + 'POINT2GRID_CUSTOM_LOOP_LIST': "four, five",}, 'point2grid', ['four', 'five']), + ] +) +@pytest.mark.util +def test_get_custom_string_list(metplus_config, conf_items, met_tool, expected_result): + config = metplus_config + for conf_key, conf_value in conf_items.items(): + config.set('config', conf_key, conf_value) + + assert get_custom_string_list(config, met_tool) == expected_result + + +@pytest.mark.parametrize( + 'input_list, expected_list', [ + ('Point2Grid', ['Point2Grid']), + # MET documentation syntax (with dashes) + ('Pcp-Combine, Grid-Stat, Ensemble-Stat', ['PCPCombine', + 'GridStat', + 'EnsembleStat']), + ('Point-Stat', ['PointStat']), + ('Mode, MODE Time Domain', ['MODE', + 'MTD']), + # actual tool name (lower case underscore) + ('point_stat, grid_stat, ensemble_stat', ['PointStat', + 'GridStat', + 'EnsembleStat']), + ('mode, mtd', ['MODE', + 'MTD']), + ('ascii2nc, pb2nc, regrid_data_plane', ['ASCII2NC', + 'PB2NC', + 'RegridDataPlane']), + ('pcp_combine, tc_pairs, tc_stat', ['PCPCombine', + 'TCPairs', + 'TCStat']), + ('gen_vx_mask, stat_analysis, series_analysis', ['GenVxMask', + 'StatAnalysis', + 'SeriesAnalysis']), + # old capitalization format + ('PcpCombine, Ascii2Nc, TcStat, TcPairs', ['PCPCombine', + 'ASCII2NC', + 'TCStat', + 'TCPairs']), + ] +) +@pytest.mark.util +def test_get_process_list(metplus_config, input_list, expected_list): + conf = metplus_config + conf.set('config', 'PROCESS_LIST', input_list) + process_list = get_process_list(conf) + output_list = [item[0] for item in process_list] + assert output_list == expected_list + + +@pytest.mark.parametrize( + 'input_list, expected_list', [ + # no instances + ('Point2Grid', [('Point2Grid', None)]), + # one with instance one without + ('PcpCombine, GridStat(my_instance)', [('PCPCombine', None), + ('GridStat', 'my_instance')]), + # duplicate process, one with instance one without + ('TCStat, ExtractTiles, TCStat(for_series), SeriesAnalysis', ( + [('TCStat',None), + ('ExtractTiles',None), + ('TCStat', 'for_series'), + ('SeriesAnalysis',None),])), + # two processes, both with instances + ('mode(uno), mtd(dos)', [('MODE', 'uno'), + ('MTD', 'dos')]), + # lower-case names, first with instance, second without + ('ascii2nc(some_name), pb2nc', [('ASCII2NC', 'some_name'), + ('PB2NC', None)]), + # duplicate process, both with different instances + ('tc_stat(one), tc_pairs, tc_stat(two)', [('TCStat', 'one'), + ('TCPairs', None), + ('TCStat', 'two')]), + ] +) +@pytest.mark.util +def test_get_process_list_instances(metplus_config, input_list, expected_list): + conf = metplus_config + conf.set('config', 'PROCESS_LIST', input_list) + output_list = get_process_list(conf) + assert output_list == expected_list + + +@pytest.mark.parametrize( + 'input_dict, expected_list', [ + ({'init': datetime(2019, 2, 1, 6), + 'lead': 7200, }, + [ + {'index': '1', + 'fcst_name': 'FNAME_2019', + 'fcst_level': 'Z06', + 'obs_name': 'ONAME_2019', + 'obs_level': 'L06', + }, + {'index': '1', + 'fcst_name': 'FNAME_2019', + 'fcst_level': 'Z08', + 'obs_name': 'ONAME_2019', + 'obs_level': 'L08', + }, + ]), + ({'init': datetime(2021, 4, 13, 9), + 'lead': 10800, }, + [ + {'index': '1', + 'fcst_name': 'FNAME_2021', + 'fcst_level': 'Z09', + 'obs_name': 'ONAME_2021', + 'obs_level': 'L09', + }, + {'index': '1', + 'fcst_name': 'FNAME_2021', + 'fcst_level': 'Z12', + 'obs_name': 'ONAME_2021', + 'obs_level': 'L12', + }, + ]), + ] +) +@pytest.mark.util +def test_sub_var_list(metplus_config, input_dict, expected_list): + config = metplus_config + config.set('config', 'FCST_VAR1_NAME', 'FNAME_{init?fmt=%Y}') + config.set('config', 'FCST_VAR1_LEVELS', 'Z{init?fmt=%H}, Z{valid?fmt=%H}') + config.set('config', 'OBS_VAR1_NAME', 'ONAME_{init?fmt=%Y}') + config.set('config', 'OBS_VAR1_LEVELS', 'L{init?fmt=%H}, L{valid?fmt=%H}') + + time_info = ti_calculate(input_dict) + + actual_temp = parse_var_list(config) + + pp = pprint.PrettyPrinter() + print(f'Actual var list (before sub):') + pp.pprint(actual_temp) + + actual_list = sub_var_list(actual_temp, time_info) + print(f'Actual var list (after sub):') + pp.pprint(actual_list) + + assert len(actual_list) == len(expected_list) + for actual, expected in zip(actual_list, expected_list): + for key, value in expected.items(): + assert actual.get(key) == value diff --git a/internal/tests/pytests/util/config_validate/test_config_validate.py b/internal/tests/pytests/util/config_validate/test_config_validate.py new file mode 100644 index 0000000000..0cd4a3193c --- /dev/null +++ b/internal/tests/pytests/util/config_validate/test_config_validate.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +import pytest + +import pprint +import os +from datetime import datetime + +from metplus.util.config_validate import * + + +@pytest.mark.parametrize( + 'item_list, extension, is_valid', [ + (['FCST'], 'NAME', False), + (['OBS'], 'NAME', False), + (['FCST', 'OBS'], 'NAME', True), + (['BOTH'], 'NAME', True), + (['FCST', 'OBS', 'BOTH'], 'NAME', False), + (['FCST', 'ENS'], 'NAME', False), + (['OBS', 'ENS'], 'NAME', False), + (['FCST', 'OBS', 'ENS'], 'NAME', True), + (['BOTH', 'ENS'], 'NAME', True), + (['FCST', 'OBS', 'BOTH', 'ENS'], 'NAME', False), + + (['FCST', 'OBS'], 'THRESH', True), + (['BOTH'], 'THRESH', True), + (['FCST', 'OBS', 'BOTH'], 'THRESH', False), + (['FCST', 'OBS', 'ENS'], 'THRESH', True), + (['BOTH', 'ENS'], 'THRESH', True), + (['FCST', 'OBS', 'BOTH', 'ENS'], 'THRESH', False), + + (['FCST'], 'OPTIONS', True), + (['OBS'], 'OPTIONS', True), + (['FCST', 'OBS'], 'OPTIONS', True), + (['BOTH'], 'OPTIONS', True), + (['FCST', 'OBS', 'BOTH'], 'OPTIONS', False), + (['FCST', 'ENS'], 'OPTIONS', True), + (['OBS', 'ENS'], 'OPTIONS', True), + (['FCST', 'OBS', 'ENS'], 'OPTIONS', True), + (['BOTH', 'ENS'], 'OPTIONS', True), + (['FCST', 'OBS', 'BOTH', 'ENS'], 'OPTIONS', False), + + (['FCST', 'OBS', 'BOTH'], 'LEVELS', False), + (['FCST', 'OBS'], 'LEVELS', True), + (['BOTH'], 'LEVELS', True), + (['FCST', 'OBS', 'ENS'], 'LEVELS', True), + (['BOTH', 'ENS'], 'LEVELS', True), + + ] +) +@pytest.mark.util +def test_is_var_item_valid(metplus_config, item_list, extension, is_valid): + conf = metplus_config + assert is_var_item_valid(item_list, '1', extension, conf)[0] == is_valid + + +@pytest.mark.parametrize( + 'item_list, configs_to_set, is_valid', [ + + (['FCST'], {'FCST_VAR1_LEVELS': 'A06', + 'OBS_VAR1_NAME': 'script_name.py something else'}, True), + (['FCST'], {'FCST_VAR1_LEVELS': 'A06', + 'OBS_VAR1_NAME': 'APCP'}, False), + (['OBS'], {'OBS_VAR1_LEVELS': '"(*,*)"', + 'FCST_VAR1_NAME': 'script_name.py something else'}, True), + (['OBS'], {'OBS_VAR1_LEVELS': '"(*,*)"', + 'FCST_VAR1_NAME': 'APCP'}, False), + + (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06', + 'OBS_VAR1_NAME': 'script_name.py something else'}, True), + (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06', + 'OBS_VAR1_NAME': 'APCP'}, False), + (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(*,*)"', + 'FCST_VAR1_NAME': 'script_name.py something else'}, True), + (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(*,*)"', + 'FCST_VAR1_NAME': 'APCP'}, False), + + (['FCST'], {'FCST_VAR1_LEVELS': 'A06, A12', + 'OBS_VAR1_NAME': 'script_name.py something else'}, False), + (['FCST'], {'FCST_VAR1_LEVELS': 'A06, A12', + 'OBS_VAR1_NAME': 'APCP'}, False), + (['OBS'], {'OBS_VAR1_LEVELS': '"(0,*,*)", "(1,*,*)"', + 'FCST_VAR1_NAME': 'script_name.py something else'}, False), + + (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06, A12', + 'OBS_VAR1_NAME': 'script_name.py something else'}, False), + (['FCST', 'ENS'], {'FCST_VAR1_LEVELS': 'A06, A12', + 'OBS_VAR1_NAME': 'APCP'}, False), + (['OBS', 'ENS'], {'OBS_VAR1_LEVELS': '"(0,*,*)", "(1,*,*)"', + 'FCST_VAR1_NAME': 'script_name.py something else'}, False), + + ] +) +@pytest.mark.util +def test_is_var_item_valid_levels(metplus_config, item_list, configs_to_set, is_valid): + conf = metplus_config + for key, value in configs_to_set.items(): + conf.set('config', key, value) + + assert is_var_item_valid(item_list, '1', 'LEVELS', conf)[0] == is_valid diff --git a/internal/tests/pytests/util/string_manip/test_util_string_manip.py b/internal/tests/pytests/util/string_manip/test_util_string_manip.py index c5c27b3e56..22453a53fe 100644 --- a/internal/tests/pytests/util/string_manip/test_util_string_manip.py +++ b/internal/tests/pytests/util/string_manip/test_util_string_manip.py @@ -2,6 +2,7 @@ import pytest +import pprint from csv import reader from metplus.util.string_manip import * @@ -371,3 +372,60 @@ def test_format_thresh(expression, expected_result): @pytest.mark.util def test_format_level(level, expected_result): assert format_level(level) == expected_result + + +@pytest.mark.parametrize( + 'regex,index,id,expected_result', [ + # 0: No ID + (r'^FCST_VAR(\d+)_NAME$', 1, None, + {'1': [None], + '2': [None], + '4': [None]}), + # 1: ID and index 2 + (r'(\w+)_VAR(\d+)_NAME', 2, 1, + {'1': ['FCST'], + '2': ['FCST'], + '4': ['FCST']}), + # 2: index 1, ID 2, multiple identifiers + (r'^FCST_VAR(\d+)_(\w+)$', 1, 2, + {'1': ['NAME', 'LEVELS'], + '2': ['NAME'], + '4': ['NAME']}), + # 3: command that StatAnalysis wrapper uses + (r'MODEL(\d+)$', 1, None, + {'1': [None], + '2': [None],}), + # 4: TCPairs conensus logic + (r'^TC_PAIRS_CONSENSUS(\d+)_(\w+)$', 1, 2, + {'1': ['NAME', 'MEMBERS', 'REQUIRED', 'MIN_REQ'], + '2': ['NAME', 'MEMBERS', 'REQUIRED', 'MIN_REQ']}), + ] +) +@pytest.mark.util +def test_find_indices_in_config_section(metplus_config, regex, index, + id, expected_result): + config = metplus_config + config.set('config', 'FCST_VAR1_NAME', 'name1') + config.set('config', 'FCST_VAR1_LEVELS', 'level1') + config.set('config', 'FCST_VAR2_NAME', 'name2') + config.set('config', 'FCST_VAR4_NAME', 'name4') + config.set('config', 'MODEL1', 'model1') + config.set('config', 'MODEL2', 'model2') + + config.set('config', 'TC_PAIRS_CONSENSUS1_NAME', 'name1') + config.set('config', 'TC_PAIRS_CONSENSUS1_MEMBERS', 'member1') + config.set('config', 'TC_PAIRS_CONSENSUS1_REQUIRED', 'True') + config.set('config', 'TC_PAIRS_CONSENSUS1_MIN_REQ', '1') + config.set('config', 'TC_PAIRS_CONSENSUS2_NAME', 'name2') + config.set('config', 'TC_PAIRS_CONSENSUS2_MEMBERS', 'member2') + config.set('config', 'TC_PAIRS_CONSENSUS2_REQUIRED', 'True') + config.set('config', 'TC_PAIRS_CONSENSUS2_MIN_REQ', '2') + + indices = find_indices_in_config_section(regex, config, index_index=index, + id_index=id) + + pp = pprint.PrettyPrinter() + print(f'Indices:') + pp.pprint(indices) + + assert indices == expected_result diff --git a/metplus/util/__init__.py b/metplus/util/__init__.py index 58df7821c6..752469810f 100644 --- a/metplus/util/__init__.py +++ b/metplus/util/__init__.py @@ -4,8 +4,10 @@ from .system_util import * from .time_util import * from .string_template_substitution import * -from .doc_util import * +from .config_util import * from .config_metplus import * +from .config_validate import * +from .doc_util import * from .run_util import * from .met_config import * from .time_looping import * diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index 5e9b02c07c..b1f8431c28 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -23,11 +23,12 @@ from produtil.config import ProdConfig from .constants import RUNTIME_CONFS, MISSING_DATA_VALUE -from .string_template_substitution import get_tags, do_string_sub -from .string_manip import getlist, remove_quotes, is_python_script -from .string_manip import validate_thresholds +from .string_template_substitution import do_string_sub +from .string_manip import getlist, remove_quotes +from .string_manip import validate_thresholds, find_indices_in_config_section from .system_util import mkdir_p -from .doc_util import get_wrapper_name +from .config_util import is_loop_by_init +from .config_validate import validate_field_info_configs """!Creates the initial METplus directory structure, loads information into each job. @@ -48,15 +49,8 @@ ''' __all__ = [ 'setup', - 'get_custom_string_list', - 'find_indices_in_config_section', 'parse_var_list', - 'sub_var_list', - 'get_process_list', - 'validate_configuration_variables', - 'is_loop_by_init', - 'handle_tmp_dir', - 'log_runtime_banner', + 'replace_config_from_section', ] '''!@var METPLUS_BASE @@ -90,6 +84,7 @@ # set all loggers to use UTC logging.Formatter.converter = time.gmtime + def setup(args, logger=None, base_confs=None): """!The METplus setup function. @param args list of configuration files or configuration @@ -116,6 +111,7 @@ def setup(args, logger=None, base_confs=None): return config + def _get_default_config_list(parm_base=None): """! Get list of default METplus config files. Look through BASE_CONFS list and check if each file exists under the parm base. Add each to a list @@ -145,6 +141,7 @@ def _get_default_config_list(parm_base=None): return default_config_list + def _parse_launch_args(args, logger): """! Parsed arguments to scripts that launch the METplus wrappers. @@ -210,6 +207,7 @@ def _parse_launch_args(args, logger): return override_list + def launch(config_list): """! Process configuration files and explicit configuration variables overrides. Subsequent configuration files override values in previously @@ -275,6 +273,7 @@ def launch(config_list): return config + def _set_logvars(config, logger=None): """!Sets and adds the LOG_METPLUS and LOG_TIMESTAMP to the config object. If LOG_METPLUS was already defined by the @@ -339,6 +338,7 @@ def _set_logvars(config, logger=None): # expand LOG_METPLUS to ensure it is available config.set('config', 'LOG_METPLUS', metpluslog) + def get_logger(config, sublog=None): """!This function will return a logger with a formatted file handler for writing to the LOG_METPLUS and it sets the LOG_LEVEL. If LOG_METPLUS is @@ -414,6 +414,7 @@ def get_logger(config, sublog=None): config.logger = logger return logger + def replace_config_from_section(config, section, required=True): """! Check if config has a section named [section] If it does, create a new METplusConfig object, set each value from the input config, then @@ -460,6 +461,7 @@ def replace_config_from_section(config, section, required=True): return new_config + class METplusConfig(ProdConfig): """! Configuration class to store configuration values read from METplus config files. @@ -937,618 +939,6 @@ def format(self, record): return output -def validate_configuration_variables(config, force_check=False): - - all_sed_cmds = [] - # check for deprecated config items and warn user to remove/replace them - deprecated_isOK, sed_cmds = check_for_deprecated_config(config) - all_sed_cmds.extend(sed_cmds) - - # check for deprecated env vars in MET config files and warn user to remove/replace them - deprecatedMET_isOK, sed_cmds = check_for_deprecated_met_config(config) - all_sed_cmds.extend(sed_cmds) - - # validate configuration variables - field_isOK, sed_cmds = validate_field_info_configs(config, force_check) - all_sed_cmds.extend(sed_cmds) - - # check that OUTPUT_BASE is not set to the exact same value as INPUT_BASE - inoutbase_isOK = True - input_real_path = os.path.realpath(config.getdir_nocheck('INPUT_BASE', '')) - output_real_path = os.path.realpath(config.getdir('OUTPUT_BASE')) - if input_real_path == output_real_path: - config.logger.error(f"INPUT_BASE AND OUTPUT_BASE are set to the exact same path: {input_real_path}") - config.logger.error("Please change one of these paths to avoid risk of losing input data") - inoutbase_isOK = False - - check_user_environment(config) - - return deprecated_isOK, field_isOK, inoutbase_isOK, deprecatedMET_isOK, all_sed_cmds - -def check_for_deprecated_config(config): - """!Checks user configuration files and reports errors or warnings if any deprecated variable - is found. If an alternate variable name can be suggested, add it to the 'alt' section - If the alternate cannot be literally substituted for the old name, set copy to False - Args: - @config : METplusConfig object to evaluate - Returns: - A tuple containing a boolean if the configuration is suitable to run or not and - if it is not correct, the 2nd item is a list of sed commands that can be run to help - fix the incorrect configuration variables - """ - - # key is the name of the depreacted variable that is no longer allowed in any config files - # value is a dictionary containing information about what to do with the deprecated config - # 'sec' is the section of the config file where the replacement resides, i.e. config, dir, - # filename_templates - # 'alt' is the alternative name for the deprecated config. this can be a single variable name or - # text to describe multiple variables or how to handle it. Set to None to tell the user to - # just remove the variable - # 'copy' is an optional item (defaults to True). set this to False if one cannot simply replace - # the deprecated config variable name with the value in 'alt' - # 'req' is an optional item (defaults to True). this to False to report a warning for the - # deprecated config and allow execution to continue. this is generally no longer used - # because we are requiring users to update the config files. if used, the developer must - # modify the code to handle both variables accordingly - deprecated_dict = { - 'LOOP_BY_INIT' : {'sec' : 'config', 'alt' : 'LOOP_BY', 'copy': False}, - 'PREPBUFR_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : None}, - 'PREPBUFR_FILE_REGEX' : {'sec' : 'regex_pattern', 'alt' : None}, - 'OBS_INPUT_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : 'OBS_POINT_STAT_INPUT_DIR', 'copy': False}, - 'FCST_INPUT_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : 'FCST_POINT_STAT_INPUT_DIR', 'copy': False}, - 'FCST_INPUT_FILE_REGEX' : - {'sec' : 'regex_pattern', 'alt' : 'FCST_POINT_STAT_INPUT_TEMPLATE', 'copy': False}, - 'OBS_INPUT_FILE_REGEX' : {'sec' : 'regex_pattern', 'alt' : 'OBS_POINT_STAT_INPUT_TEMPLATE', 'copy': False}, - 'PREPBUFR_DATA_DIR' : {'sec' : 'dir', 'alt' : 'PB2NC_INPUT_DIR'}, - 'PREPBUFR_MODEL_DIR_NAME' : {'sec' : 'dir', 'alt' : 'PB2NC_INPUT_DIR', 'copy': False}, - 'OBS_INPUT_FILE_TMPL' : - {'sec' : 'filename_templates', 'alt' : 'OBS_POINT_STAT_INPUT_TEMPLATE'}, - 'FCST_INPUT_FILE_TMPL' : - {'sec' : 'filename_templates', 'alt' : 'FCST_POINT_STAT_INPUT_TEMPLATE'}, - 'NC_FILE_TMPL' : {'sec' : 'filename_templates', 'alt' : 'PB2NC_OUTPUT_TEMPLATE'}, - 'FCST_INPUT_DIR' : {'sec' : 'dir', 'alt' : 'FCST_POINT_STAT_INPUT_DIR'}, - 'OBS_INPUT_DIR' : {'sec' : 'dir', 'alt' : 'OBS_POINT_STAT_INPUT_DIR'}, - 'REGRID_TO_GRID' : {'sec' : 'config', 'alt' : 'POINT_STAT_REGRID_TO_GRID'}, - 'FCST_HR_START' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'FCST_HR_END' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'FCST_HR_INTERVAL' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'START_DATE' : {'sec' : 'config', 'alt' : 'INIT_BEG or VALID_BEG', 'copy': False}, - 'END_DATE' : {'sec' : 'config', 'alt' : 'INIT_END or VALID_END', 'copy': False}, - 'INTERVAL_TIME' : {'sec' : 'config', 'alt' : 'INIT_INCREMENT or VALID_INCREMENT', 'copy': False}, - 'BEG_TIME' : {'sec' : 'config', 'alt' : 'INIT_BEG or VALID_BEG', 'copy': False}, - 'END_TIME' : {'sec' : 'config', 'alt' : 'INIT_END or VALID_END', 'copy': False}, - 'START_HOUR' : {'sec' : 'config', 'alt' : 'INIT_BEG or VALID_BEG', 'copy': False}, - 'END_HOUR' : {'sec' : 'config', 'alt' : 'INIT_END or VALID_END', 'copy': False}, - 'OBS_BUFR_VAR_LIST' : {'sec' : 'config', 'alt' : 'PB2NC_OBS_BUFR_VAR_LIST'}, - 'TIME_SUMMARY_FLAG' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_FLAG'}, - 'TIME_SUMMARY_BEG' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_BEG'}, - 'TIME_SUMMARY_END' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_END'}, - 'TIME_SUMMARY_VAR_NAMES' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_VAR_NAMES'}, - 'TIME_SUMMARY_TYPE' : {'sec' : 'config', 'alt' : 'PB2NC_TIME_SUMMARY_TYPE'}, - 'OVERWRITE_NC_OUTPUT' : {'sec' : 'config', 'alt' : 'PB2NC_SKIP_IF_OUTPUT_EXISTS', 'copy': False}, - 'VERTICAL_LOCATION' : {'sec' : 'config', 'alt' : 'PB2NC_VERTICAL_LOCATION'}, - 'VERIFICATION_GRID' : {'sec' : 'config', 'alt' : 'REGRID_DATA_PLANE_VERIF_GRID'}, - 'WINDOW_RANGE_BEG' : {'sec' : 'config', 'alt' : 'OBS_WINDOW_BEGIN'}, - 'WINDOW_RANGE_END' : {'sec' : 'config', 'alt' : 'OBS_WINDOW_END'}, - 'OBS_EXACT_VALID_TIME' : - {'sec' : 'config', 'alt' : 'OBS_WINDOW_BEGIN and OBS_WINDOW_END', 'copy': False}, - 'FCST_EXACT_VALID_TIME' : - {'sec' : 'config', 'alt' : 'FCST_WINDOW_BEGIN and FCST_WINDOW_END', 'copy': False}, - 'PCP_COMBINE_METHOD' : - {'sec' : 'config', 'alt' : 'FCST_PCP_COMBINE_METHOD and/or OBS_PCP_COMBINE_METHOD', 'copy': False}, - 'FHR_BEG' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'FHR_END' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'FHR_INC' : {'sec' : 'config', 'alt' : 'LEAD_SEQ', 'copy': False}, - 'FHR_GROUP_BEG' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_[N]', 'copy': False}, - 'FHR_GROUP_END' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_[N]', 'copy': False}, - 'FHR_GROUP_LABELS' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_[N]_LABEL', 'copy': False}, - 'CYCLONE_OUT_DIR' : {'sec' : 'dir', 'alt' : 'CYCLONE_OUTPUT_DIR'}, - 'ENSEMBLE_STAT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'ENSEMBLE_STAT_OUTPUT_DIR'}, - 'EXTRACT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'EXTRACT_TILES_OUTPUT_DIR'}, - 'GRID_STAT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'GRID_STAT_OUTPUT_DIR'}, - 'MODE_OUT_DIR' : {'sec' : 'dir', 'alt' : 'MODE_OUTPUT_DIR'}, - 'MTD_OUT_DIR' : {'sec' : 'dir', 'alt' : 'MTD_OUTPUT_DIR'}, - 'SERIES_INIT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_OUTPUT_DIR'}, - 'SERIES_LEAD_OUT_DIR' : {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_OUTPUT_DIR'}, - 'SERIES_INIT_FILTERED_OUT_DIR' : - {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, - 'SERIES_LEAD_FILTERED_OUT_DIR' : - {'sec' : 'dir', 'alt' : 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, - 'STAT_ANALYSIS_OUT_DIR' : - {'sec' : 'dir', 'alt' : 'STAT_ANALYSIS_OUTPUT_DIR'}, - 'TCMPR_PLOT_OUT_DIR' : {'sec' : 'dir', 'alt' : 'TCMPR_PLOT_OUTPUT_DIR'}, - 'FCST_MIN_FORECAST' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_MIN'}, - 'FCST_MAX_FORECAST' : {'sec' : 'config', 'alt' : 'LEAD_SEQ_MAX'}, - 'OBS_MIN_FORECAST' : {'sec' : 'config', 'alt' : 'OBS_PCP_COMBINE_MIN_LEAD'}, - 'OBS_MAX_FORECAST' : {'sec' : 'config', 'alt' : 'OBS_PCP_COMBINE_MAX_LEAD'}, - 'FCST_INIT_INTERVAL' : {'sec' : 'config', 'alt' : None}, - 'OBS_INIT_INTERVAL' : {'sec' : 'config', 'alt' : None}, - 'FCST_DATA_INTERVAL' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_DATA_INTERVAL'}, - 'OBS_DATA_INTERVAL' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_DATA_INTERVAL'}, - 'FCST_IS_DAILY_FILE' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_IS_DAILY_FILE'}, - 'OBS_IS_DAILY_FILE' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_IS_DAILY_FILE'}, - 'FCST_TIMES_PER_FILE' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_TIMES_PER_FILE'}, - 'OBS_TIMES_PER_FILE' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_TIMES_PER_FILE'}, - 'FCST_LEVEL' : {'sec' : '', 'alt' : 'FCST_PCP_COMBINE_INPUT_ACCUMS', 'copy': False}, - 'OBS_LEVEL' : {'sec' : '', 'alt' : 'OBS_PCP_COMBINE_INPUT_ACCUMS', 'copy': False}, - 'MODE_FCST_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'FCST_MODE_CONV_RADIUS'}, - 'MODE_FCST_CONV_THRESH' : {'sec' : 'config', 'alt' : 'FCST_MODE_CONV_THRESH'}, - 'MODE_FCST_MERGE_FLAG' : {'sec' : 'config', 'alt' : 'FCST_MODE_MERGE_FLAG'}, - 'MODE_FCST_MERGE_THRESH' : {'sec' : 'config', 'alt' : 'FCST_MODE_MERGE_THRESH'}, - 'MODE_OBS_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'OBS_MODE_CONV_RADIUS'}, - 'MODE_OBS_CONV_THRESH' : {'sec' : 'config', 'alt' : 'OBS_MODE_CONV_THRESH'}, - 'MODE_OBS_MERGE_FLAG' : {'sec' : 'config', 'alt' : 'OBS_MODE_MERGE_FLAG'}, - 'MODE_OBS_MERGE_THRESH' : {'sec' : 'config', 'alt' : 'OBS_MODE_MERGE_THRESH'}, - 'MTD_FCST_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'FCST_MTD_CONV_RADIUS'}, - 'MTD_FCST_CONV_THRESH' : {'sec' : 'config', 'alt' : 'FCST_MTD_CONV_THRESH'}, - 'MTD_OBS_CONV_RADIUS' : {'sec' : 'config', 'alt' : 'OBS_MTD_CONV_RADIUS'}, - 'MTD_OBS_CONV_THRESH' : {'sec' : 'config', 'alt' : 'OBS_MTD_CONV_THRESH'}, - 'RM_EXE' : {'sec' : 'exe', 'alt' : 'RM'}, - 'CUT_EXE' : {'sec' : 'exe', 'alt' : 'CUT'}, - 'TR_EXE' : {'sec' : 'exe', 'alt' : 'TR'}, - 'NCAP2_EXE' : {'sec' : 'exe', 'alt' : 'NCAP2'}, - 'CONVERT_EXE' : {'sec' : 'exe', 'alt' : 'CONVERT'}, - 'NCDUMP_EXE' : {'sec' : 'exe', 'alt' : 'NCDUMP'}, - 'EGREP_EXE' : {'sec' : 'exe', 'alt' : 'EGREP'}, - 'ADECK_TRACK_DATA_DIR' : {'sec' : 'dir', 'alt' : 'TC_PAIRS_ADECK_INPUT_DIR'}, - 'BDECK_TRACK_DATA_DIR' : {'sec' : 'dir', 'alt' : 'TC_PAIRS_BDECK_INPUT_DIR'}, - 'MISSING_VAL_TO_REPLACE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_MISSING_VAL_TO_REPLACE'}, - 'MISSING_VAL' : {'sec' : 'config', 'alt' : 'TC_PAIRS_MISSING_VAL'}, - 'TRACK_DATA_SUBDIR_MOD' : {'sec' : 'dir', 'alt' : None}, - 'ADECK_FILE_PREFIX' : {'sec' : 'config', 'alt' : 'TC_PAIRS_ADECK_TEMPLATE', 'copy': False}, - 'BDECK_FILE_PREFIX' : {'sec' : 'config', 'alt' : 'TC_PAIRS_BDECK_TEMPLATE', 'copy': False}, - 'TOP_LEVEL_DIRS' : {'sec' : 'config', 'alt' : 'TC_PAIRS_READ_ALL_FILES'}, - 'TC_PAIRS_DIR' : {'sec' : 'dir', 'alt' : 'TC_PAIRS_OUTPUT_DIR'}, - 'CYCLONE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_CYCLONE'}, - 'STORM_ID' : {'sec' : 'config', 'alt' : 'TC_PAIRS_STORM_ID'}, - 'BASIN' : {'sec' : 'config', 'alt' : 'TC_PAIRS_BASIN'}, - 'STORM_NAME' : {'sec' : 'config', 'alt' : 'TC_PAIRS_STORM_NAME'}, - 'DLAND_FILE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_DLAND_FILE'}, - 'TRACK_TYPE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_REFORMAT_DECK'}, - 'FORECAST_TMPL' : {'sec' : 'filename_templates', 'alt' : 'TC_PAIRS_ADECK_TEMPLATE'}, - 'REFERENCE_TMPL' : {'sec' : 'filename_templates', 'alt' : 'TC_PAIRS_BDECK_TEMPLATE'}, - 'TRACK_DATA_MOD_FORCE_OVERWRITE' : - {'sec' : 'config', 'alt' : 'TC_PAIRS_SKIP_IF_REFORMAT_EXISTS', 'copy': False}, - 'TC_PAIRS_FORCE_OVERWRITE' : {'sec' : 'config', 'alt' : 'TC_PAIRS_SKIP_IF_OUTPUT_EXISTS', 'copy': False}, - 'GRID_STAT_CONFIG' : {'sec' : 'config', 'alt' : 'GRID_STAT_CONFIG_FILE'}, - 'MODE_CONFIG' : {'sec' : 'config', 'alt': 'MODE_CONFIG_FILE'}, - 'FCST_PCP_COMBINE_INPUT_LEVEL': {'sec': 'config', 'alt' : 'FCST_PCP_COMBINE_INPUT_ACCUMS'}, - 'OBS_PCP_COMBINE_INPUT_LEVEL': {'sec': 'config', 'alt' : 'OBS_PCP_COMBINE_INPUT_ACCUMS'}, - 'TIME_METHOD': {'sec': 'config', 'alt': 'LOOP_BY', 'copy': False}, - 'MODEL_DATA_DIR': {'sec': 'dir', 'alt': 'EXTRACT_TILES_GRID_INPUT_DIR'}, - 'STAT_LIST': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_STAT_LIST'}, - 'NLAT': {'sec': 'config', 'alt': 'EXTRACT_TILES_NLAT'}, - 'NLON': {'sec': 'config', 'alt': 'EXTRACT_TILES_NLON'}, - 'DLAT': {'sec': 'config', 'alt': 'EXTRACT_TILES_DLAT'}, - 'DLON': {'sec': 'config', 'alt': 'EXTRACT_TILES_DLON'}, - 'LON_ADJ': {'sec': 'config', 'alt': 'EXTRACT_TILES_LON_ADJ'}, - 'LAT_ADJ': {'sec': 'config', 'alt': 'EXTRACT_TILES_LAT_ADJ'}, - 'OVERWRITE_TRACK': {'sec': 'config', 'alt': 'EXTRACT_TILES_OVERWRITE_TRACK'}, - 'BACKGROUND_MAP': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_BACKGROUND_MAP'}, - 'GFS_FCST_FILE_TMPL': {'sec': 'filename_templates', 'alt': 'FCST_EXTRACT_TILES_INPUT_TEMPLATE'}, - 'GFS_ANLY_FILE_TMPL': {'sec': 'filename_templates', 'alt': 'OBS_EXTRACT_TILES_INPUT_TEMPLATE'}, - 'SERIES_BY_LEAD_FILTERED_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, - 'SERIES_BY_INIT_FILTERED_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_FILTERED_OUTPUT_DIR'}, - 'SERIES_BY_LEAD_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_OUTPUT_DIR'}, - 'SERIES_BY_INIT_OUTPUT_DIR': {'sec': 'dir', 'alt': 'SERIES_ANALYSIS_OUTPUT_DIR'}, - 'SERIES_BY_LEAD_GROUP_FCSTS': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_GROUP_FCSTS'}, - 'SERIES_ANALYSIS_BY_LEAD_CONFIG_FILE': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_CONFIG_FILE'}, - 'SERIES_ANALYSIS_BY_INIT_CONFIG_FILE': {'sec': 'config', 'alt': 'SERIES_ANALYSIS_CONFIG_FILE'}, - 'ENSEMBLE_STAT_MET_OBS_ERROR_TABLE': {'sec': 'config', 'alt': 'ENSEMBLE_STAT_MET_OBS_ERR_TABLE'}, - 'VAR_LIST': {'sec': 'config', 'alt': 'BOTH_VAR_NAME BOTH_VAR_LEVELS or SERIES_ANALYSIS_VAR_LIST', 'copy': False}, - 'SERIES_ANALYSIS_VAR_LIST': {'sec': 'config', 'alt': 'BOTH_VAR_NAME BOTH_VAR_LEVELS', 'copy': False}, - 'EXTRACT_TILES_VAR_LIST': {'sec': 'config', 'alt': ''}, - 'STAT_ANALYSIS_LOOKIN_DIR': {'sec': 'dir', 'alt': 'MODEL1_STAT_ANALYSIS_LOOKIN_DIR'}, - 'VALID_HOUR_METHOD': {'sec': 'config', 'alt': None}, - 'VALID_HOUR_BEG': {'sec': 'config', 'alt': None}, - 'VALID_HOUR_END': {'sec': 'config', 'alt': None}, - 'VALID_HOUR_INCREMENT': {'sec': 'config', 'alt': None}, - 'INIT_HOUR_METHOD': {'sec': 'config', 'alt': None}, - 'INIT_HOUR_BEG': {'sec': 'config', 'alt': None}, - 'INIT_HOUR_END': {'sec': 'config', 'alt': None}, - 'INIT_HOUR_INCREMENT': {'sec': 'config', 'alt': None}, - 'STAT_ANALYSIS_CONFIG': {'sec': 'config', 'alt': 'STAT_ANALYSIS_CONFIG_FILE'}, - 'JOB_NAME': {'sec': 'config', 'alt': 'STAT_ANALYSIS_JOB_NAME'}, - 'JOB_ARGS': {'sec': 'config', 'alt': 'STAT_ANALYSIS_JOB_ARGS'}, - 'FCST_LEAD': {'sec': 'config', 'alt': 'FCST_LEAD_LIST'}, - 'FCST_VAR_NAME': {'sec': 'config', 'alt': 'FCST_VAR_LIST'}, - 'FCST_VAR_LEVEL': {'sec': 'config', 'alt': 'FCST_VAR_LEVEL_LIST'}, - 'OBS_VAR_NAME': {'sec': 'config', 'alt': 'OBS_VAR_LIST'}, - 'OBS_VAR_LEVEL': {'sec': 'config', 'alt': 'OBS_VAR_LEVEL_LIST'}, - 'REGION': {'sec': 'config', 'alt': 'VX_MASK_LIST'}, - 'INTERP': {'sec': 'config', 'alt': 'INTERP_LIST'}, - 'INTERP_PTS': {'sec': 'config', 'alt': 'INTERP_PTS_LIST'}, - 'CONV_THRESH': {'sec': 'config', 'alt': 'CONV_THRESH_LIST'}, - 'FCST_THRESH': {'sec': 'config', 'alt': 'FCST_THRESH_LIST'}, - 'LINE_TYPE': {'sec': 'config', 'alt': 'LINE_TYPE_LIST'}, - 'STAT_ANALYSIS_DUMP_ROW_TMPL': {'sec': 'filename_templates', 'alt': 'STAT_ANALYSIS_DUMP_ROW_TEMPLATE'}, - 'STAT_ANALYSIS_OUT_STAT_TMPL': {'sec': 'filename_templates', 'alt': 'STAT_ANALYSIS_OUT_STAT_TEMPLATE'}, - 'PLOTTING_SCRIPTS_DIR': {'sec': 'dir', 'alt': 'MAKE_PLOTS_SCRIPTS_DIR'}, - 'STAT_FILES_INPUT_DIR': {'sec': 'dir', 'alt': 'MAKE_PLOTS_INPUT_DIR'}, - 'PLOTTING_OUTPUT_DIR': {'sec': 'dir', 'alt': 'MAKE_PLOTS_OUTPUT_DIR'}, - 'VERIF_CASE': {'sec': 'config', 'alt': 'MAKE_PLOTS_VERIF_CASE'}, - 'VERIF_TYPE': {'sec': 'config', 'alt': 'MAKE_PLOTS_VERIF_TYPE'}, - 'PLOT_TIME': {'sec': 'config', 'alt': 'DATE_TIME'}, - 'MODEL_NAME': {'sec': 'config', 'alt': 'MODEL'}, - 'MODEL_OBS_NAME': {'sec': 'config', 'alt': 'MODEL_OBTYPE'}, - 'MODEL_STAT_DIR': {'sec': 'dir', 'alt': 'MODEL_STAT_ANALYSIS_LOOKIN_DIR'}, - 'MODEL_NAME_ON_PLOT': {'sec': 'config', 'alt': 'MODEL_REFERENCE_NAME'}, - 'REGION_LIST': {'sec': 'config', 'alt': 'VX_MASK_LIST'}, - 'PLOT_STATS_LIST': {'sec': 'config', 'alt': 'MAKE_PLOT_STATS_LIST'}, - 'CI_METHOD': {'sec': 'config', 'alt': 'MAKE_PLOTS_CI_METHOD'}, - 'VERIF_GRID': {'sec': 'config', 'alt': 'MAKE_PLOTS_VERIF_GRID'}, - 'EVENT_EQUALIZATION': {'sec': 'config', 'alt': 'MAKE_PLOTS_EVENT_EQUALIZATION'}, - 'MTD_CONFIG': {'sec': 'config', 'alt': 'MTD_CONFIG_FILE'}, - 'CLIMO_GRID_STAT_INPUT_DIR': {'sec': 'dir', 'alt': 'GRID_STAT_CLIMO_MEAN_INPUT_DIR'}, - 'CLIMO_GRID_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', 'alt': 'GRID_STAT_CLIMO_MEAN_INPUT_TEMPLATE'}, - 'CLIMO_POINT_STAT_INPUT_DIR': {'sec': 'dir', 'alt': 'POINT_STAT_CLIMO_MEAN_INPUT_DIR'}, - 'CLIMO_POINT_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', 'alt': 'POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE'}, - 'GEMPAKTOCF_CLASSPATH': {'sec': 'exe', 'alt': 'GEMPAKTOCF_JAR', 'copy': False}, - 'CUSTOM_INGEST__OUTPUT_DIR': {'sec': 'dir', 'alt': 'PY_EMBED_INGEST__OUTPUT_DIR'}, - 'CUSTOM_INGEST__OUTPUT_TEMPLATE': {'sec': 'filename_templates', 'alt': 'PY_EMBED_INGEST__OUTPUT_TEMPLATE'}, - 'CUSTOM_INGEST__OUTPUT_GRID': {'sec': 'config', 'alt': 'PY_EMBED_INGEST__OUTPUT_GRID'}, - 'CUSTOM_INGEST__SCRIPT': {'sec': 'config', 'alt': 'PY_EMBED_INGEST__SCRIPT'}, - 'CUSTOM_INGEST__TYPE': {'sec': 'config', 'alt': 'PY_EMBED_INGEST__TYPE'}, - 'TC_STAT_RUN_VIA': {'sec': 'config', 'alt': 'TC_STAT_CONFIG_FILE', - 'copy': False}, - 'TC_STAT_CMD_LINE_JOB': {'sec': 'config', 'alt': 'TC_STAT_JOB_ARGS'}, - 'TC_STAT_JOBS_LIST': {'sec': 'config', 'alt': 'TC_STAT_JOB_ARGS'}, - 'EXTRACT_TILES_OVERWRITE_TRACK': {'sec': 'config', - 'alt': 'EXTRACT_TILES_SKIP_IF_OUTPUT_EXISTS', - 'copy': False}, - 'EXTRACT_TILES_PAIRS_INPUT_DIR': {'sec': 'dir', - 'alt': 'EXTRACT_TILES_STAT_INPUT_DIR', - 'copy': False}, - 'EXTRACT_TILES_FILTERED_OUTPUT_TEMPLATE': {'sec': 'filename_template', - 'alt': 'EXTRACT_TILES_STAT_INPUT_TEMPLATE',}, - 'EXTRACT_TILES_GRID_INPUT_DIR': {'sec': 'dir', - 'alt': 'FCST_EXTRACT_TILES_INPUT_DIR' - 'and ' - 'OBS_EXTRACT_TILES_INPUT_DIR', - 'copy': False}, - 'SERIES_ANALYSIS_FILTER_OPTS': {'sec': 'config', - 'alt': 'TC_STAT_JOB_ARGS', - 'copy': False}, - 'SERIES_ANALYSIS_INPUT_DIR': {'sec': 'dir', - 'alt': 'FCST_SERIES_ANALYSIS_INPUT_DIR ' - 'and ' - 'OBS_SERIES_ANALYSIS_INPUT_DIR'}, - 'FCST_SERIES_ANALYSIS_TILE_INPUT_TEMPLATE': {'sec': 'filename_templates', - 'alt': 'FCST_SERIES_ANALYSIS_INPUT_TEMPLATE '}, - 'OBS_SERIES_ANALYSIS_TILE_INPUT_TEMPLATE': {'sec': 'filename_templates', - 'alt': 'OBS_SERIES_ANALYSIS_INPUT_TEMPLATE '}, - 'EXTRACT_TILES_STAT_INPUT_DIR': {'sec': 'dir', - 'alt': 'EXTRACT_TILES_TC_STAT_INPUT_DIR',}, - 'EXTRACT_TILES_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', - 'alt': 'EXTRACT_TILES_TC_STAT_INPUT_TEMPLATE',}, - 'SERIES_ANALYSIS_STAT_INPUT_DIR': {'sec': 'dir', - 'alt': 'SERIES_ANALYSIS_TC_STAT_INPUT_DIR', }, - 'SERIES_ANALYSIS_STAT_INPUT_TEMPLATE': {'sec': 'filename_templates', - 'alt': 'SERIES_ANALYSIS_TC_STAT_INPUT_TEMPLATE', }, - } - - # template '' : {'sec' : '', 'alt' : '', 'copy': True}, - - logger = config.logger - - # create list of errors and warnings to report for deprecated configs - e_list = [] - w_list = [] - all_sed_cmds = [] - - for old, depr_info in deprecated_dict.items(): - if isinstance(depr_info, dict): - - # check if is found in the old item, use regex to find variables if found - if '' in old: - old_regex = old.replace('', r'(\d+)') - indices = find_indices_in_config_section(old_regex, - config, - index_index=1).keys() - for index in indices: - old_with_index = old.replace('', index) - if depr_info['alt']: - alt_with_index = depr_info['alt'].replace('', index) - else: - alt_with_index = '' - - handle_deprecated(old_with_index, alt_with_index, depr_info, - config, all_sed_cmds, w_list, e_list) - else: - handle_deprecated(old, depr_info['alt'], depr_info, - config, all_sed_cmds, w_list, e_list) - - - # check all templates and error if any deprecated tags are used - # value of dict is replacement tag, set to None if no replacement exists - # deprecated tags: region (replace with basin) - deprecated_tags = {'region' : 'basin'} - template_vars = config.keys('config') - template_vars = [tvar for tvar in template_vars if tvar.endswith('_TEMPLATE')] - for temp_var in template_vars: - template = config.getraw('filename_templates', temp_var) - tags = get_tags(template) - - for depr_tag, replace_tag in deprecated_tags.items(): - if depr_tag in tags: - e_msg = 'Deprecated tag {{{}}} found in {}.'.format(depr_tag, - temp_var) - if replace_tag is not None: - e_msg += ' Replace with {{{}}}'.format(replace_tag) - - e_list.append(e_msg) - - # if any warning exist, report them - if w_list: - for warning_msg in w_list: - logger.warning(warning_msg) - - # if any errors exist, report them and exit - if e_list: - logger.error('DEPRECATED CONFIG ITEMS WERE FOUND. ' +\ - 'PLEASE REMOVE/REPLACE THEM FROM CONFIG FILES') - for error_msg in e_list: - logger.error(error_msg) - return False, all_sed_cmds - - return True, [] - -def check_for_deprecated_met_config(config): - sed_cmds = [] - all_good = True - - # check if *_CONFIG_FILE if set in the METplus config file and check for - # deprecated environment variables in those files - met_config_keys = [key for key in config.keys('config') - if key.endswith('CONFIG_FILE')] - - for met_config_key in met_config_keys: - met_tool = met_config_key.replace('_CONFIG_FILE', '') - - # get custom loop list to check if multiple config files are used based on the custom string - custom_list = get_custom_string_list(config, met_tool) - - for custom_string in custom_list: - met_config = config.getraw('config', met_config_key) - if not met_config: - continue - - met_config_file = do_string_sub(met_config, custom=custom_string) - - if not check_for_deprecated_met_config_file(config, met_config_file, sed_cmds, met_tool): - all_good = False - - return all_good, sed_cmds - -def check_for_deprecated_met_config_file(config, met_config, sed_cmds, met_tool): - - all_good = True - if not os.path.exists(met_config): - config.logger.error(f"Config file does not exist: {met_config}") - return False - - deprecated_met_list = ['MET_VALID_HHMM', 'GRID_VX', 'CONFIG_DIR'] - deprecated_output_prefix_list = ['FCST_VAR', 'OBS_VAR'] - config.logger.debug(f"Checking for deprecated environment variables in: {met_config}") - - with open(met_config, 'r') as file_handle: - lines = file_handle.read().splitlines() - - for line in lines: - for deprecated_item in deprecated_met_list: - if '${' + deprecated_item + '}' in line: - all_good = False - config.logger.error("Please remove deprecated environment variable " - f"${{{deprecated_item}}} found in MET config file: " - f"{met_config}") - - if deprecated_item == 'MET_VALID_HHMM' and 'file_name' in line: - config.logger.error(f"Set {met_tool}_CLIMO_MEAN_INPUT_[DIR/TEMPLATE] in a " - "METplus config file to set CLIMO_MEAN_FILE in a MET config") - new_line = " file_name = [ ${CLIMO_MEAN_FILE} ];" - - # escape [ and ] because they are special characters in sed commands - old_line = line.rstrip().replace('[', r'\[').replace(']', r'\]') - - sed_cmds.append(f"sed -i 's|^{old_line}|{new_line}|g' {met_config}") - add_line = f"{met_tool}_CLIMO_MEAN_INPUT_TEMPLATE" - sed_cmds.append(f"#Add {add_line}") - break - - if 'to_grid' in line: - config.logger.error("MET to_grid variable should reference " - "${REGRID_TO_GRID} environment variable") - new_line = " to_grid = ${REGRID_TO_GRID};" - - # escape [ and ] because they are special characters in sed commands - old_line = line.rstrip().replace('[', r'\[').replace(']', r'\]') - - sed_cmds.append(f"sed -i 's|^{old_line}|{new_line}|g' {met_config}") - config.logger.info(f"Be sure to set {met_tool}_REGRID_TO_GRID to the correct value.") - add_line = f"{met_tool}_REGRID_TO_GRID" - sed_cmds.append(f"#Add {add_line}") - break - - - for deprecated_item in deprecated_output_prefix_list: - # if deprecated item found in output prefix or to_grid line, replace line to use - # env var OUTPUT_PREFIX or REGRID_TO_GRID - if '${' + deprecated_item + '}' in line and 'output_prefix' in line: - config.logger.error("output_prefix variable should reference " - "${OUTPUT_PREFIX} environment variable") - new_line = "output_prefix = \"${OUTPUT_PREFIX}\";" - - # escape [ and ] because they are special characters in sed commands - old_line = line.rstrip().replace('[', r'\[').replace(']', r'\]') - - sed_cmds.append(f"sed -i 's|^{old_line}|{new_line}|g' {met_config}") - config.logger.info(f"You will need to add {met_tool}_OUTPUT_PREFIX to the METplus config file" - f" that sets {met_tool}_CONFIG_FILE. Set it to:") - output_prefix = _replace_output_prefix(line) - add_line = f"{met_tool}_OUTPUT_PREFIX = {output_prefix}" - config.logger.info(add_line) - sed_cmds.append(f"#Add {add_line}") - all_good = False - break - - return all_good - -def validate_field_info_configs(config, force_check=False): - """!Verify that config variables with _VAR_ in them are valid. Returns True if all are valid. - Returns False if any items are invalid""" - - variable_extensions = ['NAME', 'LEVELS', 'THRESH', 'OPTIONS'] - all_good = True, [] - - if skip_field_info_validation(config) and not force_check: - return True, [] - - # keep track of all sed commands to replace config variable names - all_sed_cmds = [] - - for ext in variable_extensions: - # find all _VAR_ keys in the conf files - data_types_and_indices = find_indices_in_config_section(r"(\w+)_VAR(\d+)_"+ext, - config, - index_index=2, - id_index=1) - - # if BOTH_VAR_ is used, set FCST and OBS to the same value - # if FCST or OBS is used, the other must be present as well - # if BOTH and either FCST or OBS are set, report an error - # get other data type - for index, data_type_list in data_types_and_indices.items(): - - is_valid, err_msgs, sed_cmds = is_var_item_valid(data_type_list, index, ext, config) - if not is_valid: - for err_msg in err_msgs: - config.logger.error(err_msg) - all_sed_cmds.extend(sed_cmds) - all_good = False - - # make sure FCST and OBS have the same number of levels if coming from separate variables - elif ext == 'LEVELS' and all(item in ['FCST', 'OBS'] for item in data_type_list): - fcst_levels = getlist(config.getraw('config', f"FCST_VAR{index}_LEVELS", '')) - - # add empty string if no levels are found because python embedding items do not need - # to include a level, but the other item may have a level and the numbers need to match - if not fcst_levels: - fcst_levels.append('') - - obs_levels = getlist(config.getraw('config', f"OBS_VAR{index}_LEVELS", '')) - if not obs_levels: - obs_levels.append('') - - if len(fcst_levels) != len(obs_levels): - config.logger.error(f"FCST_VAR{index}_LEVELS and OBS_VAR{index}_LEVELS do not have " - "the same number of elements") - all_good = False - - return all_good, all_sed_cmds - -def check_user_environment(config): - """!Check if any environment variables set in [user_env_vars] are already set in - the user's environment. Warn them that it will be overwritten from the conf if it is""" - if not config.has_section('user_env_vars'): - return - - for env_var in config.keys('user_env_vars'): - if env_var in os.environ: - msg = '{} is already set in the environment. '.format(env_var) +\ - 'Overwriting from conf file' - config.logger.warning(msg) - -def find_indices_in_config_section(regex, config, sec='config', - index_index=1, id_index=None): - """! Use regular expression to get all config variables that match and - are set in the user's configuration. This is used to handle config - variables that have multiple indices, i.e. FCST_VAR1_NAME, FCST_VAR2_NAME, - etc. - - @param regex regular expression to use to find variables - @param config METplusConfig object to search - @param sec (optional) config file section to search. Defaults to config - @param index_index 1 based number that is the regex match index for the - index number (default is 1) - @param id_index 1 based number that is the regex match index for the - identifier. Defaults to None which does not extract an indentifier - - number and the first match is used as an identifier - @returns dictionary where keys are the index number and the value is a - list of identifiers (if noID=True) or a list containing None - """ - # regex expression must have 2 () items and the 2nd item must be the index - all_conf = config.keys(sec) - indices = {} - regex = re.compile(regex) - for conf in all_conf: - result = regex.match(conf) - if result is None: - continue - - index = result.group(index_index) - if id_index: - identifier = result.group(id_index) - else: - identifier = None - - if index not in indices: - indices[index] = [identifier] - else: - indices[index].append(identifier) - - return indices - -def handle_deprecated(old, alt, depr_info, config, all_sed_cmds, w_list, e_list): - sec = depr_info['sec'] - config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') - # if deprecated config item is found - if config.has_option(sec, old): - # if it is not required to remove, add to warning list - if 'req' in depr_info.keys() and depr_info['req'] is False: - msg = '[{}] {} is deprecated and will be '.format(sec, old) + \ - 'removed in a future version of METplus' - if alt: - msg += ". Please replace with {}".format(alt) - w_list.append(msg) - # if it is required to remove, add to error list - else: - if not alt: - e_list.append("[{}] {} should be removed".format(sec, old)) - else: - e_list.append("[{}] {} should be replaced with {}".format(sec, old, alt)) - - if 'copy' not in depr_info.keys() or depr_info['copy']: - for config_file in config_files: - all_sed_cmds.append(f"sed -i 's|^{old}|{alt}|g' {config_file}") - all_sed_cmds.append(f"sed -i 's|{{{old}}}|{{{alt}}}|g' {config_file}") - -def get_custom_string_list(config, met_tool): - var_name = 'CUSTOM_LOOP_LIST' - custom_loop_list = config.getstr_nocheck('config', - f'{met_tool.upper()}_{var_name}', - config.getstr_nocheck('config', - var_name, - '')) - custom_loop_list = getlist(custom_loop_list) - if not custom_loop_list: - custom_loop_list.append('') - - return custom_loop_list - -def _replace_output_prefix(line): - op_replacements = {'${MODEL}': '{MODEL}', - '${FCST_VAR}': '{CURRENT_FCST_NAME}', - '${OBTYPE}': '{OBTYPE}', - '${OBS_VAR}': '{CURRENT_OBS_NAME}', - '${LEVEL}': '{CURRENT_FCST_LEVEL}', - '${FCST_TIME}': '{lead?fmt=%3H}', - } - prefix = line.split('=')[1].strip().rstrip(';').strip('"') - for key, value, in op_replacements.items(): - prefix = prefix.replace(key, value) - - return prefix def parse_var_list(config, time_info=None, data_type=None, met_tool=None, levels_as_list=False): @@ -1820,59 +1210,6 @@ def _format_var_items(field_configs, time_info=None): return var_items -def skip_field_info_validation(config): - """!Check config to see if having corresponding FCST/OBS variables is necessary. If process list only - contains reformatter wrappers, don't validate field info. Also, if MTD is in the process list and - it is configured to only process either FCST or OBS, validation is unnecessary.""" - - reformatters = ['PCPCombine', 'RegridDataPlane'] - process_list = [item[0] for item in get_process_list(config)] - - # if running MTD in single mode, you don't need matching FCST/OBS - if ('MTD' in process_list and - config.getbool('config', 'MTD_SINGLE_RUN', False)): - return True - - # if running any app other than the reformatters, you need matching FCST/OBS, so don't skip - if [item for item in process_list if item not in reformatters]: - return False - - return True - -def get_process_list(config): - """!Read process list, Extract instance string if specified inside - parenthesis. Remove dashes/underscores and change to lower case, - then map the name to the correct wrapper name - - @param config METplusConfig object to read PROCESS_LIST value - @returns list of tuple containing process name and instance identifier - (None if no instance was set) - """ - # get list of processes - process_list = getlist(config.getstr('config', 'PROCESS_LIST')) - - out_process_list = [] - # for each item remove dashes, underscores, and cast to lower-case - for process in process_list: - # if instance is specified, extract the text inside parenthesis - match = re.match(r'(.*)\((.*)\)', process) - if match: - instance = match.group(2) - process_name = match.group(1) - else: - instance = None - process_name = process - - wrapper_name = get_wrapper_name(process_name) - if wrapper_name is None: - config.logger.warning(f"PROCESS_LIST item {process_name} " - "may be invalid.") - wrapper_name = process_name - - out_process_list.append((wrapper_name, instance)) - - return out_process_list - def get_field_search_prefixes(data_type, met_tool=None): """! Get list of prefixes to search for field variables. @@ -1902,67 +1239,6 @@ def get_field_search_prefixes(data_type, met_tool=None): return search_prefixes -def is_var_item_valid(item_list, index, ext, config): - """!Given a list of data types (FCST, OBS, ENS, or BOTH) check if the - combination is valid. - If BOTH is found, FCST and OBS should not be found. - If FCST or OBS is found, the other must also be found. - @param item_list list of data types that were found for a given index - @param index number following _VAR in the variable name - @param ext extension to check, i.e. NAME, LEVELS, THRESH, or OPTIONS - @param config METplusConfig instance - @returns tuple containing boolean if var item is valid, list of error - messages and list of sed commands to help the user update their old - configuration files - """ - - full_ext = f"_VAR{index}_{ext}" - msg = [] - sed_cmds = [] - if 'BOTH' in item_list and ('FCST' in item_list or 'OBS' in item_list): - - msg.append(f"Cannot set FCST{full_ext} or OBS{full_ext} if BOTH{full_ext} is set.") - elif ext == 'THRESH': - # allow thresholds unless BOTH and (FCST or OBS) are set - pass - - elif 'FCST' in item_list and 'OBS' not in item_list: - # if FCST level has 1 item and OBS name is a python embedding script, - # don't report error - level_list = getlist(config.getraw('config', - f'FCST_VAR{index}_LEVELS', - '')) - other_name = config.getraw('config', f'OBS_VAR{index}_NAME', '') - skip_error_for_py_embed = ext == 'LEVELS' and is_python_script(other_name) and len(level_list) == 1 - # do not report error for OPTIONS since it isn't required to be the same length - if ext not in ['OPTIONS'] and not skip_error_for_py_embed: - msg.append(f"If FCST{full_ext} is set, you must either set OBS{full_ext} or " - f"change FCST{full_ext} to BOTH{full_ext}") - - config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') - for config_file in config_files: - sed_cmds.append(f"sed -i 's|^FCST{full_ext}|BOTH{full_ext}|g' {config_file}") - sed_cmds.append(f"sed -i 's|{{FCST{full_ext}}}|{{BOTH{full_ext}}}|g' {config_file}") - - elif 'OBS' in item_list and 'FCST' not in item_list: - # if OBS level has 1 item and FCST name is a python embedding script, - # don't report error - level_list = getlist(config.getraw('config', - f'OBS_VAR{index}_LEVELS', - '')) - other_name = config.getraw('config', f'FCST_VAR{index}_NAME', '') - skip_error_for_py_embed = ext == 'LEVELS' and is_python_script(other_name) and len(level_list) == 1 - - if ext not in ['OPTIONS'] and not skip_error_for_py_embed: - msg.append(f"If OBS{full_ext} is set, you must either set FCST{full_ext} or " - f"change OBS{full_ext} to BOTH{full_ext}") - - config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') - for config_file in config_files: - sed_cmds.append(f"sed -i 's|^OBS{full_ext}|BOTH{full_ext}|g' {config_file}") - sed_cmds.append(f"sed -i 's|{{OBS{full_ext}}}|{{BOTH{full_ext}}}|g' {config_file}") - - return not bool(msg), msg, sed_cmds def get_field_config_variables(config, index, search_prefixes): """! Search for variables that are set in the config that correspond to @@ -2026,170 +1302,3 @@ def get_field_config_variables(config, index, search_prefixes): break return field_configs - - -def is_loop_by_init(config): - """!Check config variables to determine if looping by valid or init time""" - if config.has_option('config', 'LOOP_BY'): - loop_by = config.getstr('config', 'LOOP_BY').lower() - if loop_by in ['init', 'retro']: - return True - elif loop_by in ['valid', 'realtime']: - return False - - if config.has_option('config', 'LOOP_BY_INIT'): - return config.getbool('config', 'LOOP_BY_INIT') - - msg = 'MUST SET LOOP_BY to VALID, INIT, RETRO, or REALTIME' - if config.logger is None: - print(msg) - else: - config.logger.error(msg) - - return None - - -def handle_tmp_dir(config): - """! if env var MET_TMP_DIR is set, override config TMP_DIR with value - if it differs from what is set - get config temp dir using getdir_nocheck to bypass check for /path/to - this is done so the user can set env MET_TMP_DIR instead of config TMP_DIR - and config TMP_DIR will be set automatically""" - handle_env_var_config(config, 'MET_TMP_DIR', 'TMP_DIR') - - # create temp dir if it doesn't exist already - # this will fail if TMP_DIR is not set correctly and - # env MET_TMP_DIR was not set - mkdir_p(config.getdir('TMP_DIR')) - - -def handle_env_var_config(config, env_var_name, config_name): - """! If environment variable is set, use that value - for the config variable and warn if the previous config value differs - - @param config METplusConfig object to read - @param env_var_name name of environment variable to read - @param config_name name of METplus config variable to check - """ - env_var_value = os.environ.get(env_var_name, '') - config_value = config.getdir_nocheck(config_name, '') - - # do nothing if environment variable is not set - if not env_var_value: - return - - # override config config variable to environment variable value - config.set('config', config_name, env_var_value) - - # if config config value differed from environment variable value, warn - if config_value == env_var_value: - return - - config.logger.warning(f'Config variable {config_name} ({config_value}) ' - 'will be overridden by the environment variable ' - f'{env_var_name} ({env_var_value})') - - -def write_all_commands(all_commands, config): - """! Write all commands that were run to a file in the log - directory. This includes the environment variables that - were set before each command. - - @param all_commands list of tuples with command run and - list of environment variables that were set - @param config METplusConfig object used to write log output - and get the log timestamp to name the output file - @returns False if no commands were provided, True otherwise - """ - if not all_commands: - config.logger.error("No commands were run. " - "Skip writing all_commands file") - return False - - log_timestamp = config.getstr('config', 'LOG_TIMESTAMP') - filename = os.path.join(config.getdir('LOG_DIR'), - f'.all_commands.{log_timestamp}') - config.logger.debug(f"Writing all commands and environment to {filename}") - with open(filename, 'w') as file_handle: - for command, envs in all_commands: - for env in envs: - file_handle.write(f"{env}\n") - - file_handle.write("COMMAND:\n") - file_handle.write(f"{command}\n\n") - - return True - - -def write_final_conf(config): - """! Write final conf file including default values that were set during - run. Move variables that are specific to the user's run to the [runtime] - section to avoid issues such as overwriting existing log files. - - @param config METplusConfig object to write to file - """ - final_conf = config.getstr('config', 'METPLUS_CONF') - - # remove variables that start with CURRENT - config.remove_current_vars() - - # move runtime variables to [runtime] section - config.move_runtime_configs() - - config.logger.info('Overwrite final conf here: %s' % (final_conf,)) - with open(final_conf, 'wt') as conf_file: - config.write(conf_file) - - -def log_runtime_banner(config, time_input, process): - loop_by = time_input['loop_by'] - run_time = time_input[loop_by].strftime("%Y-%m-%d %H:%M") - - process_name = process.__class__.__name__ - if process.instance: - process_name = f"{process_name}({process.instance})" - - config.logger.info("****************************************") - config.logger.info(f"* Running METplus {process_name}") - config.logger.info(f"* at {loop_by} time: {run_time}") - config.logger.info("****************************************") - - -def sub_var_list(var_list, time_info): - """! Perform string substitution on var list values with time info - - @param var_list list of field info to substitute values into - @param time_info dictionary containing time information - @returns var_list with values substituted - """ - if not var_list: - return [] - - out_var_list = [] - for var_info in var_list: - out_var_info = _sub_var_info(var_info, time_info) - out_var_list.append(out_var_info) - - return out_var_list - - -def _sub_var_info(var_info, time_info): - if not var_info: - return {} - - out_var_info = {} - for key, value in var_info.items(): - if isinstance(value, list): - out_value = [] - for item in value: - out_value.append(do_string_sub(item, - skip_missing_tags=True, - **time_info)) - else: - out_value = do_string_sub(value, - skip_missing_tags=True, - **time_info) - - out_var_info[key] = out_value - - return out_var_info diff --git a/metplus/util/config_util.py b/metplus/util/config_util.py new file mode 100644 index 0000000000..0a08aeff15 --- /dev/null +++ b/metplus/util/config_util.py @@ -0,0 +1,239 @@ +import os +import re + +from .constants import LOWER_TO_WRAPPER_NAME +from .string_manip import getlist +from .string_template_substitution import do_string_sub +from .system_util import mkdir_p + + +def get_wrapper_name(process_name): + """! Determine name of wrapper from string that may not contain the correct + capitalization, i.e. Pcp-Combine translates to PCPCombine + + @param process_name string that was listed in the PROCESS_LIST + @returns name of wrapper (without 'Wrapper' at the end) and None if + name cannot be determined + """ + lower_process = (process_name.replace('-', '').replace('_', '') + .replace(' ', '').lower()) + if lower_process in LOWER_TO_WRAPPER_NAME.keys(): + return LOWER_TO_WRAPPER_NAME[lower_process] + + return None + + +def get_process_list(config): + """!Read process list, Extract instance string if specified inside + parenthesis. Remove dashes/underscores and change to lower case, + then map the name to the correct wrapper name + + @param config METplusConfig object to read PROCESS_LIST value + @returns list of tuple containing process name and instance identifier + (None if no instance was set) + """ + # get list of processes + process_list = getlist(config.getstr('config', 'PROCESS_LIST')) + + out_process_list = [] + # for each item remove dashes, underscores, and cast to lower-case + for process in process_list: + # if instance is specified, extract the text inside parenthesis + match = re.match(r'(.*)\((.*)\)', process) + if match: + instance = match.group(2) + process_name = match.group(1) + else: + instance = None + process_name = process + + wrapper_name = get_wrapper_name(process_name) + if wrapper_name is None: + config.logger.warning(f"PROCESS_LIST item {process_name} " + "may be invalid.") + wrapper_name = process_name + + out_process_list.append((wrapper_name, instance)) + + return out_process_list + + +def get_custom_string_list(config, met_tool): + var_name = 'CUSTOM_LOOP_LIST' + custom_loop_list = config.getstr_nocheck('config', + f'{met_tool.upper()}_{var_name}', + config.getstr_nocheck('config', + var_name, + '')) + custom_loop_list = getlist(custom_loop_list) + if not custom_loop_list: + custom_loop_list.append('') + + return custom_loop_list + + +def is_loop_by_init(config): + """!Check config variables to determine if looping by valid or init time""" + if config.has_option('config', 'LOOP_BY'): + loop_by = config.getstr('config', 'LOOP_BY').lower() + if loop_by in ['init', 'retro']: + return True + elif loop_by in ['valid', 'realtime']: + return False + + if config.has_option('config', 'LOOP_BY_INIT'): + return config.getbool('config', 'LOOP_BY_INIT') + + msg = 'MUST SET LOOP_BY to VALID, INIT, RETRO, or REALTIME' + if config.logger is None: + print(msg) + else: + config.logger.error(msg) + + return None + + +def handle_tmp_dir(config): + """! if env var MET_TMP_DIR is set, override config TMP_DIR with value + if it differs from what is set + get config temp dir using getdir_nocheck to bypass check for /path/to + this is done so the user can set env MET_TMP_DIR instead of config TMP_DIR + and config TMP_DIR will be set automatically""" + handle_env_var_config(config, 'MET_TMP_DIR', 'TMP_DIR') + + # create temp dir if it doesn't exist already + # this will fail if TMP_DIR is not set correctly and + # env MET_TMP_DIR was not set + mkdir_p(config.getdir('TMP_DIR')) + + +def handle_env_var_config(config, env_var_name, config_name): + """! If environment variable is set, use that value + for the config variable and warn if the previous config value differs + + @param config METplusConfig object to read + @param env_var_name name of environment variable to read + @param config_name name of METplus config variable to check + """ + env_var_value = os.environ.get(env_var_name, '') + config_value = config.getdir_nocheck(config_name, '') + + # do nothing if environment variable is not set + if not env_var_value: + return + + # override config config variable to environment variable value + config.set('config', config_name, env_var_value) + + # if config config value differed from environment variable value, warn + if config_value == env_var_value: + return + + config.logger.warning(f'Config variable {config_name} ({config_value}) ' + 'will be overridden by the environment variable ' + f'{env_var_name} ({env_var_value})') + + +def log_runtime_banner(config, time_input, process): + loop_by = time_input['loop_by'] + run_time = time_input[loop_by].strftime("%Y-%m-%d %H:%M") + + process_name = process.__class__.__name__ + if process.instance: + process_name = f"{process_name}({process.instance})" + + config.logger.info("****************************************") + config.logger.info(f"* Running METplus {process_name}") + config.logger.info(f"* at {loop_by} time: {run_time}") + config.logger.info("****************************************") + + +def write_final_conf(config): + """! Write final conf file including default values that were set during + run. Move variables that are specific to the user's run to the [runtime] + section to avoid issues such as overwriting existing log files. + + @param config METplusConfig object to write to file + """ + final_conf = config.getstr('config', 'METPLUS_CONF') + + # remove variables that start with CURRENT + config.remove_current_vars() + + # move runtime variables to [runtime] section + config.move_runtime_configs() + + config.logger.info('Overwrite final conf here: %s' % (final_conf,)) + with open(final_conf, 'wt') as conf_file: + config.write(conf_file) + + +def write_all_commands(all_commands, config): + """! Write all commands that were run to a file in the log + directory. This includes the environment variables that + were set before each command. + + @param all_commands list of tuples with command run and + list of environment variables that were set + @param config METplusConfig object used to write log output + and get the log timestamp to name the output file + @returns False if no commands were provided, True otherwise + """ + if not all_commands: + config.logger.error("No commands were run. " + "Skip writing all_commands file") + return False + + log_timestamp = config.getstr('config', 'LOG_TIMESTAMP') + filename = os.path.join(config.getdir('LOG_DIR'), + f'.all_commands.{log_timestamp}') + config.logger.debug(f"Writing all commands and environment to {filename}") + with open(filename, 'w') as file_handle: + for command, envs in all_commands: + for env in envs: + file_handle.write(f"{env}\n") + + file_handle.write("COMMAND:\n") + file_handle.write(f"{command}\n\n") + + return True + + +def sub_var_list(var_list, time_info): + """! Perform string substitution on var list values with time info + + @param var_list list of field info to substitute values into + @param time_info dictionary containing time information + @returns var_list with values substituted + """ + if not var_list: + return [] + + out_var_list = [] + for var_info in var_list: + out_var_info = _sub_var_info(var_info, time_info) + out_var_list.append(out_var_info) + + return out_var_list + + +def _sub_var_info(var_info, time_info): + if not var_info: + return {} + + out_var_info = {} + for key, value in var_info.items(): + if isinstance(value, list): + out_value = [] + for item in value: + out_value.append(do_string_sub(item, + skip_missing_tags=True, + **time_info)) + else: + out_value = do_string_sub(value, + skip_missing_tags=True, + **time_info) + + out_var_info[key] = out_value + + return out_var_info diff --git a/metplus/util/config_validate.py b/metplus/util/config_validate.py new file mode 100644 index 0000000000..e005763150 --- /dev/null +++ b/metplus/util/config_validate.py @@ -0,0 +1,351 @@ +import os + +from .constants import DEPRECATED_DICT, DEPRECATED_MET_LIST +from .constants import UPGRADE_INSTRUCTIONS_URL +from .string_manip import find_indices_in_config_section, getlist +from .string_manip import is_python_script +from .string_template_substitution import do_string_sub +from .config_util import get_process_list, get_custom_string_list + + +def validate_config_variables(config): + + all_sed_cmds = [] + # check for deprecated config items and warn user to remove/replace them + deprecated_is_ok, sed_cmds = check_for_deprecated_config(config) + all_sed_cmds.extend(sed_cmds) + + # check for deprecated env vars in MET config files and + # warn user to remove/replace them + deprecated_met_is_ok, sed_cmds = check_for_deprecated_met_config(config) + all_sed_cmds.extend(sed_cmds) + + # validate configuration variables + field_is_ok, sed_cmds = validate_field_info_configs(config) + all_sed_cmds.extend(sed_cmds) + + # check that OUTPUT_BASE is not set to the exact same value as INPUT_BASE + inoutbase_is_ok = True + input_real_path = os.path.realpath(config.getdir_nocheck('INPUT_BASE', '')) + output_real_path = os.path.realpath(config.getdir('OUTPUT_BASE')) + if input_real_path == output_real_path: + config.logger.error("INPUT_BASE AND OUTPUT_BASE are set to the " + f"exact same path: {input_real_path}") + config.logger.error("Please change one of these paths to avoid risk " + "of losing input data") + inoutbase_is_ok = False + + check_user_environment(config) + + is_ok = (deprecated_is_ok and field_is_ok and + inoutbase_is_ok and deprecated_met_is_ok) + + return is_ok, all_sed_cmds + + +def check_for_deprecated_config(config): + """!Checks user configuration files and reports errors or warnings if any + deprecated variable are found. If an alternate variable name can be + suggested, add it to the 'alt' section. If the alternate cannot be + literally substituted for the old name, set copy to False + + @param config METplusConfig object to evaluate + @returns A tuple containing a boolean if the configuration is suitable to + run or not and if it is not correct, the 2nd item is a list of sed + commands that can be run to help fix the incorrect config variables + """ + logger = config.logger + + # create list of errors and warnings to report for deprecated configs + e_list = [] + all_sed_cmds = [] + + # keep track of upgrade instructions to output after all vars are checked + upgrade_notes = set() + + for old, depr_info in DEPRECATED_DICT.items(): + if not isinstance(depr_info, dict): + continue + + # check if is found in old item, use regex to find vars if found + if '' not in old: + upgrade_note = handle_deprecated(old, depr_info.get('alt', ''), + depr_info, config, all_sed_cmds, + e_list) + if upgrade_note: + upgrade_notes.add(upgrade_note) + continue + + old_regex = old.replace('', r'(\d+)') + indices = find_indices_in_config_section(old_regex, + config, + index_index=1).keys() + for index in indices: + old_with_index = old.replace('', index) + alt_with_index = depr_info.get('alt', '').replace('', index) + + upgrade_note = handle_deprecated(old_with_index, alt_with_index, + depr_info, config, all_sed_cmds, + e_list) + if upgrade_note: + upgrade_notes.add(upgrade_note) + + if 'ensemble' in upgrade_notes: + short_msg = ('Please navigate to the upgrade instructions: ' + f'{UPGRADE_INSTRUCTIONS_URL}') + msg = ('EnsembleStat functionality has been moved to GenEnsProd. ' + 'The required changes to the config files depend on ' + 'the type of evaluation that is being performed. ' + f'{short_msg}') + + e_list.insert(0, msg) + e_list.append(short_msg) + + # if any errors exist, report them and exit + if e_list: + logger.error('DEPRECATED CONFIG ITEMS WERE FOUND. PLEASE FOLLOW THE ' + 'INSTRUCTIONS TO UPDATE THE CONFIG FILES') + for error_msg in e_list: + logger.error(error_msg) + return False, all_sed_cmds + + return True, [] + + +def handle_deprecated(old, alt, depr_info, config, all_sed_cmds, e_list): + sec = 'config' + config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') + + upgrade_note = None + + # if deprecated config item is found + if not config.has_option(sec, old): + return upgrade_note + + upgrade_note = depr_info.get('upgrade') + + # if it is required to remove, add to error list + if not alt: + e_list.append("{} should be removed".format(old)) + return upgrade_note + + e_list.append("{} should be replaced with {}".format(old, alt)) + + if 'copy' not in depr_info.keys() or depr_info['copy']: + for config_file in config_files: + all_sed_cmds.append(f"sed -i 's|^{old}|{alt}|g' {config_file}") + all_sed_cmds.append(f"sed -i 's|{{{old}}}|{{{alt}}}|g' {config_file}") + + return upgrade_note + + +def check_for_deprecated_met_config(config): + sed_cmds = [] + all_good = True + + # check if *_CONFIG_FILE if set in the METplus config file and check for + # deprecated environment variables in those files + met_config_keys = [key for key in config.keys('config') + if key.endswith('CONFIG_FILE')] + + for met_config_key in met_config_keys: + met_tool = met_config_key.replace('_CONFIG_FILE', '') + + # get custom loop list to check if multiple config files are used + # based on the custom string + custom_list = get_custom_string_list(config, met_tool) + + for custom_string in custom_list: + met_config = config.getraw('config', met_config_key) + if not met_config: + continue + + met_config_file = do_string_sub(met_config, custom=custom_string) + + if not check_for_deprecated_met_config_file(config, + met_config_file): + all_good = False + + return all_good, sed_cmds + + +def check_for_deprecated_met_config_file(config, met_config): + + all_good = True + if not os.path.exists(met_config): + config.logger.error(f"Config file does not exist: {met_config}") + return False + + # skip check if no deprecated variables are set + if not DEPRECATED_MET_LIST: + return all_good + + config.logger.debug("Checking for deprecated environment " + f"variables in: {met_config}") + + with open(met_config, 'r') as file_handle: + lines = file_handle.read().splitlines() + + for line in lines: + for deprecated_item in DEPRECATED_MET_LIST: + if '${' + deprecated_item + '}' not in line: + continue + all_good = False + config.logger.error("Please remove deprecated environment variable" + f" ${{{deprecated_item}}} found in MET config " + f"file: {met_config}") + + return all_good + + +def validate_field_info_configs(config): + """!Verify that config variables with _VAR_ in them are valid. + + @param config METplusConfig object to validate + @returns True if all are valid or False if any items are invalid + """ + + variable_extensions = ['NAME', 'LEVELS', 'THRESH', 'OPTIONS'] + all_good = True, [] + + if skip_field_info_validation(config): + return True, [] + + # keep track of all sed commands to replace config variable names + all_sed_cmds = [] + + for ext in variable_extensions: + # find all _VAR_ keys in the conf files + data_types_and_indices = find_indices_in_config_section(r"(\w+)_VAR(\d+)_"+ext, + config, + index_index=2, + id_index=1) + + # if BOTH_VAR_ is used, set FCST and OBS to the same value + # if FCST or OBS is used, the other must be present as well + # if BOTH and either FCST or OBS are set, report an error + # get other data type + for index, data_type_list in data_types_and_indices.items(): + + is_valid, err_msgs, sed_cmds = is_var_item_valid(data_type_list, index, ext, config) + if not is_valid: + for err_msg in err_msgs: + config.logger.error(err_msg) + all_sed_cmds.extend(sed_cmds) + all_good = False + + # make sure FCST and OBS have the same number of levels if coming from separate variables + elif ext == 'LEVELS' and all(item in ['FCST', 'OBS'] for item in data_type_list): + fcst_levels = getlist(config.getraw('config', f"FCST_VAR{index}_LEVELS", '')) + + # add empty string if no levels are found because python embedding items do not need + # to include a level, but the other item may have a level and the numbers need to match + if not fcst_levels: + fcst_levels.append('') + + obs_levels = getlist(config.getraw('config', f"OBS_VAR{index}_LEVELS", '')) + if not obs_levels: + obs_levels.append('') + + if len(fcst_levels) != len(obs_levels): + config.logger.error(f"FCST_VAR{index}_LEVELS and OBS_VAR{index}_LEVELS do not have " + "the same number of elements") + all_good = False + + return all_good, all_sed_cmds + + +def skip_field_info_validation(config): + """!Check config to see if having corresponding FCST/OBS variables is necessary. If process list only + contains reformatter wrappers, don't validate field info. Also, if MTD is in the process list and + it is configured to only process either FCST or OBS, validation is unnecessary.""" + + reformatters = ['PCPCombine', 'RegridDataPlane'] + process_list = [item[0] for item in get_process_list(config)] + + # if running MTD in single mode, you don't need matching FCST/OBS + if ('MTD' in process_list and + config.getbool('config', 'MTD_SINGLE_RUN', False)): + return True + + # if running any app other than the reformatters, you need matching FCST/OBS, so don't skip + if [item for item in process_list if item not in reformatters]: + return False + + return True + + +def check_user_environment(config): + """!Check if any environment variables set in [user_env_vars] are already set in + the user's environment. Warn them that it will be overwritten from the conf if it is""" + if not config.has_section('user_env_vars'): + return + + for env_var in config.keys('user_env_vars'): + if env_var in os.environ: + msg = '{} is already set in the environment. '.format(env_var) +\ + 'Overwriting from conf file' + config.logger.warning(msg) + + +def is_var_item_valid(item_list, index, ext, config): + """!Given a list of data types (FCST, OBS, ENS, or BOTH) check if the + combination is valid. + If BOTH is found, FCST and OBS should not be found. + If FCST or OBS is found, the other must also be found. + @param item_list list of data types that were found for a given index + @param index number following _VAR in the variable name + @param ext extension to check, i.e. NAME, LEVELS, THRESH, or OPTIONS + @param config METplusConfig instance + @returns tuple containing boolean if var item is valid, list of error + messages and list of sed commands to help the user update their old + configuration files + """ + + full_ext = f"_VAR{index}_{ext}" + msg = [] + sed_cmds = [] + if 'BOTH' in item_list and ('FCST' in item_list or 'OBS' in item_list): + + msg.append(f"Cannot set FCST{full_ext} or OBS{full_ext} if BOTH{full_ext} is set.") + elif ext == 'THRESH': + # allow thresholds unless BOTH and (FCST or OBS) are set + pass + + elif 'FCST' in item_list and 'OBS' not in item_list: + # if FCST level has 1 item and OBS name is a python embedding script, + # don't report error + level_list = getlist(config.getraw('config', + f'FCST_VAR{index}_LEVELS', + '')) + other_name = config.getraw('config', f'OBS_VAR{index}_NAME', '') + skip_error_for_py_embed = ext == 'LEVELS' and is_python_script(other_name) and len(level_list) == 1 + # do not report error for OPTIONS since it isn't required to be the same length + if ext not in ['OPTIONS'] and not skip_error_for_py_embed: + msg.append(f"If FCST{full_ext} is set, you must either set OBS{full_ext} or " + f"change FCST{full_ext} to BOTH{full_ext}") + + config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') + for config_file in config_files: + sed_cmds.append(f"sed -i 's|^FCST{full_ext}|BOTH{full_ext}|g' {config_file}") + sed_cmds.append(f"sed -i 's|{{FCST{full_ext}}}|{{BOTH{full_ext}}}|g' {config_file}") + + elif 'OBS' in item_list and 'FCST' not in item_list: + # if OBS level has 1 item and FCST name is a python embedding script, + # don't report error + level_list = getlist(config.getraw('config', + f'OBS_VAR{index}_LEVELS', + '')) + other_name = config.getraw('config', f'FCST_VAR{index}_NAME', '') + skip_error_for_py_embed = ext == 'LEVELS' and is_python_script(other_name) and len(level_list) == 1 + + if ext not in ['OPTIONS'] and not skip_error_for_py_embed: + msg.append(f"If OBS{full_ext} is set, you must either set FCST{full_ext} or " + f"change OBS{full_ext} to BOTH{full_ext}") + + config_files = config.getstr('config', 'CONFIG_INPUT', '').split(',') + for config_file in config_files: + sed_cmds.append(f"sed -i 's|^OBS{full_ext}|BOTH{full_ext}|g' {config_file}") + sed_cmds.append(f"sed -i 's|{{OBS{full_ext}}}|{{BOTH{full_ext}}}|g' {config_file}") + + return not bool(msg), msg, sed_cmds diff --git a/metplus/util/constants.py b/metplus/util/constants.py index 5e6f3dcb99..48956d8795 100644 --- a/metplus/util/constants.py +++ b/metplus/util/constants.py @@ -1,5 +1,10 @@ # Constant variables used throughout the METplus wrappers source code +UPGRADE_INSTRUCTIONS_URL = ( + 'https://metplus.readthedocs.io/en/develop/Users_Guide/' + 'release-notes.html#metplus-wrappers-upgrade-instructions' +) + # dictionary used by get_wrapper_name function to easily convert wrapper # name in many formats to the correct name of the wrapper class LOWER_TO_WRAPPER_NAME = { @@ -113,3 +118,48 @@ # we often check for None if a variable is not set, but 0 and None # have the same result in a test. 0 may be a valid integer value MISSING_DATA_VALUE = -9999 + +# Dictionary used to alert users that they are using deprecated config +# variables and need to update the configs to run METplus +# key is the name of the depreacted variable that is no longer allowed in any +# config files +# value is a dictionary containing information about what to do with the +# deprecated config +# 'alt' is the alternative name for the deprecated config. this can be a +# single variable name or text to describe multiple variables or how to +# handle it. Set to None to tell the user to just remove the variable. +# 'copy' is an optional item (defaults to True). set this to False if one +# cannot simply replace the deprecated variable name with the value in 'alt' +# 'upgrade' is an optional item where the value is a keyword that will output +# additional instructions for the user. +# Valid Values: 'ensemble' +DEPRECATED_DICT = { + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MEAN': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_STDEV': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MINUS': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_PLUS': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MIN': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_MAX': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_RANGE': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_VLD_COUNT': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_FREQUENCY': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_NEP': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_ENSEMBLE_FLAG_NMEP': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NBRHD_PROB_WIDTH': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NBRHD_PROB_SHAPE': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NBRHD_PROB_VLD_THRESH': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_VLD_THRESH': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_SHAPE': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_METHOD': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_WIDTH': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_DX': {'upgrade': 'ensemble'}, + 'ENSEMBLE_STAT_NMEP_SMOOTH_GAUSSIAN_RADIUS': {'upgrade': 'ensemble'}, +} + +# List of variables in wrapped MET config files that are no longer set +# All explicitly set wrapped MET config files found in a METplus config, +# e.g. GRID_STAT_CONFIG_FILE, will be checked for these variables +# If any of these items are found, then an error will be reported +DEPRECATED_MET_LIST = [ +] diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index d32e42f778..a61ce75a63 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -3,25 +3,7 @@ import sys import os -from . import LOWER_TO_WRAPPER_NAME - - -def get_wrapper_name(process_name): - """! Determine name of wrapper from string that may not contain the correct - capitalization, i.e. Pcp-Combine translates to PCPCombine - - @param process_name string that was listed in the PROCESS_LIST - @returns name of wrapper (without 'Wrapper' at the end) and None if - name cannot be determined - """ - lower_process = (process_name.replace('-', '') - .replace('_', '') - .replace(' ', '') - .lower()) - if lower_process in LOWER_TO_WRAPPER_NAME.keys(): - return LOWER_TO_WRAPPER_NAME[lower_process] - - return None +from .config_util import get_wrapper_name def print_doc_text(tool_name, input_dict): diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index ddb4ef71ca..347b976d90 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -9,7 +9,8 @@ from .constants import PYTHON_EMBEDDING_TYPES, CLIMO_TYPES, MISSING_DATA_VALUE from .string_manip import getlist, get_threshold_via_regex from .string_manip import remove_quotes as util_remove_quotes -from .config_metplus import find_indices_in_config_section, parse_var_list +from .string_manip import find_indices_in_config_section +from .config_metplus import parse_var_list from .field_util import format_all_field_info class METConfig: diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index fb7b743b35..3298e5fdac 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -6,17 +6,21 @@ from .constants import NO_COMMAND_WRAPPERS from .system_util import get_user_info, write_list_to_file +from .config_util import get_process_list, handle_env_var_config +from .config_util import handle_tmp_dir, write_final_conf, write_all_commands +from .config_validate import validate_config_variables from .. import get_metplus_version -from . import config_metplus +from .config_metplus import setup from . import camel_to_underscore + def pre_run_setup(config_inputs): version_number = get_metplus_version() print(f'Starting METplus v{version_number}') # Read config inputs and return a config instance - config = config_metplus.setup(config_inputs) + config = setup(config_inputs) logger = config.logger @@ -35,19 +39,23 @@ def pre_run_setup(config_inputs): logger.info(f"Config Input: {config_item}") # validate configuration variables - isOK_A, isOK_B, isOK_C, isOK_D, all_sed_cmds = config_metplus.validate_configuration_variables(config) - if not (isOK_A and isOK_B and isOK_C and isOK_D): + is_ok, all_sed_cmds = validate_config_variables(config) + if not is_ok: # if any sed commands were generated, write them to the sed file if all_sed_cmds: - sed_file = os.path.join(config.getdir('OUTPUT_BASE'), 'sed_commands.txt') + sed_file = os.path.join(config.getdir('OUTPUT_BASE'), + 'sed_commands.txt') # remove if sed file exists if os.path.exists(sed_file): os.remove(sed_file) write_list_to_file(sed_file, all_sed_cmds) - config.logger.error(f"Find/Replace commands have been generated in {sed_file}") + config.logger.error("Find/Replace commands have been " + f"generated in {sed_file}") logger.error("Correct configuration variables and rerun. Exiting.") + logger.info("Check the log file for more information: " + f"{config.getstr('config', 'LOG_METPLUS')}") sys.exit(1) if not config.getdir('MET_INSTALL_DIR', must_exist=True): @@ -60,10 +68,10 @@ def pre_run_setup(config_inputs): os.path.join(config.getdir('OUTPUT_BASE'), "stage")) # handle dir to write temporary files - config_metplus.handle_tmp_dir(config) + handle_tmp_dir(config) # handle OMP_NUM_THREADS environment variable - config_metplus.handle_env_var_config(config, + handle_env_var_config(config, env_var_name='OMP_NUM_THREADS', config_name='OMP_NUM_THREADS') @@ -76,7 +84,7 @@ def run_metplus(config): total_errors = 0 # Use config object to get the list of processes to call - process_list = config_metplus.get_process_list(config) + process_list = get_process_list(config) try: processes = [] @@ -129,27 +137,30 @@ def run_metplus(config): # if process list contains any wrapper that should run commands if any([item[0] not in NO_COMMAND_WRAPPERS for item in process_list]): # write out all commands and environment variables to file - if not config_metplus.write_all_commands(all_commands, config): + if not write_all_commands(all_commands, config): # report an error if no commands were generated total_errors += 1 # compute total number of errors that occurred and output results for process in processes: - if process.errors != 0: - process_name = process.__class__.__name__.replace('Wrapper', '') - error_msg = '{} had {} error'.format(process_name, process.errors) - if process.errors > 1: - error_msg += 's' - error_msg += '.' - logger.error(error_msg) - total_errors += process.errors + if not process.errors: + continue + process_name = process.__class__.__name__.replace('Wrapper', '') + error_msg = f'{process_name} had {process.errors} error' + if process.errors > 1: + error_msg += 's' + error_msg += '.' + logger.error(error_msg) + total_errors += process.errors return total_errors except: logger.exception("Fatal error occurred") - logger.info(f"Check the log file for more information: {config.getstr('config', 'LOG_METPLUS')}") + logger.info("Check the log file for more information: " + f"{config.getstr('config', 'LOG_METPLUS')}") return 1 + def post_run_cleanup(config, app_name, total_errors): logger = config.logger # scrub staging directory if requested @@ -169,7 +180,7 @@ def post_run_cleanup(config, app_name, total_errors): '%Y%m%d%H%M%S') # rewrite final conf so it contains all of the default values used - config_metplus.write_final_conf(config) + write_final_conf(config) # compute time it took to run end_clock_time = datetime.now() diff --git a/metplus/util/string_manip.py b/metplus/util/string_manip.py index c297dde55e..d93bda5ad2 100644 --- a/metplus/util/string_manip.py +++ b/metplus/util/string_manip.py @@ -496,3 +496,45 @@ def subset_list(full_list, subset_definition): subset_list = [subset_list] return subset_list + + +def find_indices_in_config_section(regex, config, sec='config', + index_index=1, id_index=None): + """! Use regular expression to get all config variables that match and + are set in the user's configuration. This is used to handle config + variables that have multiple indices, i.e. FCST_VAR1_NAME, FCST_VAR2_NAME, + etc. + + @param regex regular expression to use to find variables + @param config METplusConfig object to search + @param sec (optional) config file section to search. Defaults to config + @param index_index 1 based number that is the regex match index for the + index number (default is 1) + @param id_index 1 based number that is the regex match index for the + identifier. Defaults to None which does not extract an indentifier + + number and the first match is used as an identifier + @returns dictionary where keys are the index number and the value is a + list of identifiers (if noID=True) or a list containing None + """ + # regex expression must have 2 () items and the 2nd item must be the index + all_conf = config.keys(sec) + indices = {} + regex = re.compile(regex) + for conf in all_conf: + result = regex.match(conf) + if result is None: + continue + + index = result.group(index_index) + if id_index: + identifier = result.group(id_index) + else: + identifier = None + + if index not in indices: + indices[index] = [identifier] + else: + indices[index].append(identifier) + + return indices diff --git a/metplus/util/time_looping.py b/metplus/util/time_looping.py index 2cd124ff8d..4e6f275f4d 100644 --- a/metplus/util/time_looping.py +++ b/metplus/util/time_looping.py @@ -6,7 +6,8 @@ from .time_util import ti_get_hours_from_relativedelta from .time_util import ti_get_seconds_from_relativedelta from .string_template_substitution import do_string_sub -from .config_metplus import log_runtime_banner +from .config_util import log_runtime_banner + def time_generator(config): """! Generator used to read METplusConfig variables for time looping diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 6af2d49a66..ee7810468a 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -22,7 +22,7 @@ from ..util import getlist, preprocess_file, loop_over_times_and_call from ..util import do_string_sub, ti_calculate, get_seconds_from_string from ..util import get_time_from_file, shift_time_seconds -from ..util import config_metplus +from ..util import replace_config_from_section from ..util import METConfig from ..util import MISSING_DATA_VALUE from ..util import get_custom_string_list @@ -82,11 +82,8 @@ def __init__(self, config, instance=None): # if instance is set, check for a section with the same name in the # METplusConfig object. If found, copy all values into the config if instance: - self.config = ( - config_metplus.replace_config_from_section(self.config, - instance, - required=False) - ) + self.config = replace_config_from_section(self.config, instance, + required=False) self.instance = instance self.env = config.env if hasattr(config, 'env') else os.environ.copy() diff --git a/ush/run_metplus.py b/ush/run_metplus.py index 39149b7e42..1971c2f0e9 100755 --- a/ush/run_metplus.py +++ b/ush/run_metplus.py @@ -18,6 +18,7 @@ import os import sys +import traceback # add metplus directory to path so the wrappers and utilities can be found sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), @@ -126,5 +127,6 @@ def get_config_inputs_from_command_line(): produtil.setup.setup(send_dbn=False, jobname='run-METplus') main() except Exception as exc: + print(traceback.format_exc()) print('ERROR: run_metplus failed: %s' % exc) sys.exit(2) diff --git a/ush/validate_config.py b/ush/validate_config.py index bdc54beb9b..015d36662d 100755 --- a/ush/validate_config.py +++ b/ush/validate_config.py @@ -8,7 +8,7 @@ the changes made for them. History Log: Initial version Usage: Call the same as run_metplus.py, - i.e. validate_config.py -c -c + i.e. validate_config.py Parameters: None Input Files: Configuration files Output Files: @@ -24,9 +24,10 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) -from metplus.util import config_metplus, validate_configuration_variables +from metplus.util import config_metplus, validate_config_variables from run_metplus import get_config_inputs_from_command_line + def main(): config_inputs = get_config_inputs_from_command_line() @@ -34,20 +35,17 @@ def main(): config = config_metplus.setup(config_inputs) # validate configuration variables - deprecatedIsOK, fieldIsOK, inoutbaseIsOk, metIsOK, sed_cmds = validate_configuration_variables(config) - - if metIsOK: - print("No MET config files are using deprecated environment variables") + is_ok, sed_commands = validate_config_variables(config) # if everything is valid, report success and exit - if deprecatedIsOK and fieldIsOK and inoutbaseIsOk and metIsOK: + if is_ok: print("SUCCESS: Configuration passed all of the validation tests.") sys.exit(0) # if sed commands can be run, output lines that will be changed and ask # user if they want to run the sed command - if sed_cmds: - for cmd in sed_cmds: + if sed_commands: + for cmd in sed_commands: if cmd.startswith('#Add'): add_line = cmd.replace('#Add ', '') met_tool = None