diff --git a/.github/actions/run_tests/entrypoint.sh b/.github/actions/run_tests/entrypoint.sh index 7dda2d2133..3bfad88343 100644 --- a/.github/actions/run_tests/entrypoint.sh +++ b/.github/actions/run_tests/entrypoint.sh @@ -61,7 +61,7 @@ if [[ "$INPUT_CATEGORIES" == pytests* ]]; then for x in `cat $PYTESTS_GROUPS_FILEPATH`; do marker="${x//_or_/ or }" marker="${marker//not_/not }" - command+="/usr/local/envs/pytest/bin/pytest -vv --cov=../../metplus -m \"$marker\"" + command+="/usr/local/envs/pytest/bin/pytest -vv --cov=../../../metplus -m \"$marker\"" command+=";if [ \$? != 0 ]; then status=1; fi;" done command+="if [ \$status != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi" diff --git a/.github/parm/pytest_groups.txt b/.github/parm/pytest_groups.txt index 48bf9bd415..a5ca80e665 100644 --- a/.github/parm/pytest_groups.txt +++ b/.github/parm/pytest_groups.txt @@ -1,3 +1,4 @@ +run_metplus util wrapper wrapper_a diff --git a/internal/tests/pytests/conftest.py b/internal/tests/pytests/conftest.py index 97af766988..8056e4cfe4 100644 --- a/internal/tests/pytests/conftest.py +++ b/internal/tests/pytests/conftest.py @@ -4,6 +4,7 @@ import subprocess import pytest import getpass +import shutil from pathlib import Path # add METplus directory to path so the wrappers and utilities can be found @@ -19,7 +20,8 @@ if pytest_host is None: import socket pytest_host = socket.gethostname() - print(f"No hostname provided with METPLUS_PYTEST_HOST, using {pytest_host}") + print("No hostname provided with METPLUS_PYTEST_HOST, " + f"using {pytest_host}") else: print(f"METPLUS_PYTEST_HOST = {pytest_host}") @@ -33,7 +35,8 @@ # source minimum_pytest..sh script current_user = getpass.getuser() -command = shlex.split(f"env -i bash -c 'export USER={current_user} && source {minimum_pytest_file} && env'") +command = shlex.split(f"env -i bash -c 'export USER={current_user} && " + f"source {minimum_pytest_file} && env'") proc = subprocess.Popen(command, stdout=subprocess.PIPE) for line in proc.stdout: @@ -43,21 +46,70 @@ proc.communicate() +output_base = os.environ['METPLUS_TEST_OUTPUT_BASE'] +if not output_base: + print('ERROR: METPLUS_TEST_OUTPUT_BASE must be set to a path to write') + sys.exit(1) + +test_output_dir = os.path.join(output_base, 'test_output') +if os.path.exists(test_output_dir): + print(f'Removing test output dir: {test_output_dir}') + shutil.rmtree(test_output_dir) + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + """! This is used to capture the status of a test so the metplus_config + fixture can remove output data from tests that pass. + """ + # execute all other hooks to obtain the report object + outcome = yield + rep = outcome.get_result() + + # set a report attribute for each phase of a call, which can + # be "setup", "call", "teardown" + + setattr(item, "rep_" + rep.when, rep) + + +@pytest.fixture() +def metplus_config(request): + """! Create a METplus configuration object using only the minimum required + settings found in minimum_pytest.conf. This fixture checks the result of + the test it is used in and automatically removes the output that is + generated by it unless the test fails. This makes it much easier to review + the failed tests. To use this fixture, add metplus_config to the test + function arguments and set a variable called config to metplus_config, e.g. + config = metplus_config. + """ + script_dir = os.path.dirname(__file__) + args = [os.path.join(script_dir, 'minimum_pytest.conf')] + config = config_metplus.setup(args) + yield config + + # don't remove output base if test fails + if request.node.rep_call.failed: + return + config_output_base = config.getdir('OUTPUT_BASE') + if config_output_base and os.path.exists(config_output_base): + shutil.rmtree(config_output_base) + + @pytest.fixture(scope='function') -def metplus_config(): - """! Create a METplus configuration object that can be - manipulated/modified to - reflect different paths, directories, values, etc. for individual - tests. +def metplus_config_files(): + """! Create a METplus configuration object using minimum_pytest.conf + settings and any list of config files.The metplus_config fixture is + preferred because it automatically cleans up the output files generated + by the use case unless the test fails. To use this in a test, add + metplus_config_files as an argument to the test function and pass in a list + of config files to it. Example: config = metplus_config_files([my_file]) """ - def read_configs(extra_configs=[]): + def read_configs(extra_configs): # Read in minimum pytest config file and any other extra configs script_dir = os.path.dirname(__file__) minimum_conf = os.path.join(script_dir, 'minimum_pytest.conf') - args = [minimum_conf] - if extra_configs: - args.extend(extra_configs) - + args = extra_configs.copy() + args.append(minimum_conf) config = config_metplus.setup(args) return config diff --git a/internal/tests/pytests/minimum_pytest.conf b/internal/tests/pytests/minimum_pytest.conf index 5a68934956..9982acc1a4 100644 --- a/internal/tests/pytests/minimum_pytest.conf +++ b/internal/tests/pytests/minimum_pytest.conf @@ -1,6 +1,6 @@ [config] INPUT_BASE = {ENV[METPLUS_TEST_INPUT_BASE]} -OUTPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]} +OUTPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]}/test_output/{RUN_ID} MET_INSTALL_DIR = {ENV[METPLUS_TEST_MET_INSTALL_DIR]} TMP_DIR = {ENV[METPLUS_TEST_TMP_DIR]} diff --git a/internal/tests/pytests/minimum_pytest.corrinado.sh b/internal/tests/pytests/minimum_pytest.corrinado.sh deleted file mode 100644 index 0555a9f7e1..0000000000 --- a/internal/tests/pytests/minimum_pytest.corrinado.sh +++ /dev/null @@ -1,13 +0,0 @@ -export METPLUS_TEST_INPUT_BASE=${HOME}/data/METplus_Data -export METPLUS_TEST_OUTPUT_BASE=${HOME}/pytest -export METPLUS_TEST_MET_INSTALL_DIR=${HOME}/met/9.0-beta3 -export METPLUS_TEST_TMP_DIR=${METPLUS_TEST_OUTPUT_BASE}/tmp - -export METPLUS_TEST_EXE_WGRIB2=wgrib2 -export METPLUS_TEST_EXE_CUT=cut -export METPLUS_TEST_EXE_TR=tr -export METPLUS_TEST_EXE_RM=rm -export METPLUS_TEST_EXE_NCAP2=ncap2 -export METPLUS_TEST_EXE_CONVERT=convert -export METPLUS_TEST_EXE_NCDUMP=ncdump -export METPLUS_TEST_EXE_EGREP=egrep diff --git a/internal/tests/pytests/minimum_pytest.dakota.sh b/internal/tests/pytests/minimum_pytest.dakota.sh index e3c93beef7..0b66555fa9 100644 --- a/internal/tests/pytests/minimum_pytest.dakota.sh +++ b/internal/tests/pytests/minimum_pytest.dakota.sh @@ -2,12 +2,3 @@ export METPLUS_TEST_INPUT_BASE=/d3/projects/MET/METplus_Data export METPLUS_TEST_OUTPUT_BASE=/d3/personal/${USER}/pytest export METPLUS_TEST_MET_INSTALL_DIR=/d3/projects/MET/MET_releases/met-9.1_beta3 export METPLUS_TEST_TMP_DIR=${METPLUS_TEST_OUTPUT_BASE}/tmp - -export METPLUS_TEST_EXE_WGRIB2=/usr/local/bin/wgrib2 -export METPLUS_TEST_EXE_CUT=/usr/bin/cut -export METPLUS_TEST_EXE_TR=/usr/bin/tr -export METPLUS_TEST_EXE_RM=/bin/rm -export METPLUS_TEST_EXE_NCAP2=/usr/local/nco/bin/ncap2 -export METPLUS_TEST_EXE_CONVERT=/usr/bin/convert -export METPLUS_TEST_EXE_NCDUMP=/usr/local/bin/ncdump -export METPLUS_TEST_EXE_EGREP=/bin/egrep diff --git a/internal/tests/pytests/minimum_pytest.eyewall.sh b/internal/tests/pytests/minimum_pytest.eyewall.sh index b2a8a99753..06a69dd650 100644 --- a/internal/tests/pytests/minimum_pytest.eyewall.sh +++ b/internal/tests/pytests/minimum_pytest.eyewall.sh @@ -3,12 +3,3 @@ export METPLUS_TEST_OUTPUT_BASE=/d1/${USER}/pytest export METPLUS_TEST_MET_INSTALL_DIR=/usr/local/met-9.0 #export METPLUS_TEST_MET_INSTALL_DIR=/d1/CODE/MET/MET_releases/met-9.0_beta4 export METPLUS_TEST_TMP_DIR=${METPLUS_TEST_OUTPUT_BASE}/tmp - -export METPLUS_TEST_EXE_WGRIB2=/usr/local/bin/wgrib2 -export METPLUS_TEST_EXE_CUT=/usr/bin/cut -export METPLUS_TEST_EXE_TR=/usr/bin/tr -export METPLUS_TEST_EXE_RM=/bin/rm -export METPLUS_TEST_EXE_NCAP2=/usr/local/nco/bin/ncap2 -export METPLUS_TEST_EXE_CONVERT=/usr/bin/convert -export METPLUS_TEST_EXE_NCDUMP=/usr/local/bin/ncdump -export METPLUS_TEST_EXE_EGREP=/bin/egrep diff --git a/internal/tests/pytests/minimum_pytest.hera.sh b/internal/tests/pytests/minimum_pytest.hera.sh index 64407e59a0..bfb541180d 100644 --- a/internal/tests/pytests/minimum_pytest.hera.sh +++ b/internal/tests/pytests/minimum_pytest.hera.sh @@ -2,12 +2,3 @@ export METPLUS_TEST_INPUT_BASE=/home/${USER}/metplus_pytests export METPLUS_TEST_OUTPUT_BASE=/home/${USER}/metplus_pytests/out export METPLUS_TEST_MET_INSTALL_DIR=/contrib/met/8.1 export METPLUS_TEST_TMP_DIR=/tmp - -export METPLUS_TEST_EXE_WGRIB2=/apps/wgrib2/2.0.8/intel/18.0.3.222/bin/wgrib2 -export METPLUS_TEST_EXE_CUT=/usr/bin/cut -export METPLUS_TEST_EXE_TR=/usr/bin/tr -export METPLUS_TEST_EXE_RM=/usr/bin/rm -export METPLUS_TEST_EXE_NCAP2=/apps/nco/4.7.0/intel/18.0.3.051/bin/ncap2 -export METPLUS_TEST_EXE_CONVERT=/usr/bin/convert -export METPLUS_TEST_EXE_NCDUMP=/apps/netcdf/4.7.0/intel/18.0.5.274/bin/ncdump -export METPLUS_TEST_EXE_EGREP=/usr/bin/grep diff --git a/internal/tests/pytests/minimum_pytest.kiowa.sh b/internal/tests/pytests/minimum_pytest.kiowa.sh index 655f80f2d0..33cb80aa93 100644 --- a/internal/tests/pytests/minimum_pytest.kiowa.sh +++ b/internal/tests/pytests/minimum_pytest.kiowa.sh @@ -1,14 +1,5 @@ export METPLUS_TEST_INPUT_BASE=/d1/projects/METplus/METplus_Data export METPLUS_TEST_OUTPUT_BASE=/d1/personal/${USER}/pytest -export METPLUS_TEST_MET_INSTALL_DIR=/usr/local/met-9.0 +export METPLUS_TEST_MET_INSTALL_DIR=/usr/local/met #export METPLUS_TEST_MET_INSTALL_DIR=/d1/projects/MET/MET_releases/met-9.0_beta4 export METPLUS_TEST_TMP_DIR=${METPLUS_TEST_OUTPUT_BASE}/tmp -#export METPLUS_TEST_TMP_DIR=/tmp -export METPLUS_TEST_EXE_WGRIB2=/usr/local/bin/wgrib2 -export METPLUS_TEST_EXE_CUT=/usr/bin/cut -export METPLUS_TEST_EXE_TR=/usr/bin/tr -export METPLUS_TEST_EXE_RM=/bin/rm -export METPLUS_TEST_EXE_NCAP2=/usr/local/nco/bin/ncap2 -export METPLUS_TEST_EXE_CONVERT=/usr/bin/convert -export METPLUS_TEST_EXE_NCDUMP=/usr/local/bin/ncdump -export METPLUS_TEST_EXE_EGREP=/bin/egrep diff --git a/internal/tests/pytests/minimum_pytest.venus.sh b/internal/tests/pytests/minimum_pytest.venus.sh index 493f861ff1..2c4774e348 100644 --- a/internal/tests/pytests/minimum_pytest.venus.sh +++ b/internal/tests/pytests/minimum_pytest.venus.sh @@ -2,12 +2,3 @@ export METPLUS_TEST_INPUT_BASE=/gpfs/dell2/emc/verification/noscrub/$USER/METplu export METPLUS_TEST_OUTPUT_BASE=/gpfs/dell2/emc/verification/noscrub/$USER/metplus_test export METPLUS_TEST_MET_INSTALL_DIR=/gpfs/dell2/emc/verification/noscrub/$USER/met/9.0_beta4 export METPLUS_TEST_TMP_DIR=${METPLUS_TEST_OUTPUT_BASE}/tmp - -export METPLUS_TEST_EXE_WGRIB2=$WGRIB2 -export METPLUS_TEST_EXE_CUT=cut -export METPLUS_TEST_EXE_TR=tr -export METPLUS_TEST_EXE_RM=rm -export METPLUS_TEST_EXE_NCAP2=ncap2 -export METPLUS_TEST_EXE_CONVERT=convert -export METPLUS_TEST_EXE_NCDUMP=ncdump -export METPLUS_TEST_EXE_EGREP=egrep diff --git a/internal/tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py b/internal/tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py index 23d3a27153..0b7dca4e6c 100644 --- a/internal/tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py +++ b/internal/tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py @@ -99,7 +99,7 @@ def set_minimum_config_settings(config): ) @pytest.mark.plotting def test_read_loop_info(metplus_config, config_overrides, expected_loop_args): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -181,7 +181,7 @@ def test_read_loop_info(metplus_config, config_overrides, expected_loop_args): @pytest.mark.plotting def test_tcmpr_plotter_loop(metplus_config, config_overrides, expected_strings): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -278,7 +278,7 @@ def test_tcmpr_plotter(metplus_config, config_overrides, expected_string): expected_string = f' {expected_string}' for single_file in [True, False]: - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) diff --git a/internal/tests/pytests/pytest.ini b/internal/tests/pytests/pytest.ini index e9f3dd09e8..1a9aa7a977 100644 --- a/internal/tests/pytests/pytest.ini +++ b/internal/tests/pytests/pytest.ini @@ -1,5 +1,6 @@ [pytest] markers = + run_metplus: custom marker for testing run_metplus.py script util: custom marker for testing metplus/util logic wrapper_a: custom marker for testing metplus/wrapper logic - A group wrapper_b: custom marker for testing metplus/wrapper logic - B group diff --git a/internal/tests/pytests/run_metplus/test_run_metplus.py b/internal/tests/pytests/run_metplus/test_run_metplus.py new file mode 100644 index 0000000000..6567e2d8e7 --- /dev/null +++ b/internal/tests/pytests/run_metplus/test_run_metplus.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import pytest + +from pathlib import Path +import os +import shutil +from subprocess import run + +# get METplus directory relative to this file +# from this script's directory, go up 4 directories +METPLUS_DIR = str(Path(__file__).parents[4]) +RUN_METPLUS = os.path.join(METPLUS_DIR, 'ush', 'run_metplus.py') +EXAMPLE_CONF = os.path.join(METPLUS_DIR, 'parm', 'use_cases', + 'met_tool_wrapper', 'Example', 'Example.conf') +MINIMUM_CONF = os.path.join(METPLUS_DIR, 'internal', 'tests', 'pytests', + 'minimum_pytest.conf') +TEST_OUTPUT_DIR = os.path.join(os.environ['METPLUS_TEST_OUTPUT_BASE'], + 'test_output') +NEW_OUTPUT_BASE = os.path.join(TEST_OUTPUT_DIR, 'run_metplus') +OUTPUT_BASE_OVERRIDE = f"config.OUTPUT_BASE={NEW_OUTPUT_BASE}" + +@pytest.mark.run_metplus +def test_run_metplus_exists(): + """! Check that run_metplus.py script exists """ + assert os.path.exists(RUN_METPLUS) + + +@pytest.mark.parametrize( + 'command, expected_return_code', [ + ([RUN_METPLUS], 2), + ([RUN_METPLUS, EXAMPLE_CONF], 2), + ([RUN_METPLUS, EXAMPLE_CONF, MINIMUM_CONF, OUTPUT_BASE_OVERRIDE], 0), + ] +) +@pytest.mark.run_metplus +def test_run_metplus_check_return_code(command, expected_return_code): + """! Call run_metplus.py without various arguments and check that the + expected value is returned by the script. A successful run should return + 0 and a failed run should return a non-zero return code, typically 2. + """ + process = run(command) + assert process.returncode == expected_return_code + + if os.path.exists(NEW_OUTPUT_BASE): + shutil.rmtree(NEW_OUTPUT_BASE) + + +@pytest.mark.run_metplus +def test_output_dir_is_created(): + """! Check that the test output directory was created after running tests + """ + assert os.path.exists(TEST_OUTPUT_DIR) diff --git a/internal/tests/pytests/util/config/test_config.py b/internal/tests/pytests/util/config/test_config.py index 7c054ab3d8..0465bc62be 100644 --- a/internal/tests/pytests/util/config/test_config.py +++ b/internal/tests/pytests/util/config/test_config.py @@ -4,7 +4,7 @@ import os from configparser import NoOptionError -from shutil import which +from shutil import which, rmtree from metplus.util import met_util as util @@ -28,7 +28,7 @@ ) @pytest.mark.util def test_getseconds(metplus_config, input_value, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_SECONDS', input_value) @@ -57,7 +57,7 @@ def test_getseconds(metplus_config, input_value, result): ) @pytest.mark.util def test_getstr(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETSTR', input_value) @@ -82,7 +82,7 @@ def test_getstr(metplus_config, input_value, default, result): ) @pytest.mark.util def test_getdir(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETDIR', input_value) @@ -110,7 +110,7 @@ def test_getdir(metplus_config, input_value, default, result): ) @pytest.mark.util def test_getraw(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config conf.set('config', 'TEST_EXTRA', 'extra') conf.set('config', 'TEST_EXTRA2', '{TEST_EXTRA}_extra') @@ -144,7 +144,7 @@ def test_getraw(metplus_config, input_value, default, result): ) @pytest.mark.util def test_getbool(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETBOOL', input_value) @@ -167,7 +167,7 @@ def test_getbool(metplus_config, input_value, default, result): ) @pytest.mark.util def test_getexe(metplus_config, input_value, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETEXE', input_value) @@ -189,7 +189,7 @@ def test_getexe(metplus_config, input_value, result): ] ) def test_getfloat(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETFLOAT', input_value) @@ -220,7 +220,7 @@ def test_getfloat(metplus_config, input_value, default, result): ) @pytest.mark.util def test_getint(metplus_config, input_value, default, result): - conf = metplus_config() + conf = metplus_config if input_value is not None: conf.set('config', 'TEST_GETINT', input_value) @@ -241,15 +241,19 @@ def test_getint(metplus_config, input_value, default, result): ] ) @pytest.mark.util -def test_move_all_to_config_section(metplus_config, config_key, expected_result): +def test_move_all_to_config_section(metplus_config_files, config_key, + expected_result): config_files = ['config_1.conf', 'config_2.conf', 'config_3.conf', ] test_dir = os.path.dirname(__file__) config_files = [os.path.join(test_dir, item) for item in config_files] - config = metplus_config(config_files) + config = metplus_config_files(config_files) assert config.getstr('config', config_key) == expected_result + output_base = config.getdir('OUTPUT_BASE') + if output_base and os.path.exists(output_base): + rmtree(output_base) @pytest.mark.parametrize( @@ -280,11 +284,14 @@ def test_move_all_to_config_section(metplus_config, config_key, expected_result) ] ) @pytest.mark.util -def test_move_all_to_config_section_cmd_line(metplus_config, overrides, +def test_move_all_to_config_section_cmd_line(metplus_config_files, overrides, config_key, expected_result): - config = metplus_config(overrides) + config = metplus_config_files(overrides) assert config.getstr('config', config_key, '') == expected_result + output_base = config.getdir('OUTPUT_BASE') + if output_base and os.path.exists(output_base): + rmtree(output_base) @pytest.mark.parametrize( 'config_name, expected_result', [ @@ -330,13 +337,17 @@ def test_move_all_to_config_section_cmd_line(metplus_config, overrides, ] ) @pytest.mark.util -def test_getraw_nested_curly_braces(metplus_config, +def test_getraw_nested_curly_braces(metplus_config_files, config_name, expected_result): config_files = ['config_1.conf', ] test_dir = os.path.dirname(__file__) config_files = [os.path.join(test_dir, item) for item in config_files] - config = metplus_config(config_files) + config = metplus_config_files(config_files) sec, name = config_name.split('.', 1) assert config.getraw(sec, name) == expected_result + + output_base = config.getdir('OUTPUT_BASE') + if output_base and os.path.exists(output_base): + rmtree(output_base) diff --git a/internal/tests/pytests/util/config_metplus/test_config_metplus.py b/internal/tests/pytests/util/config_metplus/test_config_metplus.py index 8332aba14c..f7161d6c8b 100644 --- a/internal/tests/pytests/util/config_metplus/test_config_metplus.py +++ b/internal/tests/pytests/util/config_metplus/test_config_metplus.py @@ -72,7 +72,7 @@ def test_get_default_config_list(): @pytest.mark.util def test_find_indices_in_config_section(metplus_config, regex, index, id, expected_result): - config = metplus_config() + config = metplus_config config.set('config', 'FCST_VAR1_NAME', 'name1') config.set('config', 'FCST_VAR1_LEVELS', 'level1') config.set('config', 'FCST_VAR2_NAME', 'name2') @@ -118,7 +118,7 @@ def test_find_indices_in_config_section(metplus_config, regex, index, ) @pytest.mark.util def test_get_custom_string_list(metplus_config, conf_items, met_tool, expected_result): - config = metplus_config() + config = metplus_config for conf_key, conf_value in conf_items.items(): config.set('config', conf_key, conf_value) @@ -146,7 +146,7 @@ def test_find_var_indices_fcst(metplus_config, config_var_name, expected_indices, set_met_tool): - config = metplus_config() + config = metplus_config data_types = ['FCST'] config.set('config', config_var_name, "NAME1") met_tool = 'grid_stat' if set_met_tool else None @@ -229,7 +229,7 @@ def test_get_field_search_prefixes(data_type, met_tool, expected_out): ) @pytest.mark.util def test_is_var_item_valid(metplus_config, item_list, extension, is_valid): - conf = metplus_config() + conf = metplus_config assert config_metplus.is_var_item_valid(item_list, '1', extension, conf)[0] == is_valid @@ -272,7 +272,7 @@ def test_is_var_item_valid(metplus_config, item_list, extension, is_valid): ) @pytest.mark.util def test_is_var_item_valid_levels(metplus_config, item_list, configs_to_set, is_valid): - conf = metplus_config() + conf = metplus_config for key, value in configs_to_set.items(): conf.set('config', key, value) @@ -321,7 +321,7 @@ def test_get_field_config_variables(metplus_config, search_prefixes, config_overrides, expected_value): - config = metplus_config() + config = metplus_config index = '1' field_info_types = ['name', 'levels', 'thresh', 'options', 'output_names'] for field_info_type in field_info_types: @@ -388,7 +388,7 @@ def test_get_field_config_variables_synonyms(metplus_config, config_keys, field_key, expected_value): - config = metplus_config() + config = metplus_config index = '1' prefix = 'BOTH_REGRID_DATA_PLANE_' for key in config_keys: @@ -411,7 +411,7 @@ def test_get_field_config_variables_synonyms(metplus_config, ) @pytest.mark.util def test_parse_var_list_fcst_only(metplus_config, data_type, list_created): - conf = metplus_config() + conf = metplus_config conf.set('config', 'FCST_VAR1_NAME', "NAME1") conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12") conf.set('config', 'FCST_VAR2_NAME', "NAME2") @@ -448,7 +448,7 @@ def test_parse_var_list_fcst_only(metplus_config, data_type, list_created): ) @pytest.mark.util def test_parse_var_list_obs(metplus_config, data_type, list_created): - conf = metplus_config() + conf = metplus_config conf.set('config', 'OBS_VAR1_NAME', "NAME1") conf.set('config', 'OBS_VAR1_LEVELS', "LEVELS11, LEVELS12") conf.set('config', 'OBS_VAR2_NAME', "NAME2") @@ -485,7 +485,7 @@ def test_parse_var_list_obs(metplus_config, data_type, list_created): ) @pytest.mark.util def test_parse_var_list_both(metplus_config, data_type, list_created): - conf = metplus_config() + conf = metplus_config conf.set('config', 'BOTH_VAR1_NAME', "NAME1") conf.set('config', 'BOTH_VAR1_LEVELS', "LEVELS11, LEVELS12") conf.set('config', 'BOTH_VAR2_NAME', "NAME2") @@ -512,7 +512,7 @@ def test_parse_var_list_both(metplus_config, data_type, list_created): # field info defined in both FCST_* and OBS_* variables @pytest.mark.util def test_parse_var_list_fcst_and_obs(metplus_config): - conf = metplus_config() + conf = metplus_config conf.set('config', 'FCST_VAR1_NAME', "FNAME1") conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12") conf.set('config', 'FCST_VAR2_NAME', "FNAME2") @@ -549,7 +549,7 @@ def test_parse_var_list_fcst_and_obs(metplus_config): # VAR1 defined by FCST, VAR2 defined by OBS @pytest.mark.util def test_parse_var_list_fcst_and_obs_alternate(metplus_config): - conf = metplus_config() + conf = metplus_config conf.set('config', 'FCST_VAR1_NAME', "FNAME1") conf.set('config', 'FCST_VAR1_LEVELS', "FLEVELS11, FLEVELS12") conf.set('config', 'OBS_VAR2_NAME', "ONAME2") @@ -569,7 +569,7 @@ def test_parse_var_list_fcst_and_obs_alternate(metplus_config): ) @pytest.mark.util def test_parse_var_list_fcst_and_obs_and_both(metplus_config, data_type, list_len, name_levels): - conf = metplus_config() + conf = metplus_config conf.set('config', 'OBS_VAR1_NAME', "ONAME1") conf.set('config', 'OBS_VAR1_LEVELS', "OLEVELS11, OLEVELS12") conf.set('config', 'FCST_VAR2_NAME', "FNAME2") @@ -619,7 +619,7 @@ def test_parse_var_list_fcst_and_obs_and_both(metplus_config, data_type, list_le ) @pytest.mark.util def test_parse_var_list_fcst_only_options(metplus_config, data_type, list_len): - conf = metplus_config() + conf = metplus_config conf.set('config', 'FCST_VAR1_NAME', "NAME1") conf.set('config', 'FCST_VAR1_LEVELS', "LEVELS11, LEVELS12") conf.set('config', 'FCST_VAR1_THRESH', ">1, >2") @@ -643,7 +643,7 @@ def test_parse_var_list_fcst_only_options(metplus_config, data_type, list_len): ) @pytest.mark.util def test_find_var_indices_wrapper_specific(metplus_config, met_tool, indices): - conf = metplus_config() + conf = metplus_config data_type = 'FCST' conf.set('config', f'{data_type}_VAR1_NAME', "NAME1") conf.set('config', f'{data_type}_GRID_STAT_VAR2_NAME', "GSNAME2") @@ -659,7 +659,7 @@ def test_find_var_indices_wrapper_specific(metplus_config, met_tool, indices): # works as expected @pytest.mark.util def test_parse_var_list_ensemble(metplus_config): - config = metplus_config() + config = metplus_config config.set('config', 'ENS_VAR1_NAME', 'APCP') config.set('config', 'ENS_VAR1_LEVELS', 'A24') config.set('config', 'ENS_VAR1_THRESH', '>0.0, >=10.0') @@ -750,7 +750,7 @@ def test_parse_var_list_ensemble(metplus_config): @pytest.mark.util def test_parse_var_list_series_by(metplus_config): - config = metplus_config() + config = metplus_config config.set('config', 'BOTH_EXTRACT_TILES_VAR1_NAME', 'RH') config.set('config', 'BOTH_EXTRACT_TILES_VAR1_LEVELS', 'P850, P700') config.set('config', 'BOTH_EXTRACT_TILES_VAR1_OUTPUT_NAMES', @@ -817,8 +817,13 @@ def test_parse_var_list_series_by(metplus_config): assert actual_sa.get(key) == value +@pytest.mark.parametrize( + 'start_index', [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + ] +) @pytest.mark.util -def test_parse_var_list_priority_fcst(metplus_config): +def test_parse_var_list_priority_fcst(metplus_config, start_index): priority_list = ['FCST_GRID_STAT_VAR1_NAME', 'FCST_GRID_STAT_VAR1_INPUT_FIELD_NAME', 'FCST_GRID_STAT_VAR1_FIELD_NAME', @@ -833,22 +838,15 @@ def test_parse_var_list_priority_fcst(metplus_config): 'BOTH_VAR1_FIELD_NAME', ] time_info = {} + config = metplus_config + for key in priority_list[start_index:]: + config.set('config', key, key.lower()) - # loop through priority list, process, then pop first value off and - # process again until all items have been popped. - # This will check that list is in priority order - while(priority_list): - config = metplus_config() - for key in priority_list: - config.set('config', key, key.lower()) - - var_list = config_metplus.parse_var_list(config, time_info=time_info, - data_type='FCST', - met_tool='grid_stat') - - assert len(var_list) == 1 - assert var_list[0].get('fcst_name') == priority_list[0].lower() - priority_list.pop(0) + var_list = config_metplus.parse_var_list(config, time_info=time_info, + data_type='FCST', + met_tool='grid_stat') + assert len(var_list) == 1 + assert var_list[0].get('fcst_name') == priority_list[start_index].lower() # test that if wrapper specific field info is specified, it only gets @@ -856,7 +854,7 @@ def test_parse_var_list_priority_fcst(metplus_config): # wrapper specific field info variables are specified @pytest.mark.util def test_parse_var_list_wrapper_specific(metplus_config): - conf = metplus_config() + conf = metplus_config conf.set('config', 'FCST_VAR1_NAME', "ENAME1") conf.set('config', 'FCST_VAR1_LEVELS', "ELEVELS11, ELEVELS12") conf.set('config', 'FCST_VAR2_NAME', "ENAME2") @@ -942,7 +940,7 @@ def test_parse_var_list_wrapper_specific(metplus_config): @pytest.mark.util def test_parse_var_list_py_embed_multi_levels(metplus_config, config_overrides, expected_results): - config = metplus_config() + config = metplus_config for key, value in config_overrides.items(): config.set('config', key, value) @@ -999,7 +997,7 @@ def test_parse_var_list_py_embed_multi_levels(metplus_config, config_overrides, ) @pytest.mark.util def test_get_process_list(metplus_config, input_list, expected_list): - conf = metplus_config() + conf = metplus_config conf.set('config', 'PROCESS_LIST', input_list) process_list = config_metplus.get_process_list(conf) output_list = [item[0] for item in process_list] @@ -1033,7 +1031,7 @@ def test_get_process_list(metplus_config, input_list, expected_list): ) @pytest.mark.util def test_get_process_list_instances(metplus_config, input_list, expected_list): - conf = metplus_config() + conf = metplus_config conf.set('config', 'PROCESS_LIST', input_list) output_list = config_metplus.get_process_list(conf) assert output_list == expected_list @@ -1044,7 +1042,7 @@ def test_getraw_sub_and_nosub(metplus_config): raw_string = '{MODEL}_{CURRENT_FCST_NAME}' sub_actual = 'FCST_NAME' - config = metplus_config() + config = metplus_config config.set('config', 'MODEL', 'FCST') config.set('config', 'CURRENT_FCST_NAME', 'NAME') config.set('config', 'OUTPUT_PREFIX', raw_string) @@ -1062,7 +1060,7 @@ def test_getraw_instance_with_unset_var(metplus_config): """ pytest.skip() instance = 'my_section' - config = metplus_config() + config = metplus_config config.set('config', 'MODEL', 'FCST') config.add_section(instance) diff --git a/internal/tests/pytests/util/logging/test_logging.py b/internal/tests/pytests/util/logging/test_logging.py index 68eca3262d..085b33aac9 100644 --- a/internal/tests/pytests/util/logging/test_logging.py +++ b/internal/tests/pytests/util/logging/test_logging.py @@ -10,7 +10,7 @@ @pytest.mark.util def test_log_level(metplus_config): # Verify that the log level is set to what we indicated in the config file. - config = metplus_config() + config = metplus_config fixture_logger = config.logger # Expecting log level = INFO as set in the test config file. level = logging.getLevelName('INFO') @@ -20,7 +20,7 @@ def test_log_level(metplus_config): @pytest.mark.util def test_log_level_key(metplus_config): # Verify that the LOG_LEVEL key is in the config file - config_instance = metplus_config() + config_instance = metplus_config section = 'config' option = 'LOG_LEVEL' assert config_instance.has_option(section, option) @@ -29,7 +29,7 @@ def test_log_level_key(metplus_config): @pytest.mark.util def test_logdir_exists(metplus_config): # Verify that the expected log dir exists. - config = metplus_config() + config = metplus_config log_dir = config.get('config', 'LOG_DIR') # Verify that a logfile exists in the log dir, with a filename # like {LOG_DIR}/metplus.YYYYMMDD.log @@ -40,7 +40,7 @@ def test_logdir_exists(metplus_config): def test_logfile_exists(metplus_config): # Verify that a logfile with format metplus.log exists # We are assuming that there can be numerous files in the log directory. - config = metplus_config() + config = metplus_config log_dir = config.get('config', 'LOG_DIR') # Only check for the log file if the log directory is present if os.path.exists(log_dir): diff --git a/internal/tests/pytests/util/met_config/test_met_config.py b/internal/tests/pytests/util/met_config/test_met_config.py index 0f3adb6587..e00e5f2e9b 100644 --- a/internal/tests/pytests/util/met_config/test_met_config.py +++ b/internal/tests/pytests/util/met_config/test_met_config.py @@ -36,7 +36,7 @@ def test_read_climo_field(metplus_config, config_overrides, expected_value): app_name = 'app' for climo_type in ('MEAN', 'STDEV'): expected_var = f'{app_name}_CLIMO_{climo_type}_FIELD'.upper() - config = metplus_config() + config = metplus_config # set config values for key, value in config_overrides.items(): @@ -135,7 +135,7 @@ def test_handle_climo_dict(metplus_config, config_overrides, expected_value): app_name = 'app' for climo_type in ('MEAN', 'STDEV'): expected_var = f'METPLUS_CLIMO_{climo_type}_DICT' - config = metplus_config() + config = metplus_config output_dict = {} # set config values @@ -252,7 +252,7 @@ def test_read_climo_file_name(metplus_config, config_overrides, for climo_type in CLIMO_TYPES: prefix = f'{app_name.upper()}_CLIMO_{climo_type.upper()}_' - config = metplus_config() + config = metplus_config # set config values for key, value in config_overrides.items(): diff --git a/internal/tests/pytests/util/met_util/test_met_util.py b/internal/tests/pytests/util/met_util/test_met_util.py index 481d4f9d46..784d4b0345 100644 --- a/internal/tests/pytests/util/met_util/test_met_util.py +++ b/internal/tests/pytests/util/met_util/test_met_util.py @@ -98,7 +98,7 @@ def test_get_threshold_via_regex(key, value): ) @pytest.mark.util def test_preprocess_file_stage(metplus_config, filename, ext): - conf = metplus_config() + conf = metplus_config metplus_base = conf.getdir('METPLUS_BASE') stage_dir = conf.getdir('STAGING_DIR', os.path.join(conf.getdir('OUTPUT_BASE'), @@ -140,7 +140,7 @@ def test_preprocess_file_options(metplus_config, data_type, allow_dir, expected): - config = metplus_config() + config = metplus_config if filename == 'dir': filename = config.getdir('METPLUS_BASE') expected = filename @@ -150,7 +150,7 @@ def test_preprocess_file_options(metplus_config, def test_get_lead_sequence_lead(metplus_config): input_dict = {'valid': datetime.datetime(2019, 2, 1, 13)} - conf = metplus_config() + conf = metplus_config conf.set('config', 'LEAD_SEQ', "3,6,9,12") test_seq = util.get_lead_sequence(conf, input_dict) hour_seq = [] @@ -177,7 +177,7 @@ def test_get_lead_sequence_lead(metplus_config): @pytest.mark.util def test_get_lead_sequence_lead_list(metplus_config, key, value): input_dict = { 'valid' : datetime.datetime(2019, 2, 1, 13) } - conf = metplus_config() + conf = metplus_config conf.set('config', 'LEAD_SEQ', key) test_seq = util.get_lead_sequence(conf, input_dict) hour_seq = [] @@ -222,7 +222,7 @@ def test_get_lead_sequence_lead_list(metplus_config, key, value): ) @pytest.mark.util def test_get_lead_sequence_groups(metplus_config, config_dict, expected_list): - config = metplus_config() + config = metplus_config for key, value in config_dict.items(): config.set('config', key, value) @@ -268,7 +268,7 @@ def test_get_lead_sequence_groups(metplus_config, config_dict, expected_list): @pytest.mark.util def test_get_lead_sequence_init(metplus_config, current_hour, lead_seq): input_dict = {'valid': datetime.datetime(2019, 2, 1, current_hour)} - conf = metplus_config() + conf = metplus_config conf.set('config', 'INIT_SEQ', "0, 12") conf.set('config', 'LEAD_SEQ_MAX', 36) test_seq = util.get_lead_sequence(conf, input_dict) @@ -278,7 +278,7 @@ def test_get_lead_sequence_init(metplus_config, current_hour, lead_seq): @pytest.mark.util def test_get_lead_sequence_init_min_10(metplus_config): input_dict = {'valid': datetime.datetime(2019, 2, 1, 12)} - conf = metplus_config() + conf = metplus_config conf.set('config', 'INIT_SEQ', "0, 12") conf.set('config', 'LEAD_SEQ_MAX', 24) conf.set('config', 'LEAD_SEQ_MIN', 10) @@ -343,7 +343,7 @@ def test_round_0p5(value, expected_result): ) @pytest.mark.util def test_get_skip_times(metplus_config, skip_times_conf, expected_dict): - conf = metplus_config() + conf = metplus_config conf.set('config', 'SKIP_TIMES', skip_times_conf) assert util.get_skip_times(conf) == expected_dict @@ -362,7 +362,7 @@ def test_get_skip_times(metplus_config, skip_times_conf, expected_dict): ) @pytest.mark.util def test_get_skip_times_wrapper(metplus_config, skip_times_conf, expected_dict): - conf = metplus_config() + conf = metplus_config # set wrapper specific skip times, then ensure it is found conf.set('config', 'GRID_STAT_SKIP_TIMES', skip_times_conf) @@ -383,7 +383,7 @@ def test_get_skip_times_wrapper(metplus_config, skip_times_conf, expected_dict): ) @pytest.mark.util def test_get_skip_times_wrapper_not_used(metplus_config, skip_times_conf, expected_dict): - conf = metplus_config() + conf = metplus_config # set generic SKIP_TIMES, then request grid_stat to ensure it uses generic conf.set('config', 'SKIP_TIMES', skip_times_conf) @@ -485,7 +485,7 @@ def test_subset_list(subset_definition, expected_result): ) @pytest.mark.util def test_get_storm_ids(metplus_config, filename, expected_result): - config = metplus_config() + config = metplus_config filepath = os.path.join(config.getdir('METPLUS_BASE'), 'internal', 'tests', 'data', @@ -514,7 +514,7 @@ def test_get_storm_ids(metplus_config, filename, expected_result): @pytest.mark.util def test_get_storms(metplus_config, filename, expected_result): storm_id_index = 4 - config = metplus_config() + config = metplus_config filepath = os.path.join(config.getdir('METPLUS_BASE'), 'internal', 'tests', 'data', @@ -543,7 +543,7 @@ def test_get_storms_mtd(metplus_config): 'CO001' ] sort_column = 'OBJECT_CAT' - config = metplus_config() + config = metplus_config filepath = os.path.join(config.getdir('METPLUS_BASE'), 'internal', 'tests', 'data', @@ -644,7 +644,7 @@ def test_format_level(level, expected_result): ) @pytest.mark.util def test_sub_var_list(metplus_config, input_dict, expected_list): - config = metplus_config() + config = metplus_config config.set('config', 'FCST_VAR1_NAME', 'FNAME_{init?fmt=%Y}') config.set('config', 'FCST_VAR1_LEVELS', 'Z{init?fmt=%H}, Z{valid?fmt=%H}') config.set('config', 'OBS_VAR1_NAME', 'ONAME_{init?fmt=%Y}') diff --git a/internal/tests/pytests/util/time_looping/test_time_looping.py b/internal/tests/pytests/util/time_looping/test_time_looping.py index 13ae967c5e..240a1cccd8 100644 --- a/internal/tests/pytests/util/time_looping/test_time_looping.py +++ b/internal/tests/pytests/util/time_looping/test_time_looping.py @@ -11,7 +11,7 @@ def test_get_start_and_end_times(metplus_config): end_time = '2018103109' time_format = '%Y%m%d%H' for prefix in ['INIT', 'VALID']: - config = metplus_config() + config = metplus_config config.set('config', 'LOOP_BY', prefix) config.set('config', f'{prefix}_TIME_FMT', time_format) config.set('config', f'{prefix}_BEG', start_time) @@ -25,7 +25,7 @@ def test_get_start_and_end_times(metplus_config): def test_get_start_and_end_times_now(metplus_config): time_format = '%Y%m%d%H%M%S' for prefix in ['INIT', 'VALID']: - config = metplus_config() + config = metplus_config config.set('config', 'LOOP_BY', prefix) config.set('config', f'{prefix}_TIME_FMT', time_format) config.set('config', f'{prefix}_BEG', '{now?fmt=%Y%m%d%H%M%S?shift=-1d}') @@ -43,7 +43,7 @@ def test_get_start_and_end_times_now(metplus_config): def test_get_start_and_end_times_today(metplus_config): time_format = '%Y%m%d' for prefix in ['INIT', 'VALID']: - config = metplus_config() + config = metplus_config config.set('config', 'LOOP_BY', prefix) config.set('config', f'{prefix}_TIME_FMT', time_format) config.set('config', f'{prefix}_BEG', '{today}') @@ -60,7 +60,7 @@ def test_get_start_and_end_times_today(metplus_config): @pytest.mark.util def test_time_generator_list(metplus_config): for prefix in ['INIT', 'VALID']: - config = metplus_config() + config = metplus_config config.set('config', 'LOOP_BY', prefix) config.set('config', f'{prefix}_TIME_FMT', '%Y%m%d%H') config.set('config', f'{prefix}_LIST', '2021020104, 2021103121') @@ -83,7 +83,7 @@ def test_time_generator_list(metplus_config): @pytest.mark.util def test_time_generator_increment(metplus_config): for prefix in ['INIT', 'VALID']: - config = metplus_config() + config = metplus_config config.set('config', 'LOOP_BY', prefix) config.set('config', f'{prefix}_TIME_FMT', '%Y%m%d%H') config.set('config', f'{prefix}_BEG', '2021020104') @@ -107,8 +107,13 @@ def test_time_generator_increment(metplus_config): assert True +@pytest.mark.parametrize( + 'prefix', [ + 'INIT', 'VALID', + ] +) @pytest.mark.util -def test_time_generator_error_check(metplus_config): +def test_time_generator_error_check_list(metplus_config, prefix): """! Test that None is returned by the time generator when the time looping config variables are not set properly. Tests: Missing LOOP_BY, @@ -120,65 +125,82 @@ def test_time_generator_error_check(metplus_config): _BEG is after _END, """ time_fmt = '%Y%m%d%H' - for prefix in ['INIT', 'VALID']: - config = metplus_config() + config = metplus_config - # unset LOOP_BY - assert next(time_generator(config)) is None - config.set('config', 'LOOP_BY', prefix) + # unset LOOP_BY + assert next(time_generator(config)) is None + config.set('config', 'LOOP_BY', prefix) - # unset _TIME_FMT - assert next(time_generator(config)) is None - config.set('config', f'{prefix}_TIME_FMT', time_fmt) + # unset _TIME_FMT + assert next(time_generator(config)) is None + config.set('config', f'{prefix}_TIME_FMT', time_fmt) - # test [INIT/VALID]_LIST configurations + # test [INIT/VALID]_LIST configurations - # empty _LIST - config.set('config', f'{prefix}_LIST', '') - assert next(time_generator(config)) is None + # empty _LIST + config.set('config', f'{prefix}_LIST', '') + assert next(time_generator(config)) is None - # list value doesn't match format - config.set('config', f'{prefix}_LIST', '202102010412') - assert next(time_generator(config)) is None + # list value doesn't match format + config.set('config', f'{prefix}_LIST', '202102010412') + assert next(time_generator(config)) is None - # 2nd list value doesn't match format - config.set('config', f'{prefix}_LIST', '2021020104, 202102010412') - expected_time = datetime.strptime('2021020104', time_fmt) - generator = time_generator(config) - assert next(generator)[prefix.lower()] == expected_time - assert next(generator) is None + # 2nd list value doesn't match format + config.set('config', f'{prefix}_LIST', '2021020104, 202102010412') + expected_time = datetime.strptime('2021020104', time_fmt) + generator = time_generator(config) + assert next(generator)[prefix.lower()] == expected_time + assert next(generator) is None - # good _LIST - config.set('config', f'{prefix}_LIST', '2021020104') - assert next(time_generator(config))[prefix.lower()] == expected_time + # good _LIST + config.set('config', f'{prefix}_LIST', '2021020104') + assert next(time_generator(config))[prefix.lower()] == expected_time - # get a fresh config object to test BEG/END configurations - config = metplus_config() - config.set('config', 'LOOP_BY', prefix) - config.set('config', f'{prefix}_TIME_FMT', time_fmt) - # _BEG doesn't match time format (too long) - config.set('config', f'{prefix}_BEG', '202110311259') - config.set('config', f'{prefix}_END', '2021112012') +@pytest.mark.parametrize( + 'prefix', [ + 'INIT', 'VALID', + ] +) +@pytest.mark.util +def test_time_generator_error_check_beg_end(metplus_config, prefix): + """! Test that None is returned by the time generator when + the time looping config variables are not set properly. Tests: + Missing LOOP_BY, + Missing [INIT/VALID]_TIME_FMT, + Empty [INIT/VALID]_LIST (if set), + List value doesn't match time format, + _BEG or _END value doesn't match format, + _INCREMENT is less than 60 seconds, + _BEG is after _END, + """ + time_fmt = '%Y%m%d%H' + config = metplus_config + config.set('config', 'LOOP_BY', prefix) + config.set('config', f'{prefix}_TIME_FMT', time_fmt) + + # _BEG doesn't match time format (too long) + config.set('config', f'{prefix}_BEG', '202110311259') + config.set('config', f'{prefix}_END', '2021112012') - assert next(time_generator(config)) is None - config.set('config', f'{prefix}_BEG', '2021103112') + assert next(time_generator(config)) is None + config.set('config', f'{prefix}_BEG', '2021103112') - # unset _END uses _BEG value, so it should succeed - assert next(time_generator(config)) is not None + # unset _END uses _BEG value, so it should succeed + assert next(time_generator(config)) is not None - # _END doesn't match time format (too long) - config.set('config', f'{prefix}_END', '202111201259') + # _END doesn't match time format (too long) + config.set('config', f'{prefix}_END', '202111201259') - assert next(time_generator(config)) is None - config.set('config', f'{prefix}_END', '2021112012') - assert next(time_generator(config)) is not None + assert next(time_generator(config)) is None + config.set('config', f'{prefix}_END', '2021112012') + assert next(time_generator(config)) is not None - # _INCREMENT is less than 60 seconds - config.set('config', f'{prefix}_INCREMENT', '10S') - assert next(time_generator(config)) is None - config.set('config', f'{prefix}_INCREMENT', '1d') + # _INCREMENT is less than 60 seconds + config.set('config', f'{prefix}_INCREMENT', '10S') + assert next(time_generator(config)) is None + config.set('config', f'{prefix}_INCREMENT', '1d') - # _END time comes before _BEG time - config.set('config', f'{prefix}_END', '2020112012') - assert next(time_generator(config)) is None + # _END time comes before _BEG time + config.set('config', f'{prefix}_END', '2020112012') + assert next(time_generator(config)) is None diff --git a/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py b/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py index f492f1f349..0af69609ad 100644 --- a/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py @@ -3,20 +3,37 @@ import pytest import os +import shutil from metplus.wrappers.ascii2nc_wrapper import ASCII2NCWrapper -def ascii2nc_wrapper(metplus_config, config_path=None, config_overrides=None): - config = metplus_config() - - if config_path: - parm_base = config.getdir('PARM_BASE') - config_full_path = os.path.join(parm_base, config_path) - config = metplus_config([config_full_path]) - - overrides = {'DO_NOT_RUN_EXE': True, - 'INPUT_MUST_EXIST': False} +def ascii2nc_wrapper(metplus_config, config_overrides=None): + config = metplus_config + overrides = { + 'DO_NOT_RUN_EXE': True, + 'INPUT_MUST_EXIST': False, + 'PROCESS_LIST': 'ASCII2NC', + 'LOOP_BY': 'VALID', + 'VALID_TIME_FMT': '%Y%m%d%H', + 'VALID_BEG': '2010010112', + 'VALID_END': '2010010112', + 'VALID_INCREMENT': '1M', + 'ASCII2NC_INPUT_TEMPLATE': '{INPUT_BASE}/met_test/data/sample_obs/ascii/precip24_{valid?fmt=%Y%m%d%H}.ascii', + 'ASCII2NC_OUTPUT_TEMPLATE': '{OUTPUT_BASE}/ascii2nc/precip24_{valid?fmt=%Y%m%d%H}.nc', + 'ASCII2NC_CONFIG_FILE': '{PARM_BASE}/met_config/Ascii2NcConfig_wrapped', + 'ASCII2NC_TIME_SUMMARY_FLAG': 'False', + 'ASCII2NC_TIME_SUMMARY_RAW_DATA': 'False', + 'ASCII2NC_TIME_SUMMARY_BEG': '000000', + 'ASCII2NC_TIME_SUMMARY_END': '235959', + 'ASCII2NC_TIME_SUMMARY_STEP': '300', + 'ASCII2NC_TIME_SUMMARY_WIDTH': '600', + 'ASCII2NC_TIME_SUMMARY_GRIB_CODES': '11, 204, 211', + 'ASCII2NC_TIME_SUMMARY_VAR_NAMES': '', + 'ASCII2NC_TIME_SUMMARY_TYPES': 'min, max, range, mean, stdev, median, p80', + 'ASCII2NC_TIME_SUMMARY_VALID_FREQ': '0', + 'ASCII2NC_TIME_SUMMARY_VALID_THRESH': '0.0', + } if config_overrides: for key, value in config_overrides.items(): overrides[key] = value @@ -27,8 +44,7 @@ def ascii2nc_wrapper(metplus_config, config_path=None, config_overrides=None): for key, value in overrides.items(): config.set(instance, key, value) - return ASCII2NCWrapper(config, - instance=instance) + return ASCII2NCWrapper(config, instance=instance) @pytest.mark.parametrize( @@ -142,11 +158,7 @@ def ascii2nc_wrapper(metplus_config, config_path=None, config_overrides=None): @pytest.mark.wrapper def test_ascii2nc_wrapper(metplus_config, config_overrides, env_var_values): - wrapper = ( - ascii2nc_wrapper(metplus_config, - 'use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf', - config_overrides) - ) + wrapper = ascii2nc_wrapper(metplus_config, config_overrides) assert wrapper.isOK input_path = wrapper.config.getraw('config', 'ASCII2NC_INPUT_TEMPLATE') @@ -171,7 +183,7 @@ def test_ascii2nc_wrapper(metplus_config, config_overrides, f"-config {config_file} " f"{verbosity}") - assert(all_commands[0][0] == expected_cmd) + assert all_commands[0][0] == expected_cmd env_vars = all_commands[0][1] # check that environment variables were set properly @@ -182,16 +194,20 @@ def test_ascii2nc_wrapper(metplus_config, config_overrides, for env_var_key in env_var_keys: match = next((item for item in env_vars if item.startswith(env_var_key)), None) - assert (match is not None) + assert match is not None value = match.split('=', 1)[1] - assert (env_var_values.get(env_var_key, '') == value) + assert env_var_values.get(env_var_key, '') == value + + output_base = wrapper.config.getdir('OUTPUT_BASE') + if output_base: + shutil.rmtree(output_base) @pytest.mark.wrapper def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config config.set('config', 'INPUT_MUST_EXIST', False) wrapper = ASCII2NCWrapper(config) diff --git a/internal/tests/pytests/wrappers/command_builder/test_command_builder.py b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py index fa95f7e037..c62609b983 100644 --- a/internal/tests/pytests/wrappers/command_builder/test_command_builder.py +++ b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py @@ -24,7 +24,7 @@ def get_data_dir(config): ) @pytest.mark.wrapper def test_find_data_no_dated(metplus_config, data_type): - config = metplus_config() + config = metplus_config pcw = CommandBuilder(config) v = {} @@ -55,7 +55,7 @@ def test_find_data_no_dated(metplus_config, data_type): ) @pytest.mark.wrapper def test_find_data_not_a_path(metplus_config, data_type): - config = metplus_config() + config = metplus_config pcw = CommandBuilder(config) task_info = {} @@ -73,7 +73,7 @@ def test_find_data_not_a_path(metplus_config, data_type): @pytest.mark.wrapper def test_find_obs_no_dated(metplus_config): - config = metplus_config() + config = metplus_config pcw = CommandBuilder(config) v = {} @@ -93,7 +93,7 @@ def test_find_obs_no_dated(metplus_config): @pytest.mark.wrapper def test_find_obs_dated(metplus_config): - config = metplus_config() + config = metplus_config pcw = CommandBuilder(config) v = {} @@ -123,7 +123,7 @@ def test_find_obs_dated(metplus_config): ) @pytest.mark.wrapper def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds): - config = metplus_config() + config = metplus_config pcw = CommandBuilder(config) v = {} @@ -150,7 +150,7 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds) @pytest.mark.wrapper def test_find_obs_dated_previous_day(metplus_config): - config = metplus_config() + config = metplus_config pcw = CommandBuilder(config) v = {} @@ -170,7 +170,7 @@ def test_find_obs_dated_previous_day(metplus_config): @pytest.mark.wrapper def test_find_obs_dated_next_day(metplus_config): - config = metplus_config() + config = metplus_config pcw = CommandBuilder(config) v = {} @@ -209,7 +209,7 @@ def test_find_obs_dated_next_day(metplus_config): ) @pytest.mark.wrapper def test_override_by_instance(metplus_config, section_items): - config = metplus_config() + config = metplus_config # set config variables to default for key in section_items: @@ -240,7 +240,7 @@ def test_override_by_instance(metplus_config, section_items): ) @pytest.mark.wrapper def test_write_list_file(metplus_config, filename, file_list, output_dir): - config = metplus_config() + config = metplus_config cbw = CommandBuilder(config) # use output_dir relative to OUTPUT_BASE if it is specified @@ -290,7 +290,7 @@ def test_write_list_file(metplus_config, filename, file_list, output_dir): ) @pytest.mark.wrapper def test_handle_description(metplus_config, config_overrides, expected_value): - config = metplus_config() + config = metplus_config # set config values for key, value in config_overrides.items(): @@ -332,7 +332,7 @@ def test_handle_description(metplus_config, config_overrides, expected_value): @pytest.mark.wrapper def test_handle_regrid_old(metplus_config, config_overrides, set_to_grid, expected_dict): - config = metplus_config() + config = metplus_config # set config values for key, value in config_overrides.items(): @@ -381,7 +381,7 @@ def test_handle_regrid_old(metplus_config, config_overrides, set_to_grid, ) @pytest.mark.wrapper def test_handle_regrid_new(metplus_config, config_overrides, expected_output): - config = metplus_config() + config = metplus_config # set config values for key, value in config_overrides.items(): @@ -418,7 +418,7 @@ def test_handle_regrid_new(metplus_config, config_overrides, expected_output): @pytest.mark.wrapper def test_add_met_config_string(metplus_config, mp_config_name, met_config_name, c_dict_key, remove_quotes, expected_output): - cbw = CommandBuilder(metplus_config()) + cbw = CommandBuilder(metplus_config) # set some config variables to test cbw.config.set('config', 'TEST_STRING_1', 'value_1') @@ -472,7 +472,7 @@ def test_add_met_config_string(metplus_config, mp_config_name, met_config_name, @pytest.mark.wrapper def test_add_met_config_bool(metplus_config, mp_config_name, met_config_name, c_dict_key, uppercase, expected_output, is_ok): - cbw = CommandBuilder(metplus_config()) + cbw = CommandBuilder(metplus_config) # set some config variables to test cbw.config.set('config', 'TEST_BOOL_1', True) @@ -520,7 +520,7 @@ def test_add_met_config_bool(metplus_config, mp_config_name, met_config_name, @pytest.mark.wrapper def test_add_met_config_int(metplus_config, mp_config_name, met_config_name, c_dict_key, expected_output, is_ok): - cbw = CommandBuilder(metplus_config()) + cbw = CommandBuilder(metplus_config) # set some config variables to test cbw.config.set('config', 'TEST_INT_1', 7) @@ -563,7 +563,7 @@ def test_add_met_config_int(metplus_config, mp_config_name, met_config_name, @pytest.mark.wrapper def test_add_met_config_float(metplus_config, mp_config_name, met_config_name, c_dict_key, expected_output, is_ok): - cbw = CommandBuilder(metplus_config()) + cbw = CommandBuilder(metplus_config) # set some config variables to test cbw.config.set('config', 'TEST_FLOAT_1', 7.0) @@ -612,7 +612,7 @@ def test_add_met_config_float(metplus_config, mp_config_name, met_config_name, @pytest.mark.wrapper def test_add_met_config_thresh(metplus_config, mp_config_name, met_config_name, c_dict_key, expected_output, is_ok): - cbw = CommandBuilder(metplus_config()) + cbw = CommandBuilder(metplus_config) # set some config variables to test cbw.config.set('config', 'TEST_THRESH_1', 'gt74') @@ -664,7 +664,7 @@ def test_add_met_config_thresh(metplus_config, mp_config_name, met_config_name, @pytest.mark.wrapper def test_add_met_config_list(metplus_config, mp_config_name, met_config_name, c_dict_key, remove_quotes, expected_output): - cbw = CommandBuilder(metplus_config()) + cbw = CommandBuilder(metplus_config) # set some config variables to test cbw.config.set('config', 'TEST_LIST_1', 'value_1, value2') @@ -705,7 +705,7 @@ def test_add_met_config_list(metplus_config, mp_config_name, met_config_name, @pytest.mark.wrapper def test_add_met_config_list_allow_empty(metplus_config, mp_config_name, allow_empty, expected_output): - cbw = CommandBuilder(metplus_config()) + cbw = CommandBuilder(metplus_config) # set some config variables to test cbw.config.set('config', 'TEST_LIST_1', '') @@ -731,7 +731,7 @@ def test_add_met_config_dict(metplus_config): end = 5 expected_value = f'{dict_name} = {{beg = -3;end = 5;}}' - config = metplus_config() + config = metplus_config config.set('config', 'TC_GEN_FCST_HR_WINDOW_BEG', beg) config.set('config', 'TC_GEN_FCST_HR_WINDOW_END', end) cbw = CommandBuilder(config) @@ -755,7 +755,7 @@ def test_add_met_config_window(metplus_config): end = 5 expected_value = f'{dict_name} = {{beg = -3;end = 5;}}' - config = metplus_config() + config = metplus_config config.set('config', 'TC_GEN_FCST_HR_WINDOW_BEG', beg) config.set('config', 'TC_GEN_FCST_HR_WINDOW_END', end) cbw = CommandBuilder(config) @@ -769,7 +769,7 @@ def test_add_met_config_window(metplus_config): @pytest.mark.wrapper def test_add_met_config(metplus_config): - config = metplus_config() + config = metplus_config value = 5 config.set('config', 'TC_GEN_VALID_FREQUENCY', value) cbw = CommandBuilder(config) @@ -795,7 +795,7 @@ def test_add_met_config_dict_nested(metplus_config): f'{{var1 = {sub_dict_value1};var2 = {sub_dict_value2};}}}}' ) - config = metplus_config() + config = metplus_config config.set('config', 'APP_OUTER_BEG', beg) config.set('config', 'APP_OUTER_END', end) config.set('config', 'APP_OUTER_INNER_VAR1', sub_dict_value1) @@ -832,7 +832,7 @@ def test_get_field_info_extra(metplus_config, extra, expected_value): d_type = 'FCST' name = 'name' level = '"(*,*)"' - config = metplus_config() + config = metplus_config wrapper = CommandBuilder(config) actual_value = wrapper.get_field_info( d_type=d_type, diff --git a/internal/tests/pytests/wrappers/compare_gridded/test_compare_gridded.py b/internal/tests/pytests/wrappers/compare_gridded/test_compare_gridded.py index 82b4fa2059..0eaf8d5ab9 100644 --- a/internal/tests/pytests/wrappers/compare_gridded/test_compare_gridded.py +++ b/internal/tests/pytests/wrappers/compare_gridded/test_compare_gridded.py @@ -13,7 +13,7 @@ def compare_gridded_wrapper(metplus_config): files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" - config = metplus_config() + config = metplus_config return CompareGriddedWrapper(config) diff --git a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py index 7a745cb92a..9328c3112a 100644 --- a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py @@ -80,7 +80,7 @@ def set_minimum_config_settings(config, set_fields=True): def test_ensemble_stat_field_info(metplus_config, config_overrides, env_var_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config, set_fields=False) @@ -136,7 +136,7 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, """ old_env_vars = ['CLIMO_MEAN_FILE', 'CLIMO_STDEV_FILE'] - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -564,7 +564,7 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, def test_ensemble_stat_single_field(metplus_config, config_overrides, env_var_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -619,7 +619,7 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides, def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'EnsembleStatConfig_wrapped') @@ -641,7 +641,7 @@ def test_get_config_file(metplus_config): @pytest.mark.wrapper_c def test_ensemble_stat_fill_missing(metplus_config, config_overrides, expected_num_files): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) diff --git a/internal/tests/pytests/wrappers/extract_tiles/extract_tiles_test.conf b/internal/tests/pytests/wrappers/extract_tiles/extract_tiles_test.conf deleted file mode 100644 index 4f34897b33..0000000000 --- a/internal/tests/pytests/wrappers/extract_tiles/extract_tiles_test.conf +++ /dev/null @@ -1,74 +0,0 @@ -# -# CONFIGURATION -# -[config] - -# Loop over each process in the process list (set in PROCESS_LIST) for all times in the time window of -# interest. -LOOP_ORDER = processes - -PROCESS_LIST = ExtractTiles - -# The init time begin and end times, increment -LOOP_BY = INIT -INIT_TIME_FMT = %Y%m%d -INIT_BEG = 20141214 -INIT_END = 20141214 - -# This is the step-size. Increment in seconds from the begin time to the end -# time -INIT_INCREMENT = 21600 ;; set to every 6 hours=21600 seconds - -# A list of times to include, in format YYYYMMDD_hh -INIT_INCLUDE = - -# A list of times to exclude, in format YYYYMMDD_hh -INIT_EXCLUDE = - -# Constants used in creating the tile grid, used by extract tiles -EXTRACT_TILES_NLAT = 60 -EXTRACT_TILES_NLON = 60 - -# Resolution of data in degrees, used by extract tiles -EXTRACT_TILES_DLAT = 0.5 -EXTRACT_TILES_DLON = 0.5 - -# Degrees to subtract from the center lat and lon to -# calculate the lower left lat (lat_ll) and lower -# left lon (lon_ll) for a grid that is 2n X 2m, -# where n = EXTRACT_TILES_LAT_ADJ degrees and m = EXTRACT_TILES_LON_ADJ degrees. -# For this case, where n=15 and m=15, this results -# in a 30 deg X 30 deg grid. Used by extract tiles -EXTRACT_TILES_LON_ADJ = 15 -EXTRACT_TILES_LAT_ADJ = 15 - -#EXTRACT_TILES_FILTER_OPTS = -EXTRACT_TILES_FILTER_OPTS = -basin ML -SERIES_ANALYSIS_FILTER_OPTS = -init_beg {INIT_BEG} -init_end {INIT_END} - -# OVERWRITE OPTIONS -# Don't overwrite filter files if they already exist. -# Set to no if you do NOT want to override existing files -# Set to yes if you do want to override existing files -EXTRACT_TILES_OVERWRITE_TRACK = no - -# if = initializes to an empty string '' or list [], indicating all vars are to be considered -EXTRACT_TILES_VAR_LIST = - -# -# FILENAME TEMPLATES -# -[filename_templates] -# Define the format of the filenames -FCST_EXTRACT_TILES_INPUT_TEMPLATE = gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%HHH}.grb2 -OBS_EXTRACT_TILES_INPUT_TEMPLATE = gfs_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H}00_000.grb2 - -[dir] -# Location of your model data of interest -EXTRACT_TILES_GRID_INPUT_DIR = {INPUT_BASE}/cyclone_track_feature/reduced_model_data - -EXTRACT_TILES_PAIRS_INPUT_DIR = {OUTPUT_BASE}/tc_pairs - -# Use this setting to separate the filtered track files from -# the series analysis directory. -EXTRACT_TILES_OUTPUT_DIR = {OUTPUT_BASE}/extract_tiles diff --git a/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py b/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py index 41d54886f0..aa71f0eb56 100644 --- a/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py +++ b/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py @@ -8,17 +8,32 @@ from metplus.wrappers.extract_tiles_wrapper import ExtractTilesWrapper -def get_config(metplus_config): - extra_configs = [] - extra_configs.append(os.path.join(os.path.dirname(__file__), - 'extract_tiles_test.conf')) - return metplus_config(extra_configs) - - def extract_tiles_wrapper(metplus_config): - config = get_config(metplus_config) + config = metplus_config + config.set('config', 'PROCESS_LIST', 'ExtractTiles') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d') + config.set('config', 'INIT_BEG', '20141214') + config.set('config', 'INIT_END', '20141214') + config.set('config', 'INIT_INCREMENT', '21600') + config.set('config', 'EXTRACT_TILES_NLAT', '60') + config.set('config', 'EXTRACT_TILES_NLON', '60') + config.set('config', 'EXTRACT_TILES_DLAT', '0.5') + config.set('config', 'EXTRACT_TILES_DLON', '0.5') + config.set('config', 'EXTRACT_TILES_LAT_ADJ', '15') + config.set('config', 'EXTRACT_TILES_LON_ADJ', '15') + config.set('config', 'EXTRACT_TILES_FILTER_OPTS', '-basin ML') + config.set('config', 'FCST_EXTRACT_TILES_INPUT_TEMPLATE', + 'gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%HHH}.grb2') + config.set('config', 'OBS_EXTRACT_TILES_INPUT_TEMPLATE', + 'gfs_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H}00_000.grb2') + config.set('config', 'EXTRACT_TILES_GRID_INPUT_DIR', + '{INPUT_BASE}/cyclone_track_feature/reduced_model_data') + config.set('config', 'EXTRACT_TILES_PAIRS_INPUT_DIR', + '{OUTPUT_BASE}/tc_pairs') + config.set('config', 'EXTRACT_TILES_OUTPUT_DIR', + '{OUTPUT_BASE}/extract_tiles') - config.set('config', 'LOOP_ORDER', 'processes') wrapper = ExtractTilesWrapper(config) return wrapper diff --git a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py index f8ed2b4639..49990fc528 100644 --- a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -362,7 +362,7 @@ def handle_input_dir(config): def test_gen_ens_prod_single_field(metplus_config, config_overrides, env_var_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -440,7 +440,7 @@ def test_gen_ens_prod_single_field(metplus_config, config_overrides, ) @pytest.mark.wrapper def test_get_config_file(metplus_config, use_default_config_file): - config = metplus_config() + config = metplus_config if use_default_config_file: config_file = os.path.join(config.getdir('PARM_BASE'), @@ -463,7 +463,7 @@ def test_get_config_file(metplus_config, use_default_config_file): @pytest.mark.wrapper def test_gen_ens_prod_fill_missing(metplus_config, config_overrides, expected_num_files): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) handle_input_dir(config) diff --git a/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py b/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py index b584f75499..fa1d660f90 100644 --- a/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py +++ b/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py @@ -16,7 +16,7 @@ def gen_vx_mask_wrapper(metplus_config): files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) return GenVxMaskWrapper(config) diff --git a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py index 61fd599150..72e3a878d7 100644 --- a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py +++ b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py @@ -68,7 +68,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expected_subset): """! Test to ensure that get_all_files only gets the files that are relevant to the runtime settings and not every file in the directory """ - config = metplus_config() + config = metplus_config config.set('config', 'LOOP_BY', 'INIT') config.set('config', 'GRID_DIAG_RUNTIME_FREQ', 'RUN_ONCE') config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H%M%S') @@ -169,7 +169,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expected_subset): ) @pytest.mark.wrapper def test_get_list_file_name(metplus_config, time_info, expected_filename): - wrapper = GridDiagWrapper(metplus_config()) + wrapper = GridDiagWrapper(metplus_config) assert(wrapper.get_list_file_name(time_info, 'input0') == expected_filename) @@ -177,7 +177,7 @@ def test_get_list_file_name(metplus_config, time_info, expected_filename): def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'GridDiagConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py index c10d914872..9a0caed0b7 100644 --- a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py @@ -88,7 +88,7 @@ def set_minimum_config_settings(config): @pytest.mark.wrapper_b def test_grid_stat_is_prob(metplus_config, config_overrides, expected_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -132,7 +132,7 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, """ old_env_vars = ['CLIMO_MEAN_FILE', 'CLIMO_STDEV_FILE'] - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -695,7 +695,7 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, def test_grid_stat_single_field(metplus_config, config_overrides, env_var_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -750,7 +750,7 @@ def test_grid_stat_single_field(metplus_config, config_overrides, def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'GridStatConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py index 1e06c77152..99a98db0fc 100644 --- a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py @@ -186,7 +186,7 @@ def set_minimum_config_settings(config): @pytest.mark.wrapper def test_ioda2nc_wrapper(metplus_config, config_overrides, env_var_values, extra_args): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -238,7 +238,7 @@ def test_ioda2nc_wrapper(metplus_config, config_overrides, @pytest.mark.wrapper def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config config.set('config', 'INPUT_MUST_EXIST', False) wrapper = IODA2NCWrapper(config) diff --git a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py index 6f31714186..1ca480b49d 100644 --- a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py +++ b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py @@ -318,7 +318,7 @@ def set_minimum_config_settings(config): @pytest.mark.wrapper_a def test_mode_single_field(metplus_config, config_overrides, expected_output): - config = metplus_config() + config = metplus_config # set config variables needed to run set_minimum_config_settings(config) @@ -401,7 +401,7 @@ def test_mode_single_field(metplus_config, config_overrides, @pytest.mark.wrapper_a def test_mode_multi_variate(metplus_config, config_overrides, expected_output): - config = metplus_config() + config = metplus_config # set config variables needed to run set_minimum_config_settings(config) @@ -518,7 +518,7 @@ def test_config_synonyms(metplus_config, config_name, env_var_name, elif var_type == 'float': in_value = out_value = 4.0 - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) config.set('config', config_name, in_value) wrapper = MODEWrapper(config) @@ -533,7 +533,7 @@ def test_config_synonyms(metplus_config, config_name, env_var_name, def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'MODEConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py index 135c62031f..420e6d40fc 100644 --- a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py +++ b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py @@ -19,7 +19,7 @@ def mtd_wrapper(metplus_config, lead_seq=None): files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) config.set('config', 'BOTH_VAR1_NAME', 'APCP') config.set('config', 'BOTH_VAR1_LEVELS', 'A06') @@ -195,7 +195,7 @@ def test_mtd_single(metplus_config): def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'MTDConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/pb2nc/conf1 b/internal/tests/pytests/wrappers/pb2nc/conf1 deleted file mode 100644 index a1c694fcdb..0000000000 --- a/internal/tests/pytests/wrappers/pb2nc/conf1 +++ /dev/null @@ -1,72 +0,0 @@ -[config] -## Configuration-related settings such as the process list, begin and end times, etc. -PROCESS_LIST = PB2NC - -## LOOP_ORDER -## Options are: processes, times -## Looping by time- runs all items in the PROCESS_LIST for each -## initialization time and repeats until all times have been evaluated. -## Looping by processes- run each item in the PROCESS_LIST for all -## specified initialization times then repeat for the next item in the -## PROCESS_LIST. -#LOOP_ORDER = processes - -# Logging levels: DEBUG, INFO, WARN, ERROR (most verbose is DEBUG) -#LOG_LEVEL = DEBUG - -## MET Configuration files for pb2nc -PB2NC_CONFIG_FILE = {PARM_BASE}/met_config/PB2NCConfig_wrapped - -PB2NC_SKIP_IF_OUTPUT_EXISTS = True - -#LOOP_BY = VALID -#VALID_TIME_FMT = %Y%m%d -#VALID_BEG = 20170601 -#VALID_END = 20170603 -#VALID_INCREMENT = 86400 - -#LEAD_SEQ = 0 - - -# For both pb2nc and point_stat, the obs_window dictionary: -#OBS_WINDOW_BEGIN = -2700 -#OBS_WINDOW_END = 2700 - -# Either conus_sfc or upper_air -PB2NC_VERTICAL_LOCATION = conus_sfc - -# -# PB2NC -# -# These are appended with PB2NC to differentiate the GRID, POLY, and MESSAGE_TYPE for point_stat. -PB2NC_GRID = -PB2NC_POLY = -PB2NC_STATION_ID = -PB2NC_MESSAGE_TYPE = - -# Leave empty to process all -PB2NC_OBS_BUFR_VAR_LIST = PMO, TOB, TDO, UOB, VOB, PWO, TOCC, D_RH - -#*********** -# ***NOTE*** -#*********** -# SET TIME_SUMMARY_FLAG to False. There is a bug in met-6.1. -## For defining the time periods for summarization -# False for no time summary, True otherwise -PB2NC_TIME_SUMMARY_FLAG = False -PB2NC_TIME_SUMMARY_BEG = 000000 ;; start time of time summary in HHMMSS format -PB2NC_TIME_SUMMARY_END = 235959 ;; end time of time summary in HHMMSS format -PB2NC_TIME_SUMMARY_VAR_NAMES = PMO,TOB,TDO,UOB,VOB,PWO,TOCC -PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 ;; a list of the statistics to summarize - -# Model/fcst and obs name, e.g. GFS, NAM, GDAS, etc. -#MODEL_NAME = gfs -#OBS_NAME = nam - -[dir] -PB2NC_INPUT_DIR = {INPUT_BASE}/grid_to_obs/prepbufr/nam - -[filename_templates] -PB2NC_INPUT_TEMPLATE = t{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H} - -PB2NC_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/nam.{valid?fmt=%Y%m%d%H}.nc \ No newline at end of file diff --git a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py index 9f98f0d80e..8096cc1e4c 100644 --- a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py @@ -15,19 +15,16 @@ def pb2nc_wrapper(metplus_config): """! Returns a default PB2NCWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration files. Subsequent tests can customize the final METplus configuration - to over-ride these /path/to values.""" - - # PB2NCWrapper with configuration values determined by what is set in - # the pb2nc_test.conf file. - extra_configs = [] - extra_configs.append(os.path.join(os.path.dirname(__file__), 'conf1')) - config = metplus_config(extra_configs) + to over-ride these /path/to values. + """ + config = metplus_config + config.set('config', 'PB2NC_INPUT_TEMPLATE', + 't{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}') return PB2NCWrapper(config) @pytest.mark.parametrize( - # key = grid_id, value = expected reformatted grid id - 'exists, skip, run', [ + 'exists, skip, run', [ (True, True, False), (True, False, True), (False, True, True), @@ -67,12 +64,12 @@ def test_find_and_check_output_file_skip(metplus_config, exists, skip, run): # --------------------- @pytest.mark.parametrize( # list of input files - 'infiles', [ - [], - ['file1'], - ['file1', 'file2'], - ['file1', 'file2', 'file3'], - ] + 'infiles', [ + [], + ['file1'], + ['file1', 'file2'], + ['file1', 'file2', 'file3'], + ] ) @pytest.mark.wrapper def test_get_command(metplus_config, infiles): @@ -101,12 +98,12 @@ def test_get_command(metplus_config, infiles): @pytest.mark.parametrize( # offset = list of offsets to search # offset_to_find = expected offset file to find, None if no files should be found - 'offsets, offset_to_find', [ - ([6, 5, 4, 3], 5), - ([6, 4, 3], 3), - ([2, 3, 4, 5, 6], 3), - ([2, 4, 6], None), - ] + 'offsets, offset_to_find', [ + ([6, 5, 4, 3], 5), + ([6, 4, 3], 3), + ([2, 3, 4, 5, 6], 3), + ([2, 4, 6], None), + ] ) @pytest.mark.wrapper def test_find_input_files(metplus_config, offsets, offset_to_find): @@ -271,7 +268,7 @@ def test_find_input_files(metplus_config, offsets, offset_to_find): def test_pb2nc_all_fields(metplus_config, config_overrides, env_var_values): input_dir = '/some/input/dir' - config = metplus_config() + config = metplus_config # set config variables to prevent command from running and bypass check # if input files actually exist @@ -343,7 +340,7 @@ def test_pb2nc_all_fields(metplus_config, config_overrides, def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'PB2NCConfig_wrapped') @@ -361,7 +358,7 @@ def test_pb2nc_file_window(metplus_config): begin_value = -3600 end_value = 3600 - config = metplus_config() + config = metplus_config config.set('config', 'PB2NC_FILE_WINDOW_BEGIN', begin_value) config.set('config', 'PB2NC_FILE_WINDOW_END', end_value) wrapper = PB2NCWrapper(config) diff --git a/internal/tests/pytests/wrappers/pcp_combine/test1.conf b/internal/tests/pytests/wrappers/pcp_combine/test1.conf deleted file mode 100644 index 0d50280991..0000000000 --- a/internal/tests/pytests/wrappers/pcp_combine/test1.conf +++ /dev/null @@ -1,35 +0,0 @@ -[config] -FCST_PCP_COMBINE_INPUT_ACCUMS = 6 -FCST_PCP_COMBINE_INPUT_NAMES = P06M_NONE -FCST_PCP_COMBINE_INPUT_LEVELS = "(*,*)" - -OBS_PCP_COMBINE_INPUT_ACCUMS = 1 -OBS_PCP_COMBINE_INPUT_NAMES = P01M_NONE - -OBS_PCP_COMBINE_DATA_INTERVAL = 1 -OBS_PCP_COMBINE_TIMES_PER_FILE = 4 - -FCST_PCP_COMBINE_INPUT_DATATYPE = NETCDF -OBS_PCP_COMBINE_INPUT_DATATYPE = NETCDF - -FCST_PCP_COMBINE_RUN = True - -FCST_PCP_COMBINE_METHOD = ADD - -OBS_PCP_COMBINE_RUN = True - -OBS_PCP_COMBINE_METHOD = ADD - -[dir] -OBS_PCP_COMBINE_INPUT_DIR = {METPLUS_BASE}/internal/tests/data/accum -OBS_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/internal/tests/data/fakeout - -FCST_PCP_COMBINE_INPUT_DIR = {METPLUS_BASE}/internal/tests/data/fcst -FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/internal/tests/data/fakeout - -[filename_templates] -OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}.{level?fmt=%HH}h -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/outfile.{valid?fmt=%Y%m%d%H}_A{level?fmt=%HH}h -FCST_PCP_COMBINE_INPUT_TEMPLATE = {init?fmt=%Y%m%d}/file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc -FCST2_PCP_COMBINE_INPUT_TEMPLATE = file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}_A{level?fmt=%HHH}.nc \ No newline at end of file diff --git a/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py index 19be7d9b24..3cfe0e6764 100644 --- a/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py +++ b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py @@ -16,17 +16,41 @@ def get_test_data_dir(config, subdir=None): top_dir = os.path.join(top_dir, subdir) return top_dir + def pcp_combine_wrapper(metplus_config, d_type): """! Returns a default PCPCombineWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" + config = metplus_config + config.set('config', 'FCST_PCP_COMBINE_INPUT_ACCUMS', '6') + config.set('config', 'FCST_PCP_COMBINE_INPUT_NAMES', 'P06M_NONE') + config.set('config', 'FCST_PCP_COMBINE_INPUT_LEVELS', '"(*,*)"') + config.set('config', 'OBS_PCP_COMBINE_INPUT_ACCUMS', '1') + config.set('config', 'OBS_PCP_COMBINE_INPUT_NAMES', 'P01M_NONE') + config.set('config', 'OBS_PCP_COMBINE_DATA_INTERVAL', '1') + config.set('config', 'OBS_PCP_COMBINE_TIMES_PER_FILE', '4') + config.set('config', 'FCST_PCP_COMBINE_METHOD', 'ADD') + config.set('config', 'OBS_PCP_COMBINE_METHOD', 'ADD') + config.set('config', 'OBS_PCP_COMBINE_INPUT_DIR', + '{METPLUS_BASE}/internal/tests/data/accum') + config.set('config', 'OBS_PCP_COMBINE_OUTPUT_DIR', + '{OUTPUT_BASE}/internal/tests/data/fakeout') + config.set('config', 'FCST_PCP_COMBINE_INPUT_DIR', + '{METPLUS_BASE}/internal/tests/data/fcst') + config.set('config', 'FCST_PCP_COMBINE_OUTPUT_DIR', + '{OUTPUT_BASE}/internal/tests/data/fakeout') + config.set('config', 'OBS_PCP_COMBINE_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}.{level?fmt=%HH}h') + config.set('config', 'OBS_PCP_COMBINE_OUTPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/outfile.{valid?fmt=%Y%m%d%H}_A{level?fmt=%HH}h') + config.set('config', 'FCST_PCP_COMBINE_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d}/file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc') + config.set('config', 'FCST2_PCP_COMBINE_INPUT_TEMPLATE', + 'file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc') + config.set('config', 'FCST_PCP_COMBINE_OUTPUT_TEMPLATE', + '{valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}_A{level?fmt=%HHH}.nc') - # PCPCombineWrapper with configuration values determined by what is set in - # the test1.conf file. - extra_configs = [] - extra_configs.append(os.path.join(os.path.dirname(__file__), 'test1.conf')) - config = metplus_config(extra_configs) if d_type == "FCST": config.set('config', 'FCST_PCP_COMBINE_RUN', True) elif d_type == "OBS": @@ -220,7 +244,7 @@ def test_pcp_combine_add_subhourly(metplus_config): fcst_level = 'Surface' fcst_output_name = 'A001500' fcst_fmt = f'\'name="{fcst_name}"; level="{fcst_level}";\'' - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -285,7 +309,7 @@ def test_pcp_combine_add_subhourly(metplus_config): @pytest.mark.wrapper def test_pcp_combine_bucket(metplus_config): fcst_output_name = 'APCP' - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -365,7 +389,7 @@ def test_pcp_combine_derive(metplus_config, config_overrides, extra_fields): fcst_name = 'APCP' fcst_level = 'A03' fcst_fmt = f'-field \'name="{fcst_name}"; level="{fcst_level}";\'' - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -438,7 +462,7 @@ def test_pcp_combine_derive(metplus_config, config_overrides, extra_fields): def test_pcp_combine_loop_custom(metplus_config): fcst_name = 'APCP' ens_list = ['ens1', 'ens2', 'ens3', 'ens4', 'ens5', 'ens6'] - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -500,7 +524,7 @@ def test_pcp_combine_loop_custom(metplus_config): @pytest.mark.wrapper def test_pcp_combine_subtract(metplus_config): - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -563,7 +587,7 @@ def test_pcp_combine_sum_subhourly(metplus_config): fcst_level = 'Surface' fcst_output_name = 'A001500' fcst_fmt = f'-field \'name="{fcst_name}"; level="{fcst_level}";\'' - config = metplus_config() + config = metplus_config test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, @@ -641,7 +665,7 @@ def test_pcp_combine_sum_subhourly(metplus_config): def test_handle_name_argument(metplus_config, output_name, extra_output, expected_results): data_src = 'FCST' - config = metplus_config() + config = metplus_config wrapper = PCPCombineWrapper(config) wrapper.c_dict[data_src + '_EXTRA_OUTPUT_NAMES'] = extra_output wrapper._handle_name_argument(output_name, data_src) @@ -680,7 +704,7 @@ def test_handle_name_argument(metplus_config, output_name, extra_output, @pytest.mark.wrapper def test_get_extra_fields(metplus_config, names, levels, expected_args): data_src = 'FCST' - config = metplus_config() + config = metplus_config config.set('config', 'FCST_PCP_COMBINE_RUN', True) config.set('config', 'FCST_PCP_COMBINE_METHOD', 'ADD') config.set('config', 'FCST_PCP_COMBINE_EXTRA_NAMES', names) @@ -697,7 +721,7 @@ def test_get_extra_fields(metplus_config, names, levels, expected_args): @pytest.mark.wrapper def test_add_method_single_file(metplus_config): data_src = 'FCST' - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) config.set('config', 'INPUT_MUST_EXIST', False) @@ -767,7 +791,7 @@ def test_subtract_method_zero_accum(metplus_config): input_level = '"(*,*)"' in_dir = '/some/input/dir' out_dir = '/some/output/dir' - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) config.set('config', 'INPUT_MUST_EXIST', False) diff --git a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py index 3e779d81a3..fd05410467 100644 --- a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py +++ b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py @@ -217,7 +217,7 @@ def set_minimum_config_settings(config): @pytest.mark.wrapper_c def test_plot_point_obs(metplus_config, config_overrides, env_var_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -288,7 +288,7 @@ def test_plot_point_obs(metplus_config, config_overrides, env_var_values): def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'PlotPointObsConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py index e4b27f5d60..c0f51d47fd 100644 --- a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py +++ b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py @@ -13,7 +13,7 @@ def p2g_wrapper(metplus_config): files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) return Point2GridWrapper(config) diff --git a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py index 84ddf4e98c..2c574d2476 100755 --- a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py @@ -41,7 +41,7 @@ def set_minimum_config_settings(config): @pytest.mark.wrapper_a def test_met_dictionary_in_var_options(metplus_config): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) config.set('config', 'BOTH_VAR1_NAME', 'name') @@ -514,7 +514,7 @@ def test_point_stat_all_fields(metplus_config, config_overrides, fcst_fmts.append(fcst_fmt) obs_fmts.append(obs_fmt) - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) for index, (fcst, obs) in enumerate(zip(fcsts, obss)): @@ -583,7 +583,7 @@ def test_point_stat_all_fields(metplus_config, config_overrides, def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'PointStatConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py b/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py index 9e6e436bf0..bce689fb23 100644 --- a/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py +++ b/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py @@ -15,7 +15,7 @@ def rdp_wrapper(metplus_config): files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) return RegridDataPlaneWrapper(config) @@ -61,7 +61,7 @@ def rdp_wrapper(metplus_config): def test_set_field_command_line_arguments(metplus_config, field_info, expected_arg): data_type = 'FCST' - config = metplus_config() + config = metplus_config rdp = RegridDataPlaneWrapper(config) @@ -128,7 +128,7 @@ def test_set_field_command_line_arguments(metplus_config, field_info, expected_a def test_get_output_names(metplus_config, var_list, expected_names): data_type = 'FCST' - rdp = RegridDataPlaneWrapper(metplus_config()) + rdp = RegridDataPlaneWrapper(metplus_config) assert rdp.get_output_names(var_list, data_type) == expected_names diff --git a/internal/tests/pytests/wrappers/runtime_freq/test_runtime_freq.py b/internal/tests/pytests/wrappers/runtime_freq/test_runtime_freq.py index 0c589349c3..0d0f218cff 100644 --- a/internal/tests/pytests/wrappers/runtime_freq/test_runtime_freq.py +++ b/internal/tests/pytests/wrappers/runtime_freq/test_runtime_freq.py @@ -102,7 +102,7 @@ ) @pytest.mark.wrapper def test_compare_time_info(metplus_config, runtime, filetime, expected_result): - config = metplus_config() + config = metplus_config wrapper = RuntimeFreqWrapper(config) actual_result = wrapper.compare_time_info(runtime, filetime) diff --git a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index d588d2425f..228e89114c 100644 --- a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -37,11 +37,27 @@ def get_input_dirs(config): def series_analysis_wrapper(metplus_config, config_overrides=None): - extra_configs = [] - extra_configs.append(os.path.join(os.path.dirname(__file__), - 'series_test.conf')) - config = metplus_config(extra_configs) - config.set('config', 'LOOP_ORDER', 'processes') + config = metplus_config + config.set('config', 'SERIES_ANALYSIS_STAT_LIST', 'TOTAL, FBAR, OBAR, ME') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d') + config.set('config', 'INIT_BEG', '20141214') + config.set('config', 'INIT_END', '20141214') + config.set('config', 'INIT_INCREMENT', '21600') + config.set('config', 'SERIES_ANALYSIS_BACKGROUND_MAP', 'no') + config.set('config', 'FCST_SERIES_ANALYSIS_INPUT_TEMPLATE', + ('{init?fmt=%Y%m%d_%H}/{storm_id}/FCST_TILE_F{lead?fmt=%3H}_' + 'gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%3H}.nc')) + config.set('config', 'OBS_SERIES_ANALYSIS_INPUT_TEMPLATE', + ('{init?fmt=%Y%m%d_%H}/{storm_id}/OBS_TILE_F{lead?fmt=%3H}_gfs' + '_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%3H}.nc')) + config.set('config', 'EXTRACT_TILES_OUTPUT_DIR', + '{OUTPUT_BASE}/extract_tiles') + config.set('config', 'FCST_SERIES_ANALYSIS_INPUT_DIR', + '{EXTRACT_TILES_OUTPUT_DIR}') + config.set('config', 'OBS_SERIES_ANALYSIS_INPUT_DIR', + '{EXTRACT_TILES_OUTPUT_DIR}') + config.set('config', 'SERIES_ANALYSIS_OUTPUT_DIR', + '{OUTPUT_BASE}/series_analysis_init') if config_overrides: for key, value in config_overrides.items(): config.set('config', key, value) @@ -297,7 +313,7 @@ def set_minimum_config_settings(config): def test_series_analysis_single_field(metplus_config, config_overrides, env_var_values): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -851,7 +867,7 @@ def test_get_netcdf_min_max(metplus_config): def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'SeriesAnalysisConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py index b5a9552da9..76c277fd93 100644 --- a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py +++ b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py @@ -16,17 +16,43 @@ pp = pprint.PrettyPrinter() + def stat_analysis_wrapper(metplus_config): """! Returns a default StatAnalysisWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration files. Subsequent tests can customize the final METplus configuration to over-ride these /path/to values.""" + config = metplus_config - # Default, empty StatAnalysisWrapper with some configuration values set - # to /path/to: - extra_configs = [TEST_CONF] - config = metplus_config(extra_configs) handle_tmp_dir(config) + config.set('config', 'PROCESS_LIST', 'StatAnalysis') + config.set('config', 'STAT_ANALYSIS_OUTPUT_DIR', + '{OUTPUT_BASE}/stat_analysis') + config.set('config', 'MODEL1_STAT_ANALYSIS_LOOKIN_DIR', + '{METPLUS_BASE}/internal/tests/data/stat_data') + config.set('config', 'LOOP_BY', 'VALID') + config.set('config', 'VALID_TIME_FMT', '%Y%m%d') + config.set('config', 'VALID_BEG', '20190101') + config.set('config', 'VALID_END', '20190101') + config.set('config', 'VALID_INCREMENT', '86400') + config.set('config', 'MODEL1', 'MODEL_TEST') + config.set('config', 'MODEL1_REFERENCE_NAME', 'MODELTEST') + config.set('config', 'MODEL1_OBTYPE', 'MODEL_TEST_ANL') + config.set('config', 'STAT_ANALYSIS_CONFIG_FILE', + '{PARM_BASE}/met_config/STATAnalysisConfig_wrapped') + config.set('config', 'STAT_ANALYSIS_JOB_NAME', 'filter') + config.set('config', 'STAT_ANALYSIS_JOB_ARGS', '-dump_row [dump_row_file]') + config.set('config', 'MODEL_LIST', '{MODEL1}') + config.set('config', 'FCST_VALID_HOUR_LIST', '00') + config.set('config', 'FCST_INIT_HOUR_LIST', '00, 06, 12, 18') + config.set('config', 'GROUP_LIST_ITEMS', 'FCST_INIT_HOUR_LIST') + config.set('config', 'LOOP_LIST_ITEMS', 'FCST_VALID_HOUR_LIST, MODEL_LIST') + config.set('config', 'MODEL1_STAT_ANALYSIS_DUMP_ROW_TEMPLATE', + ('{fcst_valid_hour?fmt=%H}Z/{MODEL1}/' + '{MODEL1}_{valid?fmt=%Y%m%d}.stat')) + config.set('config', 'MODEL1_STAT_ANALYSIS_OUT_STAT_TEMPLATE', + ('{model?fmt=%s}_{obtype?fmt=%s}_valid{valid?fmt=%Y%m%d}' + '{valid_hour?fmt=%H}_init{fcst_init_hour?fmt=%s}.stat')) return StatAnalysisWrapper(config) @@ -148,9 +174,9 @@ def set_minimum_config_settings(config): ] ) @pytest.mark.wrapper_d -def test_stat_analysis_env_vars(metplus_config, config_overrides, - expected_env_vars): - config = metplus_config() +def test_valid_init_env_vars(metplus_config, config_overrides, + expected_env_vars): + config = metplus_config set_minimum_config_settings(config) config.set('config', 'INIT_END', '20221015') for key, value in config_overrides.items(): @@ -207,7 +233,7 @@ def test_stat_analysis_env_vars(metplus_config, config_overrides, @pytest.mark.wrapper_d def test_check_required_job_template(metplus_config, config_overrides, expected_result): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) for key, value in config_overrides.items(): config.set('config', key, value) @@ -292,10 +318,11 @@ def test_check_required_job_template(metplus_config, config_overrides, ) @pytest.mark.wrapper_d def test_get_runtime_settings(metplus_config, c_dict, expected_result): - config = metplus_config() + config = metplus_config wrapper = StatAnalysisWrapper(config) runtime_settings = wrapper._get_runtime_settings(c_dict) + pp.pprint(runtime_settings) assert runtime_settings == expected_result @@ -315,7 +342,7 @@ def test_get_runtime_settings(metplus_config, c_dict, expected_result): @pytest.mark.wrapper_d def test_format_conf_list(metplus_config, list_name, config_overrides, expected_value): - config = metplus_config() + config = metplus_config for key, value in config_overrides.items(): config.set('config', key, value) @@ -587,6 +614,7 @@ def test_build_stringsub_dict(metplus_config, lists_to_loop, c_dict_overrides, if item not in lists_to_loop] config_dict['LISTS_TO_GROUP'] = lists_to_group config_dict['LISTS_TO_LOOP'] = lists_to_loop + test_stringsub_dict = st._build_stringsub_dict(config_dict) print(test_stringsub_dict) @@ -869,7 +897,7 @@ def test_run_stat_analysis(metplus_config): ) @pytest.mark.wrapper_d def test_get_level_list(metplus_config, data_type, config_list, expected_list): - config = metplus_config() + config = metplus_config config.set('config', f'{data_type}_LEVEL_LIST', config_list) saw = StatAnalysisWrapper(config) @@ -880,7 +908,7 @@ def test_get_level_list(metplus_config, data_type, config_list, expected_list): @pytest.mark.wrapper_d def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config config.set('config', 'INPUT_MUST_EXIST', False) wrapper = StatAnalysisWrapper(config) diff --git a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py index 248228fa92..b3cb191fa0 100644 --- a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py @@ -293,7 +293,7 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values): expected_edeck_count = 6 expected_shape_count = 5 - config = metplus_config() + config = metplus_config test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), 'internal', 'tests', @@ -423,7 +423,7 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values): def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', 'TCGenConfig_wrapped') diff --git a/internal/tests/pytests/wrappers/tc_pairs/tc_pairs_wrapper_test.conf b/internal/tests/pytests/wrappers/tc_pairs/tc_pairs_wrapper_test.conf deleted file mode 100644 index 8c574d2e5f..0000000000 --- a/internal/tests/pytests/wrappers/tc_pairs/tc_pairs_wrapper_test.conf +++ /dev/null @@ -1,105 +0,0 @@ -# -# CONFIGURATION -# -[config] -LOOP_METHOD = processes -# Configuration files -TC_PAIRS_CONFIG_FILE = {PARM_BASE}/met_config/TCPairsConfig_wrapped - -PROCESS_LIST = TCPairs - -# The init time begin and end times, increment, and last init hour. -INIT_TIME_FMT = %Y%m%d -INIT_BEG = 20141201 -INIT_END = 20141231 -INIT_INCREMENT = 21600 ;; set to every 6 hours=21600 seconds -TC_PAIRS_INIT_INCLUDE = -TC_PAIRS_INIT_EXCLUDE = - -TC_PAIRS_VALID_BEG = -TC_PAIRS_VALID_END = - -TC_PAIRS_READ_ALL_FILES = no - -# set to true or yes to reformat track data into ATCF format expected by tc_pairs -TC_PAIRS_REFORMAT_DECK = yes -TC_PAIRS_REFORMAT_TYPE = SBU - - -# TC PAIRS filtering options -TC_PAIRS_MISSING_VAL_TO_REPLACE = -99 -TC_PAIRS_MISSING_VAL = -9999 - - -# OVERWRITE OPTIONS -# Don't overwrite filter files if they already exist. -# Set to no if you do NOT want to override existing files -# Set to yes if you do want to override existing files -#OVERWRITE_TRACK = yes -TC_PAIRS_SKIP_IF_REFORMAT_EXISTS = no -TC_PAIRS_SKIP_IF_OUTPUT_EXISTS = no - -# List of models to be used (white space or comma separated) eg: DSHP, LGEM, HWRF -# If no models are listed, then process all models in the input file(s). -MODEL = - -# List of storm ids of interest (space or comma separated) e.g.: AL112012, AL122012 -# If no storm ids are listed, then process all storm ids in the input file(s). -TC_PAIRS_STORM_ID = - -# Basins (of origin/region). Indicate with space or comma-separated list of regions, eg. AL: for North Atlantic, -# WP: Western North Pacific, CP: Central North Pacific, SH: Southern Hemisphere, IO: North Indian Ocean, LS: Southern -# Hemisphere -TC_PAIRS_BASIN = - -# Cyclone, a space or comma-separated list of cyclone numbers. If left empty, all cyclones will be used. -TC_PAIRS_CYCLONE = - -# Storm name, a space or comma-separated list of storm names to evaluate. If left empty, all storms will be used. -TC_PAIRS_STORM_NAME = - -# DLAND file, the full path of the file that contains the gridded representation of the -# minimum distance from land. -TC_PAIRS_DLAND_FILE = MET_BASE/tc_data/dland_global_tenth_degree.nc - - -# -# FILENAME TEMPLATES -# -[filename_templates] -# We DO NOT want to interpret time info or expand{} these values. -# Use, getraw('filename_templates','FCST_EXTRACT_TILES_INPUT_TEMPLATE') to get -# 'gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%HHH}.grb2' -# FCST_EXTRACT_TILES_INPUT_TEMPLATE = gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%HHH}.grb2 -# GFS_FCST_NC_FILE_TMPL = gfs_4_{init?fmt=%Y%m%d}_{init?fmt=%H}00_{lead?fmt=%HHH}.nc -# OBS_EXTRACT_TILES_INPUT_TEMPLATE = gfs_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H}00_000.grb2 -# GFS_ANLY_NC_FILE_TMPL = gfs_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H}00_000.nc - -TC_PAIRS_ADECK_TEMPLATE = {date?fmt=%Y%m}/a{basin?fmt=%s}q{date?fmt=%Y%m}*.gfso.{cyclone?fmt=%s} -TC_PAIRS_BDECK_TEMPLATE = {date?fmt=%Y%m}/b{basin?fmt=%s}q{date?fmt=%Y%m}*.gfso.{cyclone?fmt=%s} -TC_PAIRS_OUTPUT_TEMPLATE = {date?fmt=%Y%m}/{basin?fmt=%s}q{date?fmt=%Y%m%d%H}.gfso.{cyclone?fmt=%s} - -# -# DIRECTORIES -# -[dir] - -# Location of your model data of interest -#EXTRACT_TILES_GRID_INPUT_DIR = {METPLUS_BASE}/sample_data/GFS/reduced_model_data -#EXTRACT_TILES_GRID_INPUT_DIR = /d1/SBU/GFS/reduced_model_data -# Commonly used base METplus variables - -# track data, set to your data source -TC_PAIRS_ADECK_INPUT_DIR = {INPUT_BASE}/met_test/new/track_data -TC_PAIRS_BDECK_INPUT_DIR = {INPUT_BASE}/met_test/new/track_data - - -#TRACK_DATA_DIR = {METPLUS_BASE}/sample_data/GFS/track_data -#TC_PAIRS_ADECK_INPUT_DIR = /d1/SBU/GFS/track_data -#TC_PAIRS_ADECK_INPUT_DIR = /d1/METplus_TC/adeck -#TC_PAIRS_BDECK_INPUT_DIR = /d1/SBU/GFS/track_data -#TC_PAIRS_BDECK_INPUT_DIR = /d1/METplus_TC/bdeck -TC_PAIRS_REFORMAT_DIR = {OUTPUT_BASE}/track_data_atcf -#TRACK_DATA_SUBDIR_MOD = {PROJ_DIR}/track_data_atcf -TC_PAIRS_OUTPUT_DIR = {OUTPUT_BASE}/tc_pairs - diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index 8f9de6d6ff..b7ad438f2a 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -68,7 +68,7 @@ def set_minimum_config_settings(config, loop_by='INIT'): ) def test_read_storm_info(metplus_config, config_overrides, isOK): """! Check if error is thrown if storm_id and basin or cyclone are set """ - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) # set config variable overrides @@ -94,7 +94,7 @@ def test_parse_storm_id(metplus_config, storm_id, basin, cyclone): Check that it returns wildcard expressions basin and cyclone cannot be parsed from storm ID """ - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -141,7 +141,7 @@ def test_get_bdeck(metplus_config, basin, cyclone, expected_files, combinations of basin/cyclone inputs """ time_info = {'date': datetime(2014, 12, 31, 18)} - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -194,7 +194,7 @@ def test_get_basin_cyclone_from_bdeck(metplus_config, template, filename, expected_basin = other_basin if other_basin else 'al' expected_cyclone = other_cyclone if other_cyclone else '1009' time_info = {'date': datetime(2014, 12, 31, 18)} - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) wrapper = TCPairsWrapper(config) @@ -245,7 +245,7 @@ def test_get_basin_cyclone_from_bdeck(metplus_config, template, filename, @pytest.mark.wrapper def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, storm_type, values_to_check): - config = metplus_config() + config = metplus_config set_minimum_config_settings(config) @@ -297,47 +297,48 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, @pytest.mark.parametrize( - 'config_overrides, env_var_values', [ + 'loop_by, config_overrides, env_var_values', [ + # LOOP_BY = INIT # 0: no config overrides that set env vars - ({}, {}), + ('INIT', {}, {}), # 1: description - ({'TC_PAIRS_DESC': 'my_desc'}, + ('INIT', {'TC_PAIRS_DESC': 'my_desc'}, {'METPLUS_DESC': 'desc = "my_desc";'}), # 2: only basin that corresponds to existing test file is used - ({'TC_PAIRS_BASIN': 'AL, ML'}, + ('INIT', {'TC_PAIRS_BASIN': 'AL, ML'}, {'METPLUS_BASIN': 'basin = ["ML"];'}), # 3: only cyclone that corresponds to existing test file is used - ({'TC_PAIRS_CYCLONE': '1005, 0104'}, + ('INIT', {'TC_PAIRS_CYCLONE': '1005, 0104'}, {'METPLUS_CYCLONE': 'cyclone = ["0104"];'}), # 4: model list - ({'MODEL': 'MOD1, MOD2'}, + ('INIT', {'MODEL': 'MOD1, MOD2'}, {'METPLUS_MODEL': 'model = ["MOD1", "MOD2"];'}), # 5: init begin - ({'TC_PAIRS_INIT_BEG': '20141031_14'}, + ('INIT', {'TC_PAIRS_INIT_BEG': '20141031_14'}, {'METPLUS_INIT_BEG': 'init_beg = "20141031_14";'}), # 6: init end - ({'TC_PAIRS_INIT_END': '20151031_14'}, + ('INIT', {'TC_PAIRS_INIT_END': '20151031_14'}, {'METPLUS_INIT_END': 'init_end = "20151031_14";'}), # 7: dland file - ({'TC_PAIRS_DLAND_FILE': 'my_dland.nc'}, + ('INIT', {'TC_PAIRS_DLAND_FILE': 'my_dland.nc'}, {'METPLUS_DLAND_FILE': 'dland_file = "my_dland.nc";'}), # 8: init_exc - ({'TC_PAIRS_INIT_EXCLUDE': '20141031_14'}, + ('INIT', {'TC_PAIRS_INIT_EXCLUDE': '20141031_14'}, {'METPLUS_INIT_EXC': 'init_exc = ["20141031_14"];'}), # 9: init_inc - ({'TC_PAIRS_INIT_INCLUDE': '20141031_14'}, + ('INIT', {'TC_PAIRS_INIT_INCLUDE': '20141031_14'}, {'METPLUS_INIT_INC': 'init_inc = ["20141031_14"];'}), # 10: storm name - ({'TC_PAIRS_STORM_NAME': 'KATRINA, OTHER'}, + ('INIT', {'TC_PAIRS_STORM_NAME': 'KATRINA, OTHER'}, {'METPLUS_STORM_NAME': 'storm_name = ["KATRINA", "OTHER"];'}), # 11: valid begin - ({'TC_PAIRS_VALID_BEG': '20141031_14'}, + ('INIT', {'TC_PAIRS_VALID_BEG': '20141031_14'}, {'METPLUS_VALID_BEG': 'valid_beg = "20141031_14";'}), # 12: valid end - ({'TC_PAIRS_VALID_END': '20141031_14'}, + ('INIT', {'TC_PAIRS_VALID_END': '20141031_14'}, {'METPLUS_VALID_END': 'valid_end = "20141031_14";'}), # 13: consensus 1 dictionary - ({'TC_PAIRS_CONSENSUS1_NAME': 'name1', + ('INIT', {'TC_PAIRS_CONSENSUS1_NAME': 'name1', 'TC_PAIRS_CONSENSUS1_MEMBERS': 'member1a, member1b', 'TC_PAIRS_CONSENSUS1_REQUIRED': 'true, false', 'TC_PAIRS_CONSENSUS1_MIN_REQ': '1'}, @@ -346,7 +347,7 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, 'required = [true, false];min_req = 1;}];' )}), # 14: consensus 2 dictionaries - ({'TC_PAIRS_CONSENSUS1_NAME': 'name1', + ('INIT', {'TC_PAIRS_CONSENSUS1_NAME': 'name1', 'TC_PAIRS_CONSENSUS1_MEMBERS': 'member1a, member1b', 'TC_PAIRS_CONSENSUS1_REQUIRED': 'true, false', 'TC_PAIRS_CONSENSUS1_MIN_REQ': '1', @@ -363,199 +364,283 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, 'required = [false, true];min_req = 2;}];' )}), # 15: valid_exc - ({'TC_PAIRS_VALID_EXCLUDE': '20141031_14'}, + ('INIT', {'TC_PAIRS_VALID_EXCLUDE': '20141031_14'}, {'METPLUS_VALID_EXC': 'valid_exc = ["20141031_14"];'}), # 16: valid_inc - ({'TC_PAIRS_VALID_INCLUDE': '20141031_14'}, + ('INIT', {'TC_PAIRS_VALID_INCLUDE': '20141031_14'}, {'METPLUS_VALID_INC': 'valid_inc = ["20141031_14"];'}), # 17: write_valid - ({'TC_PAIRS_WRITE_VALID': '20141031_14'}, + ('INIT', {'TC_PAIRS_WRITE_VALID': '20141031_14'}, {'METPLUS_WRITE_VALID': 'write_valid = ["20141031_14"];'}), # 18: check_dup - ({'TC_PAIRS_CHECK_DUP': 'False', }, + ('INIT', {'TC_PAIRS_CHECK_DUP': 'False', }, {'METPLUS_CHECK_DUP': 'check_dup = FALSE;'}), # 19: interp12 - ({'TC_PAIRS_INTERP12': 'replace', }, + ('INIT', {'TC_PAIRS_INTERP12': 'replace', }, {'METPLUS_INTERP12': 'interp12 = REPLACE;'}), # 20 match_points - ({'TC_PAIRS_MATCH_POINTS': 'False', }, + ('INIT', {'TC_PAIRS_MATCH_POINTS': 'False', }, + {'METPLUS_MATCH_POINTS': 'match_points = FALSE;'}), + # LOOP_BY = VALID + # 21: no config overrides that set env vars + ('VALID', {}, {}), + # 22: description + ('VALID', {'TC_PAIRS_DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + # 23: only basin that corresponds to existing test file is used + ('VALID', {'TC_PAIRS_BASIN': 'AL, ML'}, + {'METPLUS_BASIN': 'basin = ["ML"];'}), + # 24: only cyclone that corresponds to existing test file is used + ('VALID', {'TC_PAIRS_CYCLONE': '1005, 0104'}, + {'METPLUS_CYCLONE': 'cyclone = ["0104"];'}), + # 25: model list + ('VALID', {'MODEL': 'MOD1, MOD2'}, + {'METPLUS_MODEL': 'model = ["MOD1", "MOD2"];'}), + # 26: init begin + ('VALID', {'TC_PAIRS_INIT_BEG': '20141031_14'}, + {'METPLUS_INIT_BEG': 'init_beg = "20141031_14";'}), + # 27: init end + ('VALID', {'TC_PAIRS_INIT_END': '20151031_14'}, + {'METPLUS_INIT_END': 'init_end = "20151031_14";'}), + # 28: dland file + ('VALID', {'TC_PAIRS_DLAND_FILE': 'my_dland.nc'}, + {'METPLUS_DLAND_FILE': 'dland_file = "my_dland.nc";'}), + # 29: init_exc + ('VALID', {'TC_PAIRS_INIT_EXCLUDE': '20141031_14'}, + {'METPLUS_INIT_EXC': 'init_exc = ["20141031_14"];'}), + # 30: init_inc + ('VALID', {'TC_PAIRS_INIT_INCLUDE': '20141031_14'}, + {'METPLUS_INIT_INC': 'init_inc = ["20141031_14"];'}), + # 31: storm name + ('VALID', {'TC_PAIRS_STORM_NAME': 'KATRINA, OTHER'}, + {'METPLUS_STORM_NAME': 'storm_name = ["KATRINA", "OTHER"];'}), + # 32: valid begin + ('VALID', {'TC_PAIRS_VALID_BEG': '20141031_14'}, + {'METPLUS_VALID_BEG': 'valid_beg = "20141031_14";'}), + # 33: valid end + ('VALID', {'TC_PAIRS_VALID_END': '20141031_14'}, + {'METPLUS_VALID_END': 'valid_end = "20141031_14";'}), + # 34: consensus 1 dictionary + ('VALID', {'TC_PAIRS_CONSENSUS1_NAME': 'name1', + 'TC_PAIRS_CONSENSUS1_MEMBERS': 'member1a, member1b', + 'TC_PAIRS_CONSENSUS1_REQUIRED': 'true, false', + 'TC_PAIRS_CONSENSUS1_MIN_REQ': '1'}, + {'METPLUS_CONSENSUS_LIST': ( + 'consensus = [{name = "name1";members = ["member1a", "member1b"];' + 'required = [true, false];min_req = 1;}];' + )}), + # 35: consensus 2 dictionaries + ('VALID', {'TC_PAIRS_CONSENSUS1_NAME': 'name1', + 'TC_PAIRS_CONSENSUS1_MEMBERS': 'member1a, member1b', + 'TC_PAIRS_CONSENSUS1_REQUIRED': 'true, false', + 'TC_PAIRS_CONSENSUS1_MIN_REQ': '1', + 'TC_PAIRS_CONSENSUS2_NAME': 'name2', + 'TC_PAIRS_CONSENSUS2_MEMBERS': 'member2a, member2b', + 'TC_PAIRS_CONSENSUS2_REQUIRED': 'false, true', + 'TC_PAIRS_CONSENSUS2_MIN_REQ': '2' + }, + {'METPLUS_CONSENSUS_LIST': ( + 'consensus = [' + '{name = "name1";members = ["member1a", "member1b"];' + 'required = [true, false];min_req = 1;}' + '{name = "name2";members = ["member2a", "member2b"];' + 'required = [false, true];min_req = 2;}];' + )}), + # 36: valid_exc + ('VALID', {'TC_PAIRS_VALID_EXCLUDE': '20141031_14'}, + {'METPLUS_VALID_EXC': 'valid_exc = ["20141031_14"];'}), + # 37: valid_inc + ('VALID', {'TC_PAIRS_VALID_INCLUDE': '20141031_14'}, + {'METPLUS_VALID_INC': 'valid_inc = ["20141031_14"];'}), + # 38: write_valid + ('VALID', {'TC_PAIRS_WRITE_VALID': '20141031_14'}, + {'METPLUS_WRITE_VALID': 'write_valid = ["20141031_14"];'}), + # 39: check_dup + ('VALID', {'TC_PAIRS_CHECK_DUP': 'False', }, + {'METPLUS_CHECK_DUP': 'check_dup = FALSE;'}), + # 40: interp12 + ('VALID', {'TC_PAIRS_INTERP12': 'replace', }, + {'METPLUS_INTERP12': 'interp12 = REPLACE;'}), + # 41 match_points + ('VALID', {'TC_PAIRS_MATCH_POINTS': 'False', }, {'METPLUS_MATCH_POINTS': 'match_points = FALSE;'}), - ] ) @pytest.mark.wrapper -def test_tc_pairs_loop_order_processes(metplus_config, config_overrides, - env_var_values): - # run using init and valid time variables - for loop_by in ['INIT', 'VALID']: - remove_beg = remove_end = remove_match_points = False - config = metplus_config() - - set_minimum_config_settings(config, loop_by) - - test_data_dir = get_data_dir(config) - bdeck_dir = os.path.join(test_data_dir, 'bdeck') - adeck_dir = os.path.join(test_data_dir, 'adeck') - - config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) - config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir) - - # LOOP_ORDER processes runs once, times runs once per time - config.set('config', 'LOOP_ORDER', 'processes') - - # set config variable overrides - for key, value in config_overrides.items(): - config.set('config', key, value) - - if f'METPLUS_{loop_by}_BEG' not in env_var_values: - env_var_values[f'METPLUS_{loop_by}_BEG'] = ( - f'{loop_by.lower()}_beg = "{run_times[0]}";' - ) - remove_beg = True - - if f'METPLUS_{loop_by}_END' not in env_var_values: - env_var_values[f'METPLUS_{loop_by}_END'] = ( - f'{loop_by.lower()}_end = "{run_times[-1]}";' - ) - remove_end = True - - if f'METPLUS_MATCH_POINTS' not in env_var_values: - env_var_values[f'METPLUS_MATCH_POINTS'] = ( - 'match_points = TRUE;' - ) - remove_match_points = True - - wrapper = TCPairsWrapper(config) - assert wrapper.isOK - - app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) - verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" - config_file = wrapper.c_dict.get('CONFIG_FILE') - out_dir = wrapper.c_dict.get('OUTPUT_DIR') - expected_cmds = [(f"{app_path} {verbosity} " - f"-bdeck {bdeck_dir}/bmlq2014123118.gfso.0104 " - f"-adeck {adeck_dir}/amlq2014123118.gfso.0104 " - f"-config {config_file} " - f"-out {out_dir}/mlq2014121318.gfso.0104"), - ] - - - all_cmds = wrapper.run_all_times() - print(f"ALL COMMANDS: {all_cmds}") - assert len(all_cmds) == len(expected_cmds) - - for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): - # ensure commands are generated as expected - assert cmd == expected_cmd - - # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: - match = next((item for item in env_vars if - item.startswith(env_var_key)), None) - assert match is not None - print(f'Checking env var: {env_var_key}') - actual_value = match.split('=', 1)[1] - assert env_var_values.get(env_var_key, '') == actual_value - - if remove_beg: - del env_var_values[f'METPLUS_{loop_by}_BEG'] - if remove_end: - del env_var_values[f'METPLUS_{loop_by}_END'] - if remove_match_points: - del env_var_values['METPLUS_MATCH_POINTS'] +def test_tc_pairs_loop_order_processes(metplus_config, loop_by, + config_overrides, env_var_values): + config = metplus_config + remove_beg = remove_end = remove_match_points = False + + set_minimum_config_settings(config, loop_by) + + test_data_dir = get_data_dir(config) + bdeck_dir = os.path.join(test_data_dir, 'bdeck') + adeck_dir = os.path.join(test_data_dir, 'adeck') + + config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) + config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir) + + # LOOP_ORDER processes runs once, times runs once per time + config.set('config', 'LOOP_ORDER', 'processes') + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + if f'METPLUS_{loop_by}_BEG' not in env_var_values: + env_var_values[f'METPLUS_{loop_by}_BEG'] = ( + f'{loop_by.lower()}_beg = "{run_times[0]}";' + ) + remove_beg = True + + if f'METPLUS_{loop_by}_END' not in env_var_values: + env_var_values[f'METPLUS_{loop_by}_END'] = ( + f'{loop_by.lower()}_end = "{run_times[-1]}";' + ) + remove_end = True + + if f'METPLUS_MATCH_POINTS' not in env_var_values: + env_var_values[f'METPLUS_MATCH_POINTS'] = ( + 'match_points = TRUE;' + ) + remove_match_points = True + + wrapper = TCPairsWrapper(config) + assert wrapper.isOK + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + expected_cmds = [(f"{app_path} {verbosity} " + f"-bdeck {bdeck_dir}/bmlq2014123118.gfso.0104 " + f"-adeck {adeck_dir}/amlq2014123118.gfso.0104 " + f"-config {config_file} " + f"-out {out_dir}/mlq2014121318.gfso.0104"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + assert len(all_cmds) == len(expected_cmds) + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd + + # check that environment variables were set properly + for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + print(f'Checking env var: {env_var_key}') + actual_value = match.split('=', 1)[1] + assert env_var_values.get(env_var_key, '') == actual_value + + if remove_beg: + del env_var_values[f'METPLUS_{loop_by}_BEG'] + if remove_end: + del env_var_values[f'METPLUS_{loop_by}_END'] + if remove_match_points: + del env_var_values['METPLUS_MATCH_POINTS'] @pytest.mark.parametrize( - 'config_overrides, env_var_values', [ - # 0: no config overrides that set env vars - ({}, {}), + 'loop_by, config_overrides, env_var_values', [ + # 0: no config overrides that set env vars loop by = INIT + ('INIT', {}, {}), # 1: storm_id list - ({'TC_PAIRS_STORM_ID': 'AL092014, ML082015'}, + ('INIT', {'TC_PAIRS_STORM_ID': 'AL092014, ML082015'}, {'METPLUS_STORM_ID': 'storm_id = ["AL092014", "ML082015"];'}), # 2: basin list - ({'TC_PAIRS_BASIN': 'AL, ML'}, + ('INIT', {'TC_PAIRS_BASIN': 'AL, ML'}, {'METPLUS_BASIN': 'basin = ["AL", "ML"];'}), # 3: cyclone list - ({'TC_PAIRS_CYCLONE': '1005, 0104'}, + ('INIT', {'TC_PAIRS_CYCLONE': '1005, 0104'}, + {'METPLUS_CYCLONE': 'cyclone = ["1005", "0104"];'}), + # 4: no config overrides that set env vars loop by = VALID + ('VALID', {}, {}), + # 5: storm_id list + ('VALID', {'TC_PAIRS_STORM_ID': 'AL092014, ML082015'}, + {'METPLUS_STORM_ID': 'storm_id = ["AL092014", "ML082015"];'}), + # 6: basin list + ('VALID', {'TC_PAIRS_BASIN': 'AL, ML'}, + {'METPLUS_BASIN': 'basin = ["AL", "ML"];'}), + # 7: cyclone list + ('VALID', {'TC_PAIRS_CYCLONE': '1005, 0104'}, {'METPLUS_CYCLONE': 'cyclone = ["1005", "0104"];'}), ] ) @pytest.mark.wrapper -def test_tc_pairs_read_all_files(metplus_config, config_overrides, +def test_tc_pairs_read_all_files(metplus_config, loop_by, config_overrides, env_var_values): - # run using init and valid time variables - for loop_by in ['INIT', 'VALID']: - config = metplus_config() + config = metplus_config + + set_minimum_config_settings(config, loop_by) - set_minimum_config_settings(config, loop_by) + test_data_dir = get_data_dir(config) + bdeck_dir = os.path.join(test_data_dir, 'bdeck') + adeck_dir = os.path.join(test_data_dir, 'adeck') - test_data_dir = get_data_dir(config) - bdeck_dir = os.path.join(test_data_dir, 'bdeck') - adeck_dir = os.path.join(test_data_dir, 'adeck') + config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) + config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir) - config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) - config.set('config', 'TC_PAIRS_ADECK_INPUT_DIR', adeck_dir) + # LOOP_ORDER processes runs once, times runs once per time + config.set('config', 'LOOP_ORDER', 'processes') - # LOOP_ORDER processes runs once, times runs once per time - config.set('config', 'LOOP_ORDER', 'processes') + config.set('config', 'TC_PAIRS_READ_ALL_FILES', True) + config.set('config', 'TC_PAIRS_OUTPUT_TEMPLATE', '') - config.set('config', 'TC_PAIRS_READ_ALL_FILES', True) - config.set('config', 'TC_PAIRS_OUTPUT_TEMPLATE', '') + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) - # set config variable overrides - for key, value in config_overrides.items(): - config.set('config', key, value) + env_var_values[f'METPLUS_{loop_by}_BEG'] = ( + f'{loop_by.lower()}_beg = "{run_times[0]}";' + ) - env_var_values[f'METPLUS_{loop_by}_BEG'] = ( - f'{loop_by.lower()}_beg = "{run_times[0]}";' - ) + env_var_values[f'METPLUS_{loop_by}_END'] = ( + f'{loop_by.lower()}_end = "{run_times[-1]}";' + ) - env_var_values[f'METPLUS_{loop_by}_END'] = ( - f'{loop_by.lower()}_end = "{run_times[-1]}";' - ) + env_var_values['METPLUS_MATCH_POINTS'] = ( + 'match_points = TRUE;' + ) - env_var_values['METPLUS_MATCH_POINTS'] = ( - 'match_points = TRUE;' - ) + wrapper = TCPairsWrapper(config) + assert wrapper.isOK - wrapper = TCPairsWrapper(config) - assert wrapper.isOK - - app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) - verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" - config_file = wrapper.c_dict.get('CONFIG_FILE') - out_dir = wrapper.c_dict.get('OUTPUT_DIR') - expected_cmds = [(f"{app_path} {verbosity} " - f"-bdeck {bdeck_dir} " - f"-adeck {adeck_dir} " - f"-config {config_file} " - f"-out {out_dir}/tc_pairs"), - ] - - all_cmds = wrapper.run_all_times() - print(f"ALL COMMANDS: {all_cmds}") - assert len(all_cmds) == len(expected_cmds) - - for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): - # check that environment variables were set properly - for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: - match = next((item for item in env_vars if - item.startswith(env_var_key)), None) - assert match is not None - print(f'Checking env var: {env_var_key}') - actual_value = match.split('=', 1)[1] - assert env_var_values.get(env_var_key, '') == actual_value - - # unset begin and end for next loop - del env_var_values[f'METPLUS_{loop_by}_BEG'] - del env_var_values[f'METPLUS_{loop_by}_END'] + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + expected_cmds = [(f"{app_path} {verbosity} " + f"-bdeck {bdeck_dir} " + f"-adeck {adeck_dir} " + f"-config {config_file} " + f"-out {out_dir}/tc_pairs"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + assert len(all_cmds) == len(expected_cmds) + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # check that environment variables were set properly + for env_var_key in wrapper.WRAPPER_ENV_VAR_KEYS: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + print(f'Checking env var: {env_var_key}') + actual_value = match.split('=', 1)[1] + assert env_var_values.get(env_var_key, '') == actual_value @pytest.mark.wrapper def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config config.set('config', 'INIT_TIME_FMT', time_fmt) config.set('config', 'INIT_BEG', run_times[0]) default_config_file = os.path.join(config.getdir('PARM_BASE'), diff --git a/internal/tests/pytests/wrappers/tc_stat/tc_stat_conf.conf b/internal/tests/pytests/wrappers/tc_stat/tc_stat_conf.conf deleted file mode 100755 index ea35a5b14a..0000000000 --- a/internal/tests/pytests/wrappers/tc_stat/tc_stat_conf.conf +++ /dev/null @@ -1,173 +0,0 @@ -# -# PRECONDITION: REQUIRES INSTALLATION OF R on user system -# - -# -# CONFIGURATION -# -[config] -# set looping method to processes-each 'task' in the process list runs to -# completion (for all init times) before the next 'task' is run -LOOP_METHOD = processes - -# List of 'tasks' to run -PROCESS_LIST = TcStat - -# The init time begin and end times, increment, and last init hour. -INIT_BEG = 20150301 -INIT_END = 20150304 -# This is the step-size. Increment in seconds from the begin time to the end time -INIT_INCREMENT = 21600 ;; set to every 6 hours=21600 seconds - -# This is the last hour in your initialization time that you want to include in your time window -#INIT_HOUR_END = 18 - -# A list of times to include, in format YYYYMMDD_hh -#INIT_INCLUDE = - -# A list of times to exclude, in format YYYYMMDD_hh -#INIT_EXCLUDE = - -# -# Specify model valid time window in format YYYYMM[DD[_hh]]. Only tracks that fall within the valid time window will -# be used. -# -#VALID_BEG = -#VALID_END = - -# Run tc_stat using a config file or as command line -# if running via MET tc_stat config file, set to CONFIG. Leave blank or -# anything other than CONFIG if running via command line. -TC_STAT_CONFIG_FILE = {PARM_BASE}/met_config/TCStatConfig_wrapped - - -# !!!!!!!IMPORTANT!!!!!! -# Please refer to the README_TC located in ${MET_INSTALL_DIR}/share/met/config -# for details on setting up your analysis jobs. - -# For arithmetic expressions such as: -# -column 'ABS(AMSLP-BMSLP)', enclose the expression in ''. Notice that there are no -# whitespaces within the arithmetic expression. White spaces are to be used to -# separate options from values (e.g. -job summary -by AMODEL,LEAD,AMSLP -init_hour 00 -column 'AMSLP-BMSLP'). -# eg. -lookin {OUTPUT_BASE}/tc_pairs -job filter -dump_row {OUTPUT_BASE}/tc_stat_filter.out -basin ML -init_hr 00 -# or -lookin {OUTPUT_BASE}/tc_pairs -job summary -by AMODEL,LEAD -column AMSLP -column AMAX_WIND -column 'ABS(AMAX_WIND-BMAX_WIND)' -out {OUTPUT_BASE}/tc_stat/tc_stat_summary.tcst - -# Only if TC_STAT_RUN_VIA = CLI -# TC_STAT_CMD_LINE_JOB = -job filter -dump_row {OUTPUT_BASE}/tc_stat/tc_stat_filter.out -basin ML -init_hour 00 - -#TC_STAT_RUN_VIA=COMMAND so no need to define this, but you MUST define -# TC_STAT_JOBS_LIST - -# -# FILL in the following values if running multiple jobs which -# requires a MET tc_stat config file. -# -# These all map to the options in the default TC-Stat config file, except these -# are pre-pended with TC_STAT to avoid clashing with any other similarly -# named options from other MET tools (eg TC_STAT_AMODEL corresponds to the -# amodel option in the default MET tc-stat config file, whereas AMODEL -# corresponds to the amodel option in the MET tc-pairs config file). - -# Stratify by these columns: -TC_STAT_AMODEL = -TC_STAT_BMODEL = -TC_STAT_DESC = -TC_STAT_STORM_ID = -TC_STAT_BASIN = -TC_STAT_CYCLONE = -TC_STAT_STORM_NAME = - -# Stratify by init times via a comma-separate list of init times to -# include or exclude. Time format defined as YYYYMMDD_HH or YYYYMMDD_HHmmss -TC_STAT_INIT_BEG = 20170705 -TC_STAT_INIT_END = 20170901 -TC_STAT_INIT_INCLUDE = -TC_STAT_INIT_EXCLUDE = -TC_STAT_INIT_HOUR = 00 -TC_STAT_INIT_MASK = -TC_STAT_VALID_MASK = -TC_STAT_VALID_BEG = -TC_STAT_VALID_END = -TC_STAT_VALID_INCLUDE = -TC_STAT_VALID_EXCLUDE = -TC_STAT_LEAD_REQ = - -# Stratify by the valid time and lead time via comma-separated list of -# times in format HH[MMSS] -TC_STAT_VALID_HOUR = -TC_STAT_LEAD = - -# Stratify over the watch_warn column in the tcst file. Setting this to -# 'ALL' will match HUWARN, HUWATCH, TSWARN, TSWATCH -TC_STAT_TRACK_WATCH_WARN = - -# Stratify by applying thresholds to numeric data columns. Specify with -# comma-separated list of column names and thresholds to be applied. -# The length of TC_STAT_COLUMN_THRESH_NAME should be the same as -# TC_STAT_COLUMN_THRESH_VAL. -TC_STAT_COLUMN_THRESH_NAME = -TC_STAT_COLUMN_THRESH_VAL = - -# Stratify by a list of comma-separated columns names and values corresponding -# to non-numeric data columns of the values of interest. -TC_STAT_COLUMN_STR_NAME = -TC_STAT_COLUMN_STR_VAL = - -# Stratify by applying thresholds to numeric data columns only when lead=0. -# If lead=0 and the value does not meet the threshold, discard the entire -# track. The length of TC_STAT_INIT_THRESH_NAME must equal the length of -# TC_STAT_INIT_THRESH_VAL. -TC_STAT_INIT_THRESH_NAME = -TC_STAT_INIT_THRESH_VAL = - -# Stratify by applying thresholds to numeric data columns only when lead = 0. -# If lead = 0 but the value doesn't meet the threshold, discard the entire -# track. -TC_STAT_INIT_STR_NAME = -TC_STAT_INIT_STR_VAL = - -# Excludes any points where distance to land is <=0. When set to TRUE, once land -# is encountered, the remainder of the forecast track is NOT used for the -# verification, even if the track moves back over water. -TC_STAT_WATER_ONLY = false - -# TRUE or FALSE. To specify whether only those track points occurring near -# landfall should be retained. Landfall is the last bmodel track point before -# the distance to land switches from water to land. -TC_STAT_LANDFALL = false - - -# Define the landfall retention window, which is defined as the hours offset -# from the time of landfall. Format is in HH[MMSS]. Default TC_STAT_LANDFALL_BEG -# is set to -24, and TC_STAT_LANDFALL_END is set to 00 -TC_STAT_LANDFALL_BEG = -24 -TC_STAT_LANDFALL_END = 00 - -# Specify whether only those track points common to both the ADECK and BDECK -# tracks should be written out -TC_STAT_MATCH_POINTS = false - -# IMPORTANT Refer to the README_TC for details on setting up analysis -# jobs (located in {MET_INSTALL_DIR}/share/met/config - -# Separate each option and value with whitespace, and each job with a whitespace. -# No whitespace within arithmetic expressions or lists of items -# (e.g. -by AMSLP,AMODEL,LEAD -column '(AMAX_WIND-BMAX_WIND)') -# Enclose your arithmetic expressions with '' and separate each job -# by whitespace: -# -job filter -dump_row /path/to, -job summary -line_type TCMPR -column 'ABS(AMAX_WIND-BMAX_WIND)' -out {OUTPUT_BASE}/tc_stat/file.tcst - -TC_STAT_JOB_ARGS = -job summary -line_type TCMPR -column 'ABS(AMAX_WIND-BMAX_WIND)' -dump_row {OUTPUT_BASE}/tc_stat/tc_stat_summary.tcst - - -# -# DIRECTORIES -# -[dir] - -# TC-Stat input data (uses output from tc-pairs) -TC_STAT_LOOKIN_DIR = {INPUT_BASE}/met_test/tc_pairs - -# TC-Stat output data (creates .tcst ASCII files which can be read or used as -# input to TCMPR_Plotter_wrapper (the Python wrapper to plot_tcmpr.R) to create plots. -TC_STAT_OUTPUT_DIR = {OUTPUT_BASE}/tc_stat diff --git a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py index fe66cff3f8..ac7001898a 100644 --- a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py @@ -11,10 +11,25 @@ def get_config(metplus_config): - extra_configs = [] - extra_configs.append(os.path.join(os.path.dirname(__file__), - 'tc_stat_conf.conf')) - return metplus_config(extra_configs) + # extra_configs = [] + # extra_configs.append(os.path.join(os.path.dirname(__file__), + # 'tc_stat_conf.conf')) + config = metplus_config + config.set('config', 'PROCESS_LIST', 'TCStat') + config.set('config', 'INIT_BEG', '20150301') + config.set('config', 'INIT_END', '20150304') + config.set('config', 'INIT_INCREMENT', '21600') + config.set('config', 'TC_STAT_INIT_BEG', '20170705') + config.set('config', 'TC_STAT_INIT_END', '20170901') + config.set('config', 'TC_STAT_INIT_HOUR', '00') + config.set('config', 'TC_STAT_JOB_ARGS', + ("-job summary -line_type TCMPR -column " + "'ABS(AMAX_WIND-BMAX_WIND)' " + "-dump_row {OUTPUT_BASE}/tc_stat/tc_stat_summary.tcst")) + config.set('config', 'TC_STAT_LOOKIN_DIR', + '{INPUT_BASE}/met_test/tc_pairs') + config.set('config', 'TC_STAT_OUTPUT_DIR', '{OUTPUT_BASE}/tc_stat') + return config def tc_stat_wrapper(metplus_config): @@ -265,7 +280,7 @@ def test_handle_jobs_create_parent_dir(metplus_config, jobs, init_dt, def test_get_config_file(metplus_config): fake_config_name = '/my/config/file' - config = metplus_config() + config = metplus_config default_config_file = os.path.join(config.getdir('PARM_BASE'), 'met_config', diff --git a/internal/tests/pytests/wrappers/user_script/test_user_script.py b/internal/tests/pytests/wrappers/user_script/test_user_script.py index b45fe88108..060f8f2a32 100644 --- a/internal/tests/pytests/wrappers/user_script/test_user_script.py +++ b/internal/tests/pytests/wrappers/user_script/test_user_script.py @@ -347,7 +347,7 @@ def set_run_type_info(config, run_type): @pytest.mark.wrapper def test_run_user_script_all_times(metplus_config, input_configs, run_types, expected_cmds): - config = metplus_config() + config = metplus_config config.set('config', 'DO_NOT_RUN_EXE', True) for key, value in input_configs.items(): diff --git a/metplus/wrappers/stat_analysis_wrapper.py b/metplus/wrappers/stat_analysis_wrapper.py index c1446c90ea..503869948d 100755 --- a/metplus/wrappers/stat_analysis_wrapper.py +++ b/metplus/wrappers/stat_analysis_wrapper.py @@ -191,7 +191,6 @@ def create_c_dict(self): # read any [FCST/OBS]_VAR_* variables if they are set c_dict['VAR_LIST'] = parse_var_list(self.config) - c_dict['MODEL_INFO_LIST'] = self._parse_model_info() # if MODEL_LIST was not set, populate it from the model info list @@ -666,6 +665,7 @@ def _build_stringsub_hours(self, list_name, config_dict, stringsub_dict): stringsub_dict[f'{generic_list}_end'] = ( stringsub_dict[f'{sub_name}_end'] ) + if (stringsub_dict[f'{generic_list}_beg'] == stringsub_dict[f'{generic_list}_end']): stringsub_dict[generic_list] = (