diff --git a/.codacy.yml b/.codacy.yml index 06a0ea342f..a4d0293fd5 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -21,5 +21,6 @@ engines: exclude_paths: [ 'doc/sphinx/**', 'esmvaltool/cmor/tables/**', - 'tests/**' + 'tests/**', + 'esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/test_configure.py' ] diff --git a/esmvaltool/utils/recipe_test_workflow/doc/source/user_guide/workflow.rst b/esmvaltool/utils/recipe_test_workflow/doc/source/user_guide/workflow.rst index ca52d19711..3bb4a9b7e5 100644 --- a/esmvaltool/utils/recipe_test_workflow/doc/source/user_guide/workflow.rst +++ b/esmvaltool/utils/recipe_test_workflow/doc/source/user_guide/workflow.rst @@ -20,7 +20,7 @@ The |RTW| performs the following steps: ``get_esmval`` :Description: - Either clones the latest versions of |ESMValTool| and |ESMValCore| from GitHub, + Either clones the latest versions of |ESMValTool| and |ESMValCore| from GitHub, or gets the latest container image from DockerHub and converts to a singularity image, depending on ``SITE``. :Runs on: @@ -35,13 +35,13 @@ The |RTW| performs the following steps: ``configure`` :Description: - Creates the |ESMValTool| user configuration file + Creates the |ESMValTool| user configuration file and validates it. :Runs on: Localhost :Executes: The ``configure.py`` script from the |Rose| app :Details: - ``configure`` should run at the start of each cycle after + ``configure`` should run at the start of each cycle after ``install_env_file`` has completed. ``process`` diff --git a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/__init__.py b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py index bdd4f419f0..380dfeb2df 100755 --- a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py +++ b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py @@ -4,6 +4,7 @@ import pprint import yaml +from esmvalcore.config._config_validators import ValidationError, _validators def main(): @@ -17,6 +18,9 @@ def main(): # 'configure' task. config_values = get_config_values_from_task_env() + # Validate the user config file content. + validate_user_config_file(config_values) + # Update the configuration from OS environment. user_config_path = os.environ["USER_CONFIG_PATH"] config_values["config_file"] = user_config_path @@ -80,6 +84,49 @@ def get_config_values_from_task_env(): return config_values_from_task_env +def validate_user_config_file(user_config_file_content): + """Validate a user config with ``ESMValCore.config._validators`` functions. + + Parameters + ---------- + user_config_file_content: dict + An ESMValTool user configuration file loaded in memory as a Python + dictionary. + + Raises + ------ + KeyError + If ``user_config_file_content`` includes a key for which there is no + validator listed in ``_validators``, + ValidationError + If any of the called validation functions raise a ValidationError. + """ + errors = [ + "There were validation errors in your user configuration file. See " + "details below.\n" + ] + for user_config_key, usr_config_value in user_config_file_content.items(): + try: + validatation_function = _validators[user_config_key] + except KeyError as err: + errors.append( + f'Key Error for {user_config_key.upper()}. May not be a valid ' + f'ESMValTool user configuration key\nERROR: {err}\n') + else: + try: + print(f'Validating {user_config_key.upper()} with value ' + f'"{usr_config_value}" using function ' + f'{validatation_function.__name__.upper()}.') + validatation_function(usr_config_value) + except ValidationError as err: + errors.append( + f'Validation error for {user_config_key.upper()} with ' + f'value "{usr_config_value}"\nERROR: {err}\n') + if len(errors) > 1: + raise ValidationError("\n".join(errors)) + print("All validation checks passed.") + + def write_yaml(file_path, contents): """Write ``contents`` to the YAML file ``file_path``. diff --git a/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/test_configure.py b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/test_configure.py new file mode 100644 index 0000000000..68decf7f7d --- /dev/null +++ b/esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/test_configure.py @@ -0,0 +1,76 @@ +import pytest +from bin.configure import validate_user_config_file +from esmvalcore.config._config_validators import ValidationError + + +def test_validate_user_config_file(): + mock_valid_config = { + "output_dir": "~/esmvaltool_output", + "auxiliary_data_dir": "~/auxiliary_data", + "search_esgf": "never", + "download_dir": "~/climate_data", + "max_parallel_tasks": None, + "log_level": "info", + "exit_on_warning": True, + "output_file_type": "png", + } + # No assert statement is needed. If the function call errors Pytest + # considers the test failed. + validate_user_config_file(mock_valid_config) + + +def test_validate_user_config_file_one_validation_error(): + mock_one_invalid_config = { + "output_dir": "~/esmvaltool_output", + "auxiliary_data_dir": "~/auxiliary_data", + "search_esgf": "never", + "download_dir": "~/climate_data", + "max_parallel_tasks": None, + "log_level": "info", + "exit_on_warning": 100, + "output_file_type": "png", + } + with pytest.raises( + ValidationError, + match='Validation error for EXIT_ON_WARNING with value "100"\n' + 'ERROR: Could not convert `100` to `bool`\n'): + validate_user_config_file(mock_one_invalid_config) + + +def test_validate_user_config_file_two_validation_errors(): + mock_two_invalids_config = { + "output_dir": 111, + "auxiliary_data_dir": "~/auxiliary_data", + "search_esgf": "never", + "download_dir": "~/climate_data", + "max_parallel_tasks": None, + "log_level": "info", + "exit_on_warning": 100, + "output_file_type": "png", + } + with pytest.raises( + ValidationError, + match='Validation error for OUTPUT_DIR with value "111"\nERROR: ' + 'Expected a path, but got 111\n\nValidation error for ' + 'EXIT_ON_WARNING with value "100"\nERROR: Could not convert `100` ' + 'to `bool`\n'): + validate_user_config_file(mock_two_invalids_config) + + +def test_validate_user_config_file_key_error(): + mock_one_key_error = { + "output_dir": "~/esmvaltool_output", + "auxiliary_data_dir": "~/auxiliary_data", + "search_esgf": "never", + "download_dir": "~/climate_data", + "one_rogue_field": 90210, + "max_parallel_tasks": None, + "log_level": "info", + "exit_on_warning": True, + "output_file_type": "png", + } + with pytest.raises( + ValidationError, + match="Key Error for ONE_ROGUE_FIELD. May not be a valid " + "ESMValTool user configuration key\nERROR: 'one_rogue_field'\n"): + validate_user_config_file(mock_one_key_error)