From bc05a1cd096af61b27197914789f6ab61d492f5e Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:40:55 +0200 Subject: [PATCH 01/19] Merge configuration object from multiple files (instead of one single file) (#2448) Co-authored-by: Bouwe Andela --- doc/api/esmvalcore.config.rst | 60 ++- doc/contributing.rst | 2 +- doc/develop/fixing_data.rst | 23 +- doc/quickstart/configure.rst | 444 ++++++++++-------- doc/quickstart/find_data.rst | 79 ++-- doc/quickstart/install.rst | 8 +- doc/quickstart/output.rst | 37 +- doc/quickstart/run.rst | 23 +- doc/recipe/overview.rst | 15 +- doc/recipe/preprocessor.rst | 11 +- esmvalcore/_main.py | 254 ++++++---- esmvalcore/_recipe/recipe.py | 20 +- esmvalcore/cmor/_fixes/icon/_base_fixes.py | 5 +- esmvalcore/config/__init__.py | 11 +- esmvalcore/config/_config.py | 5 +- esmvalcore/config/_config_object.py | 296 ++++++++++-- esmvalcore/config/_config_validators.py | 69 ++- .../configurations/defaults}/config-user.yml | 10 +- .../configurations/defaults/more_options.yml | 9 + esmvalcore/local.py | 4 +- tests/conftest.py | 24 + tests/integration/conftest.py | 13 - tests/integration/test_deprecated_config.py | 14 +- tests/integration/test_diagnostic_run.py | 72 ++- tests/integration/test_main.py | 175 ++++--- .../experimental/test_run_recipe.py | 9 +- tests/unit/config/test_config.py | 47 +- tests/unit/config/test_config_object.py | 267 +++++++++-- tests/unit/config/test_config_validator.py | 9 + tests/unit/config/test_esgf_pyclient.py | 2 +- tests/unit/conftest.py | 14 - tests/unit/main/test_esmvaltool.py | 72 ++- tests/unit/test_dataset.py | 4 +- 33 files changed, 1414 insertions(+), 693 deletions(-) rename esmvalcore/{ => config/configurations/defaults}/config-user.yml (96%) create mode 100644 esmvalcore/config/configurations/defaults/more_options.yml create mode 100644 tests/conftest.py delete mode 100644 tests/unit/conftest.py diff --git a/doc/api/esmvalcore.config.rst b/doc/api/esmvalcore.config.rst index 659d574509..9b01587263 100644 --- a/doc/api/esmvalcore.config.rst +++ b/doc/api/esmvalcore.config.rst @@ -1,13 +1,15 @@ +.. _api_configuration: + Configuration ============= This section describes the :py:class:`~esmvalcore.config` module. -Config -****** +CFG +*** -Configuration of ESMValCore/Tool is done via the :py:class:`~esmvalcore.config.Config` object. -The global configuration can be imported from the :py:mod:`esmvalcore.config` module as :py:data:`~esmvalcore.config.CFG`: +Configuration of ESMValCore/Tool is done via :py:data:`~esmvalcore.config.CFG` +object: .. code-block:: python @@ -16,7 +18,6 @@ The global configuration can be imported from the :py:mod:`esmvalcore.config` mo Config({'auxiliary_data_dir': PosixPath('/home/user/auxiliary_data'), 'compress_netcdf': False, 'config_developer_file': None, - 'config_file': PosixPath('/home/user/.esmvaltool/config-user.yml'), 'drs': {'CMIP5': 'default', 'CMIP6': 'default'}, 'exit_on_warning': False, 'log_level': 'info', @@ -30,9 +31,10 @@ The global configuration can be imported from the :py:mod:`esmvalcore.config` mo 'default': '~/default_inputpath'}, 'save_intermediary_cubes': False) -The parameters for the user configuration file are listed :ref:`here `. +All configuration parameters are listed :ref:`here `. -:py:data:`~esmvalcore.config.CFG` is essentially a python dictionary with a few extra functions, similar to :py:data:`matplotlib.rcParams`. +:py:data:`~esmvalcore.config.CFG` is essentially a python dictionary with a few +extra functions, similar to :py:data:`matplotlib.rcParams`. This means that values can be updated like this: .. code-block:: python @@ -41,8 +43,10 @@ This means that values can be updated like this: >>> CFG['output_dir'] PosixPath('/home/user/esmvaltool_output') -Notice that :py:data:`~esmvalcore.config.CFG` automatically converts the path to an instance of ``pathlib.Path`` and expands the home directory. -All values entered into the config are validated to prevent mistakes, for example, it will warn you if you make a typo in the key: +Notice that :py:data:`~esmvalcore.config.CFG` automatically converts the path +to an instance of :class:`pathlib.Path` and expands the home directory. +All values entered into the config are validated to prevent mistakes, for +example, it will warn you if you make a typo in the key: .. code-block:: python @@ -56,7 +60,8 @@ Or, if the value entered cannot be converted to the expected type: >>> CFG['max_parallel_tasks'] = '🐜' InvalidConfigParameter: Key `max_parallel_tasks`: Could not convert '🐜' to int -:py:class:`~esmvalcore.config.Config` is also flexible, so it tries to correct the type of your input if possible: +:py:data:`~esmvalcore.config.CFG` is also flexible, so it tries to correct the +type of your input if possible: .. code-block:: python @@ -64,35 +69,44 @@ Or, if the value entered cannot be converted to the expected type: >>> type(CFG['max_parallel_tasks']) int -By default, the config is loaded from the default location (``/home/user/.esmvaltool/config-user.yml``). -If it does not exist, it falls back to the default values. -to load a different file: +By default, the configuration is loaded from YAML files in the user's home +directory at ``~/.config/esmvaltool``. +If set, this can be overwritten with the ``ESMVALTOOL_CONFIG_DIR`` environment +variable. +Defaults for options that are not specified explicitly are listed :ref:`here +`. +To reload the current configuration object according to these rules, use: .. code-block:: python - >>> CFG.load_from_file('~/my-config.yml') + >>> CFG.reload() -Or to reload the current config: +To load the configuration object from custom directories, use: .. code-block:: python - >>> CFG.reload() + >>> dirs = ['my/default/config', 'my/custom/config'] + >>> CFG.load_from_dirs(dirs) Session ******* Recipes and diagnostics will be run in their own directories. -This behaviour can be controlled via the :py:data:`~esmvalcore.config.Session` object. -A :py:data:`~esmvalcore.config.Session` can be initiated from the global :py:class:`~esmvalcore.config.Config`. +This behavior can be controlled via the :py:data:`~esmvalcore.config.Session` +object. +A :py:data:`~esmvalcore.config.Session` must always be initiated from the +global :py:data:`~esmvalcore.config.CFG` object: .. code-block:: python >>> session = CFG.start_session(name='my_session') A :py:data:`~esmvalcore.config.Session` is very similar to the config. -It is also a dictionary, and copies all the keys from the :py:class:`~esmvalcore.config.Config`. -At this moment, ``session`` is essentially a copy of :py:data:`~esmvalcore.config.CFG`: +It is also a dictionary, and copies all the keys from the +:py:data:`~esmvalcore.config.CFG` object. +At this moment, ``session`` is essentially a copy of +:py:data:`~esmvalcore.config.CFG`: .. code-block:: python @@ -102,7 +116,8 @@ At this moment, ``session`` is essentially a copy of :py:data:`~esmvalcore.confi >>> print(session == CFG) # False False -A :py:data:`~esmvalcore.config.Session` also knows about the directories where the data will stored. +A :py:data:`~esmvalcore.config.Session` also knows about the directories where +the data will stored. The session name is used to prefix the directories. .. code-block:: python @@ -118,7 +133,8 @@ The session name is used to prefix the directories. >>> session.plot_dir /home/user/my_output_dir/my_session_20201203_155821/plots -Unlike the global configuration, of which only one can exist, multiple sessions can be initiated from :py:class:`~esmvalcore.config.Config`. +Unlike the global configuration, of which only one can exist, multiple sessions +can be initiated from :py:data:`~esmvalcore.config.CFG`. API reference diff --git a/doc/contributing.rst b/doc/contributing.rst index ee47974e90..a21a005e72 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -571,7 +571,7 @@ users. When making changes, e.g. to the :ref:`recipe format `, the :ref:`diagnostic script interface `, the public -:ref:`Python API `, or the :ref:`configuration file format `, +:ref:`Python API `, or the :ref:`configuration format `, keep in mind that this may affect many users. To keep the tool user friendly, try to avoid making changes that are not backward compatible, i.e. changes that require users to change their existing diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 174be1815d..68b6e27221 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -329,9 +329,9 @@ severity. From highest to lowest: Users can have control about which levels of issues are interpreted as errors, and therefore make the checker fail or warnings or debug messages. -For this purpose there is an optional command line option `--check-level` -that can take a number of values, listed below from the lowest level of -strictness to the highest: +For this purpose there is an optional :ref:`configuration option +` ``check_level`` that can take a number of values, listed +below from the lowest level of strictness to the highest: - ``ignore``: all issues, regardless of severity, will be reported as warnings. Checker will never fail. Use this at your own risk. @@ -375,8 +375,8 @@ To allow ESMValCore to locate the data files, use the following steps: - If you want to use the ``native6`` project (recommended for datasets whose input files can be easily moved to the usual ``native6`` directory - structure given by the ``rootpath`` in your :ref:`user configuration - file`; this is usually the case for native reanalysis/observational + structure given by the :ref:`configuration option ` + ``rootpath``; this is usually the case for native reanalysis/observational datasets): The entry ``native6`` of ``config-developer.yml`` should be complemented @@ -399,8 +399,8 @@ To allow ESMValCore to locate the data files, use the following steps: To find your native data (e.g., called ``MYDATA``) that is for example located in ``{rootpath}/MYDATA/amip/run1/42-0/atm/run1_1979.nc`` - (``{rootpath}`` is ESMValTool's ``rootpath`` for the project ``native6`` - defined in your :ref:`user configuration file`), use the following dataset + (``{rootpath}`` is ESMValTool's ``rootpath`` :ref:`configuration option + ` for the project ``native6``), use the following dataset entry in your recipe .. code-block:: yaml @@ -408,8 +408,8 @@ To allow ESMValCore to locate the data files, use the following steps: datasets: - {project: native6, dataset: MYDATA, exp: amip, simulation: run1, version: 42-0, type: atm} - and make sure to use the following DRS for the project ``native6`` in your - :ref:`user configuration file`: + and make sure to use the following :ref:`configuration option + ` ``drs``: .. code-block:: yaml @@ -437,9 +437,8 @@ To allow ESMValCore to locate the data files, use the following steps: To find your ICON data that is for example located in files like ``{rootpath}/amip/amip_atm_2d_ml_20000101T000000Z.nc`` (``{rootpath}`` is - ESMValTool ``rootpath`` for the project ``ICON`` defined in your - :ref:`user configuration file`), use the following dataset entry in your - recipe: + ESMValCore's :ref:`configuration option ` ``rootpath`` for + the project ``ICON``), use the following dataset entry in your recipe: .. code-block:: yaml diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 37e6efd230..c65fdbd1c5 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -1,203 +1,273 @@ .. _config: -******************* -Configuration files -******************* +************* +Configuration +************* + +.. _config_overview: Overview ======== -There are several configuration files in ESMValCore: +Similar to `Dask `__, +ESMValCore provides one single configuration object that consists of a single +nested dictionary for its configuration. -* ``config-user.yml``: sets a number of user-specific options like desired - graphical output format, root paths to data, etc.; -* ``config-developer.yml``: sets a number of standardized file-naming and paths - to data formatting; +.. note:: -and one configuration file which is distributed with ESMValTool: + In v2.12.0, a redesign process of ESMValTool/Core's configuration started. + Its main aim is to simplify the configuration by moving from many different + configuration files for individual components to one configuration object + that consists of a single nested dictionary (similar to `Dask's configuration + `__). + This change will not be implemented in one large pull request but rather in a + step-by-step procedure. + Thus, the configuration might appear inconsistent until this redesign is + finished. + A detailed plan for this new configuration is outlined in :issue:`2371`. -* ``config-references.yml``: stores information on diagnostic and recipe authors and - scientific journals references; -.. _user configuration file: +.. _config_for_cli: -User configuration file -======================= +Specify configuration for ``esmvaltool`` command line tool +========================================================== +When running recipes via the :ref:`command line `, configuration +options can be specified via YAML files and command line arguments. -The ``config-user.yml`` configuration file contains all the global level -information needed by ESMValCore. It can be reused as many times the user needs -to before changing any of the options stored in it. This file is essentially -the gateway between the user and the machine-specific instructions to -``esmvaltool``. By default, esmvaltool looks for it in the home directory, -inside the ``.esmvaltool`` folder. -Users can get a copy of this file with default values by running - -.. code-block:: bash +.. _config_yaml_files: - esmvaltool config get-config-user --path=${TARGET_FOLDER} +YAML files +---------- -If the option ``--path`` is omitted, the file will be created in -``${HOME}/.esmvaltool`` +:ref:`Configuration options ` can be specified via YAML files +(i.e., ``*.yaml`` and ``*.yml``). -The following shows the default settings from the ``config-user.yml`` file -with explanations in a commented line above each option. If only certain values -are allowed for an option, these are listed after ``---``. The option in square -brackets is the default value, i.e., the one that is used if this option is -omitted in the file. +A file could look like this (for example, located at +``~/.config/esmvaltool/config.yml``): .. code-block:: yaml - # Destination directory where all output will be written - # Includes log files and performance stats. output_dir: ~/esmvaltool_output + search_esgf: when_missing + download_dir: ~/downloaded_data + +These files can live in any of the following locations: + +1. The directory specified via the ``--config_dir`` command line argument. - # Auxiliary data directory - # Used by some recipes to look for additional datasets. - auxiliary_data_dir: ~/auxiliary_data - - # Automatic data download from ESGF --- [never]/when_missing/always - # Use automatic download of missing CMIP3, CMIP5, CMIP6, CORDEX, and obs4MIPs - # data from ESGF. ``never`` disables this feature, which is useful if you are - # working on a computer without an internet connection, or if you have limited - # disk space. ``when_missing`` enables the automatic download for files that - # are not available locally. ``always`` will always check ESGF for the latest - # version of a file, and will only use local files if they correspond to that - # latest version. - search_esgf: never - - # Directory for storing downloaded climate data - # Make sure to use a directory where you can store multiple GBs of data. Your - # home directory on a HPC is usually not suited for this purpose, so please - # change the default value in this case! - download_dir: ~/climate_data - - # Rootpaths to the data from different projects - # This default setting will work if files have been downloaded by ESMValTool - # via ``search_esgf``. Lists are also possible. For site-specific entries, - # see the default ``config-user.yml`` file that can be installed with the - # command ``esmvaltool config get_config_user``. For each project, this can - # be either a single path or a list of paths. Comment out these when using a - # site-specific path. - rootpath: - default: ~/climate_data - - # Directory structure for input data --- [default]/ESGF/BADC/DKRZ/ETHZ/etc. - # This default setting will work if files have been downloaded by ESMValTool - # via ``search_esgf``. See ``config-developer.yml`` for definitions. Comment - # out/replace as per needed. - drs: - CMIP3: ESGF - CMIP5: ESGF - CMIP6: ESGF - CORDEX: ESGF - obs4MIPs: ESGF - - # Run at most this many tasks in parallel --- [null]/1/2/3/4/... - # Set to ``null`` to use the number of available CPUs. If you run out of - # memory, try setting max_parallel_tasks to ``1`` and check the amount of - # memory you need for that by inspecting the file ``run/resource_usage.txt`` in - # the output directory. Using the number there you can increase the number of - # parallel tasks again to a reasonable number for the amount of memory - # available in your system. - max_parallel_tasks: null - - # Log level of the console --- debug/[info]/warning/error - # For much more information printed to screen set log_level to ``debug``. - log_level: info - - # Exit on warning --- true/[false] - # Only used in NCL diagnostic scripts. - exit_on_warning: false - - # Plot file format --- [png]/pdf/ps/eps/epsi - output_file_type: png - - # Remove the ``preproc`` directory if the run was successful --- [true]/false - # By default this option is set to ``true``, so all preprocessor output files - # will be removed after a successful run. Set to ``false`` if you need those files. - remove_preproc_dir: true - - # Use netCDF compression --- true/[false] - compress_netcdf: false - - # Save intermediary cubes in the preprocessor --- true/[false] - # Setting this to ``true`` will save the output cube from each preprocessing - # step. These files are numbered according to the preprocessing order. - save_intermediary_cubes: false - - # Use a profiling tool for the diagnostic run --- [false]/true - # A profiler tells you which functions in your code take most time to run. - # For this purpose we use ``vprof``, see below for notes. Only available for - # Python diagnostics. - profile_diagnostic: false - - # Path to custom ``config-developer.yml`` file - # This can be used to customise project configurations. See - # ``config-developer.yml`` for an example. Set to ``null`` to use the default. - config_developer_file: null - -The ``search_esgf`` setting can be used to disable or enable automatic -downloads from ESGF. -If ``search_esgf`` is set to ``never``, the tool does not download any data -from the ESGF. -If ``search_esgf`` is set to ``when_missing``, the tool will download any CMIP3, -CMIP5, CMIP6, CORDEX, and obs4MIPs data that is required to run a recipe but -not available locally and store it in ``download_dir`` using the ``ESGF`` -directory structure defined in the :ref:`config-developer`. -If ``search_esgf`` is set to ``always``, the tool will first check the ESGF for -the needed data, regardless of any local data availability; if the data found -on ESGF is newer than the local data (if any) or the user specifies a version -of the data that is available only from the ESGF, then that data will be -downloaded; otherwise, local data will be used. - -The ``auxiliary_data_dir`` setting is the path to place any required -additional auxiliary data files. This is necessary because certain -Python toolkits, such as cartopy, will attempt to download data files at run -time, typically geographic data files such as coastlines or land surface maps. -This can fail if the machine does not have access to the wider internet. This -location allows the user to specify where to find such files if they can not be -downloaded at runtime. The example user configuration file already contains two valid -locations for ``auxiliary_data_dir`` directories on CEDA-JASMIN and DKRZ, and a number -of such maps and shapefiles (used by current diagnostics) are already there. You will -need ``esmeval`` group workspace membership to access the JASMIN one (see -`instructions `_ -how to gain access to the group workspace. +2. The user configuration directory: by default ``~/.config/esmvaltool``, but + this can be changed with the ``ESMVALTOOL_CONFIG_DIR`` environment variable. + If ``~/.config/esmvaltool`` does not exist, this will be silently ignored. + +ESMValCore searches for all YAML files within each of these directories and +merges them together using :func:`dask.config.collect`. +This properly considers nested objects; see :func:`dask.config.update` for +details. +Preference follows the order in the list above (i.e., the directory specified +via command line argument is preferred over the user configuration directory). +Within a directory, files are sorted alphabetically, and later files (e.g., +``z.yml``) will take precedence over earlier files (e.g., ``a.yml``). .. warning:: - This setting is not for model or observational datasets, rather it is for - extra data files such as shapefiles or other data sources needed by the diagnostics. + ESMValCore will read **all** YAML files in these configuration directories. + Thus, other YAML files in this directory which are not valid configuration + files (like the old ``config-developer.yml`` files) will lead to errors. + Make sure to move these files to a different directory. -The ``profile_diagnostic`` setting triggers profiling of Python diagnostics, -this will tell you which functions in the diagnostic took most time to run. -For this purpose we use `vprof `_. -For each diagnostic script in the recipe, the profiler writes a ``.json`` file -that can be used to plot a -`flame graph `__ -of the profiling information by running +To get a copy of the default configuration file, you can run .. code-block:: bash - vprof --input-file esmvaltool_output/recipe_output/run/diagnostic/script/profile.json + esmvaltool config get_config_user --path=/target/file.yml -Note that it is also possible to use vprof to understand other resources used -while running the diagnostic, including execution time of different code blocks -and memory usage. +If the option ``--path`` is omitted, the file will be copied to +``~/.config/esmvaltool/config-user.yml``. -A detailed explanation of the data finding-related sections of the -``config-user.yml`` (``rootpath`` and ``drs``) is presented in the -:ref:`data-retrieval` section. This section relates directly to the data -finding capabilities of ESMValCore and are very important to be understood by -the user. -.. note:: +Command line arguments +---------------------- + +All :ref:`configuration options ` can also be given as command +line arguments to the ``esmvaltool`` executable. + +Example: + +.. code-block:: bash + + esmvaltool run --search_esgf=when_missing --max_parallel_tasks=2 /path/to/recipe.yml + +Options given via command line arguments will always take precedence over +options specified via YAML files. + + +.. _config_for_api: + +Specify/access configuration for Python API +=========================================== + +When running recipes with the :ref:`experimental Python API +`, configuration options can be specified and accessed via +the :py:data:`~esmvalcore.config.CFG` object. +For example: + +.. code-block:: python + + >>> from esmvalcore.config import CFG + >>> CFG['output_dir'] = '~/esmvaltool_output' + >>> CFG['output_dir'] + PosixPath('/home/user/esmvaltool_output') + +This will also consider YAML configuration files in the user configuration +directory (by default ``~/.config/esmvaltool``, but this can be changed with +the ``ESMVALTOOL_CONFIG_DIR`` environment variable). + +More information about this can be found :ref:`here `. + + +.. _config_options: + +Configuration options +===================== + +Note: the following entries use Python syntax. +For example, Python's ``None`` is YAML's ``null``, Python's ``True`` is YAML's +``true``, and Python's ``False`` is YAML's ``false``. + ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| Option | Description | Type | Default value | ++===============================+========================================+=============================+========================================+ +| ``auxiliary_data_dir`` | Directory where auxiliary data is | :obj:`str` | ``~/auxiliary_data`` | +| | stored [#f1]_ | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``check_level`` | Sensitivity of the CMOR check | :obj:`str` | ``default`` | +| | (``debug``, ``strict``, ``default`` | | | +| | ``relaxed``, ``ignore``), see | | | +| | :ref:`cmor_check_strictness` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``compress_netcdf`` | Use netCDF compression | :obj:`bool` | ``False`` | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``config_developer_file`` | Path to custom | :obj:`str` | ``None`` (default file) | +| | :ref:`config-developer` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``diagnostics`` | Only run the selected diagnostics from | :obj:`list` or :obj:`str` | ``None`` (all diagnostics) | +| | the recipe, see :ref:`running` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``download_dir`` | Directory where downloaded data will | :obj:`str` | ``~/climate_data`` | +| | be stored [#f4]_ | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``drs`` | Directory structure for input data | :obj:`dict` | ``{CMIP3: ESGF, CMIP5: ESGF, CMIP6: | +| | [#f2]_ | | ESGF, CORDEX: ESGF, obs4MIPs: ESGF}`` | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``exit_on_warning`` | Exit on warning (only used in NCL | :obj:`bool` | ``False`` | +| | diagnostic scripts) | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``extra_facets_dir`` | Additional custom directory for | :obj:`list` of :obj:`str` | ``[]`` | +| | :ref:`extra_facets` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``log_level`` | Log level of the console (``debug``, | :obj:`str` | ``info`` | +| | ``info``, ``warning``, ``error``) | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``max_datasets`` | Maximum number of datasets to use, see | :obj:`int` | ``None`` (all datasets from recipe) | +| | :ref:`running` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``max_parallel_tasks`` | Maximum number of parallel processes, | :obj:`int` | ``None`` (number of available CPUs) | +| | see also :ref:`task_priority` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``max_years`` | Maximum number of years to use, see | :obj:`int` | ``None`` (all years from recipe) | +| | :ref:`running` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``output_dir`` | Directory where all output will be | :obj:`str` | ``~/esmvaltool_output`` | +| | written, see :ref:`outputdata` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``output_file_type`` | Plot file type | :obj:`str` | ``png`` | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``profile_diagnostic`` | Use a profiling tool for the | :obj:`bool` | ``False`` | +| | diagnostic run [#f3]_ | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``remove_preproc_dir`` | Remove the ``preproc`` directory if | :obj:`bool` | ``True`` | +| | the run was successful, see also | | | +| | :ref:`preprocessed_datasets` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``resume_from`` | Resume previous run(s) by using | :obj:`list` of :obj:`str` | ``[]`` | +| | preprocessor output files from these | | | +| | output directories, see :ref:`running` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``rootpath`` | Rootpaths to the data from different | :obj:`dict` | ``{default: ~/climate_data}`` | +| | projects [#f2]_ | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``run_diagnostic`` | Run diagnostic scripts, see | :obj:`bool` | ``True`` | +| | :ref:`running` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``save_intermediary_cubes`` | Save intermediary cubes from the | :obj:`bool` | ``False`` | +| | preprocessor, see also | | | +| | :ref:`preprocessed_datasets` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``search_esgf`` | Automatic data download from ESGF | :obj:`str` | ``never`` | +| | (``never``, ``when_missing``, | | | +| | ``always``) [#f4]_ | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ +| ``skip_nonexistent`` | Skip non-existent datasets, see | :obj:`bool` | ``False`` | +| | :ref:`running` | | | ++-------------------------------+----------------------------------------+-----------------------------+----------------------------------------+ + +.. [#f1] The ``auxiliary_data_dir`` setting is the path to place any required + additional auxiliary data files. + This is necessary because certain Python toolkits, such as cartopy, will + attempt to download data files at run time, typically geographic data files + such as coastlines or land surface maps. + This can fail if the machine does not have access to the wider internet. + This location allows the user to specify where to find such files if they + can not be downloaded at runtime. + The example configuration file already contains two valid locations for + ``auxiliary_data_dir`` directories on CEDA-JASMIN and DKRZ, and a number of + such maps and shapefiles (used by current diagnostics) are already there. + You will need ``esmeval`` group workspace membership to access the JASMIN + one (see `instructions + `_ + how to gain access to the group workspace. + + .. warning:: + + This setting is not for model or observational datasets, rather it is + for extra data files such as shapefiles or other data sources needed by + the diagnostics. +.. [#f2] A detailed explanation of the data finding-related options ``drs`` + and ``rootpath`` is presented in the :ref:`data-retrieval` section. + These sections relate directly to the data finding capabilities of + ESMValCore and are very important to be understood by the user. +.. [#f3] The ``profile_diagnostic`` setting triggers profiling of Python + diagnostics, this will tell you which functions in the diagnostic took most + time to run. + For this purpose we use `vprof `_. + For each diagnostic script in the recipe, the profiler writes a ``.json`` + file that can be used to plot a `flame graph + `__ of the profiling + information by running + + .. code-block:: bash + + vprof --input-file esmvaltool_output/recipe_output/run/diagnostic/script/profile.json + + Note that it is also possible to use vprof to understand other resources + used while running the diagnostic, including execution time of different + code blocks and memory usage. +.. [#f4] The ``search_esgf`` setting can be used to disable or enable automatic + downloads from ESGF. + If ``search_esgf`` is set to ``never``, the tool does not download any data + from the ESGF. + If ``search_esgf`` is set to ``when_missing``, the tool will download any + CMIP3, CMIP5, CMIP6, CORDEX, and obs4MIPs data that is required to run a + recipe but not available locally and store it in ``download_dir`` using the + ``ESGF`` directory structure defined in the :ref:`config-developer`. + If ``search_esgf`` is set to ``always``, the tool will first check the ESGF + for the needed data, regardless of any local data availability; if the data + found on ESGF is newer than the local data (if any) or the user specifies a + version of the data that is available only from the ESGF, then that data + will be downloaded; otherwise, local data will be used. - You can choose your ``config-user.yml`` file at run time, so you could have several of - them available with different purposes. One for a formalised run, another for - debugging, etc. You can even provide any config user value as a run flag - ``--argument_name argument_value`` .. _config-dask: @@ -397,7 +467,7 @@ Configuring Dask for debugging For debugging purposes, it can be useful to disable all parallelism, as this will often result in more clear error messages. This can be achieved by -settings ``max_parallel_tasks: 1`` in config-user.yml, +setting ``max_parallel_tasks: 1`` in the configuration, commenting out or removing all content of ``~/.esmvaltool/dask.yml``, and creating a file called ``~/.config/dask/dask.yml`` with the following content: @@ -419,12 +489,10 @@ ESGF configuration The ``esmvaltool run`` command can automatically download the files required to run a recipe from ESGF for the projects CMIP3, CMIP5, CMIP6, CORDEX, and obs4MIPs. -The downloaded files will be stored in the ``download_dir`` specified in the -:ref:`user configuration file`. -To enable automatic downloads from ESGF, set ``search_esgf: when_missing`` or -``search_esgf: always`` in the :ref:`user configuration file`, or provide the -corresponding command line arguments ``--search_esgf=when_missing`` or -``--search_esgf=always`` when running the recipe. +The downloaded files will be stored in the directory specified via the +:ref:`configuration option ` ``download_dir``. +To enable automatic downloads from ESGF, use the :ref:`configuration options +` ``search_esgf: when_missing`` or ``search_esgf: always``. .. note:: @@ -534,22 +602,27 @@ out by CMOR and DRS. For a detailed description of these standards and their adoption in ESMValCore, we refer the user to :ref:`CMOR-DRS` section where we relate these standards to the data retrieval mechanism of the ESMValCore. -By default, esmvaltool looks for it in the home directory, -inside the '.esmvaltool' folder. - Users can get a copy of this file with default values by running .. code-block:: bash - esmvaltool config get-config-developer --path=${TARGET_FOLDER} + esmvaltool config get_config_developer --path=${TARGET_FOLDER} If the option ``--path`` is omitted, the file will be created in -```${HOME}/.esmvaltool``. +``~/.esmvaltool``. .. note:: - Remember to change your config-user file if you want to use a custom - config-developer. + Remember to change the configuration option ``config_developer_file`` if you + want to use a custom config developer file. + +.. warning:: + + For now, make sure that the custom ``config-developer.yml`` is **not** saved + in the ESMValTool/Core configuration directories (see + :ref:`config_yaml_files` for details). + This will change in the future due to the :ref:`redesign of ESMValTool/Core's + configuration `. Example of the CMIP6 project configuration: @@ -894,16 +967,15 @@ to support a particular use-case within the ESMValCore project, they will be provided in the sub-folder `extra_facets` inside the package :mod:`esmvalcore.config`. If they are used from the user side, they can be either placed in `~/.esmvaltool/extra_facets` or in any other directory of the users -choosing. In that case this directory must be added to the `config-user.yml` -file under the `extra_facets_dir` setting, which can take a single directory or -a list of directories. +choosing. In that case, the configuration option ``extra_facets_dir`` must be +set, which can take a single directory or a list of directories. The order in which the directories are searched is 1. The internal directory `esmvalcore.config/extra_facets` 2. The default user directory `~/.esmvaltool/extra_facets` -3. The custom user directories in the order in which they are given in - `config-user.yml`. +3. The custom user directories given by the configuration option + ``extra_facets_dir`` The extra facets files within each of these directories are processed in lexicographical order according to their file name. diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index e9077884f2..b7708fd95f 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -8,7 +8,7 @@ Overview ======== Data discovery and retrieval is the first step in any evaluation process; ESMValCore uses a `semi-automated` data finding mechanism with inputs from both -the user configuration file and the recipe file: this means that the user will +the configuration and the recipe file: this means that the user will have to provide the tool with a set of parameters related to the data needed and once these parameters have been provided, the tool will automatically find the right data. We will detail below the data finding and retrieval process and @@ -105,8 +105,8 @@ Supported native reanalysis/observational datasets The following native reanalysis/observational datasets are supported under the ``native6`` project. To use these datasets, put the files containing the data in the directory that -you have configured for the ``native6`` project in your :ref:`user -configuration file`, in a subdirectory called +you have :ref:`configured ` for the ``rootpath`` of the +``native6`` project, in a subdirectory called ``Tier{tier}/{dataset}/{version}/{frequency}/{short_name}``. Replace the items in curly braces by the values used in the variable/dataset definition in the :ref:`recipe `. @@ -183,7 +183,7 @@ The default naming conventions for input directories and files for CESM are * input files: ``{case}.{scomp}.{type}.{string}*nc`` as configured in the :ref:`config-developer file ` (using the -default DRS ``drs: default`` in the :ref:`user configuration file`). +:ref:`configuration option ` ``drs: default``). More information about CESM naming conventions are given `here `__. @@ -262,7 +262,7 @@ The default naming conventions for input directories and files for EMAC are * input files: ``{exp}*{channel}{postproc_flag}.nc`` as configured in the :ref:`config-developer file ` (using the -default DRS ``drs: default`` in the :ref:`user configuration file`). +:ref:`configuration option ` ``drs: default``). Thus, example dataset entries could look like this: @@ -335,7 +335,7 @@ The default naming conventions for input directories and files for ICON are * input files: ``{exp}_{var_type}*.nc`` as configured in the :ref:`config-developer file ` (using the -default DRS ``drs: default`` in the :ref:`user configuration file`). +:ref:`configuration option ` ``drs: default``). Thus, example dataset entries could look like this: @@ -383,11 +383,10 @@ is always disabled. Usually, ESMValCore will need the corresponding ICON grid file of your simulation to work properly (examples: setting latitude/longitude coordinates if these are not yet present, UGRIDization [see below], etc.). -This grid file can either be specified as absolute or relative (to -``auxiliary_data_dir`` as defined in the :ref:`user configuration file`) path -with the facet ``horizontal_grid`` in the recipe or the extra facets (see -below), or retrieved automatically from the `grid_file_uri` attribute of the -input files. +This grid file can either be specified as absolute or relative (to the +:ref:`configuration option ` ``auxiliary_data_dir``) path with +the facet ``horizontal_grid`` in the recipe or the extra facets (see below), or +retrieved automatically from the `grid_file_uri` attribute of the input files. In the latter case, ESMValCore first searches the input directories specified for ICON for a grid file with that name, and if that was not successful, tries to download the file and cache it. @@ -417,8 +416,8 @@ If neither of these variables are available in the input files, it is possible to specify the location of files that include the corresponding `zg` or `zghalf` variables with the facets ``zg_file`` and/or ``zghalf_file`` in the recipe or the extra facets. -The paths to these files can be specified absolute or relative (to -``auxiliary_data_dir`` as defined in the :ref:`user configuration file`). +The paths to these files can be specified absolute or relative (to the +:ref:`configuration option ` ``auxiliary_data_dir``). .. hint:: @@ -453,10 +452,8 @@ Supported keys for extra facets are: Key Description Default value if not specified =================== ================================ =================================== ``horizontal_grid`` Absolute or relative (to If not given, use file attribute - ``auxiliary_data_dir`` defined ``grid_file_uri`` to retrieve ICON - in the grid file (see details above) - :ref:`user configuration file`) - path to the ICON grid file + ``auxiliary_data_dir``) ``grid_file_uri`` to retrieve ICON + path to the ICON grid file grid file (see details above) ``latitude`` Standard name of the latitude ``latitude`` coordinate in the raw input file @@ -479,17 +476,13 @@ Key Description Default value if not specif variable in the raw input in extra facets or recipe if file default DRS is used) ``zg_file`` Absolute or relative (to If possible, use `zg` variable - ``auxiliary_data_dir`` defined provided by the raw input file - in the - :ref:`user configuration file`) - path to the input file that - contains `zg` + ``auxiliary_data_dir``) path to provided by the raw input file + the the input file that contains + `zg` ``zghalf_file`` Absolute or relative (to If possible, use `zghalf` variable - ``auxiliary_data_dir`` defined provided by the raw input file - in the - :ref:`user configuration file`) - path to the input file that - contains `zghalf` + ``auxiliary_data_dir``) path to provided by the raw input file + the the input file that contains + `zghalf` =================== ================================ =================================== .. hint:: @@ -630,20 +623,18 @@ retrieval parameters is explained below. Enabling automatic downloads from the ESGF ------------------------------------------ -To enable automatic downloads from ESGF, set ``search_esgf: when_missing`` (use -local files whenever possible) or ``search_esgf: always`` (always search ESGF -for latest version of files and only use local data if it is the latest -version) in the :ref:`user configuration file`, or provide the corresponding -command line arguments ``--search_esgf=when_missing`` or -``--search_esgf=always`` when running the recipe. -The files will be stored in the ``download_dir`` set in -the :ref:`user configuration file`. +To enable automatic downloads from ESGF, use the :ref:`configuration option +` ``search_esgf: when_missing`` (use local files +whenever possible) or ``search_esgf: always`` (always search ESGF for latest +version of files and only use local data if it is the latest version). +The files will be stored in the directory specified via the :ref:`configuration +option ` ``download_dir``. Setting the correct root paths ------------------------------ The first step towards providing ESMValCore the correct set of parameters for -data retrieval is setting the root paths to the data. This is done in the user -configuration file ``config-user.yml``. The two sections where the user will +data retrieval is setting the root paths to the data. This is done in the +configuration. The two sections where the user will set the paths are ``rootpath`` and ``drs``. ``rootpath`` contains pointers to ``CMIP``, ``OBS``, ``default`` and ``RAWOBS`` root paths; ``drs`` sets the type of directory structure the root paths are structured by. It is important to @@ -651,10 +642,8 @@ first discuss the ``drs`` parameter: as we've seen in the previous section, the DRS as a standard is used for both file naming conventions and for directory structures. -.. _config-user-drs: - -Explaining ``config-user/drs: CMIP5:`` or ``config-user/drs: CMIP6:`` ---------------------------------------------------------------------- +Explaining ``drs: CMIP5:`` or ``drs: CMIP6:`` +--------------------------------------------- Whereas ESMValCore will by default use the CMOR standard for file naming (please refer above), by setting the ``drs`` parameter the user tells the tool what type of root paths they need the data from, e.g.: @@ -697,10 +686,10 @@ The names of the directories trees that can be used under `drs` are defined in versions of the same file because the files typically have the same name for different versions. -.. _config-user-rootpath: +.. _config_option_rootpath: -Explaining ``config-user/rootpath:`` ------------------------------------- +Explaining ``rootpath:`` +------------------------ ``rootpath`` identifies the root directory for different data types (``ROOT`` as we used it above): @@ -786,7 +775,7 @@ The data finding feature will use this information to find data for **all** the Recap and example ================= Let us look at a practical example for a recap of the information above: -suppose you are using a ``config-user.yml`` that has the following entries for +suppose you are using configuration that has the following entries for data finding: .. code-block:: yaml diff --git a/doc/quickstart/install.rst b/doc/quickstart/install.rst index 0a821a0df9..c190f35e1e 100644 --- a/doc/quickstart/install.rst +++ b/doc/quickstart/install.rst @@ -103,10 +103,10 @@ For example, the following command would run a recipe .. code-block:: bash - docker run -e HOME -v "$HOME":"$HOME" -v /data:/data esmvalgroup/esmvalcore:stable -c ~/config-user.yml ~/recipes/recipe_example.yml + docker run -e HOME -v "$HOME":"$HOME" -v /data:/data esmvalgroup/esmvalcore:stable ~/recipes/recipe_example.yml with the environmental variable ``$HOME`` available inside the container and the data -in the directories ``$HOME`` and ``/data``, so these can be used to find the configuration file, recipe, and data. +in the directories ``$HOME`` and ``/data``, so these can be used to find the configuration, recipe, and data. It might be useful to define a `bash alias `_ @@ -131,7 +131,7 @@ following command .. code-block:: bash - singularity run docker://esmvalgroup/esmvalcore:stable -c ~/config-user.yml ~/recipes/recipe_example.yml + singularity run docker://esmvalgroup/esmvalcore:stable ~/recipes/recipe_example.yml Note that the container does not see the data available in the host by default. You can make host data available with ``-B /path:/path/in/container``. @@ -158,7 +158,7 @@ To run the container using the image file ``esmvalcore.sif`` use: .. code-block:: bash - singularity run esmvalcore.sif -c ~/config-user.yml ~/recipes/recipe_example.yml + singularity run esmvalcore.sif ~/recipes/recipe_example.yml .. _installation-from-source: diff --git a/doc/quickstart/output.rst b/doc/quickstart/output.rst index c30e59c046..2698456c6b 100644 --- a/doc/quickstart/output.rst +++ b/doc/quickstart/output.rst @@ -3,9 +3,10 @@ Output ****** -ESMValTool automatically generates a new output directory with every run. The -location is determined by the output_dir option in the config-user.yml file, -the recipe name, and the date and time, using the the format: ``YYYYMMDD_HHMMSS``. +ESMValTool automatically generates a new output directory with every run. +The location is determined by the ``output_dir`` :ref:`configuration option +`, the recipe name, and the date and time, using the the +format: ``YYYYMMDD_HHMMSS``. For instance, a typical output location would be: ``output_directory/recipe_ocean_amoc_20190118_1027/`` @@ -27,6 +28,8 @@ A summary of the output is produced in the file: ``index.html`` +.. _preprocessed_datasets: + Preprocessed datasets ===================== @@ -34,13 +37,13 @@ The preprocessed datasets will be stored to the preproc/ directory. Each variable in each diagnostic will have its own the `metadata.yml`_ interface files saved in the preproc directory. -If the option ``save_intermediary_cubes`` is set to ``true`` in the -config-user.yml file, then the intermediary cubes will also be saved here. -This option is set to false in the default ``config-user.yml`` file. +If the :ref:`configuration option ` ``save_intermediary_cubes`` +is set to ``true``, then the intermediary cubes will also be saved here +(default: ``false``). -If the option ``remove_preproc_dir`` is set to ``true`` in the config-user.yml -file, then the preproc directory will be deleted after the run completes. This -option is set to true in the default ``config-user.yml`` file. +If the :ref:`configuration option ` ``remove_preproc_dir`` is +set to ``true``, then the preproc directory will be deleted after the run +completes (default: ``true``). Run @@ -70,9 +73,9 @@ the results should be saved to the work directory. Plots ===== -The plots directory is where diagnostics save their output figures. These -plots are saved in the format requested by the option `output_file_type` in the -config-user.yml file. +The plots directory is where diagnostics save their output figures. These +plots are saved in the format requested by the :ref:`configuration option +` ``output_file_type``. Settings.yml @@ -82,10 +85,10 @@ The settings.yml file is automatically generated by ESMValTool. Each diagnostic will produce a unique settings.yml file. The settings.yml file passes several global level keys to diagnostic scripts. -This includes several flags from the config-user.yml file (such as -'log_level'), several paths which are specific to the -diagnostic being run (such as 'plot_dir' and 'run_dir') and the location on -disk of the metadata.yml file (described below). +This includes several flags from the global configuration (such as +``log_level``), several paths which are specific to the +diagnostic being run (such as ``plot_dir`` and ``run_dir``) and the location on +disk of the ``metadata.yml`` file (described below). .. code-block:: yaml @@ -113,7 +116,7 @@ The metadata.yml files is automatically generated by ESMValTool. Along with the settings.yml file, it passes all the paths, boolean flags, and additional arguments that your diagnostic needs to know in order to run. -The metadata is loaded from cfg as a dictionairy object in python diagnostics. +The metadata is loaded from cfg as a dictionary object in python diagnostics. Here is an example metadata.yml file: diff --git a/doc/quickstart/run.rst b/doc/quickstart/run.rst index fec474f290..61709bc778 100644 --- a/doc/quickstart/run.rst +++ b/doc/quickstart/run.rst @@ -46,24 +46,27 @@ and run that. To work with installed recipes, the ESMValTool package provides the ``esmvaltool recipes`` command, see :ref:`esmvaltool:recipes_command`. -If the configuration file is not in the default location -``~/.esmvaltool/config-user.yml``, you can pass its path explicitly: +By default, ESMValTool searches for :ref:`configuration files +` in ``~/.config/esmvaltool``. +If you'd like to use a custom location, you can specify this via the +``--config_dir`` command line argument: .. code:: bash - esmvaltool run --config_file /path/to/config-user.yml recipe_example.yml + esmvaltool run --config_dir /path/to/custom_config recipe_example.yml -It is also possible to explicitly change values from the config file using flags: +It is also possible to explicitly set configuration options with command line +arguments: .. code:: bash esmvaltool run --argument_name argument_value recipe_example.yml -To automatically download the files required to run a recipe from ESGF, set -``search_esgf`` to ``when_missing`` (use local files whenever possible) or -``always`` (always search ESGF for latest version of files and only use local -data if it is the latest version) in the :ref:`user configuration file` or run -the tool with the corresponding commands +To automatically download the files required to run a recipe from ESGF, use the +:ref:`configuration option ` ``search_esgf=when_missing`` (use +local files whenever possible) or ``search_esgf=always`` (always search ESGF +for latest version of files and only use local data if it is the latest +version): .. code:: bash @@ -123,7 +126,7 @@ To run only the preprocessor tasks from a recipe, use .. note:: Only preprocessing :ref:`tasks ` that completed successfully - can be re-used with the ``--resume_from`` option. + can be reused with the ``--resume_from`` option. Preprocessing tasks that completed successfully, contain a file called :ref:`metadata.yml ` in their output directory. diff --git a/doc/recipe/overview.rst b/doc/recipe/overview.rst index e0d63dc06b..dd0f5f643c 100644 --- a/doc/recipe/overview.rst +++ b/doc/recipe/overview.rst @@ -3,8 +3,8 @@ Overview ******** -After ``config-user.yml``, the ``recipe.yml`` is the second file the user needs -to pass to ``esmvaltool`` as command line option, at each run time point. +The recipe is the main control file of ESMValTool. +It is the only required argument for the ``esmvaltool`` command line program. Recipes contain the data and data analysis information and instructions needed to run the diagnostic(s), as well as specific diagnostic-related instructions. @@ -130,9 +130,9 @@ See :ref:`CMOR-DRS` for more information on this kind of file organization. When (some) files are available locally, the tool will not automatically look for more files on ESGF. -To populate a recipe with all available datasets from ESGF, ``search_esgf`` -should be set to ``always`` in the :ref:`user configuration file`. +To populate a recipe with all available datasets from ESGF, the +:ref:`configuration option ` ``search_esgf`` should be set to +``always``. For more control over which datasets are selected, it is recommended to use a Python script or `Jupyter notebook `_ to compose @@ -544,11 +544,14 @@ script will receive the preprocessed air temperature data script will receive the results of diagnostic_a.py and the preprocessed precipitation data (has ancestors ``diagnostic_1/script_a`` and ``diagnostic_2/precip``). +.. _task_priority: + Task priority ------------- Tasks are assigned a priority, with tasks appearing earlier on in the recipe getting higher priority. The tasks will be executed sequentially or in parallel, -depending on the setting of ``max_parallel_tasks`` in the :ref:`user configuration file`. +depending on the :ref:`configuration option ` +``max_parallel_tasks``. When there are fewer than ``max_parallel_tasks`` running, tasks will be started according to their priority. For obvious reasons, only tasks that are not waiting for ancestor tasks can be started. This feature makes it possible to diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index ddd9d2b472..a02bb4a566 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -611,7 +611,7 @@ See also :func:`esmvalcore.preprocessor.weighting_landsea_fraction`. .. _masking: Masking -======== +======= Introduction to masking ----------------------- @@ -1918,9 +1918,10 @@ Parameters: region to be extracted. If the file contains multiple shapes behaviour depends on the ``decomposed`` parameter. - This path can be relative to ``auxiliary_data_dir`` defined in the - :ref:`user configuration file` or relative to - ``esmvalcore/preprocessor/shapefiles`` (in that priority order). + This path can be relative to the directory specified via the + :ref:`configuration option ` ``auxiliary_data_dir`` or + relative to ``esmvalcore/preprocessor/shapefiles`` (in that priority + order). Alternatively, a string (see "Shapefile name" below) can be given to load one of the following shapefiles that are shipped with ESMValCore: @@ -2422,7 +2423,7 @@ See also :func:`esmvalcore.preprocessor.linear_trend_stderr`. .. _detrend: Detrend -======== +======= ESMValCore also supports detrending along any dimension using the preprocessor function 'detrend'. diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 32b692e070..d0bd6fcf10 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -28,10 +28,13 @@ """ # pylint: disable=import-outside-toplevel +from __future__ import annotations + import logging import os import sys from pathlib import Path +from typing import Optional if (sys.version_info.major, sys.version_info.minor) < (3, 10): from importlib_metadata import entry_points @@ -119,7 +122,7 @@ def process_recipe(recipe_file: Path, session): ) logger.info( "If you experience memory problems, try reducing " - "'max_parallel_tasks' in your user configuration file." + "'max_parallel_tasks' in your configuration." ) check_distributed_config() @@ -159,64 +162,94 @@ class Config: """ @staticmethod - def _copy_config_file(filename, overwrite, path): + def _copy_config_file( + in_file: Path, + out_file: Path, + overwrite: bool, + ): + """Copy a configuration file.""" import shutil from .config._logging import configure_logging configure_logging(console_log_level="info") - if not path: - path = os.path.join(os.path.expanduser("~/.esmvaltool"), filename) - if os.path.isfile(path): + + if out_file.is_file(): if overwrite: - logger.info("Overwriting file %s.", path) + logger.info("Overwriting file %s.", out_file) else: - logger.info("Copy aborted. File %s already exists.", path) + logger.info("Copy aborted. File %s already exists.", out_file) return - target_folder = os.path.dirname(path) - if not os.path.isdir(target_folder): + target_folder = out_file.parent + if not target_folder.is_dir(): logger.info("Creating folder %s", target_folder) - os.makedirs(target_folder) + target_folder.mkdir(parents=True, exist_ok=True) - conf_file = os.path.join(os.path.dirname(__file__), filename) - logger.info("Copying file %s to path %s.", conf_file, path) - shutil.copy2(conf_file, path) + logger.info("Copying file %s to path %s.", in_file, out_file) + shutil.copy2(in_file, out_file) logger.info("Copy finished.") @classmethod - def get_config_user(cls, overwrite=False, path=None): - """Copy default config-user.yml file to a given path. + def get_config_user( + cls, + overwrite: bool = False, + path: Optional[str | Path] = None, + ) -> None: + """Copy default configuration to a given path. - Copy default config-user.yml file to a given path or, if a path is - not provided, install it in the default `${HOME}/.esmvaltool` folder. + Copy default configuration to a given path or, if a `path` is not + provided, install it in the default `~/.config/esmvaltool/` directory. Parameters ---------- - overwrite: boolean + overwrite: Overwrite an existing file. - path: str + path: If not provided, the file will be copied to - .esmvaltool in the user's home. + `~/.config/esmvaltool/`. + """ - cls._copy_config_file("config-user.yml", overwrite, path) + from .config._config_object import DEFAULT_CONFIG_DIR + + in_file = DEFAULT_CONFIG_DIR / "config-user.yml" + if path is None: + out_file = ( + Path.home() / ".config" / "esmvaltool" / "config-user.yml" + ) + else: + out_file = Path(path) + if not out_file.suffix: # out_file looks like a directory + out_file = out_file / "config-user.yml" + cls._copy_config_file(in_file, out_file, overwrite) @classmethod - def get_config_developer(cls, overwrite=False, path=None): + def get_config_developer( + cls, + overwrite: bool = False, + path: Optional[str | Path] = None, + ) -> None: """Copy default config-developer.yml file to a given path. Copy default config-developer.yml file to a given path or, if a path is - not provided, install it in the default `${HOME}/.esmvaltool` folder. + not provided, install it in the default `~/.esmvaltool` folder. Parameters ---------- overwrite: boolean Overwrite an existing file. path: str - If not provided, the file will be copied to - .esmvaltool in the user's home. + If not provided, the file will be copied to `~/.esmvaltool`. + """ - cls._copy_config_file("config-developer.yml", overwrite, path) + in_file = Path(__file__).parent / "config-developer.yml" + if path is None: + out_file = Path.home() / ".esmvaltool" / "config-developer.yml" + else: + out_file = Path(path) + if not out_file.suffix: # out_file looks like a directory + out_file = out_file / "config-developer.yml" + cls._copy_config_file(in_file, out_file, overwrite) class Recipes: @@ -358,91 +391,75 @@ def version(self): for project, version in self._extra_packages.items(): print(f"{project}: {version}") - def run( - self, - recipe, - config_file=None, - resume_from=None, - max_datasets=None, - max_years=None, - skip_nonexistent=None, - search_esgf=None, - diagnostics=None, - check_level=None, - **kwargs, - ): + def run(self, recipe, **kwargs): """Execute an ESMValTool recipe. `esmvaltool run` executes the given recipe. To see a list of available recipes or create a local copy of any of them, use the `esmvaltool recipes` command group. - Parameters - ---------- - recipe : str - Recipe to run, as either the name of an installed recipe or the - path to a non-installed one. - config_file: str, optional - Configuration file to use. Can be given as absolute or relative - path. In the latter case, search in the current working directory - and `${HOME}/.esmvaltool` (in that order). If not provided, the - file `${HOME}/.esmvaltool/config-user.yml` will be used. - resume_from: list(str), optional - Resume one or more previous runs by using preprocessor output files - from these output directories. - max_datasets: int, optional - Maximum number of datasets to use. - max_years: int, optional - Maximum number of years to use. - skip_nonexistent: bool, optional - If True, the run will not fail if some datasets are not available. - search_esgf: str, optional - If `never`, disable automatic download of data from the ESGF. If - `when_missing`, enable the automatic download of files that are not - available locally. If `always`, always check ESGF for the latest - version of a file, and only use local files if they correspond to - that latest version. - diagnostics: list(str), optional - Only run the selected diagnostics from the recipe. To provide more - than one diagnostic to filter use the syntax 'diag1 diag2/script1' - or '("diag1", "diag2/script1")' and pay attention to the quotes. - check_level: str, optional - Configure the sensitivity of the CMOR check. Possible values are: - `ignore` (all errors will be reported as warnings), - `relaxed` (only fail if there are critical errors), - default (fail if there are any errors), - strict (fail if there are any warnings). + A list of possible flags is given here: + https://docs.esmvaltool.org/projects/ESMValCore/en/latest/quickstart/configure.html#configuration-options + """ from .config import CFG + from .config._config_object import _get_all_config_dirs + from .exceptions import InvalidConfigParameter + + cli_config_dir = kwargs.pop("config_dir", None) + if cli_config_dir is not None: + cli_config_dir = Path(cli_config_dir).expanduser().absolute() + if not cli_config_dir.is_dir(): + raise NotADirectoryError( + f"Invalid --config_dir given: {cli_config_dir} is not an " + f"existing directory" + ) + # TODO: remove in v2.14.0 # At this point, --config_file is already parsed if a valid file has # been given (see # https://github.com/ESMValGroup/ESMValCore/issues/2280), but no error # has been raised if the file does not exist. Thus, reload the file # here with `load_from_file` to make sure a proper error is raised. - CFG.load_from_file(config_file) + if "config_file" in kwargs: + cli_config_dir = kwargs["config_file"] + CFG.load_from_file(kwargs["config_file"]) + + # New in v2.12.0: read additional configuration directory given by CLI + # argument + if CFG.get("config_file") is None: # remove in v2.14.0 + config_dirs = _get_all_config_dirs(cli_config_dir) + try: + CFG.load_from_dirs(config_dirs) + + # Potential errors must come from --config_dir (i.e., + # cli_config_dir) since other sources have already been read (and + # validated) when importing the module with `from .config import + # CFG` + except InvalidConfigParameter as exc: + raise InvalidConfigParameter( + f"Failed to parse configuration directory " + f"{cli_config_dir} (command line argument): " + f"{str(exc)}" + ) from exc recipe = self._get_recipe(recipe) session = CFG.start_session(recipe.stem) - if check_level is not None: - session["check_level"] = check_level - if diagnostics is not None: - session["diagnostics"] = diagnostics - if max_datasets is not None: - session["max_datasets"] = max_datasets - if max_years is not None: - session["max_years"] = max_years - if search_esgf is not None: - session["search_esgf"] = search_esgf - if skip_nonexistent is not None: - session["skip_nonexistent"] = skip_nonexistent - session["resume_from"] = parse_resume(resume_from, recipe) session.update(kwargs) + session["resume_from"] = parse_resume(session["resume_from"], recipe) + + self._run(recipe, session, cli_config_dir) + + # Print warnings about deprecated configuration options again + # TODO: remove in v2.14.0 + if CFG.get("config_file") is not None: + CFG.reload() - self._run(recipe, session) - # Print warnings about deprecated configuration options again: - CFG.reload() + # New in v2.12.0 + else: + config_dirs = _get_all_config_dirs(cli_config_dir) # remove v2.14 + CFG.load_from_dirs(config_dirs) @staticmethod def _create_session_dir(session): @@ -464,7 +481,12 @@ def _create_session_dir(session): " unable to find alternative, aborting to prevent data loss." ) - def _run(self, recipe: Path, session) -> None: + def _run( + self, + recipe: Path, + session, + cli_config_dir: Optional[Path], + ) -> None: """Run `recipe` using `session`.""" self._create_session_dir(session) session.run_dir.mkdir() @@ -475,7 +497,7 @@ def _run(self, recipe: Path, session) -> None: log_files = configure_logging( output_dir=session.run_dir, console_log_level=session["log_level"] ) - self._log_header(session["config_file"], log_files) + self._log_header(log_files, cli_config_dir) # configure resource logger and run program from ._task import resource_usage_logger @@ -509,7 +531,7 @@ def _clean_preproc(session): logger.debug( "If this data is further needed, then set " "`save_intermediary_cubes` to `true` and `remove_preproc_dir` " - "to `false` in your user configuration file" + "to `false` in your configuration" ) shutil.rmtree(session._fixed_file_dir) @@ -519,8 +541,7 @@ def _clean_preproc(session): ) logger.info( "If this data is further needed, then set " - "`remove_preproc_dir` to `false` in your user configuration " - "file" + "`remove_preproc_dir` to `false` in your configuration" ) shutil.rmtree(session.preproc_dir) @@ -535,7 +556,41 @@ def _get_recipe(recipe) -> Path: recipe = Path(os.path.expandvars(recipe)).expanduser().absolute() return recipe - def _log_header(self, config_file, log_files): + @staticmethod + def _get_config_info(cli_config_dir): + """Get information about config files for logging.""" + from .config import CFG + from .config._config_object import ( + DEFAULT_CONFIG_DIR, + _get_all_config_dirs, + _get_all_config_sources, + ) + + # TODO: remove in v2.14.0 + if CFG.get("config_file") is not None: + config_info = [ + (DEFAULT_CONFIG_DIR, "defaults"), + (CFG["config_file"], "single configuration file [deprecated]"), + ] + + # New in v2.12.0 + else: + config_dirs = [] + for path in _get_all_config_dirs(cli_config_dir): + if not path.is_dir(): + config_dirs.append(f"{path} [NOT AN EXISTING DIRECTORY]") + else: + config_dirs.append(str(path)) + config_info = list( + zip( + config_dirs, + _get_all_config_sources(cli_config_dir), + ) + ) + + return "\n".join(f"{i[0]} ({i[1]})" for i in config_info) + + def _log_header(self, log_files, cli_config_dir): from . import __version__ logger.info(HEADER) @@ -545,7 +600,10 @@ def _log_header(self, config_file, log_files): for project, version in self._extra_packages.items(): logger.info("%s: %s", project, version) logger.info("----------------") - logger.info("Using config file %s", config_file) + logger.info( + "Reading configuration files from:\n%s", + self._get_config_info(cli_config_dir), + ) logger.info("Writing program log files to:\n%s", "\n".join(log_files)) diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 06bb2fd1a4..41002bbc1b 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -785,23 +785,21 @@ def _log_recipe_errors(self, exc): isinstance(err, InputFilesNotFound) for err in exc.failed_tasks ): logger.error( - "Not all input files required to run the recipe could be" - " found." + "Not all input files required to run the recipe could be " + "found." ) logger.error( - "If the files are available locally, please check" - " your `rootpath` and `drs` settings in your user " - "configuration file %s", - self.session["config_file"], + "If the files are available locally, please check " + "your `rootpath` and `drs` settings in your configuration " + "file(s)" ) logger.error( "To automatically download the required files to " - "`download_dir: %s`, set `search_esgf: when_missing` or " - "`search_esgf: always` in %s, or run the recipe with the " - "extra command line argument --search_esgf=when_missing or " - "--search_esgf=always", + "`download_dir: %s`, use `search_esgf: when_missing` or " + "`search_esgf: always` in your configuration file(s), or run " + "the recipe with the command line argument " + "--search_esgf=when_missing or --search_esgf=always", self.session["download_dir"], - self.session["config_file"], ) logger.info( "Note that automatic download is only available for files" diff --git a/esmvalcore/cmor/_fixes/icon/_base_fixes.py b/esmvalcore/cmor/_fixes/icon/_base_fixes.py index be77c9d6c8..9c551ef4a0 100644 --- a/esmvalcore/cmor/_fixes/icon/_base_fixes.py +++ b/esmvalcore/cmor/_fixes/icon/_base_fixes.py @@ -221,9 +221,8 @@ def add_additional_cubes(self, cubes): Note ---- - Files can be specified as absolute or relative (to - ``auxiliary_data_dir`` as defined in the :ref:`user configuration - file`) paths. + Files can be specified as absolute or relative (to the configuration + option ``auxiliary_data_dir``) paths. Parameters ---------- diff --git a/esmvalcore/config/__init__.py b/esmvalcore/config/__init__.py index f9a632b75c..5d23c6b0e2 100644 --- a/esmvalcore/config/__init__.py +++ b/esmvalcore/config/__init__.py @@ -2,11 +2,14 @@ .. data:: CFG - ESMValCore configuration. + Global ESMValCore configuration object of type + :class:`esmvalcore.config.Config`. - By default, this will be loaded from the file - ``~/.esmvaltool/config-user.yml``. If used within the ``esmvaltool`` - program, this will respect the ``--config_file`` argument. + By default, this will be loaded from YAML files in the user configuration + directory (by default ``~/.config/esmvaltool``, but this can be changed + with the ``ESMVALTOOL_CONFIG_DIR`` environment variable) similar to the way + `Dask handles configuration + `__. """ diff --git a/esmvalcore/config/_config.py b/esmvalcore/config/_config.py index 71617c625e..6df9e9bf52 100644 --- a/esmvalcore/config/_config.py +++ b/esmvalcore/config/_config.py @@ -1,4 +1,4 @@ -"""Functions dealing with config-user.yml / config-developer.yml.""" +"""Functions dealing with config-developer.yml and extra facets.""" from __future__ import annotations @@ -52,7 +52,8 @@ def _load_extra_facets(project, extra_facets_dir): def get_extra_facets(dataset, extra_facets_dir): - """Read configuration files with additional variable information.""" + """Read files with additional variable information ("extra facets").""" + extra_facets_dir = tuple(extra_facets_dir) project_details = _load_extra_facets( dataset.facets["project"], extra_facets_dir, diff --git a/esmvalcore/config/_config_object.py b/esmvalcore/config/_config_object.py index dc78506215..dfe784ef58 100644 --- a/esmvalcore/config/_config_object.py +++ b/esmvalcore/config/_config_object.py @@ -4,29 +4,67 @@ import os import sys +import warnings +from collections.abc import Iterable from datetime import datetime from pathlib import Path -from types import MappingProxyType from typing import Optional +import dask.config import yaml import esmvalcore -from esmvalcore.cmor.check import CheckLevels -from esmvalcore.exceptions import InvalidConfigParameter - -from ._config_validators import ( +from esmvalcore.config._config_validators import ( _deprecated_options_defaults, _deprecators, _validators, ) -from ._validated_config import ValidatedConfig +from esmvalcore.config._validated_config import ValidatedConfig +from esmvalcore.exceptions import ( + ESMValCoreDeprecationWarning, + InvalidConfigParameter, +) URL = ( "https://docs.esmvaltool.org/projects/" "ESMValCore/en/latest/quickstart/configure.html" ) +# Configuration directory in which defaults are stored +DEFAULT_CONFIG_DIR = ( + Path(esmvalcore.__file__).parent / "config" / "configurations" / "defaults" +) + + +def _get_user_config_dir() -> Path: + """Get user configuration directory.""" + if "ESMVALTOOL_CONFIG_DIR" in os.environ: + user_config_dir = ( + Path(os.environ["ESMVALTOOL_CONFIG_DIR"]).expanduser().absolute() + ) + if not user_config_dir.is_dir(): + raise NotADirectoryError( + f"Invalid configuration directory specified via " + f"ESMVALTOOL_CONFIG_DIR environment variable: " + f"{user_config_dir} is not an existing directory" + ) + return user_config_dir + return Path.home() / ".config" / "esmvaltool" + + +def _get_user_config_source() -> str: + """Get source of user configuration directory.""" + if "ESMVALTOOL_CONFIG_DIR" in os.environ: + return "ESMVALTOOL_CONFIG_DIR environment variable" + return "default user configuration directory" + + +# User configuration directory +USER_CONFIG_DIR = _get_user_config_dir() + +# Source of user configuration directory +USER_CONFIG_SOURCE = _get_user_config_source() + class Config(ValidatedConfig): """ESMValTool configuration object. @@ -36,6 +74,7 @@ class Config(ValidatedConfig): """ + # TODO: remove in v2.14.0 _DEFAULT_USER_CONFIG_DIR = Path.home() / ".esmvaltool" _validate = _validators @@ -46,6 +85,16 @@ class Config(ValidatedConfig): ("rootpath", URL), ) + def __init__(self, *args, **kwargs): + """Initialize class instance.""" + super().__init__(*args, **kwargs) + msg = ( + "Do not instantiate `Config` objects directly, this will lead " + "to unexpected behavior. Use `esmvalcore.config.CFG` instead." + ) + warnings.warn(msg, UserWarning) + + # TODO: remove in v2.14.0 @classmethod def _load_user_config( cls, @@ -69,8 +118,15 @@ def _load_user_config( configuration file is given (relevant if used within a script or notebook). """ - new = cls() - new.update(CFG_DEFAULT) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="Do not instantiate `Config` objects directly", + category=UserWarning, + module="esmvalcore", + ) + new = cls() + new.update(Config._load_default_config()) config_user_path = cls._get_config_user_path(filename) @@ -93,31 +149,26 @@ def _load_user_config( return new + # TODO: remove in v2.14.0 @classmethod def _load_default_config(cls): """Load the default configuration.""" - new = cls() - - package_config_user_path = ( - Path(esmvalcore.__file__).parent / "config-user.yml" - ) - mapping = cls._read_config_file(package_config_user_path) - - # Add defaults that are not available in esmvalcore/config-user.yml - mapping["check_level"] = CheckLevels.DEFAULT - mapping["config_file"] = package_config_user_path - mapping["diagnostics"] = None - mapping["extra_facets_dir"] = tuple() - mapping["max_datasets"] = None - mapping["max_years"] = None - mapping["resume_from"] = [] - mapping["run_diagnostic"] = True - mapping["skip_nonexistent"] = False + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="Do not instantiate `Config` objects directly", + category=UserWarning, + module="esmvalcore", + ) + new = cls() + paths = [DEFAULT_CONFIG_DIR] + mapping = dask.config.collect(paths=paths, env={}) new.update(mapping) return new + # TODO: remove in v2.14.0 @staticmethod def _read_config_file(config_user_path: Path) -> dict: """Read configuration file and store settings in a dictionary.""" @@ -131,6 +182,7 @@ def _read_config_file(config_user_path: Path) -> dict: return cfg + # TODO: remove in v2.14.0 @staticmethod def _get_config_user_path( filename: Optional[os.PathLike | str] = None, @@ -201,6 +253,7 @@ def _get_config_user_path( return config_user + # TODO: remove in v2.14.0 @staticmethod def _get_config_path_from_cli() -> None | str: """Try to get configuration path from CLI arguments. @@ -237,25 +290,126 @@ def _get_config_path_from_cli() -> None | str: return None + # TODO: remove in v2.14.0 def load_from_file( self, filename: Optional[os.PathLike | str] = None, ) -> None: - """Load user configuration from the given file.""" + """Load user configuration from the given file. + + .. deprecated:: 2.12.0 + This method has been deprecated in ESMValCore version 2.14.0 and is + scheduled for removal in version 2.14.0. Please use + `CFG.load_from_dirs()` instead. + + Parameters + ---------- + filename: + YAML file to load. + + """ + msg = ( + "The method `CFG.load_from_file()` has been deprecated in " + "ESMValCore version 2.12.0 and is scheduled for removal in " + "version 2.14.0. Please use `CFG.load_from_dirs()` instead." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) self.clear() self.update(Config._load_user_config(filename)) - def reload(self): - """Reload the config file.""" - if "config_file" not in self: - raise ValueError( - "Cannot reload configuration, option 'config_file' is " - "missing; make sure to only use the `CFG` object from the " - "`esmvalcore.config` module" + def load_from_dirs(self, dirs: Iterable[str | Path]) -> None: + """Load configuration object from directories. + + This searches for all YAML files within the given directories and + merges them together using :func:`dask.config.collect`. Nested objects + are properly considered; see :func:`dask.config.update` for details. + Values in the latter directories are preferred to those in the former. + + Options that are not explicitly specified via YAML files are set to the + :ref:`default values `. + + Note + ---- + Just like :func:`dask.config.collect`, this silently ignores + non-existing directories. + + Parameters + ---------- + dirs: + A list of directories to search for YAML configuration files. + + Raises + ------ + esmvalcore.exceptions.InvalidConfigParameter + Invalid configuration option given. + + """ + dirs_str: list[str] = [] + + # Always consider default options; these have the lowest priority + dirs_str.append(str(DEFAULT_CONFIG_DIR)) + + for config_dir in dirs: + config_dir = Path(config_dir).expanduser().absolute() + dirs_str.append(str(config_dir)) + + new_config_dict = dask.config.collect(paths=dirs_str, env={}) + self.clear() + self.update(new_config_dict) + + self.check_missing() + + def reload(self) -> None: + """Reload the configuration object. + + This will read all YAML files in the user configuration directory (by + default ``~/.config/esmvaltool``, but this can be changed with the + ``ESMVALTOOL_CONFIG_DIR`` environment variable) and merges them + together using :func:`dask.config.collect`. Nested objects are properly + considered; see :func:`dask.config.update` for details. + + Options that are not explicitly specified via YAML files are set to the + :ref:`default values `. + + Note + ---- + If the user configuration directory does not exist, this will be + silently ignored. + + Raises + ------ + esmvalcore.exceptions.InvalidConfigParameter + Invalid configuration option given. + + """ + # TODO: remove in v2.14.0 + self.clear() + _deprecated_config_user_path = Config._get_config_user_path() + if _deprecated_config_user_path.is_file(): + deprecation_msg = ( + f"Usage of the single configuration file " + f"~/.esmvaltool/config-user.yml or specifying it via CLI " + f"argument `--config_file` has been deprecated in ESMValCore " + f"version 2.12.0 and is scheduled for removal in version " + f"2.14.0. Please run `mkdir -p ~/.config/esmvaltool && mv " + f"{_deprecated_config_user_path} ~/.config/esmvaltool` (or " + f"alternatively use a custom `--config_dir`) and omit " + f"`--config_file`." ) - self.load_from_file(self["config_file"]) + warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning) + self.update(Config._load_user_config(raise_exception=False)) + return - def start_session(self, name: str): + # New since v2.12.0 + try: + self.load_from_dirs([USER_CONFIG_DIR]) + except InvalidConfigParameter as exc: + raise InvalidConfigParameter( + f"Failed to parse configuration directory {USER_CONFIG_DIR} " + f"({USER_CONFIG_SOURCE}): {str(exc)}" + ) from exc + + def start_session(self, name: str) -> Session: """Start a new session from this configuration object. Parameters @@ -267,7 +421,15 @@ def start_session(self, name: str): ------- Session """ - return Session(config=self.copy(), name=name) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="Do not instantiate `Session` objects directly", + category=UserWarning, + module="esmvalcore", + ) + session = Session(config=self.copy(), name=name) + return session class Session(ValidatedConfig): @@ -302,6 +464,12 @@ def __init__(self, config: dict, name: str = "session"): super().__init__(config) self.session_name: str | None = None self.set_session_name(name) + msg = ( + "Do not instantiate `Session` objects directly, this will lead " + "to unexpected behavior. Use " + "`esmvalcore.config.CFG.start_session` instead." + ) + warnings.warn(msg, UserWarning) def set_session_name(self, name: str = "session"): """Set the name for the session. @@ -337,9 +505,24 @@ def run_dir(self): """Return run directory.""" return self.session_dir / self.relative_run_dir + # TODO: remove in v2.14.0 @property def config_dir(self): - """Return user config directory.""" + """Return user config directory. + + .. deprecated:: 2.12.0 + This attribute has been deprecated in ESMValCore version 2.12.0 and + is scheduled for removal in version 2.14.0. + + """ + msg = ( + "The attribute `Session.config_dir` has been deprecated in " + "ESMValCore version 2.12.0 and is scheduled for removal in " + "version 2.14.0." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + if self.get("config_file") is None: + return None return Path(self["config_file"]).parent @property @@ -363,6 +546,41 @@ def _fixed_file_dir(self): return self.session_dir / self._relative_fixed_file_dir +def _get_all_config_dirs(cli_config_dir: Optional[Path]) -> list[Path]: + """Get all configuration directories.""" + config_dirs: list[Path] = [ + DEFAULT_CONFIG_DIR, + USER_CONFIG_DIR, + ] + if cli_config_dir is not None: + config_dirs.append(cli_config_dir) + return config_dirs + + +def _get_all_config_sources(cli_config_dir: Optional[Path]) -> list[str]: + """Get all sources of configuration directories.""" + config_sources: list[str] = [ + "defaults", + USER_CONFIG_SOURCE, + ] + if cli_config_dir is not None: + config_sources.append("command line argument") + return config_sources + + +def _get_global_config() -> Config: + """Get global configuration object.""" + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="Do not instantiate `Config` objects directly", + category=UserWarning, + module="esmvalcore", + ) + config_obj = Config() + config_obj.reload() + return config_obj + + # Initialize configuration objects -CFG_DEFAULT = MappingProxyType(Config._load_default_config()) -CFG = Config._load_user_config(raise_exception=False) +CFG = _get_global_config() diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 23034ce5c2..9cc85bee5e 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -204,10 +204,6 @@ def chained(value): validate_path, docstring="Return a list of paths." ) -validate_pathtuple = _listify_validator( - validate_path, docstring="Return a tuple of paths.", return_type=tuple -) - validate_int_positive = _chain_validator(validate_int, validate_positive) validate_int_positive_or_none = _make_type_validator( validate_int_positive, allow_none=True @@ -222,7 +218,7 @@ def validate_rootpath(value): if key == "obs4mips": logger.warning( "Correcting capitalization, project 'obs4mips' should be " - "written as 'obs4MIPs' in 'rootpath' in config-user.yml" + "written as 'obs4MIPs' in configured 'rootpath'" ) key = "obs4MIPs" if isinstance(paths, Path): @@ -247,7 +243,7 @@ def validate_drs(value): if key == "obs4mips": logger.warning( "Correcting capitalization, project 'obs4mips' should be " - "written as 'obs4MIPs' in 'drs' in config-user.yml" + "written as 'obs4MIPs' in configured 'drs'" ) key = "obs4MIPs" new_mapping[key] = validate_string(drs) @@ -306,34 +302,47 @@ def validate_diagnostics( } +# TODO: remove in v2.14.0 +def validate_extra_facets_dir(value): + """Validate extra_facets_dir.""" + if isinstance(value, tuple): + msg = ( + "Specifying `extra_facets_dir` as tuple has been deprecated in " + "ESMValCore version 2.12.0 and is scheduled for removal in " + "version 2.14.0. Please use a list instead." + ) + warnings.warn(msg, ESMValCoreDeprecationWarning) + value = list(value) + return validate_pathlist(value) + + _validators = { - # From user config "auxiliary_data_dir": validate_path, + "check_level": validate_check_level, "compress_netcdf": validate_bool, "config_developer_file": validate_config_developer, + "diagnostics": validate_diagnostics, "download_dir": validate_path, "drs": validate_drs, "exit_on_warning": validate_bool, - "extra_facets_dir": validate_pathtuple, + "extra_facets_dir": validate_extra_facets_dir, "log_level": validate_string, + "max_datasets": validate_int_positive_or_none, "max_parallel_tasks": validate_int_or_none, + "max_years": validate_int_positive_or_none, "output_dir": validate_path, "output_file_type": validate_string, "profile_diagnostic": validate_bool, "remove_preproc_dir": validate_bool, + "resume_from": validate_pathlist, "rootpath": validate_rootpath, "run_diagnostic": validate_bool, "save_intermediary_cubes": validate_bool, "search_esgf": validate_search_esgf, - # From CLI - "check_level": validate_check_level, - "diagnostics": validate_diagnostics, - "max_datasets": validate_int_positive_or_none, - "max_years": validate_int_positive_or_none, - "resume_from": validate_pathlist, "skip_nonexistent": validate_bool, # From recipe "write_ncl_interface": validate_bool, + # TODO: remove in v2.14.0 # config location "config_file": validate_path, } @@ -365,12 +374,40 @@ def _handle_deprecation( warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning) +# TODO: remove in v2.14.0 +def deprecate_config_file(validated_config, value, validated_value): + """Deprecate ``config_file`` option. + + Parameters + ---------- + validated_config: ValidatedConfig + ``ValidatedConfig`` instance which will be modified in place. + value: Any + Raw input value for ``config_file`` option. + validated_value: Any + Validated value for ``config_file`` option. + + """ + validated_config # noqa + value # noqa + validated_value # noqa + option = "config_file" + deprecated_version = "2.12.0" + remove_version = "2.14.0" + more_info = " Please use the option `config_dir` instead." + _handle_deprecation(option, deprecated_version, remove_version, more_info) + + # Example usage: see removed files in # https://github.com/ESMValGroup/ESMValCore/pull/2213 -_deprecators: dict[str, Callable] = {} +_deprecators: dict[str, Callable] = { + "config_file": deprecate_config_file, # TODO: remove in v2.14.0 +} # Default values for deprecated options # Example usage: see removed files in # https://github.com/ESMValGroup/ESMValCore/pull/2213 -_deprecated_options_defaults: dict[str, Any] = {} +_deprecated_options_defaults: dict[str, Any] = { + "config_file": None, # TODO: remove in v2.14.0 +} diff --git a/esmvalcore/config-user.yml b/esmvalcore/config/configurations/defaults/config-user.yml similarity index 96% rename from esmvalcore/config-user.yml rename to esmvalcore/config/configurations/defaults/config-user.yml index ecdee818fc..39cffb67fb 100644 --- a/esmvalcore/config-user.yml +++ b/esmvalcore/config/configurations/defaults/config-user.yml @@ -1,5 +1,5 @@ ############################################################################### -# Example user configuration file for ESMValTool +# Default configuration settings ############################################################################### # # Note for users: @@ -13,14 +13,6 @@ # file. # ############################################################################### -# -# Note for developers: -# ------------------- -# Two identical copies of this file (``ESMValTool/config-user-example.yml`` and -# ``ESMValCore/esmvalcore/config-user.yml``) exist. If you change one of it, -# make sure to apply the changes to the other. -# -############################################################################### --- # Destination directory where all output will be written diff --git a/esmvalcore/config/configurations/defaults/more_options.yml b/esmvalcore/config/configurations/defaults/more_options.yml new file mode 100644 index 0000000000..c61a70a493 --- /dev/null +++ b/esmvalcore/config/configurations/defaults/more_options.yml @@ -0,0 +1,9 @@ +# Other options not included in config-user.yml +check_level: default +diagnostics: null +extra_facets_dir: [] +max_datasets: null +max_years: null +resume_from: [] +run_diagnostic: true +skip_nonexistent: false diff --git a/esmvalcore/local.py b/esmvalcore/local.py index 61c2782b58..e94a998ed5 100644 --- a/esmvalcore/local.py +++ b/esmvalcore/local.py @@ -469,7 +469,7 @@ def _get_data_sources(project: str) -> list[DataSource]: nonexistent = tuple(p for p in paths if not os.path.exists(p)) if nonexistent and (key, nonexistent) not in _ROOTPATH_WARNED: logger.warning( - "'%s' rootpaths '%s' set in config-user.yml do not exist", + "Configured '%s' rootpaths '%s' do not exist", key, ", ".join(str(p) for p in nonexistent), ) @@ -490,7 +490,7 @@ def _get_data_sources(project: str) -> list[DataSource]: raise KeyError( f"No '{project}' or 'default' path specified under 'rootpath' in " - "the user configuration." + "the configuration." ) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..5fd7be7460 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,24 @@ +from copy import deepcopy +from pathlib import Path + +import pytest + +from esmvalcore.config import CFG + + +@pytest.fixture +def cfg_default(mocker): + """Configuration object with defaults.""" + cfg = deepcopy(CFG) + cfg.load_from_dirs([]) + return cfg + + +@pytest.fixture +def session(tmp_path: Path, cfg_default, monkeypatch): + """Session object with default settings.""" + for key, value in cfg_default.items(): + monkeypatch.setitem(CFG, key, deepcopy(value)) + monkeypatch.setitem(CFG, "rootpath", {"default": {tmp_path: "default"}}) + monkeypatch.setitem(CFG, "output_dir", tmp_path / "esmvaltool_output") + return CFG.start_session("recipe_test") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8787b345ee..e32e3ca3fa 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -5,8 +5,6 @@ import pytest import esmvalcore.local -from esmvalcore.config import CFG -from esmvalcore.config._config_object import CFG_DEFAULT from esmvalcore.local import ( LocalFile, _replace_tags, @@ -15,17 +13,6 @@ ) -@pytest.fixture -def session(tmp_path: Path, monkeypatch): - CFG.clear() - CFG.update(CFG_DEFAULT) - monkeypatch.setitem(CFG, "rootpath", {"default": {tmp_path: "default"}}) - - session = CFG.start_session("recipe_test") - session["output_dir"] = tmp_path / "esmvaltool_output" - return session - - def create_test_file(filename, tracking_id=None): dirname = os.path.dirname(filename) if not os.path.exists(dirname): diff --git a/tests/integration/test_deprecated_config.py b/tests/integration/test_deprecated_config.py index cf50f2ea4c..0ae313511f 100644 --- a/tests/integration/test_deprecated_config.py +++ b/tests/integration/test_deprecated_config.py @@ -1,8 +1,6 @@ import warnings -from pathlib import Path -import esmvalcore -from esmvalcore.config import CFG, Config +from esmvalcore.config import CFG from esmvalcore.exceptions import ESMValCoreDeprecationWarning @@ -12,13 +10,3 @@ def test_no_deprecation_default_cfg(): warnings.simplefilter("error", category=ESMValCoreDeprecationWarning) CFG.reload() CFG.start_session("my_session") - - -def test_no_deprecation_user_cfg(): - """Test that user config does not raise any deprecation warnings.""" - config_file = Path(esmvalcore.__file__).parent / "config-user.yml" - with warnings.catch_warnings(): - warnings.simplefilter("error", category=ESMValCoreDeprecationWarning) - cfg = Config(CFG.copy()) - cfg.load_from_file(config_file) - cfg.start_session("my_session") diff --git a/tests/integration/test_diagnostic_run.py b/tests/integration/test_diagnostic_run.py index e66cd925c2..285f86fd15 100644 --- a/tests/integration/test_diagnostic_run.py +++ b/tests/integration/test_diagnostic_run.py @@ -186,12 +186,80 @@ def test_diagnostic_run(tmp_path, script_file, script): # ensure that tags are cleared TAGS.clear() - config_user_file = write_config_user_file(tmp_path) + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True, exist_ok=True) + write_config_user_file(config_dir) + + with arguments( + "esmvaltool", + "run", + "--config_dir", + str(config_dir), + str(recipe_file), + ): + run() + + check(result_file) + + +# TODO: remove in v2.14.0 +@pytest.mark.parametrize( + "script_file, script", + [ + pytest.param( + script_file, + script, + marks=[ + pytest.mark.installation, + pytest.mark.xfail( + interpreter_not_installed(script_file), + run=False, + reason="Interpreter not available", + ), + ], + ) + for script_file, script in SCRIPTS.items() + if script_file != "null" + ], +) +def test_diagnostic_run_old_config(tmp_path, script_file, script): + recipe_file = tmp_path / "recipe_test.yml" + script_file = tmp_path / script_file + result_file = tmp_path / "result.yml" + + # Write script to file + script_file.write_text(str(script)) + + # Create recipe + recipe = dedent( + """ + documentation: + title: Recipe without data + description: Recipe with no data. + authors: [andela_bouwe] + + diagnostics: + diagnostic_name: + scripts: + script_name: + script: {} + setting_name: {} + """.format(script_file, result_file) + ) + recipe_file.write_text(str(recipe)) + + # ensure that tags are cleared + TAGS.clear() + + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True, exist_ok=True) + config_file = write_config_user_file(config_dir) + with arguments( "esmvaltool", "run", "--config_file", - config_user_file, + str(config_file), str(recipe_file), ): run() diff --git a/tests/integration/test_main.py b/tests/integration/test_main.py index e0838fd3e2..b15a0e6129 100644 --- a/tests/integration/test_main.py +++ b/tests/integration/test_main.py @@ -9,13 +9,14 @@ import os import shutil import sys +from pathlib import Path from textwrap import dedent from unittest.mock import patch import pytest -import yaml from fire.core import FireExit +import esmvalcore.config._config from esmvalcore._main import Config, ESMValTool, Recipes, run from esmvalcore.exceptions import RecipeError @@ -34,8 +35,10 @@ def empty(*args, **kwargs): def arguments(*args): backup = sys.argv sys.argv = list(args) - yield - sys.argv = backup + try: + yield + finally: + sys.argv = backup def test_setargs(): @@ -62,8 +65,11 @@ def test_run(): run() -def test_empty_run(tmp_path): +def test_empty_run(tmp_path, monkeypatch): """Test real run with no diags.""" + monkeypatch.delitem( # TODO: remove in v2.14.0 + esmvalcore.config.CFG._mapping, "config_file", raising=False + ) recipe_file = tmp_path / "recipe.yml" content = dedent(""" documentation: @@ -79,17 +85,14 @@ def test_empty_run(tmp_path): diagnostics: null """) recipe_file.write_text(content) - Config.get_config_user(path=tmp_path) log_dir = f"{tmp_path}/esmvaltool_output" - config_file = f"{tmp_path}/config-user.yml" - with open(config_file, "r+", encoding="utf-8") as file: - config = yaml.safe_load(file) - config["output_dir"] = log_dir - yaml.safe_dump(config, file, sort_keys=False) + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True, exist_ok=True) + config_file = config_dir / "config.yml" + config_file.write_text(f"output_dir: {log_dir}") + with pytest.raises(RecipeError) as exc: - ESMValTool().run( - recipe_file, config_file=f"{tmp_path}/config-user.yml" - ) + ESMValTool().run(recipe_file, config_dir=config_dir) assert str(exc.value) == "The given recipe does not have any diagnostic." log_file = os.path.join( log_dir, os.listdir(log_dir)[0], "run", "main_log.txt" @@ -103,65 +106,6 @@ def test_empty_run(tmp_path): assert not filled_recipe -@patch("esmvalcore._main.ESMValTool.run", new=wrapper(ESMValTool.run)) -def test_run_with_config(): - with arguments( - "esmvaltool", "run", "recipe.yml", "--config_file", "config.yml" - ): - run() - - -@patch("esmvalcore._main.ESMValTool.run", new=wrapper(ESMValTool.run)) -def test_run_with_max_years(): - with arguments( - "esmvaltool", - "run", - "recipe.yml", - "--config_file=config.yml", - "--max_years=2", - ): - run() - - -@patch("esmvalcore._main.ESMValTool.run", new=wrapper(ESMValTool.run)) -def test_run_with_max_datasets(): - with arguments("esmvaltool", "run", "recipe.yml", "--max_datasets=2"): - run() - - -@patch("esmvalcore._main.ESMValTool.run", new=wrapper(ESMValTool.run)) -def test_run_with_search_esgf(): - with arguments("esmvaltool", "run", "recipe.yml", "--search_esgf=always"): - run() - - -@patch("esmvalcore._main.ESMValTool.run", new=wrapper(ESMValTool.run)) -def test_run_with_check_level(): - with arguments("esmvaltool", "run", "recipe.yml", "--check_level=default"): - run() - - -@patch("esmvalcore._main.ESMValTool.run", new=wrapper(ESMValTool.run)) -def test_run_with_skip_nonexistent(): - with arguments( - "esmvaltool", "run", "recipe.yml", "--skip_nonexistent=True" - ): - run() - - -@patch("esmvalcore._main.ESMValTool.run", new=wrapper(ESMValTool.run)) -def test_run_with_diagnostics(): - with arguments("esmvaltool", "run", "recipe.yml", "--diagnostics=[badt]"): - run() - - -@patch("esmvalcore._main.ESMValTool.run", new=wrapper(ESMValTool.run)) -def test_run_fails_with_other_params(): - with arguments("esmvaltool", "run", "recipe.yml", "--extra_param=dfa"): - with pytest.raises(SystemExit): - run() - - def test_recipes_get(tmp_path, monkeypatch): """Test version command.""" src_recipe = tmp_path / "recipe.yml" @@ -199,6 +143,66 @@ def test_get_config_developer(): run() +def test_get_config_developer_no_path(): + """Test version command.""" + with arguments("esmvaltool", "config", "get_config_developer"): + run() + config_file = Path.home() / ".esmvaltool" / "config-developer.yml" + assert config_file.is_file() + + +def test_get_config_developer_path(tmp_path): + """Test version command.""" + new_path = tmp_path / "subdir" + with arguments( + "esmvaltool", "config", "get_config_developer", f"--path={new_path}" + ): + run() + assert (new_path / "config-developer.yml").is_file() + + +def test_get_config_developer_overwrite(tmp_path): + """Test version command.""" + config_developer = tmp_path / "config-developer.yml" + config_developer.write_text("old text") + with arguments( + "esmvaltool", + "config", + "get_config_developer", + f"--path={tmp_path}", + "--overwrite", + ): + run() + assert config_developer.read_text() != "old text" + + +def test_get_config_developer_no_overwrite(tmp_path): + """Test version command.""" + config_developer = tmp_path / "configuration_file.yml" + config_developer.write_text("old text") + with arguments( + "esmvaltool", + "config", + "get_config_developer", + f"--path={config_developer}", + ): + run() + assert config_developer.read_text() == "old text" + + +@patch( + "esmvalcore._main.Config.get_config_developer", + new=wrapper(Config.get_config_developer), +) +def test_get_config_developer_bad_option_fails(): + """Test version command.""" + with arguments( + "esmvaltool", "config", "get_config_developer", "--bad_option=path" + ): + with pytest.raises(FireExit): + run() + + @patch( "esmvalcore._main.Config.get_config_user", new=wrapper(Config.get_config_user), @@ -209,19 +213,28 @@ def test_get_config_user(): run() +def test_get_config_user_no_path(): + """Test version command.""" + with arguments("esmvaltool", "config", "get_config_user"): + run() + config_file = Path.home() / ".config" / "esmvaltool" / "config-user.yml" + assert config_file.is_file() + + def test_get_config_user_path(tmp_path): """Test version command.""" + new_path = tmp_path / "subdir" with arguments( - "esmvaltool", "config", "get_config_user", f"--path={tmp_path}" + "esmvaltool", "config", "get_config_user", f"--path={new_path}" ): run() - assert (tmp_path / "config-user.yml").is_file() + assert (new_path / "config-user.yml").is_file() def test_get_config_user_overwrite(tmp_path): """Test version command.""" config_user = tmp_path / "config-user.yml" - config_user.touch() + config_user.write_text("old text") with arguments( "esmvaltool", "config", @@ -230,6 +243,18 @@ def test_get_config_user_overwrite(tmp_path): "--overwrite", ): run() + assert config_user.read_text() != "old text" + + +def test_get_config_user_no_overwrite(tmp_path): + """Test version command.""" + config_user = tmp_path / "configuration_file.yml" + config_user.write_text("old text") + with arguments( + "esmvaltool", "config", "get_config_user", f"--path={config_user}" + ): + run() + assert config_user.read_text() == "old text" @patch( diff --git a/tests/sample_data/experimental/test_run_recipe.py b/tests/sample_data/experimental/test_run_recipe.py index 2abdd22197..141cc74c57 100644 --- a/tests/sample_data/experimental/test_run_recipe.py +++ b/tests/sample_data/experimental/test_run_recipe.py @@ -12,7 +12,6 @@ import pytest import esmvalcore._task -from esmvalcore.config._config_object import CFG_DEFAULT from esmvalcore.config._diagnostics import TAGS from esmvalcore.exceptions import RecipeError from esmvalcore.experimental import CFG, Recipe, get_recipe @@ -59,7 +58,9 @@ def recipe(): @pytest.mark.use_sample_data @pytest.mark.parametrize("ssh", (True, False)) @pytest.mark.parametrize("task", (None, "example/ta")) -def test_run_recipe(monkeypatch, task, ssh, recipe, tmp_path, caplog): +def test_run_recipe( + monkeypatch, cfg_default, task, ssh, recipe, tmp_path, caplog +): """Test running a basic recipe using sample data. Recipe contains no provenance and no diagnostics. @@ -79,9 +80,7 @@ def test_run_recipe(monkeypatch, task, ssh, recipe, tmp_path, caplog): sample_data_config = esmvaltool_sample_data.get_rootpaths() monkeypatch.setitem(CFG, "rootpath", sample_data_config["rootpath"]) monkeypatch.setitem(CFG, "drs", {"CMIP6": "SYNDA"}) - session = CFG.start_session(recipe.path.stem) - session.clear() - session.update(CFG_DEFAULT) + session = cfg_default.start_session(recipe.path.stem) session["output_dir"] = tmp_path / "esmvaltool_output" session["max_parallel_tasks"] = 1 session["remove_preproc_dir"] = False diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 513fd20595..194724a317 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -4,7 +4,6 @@ import pytest import yaml -import esmvalcore from esmvalcore.cmor.check import CheckLevels from esmvalcore.config import CFG, _config, _config_validators from esmvalcore.config._config import ( @@ -15,7 +14,7 @@ importlib_files, ) from esmvalcore.dataset import Dataset -from esmvalcore.exceptions import RecipeError +from esmvalcore.exceptions import ESMValCoreDeprecationWarning, RecipeError TEST_DEEP_UPDATE = [ ([{}], {}), @@ -167,32 +166,19 @@ def test_get_project_config(mocker): _config.get_project_config("non-existent-project") -CONFIG_USER_FILE = importlib_files("esmvalcore") / "config-user.yml" - - -@pytest.fixture -def default_config(): - # Load default configuration - CFG.load_from_file(CONFIG_USER_FILE) - # Run test - yield - # Restore default configuration - CFG.load_from_file(CONFIG_USER_FILE) - - -def test_load_default_config(monkeypatch, default_config): +def test_load_default_config(cfg_default, monkeypatch): """Test that the default configuration can be loaded.""" project_cfg = {} monkeypatch.setattr(_config, "CFG", project_cfg) default_dev_file = importlib_files("esmvalcore") / "config-developer.yml" - cfg = CFG.start_session("recipe_example") + + session = cfg_default.start_session("recipe_example") default_cfg = { "auxiliary_data_dir": Path.home() / "auxiliary_data", "check_level": CheckLevels.DEFAULT, "compress_netcdf": False, "config_developer_file": default_dev_file, - "config_file": CONFIG_USER_FILE, "diagnostics": None, "download_dir": Path.home() / "climate_data", "drs": { @@ -203,7 +189,7 @@ def test_load_default_config(monkeypatch, default_config): "obs4MIPs": "ESGF", }, "exit_on_warning": False, - "extra_facets_dir": tuple(), + "extra_facets_dir": [], "log_level": "info", "max_datasets": None, "max_parallel_tasks": None, @@ -229,38 +215,39 @@ def test_load_default_config(monkeypatch, default_config): "config_dir", } # Check that only allowed keys are in it - assert set(default_cfg) == set(cfg) + assert set(default_cfg) == set(session) # Check that all required directories are available - assert all(hasattr(cfg, attr) for attr in directory_attrs) + assert all(hasattr(session, attr) for attr in directory_attrs) # Check default values for key in default_cfg: - assert cfg[key] == default_cfg[key] + assert session[key] == default_cfg[key] # Check output directories - assert str(cfg.session_dir).startswith( + assert str(session.session_dir).startswith( str(Path.home() / "esmvaltool_output" / "recipe_example") ) for path in ("preproc", "work", "run"): - assert getattr(cfg, path + "_dir") == cfg.session_dir / path - assert cfg.plot_dir == cfg.session_dir / "plots" - assert cfg.config_dir == Path(esmvalcore.__file__).parent + assert getattr(session, path + "_dir") == session.session_dir / path + assert session.plot_dir == session.session_dir / "plots" + with pytest.warns(ESMValCoreDeprecationWarning): + assert session.config_dir is None # Check that projects were configured assert project_cfg -def test_rootpath_obs4mips_case_correction(default_config): +def test_rootpath_obs4mips_case_correction(monkeypatch): """Test that the name of the obs4MIPs project is correct in rootpath.""" - CFG["rootpath"] = {"obs4mips": "/path/to/data"} + monkeypatch.setitem(CFG, "rootpath", {"obs4mips": "/path/to/data"}) assert "obs4mips" not in CFG["rootpath"] assert CFG["rootpath"]["obs4MIPs"] == [Path("/path/to/data")] -def test_drs_obs4mips_case_correction(default_config): +def test_drs_obs4mips_case_correction(monkeypatch): """Test that the name of the obs4MIPs project is correct in rootpath.""" - CFG["drs"] = {"obs4mips": "ESGF"} + monkeypatch.setitem(CFG, "drs", {"obs4mips": "ESGF"}) assert "obs4mips" not in CFG["drs"] assert CFG["drs"]["obs4MIPs"] == "ESGF" diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py index ac301fb43e..fa6c3111b3 100644 --- a/tests/unit/config/test_config_object.py +++ b/tests/unit/config/test_config_object.py @@ -1,28 +1,21 @@ -import contextlib import os -import sys from collections.abc import MutableMapping -from copy import deepcopy from pathlib import Path +from textwrap import dedent import pytest import esmvalcore import esmvalcore.config._config_object from esmvalcore.config import Config, Session -from esmvalcore.exceptions import InvalidConfigParameter +from esmvalcore.config._config_object import DEFAULT_CONFIG_DIR +from esmvalcore.exceptions import ( + ESMValCoreDeprecationWarning, + InvalidConfigParameter, +) from tests.integration.test_main import arguments -@contextlib.contextmanager -def environment(**kwargs): - """Temporary environment variables.""" - backup = deepcopy(os.environ) - os.environ = kwargs - yield - os.environ = backup - - def test_config_class(): config = { "log_level": "info", @@ -69,14 +62,17 @@ def test_config_init(): assert isinstance(config, MutableMapping) +# TODO: remove in v2.14.0 def test_load_from_file(monkeypatch): - default_config_file = Path(esmvalcore.__file__).parent / "config-user.yml" + default_config_file = DEFAULT_CONFIG_DIR / "config-user.yml" config = Config() assert not config - config.load_from_file(default_config_file) + with pytest.warns(ESMValCoreDeprecationWarning): + config.load_from_file(default_config_file) assert config +# TODO: remove in v2.14.0 def test_load_from_file_filenotfound(monkeypatch): """Test `Config.load_from_file`.""" config = Config() @@ -88,6 +84,7 @@ def test_load_from_file_filenotfound(monkeypatch): config.load_from_file("not_existent_file.yml") +# TODO: remove in v2.14.0 def test_load_from_file_invalidconfigparameter(monkeypatch, tmp_path): """Test `Config.load_from_file`.""" monkeypatch.chdir(tmp_path) @@ -111,23 +108,31 @@ def test_config_key_error(): config["invalid_key"] -def test_reload(): +def test_reload(cfg_default, monkeypatch, tmp_path): """Test `Config.reload`.""" - cfg_path = Path(esmvalcore.__file__).parent / "config-user.yml" - config = Config(config_file=cfg_path) - config.reload() - assert config["config_file"] == cfg_path + monkeypatch.setattr( + esmvalcore.config._config_object, + "USER_CONFIG_DIR", + tmp_path / "this" / "is" / "an" / "empty" / "dir", + ) + cfg = Config() + cfg.reload() -def test_reload_fail(): + assert cfg == cfg_default + + +def test_reload_fail(monkeypatch, tmp_path): """Test `Config.reload`.""" - config = Config() - msg = ( - "Cannot reload configuration, option 'config_file' is missing; make " - "sure to only use the `CFG` object from the `esmvalcore.config` module" + config_file = tmp_path / "invalid_config_file.yml" + config_file.write_text("invalid_option: 1") + monkeypatch.setattr( + esmvalcore.config._config_object, "USER_CONFIG_DIR", tmp_path ) - with pytest.raises(ValueError, match=msg): - config.reload() + cfg = Config() + + with pytest.raises(InvalidConfigParameter): + cfg.reload() def test_session(): @@ -146,6 +151,14 @@ def test_session_key_error(): session["invalid_key"] +# TODO: remove in v2.14.0 +def test_session_config_dir(): + session = Session({"config_file": "/path/to/config.yml"}) + with pytest.warns(ESMValCoreDeprecationWarning): + config_dir = session.config_dir + assert config_dir == Path("/path/to") + + TEST_GET_CFG_PATH = [ (None, None, None, "~/.esmvaltool/config-user.yml", False), ( @@ -158,7 +171,7 @@ def test_session_key_error(): ( None, None, - ("esmvaltool", "run", "--max-parallel-tasks=4"), + ("esmvaltool", "run", "--max_parallel_tasks=4"), "~/.esmvaltool/config-user.yml", True, ), @@ -264,6 +277,7 @@ def test_session_key_error(): ] +# TODO: remove in v2.14.0 @pytest.mark.parametrize( "filename,env,cli_args,output,env_var_set", TEST_GET_CFG_PATH ) @@ -271,21 +285,24 @@ def test_get_config_user_path( filename, env, cli_args, output, env_var_set, monkeypatch, tmp_path ): """Test `Config._get_config_user_path`.""" + monkeypatch.delenv("_ESMVALTOOL_USER_CONFIG_FILE_", raising=False) + # Create empty test file monkeypatch.chdir(tmp_path) (tmp_path / "existing_cfg.yml").write_text("") - if env is None: - env = {} - if cli_args is None: - cli_args = sys.argv - if output == "existing_cfg.yml": output = tmp_path / "existing_cfg.yml" else: output = Path(output).expanduser() - with environment(**env), arguments(*cli_args): + if env is not None: + for key, val in env.items(): + monkeypatch.setenv(key, val) + if cli_args is None: + cli_args = ["python"] + + with arguments(*cli_args): config_path = Config._get_config_user_path(filename) if env_var_set: assert os.environ["_ESMVALTOOL_USER_CONFIG_FILE_"] == str(output) @@ -295,6 +312,7 @@ def test_get_config_user_path( assert config_path == output +# TODO: remove in v2.14.0 def test_load_user_config_filenotfound(): """Test `Config._load_user_config`.""" expected_path = Path.home() / ".esmvaltool" / "not_existent_file.yml" @@ -303,6 +321,13 @@ def test_load_user_config_filenotfound(): Config._load_user_config("not_existent_file.yml") +# TODO: remove in v2.14.0 +def test_load_user_config_no_exception(): + """Test `Config._load_user_config`.""" + Config._load_user_config("not_existent_file.yml", raise_exception=False) + + +# TODO: remove in v2.14.0 def test_load_user_config_invalidconfigparameter(monkeypatch, tmp_path): """Test `Config._load_user_config`.""" monkeypatch.chdir(tmp_path) @@ -315,3 +340,177 @@ def test_load_user_config_invalidconfigparameter(monkeypatch, tmp_path): ) with pytest.raises(InvalidConfigParameter, match=msg): Config._load_user_config(cfg_path) + + +def test_get_user_config_dir_and_source_with_env(tmp_path, monkeypatch): + """Test `_get_user_config_dir` and `_get_user_config_source`.""" + monkeypatch.setenv("ESMVALTOOL_CONFIG_DIR", str(tmp_path)) + + config_dir = esmvalcore.config._config_object._get_user_config_dir() + config_src = esmvalcore.config._config_object._get_user_config_source() + + assert config_dir == tmp_path + assert config_src == "ESMVALTOOL_CONFIG_DIR environment variable" + + +def test_get_user_config_dir_and_source_no_env(tmp_path, monkeypatch): + """Test `_get_user_config_dir` and `_get_user_config_source`.""" + monkeypatch.delenv("ESMVALTOOL_CONFIG_DIR", raising=False) + + config_dir = esmvalcore.config._config_object._get_user_config_dir() + config_src = esmvalcore.config._config_object._get_user_config_source() + + assert config_dir == Path("~/.config/esmvaltool").expanduser() + assert config_src == "default user configuration directory" + + +def test_get_user_config_dir_with_env_fail(tmp_path, monkeypatch): + """Test `_get_user_config_dir` and `_get_user_config_source`.""" + empty_path = tmp_path / "this" / "does" / "not" / "exist" + monkeypatch.setenv("ESMVALTOOL_CONFIG_DIR", str(empty_path)) + + msg = ( + "Invalid configuration directory specified via ESMVALTOOL_CONFIG_DIR " + "environment variable:" + ) + with pytest.raises(NotADirectoryError, match=msg): + esmvalcore.config._config_object._get_user_config_dir() + + +# TODO: remove in v2.14.0 +def test_get_global_config_deprecated(mocker, tmp_path): + """Test ``_get_global_config``.""" + config_file = tmp_path / "old_config_user.yml" + config_file.write_text("output_dir: /new/output/dir") + mocker.patch.object( + esmvalcore.config._config_object.Config, + "_get_config_user_path", + return_value=config_file, + ) + with pytest.warns(ESMValCoreDeprecationWarning): + cfg = esmvalcore.config._config_object._get_global_config() + + assert cfg["output_dir"] == Path("/new/output/dir") + + +@pytest.mark.parametrize( + "dirs,output_file_type,rootpath", + [ + ([], "png", {"default": "~/climate_data"}), + (["/this/path/does/not/exist"], "png", {"default": "~/climate_data"}), + (["{tmp_path}/config1"], "1", {"default": "1", "1": "1"}), + ( + ["{tmp_path}/config1", "/this/path/does/not/exist"], + "1", + {"default": "1", "1": "1"}, + ), + ( + ["{tmp_path}/config1", "{tmp_path}/config2"], + "2b", + {"default": "2b", "1": "1", "2": "2b"}, + ), + ( + ["{tmp_path}/config2", "{tmp_path}/config1"], + "1", + {"default": "1", "1": "1", "2": "2b"}, + ), + ], +) +def test_load_from_dirs_always_default( + dirs, output_file_type, rootpath, tmp_path +): + """Test `Config.load_from_dirs`.""" + config1 = tmp_path / "config1" / "1.yml" + config2a = tmp_path / "config2" / "2a.yml" + config2b = tmp_path / "config2" / "2b.yml" + config1.parent.mkdir(parents=True, exist_ok=True) + config2a.parent.mkdir(parents=True, exist_ok=True) + config1.write_text( + dedent( + """ + output_file_type: '1' + rootpath: + default: '1' + '1': '1' + """ + ) + ) + config2a.write_text( + dedent( + """ + output_file_type: '2a' + rootpath: + default: '2a' + '2': '2a' + """ + ) + ) + config2b.write_text( + dedent( + """ + output_file_type: '2b' + rootpath: + default: '2b' + '2': '2b' + """ + ) + ) + + config_dirs = [] + for dir_ in dirs: + config_dirs.append(dir_.format(tmp_path=str(tmp_path))) + for name, path in rootpath.items(): + path = Path(path).expanduser().absolute() + rootpath[name] = [path] + + cfg = Config() + assert not cfg + + cfg.load_from_dirs(config_dirs) + + assert cfg["output_file_type"] == output_file_type + assert cfg["rootpath"] == rootpath + + +@pytest.mark.parametrize( + "cli_config_dir,output", + [ + (None, [DEFAULT_CONFIG_DIR, "~/.config/esmvaltool"]), + (Path("/c"), [DEFAULT_CONFIG_DIR, "~/.config/esmvaltool", "/c"]), + ], +) +def test_get_all_config_dirs(cli_config_dir, output, monkeypatch): + """Test `_get_all_config_dirs`.""" + monkeypatch.delenv("ESMVALTOOL_CONFIG_DIR", raising=False) + excepted = [] + for out in output: + excepted.append(Path(out).expanduser().absolute()) + + config_dirs = esmvalcore.config._config_object._get_all_config_dirs( + cli_config_dir + ) + + assert config_dirs == excepted + + +@pytest.mark.parametrize( + "cli_config_dir,output", + [ + (None, ["defaults", "default user configuration directory"]), + ( + Path("/c"), + [ + "defaults", + "default user configuration directory", + "command line argument", + ], + ), + ], +) +def test_get_all_config_sources(cli_config_dir, output, monkeypatch): + """Test `_get_all_config_sources`.""" + monkeypatch.delenv("ESMVALTOOL_CONFIG_DIR", raising=False) + config_srcs = esmvalcore.config._config_object._get_all_config_sources( + cli_config_dir + ) + assert config_srcs == output diff --git a/tests/unit/config/test_config_validator.py b/tests/unit/config/test_config_validator.py index 1a8283ce4b..eb2bad19cd 100644 --- a/tests/unit/config/test_config_validator.py +++ b/tests/unit/config/test_config_validator.py @@ -6,6 +6,7 @@ import esmvalcore from esmvalcore import __version__ as current_version +from esmvalcore.config import CFG from esmvalcore.config._config_validators import ( _handle_deprecation, _listify_validator, @@ -331,3 +332,11 @@ def test_validate_config_developer(tmp_path): # Restore original config-developer file validate_config_developer(None) + + +# TODO: remove in v2.14.0 +def test_extra_facets_dir_tuple_deprecated(monkeypatch): + """Test extra_facets_dir.""" + with pytest.warns(ESMValCoreDeprecationWarning): + monkeypatch.setitem(CFG, "extra_facets_dir", ("/extra/facets",)) + assert CFG["extra_facets_dir"] == [Path("/extra/facets")] diff --git a/tests/unit/config/test_esgf_pyclient.py b/tests/unit/config/test_esgf_pyclient.py index f23813bf71..4f71674b58 100644 --- a/tests/unit/config/test_esgf_pyclient.py +++ b/tests/unit/config/test_esgf_pyclient.py @@ -46,7 +46,7 @@ def test_read_config_file(monkeypatch, tmp_path): def test_read_v25_config_file(monkeypatch, tmp_path): """Test function read_config_file for v2.5 and earlier. - For v2.5 and earlier, the config-file contained a single `url` + For v2.5 and earlier, the ESGF config file contained a single `url` instead of a list of `urls` to specify the ESGF index node. """ cfg_file = tmp_path / "esgf-pyclient.yml" diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py deleted file mode 100644 index edc9340fb9..0000000000 --- a/tests/unit/conftest.py +++ /dev/null @@ -1,14 +0,0 @@ -import copy - -import pytest - -from esmvalcore.config import CFG -from esmvalcore.config._config_object import CFG_DEFAULT - - -@pytest.fixture -def session(tmp_path, monkeypatch): - for key, value in CFG_DEFAULT.items(): - monkeypatch.setitem(CFG, key, copy.deepcopy(value)) - monkeypatch.setitem(CFG, "output_dir", tmp_path / "esmvaltool_output") - return CFG.start_session("recipe_test") diff --git a/tests/unit/main/test_esmvaltool.py b/tests/unit/main/test_esmvaltool.py index b6a5b96599..e498cef670 100644 --- a/tests/unit/main/test_esmvaltool.py +++ b/tests/unit/main/test_esmvaltool.py @@ -8,11 +8,12 @@ import esmvalcore._main import esmvalcore._task import esmvalcore.config +import esmvalcore.config._config_object import esmvalcore.config._logging import esmvalcore.esgf from esmvalcore import __version__ from esmvalcore._main import HEADER, ESMValTool -from esmvalcore.exceptions import RecipeError +from esmvalcore.exceptions import InvalidConfigParameter, RecipeError LOGGER = logging.getLogger(__name__) @@ -22,9 +23,10 @@ def cfg(mocker, tmp_path): """Mock `esmvalcore.config.CFG`.""" session = mocker.MagicMock() - cfg_dict = {} + cfg_dict = {"resume_from": []} session.__getitem__.side_effect = cfg_dict.__getitem__ session.__setitem__.side_effect = cfg_dict.__setitem__ + session.update.side_effect = cfg_dict.update output_dir = tmp_path / "esmvaltool_output" session.session_dir = output_dir / "recipe_test" @@ -54,7 +56,7 @@ def session(cfg): ("check_level", "strict"), ], ) -def test_run_command_line_config(mocker, cfg, argument, value): +def test_run_command_line_config(mocker, cfg, argument, value, tmp_path): """Check that the configuration is updated from the command line.""" mocker.patch.object( esmvalcore.config, @@ -65,17 +67,19 @@ def test_run_command_line_config(mocker, cfg, argument, value): program = ESMValTool() recipe_file = "/path/to/recipe_test.yml" - config_file = "/path/to/config-user.yml" + config_dir = tmp_path / "config" + config_dir.mkdir(parents=True, exist_ok=True) mocker.patch.object(program, "_get_recipe", return_value=Path(recipe_file)) mocker.patch.object(program, "_run") - program.run(recipe_file, config_file, **{argument: value}) + program.run(recipe_file, config_dir=config_dir, **{argument: value}) - cfg.load_from_file.assert_called_with(config_file) cfg.start_session.assert_called_once_with(Path(recipe_file).stem) program._get_recipe.assert_called_with(recipe_file) - program._run.assert_called_with(program._get_recipe.return_value, session) + program._run.assert_called_with( + program._get_recipe.return_value, session, config_dir + ) assert session[argument] == value @@ -84,7 +88,6 @@ def test_run_command_line_config(mocker, cfg, argument, value): def test_run(mocker, session, search_esgf): session["search_esgf"] = search_esgf session["log_level"] = "default" - session["config_file"] = "/path/to/config-user.yml" session["remove_preproc_dir"] = True session["save_intermediary_cubes"] = False session.cmor_log.read_text.return_value = "WARNING: attribute not present" @@ -113,7 +116,7 @@ def test_run(mocker, session, search_esgf): create_autospec=True, ) - ESMValTool()._run(recipe, session=session) + ESMValTool()._run(recipe, session=session, cli_config_dir=None) # Check that the correct functions have been called esmvalcore.config._logging.configure_logging.assert_called_once_with( @@ -150,6 +153,36 @@ def test_run_session_dir_exists_alternative_fails(mocker, session): program._create_session_dir(session) +def test_run_missing_config_dir(tmp_path): + """Test `ESMValTool.run`.""" + config_dir = tmp_path / "path" / "does" / "not" / "exist" + program = ESMValTool() + + msg = ( + f"Invalid --config_dir given: {config_dir} is not an existing " + f"directory" + ) + with pytest.raises(NotADirectoryError, match=msg): + program.run("/recipe_dir/recipe_test.yml", config_dir=config_dir) + + +def test_run_invalid_config_dir(monkeypatch, tmp_path): + """Test `ESMValTool.run`.""" + monkeypatch.delitem( # TODO: remove in v2.14.0 + esmvalcore.config.CFG._mapping, "config_file", raising=False + ) + config_path = tmp_path / "config.yml" + config_path.write_text("invalid: option") + program = ESMValTool() + + msg = ( + rf"Failed to parse configuration directory {tmp_path} \(command line " + rf"argument\): `invalid` is not a valid config parameter." + ) + with pytest.raises(InvalidConfigParameter, match=msg): + program.run("/recipe_dir/recipe_test.yml", config_dir=tmp_path) + + def test_clean_preproc_dir(session): session.preproc_dir.mkdir(parents=True) session._fixed_file_dir.mkdir(parents=True) @@ -173,16 +206,25 @@ def test_do_not_clean_preproc_dir(session): @mock.patch("esmvalcore._main.entry_points") -def test_header(mock_entry_points, caplog): +def test_header(mock_entry_points, monkeypatch, tmp_path, caplog): + tmp_path.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr( + esmvalcore.config._config_object, "USER_CONFIG_DIR", tmp_path + ) + monkeypatch.setattr( + esmvalcore.config._config_object, "USER_CONFIG_SOURCE", "SOURCE" + ) entry_point = mock.Mock() entry_point.dist.name = "MyEntry" entry_point.dist.version = "v42.42.42" entry_point.name = "Entry name" mock_entry_points.return_value = [entry_point] + cli_config_dir = tmp_path / "this" / "does" / "not" / "exist" + with caplog.at_level(logging.INFO): ESMValTool()._log_header( - "path_to_config_file", ["path_to_log_file1", "path_to_log_file2"], + cli_config_dir, ) assert len(caplog.messages) == 8 @@ -192,7 +234,13 @@ def test_header(mock_entry_points, caplog): assert caplog.messages[3] == f"ESMValCore: {__version__}" assert caplog.messages[4] == "MyEntry: v42.42.42" assert caplog.messages[5] == "----------------" - assert caplog.messages[6] == "Using config file path_to_config_file" + assert caplog.messages[6] == ( + f"Reading configuration files from:\n" + f"{Path(esmvalcore.__file__).parent}/config/configurations/defaults " + f"(defaults)\n" + f"{tmp_path} (SOURCE)\n" + f"{cli_config_dir} [NOT AN EXISTING DIRECTORY] (command line argument)" + ) assert caplog.messages[7] == ( "Writing program log files to:\n" "path_to_log_file1\n" diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 66d23306ec..8408c622b9 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -9,7 +9,7 @@ import esmvalcore.dataset import esmvalcore.local from esmvalcore.cmor.check import CheckLevels -from esmvalcore.config import CFG +from esmvalcore.config import CFG, Session from esmvalcore.dataset import Dataset from esmvalcore.esgf import ESGFFile from esmvalcore.exceptions import InputFilesNotFound, RecipeError @@ -112,7 +112,7 @@ def test_session_setter(): ds.session - assert isinstance(ds.session, esmvalcore.config.Session) + assert isinstance(ds.session, Session) assert ds.session == ds.supplementaries[0].session From e7a78567012622c6569402856fcd373ff3376aae Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Thu, 3 Oct 2024 12:44:53 +0200 Subject: [PATCH 02/19] Ignore reformatting when viewing git blame (#2539) --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..93ce6ae46a --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Use ruff formatter (#2524) +436558caacda69d4966a5aff35959ce9188cac37 From 5b283729c37879c3831ebb1428d056b9d9d7c05f Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 3 Oct 2024 13:44:30 +0100 Subject: [PATCH 03/19] add a pre-commit badge to README (#2534) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f3e5c693bf..667966da1b 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ [![Docker Build Status](https://img.shields.io/docker/cloud/build/esmvalgroup/esmvalcore)](https://hub.docker.com/r/esmvalgroup/esmvalcore/) [![Anaconda-Server Badge](https://img.shields.io/conda/vn/conda-forge/ESMValCore?color=blue&label=conda-forge&logo=conda-forge&logoColor=white)](https://anaconda.org/conda-forge/esmvalcore) [![Github Actions Test](https://github.com/ESMValGroup/ESMValCore/actions/workflows/run-tests.yml/badge.svg)](https://github.com/ESMValGroup/ESMValCore/actions/workflows/run-tests.yml) +[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/ESMValGroup/ESMValCore/main.svg)](https://results.pre-commit.ci/latest/github/ESMValGroup/ESMValCore/main) ![esmvaltoollogo](https://raw.githubusercontent.com/ESMValGroup/ESMValCore/main/doc/figures/ESMValTool-logo-2-glow.png) From 97de18dd02ffde0626c144b9e79b66ea37b7bf3c Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Mon, 7 Oct 2024 13:22:48 +0100 Subject: [PATCH 04/19] Free esmpy of ` >=8.6.0` pin and pin `iris-grib >=0.20.0` (#2542) Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com> --- environment.yml | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index eaf317965f..636883f04a 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - dask-jobqueue - distributed - esgf-pyclient >=0.3.1 - - esmpy >=8.6.0 # github.com/SciTools-incubator/iris-esmf-regrid/pull/342 + - esmpy !=8.1.0 - filelock - fiona - fire @@ -20,7 +20,7 @@ dependencies: - humanfriendly - iris >=3.10.0 - iris-esmf-regrid >=0.11.0 - - iris-grib + - iris-grib >=0.20.0 # github.com/ESMValGroup/ESMValCore/issues/2535 - isodate - jinja2 - libnetcdf !=4.9.1 # to avoid hdf5 warnings diff --git a/setup.py b/setup.py index 32d2e15e27..15ea79aae3 100755 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ "fire", "geopy", "humanfriendly", - "iris-grib", + "iris-grib>=0.20.0", # github.com/ESMValGroup/ESMValCore/issues/2535 "isodate", "jinja2", "nc-time-axis", # needed by iris.plot From 6f347916a622d19b9cd34d000b62c5dd7740476c Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Mon, 7 Oct 2024 18:02:04 +0200 Subject: [PATCH 05/19] Fix tests if deprecated `~/.esmvaltool/config-user.yml` file is available (#2543) --- tests/conftest.py | 12 +++++++- tests/unit/config/test_config_object.py | 37 ++++++++++++++++++------- tests/unit/main/test_esmvaltool.py | 14 ++++++---- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5fd7be7460..0d385fb2f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ import pytest -from esmvalcore.config import CFG +from esmvalcore.config import CFG, Config @pytest.fixture @@ -22,3 +22,13 @@ def session(tmp_path: Path, cfg_default, monkeypatch): monkeypatch.setitem(CFG, "rootpath", {"default": {tmp_path: "default"}}) monkeypatch.setitem(CFG, "output_dir", tmp_path / "esmvaltool_output") return CFG.start_session("recipe_test") + + +# TODO: remove in v2.14.0 +@pytest.fixture(autouse=True) +def ignore_old_config_user(tmp_path, monkeypatch): + """Ignore potentially existing old config-user.yml file in all tests.""" + nonexistent_config_dir = tmp_path / "nonexistent_config_dir" + monkeypatch.setattr( + Config, "_DEFAULT_USER_CONFIG_DIR", nonexistent_config_dir + ) diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py index fa6c3111b3..0a8a73f063 100644 --- a/tests/unit/config/test_config_object.py +++ b/tests/unit/config/test_config_object.py @@ -73,12 +73,14 @@ def test_load_from_file(monkeypatch): # TODO: remove in v2.14.0 -def test_load_from_file_filenotfound(monkeypatch): +def test_load_from_file_filenotfound(monkeypatch, tmp_path): """Test `Config.load_from_file`.""" config = Config() assert not config - expected_path = Path.home() / ".esmvaltool" / "not_existent_file.yml" + expected_path = ( + tmp_path / "nonexistent_config_dir" / "not_existent_file.yml" + ) msg = f"Config file '{expected_path}' does not exist" with pytest.raises(FileNotFoundError, match=msg): config.load_from_file("not_existent_file.yml") @@ -110,6 +112,9 @@ def test_config_key_error(): def test_reload(cfg_default, monkeypatch, tmp_path): """Test `Config.reload`.""" + # TODO: remove in v2.14.0 + monkeypatch.delenv("_ESMVALTOOL_USER_CONFIG_FILE_", raising=False) + monkeypatch.setattr( esmvalcore.config._config_object, "USER_CONFIG_DIR", @@ -124,6 +129,9 @@ def test_reload(cfg_default, monkeypatch, tmp_path): def test_reload_fail(monkeypatch, tmp_path): """Test `Config.reload`.""" + # TODO: remove in v2.14.0 + monkeypatch.delenv("_ESMVALTOOL_USER_CONFIG_FILE_", raising=False) + config_file = tmp_path / "invalid_config_file.yml" config_file.write_text("invalid_option: 1") monkeypatch.setattr( @@ -160,26 +168,32 @@ def test_session_config_dir(): TEST_GET_CFG_PATH = [ - (None, None, None, "~/.esmvaltool/config-user.yml", False), + ( + None, + None, + None, + "{tmp_path}/nonexistent_config_dir/config-user.yml", + False, + ), ( None, None, ("any_other_module", "--config_file=cli.yml"), - "~/.esmvaltool/config-user.yml", + "{tmp_path}/nonexistent_config_dir/config-user.yml", False, ), ( None, None, ("esmvaltool", "run", "--max_parallel_tasks=4"), - "~/.esmvaltool/config-user.yml", + "{tmp_path}/nonexistent_config_dir/config-user.yml", True, ), ( None, None, ("esmvaltool", "--config_file"), - "~/.esmvaltool/config-user.yml", + "{tmp_path}/nonexistent_config_dir/config-user.yml", True, ), ( @@ -214,7 +228,7 @@ def test_session_config_dir(): None, None, ("esmvaltool", "run", "--config-file=relative_cli.yml"), - "~/.esmvaltool/relative_cli.yml", + "{tmp_path}/nonexistent_config_dir/relative_cli.yml", True, ), ( @@ -264,7 +278,7 @@ def test_session_config_dir(): "filename.yml", None, None, - "~/.esmvaltool/filename.yml", + "{tmp_path}/nonexistent_config_dir/filename.yml", False, ), ( @@ -285,6 +299,7 @@ def test_get_config_user_path( filename, env, cli_args, output, env_var_set, monkeypatch, tmp_path ): """Test `Config._get_config_user_path`.""" + output = output.format(tmp_path=tmp_path) monkeypatch.delenv("_ESMVALTOOL_USER_CONFIG_FILE_", raising=False) # Create empty test file @@ -313,9 +328,11 @@ def test_get_config_user_path( # TODO: remove in v2.14.0 -def test_load_user_config_filenotfound(): +def test_load_user_config_filenotfound(tmp_path): """Test `Config._load_user_config`.""" - expected_path = Path.home() / ".esmvaltool" / "not_existent_file.yml" + expected_path = ( + tmp_path / "nonexistent_config_dir" / "not_existent_file.yml" + ) msg = f"Config file '{expected_path}' does not exist" with pytest.raises(FileNotFoundError, match=msg): Config._load_user_config("not_existent_file.yml") diff --git a/tests/unit/main/test_esmvaltool.py b/tests/unit/main/test_esmvaltool.py index e498cef670..7b9cb29662 100644 --- a/tests/unit/main/test_esmvaltool.py +++ b/tests/unit/main/test_esmvaltool.py @@ -205,8 +205,11 @@ def test_do_not_clean_preproc_dir(session): assert session._fixed_file_dir.exists() +@mock.patch("esmvalcore._main.ESMValTool._get_config_info") @mock.patch("esmvalcore._main.entry_points") -def test_header(mock_entry_points, monkeypatch, tmp_path, caplog): +def test_header( + mock_entry_points, mock_get_config_info, monkeypatch, tmp_path, caplog +): tmp_path.mkdir(parents=True, exist_ok=True) monkeypatch.setattr( esmvalcore.config._config_object, "USER_CONFIG_DIR", tmp_path @@ -221,6 +224,9 @@ def test_header(mock_entry_points, monkeypatch, tmp_path, caplog): mock_entry_points.return_value = [entry_point] cli_config_dir = tmp_path / "this" / "does" / "not" / "exist" + # TODO: remove in v2.14.0 + mock_get_config_info.return_value = "config_dir (SOURCE)" + with caplog.at_level(logging.INFO): ESMValTool()._log_header( ["path_to_log_file1", "path_to_log_file2"], @@ -235,11 +241,7 @@ def test_header(mock_entry_points, monkeypatch, tmp_path, caplog): assert caplog.messages[4] == "MyEntry: v42.42.42" assert caplog.messages[5] == "----------------" assert caplog.messages[6] == ( - f"Reading configuration files from:\n" - f"{Path(esmvalcore.__file__).parent}/config/configurations/defaults " - f"(defaults)\n" - f"{tmp_path} (SOURCE)\n" - f"{cli_config_dir} [NOT AN EXISTING DIRECTORY] (command line argument)" + "Reading configuration files from:\nconfig_dir (SOURCE)" ) assert caplog.messages[7] == ( "Writing program log files to:\n" From 99c6a7be1e995f8b4fcb61f3a44bd46aa828c397 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:25:33 +0200 Subject: [PATCH 06/19] Add public `Config.update_from_dirs()` method (#2538) Co-authored-by: Bouwe Andela --- doc/api/esmvalcore.config.rst | 7 ++ esmvalcore/_main.py | 10 +-- esmvalcore/config/_config_object.py | 56 +++++++++--- tests/unit/config/test_config_object.py | 111 ++++++++++++++++++------ 4 files changed, 141 insertions(+), 43 deletions(-) diff --git a/doc/api/esmvalcore.config.rst b/doc/api/esmvalcore.config.rst index 9b01587263..231140effd 100644 --- a/doc/api/esmvalcore.config.rst +++ b/doc/api/esmvalcore.config.rst @@ -88,6 +88,13 @@ To load the configuration object from custom directories, use: >>> dirs = ['my/default/config', 'my/custom/config'] >>> CFG.load_from_dirs(dirs) +To update the existing configuration object from custom directories, use: + +.. code-block:: python + + >>> dirs = ['my/default/config', 'my/custom/config'] + >>> CFG.update_from_dirs(dirs) + Session ******* diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index d0bd6fcf10..451f228ae8 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -403,7 +403,6 @@ def run(self, recipe, **kwargs): """ from .config import CFG - from .config._config_object import _get_all_config_dirs from .exceptions import InvalidConfigParameter cli_config_dir = kwargs.pop("config_dir", None) @@ -427,10 +426,9 @@ def run(self, recipe, **kwargs): # New in v2.12.0: read additional configuration directory given by CLI # argument - if CFG.get("config_file") is None: # remove in v2.14.0 - config_dirs = _get_all_config_dirs(cli_config_dir) + if CFG.get("config_file") is None and cli_config_dir is not None: try: - CFG.load_from_dirs(config_dirs) + CFG.update_from_dirs([cli_config_dir]) # Potential errors must come from --config_dir (i.e., # cli_config_dir) since other sources have already been read (and @@ -458,8 +456,8 @@ def run(self, recipe, **kwargs): # New in v2.12.0 else: - config_dirs = _get_all_config_dirs(cli_config_dir) # remove v2.14 - CFG.load_from_dirs(config_dirs) + if cli_config_dir is not None: + CFG.update_from_dirs([cli_config_dir]) @staticmethod def _create_session_dir(session): diff --git a/esmvalcore/config/_config_object.py b/esmvalcore/config/_config_object.py index dfe784ef58..baa344f829 100644 --- a/esmvalcore/config/_config_object.py +++ b/esmvalcore/config/_config_object.py @@ -317,8 +317,17 @@ def load_from_file( self.clear() self.update(Config._load_user_config(filename)) + @staticmethod + def _get_config_dict_from_dirs(dirs: Iterable[str | Path]) -> dict: + """Get configuration :obj:`dict` from directories.""" + dirs_str: list[str] = [] + for config_dir in dirs: + config_dir = Path(config_dir).expanduser().absolute() + dirs_str.append(str(config_dir)) + return dask.config.collect(paths=dirs_str, env={}) + def load_from_dirs(self, dirs: Iterable[str | Path]) -> None: - """Load configuration object from directories. + """Clear and load configuration object from directories. This searches for all YAML files within the given directories and merges them together using :func:`dask.config.collect`. Nested objects @@ -344,23 +353,17 @@ def load_from_dirs(self, dirs: Iterable[str | Path]) -> None: Invalid configuration option given. """ - dirs_str: list[str] = [] - # Always consider default options; these have the lowest priority - dirs_str.append(str(DEFAULT_CONFIG_DIR)) + dirs = [DEFAULT_CONFIG_DIR] + list(dirs) - for config_dir in dirs: - config_dir = Path(config_dir).expanduser().absolute() - dirs_str.append(str(config_dir)) - - new_config_dict = dask.config.collect(paths=dirs_str, env={}) + new_config_dict = self._get_config_dict_from_dirs(dirs) self.clear() self.update(new_config_dict) self.check_missing() def reload(self) -> None: - """Reload the configuration object. + """Clear and reload the configuration object. This will read all YAML files in the user configuration directory (by default ``~/.config/esmvaltool``, but this can be changed with the @@ -431,6 +434,39 @@ def start_session(self, name: str) -> Session: session = Session(config=self.copy(), name=name) return session + def update_from_dirs(self, dirs: Iterable[str | Path]) -> None: + """Update configuration object from directories. + + This will first search for all YAML files within the given directories + and merge them together using :func:`dask.config.collect` (if identical + values are provided in multiple files, the value from the last file + will be used). Then, the current configuration is merged with these + new configuration options using :func:`dask.config.merge` (new values + are preferred over old values). Nested objects are properly considered; + see :func:`dask.config.update` for details. + + Note + ---- + Just like :func:`dask.config.collect`, this silently ignores + non-existing directories. + + Parameters + ---------- + dirs: + A list of directories to search for YAML configuration files. + + Raises + ------ + esmvalcore.exceptions.InvalidConfigParameter + Invalid configuration option given. + + """ + new_config_dict = self._get_config_dict_from_dirs(dirs) + merged_config_dict = dask.config.merge(self, new_config_dict) + self.update(merged_config_dict) + + self.check_missing() + class Session(ValidatedConfig): """Container class for session configuration and directory information. diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py index 0a8a73f063..02da498562 100644 --- a/tests/unit/config/test_config_object.py +++ b/tests/unit/config/test_config_object.py @@ -410,33 +410,8 @@ def test_get_global_config_deprecated(mocker, tmp_path): assert cfg["output_dir"] == Path("/new/output/dir") -@pytest.mark.parametrize( - "dirs,output_file_type,rootpath", - [ - ([], "png", {"default": "~/climate_data"}), - (["/this/path/does/not/exist"], "png", {"default": "~/climate_data"}), - (["{tmp_path}/config1"], "1", {"default": "1", "1": "1"}), - ( - ["{tmp_path}/config1", "/this/path/does/not/exist"], - "1", - {"default": "1", "1": "1"}, - ), - ( - ["{tmp_path}/config1", "{tmp_path}/config2"], - "2b", - {"default": "2b", "1": "1", "2": "2b"}, - ), - ( - ["{tmp_path}/config2", "{tmp_path}/config1"], - "1", - {"default": "1", "1": "1", "2": "2b"}, - ), - ], -) -def test_load_from_dirs_always_default( - dirs, output_file_type, rootpath, tmp_path -): - """Test `Config.load_from_dirs`.""" +def _setup_config_dirs(tmp_path): + """Setup test configuration directories.""" config1 = tmp_path / "config1" / "1.yml" config2a = tmp_path / "config2" / "2a.yml" config2b = tmp_path / "config2" / "2b.yml" @@ -473,6 +448,36 @@ def test_load_from_dirs_always_default( ) ) + +@pytest.mark.parametrize( + "dirs,output_file_type,rootpath", + [ + ([], "png", {"default": "~/climate_data"}), + (["/this/path/does/not/exist"], "png", {"default": "~/climate_data"}), + (["{tmp_path}/config1"], "1", {"default": "1", "1": "1"}), + ( + ["{tmp_path}/config1", "/this/path/does/not/exist"], + "1", + {"default": "1", "1": "1"}, + ), + ( + ["{tmp_path}/config1", "{tmp_path}/config2"], + "2b", + {"default": "2b", "1": "1", "2": "2b"}, + ), + ( + ["{tmp_path}/config2", "{tmp_path}/config1"], + "1", + {"default": "1", "1": "1", "2": "2b"}, + ), + ], +) +def test_load_from_dirs_always_default( + dirs, output_file_type, rootpath, tmp_path +): + """Test `Config.load_from_dirs`.""" + _setup_config_dirs(tmp_path) + config_dirs = [] for dir_ in dirs: config_dirs.append(dir_.format(tmp_path=str(tmp_path))) @@ -482,11 +487,14 @@ def test_load_from_dirs_always_default( cfg = Config() assert not cfg + cfg["rootpath"] = {"X": "x"} + cfg["search_esgf"] = "when_missing" cfg.load_from_dirs(config_dirs) assert cfg["output_file_type"] == output_file_type assert cfg["rootpath"] == rootpath + assert cfg["search_esgf"] == "never" @pytest.mark.parametrize( @@ -531,3 +539,52 @@ def test_get_all_config_sources(cli_config_dir, output, monkeypatch): cli_config_dir ) assert config_srcs == output + + +@pytest.mark.parametrize( + "dirs,output_file_type,rootpath", + [ + ([], None, {"X": "x"}), + (["/this/path/does/not/exist"], None, {"X": "x"}), + (["{tmp_path}/config1"], "1", {"default": "1", "1": "1", "X": "x"}), + ( + ["{tmp_path}/config1", "/this/path/does/not/exist"], + "1", + {"default": "1", "1": "1", "X": "x"}, + ), + ( + ["{tmp_path}/config1", "{tmp_path}/config2"], + "2b", + {"default": "2b", "1": "1", "2": "2b", "X": "x"}, + ), + ( + ["{tmp_path}/config2", "{tmp_path}/config1"], + "1", + {"default": "1", "1": "1", "2": "2b", "X": "x"}, + ), + ], +) +def test_update_from_dirs(dirs, output_file_type, rootpath, tmp_path): + """Test `Config.update_from_dirs`.""" + _setup_config_dirs(tmp_path) + + config_dirs = [] + for dir_ in dirs: + config_dirs.append(dir_.format(tmp_path=str(tmp_path))) + for name, path in rootpath.items(): + path = Path(path).expanduser().absolute() + rootpath[name] = [path] + + cfg = Config() + assert not cfg + cfg["rootpath"] = {"X": "x"} + cfg["search_esgf"] = "when_missing" + + cfg.update_from_dirs(config_dirs) + + if output_file_type is None: + assert "output_file_type" not in cfg + else: + assert cfg["output_file_type"] == output_file_type + assert cfg["rootpath"] == rootpath + assert cfg["search_esgf"] == "when_missing" From fd82b43e495cfe3a15650c369475feea0e9c54f8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 21:53:30 +0200 Subject: [PATCH 07/19] [pre-commit.ci] pre-commit autoupdate (#2544) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6dac9fed16..bfc435ab81 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ exclude: | repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-ast @@ -33,7 +33,7 @@ repos: - id: codespell additional_dependencies: [tomli] # required for Python 3.10 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.6.8" + rev: "v0.6.9" hooks: - id: ruff args: [--fix] From 892d4a3ab3757df4c90408d6326ebb71d9e61c76 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:35:21 +0200 Subject: [PATCH 08/19] Remove deprecated CMOR fix/check code (#2552) --- esmvalcore/cmor/check.py | 53 ------ esmvalcore/cmor/fix.py | 79 -------- esmvalcore/dataset.py | 2 - .../cmor/_fixes/obs4mips/test_airs_2_0.py | 9 +- tests/integration/cmor/test_fix.py | 46 +---- tests/unit/cmor/test_cmor_check.py | 29 --- tests/unit/cmor/test_fix.py | 177 +++++++----------- tests/unit/test_dataset.py | 2 - 8 files changed, 68 insertions(+), 329 deletions(-) diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 342d0fff66..a75dcdaab4 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import warnings from collections import namedtuple from collections.abc import Callable from enum import IntEnum @@ -20,7 +19,6 @@ from iris.coords import Coord from iris.cube import Cube -from esmvalcore.cmor._fixes.fix import GenericFix from esmvalcore.cmor._utils import ( _get_alternative_generic_lev_coord, _get_generic_lev_coord_names, @@ -28,7 +26,6 @@ _get_simplified_calendar, ) from esmvalcore.cmor.table import CoordinateInfo, get_var_info -from esmvalcore.exceptions import ESMValCoreDeprecationWarning from esmvalcore.iris_helpers import has_unstructured_grid @@ -70,18 +67,6 @@ class CMORCheck: fail_on_error: bool If true, CMORCheck stops on the first error. If false, it collects all possible errors before stopping. - automatic_fixes: bool - If True, CMORCheck will try to apply automatic fixes for any - detected error, if possible. - - .. deprecated:: 2.10.0 - This option has been deprecated in ESMValCore version 2.10.0 and is - scheduled for removal in version 2.12.0. Please use the functions - :func:`~esmvalcore.preprocessor.fix_metadata`, - :func:`~esmvalcore.preprocessor.fix_data`, or - :meth:`esmvalcore.dataset.Dataset.load` (which automatically - includes the first two functions) instead. Fixes and CMOR checks - have been clearly separated in ESMValCore version 2.10.0. check_level: CheckLevels Level of strictness of the checks. @@ -104,7 +89,6 @@ def __init__( frequency=None, fail_on_error=False, check_level=CheckLevels.DEFAULT, - automatic_fixes=False, ): self._cube = cube self._failerr = fail_on_error @@ -118,26 +102,6 @@ def __init__( if not frequency: frequency = self._cmor_var.frequency self.frequency = frequency - self.automatic_fixes = automatic_fixes - - # Deprecate automatic_fixes (remove in v2.12) - if automatic_fixes: - msg = ( - "The option `automatic_fixes` has been deprecated in " - "ESMValCore version 2.10.0 and is scheduled for removal in " - "version 2.12.0. Please use the functions " - "esmvalcore.preprocessor.fix_metadata(), " - "esmvalcore.preprocessor.fix_data(), or " - "esmvalcore.dataset.Dataset.load() (which automatically " - "includes the first two functions) instead. Fixes and CMOR " - "checks have been clearly separated in ESMValCore version " - "2.10.0." - ) - warnings.warn(msg, ESMValCoreDeprecationWarning) - - # TODO: remove in v2.12 - - self._generic_fix = GenericFix(var_info, frequency=frequency) @cached_property def _unstructured_grid(self) -> bool: @@ -171,10 +135,6 @@ def check_metadata(self, logger: Optional[logging.Logger] = None) -> Cube: if logger is not None: self._logger = logger - # TODO: remove in v2.12 - if self.automatic_fixes: - [self._cube] = self._generic_fix.fix_metadata([self._cube]) - self._check_var_metadata() self._check_fill_value() self._check_multiple_coords_same_stdname() @@ -220,10 +180,6 @@ def check_data(self, logger: Optional[logging.Logger] = None) -> Cube: if logger is not None: self._logger = logger - # TODO: remove in v2.12 - if self.automatic_fixes: - self._cube = self._generic_fix.fix_data(self._cube) - self._check_coords_data() self.report_debug_messages() @@ -345,7 +301,6 @@ def _check_var_metadata(self): def _get_effective_units(self): """Get effective units.""" - # TODO: remove entire function in v2.12 if self._cmor_var.units.lower() == "psu": units = "1.0" else: @@ -606,12 +561,6 @@ def _check_coords_data(self): except iris.exceptions.CoordinateNotFoundError: continue - # TODO: remove in v2.12 - if self.automatic_fixes: - (self._cube, coord) = self._generic_fix._fix_coord_direction( - self._cube, coordinate, coord - ) - self._check_coord_monotonicity_and_direction( coordinate, coord, var_name ) @@ -935,7 +884,6 @@ def _get_cmor_checker( frequency: None | str = None, fail_on_error: bool = False, check_level: CheckLevels = CheckLevels.DEFAULT, - automatic_fixes: bool = False, # TODO: remove in v2.12 ) -> Callable[[Cube], CMORCheck]: """Get a CMOR checker.""" var_info = get_var_info(project, mip, short_name) @@ -947,7 +895,6 @@ def _checker(cube: Cube) -> CMORCheck: frequency=frequency, fail_on_error=fail_on_error, check_level=check_level, - automatic_fixes=automatic_fixes, ) return _checker diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index 2e3209897d..ab81353cfb 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -8,7 +8,6 @@ from __future__ import annotations import logging -import warnings from collections import defaultdict from collections.abc import Sequence from pathlib import Path @@ -17,8 +16,6 @@ from iris.cube import Cube, CubeList from esmvalcore.cmor._fixes.fix import Fix -from esmvalcore.cmor.check import CheckLevels, _get_cmor_checker -from esmvalcore.exceptions import ESMValCoreDeprecationWarning if TYPE_CHECKING: from ..config import Session @@ -109,7 +106,6 @@ def fix_metadata( dataset: str, mip: str, frequency: Optional[str] = None, - check_level: CheckLevels = CheckLevels.DEFAULT, session: Optional[Session] = None, **extra_facets, ) -> CubeList: @@ -132,17 +128,6 @@ def fix_metadata( Variable's MIP. frequency: Variable's data frequency, if available. - check_level: - Level of strictness of the checks. - - .. deprecated:: 2.10.0 - This option has been deprecated in ESMValCore version 2.10.0 and is - scheduled for removal in version 2.12.0. Please use the functions - :func:`~esmvalcore.preprocessor.cmor_check_metadata`, - :func:`~esmvalcore.preprocessor.cmor_check_data`, or - :meth:`~esmvalcore.cmor.check.cmor_check` instead. This function - will no longer perform CMOR checks. Fixes and CMOR checks have been - clearly separated in ESMValCore version 2.10.0. session: Current session which includes configuration and directory information. **extra_facets: @@ -155,20 +140,6 @@ def fix_metadata( Fixed cubes. """ - # Deprecate CMOR checks (remove in v2.12) - if check_level != CheckLevels.DEFAULT: - msg = ( - "The option `check_level` has been deprecated in ESMValCore " - "version 2.10.0 and is scheduled for removal in version 2.12.0. " - "Please use the functions " - "esmvalcore.preprocessor.cmor_check_metadata, " - "esmvalcore.preprocessor.cmor_check_data, or " - "esmvalcore.cmor.check.cmor_check instead. This function will no " - "longer perform CMOR checks. Fixes and CMOR checks have been " - "clearly separated in ESMValCore version 2.10.0." - ) - warnings.warn(msg, ESMValCoreDeprecationWarning) - # Update extra_facets with variable information given as regular arguments # to this function extra_facets.update( @@ -207,18 +178,6 @@ def fix_metadata( # returns a single cube cube = cube_list[0] - # Perform CMOR checks - # TODO: remove in v2.12 - checker = _get_cmor_checker( - project, - mip, - short_name, - frequency, - fail_on_error=False, - check_level=check_level, - ) - cube = checker(cube).check_metadata() - cube.attributes.pop("source_file", None) fixed_cubes.append(cube) @@ -232,7 +191,6 @@ def fix_data( dataset: str, mip: str, frequency: Optional[str] = None, - check_level: CheckLevels = CheckLevels.DEFAULT, session: Optional[Session] = None, **extra_facets, ) -> Cube: @@ -257,17 +215,6 @@ def fix_data( Variable's MIP. frequency: Variable's data frequency, if available. - check_level: - Level of strictness of the checks. - - .. deprecated:: 2.10.0 - This option has been deprecated in ESMValCore version 2.10.0 and is - scheduled for removal in version 2.12.0. Please use the functions - :func:`~esmvalcore.preprocessor.cmor_check_metadata`, - :func:`~esmvalcore.preprocessor.cmor_check_data`, or - :meth:`~esmvalcore.cmor.check.cmor_check` instead. This function - will no longer perform CMOR checks. Fixes and CMOR checks have been - clearly separated in ESMValCore version 2.10.0. session: Current session which includes configuration and directory information. **extra_facets: @@ -280,20 +227,6 @@ def fix_data( Fixed cube. """ - # Deprecate CMOR checks (remove in v2.12) - if check_level != CheckLevels.DEFAULT: - msg = ( - "The option `check_level` has been deprecated in ESMValCore " - "version 2.10.0 and is scheduled for removal in version 2.12.0. " - "Please use the functions " - "esmvalcore.preprocessor.cmor_check_metadata, " - "esmvalcore.preprocessor.cmor_check_data, or " - "esmvalcore.cmor.check.cmor_check instead. This function will no " - "longer perform CMOR checks. Fixes and CMOR checks have been " - "clearly separated in ESMValCore version 2.10.0." - ) - warnings.warn(msg, ESMValCoreDeprecationWarning) - # Update extra_facets with variable information given as regular arguments # to this function extra_facets.update( @@ -317,16 +250,4 @@ def fix_data( ): cube = fix.fix_data(cube) - # Perform CMOR checks - # TODO: remove in v2.12 - checker = _get_cmor_checker( - project, - mip, - short_name, - frequency, - fail_on_error=False, - check_level=check_level, - ) - cube = checker(cube).check_data() - return cube diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index c436edfe8b..6717485e38 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -761,7 +761,6 @@ def _load(self) -> Cube: ), } settings["fix_metadata"] = { - "check_level": self.session["check_level"], "session": self.session, **self.facets, } @@ -778,7 +777,6 @@ def _load(self) -> Cube: "timerange": self.facets["timerange"], } settings["fix_data"] = { - "check_level": self.session["check_level"], "session": self.session, **self.facets, } diff --git a/tests/integration/cmor/_fixes/obs4mips/test_airs_2_0.py b/tests/integration/cmor/_fixes/obs4mips/test_airs_2_0.py index d82aba1640..8370f15e97 100644 --- a/tests/integration/cmor/_fixes/obs4mips/test_airs_2_0.py +++ b/tests/integration/cmor/_fixes/obs4mips/test_airs_2_0.py @@ -20,14 +20,7 @@ def test_fix_metadata_hur(): ] ) - fixed_cubes = fix_metadata( - cubes, - "hur", - "obs4MIPs", - "AIRS-2-0", - "Amon", - check_level=5, - ) + fixed_cubes = fix_metadata(cubes, "hur", "obs4MIPs", "AIRS-2-0", "Amon") assert len(fixed_cubes) == 1 fixed_cube = fixed_cubes[0] diff --git a/tests/integration/cmor/test_fix.py b/tests/integration/cmor/test_fix.py index 63b4767841..74a0a08841 100644 --- a/tests/integration/cmor/test_fix.py +++ b/tests/integration/cmor/test_fix.py @@ -8,8 +8,7 @@ from iris.coords import AuxCoord, DimCoord from iris.cube import Cube, CubeList -from esmvalcore.cmor.check import CheckLevels, CMORCheckError -from esmvalcore.exceptions import ESMValCoreDeprecationWarning +from esmvalcore.cmor.check import CMORCheckError from esmvalcore.preprocessor import ( cmor_check_data, cmor_check_metadata, @@ -18,25 +17,6 @@ ) -# TODO: remove in v2.12 -@pytest.fixture(autouse=True) -def disable_fix_cmor_checker(mocker): - """Disable the CMOR checker in fixes (will be default in v2.12).""" - - class MockChecker: - def __init__(self, cube): - self._cube = cube - - def check_metadata(self): - return self._cube - - def check_data(self): - return self._cube - - mock = mocker.patch("esmvalcore.cmor.fix._get_cmor_checker") - mock.return_value = MockChecker - - class TestGenericFix: """Tests for ``GenericFix``.""" @@ -887,27 +867,3 @@ def test_fix_data_amon_tas(self): assert self.mock_debug.call_count == 0 assert self.mock_warning.call_count == 0 - - def test_deprecate_check_level_fix_metadata(self): - """Test deprecation of check level in ``fix_metadata``.""" - with pytest.warns(ESMValCoreDeprecationWarning): - fix_metadata( - self.cubes_4d, - "ta", - "CMIP6", - "MODEL", - "Amon", - check_level=CheckLevels.RELAXED, - ) - - def test_deprecate_check_level_fix_data(self): - """Test deprecation of check level in ``fix_data``.""" - with pytest.warns(ESMValCoreDeprecationWarning): - fix_metadata( - self.cubes_4d, - "ta", - "CMIP6", - "MODEL", - "Amon", - check_level=CheckLevels.RELAXED, - ) diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index 331f3b6273..a7822ec03b 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -20,7 +20,6 @@ CMORCheckError, _get_cmor_checker, ) -from esmvalcore.exceptions import ESMValCoreDeprecationWarning logger = logging.getLogger(__name__) @@ -299,14 +298,6 @@ def test_valid_generic_level(self): checker.check_metadata() checker.check_data() - # TODO: remove in v2.12 - def test_valid_generic_level_automatic_fixes(self): - """Test valid generic level coordinate with automatic fixes.""" - self._setup_generic_level_var() - checker = CMORCheck(self.cube, self.var_info, automatic_fixes=True) - checker.check_metadata() - checker.check_data() - def test_invalid_generic_level(self): """Test invalid generic level coordinate.""" self._setup_generic_level_var() @@ -680,20 +671,6 @@ def test_non_decreasing(self): self.var_info.coordinates["lat"].stored_direction = "decreasing" self._check_fails_in_metadata() - # TODO: remove in v2.12 - def test_non_decreasing_automatic_fix_metadata(self): - """Automatic fix for decreasing coordinate.""" - self.var_info.coordinates["lat"].stored_direction = "decreasing" - checker = CMORCheck(self.cube, self.var_info, automatic_fixes=True) - checker.check_metadata() - - # TODO: remove in v2.12 - def test_non_decreasing_automatic_fix_data(self): - """Automatic fix for decreasing coordinate.""" - self.var_info.coordinates["lat"].stored_direction = "decreasing" - checker = CMORCheck(self.cube, self.var_info, automatic_fixes=True) - checker.check_data() - def test_lat_non_monotonic(self): """Test fail for non monotonic latitude.""" lat = self.cube.coord("latitude") @@ -1258,11 +1235,5 @@ def test_get_cmor_checker_invalid_project_fail(): _get_cmor_checker("INVALID_PROJECT", "mip", "short_name", "frequency") -def test_deprecate_automatic_fixes(): - """Test deprecation of automatic_fixes.""" - with pytest.warns(ESMValCoreDeprecationWarning): - CMORCheck("cube", "var_info", "frequency", automatic_fixes=True) - - if __name__ == "__main__": unittest.main() diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index 01038d2786..8c005f1400 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -117,12 +117,9 @@ class TestFixMetadata: def setUp(self): """Prepare for testing.""" self.cube = self._create_mock_cube() - self.intermediate_cube = self._create_mock_cube() self.fixed_cube = self._create_mock_cube() self.mock_fix = Mock() - self.mock_fix.fix_metadata.return_value = [self.intermediate_cube] - self.checker = Mock() - self.check_metadata = self.checker.return_value.check_metadata + self.mock_fix.fix_metadata.return_value = [self.fixed_cube] self.expected_get_fixes_call = { "project": "project", "dataset": "model", @@ -148,81 +145,58 @@ def _create_mock_cube(var_name="short_name"): def test_fix(self): """Check that the returned fix is applied.""" - self.check_metadata.side_effect = lambda: self.fixed_cube with patch( "esmvalcore.cmor._fixes.fix.Fix.get_fixes", return_value=[self.mock_fix], ) as mock_get_fixes: - with patch( - "esmvalcore.cmor.fix._get_cmor_checker", - return_value=self.checker, - ): - cube_returned = fix_metadata( - cubes=[self.cube], - short_name="short_name", - project="project", - dataset="model", - mip="mip", - frequency="frequency", - session=sentinel.session, - )[0] - self.checker.assert_called_once_with(self.intermediate_cube) - self.check_metadata.assert_called_once_with() - assert cube_returned is not self.cube - assert cube_returned is not self.intermediate_cube - assert cube_returned is self.fixed_cube - mock_get_fixes.assert_called_once_with( - **self.expected_get_fixes_call - ) + cube_returned = fix_metadata( + cubes=[self.cube], + short_name="short_name", + project="project", + dataset="model", + mip="mip", + frequency="frequency", + session=sentinel.session, + )[0] + assert cube_returned is not self.cube + assert cube_returned is self.fixed_cube + mock_get_fixes.assert_called_once_with( + **self.expected_get_fixes_call + ) def test_nofix(self): """Check that the same cube is returned if no fix is available.""" - self.check_metadata.side_effect = lambda: self.cube with patch( "esmvalcore.cmor._fixes.fix.Fix.get_fixes", return_value=[] ) as mock_get_fixes: - with patch( - "esmvalcore.cmor.fix._get_cmor_checker", - return_value=self.checker, - ): - cube_returned = fix_metadata( - cubes=[self.cube], - short_name="short_name", - project="project", - dataset="model", - mip="mip", - frequency="frequency", - session=sentinel.session, - )[0] - self.checker.assert_called_once_with(self.cube) - self.check_metadata.assert_called_once_with() - assert cube_returned is self.cube - assert cube_returned is not self.intermediate_cube - assert cube_returned is not self.fixed_cube - mock_get_fixes.assert_called_once_with( - **self.expected_get_fixes_call - ) + cube_returned = fix_metadata( + cubes=[self.cube], + short_name="short_name", + project="project", + dataset="model", + mip="mip", + frequency="frequency", + session=sentinel.session, + )[0] + assert cube_returned is self.cube + assert cube_returned is not self.fixed_cube + mock_get_fixes.assert_called_once_with( + **self.expected_get_fixes_call + ) def test_select_var(self): """Check that the same cube is returned if no fix is available.""" - self.check_metadata.side_effect = lambda: self.cube with patch( "esmvalcore.cmor._fixes.fix.Fix.get_fixes", return_value=[] ): - with patch( - "esmvalcore.cmor.fix._get_cmor_checker", - return_value=self.checker, - ): - cube_returned = fix_metadata( - cubes=[self.cube, self._create_mock_cube("extra")], - short_name="short_name", - project="CMIP6", - dataset="model", - mip="mip", - )[0] - self.checker.assert_called_once_with(self.cube) - self.check_metadata.assert_called_once_with() - assert cube_returned is self.cube + cube_returned = fix_metadata( + cubes=[self.cube, self._create_mock_cube("extra")], + short_name="short_name", + project="CMIP6", + dataset="model", + mip="mip", + )[0] + assert cube_returned is self.cube def test_select_var_failed_if_bad_var_name(self): """Check that an error is raised if short_names do not match.""" @@ -247,12 +221,9 @@ class TestFixData: def setUp(self): """Prepare for testing.""" self.cube = Mock() - self.intermediate_cube = Mock() self.fixed_cube = Mock() self.mock_fix = Mock() - self.mock_fix.fix_data.return_value = self.intermediate_cube - self.checker = Mock() - self.check_data = self.checker.return_value.check_data + self.mock_fix.fix_data.return_value = self.fixed_cube self.expected_get_fixes_call = { "project": "project", "dataset": "model", @@ -271,57 +242,41 @@ def setUp(self): def test_fix(self): """Check that the returned fix is applied.""" - self.check_data.side_effect = lambda: self.fixed_cube with patch( "esmvalcore.cmor._fixes.fix.Fix.get_fixes", return_value=[self.mock_fix], ) as mock_get_fixes: - with patch( - "esmvalcore.cmor.fix._get_cmor_checker", - return_value=self.checker, - ): - cube_returned = fix_data( - self.cube, - short_name="short_name", - project="project", - dataset="model", - mip="mip", - frequency="frequency", - session=sentinel.session, - ) - self.checker.assert_called_once_with(self.intermediate_cube) - self.check_data.assert_called_once_with() - assert cube_returned is not self.cube - assert cube_returned is not self.intermediate_cube - assert cube_returned is self.fixed_cube - mock_get_fixes.assert_called_once_with( - **self.expected_get_fixes_call - ) + cube_returned = fix_data( + self.cube, + short_name="short_name", + project="project", + dataset="model", + mip="mip", + frequency="frequency", + session=sentinel.session, + ) + assert cube_returned is not self.cube + assert cube_returned is self.fixed_cube + mock_get_fixes.assert_called_once_with( + **self.expected_get_fixes_call + ) def test_nofix(self): """Check that the same cube is returned if no fix is available.""" - self.check_data.side_effect = lambda: self.cube with patch( "esmvalcore.cmor._fixes.fix.Fix.get_fixes", return_value=[] ) as mock_get_fixes: - with patch( - "esmvalcore.cmor.fix._get_cmor_checker", - return_value=self.checker, - ): - cube_returned = fix_data( - self.cube, - short_name="short_name", - project="project", - dataset="model", - mip="mip", - frequency="frequency", - session=sentinel.session, - ) - self.checker.assert_called_once_with(self.cube) - self.check_data.assert_called_once_with() - assert cube_returned is self.cube - assert cube_returned is not self.intermediate_cube - assert cube_returned is not self.fixed_cube - mock_get_fixes.assert_called_once_with( - **self.expected_get_fixes_call - ) + cube_returned = fix_data( + self.cube, + short_name="short_name", + project="project", + dataset="model", + mip="mip", + frequency="frequency", + session=sentinel.session, + ) + assert cube_returned is self.cube + assert cube_returned is not self.fixed_cube + mock_get_fixes.assert_called_once_with( + **self.expected_get_fixes_call + ) diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 8408c622b9..1348dc0ebf 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -1735,7 +1735,6 @@ def mock_preprocess( "timerange": "2000/2005", }, "fix_metadata": { - "check_level": CheckLevels.DEFAULT, "session": session, "dataset": "CanESM2", "ensemble": "r1i1p1", @@ -1757,7 +1756,6 @@ def mock_preprocess( "timerange": "2000/2005", }, "fix_data": { - "check_level": CheckLevels.DEFAULT, "session": session, "dataset": "CanESM2", "ensemble": "r1i1p1", From da3440e4744726f3f645ba3a3f240e9d8ac17d16 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:45:54 +0200 Subject: [PATCH 09/19] Remove deprecated statistical operators (#2553) Co-authored-by: Valeriu Predoi --- esmvalcore/preprocessor/_shared.py | 27 ------------------- .../_multimodel/test_multimodel.py | 7 ----- tests/unit/preprocessor/test_shared.py | 24 +++-------------- 3 files changed, 3 insertions(+), 55 deletions(-) diff --git a/esmvalcore/preprocessor/_shared.py b/esmvalcore/preprocessor/_shared.py index 04490bdda4..49272771b5 100644 --- a/esmvalcore/preprocessor/_shared.py +++ b/esmvalcore/preprocessor/_shared.py @@ -7,7 +7,6 @@ from __future__ import annotations import logging -import re import warnings from collections import defaultdict from collections.abc import Callable, Iterable @@ -22,7 +21,6 @@ from iris.exceptions import CoordinateMultiDimError, CoordinateNotFoundError from iris.util import broadcast_to_shape -from esmvalcore.exceptions import ESMValCoreDeprecationWarning from esmvalcore.iris_helpers import has_regular_grid from esmvalcore.typing import DataType @@ -74,31 +72,6 @@ def get_iris_aggregator( cap_operator = operator.upper() aggregator_kwargs = dict(operator_kwargs) - # Deprecations - if cap_operator == "STD": - msg = ( - f"The operator '{operator}' for computing the standard deviation " - f"has been deprecated in ESMValCore version 2.10.0 and is " - f"scheduled for removal in version 2.12.0. Please use 'std_dev' " - f"instead. This is an exact replacement." - ) - warnings.warn(msg, ESMValCoreDeprecationWarning) - operator = "std_dev" - cap_operator = "STD_DEV" - elif re.match(r"^(P\d{1,2})(\.\d*)?$", cap_operator): - msg = ( - f"Specifying percentile operators with the syntax 'pXX.YY' (here: " - f"'{operator}') has been deprecated in ESMValCore version 2.10.0 " - f"and is scheduled for removal in version 2.12.0. Please use " - f"`operator='percentile'` with the keyword argument " - f"`percent=XX.YY` instead. Example: `percent=95.0` for 'p95.0'. " - f"This is an exact replacement." - ) - warnings.warn(msg, ESMValCoreDeprecationWarning) - aggregator_kwargs["percent"] = float(operator[1:]) - operator = "percentile" - cap_operator = "PERCENTILE" - # Check if valid aggregator is found if not hasattr(iris.analysis, cap_operator): raise ValueError( diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 5bc5513451..39c11c944c 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -247,23 +247,17 @@ def get_cube_for_equal_coords_test(num_cubes): ("full", "mean", (5, 5, 3)), ("full", {"operator": "mean"}, (5, 5, 3)), ("full", "std_dev", (5.656854249492381, 4, 2.8284271247461903)), - ("full", "std", (5.656854249492381, 4, 2.8284271247461903)), ("full", "min", (1, 1, 1)), ("full", "max", (9, 9, 5)), ("full", "median", (5, 5, 3)), ("full", {"operator": "percentile", "percent": 50.0}, (5, 5, 3)), - ("full", "p50", (5, 5, 3)), - ("full", "p99.5", (8.96, 8.96, 4.98)), ("full", "peak", (9, 9, 5)), ("overlap", "mean", (5, 5)), ("overlap", "std_dev", (5.656854249492381, 4)), - ("overlap", "std", (5.656854249492381, 4)), ("overlap", "min", (1, 1)), ("overlap", "max", (9, 9)), ("overlap", "median", (5, 5)), ("overlap", {"operator": "percentile", "percent": 50.0}, (5, 5)), - ("overlap", "p50", (5, 5)), - ("overlap", "p99.5", (8.96, 8.96)), ("overlap", "peak", (9, 9)), # test multiple statistics ("overlap", ("min", "max"), ((1, 1), (9, 9))), @@ -1470,7 +1464,6 @@ def test_empty_input_ensemble_statistics(): {"operator": "median"}, "min", "max", - "p42.314", {"operator": "percentile", "percent": 42.314}, "std_dev", ] diff --git a/tests/unit/preprocessor/test_shared.py b/tests/unit/preprocessor/test_shared.py index 02a48991ba..b0a990c45d 100644 --- a/tests/unit/preprocessor/test_shared.py +++ b/tests/unit/preprocessor/test_shared.py @@ -11,7 +11,6 @@ from iris.coords import AuxCoord from iris.cube import Cube -from esmvalcore.exceptions import ESMValCoreDeprecationWarning from esmvalcore.preprocessor import PreprocessorFile from esmvalcore.preprocessor._shared import ( _compute_area_weights, @@ -91,16 +90,6 @@ def test_get_iris_aggregator_percentile(operator, kwargs): assert agg_kwargs == kwargs -@pytest.mark.parametrize("kwargs", [{}, {"alphap": 0.5}]) -@pytest.mark.parametrize("operator", ["p10", "P10.5"]) -def test_get_iris_aggregator_pxxyy(operator, kwargs): - """Test ``get_iris_aggregator``.""" - with pytest.warns(ESMValCoreDeprecationWarning): - (agg, agg_kwargs) = get_iris_aggregator(operator, **kwargs) - assert agg == iris.analysis.PERCENTILE - assert agg_kwargs == {"percent": float(operator[1:]), **kwargs} - - @pytest.mark.parametrize("kwargs", [{}, {"weights": True}]) @pytest.mark.parametrize("operator", ["rms", "rMs", "RMS"]) def test_get_iris_aggregator_rms(operator, kwargs): @@ -111,18 +100,11 @@ def test_get_iris_aggregator_rms(operator, kwargs): @pytest.mark.parametrize("kwargs", [{}, {"ddof": 1}]) -@pytest.mark.parametrize("operator", ["std", "STD", "std_dev", "STD_DEV"]) +@pytest.mark.parametrize("operator", ["std_dev", "STD_DEV"]) def test_get_iris_aggregator_std(operator, kwargs): """Test ``get_iris_aggregator``.""" - if operator.lower() == "std": - with pytest.warns(ESMValCoreDeprecationWarning): - (agg, agg_kwargs) = get_iris_aggregator(operator, **kwargs) - else: - with warnings.catch_warnings(): - warnings.simplefilter( - "error", category=ESMValCoreDeprecationWarning - ) - (agg, agg_kwargs) = get_iris_aggregator(operator, **kwargs) + with warnings.catch_warnings(): + (agg, agg_kwargs) = get_iris_aggregator(operator, **kwargs) assert agg == iris.analysis.STD_DEV assert agg_kwargs == kwargs From 3f9f1fe21eef10dc49a6eefad61223c1d09fc83b Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Fri, 11 Oct 2024 16:12:14 +0100 Subject: [PATCH 10/19] reformat datetime strings be in line with new `isodate==0.7.0` and actual ISO8601 and pin `isodate>=0.7.0` (#2546) Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com> --- environment.yml | 2 +- esmvalcore/_recipe/check.py | 63 ++++++++++++++-------- esmvalcore/preprocessor/_time.py | 22 ++++---- setup.py | 2 +- tests/integration/recipe/test_check.py | 23 ++++++-- tests/integration/recipe/test_recipe.py | 6 +-- tests/unit/preprocessor/_time/test_time.py | 19 +++++++ 7 files changed, 97 insertions(+), 40 deletions(-) diff --git a/environment.yml b/environment.yml index 636883f04a..5cf256c22b 100644 --- a/environment.yml +++ b/environment.yml @@ -21,7 +21,7 @@ dependencies: - iris >=3.10.0 - iris-esmf-regrid >=0.11.0 - iris-grib >=0.20.0 # github.com/ESMValGroup/ESMValCore/issues/2535 - - isodate + - isodate >=0.7.0 # incompatible with very old 0.6.1 - jinja2 - libnetcdf !=4.9.1 # to avoid hdf5 warnings - nc-time-axis diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index 4e4fa6d2b9..e4f60e9d7f 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -366,20 +366,33 @@ def _check_delimiter(timerange): def _check_duration_periods(timerange): - try: - isodate.parse_duration(timerange[0]) - except ValueError: - pass - else: + # isodate duration must always start with P + if timerange[0].startswith("P") and timerange[1].startswith("P"): + raise RecipeError( + "Invalid value encountered for `timerange`. " + "Cannot set both the beginning and the end " + "as duration periods." + ) + + if timerange[0].startswith("P"): + try: + isodate.parse_duration(timerange[0]) + except isodate.isoerror.ISO8601Error as exc: + raise RecipeError( + "Invalid value encountered for `timerange`. " + f"{timerange[0]} is not valid duration according to ISO 8601." + + "\n" + + str(exc) + ) + elif timerange[1].startswith("P"): try: isodate.parse_duration(timerange[1]) - except ValueError: - pass - else: + except isodate.isoerror.ISO8601Error as exc: raise RecipeError( "Invalid value encountered for `timerange`. " - "Cannot set both the beginning and the end " - "as duration periods." + f"{timerange[1]} is not valid duration according to ISO 8601." + + "\n" + + str(exc) ) @@ -391,20 +404,26 @@ def _check_format_years(date): def _check_timerange_values(date, timerange): + # Wildcards are fine + if date == "*": + return + # P must always be in a duration string + # if T in date, that is a datetime; otherwise it's date try: - isodate.parse_date(date) - except ValueError: - try: + if date.startswith("P"): isodate.parse_duration(date) - except ValueError as exc: - if date != "*": - raise RecipeError( - "Invalid value encountered for `timerange`. " - "Valid value must follow ISO 8601 standard " - "for dates and duration periods, or be " - "set to '*' to load available years. " - f"Got {timerange} instead." - ) from exc + elif "T" in date: + isodate.parse_datetime(date) + else: + isodate.parse_date(date) + except isodate.isoerror.ISO8601Error as exc: + raise RecipeError( + "Invalid value encountered for `timerange`. " + "Valid value must follow ISO 8601 standard " + "for dates and duration periods, or be " + "set to '*' to load available years. " + f"Got {timerange} instead." + "\n" + str(exc) + ) def valid_time_selection(timerange): diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index fd9d4c16ac..522a312f5e 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -117,17 +117,17 @@ def _parse_start_date(date): """Parse start of the input `timerange` tag given in ISO 8601 format. Returns a datetime.datetime object. + + Raises an ISO8601 parser error if data can not be parsed. """ if date.startswith("P"): start_date = isodate.parse_duration(date) + elif "T" in date: + start_date = isodate.parse_datetime(date) else: - try: - start_date = isodate.parse_datetime(date) - except isodate.isoerror.ISO8601Error: - start_date = isodate.parse_date(date) - start_date = datetime.datetime.combine( - start_date, datetime.time.min - ) + start_date = isodate.parse_date(date) + start_date = datetime.datetime.combine(start_date, datetime.time.min) + return start_date @@ -135,6 +135,8 @@ def _parse_end_date(date): """Parse end of the input `timerange` given in ISO 8601 format. Returns a datetime.datetime object. + + Raises an ISO8601 parser error if data can not be parsed. """ if date.startswith("P"): end_date = isodate.parse_duration(date) @@ -145,9 +147,9 @@ def _parse_end_date(date): month, year = get_next_month(int(date[4:]), int(date[0:4])) end_date = datetime.datetime(year, month, 1, 0, 0, 0) else: - try: + if "T" in date: end_date = isodate.parse_datetime(date) - except isodate.ISO8601Error: + else: end_date = isodate.parse_date(date) end_date = datetime.datetime.combine( end_date, datetime.time.min @@ -268,6 +270,8 @@ def clip_timerange(cube: Cube, timerange: str) -> Cube: ------ ValueError Time ranges are outside the cube's time limits. + isodate.isoerror.ISO8601Error: + Start/end times can not be parsed by isodate. """ start_date = _parse_start_date(timerange.split("/")[0]) diff --git a/setup.py b/setup.py index 15ea79aae3..83c3634428 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ "geopy", "humanfriendly", "iris-grib>=0.20.0", # github.com/ESMValGroup/ESMValCore/issues/2535 - "isodate", + "isodate>=0.7.0", # incompatible with very old 0.6.1 "jinja2", "nc-time-axis", # needed by iris.plot "nested-lookup", diff --git a/tests/integration/recipe/test_check.py b/tests/integration/recipe/test_check.py index b118162c15..d168a3e010 100644 --- a/tests/integration/recipe/test_check.py +++ b/tests/integration/recipe/test_check.py @@ -122,15 +122,15 @@ def test_data_availability_no_data(mock_logger, dirnames, filenames, error): "*", "1990/1992", "19900101/19920101", - "19900101T12H00M00S/19920101T12H00M00", + "19900101T120000/19920101T120000", "1990/*", "*/1992", "1990/P2Y", "19900101/P2Y2M1D", - "19900101TH00M00S/P2Y2M1DT12H00M00S", + "19900101T0000/P2Y2M1DT12H00M00S", "P2Y/1992", "P2Y2M1D/19920101", - "P2Y2M1D/19920101T12H00M00S", + "P2Y2M1D/19920101T120000", "P2Y/*", "P2Y2M1D/*", "P2Y21DT12H00M00S/*", @@ -159,13 +159,28 @@ def test_valid_time_selection(timerange): "199035345/19923463164526", "Invalid value encountered for `timerange`. Valid value must follow " "ISO 8601 standard for dates and duration periods, or be set to '*' " - "to load available years. Got ['199035345', '19923463164526'] instead.", + "to load available years. Got ['199035345', '19923463164526'] instead.\n" + "Unrecognised ISO 8601 date format: '199035345'", ), ( "P11Y/P42Y", "Invalid value encountered for `timerange`. Cannot set both " "the beginning and the end as duration periods.", ), + ( + "P11X/19923463164526", + "Invalid value encountered for `timerange`. " + "P11X is not valid duration according to ISO 8601.\n" + "ISO 8601 time designator 'T' missing. " + "Unable to parse datetime string '11X'", + ), + ( + "19923463164526/P11X", + "Invalid value encountered for `timerange`. " + "P11X is not valid duration according to ISO 8601.\n" + "ISO 8601 time designator 'T' missing. " + "Unable to parse datetime string '11X'", + ), ] diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index 75bed13e9f..ae1ff9b5b7 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -740,12 +740,12 @@ def test_empty_variable(tmp_path, patched_datafinder, session): ("1990/P2Y", "1990-P2Y"), ("19900101/P2Y2M1D", "19900101-P2Y2M1D"), ( - "19900101TH00M00S/P2Y2M1DT12H00M00S", - "19900101TH00M00S-P2Y2M1DT12H00M00S", + "19900101T0000/P2Y2M1DT12H00M00S", + "19900101T0000-P2Y2M1DT12H00M00S", ), ("P2Y/1992", "P2Y-1992"), ("P1Y2M1D/19920101", "P1Y2M1D-19920101"), - ("P1Y2M1D/19920101T12H00M00S", "P1Y2M1D-19920101T12H00M00S"), + ("P1Y2M1D/19920101T120000", "P1Y2M1D-19920101T120000"), ("P2Y/*", "P2Y-2019"), ("P2Y2M1D/*", "P2Y2M1D-2019"), ("P2Y21DT12H00M00S/*", "P2Y21DT12H00M00S-2019"), diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index e8ddb5aae0..9ce5e008b2 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -12,6 +12,7 @@ import iris.coords import iris.exceptions import iris.fileformats +import isodate import numpy as np import pytest from cf_units import Unit @@ -486,6 +487,24 @@ def test_clip_timerange_single_year_4d(self): coord_name ) + def test_clip_timerange_start_date_invalid_isodate(self): + cube = self._create_cube( + [[[[0.0, 1.0]]]], [150.0], [[0.0, 365.0]], "standard" + ) + with pytest.raises(isodate.isoerror.ISO8601Error) as exc: + clip_timerange(cube, "1950010101/1950") + mssg = "Unrecognised ISO 8601 date format: '1950010101'" + assert mssg in str(exc) + + def test_clip_timerange_end_date_invalid_isodate(self): + cube = self._create_cube( + [[[[0.0, 1.0]]]], [150.0], [[0.0, 365.0]], "standard" + ) + with pytest.raises(isodate.isoerror.ISO8601Error) as exc: + clip_timerange(cube, "1950/1950010101") + mssg = "Unrecognised ISO 8601 date format: '1950010101'" + assert mssg in str(exc) + class TestExtractSeason(tests.Test): """Tests for extract_season.""" From e4e6b9be0683527498252fc4aaebbd440ec58fbf Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 14 Oct 2024 16:02:35 +0200 Subject: [PATCH 11/19] Disable upstream tests on commits (#2548) --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4be031b9a3..4ff0350e4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -236,7 +236,6 @@ workflows: - run_tests - test_installation_from_source_develop_mode - test_installation_from_source_test_mode - - test_with_upstream_developments nightly: triggers: From bd36519b5140c24736aa63311d7674e5dc193450 Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 14 Oct 2024 18:16:17 +0200 Subject: [PATCH 12/19] Merge input cubes only once when computing lazy multimodel statistics (#2518) Co-authored-by: Valeriu Predoi --- esmvalcore/preprocessor/_multimodel.py | 46 +++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index dcce65ebd3..b790b45117 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -484,7 +484,8 @@ def _compute_eager( input_slices = cubes # scalar cubes else: input_slices = [cube[chunk] for cube in cubes] - result_slice = _compute(input_slices, operator=operator, **kwargs) + combined_cube = _combine(input_slices) + result_slice = _compute(combined_cube, operator=operator, **kwargs) result_slices.append(result_slice) try: @@ -503,10 +504,13 @@ def _compute_eager( return result_cube -def _compute(cubes: list, *, operator: iris.analysis.Aggregator, **kwargs): +def _compute( + cube: iris.cube.Cube, + *, + operator: iris.analysis.Aggregator, + **kwargs, +): """Compute statistic.""" - cube = _combine(cubes) - with warnings.catch_warnings(): warnings.filterwarnings( "ignore", @@ -531,8 +535,6 @@ def _compute(cubes: list, *, operator: iris.analysis.Aggregator, **kwargs): # Remove concatenation dimension added by _combine result_cube.remove_coord(CONCAT_DIM) - for cube in cubes: - cube.remove_coord(CONCAT_DIM) # some iris aggregators modify dtype, see e.g. # https://numpy.org/doc/stable/reference/generated/numpy.ma.average.html @@ -545,7 +547,6 @@ def _compute(cubes: list, *, operator: iris.analysis.Aggregator, **kwargs): method=cell_method.method, coords=cell_method.coord_names, intervals=cell_method.intervals, - comments=f"input_cubes: {len(cubes)}", ) result_cube.add_cell_method(updated_method) return result_cube @@ -602,27 +603,26 @@ def _multicube_statistics( # Calculate statistics statistics_cubes = {} lazy_input = any(cube.has_lazy_data() for cube in cubes) - for stat in statistics: - (stat_id, result_cube) = _compute_statistic(cubes, lazy_input, stat) + combined_cube = None + for statistic in statistics: + stat_id = _get_stat_identifier(statistic) + logger.debug("Multicube statistics: computing: %s", stat_id) + + (operator, kwargs) = _get_operator_and_kwargs(statistic) + (agg, agg_kwargs) = get_iris_aggregator(operator, **kwargs) + if lazy_input and agg.lazy_func is not None: + if combined_cube is None: + # Merge input cubes only once as this is can be computationally + # expensive. + combined_cube = _combine(cubes) + result_cube = _compute(combined_cube, operator=agg, **agg_kwargs) + else: + result_cube = _compute_eager(cubes, operator=agg, **agg_kwargs) statistics_cubes[stat_id] = result_cube return statistics_cubes -def _compute_statistic(cubes, lazy_input, statistic): - """Compute a single statistic.""" - stat_id = _get_stat_identifier(statistic) - logger.debug("Multicube statistics: computing: %s", stat_id) - - (operator, kwargs) = _get_operator_and_kwargs(statistic) - (agg, agg_kwargs) = get_iris_aggregator(operator, **kwargs) - if lazy_input and agg.lazy_func is not None: - result_cube = _compute(cubes, operator=agg, **agg_kwargs) - else: - result_cube = _compute_eager(cubes, operator=agg, **agg_kwargs) - return (stat_id, result_cube) - - def _multiproduct_statistics( products, statistics, From 7ff491cdd86bacb65d7e3d832d712cd6d2819cfc Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Mon, 14 Oct 2024 18:33:10 +0200 Subject: [PATCH 13/19] Disable collecting test coverage by default (#2456) Co-authored-by: Valeriu Predoi --- .circleci/config.yml | 11 ++++++++--- doc/contributing.rst | 13 ++++++++++--- setup.cfg | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4ff0350e4e..8232620e35 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,6 +21,10 @@ commands: circleci step halt fi test_and_report: + parameters: + args: + type: string + default: "" steps: - run: name: Run tests @@ -28,7 +32,7 @@ commands: mkdir -p test-reports . /opt/conda/etc/profile.d/conda.sh conda activate esmvaltool - pytest -n 4 --junitxml=test-reports/report.xml + pytest -n 4 --junitxml=test-reports/report.xml << parameters.args >> esmvaltool version - store_test_results: path: test-reports/report.xml @@ -127,8 +131,9 @@ jobs: . /opt/conda/etc/profile.d/conda.sh mkdir /logs conda activate esmvaltool - pip install .[test] |& tee -a /logs/install.txt - - test_and_report + pip install .[test] > /logs/install.txt 2>&1 + - test_and_report: + args: --cov - save_cache: key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }} paths: diff --git a/doc/contributing.rst b/doc/contributing.rst index a21a005e72..8737fda13e 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -470,9 +470,16 @@ successful. Test coverage ~~~~~~~~~~~~~ -To check which parts of your code are `covered by unit tests`_, open the file -``test-reports/coverage_html/index.html`` (available after running a ``pytest`` -command) and browse to the relevant file. +To check which parts of your code are `covered by unit tests`_, run the command + +.. code-block:: bash + + pytest --cov + +and open the file ``test-reports/coverage_html/index.html`` and browse to the +relevant file. Note that tracking code coverage slows down the test runs, +therefore it is disabled by default and needs to be requested by providing +``pytest`` with the ``--cov`` flag. CircleCI will upload the coverage results from running the tests to codecov and Codacy. diff --git a/setup.cfg b/setup.cfg index 15d02392d7..995fd69c9e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,6 @@ addopts = --doctest-modules --ignore=esmvalcore/cmor/tables/ - --cov=esmvalcore --cov-report=xml:test-reports/coverage.xml --cov-report=html:test-reports/coverage_html --html=test-reports/report.html @@ -15,6 +14,7 @@ markers = [coverage:run] parallel = true +source = esmvalcore [coverage:report] exclude_lines = pragma: no cover From a3557ecdde7d1b9e857610779105fe75049f7e04 Mon Sep 17 00:00:00 2001 From: Liza Malinina <66973360+malininae@users.noreply.github.com> Date: Tue, 15 Oct 2024 05:07:04 -0700 Subject: [PATCH 14/19] Make `start_year`, `end_year` in `extract_time` optional to obtain time blocks in each year (#2490) Co-authored-by: Elizaveta Malinina Co-authored-by: Valeriu Predoi Co-authored-by: Bouwe Andela Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com> --- .zenodo.json | 512 +++++++++++---------- CITATION.cff | 10 + doc/recipe/preprocessor.rst | 40 +- esmvalcore/preprocessor/_time.py | 59 +-- tests/unit/preprocessor/_time/test_time.py | 30 ++ 5 files changed, 365 insertions(+), 286 deletions(-) diff --git a/.zenodo.json b/.zenodo.json index f0997c6467..e45f8b7aca 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,253 +1,263 @@ { - "creators": [ - { - "affiliation": "NLeSC, Netherlands", - "name": "Andela, Bouwe", - "orcid": "0000-0001-9005-8940" - }, - { - "affiliation": "DLR, Germany", - "name": "Broetz, Bjoern" - }, - { - "affiliation": "PML, UK", - "name": "de Mora, Lee", - "orcid": "0000-0002-5080-3149" - }, - { - "affiliation": "NLeSC, Netherlands", - "name": "Drost, Niels", - "orcid": "0000-0001-9795-7981" - }, - { - "affiliation": "DLR, Germany", - "name": "Eyring, Veronika", - "orcid": "0000-0002-6887-4885" - }, - { - "affiliation": "AWI, Germany", - "name": "Koldunov, Nikolay", - "orcid": "0000-0002-3365-8146" - }, - { - "affiliation": "DLR, Germany", - "name": "Lauer, Axel", - "orcid": "0000-0002-9270-1044" - }, - { - "affiliation": "URead, UK", - "name": "Predoi, Valeriu", - "orcid": "0000-0002-9729-6578" - }, - { - "affiliation": "DLR, Germany", - "name": "Righi, Mattia", - "orcid": "0000-0003-3827-5950" - }, - { - "affiliation": "DLR, Germany", - "name": "Schlund, Manuel", - "orcid": "0000-0001-5251-0158" - }, - { - "affiliation": "BSC, Spain", - "name": "Vegas-Regidor, Javier", - "orcid": "0000-0003-0096-4291" - }, - { - "affiliation": "SMHI, Sweden", - "name": "Zimmermann, Klaus", - "orcid": "0000-0003-3994-2057" - }, - { - "affiliation": "DLR, Germany", - "name": "Bock, Lisa", - "orcid": "0000-0001-7058-5938" - }, - { - "affiliation": "NLeSC, Netherlands", - "name": "Diblen, Faruk" - }, - { - "affiliation": "MetOffice, UK", - "name": "Dreyer, Laura" - }, - { - "affiliation": "MetOffice, UK", - "name": "Earnshaw, Paul" - }, - { - "affiliation": "DLR, Germany", - "name": "Hassler, Birgit", - "orcid": "0000-0003-2724-709X" - }, - { - "affiliation": "MetOffice, UK", - "name": "Little, Bill" - }, - { - "affiliation": "BSC, Spain", - "name": "Loosveldt-Tomas, Saskia" - }, - { - "affiliation": "NLeSC, Netherlands", - "name": "Smeets, Stef", - "orcid": "0000-0002-5413-9038" - }, - { - "affiliation": "NLeSC, Netherlands", - "name": "Camphuijsen, Jaro", - "orcid": "0000-0002-8928-7831" - }, - { - "affiliation": "University of Bremen, Germany", - "name": "Gier, Bettina K.", - "orcid": "0000-0002-2928-8664" - }, - { - "affiliation": "University of Bremen, Germany", - "name": "Weigel, Katja", - "orcid": "0000-0001-6133-7801" - }, - { - "affiliation": "Institute for Atmospheric and Climate Science, ETH Zurich, Zurich, Switzerland", - "name": "Hauser, Mathias", - "orcid": "0000-0002-0057-4878" - }, - { - "affiliation": "Netherlands eScience Center", - "name": "Kalverla, Peter", - "orcid": "0000-0002-5025-7862" - }, - { - "affiliation": "University of Bremen, Germany", - "name": "Galytska, Evgenia", - "orcid": "0000-0001-6575-1559" - }, - { - "affiliation": "BSC, Spain", - "name": "Cos-Espuña, Pep" - }, - { - "affiliation": "Netherlands eScience Center", - "name": "Pelupessy, Inti", - "orcid": "0000-0002-8024-0412" - }, - { - "affiliation": "Max Planck Institute for Biogeochemistry, Germany", - "name": "Koirala, Sujan", - "orcid": "0000-0001-5681-1986" - }, - { - "affiliation": "Helmholtz-Zentrum Geesthacht, Germany ", - "name": "Stacke, Tobias", - "orcid": "0000-0003-4637-5337" - }, - { - "affiliation": "Netherlands eScience Center", - "name": "Alidoost, Sarah", - "orcid": "0000-0001-8407-6472" - }, - { - "affiliation": "Barcelona Supercomputing Center", - "name": "Jury, Martin", - "orcid": "0000-0003-0590-7843" - }, - { - "affiliation": "Stéphane Sénési EIRL, Colomiers, France", - "name": "Sénési, Stéphane", - "orcid": "0000-0003-0892-5967" - }, - { - "affiliation": "MetOffice, UK", - "name": "Crocker, Thomas", - "orcid": "0000-0001-7761-5546" - }, - { - "affiliation": "Netherlands eScience Center", - "name": "Vreede, Barbara", - "orcid": "0000-0002-5023-4601" - }, - { - "affiliation": "Netherlands eScience Center", - "name": "Soares Siqueira, Abel", - "orcid": "0000-0003-4451-281X" - }, - { - "affiliation": "DLR, Germany", - "name": "Kazeroni, Rémi", - "orcid": "0000-0001-7205-9528" - }, - { - "affiliation": "GEOMAR, Germany", - "name": "Hohn, David", - "orcid": "0000-0002-5317-1247" - }, - { - "affiliation": "DLR, Germany", - "name": "Bauer, Julian" - }, - { - "affiliation": "ACCESS-NRI, Australia", - "name": "Beucher, Romain", - "orcid": "0000-0003-3891-5444" - }, - { - "affiliation": "Forschungszentrum Juelich, Germany", - "name": "Benke, Joerg" - }, - { - "affiliation": "BSC, Spain", - "name": "Martin-Martinez, Eneko", - "orcid": "0000-0002-9213-7818" - }, - { - "affiliation": "DLR, Germany", - "name": "Cammarano, Diego" - }, - { - "affiliation": "ACCESS-NRI, Australia", - "name": "Yousong, Zeng", - "orcid": "0000-0002-8385-5367" - } - ], - "description": "ESMValCore: A community tool for pre-processing data from Earth system models in CMIP and running analysis scripts.", - "license": { - "id": "Apache-2.0" - }, - "title": "ESMValCore", - "communities": [ - { - "identifier": "is-enes3" - }, - { - "identifier": "dlr_de" - }, - { - "identifier": "ecfunded" - }, - { - "identifier": "nlesc" - } - ], - "grants": [ - { - "id": "10.13039/501100000780::282672" - }, - { - "id": "10.13039/501100000780::641727" - }, - { - "id": "10.13039/501100000780::641816" - }, - { - "id": "10.13039/501100000780::727862" - }, - { - "id": "10.13039/501100000780::776613" - }, - { - "id": "10.13039/501100000780::824084" - } - ] + "creators": [ + { + "affiliation": "NLeSC, Netherlands", + "name": "Andela, Bouwe", + "orcid": "0000-0001-9005-8940" + }, + { + "affiliation": "DLR, Germany", + "name": "Broetz, Bjoern" + }, + { + "affiliation": "PML, UK", + "name": "de Mora, Lee", + "orcid": "0000-0002-5080-3149" + }, + { + "affiliation": "NLeSC, Netherlands", + "name": "Drost, Niels", + "orcid": "0000-0001-9795-7981" + }, + { + "affiliation": "DLR, Germany", + "name": "Eyring, Veronika", + "orcid": "0000-0002-6887-4885" + }, + { + "affiliation": "AWI, Germany", + "name": "Koldunov, Nikolay", + "orcid": "0000-0002-3365-8146" + }, + { + "affiliation": "DLR, Germany", + "name": "Lauer, Axel", + "orcid": "0000-0002-9270-1044" + }, + { + "affiliation": "URead, UK", + "name": "Predoi, Valeriu", + "orcid": "0000-0002-9729-6578" + }, + { + "affiliation": "DLR, Germany", + "name": "Righi, Mattia", + "orcid": "0000-0003-3827-5950" + }, + { + "affiliation": "DLR, Germany", + "name": "Schlund, Manuel", + "orcid": "0000-0001-5251-0158" + }, + { + "affiliation": "BSC, Spain", + "name": "Vegas-Regidor, Javier", + "orcid": "0000-0003-0096-4291" + }, + { + "affiliation": "SMHI, Sweden", + "name": "Zimmermann, Klaus", + "orcid": "0000-0003-3994-2057" + }, + { + "affiliation": "DLR, Germany", + "name": "Bock, Lisa", + "orcid": "0000-0001-7058-5938" + }, + { + "affiliation": "NLeSC, Netherlands", + "name": "Diblen, Faruk" + }, + { + "affiliation": "MetOffice, UK", + "name": "Dreyer, Laura" + }, + { + "affiliation": "MetOffice, UK", + "name": "Earnshaw, Paul" + }, + { + "affiliation": "DLR, Germany", + "name": "Hassler, Birgit", + "orcid": "0000-0003-2724-709X" + }, + { + "affiliation": "MetOffice, UK", + "name": "Little, Bill" + }, + { + "affiliation": "BSC, Spain", + "name": "Loosveldt-Tomas, Saskia" + }, + { + "affiliation": "NLeSC, Netherlands", + "name": "Smeets, Stef", + "orcid": "0000-0002-5413-9038" + }, + { + "affiliation": "NLeSC, Netherlands", + "name": "Camphuijsen, Jaro", + "orcid": "0000-0002-8928-7831" + }, + { + "affiliation": "University of Bremen, Germany", + "name": "Gier, Bettina K.", + "orcid": "0000-0002-2928-8664" + }, + { + "affiliation": "University of Bremen, Germany", + "name": "Weigel, Katja", + "orcid": "0000-0001-6133-7801" + }, + { + "affiliation": "Institute for Atmospheric and Climate Science, ETH Zurich, Zurich, Switzerland", + "name": "Hauser, Mathias", + "orcid": "0000-0002-0057-4878" + }, + { + "affiliation": "Netherlands eScience Center", + "name": "Kalverla, Peter", + "orcid": "0000-0002-5025-7862" + }, + { + "affiliation": "University of Bremen, Germany", + "name": "Galytska, Evgenia", + "orcid": "0000-0001-6575-1559" + }, + { + "affiliation": "BSC, Spain", + "name": "Cos-Espuña, Pep" + }, + { + "affiliation": "Netherlands eScience Center", + "name": "Pelupessy, Inti", + "orcid": "0000-0002-8024-0412" + }, + { + "affiliation": "Max Planck Institute for Biogeochemistry, Germany", + "name": "Koirala, Sujan", + "orcid": "0000-0001-5681-1986" + }, + { + "affiliation": "Helmholtz-Zentrum Geesthacht, Germany ", + "name": "Stacke, Tobias", + "orcid": "0000-0003-4637-5337" + }, + { + "affiliation": "Netherlands eScience Center", + "name": "Alidoost, Sarah", + "orcid": "0000-0001-8407-6472" + }, + { + "affiliation": "Barcelona Supercomputing Center", + "name": "Jury, Martin", + "orcid": "0000-0003-0590-7843" + }, + { + "affiliation": "Stéphane Sénési EIRL, Colomiers, France", + "name": "Sénési, Stéphane", + "orcid": "0000-0003-0892-5967" + }, + { + "affiliation": "MetOffice, UK", + "name": "Crocker, Thomas", + "orcid": "0000-0001-7761-5546" + }, + { + "affiliation": "Netherlands eScience Center", + "name": "Vreede, Barbara", + "orcid": "0000-0002-5023-4601" + }, + { + "affiliation": "Netherlands eScience Center", + "name": "Soares Siqueira, Abel", + "orcid": "0000-0003-4451-281X" + }, + { + "affiliation": "DLR, Germany", + "name": "Kazeroni, Rémi", + "orcid": "0000-0001-7205-9528" + }, + { + "affiliation": "GEOMAR, Germany", + "name": "Hohn, David", + "orcid": "0000-0002-5317-1247" + }, + { + "affiliation": "DLR, Germany", + "name": "Bauer, Julian" + }, + { + "affiliation": "ACCESS-NRI, Australia", + "name": "Beucher, Romain", + "orcid": "0000-0003-3891-5444" + }, + { + "affiliation": "Forschungszentrum Juelich (FZJ), Germany", + "name": "Benke, Joerg" + }, + { + "affiliation": "BSC, Spain", + "name": "Martin-Martinez, Eneko", + "orcid": "0000-0002-9213-7818" + }, + { + "affiliation": "DLR, Germany", + "name": "Cammarano, Diego" + }, + { + "affiliation": "ACCESS-NRI, Australia", + "name": "Yousong, Zeng", + "orcid": "0000-0002-8385-5367" + }, + { + "affiliation": "ECCC, Canada", + "name": "Malinina, Elizaveta", + "orcid": "0000-0002-4102-2877" + }, + { + "affiliation": "ECCC, Canada", + "name": "Garcia Perdomo, Karen", + "orcid": "0009-0004-2333-3358" + } + ], + "description": "ESMValCore: A community tool for pre-processing data from Earth system models in CMIP and running analysis scripts.", + "license": { + "id": "Apache-2.0" + }, + "title": "ESMValCore", + "communities": [ + { + "identifier": "is-enes3" + }, + { + "identifier": "dlr_de" + }, + { + "identifier": "ecfunded" + }, + { + "identifier": "nlesc" + } + ], + "grants": [ + { + "id": "10.13039/501100000780::282672" + }, + { + "id": "10.13039/501100000780::641727" + }, + { + "id": "10.13039/501100000780::641816" + }, + { + "id": "10.13039/501100000780::727862" + }, + { + "id": "10.13039/501100000780::776613" + }, + { + "id": "10.13039/501100000780::824084" + } + ] } diff --git a/CITATION.cff b/CITATION.cff index 4d3da6e4c7..f298987f64 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -213,6 +213,16 @@ authors: family-names: Yousong given-names: Zeng orcid: "https://orcid.org/0000-0002-8385-5367" + - + affiliation: "ECCC, Canada" + family-names: Malinina + given-names: Elizaveta + orcid: "https://orcid.org/0000-0002-4102-2877" + - + affiliation: "ECCC, Canada" + family-names: Garcia Perdomo + given-names: Karen + orcid: "https://orcid.org/0009-0004-2333-3358" cff-version: 1.2.0 date-released: 2024-07-03 diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index a02bb4a566..0954b26daa 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -1307,9 +1307,9 @@ daily maximum of any given variable. ``extract_time`` ---------------- -This function subsets a dataset between two points in times. It removes all -times in the dataset before the first time and after the last time point. -The required arguments are relatively self explanatory: +This function extracts data within specific time criteria. The +preprocessor removes all times which fall outside the specified +time range. The required arguments are relatively self explanatory: * ``start_year`` * ``start_month`` @@ -1318,9 +1318,37 @@ The required arguments are relatively self explanatory: * ``end_month`` * ``end_day`` -These start and end points are set using the datasets native calendar. -All six arguments should be given as integers - the named month string -will not be accepted. +The start and end points are set using the datasets native calendar. +``start_month``, ``start_day``, ``end_month``, and ``end_day`` should +be given as integers - the named month string will not be accepted. +``start_year`` and ``end_year`` should both be either integers or +``null``. If ``start_year`` and ``end_year`` are ``null``, the date +ranges (``start_month``-``start_day`` to ``end_month``-``end_day``) +are selected in each year. For example, ranges Feb 3 - Apr 6 in each year +are selected with the following preprocessor: + +.. code-block:: yaml + + extract_time: + start_year: null + start_month: 2 + start_day: 3 + end_year: null + end_month: 4 + end_day: 6 + +And the period between Feb 3, 2001 - Apr 6, 2004 is selected as follows: + +.. code-block:: yaml + + extract_time: + start_year: 2001 + start_month: 2 + start_day: 3 + end_year: 2004 + end_month: 4 + end_day: 6 + See also :func:`esmvalcore.preprocessor.extract_time`. diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index 522a312f5e..062cdf0ba2 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -62,10 +62,10 @@ def extract_time( cube: Cube, - start_year: int, + start_year: int | None, start_month: int, start_day: int, - end_year: int, + end_year: int | None, end_month: int, end_day: int, ) -> Cube: @@ -80,13 +80,23 @@ def extract_time( cube: Input cube. start_year: - Start year. + Start year. If ``None``, the date ranges (`start_month`-`start_day` to + `end_month`-`end_day`) are selected in each year. For example, + ranges Feb 3 - Apr 6 in each year are selected if + `start_year=None`, `start_month=2`, `start_day=3`, + `end_year=None`, `end_month=4`, `end_day=6`. If `start_year` is + ``None``, `end_year` has to be ``None`` too. start_month: Start month. start_day: Start day. end_year: - End year. + End year. If ``None``, the date ranges (`start_month`-`start_day` to + `end_month`-`end_day`) are selected in each year. For example, + ranges Feb 3 - Apr 6 in each year are selected if + `start_year=None`, `start_month=2`, `start_day=3`, + `end_year=None`, `end_month=4`, `end_day=6`. If `end_year` is ``None``, + `start_year` has to be ``None`` too. end_month: End month. end_day: @@ -101,13 +111,24 @@ def extract_time( ------ ValueError Time ranges are outside the cube time limits. - """ + if start_year is not None: + start_year = int(start_year) + if end_year is not None: + end_year = int(end_year) + if (start_year is None) ^ (end_year is None): + raise ValueError( + "If start_year or end_year is None, both " + "start_year and end_year have to be None. " + f"Currently, start_year is {start_year} " + f"and end_year is {end_year}." + ) + t_1 = PartialDateTime( - year=int(start_year), month=int(start_month), day=int(start_day) + year=start_year, month=int(start_month), day=int(start_day) ) t_2 = PartialDateTime( - year=int(end_year), month=int(end_month), day=int(end_day) + year=end_year, month=int(end_month), day=int(end_day) ) return _extract_datetime(cube, t_1, t_2) @@ -221,7 +242,7 @@ def _extract_datetime( if isinstance(end_datetime.day, int) and end_datetime.day > 30: end_datetime.day = 30 - if not cube.coord_dims(time_coord): + if (not cube.coord_dims(time_coord)) or (start_datetime.year is None): constraint = iris.Constraint( time=lambda t: start_datetime <= t.point < end_datetime ) @@ -270,7 +291,7 @@ def clip_timerange(cube: Cube, timerange: str) -> Cube: ------ ValueError Time ranges are outside the cube's time limits. - isodate.isoerror.ISO8601Error: + isodate.isoerror.ISO8601Error Start/end times can not be parsed by isodate. """ @@ -330,7 +351,6 @@ def extract_season(cube: Cube, season: str) -> Cube: ------ ValueError Requested season is not present in the cube. - """ season = season.upper() @@ -384,7 +404,6 @@ def extract_month(cube: Cube, month: int) -> Cube: ------ ValueError Requested month is not present in the cube. - """ if month not in range(1, 13): raise ValueError("Please provide a month number between 1 and 12.") @@ -464,7 +483,6 @@ def hourly_statistics( ------- iris.cube.Cube Hourly statistics cube. - """ if not cube.coords("hour_group"): iris.coord_categorisation.add_categorised_coord( @@ -517,7 +535,6 @@ def daily_statistics( ------- iris.cube.Cube Daily statistics cube. - """ if not cube.coords("day_of_year"): iris.coord_categorisation.add_day_of_year(cube, "time") @@ -558,7 +575,6 @@ def monthly_statistics( ------- iris.cube.Cube Monthly statistics cube. - """ if not cube.coords("month_number"): iris.coord_categorisation.add_month_number(cube, "time") @@ -603,7 +619,6 @@ def seasonal_statistics( ------- iris.cube.Cube Seasonal statistic cube. - """ seasons = tuple(sea.upper() for sea in seasons) @@ -650,7 +665,6 @@ def spans_full_season(cube: Cube) -> list[bool]: ------- list[bool] Truth statements if time bounds are within (month*29, month*31) - """ time = cube.coord("time") num_days = [(tt.bounds[0, 1] - tt.bounds[0, 0]) for tt in time] @@ -694,7 +708,6 @@ def annual_statistics( ------- iris.cube.Cube Annual statistics cube. - """ # TODO: Add weighting in time dimension. See iris issue 3290 # https://github.com/SciTools/iris/issues/3290 @@ -736,7 +749,6 @@ def decadal_statistics( ------- iris.cube.Cube Decadal statistics cube. - """ # TODO: Add weighting in time dimension. See iris issue 3290 # https://github.com/SciTools/iris/issues/3290 @@ -1059,7 +1071,6 @@ def regrid_time( NotImplementedError An invalid `frequency` is given or `calendar` is set for a non-supported frequency. - """ # Do not overwrite input cube cube = cube.copy() @@ -1244,7 +1255,6 @@ def timeseries_filter( Cube does not have time coordinate. NotImplementedError: `filter_type` is not implemented. - """ try: cube.coord("time") @@ -1320,7 +1330,6 @@ def resample_hours( `interval` is not a divisor of 24; invalid `interpolate` given; or input data does not contain any target hour (if `interpolate` is ``None``). - """ allowed_intervals = (1, 2, 3, 4, 6, 12) if interval not in allowed_intervals: @@ -1419,7 +1428,6 @@ def resample_time( ------- iris.cube.Cube Cube with the new frequency. - """ time = cube.coord("time") dates = time.units.num2date(time.points) @@ -1458,7 +1466,6 @@ def _get_lst_offset(lon_coord: Coord) -> np.ndarray: ---- This function expects longitude in degrees. Can be in [0, 360] or [-180, 180] format. - """ # Make sure that longitude is in degrees and shift it to [-180, 180] first # (do NOT overwrite input coordinate) @@ -1475,7 +1482,6 @@ def _get_lsts(time_coord: DimCoord, lon_coord: Coord) -> np.ndarray: ---- LSTs outside of the time bins given be the time coordinate bounds are put into a bin below/above the time coordinate. - """ # Pad time coordinate with 1 time step at both sides for the bins for LSTs # outside of the time coordinate @@ -1521,7 +1527,6 @@ def _get_time_index_and_mask( (LSTs) are given. E.g., for hourly data with first time point 01:00:00 UTC, LST in Berlin is already 02:00:00 (assuming no daylight saving time). Thus, for 01:00:00 LST on this day, there is no value for Berlin. - """ # Make sure that time coordinate has bounds (these are necessary for the # binning) and uses 'hours' as reference units @@ -1586,7 +1591,6 @@ def _transform_to_lst_eager( reorder the data along the time axis based on the longitude axis. `mask` is 2D with shape (time, lon) that will be applied to the final data. - """ # Apart from the time index, all other dimensions will stay the same; this # is ensured with np.ogrid @@ -1630,7 +1634,6 @@ def _transform_to_lst_lazy( reorder the data along the time axis based on the longitude axis. `mask` is 2D with shape (time, lon) that will be applied to the final data. - """ new_data = da.apply_gufunc( _transform_to_lst_eager, @@ -1661,7 +1664,6 @@ def _transform_arr_to_lst( ---- This function either calls `_transform_to_lst_eager` or `_transform_to_lst_lazy` depending on the type of input data. - """ if isinstance(data, np.ndarray): func = _transform_to_lst_eager # type: ignore @@ -1843,7 +1845,6 @@ def local_solar_time(cube: Cube) -> Cube: Input cube has multidimensional `longitude` coordinate. ValueError `time` coordinate of input cube is not monotonically increasing. - """ # Make sure that cube has valid time and longitude coordinates _check_cube_coords(cube) diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index 9ce5e008b2..d3518e348d 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -156,6 +156,14 @@ def test_extract_time_non_gregorian_day(self): sliced = extract_time(cube, 1950, 2, 30, 1950, 3, 1) assert_array_equal(np.array([59]), sliced.coord("time").points) + def test_extract_time_none_years(self): + """Test extract slice if both end and start year are None.""" + sliced = extract_time(self.cube, None, 2, 5, None, 4, 17) + assert_array_equal( + np.array([45.0, 75.0, 105.0, 405.0, 435.0, 465.0]), + sliced.coord("time").points, + ) + def test_extract_time_no_slice(self): """Test fail of extract_time.""" self.cube.coord("time").guess_bounds() @@ -181,6 +189,28 @@ def test_extract_time_no_time(self): sliced = extract_time(cube, 1950, 1, 1, 1950, 12, 31) assert cube == sliced + def test_extract_time_start_none_year(self): + """Test extract_time when only start_year is None.""" + cube = self.cube.coord("time").guess_bounds() + msg = ( + "If start_year or end_year is None, both start_year and " + "end_year have to be None. Currently, start_year is None and " + "end_year is 1950." + ) + with pytest.raises(ValueError, match=msg): + extract_time(cube, None, 1, 1, 1950, 2, 1) + + def test_extract_time_end_none_year(self): + """Test extract_time when only end_year is None.""" + cube = self.cube.coord("time").guess_bounds() + msg = ( + "If start_year or end_year is None, both start_year and " + "end_year have to be None. Currently, start_year is 1950 and " + "end_year is None." + ) + with pytest.raises(ValueError, match=msg): + extract_time(cube, 1950, 1, 1, None, 2, 1) + class TestClipTimerange(tests.Test): """Tests for clip_timerange.""" From 1227b2f36374d1dea5b3a0bb9f69c8643b73dd2e Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 16 Oct 2024 13:41:43 +0200 Subject: [PATCH 15/19] Enable ruff flake8-bugbear rule (#2536) Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com> --- doc/contributing.rst | 2 +- doc/recipe/preprocessor.rst | 4 +- esmvalcore/_recipe/check.py | 20 ++--- esmvalcore/_recipe/recipe.py | 12 +-- esmvalcore/cmor/_fixes/fix.py | 4 +- esmvalcore/config/_config_object.py | 12 +-- esmvalcore/config/_config_validators.py | 4 +- esmvalcore/config/_validated_config.py | 1 + esmvalcore/experimental/_warnings.py | 1 + esmvalcore/preprocessor/__init__.py | 4 +- esmvalcore/preprocessor/_derive/__init__.py | 2 +- esmvalcore/preprocessor/_derive/ctotal.py | 4 +- esmvalcore/preprocessor/_derive/ohc.py | 2 +- esmvalcore/preprocessor/_io.py | 2 +- esmvalcore/preprocessor/_mapping.py | 4 +- esmvalcore/preprocessor/_regrid.py | 4 +- esmvalcore/preprocessor/_regrid_esmpy.py | 10 +-- esmvalcore/preprocessor/_shared.py | 2 +- pyproject.toml | 5 ++ .../integration/preprocessor/_io/test_load.py | 6 +- tests/integration/recipe/test_check.py | 75 +++++++++++++++++++ tests/integration/recipe/test_recipe.py | 6 +- tests/integration/test_task.py | 8 +- .../experimental/test_run_recipe.py | 2 +- tests/unit/cmor/test_fix.py | 2 +- tests/unit/preprocessor/_area/test_area.py | 2 +- .../_multimodel/test_multimodel.py | 6 +- tests/unit/preprocessor/_other/test_other.py | 24 +++--- tests/unit/test_dataset.py | 2 +- tests/unit/test_iris_helpers.py | 8 +- 30 files changed, 164 insertions(+), 76 deletions(-) diff --git a/doc/contributing.rst b/doc/contributing.rst index 8737fda13e..81a8ffb012 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -609,7 +609,7 @@ that feature should be removed in version 2.7: "ESMValCore version 2.5 and is scheduled for removal in " "version 2.7. Add additional text (e.g., description of " "alternatives) here.") - warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning) + warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning, stacklevel=2) # Other code diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 0954b26daa..92bd400e61 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -611,7 +611,7 @@ See also :func:`esmvalcore.preprocessor.weighting_landsea_fraction`. .. _masking: Masking -======= +======== Introduction to masking ----------------------- @@ -2451,7 +2451,7 @@ See also :func:`esmvalcore.preprocessor.linear_trend_stderr`. .. _detrend: Detrend -======= +======== ESMValCore also supports detrending along any dimension using the preprocessor function 'detrend'. diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index e4f60e9d7f..9a26ef4561 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -44,12 +44,12 @@ def ncl_version(): try: cmd = [ncl, "-V"] version = subprocess.check_output(cmd, universal_newlines=True) - except subprocess.CalledProcessError: - logger.error("Failed to execute '%s'", " ".join(" ".join(cmd))) + except subprocess.CalledProcessError as exc: + logger.error("Failed to execute '%s'", " ".join(cmd)) raise RecipeError( "Recipe contains NCL scripts, but your NCL " "installation appears to be broken." - ) + ) from exc version = version.strip() logger.info("Found NCL version %s", version) @@ -383,7 +383,7 @@ def _check_duration_periods(timerange): f"{timerange[0]} is not valid duration according to ISO 8601." + "\n" + str(exc) - ) + ) from exc elif timerange[1].startswith("P"): try: isodate.parse_duration(timerange[1]) @@ -393,7 +393,7 @@ def _check_duration_periods(timerange): f"{timerange[1]} is not valid duration according to ISO 8601." + "\n" + str(exc) - ) + ) from exc def _check_format_years(date): @@ -423,7 +423,7 @@ def _check_timerange_values(date, timerange): "for dates and duration periods, or be " "set to '*' to load available years. " f"Got {timerange} instead." + "\n" + str(exc) - ) + ) from exc def valid_time_selection(timerange): @@ -584,7 +584,7 @@ def _check_regular_stat(step, step_settings): try: get_iris_aggregator(operator, **operator_kwargs) except ValueError as exc: - raise RecipeError(f"Invalid options for {step}: {exc}") + raise RecipeError(f"Invalid options for {step}: {exc}") from exc def _check_mm_stat(step, step_settings): @@ -594,11 +594,11 @@ def _check_mm_stat(step, step_settings): try: (operator, kwargs) = _get_operator_and_kwargs(stat) except ValueError as exc: - raise RecipeError(str(exc)) + raise RecipeError(str(exc)) from exc try: get_iris_aggregator(operator, **kwargs) except ValueError as exc: - raise RecipeError(f"Invalid options for {step}: {exc}") + raise RecipeError(f"Invalid options for {step}: {exc}") from exc def regridding_schemes(settings: dict): @@ -645,4 +645,4 @@ def regridding_schemes(settings: dict): f"https://docs.esmvaltool.org/projects/ESMValCore/en/latest" f"/recipe/preprocessor.html#generic-regridding-schemes for " f"details." - ) + ) from exc diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 41002bbc1b..55e789d6f4 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -418,11 +418,11 @@ def _update_multiproduct(input_products, order, preproc_dir, step): called from the input products, the products that are created here need to be added to their ancestors products' settings (). """ - products = {p for p in input_products if step in p.settings} - if not products: + multiproducts = {p for p in input_products if step in p.settings} + if not multiproducts: return input_products, {} - settings = list(products)[0].settings[step] + settings = list(multiproducts)[0].settings[step] if step == "ensemble_statistics": check.ensemble_statistics_preproc(settings) @@ -431,14 +431,16 @@ def _update_multiproduct(input_products, order, preproc_dir, step): check.multimodel_statistics_preproc(settings) grouping = settings.get("groupby", None) - downstream_settings = _get_downstream_settings(step, order, products) + downstream_settings = _get_downstream_settings(step, order, multiproducts) relevant_settings = { "output_products": defaultdict(dict) } # pass to ancestors output_products = set() - for identifier, products in _group_products(products, by_key=grouping): + for identifier, products in _group_products( + multiproducts, by_key=grouping + ): common_attributes = _get_common_attributes(products, settings) statistics = settings.get("statistics", []) diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 5aa41f6486..973ac57d0b 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -141,7 +141,7 @@ def get_cube_from_list( Raises ------ - Exception + ValueError No cube is found. Returns @@ -155,7 +155,7 @@ def get_cube_from_list( for cube in cubes: if cube.var_name == short_name: return cube - raise Exception(f'Cube for variable "{short_name}" not found') + raise ValueError(f'Cube for variable "{short_name}" not found') def fix_data(self, cube: Cube) -> Cube: """Apply fixes to the data of the cube. diff --git a/esmvalcore/config/_config_object.py b/esmvalcore/config/_config_object.py index baa344f829..489e2301b2 100644 --- a/esmvalcore/config/_config_object.py +++ b/esmvalcore/config/_config_object.py @@ -92,7 +92,7 @@ def __init__(self, *args, **kwargs): "Do not instantiate `Config` objects directly, this will lead " "to unexpected behavior. Use `esmvalcore.config.CFG` instead." ) - warnings.warn(msg, UserWarning) + warnings.warn(msg, UserWarning, stacklevel=2) # TODO: remove in v2.14.0 @classmethod @@ -313,7 +313,7 @@ def load_from_file( "ESMValCore version 2.12.0 and is scheduled for removal in " "version 2.14.0. Please use `CFG.load_from_dirs()` instead." ) - warnings.warn(msg, ESMValCoreDeprecationWarning) + warnings.warn(msg, ESMValCoreDeprecationWarning, stacklevel=2) self.clear() self.update(Config._load_user_config(filename)) @@ -399,7 +399,9 @@ def reload(self) -> None: f"alternatively use a custom `--config_dir`) and omit " f"`--config_file`." ) - warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning) + warnings.warn( + deprecation_msg, ESMValCoreDeprecationWarning, stacklevel=2 + ) self.update(Config._load_user_config(raise_exception=False)) return @@ -505,7 +507,7 @@ def __init__(self, config: dict, name: str = "session"): "to unexpected behavior. Use " "`esmvalcore.config.CFG.start_session` instead." ) - warnings.warn(msg, UserWarning) + warnings.warn(msg, UserWarning, stacklevel=2) def set_session_name(self, name: str = "session"): """Set the name for the session. @@ -556,7 +558,7 @@ def config_dir(self): "ESMValCore version 2.12.0 and is scheduled for removal in " "version 2.14.0." ) - warnings.warn(msg, ESMValCoreDeprecationWarning) + warnings.warn(msg, ESMValCoreDeprecationWarning, stacklevel=2) if self.get("config_file") is None: return None return Path(self["config_file"]).parent diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index 9cc85bee5e..dd3f2d268d 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -311,7 +311,7 @@ def validate_extra_facets_dir(value): "ESMValCore version 2.12.0 and is scheduled for removal in " "version 2.14.0. Please use a list instead." ) - warnings.warn(msg, ESMValCoreDeprecationWarning) + warnings.warn(msg, ESMValCoreDeprecationWarning, stacklevel=2) value = list(value) return validate_pathlist(value) @@ -371,7 +371,7 @@ def _handle_deprecation( f"been deprecated in ESMValCore version {deprecated_version} and is " f"scheduled for removal in version {remove_version}.{more_info}" ) - warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning) + warnings.warn(deprecation_msg, ESMValCoreDeprecationWarning, stacklevel=2) # TODO: remove in v2.14.0 diff --git a/esmvalcore/config/_validated_config.py b/esmvalcore/config/_validated_config.py index 898abf3bb8..dca0a543f4 100644 --- a/esmvalcore/config/_validated_config.py +++ b/esmvalcore/config/_validated_config.py @@ -121,6 +121,7 @@ def check_missing(self): warnings.warn( f"`{key}` is not defined{more_info}", MissingConfigParameter, + stacklevel=1, ) def copy(self): diff --git a/esmvalcore/experimental/_warnings.py b/esmvalcore/experimental/_warnings.py index ddc474f568..548dc4e853 100644 --- a/esmvalcore/experimental/_warnings.py +++ b/esmvalcore/experimental/_warnings.py @@ -14,4 +14,5 @@ def _warning_formatter(message, category, filename, lineno, line=None): "\n Thank you for trying out the new ESMValCore API." "\n Note that this API is experimental and may be subject to change." "\n More info: https://github.com/ESMValGroup/ESMValCore/issues/498", + stacklevel=1, ) diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 3429078a5d..851aae49f0 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -717,13 +717,13 @@ def _run(self, _): if step in product.settings: product.apply(step, self.debug) if block == blocks[-1]: - product.cubes # pylint: disable=pointless-statement + product.cubes # noqa: B018 pylint: disable=pointless-statement product.close() saved.add(product.filename) for product in self.products: if product.filename not in saved: - product.cubes # pylint: disable=pointless-statement + product.cubes # noqa: B018 pylint: disable=pointless-statement product.close() metadata_files = write_metadata( diff --git a/esmvalcore/preprocessor/_derive/__init__.py b/esmvalcore/preprocessor/_derive/__init__.py index 065845ef4d..add5d822e6 100644 --- a/esmvalcore/preprocessor/_derive/__init__.py +++ b/esmvalcore/preprocessor/_derive/__init__.py @@ -27,7 +27,7 @@ def _get_all_derived_variables(): module = importlib.import_module( f"esmvalcore.preprocessor._derive.{short_name}" ) - derivers[short_name] = getattr(module, "DerivedVariable") + derivers[short_name] = module.DerivedVariable return derivers diff --git a/esmvalcore/preprocessor/_derive/ctotal.py b/esmvalcore/preprocessor/_derive/ctotal.py index 8d8d00faef..159289f13e 100644 --- a/esmvalcore/preprocessor/_derive/ctotal.py +++ b/esmvalcore/preprocessor/_derive/ctotal.py @@ -37,12 +37,12 @@ def calculate(cubes): c_soil_cube = cubes.extract_cube( Constraint(name="soil_mass_content_of_carbon") ) - except iris.exceptions.ConstraintMismatchError: + except iris.exceptions.ConstraintMismatchError as exc: raise ValueError( f"No cube from {cubes} can be loaded with " f"standard name CMIP5: soil_carbon_content " f"or CMIP6: soil_mass_content_of_carbon" - ) + ) from exc c_veg_cube = cubes.extract_cube( Constraint(name="vegetation_carbon_content") ) diff --git a/esmvalcore/preprocessor/_derive/ohc.py b/esmvalcore/preprocessor/_derive/ohc.py index 05590c9f3b..6cea2b06f5 100644 --- a/esmvalcore/preprocessor/_derive/ohc.py +++ b/esmvalcore/preprocessor/_derive/ohc.py @@ -74,7 +74,7 @@ def calculate(cubes): contains_dimension=t_coord_dim, dim_coords=False ) ] - for coord, dims in dim_coords + aux_coords: + for coord, _ in dim_coords + aux_coords: cube.remove_coord(coord) new_cube = cube * volume new_cube *= RHO_CP diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index d30255ec13..5f83b1946c 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -321,7 +321,7 @@ def _sort_cubes_by_time(cubes): msg = "One or more cubes {} are missing".format( cubes ) + " time coordinate: {}".format(str(exc)) - raise ValueError(msg) + raise ValueError(msg) from exc except TypeError as error: msg = ( "Cubes cannot be sorted " diff --git a/esmvalcore/preprocessor/_mapping.py b/esmvalcore/preprocessor/_mapping.py index ccbeed2816..a84df1e67e 100644 --- a/esmvalcore/preprocessor/_mapping.py +++ b/esmvalcore/preprocessor/_mapping.py @@ -57,10 +57,10 @@ def ref_to_dims_index_as_index(cube, ref): """Get dim for index ref.""" try: dim = int(ref) - except (ValueError, TypeError): + except (ValueError, TypeError) as exc: raise ValueError( "{} Incompatible type {} for slicing".format(ref, type(ref)) - ) + ) from exc if dim < 0 or dim > cube.ndim: msg = ( "Requested an iterator over a dimension ({}) " diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index a8558f6ee1..2fc34e4d85 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -588,7 +588,7 @@ def _load_scheme(src_cube: Cube, tgt_cube: Cube, scheme: str | dict): "version 2.11.0, ESMValCore is able to determine the most " "suitable regridding scheme based on the input data." ) - warnings.warn(msg, ESMValCoreDeprecationWarning) + warnings.warn(msg, ESMValCoreDeprecationWarning, stacklevel=2) scheme = "nearest" if scheme == "linear_extrapolate": @@ -601,7 +601,7 @@ def _load_scheme(src_cube: Cube, tgt_cube: Cube, scheme: str | dict): "latest/recipe/preprocessor.html#generic-regridding-schemes)." "This is an exact replacement." ) - warnings.warn(msg, ESMValCoreDeprecationWarning) + warnings.warn(msg, ESMValCoreDeprecationWarning, stacklevel=2) scheme = "linear" loaded_scheme = Linear(extrapolation_mode="extrapolate") logger.debug("Loaded regridding scheme %s", loaded_scheme) diff --git a/esmvalcore/preprocessor/_regrid_esmpy.py b/esmvalcore/preprocessor/_regrid_esmpy.py index b2cb559406..e4d0b40ba6 100755 --- a/esmvalcore/preprocessor/_regrid_esmpy.py +++ b/esmvalcore/preprocessor/_regrid_esmpy.py @@ -8,7 +8,7 @@ try: import ESMF as esmpy # noqa: N811 except ImportError: - raise exc + raise exc from None import warnings import iris @@ -78,8 +78,8 @@ def __init__( ): """Initialize class instance.""" # These regridders are not lazy, so load source and target data once. - src_cube.data # pylint: disable=pointless-statement - tgt_cube.data # pylint: disable=pointless-statement + src_cube.data # # noqa: B018 pylint: disable=pointless-statement + tgt_cube.data # # noqa: B018 pylint: disable=pointless-statement self.src_cube = src_cube self.tgt_cube = tgt_cube self.method = method @@ -100,7 +100,7 @@ def __call__(self, cube: Cube) -> Cube: """ # These regridders are not lazy, so load source data once. - cube.data # pylint: disable=pointless-statement + cube.data # # noqa: B018 pylint: disable=pointless-statement src_rep, dst_rep = get_grid_representants(cube, self.tgt_cube) regridder = build_regridder( src_rep, dst_rep, self.method, mask_threshold=self.mask_threshold @@ -140,7 +140,7 @@ def __init__(self, mask_threshold: float = 0.99): "`esmvalcore.preprocessor.regrid_schemes.IrisESMFRegrid` " "instead." ) - warnings.warn(msg, ESMValCoreDeprecationWarning) + warnings.warn(msg, ESMValCoreDeprecationWarning, stacklevel=2) self.mask_threshold = mask_threshold def __repr__(self) -> str: diff --git a/esmvalcore/preprocessor/_shared.py b/esmvalcore/preprocessor/_shared.py index 49272771b5..2355215800 100644 --- a/esmvalcore/preprocessor/_shared.py +++ b/esmvalcore/preprocessor/_shared.py @@ -95,7 +95,7 @@ def get_iris_aggregator( except (ValueError, TypeError) as exc: raise ValueError( f"Invalid kwargs for operator '{operator}': {str(exc)}" - ) + ) from exc return (aggregator, aggregator_kwargs) diff --git a/pyproject.toml b/pyproject.toml index 5a45ca2ab9..5abbbaa1d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ disable = [ line-length = 79 [tool.ruff.lint] select = [ + "B", "E", # pycodestyle "F", # pyflakes "I", # isort @@ -42,5 +43,9 @@ select = [ ignore = [ "E501", # Disable line-too-long as this is taken care of by the formatter. ] +[tool.ruff.lint.per-file-ignores] +"tests/**.py" = [ + "B011", # `assert False` is valid test code. +] [tool.ruff.lint.isort] known-first-party = ["esmvalcore"] diff --git a/tests/integration/preprocessor/_io/test_load.py b/tests/integration/preprocessor/_io/test_load.py index 1df7c3bb55..4c76ba2651 100644 --- a/tests/integration/preprocessor/_io/test_load.py +++ b/tests/integration/preprocessor/_io/test_load.py @@ -90,7 +90,7 @@ def test_callback_remove_attributes_from_coords(self): ) for coord in cube.coords(): for attr in attributes: - self.assertTrue(attr not in cube.attributes) + self.assertTrue(attr not in coord.attributes) def test_callback_fix_lat_units(self): """Test callback for fixing units.""" @@ -118,7 +118,9 @@ def test_fail_empty_cubes(self, mock_load_raw): def load_with_warning(*_, **__): """Mock load with a warning.""" warnings.warn( - "This is a custom expected warning", category=UserWarning + "This is a custom expected warning", + category=UserWarning, + stacklevel=2, ) return CubeList([Cube(0)]) diff --git a/tests/integration/recipe/test_check.py b/tests/integration/recipe/test_check.py index d168a3e010..1be8324037 100644 --- a/tests/integration/recipe/test_check.py +++ b/tests/integration/recipe/test_check.py @@ -1,6 +1,7 @@ """Integration tests for :mod:`esmvalcore._recipe.check`.""" import os.path +import subprocess from pathlib import Path from typing import Any, List from unittest import mock @@ -15,6 +16,80 @@ from esmvalcore.exceptions import RecipeError from esmvalcore.preprocessor import PreprocessorFile + +def test_ncl_version(mocker): + ncl = "/path/to/ncl" + mocker.patch.object( + check, + "which", + autospec=True, + return_value=ncl, + ) + mocker.patch.object( + check.subprocess, + "check_output", + autospec=True, + return_value="6.6.2\n", + ) + check.ncl_version() + + +def test_ncl_version_too_low(mocker): + ncl = "/path/to/ncl" + mocker.patch.object( + check, + "which", + autospec=True, + return_value=ncl, + ) + mocker.patch.object( + check.subprocess, + "check_output", + autospec=True, + return_value="6.3.2\n", + ) + with pytest.raises( + RecipeError, + match="NCL version 6.4 or higher is required", + ): + check.ncl_version() + + +def test_ncl_version_no_ncl(mocker): + mocker.patch.object( + check, + "which", + autospec=True, + return_value=None, + ) + with pytest.raises( + RecipeError, + match="cannot find an NCL installation", + ): + check.ncl_version() + + +def test_ncl_version_broken(mocker): + ncl = "/path/to/ncl" + mocker.patch.object( + check, + "which", + autospec=True, + return_value=ncl, + ) + mocker.patch.object( + check.subprocess, + "check_output", + autospec=True, + side_effect=subprocess.CalledProcessError(1, [ncl, "-V"]), + ) + with pytest.raises( + RecipeError, + match="NCL installation appears to be broken", + ): + check.ncl_version() + + ERR_ALL = "Looked for files matching%s" ERR_RANGE = "No input data available for years {} in files:\n{}" VAR = { diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index ae1ff9b5b7..5c58e9dc1a 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -1750,7 +1750,7 @@ def test_extract_shape_raises( def _test_output_product_consistency(products, preprocessor, statistics): product_out = defaultdict(list) - for i, product in enumerate(products): + for product in products: settings = product.settings.get(preprocessor) if settings: output_products = settings["output_products"] @@ -1760,7 +1760,7 @@ def _test_output_product_consistency(products, preprocessor, statistics): product_out[identifier, statistic].append(preproc_file) # Make sure that output products are consistent - for (identifier, statistic), value in product_out.items(): + for (_, statistic), value in product_out.items(): assert statistic in statistics assert len(set(value)) == 1, "Output products are not equal" @@ -1908,7 +1908,7 @@ def test_multi_model_statistics_exclude(tmp_path, patched_datafinder, session): assert len(product_out) == len(statistics) assert "OBS" not in product_out - for id, prods in product_out: + for id, _ in product_out: assert id != "OBS" assert id == "CMIP5" task._initialize_product_provenance() diff --git a/tests/integration/test_task.py b/tests/integration/test_task.py index 2fb56b2cc4..cb8c632fbf 100644 --- a/tests/integration/test_task.py +++ b/tests/integration/test_task.py @@ -341,7 +341,9 @@ def _get_diagnostic_tasks(tmp_path, diagnostic_text, extension): def test_diagnostic_run_task(monkeypatch, executable, diag_text, tmp_path): """Run DiagnosticTask that will not fail.""" - def _run(self, input_filesi=[]): + def _run(self, input_filesi=None): + if input_filesi is None: + input_filesi = [] print(f"running task {self.name}") task = _get_diagnostic_tasks(tmp_path, diag_text, executable[1]) @@ -356,7 +358,9 @@ def test_diagnostic_run_task_fail( ): """Run DiagnosticTask that will fail.""" - def _run(self, input_filesi=[]): + def _run(self, input_filesi=None): + if input_filesi is None: + input_filesi = [] print(f"running task {self.name}") task = _get_diagnostic_tasks(tmp_path, diag_text[0], executable[1]) diff --git a/tests/sample_data/experimental/test_run_recipe.py b/tests/sample_data/experimental/test_run_recipe.py index 141cc74c57..d0d6b079e7 100644 --- a/tests/sample_data/experimental/test_run_recipe.py +++ b/tests/sample_data/experimental/test_run_recipe.py @@ -95,7 +95,7 @@ def test_run_recipe( assert isinstance(output.read_main_log(), str) assert isinstance(output.read_main_log_debug(), str) - for task, task_output in output.items(): + for _, task_output in output.items(): assert isinstance(task_output, TaskOutput) assert len(task_output) > 0 diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index 8c005f1400..4279d60df6 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -101,7 +101,7 @@ def test_get_second_cube(self): def test_get_default_raises(self): """Check that the default raises (Fix is not a cube).""" - with pytest.raises(Exception): + with pytest.raises(ValueError): self.fix.get_cube_from_list(self.cubes) def test_get_default(self): diff --git a/tests/unit/preprocessor/_area/test_area.py b/tests/unit/preprocessor/_area/test_area.py index 9e88002aaa..99eaf3f150 100644 --- a/tests/unit/preprocessor/_area/test_area.py +++ b/tests/unit/preprocessor/_area/test_area.py @@ -1333,7 +1333,7 @@ def test_update_shapefile_path_abs(session, tmp_path): # Test with Path and str object for shapefile_in in (shapefile, str(shapefile)): - shapefile_out = _update_shapefile_path(shapefile, session=session) + shapefile_out = _update_shapefile_path(shapefile_in, session=session) assert isinstance(shapefile_out, Path) assert shapefile_out == shapefile diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 39c11c944c..653cd61038 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -195,11 +195,7 @@ def get_cubes_for_validation_test(frequency, lazy=False): def get_cube_for_equal_coords_test(num_cubes): """Set up cubes with equal auxiliary coordinates.""" - cubes = [] - - for num in range(num_cubes): - cube = generate_cube_from_dates("monthly") - cubes.append(cube) + cubes = [generate_cube_from_dates("monthly") for _ in range(num_cubes)] # Create cubes that have one exactly equal coordinate ('year'), one # coordinate with matching names ('m') and one coordinate with non-matching diff --git a/tests/unit/preprocessor/_other/test_other.py b/tests/unit/preprocessor/_other/test_other.py index a2237bfb6a..c50bed0a83 100644 --- a/tests/unit/preprocessor/_other/test_other.py +++ b/tests/unit/preprocessor/_other/test_other.py @@ -120,9 +120,9 @@ def test_histogram_defaults(cube, lazy): ) np.testing.assert_allclose(result.data.mask, [False] * 10) bin_coord = result.coord("air_temperature") - bin_coord.shape == (10,) - bin_coord.dtype == np.float64 - bin_coord.bounds_dtype == np.float64 + assert bin_coord.shape == (10,) + assert bin_coord.dtype == np.float64 + assert bin_coord.bounds_dtype == np.float64 np.testing.assert_allclose( bin_coord.points, [0.35, 1.05, 1.75, 2.45, 3.15, 3.85, 4.55, 5.25, 5.95, 6.65], @@ -196,9 +196,9 @@ def test_histogram_over_time(cube, lazy, weights, normalization): np.testing.assert_allclose(result.data, expected_data) np.testing.assert_allclose(result.data.mask, expected_data.mask) bin_coord = result.coord("air_temperature") - bin_coord.shape == (10,) - bin_coord.dtype == np.float64 - bin_coord.bounds_dtype == np.float64 + assert bin_coord.shape == (3,) + assert bin_coord.dtype == np.float64 + assert bin_coord.bounds_dtype == np.float64 np.testing.assert_allclose(bin_coord.points, [5.5, 7.5, 9.5]) np.testing.assert_allclose( bin_coord.bounds, @@ -231,9 +231,9 @@ def test_histogram_fully_masked(cube, lazy, normalization): ) np.testing.assert_equal(result.data.mask, [True] * 10) bin_coord = result.coord("air_temperature") - bin_coord.shape == (10,) - bin_coord.dtype == np.float64 - bin_coord.bounds_dtype == np.float64 + assert bin_coord.shape == (10,) + assert bin_coord.dtype == np.float64 + assert bin_coord.bounds_dtype == np.float64 np.testing.assert_allclose( bin_coord.points, [0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5], @@ -303,9 +303,9 @@ def test_histogram_weights(cube, lazy, weights, normalization): np.testing.assert_allclose(result.data, expected_data) np.testing.assert_allclose(result.data.mask, expected_data.mask) bin_coord = result.coord("air_temperature") - bin_coord.shape == (10,) - bin_coord.dtype == np.float64 - bin_coord.bounds_dtype == np.float64 + assert bin_coord.shape == (3,) + assert bin_coord.dtype == np.float64 + assert bin_coord.bounds_dtype == np.float64 np.testing.assert_allclose(bin_coord.points, [1.0, 3.0, 6.0]) np.testing.assert_allclose( bin_coord.bounds, diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 1348dc0ebf..232803b627 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -110,7 +110,7 @@ def test_session_setter(): assert ds._session is None assert ds.supplementaries[0]._session is None - ds.session + ds.session # noqa: B018 assert isinstance(ds.session, Session) assert ds.session == ds.supplementaries[0].session diff --git a/tests/unit/test_iris_helpers.py b/tests/unit/test_iris_helpers.py index ccfd6fbbf6..1b5066ae51 100644 --- a/tests/unit/test_iris_helpers.py +++ b/tests/unit/test_iris_helpers.py @@ -329,10 +329,10 @@ def test_rechunk_cube_partly_lazy(cube_3d, complete_dims): input_cube = cube_3d.copy() # Realize some arrays - input_cube.data - input_cube.coord("xyz").points - input_cube.coord("xyz").bounds - input_cube.cell_measure("cell_measure").data + input_cube.data # noqa: B018 + input_cube.coord("xyz").points # noqa: B018 + input_cube.coord("xyz").bounds # noqa: B018 + input_cube.cell_measure("cell_measure").data # noqa: B018 result = rechunk_cube(input_cube, complete_dims, remaining_dims=2) From 12798741cd4422adc1b7a6ecbc987dbc6e488388 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Oct 2024 14:14:22 +0100 Subject: [PATCH 16/19] retire Mambaforge (#2556) --- .circleci/config.yml | 10 +++++----- .github/workflows/create-condalock-file.yml | 1 - .github/workflows/install-from-conda.yml | 2 -- .github/workflows/install-from-pypi.yml | 2 -- .github/workflows/install-from-source.yml | 2 -- .github/workflows/run-tests-monitor.yml | 2 -- .github/workflows/run-tests.yml | 2 -- 7 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8232620e35..bbfdd87fb8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -154,7 +154,7 @@ jobs: test_installation_from_source_test_mode: # Test installation from source docker: - - image: condaforge/mambaforge + - image: condaforge/miniforge3 resource_class: large steps: - test_installation_from_source @@ -162,7 +162,7 @@ jobs: test_installation_from_source_develop_mode: # Test development installation docker: - - image: condaforge/mambaforge + - image: condaforge/miniforge3 resource_class: large steps: - test_installation_from_source: @@ -172,7 +172,7 @@ jobs: test_with_upstream_developments: # Test with development versions of upstream packages docker: - - image: condaforge/mambaforge + - image: condaforge/miniforge3 resource_class: large steps: - test_installation_from_source: @@ -192,7 +192,7 @@ jobs: # Test conda package installation working_directory: /esmvaltool docker: - - image: condaforge/mambaforge + - image: condaforge/miniforge3 resource_class: medium steps: - run: @@ -214,7 +214,7 @@ jobs: build_documentation: # Test building documentation docker: - - image: condaforge/mambaforge + - image: condaforge/miniforge3 resource_class: medium steps: - checkout diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml index 5e1eaec889..83c9c70c8f 100644 --- a/.github/workflows/create-condalock-file.yml +++ b/.github/workflows/create-condalock-file.yml @@ -29,7 +29,6 @@ jobs: activate-environment: esmvaltool-fromlock python-version: "3.12" miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - name: Update and show conda config run: | diff --git a/.github/workflows/install-from-conda.yml b/.github/workflows/install-from-conda.yml index 88e78619ea..d77ef193aa 100644 --- a/.github/workflows/install-from-conda.yml +++ b/.github/workflows/install-from-conda.yml @@ -49,7 +49,6 @@ jobs: with: python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p conda_install_linux_artifacts_python_${{ matrix.python-version }} - name: Record versions @@ -85,7 +84,6 @@ jobs: architecture: ${{ matrix.architecture }} python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p conda_install_osx_artifacts_python_${{ matrix.python-version }} - name: Record versions diff --git a/.github/workflows/install-from-pypi.yml b/.github/workflows/install-from-pypi.yml index 1e326c89eb..bb18afa7c5 100644 --- a/.github/workflows/install-from-pypi.yml +++ b/.github/workflows/install-from-pypi.yml @@ -52,7 +52,6 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p pip_install_linux_artifacts_python_${{ matrix.python-version }} - name: Record versions @@ -90,7 +89,6 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p pip_install_osx_artifacts_python_${{ matrix.python-version }} - name: Record versions diff --git a/.github/workflows/install-from-source.yml b/.github/workflows/install-from-source.yml index 7cb8c7d629..0d193bd79a 100644 --- a/.github/workflows/install-from-source.yml +++ b/.github/workflows/install-from-source.yml @@ -50,7 +50,6 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p source_install_linux_artifacts_python_${{ matrix.python-version }} - name: Record versions @@ -89,7 +88,6 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p source_install_osx_artifacts_python_${{ matrix.python-version }} - name: Record versions diff --git a/.github/workflows/run-tests-monitor.yml b/.github/workflows/run-tests-monitor.yml index 160bdd2850..27c0853730 100644 --- a/.github/workflows/run-tests-monitor.yml +++ b/.github/workflows/run-tests-monitor.yml @@ -35,7 +35,6 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p test_linux_artifacts_python_${{ matrix.python-version }} - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt @@ -70,7 +69,6 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p test_osx_artifacts_python_${{ matrix.python-version }} - run: conda --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 73e15c100c..743ac00f45 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -52,7 +52,6 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p test_linux_artifacts_python_${{ matrix.python-version }} - run: conda --version 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt @@ -90,7 +89,6 @@ jobs: environment-file: environment.yml python-version: ${{ matrix.python-version }} miniforge-version: "latest" - miniforge-variant: Mambaforge use-mamba: true - run: mkdir -p test_osx_artifacts_python_${{ matrix.python-version }} - run: conda --version 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt From e69ec5dfd9e1d917e6426bf1ab21d3551d00857d Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 16 Oct 2024 16:08:53 +0200 Subject: [PATCH 17/19] Enable ruff pydocstyle linter rule (#2547) Co-authored-by: Manuel Schlund <32543114+schlunma@users.noreply.github.com> --- esmvalcore/cmor/_fixes/__init__.py | 2 +- esmvalcore/cmor/_fixes/cmip5/gfdl_cm3.py | 4 +- esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py | 4 +- esmvalcore/cmor/_fixes/cmip5/gfdl_esm2m.py | 2 +- pyproject.toml | 10 ++ tests/__init__.py | 6 +- tests/conftest.py | 2 +- .../cmor/_fixes/cmip5/test_bnu_esm.py | 2 +- .../cmor/_fixes/cmip5/test_cnrm_cm5.py | 4 +- .../cmor/_fixes/cmip5/test_ec_earth.py | 16 +- .../cmor/_fixes/cmip5/test_fio_esm.py | 4 +- .../cmor/_fixes/cmip5/test_gfdl_cm2p1.py | 6 +- .../cmor/_fixes/cmip5/test_gfdl_cm3.py | 4 +- .../cmor/_fixes/cmip5/test_gfdl_esm2g.py | 8 +- .../cmor/_fixes/cmip5/test_gfdl_esm2m.py | 6 +- .../cmor/_fixes/cmip5/test_hadgem2_cc.py | 4 +- .../cmor/_fixes/cmip5/test_miroc_esm_chem.py | 2 +- .../cmor/_fixes/cmip5/test_mri_esm1.py | 2 +- .../cmor/_fixes/cmip5/test_noresm1_me.py | 2 +- .../integration/cmor/_fixes/icon/test_icon.py | 4 +- .../cmor/_fixes/ipslcm/test_ipsl_cm6.py | 1 - tests/integration/cmor/_fixes/test_common.py | 1 - .../cmor/_fixes/test_data/create_test_data.py | 2 +- tests/integration/cmor/test_fix.py | 2 +- .../preprocessor/_derive/test_sithick.py | 2 - .../integration/preprocessor/_io/test_save.py | 2 +- .../preprocessor/_mask/__init__.py | 2 +- .../preprocessor/_regrid/__init__.py | 5 +- .../_regrid/test_extract_coordinate_points.py | 25 ++- .../_regrid/test_extract_levels.py | 3 +- .../_regrid/test_extract_location.py | 10 +- .../_regrid/test_extract_point.py | 26 ++- .../_regrid/test_get_cmor_levels.py | 7 +- .../_regrid/test_get_file_levels.py | 3 +- .../preprocessor/_regrid/test_regrid.py | 6 +- .../_regrid/test_regrid_schemes.py | 4 +- .../_supplementary_vars/test_register.py | 2 +- tests/integration/recipe/test_recipe.py | 2 - tests/integration/test_task.py | 2 +- tests/unit/cmor/test_cmor_check.py | 170 +++++++++++------- tests/unit/cmor/test_generic_fix.py | 2 +- tests/unit/config/test_config_object.py | 2 +- tests/unit/experimental/test_recipe_output.py | 4 - tests/unit/local/test_select_files.py | 6 +- tests/unit/local/test_time.py | 18 +- tests/unit/preprocessor/_area/test_area.py | 6 +- .../test_compare_with_refs.py | 2 +- tests/unit/preprocessor/_mask/test_mask.py | 2 +- tests/unit/preprocessor/_regrid/__init__.py | 15 +- .../preprocessor/_regrid/test__create_cube.py | 6 +- .../preprocessor/_regrid/test__stock_cube.py | 6 +- .../_regrid/test_extract_point.py | 6 +- .../_regrid/test_extract_regional_grid.py | 2 +- .../unit/preprocessor/_regrid/test_regrid.py | 23 ++- tests/unit/preprocessor/_time/test_time.py | 8 +- .../unit/preprocessor/_volume/test_volume.py | 31 ++-- tests/unit/preprocessor/test_shared.py | 2 +- tests/unit/recipe/test_recipe.py | 2 +- tests/unit/test_naming.py | 12 +- 59 files changed, 264 insertions(+), 262 deletions(-) diff --git a/esmvalcore/cmor/_fixes/__init__.py b/esmvalcore/cmor/_fixes/__init__.py index e50e749659..4327b12471 100644 --- a/esmvalcore/cmor/_fixes/__init__.py +++ b/esmvalcore/cmor/_fixes/__init__.py @@ -1,5 +1,5 @@ """ -Automatic fixes for input data +Automatic fixes for input data. Module to apply automatic fixes at different levels to input data for known errors. diff --git a/esmvalcore/cmor/_fixes/cmip5/gfdl_cm3.py b/esmvalcore/cmor/_fixes/cmip5/gfdl_cm3.py index f2cf4b7101..3938d20e0a 100644 --- a/esmvalcore/cmor/_fixes/cmip5/gfdl_cm3.py +++ b/esmvalcore/cmor/_fixes/cmip5/gfdl_cm3.py @@ -9,7 +9,7 @@ class AllVars(BaseAllVars): class Areacello(Fix): - """Fixes for areacello""" + """Fixes for areacello.""" def fix_metadata(self, cubes): """ @@ -56,7 +56,7 @@ def fix_data(self, cube): class Tos(Fix): - """Fixes for tos""" + """Fixes for tos.""" def fix_metadata(self, cubes): """ diff --git a/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py b/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py index f3722c9d07..cf199a3229 100644 --- a/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py +++ b/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2g.py @@ -37,7 +37,7 @@ def fix_metadata(self, cubes): class Areacello(Fix): - """Fixes for areacello""" + """Fixes for areacello.""" def fix_metadata(self, cubes): """ @@ -116,6 +116,7 @@ def fix_metadata(self, cubes): Parameters ---------- cubes: iris.cube.CubeList + Returns ------- iris.cube.CubeList @@ -137,6 +138,7 @@ def fix_metadata(self, cubes): Parameters ---------- cubes: iris.cube.CubeList + Returns ------- iris.cube.CubeList diff --git a/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2m.py b/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2m.py index e358bf525c..e7993c8903 100644 --- a/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2m.py +++ b/esmvalcore/cmor/_fixes/cmip5/gfdl_esm2m.py @@ -9,7 +9,7 @@ class AllVars(BaseAllVars): class Areacello(Fix): - """Fixes for areacello""" + """Fixes for areacello.""" def fix_metadata(self, cubes): """ diff --git a/pyproject.toml b/pyproject.toml index 5abbbaa1d1..6234370689 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ line-length = 79 [tool.ruff.lint] select = [ "B", + "D", # pydocstyle "E", # pycodestyle "F", # pyflakes "I", # isort @@ -42,10 +43,19 @@ select = [ ] ignore = [ "E501", # Disable line-too-long as this is taken care of by the formatter. + "D105", # Disable Missing docstring in magic method as these are well defined. ] [tool.ruff.lint.per-file-ignores] "tests/**.py" = [ "B011", # `assert False` is valid test code. + # Docstrings in tests are only needed if the code is not self-explanatory. + "D100", # Missing docstring in public module + "D101", # Missing docstring in public class + "D102", # Missing docstring in public method + "D103", # Missing docstring in public function + "D104", # Missing docstring in public package ] [tool.ruff.lint.isort] known-first-party = ["esmvalcore"] +[tool.ruff.lint.pydocstyle] +convention = "numpy" diff --git a/tests/__init__.py b/tests/__init__.py index 7c55516496..e24d27357e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -19,11 +19,7 @@ class Test(unittest.TestCase): """Provides esmvaltool specific testing functionality.""" def _remove_testcase_patches(self): - """ - Helper method to remove per-testcase patches installed by - :meth:`patch`. - - """ + """Remove per-testcase patches installed by :meth:`patch`.""" # Remove all patches made, ignoring errors. for patch in self.testcase_patches: patch.stop() diff --git a/tests/conftest.py b/tests/conftest.py index 0d385fb2f4..d973b76695 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ @pytest.fixture def cfg_default(mocker): - """Configuration object with defaults.""" + """Create a configuration object with default values.""" cfg = deepcopy(CFG) cfg.load_from_dirs([]) return cfg diff --git a/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py b/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py index a48dd9860b..1921282378 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_bnu_esm.py @@ -29,7 +29,7 @@ def setUp(self): self.fix = Cl(None) def test_get(self): - """Test fix get""" + """Test fix get.""" fix = Fix.get_fixes("CMIP5", "BNU-ESM", "Amon", "cl") assert fix == [Cl(None), GenericFix(None)] diff --git a/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py b/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py index cc7c9a8059..12cdf8fd53 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py +++ b/tests/integration/cmor/_fixes/cmip5/test_cnrm_cm5.py @@ -19,7 +19,7 @@ def setUp(self): self.fix = Msftmyz(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "CNRM-CM5", "Amon", "msftmyz"), [Msftmyz(None), GenericFix(None)], @@ -41,7 +41,7 @@ def setUp(self): self.fix = Msftmyzba(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "CNRM-CM5", "Amon", "msftmyzba"), [Msftmyzba(None), GenericFix(None)], diff --git a/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py b/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py index 9fdd838268..8390e9385b 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py +++ b/tests/integration/cmor/_fixes/cmip5/test_ec_earth.py @@ -28,7 +28,7 @@ def setUp(self): self.fix = Sic(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "EC-EARTH", "Amon", "sic"), [Sic(None), GenericFix(None)], @@ -50,7 +50,7 @@ def setUp(self): self.fix = Sftlf(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "EC-EARTH", "Amon", "sftlf"), [Sftlf(None), GenericFix(None)], @@ -68,7 +68,6 @@ class TestTas(unittest.TestCase): def setUp(self): """Prepare tests.""" - height_coord = DimCoord( 2.0, standard_name="height", @@ -99,7 +98,7 @@ def setUp(self): self.fix = Tas(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "EC-EARTH", "Amon", "tas"), [Tas(None), GenericFix(None)], @@ -107,7 +106,6 @@ def test_get(self): def test_tas_fix_metadata(self): """Test metadata fix.""" - out_cube_without = self.fix.fix_metadata(self.cube_without) # make sure this does not raise an error @@ -131,7 +129,6 @@ class TestAreacello(unittest.TestCase): def setUp(self): """Prepare tests.""" - latitude = Cube( np.ones((2, 2)), standard_name="latitude", @@ -163,7 +160,7 @@ def setUp(self): self.fix = Areacello(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "EC-EARTH", "Omon", "areacello"), [Areacello(None), GenericFix(None)], @@ -171,7 +168,6 @@ def test_get(self): def test_areacello_fix_metadata(self): """Test metadata fix.""" - out_cube = self.fix.fix_metadata(self.cubes) assert len(out_cube) == 1 @@ -184,7 +180,6 @@ class TestPr(unittest.TestCase): def setUp(self): """Prepare tests.""" - wrong_time_coord = AuxCoord( points=[1.0, 2.0, 1.0, 2.0, 3.0], var_name="time", @@ -226,7 +221,7 @@ def setUp(self): self.fix = Pr(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "EC-EARTH", "Amon", "pr"), [Pr(None), GenericFix(None)], @@ -234,7 +229,6 @@ def test_get(self): def test_pr_fix_metadata(self): """Test metadata fix.""" - out_wrong_cube = self.fix.fix_metadata(self.wrong_cube) out_correct_cube = self.fix.fix_metadata(self.correct_cube) diff --git a/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py b/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py index 2da5fd742f..a9d582f98d 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py +++ b/tests/integration/cmor/_fixes/cmip5/test_fio_esm.py @@ -31,7 +31,7 @@ def setUp(self): self.fix = Ch4(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "FIO-ESM", "Amon", "ch4"), [Ch4(None), GenericFix(None)], @@ -53,7 +53,7 @@ def setUp(self): self.fix = Co2(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "FIO-ESM", "Amon", "co2"), [Co2(None), GenericFix(None)], diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py index 0ddcfdcacf..77b2521cb1 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm2p1.py @@ -44,7 +44,7 @@ def setUp(self): self.fix = Sftof(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-CM2P1", "fx", "sftof"), [Sftof(None), AllVars(None), GenericFix(None)], @@ -67,7 +67,7 @@ def setUp(self): self.fix = Areacello(self.vardef) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-CM2P1", "Amon", "areacello"), [ @@ -114,7 +114,7 @@ def setUp(self): self.fix = Sit(self.var_info_mock) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-CM2P1", "OImon", "sit"), [Sit(self.var_info_mock), AllVars(None), GenericFix(None)], diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py index fffe1812ee..d7647dfee1 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_cm3.py @@ -20,7 +20,7 @@ def setUp(self): self.fix = Sftof(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-CM3", "fx", "sftof"), [Sftof(None), AllVars(None), GenericFix(None)], @@ -43,7 +43,7 @@ def setUp(self): self.fix = Areacello(self.vardef) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-CM3", "Amon", "areacello"), [ diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py index 30421411e3..967ea83a2d 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2g.py @@ -79,7 +79,7 @@ def setUp(self): self.fix = Co2(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-ESM2G", "Amon", "co2"), [Co2(None), AllVars(None), GenericFix(None)], @@ -102,7 +102,7 @@ def setUp(self): self.fix = Usi(self.vardef) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-ESM2G", "day", "usi"), [Usi(self.vardef), AllVars(self.vardef), GenericFix(self.vardef)], @@ -124,7 +124,7 @@ def setUp(self): self.fix = Vsi(self.vardef) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-ESM2G", "day", "vsi"), [Vsi(self.vardef), AllVars(self.vardef), GenericFix(self.vardef)], @@ -146,7 +146,7 @@ def setUp(self): self.fix = Areacello(self.vardef) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-ESM2G", "fx", "areacello"), [ diff --git a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py index 29f3ba4c7e..428bc72434 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py +++ b/tests/integration/cmor/_fixes/cmip5/test_gfdl_esm2m.py @@ -25,7 +25,7 @@ def setUp(self): self.fix = Sftof(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-ESM2M", "fx", "sftof"), [Sftof(None), AllVars(None), GenericFix(None)], @@ -47,7 +47,7 @@ def setUp(self): self.fix = Co2(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-ESM2M", "Amon", "co2"), [Co2(None), AllVars(None), GenericFix(None)], @@ -70,7 +70,7 @@ def setUp(self): self.fix = Areacello(self.vardef) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "GFDL-ESM2M", "fx", "areacello"), [ diff --git a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py index 13642a35ad..fd9a4c1ae2 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py +++ b/tests/integration/cmor/_fixes/cmip5/test_hadgem2_cc.py @@ -11,7 +11,7 @@ class TestAllVars(unittest.TestCase): """Test allvars fixes.""" def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "HADGEM2-CC", "Amon", "tas"), [AllVars(None), GenericFix(None)], @@ -22,7 +22,7 @@ class TestO2(unittest.TestCase): """Test o2 fixes.""" def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "HADGEM2-CC", "Amon", "o2"), [O2(None), AllVars(None), GenericFix(None)], diff --git a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py index 21624bcc44..9f97232c55 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py +++ b/tests/integration/cmor/_fixes/cmip5/test_miroc_esm_chem.py @@ -19,7 +19,7 @@ def setUp(self): self.fix = Tro3(None) def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "MIROC-ESM-CHEM", "Amon", "tro3"), [Tro3(None), GenericFix(None)], diff --git a/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py b/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py index a847a30f15..16969fc6d8 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py +++ b/tests/integration/cmor/_fixes/cmip5/test_mri_esm1.py @@ -11,7 +11,7 @@ class TestMsftmyz(unittest.TestCase): """Test msftmyz fixes.""" def test_get(self): - """Test fix get""" + """Test fix get.""" self.assertListEqual( Fix.get_fixes("CMIP5", "MRI-ESM1", "Amon", "msftmyz"), [Msftmyz(None), GenericFix(None)], diff --git a/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py b/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py index 87d94760a1..bedac4aa50 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py +++ b/tests/integration/cmor/_fixes/cmip5/test_noresm1_me.py @@ -72,7 +72,7 @@ def test_tas(cubes_in, cubes_out): def test_get(): - """Test fix get""" + """Test fix get.""" assert Fix.get_fixes("CMIP5", "NORESM1-ME", "Amon", "tas") == [ Tas(None), GenericFix(None), diff --git a/tests/integration/cmor/_fixes/icon/test_icon.py b/tests/integration/cmor/_fixes/icon/test_icon.py index b87c052008..cfd3ea6726 100644 --- a/tests/integration/cmor/_fixes/icon/test_icon.py +++ b/tests/integration/cmor/_fixes/icon/test_icon.py @@ -1,4 +1,4 @@ -"""Tests for the ICON on-the-fly CMORizer.""" +"""Test the ICON on-the-fly CMORizer.""" from copy import deepcopy from datetime import datetime @@ -134,7 +134,7 @@ def cubes_2d_lat_lon_grid(): @pytest.fixture def simple_unstructured_cube(): - """Simple cube with unstructured grid.""" + """Create a cube with an unstructured grid.""" time_coord = DimCoord( [0], var_name="time", diff --git a/tests/integration/cmor/_fixes/ipslcm/test_ipsl_cm6.py b/tests/integration/cmor/_fixes/ipslcm/test_ipsl_cm6.py index f13c329d49..4b2e83784f 100644 --- a/tests/integration/cmor/_fixes/ipslcm/test_ipsl_cm6.py +++ b/tests/integration/cmor/_fixes/ipslcm/test_ipsl_cm6.py @@ -19,7 +19,6 @@ def test_get_tas_fix(): @pytest.fixture def cubes(): """``tas`` cube.""" - cube = iris.cube.Cube( [200.0], # chilly, isn't it ? var_name="tas", diff --git a/tests/integration/cmor/_fixes/test_common.py b/tests/integration/cmor/_fixes/test_common.py index 075335d7d8..781c5df0f9 100644 --- a/tests/integration/cmor/_fixes/test_common.py +++ b/tests/integration/cmor/_fixes/test_common.py @@ -305,7 +305,6 @@ def get_tos_cubes(wrong_ij_names=False, ij_bounds=False): def get_tos_regular_grid_cubes(): """Cubes containing tos variable.""" - time_coord = iris.coords.DimCoord( 1.0, bounds=[0.0, 2.0], diff --git a/tests/integration/cmor/_fixes/test_data/create_test_data.py b/tests/integration/cmor/_fixes/test_data/create_test_data.py index 35dc40351b..c4ff26ab4b 100644 --- a/tests/integration/cmor/_fixes/test_data/create_test_data.py +++ b/tests/integration/cmor/_fixes/test_data/create_test_data.py @@ -456,7 +456,7 @@ def save_gfdl_cm4_cl_file(save_path): def main(): - """Main function to create datasets.""" + """Create all datasets.""" save_path = os.path.dirname(os.path.abspath(__file__)) save_cl_file_with_a(save_path) save_cl_file_with_ap(save_path) diff --git a/tests/integration/cmor/test_fix.py b/tests/integration/cmor/test_fix.py index 74a0a08841..4af47a44d7 100644 --- a/tests/integration/cmor/test_fix.py +++ b/tests/integration/cmor/test_fix.py @@ -22,7 +22,7 @@ class TestGenericFix: @pytest.fixture(autouse=True) def setup(self, mocker): - """Setup tests.""" + """Set up tests.""" self.mock_debug = mocker.patch( "esmvalcore.cmor._fixes.fix.GenericFix._debug_msg", autospec=True ) diff --git a/tests/integration/preprocessor/_derive/test_sithick.py b/tests/integration/preprocessor/_derive/test_sithick.py index d0523d6d7d..4b59a71cf4 100644 --- a/tests/integration/preprocessor/_derive/test_sithick.py +++ b/tests/integration/preprocessor/_derive/test_sithick.py @@ -8,7 +8,6 @@ def test_sispeed_calculation(): """Test calculation of `sithick`.""" - siconc = Cube(np.full((2, 2), 0.5), "sea_ice_area_fraction", units="1.0") sivol = Cube(np.full((2, 2), 0.5), "sea_ice_thickness") @@ -20,7 +19,6 @@ def test_sispeed_calculation(): def test_sispeed_calculation_percent(): """Test calculation of `sithick` with sit in %.""" - siconc = Cube(np.full((2, 2), 50.0), "sea_ice_area_fraction", units="%") sivol = Cube(np.full((2, 2), 0.5), "sea_ice_thickness") diff --git a/tests/integration/preprocessor/_io/test_save.py b/tests/integration/preprocessor/_io/test_save.py index 0bfd0c6958..0e4f6b4366 100644 --- a/tests/integration/preprocessor/_io/test_save.py +++ b/tests/integration/preprocessor/_io/test_save.py @@ -1,4 +1,4 @@ -"""Integration tests for :func:`esmvalcore.preprocessor.save`""" +"""Integration tests for :func:`esmvalcore.preprocessor.save`.""" import iris import netCDF4 diff --git a/tests/integration/preprocessor/_mask/__init__.py b/tests/integration/preprocessor/_mask/__init__.py index 9886ea9eed..97e5350864 100644 --- a/tests/integration/preprocessor/_mask/__init__.py +++ b/tests/integration/preprocessor/_mask/__init__.py @@ -1,5 +1,5 @@ """ -Test _mask.py +Test _mask.py. Integration tests for the esmvalcore.preprocessor._mask module """ diff --git a/tests/integration/preprocessor/_regrid/__init__.py b/tests/integration/preprocessor/_regrid/__init__.py index e19e118a0f..bb8eab5e54 100644 --- a/tests/integration/preprocessor/_regrid/__init__.py +++ b/tests/integration/preprocessor/_regrid/__init__.py @@ -1,4 +1 @@ -""" -Integration tests for the :mod:`esmvalcore.preprocessor._regrid` module. - -""" +"""Integration tests for the :mod:`esmvalcore.preprocessor._regrid` module.""" diff --git a/tests/integration/preprocessor/_regrid/test_extract_coordinate_points.py b/tests/integration/preprocessor/_regrid/test_extract_coordinate_points.py index d5cad7c9ec..1cf039b7b4 100644 --- a/tests/integration/preprocessor/_regrid/test_extract_coordinate_points.py +++ b/tests/integration/preprocessor/_regrid/test_extract_coordinate_points.py @@ -1,8 +1,4 @@ -""" -Integration tests for the :func:`esmvalcore.preprocessor.regrid.regrid` -function. - -""" +"""Integration tests for :func:`esmvalcore.preprocessor.regrid`.""" import unittest @@ -23,7 +19,7 @@ def setUp(self): self.cs = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) def test_extract_point__single_linear(self): - """Test linear interpolation when extracting a single point""" + """Test linear interpolation when extracting a single point.""" point = extract_coordinate_points( self.cube, {"grid_latitude": 2.1, "grid_longitude": 2.1}, @@ -71,8 +67,7 @@ def test_extract_point__single_linear(self): self.assert_array_equal(point.data, masked) def test_extract_point__single_nearest(self): - """Test nearest match when extracting a single point""" - + """Test nearest match when extracting a single point.""" point = extract_coordinate_points( self.cube, {"grid_latitude": 2.1, "grid_longitude": 2.1}, @@ -108,8 +103,7 @@ def test_extract_point__single_nearest(self): self.assert_array_equal(point.data, masked) def test_extract_point__multiple_linear(self): - """Test linear interpolation for an array of one coordinate""" - + """Test linear interpolation for an array of one coordinate.""" # Test points on the grid edges, on a grid point, halfway and # one in between. coords = self.cube.coords(dim_coords=True) @@ -176,8 +170,7 @@ def test_extract_point__multiple_linear(self): self.assert_array_equal(point.data, masked) def test_extract_point__multiple_nearest(self): - """Test nearest match for an array of one coordinate""" - + """Test nearest match for an array of one coordinate.""" point = extract_coordinate_points( self.cube, {"grid_latitude": [1, 1.1, 1.5, 1.501, 2, 4], "grid_longitude": 2}, @@ -231,8 +224,10 @@ def test_extract_point__multiple_nearest(self): self.assert_array_equal(point.data, masked) def test_extract_point__multiple_both_linear(self): - """Test for both latitude and longitude arrays, with - linear interpolation""" + """Test for both latitude and longitude arrays. + + With linear interpolation. + """ point = extract_coordinate_points( self.cube, { @@ -266,7 +261,7 @@ def test_extract_point__multiple_both_linear(self): np.testing.assert_allclose(point.data, result) def test_extract_point__multiple_both_nearest(self): - """Test for both latitude and longitude arrays, with nearest match""" + """Test for both latitude and longitude arrays, with nearest match.""" point = extract_coordinate_points( self.cube, { diff --git a/tests/integration/preprocessor/_regrid/test_extract_levels.py b/tests/integration/preprocessor/_regrid/test_extract_levels.py index 000931637d..82c9991a64 100644 --- a/tests/integration/preprocessor/_regrid/test_extract_levels.py +++ b/tests/integration/preprocessor/_regrid/test_extract_levels.py @@ -1,5 +1,4 @@ -"""Integration tests for the -:func:`esmvalcore.preprocessor.regrid.extract_levels` function.""" +"""Tests for :func:`esmvalcore.preprocessor.regrid.extract_levels` function.""" import unittest diff --git a/tests/integration/preprocessor/_regrid/test_extract_location.py b/tests/integration/preprocessor/_regrid/test_extract_location.py index b5c316bd39..3b3274fb82 100644 --- a/tests/integration/preprocessor/_regrid/test_extract_location.py +++ b/tests/integration/preprocessor/_regrid/test_extract_location.py @@ -117,10 +117,12 @@ def test_no_scheme_parameter(test_cube): @patch("esmvalcore.preprocessor._regrid.ssl.create_default_context") def test_create_default_ssl_context_raises_exception(mock_create, test_cube): - """Test the original way 'extract_location' worked before adding the - default SSL context, see - https://github.com/ESMValGroup/ESMValCore/issues/2012 for more - information.""" + """Test the original way 'extract_location' worked. + + Test the way `extract_location` worked before adding the default SSL + context, see https://github.com/ESMValGroup/ESMValCore/issues/2012 for more + information. + """ mock_create.side_effect = ssl.SSLSyscallError extract_location(test_cube, scheme="nearest", location="Peñacaballera") mock_create.assert_called_once() diff --git a/tests/integration/preprocessor/_regrid/test_extract_point.py b/tests/integration/preprocessor/_regrid/test_extract_point.py index 132d3fb8dd..dc139bd8d7 100644 --- a/tests/integration/preprocessor/_regrid/test_extract_point.py +++ b/tests/integration/preprocessor/_regrid/test_extract_point.py @@ -1,8 +1,4 @@ -""" -Integration tests for the :func:`esmvalcore.preprocessor.regrid.regrid` -function. - -""" +"""Integration tests for :func:`esmvalcore.preprocessor.regrid`.""" import unittest @@ -23,8 +19,7 @@ def setUp(self): self.cs = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) def test_extract_point__single_linear(self): - """Test linear interpolation when extracting a single point""" - + """Test linear interpolation when extracting a single point.""" point = extract_point(self.cube, 2.1, 2.1, scheme="linear") self.assertEqual(point.shape, (3,)) np.testing.assert_allclose(point.data, [5.5, 21.5, 37.5]) @@ -58,8 +53,7 @@ def test_extract_point__single_linear(self): assert point.data.mask.all() def test_extract_point__single_nearest(self): - """Test nearest match when extracting a single point""" - + """Test nearest match when extracting a single point.""" point = extract_point(self.cube, 2.1, 2.1, scheme="nearest") self.assertEqual(point.shape, (3,)) np.testing.assert_allclose(point.data, [5, 21, 37]) @@ -79,8 +73,7 @@ def test_extract_point__single_nearest(self): self.assert_array_equal(point.data, masked) def test_extract_point__multiple_linear(self): - """Test linear interpolation for an array of one coordinate""" - + """Test linear interpolation for an array of one coordinate.""" # Test points on the grid edges, on a grid point, halfway and # one in between. coords = self.cube.coords(dim_coords=True) @@ -135,8 +128,7 @@ def test_extract_point__multiple_linear(self): self.assert_array_equal(point.data, masked) def test_extract_point__multiple_nearest(self): - """Test nearest match for an array of one coordinate""" - + """Test nearest match for an array of one coordinate.""" point = extract_point( self.cube, [1, 1.1, 1.5, 1.501, 2, 4], 2, scheme="nearest" ) @@ -178,8 +170,10 @@ def test_extract_point__multiple_nearest(self): self.assert_array_equal(point.data, masked) def test_extract_point__multiple_both_linear(self): - """Test for both latitude and longitude arrays, with - linear interpolation""" + """Test for both latitude and longitude arrays. + + Uses linear interpolation. + """ point = extract_point( self.cube, [0, 1.1, 1.5, 1.51, 4, 5], @@ -211,7 +205,7 @@ def test_extract_point__multiple_both_linear(self): np.testing.assert_allclose(point.data, result) def test_extract_point__multiple_both_nearest(self): - """Test for both latitude and longitude arrays, with nearest match""" + """Test for both latitude and longitude arrays, with nearest match.""" point = extract_point( self.cube, [0, 1.1, 1.5, 1.51, 4, 5], diff --git a/tests/integration/preprocessor/_regrid/test_get_cmor_levels.py b/tests/integration/preprocessor/_regrid/test_get_cmor_levels.py index af18135070..e2c14c39fd 100644 --- a/tests/integration/preprocessor/_regrid/test_get_cmor_levels.py +++ b/tests/integration/preprocessor/_regrid/test_get_cmor_levels.py @@ -1,9 +1,4 @@ -""" -Integration tests for the :func: -`esmvalcore.preprocessor.regrid.get_cmor_levels` -function. - -""" +"""Tests for :func:`esmvalcore.preprocessor.regrid.get_cmor_levels`.""" import unittest diff --git a/tests/integration/preprocessor/_regrid/test_get_file_levels.py b/tests/integration/preprocessor/_regrid/test_get_file_levels.py index ed5069e2a9..3421fb8939 100644 --- a/tests/integration/preprocessor/_regrid/test_get_file_levels.py +++ b/tests/integration/preprocessor/_regrid/test_get_file_levels.py @@ -1,5 +1,4 @@ -"""Integration test for -:func:`esmvalcore.preprocessor.regrid.get_reference_levels`.""" +"""Tests for :func:`esmvalcore.preprocessor.regrid.get_reference_levels`.""" import iris.coords import iris.cube diff --git a/tests/integration/preprocessor/_regrid/test_regrid.py b/tests/integration/preprocessor/_regrid/test_regrid.py index bf39ee9ff2..05f6a475db 100644 --- a/tests/integration/preprocessor/_regrid/test_regrid.py +++ b/tests/integration/preprocessor/_regrid/test_regrid.py @@ -1,8 +1,4 @@ -""" -Integration tests for the :func:`esmvalcore.preprocessor.regrid.regrid` -function. - -""" +"""Integration tests for :func:`esmvalcore.preprocessor.regrid`.""" import iris import numpy as np diff --git a/tests/integration/preprocessor/_regrid/test_regrid_schemes.py b/tests/integration/preprocessor/_regrid/test_regrid_schemes.py index 2685d33207..6e548238d1 100644 --- a/tests/integration/preprocessor/_regrid/test_regrid_schemes.py +++ b/tests/integration/preprocessor/_regrid/test_regrid_schemes.py @@ -11,14 +11,14 @@ def set_data_to_const(cube, _, const=1.0): - """Dummy function to test ``GenericFuncScheme``.""" + """Compute something to test ``GenericFuncScheme``.""" cube = cube.copy(np.full(cube.shape, const)) return cube @pytest.fixture def generic_func_scheme(): - """Generic function scheme.""" + """Create a GenericFunctionScheme.""" return GenericFuncScheme(set_data_to_const, const=2) diff --git a/tests/integration/preprocessor/_supplementary_vars/test_register.py b/tests/integration/preprocessor/_supplementary_vars/test_register.py index 9512b5067e..8e9ce87685 100644 --- a/tests/integration/preprocessor/_supplementary_vars/test_register.py +++ b/tests/integration/preprocessor/_supplementary_vars/test_register.py @@ -28,7 +28,7 @@ def test_func(): def test_register_invalid_fails(): - """test that registering an invalid requirement fails.""" + """Test that registering an invalid requirement fails.""" with pytest.raises(NotImplementedError): @_supplementary_vars.register_supplementaries( diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index 5c58e9dc1a..f486db1657 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -569,7 +569,6 @@ def test_default_preprocessor_custom_order( tmp_path, patched_datafinder, session ): """Test if default settings are used when ``custom_order`` is ``True``.""" - content = dedent(""" preprocessors: default_custom_order: @@ -631,7 +630,6 @@ def test_invalid_preprocessor(tmp_path, patched_datafinder, session): def test_disable_preprocessor_function(tmp_path, patched_datafinder, session): """Test if default settings are used when ``custom_order`` is ``True``.""" - content = dedent(""" datasets: - dataset: HadGEM3-GC31-LL diff --git a/tests/integration/test_task.py b/tests/integration/test_task.py index cb8c632fbf..9570ec8e58 100644 --- a/tests/integration/test_task.py +++ b/tests/integration/test_task.py @@ -47,7 +47,7 @@ def _run(self, input_files): @pytest.fixture def example_tasks(tmp_path): - """Example tasks for testing the task runners.""" + """Create example tasks for testing the task runners.""" tasks = TaskSet() for i in range(3): task = MockBaseTask( diff --git a/tests/unit/cmor/test_cmor_check.py b/tests/unit/cmor/test_cmor_check.py index a7822ec03b..0dac7d2911 100644 --- a/tests/unit/cmor/test_cmor_check.py +++ b/tests/unit/cmor/test_cmor_check.py @@ -138,7 +138,7 @@ def test_warning_fail_on_error(self): ) def test_report_debug_message(self): - """ "Test report debug message function""" + """Test report debug message function.""" checker = CMORCheck(self.cube, self.var_info) self.assertFalse(checker.has_debug_messages()) checker.report_debug_message("New debug message") @@ -277,8 +277,7 @@ def test_rank_unstructured_grid(self): self._check_cube() def test_bad_generic_level(self): - """Test check fails in metadata if generic level coord - has wrong var_name.""" + """Test check fails if generic level coord has wrong var_name.""" depth_coord = CoordinateInfoMock("depth") depth_coord.axis = "Z" depth_coord.generic_lev_name = "olevel" @@ -344,26 +343,34 @@ def test_generic_level_invalid_alternative(self): self._check_fails_in_metadata() def test_check_bad_var_standard_name_strict_flag(self): - """Test check fails for a bad variable standard_name with - --cmor-check strict.""" + """Test check fails for a bad variable standard_name. + + With --cmor-check strict. + """ self.cube.standard_name = "wind_speed" self._check_fails_in_metadata() def test_check_bad_var_long_name_strict_flag(self): - """Test check fails for a bad variable long_name with - --cmor-check strict.""" + """Test check fails for a bad variable long_name. + + With --cmor-check strict. + """ self.cube.long_name = "Near-Surface Wind Speed" self._check_fails_in_metadata() def test_check_bad_var_units_strict_flag(self): - """Test check fails for a bad variable units with - --cmor-check strict.""" + """Test check fails for a bad variable units. + + With --cmor-check strict. + """ self.cube.units = "kg" self._check_fails_in_metadata() def test_check_bad_attributes_strict_flag(self): - """Test check fails for a bad variable attribute with - --cmor-check strict.""" + """Test check fails for a bad variable attribute. + + With --cmor-check strict. + """ self.var_info.standard_name = "surface_upward_latent_heat_flux" self.var_info.positive = "up" self.cube = self.get_cube(self.var_info) @@ -371,67 +378,78 @@ def test_check_bad_attributes_strict_flag(self): self._check_fails_in_metadata() def test_check_bad_rank_strict_flag(self): - """Test check fails for a bad variable rank with - --cmor-check strict.""" + """Test check fails for a bad variable rank with --cmor-check strict.""" lat = iris.coords.AuxCoord.from_coord(self.cube.coord("latitude")) self.cube.remove_coord("latitude") self.cube.add_aux_coord(lat, self.cube.coord_dims("longitude")) self._check_fails_in_metadata() def test_check_bad_coord_var_name_strict_flag(self): - """Test check fails for bad coord var_name with - --cmor-check strict""" + """Test check fails for bad coord var_name. + + With --cmor-check strict. + """ self.var_info.table_type = "CMIP5" self.cube.coord("longitude").var_name = "bad_name" self._check_fails_in_metadata() def test_check_missing_lon_strict_flag(self): - """Test check fails for missing longitude with --cmor-check strict""" + """Test check fails for missing longitude with --cmor-check strict.""" self.var_info.table_type = "CMIP5" self.cube.remove_coord("longitude") self._check_fails_in_metadata() def test_check_missing_lat_strict_flag(self): - """Test check fails for missing latitude with --cmor-check strict""" + """Test check fails for missing latitude with --cmor-check strict.""" self.var_info.table_type = "CMIP5" self.cube.remove_coord("latitude") self._check_fails_in_metadata() def test_check_missing_time_strict_flag(self): - """Test check fails for missing time with --cmor-check strict""" + """Test check fails for missing time with --cmor-check strict.""" self.var_info.table_type = "CMIP5" self.cube.remove_coord("time") self._check_fails_in_metadata() def test_check_missing_coord_strict_flag(self): - """Test check fails for missing coord other than lat and lon - with --cmor-check strict""" + """Test check fails for missing coord other than lat and lon. + + With --cmor-check relaxed. + """ self.var_info.coordinates.update( {"height2m": CoordinateInfoMock("height2m")} ) self._check_fails_in_metadata() def test_check_bad_var_standard_name_relaxed_flag(self): - """Test check reports warning for a bad variable standard_name with - --cmor-check relaxed.""" + """Test check reports warning for a bad variable standard_name. + + With --cmor-check relaxed. + """ self.cube.standard_name = "wind_speed" self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_var_long_name_relaxed_flag(self): - """Test check reports warning for a bad variable long_name with - --cmor-check relaxed.""" + """Test check reports warning for a bad variable long_name. + + With --cmor-check relaxed. + """ self.cube.long_name = "Near-Surface Wind Speed" self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_var_units_relaxed_flag(self): - """Test check reports warning for a bad variable units with - --cmor-check relaxed.""" + """Test check reports warning for a bad variable units. + + With --cmor-check relaxed. + """ self.cube.units = "kg" self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_attributes_relaxed_flag(self): - """Test check report warnings for a bad variable attribute with - --cmor-check relaxed.""" + """Test check report warnings for a bad variable attribute. + + With --cmor-check relaxed. + """ self.var_info.standard_name = "surface_upward_latent_heat_flux" self.var_info.positive = "up" self.cube = self.get_cube(self.var_info) @@ -439,67 +457,81 @@ def test_check_bad_attributes_relaxed_flag(self): self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_rank_relaxed_flag(self): - """Test check report warnings for a bad variable rank with - --cmor-check relaxed.""" + """Test check report warnings for a bad variable rank. + + With --cmor-check relaxed. + """ lat = iris.coords.AuxCoord.from_coord(self.cube.coord("latitude")) self.cube.remove_coord("latitude") self.cube.add_aux_coord(lat, self.cube.coord_dims("longitude")) self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_coord_standard_name_relaxed_flag(self): - """Test check reports warning for bad coord var_name with - --cmor-check relaxed""" + """Test check reports warning for bad coord var_name. + + With --cmor-check relaxed. + """ self.var_info.table_type = "CMIP5" self.cube.coord("longitude").var_name = "bad_name" self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_missing_lon_relaxed_flag(self): - """Test check fails for missing longitude with --cmor-check relaxed""" + """Test check fails for missing longitude with --cmor-check relaxed.""" self.var_info.table_type = "CMIP5" self.cube.remove_coord("longitude") self._check_fails_in_metadata(check_level=CheckLevels.RELAXED) def test_check_missing_lat_relaxed_flag(self): - """Test check fails for missing latitude with --cmor-check relaxed""" + """Test check fails for missing latitude with --cmor-check relaxed.""" self.var_info.table_type = "CMIP5" self.cube.remove_coord("latitude") self._check_fails_in_metadata(check_level=CheckLevels.RELAXED) def test_check_missing_time_relaxed_flag(self): - """Test check fails for missing latitude with --cmor-check relaxed""" + """Test check fails for missing latitude with --cmor-check relaxed.""" self.var_info.table_type = "CMIP5" self.cube.remove_coord("time") self._check_fails_in_metadata(check_level=CheckLevels.RELAXED) def test_check_missing_coord_relaxed_flag(self): - """Test check reports warning for missing coord other than lat and lon - with --cmor-check relaxed""" + """Test check reports warning for missing coord. + + For a coordinate other than lat and lon, with --cmor-check relaxed. + """ self.var_info.coordinates.update( {"height2m": CoordinateInfoMock("height2m")} ) self._check_warnings_on_metadata(check_level=CheckLevels.RELAXED) def test_check_bad_var_standard_name_none_flag(self): - """Test check reports warning for a bad variable standard_name with - --cmor-check ignore.""" + """Test check reports warning for a bad variable standard_name. + + With --cmor-check ignore. + """ self.cube.standard_name = "wind_speed" self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_bad_var_long_name_none_flag(self): - """Test check reports warning for a bad variable long_name with - --cmor-check ignore.""" + """Test check reports warning for a bad variable long_name. + + With --cmor-check ignore. + """ self.cube.long_name = "Near-Surface Wind Speed" self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_bad_var_units_none_flag(self): - """Test check reports warning for a bad variable unit with - --cmor-check ignore.""" + """Test check reports warning for a bad variable unit. + + With --cmor-check ignore. + """ self.cube.units = "kg" self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_bad_attributes_none_flag(self): - """Test check reports warning for a bad variable attribute with - --cmor-check ignore.""" + """Test check reports warning for a bad variable attribute. + + With --cmor-check ignore. + """ self.var_info.standard_name = "surface_upward_latent_heat_flux" self.var_info.positive = "up" self.cube = self.get_cube(self.var_info) @@ -507,44 +539,57 @@ def test_check_bad_attributes_none_flag(self): self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_bad_rank_none_flag(self): - """Test check reports warning for a bad variable rank with - --cmor-check ignore.""" + """Test check reports warning for a bad variable rank. + + With --cmor-check ignore. + """ lat = iris.coords.AuxCoord.from_coord(self.cube.coord("latitude")) self.cube.remove_coord("latitude") self.cube.add_aux_coord(lat, self.cube.coord_dims("longitude")) self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_bad_coord_standard_name_none_flag(self): - """Test check reports warning for bad coord var_name with - --cmor-check ignore.""" + """Test check reports warning for bad coord var_name. + + With --cmor-check ignore. + """ self.var_info.table_type = "CMIP5" self.cube.coord("longitude").var_name = "bad_name" self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_missing_lon_none_flag(self): - """Test check reports warning for missing longitude with - --cmor-check ignore""" + """Test check reports warning for missing longitude. + + With --cmor-check ignore. + """ self.var_info.table_type = "CMIP5" self.cube.remove_coord("longitude") self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_missing_lat_none_flag(self): - """Test check reports warning for missing latitude with - --cmor-check ignore""" + """Test check reports warning for missing latitude. + + With --cmor-check ignore. + """ self.var_info.table_type = "CMIP5" self.cube.remove_coord("latitude") self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_missing_time_none_flag(self): - """Test check reports warning for missing time - with --cmor-check ignore""" + """Test check reports warning for missing time. + + With --cmor-check ignore. + """ self.var_info.table_type = "CMIP5" self.cube.remove_coord("time") self._check_warnings_on_metadata(check_level=CheckLevels.IGNORE) def test_check_missing_coord_none_flag(self): - """Test check reports warning for missing coord other than lat, lon and - time with --cmor-check ignore""" + """Test check reports warning for missing coord. + + For a coordinate other than lat, lon and time with + --cmor-check ignore. + """ self.var_info.coordinates.update( {"height2m": CoordinateInfoMock("height2m")} ) @@ -783,8 +828,7 @@ def test_bad_standard_name(self): self._check_fails_in_metadata() def test_bad_out_name_region_area_type(self): - """Debug message if region/area_type AuxCoord has bad var_name at - metadata.""" + """Test debug message if region/area_type AuxCoord has bad var_name.""" region_coord = CoordinateInfoMock("basin") region_coord.standard_name = "region" self.var_info.coordinates["region"] = region_coord @@ -795,19 +839,19 @@ def test_bad_out_name_region_area_type(self): self._check_debug_messages_on_metadata() def test_bad_out_name_onedim_latitude(self): - """Warning if onedimensional lat has bad var_name at metadata""" + """Warning if onedimensional lat has bad var_name at metadata.""" self.var_info.table_type = "CMIP6" self.cube.coord("latitude").var_name = "bad_name" self._check_fails_in_metadata() def test_bad_out_name_onedim_longitude(self): - """Warning if onedimensional lon has bad var_name at metadata""" + """Warning if onedimensional lon has bad var_name at metadata.""" self.var_info.table_type = "CMIP6" self.cube.coord("longitude").var_name = "bad_name" self._check_fails_in_metadata() def test_bad_out_name_other(self): - """Warning if general coordinate has bad var_name at metadata""" + """Warning if general coordinate has bad var_name at metadata.""" self.var_info.table_type = "CMIP6" self.cube.coord("time").var_name = "bad_name" self._check_fails_in_metadata() @@ -1034,7 +1078,7 @@ def _get_unstructed_grid_cube(self, n_bounds=2): return cube def _setup_generic_level_var(self): - """Setup var_info and cube with generic alevel coordinate.""" + """Set up var_info and cube with generic alevel coordinate.""" self.var_info.coordinates.pop("depth") self.var_info.coordinates.pop("air_pressure") diff --git a/tests/unit/cmor/test_generic_fix.py b/tests/unit/cmor/test_generic_fix.py index e1ab742619..fcc317794b 100644 --- a/tests/unit/cmor/test_generic_fix.py +++ b/tests/unit/cmor/test_generic_fix.py @@ -12,7 +12,7 @@ @pytest.fixture def generic_fix(): - """Generic fix object.""" + """Create a GenericFix object.""" vardef = get_var_info("CMIP6", "CFmon", "ta") extra_facets = {"short_name": "ta", "project": "CMIP6", "dataset": "MODEL"} return GenericFix(vardef, extra_facets=extra_facets) diff --git a/tests/unit/config/test_config_object.py b/tests/unit/config/test_config_object.py index 02da498562..09fc93ed3a 100644 --- a/tests/unit/config/test_config_object.py +++ b/tests/unit/config/test_config_object.py @@ -411,7 +411,7 @@ def test_get_global_config_deprecated(mocker, tmp_path): def _setup_config_dirs(tmp_path): - """Setup test configuration directories.""" + """Set up test configuration directories.""" config1 = tmp_path / "config1" / "1.yml" config2a = tmp_path / "config2" / "2a.yml" config2b = tmp_path / "config2" / "2b.yml" diff --git a/tests/unit/experimental/test_recipe_output.py b/tests/unit/experimental/test_recipe_output.py index 7d64935596..ca8f333e21 100644 --- a/tests/unit/experimental/test_recipe_output.py +++ b/tests/unit/experimental/test_recipe_output.py @@ -32,7 +32,6 @@ def test_diagnostic_output_repr(mocker): def test_recipe_output_add_to_filters(): """Coverage test for `RecipeOutput._add_to_filters`.""" - filters = {} valid_attr = recipe_output.RecipeOutput.FILTER_ATTRS[0] @@ -57,7 +56,6 @@ def test_recipe_output_add_to_filters(): def test_recipe_output_add_to_filters_no_attributes(): """Test `RecipeOutput._add_to_filters` with no attributes.""" - filters = {} recipe_output.RecipeOutput._add_to_filters(filters, {}) assert len(filters) == 0 @@ -65,7 +63,6 @@ def test_recipe_output_add_to_filters_no_attributes(): def test_recipe_output_add_to_filters_no_valid_attributes(): """Test `RecipeOutput._add_to_filters` with no valid attributes.""" - filters = {} invalid = "invalid_attribute" recipe_output.RecipeOutput._add_to_filters(filters, {invalid: "value"}) @@ -77,7 +74,6 @@ def test_recipe_output_add_to_filters_no_valid_attributes(): def test_recipe_output_sort_filters(): """Coverage test for `RecipeOutput._sort_filters`.""" - filters = {} valid_attr = recipe_output.RecipeOutput.FILTER_ATTRS[0] unsorted_attributes = ["1", "2", "4", "value", "3"] diff --git a/tests/unit/local/test_select_files.py b/tests/unit/local/test_select_files.py index 674912060c..570a393c8c 100644 --- a/tests/unit/local/test_select_files.py +++ b/tests/unit/local/test_select_files.py @@ -113,8 +113,7 @@ def test_select_files_sub_daily_resolution(): def test_select_files_time_period(): - """Test file selection works with time range given as duration periods of - various resolution.""" + """Test file selection works with `timerange` given as a period.""" filename_date = "pr_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i1p1f1_gr_" filename_datetime = ( "psl_6hrPlev_EC-Earth3_dcppA-hindcast_s1960-r1i1p1f1_gr_" @@ -153,8 +152,7 @@ def test_select_files_time_period(): def test_select_files_varying_format(): - """Test file selection works with time range of various time resolutions - and formats.""" + """Test file selection works with various `timerange`s.""" filename = "psl_6hrPlev_EC-Earth3_dcppA-hindcast_s1960-r1i1p1f1_gr_" files = [ diff --git a/tests/unit/local/test_time.py b/tests/unit/local/test_time.py index 31afbedee3..b6a29f450c 100644 --- a/tests/unit/local/test_time.py +++ b/tests/unit/local/test_time.py @@ -212,8 +212,10 @@ def test_fails_if_no_date_present(): def test_get_timerange_from_years(): - """Test a `timerange` tag with value `start_year/end_year` can be built - from tags `start_year` and `end_year`.""" + """Test a `timerange` with value `start_year/end_year` can be built. + + From `start_year` and `end_year`. + """ variable = {"start_year": 2000, "end_year": 2002} _replace_years_with_timerange(variable) @@ -224,8 +226,10 @@ def test_get_timerange_from_years(): def test_get_timerange_from_start_year(): - """Test a `timerange` tag with value `start_year/start_year` can be built - from tag `start_year` when an `end_year` is not given.""" + """Test a `timerange` with value `start_year/start_year` can be built. + + From `start_year` when an `end_year` is not given. + """ variable = {"start_year": 2000} _replace_years_with_timerange(variable) @@ -235,8 +239,10 @@ def test_get_timerange_from_start_year(): def test_get_timerange_from_end_year(): - """Test a `timerange` tag with value `end_year/end_year` can be built from - tag `end_year` when a `start_year` is not given.""" + """Test a `timerange` with value `end_year/end_year` can be built. + + From `end_year` when a `start_year` is not given. + """ variable = {"end_year": 2002} _replace_years_with_timerange(variable) diff --git a/tests/unit/preprocessor/_area/test_area.py b/tests/unit/preprocessor/_area/test_area.py index 99eaf3f150..ec741b629a 100644 --- a/tests/unit/preprocessor/_area/test_area.py +++ b/tests/unit/preprocessor/_area/test_area.py @@ -233,8 +233,10 @@ def test_extract_region(self): self.assert_array_equal(result.data, expected) def test_extract_region_mean(self): - """Test for extracting a region and performing the area mean of a 2D - field.""" + """Test for extracting a region and performing the area mean. + + Use a 2D field. + """ cube = guess_bounds(self.grid, ["longitude", "latitude"]) grid_areas = iris.analysis.cartography.area_weights(cube) measure = iris.coords.CellMeasure( diff --git a/tests/unit/preprocessor/_compare_with_refs/test_compare_with_refs.py b/tests/unit/preprocessor/_compare_with_refs/test_compare_with_refs.py index 631eac916d..003cba71dc 100644 --- a/tests/unit/preprocessor/_compare_with_refs/test_compare_with_refs.py +++ b/tests/unit/preprocessor/_compare_with_refs/test_compare_with_refs.py @@ -78,7 +78,7 @@ def regular_cubes(): @pytest.fixture def ref_cubes(): - """Reference cubes.""" + """Create reference cubes.""" cube_data = np.full((2, 2, 2), 2.0) cube_data[1, 1, 1] = 4.0 cube = get_3d_cube( diff --git a/tests/unit/preprocessor/_mask/test_mask.py b/tests/unit/preprocessor/_mask/test_mask.py index 9eca669d38..59b383c59a 100644 --- a/tests/unit/preprocessor/_mask/test_mask.py +++ b/tests/unit/preprocessor/_mask/test_mask.py @@ -115,7 +115,7 @@ def test_get_fx_mask(self): self.assert_array_equal(expected, computed) def test_mask_glaciated(self): - """Test to mask glaciated (NE mask)""" + """Test to mask glaciated (NE mask).""" result = mask_glaciated(self.arr, mask_out="glaciated") expected = np.ma.masked_array( self.data2, mask=np.array([[True, True], [False, False]]) diff --git a/tests/unit/preprocessor/_regrid/__init__.py b/tests/unit/preprocessor/_regrid/__init__.py index a6869d33cf..a59bbc5662 100644 --- a/tests/unit/preprocessor/_regrid/__init__.py +++ b/tests/unit/preprocessor/_regrid/__init__.py @@ -1,7 +1,4 @@ -""" -Unit tests for the :mod:`esmvalcore.preprocessor.regrid` module. - -""" +"""Unit tests for the :mod:`esmvalcore.preprocessor.regrid` module.""" from typing import Literal @@ -12,10 +9,7 @@ def _make_vcoord(data, dtype=None): - """ - Create a synthetic test vertical coordinate. - - """ + """Create a synthetic test vertical coordinate.""" if dtype is None: dtype = np.dtype("int8") @@ -49,10 +43,7 @@ def _make_cube( dtype=None, grid: Literal["regular", "rotated", "mesh"] = "regular", ): - """ - Create a 3d synthetic test cube. - - """ + """Create a 3d synthetic test cube.""" if dtype is None: dtype = np.dtype("int8") diff --git a/tests/unit/preprocessor/_regrid/test__create_cube.py b/tests/unit/preprocessor/_regrid/test__create_cube.py index 0308df61ae..4ec1e97b60 100644 --- a/tests/unit/preprocessor/_regrid/test__create_cube.py +++ b/tests/unit/preprocessor/_regrid/test__create_cube.py @@ -1,8 +1,4 @@ -""" -Unit tests for the :func:`esmvalcore.preprocessor.regrid._create_cube` -function. - -""" +"""Unit tests for :func:`esmvalcore.preprocessor.regrid._create_cube`.""" import unittest diff --git a/tests/unit/preprocessor/_regrid/test__stock_cube.py b/tests/unit/preprocessor/_regrid/test__stock_cube.py index 220c7195e8..df2c50c330 100644 --- a/tests/unit/preprocessor/_regrid/test__stock_cube.py +++ b/tests/unit/preprocessor/_regrid/test__stock_cube.py @@ -1,8 +1,4 @@ -""" -Unit tests for the :func:`esmvalcore.preprocessor.regrid._stock_cube` -function. - -""" +"""Unit tests for :func:`esmvalcore.preprocessor.regrid._stock_cube`.""" import unittest from unittest import mock diff --git a/tests/unit/preprocessor/_regrid/test_extract_point.py b/tests/unit/preprocessor/_regrid/test_extract_point.py index f131463f3e..b266835b8f 100644 --- a/tests/unit/preprocessor/_regrid/test_extract_point.py +++ b/tests/unit/preprocessor/_regrid/test_extract_point.py @@ -1,8 +1,4 @@ -""" -Unit tests for the -:func:`esmvalcore.preprocessor.regrid.extract_point` function. - -""" +"""Unit tests for :func:`esmvalcore.preprocessor.extract_point`.""" import unittest from unittest import mock diff --git a/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py b/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py index 15055a7658..6ff0df6fec 100644 --- a/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py +++ b/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py @@ -1,4 +1,4 @@ -"""Unit test for :func:`esmvalcore.preprocessor._regrid`""" +"""Unit test for :func:`esmvalcore.preprocessor._regrid`.""" from decimal import Decimal diff --git a/tests/unit/preprocessor/_regrid/test_regrid.py b/tests/unit/preprocessor/_regrid/test_regrid.py index 69589a7a8c..3521a35053 100644 --- a/tests/unit/preprocessor/_regrid/test_regrid.py +++ b/tests/unit/preprocessor/_regrid/test_regrid.py @@ -1,5 +1,4 @@ -"""Unit tests for the :func:`esmvalcore.preprocessor.regrid.regrid` -function.""" +"""Unit tests for :func:`esmvalcore.preprocessor.regrid`.""" import dask import dask.array as da @@ -25,7 +24,7 @@ def clear_regridder_cache(monkeypatch): def _make_coord(start: float, stop: float, step: int, *, name: str): - """Helper function for creating a coord.""" + """Create a latitude or longitude coordinate with bounds.""" coord = iris.coords.DimCoord( np.linspace(start, stop, step), standard_name=name, @@ -36,7 +35,7 @@ def _make_coord(start: float, stop: float, step: int, *, name: str): def _make_cube(*, lat: tuple, lon: tuple): - """Helper function for creating a cube.""" + """Create a cube with a latitude and longitude dimension.""" lat_coord = _make_coord(*lat, name="latitude") lon_coord = _make_coord(*lon, name="longitude") @@ -83,7 +82,7 @@ def cube_30x30(): @pytest.mark.parametrize("cache_weights", [True, False]) @pytest.mark.parametrize("scheme", SCHEMES) def test_builtin_regridding(scheme, cache_weights, cube_10x10, cube_30x30): - """Test `regrid.`""" + """Test `regrid.`.""" _cached_regridders = esmvalcore.preprocessor._regrid._CACHED_REGRIDDERS assert _cached_regridders == {} @@ -104,7 +103,7 @@ def test_builtin_regridding(scheme, cache_weights, cube_10x10, cube_30x30): @pytest.mark.parametrize("scheme", SCHEMES) def test_invalid_target_grid(scheme, cube_10x10, mocker): - """Test `regrid.`""" + """Test `regrid.`.""" target_grid = mocker.sentinel.target_grid msg = "Expecting a cube" with pytest.raises(ValueError, match=msg): @@ -112,7 +111,7 @@ def test_invalid_target_grid(scheme, cube_10x10, mocker): def test_invalid_scheme(cube_10x10, cube_30x30): - """Test `regrid.`""" + """Test `regrid.`.""" msg = ( "Regridding scheme 'wibble' not available for regular data, " "expected one of: area_weighted, linear, nearest" @@ -122,14 +121,14 @@ def test_invalid_scheme(cube_10x10, cube_30x30): def test_regrid_generic_missing_reference(cube_10x10, cube_30x30): - """Test `regrid.`""" + """Test `regrid.`.""" msg = "No reference specified for generic regridding." with pytest.raises(ValueError, match=msg): regrid(cube_10x10, cube_30x30, {}) def test_regrid_generic_invalid_reference(cube_10x10, cube_30x30): - """Test `regrid.`""" + """Test `regrid.`.""" msg = "Could not import specified generic regridding module." with pytest.raises(ValueError, match=msg): regrid(cube_10x10, cube_30x30, {"reference": "this.does:not.exist"}) @@ -137,7 +136,7 @@ def test_regrid_generic_invalid_reference(cube_10x10, cube_30x30): @pytest.mark.parametrize("cache_weights", [True, False]) def test_regrid_generic_regridding(cache_weights, cube_10x10, cube_30x30): - """Test `regrid.`""" + """Test `regrid.`.""" _cached_regridders = esmvalcore.preprocessor._regrid._CACHED_REGRIDDERS assert _cached_regridders == {} @@ -401,7 +400,7 @@ def test_no_rechunk_non_lazy(): @pytest.mark.parametrize("scheme", SCHEMES) def test_regridding_weights_use_cache(scheme, cube_10x10, cube_30x30, mocker): - """Test `regrid.`""" + """Test `regrid.`.""" _cached_regridders = esmvalcore.preprocessor._regrid._CACHED_REGRIDDERS assert _cached_regridders == {} @@ -429,7 +428,7 @@ def test_regridding_weights_use_cache(scheme, cube_10x10, cube_30x30, mocker): def test_clear_regridding_weights_cache(): - """Test `regrid.cache_clear().`""" + """Test `regrid.cache_clear().`.""" _cached_regridders = esmvalcore.preprocessor._regrid._CACHED_REGRIDDERS _cached_regridders["test"] = "test" diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index d3518e348d..54db6ed2b8 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -351,8 +351,10 @@ def test_clip_timerange_daily(self): assert sliced_backward.coord("time").cell(0).point.day == 1 def test_clip_timerange_duration_seconds(self): - """Test timerange with duration periods with resolution up to - seconds.""" + """Test clip_timerange. + + Test with duration periods with resolution up to seconds. + """ data = np.arange(8) times = np.arange(0, 48, 6) calendars = [ @@ -1305,7 +1307,7 @@ def test_sum(self): @pytest.fixture def cube_1d_time(): - """Simple 1D cube with time coordinate of length one.""" + """Create a 1D cube with a time coordinate of length one.""" units = Unit("days since 2000-01-01", calendar="standard") time_coord = iris.coords.DimCoord( units.date2num(datetime(2024, 1, 26, 14, 57, 28)), diff --git a/tests/unit/preprocessor/_volume/test_volume.py b/tests/unit/preprocessor/_volume/test_volume.py index 682d42fd45..1dec034b4e 100644 --- a/tests/unit/preprocessor/_volume/test_volume.py +++ b/tests/unit/preprocessor/_volume/test_volume.py @@ -441,8 +441,7 @@ def test_extract_volume_error(self): ) def test_extract_volume_mean(self): - """Test to extract the top two layers and compute the weighted average - of a cube.""" + """Test extracting the top layers and computing the weighted mean.""" grid_volume = calculate_volume(self.grid_4d) assert isinstance(grid_volume, np.ndarray) measure = iris.coords.CellMeasure( @@ -479,10 +478,10 @@ def test_volume_statistics(self): self.assertFalse(result.cell_measures("ocean_volume")) def test_volume_nolevbounds(self): - """Test to take the volume weighted average of a cube with no bounds - in the z axis. - """ + """Test to take the volume weighted average of a cube. + Test a cube with no bounds in the z axis. + """ self.assertFalse(self.grid_4d_znobounds.coord(axis="z").has_bounds()) result = volume_statistics(self.grid_4d_znobounds, "mean") @@ -493,7 +492,7 @@ def test_volume_nolevbounds(self): self.assertFalse(result.cell_measures("ocean_volume")) def test_calculate_volume_lazy(self): - """Test that calculate_volume returns a lazy volume + """Test that calculate_volume returns a lazy volume. The volume chunks should match those of the input cube for computational efficiency. @@ -553,16 +552,22 @@ def test_volume_statistics_long(self): self.assertEqual(result.units, "kg m-3") def test_volume_statistics_masked_level(self): - """Test to take the volume weighted average of a (2,3,2,2) cube where - the last depth level is fully masked.""" + """Test to take the volume weighted average. + + This is a test for a (2,3,2,2) cube where the last depth level is fully + masked. + """ self.grid_4d.data[:, -1, :, :] = np.ma.masked_all((2, 2, 2)) result = volume_statistics(self.grid_4d, "mean") expected = np.ma.array([1.0, 1.0], mask=False) self.assert_array_equal(result.data, expected) def test_volume_statistics_masked_timestep(self): - """Test to take the volume weighted average of a (2,3,2,2) cube where - the first timestep is fully masked.""" + """Test taking the volume weighted average. + + This is a test for a (2,3,2,2) cube where the first timestep is fully + masked. + """ self.grid_4d.data[0, :, :, :] = np.ma.masked_all((3, 2, 2)) result = volume_statistics(self.grid_4d, "mean") expected = np.ma.array([1.0, 1], mask=[True, False]) @@ -653,8 +658,7 @@ def test_volume_statistics_2d_lat_cellarea(self): self.assertEqual(result.units, "kg m-3") def test_volume_statistics_invalid_bounds(self): - """Test z-axis bounds is not 2 in last dimension""" - + """Test z-axis bounds is not 2 in last dimension.""" with self.assertRaises(ValueError) as err: volume_statistics(self.grid_invalid_z_bounds, "mean") assert ( @@ -663,8 +667,7 @@ def test_volume_statistics_invalid_bounds(self): ) in str(err.exception) def test_volume_statistics_invalid_units(self): - """Test z-axis units cannot be converted to m""" - + """Test z-axis units cannot be converted to m.""" with self.assertRaises(ValueError) as err: volume_statistics(self.grid_4d_sigma_space, "mean") assert ( diff --git a/tests/unit/preprocessor/test_shared.py b/tests/unit/preprocessor/test_shared.py index b0a990c45d..90dd04135b 100644 --- a/tests/unit/preprocessor/test_shared.py +++ b/tests/unit/preprocessor/test_shared.py @@ -181,7 +181,7 @@ def test_aggregator_accept_weights(aggregator, result): @preserve_float_dtype def _dummy_func(obj, arg, kwarg=2.0): - """Dummy function to test `preserve_float_dtype`.""" + """Compute something to test `preserve_float_dtype`.""" obj = obj * arg * kwarg if isinstance(obj, Cube): obj.data = obj.core_data().astype(np.float64) diff --git a/tests/unit/recipe/test_recipe.py b/tests/unit/recipe/test_recipe.py index 4dd142bc43..9934f02d3b 100644 --- a/tests/unit/recipe/test_recipe.py +++ b/tests/unit/recipe/test_recipe.py @@ -20,7 +20,7 @@ class MockRecipe(_recipe.Recipe): """Mocked Recipe class with simple constructor.""" def __init__(self, cfg, diagnostics): - """Simple constructor used for testing.""" + """Create a mock recipe for testing.""" self.session = cfg self.diagnostics = diagnostics diff --git a/tests/unit/test_naming.py b/tests/unit/test_naming.py index da453eee4f..faa97a431a 100644 --- a/tests/unit/test_naming.py +++ b/tests/unit/test_naming.py @@ -1,20 +1,20 @@ -"""Checks to ensure that files follow the naming convention""" +"""Checks to ensure that files follow the naming convention.""" import os import unittest class TestNaming(unittest.TestCase): - """Test naming of files and folders""" + """Test naming of files and folders.""" def setUp(self): - """Prepare tests""" + """Prepare tests.""" folder = os.path.join(__file__, "..", "..", "..") self.esmvaltool_folder = os.path.abspath(folder) def test_windows_reserved_names(self): """ - Check that no file or folder uses a Windows reserved name + Check that no file or folder uses a Windows reserved name. Files can not differ from a reserved name by the extension only """ @@ -61,7 +61,7 @@ def test_windows_reserved_names(self): def test_avoid_casing_collisions(self): """ - Check that there are no names differing only in the capitalization + Check that there are no names differing only in the capitalization. This includes folders differing from files """ @@ -76,7 +76,7 @@ def test_avoid_casing_collisions(self): def test_no_namelist(self): """ - Check that there are no namelist references in file and folder names + Check that there are no namelist references in file and folder names. This will help us to avoid bad merges with stale branches """ From 56a9756c5e7c37e491659c2115b6f31071fadf1b Mon Sep 17 00:00:00 2001 From: Bouwe Andela Date: Wed, 16 Oct 2024 16:35:39 +0200 Subject: [PATCH 18/19] Use pyproject.toml instead of setup.py/setup.cfg (#2540) Co-authored-by: Valeriu Predoi --- .circleci/install_triggers | 2 - doc/contributing.rst | 4 +- environment.yml | 2 +- esmvalcore/_main.py | 7 +- esmvalcore/_recipe/check.py | 2 +- esmvalcore/_recipe/from_datasets.py | 2 +- esmvalcore/_task.py | 14 +- esmvalcore/preprocessor/_multimodel.py | 4 +- esmvalcore/preprocessor/_time.py | 5 +- esmvalcore/preprocessor/_volume.py | 5 +- pyproject.toml | 210 +++++++++++++-- setup.cfg | 34 +-- setup.py | 239 ------------------ .../cmor/_fixes/cmip6/test_cesm2.py | 5 - .../cmor/_fixes/cmip6/test_cesm2_waccm.py | 4 - .../cmor/_fixes/cmip6/test_gfdl_cm4.py | 1 - .../integration/esgf/test_search_download.py | 4 +- .../multimodel_statistics/test_multimodel.py | 2 +- tests/unit/esgf/test_search.py | 2 +- .../_multimodel/test_multimodel.py | 3 +- tests/unit/preprocessor/_regrid/__init__.py | 2 + .../_regrid/test_extract_regional_grid.py | 4 +- tests/unit/preprocessor/_time/test_time.py | 4 +- tests/unit/task/test_diagnostic_task.py | 2 +- 24 files changed, 231 insertions(+), 332 deletions(-) delete mode 100755 setup.py diff --git a/.circleci/install_triggers b/.circleci/install_triggers index 6945bf8979..c1c5a829d4 100644 --- a/.circleci/install_triggers +++ b/.circleci/install_triggers @@ -1,5 +1,3 @@ ^\.circleci/ ^environment\.yml$ ^pyproject.toml$ -^setup\.py$ -^setup\.cfg$ diff --git a/doc/contributing.rst b/doc/contributing.rst index 81a8ffb012..de7d319cc3 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -443,7 +443,7 @@ previous command. To only run tests from a single file, run the command pytest tests/unit/test_some_file.py If you would like to avoid loading the default pytest configuration from -`setup.cfg `_ +`pyproject.toml `_ because this can be a bit slow for running just a few tests, use .. code-block:: bash @@ -660,7 +660,7 @@ the following files: - ``environment.yml`` contains all the development dependencies; these are all from `conda-forge `_ -- ``setup.py`` +- ``pyproject.toml`` contains all Python dependencies, regardless of their installation source Note that packages may have a different name on diff --git a/environment.yml b/environment.yml index 5cf256c22b..cacd62710b 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - dask-jobqueue - distributed - esgf-pyclient >=0.3.1 - - esmpy !=8.1.0 + - esmpy - filelock - fiona - fire diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 451f228ae8..30e2668bdb 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -33,14 +33,10 @@ import logging import os import sys +from importlib.metadata import entry_points from pathlib import Path from typing import Optional -if (sys.version_info.major, sys.version_info.minor) < (3, 10): - from importlib_metadata import entry_points -else: - from importlib.metadata import entry_points # type: ignore - import fire # set up logging @@ -583,6 +579,7 @@ def _get_config_info(cli_config_dir): zip( config_dirs, _get_all_config_sources(cli_config_dir), + strict=False, ) ) diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index 9a26ef4561..5001a5d371 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -171,7 +171,7 @@ def _group_years(years): ends.append(year) ranges = [] - for start, end in zip(starts, ends): + for start, end in zip(starts, ends, strict=False): ranges.append(f"{start}" if start == end else f"{start}-{end}") return ", ".join(ranges) diff --git a/esmvalcore/_recipe/from_datasets.py b/esmvalcore/_recipe/from_datasets.py index 60384c8026..f68bd9e096 100644 --- a/esmvalcore/_recipe/from_datasets.py +++ b/esmvalcore/_recipe/from_datasets.py @@ -219,7 +219,7 @@ def _group_ensemble_names(ensemble_names: Iterable[str]) -> list[str]: groups = [] for ensemble_range in ensemble_ranges: txt = "" - for name, value in zip("ripf", ensemble_range): + for name, value in zip("ripf", ensemble_range, strict=False): txt += name if value[0] == value[1]: txt += f"{value[0]}" diff --git a/esmvalcore/_task.py b/esmvalcore/_task.py index dc15718169..2a908583b6 100644 --- a/esmvalcore/_task.py +++ b/esmvalcore/_task.py @@ -109,9 +109,12 @@ def _get_resource_usage(process, start_time, children=True): continue # Create and yield log entry - entries = [sum(entry) for entry in zip(*cache.values())] + entries = [sum(entry) for entry in zip(*cache.values(), strict=False)] entries.insert(0, time.time() - start_time) - entries = [round(entry, p) for entry, p in zip(entries, precision)] + entries = [ + round(entry, p) + for entry, p in zip(entries, precision, strict=False) + ] entries.insert(0, datetime.datetime.utcnow()) max_memory = max(max_memory, entries[4]) yield (fmt.format(*entries), max_memory) @@ -609,9 +612,10 @@ def _run(self, input_files): returncode = None - with resource_usage_logger(process.pid, self.resource_log), open( - self.log, "ab" - ) as log: + with ( + resource_usage_logger(process.pid, self.resource_log), + open(self.log, "ab") as log, + ): last_line = [""] while returncode is None: returncode = process.poll() diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index b790b45117..56cea1e936 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -82,13 +82,13 @@ def _unify_time_coordinates(cubes): # monthly data dates = [ datetime(year, month, 15, 0, 0, 0) - for year, month in zip(years, months) + for year, month in zip(years, months, strict=False) ] elif 0 not in np.diff(days): # daily data dates = [ datetime(year, month, day, 0, 0, 0) - for year, month, day in zip(years, months, days) + for year, month, day in zip(years, months, days, strict=False) ] if coord.units != t_unit: logger.warning( diff --git a/esmvalcore/preprocessor/_time.py b/esmvalcore/preprocessor/_time.py index 062cdf0ba2..a0e8c9e54d 100644 --- a/esmvalcore/preprocessor/_time.py +++ b/esmvalcore/preprocessor/_time.py @@ -672,7 +672,10 @@ def spans_full_season(cube: Cube) -> list[bool]: seasons = cube.coord("clim_season").points tar_days = [(len(sea) * 29, len(sea) * 31) for sea in seasons] - return [dt[0] <= dn <= dt[1] for dn, dt in zip(num_days, tar_days)] + return [ + dt[0] <= dn <= dt[1] + for dn, dt in zip(num_days, tar_days, strict=False) + ] full_seasons = spans_full_season(result) result = result[full_seasons] diff --git a/esmvalcore/preprocessor/_volume.py b/esmvalcore/preprocessor/_volume.py index 169dcb3bba..8d56d7b51a 100644 --- a/esmvalcore/preprocessor/_volume.py +++ b/esmvalcore/preprocessor/_volume.py @@ -510,7 +510,10 @@ def extract_transect( ) for dim_name, dim_cut, coord in zip( - ["latitude", "longitude"], [latitude, longitude], [lats, lons] + ["latitude", "longitude"], + [latitude, longitude], + [lats, lons], + strict=False, ): # #### # Look for the first coordinate. diff --git a/pyproject.toml b/pyproject.toml index 6234370689..797e9bc81b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,34 +1,174 @@ [build-system] -requires = ["setuptools >= 40.6.0", "wheel", "setuptools_scm>=6.2"] +requires = [ + "setuptools >= 40.6.0", + "setuptools_scm>=6.2", +] build-backend = "setuptools.build_meta" +[project] +authors = [ + {name = "ESMValTool Development Team", email = "esmvaltool-dev@listserv.dfn.de"} +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Atmospheric Science", + "Topic :: Scientific/Engineering :: GIS", + "Topic :: Scientific/Engineering :: Hydrology", + "Topic :: Scientific/Engineering :: Physics", +] +dynamic = [ + "readme", + "version", +] +dependencies = [ + "cartopy", + "cf-units", + "dask[array,distributed]!=2024.8.0", # ESMValCore/issues/2503 + "dask-jobqueue", + "esgf-pyclient>=0.3.1", + "esmf-regrid>=0.11.0", + "esmpy", # not on PyPI + "filelock", + "fiona", + "fire", + "geopy", + "humanfriendly", + "iris-grib>=0.20.0", # github.com/ESMValGroup/ESMValCore/issues/2535 + "isodate>=0.7.0", + "jinja2", + "nc-time-axis", # needed by iris.plot + "nested-lookup", + "netCDF4", + "numpy!=1.24.3,<2.0.0", # avoid pulling 2.0.0rc1 + "packaging", + "pandas", + "pillow", + "prov", + "psutil", + "py-cordex", + "pybtex", + "pyyaml", + "requests", + "scipy>=1.6", + "scitools-iris>=3.10.0", + "shapely>=2.0.0", + "stratify>=0.3", + "yamale", +] +description = "A community tool for pre-processing data from Earth system models in CMIP and running analysis scripts" +license = {text = "Apache License, Version 2.0"} +name = "ESMValCore" +requires-python = ">=3.10" + +[project.optional-dependencies] +test = [ + "pytest>6.0.0", + "pytest-cov>=2.10.1", + "pytest-env", + "pytest-html!=2.1.0", + "pytest-metadata>=1.5.1", + "pytest-mock", + "pytest-xdist", + "ESMValTool_sample_data==0.0.3", +] +doc = [ + "autodocsumm>=0.2.2", + "ipython", + "nbsphinx", + "sphinx>=6.1.3", + "pydata_sphinx_theme", +] +develop = [ + "esmvalcore[test,doc]", + "pre-commit", + "pylint", + "pydocstyle", + "vprof", +] + +[project.scripts] +esmvaltool = "esmvalcore._main:run" + +[project.urls] +Code = "https://github.com/ESMValGroup/ESMValCore" +Community = "https://github.com/ESMValGroup/Community" +Documentation = "https://docs.esmvaltool.org" +Homepage = "https://esmvaltool.org" +Issues = "https://github.com/ESMValGroup/ESMValCore/issues" + +[tool.setuptools] +include-package-data = true +license-files = ["LICENSE"] +packages = ["esmvalcore"] +zip-safe = false + +[tool.setuptools.dynamic] +readme = {file = "README.md", content-type = "text/markdown"} + [tool.setuptools_scm] version_scheme = "release-branch-semver" -[tool.codespell] -skip = "*.ipynb,esmvalcore/config/extra_facets/ipslcm-mappings.yml" -ignore-words-list = "vas,hist,oce" +# Configure tests -[tool.pylint.main] -jobs = 1 # Running more than one job in parallel crashes prospector. -ignore-paths = [ - "doc/conf.py", # Sphinx configuration file +[tool.pytest.ini_options] +addopts = [ + "-ra", + "--strict-config", + "--strict-markers", + "--doctest-modules", + "--ignore=esmvalcore/cmor/tables/", + "--cov-report=xml:test-reports/coverage.xml", + "--cov-report=html:test-reports/coverage_html", + "--html=test-reports/report.html", ] -[tool.pylint.basic] -good-names = [ - "_", # Used by convention for unused variables - "i", "j", "k", # Used by convention for indices - "logger", # Our preferred name for the logger +log_cli_level = "INFO" +env = {MPLBACKEND = "Agg"} +log_level = "WARNING" +minversion = "6" +markers = [ + "installation: Test requires installation of dependencies", + "use_sample_data: Run functional tests using real data", ] -[tool.pylint.format] -max-line-length = 79 -[tool.pylint."messages control"] -disable = [ - "import-error", # Needed because Codacy does not install dependencies - "file-ignored", # Disable messages about disabling checks - "line-too-long", # Disable line-too-long as this is taken care of by the formatter. - "locally-disabled", # Disable messages about disabling checks +testpaths = ["tests"] +xfail_strict = true + +[tool.coverage.run] +parallel = true +source = ["esmvalcore"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", ] + +# Configure type checks + +[tool.mypy] +# See https://mypy.readthedocs.io/en/stable/config_file.html +ignore_missing_imports = true +enable_error_code = [ + "truthy-bool", +] + +# Configure linters + +[tool.codespell] +skip = "*.ipynb,esmvalcore/config/extra_facets/ipslcm-mappings.yml" +ignore-words-list = "vas,hist,oce" + [tool.ruff] line-length = 79 [tool.ruff.lint] @@ -59,3 +199,31 @@ ignore = [ known-first-party = ["esmvalcore"] [tool.ruff.lint.pydocstyle] convention = "numpy" + +# Configure linters that are run by Prospector +# TODO: remove once we have enabled all ruff rules for the tools provided by +# Prospector, see https://github.com/ESMValGroup/ESMValCore/issues/2528. + +[tool.pylint.main] +jobs = 1 # Running more than one job in parallel crashes prospector. +ignore-paths = [ + "doc/conf.py", # Sphinx configuration file +] +[tool.pylint.basic] +good-names = [ + "_", # Used by convention for unused variables + "i", "j", "k", # Used by convention for indices + "logger", # Our preferred name for the logger +] +[tool.pylint.format] +max-line-length = 79 +[tool.pylint."messages control"] +disable = [ + "import-error", # Needed because Codacy does not install dependencies + "file-ignored", # Disable messages about disabling checks + "line-too-long", # Disable line-too-long as this is taken care of by the formatter. + "locally-disabled", # Disable messages about disabling checks +] + +[tool.pydocstyle] +convention = "numpy" diff --git a/setup.cfg b/setup.cfg index 995fd69c9e..0c5e89a3e0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,25 +1,6 @@ -[tool:pytest] -addopts = - --doctest-modules - --ignore=esmvalcore/cmor/tables/ - --cov-report=xml:test-reports/coverage.xml - --cov-report=html:test-reports/coverage_html - --html=test-reports/report.html -env = - MPLBACKEND = Agg -log_level = WARNING -markers = - installation: Test requires installation of dependencies - use_sample_data: Run functional tests using real data - -[coverage:run] -parallel = true -source = esmvalcore -[coverage:report] -exclude_lines = - pragma: no cover - if __name__ == .__main__.: - if TYPE_CHECKING: +# Configure linters that are run by Prospector +# TODO: remove once we have enabled all ruff rules for the tools provided by +# Prospector, see https://github.com/ESMValGroup/ESMValCore/issues/2528. [pycodestyle] # ignore rules that conflict with ruff formatter @@ -27,12 +8,3 @@ exclude_lines = # E501: https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules # W503: https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes ignore = E203,E501,W503 - -[pydocstyle] -convention = numpy - -[mypy] -# see mypy.readthedocs.io/en/stable/command_line.html -python_version = 3.12 -ignore_missing_imports = True -files = esmvalcore, tests diff --git a/setup.py b/setup.py deleted file mode 100755 index 83c3634428..0000000000 --- a/setup.py +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env python -"""ESMValTool installation script.""" -# This script only installs dependencies available on PyPI -# -# Dependencies that need to be installed some other way (e.g. conda): -# - ncl -# - iris -# - python-stratify - -import json -import os -import re -import sys -from pathlib import Path - -from setuptools import Command, setup - -PACKAGES = [ - "esmvalcore", -] - -REQUIREMENTS = { - # Installation script (this file) dependencies - "setup": [ - "setuptools_scm", - ], - # Installation dependencies - # Use with pip install . to install from source - "install": [ - "cartopy", - "cf-units", - "dask[array,distributed]!=2024.8.0", # ESMValCore/issues/2503 - "dask-jobqueue", - "esgf-pyclient>=0.3.1", - "esmf-regrid>=0.11.0", - "esmpy!=8.1.0", # not on PyPI - "filelock", - "fiona", - "fire", - "geopy", - "humanfriendly", - "iris-grib>=0.20.0", # github.com/ESMValGroup/ESMValCore/issues/2535 - "isodate>=0.7.0", # incompatible with very old 0.6.1 - "jinja2", - "nc-time-axis", # needed by iris.plot - "nested-lookup", - "netCDF4", - "numpy!=1.24.3,<2.0.0", # avoid pulling 2.0.0rc1 - "packaging", - "pandas", - "pillow", - "prov", - "psutil", - "py-cordex", - "pybtex", - "pyyaml", - "requests", - "scipy>=1.6", - "scitools-iris>=3.10.0", - "shapely>=2.0.0", - "stratify>=0.3", - "yamale", - ], - # Test dependencies - "test": [ - "pytest>=3.9,!=6.0.0rc1,!=6.0.0", - "pytest-cov>=2.10.1", - "pytest-env", - "pytest-html!=2.1.0", - "pytest-metadata>=1.5.1", - "pytest-mock", - "pytest-xdist", - "ESMValTool_sample_data==0.0.3", - ], - # Documentation dependencies - "doc": [ - "autodocsumm>=0.2.2", - "ipython", - "nbsphinx", - "sphinx>=6.1.3", - "pydata_sphinx_theme", - ], - # Development dependencies - # Use pip install -e .[develop] to install in development mode - "develop": [ - "pre-commit", - "pylint", - "pydocstyle", - "vprof", - ], -} - - -def discover_python_files(paths, ignore): - """Discover Python files.""" - - def _ignore(path): - """Return True if `path` should be ignored, False otherwise.""" - return any(re.match(pattern, path) for pattern in ignore) - - for path in sorted(set(paths)): - for root, _, files in os.walk(path): - if _ignore(path): - continue - for filename in files: - filename = os.path.join(root, filename) - if filename.lower().endswith(".py") and not _ignore(filename): - yield filename - - -class CustomCommand(Command): - """Custom Command class.""" - - def install_deps_temp(self): - """Try to temporarily install packages needed to run the command.""" - if self.distribution.install_requires: - self.distribution.fetch_build_eggs( - self.distribution.install_requires - ) - if self.distribution.tests_require: - self.distribution.fetch_build_eggs(self.distribution.tests_require) - - -class RunLinter(CustomCommand): - """Class to run a linter and generate reports.""" - - user_options: list = [] - - def initialize_options(self): - """Do nothing.""" - - def finalize_options(self): - """Do nothing.""" - - def run(self): - """Run prospector and generate a report.""" - check_paths = PACKAGES + [ - "setup.py", - "tests", - ] - ignore = [ - "doc/", - ] - - # try to install missing dependencies and import prospector - try: - from prospector.run import main - except ImportError: - # try to install and then import - self.distribution.fetch_build_eggs(["prospector[with_pyroma]"]) - from prospector.run import main - - self.install_deps_temp() - - # run linter - - # change working directory to package root - package_root = os.path.abspath(os.path.dirname(__file__)) - os.chdir(package_root) - - # write command line - files = discover_python_files(check_paths, ignore) - sys.argv = ["prospector"] - sys.argv.extend(files) - - # run prospector - errno = main() - - sys.exit(errno) - - -def read_authors(filename): - """Read the list of authors from .zenodo.json file.""" - with Path(filename).open(encoding="utf-8") as file: - info = json.load(file) - authors = [] - for author in info["creators"]: - name = " ".join(author["name"].split(",")[::-1]).strip() - authors.append(name) - return ", ".join(authors) - - -def read_description(filename): - """Read the description from .zenodo.json file.""" - with Path(filename).open(encoding="utf-8") as file: - info = json.load(file) - return info["description"] - - -setup( - name="ESMValCore", - author=read_authors(".zenodo.json"), - description=read_description(".zenodo.json"), - long_description=Path("README.md").read_text(encoding="utf-8"), - long_description_content_type="text/markdown", - url="https://www.esmvaltool.org", - download_url="https://github.com/ESMValGroup/ESMValCore", - license="Apache License, Version 2.0", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: English", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Atmospheric Science", - "Topic :: Scientific/Engineering :: GIS", - "Topic :: Scientific/Engineering :: Hydrology", - "Topic :: Scientific/Engineering :: Physics", - ], - packages=PACKAGES, - # Include all version controlled files - include_package_data=True, - setup_requires=REQUIREMENTS["setup"], - install_requires=REQUIREMENTS["install"], - tests_require=REQUIREMENTS["test"], - extras_require={ - "develop": REQUIREMENTS["develop"] - + REQUIREMENTS["test"] - + REQUIREMENTS["doc"], - "test": REQUIREMENTS["test"], - "doc": REQUIREMENTS["doc"], - }, - entry_points={ - "console_scripts": [ - "esmvaltool = esmvalcore._main:run", - ], - }, - cmdclass={ - "lint": RunLinter, - }, - zip_safe=False, -) diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py index 1624a688ea..5d504f6084 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2.py @@ -1,7 +1,6 @@ """Tests for the fixes of CESM2.""" import os -import sys import unittest.mock import iris @@ -61,10 +60,6 @@ def test_get_cl_fix(): ) -@pytest.mark.sequential -@pytest.mark.skipif( - sys.version_info < (3, 7, 6), reason="requires python3.7.6 or newer" -) @unittest.mock.patch( "esmvalcore.cmor._fixes.cmip6.cesm2.Fix.get_fixed_filepath", autospec=True ) diff --git a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py index dd400c8ab1..363bf0d80c 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py +++ b/tests/integration/cmor/_fixes/cmip6/test_cesm2_waccm.py @@ -1,7 +1,6 @@ """Tests for the fixes of CESM2-WACCM.""" import os -import sys import unittest.mock import iris @@ -36,9 +35,6 @@ def test_cl_fix(): assert issubclass(Cl, BaseCl) -@pytest.mark.skipif( - sys.version_info < (3, 7, 6), reason="requires python3.7.6 or newer" -) @unittest.mock.patch( "esmvalcore.cmor._fixes.cmip6.cesm2.Fix.get_fixed_filepath", autospec=True ) diff --git a/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py b/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py index 7e5fb81110..5f845d6d3d 100644 --- a/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py +++ b/tests/integration/cmor/_fixes/cmip6/test_gfdl_cm4.py @@ -49,7 +49,6 @@ def test_get_cl_fix(): ) -@pytest.mark.sequential def test_cl_fix_metadata(test_data_path): """Test ``fix_metadata`` for ``cl``.""" nc_path = test_data_path / "gfdl_cm4_cl.nc" diff --git a/tests/integration/esgf/test_search_download.py b/tests/integration/esgf/test_search_download.py index 65f17b3ef1..5029d75b42 100644 --- a/tests/integration/esgf/test_search_download.py +++ b/tests/integration/esgf/test_search_download.py @@ -170,7 +170,7 @@ def test_mock_search(variable, mocker): assert False, "Wrote expected results, please check." assert len(files) == len(expected_files) - for found_file, expected in zip(files, expected_files): + for found_file, expected in zip(files, expected_files, strict=False): assert found_file.name == expected["name"] assert found_file.local_file(Path()) == Path(expected["local_file"]) assert found_file.dataset == expected["dataset"] @@ -295,7 +295,7 @@ def test_real_search_many(): ] for variable, files, datasets in zip( - VARIABLES, expected_files, expected_datasets + VARIABLES, expected_files, expected_datasets, strict=False ): result = find_files(**variable) found_files = [file.name for file in result] diff --git a/tests/sample_data/multimodel_statistics/test_multimodel.py b/tests/sample_data/multimodel_statistics/test_multimodel.py index 3a9223c15f..815789674f 100644 --- a/tests/sample_data/multimodel_statistics/test_multimodel.py +++ b/tests/sample_data/multimodel_statistics/test_multimodel.py @@ -33,7 +33,7 @@ def assert_array_almost_equal(this, other, rtol=1e-7): def assert_coords_equal(this: list, other: list): """Assert coords list `this` equals coords list `other`.""" - for this_coord, other_coord in zip(this, other): + for this_coord, other_coord in zip(this, other, strict=False): np.testing.assert_equal(this_coord.points, other_coord.points) assert this_coord.var_name == other_coord.var_name assert this_coord.standard_name == other_coord.standard_name diff --git a/tests/unit/esgf/test_search.py b/tests/unit/esgf/test_search.py index 65cd53f1cc..66d0551a8e 100644 --- a/tests/unit/esgf/test_search.py +++ b/tests/unit/esgf/test_search.py @@ -115,7 +115,7 @@ @pytest.mark.parametrize( - "our_facets, esgf_facets", zip(OUR_FACETS, ESGF_FACETS) + "our_facets, esgf_facets", zip(OUR_FACETS, ESGF_FACETS, strict=False) ) def test_get_esgf_facets(our_facets, esgf_facets): """Test that facet translation by get_esgf_facets works as expected.""" diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 653cd61038..de379983df 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -691,7 +691,6 @@ def generate_cubes_with_non_overlapping_timecoords(): ) -@pytest.mark.xfail(reason="Multimodel statistics returns the original cubes.") def test_edge_case_time_no_overlap_fail(): """Test case when time coords do not overlap using span='overlap'. @@ -913,7 +912,7 @@ def test_ignore_tas_scalar_height_coord(): tas_2m = generate_cube_from_dates("monthly") tas_1p5m = generate_cube_from_dates("monthly") - for cube, height in zip([tas_2m, tas_1p5m], [2.0, 1.5]): + for cube, height in zip([tas_2m, tas_1p5m], [2.0, 1.5], strict=False): cube.rename("air_temperature") cube.attributes["short_name"] = "tas" cube.add_aux_coord( diff --git a/tests/unit/preprocessor/_regrid/__init__.py b/tests/unit/preprocessor/_regrid/__init__.py index a59bbc5662..c2b79e2c0b 100644 --- a/tests/unit/preprocessor/_regrid/__init__.py +++ b/tests/unit/preprocessor/_regrid/__init__.py @@ -125,6 +125,7 @@ def _make_cube( for a, name in zip( np.meshgrid(node_data_x, node_data_y), ["longitude", "latitude"], + strict=False, ) ] face_data_x = np.arange(x) + 1 @@ -134,6 +135,7 @@ def _make_cube( for a, name in zip( np.meshgrid(face_data_x, face_data_y), ["longitude", "latitude"], + strict=False, ) ] # Build the face connectivity indices by creating an array of squares diff --git a/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py b/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py index 6ff0df6fec..dfa3fb75d8 100644 --- a/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py +++ b/tests/unit/preprocessor/_regrid/test_extract_regional_grid.py @@ -32,7 +32,7 @@ def clear_lru_cache(): "step_latitude", ) PASSING_SPECS = tuple( - dict(zip(SPEC_KEYS, spec)) + dict(zip(SPEC_KEYS, spec, strict=False)) for spec in ( (0, 360, 5, -90, 90, 5), (0, 360, 20, -90, 90, 20), @@ -52,7 +52,7 @@ def clear_lru_cache(): ) FAILING_SPECS = tuple( - dict(zip(SPEC_KEYS, spec)) + dict(zip(SPEC_KEYS, spec, strict=False)) for spec in ( # (0, 360, 5, -90, 90, 5), (0, 360, 5, -90, 180, 5), diff --git a/tests/unit/preprocessor/_time/test_time.py b/tests/unit/preprocessor/_time/test_time.py index 54db6ed2b8..6a9d78747d 100644 --- a/tests/unit/preprocessor/_time/test_time.py +++ b/tests/unit/preprocessor/_time/test_time.py @@ -1800,7 +1800,9 @@ def test_anomalies_preserve_metadata(period, reference, standardize=False): metadata = copy.deepcopy(cube.metadata) result = anomalies(cube, period, reference, standardize=standardize) assert result.metadata == metadata - for coord_cube, coord_res in zip(cube.coords(), result.coords()): + for coord_cube, coord_res in zip( + cube.coords(), result.coords(), strict=False + ): if coord_cube.has_bounds() and coord_res.has_bounds(): assert_array_equal(coord_cube.bounds, coord_res.bounds) assert coord_cube == coord_res diff --git a/tests/unit/task/test_diagnostic_task.py b/tests/unit/task/test_diagnostic_task.py index 78b509b040..ca48e9d0ff 100644 --- a/tests/unit/task/test_diagnostic_task.py +++ b/tests/unit/task/test_diagnostic_task.py @@ -236,7 +236,7 @@ def test_collect_provenance(mocker, diagnostic_task): def assert_warned(log, msgs): """Check that messages have been logged.""" assert len(log.records) == len(msgs) - for msg, record in zip(msgs, log.records): + for msg, record in zip(msgs, log.records, strict=False): for snippet in msg: assert snippet in record.message From f622e408e9fb9919c69870dc52b52cddc8b0767f Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 16 Oct 2024 15:37:07 +0100 Subject: [PATCH 19/19] use `miniforge3` for our docker builds instead of `mambaforge` (#2558) --- docker/Dockerfile | 2 +- docker/Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 70de4d36d9..1a47018639 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ # To build this container, go to ESMValCore root folder and execute: # docker build -t esmvalcore:latest . -f docker/Dockerfile -FROM condaforge/mambaforge +FROM condaforge/miniforge3 WORKDIR /src/ESMValCore COPY environment.yml . diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index f709cf6a42..c72ac46995 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -1,7 +1,7 @@ # To build this container, go to ESMValCore root folder and execute: # This container is used to run the tests on CircleCI. # docker build -t esmvalcore:development . -f docker/Dockerfile.dev -FROM condaforge/mambaforge +FROM condaforge/miniforge3 WORKDIR /src/ESMValCore RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y curl git ssh && apt clean