Merge branch 'main' into Add_ch4_surface_stations_Core

ESMValGroup · Aug 10, 2023 · eefe791 · eefe791
2 parents 03be702 + 6b43ca2
commit eefe791
Show file tree

Hide file tree

Showing 10 changed files with 261 additions and 72 deletions.
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -9,9 +9,16 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "mambaforge-4.10"
+    # updated and deployed from Aug 1, 2023
+    python: "mambaforge-22.9"
   jobs:
+    pre_create_environment:
+      # update mamba just in case
+      - mamba update --yes --quiet --name=base mamba
+      - mamba --version
     post_create_environment:
+      # use conda run executable wrapper to have all env variables
+      - conda run -n ${CONDA_DEFAULT_ENV} mamba --version
       - conda run -n ${CONDA_DEFAULT_ENV} pip install . --no-deps
 
 # Declare the requirements required to build your docs

diff --git a/conda-linux-64.lock b/conda-linux-64.lock
@@ -345,7 +345,7 @@ https://conda.anaconda.org/conda-forge/linux-64/kealib-1.5.1-h3e6883b_4.conda#ef
 https://conda.anaconda.org/conda-forge/linux-64/libspatialite-5.0.1-hca56755_27.conda#918a735059cab21b96fc13a8d04fbcd8
 https://conda.anaconda.org/conda-forge/linux-64/matplotlib-base-3.7.2-py311h54ef318_0.conda#2631a9e423855fb586c05f8a5ee8b177
 https://conda.anaconda.org/conda-forge/linux-64/pandas-2.0.3-py311h320fe9a_1.conda#5f92f46bd33917832a99d1660b4075ac
-https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.9.1-pyhd8ed1ab_0.conda#044e7a1e0ad42c4e67110bd078150a63
+https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.10.0-pyhd8ed1ab_0.conda#0809187ef9b89a3d94a5c24d13936236
 https://conda.anaconda.org/conda-forge/linux-64/poppler-23.05.0-hd18248d_1.conda#09e0de1aa7330fe697eed76eaeef666d
 https://conda.anaconda.org/conda-forge/noarch/pybtex-0.24.0-pyhd8ed1ab_2.tar.bz2#2099b86a7399c44c0c61cdb6de6915ba
 https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.6.0-py311ha169711_1.conda#92633556d37e88ce45193374d408072c
@@ -389,7 +389,7 @@ https://conda.anaconda.org/conda-forge/linux-64/gtk2-2.24.33-h90689f9_2.tar.bz2#
 https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.3.0-pyhd8ed1ab_0.conda#1d018ee4ab13217e2544f795eb0a6798
 https://conda.anaconda.org/conda-forge/linux-64/librsvg-2.56.3-h98fae49_0.conda#620e754f4344f4c27259ff460a2b9c50
 https://conda.anaconda.org/conda-forge/noarch/myproxyclient-2.1.0-pyhd8ed1ab_2.tar.bz2#363b0816e411feb0df925d4f224f026a
-https://conda.anaconda.org/conda-forge/noarch/nbformat-5.9.1-pyhd8ed1ab_0.conda#3ec35d84fc1775215061517eb4660693
+https://conda.anaconda.org/conda-forge/noarch/nbformat-5.9.2-pyhd8ed1ab_0.conda#61ba076de6530d9301a0053b02f093d2
 https://conda.anaconda.org/conda-forge/noarch/pep8-naming-0.10.0-pyh9f0ad1d_0.tar.bz2#b3c5536e4f9f58a4b16adb6f1e11732d
 https://conda.anaconda.org/conda-forge/noarch/pre-commit-3.3.3-pyha770c72_0.conda#dd64a0e440754ed97610b3e6b502b6b1
 https://conda.anaconda.org/conda-forge/noarch/prompt_toolkit-3.0.39-hd8ed1ab_0.conda#4bbbe67d5df19db30f04b8e344dc9976

diff --git a/doc/conf.py b/doc/conf.py
@@ -77,6 +77,9 @@
     'autosummary': True,
 }
 
+# Show type hints in function signature AND docstring
+autodoc_typehints = 'both'
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
 
@@ -385,7 +388,7 @@
 # The format is a list of tuples containing the path and title.
 # epub_pre_files = []
 
-# HTML files shat should be inserted after the pages created by sphinx.
+# HTML files that should be inserted after the pages created by sphinx.
 # The format is a list of tuples containing the path and title.
 # epub_post_files = []
 

diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst
@@ -126,12 +126,12 @@ Then we have to create the class for the fix deriving from
 Next we must choose the method to use between the ones offered by the
 Fix class:
 
-- ``fix_file`` : should be used only to fix errors that prevent data loading.
+- ``fix_file``: should be used only to fix errors that prevent data loading.
   As a rule of thumb, you should only use it if the execution halts before
   reaching the checks.
 
-- ``fix_metadata`` : you want to change something in the cube that is not
-  the data (e.g variable or coordinate names, data units).
+- ``fix_metadata``: you want to change something in the cube that is not
+  the data (e.g., variable or coordinate names, data units).
 
 - ``fix_data``: you need to fix the data. Beware: coordinates data values are
   part of the metadata.
@@ -253,6 +253,7 @@ with the actual data units.
 .. code-block:: python
 
     def fix_metadata(self, cubes):
+        cube = self.get_cube_from_list(cubes)
         cube.units = 'real_units'
 
 
@@ -267,6 +268,7 @@ For example:
 .. code-block:: python
 
     def fix_metadata(self, cubes):
+        cube = self.get_cube_from_list(cubes)
         cube.units = self.vardef.units
 
 To learn more about what is available in these definitions, see:
@@ -289,7 +291,7 @@ missing coordinate you can create a fix for this model:
 .. code-block:: python
 
     def fix_metadata(self, cubes):
-        coord_cube = cubes.extract_strict('COORDINATE_NAME')
+        coord_cube = cubes.extract_cube('COORDINATE_NAME')
         # Usually this will correspond to an auxiliary coordinate
         # because the most common error is to forget adding it to the
         # coordinates attribute
@@ -302,9 +304,9 @@ missing coordinate you can create a fix for this model:
         }
 
         # It may also have bounds as another cube
-        coord.bounds = cubes.extract_strict('BOUNDS_NAME').data
+        coord.bounds = cubes.extract_cube('BOUNDS_NAME').data
 
-        data_cube = cubes.extract_strict('VAR_NAME')
+        data_cube = cubes.extract_cube('VAR_NAME')
         data_cube.add_aux_coord(coord, DIMENSIONS_INDEX_TUPLE)
         return [data_cube]
 

diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst
@@ -136,18 +136,20 @@ ESMValCore deals with those issues by applying specific fixes for those
 datasets that require them. Fixes are applied at three different preprocessor
 steps:
 
-    - fix_file: apply fixes directly to a copy of the file. Copying the files
-      is costly, so only errors that prevent Iris to load the file are fixed
-      here. See :func:`esmvalcore.preprocessor.fix_file`
-
-    - fix_metadata: metadata fixes are done just before concatenating the cubes
-      loaded from different files in the final one. Automatic metadata fixes
-      are also applied at this step. See
-      :func:`esmvalcore.preprocessor.fix_metadata`
-
-    - fix_data: data fixes are applied before starting any operation that will
-      alter the data itself. Automatic data fixes are also applied at this step.
-      See :func:`esmvalcore.preprocessor.fix_data`
+    - ``fix_file``: apply fixes directly to a copy of the file.
+      Copying the files is costly, so only errors that prevent Iris to load the
+      file are fixed here.
+      See :func:`esmvalcore.preprocessor.fix_file`.
+
+    - ``fix_metadata``: metadata fixes are done just before concatenating the
+      cubes loaded from different files in the final one.
+      Automatic metadata fixes are also applied at this step.
+      See :func:`esmvalcore.preprocessor.fix_metadata`.
+
+    - ``fix_data``: data fixes are applied before starting any operation that
+      will alter the data itself.
+      Automatic data fixes are also applied at this step.
+      See :func:`esmvalcore.preprocessor.fix_data`.
 
 To get an overview on data fixes and how to implement new ones, please go to
 :ref:`fixing_data`.

diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py
@@ -5,7 +5,9 @@
 import inspect
 import tempfile
 from pathlib import Path
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Any, Optional
+
+from iris.cube import Cube, CubeList
 
 from ..table import CMOR_TABLES
 
@@ -22,17 +24,17 @@ def __init__(
         vardef: VariableInfo,
         extra_facets: Optional[dict] = None,
         session: Optional[Session] = None,
-    ):
+    ) -> None:
         """Initialize fix object.
 
         Parameters
         ----------
-        vardef: VariableInfo
+        vardef:
             CMOR table entry.
-        extra_facets: dict, optional
+        extra_facets:
             Extra facets are mainly used for data outside of the big projects
             like CMIP, CORDEX, obs4MIPs. For details, see :ref:`extra_facets`.
-        session: Session, optional
+        session:
             Current session which includes configuration and directory
             information.
 
@@ -48,7 +50,7 @@ def fix_file(
         filepath: Path,
         output_dir: Path,
         add_unique_suffix: bool = False,
-    ) -> Path:
+    ) -> str | Path:
         """Apply fixes to the files prior to creating the cube.
 
         Should be used only to fix errors that prevent loading or cannot be
@@ -57,24 +59,24 @@ def fix_file(
 
         Parameters
         ----------
-        filepath: Path
+        filepath:
             File to fix.
-        output_dir: Path
+        output_dir:
             Output directory for fixed files.
-        add_unique_suffix: bool, optional (default: False)
+        add_unique_suffix:
             Adds a unique suffix to `output_dir` for thread safety.
 
         Returns
         -------
-        Path
+        str or pathlib.Path
             Path to the corrected file. It can be different from the original
             filepath if a fix has been applied, but if not it should be the
             original filepath.
 
         """
         return filepath
 
-    def fix_metadata(self, cubes):
+    def fix_metadata(self, cubes: CubeList) -> CubeList:
         """Apply fixes to the metadata of the cube.
 
         Changes applied here must not require data loading.
@@ -83,7 +85,7 @@ def fix_metadata(self, cubes):
 
         Parameters
         ----------
-        cubes: iris.cube.CubeList
+        cubes:
             Cubes to fix.
 
         Returns
@@ -94,26 +96,30 @@ def fix_metadata(self, cubes):
         """
         return cubes
 
-    def get_cube_from_list(self, cubes, short_name=None):
+    def get_cube_from_list(
+        self,
+        cubes: CubeList,
+        short_name: Optional[str] = None,
+    ) -> Cube:
         """Get a cube from the list with a given short name.
 
         Parameters
         ----------
-        cubes : iris.cube.CubeList
+        cubes:
             List of cubes to search.
-        short_name : str or None
+        short_name:
             Cube's variable short name. If `None`, `short name` is the class
             name.
 
         Raises
         ------
         Exception
-            If no cube is found.
+            No cube is found.
 
         Returns
         -------
-        iris.Cube
-            Variable's cube
+        iris.cube.Cube
+            Variable's cube.
 
         """
         if short_name is None:
@@ -123,14 +129,14 @@ def get_cube_from_list(self, cubes, short_name=None):
                 return cube
         raise Exception(f'Cube for variable "{short_name}" not found')
 
-    def fix_data(self, cube):
+    def fix_data(self, cube: Cube) -> Cube:
         """Apply fixes to the data of the cube.
 
         These fixes should be applied before checking the data.
 
         Parameters
         ----------
-        cube: iris.cube.Cube
+        cube:
             Cube to fix.
 
         Returns
@@ -141,11 +147,11 @@ def fix_data(self, cube):
         """
         return cube
 
-    def __eq__(self, other):
+    def __eq__(self, other: Any) -> bool:
         """Fix equality."""
         return isinstance(self, other.__class__)
 
-    def __ne__(self, other):
+    def __ne__(self, other: Any) -> bool:
         """Fix inequality."""
         return not self.__eq__(other)
 
@@ -173,18 +179,18 @@ def get_fixes(
 
         Parameters
         ----------
-        project: str
+        project:
             Project of the dataset.
-        dataset: str
+        dataset:
             Name of the dataset.
-        mip: str
+        mip:
             Variable's MIP.
-        short_name: str
+        short_name:
             Variable's short name.
-        extra_facets: dict, optional
+        extra_facets:
             Extra facets are mainly used for data outside of the big projects
             like CMIP, CORDEX, obs4MIPs. For details, see :ref:`extra_facets`.
-        session: Session, optional
+        session:
             Current session which includes configuration and directory
             information.
 
@@ -248,12 +254,12 @@ def get_fixed_filepath(
 
         Parameters
         ----------
-        output_dir: Path
+        output_dir:
             Output directory for fixed files. Will be created if it does not
             exist yet.
-        filepath: str or Path
+        filepath:
             Original path.
-        add_unique_suffix: bool, optional (default: False)
+        add_unique_suffix:
             Adds a unique suffix to `output_dir` for thread safety.
 
         Returns

diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py
@@ -32,7 +32,7 @@ def fix_file(
     add_unique_suffix: bool = False,
     session: Optional[Session] = None,
     **extra_facets,
-) -> Path:
+) -> str | Path:
     """Fix files before ESMValTool can load them.
 
     This fixes are only for issues that prevent iris from loading the cube or
@@ -42,30 +42,31 @@ def fix_file(
 
     Parameters
     ----------
-    file: Path
+    file:
         Path to the original file.
-    short_name: str
+    short_name:
         Variable's short name.
-    project: str
+    project:
         Project of the dataset.
-    dataset: str
+    dataset:
         Name of the dataset.
-    mip: str
+    mip:
         Variable's MIP.
-    output_dir: Path
+    output_dir:
         Output directory for fixed files.
-    add_unique_suffix: bool, optional (default: False)
+    add_unique_suffix:
         Adds a unique suffix to `output_dir` for thread safety.
-    session: Session, optional
+    session:
         Current session which includes configuration and directory information.
-    **extra_facets: dict, optional
+    **extra_facets:
         Extra facets are mainly used for data outside of the big projects like
         CMIP, CORDEX, obs4MIPs. For details, see :ref:`extra_facets`.
 
     Returns
     -------
-    Path:
+    str or pathlib.Path
         Path to the fixed file.
+
     """
     # Update extra_facets with variable information given as regular arguments
     # to this function