Merge branch 'main' into multi_model_stats_equalise_coords

doc/changelog.rst

@@ -2,6 +2,34 @@ Changelog
 =========
 
 
+.. _changelog-v2-7-1:
+
+
+v2.7.1
+------
+Highlights
+~~~~~~~~~~
+
+This is a bugfix release where we unpin `cf-units` to allow the latest `iris=3.4.0` to be installed. It also includes an update to the default configuration used when searching the ESGF for files, to account for a recent change of the CEDA ESGF index node hostname. The changelog contains only changes that were made to the ``main`` branch.
+
+Installation
+~~~~~~~~~~~~
+
+- Set the version number on the development branches to one minor version more than the previous release (`#1854 <https://github.com/ESMValGroup/ESMValCore/pull/1854>`__) `Bouwe Andela <https://github.com/bouweandela>`__
+- Unpin cf-units (`#1770 <https://github.com/ESMValGroup/ESMValCore/pull/1770>`__) `Bouwe Andela <https://github.com/bouweandela>`__
+
+Bug fixes
+~~~~~~~~~
+
+- Improve error handling if an esgf index node is offline (`#1834 <https://github.com/ESMValGroup/ESMValCore/pull/1834>`__) `Bouwe Andela <https://github.com/bouweandela>`__
+
+Automatic testing
+~~~~~~~~~~~~~~~~~
+
+- Removed unnecessary test that fails with iris 3.4.0 (`#1846 <https://github.com/ESMValGroup/ESMValCore/pull/1846>`__) `Manuel Schlund <https://github.com/schlunma>`__
+- Update CEDA ESGF index node hostname (`#1838 <https://github.com/ESMValGroup/ESMValCore/pull/1838>`__) `Valeriu Predoi <https://github.com/valeriupredoi>`__
+
+
 .. _changelog-v2-7-0:
 
 

doc/quickstart/find_data.rst

@@ -363,6 +363,23 @@ Key           Description                   Default value if not specified
               file                          default DRS is used)
 ============= ============================= =================================
 
+.. hint::
+
+   To use the :func:`~esmvalcore.preprocessor.extract_levels` preprocessor on
+   native ICON data, you need to specify the name of the vertical coordinate
+   (e.g., ``coordinate: air_pressure``) since native ICON output usually
+   provides a 3D air pressure field instead of a simple 1D vertical coordinate.
+   Example:
+
+   .. code-block:: yaml
+
+    preprocessors:
+      extract_500hPa_level_from_icon:
+        extract_levels:
+          levels: 50000
+          scheme: linear
+          coordinate: air_pressure
+
 .. hint::
 
    In order to read cell area files (``areacella`` and ``areacello``), one

esmvalcore/_provenance.py

@@ -2,6 +2,7 @@
 import copy
 import logging
 import os
+from functools import total_ordering
 
 from netCDF4 import Dataset
 from PIL import Image
@@ -98,6 +99,7 @@ def get_task_provenance(task, recipe_entity):
     return activity
 
 
+@total_ordering
 class TrackedFile:
     """File with provenance tracking."""
 
@@ -141,6 +143,18 @@ def __repr__(self):
         """Return representation string (e.g., used by ``pformat``)."""
         return f"{self.__class__.__name__}: {self.filename}"
 
+    def __eq__(self, other):
+        """Check if `other` equals `self`."""
+        return hasattr(other, 'filename') and self.filename == other.filename
+
+    def __lt__(self, other):
+        """Check if `other` should be sorted before `self`."""
+        return hasattr(other, 'filename') and self.filename < other.filename
+
+    def __hash__(self):
+        """Return a unique hash for the file."""
+        return hash(self.filename)
+
     def copy_provenance(self):
         """Create a copy with identical provenance information."""
         if self.provenance is None:

esmvalcore/_recipe/recipe.py

@@ -1915,7 +1915,7 @@ def get_output(self) -> dict:
         output['recipe_data'] = self._raw_recipe
         output['task_output'] = {}
 
-        for task in self.tasks.flatten():
+        for task in sorted(self.tasks.flatten(), key=lambda t: t.priority):
             if self._cfg['remove_preproc_dir'] and isinstance(
                     task, PreprocessingTask):
                 # Skip preprocessing tasks that are deleted afterwards

esmvalcore/_task.py

@@ -274,7 +274,7 @@ def get_product_attributes(self) -> dict:
         """Return a mapping of product attributes."""
         return {
             product.filename: product.attributes
-            for product in self.products
+            for product in sorted(self.products)
         }
 
     def print_ancestors(self):

tests/unit/test_provenance.py

@@ -0,0 +1,24 @@
+"""Test `esmvalcore._provenance`."""
+from esmvalcore._provenance import TrackedFile
+
+
+def test_set():
+    assert {
+        TrackedFile('file1.nc', attributes={}),
+        TrackedFile('file1.nc', attributes={}),
+        TrackedFile('file2.nc', attributes={}),
+    } == {
+        TrackedFile('file1.nc', attributes={}),
+        TrackedFile('file2.nc', attributes={}),
+    }
+
+
+def test_sort():
+    file1 = TrackedFile('file1.nc', attributes={})
+    file2 = TrackedFile('file2.nc', attributes={})
+    assert sorted([file2, file1]) == [file1, file2]
+
+
+def test_equals():
+    file = TrackedFile('file.nc', attributes={})
+    assert file == TrackedFile('file.nc', attributes={})