Skip to content

Commit

Permalink
Merge pull request #5361 from SciTools/v3.6.x
Browse files Browse the repository at this point in the history 
v3.6.x merge-back
  • Loading branch information
@trexfeathers
trexfeathers authored Jun 27, 2023
2 parents bbba807 + d6c6c99 commit dab88e8
Show file tree
Hide file tree
Showing 12 changed files with 818 additions and 330 deletions.
54 changes: 53 additions & 1 deletion docs/src/whatsnew/3.6.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,56 @@ This document explains the changes made to Iris for this release
or feature requests for improving Iris. Enjoy!


v3.6.1 (26 June 2023)
=====================

.. dropdown:: v3.6.1 Patches
:color: primary
:icon: alert
:animate: fade-in

📢 **Announcements**

Welcome and congratulations to `@sloosvel`_ who made their first contribution to
Iris! 🎉

The patches in this release of Iris include:

✨ **Features**

#. `@rcomer`_ rewrote :func:`~iris.util.broadcast_to_shape` so it now handles
lazy data. This pull-request has been included to support :pull:`5341`.
(:pull:`5307`) [``pre-v3.7.0``]

🐛 **Bugs Fixed**

#. `@stephenworsley`_ fixed :meth:`~iris.cube.Cube.convert_units` to allow unit
conversion of lazy data when using a `Distributed`_ scheduler.
(:issue:`5347`, :pull:`5349`)

#. `@schlunma`_ fixed a bug in the concatenation of cubes with aux factories
which could lead to a `KeyError` due to dependencies that have not been
properly updated.
(:issue:`5339`, :pull:`5340`)

#. `@schlunma`_ fixed a bug which realized all weights during weighted
aggregation. Now weighted aggregation is fully lazy again.
(:issue:`5338`, :pull:`5341`)

🚀 **Performance Enhancements**

#. `@sloosvel`_ improved :meth:`~iris.cube.CubeList.concatenate_cube` and
:meth:`~iris.cube.CubeList.concatenate` to ensure that lazy auxiliary coordinate
points and bounds are not realized. This change now allows cubes with
high-resolution auxiliary coordinates to concatenate successfully whilst using a
minimal in-core memory footprint.
(:issue:`5115`, :pull:`5142`)

Note that, the above contribution labelled with ``pre-v3.7.0`` is part of the
forthcoming Iris ``v3.7.0`` release, but requires to be included in this patch
release.


📢 Announcements
================

Expand Down Expand Up @@ -169,6 +219,7 @@ This document explains the changes made to Iris for this release
core dev names are automatically included by the common_links.inc:
.. _@fnattino: https://github.com/fnattino
.. _@sloosvel: https://github.com/sloosvel


.. comment
Expand All @@ -180,4 +231,5 @@ This document explains the changes made to Iris for this release
.. _PEP-0621: https://peps.python.org/pep-0621/
.. _pypa/build: https://pypa-build.readthedocs.io/en/stable/
.. _NEP29: https://numpy.org/neps/nep-0029-deprecation_policy.html
.. _Contributor Covenant: https://www.contributor-covenant.org/version/2/1/code_of_conduct/
.. _Contributor Covenant: https://www.contributor-covenant.org/version/2/1/code_of_conduct/
.. _Distributed: https://distributed.dask.org/en/stable/
134 changes: 90 additions & 44 deletions lib/iris/_concatenate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,15 @@ def __new__(mcs, coord, dims):
"""
defn = coord.metadata
points_dtype = coord.points.dtype
bounds_dtype = coord.bounds.dtype if coord.bounds is not None else None
points_dtype = coord.core_points().dtype
bounds_dtype = (
coord.core_bounds().dtype
if coord.core_bounds() is not None
else None
)
kwargs = {}
# Add scalar flag metadata.
kwargs["scalar"] = coord.points.size == 1
kwargs["scalar"] = coord.core_points().size == 1
# Add circular flag metadata for dimensional coordinates.
if hasattr(coord, "circular"):
kwargs["circular"] = coord.circular
Expand Down Expand Up @@ -700,18 +704,27 @@ def _cmp(coord, other):
"""
# A candidate axis must have non-identical coordinate points.
candidate_axis = not array_equal(coord.points, other.points)
candidate_axis = not array_equal(
coord.core_points(), other.core_points()
)

if candidate_axis:
# Ensure both have equal availability of bounds.
result = (coord.bounds is None) == (other.bounds is None)
result = (coord.core_bounds() is None) == (
other.core_bounds() is None
)
else:
if coord.bounds is not None and other.bounds is not None:
if (
coord.core_bounds() is not None
and other.core_bounds() is not None
):
# Ensure equality of bounds.
result = array_equal(coord.bounds, other.bounds)
result = array_equal(coord.core_bounds(), other.core_bounds())
else:
# Ensure both have equal availability of bounds.
result = coord.bounds is None and other.bounds is None
result = (
coord.core_bounds() is None and other.core_bounds() is None
)

return result, candidate_axis

Expand Down Expand Up @@ -851,9 +864,13 @@ def concatenate(self):
# Concatenate the new dimension coordinate.
dim_coords_and_dims = self._build_dim_coordinates()

# Concatenate the new auxiliary coordinates.
# Concatenate the new auxiliary coordinates (does NOT include
# scalar coordinates!).
aux_coords_and_dims = self._build_aux_coordinates()

# Concatenate the new scalar coordinates.
scalar_coords = self._build_scalar_coordinates()

# Concatenate the new cell measures
cell_measures_and_dims = self._build_cell_measures()

Expand All @@ -862,18 +879,21 @@ def concatenate(self):

# Concatenate the new aux factories
aux_factories = self._build_aux_factories(
dim_coords_and_dims, aux_coords_and_dims
dim_coords_and_dims, aux_coords_and_dims, scalar_coords
)

# Concatenate the new data payload.
data = self._build_data()

# Build the new cube.
all_aux_coords_and_dims = aux_coords_and_dims + [
(scalar_coord, ()) for scalar_coord in scalar_coords
]
kwargs = cube_signature.defn._asdict()
cube = iris.cube.Cube(
data,
dim_coords_and_dims=dim_coords_and_dims,
aux_coords_and_dims=aux_coords_and_dims,
aux_coords_and_dims=all_aux_coords_and_dims,
cell_measures_and_dims=cell_measures_and_dims,
ancillary_variables_and_dims=ancillary_variables_and_dims,
aux_factories=aux_factories,
Expand Down Expand Up @@ -1095,7 +1115,7 @@ def _build_aux_coordinates(self):
# Concatenate the points together.
dim = dims.index(self.axis)
points = [
skton.signature.aux_coords_and_dims[i].coord.points
skton.signature.aux_coords_and_dims[i].coord.core_points()
for skton in skeletons
]
points = np.concatenate(tuple(points), axis=dim)
Expand All @@ -1104,7 +1124,9 @@ def _build_aux_coordinates(self):
bnds = None
if coord.has_bounds():
bnds = [
skton.signature.aux_coords_and_dims[i].coord.bounds
skton.signature.aux_coords_and_dims[
i
].coord.core_bounds()
for skton in skeletons
]
bnds = np.concatenate(tuple(bnds), axis=dim)
Expand Down Expand Up @@ -1132,12 +1154,22 @@ def _build_aux_coordinates(self):

aux_coords_and_dims.append((coord.copy(), dims))

# Generate all the scalar coordinates for the new concatenated cube.
for coord in cube_signature.scalar_coords:
aux_coords_and_dims.append((coord.copy(), ()))

return aux_coords_and_dims

def _build_scalar_coordinates(self):
"""
Generate the scalar coordinates for the new concatenated cube.
Returns:
A list of scalar coordinates.
"""
scalar_coords = []
for coord in self._cube_signature.scalar_coords:
scalar_coords.append(coord.copy())

return scalar_coords

def _build_cell_measures(self):
"""
Generate the cell measures with associated dimension(s)
Expand Down Expand Up @@ -1216,7 +1248,9 @@ def _build_ancillary_variables(self):

return ancillary_variables_and_dims

def _build_aux_factories(self, dim_coords_and_dims, aux_coords_and_dims):
def _build_aux_factories(
self, dim_coords_and_dims, aux_coords_and_dims, scalar_coords
):
"""
Generate the aux factories for the new concatenated cube.
Expand All @@ -1230,6 +1264,9 @@ def _build_aux_factories(self, dim_coords_and_dims, aux_coords_and_dims):
A list of auxiliary coordinates and dimension(s) tuple pairs from
the concatenated cube.
* scalar_coords:
A list of scalar coordinates from the concatenated cube.
Returns:
A list of :class:`iris.aux_factory.AuxCoordFactory`.
Expand All @@ -1240,35 +1277,44 @@ def _build_aux_factories(self, dim_coords_and_dims, aux_coords_and_dims):
old_aux_coords = [a[0] for a in cube_signature.aux_coords_and_dims]
new_dim_coords = [d[0] for d in dim_coords_and_dims]
new_aux_coords = [a[0] for a in aux_coords_and_dims]
scalar_coords = cube_signature.scalar_coords
old_scalar_coords = cube_signature.scalar_coords
new_scalar_coords = scalar_coords

aux_factories = []

# Generate all the factories for the new concatenated cube.
for i, (coord, dims, factory) in enumerate(
cube_signature.derived_coords_and_dims
):
# Check whether the derived coordinate of the factory spans the
# nominated dimension of concatenation.
if self.axis in dims:
# Update the dependencies of the factory with coordinates of
# the concatenated cube. We need to check all coordinate types
# here (dim coords, aux coords, and scalar coords).
new_dependencies = {}
for old_dependency in factory.dependencies.values():
if old_dependency in old_dim_coords:
dep_idx = old_dim_coords.index(old_dependency)
new_dependency = new_dim_coords[dep_idx]
elif old_dependency in old_aux_coords:
dep_idx = old_aux_coords.index(old_dependency)
new_dependency = new_aux_coords[dep_idx]
else:
dep_idx = scalar_coords.index(old_dependency)
new_dependency = scalar_coords[dep_idx]
new_dependencies[id(old_dependency)] = new_dependency
for _, _, factory in cube_signature.derived_coords_and_dims:
# Update the dependencies of the factory with coordinates of
# the concatenated cube. We need to check all coordinate types
# here (dim coords, aux coords, and scalar coords).

# Note: in contrast to other _build_... methods of this class, we
# do NOT need to distinguish between aux factories that span the
# nominated concatenation axis and aux factories that do not. The
# reason is that ALL aux factories need to be updated with the new
# coordinates of the concatenated cube (passed to this function via
# dim_coords_and_dims, aux_coords_and_dims, scalar_coords [these
# contain ALL new coordinates, not only the ones spanning the
# concatenation dimension]), so no special treatment for the aux
# factories that span the concatenation dimension is necessary. If
# not all aux factories are properly updated with references to the
# new coordinates, this may lead to KeyErrors (see
# https://github.com/SciTools/iris/issues/5339).
new_dependencies = {}
for old_dependency in factory.dependencies.values():
if old_dependency in old_dim_coords:
dep_idx = old_dim_coords.index(old_dependency)
new_dependency = new_dim_coords[dep_idx]
elif old_dependency in old_aux_coords:
dep_idx = old_aux_coords.index(old_dependency)
new_dependency = new_aux_coords[dep_idx]
else:
dep_idx = old_scalar_coords.index(old_dependency)
new_dependency = new_scalar_coords[dep_idx]
new_dependencies[id(old_dependency)] = new_dependency

# Create new factory with the updated dependencies.
factory = factory.updated(new_dependencies)
# Create new factory with the updated dependencies.
factory = factory.updated(new_dependencies)

aux_factories.append(factory)

Expand Down Expand Up @@ -1307,7 +1353,7 @@ def _build_dim_coordinates(self):

# Concatenate the points together for the nominated dimension.
points = [
skeleton.signature.dim_coords[dim_ind].points
skeleton.signature.dim_coords[dim_ind].core_points()
for skeleton in skeletons
]
points = np.concatenate(tuple(points))
Expand All @@ -1316,7 +1362,7 @@ def _build_dim_coordinates(self):
bounds = None
if self._cube_signature.dim_coords[dim_ind].has_bounds():
bounds = [
skeleton.signature.dim_coords[dim_ind].bounds
skeleton.signature.dim_coords[dim_ind].core_bounds()
for skeleton in skeletons
]
bounds = np.concatenate(tuple(bounds))
Expand Down
Loading

0 comments on commit dab88e8

Please sign in to comment.