From 4656b3928052619491a3b3c60c13af6198d23b10 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 23 Nov 2022 18:14:59 +0100 Subject: [PATCH 01/25] First implementation for smarter weighted aggregation --- lib/iris/analysis/__init__.py | 123 +++++++++++++++++++++++++++++++--- lib/iris/cube.py | 13 ++++ 2 files changed, 126 insertions(+), 10 deletions(-) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index c80469c3c7..744098fe68 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -41,6 +41,7 @@ from functools import wraps import warnings +from cf_units import Unit import dask.array as da import numpy as np import numpy.ma as ma @@ -56,6 +57,7 @@ from iris.analysis._regrid import CurvilinearRegridder, RectilinearRegridder import iris.coords from iris.exceptions import LazyAggregatorError +import iris.util __all__ = ( "Aggregator", @@ -467,7 +469,7 @@ def __init__( Kwargs: * units_func (callable): - | *Call signature*: (units) + | *Call signature*: (units, \**kwargs) If provided, called to convert a cube's units. Returns an :class:`cf_units.Unit`, or a @@ -625,7 +627,7 @@ def update_metadata(self, cube, coords, **kwargs): """ # Update the units if required. if self.units_func is not None: - cube.units = self.units_func(cube.units) + cube.units = self.units_func(cube.units, **kwargs) def post_process(self, collapsed_cube, data_result, coords, **kwargs): """ @@ -693,13 +695,13 @@ class PercentileAggregator(_Aggregator): """ def __init__(self, units_func=None, **kwargs): - """ + r""" Create a percentile aggregator. Kwargs: * units_func (callable): - | *Call signature*: (units) + | *Call signature*: (units, \**kwargs) If provided, called to convert a cube's units. Returns an :class:`cf_units.Unit`, or a @@ -934,13 +936,13 @@ class WeightedPercentileAggregator(PercentileAggregator): """ def __init__(self, units_func=None, lazy_func=None, **kwargs): - """ + r""" Create a weighted percentile aggregator. Kwargs: * units_func (callable): - | *Call signature*: (units) + | *Call signature*: (units, \**kwargs) If provided, called to convert a cube's units. Returns an :class:`cf_units.Unit`, or a @@ -1172,6 +1174,92 @@ def post_process(self, collapsed_cube, data_result, coords, **kwargs): return result +class Weights(np.ndarray): + """Class for handling weights for weighted aggregation. + + Since it inherits from :numpy:`ndarray`, all common methods and properties + of this are available. + + Details on subclassing :numpy:`ndarray` are given here: + https://numpy.org/doc/stable/user/basics.subclassing.html + + """ + + def __new__(cls, weights, cube): + """Create class instance. + + Args: + + * weights (Weights, numpy.ndarray, Cube, string): + If given as :class:`iris.analysis.Weights`, simply use this. If + given as a :class:`numpy.ndarray`, use this directly (assume units + of `1`). If given as a :class:`iris.cube.Cube`, use its data and + units. If given as a :obj:`str`, assume this is the name of a cell + measure from ``cube`` and its data and units. + * cube (Cube): + Input cube for aggregation. If weights is given as :obj:`str`, try + to extract a cell measure with the corresponding name from this + cube. Otherwise, this argument is ignored. + + """ + # Weights is Weights + if isinstance(weights, cls): + obj = weights + + # Weights is a cube + # Note: to avoid circular imports of Cube we use duck typing using the + # "hasattr" syntax here + elif hasattr(weights, "add_aux_coord"): + obj = np.asarray(weights.data).view(cls) + obj.units = weights.units + + # Weights is a string + elif isinstance(weights, str): + cell_measure = cube.cell_measure(weights) + if cell_measure.shape != cube.shape: + arr = iris.util.broadcast_to_shape( + cell_measure.data, # fails for dask arrays + cube.shape, + cube.cell_measure_dims(cell_measure), + ) + else: + arr = cell_measure.data + obj = np.asarray(arr).view(cls) + obj.units = cell_measure.units + + # Remaining types (e.g., np.ndarray): try to convert to ndarray. + else: + obj = np.asarray(weights).view(cls) + obj.units = Unit("1") + + return obj + + def __array_finalize__(self, obj): + """See https://numpy.org/doc/stable/user/basics.subclassing.html.""" + if obj is None: + return + self.units = getattr(obj, "units", Unit("1")) + + @classmethod + def update_kwargs(cls, kwargs, cube): + """Update ``weights`` keyword argument in-place. + + Args: + + * kwargs (dict): + Keyword arguments that will be updated in-place if a ``weights`` + keyword is present. + * cube (Cube): + Input cube for aggregation. If weights is given as :obj:`str`, try + to extract a cell measure with the corresponding name from this + cube. Otherwise, this argument is ignored. + + """ + if "weights" not in kwargs: + return + kwargs["weights"] = cls(kwargs["weights"], cube) + + def create_weighted_aggregator_fn(aggregator_fn, axis, **kwargs): """Return an aggregator function that can explicitely handle weights. @@ -1630,6 +1718,16 @@ def _sum(array, **kwargs): return rvalue +def _sum_units_func(units, **kwargs): + """Multiply original units with weight units if possible.""" + if "weights" not in kwargs: + return units + weights = kwargs["weights"] + if not hasattr(weights, "units"): + return units + return units * weights.units + + def _peak(array, **kwargs): def column_segments(column): nan_indices = np.where(np.isnan(column))[0] @@ -1745,7 +1843,7 @@ def interp_order(length): COUNT = Aggregator( "count", _count, - units_func=lambda units: 1, + units_func=lambda units, **kwargs: 1, lazy_func=_build_dask_mdtol_function(_count), ) """ @@ -1777,7 +1875,7 @@ def interp_order(length): MAX_RUN = Aggregator( None, iris._lazy_data.non_lazy(_lazy_max_run), - units_func=lambda units: 1, + units_func=lambda units, **kwargs: 1, lazy_func=_build_dask_mdtol_function(_lazy_max_run), ) """ @@ -2021,7 +2119,11 @@ def interp_order(length): """ -PROPORTION = Aggregator("proportion", _proportion, units_func=lambda units: 1) +PROPORTION = Aggregator( + "proportion", + _proportion, + units_func=lambda units, **kwargs: 1, +) """ An :class:`~iris.analysis.Aggregator` instance that calculates the proportion, as a fraction, of :class:`~iris.cube.Cube` data occurrences @@ -2122,6 +2224,7 @@ def interp_order(length): SUM = WeightedAggregator( "sum", _sum, + units_func=_sum_units_func, lazy_func=_build_dask_mdtol_function(_sum), ) """ @@ -2159,7 +2262,7 @@ def interp_order(length): VARIANCE = Aggregator( "variance", ma.var, - units_func=lambda units: units * units, + units_func=lambda units, **kwargs: units * units, lazy_func=_build_dask_mdtol_function(da.var), ddof=1, ) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index abe37c35fb..61811a3f15 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -28,6 +28,7 @@ import iris._lazy_data as _lazy import iris._merge import iris.analysis +from iris.analysis import Weights from iris.analysis.cartography import wrap_lons import iris.analysis.maths import iris.aux_factory @@ -3802,6 +3803,10 @@ def collapsed(self, coords, aggregator, **kwargs): cube.collapsed(['latitude', 'longitude'], iris.analysis.VARIANCE) """ + # Update weights kwargs (if necessary) to handle different types of + # weights + Weights.update_kwargs(kwargs, self) + # Convert any coordinate names to coordinates coords = self._as_list_of_coords(coords) @@ -4032,6 +4037,10 @@ def aggregated_by( STASH m01s00i024 """ + # Update weights kwargs (if necessary) to handle different types of + # weights + Weights.update_kwargs(kwargs, self) + groupby_coords = [] dimension_to_groupby = None @@ -4358,6 +4367,10 @@ def rolling_window(self, coord, aggregator, window, **kwargs): possible windows of size 3 from the original cube. """ + # Update weights kwargs (if necessary) to handle different types of + # weights + Weights.update_kwargs(kwargs, self) + coord = self._as_list_of_coords(coord)[0] if getattr(coord, "circular", False): From fcde56376b04aea2c177eee42fe3764cd4ea2a3b Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 24 Nov 2022 18:39:48 +0100 Subject: [PATCH 02/25] Allowed arbitrary _DimensionalMetadata for weights --- lib/iris/analysis/__init__.py | 52 ++++++++++++++++++++++------------- lib/iris/cube.py | 27 ++++++++++++++---- 2 files changed, 54 insertions(+), 25 deletions(-) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index 744098fe68..59929a2603 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -56,6 +56,7 @@ ) from iris.analysis._regrid import CurvilinearRegridder, RectilinearRegridder import iris.coords +from iris.coords import _DimensionalMetadata from iris.exceptions import LazyAggregatorError import iris.util @@ -1177,8 +1178,8 @@ def post_process(self, collapsed_cube, data_result, coords, **kwargs): class Weights(np.ndarray): """Class for handling weights for weighted aggregation. - Since it inherits from :numpy:`ndarray`, all common methods and properties - of this are available. + This subclasses :numpy:`ndarray`; thus, all methods and properties of + :numpy:`ndarray` (e.g., `shape`, `ndim`, `view()`, etc.) are available. Details on subclassing :numpy:`ndarray` are given here: https://numpy.org/doc/stable/user/basics.subclassing.html @@ -1190,42 +1191,55 @@ def __new__(cls, weights, cube): Args: - * weights (Weights, numpy.ndarray, Cube, string): + * weights (Weights, Cube, string, _DimensionalMetadata, array-like): If given as :class:`iris.analysis.Weights`, simply use this. If - given as a :class:`numpy.ndarray`, use this directly (assume units - of `1`). If given as a :class:`iris.cube.Cube`, use its data and - units. If given as a :obj:`str`, assume this is the name of a cell - measure from ``cube`` and its data and units. + given as a :class:`iris.cube.Cube`, use its data and units. If + given as a :obj:`str` or :class:`iris.coords._DimensionalMetadata`, + assume this is (the name of) a + :class:`iris.coords._DimensionalMetadata` object of the cube (i.e., + one of :func:`iris.cube.Cube.coords`, + :func:`iris.cube.Cube.cell_measures`, or + :func:`iris.cube.Cube.ancillary_variables`). If given as an + array-like object, use this directly and assume units of `1`. * cube (Cube): - Input cube for aggregation. If weights is given as :obj:`str`, try - to extract a cell measure with the corresponding name from this - cube. Otherwise, this argument is ignored. + Input cube for aggregation. If weights is given as :obj:`str` or + :class:`iris.coords._DimensionalMetadata`, try to extract the + :class:`iris.coords._DimensionalMetadata` object and corresponding + dimensional mappings from this cube. Otherwise, this argument is + ignored. """ # Weights is Weights + # --> Simple return this object if isinstance(weights, cls): obj = weights # Weights is a cube # Note: to avoid circular imports of Cube we use duck typing using the # "hasattr" syntax here + # --> Extract data and units from cube elif hasattr(weights, "add_aux_coord"): obj = np.asarray(weights.data).view(cls) obj.units = weights.units - # Weights is a string - elif isinstance(weights, str): - cell_measure = cube.cell_measure(weights) - if cell_measure.shape != cube.shape: + # Weights is a string or _DimensionalMetadata object + # --> Extract _DimensionalMetadata object from cube, broadcast it to + # correct shape using the corresponding dimensional mapping, and use + # its data and units + elif isinstance(weights, (str, _DimensionalMetadata)): + dim_metadata = cube._dimensional_metadata(weights) + if isinstance(dim_metadata, iris.coords.Coord): + arr = dim_metadata.points + else: + arr = dim_metadata.data + if dim_metadata.shape != cube.shape: arr = iris.util.broadcast_to_shape( - cell_measure.data, # fails for dask arrays + arr, cube.shape, - cube.cell_measure_dims(cell_measure), + dim_metadata.cube_dims(cube), ) - else: - arr = cell_measure.data obj = np.asarray(arr).view(cls) - obj.units = cell_measure.units + obj.units = dim_metadata.units # Remaining types (e.g., np.ndarray): try to convert to ndarray. else: diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 61811a3f15..c73ab8fbe7 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -4079,11 +4079,18 @@ def aggregated_by( f"that is aggregated, got {len(weights):d}, expected " f"{self.shape[dimension_to_groupby]:d}" ) - weights = iris.util.broadcast_to_shape( - weights, - self.shape, - (dimension_to_groupby,), + + # iris.util.broadcast_to_shape does not preserve Weights type + weights_units = weights.units + weights = Weights( + iris.util.broadcast_to_shape( + weights, + self.shape, + (dimension_to_groupby,), + ), + self, ) + weights.units = weights_units if weights.shape != self.shape: raise ValueError( f"Weights must either be 1D or have the same shape as the " @@ -4472,9 +4479,17 @@ def rolling_window(self, coord, aggregator, window, **kwargs): "as the window." ) kwargs = dict(kwargs) - kwargs["weights"] = iris.util.broadcast_to_shape( - weights, rolling_window_data.shape, (dimension + 1,) + + # iris.util.broadcast_to_shape does not preserve Weights type + weights_units = weights.units + weights = Weights( + iris.util.broadcast_to_shape( + weights, rolling_window_data.shape, (dimension + 1,) + ), + self, ) + weights.units = weights_units + kwargs["weights"] = weights data_result = aggregator.aggregate( rolling_window_data, axis=dimension + 1, **kwargs ) From ae4efbb9f1242e81564395cb37fe1c69f36548cc Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 24 Nov 2022 18:41:14 +0100 Subject: [PATCH 03/25] Added tests --- lib/iris/tests/test_aggregate_by.py | 148 +++++++++++++++++ lib/iris/tests/test_analysis.py | 108 +++++++++++++ lib/iris/tests/test_lazy_aggregate_by.py | 62 ++++++++ lib/iris/tests/unit/cube/test_Cube.py | 193 +++++++++++++++++++++-- 4 files changed, 501 insertions(+), 10 deletions(-) diff --git a/lib/iris/tests/test_aggregate_by.py b/lib/iris/tests/test_aggregate_by.py index 90bf0e5d4e..9cb1ba2d52 100644 --- a/lib/iris/tests/test_aggregate_by.py +++ b/lib/iris/tests/test_aggregate_by.py @@ -1328,5 +1328,153 @@ def test_weights_fail_with_non_weighted_aggregator(self): ) +# Simply redo the tests of TestAggregateBy with other cubes as weights +# Note: other weights types are not tested this way here since this would +# require adding dimensional metadata objects to the cubes, which would change +# the the CMLs of all resulting cubes + + +class TestAggregateByWeightedByCube(TestAggregateBy): + def setUp(self): + super().setUp() + + self.weights_single_original = self.weights_single + self.weights_multi_original = self.weights_multi + + self.weights_single = self.cube_single[:, 0, 0].copy( + self.weights_single_original + ) + self.weights_single.units = "m2" + self.weights_multi = self.cube_multi[:, 0, 0].copy( + self.weights_multi_original + ) + self.weights_multi.units = "m2" + + def test_weighted_sum_single(self): + aggregateby_cube = self.cube_single.aggregated_by( + "height", + iris.analysis.SUM, + weights=self.weights_single, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + aggregateby_cube = self.cube_single.aggregated_by( + self.coord_z_single, + iris.analysis.SUM, + weights=self.weights_single, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + def test_weighted_sum_multi(self): + aggregateby_cube = self.cube_multi.aggregated_by( + ["height", "level"], + iris.analysis.SUM, + weights=self.weights_multi, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + aggregateby_cube = self.cube_multi.aggregated_by( + ["level", "height"], + iris.analysis.SUM, + weights=self.weights_multi, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + aggregateby_cube = self.cube_multi.aggregated_by( + [self.coord_z1_multi, self.coord_z2_multi], + iris.analysis.SUM, + weights=self.weights_multi, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + aggregateby_cube = self.cube_multi.aggregated_by( + [self.coord_z2_multi, self.coord_z1_multi], + iris.analysis.SUM, + weights=self.weights_multi, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + +class TestAggregateByWeightedByObj(tests.IrisTest): + def setUp(self): + self.dim_coord = iris.coords.DimCoord( + [0, 1, 2], standard_name="latitude", units="degrees" + ) + self.aux_coord = iris.coords.AuxCoord( + [0, 1, 1], long_name="auxcoord", units="kg" + ) + self.cell_measure = iris.coords.CellMeasure( + [0, 0, 0], standard_name="cell_area", units="m2" + ) + self.ancillary_variable = iris.coords.AncillaryVariable( + [1, 1, 1], var_name="ancvar", units="kg" + ) + self.cube = iris.cube.Cube( + [1, 2, 3], + standard_name="air_temperature", + units="K", + dim_coords_and_dims=[(self.dim_coord, 0)], + aux_coords_and_dims=[(self.aux_coord, 0)], + cell_measures_and_dims=[(self.cell_measure, 0)], + ancillary_variables_and_dims=[(self.ancillary_variable, 0)], + ) + + def test_weighting_with_str_dim_coord(self): + res_cube = self.cube.aggregated_by( + "auxcoord", iris.analysis.SUM, weights="latitude" + ) + np.testing.assert_array_equal(res_cube.data, [0, 8]) + self.assertEqual(res_cube.units, "K degrees") + + def test_weighting_with_str_aux_coord(self): + res_cube = self.cube.aggregated_by( + "auxcoord", iris.analysis.SUM, weights="auxcoord" + ) + np.testing.assert_array_equal(res_cube.data, [0, 5]) + self.assertEqual(res_cube.units, "K kg") + + def test_weighting_with_str_cell_measure(self): + res_cube = self.cube.aggregated_by( + "auxcoord", iris.analysis.SUM, weights="cell_area" + ) + np.testing.assert_array_equal(res_cube.data, [0, 0]) + self.assertEqual(res_cube.units, "K m2") + + def test_weighting_with_str_ancillary_variable(self): + res_cube = self.cube.aggregated_by( + "auxcoord", iris.analysis.SUM, weights="ancvar" + ) + np.testing.assert_array_equal(res_cube.data, [1, 5]) + self.assertEqual(res_cube.units, "K kg") + + def test_weighting_with_dim_coord(self): + res_cube = self.cube.aggregated_by( + "auxcoord", iris.analysis.SUM, weights=self.dim_coord + ) + np.testing.assert_array_equal(res_cube.data, [0, 8]) + self.assertEqual(res_cube.units, "K degrees") + + def test_weighting_with_aux_coord(self): + res_cube = self.cube.aggregated_by( + "auxcoord", iris.analysis.SUM, weights=self.aux_coord + ) + np.testing.assert_array_equal(res_cube.data, [0, 5]) + self.assertEqual(res_cube.units, "K kg") + + def test_weighting_with_cell_measure(self): + res_cube = self.cube.aggregated_by( + "auxcoord", iris.analysis.SUM, weights=self.cell_measure + ) + np.testing.assert_array_equal(res_cube.data, [0, 0]) + self.assertEqual(res_cube.units, "K m2") + + def test_weighting_with_ancillary_variable(self): + res_cube = self.cube.aggregated_by( + "auxcoord", iris.analysis.SUM, weights=self.ancillary_variable + ) + np.testing.assert_array_equal(res_cube.data, [1, 5]) + self.assertEqual(res_cube.units, "K kg") + + if __name__ == "__main__": unittest.main() diff --git a/lib/iris/tests/test_analysis.py b/lib/iris/tests/test_analysis.py index e0a5d0971e..d25927ef1b 100644 --- a/lib/iris/tests/test_analysis.py +++ b/lib/iris/tests/test_analysis.py @@ -1702,5 +1702,113 @@ def test_weights_in_kwargs(self): self.assertEqual(kwargs, {"test_kwarg": "test", "weights": "ignored"}) +class TestWeights(tests.IrisTest): + def setUp(self): + self.lat = iris.coords.DimCoord( + [0, 1], standard_name="latitude", units="degrees" + ) + self.lon = iris.coords.DimCoord( + [0, 1, 2], standard_name="longitude", units="degrees" + ) + self.cell_measure = iris.coords.CellMeasure( + np.arange(6).reshape(2, 3), standard_name="cell_area", units="m2" + ) + self.aux_coord = iris.coords.AuxCoord( + [3, 4], long_name="auxcoord", units="s" + ) + self.ancillary_variable = iris.coords.AncillaryVariable( + [5, 6, 7], var_name="ancvar", units="kg" + ) + self.cube = iris.cube.Cube( + np.arange(6).reshape(2, 3), + standard_name="air_temperature", + units="K", + dim_coords_and_dims=[(self.lat, 0), (self.lon, 1)], + aux_coords_and_dims=[(self.aux_coord, 0)], + cell_measures_and_dims=[(self.cell_measure, (0, 1))], + ancillary_variables_and_dims=[(self.ancillary_variable, 1)], + ) + + def test_init_with_weights(self): + weights = iris.analysis.Weights([], self.cube) + new_weights = iris.analysis.Weights(weights, self.cube) + self.assertTrue(isinstance(new_weights, iris.analysis.Weights)) + self.assertTrue(new_weights is weights) + + def test_init_with_cube(self): + weights = iris.analysis.Weights(self.cube, self.cube) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) + self.assertEqual(weights.units, "K") + + def test_init_with_str_dim_coord(self): + weights = iris.analysis.Weights("latitude", self.cube) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) + self.assertEqual(weights.units, "degrees") + + def test_init_with_str_aux_coord(self): + weights = iris.analysis.Weights("auxcoord", self.cube) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) + self.assertEqual(weights.units, "s") + + def test_init_with_str_ancillary_variable(self): + weights = iris.analysis.Weights("ancvar", self.cube) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) + self.assertEqual(weights.units, "kg") + + def test_init_with_str_cell_measure(self): + weights = iris.analysis.Weights("cell_area", self.cube) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) + self.assertEqual(weights.units, "m2") + + def test_init_with_dim_coord(self): + weights = iris.analysis.Weights(self.lat, self.cube) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) + self.assertEqual(weights.units, "degrees") + + def test_init_with_aux_coord(self): + weights = iris.analysis.Weights(self.aux_coord, self.cube) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) + self.assertEqual(weights.units, "s") + + def test_init_with_ancillary_variable(self): + weights = iris.analysis.Weights(self.ancillary_variable, self.cube) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) + self.assertEqual(weights.units, "kg") + + def test_init_with_cell_measure(self): + weights = iris.analysis.Weights(self.cell_measure, self.cube) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) + self.assertEqual(weights.units, "m2") + + def test_init_with_list(self): + weights = iris.analysis.Weights([1, 2, 3], self.cube) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [1, 2, 3]) + self.assertEqual(weights.units, "1") + + def test_init_with_ndarray(self): + weights = iris.analysis.Weights(np.zeros((5, 5)), self.cube) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, np.zeros((5, 5))) + self.assertEqual(weights.units, "1") + + def test_init_with_invalid_obj(self): + self.assertRaises( + KeyError, + iris.analysis.Weights, + "invalid_obj", + self.cube, + ) + + if __name__ == "__main__": tests.main() diff --git a/lib/iris/tests/test_lazy_aggregate_by.py b/lib/iris/tests/test_lazy_aggregate_by.py index d1ebc9a36a..9debebed92 100644 --- a/lib/iris/tests/test_lazy_aggregate_by.py +++ b/lib/iris/tests/test_lazy_aggregate_by.py @@ -6,6 +6,7 @@ import unittest from iris._lazy_data import as_lazy_data +from iris.analysis import SUM from iris.tests import test_aggregate_by @@ -44,5 +45,66 @@ def tearDown(self): assert self.cube_easy.has_lazy_data() +class TestLazyAggregateByWeightedByCube(TestLazyAggregateBy): + def setUp(self): + super().setUp() + + self.weights_single_original = self.weights_single + self.weights_multi_original = self.weights_multi + + self.weights_single = self.cube_single[:, 0, 0].copy( + self.weights_single_original + ) + self.weights_single.units = "m2" + self.weights_multi = self.cube_multi[:, 0, 0].copy( + self.weights_multi_original + ) + self.weights_multi.units = "m2" + + def test_weighted_sum_single(self): + aggregateby_cube = self.cube_single.aggregated_by( + "height", + SUM, + weights=self.weights_single, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + aggregateby_cube = self.cube_single.aggregated_by( + self.coord_z_single, + SUM, + weights=self.weights_single, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + def test_weighted_sum_multi(self): + aggregateby_cube = self.cube_multi.aggregated_by( + ["height", "level"], + SUM, + weights=self.weights_multi, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + aggregateby_cube = self.cube_multi.aggregated_by( + ["level", "height"], + SUM, + weights=self.weights_multi, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + aggregateby_cube = self.cube_multi.aggregated_by( + [self.coord_z1_multi, self.coord_z2_multi], + SUM, + weights=self.weights_multi, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + aggregateby_cube = self.cube_multi.aggregated_by( + [self.coord_z2_multi, self.coord_z1_multi], + SUM, + weights=self.weights_multi, + ) + self.assertEqual(aggregateby_cube.units, "kelvin m2") + + if __name__ == "__main__": unittest.main() diff --git a/lib/iris/tests/unit/cube/test_Cube.py b/lib/iris/tests/unit/cube/test_Cube.py index 8e9e00dce8..b5ddce2950 100644 --- a/lib/iris/tests/unit/cube/test_Cube.py +++ b/lib/iris/tests/unit/cube/test_Cube.py @@ -20,7 +20,7 @@ from iris._lazy_data import as_lazy_data import iris.analysis -from iris.analysis import MEAN, Aggregator, WeightedAggregator +from iris.analysis import MEAN, SUM, Aggregator, WeightedAggregator import iris.aux_factory from iris.aux_factory import HybridHeightFactory from iris.common.metadata import BaseMetadata @@ -342,18 +342,19 @@ def test_non_lazy_aggregator(self): self.assertArrayEqual(result.data, np.mean(self.data, axis=1)) -class Test_collapsed__multidim_weighted(tests.IrisTest): +class Test_collapsed__multidim_weighted_with_arr(tests.IrisTest): def setUp(self): self.data = np.arange(6.0).reshape((2, 3)) self.lazydata = as_lazy_data(self.data) # Test cubes wth (same-valued) real and lazy data - cube_real = Cube(self.data) + cube_real = Cube(self.data, units="m") for i_dim, name in enumerate(("y", "x")): npts = cube_real.shape[i_dim] coord = DimCoord(np.arange(npts), long_name=name) cube_real.add_dim_coord(coord, i_dim) self.cube_real = cube_real self.cube_lazy = cube_real.copy(data=self.lazydata) + self.cube_lazy.units = "kg" # Test weights and expected result for a y-collapse self.y_weights = np.array([0.3, 0.5]) self.full_weights_y = np.broadcast_to( @@ -375,6 +376,7 @@ def test_weighted_fullweights_real_y(self): self.assertArrayAlmostEqual( cube_collapsed.data, self.expected_result_y ) + self.assertEqual(cube_collapsed.units, "m") def test_weighted_fullweights_lazy_y(self): # Full-shape weights, lazy data : Check lazy result, same values as real calc. @@ -385,6 +387,7 @@ def test_weighted_fullweights_lazy_y(self): self.assertArrayAlmostEqual( cube_collapsed.data, self.expected_result_y ) + self.assertEqual(cube_collapsed.units, "kg") def test_weighted_1dweights_real_y(self): # 1-D weights, real data : Check same results as full-shape. @@ -394,6 +397,7 @@ def test_weighted_1dweights_real_y(self): self.assertArrayAlmostEqual( cube_collapsed.data, self.expected_result_y ) + self.assertEqual(cube_collapsed.units, "m") def test_weighted_1dweights_lazy_y(self): # 1-D weights, lazy data : Check lazy result, same values as real calc. @@ -404,6 +408,7 @@ def test_weighted_1dweights_lazy_y(self): self.assertArrayAlmostEqual( cube_collapsed.data, self.expected_result_y ) + self.assertEqual(cube_collapsed.units, "kg") def test_weighted_fullweights_real_x(self): # Full weights, real data, ** collapse X ** : as for 'y' case above @@ -413,6 +418,7 @@ def test_weighted_fullweights_real_x(self): self.assertArrayAlmostEqual( cube_collapsed.data, self.expected_result_x ) + self.assertEqual(cube_collapsed.units, "m") def test_weighted_fullweights_lazy_x(self): # Full weights, lazy data, ** collapse X ** : as for 'y' case above @@ -423,6 +429,7 @@ def test_weighted_fullweights_lazy_x(self): self.assertArrayAlmostEqual( cube_collapsed.data, self.expected_result_x ) + self.assertEqual(cube_collapsed.units, "kg") def test_weighted_1dweights_real_x(self): # 1-D weights, real data, ** collapse X ** : as for 'y' case above @@ -432,6 +439,7 @@ def test_weighted_1dweights_real_x(self): self.assertArrayAlmostEqual( cube_collapsed.data, self.expected_result_x ) + self.assertEqual(cube_collapsed.units, "m") def test_weighted_1dweights_lazy_x(self): # 1-D weights, lazy data, ** collapse X ** : as for 'y' case above @@ -442,6 +450,148 @@ def test_weighted_1dweights_lazy_x(self): self.assertArrayAlmostEqual( cube_collapsed.data, self.expected_result_x ) + self.assertEqual(cube_collapsed.units, "kg") + + def test_weighted_sum_fullweights_adapt_units_real_y(self): + # Check that units are adapated correctly ('m' * '1' = 'm') + cube_collapsed = self.cube_real.collapsed( + "y", SUM, weights=self.full_weights_y + ) + self.assertEqual(cube_collapsed.units, "m") + + def test_weighted_sum_fullweights_adapt_units_lazy_y(self): + # Check that units are adapated correctly ('kg' * '1' = 'kg') + cube_collapsed = self.cube_lazy.collapsed( + "y", SUM, weights=self.full_weights_y + ) + self.assertEqual(cube_collapsed.units, "kg") + + def test_weighted_sum_1dweights_adapt_units_real_y(self): + # Check that units are adapated correctly ('m' * '1' = 'm') + # Note: the same test with lazy data fails: + # https://github.com/SciTools/iris/issues/5083 + cube_collapsed = self.cube_real.collapsed( + "y", SUM, weights=self.y_weights + ) + self.assertEqual(cube_collapsed.units, "m") + + def test_weighted_sum_with_unkown_units_real_y(self): + # Check that units are adapated correctly ('unknown' * '1' = 'unknown') + # Note: does not need to be adapted in subclasses since 'unknown' + # multiplied by any unit is 'unknown' + self.cube_real.units = "unknown" + cube_collapsed = self.cube_real.collapsed( + "y", + SUM, + weights=self.full_weights_y, + ) + self.assertEqual(cube_collapsed.units, "unknown") + + def test_weighted_sum_with_unkown_units_lazy_y(self): + # Check that units are adapated correctly ('unknown' * '1' = 'unknown') + # Note: does not need to be adapted in subclasses since 'unknown' + # multiplied by any unit is 'unknown' + self.cube_lazy.units = "unknown" + cube_collapsed = self.cube_lazy.collapsed( + "y", + SUM, + weights=self.full_weights_y, + ) + self.assertEqual(cube_collapsed.units, "unknown") + + +# Simply redo the tests of Test_collapsed__multidim_weighted_with_arr with +# other allowed objects for weights + + +class Test_collapsed__multidim_weighted_with_cube( + Test_collapsed__multidim_weighted_with_arr +): + def setUp(self): + super().setUp() + + self.y_weights_original = self.y_weights + self.full_weights_y_original = self.full_weights_y + self.x_weights_original = self.x_weights + self.full_weights_x_original = self.full_weights_x + + self.y_weights = self.cube_real[:, 0].copy(self.y_weights_original) + self.y_weights.units = "m2" + self.full_weights_y = self.cube_real.copy(self.full_weights_y_original) + self.full_weights_y.units = "m2" + self.x_weights = self.cube_real[0, :].copy(self.x_weights_original) + self.full_weights_x = self.cube_real.copy(self.full_weights_x_original) + + def test_weighted_sum_fullweights_adapt_units_real_y(self): + # Check that units are adapated correctly ('m' * 'm2' = 'm3') + cube_collapsed = self.cube_real.collapsed( + "y", SUM, weights=self.full_weights_y + ) + self.assertEqual(cube_collapsed.units, "m3") + + def test_weighted_sum_fullweights_adapt_units_lazy_y(self): + # Check that units are adapated correctly ('kg' * 'm2' = 'kg m2') + cube_collapsed = self.cube_lazy.collapsed( + "y", SUM, weights=self.full_weights_y + ) + self.assertEqual(cube_collapsed.units, "kg m2") + + def test_weighted_sum_1dweights_adapt_units_real_y(self): + # Check that units are adapated correctly ('m' * 'm2' = 'm3') + # Note: the same test with lazy data fails: + # https://github.com/SciTools/iris/issues/5083 + cube_collapsed = self.cube_real.collapsed( + "y", SUM, weights=self.y_weights + ) + self.assertEqual(cube_collapsed.units, "m3") + + +class Test_collapsed__multidim_weighted_with_str( + Test_collapsed__multidim_weighted_with_cube +): + def setUp(self): + super().setUp() + + self.full_weights_y = "full_y" + self.full_weights_x = "full_x" + self.y_weights = "y" + self.x_weights = "1d_x" + + self.dim_metadata_full_y = iris.coords.CellMeasure( + self.full_weights_y_original, + long_name=self.full_weights_y, + units="m2", + ) + self.dim_metadata_full_x = iris.coords.AuxCoord( + self.full_weights_x_original, + long_name=self.full_weights_x, + units="m2", + ) + self.dim_metadata_1d_y = iris.coords.DimCoord( + self.y_weights_original, long_name=self.y_weights, units="m2" + ) + self.dim_metadata_1d_x = iris.coords.AncillaryVariable( + self.x_weights_original, long_name=self.x_weights, units="m2" + ) + + for cube in (self.cube_real, self.cube_lazy): + cube.add_cell_measure(self.dim_metadata_full_y, (0, 1)) + cube.add_aux_coord(self.dim_metadata_full_x, (0, 1)) + cube.remove_coord("y") + cube.add_dim_coord(self.dim_metadata_1d_y, 0) + cube.add_ancillary_variable(self.dim_metadata_1d_x, 1) + + +class Test_collapsed__multidim_weighted_with_dim_metadata( + Test_collapsed__multidim_weighted_with_str +): + def setUp(self): + super().setUp() + + self.full_weights_y = self.dim_metadata_full_y + self.full_weights_x = self.dim_metadata_full_x + self.y_weights = self.dim_metadata_1d_y + self.x_weights = self.dim_metadata_1d_x class Test_collapsed__cellmeasure_ancils(tests.IrisTest): @@ -604,16 +754,14 @@ def _assert_cube_as_expected(self, cube): def test_collapsed_lat_with_3_bounds(self): """Collapse latitude with 3 bounds.""" with mock.patch("warnings.warn") as warn: - collapsed_cube = self.cube.collapsed("latitude", iris.analysis.SUM) + collapsed_cube = self.cube.collapsed("latitude", SUM) self._assert_warn_cannot_check_contiguity(warn) self._assert_cube_as_expected(collapsed_cube) def test_collapsed_lon_with_3_bounds(self): """Collapse longitude with 3 bounds.""" with mock.patch("warnings.warn") as warn: - collapsed_cube = self.cube.collapsed( - "longitude", iris.analysis.SUM - ) + collapsed_cube = self.cube.collapsed("longitude", SUM) self._assert_warn_cannot_check_contiguity(warn) self._assert_cube_as_expected(collapsed_cube) @@ -621,7 +769,7 @@ def test_collapsed_lat_lon_with_3_bounds(self): """Collapse latitude and longitude with 3 bounds.""" with mock.patch("warnings.warn") as warn: collapsed_cube = self.cube.collapsed( - ["latitude", "longitude"], iris.analysis.SUM + ["latitude", "longitude"], SUM ) self._assert_warn_cannot_check_contiguity(warn) self._assert_cube_as_expected(collapsed_cube) @@ -741,9 +889,9 @@ def test_different_array_attrs_incompatible(self): class Test_rolling_window(tests.IrisTest): def setUp(self): - self.cube = Cube(np.arange(6)) + self.cube = Cube(np.arange(6), units="kg") self.multi_dim_cube = Cube(np.arange(36).reshape(6, 6)) - val_coord = DimCoord([0, 1, 2, 3, 4, 5], long_name="val") + val_coord = DimCoord([0, 1, 2, 3, 4, 5], long_name="val", units="s") month_coord = AuxCoord( ["jan", "feb", "mar", "apr", "may", "jun"], long_name="month" ) @@ -770,6 +918,7 @@ def test_string_coord(self): np.array([1, 2, 3, 4]), bounds=np.array([[0, 2], [1, 3], [2, 4], [3, 5]]), long_name="val", + units="s", ) month_coord = AuxCoord( np.array( @@ -818,6 +967,30 @@ def test_ancillary_variables_and_cell_measures_removed(self): self.assertEqual(res_cube.ancillary_variables(), []) self.assertEqual(res_cube.cell_measures(), []) + def test_weights_arr(self): + weights = [0, 0, 1, 0, 2] + res_cube = self.cube.rolling_window("val", SUM, 5, weights=weights) + np.testing.assert_array_equal(res_cube.data, [10, 13]) + self.assertEqual(res_cube.units, "kg") + + def test_weights_cube(self): + weights = Cube([0, 0, 1, 0, 2], units="m2") + res_cube = self.cube.rolling_window("val", SUM, 5, weights=weights) + np.testing.assert_array_equal(res_cube.data, [10, 13]) + self.assertEqual(res_cube.units, "kg m2") + + def test_weights_str(self): + weights = "val" + res_cube = self.cube.rolling_window("val", SUM, 6, weights=weights) + np.testing.assert_array_equal(res_cube.data, [55]) + self.assertEqual(res_cube.units, "kg s") + + def test_weights_dim_coord(self): + weights = self.cube.coord("val") + res_cube = self.cube.rolling_window("val", SUM, 6, weights=weights) + np.testing.assert_array_equal(res_cube.data, [55]) + self.assertEqual(res_cube.units, "kg s") + class Test_slices_dim_order(tests.IrisTest): """ From c3dd61d56fd05e87d45ea3e256790dc640a344dc Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 25 Nov 2022 09:45:26 +0100 Subject: [PATCH 04/25] Added units argument to constructor of Weights --- lib/iris/analysis/__init__.py | 15 +++++- lib/iris/cube.py | 9 ++-- lib/iris/tests/test_analysis.py | 89 +++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 8 deletions(-) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index 59929a2603..f6bcd83c60 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1186,7 +1186,7 @@ class Weights(np.ndarray): """ - def __new__(cls, weights, cube): + def __new__(cls, weights, cube, units=None): """Create class instance. Args: @@ -1200,13 +1200,20 @@ def __new__(cls, weights, cube): one of :func:`iris.cube.Cube.coords`, :func:`iris.cube.Cube.cell_measures`, or :func:`iris.cube.Cube.ancillary_variables`). If given as an - array-like object, use this directly and assume units of `1`. + array-like object, use this directly and assume units of `1`. If + `units` is given, ignore all units derived above and use the ones + given by `units`. * cube (Cube): Input cube for aggregation. If weights is given as :obj:`str` or :class:`iris.coords._DimensionalMetadata`, try to extract the :class:`iris.coords._DimensionalMetadata` object and corresponding dimensional mappings from this cube. Otherwise, this argument is ignored. + * units (string, Unit): + If ``None``, use units derived from `weights`. Otherwise, overwrite + the units derived from `weights` and use `units`. Warning: if + `weights` has been given as `Weights` object and `units` is used, + this will also overwrite the original instance. """ # Weights is Weights @@ -1246,6 +1253,10 @@ def __new__(cls, weights, cube): obj = np.asarray(weights).view(cls) obj.units = Unit("1") + # Overwrite units from units argument if necessary + if units is not None: + obj.units = units + return obj def __array_finalize__(self, obj): diff --git a/lib/iris/cube.py b/lib/iris/cube.py index c73ab8fbe7..249e6e03ef 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -4081,7 +4081,6 @@ def aggregated_by( ) # iris.util.broadcast_to_shape does not preserve Weights type - weights_units = weights.units weights = Weights( iris.util.broadcast_to_shape( weights, @@ -4089,8 +4088,8 @@ def aggregated_by( (dimension_to_groupby,), ), self, + units=weights.units, ) - weights.units = weights_units if weights.shape != self.shape: raise ValueError( f"Weights must either be 1D or have the same shape as the " @@ -4481,15 +4480,13 @@ def rolling_window(self, coord, aggregator, window, **kwargs): kwargs = dict(kwargs) # iris.util.broadcast_to_shape does not preserve Weights type - weights_units = weights.units - weights = Weights( + kwargs["weights"] = Weights( iris.util.broadcast_to_shape( weights, rolling_window_data.shape, (dimension + 1,) ), self, + units=weights.units, ) - weights.units = weights_units - kwargs["weights"] = weights data_result = aggregator.aggregate( rolling_window_data, axis=dimension + 1, **kwargs ) diff --git a/lib/iris/tests/test_analysis.py b/lib/iris/tests/test_analysis.py index d25927ef1b..fcccc81868 100644 --- a/lib/iris/tests/test_analysis.py +++ b/lib/iris/tests/test_analysis.py @@ -1734,6 +1734,16 @@ def test_init_with_weights(self): new_weights = iris.analysis.Weights(weights, self.cube) self.assertTrue(isinstance(new_weights, iris.analysis.Weights)) self.assertTrue(new_weights is weights) + np.testing.assert_array_equal(new_weights, []) + self.assertTrue(new_weights.units, "K") + + def test_init_with_weights_and_units(self): + weights = iris.analysis.Weights([], self.cube) + new_weights = iris.analysis.Weights(weights, self.cube, units="J") + self.assertTrue(isinstance(new_weights, iris.analysis.Weights)) + self.assertTrue(new_weights is weights) + np.testing.assert_array_equal(new_weights, []) + self.assertTrue(new_weights.units, "J") def test_init_with_cube(self): weights = iris.analysis.Weights(self.cube, self.cube) @@ -1741,66 +1751,136 @@ def test_init_with_cube(self): np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) self.assertEqual(weights.units, "K") + def test_init_with_cube_and_units(self): + weights = iris.analysis.Weights(self.cube, self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) + self.assertEqual(weights.units, "J") + def test_init_with_str_dim_coord(self): weights = iris.analysis.Weights("latitude", self.cube) self.assertTrue(isinstance(weights, iris.analysis.Weights)) np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) self.assertEqual(weights.units, "degrees") + def test_init_with_str_dim_coord_and_units(self): + weights = iris.analysis.Weights("latitude", self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) + self.assertEqual(weights.units, "J") + def test_init_with_str_aux_coord(self): weights = iris.analysis.Weights("auxcoord", self.cube) self.assertTrue(isinstance(weights, iris.analysis.Weights)) np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) self.assertEqual(weights.units, "s") + def test_init_with_str_aux_coord_and_units(self): + weights = iris.analysis.Weights("auxcoord", self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) + self.assertEqual(weights.units, "J") + def test_init_with_str_ancillary_variable(self): weights = iris.analysis.Weights("ancvar", self.cube) self.assertTrue(isinstance(weights, iris.analysis.Weights)) np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) self.assertEqual(weights.units, "kg") + def test_init_with_str_ancillary_variable_and_units(self): + weights = iris.analysis.Weights("ancvar", self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) + self.assertEqual(weights.units, "J") + def test_init_with_str_cell_measure(self): weights = iris.analysis.Weights("cell_area", self.cube) self.assertTrue(isinstance(weights, iris.analysis.Weights)) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) self.assertEqual(weights.units, "m2") + def test_init_with_str_cell_measure_and_units(self): + weights = iris.analysis.Weights("cell_area", self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) + self.assertEqual(weights.units, "J") + def test_init_with_dim_coord(self): weights = iris.analysis.Weights(self.lat, self.cube) self.assertTrue(isinstance(weights, iris.analysis.Weights)) np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) self.assertEqual(weights.units, "degrees") + def test_init_with_dim_coord_and_units(self): + weights = iris.analysis.Weights(self.lat, self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) + self.assertEqual(weights.units, "J") + def test_init_with_aux_coord(self): weights = iris.analysis.Weights(self.aux_coord, self.cube) self.assertTrue(isinstance(weights, iris.analysis.Weights)) np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) self.assertEqual(weights.units, "s") + def test_init_with_aux_coord_and_units(self): + weights = iris.analysis.Weights(self.aux_coord, self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) + self.assertEqual(weights.units, "J") + def test_init_with_ancillary_variable(self): weights = iris.analysis.Weights(self.ancillary_variable, self.cube) self.assertTrue(isinstance(weights, iris.analysis.Weights)) np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) self.assertEqual(weights.units, "kg") + def test_init_with_ancillary_variable_and_units(self): + weights = iris.analysis.Weights( + self.ancillary_variable, self.cube, units="J" + ) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) + self.assertEqual(weights.units, "J") + def test_init_with_cell_measure(self): weights = iris.analysis.Weights(self.cell_measure, self.cube) self.assertTrue(isinstance(weights, iris.analysis.Weights)) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) self.assertEqual(weights.units, "m2") + def test_init_with_cell_measure_and_units(self): + weights = iris.analysis.Weights( + self.cell_measure, self.cube, units="J" + ) + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) + self.assertEqual(weights.units, "J") + def test_init_with_list(self): weights = iris.analysis.Weights([1, 2, 3], self.cube) self.assertTrue(isinstance(weights, iris.analysis.Weights)) np.testing.assert_array_equal(weights, [1, 2, 3]) self.assertEqual(weights.units, "1") + def test_init_with_list_and_units(self): + weights = iris.analysis.Weights([1, 2, 3], self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, [1, 2, 3]) + self.assertEqual(weights.units, "J") + def test_init_with_ndarray(self): weights = iris.analysis.Weights(np.zeros((5, 5)), self.cube) self.assertTrue(isinstance(weights, iris.analysis.Weights)) np.testing.assert_array_equal(weights, np.zeros((5, 5))) self.assertEqual(weights.units, "1") + def test_init_with_ndarray_and_units(self): + weights = iris.analysis.Weights(np.zeros((5, 5)), self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis.Weights)) + np.testing.assert_array_equal(weights, np.zeros((5, 5))) + self.assertEqual(weights.units, "J") + def test_init_with_invalid_obj(self): self.assertRaises( KeyError, @@ -1809,6 +1889,15 @@ def test_init_with_invalid_obj(self): self.cube, ) + def test_init_with_invalid_obj_and_units(self): + self.assertRaises( + KeyError, + iris.analysis.Weights, + "invalid_obj", + self.cube, + units="J", + ) + if __name__ == "__main__": tests.main() From 401d49d0eecb9f975bbdd8da7c144d98de1f5591 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 25 Nov 2022 10:52:04 +0100 Subject: [PATCH 05/25] Added documentation --- .../general/plot_custom_aggregation.py | 2 +- docs/src/userguide/cube_statistics.rst | 85 +++++++++++++++++++ lib/iris/analysis/__init__.py | 6 +- lib/iris/cube.py | 39 ++++++--- 4 files changed, 115 insertions(+), 17 deletions(-) diff --git a/docs/gallery_code/general/plot_custom_aggregation.py b/docs/gallery_code/general/plot_custom_aggregation.py index 5fba3669b6..6ef6075fb3 100644 --- a/docs/gallery_code/general/plot_custom_aggregation.py +++ b/docs/gallery_code/general/plot_custom_aggregation.py @@ -72,7 +72,7 @@ def main(): # Make an aggregator from the user function. SPELL_COUNT = Aggregator( - "spell_count", count_spells, units_func=lambda units: 1 + "spell_count", count_spells, units_func=lambda units, **kwargs: 1 ) # Define the parameters of the test. diff --git a/docs/src/userguide/cube_statistics.rst b/docs/src/userguide/cube_statistics.rst index 08297c2a51..6e552f7562 100644 --- a/docs/src/userguide/cube_statistics.rst +++ b/docs/src/userguide/cube_statistics.rst @@ -155,6 +155,50 @@ Several examples of area averaging exist in the gallery which may be of interest including an example on taking a :ref:`global area-weighted mean `. +In addition to plain arrays, weights can also be given as cubes or (names of) +:meth:`~iris.cube.Cube.coords`, :meth:`~iris.cube.Cube.cell_measures`, or +:meth:`~iris.cube.Cube.ancillary_variables`. +This has the advantage of correct unit handling, e.g., for area-weighted sums +the units of the resulting cube are multiplied by an area unit: + +.. doctest:: + + >>> from iris.coords import CellMeasure + >>> cell_areas = CellMeasure( + ... grid_areas, + ... standard_name='cell_area', + ... units='m2', + ... measure='area', + ... ) + >>> cube.add_cell_measure(cell_areas, (0, 1, 2, 3)) + >>> area_weighted_sum = cube.collapsed( + ... ['grid_longitude', 'grid_latitude'], + ... iris.analysis.SUM, + ... weights='cell_area' + ... ) + >>> print(area_weighted_sum) + air_potential_temperature / (m2.K) (time: 3; model_level_number: 7) + Dimension coordinates: + time x - + model_level_number - x + Auxiliary coordinates: + forecast_period x - + level_height - x + sigma - x + Derived coordinates: + altitude - x + Scalar coordinates: + forecast_reference_time 2009-11-19 04:00:00 + grid_latitude 1.5145501 degrees, bound=(0.13755022, 2.89155) degrees + grid_longitude 358.74948 degrees, bound=(357.48724, 360.01172) degrees + surface_altitude 399.625 m, bound=(-14.0, 813.25) m + Cell methods: + sum grid_longitude, grid_latitude + Attributes: + STASH m01s00i004 + source 'Data from Met Office Unified Model' + um_version '7.3' + .. _cube-statistics-aggregated-by: Partially Reducing Data Dimensions @@ -338,3 +382,44 @@ from jja-2006 to jja-2010: mam 2010 jja 2010 +Moreover, :meth:`Cube.aggregated_by ` supports +weighted aggregation. +For example, this is helpful for an aggregation over a monthly time +coordinate that consists of months with different numbers of days. +Similar to :meth:`Cube.collapsed `, weights can be +given as arrays, cubes, or as (names of) :meth:`~iris.cube.Cube.coords`, +:meth:`~iris.cube.Cube.cell_measures`, or +:meth:`~iris.cube.Cube.ancillary_variables`. +When weights are not given as arrays, units are correctly handled for weighted +sums, i.e., the original unit of the cube is multiplied by the units of the +weights. +The following example shows a weighted sum (notice the change of the units): + +.. doctest:: aggregation + + >>> from iris.coords import AncillaryVariable + >>> time_weights = AncillaryVariable( + ... cube.coord("time").bounds[:, 1] - cube.coord("time").bounds[:, 0], + ... long_name="Time Weights", + ... units="hours", + ... ) + >>> cube.add_ancillary_variable(time_weights, 0) + >>> seasonal_sum = cube.aggregated_by("clim_season", iris.analysis.SUM, weights="Time Weights") + >>> print(seasonal_sum) + surface_temperature / (3600 s.K) (-- : 4; latitude: 18; longitude: 432) + Dimension coordinates: + latitude - x - + longitude - - x + Auxiliary coordinates: + clim_season x - - + forecast_reference_time x - - + season_year x - - + time x - - + Scalar coordinates: + forecast_period 0 hours + Cell methods: + mean month, year + sum clim_season + Attributes: + Conventions 'CF-1.5' + STASH m01s00i024 diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index f6bcd83c60..6acbecba4d 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1197,9 +1197,9 @@ def __new__(cls, weights, cube, units=None): given as a :obj:`str` or :class:`iris.coords._DimensionalMetadata`, assume this is (the name of) a :class:`iris.coords._DimensionalMetadata` object of the cube (i.e., - one of :func:`iris.cube.Cube.coords`, - :func:`iris.cube.Cube.cell_measures`, or - :func:`iris.cube.Cube.ancillary_variables`). If given as an + one of :meth:`iris.cube.Cube.coords`, + :meth:`iris.cube.Cube.cell_measures`, or + :meth:`iris.cube.Cube.ancillary_variables`). If given as an array-like object, use this directly and assume units of `1`. If `units` is given, ignore all units derived above and use the ones given by `units`. diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 249e6e03ef..e9ecbf8cf2 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -2701,7 +2701,7 @@ def subset(self, coord): else: if len(self.coord_dims(coord_to_extract)) > 1: - msg = "Currently, only 1D coords can be used to subset a cube" + msg = "Currently, only 1d coords can be used to subset a cube" raise iris.exceptions.CoordinateMultiDimError(msg) # Identify the dimension of the cube which this coordinate # references @@ -3722,9 +3722,15 @@ def collapsed(self, coords, aggregator, **kwargs): sum :data:`~iris.analysis.SUM`. Weighted aggregations support an optional *weights* keyword argument. - If set, this should be supplied as an array of weights whose shape - matches the cube. Values for latitude-longitude area weights may be - calculated using :func:`iris.analysis.cartography.area_weights`. + If set, this can be supplied as an array, cube, or (names of) + :meth:`~iris.cube.Cube.coords`, :meth:`~iris.cube.Cube.cell_measures`, + or :meth:`~iris.cube.Cube.ancillary_variables`. In all cases, the + weights should be 1d (for collapsing over a 1d coordinate) or match the + shape of the cube. When weights are not given as arrays, units are + correctly handled for weighted sums, i.e., the original unit of the + cube is multiplied by the units of the weights. Values for + latitude-longitude area weights may be calculated using + :func:`iris.analysis.cartography.area_weights`. Some Iris aggregators support "lazy" evaluation, meaning that cubes resulting from this method may represent data arrays which are @@ -3975,10 +3981,14 @@ def aggregated_by( also be supplied. These include :data:`~iris.analysis.MEAN` and :data:`~iris.analysis.SUM`. - Weighted aggregations support an optional *weights* keyword argument. If - set, this should be supplied as an array of weights whose shape matches - the cube or as 1D array whose length matches the dimension over which is - aggregated. + Weighted aggregations support an optional *weights* keyword argument. + If set, this can be supplied as an array, cube, or (names of) + :meth:`~iris.cube.Cube.coords`, :meth:`~iris.cube.Cube.cell_measures`, + or :meth:`~iris.cube.Cube.ancillary_variables`. In all cases, the + weights should be 1d or match the shape of the cube. When weights are + not given as arrays, units are correctly handled for weighted sums, + i.e., the original unit of the cube is multiplied by the units of the + weights. Parameters ---------- @@ -4067,7 +4077,7 @@ def aggregated_by( groupby_coords.append(coord) # Check shape of weights. These must either match the shape of the cube - # or be 1D (in this case, their length must be equal to the length of the + # or be 1d (in this case, their length must be equal to the length of the # dimension we are aggregating over). weights = kwargs.get("weights") return_weights = kwargs.get("returned", False) @@ -4075,7 +4085,7 @@ def aggregated_by( if weights.ndim == 1: if len(weights) != self.shape[dimension_to_groupby]: raise ValueError( - f"1D weights must have the same length as the dimension " + f"1d weights must have the same length as the dimension " f"that is aggregated, got {len(weights):d}, expected " f"{self.shape[dimension_to_groupby]:d}" ) @@ -4092,7 +4102,7 @@ def aggregated_by( ) if weights.shape != self.shape: raise ValueError( - f"Weights must either be 1D or have the same shape as the " + f"Weights must either be 1d or have the same shape as the " f"cube, got shape {weights.shape} for weights, " f"{self.shape} for cube" ) @@ -4304,8 +4314,11 @@ def rolling_window(self, coord, aggregator, window, **kwargs): * kwargs: Aggregator and aggregation function keyword arguments. The weights - argument to the aggregator, if any, should be a 1d array with the - same length as the chosen window. + argument to the aggregator, if any, should be a 1d array, cube, or + (names of) :meth:`~iris.cube.Cube.coords`, + :meth:`~iris.cube.Cube.cell_measures`, or + :meth:`~iris.cube.Cube.ancillary_variables` with the same length as + the chosen window. Returns: :class:`iris.cube.Cube`. From 546f20463ca299bd5c249d200980e85ea371d05b Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 25 Nov 2022 11:03:25 +0100 Subject: [PATCH 06/25] Fixed tests and added Weights class to API doc --- lib/iris/analysis/__init__.py | 1 + lib/iris/cube.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index 6acbecba4d..55243d0328 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -84,6 +84,7 @@ "VARIANCE", "WPERCENTILE", "WeightedAggregator", + "Weights", "clear_phenomenon_identity", "create_weighted_aggregator_fn", ) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index e9ecbf8cf2..d39cf52a11 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -2701,7 +2701,7 @@ def subset(self, coord): else: if len(self.coord_dims(coord_to_extract)) > 1: - msg = "Currently, only 1d coords can be used to subset a cube" + msg = "Currently, only 1D coords can be used to subset a cube" raise iris.exceptions.CoordinateMultiDimError(msg) # Identify the dimension of the cube which this coordinate # references From c694c4bb59f5905f740207971bbc8dd71154a1ad Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 25 Nov 2022 11:07:50 +0100 Subject: [PATCH 07/25] Fixed tests again --- lib/iris/cube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index d39cf52a11..17bc9f90b4 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -2701,7 +2701,7 @@ def subset(self, coord): else: if len(self.coord_dims(coord_to_extract)) > 1: - msg = "Currently, only 1D coords can be used to subset a cube" + msg = "Currently, only 1D coords can be used to subset a cube" raise iris.exceptions.CoordinateMultiDimError(msg) # Identify the dimension of the cube which this coordinate # references From 6dff4a8d0d44e1f2d8e46ae976fd8eca5bbdfd97 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 25 Nov 2022 12:06:36 +0100 Subject: [PATCH 08/25] Fixed bug which lead to errors when weights was explicitly set to None --- lib/iris/analysis/__init__.py | 6 ++++-- lib/iris/cube.py | 6 +++--- lib/iris/tests/test_aggregate_by.py | 21 +++++++++++++++++++++ lib/iris/tests/test_analysis.py | 19 +++++++++++++++++++ lib/iris/tests/unit/cube/test_Cube.py | 18 ++++++++++++++++++ 5 files changed, 65 insertions(+), 5 deletions(-) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index 55243d0328..b221468be6 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1273,8 +1273,8 @@ def update_kwargs(cls, kwargs, cube): Args: * kwargs (dict): - Keyword arguments that will be updated in-place if a ``weights`` - keyword is present. + Keyword arguments that will be updated in-place if a `weights` + keyword is present which is not ``None``. * cube (Cube): Input cube for aggregation. If weights is given as :obj:`str`, try to extract a cell measure with the corresponding name from this @@ -1283,6 +1283,8 @@ def update_kwargs(cls, kwargs, cube): """ if "weights" not in kwargs: return + if kwargs["weights"] is None: + return kwargs["weights"] = cls(kwargs["weights"], cube) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 17bc9f90b4..92a37f86b2 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -4077,7 +4077,7 @@ def aggregated_by( groupby_coords.append(coord) # Check shape of weights. These must either match the shape of the cube - # or be 1d (in this case, their length must be equal to the length of the + # or be 1D (in this case, their length must be equal to the length of the # dimension we are aggregating over). weights = kwargs.get("weights") return_weights = kwargs.get("returned", False) @@ -4085,7 +4085,7 @@ def aggregated_by( if weights.ndim == 1: if len(weights) != self.shape[dimension_to_groupby]: raise ValueError( - f"1d weights must have the same length as the dimension " + f"1D weights must have the same length as the dimension " f"that is aggregated, got {len(weights):d}, expected " f"{self.shape[dimension_to_groupby]:d}" ) @@ -4102,7 +4102,7 @@ def aggregated_by( ) if weights.shape != self.shape: raise ValueError( - f"Weights must either be 1d or have the same shape as the " + f"Weights must either be 1D or have the same shape as the " f"cube, got shape {weights.shape} for weights, " f"{self.shape} for cube" ) diff --git a/lib/iris/tests/test_aggregate_by.py b/lib/iris/tests/test_aggregate_by.py index 9cb1ba2d52..1e057e3676 100644 --- a/lib/iris/tests/test_aggregate_by.py +++ b/lib/iris/tests/test_aggregate_by.py @@ -413,6 +413,27 @@ def test_single(self): aggregateby_cube.data, self.single_rms_expected ) + def test_single_weights_none(self): + # mean group-by with single coordinate name. + aggregateby_cube = self.cube_single.aggregated_by( + "height", iris.analysis.MEAN, weights=None + ) + self.assertCML( + aggregateby_cube, ("analysis", "aggregated_by", "single.cml") + ) + + # mean group-by with single coordinate. + aggregateby_cube = self.cube_single.aggregated_by( + self.coord_z_single, iris.analysis.MEAN, weights=None + ) + self.assertCML( + aggregateby_cube, ("analysis", "aggregated_by", "single.cml") + ) + + np.testing.assert_almost_equal( + aggregateby_cube.data, self.single_expected + ) + def test_weighted_single(self): # weighted mean group-by with single coordinate name. aggregateby_cube = self.cube_single.aggregated_by( diff --git a/lib/iris/tests/test_analysis.py b/lib/iris/tests/test_analysis.py index fcccc81868..377b0c6ad5 100644 --- a/lib/iris/tests/test_analysis.py +++ b/lib/iris/tests/test_analysis.py @@ -1898,6 +1898,25 @@ def test_init_with_invalid_obj_and_units(self): units="J", ) + def test_update_kwargs_no_weights(self): + kwargs = {"test": [1, 2, 3]} + iris.analysis.Weights.update_kwargs(kwargs, self.cube) + self.assertEqual(kwargs, {"test": [1, 2, 3]}) + + def test_update_kwargs_weights_none(self): + kwargs = {"test": [1, 2, 3], "weights": None} + iris.analysis.Weights.update_kwargs(kwargs, self.cube) + self.assertEqual(kwargs, {"test": [1, 2, 3], "weights": None}) + + def test_update_kwargs_weights(self): + kwargs = {"test": [1, 2, 3], "weights": [1, 2]} + iris.analysis.Weights.update_kwargs(kwargs, self.cube) + self.assertEqual(len(kwargs), 2) + self.assertEqual(kwargs["test"], [1, 2, 3]) + self.assertTrue(isinstance(kwargs["weights"], iris.analysis.Weights)) + np.testing.assert_array_equal(kwargs["weights"], [1, 2]) + self.assertEqual(kwargs["weights"].units, "1") + if __name__ == "__main__": tests.main() diff --git a/lib/iris/tests/unit/cube/test_Cube.py b/lib/iris/tests/unit/cube/test_Cube.py index b5ddce2950..3309ea5744 100644 --- a/lib/iris/tests/unit/cube/test_Cube.py +++ b/lib/iris/tests/unit/cube/test_Cube.py @@ -320,18 +320,36 @@ def test_dim0_lazy(self): self.assertArrayAlmostEqual(cube_collapsed.data, [1.5, 2.5, 3.5]) self.assertFalse(cube_collapsed.has_lazy_data()) + def test_dim0_lazy_weights_none(self): + cube_collapsed = self.cube.collapsed("y", MEAN, weights=None) + self.assertTrue(cube_collapsed.has_lazy_data()) + self.assertArrayAlmostEqual(cube_collapsed.data, [1.5, 2.5, 3.5]) + self.assertFalse(cube_collapsed.has_lazy_data()) + def test_dim1_lazy(self): cube_collapsed = self.cube.collapsed("x", MEAN) self.assertTrue(cube_collapsed.has_lazy_data()) self.assertArrayAlmostEqual(cube_collapsed.data, [1.0, 4.0]) self.assertFalse(cube_collapsed.has_lazy_data()) + def test_dim1_lazy_weights_none(self): + cube_collapsed = self.cube.collapsed("x", MEAN, weights=None) + self.assertTrue(cube_collapsed.has_lazy_data()) + self.assertArrayAlmostEqual(cube_collapsed.data, [1.0, 4.0]) + self.assertFalse(cube_collapsed.has_lazy_data()) + def test_multidims(self): # Check that MEAN works with multiple dims. cube_collapsed = self.cube.collapsed(("x", "y"), MEAN) self.assertTrue(cube_collapsed.has_lazy_data()) self.assertArrayAllClose(cube_collapsed.data, 2.5) + def test_multidims_weights_none(self): + # Check that MEAN works with multiple dims. + cube_collapsed = self.cube.collapsed(("x", "y"), MEAN, weights=None) + self.assertTrue(cube_collapsed.has_lazy_data()) + self.assertArrayAllClose(cube_collapsed.data, 2.5) + def test_non_lazy_aggregator(self): # An aggregator which doesn't have a lazy function should still work. dummy_agg = Aggregator( From c820183547c1dd57f192ada7ef14c59acbea2923 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 25 Nov 2022 12:19:47 +0100 Subject: [PATCH 09/25] Fixed bug in doc --- lib/iris/analysis/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index b221468be6..3034eef16b 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1179,10 +1179,11 @@ def post_process(self, collapsed_cube, data_result, coords, **kwargs): class Weights(np.ndarray): """Class for handling weights for weighted aggregation. - This subclasses :numpy:`ndarray`; thus, all methods and properties of - :numpy:`ndarray` (e.g., `shape`, `ndim`, `view()`, etc.) are available. + This subclasses :class:`numpy.ndarray`; thus, all methods and properties of + :class:`numpy.ndarray` (e.g., `shape`, `ndim`, `view()`, etc.) are + available. - Details on subclassing :numpy:`ndarray` are given here: + Details on subclassing :class:`numpy.ndarray` are given here: https://numpy.org/doc/stable/user/basics.subclassing.html """ From b4e2e8379e54a1b7624d5a54b7caf642d8c2b7c4 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 25 Nov 2022 12:37:09 +0100 Subject: [PATCH 10/25] Try 1 to fix doctest --- docs/src/userguide/cube_statistics.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/userguide/cube_statistics.rst b/docs/src/userguide/cube_statistics.rst index 6e552f7562..79478e4ac6 100644 --- a/docs/src/userguide/cube_statistics.rst +++ b/docs/src/userguide/cube_statistics.rst @@ -14,7 +14,7 @@ Cube Statistics Collapsing Entire Data Dimensions --------------------------------- -.. testsetup:: +.. testsetup:: collapsing import iris filename = iris.sample_data_path('uk_hires.pp') @@ -125,7 +125,7 @@ in order to calculate the area of the grid boxes:: These areas can now be passed to the ``collapsed`` method as weights: -.. doctest:: +.. doctest:: collapsing >>> new_cube = cube.collapsed(['grid_longitude', 'grid_latitude'], iris.analysis.MEAN, weights=grid_areas) >>> print(new_cube) @@ -161,7 +161,7 @@ In addition to plain arrays, weights can also be given as cubes or (names of) This has the advantage of correct unit handling, e.g., for area-weighted sums the units of the resulting cube are multiplied by an area unit: -.. doctest:: +.. doctest:: collapsing >>> from iris.coords import CellMeasure >>> cell_areas = CellMeasure( From 43ff2db8a8eb5c020213d6cd4196164525eb4785 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 25 Nov 2022 13:26:13 +0100 Subject: [PATCH 11/25] Do not import inherited functions for Weights anymore to avoid doctest failures --- .../contributing_documentation_full.rst | 3 +++ docs/src/sphinxext/generate_package_rst.py | 14 +++++++++++++- docs/src/userguide/cube_statistics.rst | 4 ++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/src/developers_guide/contributing_documentation_full.rst b/docs/src/developers_guide/contributing_documentation_full.rst index a470def683..cfd8ae7b1f 100755 --- a/docs/src/developers_guide/contributing_documentation_full.rst +++ b/docs/src/developers_guide/contributing_documentation_full.rst @@ -139,6 +139,9 @@ If there is a particularly troublesome module that breaks the ``make html`` you can exclude the module from the API documentation. Add the entry to the ``exclude_modules`` tuple list in the ``docs/src/sphinxext/generate_package_rst.py`` file. +If you want to avoid adding all methods of a parent class to a subclass, you +can add that class to the ``classes_no_inherited_members`` list at the top of +the ``docs/src/sphinxext/generate_package_rst.py`` file. .. _contributing.documentation.gallery: diff --git a/docs/src/sphinxext/generate_package_rst.py b/docs/src/sphinxext/generate_package_rst.py index 8f4119944f..b50403b901 100644 --- a/docs/src/sphinxext/generate_package_rst.py +++ b/docs/src/sphinxext/generate_package_rst.py @@ -17,6 +17,12 @@ ] +# List of classes for which no inherited members are shown +classes_no_inherited_members = [ + "iris.analysis.Weights", # avoid showing all methods of np.ndarray +] + + # print to stdout, including the name of the python file def autolog(message): print("[{}] {}".format(ntpath.basename(__file__), message)) @@ -32,7 +38,7 @@ def autolog(message): .. autoclass:: {object_name} :members: :undoc-members: - :inherited-members: + {inherited_members_setting} """, "function": """ @@ -160,10 +166,16 @@ def sort_key(arg): lines = [] for element, obj in document_these: object_name = import_name + "." + element + if object_name in classes_no_inherited_members: + inherited_members_setting = ":no-inherited-members:" + else: + inherited_members_setting = ":inherited-members:" obj_content = document_dict[lookup_object_type(obj)].format( + inherited_members_setting=inherited_members_setting, object_name=object_name, object_name_header_line="+" * len(object_name), object_docstring=inspect.getdoc(obj), + ) lines.append(obj_content) diff --git a/docs/src/userguide/cube_statistics.rst b/docs/src/userguide/cube_statistics.rst index 79478e4ac6..9dc21f91b5 100644 --- a/docs/src/userguide/cube_statistics.rst +++ b/docs/src/userguide/cube_statistics.rst @@ -141,8 +141,8 @@ These areas can now be passed to the ``collapsed`` method as weights: altitude - x Scalar coordinates: forecast_reference_time 2009-11-19 04:00:00 - grid_latitude 1.5145501 degrees, bound=(0.14430022, 2.8848) degrees - grid_longitude 358.74948 degrees, bound=(357.494, 360.00497) degrees + grid_latitude 1.5145501 degrees, bound=(0.13755022, 2.89155) degrees + grid_longitude 358.74948 degrees, bound=(357.48724, 360.01172) degrees surface_altitude 399.625 m, bound=(-14.0, 813.25) m Cell methods: mean grid_longitude, grid_latitude From 86707882a242a50b847ad6b7ead2337f2f1b65cb Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 6 Mar 2023 10:54:20 +0100 Subject: [PATCH 12/25] Made Weights class private --- .../contributing_documentation_full.rst | 3 - docs/src/sphinxext/generate_package_rst.py | 14 +- lib/iris/analysis/__init__.py | 23 +--- lib/iris/cube.py | 18 +-- lib/iris/tests/test_analysis.py | 122 +++++++++--------- 5 files changed, 80 insertions(+), 100 deletions(-) diff --git a/docs/src/developers_guide/contributing_documentation_full.rst b/docs/src/developers_guide/contributing_documentation_full.rst index cfd8ae7b1f..a470def683 100755 --- a/docs/src/developers_guide/contributing_documentation_full.rst +++ b/docs/src/developers_guide/contributing_documentation_full.rst @@ -139,9 +139,6 @@ If there is a particularly troublesome module that breaks the ``make html`` you can exclude the module from the API documentation. Add the entry to the ``exclude_modules`` tuple list in the ``docs/src/sphinxext/generate_package_rst.py`` file. -If you want to avoid adding all methods of a parent class to a subclass, you -can add that class to the ``classes_no_inherited_members`` list at the top of -the ``docs/src/sphinxext/generate_package_rst.py`` file. .. _contributing.documentation.gallery: diff --git a/docs/src/sphinxext/generate_package_rst.py b/docs/src/sphinxext/generate_package_rst.py index b50403b901..8f4119944f 100644 --- a/docs/src/sphinxext/generate_package_rst.py +++ b/docs/src/sphinxext/generate_package_rst.py @@ -17,12 +17,6 @@ ] -# List of classes for which no inherited members are shown -classes_no_inherited_members = [ - "iris.analysis.Weights", # avoid showing all methods of np.ndarray -] - - # print to stdout, including the name of the python file def autolog(message): print("[{}] {}".format(ntpath.basename(__file__), message)) @@ -38,7 +32,7 @@ def autolog(message): .. autoclass:: {object_name} :members: :undoc-members: - {inherited_members_setting} + :inherited-members: """, "function": """ @@ -166,16 +160,10 @@ def sort_key(arg): lines = [] for element, obj in document_these: object_name = import_name + "." + element - if object_name in classes_no_inherited_members: - inherited_members_setting = ":no-inherited-members:" - else: - inherited_members_setting = ":inherited-members:" obj_content = document_dict[lookup_object_type(obj)].format( - inherited_members_setting=inherited_members_setting, object_name=object_name, object_name_header_line="+" * len(object_name), object_docstring=inspect.getdoc(obj), - ) lines.append(obj_content) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index 3034eef16b..5a6d108c14 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -84,7 +84,6 @@ "VARIANCE", "WPERCENTILE", "WeightedAggregator", - "Weights", "clear_phenomenon_identity", "create_weighted_aggregator_fn", ) @@ -1176,7 +1175,7 @@ def post_process(self, collapsed_cube, data_result, coords, **kwargs): return result -class Weights(np.ndarray): +class _Weights(np.ndarray): """Class for handling weights for weighted aggregation. This subclasses :class:`numpy.ndarray`; thus, all methods and properties of @@ -1193,9 +1192,8 @@ def __new__(cls, weights, cube, units=None): Args: - * weights (Weights, Cube, string, _DimensionalMetadata, array-like): - If given as :class:`iris.analysis.Weights`, simply use this. If - given as a :class:`iris.cube.Cube`, use its data and units. If + * weights (Cube, string, _DimensionalMetadata, array-like): + If given as a :class:`iris.cube.Cube`, use its data and units. If given as a :obj:`str` or :class:`iris.coords._DimensionalMetadata`, assume this is (the name of) a :class:`iris.coords._DimensionalMetadata` object of the cube (i.e., @@ -1213,25 +1211,18 @@ def __new__(cls, weights, cube, units=None): ignored. * units (string, Unit): If ``None``, use units derived from `weights`. Otherwise, overwrite - the units derived from `weights` and use `units`. Warning: if - `weights` has been given as `Weights` object and `units` is used, - this will also overwrite the original instance. + the units derived from `weights` and use `units`. """ - # Weights is Weights - # --> Simple return this object - if isinstance(weights, cls): - obj = weights - - # Weights is a cube + # `weights` is a cube # Note: to avoid circular imports of Cube we use duck typing using the # "hasattr" syntax here # --> Extract data and units from cube - elif hasattr(weights, "add_aux_coord"): + if hasattr(weights, "add_aux_coord"): obj = np.asarray(weights.data).view(cls) obj.units = weights.units - # Weights is a string or _DimensionalMetadata object + # `weights`` is a string or _DimensionalMetadata object # --> Extract _DimensionalMetadata object from cube, broadcast it to # correct shape using the corresponding dimensional mapping, and use # its data and units diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 92a37f86b2..fcd7b5b828 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -28,7 +28,7 @@ import iris._lazy_data as _lazy import iris._merge import iris.analysis -from iris.analysis import Weights +from iris.analysis import _Weights from iris.analysis.cartography import wrap_lons import iris.analysis.maths import iris.aux_factory @@ -3728,7 +3728,7 @@ def collapsed(self, coords, aggregator, **kwargs): weights should be 1d (for collapsing over a 1d coordinate) or match the shape of the cube. When weights are not given as arrays, units are correctly handled for weighted sums, i.e., the original unit of the - cube is multiplied by the units of the weights. Values for + cube is multiplied by the units of the weights. Values for latitude-longitude area weights may be calculated using :func:`iris.analysis.cartography.area_weights`. @@ -3811,7 +3811,7 @@ def collapsed(self, coords, aggregator, **kwargs): """ # Update weights kwargs (if necessary) to handle different types of # weights - Weights.update_kwargs(kwargs, self) + _Weights.update_kwargs(kwargs, self) # Convert any coordinate names to coordinates coords = self._as_list_of_coords(coords) @@ -4049,7 +4049,7 @@ def aggregated_by( """ # Update weights kwargs (if necessary) to handle different types of # weights - Weights.update_kwargs(kwargs, self) + _Weights.update_kwargs(kwargs, self) groupby_coords = [] dimension_to_groupby = None @@ -4090,8 +4090,8 @@ def aggregated_by( f"{self.shape[dimension_to_groupby]:d}" ) - # iris.util.broadcast_to_shape does not preserve Weights type - weights = Weights( + # iris.util.broadcast_to_shape does not preserve _Weights type + weights = _Weights( iris.util.broadcast_to_shape( weights, self.shape, @@ -4388,7 +4388,7 @@ def rolling_window(self, coord, aggregator, window, **kwargs): """ # Update weights kwargs (if necessary) to handle different types of # weights - Weights.update_kwargs(kwargs, self) + _Weights.update_kwargs(kwargs, self) coord = self._as_list_of_coords(coord)[0] @@ -4492,8 +4492,8 @@ def rolling_window(self, coord, aggregator, window, **kwargs): ) kwargs = dict(kwargs) - # iris.util.broadcast_to_shape does not preserve Weights type - kwargs["weights"] = Weights( + # iris.util.broadcast_to_shape does not preserve _Weights type + kwargs["weights"] = _Weights( iris.util.broadcast_to_shape( weights, rolling_window_data.shape, (dimension + 1,) ), diff --git a/lib/iris/tests/test_analysis.py b/lib/iris/tests/test_analysis.py index 377b0c6ad5..75ec231a78 100644 --- a/lib/iris/tests/test_analysis.py +++ b/lib/iris/tests/test_analysis.py @@ -1730,161 +1730,165 @@ def setUp(self): ) def test_init_with_weights(self): - weights = iris.analysis.Weights([], self.cube) - new_weights = iris.analysis.Weights(weights, self.cube) - self.assertTrue(isinstance(new_weights, iris.analysis.Weights)) - self.assertTrue(new_weights is weights) + weights = iris.analysis._Weights([], self.cube) + new_weights = iris.analysis._Weights(weights, self.cube) + self.assertTrue(isinstance(new_weights, iris.analysis._Weights)) + self.assertTrue(new_weights is not weights) np.testing.assert_array_equal(new_weights, []) - self.assertTrue(new_weights.units, "K") + self.assertTrue(new_weights.units, "1") + self.assertTrue(weights.units, "1") def test_init_with_weights_and_units(self): - weights = iris.analysis.Weights([], self.cube) - new_weights = iris.analysis.Weights(weights, self.cube, units="J") - self.assertTrue(isinstance(new_weights, iris.analysis.Weights)) - self.assertTrue(new_weights is weights) + weights = iris.analysis._Weights([], self.cube) + new_weights = iris.analysis._Weights(weights, self.cube, units="J") + self.assertTrue(isinstance(new_weights, iris.analysis._Weights)) + self.assertTrue(new_weights is not weights) np.testing.assert_array_equal(new_weights, []) self.assertTrue(new_weights.units, "J") + self.assertTrue(weights.units, "1") def test_init_with_cube(self): - weights = iris.analysis.Weights(self.cube, self.cube) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights(self.cube, self.cube) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) self.assertEqual(weights.units, "K") def test_init_with_cube_and_units(self): - weights = iris.analysis.Weights(self.cube, self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights(self.cube, self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) self.assertEqual(weights.units, "J") def test_init_with_str_dim_coord(self): - weights = iris.analysis.Weights("latitude", self.cube) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights("latitude", self.cube) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) self.assertEqual(weights.units, "degrees") def test_init_with_str_dim_coord_and_units(self): - weights = iris.analysis.Weights("latitude", self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights("latitude", self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) self.assertEqual(weights.units, "J") def test_init_with_str_aux_coord(self): - weights = iris.analysis.Weights("auxcoord", self.cube) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights("auxcoord", self.cube) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) self.assertEqual(weights.units, "s") def test_init_with_str_aux_coord_and_units(self): - weights = iris.analysis.Weights("auxcoord", self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights("auxcoord", self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) self.assertEqual(weights.units, "J") def test_init_with_str_ancillary_variable(self): - weights = iris.analysis.Weights("ancvar", self.cube) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights("ancvar", self.cube) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) self.assertEqual(weights.units, "kg") def test_init_with_str_ancillary_variable_and_units(self): - weights = iris.analysis.Weights("ancvar", self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights("ancvar", self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) self.assertEqual(weights.units, "J") def test_init_with_str_cell_measure(self): - weights = iris.analysis.Weights("cell_area", self.cube) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights("cell_area", self.cube) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) self.assertEqual(weights.units, "m2") def test_init_with_str_cell_measure_and_units(self): - weights = iris.analysis.Weights("cell_area", self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights("cell_area", self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) self.assertEqual(weights.units, "J") def test_init_with_dim_coord(self): - weights = iris.analysis.Weights(self.lat, self.cube) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights(self.lat, self.cube) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) self.assertEqual(weights.units, "degrees") def test_init_with_dim_coord_and_units(self): - weights = iris.analysis.Weights(self.lat, self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights(self.lat, self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) self.assertEqual(weights.units, "J") def test_init_with_aux_coord(self): - weights = iris.analysis.Weights(self.aux_coord, self.cube) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights(self.aux_coord, self.cube) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) self.assertEqual(weights.units, "s") def test_init_with_aux_coord_and_units(self): - weights = iris.analysis.Weights(self.aux_coord, self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights(self.aux_coord, self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) self.assertEqual(weights.units, "J") def test_init_with_ancillary_variable(self): - weights = iris.analysis.Weights(self.ancillary_variable, self.cube) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights(self.ancillary_variable, self.cube) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) self.assertEqual(weights.units, "kg") def test_init_with_ancillary_variable_and_units(self): - weights = iris.analysis.Weights( + weights = iris.analysis._Weights( self.ancillary_variable, self.cube, units="J" ) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) self.assertEqual(weights.units, "J") def test_init_with_cell_measure(self): - weights = iris.analysis.Weights(self.cell_measure, self.cube) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights(self.cell_measure, self.cube) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) self.assertEqual(weights.units, "m2") def test_init_with_cell_measure_and_units(self): - weights = iris.analysis.Weights( + weights = iris.analysis._Weights( self.cell_measure, self.cube, units="J" ) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) self.assertEqual(weights.units, "J") def test_init_with_list(self): - weights = iris.analysis.Weights([1, 2, 3], self.cube) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights([1, 2, 3], self.cube) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [1, 2, 3]) self.assertEqual(weights.units, "1") def test_init_with_list_and_units(self): - weights = iris.analysis.Weights([1, 2, 3], self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights([1, 2, 3], self.cube, units="J") + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, [1, 2, 3]) self.assertEqual(weights.units, "J") def test_init_with_ndarray(self): - weights = iris.analysis.Weights(np.zeros((5, 5)), self.cube) - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights(np.zeros((5, 5)), self.cube) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, np.zeros((5, 5))) self.assertEqual(weights.units, "1") def test_init_with_ndarray_and_units(self): - weights = iris.analysis.Weights(np.zeros((5, 5)), self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis.Weights)) + weights = iris.analysis._Weights( + np.zeros((5, 5)), self.cube, units="J" + ) + self.assertTrue(isinstance(weights, iris.analysis._Weights)) np.testing.assert_array_equal(weights, np.zeros((5, 5))) self.assertEqual(weights.units, "J") def test_init_with_invalid_obj(self): self.assertRaises( KeyError, - iris.analysis.Weights, + iris.analysis._Weights, "invalid_obj", self.cube, ) @@ -1892,7 +1896,7 @@ def test_init_with_invalid_obj(self): def test_init_with_invalid_obj_and_units(self): self.assertRaises( KeyError, - iris.analysis.Weights, + iris.analysis._Weights, "invalid_obj", self.cube, units="J", @@ -1900,20 +1904,20 @@ def test_init_with_invalid_obj_and_units(self): def test_update_kwargs_no_weights(self): kwargs = {"test": [1, 2, 3]} - iris.analysis.Weights.update_kwargs(kwargs, self.cube) + iris.analysis._Weights.update_kwargs(kwargs, self.cube) self.assertEqual(kwargs, {"test": [1, 2, 3]}) def test_update_kwargs_weights_none(self): kwargs = {"test": [1, 2, 3], "weights": None} - iris.analysis.Weights.update_kwargs(kwargs, self.cube) + iris.analysis._Weights.update_kwargs(kwargs, self.cube) self.assertEqual(kwargs, {"test": [1, 2, 3], "weights": None}) def test_update_kwargs_weights(self): kwargs = {"test": [1, 2, 3], "weights": [1, 2]} - iris.analysis.Weights.update_kwargs(kwargs, self.cube) + iris.analysis._Weights.update_kwargs(kwargs, self.cube) self.assertEqual(len(kwargs), 2) self.assertEqual(kwargs["test"], [1, 2, 3]) - self.assertTrue(isinstance(kwargs["weights"], iris.analysis.Weights)) + self.assertTrue(isinstance(kwargs["weights"], iris.analysis._Weights)) np.testing.assert_array_equal(kwargs["weights"], [1, 2]) self.assertEqual(kwargs["weights"].units, "1") From 1b458f93e5ea9facfd03c15eb46e44649cbfc81d Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 6 Mar 2023 11:10:36 +0100 Subject: [PATCH 13/25] Suggestions from code review --- lib/iris/analysis/__init__.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index 5a6d108c14..346af449e1 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1228,10 +1228,7 @@ def __new__(cls, weights, cube, units=None): # its data and units elif isinstance(weights, (str, _DimensionalMetadata)): dim_metadata = cube._dimensional_metadata(weights) - if isinstance(dim_metadata, iris.coords.Coord): - arr = dim_metadata.points - else: - arr = dim_metadata.data + arr = dim_metadata._values if dim_metadata.shape != cube.shape: arr = iris.util.broadcast_to_shape( arr, @@ -1273,11 +1270,8 @@ def update_kwargs(cls, kwargs, cube): cube. Otherwise, this argument is ignored. """ - if "weights" not in kwargs: - return - if kwargs["weights"] is None: - return - kwargs["weights"] = cls(kwargs["weights"], cube) + if kwargs.get("weights") is not None: + kwargs["weights"] = cls(kwargs["weights"], cube) def create_weighted_aggregator_fn(aggregator_fn, axis, **kwargs): @@ -1740,12 +1734,14 @@ def _sum(array, **kwargs): def _sum_units_func(units, **kwargs): """Multiply original units with weight units if possible.""" - if "weights" not in kwargs: - return units - weights = kwargs["weights"] - if not hasattr(weights, "units"): - return units - return units * weights.units + weights = kwargs.get("weights") + if weights is None: # no weights given or weights are None + result = units + elif hasattr(weights, "units"): # weights are _Weights + result = units * weights.units + else: # weights are regular np.ndarrays + result = units + return result def _peak(array, **kwargs): From 2505fb8b3008ba2e3833831736bf025acd25584e Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 6 Mar 2023 12:40:15 +0100 Subject: [PATCH 14/25] Split tests into smaller parts --- lib/iris/tests/test_aggregate_by.py | 24 ++++++++++++------------ lib/iris/tests/test_lazy_aggregate_by.py | 17 ++++++++--------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/lib/iris/tests/test_aggregate_by.py b/lib/iris/tests/test_aggregate_by.py index 1e057e3676..8f015a19f2 100644 --- a/lib/iris/tests/test_aggregate_by.py +++ b/lib/iris/tests/test_aggregate_by.py @@ -1350,28 +1350,24 @@ def test_weights_fail_with_non_weighted_aggregator(self): # Simply redo the tests of TestAggregateBy with other cubes as weights -# Note: other weights types are not tested this way here since this would -# require adding dimensional metadata objects to the cubes, which would change -# the the CMLs of all resulting cubes +# Note: other weights types (e.g., coordinates, cell measures, etc.) are not +# tested this way here since this would require adding dimensional metadata +# objects to the cubes, which would change the CMLs of all resulting cubes of +# TestAggregateBy. class TestAggregateByWeightedByCube(TestAggregateBy): def setUp(self): super().setUp() - self.weights_single_original = self.weights_single - self.weights_multi_original = self.weights_multi - self.weights_single = self.cube_single[:, 0, 0].copy( - self.weights_single_original + self.weights_single ) self.weights_single.units = "m2" - self.weights_multi = self.cube_multi[:, 0, 0].copy( - self.weights_multi_original - ) + self.weights_multi = self.cube_multi[:, 0, 0].copy(self.weights_multi) self.weights_multi.units = "m2" - def test_weighted_sum_single(self): + def test_str_aggregation_weighted_sum_single(self): aggregateby_cube = self.cube_single.aggregated_by( "height", iris.analysis.SUM, @@ -1379,6 +1375,7 @@ def test_weighted_sum_single(self): ) self.assertEqual(aggregateby_cube.units, "kelvin m2") + def test_coord_aggregation_weighted_sum_single(self): aggregateby_cube = self.cube_single.aggregated_by( self.coord_z_single, iris.analysis.SUM, @@ -1386,7 +1383,7 @@ def test_weighted_sum_single(self): ) self.assertEqual(aggregateby_cube.units, "kelvin m2") - def test_weighted_sum_multi(self): + def test_str_aggregation_weighted_sum_multi(self): aggregateby_cube = self.cube_multi.aggregated_by( ["height", "level"], iris.analysis.SUM, @@ -1394,6 +1391,7 @@ def test_weighted_sum_multi(self): ) self.assertEqual(aggregateby_cube.units, "kelvin m2") + def test_str_aggregation_rev_order_weighted_sum_multi(self): aggregateby_cube = self.cube_multi.aggregated_by( ["level", "height"], iris.analysis.SUM, @@ -1401,6 +1399,7 @@ def test_weighted_sum_multi(self): ) self.assertEqual(aggregateby_cube.units, "kelvin m2") + def test_coord_aggregation_weighted_sum_multi(self): aggregateby_cube = self.cube_multi.aggregated_by( [self.coord_z1_multi, self.coord_z2_multi], iris.analysis.SUM, @@ -1408,6 +1407,7 @@ def test_weighted_sum_multi(self): ) self.assertEqual(aggregateby_cube.units, "kelvin m2") + def test_coord_aggregation_rev_order_weighted_sum_multi(self): aggregateby_cube = self.cube_multi.aggregated_by( [self.coord_z2_multi, self.coord_z1_multi], iris.analysis.SUM, diff --git a/lib/iris/tests/test_lazy_aggregate_by.py b/lib/iris/tests/test_lazy_aggregate_by.py index 9debebed92..57b748e52f 100644 --- a/lib/iris/tests/test_lazy_aggregate_by.py +++ b/lib/iris/tests/test_lazy_aggregate_by.py @@ -49,19 +49,14 @@ class TestLazyAggregateByWeightedByCube(TestLazyAggregateBy): def setUp(self): super().setUp() - self.weights_single_original = self.weights_single - self.weights_multi_original = self.weights_multi - self.weights_single = self.cube_single[:, 0, 0].copy( - self.weights_single_original + self.weights_single ) self.weights_single.units = "m2" - self.weights_multi = self.cube_multi[:, 0, 0].copy( - self.weights_multi_original - ) + self.weights_multi = self.cube_multi[:, 0, 0].copy(self.weights_multi) self.weights_multi.units = "m2" - def test_weighted_sum_single(self): + def test_str_aggregation_weighted_sum_single(self): aggregateby_cube = self.cube_single.aggregated_by( "height", SUM, @@ -69,6 +64,7 @@ def test_weighted_sum_single(self): ) self.assertEqual(aggregateby_cube.units, "kelvin m2") + def test_coord_aggregation_weighted_sum_single(self): aggregateby_cube = self.cube_single.aggregated_by( self.coord_z_single, SUM, @@ -76,7 +72,7 @@ def test_weighted_sum_single(self): ) self.assertEqual(aggregateby_cube.units, "kelvin m2") - def test_weighted_sum_multi(self): + def test_str_aggregation_weighted_sum_multi(self): aggregateby_cube = self.cube_multi.aggregated_by( ["height", "level"], SUM, @@ -84,6 +80,7 @@ def test_weighted_sum_multi(self): ) self.assertEqual(aggregateby_cube.units, "kelvin m2") + def test_str_aggregation_rev_order_weighted_sum_multi(self): aggregateby_cube = self.cube_multi.aggregated_by( ["level", "height"], SUM, @@ -91,6 +88,7 @@ def test_weighted_sum_multi(self): ) self.assertEqual(aggregateby_cube.units, "kelvin m2") + def test_coord_aggregation_weighted_sum_multi(self): aggregateby_cube = self.cube_multi.aggregated_by( [self.coord_z1_multi, self.coord_z2_multi], SUM, @@ -98,6 +96,7 @@ def test_weighted_sum_multi(self): ) self.assertEqual(aggregateby_cube.units, "kelvin m2") + def test_coord_aggregation_rev_order_weighted_sum_multi(self): aggregateby_cube = self.cube_multi.aggregated_by( [self.coord_z2_multi, self.coord_z1_multi], SUM, From 768a0c471bfec2ace27fac2a000df27e22e9c8ca Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 6 Mar 2023 13:00:50 +0100 Subject: [PATCH 15/25] Re-wrote TestWeights as pytest test --- lib/iris/tests/test_analysis.py | 122 ++++++++++++++++---------------- 1 file changed, 62 insertions(+), 60 deletions(-) diff --git a/lib/iris/tests/test_analysis.py b/lib/iris/tests/test_analysis.py index 75ec231a78..ab265cb965 100644 --- a/lib/iris/tests/test_analysis.py +++ b/lib/iris/tests/test_analysis.py @@ -12,6 +12,7 @@ import dask.array as da import numpy as np import numpy.ma as ma +import pytest import iris import iris.analysis.cartography @@ -288,7 +289,7 @@ def test_weighted_mean(self): iris.analysis.MEAN, ) - # Test collpasing of non data coord + # Test collapsing of non data coord self.assertRaises( iris.exceptions.CoordinateCollapseError, e.collapsed, @@ -1703,7 +1704,8 @@ def test_weights_in_kwargs(self): class TestWeights(tests.IrisTest): - def setUp(self): + @pytest.fixture(autouse=True) + def setup_test_data(self): self.lat = iris.coords.DimCoord( [0, 1], standard_name="latitude", units="degrees" ) @@ -1732,158 +1734,158 @@ def setUp(self): def test_init_with_weights(self): weights = iris.analysis._Weights([], self.cube) new_weights = iris.analysis._Weights(weights, self.cube) - self.assertTrue(isinstance(new_weights, iris.analysis._Weights)) - self.assertTrue(new_weights is not weights) + assert isinstance(new_weights, iris.analysis._Weights) + assert new_weights is not weights np.testing.assert_array_equal(new_weights, []) - self.assertTrue(new_weights.units, "1") - self.assertTrue(weights.units, "1") + assert new_weights.units == "1" + assert weights.units == "1" def test_init_with_weights_and_units(self): weights = iris.analysis._Weights([], self.cube) new_weights = iris.analysis._Weights(weights, self.cube, units="J") - self.assertTrue(isinstance(new_weights, iris.analysis._Weights)) - self.assertTrue(new_weights is not weights) + assert isinstance(new_weights, iris.analysis._Weights) + assert new_weights is not weights np.testing.assert_array_equal(new_weights, []) - self.assertTrue(new_weights.units, "J") - self.assertTrue(weights.units, "1") + assert new_weights.units == "J" + assert weights.units == "1" def test_init_with_cube(self): weights = iris.analysis._Weights(self.cube, self.cube) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) - self.assertEqual(weights.units, "K") + assert weights.units == "K" def test_init_with_cube_and_units(self): weights = iris.analysis._Weights(self.cube, self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) - self.assertEqual(weights.units, "J") + assert weights.units == "J" def test_init_with_str_dim_coord(self): weights = iris.analysis._Weights("latitude", self.cube) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) - self.assertEqual(weights.units, "degrees") + assert weights.units == "degrees" def test_init_with_str_dim_coord_and_units(self): weights = iris.analysis._Weights("latitude", self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) - self.assertEqual(weights.units, "J") + assert weights.units == "J" def test_init_with_str_aux_coord(self): weights = iris.analysis._Weights("auxcoord", self.cube) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) - self.assertEqual(weights.units, "s") + assert weights.units == "s" def test_init_with_str_aux_coord_and_units(self): weights = iris.analysis._Weights("auxcoord", self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) - self.assertEqual(weights.units, "J") + assert weights.units == "J" def test_init_with_str_ancillary_variable(self): weights = iris.analysis._Weights("ancvar", self.cube) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) - self.assertEqual(weights.units, "kg") + assert weights.units == "kg" def test_init_with_str_ancillary_variable_and_units(self): weights = iris.analysis._Weights("ancvar", self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) - self.assertEqual(weights.units, "J") + assert weights.units == "J" def test_init_with_str_cell_measure(self): weights = iris.analysis._Weights("cell_area", self.cube) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) - self.assertEqual(weights.units, "m2") + assert weights.units == "m2" def test_init_with_str_cell_measure_and_units(self): weights = iris.analysis._Weights("cell_area", self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) - self.assertEqual(weights.units, "J") + assert weights.units == "J" def test_init_with_dim_coord(self): weights = iris.analysis._Weights(self.lat, self.cube) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) - self.assertEqual(weights.units, "degrees") + assert weights.units == "degrees" def test_init_with_dim_coord_and_units(self): weights = iris.analysis._Weights(self.lat, self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[0, 0, 0], [1, 1, 1]]) - self.assertEqual(weights.units, "J") + assert weights.units == "J" def test_init_with_aux_coord(self): weights = iris.analysis._Weights(self.aux_coord, self.cube) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) - self.assertEqual(weights.units, "s") + assert weights.units == "s" def test_init_with_aux_coord_and_units(self): weights = iris.analysis._Weights(self.aux_coord, self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[3, 3, 3], [4, 4, 4]]) - self.assertEqual(weights.units, "J") + assert weights.units == "J" def test_init_with_ancillary_variable(self): weights = iris.analysis._Weights(self.ancillary_variable, self.cube) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) - self.assertEqual(weights.units, "kg") + assert weights.units == "kg" def test_init_with_ancillary_variable_and_units(self): weights = iris.analysis._Weights( self.ancillary_variable, self.cube, units="J" ) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [[5, 6, 7], [5, 6, 7]]) - self.assertEqual(weights.units, "J") + assert weights.units == "J" def test_init_with_cell_measure(self): weights = iris.analysis._Weights(self.cell_measure, self.cube) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) - self.assertEqual(weights.units, "m2") + assert weights.units == "m2" def test_init_with_cell_measure_and_units(self): weights = iris.analysis._Weights( self.cell_measure, self.cube, units="J" ) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, np.arange(6).reshape(2, 3)) - self.assertEqual(weights.units, "J") + assert weights.units == "J" def test_init_with_list(self): weights = iris.analysis._Weights([1, 2, 3], self.cube) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [1, 2, 3]) - self.assertEqual(weights.units, "1") + assert weights.units == "1" def test_init_with_list_and_units(self): weights = iris.analysis._Weights([1, 2, 3], self.cube, units="J") - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, [1, 2, 3]) - self.assertEqual(weights.units, "J") + assert weights.units == "J" def test_init_with_ndarray(self): weights = iris.analysis._Weights(np.zeros((5, 5)), self.cube) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, np.zeros((5, 5))) - self.assertEqual(weights.units, "1") + assert weights.units == "1" def test_init_with_ndarray_and_units(self): weights = iris.analysis._Weights( np.zeros((5, 5)), self.cube, units="J" ) - self.assertTrue(isinstance(weights, iris.analysis._Weights)) + assert isinstance(weights, iris.analysis._Weights) np.testing.assert_array_equal(weights, np.zeros((5, 5))) - self.assertEqual(weights.units, "J") + assert weights.units == "J" def test_init_with_invalid_obj(self): self.assertRaises( @@ -1905,21 +1907,21 @@ def test_init_with_invalid_obj_and_units(self): def test_update_kwargs_no_weights(self): kwargs = {"test": [1, 2, 3]} iris.analysis._Weights.update_kwargs(kwargs, self.cube) - self.assertEqual(kwargs, {"test": [1, 2, 3]}) + assert kwargs == {"test": [1, 2, 3]} def test_update_kwargs_weights_none(self): kwargs = {"test": [1, 2, 3], "weights": None} iris.analysis._Weights.update_kwargs(kwargs, self.cube) - self.assertEqual(kwargs, {"test": [1, 2, 3], "weights": None}) + assert kwargs == {"test": [1, 2, 3], "weights": None} def test_update_kwargs_weights(self): kwargs = {"test": [1, 2, 3], "weights": [1, 2]} iris.analysis._Weights.update_kwargs(kwargs, self.cube) - self.assertEqual(len(kwargs), 2) - self.assertEqual(kwargs["test"], [1, 2, 3]) - self.assertTrue(isinstance(kwargs["weights"], iris.analysis._Weights)) + assert len(kwargs) == 2 + assert kwargs["test"] == [1, 2, 3] + assert isinstance(kwargs["weights"], iris.analysis._Weights) np.testing.assert_array_equal(kwargs["weights"], [1, 2]) - self.assertEqual(kwargs["weights"].units, "1") + assert kwargs["weights"].units == "1" if __name__ == "__main__": From 035224197a5550fc03db9723dc641bf40b78be5e Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 6 Mar 2023 13:03:01 +0100 Subject: [PATCH 16/25] Spellcheck --- lib/iris/analysis/__init__.py | 8 ++++---- lib/iris/tests/unit/cube/test_Cube.py | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index 346af449e1..7b305aa0e4 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1275,7 +1275,7 @@ def update_kwargs(cls, kwargs, cube): def create_weighted_aggregator_fn(aggregator_fn, axis, **kwargs): - """Return an aggregator function that can explicitely handle weights. + """Return an aggregator function that can explicitly handle weights. Args: @@ -1500,7 +1500,7 @@ def _weighted_quantile_1D(data, weights, quantiles, **kwargs): array or float. Calculated quantile values (set to np.nan wherever sum of weights is zero or masked) """ - # Return np.nan if no useable points found + # Return np.nan if no usable points found if np.isclose(weights.sum(), 0.0) or ma.is_masked(weights.sum()): return np.resize(np.array(np.nan), len(quantiles)) # Sort the data @@ -1637,7 +1637,7 @@ def _proportion(array, function, axis, **kwargs): # Otherwise, it is possible for numpy to return a masked array that has # a dtype for its data that is different to the dtype of the fill-value, # which can cause issues outside this function. - # Reference - tests/unit/analyis/test_PROPORTION.py Test_masked.test_ma + # Reference - tests/unit/analysis/test_PROPORTION.py Test_masked.test_ma numerator = _count(array, axis=axis, function=function, **kwargs) result = ma.asarray(numerator / total_non_masked) @@ -2920,7 +2920,7 @@ def __init__(self, mdtol=1): Both sourge and target cubes must have an XY grid defined by separate X and Y dimensions with dimension coordinates. All of the XY dimension coordinates must also be bounded, and have - the same cooordinate system. + the same coordinate system. """ if not (0 <= mdtol <= 1): diff --git a/lib/iris/tests/unit/cube/test_Cube.py b/lib/iris/tests/unit/cube/test_Cube.py index 3309ea5744..93b49048d9 100644 --- a/lib/iris/tests/unit/cube/test_Cube.py +++ b/lib/iris/tests/unit/cube/test_Cube.py @@ -364,7 +364,7 @@ class Test_collapsed__multidim_weighted_with_arr(tests.IrisTest): def setUp(self): self.data = np.arange(6.0).reshape((2, 3)) self.lazydata = as_lazy_data(self.data) - # Test cubes wth (same-valued) real and lazy data + # Test cubes with (same-valued) real and lazy data cube_real = Cube(self.data, units="m") for i_dim, name in enumerate(("y", "x")): npts = cube_real.shape[i_dim] @@ -471,21 +471,21 @@ def test_weighted_1dweights_lazy_x(self): self.assertEqual(cube_collapsed.units, "kg") def test_weighted_sum_fullweights_adapt_units_real_y(self): - # Check that units are adapated correctly ('m' * '1' = 'm') + # Check that units are adapted correctly ('m' * '1' = 'm') cube_collapsed = self.cube_real.collapsed( "y", SUM, weights=self.full_weights_y ) self.assertEqual(cube_collapsed.units, "m") def test_weighted_sum_fullweights_adapt_units_lazy_y(self): - # Check that units are adapated correctly ('kg' * '1' = 'kg') + # Check that units are adapted correctly ('kg' * '1' = 'kg') cube_collapsed = self.cube_lazy.collapsed( "y", SUM, weights=self.full_weights_y ) self.assertEqual(cube_collapsed.units, "kg") def test_weighted_sum_1dweights_adapt_units_real_y(self): - # Check that units are adapated correctly ('m' * '1' = 'm') + # Check that units are adapted correctly ('m' * '1' = 'm') # Note: the same test with lazy data fails: # https://github.com/SciTools/iris/issues/5083 cube_collapsed = self.cube_real.collapsed( @@ -494,7 +494,7 @@ def test_weighted_sum_1dweights_adapt_units_real_y(self): self.assertEqual(cube_collapsed.units, "m") def test_weighted_sum_with_unkown_units_real_y(self): - # Check that units are adapated correctly ('unknown' * '1' = 'unknown') + # Check that units are adapted correctly ('unknown' * '1' = 'unknown') # Note: does not need to be adapted in subclasses since 'unknown' # multiplied by any unit is 'unknown' self.cube_real.units = "unknown" @@ -506,7 +506,7 @@ def test_weighted_sum_with_unkown_units_real_y(self): self.assertEqual(cube_collapsed.units, "unknown") def test_weighted_sum_with_unkown_units_lazy_y(self): - # Check that units are adapated correctly ('unknown' * '1' = 'unknown') + # Check that units are adapted correctly ('unknown' * '1' = 'unknown') # Note: does not need to be adapted in subclasses since 'unknown' # multiplied by any unit is 'unknown' self.cube_lazy.units = "unknown" @@ -541,21 +541,21 @@ def setUp(self): self.full_weights_x = self.cube_real.copy(self.full_weights_x_original) def test_weighted_sum_fullweights_adapt_units_real_y(self): - # Check that units are adapated correctly ('m' * 'm2' = 'm3') + # Check that units are adapted correctly ('m' * 'm2' = 'm3') cube_collapsed = self.cube_real.collapsed( "y", SUM, weights=self.full_weights_y ) self.assertEqual(cube_collapsed.units, "m3") def test_weighted_sum_fullweights_adapt_units_lazy_y(self): - # Check that units are adapated correctly ('kg' * 'm2' = 'kg m2') + # Check that units are adapted correctly ('kg' * 'm2' = 'kg m2') cube_collapsed = self.cube_lazy.collapsed( "y", SUM, weights=self.full_weights_y ) self.assertEqual(cube_collapsed.units, "kg m2") def test_weighted_sum_1dweights_adapt_units_real_y(self): - # Check that units are adapated correctly ('m' * 'm2' = 'm3') + # Check that units are adapted correctly ('m' * 'm2' = 'm3') # Note: the same test with lazy data fails: # https://github.com/SciTools/iris/issues/5083 cube_collapsed = self.cube_real.collapsed( @@ -669,7 +669,7 @@ def _assert_warn_collapse_without_weight(self, coords, warn): self.assertIn(mock.call(msg.format(coord)), warn.call_args_list) def _assert_nowarn_collapse_without_weight(self, coords, warn): - # Ensure that warning is not rised. + # Ensure that warning is not raised. msg = "Collapsing spatial coordinate {!r} without weighting" for coord in coords: self.assertNotIn(mock.call(msg.format(coord)), warn.call_args_list) @@ -758,7 +758,7 @@ def _assert_warn_cannot_check_contiguity(self, warn): self.assertIn(mock.call(msg), warn.call_args_list) def _assert_cube_as_expected(self, cube): - """Ensure that cube data and coordiantes are as expected.""" + """Ensure that cube data and coordinates are as expected.""" self.assertArrayEqual(cube.data, np.array(3)) lat = cube.coord("latitude") @@ -1096,7 +1096,7 @@ def setUp(self): len(self.cube.coord("model_level_number").points) ) self.exp_iter_2d = np.ndindex(6, 70, 1, 1) - # Define maximum number of interations for particularly long + # Define maximum number of interactions for particularly long # (and so time-consuming) iterators. self.long_iterator_max = 5 From af9aaab56582afaa5c2efefae3f92bd43faac3be Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 6 Mar 2023 15:52:45 +0100 Subject: [PATCH 17/25] Added test for _sum_units_func --- lib/iris/tests/test_analysis.py | 39 +++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/lib/iris/tests/test_analysis.py b/lib/iris/tests/test_analysis.py index ab265cb965..0717368d98 100644 --- a/lib/iris/tests/test_analysis.py +++ b/lib/iris/tests/test_analysis.py @@ -1703,7 +1703,7 @@ def test_weights_in_kwargs(self): self.assertEqual(kwargs, {"test_kwarg": "test", "weights": "ignored"}) -class TestWeights(tests.IrisTest): +class TestWeights: @pytest.fixture(autouse=True) def setup_test_data(self): self.lat = iris.coords.DimCoord( @@ -1888,21 +1888,12 @@ def test_init_with_ndarray_and_units(self): assert weights.units == "J" def test_init_with_invalid_obj(self): - self.assertRaises( - KeyError, - iris.analysis._Weights, - "invalid_obj", - self.cube, - ) + with pytest.raises(KeyError): + iris.analysis._Weights("invalid_obj", self.cube) def test_init_with_invalid_obj_and_units(self): - self.assertRaises( - KeyError, - iris.analysis._Weights, - "invalid_obj", - self.cube, - units="J", - ) + with pytest.raises(KeyError): + iris.analysis._Weights("invalid_obj", self.cube, units="J") def test_update_kwargs_no_weights(self): kwargs = {"test": [1, 2, 3]} @@ -1924,5 +1915,25 @@ def test_update_kwargs_weights(self): assert kwargs["weights"].units == "1" +CUBE = iris.cube.Cube(0) + + +@pytest.mark.parametrize( + "kwargs,expected", + [ + ({}, "s"), + ({"test": "m"}, "s"), + ({"weights": None}, "s"), + ({"weights": [1, 2, 3]}, "s"), + ({"weights": iris.analysis._Weights([1], CUBE)}, "s"), + ({"weights": iris.analysis._Weights([1], CUBE, units="kg")}, "s kg"), + ], +) +def test_sum_units_func(kwargs, expected): + units = cf_units.Unit("s") + result = iris.analysis._sum_units_func(units, **kwargs) + assert result == expected + + if __name__ == "__main__": tests.main() From 69a854f3bb7c463cc9b647d81dcbd6294e038c3e Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 6 Mar 2023 16:22:34 +0100 Subject: [PATCH 18/25] Ensure backwards-compatibility of Aggregator.units_func --- lib/iris/analysis/__init__.py | 12 +++++-- .../tests/unit/analysis/test_Aggregator.py | 34 +++++++++++++++---- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index 7b305aa0e4..cccb01fdbe 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -39,6 +39,7 @@ from collections.abc import Iterable import functools from functools import wraps +from inspect import getfullargspec import warnings from cf_units import Unit @@ -475,6 +476,8 @@ def __init__( If provided, called to convert a cube's units. Returns an :class:`cf_units.Unit`, or a value that can be made into one. + To ensure backwards-compatibility, also accepts a callable with + call signature (units). * lazy_func (callable or None): An alternative to :data:`call_func` implementing a lazy @@ -482,7 +485,8 @@ def __init__( main operation, but should raise an error in unhandled cases. Additional kwargs:: - Passed through to :data:`call_func` and :data:`lazy_func`. + Passed through to :data:`call_func`, :data:`lazy_func`, and + :data:`units_func`. Aggregators are used by cube aggregation methods such as :meth:`~iris.cube.Cube.collapsed` and @@ -628,7 +632,11 @@ def update_metadata(self, cube, coords, **kwargs): """ # Update the units if required. if self.units_func is not None: - cube.units = self.units_func(cube.units, **kwargs) + argspec = getfullargspec(self.units_func) + if argspec.varkw is None: # old style + cube.units = self.units_func(cube.units) + else: # new style (preferred) + cube.units = self.units_func(cube.units, **kwargs) def post_process(self, collapsed_cube, data_result, coords, **kwargs): """ diff --git a/lib/iris/tests/unit/analysis/test_Aggregator.py b/lib/iris/tests/unit/analysis/test_Aggregator.py index ec837ea49a..a51c51bc23 100644 --- a/lib/iris/tests/unit/analysis/test_Aggregator.py +++ b/lib/iris/tests/unit/analysis/test_Aggregator.py @@ -5,18 +5,20 @@ # licensing details. """Unit tests for the :class:`iris.analysis.Aggregator` class instance.""" -# Import iris.tests first so that some things can be initialised before -# importing anything else. -import iris.tests as tests # isort:skip - from unittest import mock import numpy as np import numpy.ma as ma from iris.analysis import Aggregator + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +from iris.cube import Cube from iris.exceptions import LazyAggregatorError +import iris.tests as tests # isort:skip + class Test_aggregate(tests.IrisTest): # These unit tests don't call a data aggregation function, they call a @@ -286,10 +288,30 @@ def test_units_change(self): units_func = mock.Mock(return_value=mock.sentinel.new_units) aggregator = Aggregator("", None, units_func) cube = mock.Mock(units=mock.sentinel.units) - aggregator.update_metadata(cube, []) - units_func.assert_called_once_with(mock.sentinel.units) + aggregator.update_metadata(cube, [], kw1=1, kw2=2) + units_func.assert_called_once_with(mock.sentinel.units, kw1=1, kw2=2) self.assertEqual(cube.units, mock.sentinel.new_units) + def test_units_func_no_kwargs(self): + # To ensure backwards-compatibility, Aggregator also supports + # units_func that accept the single argument `units` + def units_func(units): + return units**2 + + aggregator = Aggregator("", None, units_func) + cube = Cube(0, units="s") + aggregator.update_metadata(cube, [], kw1=1, kw2=2) + self.assertEqual(cube.units, "s2") + + def test_units_func_kwargs(self): + def units_func(units, **kwargs): + return units**2 + + aggregator = Aggregator("", None, units_func) + cube = Cube(0, units="s") + aggregator.update_metadata(cube, [], kw1=1, kw2=2) + self.assertEqual(cube.units, "s2") + class Test_lazy_aggregate(tests.IrisTest): def test_kwarg_pass_through_no_kwargs(self): From 074108176bafe4a707541610acf40dcbeeacac47 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 6 Mar 2023 16:24:55 +0100 Subject: [PATCH 19/25] Restored import order of test --- lib/iris/tests/unit/analysis/test_Aggregator.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/iris/tests/unit/analysis/test_Aggregator.py b/lib/iris/tests/unit/analysis/test_Aggregator.py index a51c51bc23..45081ad07f 100644 --- a/lib/iris/tests/unit/analysis/test_Aggregator.py +++ b/lib/iris/tests/unit/analysis/test_Aggregator.py @@ -5,20 +5,19 @@ # licensing details. """Unit tests for the :class:`iris.analysis.Aggregator` class instance.""" +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests # isort:skip + from unittest import mock import numpy as np import numpy.ma as ma from iris.analysis import Aggregator - -# Import iris.tests first so that some things can be initialised before -# importing anything else. from iris.cube import Cube from iris.exceptions import LazyAggregatorError -import iris.tests as tests # isort:skip - class Test_aggregate(tests.IrisTest): # These unit tests don't call a data aggregation function, they call a From fca72f83afdefd6abdbd076229d8d6dc014c4a3d Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 6 Mar 2023 16:31:32 +0100 Subject: [PATCH 20/25] Added What's new entry --- docs/src/whatsnew/latest.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 592c713b4d..6303cc827a 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -39,6 +39,10 @@ This document explains the changes made to Iris for this release #. `@rcomer`_ enabled lazy evaluation of :obj:`~iris.analysis.RMS` calcuations with weights. (:pull:`5017`) +#. `@schlunma` allowed the usage of cubes, coordinates, cell measures, or + ancillary variables as weights for cube aggregations. This automatically + changes units if necessary. (:pull:`5084`) + 🐛 Bugs Fixed ============= From 8232c9ddce19e16ab5f81b511cc268f53cf8ef32 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 6 Mar 2023 17:48:21 +0100 Subject: [PATCH 21/25] Split further tests --- lib/iris/tests/test_aggregate_by.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/iris/tests/test_aggregate_by.py b/lib/iris/tests/test_aggregate_by.py index 8f015a19f2..e5614f6b63 100644 --- a/lib/iris/tests/test_aggregate_by.py +++ b/lib/iris/tests/test_aggregate_by.py @@ -413,7 +413,7 @@ def test_single(self): aggregateby_cube.data, self.single_rms_expected ) - def test_single_weights_none(self): + def test_str_aggregation_single_weights_none(self): # mean group-by with single coordinate name. aggregateby_cube = self.cube_single.aggregated_by( "height", iris.analysis.MEAN, weights=None @@ -421,7 +421,11 @@ def test_single_weights_none(self): self.assertCML( aggregateby_cube, ("analysis", "aggregated_by", "single.cml") ) + np.testing.assert_almost_equal( + aggregateby_cube.data, self.single_expected + ) + def test_coord_aggregation_single_weights_none(self): # mean group-by with single coordinate. aggregateby_cube = self.cube_single.aggregated_by( self.coord_z_single, iris.analysis.MEAN, weights=None @@ -429,7 +433,6 @@ def test_single_weights_none(self): self.assertCML( aggregateby_cube, ("analysis", "aggregated_by", "single.cml") ) - np.testing.assert_almost_equal( aggregateby_cube.data, self.single_expected ) From 9d8a87a0abcf395dc3a8d0b4c8701abc9f966990 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 6 Mar 2023 17:48:39 +0100 Subject: [PATCH 22/25] Optimized What's new entry --- docs/src/whatsnew/latest.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 6303cc827a..b34816e616 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -39,9 +39,11 @@ This document explains the changes made to Iris for this release #. `@rcomer`_ enabled lazy evaluation of :obj:`~iris.analysis.RMS` calcuations with weights. (:pull:`5017`) -#. `@schlunma` allowed the usage of cubes, coordinates, cell measures, or - ancillary variables as weights for cube aggregations. This automatically - changes units if necessary. (:pull:`5084`) +#. `@schlunma`_ allowed the usage of cubes, coordinates, cell measures, or + ancillary variables as weights for cube aggregations + (:meth:`iris.cube.Cube.collapsed`, :meth:`iris.cube.Cube.aggregated_by`, and + :meth:`iris.cube.Cube.rolling_window`). This automatically adapts cube units + if necessary. (:pull:`5084`) 🐛 Bugs Fixed From d1b9c561db47a8eebad2a4932a52d97864c4a6d3 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Fri, 10 Mar 2023 13:52:54 +0100 Subject: [PATCH 23/25] Apply suggestions from code review Co-authored-by: lbdreyer --- lib/iris/tests/unit/cube/test_Cube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/iris/tests/unit/cube/test_Cube.py b/lib/iris/tests/unit/cube/test_Cube.py index 93b49048d9..aa9e3b51b1 100644 --- a/lib/iris/tests/unit/cube/test_Cube.py +++ b/lib/iris/tests/unit/cube/test_Cube.py @@ -493,7 +493,7 @@ def test_weighted_sum_1dweights_adapt_units_real_y(self): ) self.assertEqual(cube_collapsed.units, "m") - def test_weighted_sum_with_unkown_units_real_y(self): + def test_weighted_sum_with_unknown_units_real_y(self): # Check that units are adapted correctly ('unknown' * '1' = 'unknown') # Note: does not need to be adapted in subclasses since 'unknown' # multiplied by any unit is 'unknown' @@ -505,7 +505,7 @@ def test_weighted_sum_with_unkown_units_real_y(self): ) self.assertEqual(cube_collapsed.units, "unknown") - def test_weighted_sum_with_unkown_units_lazy_y(self): + def test_weighted_sum_with_unknown_units_lazy_y(self): # Check that units are adapted correctly ('unknown' * '1' = 'unknown') # Note: does not need to be adapted in subclasses since 'unknown' # multiplied by any unit is 'unknown' From 17885c42f951a2f884b7d7f0901f17d993579d2f Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 10 Mar 2023 14:03:19 +0100 Subject: [PATCH 24/25] obj cannot be None in _Weights.__array_finalize__ --- lib/iris/analysis/__init__.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index cccb01fdbe..b40ea7e421 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1258,9 +1258,15 @@ def __new__(cls, weights, cube, units=None): return obj def __array_finalize__(self, obj): - """See https://numpy.org/doc/stable/user/basics.subclassing.html.""" - if obj is None: - return + """See https://numpy.org/doc/stable/user/basics.subclassing.html. + + Note + ---- + `obj` cannot be `None` here since ``_Weights.__new__`` does not call + ``super().__new__`` explicitly. + + """ + self.units = getattr(obj, "units", Unit("1")) @classmethod From ab0a20fd015f2d1c64aebcb3b98efaa4e4f52da0 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 10 Mar 2023 14:04:43 +0100 Subject: [PATCH 25/25] Removed whitespace --- lib/iris/analysis/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/iris/analysis/__init__.py b/lib/iris/analysis/__init__.py index b40ea7e421..173487cfb0 100644 --- a/lib/iris/analysis/__init__.py +++ b/lib/iris/analysis/__init__.py @@ -1266,7 +1266,6 @@ def __array_finalize__(self, obj): ``super().__new__`` explicitly. """ - self.units = getattr(obj, "units", Unit("1")) @classmethod