From ad90a12fd56dc426b06441591f4f4dabaa14a7cf Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 18 Nov 2022 18:52:58 +0100 Subject: [PATCH 1/8] Improved equalizing coords for mmm preproc --- esmvalcore/preprocessor/_multimodel.py | 106 ++++++++++++++++-- .../_multimodel/test_multimodel.py | 30 ++++- 2 files changed, 119 insertions(+), 17 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 1b994ecbad..9cc0f3f6b0 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -17,6 +17,8 @@ import iris import iris.coord_categorisation import numpy as np +from iris.common.metadata import CoordMetadata +from iris.coords import DimCoord from iris.util import equalise_attributes from esmvalcore.iris_helpers import date2num @@ -230,13 +232,8 @@ def _equalise_cell_methods(cubes): cube.cell_methods = None -def _equalise_coordinates(cubes): - """Equalise coordinates in cubes (in-place).""" - if not cubes: - return - - # If metadata of a coordinate metadata is equal for all cubes, do not - # modify it; else remove long_name and attributes. +def _get_equal_coords_metadata(cubes): + """Get metadata for exactly matching coordinates across cubes.""" equal_coords_metadata = [] for coord in cubes[0].coords(): for other_cube in cubes[1:]: @@ -248,13 +245,100 @@ def _equalise_coordinates(cubes): break else: equal_coords_metadata.append(coord.metadata) + return equal_coords_metadata + + +def _get_equal_coord_names_metadata(cubes, equal_coords_metadata): + """Get metadata for coords with matching names and units across cubes. + + Note + ---- + Ignore coordinates whose names are not unique. + + """ + equal_names_metadata = {} + for coord in cubes[0].coords(): + coord_name = coord.name() + + # Ignore exactly matching coordinates + if coord.metadata in equal_coords_metadata: + continue + + # Ignore coordinates that are not unique in original cube + if len(cubes[0].coords(coord_name)) > 1: + continue + + # Check if coordinate names and units match across all cubes + for other_cube in cubes[1:]: + + # Ignore names that do not exist in other cube/are not unique + if len(other_cube.coords(coord_name)) != 1: + break + + # Ignore names where units do not match across cubes + if coord.units != other_cube.coord(coord_name).units: + break - # Modify coordinates accordingly + # Coordinate name exists in all other cubes with identical units + # --> Filter out matching metadata + else: # Coordinate name exists in all other cubes with identical units + std_names = list( + {c.coord(coord_name).standard_name for c in cubes} + ) + long_names = list( + {c.coord(coord_name).long_name for c in cubes} + ) + var_names = list( + {c.coord(coord_name).var_name for c in cubes} + ) + equal_names_metadata[coord_name] = CoordMetadata( + standard_name=std_names[0] if len(std_names) == 1 else None, + long_name=long_names[0] if len(long_names) == 1 else None, + var_name=var_names[0] if len(var_names) == 1 else None, + units=coord.units, + attributes={}, + coord_system=None, + climatological=None, + ) + + return equal_names_metadata + + +def _equalise_coordinates(cubes): + """Equalise coordinates in cubes (in-place).""" + if not cubes: + return + + # Filter out coordinates with exactly matching metadata across all cubes + # --> these will not be modified at all + equal_coords_metadata = _get_equal_coords_metadata(cubes) + + # Filter out coordinates with matching names and units + # --> keep matching metadata of these coordinates + # Note: ignore duplicate coordinates + equal_names_metadata = _get_equal_coord_names_metadata( + cubes, + equal_coords_metadata + ) + + # Modify all coordinates of all cubes accordingly for cube in cubes: for coord in cube.coords(): - if coord.metadata not in equal_coords_metadata: - coord.long_name = None - coord.attributes = None + + # Exactly matching coordinates --> do not modify + if coord.metadata in equal_coords_metadata: + continue + + # Matching names and units --> set common metadata + if coord.name() in equal_names_metadata: + coord.metadata = equal_names_metadata[coord.name()] + continue + + # Remaining coordinates --> remove coordinate metadata + coord.long_name = None + coord.attributes = {} + if isinstance(coord, DimCoord): + coord.circular = None # Additionally remove specific scalar coordinates which are not # expected to be equal in the input cubes diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 00ced5a348..0b08cdddf8 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -134,19 +134,27 @@ def get_cube_for_equal_coords_test(num_cubes): cube = generate_cube_from_dates('monthly') cubes.append(cube) - # Create cubes that have one equal coordinate ('year') and one non-equal - # coordinate ('x') + # Create cubes that have one exactly equal coordinate ('year'), one + # coordinate with matching names ('m') and one coordinate with non-matching + # names year_coord = AuxCoord([1, 2, 3], var_name='year', long_name='year', units='1', attributes={'test': 1}) + m_coord = AuxCoord([1, 2, 3], var_name='m', long_name='m', units='s', + attributes={'test': 0}) x_coord = AuxCoord([1, 2, 3], var_name='x', long_name='x', units='s', attributes={'test': 2}) for (idx, cube) in enumerate(cubes): + new_m_coord = m_coord.copy() + new_m_coord.var_name = f'm_{idx}' new_x_coord = x_coord.copy() new_x_coord.long_name = f'x_{idx}' cube.add_aux_coord(year_coord.copy(), 0) + cube.add_aux_coord(new_m_coord, 0) cube.add_aux_coord(new_x_coord, 0) assert cube.coord('year').metadata is not year_coord.metadata assert cube.coord('year').metadata == year_coord.metadata + assert cube.coord('m').metadata is not m_coord.metadata + assert cube.coord('m').metadata != m_coord.metadata assert cube.coord(f'x_{idx}').metadata is not x_coord.metadata assert cube.coord(f'x_{idx}').metadata != x_coord.metadata @@ -482,11 +490,16 @@ def test_combine_preserve_equal_coordinates(): cubes = get_cube_for_equal_coords_test(5) merged_cube = mm._combine(cubes) - # The equal coordinate ('year') was not changed; the non-equal one ('x') - # does not have a long_name and attributes anymore + # The equal coordinate ('year') was not changed; the var_name of the + # matchine name coordinate ('m') has been removed, and the non-equal one + # ('x') does not have a long_name and attributes anymore assert merged_cube.coord('year').var_name == 'year' assert merged_cube.coord('year').standard_name is None assert merged_cube.coord('year').long_name == 'year' + assert merged_cube.coord('m').var_name is None + assert merged_cube.coord('m').standard_name is None + assert merged_cube.coord('m').long_name == 'm' + assert merged_cube.coord('m').attributes == {} assert merged_cube.coord('year').attributes == {'test': 1} assert merged_cube.coord('x').var_name == 'x' assert merged_cube.coord('x').standard_name is None @@ -911,12 +924,17 @@ def test_preserve_equal_coordinates(): stat_cube = stat_cubes['sum'] assert_array_allclose(stat_cube.data, np.ma.array([5.0, 5.0, 5.0])) - # The equal coordinate ('year') was not changed; the non-equal one ('x') - # does not have a long_name and attributes anymore + # The equal coordinate ('year') was not changed; the var_name of the + # matchine name coordinate ('m') has been removed, and the non-equal one + # ('x') does not have a long_name and attributes anymore assert stat_cube.coord('year').var_name == 'year' assert stat_cube.coord('year').standard_name is None assert stat_cube.coord('year').long_name == 'year' assert stat_cube.coord('year').attributes == {'test': 1} + assert stat_cube.coord('m').var_name is None + assert stat_cube.coord('m').standard_name is None + assert stat_cube.coord('m').long_name == 'm' + assert stat_cube.coord('m').attributes == {} assert stat_cube.coord('x').var_name == 'x' assert stat_cube.coord('x').standard_name is None assert stat_cube.coord('x').long_name is None From c5e53967df9e04086de4ae7a644a7bd61baf1065 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 21 Nov 2022 12:43:30 +0100 Subject: [PATCH 2/8] Added tests --- esmvalcore/preprocessor/_multimodel.py | 8 +- .../_multimodel/test_multimodel.py | 168 ++++++++++++++++-- 2 files changed, 158 insertions(+), 18 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 9cc0f3f6b0..aeb6859721 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -280,8 +280,8 @@ def _get_equal_coord_names_metadata(cubes, equal_coords_metadata): break # Coordinate name exists in all other cubes with identical units - # --> Filter out matching metadata - else: # Coordinate name exists in all other cubes with identical units + # --> Get metadata that is identical across all cubes + else: std_names = list( {c.coord(coord_name).standard_name for c in cubes} ) @@ -298,10 +298,10 @@ def _get_equal_coord_names_metadata(cubes, equal_coords_metadata): units=coord.units, attributes={}, coord_system=None, - climatological=None, + climatological=False, ) - return equal_names_metadata + return equal_names_metadata def _equalise_coordinates(cubes): diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 0b08cdddf8..d3eb40d374 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -11,7 +11,7 @@ import pytest from cf_units import Unit from iris.coords import AuxCoord -from iris.cube import Cube +from iris.cube import Cube, CubeList import esmvalcore.preprocessor._multimodel as mm from esmvalcore.iris_helpers import date2num @@ -485,22 +485,29 @@ def test_combine_with_scalar_coords_to_remove(scalar_coord): assert merged_cube.shape == (5, 3) -def test_combine_preserve_equal_coordinates(): +def test_combine_equal_coordinates(): """Test ``_combine`` with equal input coordinates.""" cubes = get_cube_for_equal_coords_test(5) merged_cube = mm._combine(cubes) - # The equal coordinate ('year') was not changed; the var_name of the - # matchine name coordinate ('m') has been removed, and the non-equal one - # ('x') does not have a long_name and attributes anymore + # The equal coordinate ('year') was not changed assert merged_cube.coord('year').var_name == 'year' assert merged_cube.coord('year').standard_name is None assert merged_cube.coord('year').long_name == 'year' + assert merged_cube.coord('year').attributes == {'test': 1} + + +def test_combine_non_equal_coordinates(): + """Test ``_combine`` with non-equal input coordinates.""" + cubes = get_cube_for_equal_coords_test(5) + merged_cube = mm._combine(cubes) + + # The var_name of the matchine name coordinate ('m') has been removed, and + # the non-equal one ('x') does not have a long_name and attributes anymore assert merged_cube.coord('m').var_name is None assert merged_cube.coord('m').standard_name is None assert merged_cube.coord('m').long_name == 'm' assert merged_cube.coord('m').attributes == {} - assert merged_cube.coord('year').attributes == {'test': 1} assert merged_cube.coord('x').var_name == 'x' assert merged_cube.coord('x').standard_name is None assert merged_cube.coord('x').long_name is None @@ -920,22 +927,155 @@ def test_preserve_equal_coordinates(): statistics=['sum']) assert len(stat_cubes) == 1 - assert 'sum' in stat_cubes stat_cube = stat_cubes['sum'] assert_array_allclose(stat_cube.data, np.ma.array([5.0, 5.0, 5.0])) - # The equal coordinate ('year') was not changed; the var_name of the - # matchine name coordinate ('m') has been removed, and the non-equal one - # ('x') does not have a long_name and attributes anymore + # The equal coordinate ('year') was not changed assert stat_cube.coord('year').var_name == 'year' assert stat_cube.coord('year').standard_name is None assert stat_cube.coord('year').long_name == 'year' assert stat_cube.coord('year').attributes == {'test': 1} - assert stat_cube.coord('m').var_name is None - assert stat_cube.coord('m').standard_name is None - assert stat_cube.coord('m').long_name == 'm' - assert stat_cube.coord('m').attributes == {} + + +def test_preserve_non_equal_coordinates(): + """Test ``multi_model_statistics`` with non_equal input coordinates.""" + cubes = get_cube_for_equal_coords_test(5) + stat_cubes = multi_model_statistics(cubes, span='overlap', + statistics=['sum']) + + assert len(stat_cubes) == 1 + stat_cube = stat_cubes['sum'] + assert_array_allclose(stat_cube.data, np.ma.array([5.0, 5.0, 5.0])) + + # The long_name and attributes of the non-equal coordinate ('x') have been + # removed assert stat_cube.coord('x').var_name == 'x' assert stat_cube.coord('x').standard_name is None assert stat_cube.coord('x').long_name is None assert stat_cube.coord('x').attributes == {} + + +@pytest.mark.parametrize( + 'equal_names', + [ + ['var_name'], + ['standard_name'], + ['long_name'], + ['var_name', 'standard_name'], + ['var_name', 'long_name'], + ['standard_name', 'long_name'], + ['var_name', 'standard_name', 'long_name'], + ] +) +def test_preserve_equal_name_coordinates(equal_names): + """Test ``multi_model_statistics`` with equal-name coordinates.""" + all_names = ['var_name', 'standard_name', 'long_name'] + cubes = CubeList(generate_cube_from_dates('monthly') for _ in range(5)) + + # Prepare names of coordinates of input cubes accordingly + for (idx, cube) in enumerate(cubes): + time_coord = cube.coord('time') + for name in all_names: + if name in equal_names or idx != 0: + setattr(time_coord, name, 'time') + else: # Different value for first cube if non-equal name + setattr(time_coord, name, None) + + # Use different coordinate attributes for each cube so the different + # coordinates are not exactly identical + time_coord.attributes = {'test': idx} + + stat_cubes = multi_model_statistics(cubes, span='overlap', + statistics=['sum']) + + assert len(stat_cubes) == 1 + stat_cube = stat_cubes['sum'] + assert_array_allclose(stat_cube.data, np.ma.array([5.0, 5.0, 5.0])) + + assert len(stat_cube.coords()) == 1 + time_coord = stat_cube.coords()[0] + + for name in all_names: + if name in equal_names: + assert getattr(time_coord, name) == 'time' + else: + assert getattr(time_coord, name) is None + assert time_coord.units == 'days since 1850-01-01' + assert time_coord.attributes == {} + assert time_coord.coord_system is None + assert time_coord.climatological is False + + +def test_ignore_equal_coordinates(): + """Test ``_get_equal_coord_names_metadata``.""" + cubes = CubeList(generate_cube_from_dates('monthly') for _ in range(5)) + + equal_coords_metadata = [cubes[0].coord('time').metadata] + equal_names_metadata = mm._get_equal_coord_names_metadata( + cubes, + equal_coords_metadata, + ) + + # The equal_names_metadata dict should be empty since the exactly identical + # coordinate should be ignored + assert not equal_names_metadata + + +@pytest.mark.parametrize('cube_idx', [0, 1, 2, 3, 4]) +def test_ignore_duplicate_equal_name_coordinates(cube_idx): + """Test ``_get_equal_coord_names_metadata``.""" + cubes = CubeList(generate_cube_from_dates('monthly') for _ in range(5)) + + # Add duplicate scalar coordinate + d_coord_0 = AuxCoord( + 0.0, + var_name='d', + long_name='d', + units='m', + attributes={'test': 1} + ) + d_coord_1 = AuxCoord( + 1.0, + var_name='d', + long_name='d', + units='m', + ) + for cube in cubes: + cube.add_aux_coord(d_coord_1, ()) + cubes[cube_idx].add_aux_coord(d_coord_0, ()) + + equal_names_metadata = mm._get_equal_coord_names_metadata(cubes, []) + + # The equal_names_metadata dict should only contain the equal 'time' + # dimension, not the duplicate dimension + assert len(equal_names_metadata) == 1 + assert 'time' in equal_names_metadata + + +def test_ignore_non_existing_coordinates(): + """Test ``_get_equal_coord_names_metadata``.""" + cubes = CubeList(generate_cube_from_dates('monthly') for _ in range(5)) + + # Add coordinate only for first cube + cubes[0].add_aux_coord(AuxCoord(0.0, long_name='x'), ()) + + equal_names_metadata = mm._get_equal_coord_names_metadata(cubes, []) + + # The equal_names_metadata dict should only contain the equal 'time' + # dimension, not the duplicate dimension + assert len(equal_names_metadata) == 1 + assert 'time' in equal_names_metadata + + +def test_ignore_coordinates_different_units(): + """Test ``_get_equal_coord_names_metadata``.""" + cubes = CubeList(generate_cube_from_dates('monthly') for _ in range(5)) + + # Adapt time units of one cube + cubes[3].coord('time').units = 'days since 1900-01-01' + + equal_names_metadata = mm._get_equal_coord_names_metadata(cubes, []) + + # The equal_names_metadata dict should be empty since the time units do not + # match + assert not equal_names_metadata From 1598db2c0acb1b0e0104853c8dd2f441f600c729 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 21 Nov 2022 13:07:07 +0100 Subject: [PATCH 3/8] Only adapt names for equal-name coordinates --- esmvalcore/preprocessor/_multimodel.py | 30 ++++++++++--------- .../_multimodel/test_multimodel.py | 20 ++++++------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index aeb6859721..024ac76f12 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -17,7 +17,6 @@ import iris import iris.coord_categorisation import numpy as np -from iris.common.metadata import CoordMetadata from iris.coords import DimCoord from iris.util import equalise_attributes @@ -291,20 +290,16 @@ def _get_equal_coord_names_metadata(cubes, equal_coords_metadata): var_names = list( {c.coord(coord_name).var_name for c in cubes} ) - equal_names_metadata[coord_name] = CoordMetadata( + equal_names_metadata[coord_name] = dict( standard_name=std_names[0] if len(std_names) == 1 else None, long_name=long_names[0] if len(long_names) == 1 else None, var_name=var_names[0] if len(var_names) == 1 else None, - units=coord.units, - attributes={}, - coord_system=None, - climatological=False, ) return equal_names_metadata -def _equalise_coordinates(cubes): +def _equalise_coordinate_metadata(cubes): """Equalise coordinates in cubes (in-place).""" if not cubes: return @@ -329,16 +324,23 @@ def _equalise_coordinates(cubes): if coord.metadata in equal_coords_metadata: continue - # Matching names and units --> set common metadata + # Non-exactly matching coordinates --> first, delete attributes and + # circular property + coord.attributes = {} + if isinstance(coord, DimCoord): + coord.circular = False + + # Matching names and units --> set common names if coord.name() in equal_names_metadata: - coord.metadata = equal_names_metadata[coord.name()] + equal_names = equal_names_metadata[coord.name()] + coord.standard_name = equal_names['standard_name'] + coord.long_name = equal_names['long_name'] + coord.var_name = equal_names['var_name'] continue - # Remaining coordinates --> remove coordinate metadata + # Remaining coordinates --> remove long_name + # Note: remaining differences will raise an error at a later stage coord.long_name = None - coord.attributes = {} - if isinstance(coord, DimCoord): - coord.circular = None # Additionally remove specific scalar coordinates which are not # expected to be equal in the input cubes @@ -365,7 +367,7 @@ def _combine(cubes): # merge_and_concat.html#common-issues-with-merge-and-concatenate equalise_attributes(cubes) _equalise_cell_methods(cubes) - _equalise_coordinates(cubes) + _equalise_coordinate_metadata(cubes) _equalise_fx_variables(cubes) for i, cube in enumerate(cubes): diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index d3eb40d374..cf96c1ca0a 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -514,16 +514,16 @@ def test_combine_non_equal_coordinates(): assert merged_cube.coord('x').attributes == {} -def test_equalise_coordinates_no_cubes(): - """Test that _equalise_coordinates doesn't fail with empty cubes.""" - mm._equalise_coordinates([]) +def test_equalise_coordinate_metadata_no_cubes(): + """Test _equalise_coordinate_metadata doesn't fail with empty cubes.""" + mm._equalise_coordinate_metadata([]) -def test_equalise_coordinates_one_cube(): - """Test that _equalise_coordinates doesn't fail with a single cubes.""" +def test_equalise_coordinate_metadata_one_cube(): + """Test _equalise_coordinate_metadata doesn't fail with a single cubes.""" cube = generate_cube_from_dates('monthly') new_cube = cube.copy() - mm._equalise_coordinates([new_cube]) + mm._equalise_coordinate_metadata([new_cube]) assert new_cube is not cube assert new_cube == cube @@ -1002,8 +1002,6 @@ def test_preserve_equal_name_coordinates(equal_names): assert getattr(time_coord, name) is None assert time_coord.units == 'days since 1850-01-01' assert time_coord.attributes == {} - assert time_coord.coord_system is None - assert time_coord.climatological is False def test_ignore_equal_coordinates(): @@ -1041,8 +1039,8 @@ def test_ignore_duplicate_equal_name_coordinates(cube_idx): units='m', ) for cube in cubes: - cube.add_aux_coord(d_coord_1, ()) - cubes[cube_idx].add_aux_coord(d_coord_0, ()) + cube.add_aux_coord(d_coord_0, ()) + cubes[cube_idx].add_aux_coord(d_coord_1, ()) equal_names_metadata = mm._get_equal_coord_names_metadata(cubes, []) @@ -1062,7 +1060,7 @@ def test_ignore_non_existing_coordinates(): equal_names_metadata = mm._get_equal_coord_names_metadata(cubes, []) # The equal_names_metadata dict should only contain the equal 'time' - # dimension, not the duplicate dimension + # dimension, not the coordinate that only exists for the first cube assert len(equal_names_metadata) == 1 assert 'time' in equal_names_metadata From 057b2068ce5aeb4a1aefae7a7062180f120ef989 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 21 Nov 2022 13:27:33 +0100 Subject: [PATCH 4/8] Fixed typos --- esmvalcore/preprocessor/_multimodel.py | 4 ++-- tests/unit/preprocessor/_multimodel/test_multimodel.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 024ac76f12..206c22e9c0 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -309,8 +309,8 @@ def _equalise_coordinate_metadata(cubes): equal_coords_metadata = _get_equal_coords_metadata(cubes) # Filter out coordinates with matching names and units - # --> keep matching metadata of these coordinates - # Note: ignore duplicate coordinates + # --> keep matching names of these coordinates + # Note: ignores duplicate coordinates equal_names_metadata = _get_equal_coord_names_metadata( cubes, equal_coords_metadata diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index cf96c1ca0a..325b02e9b0 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -502,8 +502,9 @@ def test_combine_non_equal_coordinates(): cubes = get_cube_for_equal_coords_test(5) merged_cube = mm._combine(cubes) - # The var_name of the matchine name coordinate ('m') has been removed, and - # the non-equal one ('x') does not have a long_name and attributes anymore + # The var_name of the matching name coordinate ('m') has been removed, and + # the non-equal one ('x') does not have a long_name anymore + # Both coordinates lost their attributes assert merged_cube.coord('m').var_name is None assert merged_cube.coord('m').standard_name is None assert merged_cube.coord('m').long_name == 'm' From 32a511b874806e5b13f54f6cd3ea0c7a45d347e3 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 29 Nov 2022 18:07:28 +0100 Subject: [PATCH 5/8] Fixed tests --- .../_multimodel/test_multimodel.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 69e6990608..840670be54 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -966,9 +966,10 @@ def test_map_to_new_time_int_coords(): assert out_cube.coord('year').bounds is None assert out_cube.coord('decade').bounds is None assert np.issubdtype(out_cube.coord('year').dtype, np.integer) + assert np.issubdtype(out_cube.coord('decade').dtype, np.integer) - def test_arbitrary_dims_5d(cubes_5d): +def test_arbitrary_dims_5d(cubes_5d): """Test ``multi_model_statistics`` with 5D cubes.""" stat_cubes = multi_model_statistics( cubes_5d, @@ -1041,7 +1042,7 @@ def test_arbitrary_dims_0d(cubes_with_arbitrary_dimensions): assert 'sum' in stat_cubes stat_cube = stat_cubes['sum'] assert stat_cube.shape == () - assert_array_allclose(stat_cube.data, np.ma.array(0.0)) assert np.issubdtype(out_cube.coord('decade').dtype, np.integer) + assert_array_allclose(stat_cube.data, np.ma.array(0.0)) def test_preserve_equal_coordinates(): @@ -1054,7 +1055,7 @@ def test_preserve_equal_coordinates(): stat_cube = stat_cubes['sum'] assert_array_allclose(stat_cube.data, np.ma.array([5.0, 5.0, 5.0])) - # The equal coordinate ('year') was not changed + # The equal coordinate 'year' was not changed assert stat_cube.coord('year').var_name == 'year' assert stat_cube.coord('year').standard_name is None assert stat_cube.coord('year').long_name == 'year' @@ -1064,6 +1065,11 @@ def test_preserve_equal_coordinates(): def test_preserve_non_equal_coordinates(): """Test ``multi_model_statistics`` with non_equal input coordinates.""" cubes = get_cube_for_equal_coords_test(5) + + # Use "circular" attribute for one cube to check that it is set to "False" + # for each cube + cubes[2].coord('time').circular = False + stat_cubes = multi_model_statistics(cubes, span='overlap', statistics=['sum']) @@ -1071,7 +1077,12 @@ def test_preserve_non_equal_coordinates(): stat_cube = stat_cubes['sum'] assert_array_allclose(stat_cube.data, np.ma.array([5.0, 5.0, 5.0])) - # The long_name and attributes of the non-equal coordinate ('x') have been + # The attributes and circular property of the non-equal coordinate 'time' + # (due to differing circular) have been removed + assert stat_cube.coord('time').attributes == {} + assert stat_cube.coord('time').circular is False + + # The long_name and attributes of the non-equal coordinate 'x' have been # removed assert stat_cube.coord('x').var_name == 'x' assert stat_cube.coord('x').standard_name is None From c8dc2363c09f63895f25cb9b3d16794f5daa50cf Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 29 Nov 2022 18:33:01 +0100 Subject: [PATCH 6/8] Added doc --- esmvalcore/preprocessor/_multimodel.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 80ce05e757..3a81a7fcc9 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -587,6 +587,31 @@ def multi_model_statistics(products, supported and can be specified like ``pXX.YY`` (for percentile ``XX.YY``; decimal part optional). + Metadata handling + ----------------- + This function can handle cubes with differing metadata: + + - **:attr:`~iris.cube.Cube.attributes`**: Differing attributes are deleted, + see :func:`iris.util.equalise_attributes`. + - **:meth:`~iris.cube.Cube.cell_methods`**: All cell methods are deleted + prior to combining cubes. + - **:meth:`~iris.cube.Cube.cell_measures`**: All cell measures are deleted + prior to combining cubes, see + :func:`esmvalcore.preprocessor.remove_fx_variables`. + - **:meth:`~iris.cube.Cube.ancillary_variables`**: All ancillary variables + are deleted prior to combining cubes, see + :func:`esmvalcore.preprocessor.remove_fx_variables`. + - **:meth:`~iris.cube.Cube.coords`**: Exactly identical coordinates are + preserved. For coordinates with equal :meth:`~iris.coords.Coord.name` and + :meth:`~iris.coords.Coord.units`, names are equalized, + :attr:`~iris.coords.Coord.attributes` deleted and + :attr:`iris.coords.Coord.circular` is set to ``False``. For all other + coordinates, :attr:`~iris.coords.Coord.long_name` is removed, + :attr:`~iris.coords.Coord.attributes` deleted and + :attr:`iris.coords.Coord.circular` is set to ``False``. Please note that + some special scalar coordinates (ancillary coordinates for derived + coordinates like `p0` and `ptop`) are removed as well. + Notes ----- Some of the operators in :py:mod:`iris.analysis` require additional From ca829451b995825d94791ce6517a5e4376386509 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 29 Nov 2022 18:34:38 +0100 Subject: [PATCH 7/8] optimize doc --- esmvalcore/preprocessor/_multimodel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 3a81a7fcc9..2e781d84d3 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -605,10 +605,10 @@ def multi_model_statistics(products, preserved. For coordinates with equal :meth:`~iris.coords.Coord.name` and :meth:`~iris.coords.Coord.units`, names are equalized, :attr:`~iris.coords.Coord.attributes` deleted and - :attr:`iris.coords.Coord.circular` is set to ``False``. For all other + :attr:`~iris.coords.Coord.circular` is set to ``False``. For all other coordinates, :attr:`~iris.coords.Coord.long_name` is removed, :attr:`~iris.coords.Coord.attributes` deleted and - :attr:`iris.coords.Coord.circular` is set to ``False``. Please note that + :attr:`~iris.coords.Coord.circular` is set to ``False``. Please note that some special scalar coordinates (ancillary coordinates for derived coordinates like `p0` and `ptop`) are removed as well. From 84b7df46685a3f93ddc589732cb4fd5afd50ad1d Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Tue, 29 Nov 2022 19:16:57 +0100 Subject: [PATCH 8/8] Fixed doc --- esmvalcore/preprocessor/_multimodel.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 2e781d84d3..948d1a694e 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -587,30 +587,29 @@ def multi_model_statistics(products, supported and can be specified like ``pXX.YY`` (for percentile ``XX.YY``; decimal part optional). - Metadata handling - ----------------- This function can handle cubes with differing metadata: - - **:attr:`~iris.cube.Cube.attributes`**: Differing attributes are deleted, + - :attr:`~iris.cube.Cube.attributes`: Differing attributes are deleted, see :func:`iris.util.equalise_attributes`. - - **:meth:`~iris.cube.Cube.cell_methods`**: All cell methods are deleted + - :attr:`~iris.cube.Cube.cell_methods`: All cell methods are deleted prior to combining cubes. - - **:meth:`~iris.cube.Cube.cell_measures`**: All cell measures are deleted + - :meth:`~iris.cube.Cube.cell_measures`: All cell measures are deleted prior to combining cubes, see :func:`esmvalcore.preprocessor.remove_fx_variables`. - - **:meth:`~iris.cube.Cube.ancillary_variables`**: All ancillary variables + - :meth:`~iris.cube.Cube.ancillary_variables`: All ancillary variables are deleted prior to combining cubes, see :func:`esmvalcore.preprocessor.remove_fx_variables`. - - **:meth:`~iris.cube.Cube.coords`**: Exactly identical coordinates are + - :meth:`~iris.cube.Cube.coords`: Exactly identical coordinates are preserved. For coordinates with equal :meth:`~iris.coords.Coord.name` and - :meth:`~iris.coords.Coord.units`, names are equalized, + :attr:`~iris.coords.Coord.units`, names are equalized, :attr:`~iris.coords.Coord.attributes` deleted and - :attr:`~iris.coords.Coord.circular` is set to ``False``. For all other + :attr:`~iris.coords.DimCoord.circular` is set to ``False``. For all other coordinates, :attr:`~iris.coords.Coord.long_name` is removed, :attr:`~iris.coords.Coord.attributes` deleted and - :attr:`~iris.coords.Coord.circular` is set to ``False``. Please note that - some special scalar coordinates (ancillary coordinates for derived - coordinates like `p0` and `ptop`) are removed as well. + :attr:`~iris.coords.DimCoord.circular` is set to ``False``. Please note + that some special scalar coordinates which are expected to differe across + cubes(ancillary coordinates for derived coordinates like `p0` and `ptop`) + are removed as well. Notes -----