[WIP] Changes to make tests pass when using conda forge channel

conda-requirements.txt

@@ -4,7 +4,7 @@
 # Mandatory dependencies
 biggus
 cartopy
-matplotlib
+matplotlib<1.9
 netcdf4
 numpy
 pyke

docs/iris/example_code/General/custom_file_loading.py

@@ -158,8 +158,8 @@ def load_NAME_III(filename):
             # Cast the x and y grid positions to floats and convert them to
             # zero based indices (the numbers are 1 based grid positions where
             # 0.5 represents half a grid point.)
-            x = float(vals[0]) - 1.5
-            y = float(vals[1]) - 1.5
+            x = int(float(vals[0]) - 1.5)
+            y = int(float(vals[1]) - 1.5)
 
             # Populate the data arrays (i.e. all columns but the leading 4).
             for i, data_array in enumerate(data_arrays):

docs/iris/src/conf.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2010 - 2016, Met Office
+# (C) British Crown Copyright 2010 - 2017, Met Office
 #
 # This file is part of Iris.
 #
@@ -56,7 +56,7 @@
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = ['sphinx.ext.autodoc',
               'sphinx.ext.coverage',
-              'sphinx.ext.pngmath',
+              'sphinx.ext.imgmath',
               'sphinx.ext.autosummary',
               'sphinx.ext.graphviz',
               'sphinx.ext.intersphinx',

lib/iris/coords.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2010 - 2016, Met Office
+# (C) British Crown Copyright 2010 - 2017, Met Office
 #
 # This file is part of Iris.
 #
@@ -329,7 +329,17 @@ def __common_cmp__(self, other, operator_method):
                     me = min(self.bound)
                 else:
                     me = max(self.bound)
-            result = operator_method(me, other)
+
+            # Work around to handle netcdftime.datetime comparison, which doesn't
+            # return NotImplemented on failure in some versions of the library
+            try:
+                result = operator_method(me, other)
+            except TypeError:
+                rop = {operator.lt: operator.gt,
+                       operator.gt: operator.lt,
+                       operator.le: operator.ge,
+                       operator.ge: operator.le}[operator_method]
+                result = rop(other, me)
 
         return result
 
@@ -492,7 +502,15 @@ def is_full_slice(s):
                         raise IndexError('Cannot index with zero length '
                                          'slice.')
                 if bounds is not None:
-                    bounds = bounds[keys + (Ellipsis, )]
+                    # Bounds will generally have an extra dimension compared
+                    # to points, so add an Ellipsis at the end, unless there
+                    # is already one, as numpy does not support double
+                    # Ellipsis.
+                    if (not isinstance(keys[-1], np.ndarray) and
+                       keys[-1] == Ellipsis):
+                        bounds = bounds[keys]
+                    else:
+                        bounds = bounds[keys + (Ellipsis, )]
 
         new_coord = self.copy(points=points, bounds=bounds)
         return new_coord

lib/iris/fileformats/grib/__init__.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2010 - 2016, Met Office
+# (C) British Crown Copyright 2010 - 2017, Met Office
 #
 # This file is part of Iris.
 #
@@ -239,7 +239,7 @@ def __init__(self, grib_message, grib_fh=None, auto_regularise=True):
             # The byte offset requires to be reset back to the first byte
             # of this message. The file pointer offset is always at the end 
             # of the current message due to the grib-api reading the message.
-            proxy = GribDataProxy(shape, np.zeros(.0).dtype, np.nan,
+            proxy = GribDataProxy(shape, np.zeros(0).dtype, np.nan,
                                   grib_fh.name,
                                   offset - message_length,
                                   auto_regularise)

lib/iris/tests/__init__.py

@@ -295,7 +295,7 @@ def assertCMLApproxData(self, cubes, reference_filename=None, **kwargs):
             if fname[-1].endswith(".cml"):
                 fname[-1] = fname[-1][:-4]
             fname[-1] += '.data.%d.json' % i
-            self.assertCubeDataAlmostEqual(cube, fname, **kwargs)
+            self.assertDataAlmostEqual(cube.data, fname, **kwargs)
         self.assertCML(cubes, reference_filename, checksum=False)
 
     def assertCDL(self, netcdf_filename, reference_filename=None, flags='-h'):
@@ -407,36 +407,36 @@ def assertTextFile(self, source_filename, reference_filename, desc="text file"):
             diff = ''.join(difflib.unified_diff(reference_text, source_text, 'Reference', 'Test result', '', '', 0))
             self.fail("%s does not match reference file: %s\n%s" % (desc, reference_filename, diff))
 
-    def assertCubeDataAlmostEqual(self, cube, reference_filename, **kwargs):
+    def assertDataAlmostEqual(self, data, reference_filename, **kwargs):
         reference_path = self.get_result_path(reference_filename)
         if self._check_reference_file(reference_path):
             kwargs.setdefault('err_msg', 'Reference file %s' % reference_path)
             with open(reference_path, 'r') as reference_file:
                 stats = json.load(reference_file)
-                self.assertEqual(stats.get('shape', []), list(cube.shape))
+                self.assertEqual(stats.get('shape', []), list(data.shape))
                 self.assertEqual(stats.get('masked', False),
-                                       isinstance(cube.data, ma.MaskedArray))
+                                       isinstance(data, ma.MaskedArray))
                 nstats = np.array((stats.get('mean', 0.), stats.get('std', 0.),
                                    stats.get('max', 0.), stats.get('min', 0.)),
                                   dtype=np.float_)
                 if math.isnan(stats.get('mean', 0.)):
-                    self.assertTrue(math.isnan(cube.data.mean()))
+                    self.assertTrue(math.isnan(data.mean()))
                 else:
-                    cube_stats = np.array((cube.data.mean(), cube.data.std(),
-                                           cube.data.max(), cube.data.min()),
+                    data_stats = np.array((data.mean(), data.std(),
+                                           data.max(), data.min()),
                                           dtype=np.float_)
-                    self.assertArrayAllClose(nstats, cube_stats, **kwargs)
+                    self.assertArrayAllClose(nstats, data_stats, **kwargs)
         else:
             self._ensure_folder(reference_path)
             logger.warning('Creating result file: %s', reference_path)
             masked = False
-            if isinstance(cube.data, ma.MaskedArray):
+            if isinstance(data, ma.MaskedArray):
                 masked = True
-            stats = {'mean': np.float_(cube.data.mean()),
-                     'std': np.float_(cube.data.std()),
-                     'max': np.float_(cube.data.max()),
-                     'min': np.float_(cube.data.min()),
-                     'shape': cube.shape, 'masked': masked}
+            stats = {'mean': np.float_(data.mean()),
+                     'std': np.float_(data.std()),
+                     'max': np.float_(data.max()),
+                     'min': np.float_(data.min()),
+                     'shape': data.shape, 'masked': masked}
             with open(reference_path, 'w') as reference_file:
                 reference_file.write(json.dumps(stats))
 

lib/iris/tests/idiff.py

@@ -41,6 +41,7 @@
 import matplotlib.pyplot as plt
 import matplotlib.image as mimg
 import matplotlib.testing.compare as mcompare
+from matplotlib.testing.exceptions import ImageComparisonFailure
 import matplotlib.widgets as mwidget
 import numpy as np
 import requests
@@ -236,7 +237,18 @@ def step_over_diffs(result_dir, action, display=True):
                         # So copy the local file to the exected file to
                         # maintain this helpfulness.
                         shutil.copy(local_fname, expected_fname)
-                mcompare.compare_images(expected_fname, result_fname, tol=0)
+                try:
+                    mcompare.compare_images(expected_fname, result_fname,
+                                            tol=0)
+                except Exception as e:
+                    if isinstance(e,  ValueError) or \
+                            isinstance(e, ImageComparisonFailure):
+                        print('Could not compare {}: {}'.format(result_fname,
+                                                                e.message))
+                        continue
+                    else:
+                        # Propagate the exception, keeping the stack trace
+                        raise
                 diff_fname = os.path.splitext(result_fname)[0] + _POSTFIX_DIFF
                 args = expected_fname, result_fname, diff_fname
                 if display:

lib/iris/tests/integration/plot/test_netcdftime.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2016, Met Office
+# (C) British Crown Copyright 2016 - 2017, Met Office
 #
 # This file is part of Iris.
 #
@@ -26,12 +26,15 @@
 # importing anything else
 import iris.tests as tests
 
+import netcdftime
+import numpy as np
+
+from iris.coords import AuxCoord
+
 from cf_units import Unit
 if tests.NC_TIME_AXIS_AVAILABLE:
     from nc_time_axis import CalendarDateTime
-import numpy as np
 
-from iris.coords import AuxCoord
 
 # Run tests in no graphics mode if matplotlib is not available.
 if tests.MPL_AVAILABLE:
@@ -47,9 +50,12 @@ def test_360_day_calendar(self):
         calendar = '360_day'
         time_unit = Unit('days since 1970-01-01 00:00', calendar=calendar)
         time_coord = AuxCoord(np.arange(n), 'time', units=time_unit)
-        expected_ydata = np.array([CalendarDateTime(time_unit.num2date(point),
-                                                    calendar)
-                                   for point in time_coord.points])
+        times = [time_unit.num2date(point) for point in time_coord.points]
+        times = [netcdftime.datetime(atime.year, atime.month, atime.day,
+                                     atime.hour, atime.minute, atime.second)
+                 for atime in times]
+        expected_ydata = np.array([CalendarDateTime(time, calendar)
+                                   for time in times])
         line1, = iplt.plot(time_coord)
         result_ydata = line1.get_ydata()
         self.assertArrayEqual(expected_ydata, result_ydata)

lib/iris/tests/results/analysis/rotated_pole.0.rlat.json

@@ -0,0 +1 @@
+{"std": 3.7195861803241614, "min": -5.593200206756592, "max": 8.72130012512207, "shape": [93, 75], "masked": false, "mean": 1.6348200228305594}

lib/iris/tests/results/analysis/rotated_pole.0.rlon.json

@@ -0,0 +1 @@
+{"std": 3.05635954577972, "min": 353.052490234375, "max": 365.08099365234375, "shape": [93, 75], "masked": false, "mean": 359.0960595703125}

lib/iris/tests/results/analysis/rotated_pole.0.x.json

@@ -0,0 +1 @@
+{"std": 5.256882991869942, "min": -16.571169372070806, "max": 7.855481068868564, "shape": [93, 75], "masked": false, "mean": -4.045853791096988}

lib/iris/tests/results/analysis/rotated_pole.0.y.json

@@ -0,0 +1 @@
+{"std": 3.711246685838892, "min": 46.42276494752364, "max": 61.22127069532664, "shape": [93, 75], "masked": false, "mean": 54.01432122690471}

lib/iris/tests/results/analysis/rotated_pole.1.rlat.json

@@ -0,0 +1 @@
+{"std": 0.10904959181607428, "min": -1.732200026512146, "max": -1.3676999807357788, "shape": [28, 26], "masked": false, "mean": -1.5499499993664878}

lib/iris/tests/results/analysis/rotated_pole.1.rlon.json

@@ -0,0 +1 @@
+{"std": 0.10125036032780034, "min": 359.8294982910156, "max": 360.1669921875, "shape": [28, 26], "masked": false, "mean": 359.99824993426984}

lib/iris/tests/results/analysis/rotated_pole.1.x.json

@@ -0,0 +1 @@
+{"std": 0.16065754648082262, "min": -2.7716267023533385, "max": -2.233964274513027, "shape": [28, 26], "masked": false, "mean": -2.5027768780735946}

lib/iris/tests/results/analysis/rotated_pole.1.y.json

@@ -0,0 +1 @@
+{"std": 0.10904936809954589, "min": 50.767481894563936, "max": 51.13229974298538, "shape": [28, 26], "masked": false, "mean": 50.94993734826177}

lib/iris/tests/results/analysis/weighted_mean_lat.cml

@@ -21,6 +21,6 @@
         <coord name="lat"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(3,)" state="loaded"/>
+    <data dtype="float32" shape="(3,)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/analysis/weighted_mean_latlon.cml

@@ -22,6 +22,6 @@
         <coord name="lon"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="()" state="loaded"/>
+    <data dtype="float32" shape="()" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/analysis/weighted_mean_lon.cml

@@ -21,6 +21,6 @@
         <coord name="lon"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(3,)" state="loaded"/>
+    <data dtype="float32" shape="(3,)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/latitude_longitude_dual_stage.cml

@@ -93,6 +93,6 @@
         <coord name="grid_longitude"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(6, 70)" state="loaded"/>
+    <data dtype="float32" shape="(6, 70)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/latitude_model_level_number_dual_stage.cml

@@ -55,6 +55,6 @@
         <coord name="model_level_number"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(6, 100)" state="loaded"/>
+    <data dtype="float32" shape="(6, 100)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/latitude_time_dual_stage.cml

@@ -99,6 +99,6 @@
         <coord name="time"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(70, 100)" state="loaded"/>
+    <data dtype="float32" shape="(70, 100)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/longitude_latitude_dual_stage.cml

@@ -93,6 +93,6 @@
         <coord name="grid_latitude"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(6, 70)" state="loaded"/>
+    <data dtype="float32" shape="(6, 70)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/longitude_model_level_number_dual_stage.cml

@@ -55,6 +55,6 @@
         <coord name="model_level_number"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(6, 100)" state="loaded"/>
+    <data dtype="float32" shape="(6, 100)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/longitude_time_dual_stage.cml

@@ -99,6 +99,6 @@
         <coord name="time"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(70, 100)" state="loaded"/>
+    <data dtype="float32" shape="(70, 100)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/model_level_number_latitude_dual_stage.cml

@@ -55,6 +55,6 @@
         <coord name="grid_latitude"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(6, 100)" state="loaded"/>
+    <data dtype="float32" shape="(6, 100)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/model_level_number_longitude_dual_stage.cml

@@ -55,6 +55,6 @@
         <coord name="grid_longitude"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(6, 100)" state="loaded"/>
+    <data dtype="float32" shape="(6, 100)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/model_level_number_time_dual_stage.cml

@@ -61,6 +61,6 @@
         <coord name="time"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(100, 100)" state="loaded"/>
+    <data dtype="float32" shape="(100, 100)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/time_latitude_dual_stage.cml

@@ -99,6 +99,6 @@
         <coord name="grid_latitude"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(70, 100)" state="loaded"/>
+    <data dtype="float32" shape="(70, 100)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/time_longitude_dual_stage.cml

@@ -99,6 +99,6 @@
         <coord name="grid_longitude"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(70, 100)" state="loaded"/>
+    <data dtype="float32" shape="(70, 100)" state="loaded"/>
   </cube>
 </cubes>

lib/iris/tests/results/cube_collapsed/time_model_level_number_dual_stage.cml

@@ -61,6 +61,6 @@
         <coord name="model_level_number"/>
       </cellMethod>
     </cellMethods>
-    <data dtype="float64" shape="(100, 100)" state="loaded"/>
+    <data dtype="float32" shape="(100, 100)" state="loaded"/>
   </cube>
 </cubes>