Python3 binary I/O

docs/iris/example_code/General/custom_file_loading.py

@@ -214,9 +214,12 @@ def NAME_to_cube(filenames, callback):
 
 # Create a format_picker specification of the NAME file format giving it a
 # priority greater than the built in NAME loader.
-_NAME_III_spec = format_picker.FormatSpecification('Name III', format_picker.LeadingLine(),
-                                      lambda line: line.startswith("NAME III"), NAME_to_cube,
-                                      priority=6)
+_NAME_III_spec = format_picker.FormatSpecification(
+    'Name III',
+    format_picker.LeadingLine(),
+    lambda line: line.startswith(b"NAME III"),
+    NAME_to_cube,
+    priority=6)
 
 # Register the NAME loader with iris
 iris.fileformats.FORMAT_AGENT.add_spec(_NAME_III_spec)

lib/iris/fileformats/__init__.py

@@ -76,7 +76,7 @@ def _pp_little_endian(filename, *args, **kwargs):
 # priority to avoid collateral damage from false positives.
 FORMAT_AGENT.add_spec(
     FormatSpecification('GRIB', MagicNumber(100),
-                        lambda header_bytes: 'GRIB' in header_bytes,
+                        lambda header_bytes: b'GRIB' in header_bytes,
                         grib.load_cubes, priority=1))
 
 
@@ -173,7 +173,7 @@ def _pp_little_endian(filename, *args, **kwargs):
 FORMAT_AGENT.add_spec(
     FormatSpecification('NAME III',
                         LeadingLine(),
-                        lambda line: line.lstrip().startswith("NAME III"),
+                        lambda line: line.lstrip().startswith(b"NAME III"),
                         name.load_cubes,
                         priority=5))
 

lib/iris/fileformats/pp.py

@@ -1270,9 +1270,13 @@ def _read_extra_data(self, pp_file, file_reader, extra_len):
             data_len = ia * PP_WORD_DEPTH
 
             if ib == 10:
-                self.field_title = ''.join(struct.unpack_from('>%dc' % data_len, file_reader(data_len))).rstrip('\00')
+                field_title = struct.unpack_from('>%ds' % data_len,
+                                                 file_reader(data_len))
+                self.field_title = field_title[0].rstrip(b'\00').decode()
             elif ib == 11:
-                self.domain_title = ''.join(struct.unpack_from('>%dc' % data_len, file_reader(data_len))).rstrip('\00')
+                domain_title = struct.unpack_from('>%ds' % data_len,
+                                                  file_reader(data_len))
+                self.domain_title = domain_title[0].rstrip(b'\00').decode()
             elif ib in EXTRA_DATA:
                 attr_name = EXTRA_DATA[ib]
                 values = np.fromfile(pp_file, dtype=np.dtype('>f%d' % PP_WORD_DEPTH), count=ia)
@@ -1467,7 +1471,8 @@ def save(self, file_handle):
         for int_code, extra_data in extra_items:
             pp_file.write(struct.pack(">L", int(int_code)))
             if isinstance(extra_data, six.string_types):
-                pp_file.write(struct.pack(">%sc" % len(extra_data), *extra_data))
+                pp_file.write(struct.pack(">%ss" % len(extra_data),
+                              extra_data.encode()))
             else:
                 extra_data = extra_data.astype(np.dtype('>f4'))
                 extra_data.tofile(pp_file)

lib/iris/tests/__init__.py

@@ -423,10 +423,6 @@ def assertString(self, string, reference_filename=None):
             reference_path = self.result_path(None, 'txt')
         else:
             reference_path = get_result_path(reference_filename)
-        # If the test string is a unicode string, encode as
-        # utf-8 before comparison to the reference string.
-        if isinstance(string, unicode):
-            string = string.encode('utf-8')
         self._check_same(string, reference_path,
                          type_comparison_name='Strings')
 
@@ -435,17 +431,17 @@ def assertRepr(self, obj, reference_filename):
 
     def _check_same(self, item, reference_path, type_comparison_name='CML'):
         if self._check_reference_file(reference_path):
-            with open(reference_path, 'r') as reference_fh:
-                reference = ''.join(reference_fh.readlines())
+            with open(reference_path, 'rb') as reference_fh:
+                reference = ''.join(part.decode('utf-8')
+                                    for part in reference_fh.readlines())
             self._assert_str_same(reference, item, reference_path,
                                   type_comparison_name)
         else:
             self._ensure_folder(reference_path)
             logger.warning('Creating result file: %s', reference_path)
-            with open(reference_path, 'w') as reference_fh:
+            with open(reference_path, 'wb') as reference_fh:
                 reference_fh.writelines(
                     part.encode('utf-8')
-                    if isinstance(part, unicode) else part
                     for part in item)
 
     def assertXMLElement(self, obj, reference_filename):

lib/iris/tests/experimental/test_raster.py

@@ -48,7 +48,7 @@ def check_tiff(self, cube, tif_header):
             iris.experimental.raster.export_geotiff(cube, temp_filename)
 
             # Check the metadata is correct.
-            with open(temp_filename) as fh:
+            with open(temp_filename, 'rb') as fh:
                 self.check_tiff_header(fh, ('experimental', 'raster',
                                             tif_header))
 

lib/iris/tests/test_file_save.py

@@ -21,6 +21,7 @@
 
 from __future__ import (absolute_import, division, print_function)
 from six.moves import (filter, input, map, range, zip)  # noqa
+import six
 
 # import iris tests first so that some things can be initialised before importing anything else
 import iris.tests as tests
@@ -135,19 +136,19 @@ def test_filehandle(self):
             save_by_filehandle(self.temp_filename1, self.temp_filename2, self.cube1, dot.save, binary_mode = True)
 
     def test_bytesio(self):
-        bio = io.BytesIO()
+        sio = six.StringIO()
 
         # Save from dot direct
         dot.save(self.cube1, self.temp_filename1)
 
         # Call save on iris
-        iris.save(self.cube1, bio, iris.io.find_saver(self.ext))
+        iris.save(self.cube1, sio, iris.io.find_saver(self.ext))
 
         with open(self.temp_filename1) as infile:
             data = infile.read()
 
         # Compare files
-        self.assertEqual(data, bio.getvalue(),
+        self.assertEquals(data, sio.getvalue(),
                          'Mismatch in data when comparing iris bytesio save '
                          'and dot.save.')
 

lib/iris/tests/test_io_init.py

@@ -97,7 +97,7 @@ def test_format_picker(self):
         # test that each filespec is identified as the expected format
         for (expected_format_name, file_spec) in test_specs:
             test_path = tests.get_data_path(file_spec)
-            with open(test_path, 'r') as test_file:
+            with open(test_path, 'rb') as test_file:
                 a = iff.FORMAT_AGENT.get_spec(test_path, test_file)
                 self.assertEqual(a.name, expected_format_name)
 
@@ -110,8 +110,8 @@ def test_format_picker_nodata(self):
         # specific to WMO bulletin headers
         header_lengths = [21, 80, 41, 42]
         for header_length in header_lengths:
-            binary_string = header_length * '\x00' + 'GRIB' + '\x00' * 100
-            with BytesIO('rw') as bh:
+            binary_string = header_length * b'\x00' + b'GRIB' + b'\x00' * 100
+            with BytesIO(b'rw') as bh:
                 bh.write(binary_string)
                 bh.name = 'fake_file_handle'
                 a = iff.FORMAT_AGENT.get_spec(bh.name, bh)

lib/iris/tests/test_util.py

@@ -21,13 +21,13 @@
 
 from __future__ import (absolute_import, division, print_function)
 from six.moves import (filter, input, map, range, zip)  # noqa
+import six
 
 # import iris tests first so that some things can be initialised before
 # importing anything else
 import iris.tests as tests
 
 import inspect
-import io
 import unittest
 
 import numpy as np
@@ -188,9 +188,9 @@ def test_identical(self):
         test_cube_a = stock.realistic_4d()
         test_cube_b = stock.realistic_4d()
 
-        return_bio = io.BytesIO()
-        iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_bio)
-        return_str = return_bio.getvalue().decode()
+        return_sio = six.StringIO()
+        iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_sio)
+        return_str = return_sio.getvalue()
 
         self.assertString(return_str, 'compatible_cubes.str.txt')
 
@@ -202,9 +202,9 @@ def test_different(self):
         test_cube_a.attributes['Conventions'] = 'CF-1.5'
         test_cube_b.attributes['Conventions'] = 'CF-1.6'
 
-        return_bio = io.BytesIO()
-        iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_bio)
-        return_str = return_bio.getvalue().decode()
+        return_sio = six.StringIO()
+        iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_sio)
+        return_str = return_sio.getvalue()
 
         self.assertString(return_str, 'incompatible_attr.str.txt')
 
@@ -214,9 +214,9 @@ def test_different(self):
 
         test_cube_a.standard_name = "relative_humidity"
 
-        return_bio = io.BytesIO()
-        iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_bio)
-        return_str = return_bio.getvalue().decode()
+        return_sio = six.StringIO()
+        iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_sio)
+        return_str = return_sio.getvalue()
 
         self.assertString(return_str, 'incompatible_name.str.txt')
 
@@ -226,19 +226,19 @@ def test_different(self):
 
         test_cube_a.units = iris.unit.Unit('m')
 
-        return_bio = io.BytesIO()
-        iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_bio)
-        return_str = return_bio.getvalue().decode()
+        return_sio = six.StringIO()
+        iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_sio)
+        return_str = return_sio.getvalue()
 
         self.assertString(return_str, 'incompatible_unit.str.txt')
 
         # test incompatible methods
         test_cube_a = stock.realistic_4d()
         test_cube_b = stock.realistic_4d().collapsed('model_level_number', iris.analysis.MEAN)
 
-        return_bio = io.BytesIO()
-        iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_bio)
-        return_str = return_bio.getvalue().decode()
+        return_sio = six.StringIO()
+        iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_sio)
+        return_str = return_sio.getvalue()
 
         self.assertString(return_str, 'incompatible_meth.str.txt')
 

lib/iris/tests/unit/fileformats/pp/test__field_gen.py

@@ -105,8 +105,7 @@ def test_invalid_header_release(self):
         # Check that an unknown LBREL value just results in a warning
         # and the end of the file iteration instead of raising an error.
         with self.temp_filename() as temp_path:
-            with open(temp_path, 'w') as f:
-                f.write(np.zeros(65, dtype='i4'))
+            np.zeros(65, dtype='i4').tofile(temp_path)
             generator = pp._field_gen(temp_path, False)
             with mock.patch('warnings.warn') as warn:
                 with self.assertRaises(StopIteration):

lib/iris/tests/unit/util/test_describe_diff.py

@@ -18,13 +18,12 @@
 
 from __future__ import (absolute_import, division, print_function)
 from six.moves import (filter, input, map, range, zip)  # noqa
+import six
 
 # import iris tests first so that some things can be initialised before
 # importing anything else
 import iris.tests as tests
 
-import io
-
 import numpy as np
 
 import iris.cube
@@ -37,9 +36,9 @@ def setUp(self):
         self.cube_b = self.cube_a.copy()
 
     def _compare_result(self, cube_a, cube_b):
-        result_bio = io.BytesIO()
-        describe_diff(cube_a, cube_b, output_file=result_bio)
-        return result_bio.getvalue()
+        result_sio = six.StringIO()
+        describe_diff(cube_a, cube_b, output_file=result_sio)
+        return result_sio.getvalue()
 
     def test_noncommon_array_attributes(self):
         # test non-common array attribute