diff --git a/docs/iris/example_code/General/custom_file_loading.py b/docs/iris/example_code/General/custom_file_loading.py index d6b41443c7..fc167050ab 100644 --- a/docs/iris/example_code/General/custom_file_loading.py +++ b/docs/iris/example_code/General/custom_file_loading.py @@ -214,9 +214,12 @@ def NAME_to_cube(filenames, callback): # Create a format_picker specification of the NAME file format giving it a # priority greater than the built in NAME loader. -_NAME_III_spec = format_picker.FormatSpecification('Name III', format_picker.LeadingLine(), - lambda line: line.startswith("NAME III"), NAME_to_cube, - priority=6) +_NAME_III_spec = format_picker.FormatSpecification( + 'Name III', + format_picker.LeadingLine(), + lambda line: line.startswith(b"NAME III"), + NAME_to_cube, + priority=6) # Register the NAME loader with iris iris.fileformats.FORMAT_AGENT.add_spec(_NAME_III_spec) diff --git a/lib/iris/fileformats/__init__.py b/lib/iris/fileformats/__init__.py index 05925b2f78..83acd13264 100644 --- a/lib/iris/fileformats/__init__.py +++ b/lib/iris/fileformats/__init__.py @@ -76,7 +76,7 @@ def _pp_little_endian(filename, *args, **kwargs): # priority to avoid collateral damage from false positives. FORMAT_AGENT.add_spec( FormatSpecification('GRIB', MagicNumber(100), - lambda header_bytes: 'GRIB' in header_bytes, + lambda header_bytes: b'GRIB' in header_bytes, grib.load_cubes, priority=1)) @@ -173,7 +173,7 @@ def _pp_little_endian(filename, *args, **kwargs): FORMAT_AGENT.add_spec( FormatSpecification('NAME III', LeadingLine(), - lambda line: line.lstrip().startswith("NAME III"), + lambda line: line.lstrip().startswith(b"NAME III"), name.load_cubes, priority=5)) diff --git a/lib/iris/fileformats/pp.py b/lib/iris/fileformats/pp.py index aaa597f867..d9e0c0b49e 100644 --- a/lib/iris/fileformats/pp.py +++ b/lib/iris/fileformats/pp.py @@ -1270,9 +1270,13 @@ def _read_extra_data(self, pp_file, file_reader, extra_len): data_len = ia * PP_WORD_DEPTH if ib == 10: - self.field_title = ''.join(struct.unpack_from('>%dc' % data_len, file_reader(data_len))).rstrip('\00') + field_title = struct.unpack_from('>%ds' % data_len, + file_reader(data_len)) + self.field_title = field_title[0].rstrip(b'\00').decode() elif ib == 11: - self.domain_title = ''.join(struct.unpack_from('>%dc' % data_len, file_reader(data_len))).rstrip('\00') + domain_title = struct.unpack_from('>%ds' % data_len, + file_reader(data_len)) + self.domain_title = domain_title[0].rstrip(b'\00').decode() elif ib in EXTRA_DATA: attr_name = EXTRA_DATA[ib] values = np.fromfile(pp_file, dtype=np.dtype('>f%d' % PP_WORD_DEPTH), count=ia) @@ -1467,7 +1471,8 @@ def save(self, file_handle): for int_code, extra_data in extra_items: pp_file.write(struct.pack(">L", int(int_code))) if isinstance(extra_data, six.string_types): - pp_file.write(struct.pack(">%sc" % len(extra_data), *extra_data)) + pp_file.write(struct.pack(">%ss" % len(extra_data), + extra_data.encode())) else: extra_data = extra_data.astype(np.dtype('>f4')) extra_data.tofile(pp_file) diff --git a/lib/iris/tests/__init__.py b/lib/iris/tests/__init__.py index d3f9043844..bc47968cef 100644 --- a/lib/iris/tests/__init__.py +++ b/lib/iris/tests/__init__.py @@ -423,10 +423,6 @@ def assertString(self, string, reference_filename=None): reference_path = self.result_path(None, 'txt') else: reference_path = get_result_path(reference_filename) - # If the test string is a unicode string, encode as - # utf-8 before comparison to the reference string. - if isinstance(string, unicode): - string = string.encode('utf-8') self._check_same(string, reference_path, type_comparison_name='Strings') @@ -435,17 +431,17 @@ def assertRepr(self, obj, reference_filename): def _check_same(self, item, reference_path, type_comparison_name='CML'): if self._check_reference_file(reference_path): - with open(reference_path, 'r') as reference_fh: - reference = ''.join(reference_fh.readlines()) + with open(reference_path, 'rb') as reference_fh: + reference = ''.join(part.decode('utf-8') + for part in reference_fh.readlines()) self._assert_str_same(reference, item, reference_path, type_comparison_name) else: self._ensure_folder(reference_path) logger.warning('Creating result file: %s', reference_path) - with open(reference_path, 'w') as reference_fh: + with open(reference_path, 'wb') as reference_fh: reference_fh.writelines( part.encode('utf-8') - if isinstance(part, unicode) else part for part in item) def assertXMLElement(self, obj, reference_filename): diff --git a/lib/iris/tests/experimental/test_raster.py b/lib/iris/tests/experimental/test_raster.py index 46d8eea2f0..e00343fc4f 100644 --- a/lib/iris/tests/experimental/test_raster.py +++ b/lib/iris/tests/experimental/test_raster.py @@ -48,7 +48,7 @@ def check_tiff(self, cube, tif_header): iris.experimental.raster.export_geotiff(cube, temp_filename) # Check the metadata is correct. - with open(temp_filename) as fh: + with open(temp_filename, 'rb') as fh: self.check_tiff_header(fh, ('experimental', 'raster', tif_header)) diff --git a/lib/iris/tests/test_file_save.py b/lib/iris/tests/test_file_save.py index 69a2d8284c..bb8123ff8a 100644 --- a/lib/iris/tests/test_file_save.py +++ b/lib/iris/tests/test_file_save.py @@ -21,6 +21,7 @@ from __future__ import (absolute_import, division, print_function) from six.moves import (filter, input, map, range, zip) # noqa +import six # import iris tests first so that some things can be initialised before importing anything else import iris.tests as tests @@ -135,19 +136,19 @@ def test_filehandle(self): save_by_filehandle(self.temp_filename1, self.temp_filename2, self.cube1, dot.save, binary_mode = True) def test_bytesio(self): - bio = io.BytesIO() + sio = six.StringIO() # Save from dot direct dot.save(self.cube1, self.temp_filename1) # Call save on iris - iris.save(self.cube1, bio, iris.io.find_saver(self.ext)) + iris.save(self.cube1, sio, iris.io.find_saver(self.ext)) with open(self.temp_filename1) as infile: data = infile.read() # Compare files - self.assertEqual(data, bio.getvalue(), + self.assertEquals(data, sio.getvalue(), 'Mismatch in data when comparing iris bytesio save ' 'and dot.save.') diff --git a/lib/iris/tests/test_io_init.py b/lib/iris/tests/test_io_init.py index b7afcb9fa7..b45617fbb9 100644 --- a/lib/iris/tests/test_io_init.py +++ b/lib/iris/tests/test_io_init.py @@ -97,7 +97,7 @@ def test_format_picker(self): # test that each filespec is identified as the expected format for (expected_format_name, file_spec) in test_specs: test_path = tests.get_data_path(file_spec) - with open(test_path, 'r') as test_file: + with open(test_path, 'rb') as test_file: a = iff.FORMAT_AGENT.get_spec(test_path, test_file) self.assertEqual(a.name, expected_format_name) @@ -110,8 +110,8 @@ def test_format_picker_nodata(self): # specific to WMO bulletin headers header_lengths = [21, 80, 41, 42] for header_length in header_lengths: - binary_string = header_length * '\x00' + 'GRIB' + '\x00' * 100 - with BytesIO('rw') as bh: + binary_string = header_length * b'\x00' + b'GRIB' + b'\x00' * 100 + with BytesIO(b'rw') as bh: bh.write(binary_string) bh.name = 'fake_file_handle' a = iff.FORMAT_AGENT.get_spec(bh.name, bh) diff --git a/lib/iris/tests/test_util.py b/lib/iris/tests/test_util.py index d9047097a3..7178fcf0c3 100644 --- a/lib/iris/tests/test_util.py +++ b/lib/iris/tests/test_util.py @@ -21,13 +21,13 @@ from __future__ import (absolute_import, division, print_function) from six.moves import (filter, input, map, range, zip) # noqa +import six # import iris tests first so that some things can be initialised before # importing anything else import iris.tests as tests import inspect -import io import unittest import numpy as np @@ -188,9 +188,9 @@ def test_identical(self): test_cube_a = stock.realistic_4d() test_cube_b = stock.realistic_4d() - return_bio = io.BytesIO() - iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_bio) - return_str = return_bio.getvalue().decode() + return_sio = six.StringIO() + iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_sio) + return_str = return_sio.getvalue() self.assertString(return_str, 'compatible_cubes.str.txt') @@ -202,9 +202,9 @@ def test_different(self): test_cube_a.attributes['Conventions'] = 'CF-1.5' test_cube_b.attributes['Conventions'] = 'CF-1.6' - return_bio = io.BytesIO() - iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_bio) - return_str = return_bio.getvalue().decode() + return_sio = six.StringIO() + iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_sio) + return_str = return_sio.getvalue() self.assertString(return_str, 'incompatible_attr.str.txt') @@ -214,9 +214,9 @@ def test_different(self): test_cube_a.standard_name = "relative_humidity" - return_bio = io.BytesIO() - iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_bio) - return_str = return_bio.getvalue().decode() + return_sio = six.StringIO() + iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_sio) + return_str = return_sio.getvalue() self.assertString(return_str, 'incompatible_name.str.txt') @@ -226,9 +226,9 @@ def test_different(self): test_cube_a.units = iris.unit.Unit('m') - return_bio = io.BytesIO() - iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_bio) - return_str = return_bio.getvalue().decode() + return_sio = six.StringIO() + iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_sio) + return_str = return_sio.getvalue() self.assertString(return_str, 'incompatible_unit.str.txt') @@ -236,9 +236,9 @@ def test_different(self): test_cube_a = stock.realistic_4d() test_cube_b = stock.realistic_4d().collapsed('model_level_number', iris.analysis.MEAN) - return_bio = io.BytesIO() - iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_bio) - return_str = return_bio.getvalue().decode() + return_sio = six.StringIO() + iris.util.describe_diff(test_cube_a, test_cube_b, output_file=return_sio) + return_str = return_sio.getvalue() self.assertString(return_str, 'incompatible_meth.str.txt') diff --git a/lib/iris/tests/unit/fileformats/pp/test__field_gen.py b/lib/iris/tests/unit/fileformats/pp/test__field_gen.py index e525cf9b0f..837fc9b6fd 100644 --- a/lib/iris/tests/unit/fileformats/pp/test__field_gen.py +++ b/lib/iris/tests/unit/fileformats/pp/test__field_gen.py @@ -105,8 +105,7 @@ def test_invalid_header_release(self): # Check that an unknown LBREL value just results in a warning # and the end of the file iteration instead of raising an error. with self.temp_filename() as temp_path: - with open(temp_path, 'w') as f: - f.write(np.zeros(65, dtype='i4')) + np.zeros(65, dtype='i4').tofile(temp_path) generator = pp._field_gen(temp_path, False) with mock.patch('warnings.warn') as warn: with self.assertRaises(StopIteration): diff --git a/lib/iris/tests/unit/util/test_describe_diff.py b/lib/iris/tests/unit/util/test_describe_diff.py index c9e3df55cc..f91d76d4e9 100644 --- a/lib/iris/tests/unit/util/test_describe_diff.py +++ b/lib/iris/tests/unit/util/test_describe_diff.py @@ -18,13 +18,12 @@ from __future__ import (absolute_import, division, print_function) from six.moves import (filter, input, map, range, zip) # noqa +import six # import iris tests first so that some things can be initialised before # importing anything else import iris.tests as tests -import io - import numpy as np import iris.cube @@ -37,9 +36,9 @@ def setUp(self): self.cube_b = self.cube_a.copy() def _compare_result(self, cube_a, cube_b): - result_bio = io.BytesIO() - describe_diff(cube_a, cube_b, output_file=result_bio) - return result_bio.getvalue() + result_sio = six.StringIO() + describe_diff(cube_a, cube_b, output_file=result_sio) + return result_sio.getvalue() def test_noncommon_array_attributes(self): # test non-common array attribute