NetCDF thread safety for v3.4.1 patch release

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ minimum_pre_commit_version: 1.21.0
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
         # Prevent giant files from being committed.
     -   id: check-added-large-files
@@ -29,32 +29,31 @@ repos:
     -   id: no-commit-to-branch
 
 -   repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 22.12.0
     hooks:
     -   id: black
         pass_filenames: false
         args: [--config=./pyproject.toml, .]
 
 -   repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
     -   id: flake8
         types: [file, python]
         args: [--config=./setup.cfg]
 
 -   repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
     -   id: isort
         types: [file, python]
         args: [--filter-files]
 
 -   repo: https://github.com/asottile/blacken-docs
-    rev: v1.12.1
+    rev: 1.13.0
     hooks:
     -   id: blacken-docs
         types: [file, rst]
-        additional_dependencies: [black==21.6b0]
 
 -   repo: https://github.com/aio-libs/sort-all
     rev: v1.2.0

benchmarks/benchmarks/experimental/ugrid/__init__.py

@@ -50,7 +50,7 @@ def time_create(self, *params):
 
 class Connectivity(UGridCommon):
     def setup(self, n_faces):
-        self.array = np.zeros([n_faces, 3], dtype=np.int)
+        self.array = np.zeros([n_faces, 3], dtype=int)
         super().setup(n_faces)
 
     def create(self):

docs/gallery_code/oceanography/plot_atlantic_profiles.py

@@ -34,7 +34,7 @@ def main():
     # the southern portion of the domain, and limit the depth of the profile
     # to 1000m.
     lon_cons = iris.Constraint(longitude=330.5)
-    lat_cons = iris.Constraint(latitude=lambda l: -10 < l < -9)
+    lat_cons = iris.Constraint(latitude=lambda lat: -10 < lat < -9)
     depth_cons = iris.Constraint(depth=lambda d: d <= 1000)
     theta_1000m = theta.extract(depth_cons & lon_cons & lat_cons)
     salinity_1000m = salinity.extract(depth_cons & lon_cons & lat_cons)

docs/src/common_links.inc

@@ -40,6 +40,7 @@
 .. _CF-UGRID: https://ugrid-conventions.github.io/ugrid-conventions/
 .. _issues on GitHub: https://github.com/SciTools/iris/issues?q=is%3Aopen+is%3Aissue+sort%3Areactions-%2B1-desc
 .. _python-stratify: https://github.com/SciTools/python-stratify
+.. _netCDF4: https://github.com/Unidata/netcdf4-python
 
 
 .. comment

docs/src/further_topics/metadata.rst

@@ -389,10 +389,10 @@ instances. Normally, this would cause issues. For example,
 
 .. doctest:: richer-metadata
 
-    >>> simply = {"one": np.int(1), "two": np.array([1.0, 2.0])}
+    >>> simply = {"one": np.int32(1), "two": np.array([1.0, 2.0])}
     >>> simply
     {'one': 1, 'two': array([1., 2.])}
-    >>> fruity = {"one": np.int(1), "two": np.array([1.0, 2.0])}
+    >>> fruity = {"one": np.int32(1), "two": np.array([1.0, 2.0])}
     >>> fruity
     {'one': 1, 'two': array([1., 2.])}
     >>> simply == fruity
@@ -419,7 +419,7 @@ However, metadata class equality is rich enough to handle this eventuality,
 
     >>> metadata1
     CubeMetadata(standard_name='air_temperature', long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'one': 1, 'two': array([1., 2.])}, cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),))
-    >>> metadata2 = cube.metadata._replace(attributes={"one": np.int(1), "two": np.array([1000.0, 2000.0])})
+    >>> metadata2 = cube.metadata._replace(attributes={"one": np.int32(1), "two": np.array([1000.0, 2000.0])})
     >>> metadata2
     CubeMetadata(standard_name='air_temperature', long_name=None, var_name='air_temperature', units=Unit('K'), attributes={'one': 1, 'two': array([1000., 2000.])}, cell_methods=(CellMethod(method='mean', coord_names=('time',), intervals=('6 hour',), comments=()),))
     >>> metadata1 == metadata2

docs/src/userguide/cube_maths.rst

@@ -5,8 +5,8 @@ Cube Maths
 ==========
 
 
-The section :doc:`navigating_a_cube` highlighted that 
-every cube has a data attribute; 
+The section :doc:`navigating_a_cube` highlighted that
+every cube has a data attribute;
 this attribute can then be manipulated directly::
 
    cube.data -= 273.15
@@ -37,7 +37,7 @@ Let's load some air temperature which runs from 1860 to 2100::
     filename = iris.sample_data_path('E1_north_america.nc')
     air_temp = iris.load_cube(filename, 'air_temperature')
 
-We can now get the first and last time slices using indexing 
+We can now get the first and last time slices using indexing
 (see :ref:`cube_indexing` for a reminder)::
 
     t_first = air_temp[0, :, :]
@@ -50,8 +50,8 @@ We can now get the first and last time slices using indexing
     t_first = air_temp[0, :, :]
     t_last = air_temp[-1, :, :]
 
-And finally we can subtract the two. 
-The result is a cube of the same size as the original two time slices, 
+And finally we can subtract the two.
+The result is a cube of the same size as the original two time slices,
 but with the data representing their difference:
 
     >>> print(t_last - t_first)
@@ -70,8 +70,8 @@ but with the data representing their difference:
 
 .. note::
 
-    Notice that the coordinates "time" and "forecast_period" have been removed 
-    from the resultant cube; 
+    Notice that the coordinates "time" and "forecast_period" have been removed
+    from the resultant cube;
     this is because these coordinates differed between the two input cubes.
 
 
@@ -174,15 +174,15 @@ broadcasting behaviour::
 Combining Multiple Phenomena to Form a New One
 ----------------------------------------------
 
-Combining cubes of potential-temperature and pressure we can calculate 
+Combining cubes of potential-temperature and pressure we can calculate
 the associated temperature using the equation:
 
 .. math::
-   
+
     T = \theta (\frac{p}{p_0}) ^ {(287.05 / 1005)}
 
-Where :math:`p` is pressure, :math:`\theta` is potential temperature, 
-:math:`p_0` is the potential temperature reference pressure 
+Where :math:`p` is pressure, :math:`\theta` is potential temperature,
+:math:`p_0` is the potential temperature reference pressure
 and :math:`T` is temperature.
 
 First, let's load pressure and potential temperature cubes::
@@ -191,7 +191,7 @@ First, let's load pressure and potential temperature cubes::
     phenomenon_names = ['air_potential_temperature', 'air_pressure']
     pot_temperature, pressure = iris.load_cubes(filename, phenomenon_names)
 
-In order to calculate :math:`\frac{p}{p_0}` we can define a coordinate which 
+In order to calculate :math:`\frac{p}{p_0}` we can define a coordinate which
 represents the standard reference pressure of 1000 hPa::
 
     import iris.coords
@@ -205,7 +205,7 @@ the :meth:`iris.coords.Coord.convert_units` method::
 
     p0.convert_units(pressure.units)
 
-Now we can combine all of this information to calculate the air temperature 
+Now we can combine all of this information to calculate the air temperature
 using the equation above::
 
     temperature = pot_temperature * ( (pressure / p0) ** (287.05 / 1005) )
@@ -219,12 +219,12 @@ The result could now be plotted using the guidance provided in the
 
 .. only:: html
 
-    A very similar example to this can be found in 
+    A very similar example to this can be found in
     :ref:`sphx_glr_generated_gallery_meteorology_plot_deriving_phenomena.py`.
 
 .. only:: latex
 
-    A very similar example to this can be found in the examples section, 
+    A very similar example to this can be found in the examples section,
     with the title "Deriving Exner Pressure and Air Temperature".
 
 .. _cube_maths_combining_units:
@@ -249,7 +249,7 @@ unit (if ``a`` had units ``'m2'`` then ``a ** 0.5`` would result in a cube
 with units ``'m'``).
 
 Iris inherits units from `cf_units <https://scitools.org.uk/cf-units/docs/latest/>`_
-which in turn inherits from `UDUNITS <https://www.unidata.ucar.edu/software/udunits/udunits-current/udunits2.html>`_.
+which in turn inherits from `UDUNITS <https://docs.unidata.ucar.edu/udunits/current/>`_.
 As well as the units UDUNITS provides, cf units also provides the units
 ``'no-unit'`` and ``'unknown'``. A unit of ``'no-unit'`` means that the
 associated data is not suitable for describing with a unit, cf units

docs/src/userguide/glossary.rst

@@ -1,3 +1,5 @@
+.. include:: ../common_links.inc
+
 .. _glossary:
 
 Glossary
@@ -125,7 +127,7 @@ Glossary
         of formats.
 
         | **Related:** :term:`CartoPy` **|** :term:`NumPy`
-        | **More information:** `Matplotlib <https://scitools.org.uk/cartopy/docs/latest/>`_
+        | **More information:**  `matplotlib`_
         |
 
    Metadata
@@ -143,9 +145,11 @@ Glossary
         When Iris loads this format, it also especially recognises and interprets data
         encoded according to the :term:`CF Conventions`.
 
+        __ `NetCDF4`_
+
         | **Related:** :term:`Fields File (FF) Format`
          **|** :term:`GRIB Format` **|** :term:`Post Processing (PP) Format`
-        | **More information:** `NetCDF-4 Python Git <https://github.com/Unidata/netcdf4-python>`_
+        | **More information:** `NetCDF-4 Python Git`__
         |
 
    NumPy

docs/src/whatsnew/2.1.rst

@@ -1,3 +1,5 @@
+.. include:: ../common_links.inc
+
 v2.1 (06 Jun 2018)
 ******************
 
@@ -67,7 +69,7 @@ Incompatible Changes
   as an alternative.
 
 * This release of Iris contains a number of updated metadata translations.
-  See this 
+  See this
   `changelist <https://github.com/SciTools/iris/commit/69597eb3d8501ff16ee3d56aef1f7b8f1c2bb316#diff-1680206bdc5cfaa83e14428f5ba0f848>`_
   for further information.
 
@@ -84,14 +86,16 @@ Internal
   calendar.
 
 * Iris updated its time-handling functionality from the
-  `netcdf4-python <http://unidata.github.io/netcdf4-python/>`_
+  `netcdf4-python`__
   ``netcdftime`` implementation to the standalone module
   `cftime <https://github.com/Unidata/cftime>`_.
   cftime is entirely compatible with netcdftime, but some issues may
   occur where users are constructing their own datetime objects.
   In this situation, simply replacing ``netcdftime.datetime`` with
   ``cftime.datetime`` should be sufficient.
 
+__ `netCDF4`_
+
 * Iris now requires version 2 of Matplotlib, and ``>=1.14`` of NumPy.
   Full requirements can be seen in the `requirements <https://github.com/SciTools/iris/>`_
   directory of the Iris' the source.

docs/src/whatsnew/3.4.rst

@@ -26,15 +26,29 @@ This document explains the changes made to Iris for this release
    * We have **begun refactoring Iris' regridding**, which has already improved
      performance and functionality, with more potential in future!
    * We have made several other significant `🚀 Performance Enhancements`_.
-   * Please note that **Iris cannot currently work with the latest NetCDF4
-     releases**. The pin is set to ``<v1.6.1``, due to incompatibility with
-     Iris' lack of thread safety. We're working hard to make Iris NetCDF
-     loading thread safe as soon as possible.
 
    And finally, get in touch with us on :issue:`GitHub<new/choose>` if you have
    any issues or feature requests for improving Iris. Enjoy!
 
 
+v3.4.1 (21 Feb 2023)
+====================
+
+.. dropdown:: :opticon:`alert` v3.4.1 Patches
+   :container: + shadow
+   :title: text-primary text-center font-weight-bold
+   :body: bg-light
+   :animate: fade-in
+
+   The patches in this release of Iris include:
+
+   #. `@trexfeathers`_ and `@pp-mo`_ made Iris' use of the `netCDF4`_ library
+      thread-safe. (:pull:`5095`)
+
+   #. `@trexfeathers`_ and `@pp-mo`_ removed the netCDF4 pin mentioned in
+      `🔗 Dependencies`_ point 3. (:pull:`5095`)
+
+
 📢 Announcements
 ================
 

lib/iris/experimental/ugrid/load.py

@@ -209,7 +209,8 @@ def load_meshes(uris, var_name=None):
 
     result = {}
     for source in valid_sources:
-        meshes_dict = _meshes_from_cf(CFUGridReader(source))
+        with CFUGridReader(source) as cf_reader:
+            meshes_dict = _meshes_from_cf(cf_reader)
         meshes = list(meshes_dict.values())
         if var_name is not None:
             meshes = list(filter(lambda m: m.var_name == var_name, meshes))

lib/iris/fileformats/cf.py

@@ -20,10 +20,10 @@
 import re
 import warnings
 
-import netCDF4
 import numpy as np
 import numpy.ma as ma
 
+from iris.fileformats.netcdf import _thread_safe_nc
 import iris.util
 
 #
@@ -1050,7 +1050,9 @@ def __init__(self, filename, warn=False, monotonic=False):
         #: Collection of CF-netCDF variables associated with this netCDF file
         self.cf_group = self.CFGroup()
 
-        self._dataset = netCDF4.Dataset(self._filename, mode="r")
+        self._dataset = _thread_safe_nc.DatasetWrapper(
+            self._filename, mode="r"
+        )
 
         # Issue load optimisation warning.
         if warn and self._dataset.file_format in [
@@ -1068,6 +1070,19 @@ def __init__(self, filename, warn=False, monotonic=False):
         self._build_cf_groups()
         self._reset()
 
+    def __enter__(self):
+        # Enable use as a context manager
+        # N.B. this **guarantees* closure of the file, when the context is exited.
+        # Note: ideally, the class would not do so much work in the __init__ call, and
+        # would do all that here, after acquiring necessary permissions/locks.
+        # But for legacy reasons, we can't do that.  So **effectively**, the context
+        # (in terms of access control) alreday started, when we created the object.
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        # When used as a context-manager, **always** close the file on exit.
+        self._close()
+
     @property
     def filename(self):
         """The file that the CFReader is reading."""
@@ -1294,10 +1309,15 @@ def _reset(self):
         for nc_var_name in self._dataset.variables.keys():
             self.cf_group[nc_var_name].cf_attrs_reset()
 
-    def __del__(self):
+    def _close(self):
         # Explicitly close dataset to prevent file remaining open.
         if self._dataset is not None:
             self._dataset.close()
+            self._dataset = None
+
+    def __del__(self):
+        # Be sure to close dataset when CFReader is destroyed / garbage-collected.
+        self._close()
 
 
 def _getncattr(dataset, attr, default=None):