Fixed (for real) MULTI-FIELD messages support. Closes #45.

CHANGELOG.rst

@@ -2,6 +2,14 @@
 Changelog for cfgrib
 ====================
 
+0.9.6.1 (unreleased)
+--------------------
+
+- Add a protocol version to the index file.
+- Fixed (for real) MULTI-FIELD messages,
+  See: `#45 <https://github.com/ecmwf/cfgrib/issues/45>`_.
+
+
 0.9.6.post1 (2019-03-07)
 ------------------------
 

README.rst

@@ -409,6 +409,7 @@ Data variables:
     r              (isobaricInhPa, y, x) float32 ...
     w              (isobaricInhPa, y, x) float32 ...
     u              (isobaricInhPa, y, x) float32 ...
+    v              (isobaricInhPa, y, x) float32 ...
 Attributes:
     GRIB_edition:            2
     GRIB_centre:             kwbc
@@ -430,6 +431,7 @@ Data variables:
     t2m                (y, x) float32 ...
     r2                 (y, x) float32 ...
     u10                (y, x) float32 ...
+    v10                (y, x) float32 ...
 Attributes:
     GRIB_edition:            2
     GRIB_centre:             kwbc
@@ -530,6 +532,7 @@ Data variables:
     pres        (y, x) float32 ...
     t           (y, x) float32 ...
     u           (y, x) float32 ...
+    v           (y, x) float32 ...
 Attributes:
     GRIB_edition:            2
     GRIB_centre:             kwbc
@@ -551,6 +554,7 @@ Data variables:
     pres        (y, x) float32 ...
     gh          (y, x) float32 ...
     u           (y, x) float32 ...
+    v           (y, x) float32 ...
 Attributes:
     GRIB_edition:            2
     GRIB_centre:             kwbc
@@ -592,6 +596,7 @@ Data variables:
     t                        (pressureFromGroundLayer, y, x) float32 ...
     r                        (pressureFromGroundLayer, y, x) float32 ...
     u                        (pressureFromGroundLayer, y, x) float32 ...
+    v                        (pressureFromGroundLayer, y, x) float32 ...
     pli                      (y, x) float32 ...
     4lftx                    (y, x) float32 ...
 Attributes:

cfgrib/messages.py

@@ -46,7 +46,7 @@
 #
 # No explicit support for MULTI-FIELD at Message level.
 #
-# bindings.codes_grib_multi_support_on()
+bindings.codes_grib_multi_support_on()
 
 
 @attr.attrs()
@@ -62,9 +62,15 @@ class Message(collections.MutableMapping):
     @classmethod
     def from_file(cls, file, offset=None, product_kind=bindings.CODES_PRODUCT_GRIB, **kwargs):
         # type: (T.IO[bytes], int, int, T.Any) -> Message
+        field_in_message = 0
+        if isinstance(offset, tuple):
+            offset, field_in_message = offset
         if offset is not None:
             file.seek(offset)
         codes_id = bindings.codes_handle_new_from_file(file, product_kind)
+        # iterate over multi-fields in the message
+        for _ in range(field_in_message):
+            codes_id = bindings.codes_handle_new_from_file(file, product_kind)
         return cls(codes_id=codes_id, **kwargs)
 
     @classmethod
@@ -266,6 +272,7 @@ def compat_create_exclusive(path, *args, **kwargs):
 
 @attr.attrs()
 class FileIndex(collections.Mapping):
+    allowed_protocol_version = '1'
     filestream = attr.attrib(type=FileStream)
     index_keys = attr.attrib(type=T.List[str])
     offsets = attr.attrib(repr=False, type=T.List[T.Tuple[T.Tuple[T.Any, ...], T.List[int]]])
@@ -278,6 +285,7 @@ def from_filestream(cls, filestream, index_keys):
         #   disabled and we may choose to remove `make_message_schema` altogether.
         schema = make_message_schema(filestream.first(), index_keys)
         offsets = collections.OrderedDict()
+        count_offsets = {}  # type: T.Dict[int, int]
         for message in filestream:
             header_values = []
             for key, args in schema.items():
@@ -292,8 +300,17 @@ def from_filestream(cls, filestream, index_keys):
                     value = tuple(value)
                 header_values.append(value)
             offset = message.message_get('offset', bindings.CODES_TYPE_LONG)
-            offsets.setdefault(tuple(header_values), []).append(offset)
-        return cls(filestream=filestream, index_keys=index_keys, offsets=list(offsets.items()))
+            if offset in count_offsets:
+                count_offsets[offset] += 1
+                offset_field = (offset, count_offsets[offset])
+            else:
+                count_offsets[offset] = 0
+                offset_field = offset
+            offsets.setdefault(tuple(header_values), []).append(offset_field)
+        self = cls(filestream=filestream, index_keys=index_keys, offsets=list(offsets.items()))
+        # record the index protocol version in the instance so it is dumped with pickle
+        self.index_protocol_version = cls.allowed_protocol_version
+        return self
 
     @classmethod
     def from_indexpath(cls, indexpath):
@@ -327,8 +344,10 @@ def from_indexpath_or_filestream(
             filestream_mtime = os.path.getmtime(filestream.path)
             if index_mtime >= filestream_mtime:
                 self = cls.from_indexpath(indexpath)
+                allowed_protocol_version = self.allowed_protocol_version
                 if getattr(self, 'index_keys', None) == index_keys and \
-                        getattr(self, 'filestream', None) == filestream:
+                        getattr(self, 'filestream', None) == filestream and \
+                        getattr(self, 'index_protocol_version', None) == allowed_protocol_version:
                     return self
                 else:
                     log.warning("Ignoring index file %r incompatible with GRIB file", indexpath)