Fix decoding multi part extension (#6904)

* fix decoding multi part extension * also fix numpy arrays
huggingface · May 17, 2024 · b3f7724 · b3f7724
1 parent b7d71ff
commit b3f7724
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/src/datasets/features/features.py b/src/datasets/features/features.py
@@ -1260,6 +1260,8 @@ def encode_nested_example(schema, obj, level=0):
         sub_schema = schema[0]
         if obj is None:
             return None
+        elif isinstance(obj, np.ndarray):
+            return encode_nested_example(schema, obj.tolist())
         else:
             if len(obj) > 0:
                 for first_elmt in obj:

diff --git a/src/datasets/packaged_modules/webdataset/webdataset.py b/src/datasets/packaged_modules/webdataset/webdataset.py
@@ -31,8 +31,8 @@ def _get_pipeline_from_tar(cls, tar_path, tar_iterator):
                 current_example["__key__"] = example_key
                 current_example["__url__"] = tar_path
                 current_example[field_name.lower()] = f.read()
-                if field_name in cls.DECODERS:
-                    current_example[field_name] = cls.DECODERS[field_name](current_example[field_name])
+                if field_name.split(".")[-1] in cls.DECODERS:
+                    current_example[field_name] = cls.DECODERS[field_name.split(".")[-1]](current_example[field_name])
         if current_example:
             yield current_example