Origin/fix missing features error

src/datasets/packaged_modules/json/json.py

@@ -82,6 +82,10 @@ def _split_generators(self, dl_manager):
 
     def _cast_table(self, pa_table: pa.Table) -> pa.Table:
         if self.config.features is not None:
+            # adding missing columns
+            for column_name in set(self.config.features) - set(pa_table.column_names):
+                type = self.config.features.arrow_schema.field(column_name).type
+                pa_table = pa_table.append_column(column_name, pa.array([None] * len(pa_table), type=type))
             # more expensive cast to support nested structures with keys in a different order
             # allows str <-> int/float or str to Audio for example
             pa_table = table_cast(pa_table, self.config.features.arrow_schema)

tests/packaged_modules/test_json.py

@@ -3,6 +3,7 @@
 import pyarrow as pa
 import pytest
 
+from datasets import Features, Value
 from datasets.packaged_modules.json.json import Json
 
 
@@ -69,3 +70,32 @@ def test_json_generate_tables(file_fixture, config_kwargs, request):
     generator = json._generate_tables([[request.getfixturevalue(file_fixture)]])
     pa_table = pa.concat_tables([table for _, table in generator])
     assert pa_table.to_pydict() == {"col_1": [1, 10], "col_2": [2, 20]}
+
+
+@pytest.mark.parametrize(
+    "file_fixture, config_kwargs",
+    [
+        (
+            "jsonl_file",
+            {"features": Features({"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")})},
+        ),
+        (
+            "json_file_with_list_of_dicts",
+            {"features": Features({"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")})},
+        ),
+        (
+            "json_file_with_list_of_dicts_field",
+            {
+                "field": "field3",
+                "features": Features(
+                    {"col_1": Value("int64"), "col_2": Value("int64"), "missing_col": Value("string")}
+                ),
+            },
+        ),
+    ],
+)
+def test_json_generate_tables_with_missing_features(file_fixture, config_kwargs, request):
+    json = Json(**config_kwargs)
+    generator = json._generate_tables([[request.getfixturevalue(file_fixture)]])
+    pa_table = pa.concat_tables([table for _, table in generator])
+    assert pa_table.to_pydict() == {"col_1": [1, 10], "col_2": [2, 20], "missing_col": [None, None]}