feat: support non streamable arrow file binary format

Signed-off-by: Mehant Kammakomati <mehant.kammakomati2@ibm.com>

src/datasets/packaged_modules/arrow/arrow.py

@@ -43,7 +43,11 @@ def _split_generators(self, dl_manager):
             if self.info.features is None:
                 for file in itertools.chain.from_iterable(files):
                     with open(file, "rb") as f:
-                        self.info.features = datasets.Features.from_arrow_schema(pa.ipc.open_stream(f).schema)
+                        try:
+                            reader = pa.ipc.open_stream(f)
+                        except pa.lib.ArrowInvalid:
+                            reader = pa.ipc.open_file(f)
+                    self.info.features = datasets.Features.from_arrow_schema(reader.schema)
                     break
             splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
         return splits
@@ -59,7 +63,12 @@ def _generate_tables(self, files):
         for file_idx, file in enumerate(itertools.chain.from_iterable(files)):
             with open(file, "rb") as f:
                 try:
-                    for batch_idx, record_batch in enumerate(pa.ipc.open_stream(f)):
+                    try:
+                        batches = pa.ipc.open_stream(f)
+                    except pa.lib.ArrowInvalid:
+                        reader = pa.ipc.open_file(f)
+                        batches = [reader.get_batch(i) for i in range(reader.num_record_batches)]
+                    for batch_idx, record_batch in enumerate(batches):
                         pa_table = pa.Table.from_batches([record_batch])
                         # Uncomment for debugging (will print the Arrow table size and elements)
                         # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")