Merge pull request #612 from bghira/main

merge
bghira · Aug 2, 2024 · 1d4e6d6 · 1d4e6d6
2 parents 76926c4 + bdd222e
commit 1d4e6d6
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/helpers/data_backend/factory.py b/helpers/data_backend/factory.py
@@ -247,7 +247,11 @@ def configure_parquet_database(backend: dict, args, data_backend: BaseDataBacken
 
     bytes_string = data_backend.read(parquet_path)
     pq = io.BytesIO(bytes_string)
-    df = pd.read_parquet(pq)
+    if parquet_path.endswith(".jsonl"):
+        df = pd.read_json(pq, lines=True)
+    else:
+        df = pd.read_parquet(pq)
+
     caption_column = parquet_config.get(
         "caption_column", args.parquet_caption_column or "description"
     )