feat: Create and push parquet to huggingface

parquet_dataset_generator.py

@@ -1,6 +1,13 @@
 import pandas as pd
 import json
 import glob
+from datasets import load_dataset
+from huggingface_hub import login
+from decouple import config
+
+
+def login_huggingface():
+    login(token=config("HF_TOKEN"))
 
 
 def combine_json_arrays(directory_path: str) -> list:
@@ -27,6 +34,12 @@ def create_parquet_from_json(json_data: list):
     df.to_parquet('internal_dataset.parquet', index=False)
 
 
+def push_parquet():
+    dataset = load_dataset('parquet', data_files='internal_dataset.parquet')
+    dataset_repo_name = 'getwithashish/internal-dept-dataset'
+    dataset.push_to_hub(dataset_repo_name)
+
+
 if __name__ == "__main__":
     raw_json_data = combine_json_arrays(".")
     create_parquet_from_json(raw_json_data)