code/process/merge_data_label_webis17_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1ea0f9ef",
   "metadata": {},
   "source": [
    "#### Merge Data and Label's jsonl files in a form of CSV based on 'id' column"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ced328b9",
   "metadata": {},
   "source": [
    "Download the original train, validation, and test Data and their Labels' 'jsonl' files from https://zenodo.org/record/5530410 and save it in 'Webis17Data' directory to run this script"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "adfcd182",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "81fa2ffd",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './Webis17Data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "55b06215",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation_truth.jsonl\n",
      "test_truth.jsonl\n",
      "train_truth.jsonl\n",
      "test_instances.jsonl\n",
      "train_instances.jsonl\n",
      ".ipynb_checkpoints\n",
      "validation_instances.jsonl\n"
     ]
    }
   ],
   "source": [
    "for f in os.listdir(path):\n",
    "    print(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "ccd57bd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_data = 'validation' # test validation train\n",
    "os.makedirs('Webis17Processed', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "eb6aef72",
   "metadata": {},
   "outputs": [],
   "source": [
    "name = path+clean_data+'_instances.jsonl'\n",
    "data = pd.read_json(name, lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "18533e73",
   "metadata": {},
   "outputs": [],
   "source": [
    "name = path+clean_data+'_truth.jsonl'\n",
    "label = pd.read_json(name, lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "53400a01",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.merge(data, label, on=\"id\", how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "44e82b0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "name = './Webis17Processed/' + clean_data + '.csv'\n",
    "df.to_csv(name, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7aad079",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}