data_prepro.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "from tqdm import tqdm\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/askd_pt.json') as f:\n",
    "    pt = json.load(f)\n",
    "\n",
    "dataset = dict()\n",
    "dataset['train_pt'] = pt['train_askd']\n",
    "dataset['validation_pt'] = pt['validation_askd']\n",
    "dataset['test_pt'] = pt['test_askd']\n",
    "\n",
    "with open('data/askd_en_augmented.json') as f:\n",
    "    en = json.load(f)\n",
    "\n",
    "dataset['train_en'] = en['train_askd']\n",
    "dataset['validation_en'] = en['validation_askd']\n",
    "dataset['test_en'] = en['test_askd']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for split in dataset:\n",
    "    for item in dataset[split]:\n",
    "        for key in [\"selftext_urls\", \"title_urls\", \"answers_urls\"]:\n",
    "            if not isinstance(item[key], list):\n",
    "                item[key] = [item[key]]\n",
    "        \n",
    "        for key in [\"a_id\", \"score\", \"text\"]:\n",
    "            if not isinstance(item['answers'][key], list):\n",
    "                item['answers'][key] = [item['answers'][key]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24256/24256 [00:00<00:00, 1048176.28it/s]\n",
      "100%|██████████| 5198/5198 [00:00<00:00, 1092120.03it/s]\n",
      "100%|██████████| 5198/5198 [00:00<00:00, 1032878.16it/s]\n",
      "100%|██████████| 141019/141019 [13:54<00:00, 169.05it/s]  \n",
      "100%|██████████| 30219/30219 [00:19<00:00, 1549.74it/s]\n",
      "100%|██████████| 30218/30218 [00:17<00:00, 1734.32it/s]\n"
     ]
    }
   ],
   "source": [
    "external_pt = list()\n",
    "external_en = list()\n",
    "\n",
    "for split in dataset:\n",
    "    for item in tqdm(dataset[split].copy()):\n",
    "        if item['answers']['a_id'] == ['0']:\n",
    "            dataset[split].remove(item)\n",
    "\n",
    "            if split.endswith('pt'):\n",
    "                external_pt.append(item)\n",
    "            else:\n",
    "                external_en.append(item)\n",
    "\n",
    "dataset['external_en'] = external_en\n",
    "dataset['external_pt'] = external_pt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for split in dataset:\n",
    "    with open(f'data/{split}.json', 'w') as f:\n",
    "        json.dump(dataset[split], f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['train_pt', 'validation_pt', 'test_pt', 'train_en', 'validation_en', 'test_en', 'external_en', 'external_pt'])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.keys()"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "5550a332701dfaf727177dbb42680f614fbec3c4b4c54b659bea7821910aed98"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}