data_preparation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "semeval_path = \"\"\n",
    "folderpth=semeval_path\n",
    "train_articles_path = semeval_path + 'datasets/train-articles/'\n",
    "dev_articles_path = semeval_path + 'datasets/dev-articles/'\n",
    "y_path_train = semeval_path + 'train-task1-SI.labels'\n",
    "y_path_dev = semeval_path + 'dev-task1-SI.labels'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os, time, pickle\n",
    "import re\n",
    "import json\n",
    "#import nltk\n",
    "from tqdm import tqdm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:85% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "\n",
    "pd.set_option('display.max_columns', 500)\n",
    "pd.set_option('display.max_rows', 500)\n",
    "display(HTML(\"<style>.container { width:85% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def indices_sentence(article_id, article_path):\n",
    "    f= open(article_path + 'article' + str(article_id) + '.txt',\"r\")\n",
    "    indices={}\n",
    "    start_index = 0\n",
    "    for i, line in enumerate(f):\n",
    "        indices[i] = {}\n",
    "        indices[i]['article_id']=article_id\n",
    "        indices[i]['span_present'] = 0\n",
    "        indices[i]['sentence'] = line\n",
    "        indices[i]['start_index'] = start_index\n",
    "        indices[i]['end_index'] = start_index + len(line)\n",
    "        start_index = indices[i]['end_index']    \n",
    "        \n",
    "    return indices\n",
    "\n",
    "def get_label_df(path):\n",
    "    label_SI = pd.read_csv(path, delimiter='\\t')\n",
    "    label_SI.loc[-1, :] = [int(i) for i in label_SI.columns.tolist()]\n",
    "    label_SI.columns=['article_id', 'begin_offset', 'end_offset']\n",
    "    label_SI.sort_values(by=['article_id', 'begin_offset'], inplace=True)\n",
    "    label_SI.reset_index(drop=True, inplace=True)\n",
    "    return label_SI\n",
    "\n",
    "def get_label_test(path):\n",
    "    label_SI = pd.read_csv(path, delimiter='\\t')\n",
    "    _dum = label_SI.columns.tolist()\n",
    "    label_SI.loc[-1, :] = [int(_dum[0]), _dum[1], int(_dum[2]), int(_dum[3])]\n",
    "    label_SI.columns=['article_id', 'tc_class','begin_offset', 'end_offset']\n",
    "    label_SI.sort_values(by=['article_id', 'begin_offset'], inplace=True)\n",
    "    label_SI.reset_index(drop=True, inplace=True)\n",
    "    return label_SI[['article_id','begin_offset', 'end_offset']]\n",
    "\n",
    "def get_tc_label_df():\n",
    "    label_TC = pd.read_csv('./datasets/train-task2-TC.labels', delimiter='\\t')\n",
    "    _dum = label_TC.columns.tolist()\n",
    "    label_TC.loc[0, :] = [int(_dum[0]), _dum[1], int(_dum[2]), int(_dum[3])]\n",
    "    label_TC.columns=['article_id', 'prpgnd_type', 'begin_offset', 'end_offset']\n",
    "    label_TC.sort_values(by=['article_id', 'begin_offset'], inplace=True)\n",
    "    label_TC.reset_index(drop=True, inplace=True)\n",
    "    uniq_TC = label_TC.prpgnd_type.unique().tolist().copy()\n",
    "    uniq_TC.sort()\n",
    "    uniq_TC = {_id:i for i, _id in enumerate(uniq_TC)}\n",
    "    return label_TC, uniq_TC\n",
    "\n",
    "def get_wordchar_indicies(sent):\n",
    "    k_l = list(sent)\n",
    "    k_l_b = [0 if i==' ' else 1 for i in k_l]\n",
    "    k_df = pd.DataFrame({'char':k_l, 'space_mark':k_l_b})\n",
    "    k_df = k_df.reset_index()\n",
    "    k_df['u1'] = k_df['space_mark'].diff()\n",
    "    k_df['u1'].fillna(1, inplace=True)\n",
    "    k_df.loc[k_df['u1']==1, 'u2']= k_df.loc[k_df['u1']==1, 'u1'].cumsum()\n",
    "    k_df.loc[k_df['u1']==-1, 'u2']= k_df.loc[k_df['u1']==-1, 'u1']\n",
    "    k_df['u2'] = k_df['u2'].ffill(axis=0)\n",
    "    k_df = k_df[k_df['u2']!=-1]\n",
    "    k_df_gb = pd.DataFrame(k_df.groupby(['u2'])['index'].min())\n",
    "    k_df_gb['last_index_word'] = k_df.groupby(['u2'])['index'].max()\n",
    "    k_df_gb = k_df_gb.reset_index().rename(columns={'u2':'word_index','index':'first_index_word'})\n",
    "    try:\n",
    "        k_df_gb['words'] = sent.split()\n",
    "    except:\n",
    "        print(k_df_gb, sent)\n",
    "    return k_df_gb\n",
    "\n",
    "def get_sentence_data(article_id, data_path, article_path):\n",
    "    se_dict = indices_sentence(article_id, article_path)\n",
    "    label_df = get_label_df(data_path)\n",
    "#     label_TC = get_tc_label_df()\n",
    "    req_labels = label_df.loc[label_df['article_id']==article_id, :]\n",
    "    req_labels.reset_index(drop=True, inplace=True)\n",
    "\n",
    "    sentence_ids = []\n",
    "    for key, value in se_dict.items():\n",
    "        \n",
    "        if value['sentence'] == '\\n':\n",
    "            se_dict[key]['word_st_index'] = [0]\n",
    "            se_dict[key]['word_en_index'] = [0]\n",
    "        else:\n",
    "            wordchar_df = get_wordchar_indicies(value['sentence'])\n",
    "            se_dict[key]['word_st_index'] = list(wordchar_df['first_index_word'])\n",
    "            se_dict[key]['word_en_index'] = list(wordchar_df['last_index_word'])\n",
    "\n",
    "        #se_dict[key]['Y1'] = np.zeros((len(value['sentence'].split()), 1))\n",
    "        #se_dict[key]['Y2'] = np.zeros((len(value['sentence'].split()),14))\n",
    "        #se_dict[key]['Y'] = np.concatenate([se_dict[key]['Y1'], se_dict[key]['Y2']], axis=1)\n",
    "        se_dict[key]['Y'] = np.zeros((len(value['sentence'].split()), 1))\n",
    "        \"\"\"\\np.concatenate(\n",
    "            [np.zeros((len(value['sentence'].split()), 1)), \n",
    "             np.zeros((len(value['sentence'].split()),14))], axis=1)\"\"\"\n",
    "        \n",
    "        k=[]\n",
    "        for b, e  in zip(req_labels['begin_offset'], req_labels['end_offset']):\n",
    "            _value = value.copy()\n",
    "            if ((value['start_index'] <= b)&(value['end_index'] >= e)):\n",
    "                sentence_ids.append([key])\n",
    "\n",
    "            if (((_value['start_index'] <= b)&(_value['end_index'] > b))&(_value['end_index'] < e)):\n",
    "\n",
    "                k.append(key)\n",
    "                key=key+1\n",
    "                _value=se_dict[key]\n",
    "                if (((_value['start_index'] <= e)&(_value['end_index'] > e))&(_value['start_index'] > b)):\n",
    "                    k.append(key)\n",
    "                else:\n",
    "                    while ((_value['start_index'] > b)&(_value['end_index'] < e)):\n",
    "                        k.append(key)\n",
    "                        key=key+1\n",
    "                        _value=se_dict[key]\n",
    "                    else:\n",
    "                        k.append(key)\n",
    "                        \n",
    "                sentence_ids.append(k)    \n",
    "\n",
    "    \n",
    "    if len(sentence_ids) == len(req_labels):\n",
    "        \n",
    "        req_labels.loc[:, 'sentence_id'] = sentence_ids\n",
    "\n",
    "        try:\n",
    "            req_labels.loc[:, 'len_s_id'] = req_labels.loc[:, 'sentence_id'].apply(lambda x: len(x))\n",
    "        except:\n",
    "            req_labels.loc[:, 'len_s_id'] = 1\n",
    "\n",
    "        multiple_spans = req_labels[req_labels['len_s_id']>1]['sentence_id'].tolist()\n",
    "        for _span_id in multiple_spans:\n",
    "            _sentence = se_dict[_span_id[0]]['sentence']\n",
    "            _st_in = se_dict[_span_id[0]]['start_index']\n",
    "            \n",
    "            _Y = se_dict[_span_id[0]]['Y']\n",
    "            _st_word = se_dict[_span_id[0]]['word_st_index']\n",
    "            _en_word = se_dict[_span_id[0]]['word_en_index']\n",
    "            \n",
    "            for _id in _span_id[1:]:\n",
    "                se_dict[_id]['only_part_span'] = 1\n",
    "                _sentence += se_dict[_id]['sentence']\n",
    "                _en_in = se_dict[_id]['end_index']\n",
    "                _Y = np.concatenate([_Y, se_dict[_id]['Y']], axis=0)\n",
    "                _st_word += se_dict[_id]['word_st_index']\n",
    "                _en_word += se_dict[_id]['word_en_index']\n",
    "                \n",
    "            se_dict[_span_id[0]]['sentence'] = _sentence\n",
    "            se_dict[_span_id[0]]['end_index'] = _en_in\n",
    "            se_dict[_span_id[0]]['Y'] = _Y\n",
    "            se_dict[_span_id[0]]['word_st_index'] = _st_word\n",
    "            se_dict[_span_id[0]]['word_en_index'] = _en_word\n",
    "    else:\n",
    "        print(sentence_ids,\"Length of sentence ids is not same as req_labels\")\n",
    "        \n",
    "    for i, s_id_list in enumerate(req_labels['sentence_id']):\n",
    "        try:\n",
    "            s_id = s_id_list[0]\n",
    "        except:\n",
    "            s_id = s_id_list\n",
    "            \n",
    "        if 'only_part_span' in se_dict[s_id].keys():\n",
    "            se_dict[s_id].pop('only_part_span')\n",
    "            \n",
    "        se_dict[s_id]['span_present'] = 1\n",
    "        sentence = se_dict[s_id]['sentence']\n",
    "        span_start = req_labels.loc[i, 'begin_offset'] - se_dict[s_id]['start_index']\n",
    "        span_end = req_labels.loc[i, 'end_offset'] - se_dict[s_id]['start_index']\n",
    "        s1 = sentence[0:span_start]\n",
    "        s2 = sentence[span_start:span_end]\n",
    "        tail_len = len(s1.split())\n",
    "        tail_san_len = len(s1.split()) + len(s2.split())\n",
    "        if tail_san_len>len(sentence.split()):\n",
    "            tail_san_len = tail_san_len - 1\n",
    "            tail_len = tail_len - 1\n",
    "        for n in range(tail_len, tail_san_len):\n",
    "            se_dict[s_id]['Y'][n, 0] = 1\n",
    "    return se_dict, req_labels\n",
    "\n",
    "def text_preprocessing(text):\n",
    "    text= text.lower()\n",
    "    text= re.sub(r'[^a-z]',' ',text)\n",
    "    text= ' '.join(text.split())\n",
    "    return(text)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 565/3262 [00:00<00:00, 5646.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thu Mar 12 13:43:18 2020\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3262/3262 [00:00<00:00, 6491.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of df_final_corr:  (57707, 9)\n",
      "0.01 Mins lapsed\n"
     ]
    }
   ],
   "source": [
    "def get_prop_df(dict_all):\n",
    "    start = time.time()\n",
    "    print(time.asctime())\n",
    "    X = []\n",
    "    Y =[]\n",
    "    Z = []\n",
    "    st = []\n",
    "    en = []\n",
    "    sn_st = []\n",
    "    sn_en = []\n",
    "    for _dict in dict_all:\n",
    "        se_dict = _dict\n",
    "        for i in range(len(se_dict)):\n",
    "            X.append(se_dict[i]['sentence'])\n",
    "            Y.append(se_dict[i]['Y'])\n",
    "            Z.append(se_dict[i]['article_id'])\n",
    "            st.append(se_dict[i]['word_st_index'])\n",
    "            en.append(se_dict[i]['word_en_index'])\n",
    "            sn_st.append(se_dict[i]['start_index'])\n",
    "            sn_en.append(se_dict[i]['end_index'])\n",
    "\n",
    "    df = pd.DataFrame()\n",
    "    df['sent'] = X\n",
    "    df['label'] = Y\n",
    "    df['article'] = Z\n",
    "    df['word_st_index'] = st\n",
    "    df['word_en_index'] = en\n",
    "    df['sent_start_index'] = sn_st\n",
    "    df['sent_end_index'] = sn_en\n",
    "    df1 = df.copy()#[df['sent'] != '\\n']\n",
    "    df1.index = range(0,len(df1))\n",
    "    word = []\n",
    "    sent_id = []\n",
    "    article_sent_id = []\n",
    "    art_id = []\n",
    "    lab_id = []\n",
    "    word_st = []\n",
    "    word_en = []\n",
    "    sent_start_index = []\n",
    "    sent_end_index = []\n",
    "\n",
    "    for i in tqdm(range(0,len(df1))):\n",
    "        sent_ls = df1['sent'].iloc[i].split()\n",
    "        art = df1['article'].iloc[i]\n",
    "        sent_start = df1['sent_start_index'].iloc[i]\n",
    "        sent_end = df1['sent_end_index'].iloc[i]\n",
    "        lab = df1['label'].iloc[i]\n",
    "        wst = df1['word_st_index'].iloc[i]\n",
    "        wen = df1['word_en_index'].iloc[i]\n",
    "        article_sent_id += list(range(len(sent_ls)))\n",
    "\n",
    "        for j in range(0,len(sent_ls)):\n",
    "            word.append(sent_ls[j])\n",
    "            lab_id.append(lab[j])\n",
    "            sent_id.append(i)\n",
    "            sent_start_index.append(sent_start)\n",
    "            sent_end_index.append(sent_end)\n",
    "            art_id.append(art)\n",
    "            word_st.append(wst[j])\n",
    "            word_en.append(wen[j])\n",
    "\n",
    "    df_final = pd.DataFrame()\n",
    "    df_final['article_id'] = art_id\n",
    "    df_final['sent_id'] = sent_id\n",
    "    df_final['word'] = word\n",
    "    df_final['label'] = lab_id\n",
    "    df_final['word_st_index'] = word_st\n",
    "    df_final['word_en_index'] = word_en\n",
    "    df_final['sent_start_index'] = sent_start_index\n",
    "    df_final['sent_end_index'] = sent_end_index\n",
    "    # df_final['article_sent_id'] = article_sent_id\n",
    "    df_final['word_corrected'] = df_final['word'].apply(lambda x: text_preprocessing(x))\n",
    "    df_final_corr = df_final[df_final['word_corrected'] != '']\n",
    "    print('shape of df_final_corr: ', df_final_corr.shape)\n",
    "    print(round((time.time()-start)/60, 2), \"Mins lapsed\")\n",
    "    return df_final_corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_article_ids = [int(file.replace('article', '').replace('.txt', '')) for file in os.listdir(train_articles_path)]\n",
    "dev_article_ids = [int(file.replace('article', '').replace('.txt', '')) for file in os.listdir(dev_articles_path)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "train_dict_all=[]\n",
    "for req_id in train_article_ids:\n",
    "    print(req_id)\n",
    "    se_dict, req_labels = get_sentence_data(req_id, y_path_train, train_articles_path)\n",
    "    train_dict_all.append(se_dict)\n",
    "print(round((time.time()-start)/60, 2), \"Mins lapsed\")\n",
    "\n",
    "start = time.time()\n",
    "dev_dict_all=[]\n",
    "for req_id in dev_article_ids:\n",
    "    print(req_id)\n",
    "    se_dict, req_labels = get_sentence_data(req_id, y_path_dev, dev_articles_path)\n",
    "    dev_dict_all.append(se_dict)\n",
    "print(round((time.time()-start)/60, 2), \"Mins lapsed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "with open(semeval_path+'data/dict_train_SI.pickle', 'wb') as handle:\n",
    "    pickle.dump(train_dict_all, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "with open(semeval_path+'data/dict_dev_SI.pickle', 'wb') as handle:\n",
    "    pickle.dump(dict_all, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_prop_df(train_dict_all).to_csv(semeval_path+'data/data_prop.csv',index=False)\n",
    "get_prop_df(dev_dict_all).to_csv(semeval_path+'data/data_prop_vDev.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}