roberta-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/roberta-base/config.json\n",
      "/kaggle/input/roberta-base/merges.txt\n",
      "/kaggle/input/roberta-base/vocab.json\n",
      "/kaggle/input/roberta-base/pytorch_model.bin\n",
      "/kaggle/input/tweet-sentiment-extraction/train.csv\n",
      "/kaggle/input/tweet-sentiment-extraction/test.csv\n",
      "/kaggle/input/tweet-sentiment-extraction/sample_submission.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import warnings\n",
    "import random\n",
    "import torch \n",
    "from torch import nn\n",
    "import torch.optim as optim\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import tokenizers\n",
    "from transformers import RobertaModel, RobertaConfig\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch. cuda. is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "def seed_everything(seed_value):\n",
    "    random.seed(seed_value)\n",
    "    np.random.seed(seed_value)\n",
    "    torch.manual_seed(seed_value)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed_value)\n",
    "    \n",
    "    if torch.cuda.is_available(): \n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value)\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = True\n",
    "\n",
    "seed = 42\n",
    "seed_everything(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TweetDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, df, max_len=96):\n",
    "        self.df = df\n",
    "        self.max_len = max_len\n",
    "        self.labeled = 'selected_text' in df\n",
    "        self.tokenizer = tokenizers.ByteLevelBPETokenizer(\n",
    "            vocab_file='../input/roberta-base/vocab.json', \n",
    "            merges_file='../input/roberta-base/merges.txt', \n",
    "            lowercase=True,\n",
    "            add_prefix_space=True)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        data = {}\n",
    "        row = self.df.iloc[index]\n",
    "        \n",
    "        ids, masks, tweet, offsets = self.get_input_data(row)\n",
    "        data['ids'] = ids\n",
    "        data['masks'] = masks\n",
    "        data['tweet'] = tweet\n",
    "        data['offsets'] = offsets\n",
    "        \n",
    "        if self.labeled:\n",
    "            start_idx, end_idx = self.get_target_idx(row, tweet, offsets)\n",
    "            data['start_idx'] = start_idx\n",
    "            data['end_idx'] = end_idx\n",
    "        \n",
    "        return data\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def get_input_data(self, row):\n",
    "        tweet = \" \" + \" \".join(row.text.lower().split())\n",
    "        encoding = self.tokenizer.encode(tweet)\n",
    "        sentiment_id = self.tokenizer.encode(row.sentiment).ids\n",
    "        ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]\n",
    "        offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]\n",
    "                \n",
    "        pad_len = self.max_len - len(ids)\n",
    "        if pad_len > 0:\n",
    "            ids += [1] * pad_len\n",
    "            offsets += [(0, 0)] * pad_len\n",
    "        \n",
    "        ids = torch.tensor(ids)\n",
    "        masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))\n",
    "        offsets = torch.tensor(offsets)\n",
    "        \n",
    "        return ids, masks, tweet, offsets\n",
    "        \n",
    "    def get_target_idx(self, row, tweet, offsets):\n",
    "        selected_text = \" \" +  \" \".join(row.selected_text.lower().split())\n",
    "\n",
    "        len_st = len(selected_text) - 1\n",
    "        idx0 = None\n",
    "        idx1 = None\n",
    "\n",
    "        for ind in (i for i, e in enumerate(tweet) if e == selected_text[1]):\n",
    "            if \" \" + tweet[ind: ind+len_st] == selected_text:\n",
    "                idx0 = ind\n",
    "                idx1 = ind + len_st - 1\n",
    "                break\n",
    "\n",
    "        char_targets = [0] * len(tweet)\n",
    "        if idx0 != None and idx1 != None:\n",
    "            for ct in range(idx0, idx1 + 1):\n",
    "                char_targets[ct] = 1\n",
    "\n",
    "        target_idx = []\n",
    "        for j, (offset1, offset2) in enumerate(offsets):\n",
    "            if sum(char_targets[offset1: offset2]) > 0:\n",
    "                target_idx.append(j)\n",
    "\n",
    "        start_idx = target_idx[0]\n",
    "        end_idx = target_idx[-1]\n",
    "        \n",
    "        return start_idx, end_idx\n",
    "        \n",
    "def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):\n",
    "    train_df = df.iloc[train_idx]\n",
    "    val_df = df.iloc[val_idx]\n",
    "\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        TweetDataset(train_df), \n",
    "        batch_size=batch_size, \n",
    "        shuffle=True, \n",
    "        num_workers=2,\n",
    "        drop_last=True)\n",
    "\n",
    "    val_loader = torch.utils.data.DataLoader(\n",
    "        TweetDataset(val_df), \n",
    "        batch_size=batch_size, \n",
    "        shuffle=False, \n",
    "        num_workers=2)\n",
    "\n",
    "    dataloaders_dict = {\"train\": train_loader, \"val\": val_loader}\n",
    "\n",
    "    return dataloaders_dict\n",
    "\n",
    "def get_test_loader(df, batch_size=32):\n",
    "    loader = torch.utils.data.DataLoader(\n",
    "        TweetDataset(df), \n",
    "        batch_size=batch_size, \n",
    "        shuffle=False, \n",
    "        num_workers=2)    \n",
    "    return loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TweetModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(TweetModel, self).__init__()\n",
    "        \n",
    "        config = RobertaConfig.from_pretrained(\n",
    "            '../input/roberta-base/config.json', output_hidden_states=True)    \n",
    "        self.roberta = RobertaModel.from_pretrained(\n",
    "            '../input/roberta-base/pytorch_model.bin', config=config)\n",
    "        self.dropout = nn.Dropout(0.5)\n",
    "        self.fc = nn.Linear(config.hidden_size, 2)\n",
    "        nn.init.normal_(self.fc.weight, std=0.02)\n",
    "        nn.init.normal_(self.fc.bias, 0)\n",
    "\n",
    "    def forward(self, input_ids, attention_mask):\n",
    "        _, _, hs = self.roberta(input_ids, attention_mask)\n",
    "         \n",
    "        x = torch.stack([hs[-1], hs[-2], hs[-3], hs[-4]])\n",
    "        x = torch.mean(x, 0)\n",
    "        x = self.dropout(x)\n",
    "        x = self.fc(x)\n",
    "        start_logits, end_logits = x.split(1, dim=-1)\n",
    "        start_logits = start_logits.squeeze(-1)\n",
    "        end_logits = end_logits.squeeze(-1)\n",
    "                \n",
    "        return start_logits, end_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(start_logits, end_logits, start_positions, end_positions):\n",
    "    ce_loss = nn.CrossEntropyLoss()\n",
    "    start_loss = ce_loss(start_logits, start_positions)\n",
    "    end_loss = ce_loss(end_logits, end_positions)    \n",
    "    total_loss = start_loss + end_loss\n",
    "    return total_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_selected_text(text, start_idx, end_idx, offsets):\n",
    "    selected_text = \"\"\n",
    "    for ix in range(start_idx, end_idx + 1):\n",
    "        selected_text += text[offsets[ix][0]: offsets[ix][1]]\n",
    "        if (ix + 1) < len(offsets) and offsets[ix][1] < offsets[ix + 1][0]:\n",
    "            selected_text += \" \"\n",
    "    return selected_text\n",
    "\n",
    "def jaccard(str1, str2): \n",
    "    a = set(str1.lower().split()) \n",
    "    b = set(str2.lower().split())\n",
    "    c = a.intersection(b)\n",
    "    return float(len(c)) / (len(a) + len(b) - len(c))\n",
    "\n",
    "def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):\n",
    "    start_pred = np.argmax(start_logits)\n",
    "    end_pred = np.argmax(end_logits)\n",
    "    if start_pred > end_pred:\n",
    "        pred = text\n",
    "    else:\n",
    "        pred = get_selected_text(text, start_pred, end_pred, offsets)\n",
    "        \n",
    "    true = get_selected_text(text, start_idx, end_idx, offsets)\n",
    "    \n",
    "    return jaccard(true, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):\n",
    "    model.cuda()\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                model.train()\n",
    "            else:\n",
    "                model.eval()\n",
    "\n",
    "            epoch_loss = 0.0\n",
    "            epoch_jaccard = 0.0\n",
    "            \n",
    "            for data in (dataloaders_dict[phase]):\n",
    "                ids = data['ids'].cuda()\n",
    "                masks = data['masks'].cuda()\n",
    "                tweet = data['tweet']\n",
    "                offsets = data['offsets'].numpy()\n",
    "                start_idx = data['start_idx'].cuda()\n",
    "                end_idx = data['end_idx'].cuda()\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "\n",
    "                    start_logits, end_logits = model(ids, masks)\n",
    "\n",
    "                    loss = criterion(start_logits, end_logits, start_idx, end_idx)\n",
    "                    \n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                    epoch_loss += loss.item() * len(ids)\n",
    "                    \n",
    "                    start_idx = start_idx.cpu().detach().numpy()\n",
    "                    end_idx = end_idx.cpu().detach().numpy()\n",
    "                    start_logits = torch.softmax(start_logits, dim=1).cpu().detach().numpy()\n",
    "                    end_logits = torch.softmax(end_logits, dim=1).cpu().detach().numpy()\n",
    "                    \n",
    "                    for i in range(len(ids)):                        \n",
    "                        jaccard_score = compute_jaccard_score(\n",
    "                            tweet[i],\n",
    "                            start_idx[i],\n",
    "                            end_idx[i],\n",
    "                            start_logits[i], \n",
    "                            end_logits[i], \n",
    "                            offsets[i])\n",
    "                        epoch_jaccard += jaccard_score\n",
    "                    \n",
    "            epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)\n",
    "            epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)\n",
    "            \n",
    "            print('Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}'.format(\n",
    "                epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard))\n",
    "    \n",
    "    torch.save(model.state_dict(), filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 3\n",
    "batch_size = 32\n",
    "skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 1\n",
      "Epoch 1/3 | train | Loss: 2.1321 | Jaccard: 0.6634\n",
      "Epoch 1/3 |  val  | Loss: 1.6480 | Jaccard: 0.7166\n",
      "Epoch 2/3 | train | Loss: 1.6013 | Jaccard: 0.7182\n",
      "Epoch 2/3 |  val  | Loss: 1.6582 | Jaccard: 0.7059\n",
      "Epoch 3/3 | train | Loss: 1.4530 | Jaccard: 0.7373\n",
      "Epoch 3/3 |  val  | Loss: 1.7208 | Jaccard: 0.7131\n",
      "Fold: 2\n",
      "Epoch 1/3 | train | Loss: 2.1968 | Jaccard: 0.6553\n",
      "Epoch 1/3 |  val  | Loss: 1.7515 | Jaccard: 0.6925\n",
      "Epoch 2/3 | train | Loss: 1.6998 | Jaccard: 0.7068\n",
      "Epoch 2/3 |  val  | Loss: 1.6465 | Jaccard: 0.7124\n",
      "Epoch 3/3 | train | Loss: 1.5439 | Jaccard: 0.7252\n",
      "Epoch 3/3 |  val  | Loss: 1.6395 | Jaccard: 0.7160\n",
      "Fold: 3\n",
      "Epoch 1/3 | train | Loss: 2.1237 | Jaccard: 0.6675\n",
      "Epoch 1/3 |  val  | Loss: 1.7006 | Jaccard: 0.7078\n",
      "Epoch 2/3 | train | Loss: 1.6322 | Jaccard: 0.7172\n",
      "Epoch 2/3 |  val  | Loss: 1.6523 | Jaccard: 0.7141\n",
      "Epoch 3/3 | train | Loss: 1.4792 | Jaccard: 0.7337\n",
      "Epoch 3/3 |  val  | Loss: 1.6673 | Jaccard: 0.7169\n",
      "Fold: 4\n",
      "Epoch 1/3 | train | Loss: 2.1617 | Jaccard: 0.6601\n",
      "Epoch 1/3 |  val  | Loss: 1.5938 | Jaccard: 0.7212\n",
      "Epoch 2/3 | train | Loss: 1.6270 | Jaccard: 0.7123\n",
      "Epoch 2/3 |  val  | Loss: 1.5249 | Jaccard: 0.7308\n",
      "Epoch 3/3 | train | Loss: 1.4600 | Jaccard: 0.7373\n",
      "Epoch 3/3 |  val  | Loss: 1.5597 | Jaccard: 0.7247\n",
      "Fold: 5\n",
      "Epoch 1/3 | train | Loss: 2.0595 | Jaccard: 0.6713\n",
      "Epoch 1/3 |  val  | Loss: 1.6322 | Jaccard: 0.7195\n",
      "Epoch 2/3 | train | Loss: 1.5929 | Jaccard: 0.7210\n",
      "Epoch 2/3 |  val  | Loss: 1.6388 | Jaccard: 0.7158\n",
      "Epoch 3/3 | train | Loss: 1.5069 | Jaccard: 0.7328\n",
      "Epoch 3/3 |  val  | Loss: 1.6546 | Jaccard: 0.7208\n",
      "Fold: 6\n",
      "Epoch 1/3 | train | Loss: 2.0796 | Jaccard: 0.6708\n",
      "Epoch 1/3 |  val  | Loss: 1.6544 | Jaccard: 0.7078\n",
      "Epoch 2/3 | train | Loss: 1.5919 | Jaccard: 0.7180\n",
      "Epoch 2/3 |  val  | Loss: 1.6450 | Jaccard: 0.7173\n",
      "Epoch 3/3 | train | Loss: 1.4351 | Jaccard: 0.7392\n",
      "Epoch 3/3 |  val  | Loss: 1.6319 | Jaccard: 0.7206\n",
      "Fold: 7\n",
      "Epoch 1/3 | train | Loss: 2.1183 | Jaccard: 0.6655\n",
      "Epoch 1/3 |  val  | Loss: 1.5971 | Jaccard: 0.7187\n",
      "Epoch 2/3 | train | Loss: 1.6268 | Jaccard: 0.7120\n",
      "Epoch 2/3 |  val  | Loss: 1.5684 | Jaccard: 0.7219\n",
      "Epoch 3/3 | train | Loss: 1.4952 | Jaccard: 0.7317\n",
      "Epoch 3/3 |  val  | Loss: 1.5916 | Jaccard: 0.7231\n",
      "Fold: 8\n",
      "Epoch 1/3 | train | Loss: 2.1916 | Jaccard: 0.6603\n",
      "Epoch 1/3 |  val  | Loss: 1.6748 | Jaccard: 0.7050\n",
      "Epoch 2/3 | train | Loss: 1.6450 | Jaccard: 0.7120\n",
      "Epoch 2/3 |  val  | Loss: 1.6062 | Jaccard: 0.7196\n",
      "Epoch 3/3 | train | Loss: 1.5097 | Jaccard: 0.7304\n",
      "Epoch 3/3 |  val  | Loss: 1.6520 | Jaccard: 0.7165\n",
      "Fold: 9\n",
      "Epoch 1/3 | train | Loss: 2.1056 | Jaccard: 0.6662\n",
      "Epoch 1/3 |  val  | Loss: 1.7136 | Jaccard: 0.7122\n",
      "Epoch 2/3 | train | Loss: 1.6063 | Jaccard: 0.7147\n",
      "Epoch 2/3 |  val  | Loss: 1.6593 | Jaccard: 0.7206\n",
      "Epoch 3/3 | train | Loss: 1.4619 | Jaccard: 0.7339\n",
      "Epoch 3/3 |  val  | Loss: 1.6499 | Jaccard: 0.7212\n",
      "Fold: 10\n",
      "Epoch 1/3 | train | Loss: 2.0821 | Jaccard: 0.6699\n",
      "Epoch 1/3 |  val  | Loss: 1.6264 | Jaccard: 0.7104\n",
      "Epoch 2/3 | train | Loss: 1.5766 | Jaccard: 0.7236\n",
      "Epoch 2/3 |  val  | Loss: 1.6017 | Jaccard: 0.7184\n",
      "Epoch 3/3 | train | Loss: 1.4345 | Jaccard: 0.7424\n",
      "Epoch 3/3 |  val  | Loss: 1.6217 | Jaccard: 0.7186\n",
      "CPU times: user 1h 24min 40s, sys: 35min 17s, total: 1h 59min 58s\n",
      "Wall time: 2h 5min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "train_df = pd.read_csv('../input/tweet-sentiment-extraction/train.csv')\n",
    "train_df['text'] = train_df['text'].astype(str)\n",
    "train_df['selected_text'] = train_df['selected_text'].astype(str)\n",
    "\n",
    "for fold, (train_idx, val_idx) in enumerate(skf.split(train_df, train_df.sentiment), start=1): \n",
    "    print(f'Fold: {fold}')\n",
    "\n",
    "    model = TweetModel()\n",
    "    optimizer = optim.AdamW(model.parameters(), lr=3e-5, betas=(0.9, 0.999))\n",
    "    criterion = loss_fn    \n",
    "    dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, batch_size)\n",
    "\n",
    "    train_model(\n",
    "        model, \n",
    "        dataloaders_dict,\n",
    "        criterion, \n",
    "        optimizer, \n",
    "        num_epochs,\n",
    "        f'roberta_fold{fold}.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 51s, sys: 41.5 s, total: 2min 33s\n",
      "Wall time: 2min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "test_df = pd.read_csv('../input/tweet-sentiment-extraction/test.csv')\n",
    "test_df['text'] = test_df['text'].astype(str)\n",
    "test_loader = get_test_loader(test_df)\n",
    "predictions = []\n",
    "models = []\n",
    "for fold in range(skf.n_splits):\n",
    "    model = TweetModel()\n",
    "    model.cuda()\n",
    "    model.load_state_dict(torch.load(f'roberta_fold{fold+1}.pth'))\n",
    "    model.eval()\n",
    "    models.append(model)\n",
    "\n",
    "for data in test_loader:\n",
    "    ids = data['ids'].cuda()\n",
    "    masks = data['masks'].cuda()\n",
    "    tweet = data['tweet']\n",
    "    offsets = data['offsets'].numpy()\n",
    "\n",
    "    start_logits = []\n",
    "    end_logits = []\n",
    "    for model in models:\n",
    "        with torch.no_grad():\n",
    "            output = model(ids, masks)\n",
    "            start_logits.append(torch.softmax(output[0], dim=1).cpu().detach().numpy())\n",
    "            end_logits.append(torch.softmax(output[1], dim=1).cpu().detach().numpy())\n",
    "\n",
    "    start_logits = np.mean(start_logits, axis=0)\n",
    "    end_logits = np.mean(end_logits, axis=0)\n",
    "    for i in range(len(ids)):    \n",
    "        start_pred = np.argmax(start_logits[i])\n",
    "        end_pred = np.argmax(end_logits[i])\n",
    "        if start_pred > end_pred:\n",
    "            pred = tweet[i]\n",
    "        else:\n",
    "            pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])\n",
    "        predictions.append(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>textID</th>\n",
       "      <th>selected_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f87dea47db</td>\n",
       "      <td>last session of the day</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>96d74cb729</td>\n",
       "      <td>exciting</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eee518ae67</td>\n",
       "      <td>such a shame!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01082688c6</td>\n",
       "      <td>happy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33987a8ee5</td>\n",
       "      <td>i like it!!</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       textID             selected_text\n",
       "0  f87dea47db   last session of the day\n",
       "1  96d74cb729                  exciting\n",
       "2  eee518ae67             such a shame!\n",
       "3  01082688c6                     happy\n",
       "4  33987a8ee5               i like it!!"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df = pd.read_csv('../input/tweet-sentiment-extraction/sample_submission.csv')\n",
    "sub_df['selected_text'] = predictions\n",
    "sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!') if len(x.split())==1 else x)\n",
    "sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('..', '.') if len(x.split())==1 else x)\n",
    "sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.') if len(x.split())==1 else x)\n",
    "sub_df.to_csv('submission.csv', index=False)\n",
    "sub_df.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}