ALBERT + BiLSTMEncoder + ALBERT-SQuAD-OUT.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ALBERT + BiLSTM Encoder + ALBERT-SQuAD-OUT\n",
    "\n",
    "Implementaion of ALBERT is taken from Hugging Face library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import time\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "import json\n",
    "import pandas as pd\n",
    "from squad import Squad\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "from transformers import (\n",
    "    AlbertConfig,\n",
    "    AlbertModel,\n",
    "    AlbertTokenizer,\n",
    "    squad_convert_examples_to_features\n",
    ")\n",
    "\n",
    "from transformers.data.processors.squad import SquadResult, SquadV2Processor, SquadExample\n",
    "from transformers.data.metrics.squad_metrics import compute_predictions_logits, get_final_text\n",
    "from evaluate_answers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = \"\"\n",
    "do_lower_case = True\n",
    "\n",
    "# Tokenizer for ALBERT's input format\n",
    "tokenizer_class = AlbertTokenizer\n",
    "tokenizer = tokenizer_class.from_pretrained(\n",
    "    \"albert-base-v2\", do_lower_case=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating train and test dataframes\n",
    "train_sq = Squad(\"./data/train-v2.0.json\")\n",
    "test_sq = Squad(\"./data/dev-v2.0.json\")\n",
    "train_df = train_sq.get_dataframe()\n",
    "test_df = test_sq.get_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_train_dataset(train_df, tokenizer):\n",
    "    \"\"\"\n",
    "    Create dataset from DataFrame\n",
    "    \n",
    "    returns: \n",
    "        dataset - pytorch dataset of training data features\n",
    "    \"\"\"\n",
    "    examples = []\n",
    "    for i, question in enumerate(train_df['content']):\n",
    "        example = SquadExample(\n",
    "            qas_id=str(i),\n",
    "            question_text=question,\n",
    "            context_text=train_df['context'][i],\n",
    "            answer_text=train_df['answer'][i],\n",
    "            start_position_character=train_df['answer_start'][i],\n",
    "            title=\"Train\",\n",
    "            is_impossible=False,\n",
    "            answers=None,\n",
    "        )\n",
    "        examples.append(example)\n",
    "    \n",
    "    features, dataset = squad_convert_examples_to_features(\n",
    "        examples=examples,\n",
    "        tokenizer=tokenizer,\n",
    "        max_seq_length=384,\n",
    "        doc_stride=128,\n",
    "        max_query_length=64,\n",
    "        is_training=True,\n",
    "        return_dataset=\"pt\",\n",
    "        threads=32,\n",
    "    )\n",
    "    \n",
    "    \n",
    "    return dataset, features, examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "convert squad examples to features: 100%|██████████| 130319/130319 [01:54<00:00, 1140.35it/s]\n",
      "add example index and unique id: 100%|██████████| 130319/130319 [00:00<00:00, 579110.50it/s]\n"
     ]
    }
   ],
   "source": [
    "dataset, _, _ = create_train_dataset(train_df, tokenizer)\n",
    "\n",
    "\n",
    "train_sampler = SequentialSampler(dataset)\n",
    "train_dataloader = DataLoader(dataset, sampler=train_sampler, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AlbertBiLSTM4QA(nn.Module):\n",
    "    def __init__(self, freeze_albert = True):\n",
    "        super(AlbertBiLSTM4QA, self).__init__()\n",
    "        \n",
    "        # create model's config\n",
    "        config_class, model_class = (AlbertConfig, AlbertModel)\n",
    "        config = config_class.from_pretrained(\"albert-base-v2\")\n",
    "        config.output_hidden_states=True\n",
    "        self.backbone = model_class.from_pretrained(\"albert-base-v2\", config=config)\n",
    "        \n",
    "        # freeze ALBERT layers if freeze_albert is True\n",
    "        if freeze_albert:\n",
    "            for param in self.backbone.embeddings.parameters():\n",
    "                param.requires_grad = False\n",
    "\n",
    "            for param in self.backbone.encoder.parameters():\n",
    "                param.requires_grad = False\n",
    "\n",
    "            for param in self.backbone.pooler.parameters():\n",
    "                param.requires_grad = False\n",
    "\n",
    "            for param in self.backbone.pooler_activation.parameters():\n",
    "                param.requires_grad = False\n",
    "        \n",
    "        self.encoder = nn.LSTM(768, 384, num_layers=3, dropout=0.2, bidirectional=True)\n",
    "        \n",
    "        self.QA = nn.Sequential(\n",
    "            nn.Linear(768,2)\n",
    "        )\n",
    "        \n",
    "    \n",
    "    def forward(self, batch, device='cpu'):\n",
    "        # inference through ALBERT\n",
    "        self.backbone.eval()\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        with torch.no_grad():\n",
    "            inputs = {\n",
    "                \"input_ids\": batch[0],\n",
    "                \"attention_mask\": batch[1],\n",
    "                \"token_type_ids\": batch[2],\n",
    "            }\n",
    "\n",
    "            features, _, _ = self.backbone(**inputs)\n",
    "        \n",
    "        # permute features dimensions to satisfy BiLSTM's input (seq_inp, batch_size, hidden_state)\n",
    "        features = features.permute(1,0,2)\n",
    "        x, _ = self.encoder(features)\n",
    "        # permute back\n",
    "        x = x.permute(1,0,2)\n",
    "        logits = self.QA(x)\n",
    "        \n",
    "        # get start and end logits also calculate loss\n",
    "        start_logits, end_logits = logits.split(1, dim=-1)\n",
    "        start_logits = start_logits.squeeze(-1).squeeze(1)\n",
    "        end_logits = end_logits.squeeze(-1).squeeze(1)\n",
    "        \n",
    "        start_positions = batch[3]\n",
    "        end_positions = batch[4]\n",
    "        \n",
    "        \n",
    "        if start_positions is not None and end_positions is not None:\n",
    "            # If we are on multi-GPU, split add a dimension\n",
    "            if len(start_positions.size()) > 1:\n",
    "                start_positions = start_positions.squeeze(-1)\n",
    "            if len(end_positions.size()) > 1:\n",
    "                end_positions = end_positions.squeeze(-1)\n",
    "            # sometimes the start/end positions are outside our model inputs, we ignore these terms\n",
    "            ignored_index = start_logits.size(1)\n",
    "            start_positions.clamp_(0, ignored_index)\n",
    "            end_positions.clamp_(0, ignored_index)\n",
    "\n",
    "            loss_fct = nn.CrossEntropyLoss(ignore_index=ignored_index)\n",
    "            start_loss = loss_fct(start_logits, start_positions)\n",
    "            end_loss = loss_fct(end_logits, end_positions)\n",
    "            total_loss = (start_loss + end_loss)/2\n",
    "        return total_loss, start_logits, end_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "device = torch.device(\"cuda:1\" if torch.cuda.is_available() else \"cpu\")\n",
    "modelA = AlbertBiLSTM4QA(freeze_albert=True).to(device)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.AdamW(modelA.parameters(), lr=3e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, epochs, optimizer):\n",
    "    model.zero_grad()\n",
    "    f = open(\"logs.txt\", \"w\")\n",
    "    for epoch in range(epochs):\n",
    "        loss = 0\n",
    "        for idx, batch in enumerate(train_dataloader):\n",
    "            model_loss ,start_logits, end_logits = model(batch,device=device)\n",
    "            loss += model_loss.item()          \n",
    "            model_loss.backward()\n",
    "            optimizer.step()\n",
    "            model.zero_grad()\n",
    "            \n",
    "            if idx % 100 == 0:\n",
    "                start_pred = torch.argmax(start_logits, dim=1).cpu()\n",
    "                end_pred = torch.argmax(end_logits, dim=1).cpu()\n",
    "                pair_accuracy = ((start_pred==batch[3])*(end_pred==batch[4])).sum().float() / len(batch[3])\n",
    "                start_accuracy = (start_pred==batch[3]).sum().float() / len(batch[3])\n",
    "                end_accuracy = (end_pred==batch[4]).sum().float() / len(batch[4])\n",
    "                string = f\"[{idx+1}/{len(train_dataloader)}]Epoch: {epoch+1}/{epochs} Loss: {model_loss.item()} Pair Accuracy: {pair_accuracy} Start Accuracy: {start_accuracy} End Accuracy: {end_accuracy}\"\n",
    "                print(string)\n",
    "                f.write(string)\n",
    "                torch.save(model.state_dict(), \"AlbertBiLSTM.pth\")\n",
    "    f.close()\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1/4132]Epoch: 1/2 Loss: 5.955636501312256 Pair Accuracy: 0.0 Start Accuracy: 0.0 End Accuracy: 0.0\n",
      "[101/4132]Epoch: 1/2 Loss: 4.423079490661621 Pair Accuracy: 0.0625 Start Accuracy: 0.21875 End Accuracy: 0.09375\n",
      "[201/4132]Epoch: 1/2 Loss: 2.959162712097168 Pair Accuracy: 0.125 Start Accuracy: 0.25 End Accuracy: 0.28125\n",
      "[301/4132]Epoch: 1/2 Loss: 3.1466174125671387 Pair Accuracy: 0.34375 Start Accuracy: 0.34375 End Accuracy: 0.5\n",
      "[401/4132]Epoch: 1/2 Loss: 3.51151967048645 Pair Accuracy: 0.09375 Start Accuracy: 0.15625 End Accuracy: 0.34375\n",
      "[501/4132]Epoch: 1/2 Loss: 3.3394079208374023 Pair Accuracy: 0.0 Start Accuracy: 0.1875 End Accuracy: 0.0\n",
      "[601/4132]Epoch: 1/2 Loss: 2.8850038051605225 Pair Accuracy: 0.0625 Start Accuracy: 0.25 End Accuracy: 0.21875\n",
      "[701/4132]Epoch: 1/2 Loss: 1.8648595809936523 Pair Accuracy: 0.28125 Start Accuracy: 0.46875 End Accuracy: 0.59375\n",
      "[801/4132]Epoch: 1/2 Loss: 1.7171783447265625 Pair Accuracy: 0.3125 Start Accuracy: 0.5 End Accuracy: 0.5\n",
      "[901/4132]Epoch: 1/2 Loss: 2.7154674530029297 Pair Accuracy: 0.15625 Start Accuracy: 0.3125 End Accuracy: 0.25\n",
      "[1001/4132]Epoch: 1/2 Loss: 2.366820812225342 Pair Accuracy: 0.15625 Start Accuracy: 0.28125 End Accuracy: 0.40625\n",
      "[1101/4132]Epoch: 1/2 Loss: 1.4861228466033936 Pair Accuracy: 0.53125 Start Accuracy: 0.625 End Accuracy: 0.5625\n",
      "[1201/4132]Epoch: 1/2 Loss: 2.5327184200286865 Pair Accuracy: 0.4375 Start Accuracy: 0.46875 End Accuracy: 0.4375\n",
      "[1301/4132]Epoch: 1/2 Loss: 2.5535993576049805 Pair Accuracy: 0.1875 Start Accuracy: 0.25 End Accuracy: 0.34375\n",
      "[1401/4132]Epoch: 1/2 Loss: 1.7709522247314453 Pair Accuracy: 0.34375 Start Accuracy: 0.46875 End Accuracy: 0.40625\n",
      "[1501/4132]Epoch: 1/2 Loss: 2.239068031311035 Pair Accuracy: 0.28125 Start Accuracy: 0.4375 End Accuracy: 0.375\n",
      "[1601/4132]Epoch: 1/2 Loss: 1.767030954360962 Pair Accuracy: 0.3125 Start Accuracy: 0.40625 End Accuracy: 0.46875\n",
      "[1701/4132]Epoch: 1/2 Loss: 2.7709107398986816 Pair Accuracy: 0.09375 Start Accuracy: 0.1875 End Accuracy: 0.21875\n",
      "[1801/4132]Epoch: 1/2 Loss: 1.5243446826934814 Pair Accuracy: 0.375 Start Accuracy: 0.5 End Accuracy: 0.65625\n",
      "[1901/4132]Epoch: 1/2 Loss: 1.950761079788208 Pair Accuracy: 0.5 Start Accuracy: 0.5625 End Accuracy: 0.5625\n",
      "[2001/4132]Epoch: 1/2 Loss: 2.0453720092773438 Pair Accuracy: 0.3125 Start Accuracy: 0.59375 End Accuracy: 0.53125\n",
      "[2101/4132]Epoch: 1/2 Loss: 1.321547508239746 Pair Accuracy: 0.5625 Start Accuracy: 0.625 End Accuracy: 0.625\n",
      "[2201/4132]Epoch: 1/2 Loss: 1.5958325862884521 Pair Accuracy: 0.5625 Start Accuracy: 0.625 End Accuracy: 0.59375\n",
      "[2301/4132]Epoch: 1/2 Loss: 2.0125861167907715 Pair Accuracy: 0.3125 Start Accuracy: 0.46875 End Accuracy: 0.46875\n",
      "[2401/4132]Epoch: 1/2 Loss: 2.542776107788086 Pair Accuracy: 0.15625 Start Accuracy: 0.25 End Accuracy: 0.40625\n",
      "[2501/4132]Epoch: 1/2 Loss: 2.080246925354004 Pair Accuracy: 0.4375 Start Accuracy: 0.5625 End Accuracy: 0.5\n",
      "[2601/4132]Epoch: 1/2 Loss: 2.0168545246124268 Pair Accuracy: 0.40625 Start Accuracy: 0.46875 End Accuracy: 0.53125\n",
      "[2701/4132]Epoch: 1/2 Loss: 1.7234925031661987 Pair Accuracy: 0.4375 Start Accuracy: 0.65625 End Accuracy: 0.4375\n",
      "[2801/4132]Epoch: 1/2 Loss: 1.768089771270752 Pair Accuracy: 0.375 Start Accuracy: 0.5625 End Accuracy: 0.40625\n",
      "[2901/4132]Epoch: 1/2 Loss: 1.6096911430358887 Pair Accuracy: 0.40625 Start Accuracy: 0.78125 End Accuracy: 0.4375\n",
      "[3001/4132]Epoch: 1/2 Loss: 1.4435831308364868 Pair Accuracy: 0.40625 Start Accuracy: 0.5625 End Accuracy: 0.5\n",
      "[3101/4132]Epoch: 1/2 Loss: 1.7811684608459473 Pair Accuracy: 0.3125 Start Accuracy: 0.5625 End Accuracy: 0.375\n",
      "[3201/4132]Epoch: 1/2 Loss: 2.3634514808654785 Pair Accuracy: 0.21875 Start Accuracy: 0.34375 End Accuracy: 0.3125\n",
      "[3301/4132]Epoch: 1/2 Loss: 2.4118409156799316 Pair Accuracy: 0.40625 Start Accuracy: 0.5 End Accuracy: 0.5\n",
      "[3401/4132]Epoch: 1/2 Loss: 2.6797611713409424 Pair Accuracy: 0.25 Start Accuracy: 0.375 End Accuracy: 0.34375\n",
      "[3501/4132]Epoch: 1/2 Loss: 3.000250816345215 Pair Accuracy: 0.09375 Start Accuracy: 0.21875 End Accuracy: 0.28125\n",
      "[3601/4132]Epoch: 1/2 Loss: 1.717771291732788 Pair Accuracy: 0.28125 Start Accuracy: 0.40625 End Accuracy: 0.59375\n",
      "[3701/4132]Epoch: 1/2 Loss: 1.3209607601165771 Pair Accuracy: 0.53125 Start Accuracy: 0.65625 End Accuracy: 0.625\n",
      "[3801/4132]Epoch: 1/2 Loss: 1.893068790435791 Pair Accuracy: 0.25 Start Accuracy: 0.53125 End Accuracy: 0.375\n",
      "[3901/4132]Epoch: 1/2 Loss: 1.5139116048812866 Pair Accuracy: 0.4375 Start Accuracy: 0.65625 End Accuracy: 0.5\n",
      "[4001/4132]Epoch: 1/2 Loss: 2.4861912727355957 Pair Accuracy: 0.25 Start Accuracy: 0.3125 End Accuracy: 0.40625\n",
      "[4101/4132]Epoch: 1/2 Loss: 1.4378962516784668 Pair Accuracy: 0.5 Start Accuracy: 0.5625 End Accuracy: 0.59375\n",
      "[1/4132]Epoch: 2/2 Loss: 2.3320794105529785 Pair Accuracy: 0.25 Start Accuracy: 0.46875 End Accuracy: 0.4375\n",
      "[101/4132]Epoch: 2/2 Loss: 1.919039249420166 Pair Accuracy: 0.3125 Start Accuracy: 0.46875 End Accuracy: 0.4375\n",
      "[201/4132]Epoch: 2/2 Loss: 1.1291687488555908 Pair Accuracy: 0.625 Start Accuracy: 0.65625 End Accuracy: 0.71875\n",
      "[301/4132]Epoch: 2/2 Loss: 1.9106686115264893 Pair Accuracy: 0.34375 Start Accuracy: 0.4375 End Accuracy: 0.4375\n",
      "[401/4132]Epoch: 2/2 Loss: 1.76743483543396 Pair Accuracy: 0.34375 Start Accuracy: 0.375 End Accuracy: 0.5\n",
      "[501/4132]Epoch: 2/2 Loss: 2.8518176078796387 Pair Accuracy: 0.09375 Start Accuracy: 0.21875 End Accuracy: 0.1875\n",
      "[601/4132]Epoch: 2/2 Loss: 2.242246150970459 Pair Accuracy: 0.21875 Start Accuracy: 0.28125 End Accuracy: 0.3125\n",
      "[701/4132]Epoch: 2/2 Loss: 1.107692837715149 Pair Accuracy: 0.5 Start Accuracy: 0.59375 End Accuracy: 0.71875\n",
      "[801/4132]Epoch: 2/2 Loss: 1.1632826328277588 Pair Accuracy: 0.46875 Start Accuracy: 0.65625 End Accuracy: 0.625\n",
      "[901/4132]Epoch: 2/2 Loss: 2.0888619422912598 Pair Accuracy: 0.28125 Start Accuracy: 0.4375 End Accuracy: 0.46875\n",
      "[1001/4132]Epoch: 2/2 Loss: 1.7475814819335938 Pair Accuracy: 0.375 Start Accuracy: 0.46875 End Accuracy: 0.46875\n",
      "[1101/4132]Epoch: 2/2 Loss: 0.9362519979476929 Pair Accuracy: 0.65625 Start Accuracy: 0.75 End Accuracy: 0.71875\n",
      "[1201/4132]Epoch: 2/2 Loss: 2.2060019969940186 Pair Accuracy: 0.40625 Start Accuracy: 0.46875 End Accuracy: 0.46875\n",
      "[1301/4132]Epoch: 2/2 Loss: 1.9896152019500732 Pair Accuracy: 0.28125 Start Accuracy: 0.3125 End Accuracy: 0.46875\n",
      "[1401/4132]Epoch: 2/2 Loss: 1.556312084197998 Pair Accuracy: 0.375 Start Accuracy: 0.46875 End Accuracy: 0.46875\n",
      "[1501/4132]Epoch: 2/2 Loss: 2.0176281929016113 Pair Accuracy: 0.3125 Start Accuracy: 0.46875 End Accuracy: 0.40625\n",
      "[1601/4132]Epoch: 2/2 Loss: 1.299136757850647 Pair Accuracy: 0.40625 Start Accuracy: 0.5625 End Accuracy: 0.6875\n",
      "[1701/4132]Epoch: 2/2 Loss: 2.4529781341552734 Pair Accuracy: 0.21875 Start Accuracy: 0.28125 End Accuracy: 0.25\n",
      "[1801/4132]Epoch: 2/2 Loss: 1.1793994903564453 Pair Accuracy: 0.40625 Start Accuracy: 0.5 End Accuracy: 0.6875\n",
      "[1901/4132]Epoch: 2/2 Loss: 1.6730186939239502 Pair Accuracy: 0.53125 Start Accuracy: 0.59375 End Accuracy: 0.5625\n",
      "[2001/4132]Epoch: 2/2 Loss: 1.9466207027435303 Pair Accuracy: 0.375 Start Accuracy: 0.5625 End Accuracy: 0.53125\n",
      "[2101/4132]Epoch: 2/2 Loss: 1.107409954071045 Pair Accuracy: 0.5625 Start Accuracy: 0.625 End Accuracy: 0.625\n",
      "[2201/4132]Epoch: 2/2 Loss: 1.320951223373413 Pair Accuracy: 0.53125 Start Accuracy: 0.625 End Accuracy: 0.5625\n",
      "[2301/4132]Epoch: 2/2 Loss: 1.9186782836914062 Pair Accuracy: 0.1875 Start Accuracy: 0.375 End Accuracy: 0.3125\n",
      "[2401/4132]Epoch: 2/2 Loss: 2.3470940589904785 Pair Accuracy: 0.25 Start Accuracy: 0.28125 End Accuracy: 0.375\n",
      "[2501/4132]Epoch: 2/2 Loss: 1.7408943176269531 Pair Accuracy: 0.46875 Start Accuracy: 0.53125 End Accuracy: 0.59375\n",
      "[2601/4132]Epoch: 2/2 Loss: 1.4122235774993896 Pair Accuracy: 0.46875 Start Accuracy: 0.53125 End Accuracy: 0.5\n",
      "[2701/4132]Epoch: 2/2 Loss: 1.5177764892578125 Pair Accuracy: 0.625 Start Accuracy: 0.71875 End Accuracy: 0.625\n",
      "[2801/4132]Epoch: 2/2 Loss: 1.4988834857940674 Pair Accuracy: 0.46875 Start Accuracy: 0.65625 End Accuracy: 0.46875\n",
      "[2901/4132]Epoch: 2/2 Loss: 1.2755310535430908 Pair Accuracy: 0.53125 Start Accuracy: 0.78125 End Accuracy: 0.5625\n",
      "[3001/4132]Epoch: 2/2 Loss: 1.3560843467712402 Pair Accuracy: 0.46875 Start Accuracy: 0.59375 End Accuracy: 0.53125\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3101/4132]Epoch: 2/2 Loss: 1.6774897575378418 Pair Accuracy: 0.28125 Start Accuracy: 0.46875 End Accuracy: 0.4375\n",
      "[3201/4132]Epoch: 2/2 Loss: 1.8956737518310547 Pair Accuracy: 0.3125 Start Accuracy: 0.40625 End Accuracy: 0.46875\n",
      "[3301/4132]Epoch: 2/2 Loss: 2.358614444732666 Pair Accuracy: 0.375 Start Accuracy: 0.53125 End Accuracy: 0.4375\n",
      "[3401/4132]Epoch: 2/2 Loss: 2.3760995864868164 Pair Accuracy: 0.21875 Start Accuracy: 0.375 End Accuracy: 0.375\n",
      "[3501/4132]Epoch: 2/2 Loss: 2.8074779510498047 Pair Accuracy: 0.1875 Start Accuracy: 0.3125 End Accuracy: 0.34375\n",
      "[3601/4132]Epoch: 2/2 Loss: 1.4766547679901123 Pair Accuracy: 0.375 Start Accuracy: 0.5625 End Accuracy: 0.5625\n",
      "[3701/4132]Epoch: 2/2 Loss: 1.2715520858764648 Pair Accuracy: 0.5 Start Accuracy: 0.625 End Accuracy: 0.59375\n",
      "[3801/4132]Epoch: 2/2 Loss: 1.5756105184555054 Pair Accuracy: 0.34375 Start Accuracy: 0.59375 End Accuracy: 0.5\n",
      "[3901/4132]Epoch: 2/2 Loss: 1.393688440322876 Pair Accuracy: 0.375 Start Accuracy: 0.5 End Accuracy: 0.46875\n",
      "[4001/4132]Epoch: 2/2 Loss: 2.2664740085601807 Pair Accuracy: 0.28125 Start Accuracy: 0.34375 End Accuracy: 0.4375\n",
      "[4101/4132]Epoch: 2/2 Loss: 1.240060806274414 Pair Accuracy: 0.5 Start Accuracy: 0.5625 End Accuracy: 0.5625\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "AlbertBiLSTM4QA(\n",
       "  (backbone): AlbertModel(\n",
       "    (embeddings): AlbertEmbeddings(\n",
       "      (word_embeddings): Embedding(30000, 128, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 128)\n",
       "      (token_type_embeddings): Embedding(2, 128)\n",
       "      (LayerNorm): LayerNorm((128,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0, inplace=False)\n",
       "    )\n",
       "    (encoder): AlbertTransformer(\n",
       "      (embedding_hidden_mapping_in): Linear(in_features=128, out_features=768, bias=True)\n",
       "      (albert_layer_groups): ModuleList(\n",
       "        (0): AlbertLayerGroup(\n",
       "          (albert_layers): ModuleList(\n",
       "            (0): AlbertLayer(\n",
       "              (full_layer_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (attention): AlbertAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0, inplace=False)\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              )\n",
       "              (ffn): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (ffn_output): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (pooler_activation): Tanh()\n",
       "  )\n",
       "  (encoder): LSTM(768, 384, num_layers=3, dropout=0.2, bidirectional=True)\n",
       "  (QA): Sequential(\n",
       "    (0): Linear(in_features=768, out_features=2, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train(modelA, 2, optimizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load saved model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "modelA.load_state_dict(torch.load(\"AlbertBiLSTM.pth\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "convert squad examples to features: 100%|██████████| 11873/11873 [00:18<00:00, 655.36it/s]\n",
      "add example index and unique id: 100%|██████████| 11873/11873 [00:00<00:00, 507123.00it/s]\n"
     ]
    }
   ],
   "source": [
    "test_dataset, test_features, test_examples = create_train_dataset(test_df, tokenizer)\n",
    "\n",
    "test_sampler = SequentialSampler(test_dataset)\n",
    "test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "\n",
    "def predict(model):\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        to_return = []\n",
    "        for idx, batch in enumerate(tqdm(test_dataloader)):\n",
    "            _ ,start_logits, end_logits = model(batch,device=device)      \n",
    "            \n",
    "            # Predict start and end\n",
    "            start_pred = torch.argmax(start_logits, dim=1).cpu()\n",
    "            end_pred = torch.argmax(end_logits, dim=1).cpu()\n",
    "            \n",
    "            for start, end in zip(start_pred, end_pred):\n",
    "                to_return.append((start.item(), end.item()))\n",
    "    return to_return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_preds(results):\n",
    "    to_return = []\n",
    "    for res, feat, example in zip(results, test_features, test_examples):\n",
    "        if res[0] == 0 and res[1] == 0:\n",
    "            to_return.append('')\n",
    "        else:\n",
    "            #  Clamp to min max start and end\n",
    "            tok_tokens = feat.tokens[res[0] : (res[1] + 1)]\n",
    "            if res[0] < min(feat.token_to_orig_map):\n",
    "                start = min(feat.token_to_orig_map)\n",
    "            elif res[0] > max(feat.token_to_orig_map):\n",
    "                start = max(feat.token_to_orig_map)\n",
    "            else:\n",
    "                start = res[0]\n",
    "                \n",
    "            if res[1] < min(feat.token_to_orig_map):\n",
    "                end = min(feat.token_to_orig_map)\n",
    "            elif res[1] > max(feat.token_to_orig_map):\n",
    "                end = max(feat.token_to_orig_map)\n",
    "            else:\n",
    "                end = res[1]\n",
    "            \n",
    "            # Convert to predicted text from albert tokenizer tokens\n",
    "            orig_doc_start = feat.token_to_orig_map[start]\n",
    "            orig_doc_end = feat.token_to_orig_map[end]\n",
    "            orig_tokens = example.doc_tokens[orig_doc_start : (orig_doc_end + 1)]\n",
    "            tok_text = tokenizer.convert_tokens_to_string(tok_tokens)\n",
    "\n",
    "            tok_text = tok_text.strip()\n",
    "            tok_text = \" \".join(tok_text.split())\n",
    "            orig_text = \" \".join(orig_tokens)\n",
    "            final_text = get_final_text(tok_text, orig_text, True, True)\n",
    "\n",
    "            to_return.append(final_text)\n",
    "    \n",
    "    answers = {}\n",
    "    for text,row in zip(to_return, test_df.loc):\n",
    "        answers[row.id] = text\n",
    "    return answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b5fe8cb6ee74e108895b51c0626821e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=384.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "my_preds = predict(modelA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = evaluate_preds(my_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"exact\": 17.05550408489851,\n",
      "  \"f1\": 18.155467079522154,\n",
      "  \"total\": 11873,\n",
      "  \"HasAns_exact\": 0.47233468286099867,\n",
      "  \"HasAns_f1\": 2.675415086903988,\n",
      "  \"HasAns_total\": 5928,\n",
      "  \"NoAns_exact\": 33.59125315391085,\n",
      "  \"NoAns_f1\": 33.59125315391085,\n",
      "  \"NoAns_total\": 5945\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# For the more representetive results we have taken script that squad owner's have written to check predictions\n",
    "dataset = test_sq.data\n",
    "preds = res\n",
    "na_probs = {k: 0.0 for k in preds}\n",
    "\n",
    "qid_to_has_ans = make_qid_to_has_ans(dataset) \n",
    "has_ans_qids = [k for k, v in qid_to_has_ans.items() if v]\n",
    "no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v]\n",
    "exact_raw, f1_raw = get_raw_scores(dataset, preds)\n",
    "exact_thresh = apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans,\n",
    "                                      1.0)\n",
    "f1_thresh = apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans,\n",
    "                                   1.0)\n",
    "out_eval = make_eval_dict(exact_thresh, f1_thresh)\n",
    "if has_ans_qids:\n",
    "    has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids)\n",
    "    merge_eval(out_eval, has_ans_eval, 'HasAns')\n",
    "if no_ans_qids:\n",
    "    no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids)\n",
    "    merge_eval(out_eval, no_ans_eval, 'NoAns')\n",
    "print(json.dumps(out_eval, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context:  The Scotland Act 1998, which was passed by the Parliament of the United Kingdom and given royal assent by Queen Elizabeth II on 19 November 1998, governs the functions and role of the Scottish Parliament and delimits its legislative competence. The Scotland Act 2012 extends the devolved competencies. For the purposes of parliamentary sovereignty, the Parliament of the United Kingdom at Westminster continues to constitute the supreme legislature of Scotland. However, under the terms of the Scotland Act, Westminster agreed to devolve some of its responsibilities over Scottish domestic policy to the Scottish Parliament. Such \"devolved matters\" include education, health, agriculture and justice. The Scotland Act enabled the Scottish Parliament to pass primary legislation on these issues. A degree of domestic authority, and all foreign policy, remain with the UK Parliament in Westminster. The Scottish Parliament has the power to pass laws and has limited tax-varying capability. Another of the roles of the Parliament is to hold the Scottish Government to account.\n",
      "\n",
      "Question:  Who gave her royal assent to the Scotland Act of 1998?\n",
      "\n",
      "Answer:  Queen Elizabeth II\n",
      "\n",
      "Predicted answer:  governs the functions\n",
      "\n",
      "//////////////////// \n",
      "\n",
      "Context:  Building activity occurred in numerous noble palaces and churches during the later decades of the 17th century. One of the best examples of this architecture are Krasiński Palace (1677–1683), Wilanów Palace (1677–1696) and St. Kazimierz Church (1688–1692). The most impressive examples of rococo architecture are Czapski Palace (1712–1721), Palace of the Four Winds (1730s) and Visitationist Church (façade 1728–1761). The neoclassical architecture in Warsaw can be described by the simplicity of the geometrical forms teamed with a great inspiration from the Roman period. Some of the best examples of the neoclassical style are the Palace on the Water (rebuilt 1775–1795), Królikarnia (1782–1786), Carmelite Church (façade 1761–1783) and Evangelical Holy Trinity Church (1777–1782). The economic growth during the first years of Congress Poland caused a rapid rise architecture. The Neoclassical revival affected all aspects of architecture, the most notable are the Great Theater (1825–1833) and buildings located at Bank Square (1825–1828).\n",
      "\n",
      "Question:  When did painting activity in the palaces and churches take place in the later decades of?\n",
      "\n",
      "Impossible to answer\n",
      "\n",
      "Predicted impossbile to answer\n",
      "\n",
      "//////////////////// \n",
      "\n",
      "Context:  Newcastle replaced him in January 1756 with Lord Loudoun, with Major General James Abercrombie as his second in command. Neither of these men had as much campaign experience as the trio of officers France sent to North America. French regular army reinforcements arrived in New France in May 1756, led by Major General Louis-Joseph de Montcalm and seconded by the Chevalier de Lévis and Colonel François-Charles de Bourlamaque, all experienced veterans from the War of the Austrian Succession. During that time in Europe, on May 18, 1756, England formally declared war on France, which expanded the war into Europe, which was later to be known as the Seven Years' War.\n",
      "\n",
      "Question:  Who was appointed as third in command to Lor Loudoun in 1756?\n",
      "\n",
      "Impossible to answer\n",
      "\n",
      "Predicted answer:  de Lévis and\n",
      "\n",
      "//////////////////// \n",
      "\n",
      "Context:  In between the French and the British, large areas were dominated by native tribes. To the north, the Mi'kmaq and the Abenaki were engaged in Father Le Loutre's War and still held sway in parts of Nova Scotia, Acadia, and the eastern portions of the province of Canada, as well as much of present-day Maine. The Iroquois Confederation dominated much of present-day Upstate New York and the Ohio Country, although the latter also included Algonquian-speaking populations of Delaware and Shawnee, as well as Iroquoian-speaking Mingo. These tribes were formally under Iroquois rule, and were limited by them in authority to make agreements.\n",
      "\n",
      "Question:  What tribes weren't in Father Le Loutre's War?\n",
      "\n",
      "Impossible to answer\n",
      "\n",
      "Predicted answer:  Nova\n",
      "\n",
      "//////////////////// \n",
      "\n",
      "Context:  In modern particle physics, forces and the acceleration of particles are explained as a mathematical by-product of exchange of momentum-carrying gauge bosons. With the development of quantum field theory and general relativity, it was realized that force is a redundant concept arising from conservation of momentum (4-momentum in relativity and momentum of virtual particles in quantum electrodynamics). The conservation of momentum can be directly derived from the homogeneity or symmetry of space and so is usually considered more fundamental than the concept of a force. Thus the currently known fundamental forces are considered more accurately to be \"fundamental interactions\".:199–128 When particle A emits (creates) or absorbs (annihilates) virtual particle B, a momentum conservation results in recoil of particle A making impression of repulsion or attraction between particles A A' exchanging by B. This description applies to all forces arising from fundamental interactions. While sophisticated mathematical descriptions are needed to predict, in full detail, the accurate result of such interactions, there is a conceptually simple way to describe such interactions through the use of Feynman diagrams. In a Feynman diagram, each matter particle is represented as a straight line (see world line) traveling through time, which normally increases up or to the right in the diagram. Matter and anti-matter particles are identical except for their direction of propagation through the Feynman diagram. World lines of particles intersect at interaction vertices, and the Feynman diagram represents any force arising from an interaction as occurring at the vertex with an associated instantaneous change in the direction of the particle world lines. Gauge bosons are emitted away from the vertex as wavy lines and, in the case of virtual particle exchange, are absorbed at an adjacent vertex.\n",
      "\n",
      "Question:  How are the particle forces and accelerations explained as by gauge bosons exchange?\n",
      "\n",
      "Answer:  mathematical by-product\n",
      "\n",
      "Predicted impossbile to answer\n",
      "\n",
      "//////////////////// \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    choice = np.random.choice(list(res))\n",
    "    row = test_df[test_df.id == choice].iloc[0]\n",
    "    print(\"Context: \", str(row.context))\n",
    "    print()\n",
    "    print(\"Question: \", str(row.content))\n",
    "    print()\n",
    "    if row.is_impossible:\n",
    "        print(\"Impossible to answer\")\n",
    "    else:\n",
    "        print(\"Answer: \", row.answer)\n",
    "    print()\n",
    "    if res[choice]:\n",
    "        print(\"Predicted answer: \", res[choice])\n",
    "    else:\n",
    "        print(\"Predicted impossbile to answer\")\n",
    "    print(\"\\n//////////////////// \\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}