rnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torchtext import datasets, data\n",
    "import torchtext\n",
    "import random\n",
    "import torch.optim as optim\n",
    "import time\n",
    "from sklearn.metrics import accuracy_score\n",
    "import torch.nn.functional as F\n",
    "from torchtext.data.utils import get_tokenizer\n",
    "import pandas as pd\n",
    "from torchtext.data import TabularDataset\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some hyperparams\n",
    "bs = 256\n",
    "min_freq = 5\n",
    "vocab_max_size = 25000\n",
    "embedding_size = 100\n",
    "lstm_hidden=200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = data.Field(lower=True,\n",
    "                  include_lengths=True,\n",
    "                  tokenize=get_tokenizer('basic_english'))\n",
    "LABEL = data.Field(sequential=False,is_target=True,unk_token=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yelp_review_polarity_csv/\n",
      "yelp_review_polarity_csv/readme.txt\n",
      "yelp_review_polarity_csv/test.csv\n",
      "yelp_review_polarity_csv/train.csv\n"
     ]
    }
   ],
   "source": [
    "torchtext.utils.download_from_url(datasets.text_classification.URLS['YelpReviewPolarity'])\n",
    "!tar -C .data -xvf .data/yelp_review_polarity_csv.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "glove_vectors = torchtext.vocab.GloVe(name='6B', dim=embedding_size,cache='.cache/embeddings')\n",
    "glove_mean, glove_std = glove_vectors.vectors.mean(), glove_vectors.vectors.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "datafields = [('label', LABEL), \n",
    "              ('text', TEXT)]\n",
    "\n",
    "train, valid = TabularDataset.splits(\n",
    "               path='.data/yelp_review_polarity_csv',\n",
    "               train='train.csv', validation='test.csv',test=None,\n",
    "               format='csv',\n",
    "               skip_header=False,\n",
    "               fields=datafields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build vocab\n",
    "TEXT.build_vocab(train, vectors = glove_vectors, max_size=vocab_max_size,min_freq=min_freq)\n",
    "LABEL.build_vocab(train)\n",
    "\n",
    "# init oov embeddings with same distribution as glove\n",
    "# skip <unk> and padding\n",
    "for i in range(2, len(TEXT.vocab.vectors)):\n",
    "    if len(TEXT.vocab.vectors[i, :].nonzero()) == 0:\n",
    "        nn.init.normal_(TEXT.vocab.vectors[i], mean=glove_mean, std=glove_std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create iterator\n",
    "train_iterator, valid_iterator = data.BucketIterator.splits((train, valid),\n",
    "                                                            batch_size=bs,\n",
    "                                                            device=device,\n",
    "                                                            shuffle=True,\n",
    "                                                            sort_key=lambda x: len(x.text),\n",
    "                                                            sort_within_batch = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit/test functions\n",
    "def fit(iterator, model, optimizer, criterion):\n",
    "    train_loss = 0\n",
    "    train_acc = 0\n",
    "    model.train()\n",
    "    all_y = []\n",
    "    all_y_hat = []\n",
    "    for batch in iterator:\n",
    "        optimizer.zero_grad()\n",
    "        y = batch.label\n",
    "        text, text_lengths = batch.text\n",
    "        y_hat = model(text, text_lengths)\n",
    "        loss = criterion(y_hat, y)\n",
    "        train_loss += loss.item()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        all_y.append(y)\n",
    "        all_y_hat.append(y_hat)\n",
    "    y = torch.cat(all_y,dim=0)\n",
    "    y_hat = torch.cat(all_y_hat,dim=0)\n",
    "    acc = accuracy_score(y.cpu(),y_hat.argmax(1).detach().cpu())\n",
    "    return train_loss / len(iterator.dataset), acc\n",
    "\n",
    "def test(iterator, model, criterion):\n",
    "    train_loss = 0\n",
    "    train_acc = 0\n",
    "    model.eval()\n",
    "    all_y = []\n",
    "    all_y_hat = []\n",
    "    for batch in iterator:\n",
    "        y = batch.label                       \n",
    "        with torch.no_grad():\n",
    "            text, text_lengths = batch.text\n",
    "            y_hat = model(text, text_lengths)\n",
    "        loss = criterion(y_hat, y)\n",
    "        train_loss += loss.item()\n",
    "        \n",
    "        all_y.append(y)\n",
    "        all_y_hat.append(y_hat)\n",
    "    y = torch.cat(all_y,dim=0)\n",
    "    y_hat = torch.cat(all_y_hat,dim=0)\n",
    "    acc = accuracy_score(y.cpu(),y_hat.argmax(1).detach().cpu())\n",
    "    return train_loss / len(iterator.dataset), acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_n_epochs(model, n, optimizer, scheduler):\n",
    "\n",
    "    criterion = nn.CrossEntropyLoss().to(device)\n",
    "    for epoch in range(n):\n",
    "        start_time = time.time()\n",
    "        train_loss, train_acc = fit(train_iterator, model, optimizer, criterion)\n",
    "        scheduler.step()\n",
    "        valid_loss, valid_acc = test(valid_iterator, model, criterion)\n",
    "\n",
    "        secs = int(time.time() - start_time)\n",
    "        mins = secs / 60\n",
    "        secs = secs % 60\n",
    "\n",
    "        print('Epoch: %d' % (epoch), \" | time in %d minutes, %d seconds\" % (mins, secs))\n",
    "        print(f'\\tTrain Loss: {train_loss:.4f}\\t|\\tAccuracy: {train_acc :.6f}')\n",
    "        print(f'\\tValidation Loss: {valid_loss:.4f}\\t|\\tAccuracy: {valid_acc:.6f}') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0  | time in 1 minutes, 0 seconds\n",
      "\tTrain Loss: 0.0008\t|\tAccuracy: 0.912532\n",
      "\tValidation Loss: 0.0011\t|\tAccuracy: 0.909000\n",
      "Epoch: 1  | time in 1 minutes, 0 seconds\n",
      "\tTrain Loss: 0.0005\t|\tAccuracy: 0.952491\n",
      "\tValidation Loss: 0.0007\t|\tAccuracy: 0.941026\n",
      "Epoch: 2  | time in 1 minutes, 0 seconds\n",
      "\tTrain Loss: 0.0004\t|\tAccuracy: 0.960500\n",
      "\tValidation Loss: 0.0005\t|\tAccuracy: 0.952053\n",
      "Epoch: 3  | time in 1 minutes, 0 seconds\n",
      "\tTrain Loss: 0.0004\t|\tAccuracy: 0.964964\n",
      "\tValidation Loss: 0.0006\t|\tAccuracy: 0.949737\n",
      "Epoch: 4  | time in 1 minutes, 0 seconds\n",
      "\tTrain Loss: 0.0003\t|\tAccuracy: 0.967975\n",
      "\tValidation Loss: 0.0004\t|\tAccuracy: 0.957842\n",
      "Epoch: 5  | time in 1 minutes, 0 seconds\n",
      "\tTrain Loss: 0.0003\t|\tAccuracy: 0.972132\n",
      "\tValidation Loss: 0.0005\t|\tAccuracy: 0.959421\n",
      "Epoch: 6  | time in 1 minutes, 0 seconds\n",
      "\tTrain Loss: 0.0003\t|\tAccuracy: 0.973209\n",
      "\tValidation Loss: 0.0005\t|\tAccuracy: 0.959316\n",
      "Epoch: 7  | time in 1 minutes, 0 seconds\n",
      "\tTrain Loss: 0.0003\t|\tAccuracy: 0.973836\n",
      "\tValidation Loss: 0.0004\t|\tAccuracy: 0.962079\n",
      "Epoch: 8  | time in 1 minutes, 0 seconds\n",
      "\tTrain Loss: 0.0003\t|\tAccuracy: 0.973543\n",
      "\tValidation Loss: 0.0004\t|\tAccuracy: 0.962237\n",
      "Epoch: 9  | time in 1 minutes, 0 seconds\n",
      "\tTrain Loss: 0.0003\t|\tAccuracy: 0.974152\n",
      "\tValidation Loss: 0.0004\t|\tAccuracy: 0.960342\n"
     ]
    }
   ],
   "source": [
    "# gru model\n",
    "class GRUNet(nn.Module):\n",
    "    def __init__(self,embeddings, embedding_dim, output_dim, pad_idx, hidden_size):\n",
    "        super().__init__()\n",
    "        self.embeddings = nn.Embedding.from_pretrained(embeddings, freeze=False, padding_idx=pad_idx)\n",
    "        self.dropout = nn.Dropout(0.5) \n",
    "        self.rnn = nn.GRU(input_size=embedding_dim,\n",
    "                          hidden_size=hidden_size, \n",
    "                          num_layers=1, batch_first=False,\n",
    "                          bidirectional=False,dropout=0.0)\n",
    "    \n",
    "        self.fc = nn.Sequential(\n",
    "            nn.BatchNorm1d(hidden_size),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(hidden_size, output_dim))\n",
    "\n",
    "    def forward(self, input, input_lengths):\n",
    "        emb = self.dropout(self.embeddings(input))\n",
    "        emb_packed = nn.utils.rnn.pack_padded_sequence(emb, input_lengths)\n",
    "        packed_output, hidden = self.rnn(emb_packed)\n",
    "        return self.fc(hidden.squeeze(0))\n",
    "\n",
    "gru_model = GRUNet(TEXT.vocab.vectors,\n",
    "                   embedding_size,\n",
    "                   len(LABEL.vocab),\n",
    "                   TEXT.vocab.stoi[TEXT.pad_token],\n",
    "                   200).to(device)\n",
    "wd=0.0000\n",
    "lr=0.001\n",
    "epochs=10\n",
    "optimizer = optim.Adam(gru_model.parameters(), lr=lr, weight_decay=wd)\n",
    "scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [5,10], gamma=0.1)\n",
    "train_n_epochs(gru_model,epochs,optimizer,scheduler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0  | time in 2 minutes, 52 seconds\n",
      "\tTrain Loss: 0.0009\t|\tAccuracy: 0.896755\n",
      "\tValidation Loss: 0.0006\t|\tAccuracy: 0.939763\n",
      "Epoch: 1  | time in 2 minutes, 52 seconds\n",
      "\tTrain Loss: 0.0005\t|\tAccuracy: 0.950484\n",
      "\tValidation Loss: 0.0004\t|\tAccuracy: 0.962263\n",
      "Epoch: 2  | time in 2 minutes, 52 seconds\n",
      "\tTrain Loss: 0.0004\t|\tAccuracy: 0.960443\n",
      "\tValidation Loss: 0.0004\t|\tAccuracy: 0.963526\n",
      "Epoch: 3  | time in 2 minutes, 52 seconds\n",
      "\tTrain Loss: 0.0004\t|\tAccuracy: 0.961823\n",
      "\tValidation Loss: 0.0004\t|\tAccuracy: 0.965158\n",
      "Epoch: 4  | time in 2 minutes, 52 seconds\n",
      "\tTrain Loss: 0.0004\t|\tAccuracy: 0.962888\n",
      "\tValidation Loss: 0.0004\t|\tAccuracy: 0.965421\n",
      "Epoch: 5  | time in 2 minutes, 51 seconds\n",
      "\tTrain Loss: 0.0004\t|\tAccuracy: 0.963963\n",
      "\tValidation Loss: 0.0004\t|\tAccuracy: 0.966263\n",
      "Epoch: 6  | time in 2 minutes, 51 seconds\n",
      "\tTrain Loss: 0.0004\t|\tAccuracy: 0.964782\n",
      "\tValidation Loss: 0.0004\t|\tAccuracy: 0.966789\n",
      "Epoch: 7  | time in 2 minutes, 50 seconds\n",
      "\tTrain Loss: 0.0004\t|\tAccuracy: 0.965277\n",
      "\tValidation Loss: 0.0004\t|\tAccuracy: 0.965921\n",
      "Epoch: 8  | time in 2 minutes, 51 seconds\n",
      "\tTrain Loss: 0.0004\t|\tAccuracy: 0.966250\n",
      "\tValidation Loss: 0.0003\t|\tAccuracy: 0.966711\n",
      "Epoch: 9  | time in 2 minutes, 51 seconds\n",
      "\tTrain Loss: 0.0003\t|\tAccuracy: 0.966741\n",
      "\tValidation Loss: 0.0003\t|\tAccuracy: 0.967789\n"
     ]
    }
   ],
   "source": [
    "# gru model\n",
    "class LSTMNet(nn.Module):\n",
    "    def __init__(self,embeddings, embedding_dim, output_dim, pad_idx, hidden_size):\n",
    "        super().__init__()\n",
    "        self.embeddings = nn.Embedding.from_pretrained(embeddings, freeze=False, padding_idx=pad_idx)\n",
    "        self.dropout = nn.Dropout(0.5) \n",
    "        self.rnn = nn.LSTM(input_size=embedding_dim,\n",
    "                           hidden_size=hidden_size, \n",
    "                           num_layers=2, batch_first=False,\n",
    "                           bidirectional=True,dropout=0.0)\n",
    "    \n",
    "        self.fc = nn.Sequential(\n",
    "            nn.BatchNorm1d(hidden_size * 2),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(hidden_size * 2, output_dim))\n",
    "\n",
    "    def forward(self, input, input_lengths):\n",
    "        emb = self.dropout(self.embeddings(input))\n",
    "        emb_packed = nn.utils.rnn.pack_padded_sequence(emb, input_lengths)\n",
    "        packed_output, (hidden,c_n) = self.rnn(emb_packed)\n",
    "        hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1))\n",
    "        return self.fc(hidden.squeeze(0))\n",
    "\n",
    "gru_model = LSTMNet(TEXT.vocab.vectors,\n",
    "                    embedding_size,\n",
    "                    len(LABEL.vocab),\n",
    "                    TEXT.vocab.stoi[TEXT.pad_token],\n",
    "                    200).to(device)\n",
    "wd=0.0000\n",
    "lr=0.001\n",
    "epochs=10\n",
    "optimizer = optim.Adam(gru_model.parameters(), lr=lr, weight_decay=wd)\n",
    "scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, [2,10], gamma=0.1)\n",
    "train_n_epochs(gru_model,epochs,optimizer,scheduler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}