code/title_generation/T5 Fine tune and Rouge.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5193622d",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install rouge-score -q\n",
    "!pip install wandb -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1f7ba12f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /home/charmichokshi4444/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "# Importing stock libraries\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\n",
    "\n",
    "# Importing the T5 modules from huggingface/transformers\n",
    "from transformers import T5Tokenizer, T5ForConditionalGeneration\n",
    "\n",
    "# WandB – Import \n",
    "import wandb\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "from rouge_score import rouge_scorer\n",
    "\n",
    "scorer = rouge_scorer.RougeScorer(['rougeL'], use_stemmer=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c6e1021",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "wandb.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "160aa788",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a custom dataset for reading the dataframe and loading it into the dataloader to pass it to the neural network at a later stage for finetuning the model and to prepare it for predictions\n",
    "\n",
    "class CustomDataset(Dataset):\n",
    "\n",
    "    def __init__(self, dataframe, tokenizer, source_len, summ_len):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.data = dataframe\n",
    "        self.source_len = source_len\n",
    "        self.summ_len = summ_len\n",
    "        self.text = self.data.text\n",
    "        self.ctext = self.data.ctext\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.text)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        ctext = str(self.ctext[index])\n",
    "        ctext = ' '.join(ctext.split())\n",
    "\n",
    "        text = str(self.text[index])\n",
    "        text = ' '.join(text.split())\n",
    "\n",
    "        source = self.tokenizer.batch_encode_plus([ctext], max_length= self.source_len, pad_to_max_length=True,return_tensors='pt')\n",
    "        target = self.tokenizer.batch_encode_plus([text], max_length= self.summ_len, pad_to_max_length=True,return_tensors='pt')\n",
    "\n",
    "        source_ids = source['input_ids'].squeeze()\n",
    "        source_mask = source['attention_mask'].squeeze()\n",
    "        target_ids = target['input_ids'].squeeze()\n",
    "        target_mask = target['attention_mask'].squeeze()\n",
    "\n",
    "        return {\n",
    "            'source_ids': source_ids.to(dtype=torch.long), \n",
    "            'source_mask': source_mask.to(dtype=torch.long), \n",
    "            'target_ids': target_ids.to(dtype=torch.long),\n",
    "            'target_ids_y': target_ids.to(dtype=torch.long)\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "34bbafc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train function to train for one epoch\n",
    "def train(epoch, tokenizer, model, device, loader, optimizer):\n",
    "    model.train()\n",
    "    for _,data in enumerate(loader, 0):\n",
    "        y = data['target_ids'].to(device, dtype = torch.long)\n",
    "        y_ids = y[:, :-1].contiguous()\n",
    "        lm_labels = y[:, 1:].clone().detach()\n",
    "        lm_labels[y[:, 1:] == tokenizer.pad_token_id] = -100\n",
    "        ids = data['source_ids'].to(device, dtype = torch.long)\n",
    "        mask = data['source_mask'].to(device, dtype = torch.long)\n",
    "\n",
    "        outputs = model(input_ids = ids, attention_mask = mask, decoder_input_ids=y_ids, labels=lm_labels)\n",
    "        loss = outputs[0]\n",
    "        \n",
    "        if _%10 == 0:\n",
    "            wandb.log({\"Training Loss\": loss.item()})\n",
    "\n",
    "        if _%500==0:\n",
    "            print(f'Epoch: {epoch}, Loss:  {loss.item()}')\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3a853748",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Validation function to validate the generation, returns both predicted and actual value\n",
    "def validate(epoch, tokenizer, model, device, loader):\n",
    "    model.eval()\n",
    "    predictions = []\n",
    "    actuals = []\n",
    "    with torch.no_grad():\n",
    "        for _, data in enumerate(loader, 0):\n",
    "            y = data['target_ids'].to(device, dtype = torch.long)\n",
    "            ids = data['source_ids'].to(device, dtype = torch.long)\n",
    "            mask = data['source_mask'].to(device, dtype = torch.long)\n",
    "\n",
    "            generated_ids = model.generate(\n",
    "                input_ids = ids,\n",
    "                attention_mask = mask, \n",
    "                max_length=150, \n",
    "                num_beams=2,\n",
    "                repetition_penalty=2.5, \n",
    "                length_penalty=1.0, \n",
    "                early_stopping=True\n",
    "                )\n",
    "            preds = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=True) for g in generated_ids]\n",
    "            target = [tokenizer.decode(t, skip_special_tokens=True, clean_up_tokenization_spaces=True)for t in y]\n",
    "            if _%100==0:\n",
    "                print(f'Completed {_}')\n",
    "\n",
    "            predictions.extend(preds)\n",
    "            actuals.extend(target)\n",
    "    return predictions, actuals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "70086206",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /home/charmichokshi4444/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "\n",
    "stop_words = set(stopwords.words('english'))\n",
    "#https://catriscode.com/2021/05/01/tweets-cleaning-with-python/\n",
    "def clean_data(tweet):\n",
    "    if type(tweet) == np.float:\n",
    "        return \"\"\n",
    "    temp = tweet.lower()\n",
    "    temp = re.sub(\"'\", \"\", temp) # to avoid removing contractions in english\n",
    "    temp = re.sub(\"@[A-Za-z0-9_]+\",\"\", temp)\n",
    "    temp = re.sub(\"#[A-Za-z0-9_]+\",\"\", temp)\n",
    "    temp = re.sub(r'http\\S+', '', temp)\n",
    "    temp = re.sub('[()!?]', ' ', temp)\n",
    "    temp = re.sub('\\[.*?\\]',' ', temp)\n",
    "    temp = re.sub(\"[^a-z0-9]\",\" \", temp)\n",
    "    temp = temp.split()\n",
    "    temp = [w for w in temp if not w in stop_words]\n",
    "    temp = \" \".join(word for word in temp)\n",
    "    return temp\n",
    "\n",
    "#https://stackoverflow.com/questions/33404752/removing-emojis-from-a-string-in-python\n",
    "def deEmojify(text):\n",
    "#     print(text)\n",
    "    regrex_pattern = re.compile(pattern = \"[\"\n",
    "        u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                           \"]+\", flags = re.UNICODE)\n",
    "    return regrex_pattern.sub(r'',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a0aadd8d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33msandstorm11\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.13.5"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/charmichokshi4444/MYCLICKBAIT/wandb/run-20221128_033215-1u2c88i8</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/sandstorm11/transformers_tutorials_summarization/runs/1u2c88i8\" target=\"_blank\">fine-sponge-22</a></strong> to <a href=\"https://wandb.ai/sandstorm11/transformers_tutorials_summarization\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:6: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                text  \\\n",
      "0  UK’s response to modern slavery leaving victim...   \n",
      "1  The \"forgotten\" Trump roast: Relive his brutal...   \n",
      "2  Tokyo's subway is shut down amid fears over an...   \n",
      "3             Ban lifted on Madrid doping laboratory   \n",
      "4  Despite the ‘Yuck Factor,’ Leeches Are Big in ...   \n",
      "\n",
      "                                               ctext  \n",
      "0  summarize: thousands modern slavery victims co...  \n",
      "1  summarize: white house correspondents dinner e...  \n",
      "2  summarize: one tokyos major subways systems sa...  \n",
      "3  summarize: share madrids anti doping laborator...  \n",
      "4  summarize: moscow small physician assistants g...  \n",
      "FULL Dataset: (16390, 2)\n",
      "TRAIN Dataset: (13112, 2)\n",
      "TEST Dataset: (3278, 2)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initiating Fine-Tuning for the model on our dataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/transformers/tokenization_utils_base.py:2310: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  FutureWarning,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, Loss:  8.9929838180542\n",
      "Epoch: 0, Loss:  3.008601427078247\n",
      "Epoch: 0, Loss:  2.458158254623413\n",
      "Epoch: 0, Loss:  2.575425863265991\n",
      "Now generating summaries on our fine tuned model for the validation dataset and saving it in a dataframe\n",
      "Completed 0\n",
      "Completed 100\n",
      "Completed 200\n",
      "Completed 300\n",
      "Completed 400\n",
      "Output Files generated for review\n",
      "Epoch: 1, Loss:  2.1646344661712646\n",
      "Epoch: 1, Loss:  2.1130573749542236\n",
      "Epoch: 1, Loss:  2.6266937255859375\n",
      "Epoch: 1, Loss:  2.509052038192749\n",
      "Now generating summaries on our fine tuned model for the validation dataset and saving it in a dataframe\n",
      "Completed 0\n",
      "Completed 100\n",
      "Completed 200\n",
      "Completed 300\n",
      "Completed 400\n",
      "Output Files generated for review\n",
      "Epoch: 2, Loss:  2.89286732673645\n",
      "Epoch: 2, Loss:  2.7665982246398926\n",
      "Epoch: 2, Loss:  1.8510767221450806\n",
      "Epoch: 2, Loss:  2.1461596488952637\n",
      "Now generating summaries on our fine tuned model for the validation dataset and saving it in a dataframe\n",
      "Completed 0\n",
      "Completed 100\n",
      "Completed 200\n",
      "Completed 300\n",
      "Completed 400\n",
      "Output Files generated for review\n",
      "Epoch: 3, Loss:  1.825895071029663\n",
      "Epoch: 3, Loss:  2.0292751789093018\n",
      "Epoch: 3, Loss:  1.848103642463684\n",
      "Epoch: 3, Loss:  1.8680893182754517\n",
      "Now generating summaries on our fine tuned model for the validation dataset and saving it in a dataframe\n",
      "Completed 0\n",
      "Completed 100\n",
      "Completed 200\n",
      "Completed 300\n",
      "Completed 400\n",
      "Output Files generated for review\n",
      "Epoch: 4, Loss:  2.1991724967956543\n",
      "Epoch: 4, Loss:  1.0352611541748047\n",
      "Epoch: 4, Loss:  1.730298399925232\n",
      "Epoch: 4, Loss:  1.6666123867034912\n",
      "Now generating summaries on our fine tuned model for the validation dataset and saving it in a dataframe\n",
      "Completed 0\n",
      "Completed 100\n",
      "Completed 200\n",
      "Completed 300\n",
      "Completed 400\n",
      "Output Files generated for review\n",
      "Epoch: 5, Loss:  1.1604466438293457\n",
      "Epoch: 5, Loss:  1.2569656372070312\n",
      "Epoch: 5, Loss:  1.3727566003799438\n",
      "Epoch: 5, Loss:  1.1557294130325317\n",
      "Now generating summaries on our fine tuned model for the validation dataset and saving it in a dataframe\n",
      "Completed 0\n",
      "Completed 100\n",
      "Completed 200\n",
      "Completed 300\n",
      "Completed 400\n",
      "Output Files generated for review\n",
      "Epoch: 6, Loss:  1.5552467107772827\n",
      "Epoch: 6, Loss:  1.3220291137695312\n",
      "Epoch: 6, Loss:  1.3708136081695557\n",
      "Epoch: 6, Loss:  1.2190841436386108\n",
      "Now generating summaries on our fine tuned model for the validation dataset and saving it in a dataframe\n",
      "Completed 0\n",
      "Completed 100\n",
      "Completed 200\n",
      "Completed 300\n",
      "Completed 400\n",
      "Output Files generated for review\n",
      "Epoch: 7, Loss:  0.8968333601951599\n",
      "Epoch: 7, Loss:  1.482319951057434\n",
      "Epoch: 7, Loss:  0.9408825039863586\n",
      "Epoch: 7, Loss:  0.7132935523986816\n",
      "Now generating summaries on our fine tuned model for the validation dataset and saving it in a dataframe\n",
      "Completed 0\n",
      "Completed 100\n",
      "Completed 200\n",
      "Completed 300\n",
      "Completed 400\n",
      "Output Files generated for review\n",
      "Epoch: 8, Loss:  0.8348583579063416\n",
      "Epoch: 8, Loss:  1.2181472778320312\n",
      "Epoch: 8, Loss:  0.9553611874580383\n",
      "Epoch: 8, Loss:  0.8580646514892578\n",
      "Now generating summaries on our fine tuned model for the validation dataset and saving it in a dataframe\n",
      "Completed 0\n",
      "Completed 100\n",
      "Completed 200\n",
      "Completed 300\n",
      "Completed 400\n",
      "Output Files generated for review\n",
      "Epoch: 9, Loss:  1.1549931764602661\n",
      "Epoch: 9, Loss:  0.9548215866088867\n",
      "Epoch: 9, Loss:  0.8084550499916077\n",
      "Epoch: 9, Loss:  0.7922202348709106\n",
      "Now generating summaries on our fine tuned model for the validation dataset and saving it in a dataframe\n",
      "Completed 0\n",
      "Completed 100\n",
      "Completed 200\n",
      "Completed 300\n",
      "Completed 400\n",
      "Output Files generated for review\n"
     ]
    }
   ],
   "source": [
    "torch.cuda.empty_cache()\n",
    "# # Setting up the device for GPU usage\n",
    "from torch import cuda\n",
    "device = 'cuda' if cuda.is_available() else 'cpu'\n",
    "# WandB – Initialize a new run\n",
    "wandb.init(project=\"transformers_tutorials_summarization\")\n",
    "\n",
    "\n",
    "# Wandb config \n",
    "config = wandb.config          # Initialize config\n",
    "config.TRAIN_BATCH_SIZE = 8    \n",
    "config.VALID_BATCH_SIZE = 8    \n",
    "config.TRAIN_EPOCHS = 10        \n",
    "config.VAL_EPOCHS = 1 \n",
    "config.LEARNING_RATE = 1e-4   \n",
    "config.SEED = 42              \n",
    "config.MAX_LEN = 512\n",
    "config.SUMMARY_LEN = 150 \n",
    "\n",
    "# Set random seeds and deterministic pytorch for reproducibility\n",
    "torch.manual_seed(config.SEED) # pytorch random seed\n",
    "np.random.seed(config.SEED) # numpy random seed\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "# tokenzier for encoding the text\n",
    "tokenizer = T5Tokenizer.from_pretrained(\"Michau/t5-base-en-generate-headline\")\n",
    "\n",
    "\n",
    "df = pd.read_csv('./TitleGenData/no_clickbait_train_data.csv')\n",
    "df = df.rename(columns={'postText':'text', 'targetParagraphs': 'ctext'})\n",
    "df = df[['text','ctext']]\n",
    "df = df.dropna()\n",
    "df['ctext'] = df['ctext'].apply(deEmojify)\n",
    "df['ctext'] = df['ctext'].apply(clean_data)\n",
    "df.ctext = 'summarize: ' + df.ctext\n",
    "print(df.head())\n",
    "\n",
    "\n",
    "# Dataset creation and 80% is train remaining is test(val) \n",
    "train_size = 0.8\n",
    "train_dataset=df.sample(frac=train_size,random_state = config.SEED)\n",
    "val_dataset=df.drop(train_dataset.index).reset_index(drop=True)\n",
    "train_dataset = train_dataset.reset_index(drop=True)\n",
    "\n",
    "print(\"FULL Dataset: {}\".format(df.shape))\n",
    "print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
    "print(\"TEST Dataset: {}\".format(val_dataset.shape))\n",
    "\n",
    "\n",
    "\n",
    "training_set = CustomDataset(train_dataset, tokenizer, config.MAX_LEN, config.SUMMARY_LEN)\n",
    "val_set = CustomDataset(val_dataset, tokenizer, config.MAX_LEN, config.SUMMARY_LEN)\n",
    "\n",
    "# parameters for dataloaders\n",
    "train_params = {\n",
    "  'batch_size': config.TRAIN_BATCH_SIZE,\n",
    "  'shuffle': True,\n",
    "  'num_workers': 0\n",
    "  }\n",
    "\n",
    "val_params = {\n",
    "  'batch_size': config.VALID_BATCH_SIZE,\n",
    "  'shuffle': False,\n",
    "  'num_workers': 0\n",
    "  }\n",
    "\n",
    "# Dataloaders for train and validation.\n",
    "training_loader = DataLoader(training_set, **train_params)\n",
    "val_loader = DataLoader(val_set, **val_params)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "model = T5ForConditionalGeneration.from_pretrained(\"Michau/t5-base-en-generate-headline\")\n",
    "model = model.to(device)\n",
    "\n",
    "# Defining the optimizer that will be used to tune the weights of the network in the training session. \n",
    "optimizer = torch.optim.AdamW(params =  model.parameters(), lr=config.LEARNING_RATE)\n",
    "\n",
    "# Log metrics with wandb\n",
    "wandb.watch(model, log=\"all\")\n",
    "# Training loop\n",
    "print('Initiating Fine-Tuning for the model on our dataset')\n",
    "\n",
    "for epoch in range(config.TRAIN_EPOCHS):\n",
    "    train(epoch, tokenizer, model, device, training_loader, optimizer)\n",
    "    torch.save(model.state_dict(), './finetuneT5/final-t5-finetuned-epoch-{}.pt'.format(epoch))\n",
    "    # Validation and saving the resulting file in a dataframe.\n",
    "    print('Now generating summaries on our fine tuned model for the validation dataset and saving it in a dataframe')\n",
    "    for vepoch in range(config.VAL_EPOCHS):\n",
    "        predictions, actuals = validate(vepoch, tokenizer, model, device, val_loader)\n",
    "        final_df = pd.DataFrame({'Generated Text':predictions,'Actual Text':actuals})\n",
    "        final_df.to_csv('./finetuneT5/fine_tuned_predictions-epoch-{}.csv'.format(epoch))\n",
    "        print('Output Files generated for review')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4879a178",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:6: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 Rouge score\n",
      "0.31175009777793916 0.2305779375785791 0.25683693955235326\n",
      "Epoch 1 Rouge score\n",
      "0.3080772083605138 0.22362719545498225 0.25157800707569883\n",
      "Epoch 2 Rouge score\n",
      "0.3139571128943229 0.23197930591981605 0.25908909252048695\n",
      "Epoch 3 Rouge score\n",
      "0.3053456100828505 0.23474932878875643 0.25736181158217875\n",
      "Epoch 4 Rouge score\n",
      "0.30878704399637136 0.23450181337142384 0.25870959977967356\n",
      "Epoch 5 Rouge score\n",
      "0.30583623616072525 0.23465087813834584 0.25792616427696813\n",
      "Epoch 6 Rouge score\n",
      "0.2976730910047489 0.2253216486152646 0.24893878897029406\n",
      "Epoch 7 Rouge score\n",
      "0.2994694282685974 0.23282905068760676 0.2539848251877404\n",
      "Epoch 8 Rouge score\n",
      "0.29991925740857966 0.22418793898921713 0.24926194586726905\n",
      "Epoch 9 Rouge score\n",
      "0.30272454317446773 0.23017937996402799 0.253891552134198\n"
     ]
    }
   ],
   "source": [
    "for e in range(config.TRAIN_EPOCHS):\n",
    "    final_df = pd.read_csv('./finetuneT5/fine_tuned_predictions-epoch-{}.csv'.format(e))\n",
    "    final_df['Actual Text'] = final_df['Actual Text'].apply(deEmojify)\n",
    "    final_df['Actual Text'] = final_df['Actual Text'].apply(clean_data)\n",
    "    ac_text = final_df['Actual Text'].tolist()\n",
    "    gen_text = final_df['Generated Text'].tolist()\n",
    "    results = {'precision': [], 'recall': [], 'fmeasure': []}\n",
    "    for (h, r) in zip(gen_text, ac_text):\n",
    "        # computing the ROUGE\n",
    "        score = scorer.score(h, r)\n",
    "        # separating the measurements\n",
    "        precision, recall, fmeasure = score['rougeL']\n",
    "        # add them to the proper list in the dictionary\n",
    "        results['precision'].append(precision)\n",
    "        results['recall'].append(recall)\n",
    "        results['fmeasure'].append(fmeasure)\n",
    "    print(\"Epoch {} Rouge score\".format(e))\n",
    "    print(sum(results['precision'])/len(results['precision']), sum(results['recall'])/len(results['recall']), sum(results['fmeasure'])/len(results['fmeasure']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fb915dce",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:6: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Default Model RougeL\n",
      "0.2657697767913373 0.25738135100269566 0.24482966585823734\n"
     ]
    }
   ],
   "source": [
    "#Calculated by running validation only on pretrained model\n",
    "print(\"Default Model RougeL\")\n",
    "print(sum(results['precision'])/len(results['precision']), sum(results['recall'])/len(results['recall']), sum(results['fmeasure'])/len(results['fmeasure']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6223b8f3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}